xnu-3248.50.21.tar.gz

[apple/xnu.git] / osfmk / vm / vm_map.c

diff --git a/osfmk/vm/vm_map.c b/osfmk/vm/vm_map.c
index 58a8c7828aec7159f3e5b1d1d90509269ae77960..840c4babfa0e6d06844d43a9abe26bc1dab81708 100644 (file)
--- a/osfmk/vm/vm_map.c
+++ b/osfmk/vm/vm_map.c
@@ -109,6 +109,7 @@
 #include <vm/vm_shared_region.h>
 #include <vm/vm_map_store.h>
 
 #include <vm/vm_shared_region.h>
 #include <vm/vm_map_store.h>
 

+

 extern u_int32_t random(void); /* from <libkern/libkern.h> */
 /* Internal prototypes
  */
 extern u_int32_t random(void); /* from <libkern/libkern.h> */
 /* Internal prototypes
  */


@@ -216,7 +217,7 @@ static kern_return_t        vm_map_wire_nested(
        vm_map_t                   map,
        vm_map_offset_t            start,
        vm_map_offset_t            end,
        vm_map_t                   map,
        vm_map_offset_t            start,
        vm_map_offset_t            end,

-       vm_prot_t                  access_type,
+       vm_prot_t                  caller_prot,

        boolean_t                  user_wire,
        pmap_t                     map_pmap, 
        vm_map_offset_t            pmap_addr,
        boolean_t                  user_wire,
        pmap_t                     map_pmap, 
        vm_map_offset_t            pmap_addr,


@@ -297,6 +298,12 @@ static kern_return_t       vm_map_can_reuse(
        vm_map_offset_t start,
        vm_map_offset_t end);
 
        vm_map_offset_t start,
        vm_map_offset_t end);
 

+#if MACH_ASSERT
+static kern_return_t   vm_map_pageout(
+       vm_map_t        map,
+       vm_map_offset_t start,
+       vm_map_offset_t end);
+#endif /* MACH_ASSERT */

 
 /*
  * Macros to copy a vm_map_entry. We must be careful to correctly
 
 /*
  * Macros to copy a vm_map_entry. We must be careful to correctly


@@ -321,6 +328,8 @@ boolean_t _vmec_reserved = (NEW)->from_reserved_zone;       \
        (NEW)->used_for_jit = FALSE;    \
        (NEW)->from_reserved_zone = _vmec_reserved;     \
        (NEW)->iokit_acct = FALSE;      \
        (NEW)->used_for_jit = FALSE;    \
        (NEW)->from_reserved_zone = _vmec_reserved;     \
        (NEW)->iokit_acct = FALSE;      \

+       (NEW)->vme_resilient_codesign = FALSE; \
+       (NEW)->vme_resilient_media = FALSE;     \

 MACRO_END
 
 #define vm_map_entry_copy_full(NEW,OLD)                        \
 MACRO_END
 
 #define vm_map_entry_copy_full(NEW,OLD)                        \


@@ -367,6 +376,8 @@ override_nx(vm_map_t map, uint32_t user_tag) /* map unused on arm */
 {
        int current_abi;
 
 {
        int current_abi;
 

+       if (map->pmap == kernel_pmap) return FALSE;
+

        /*
         * Determine if the app is running in 32 or 64 bit mode.
         */
        /*
         * Determine if the app is running in 32 or 64 bit mode.
         */


@@ -436,6 +447,7 @@ static zone_t       vm_map_entry_zone;      /* zone for vm_map_entry structures */
 static zone_t  vm_map_entry_reserved_zone;     /* zone with reserve for non-blocking
                                         * allocations */
 static zone_t  vm_map_copy_zone;       /* zone for vm_map_copy structures */
 static zone_t  vm_map_entry_reserved_zone;     /* zone with reserve for non-blocking
                                         * allocations */
 static zone_t  vm_map_copy_zone;       /* zone for vm_map_copy structures */

+zone_t         vm_map_holes_zone;      /* zone for vm map holes (vm_map_links) structures */

 
 
 /*
 
 
 /*


@@ -450,6 +462,8 @@ static void         *map_data;
 static vm_size_t       map_data_size;
 static void            *kentry_data;
 static vm_size_t       kentry_data_size;
 static vm_size_t       map_data_size;
 static void            *kentry_data;
 static vm_size_t       kentry_data_size;

+static void            *map_holes_data;
+static vm_size_t       map_holes_data_size;

 
 #define         NO_COALESCE_LIMIT  ((1024 * 128) - 1)
 
 
 #define         NO_COALESCE_LIMIT  ((1024 * 128) - 1)
 


@@ -477,7 +491,7 @@ vm_map_set_cache_attr(
                kr = KERN_INVALID_ARGUMENT;
                goto done;
        }
                kr = KERN_INVALID_ARGUMENT;
                goto done;
        }

-       object = map_entry->object.vm_object;
+       object = VME_OBJECT(map_entry);

 
        if (object == VM_OBJECT_NULL) {
                /*
 
        if (object == VM_OBJECT_NULL) {
                /*


@@ -509,87 +523,178 @@ done:
  */
 kern_return_t
 vm_map_apple_protected(
  */
 kern_return_t
 vm_map_apple_protected(

-       vm_map_t        map,
-       vm_map_offset_t start,
-       vm_map_offset_t end,
+       vm_map_t                map,
+       vm_map_offset_t         start,
+       vm_map_offset_t         end,
+       vm_object_offset_t      crypto_backing_offset,

        struct pager_crypt_info *crypt_info)
 {
        boolean_t       map_locked;
        kern_return_t   kr;
        vm_map_entry_t  map_entry;
        struct pager_crypt_info *crypt_info)
 {
        boolean_t       map_locked;
        kern_return_t   kr;
        vm_map_entry_t  map_entry;

-       memory_object_t protected_mem_obj;
+       struct vm_map_entry tmp_entry;
+       memory_object_t unprotected_mem_obj;

        vm_object_t     protected_object;
        vm_map_offset_t map_addr;
        vm_object_t     protected_object;
        vm_map_offset_t map_addr;

+       vm_map_offset_t start_aligned, end_aligned;
+       vm_object_offset_t      crypto_start, crypto_end;
+       int             vm_flags;

 
 

-       vm_map_lock_read(map);
-       map_locked = TRUE;
-
-       /* lookup the protected VM object */
-       if (!vm_map_lookup_entry(map,
-                                start,
-                                &map_entry) ||
-           map_entry->vme_end < end ||
-           map_entry->is_sub_map ||
-           !(map_entry->protection & VM_PROT_EXECUTE)) {
-               /* that memory is not properly mapped */
-               kr = KERN_INVALID_ARGUMENT;
-               goto done;
-       }
-       protected_object = map_entry->object.vm_object;
-       if (protected_object == VM_OBJECT_NULL) {
-               /* there should be a VM object here at this point */
-               kr = KERN_INVALID_ARGUMENT;
-               goto done;
-       }
+       map_locked = FALSE;
+       unprotected_mem_obj = MEMORY_OBJECT_NULL;

 
 

-       /* make sure protected object stays alive while map is unlocked */
-       vm_object_reference(protected_object);
+       start_aligned = vm_map_trunc_page(start, PAGE_MASK_64);
+       end_aligned = vm_map_round_page(end, PAGE_MASK_64);
+       start_aligned = vm_map_trunc_page(start_aligned, VM_MAP_PAGE_MASK(map));
+       end_aligned = vm_map_round_page(end_aligned, VM_MAP_PAGE_MASK(map));

 
 

-       vm_map_unlock_read(map);
-       map_locked = FALSE;
+       assert(start_aligned == start);
+       assert(end_aligned == end);

 
 

-       /*
-        * Lookup (and create if necessary) the protected memory object
-        * matching that VM object.
-        * If successful, this also grabs a reference on the memory object,
-        * to guarantee that it doesn't go away before we get a chance to map
-        * it.
-        */
-       protected_mem_obj = apple_protect_pager_setup(protected_object, crypt_info);
+       map_addr = start_aligned;
+       for (map_addr = start_aligned;
+            map_addr < end;
+            map_addr = tmp_entry.vme_end) {
+               vm_map_lock(map);
+               map_locked = TRUE;

 
 

-       /* release extra ref on protected object */
-       vm_object_deallocate(protected_object);
+               /* lookup the protected VM object */
+               if (!vm_map_lookup_entry(map,
+                                        map_addr,
+                                        &map_entry) ||
+                   map_entry->is_sub_map ||
+                   VME_OBJECT(map_entry) == VM_OBJECT_NULL ||
+                   !(map_entry->protection & VM_PROT_EXECUTE)) {
+                       /* that memory is not properly mapped */
+                       kr = KERN_INVALID_ARGUMENT;
+                       goto done;
+               }

 
 

-       if (protected_mem_obj == NULL) {
-               kr = KERN_FAILURE;
-               goto done;
-       }
+               /* get the protected object to be decrypted */
+               protected_object = VME_OBJECT(map_entry);
+               if (protected_object == VM_OBJECT_NULL) {
+                       /* there should be a VM object here at this point */
+                       kr = KERN_INVALID_ARGUMENT;
+                       goto done;
+               }
+               /* ensure protected object stays alive while map is unlocked */
+               vm_object_reference(protected_object);

 
 

-       /* map this memory object in place of the current one */
-       map_addr = start;
-       kr = vm_map_enter_mem_object(map,
-                                    &map_addr,
-                                    end - start,
-                                    (mach_vm_offset_t) 0,
-                                    VM_FLAGS_FIXED | VM_FLAGS_OVERWRITE,
-                                    (ipc_port_t) protected_mem_obj,
-                                    (map_entry->offset +
-                                     (start - map_entry->vme_start)),
-                                    TRUE,
-                                    map_entry->protection,
-                                    map_entry->max_protection,
-                                    map_entry->inheritance);
-       assert(map_addr == start);
-       /*
-        * Release the reference obtained by apple_protect_pager_setup().
-        * The mapping (if it succeeded) is now holding a reference on the
-        * memory object.
-        */
-       memory_object_deallocate(protected_mem_obj);
+               /* limit the map entry to the area we want to cover */
+               vm_map_clip_start(map, map_entry, start_aligned);
+               vm_map_clip_end(map, map_entry, end_aligned);
+
+               tmp_entry = *map_entry;
+               map_entry = VM_MAP_ENTRY_NULL; /* not valid after unlocking map */
+               vm_map_unlock(map);
+               map_locked = FALSE;
+
+               /*
+                * This map entry might be only partially encrypted
+                * (if not fully "page-aligned").
+                */
+               crypto_start = 0;
+               crypto_end = tmp_entry.vme_end - tmp_entry.vme_start;
+               if (tmp_entry.vme_start < start) {
+                       if (tmp_entry.vme_start != start_aligned) {
+                               kr = KERN_INVALID_ADDRESS;
+                       }
+                       crypto_start += (start - tmp_entry.vme_start);
+               }
+               if (tmp_entry.vme_end > end) {
+                       if (tmp_entry.vme_end != end_aligned) {
+                               kr = KERN_INVALID_ADDRESS;
+                       }
+                       crypto_end -= (tmp_entry.vme_end - end);
+               }
+
+               /*
+                * This "extra backing offset" is needed to get the decryption
+                * routine to use the right key.  It adjusts for the possibly
+                * relative offset of an interposed "4K" pager...
+                */
+               if (crypto_backing_offset == (vm_object_offset_t) -1) {
+                       crypto_backing_offset = VME_OFFSET(&tmp_entry);
+               }
+
+               /*
+                * Lookup (and create if necessary) the protected memory object
+                * matching that VM object.
+                * If successful, this also grabs a reference on the memory object,
+                * to guarantee that it doesn't go away before we get a chance to map
+                * it.
+                */
+               unprotected_mem_obj = apple_protect_pager_setup(
+                       protected_object,
+                       VME_OFFSET(&tmp_entry),
+                       crypto_backing_offset,
+                       crypt_info,
+                       crypto_start,
+                       crypto_end);
+
+               /* release extra ref on protected object */
+               vm_object_deallocate(protected_object);
+
+               if (unprotected_mem_obj == NULL) {
+                       kr = KERN_FAILURE;
+                       goto done;
+               }
+
+               vm_flags = VM_FLAGS_FIXED | VM_FLAGS_OVERWRITE;
+
+               /* map this memory object in place of the current one */
+               map_addr = tmp_entry.vme_start;
+               kr = vm_map_enter_mem_object(map,
+                                            &map_addr,
+                                            (tmp_entry.vme_end -
+                                             tmp_entry.vme_start),
+                                            (mach_vm_offset_t) 0,
+                                            vm_flags,
+                                            (ipc_port_t) unprotected_mem_obj,
+                                            0,
+                                            TRUE,
+                                            tmp_entry.protection,
+                                            tmp_entry.max_protection,
+                                            tmp_entry.inheritance);
+               assert(kr == KERN_SUCCESS);
+               assert(map_addr == tmp_entry.vme_start);
+
+#if VM_MAP_DEBUG_APPLE_PROTECT
+               printf("APPLE_PROTECT: map %p [0x%llx:0x%llx] pager %p: "
+                      "backing:[object:%p,offset:0x%llx,"
+                      "crypto_backing_offset:0x%llx,"
+                      "crypto_start:0x%llx,crypto_end:0x%llx]\n",
+                      map,
+                      (uint64_t) map_addr,
+                      (uint64_t) (map_addr + (tmp_entry.vme_end -
+                                              tmp_entry.vme_start)),
+                      unprotected_mem_obj,
+                      protected_object,
+                      VME_OFFSET(&tmp_entry),
+                      crypto_backing_offset,
+                      crypto_start,
+                      crypto_end);
+#endif /* VM_MAP_DEBUG_APPLE_PROTECT */
+                      
+               /*
+                * Release the reference obtained by
+                * apple_protect_pager_setup().
+                * The mapping (if it succeeded) is now holding a reference on
+                * the memory object.
+                */
+               memory_object_deallocate(unprotected_mem_obj);
+               unprotected_mem_obj = MEMORY_OBJECT_NULL;
+
+               /* continue with next map entry */
+               crypto_backing_offset += (tmp_entry.vme_end -
+                                         tmp_entry.vme_start);
+               crypto_backing_offset -= crypto_start;
+       }
+       kr = KERN_SUCCESS;

 
 done:
        if (map_locked) {
 
 done:
        if (map_locked) {

-               vm_map_unlock_read(map);
+               vm_map_unlock(map);

        }
        return kr;
 }
        }
        return kr;
 }


@@ -654,6 +759,10 @@ vm_map_init(
                                 16*1024, PAGE_SIZE, "VM map copies");
        zone_change(vm_map_copy_zone, Z_NOENCRYPT, TRUE);
 
                                 16*1024, PAGE_SIZE, "VM map copies");
        zone_change(vm_map_copy_zone, Z_NOENCRYPT, TRUE);
 

+       vm_map_holes_zone = zinit((vm_map_size_t) sizeof(struct vm_map_links),
+                                16*1024, PAGE_SIZE, "VM map holes");
+       zone_change(vm_map_holes_zone, Z_NOENCRYPT, TRUE);
+

        /*
         *      Cram the map and kentry zones with initial data.
         *      Set reserved_zone non-collectible to aid zone_gc().
        /*
         *      Cram the map and kentry zones with initial data.
         *      Set reserved_zone non-collectible to aid zone_gc().


@@ -668,9 +777,21 @@ vm_map_init(
        zone_change(vm_map_copy_zone, Z_CALLERACCT, FALSE); /* don't charge caller */
        zone_change(vm_map_entry_reserved_zone, Z_GZALLOC_EXEMPT, TRUE);
 
        zone_change(vm_map_copy_zone, Z_CALLERACCT, FALSE); /* don't charge caller */
        zone_change(vm_map_entry_reserved_zone, Z_GZALLOC_EXEMPT, TRUE);
 

+       zone_change(vm_map_holes_zone, Z_COLLECT, TRUE);
+       zone_change(vm_map_holes_zone, Z_EXPAND, TRUE);
+       zone_change(vm_map_holes_zone, Z_FOREIGN, TRUE);
+       zone_change(vm_map_holes_zone, Z_NOCALLOUT, TRUE);
+       zone_change(vm_map_holes_zone, Z_CALLERACCT, TRUE);
+       zone_change(vm_map_holes_zone, Z_GZALLOC_EXEMPT, TRUE);
+
+       /* 
+        * Add the stolen memory to zones, adjust zone size and stolen counts.
+        */

        zcram(vm_map_zone, (vm_offset_t)map_data, map_data_size);
        zcram(vm_map_entry_reserved_zone, (vm_offset_t)kentry_data, kentry_data_size);
        zcram(vm_map_zone, (vm_offset_t)map_data, map_data_size);
        zcram(vm_map_entry_reserved_zone, (vm_offset_t)kentry_data, kentry_data_size);

-       
+       zcram(vm_map_holes_zone, (vm_offset_t)map_holes_data, map_holes_data_size);
+       VM_PAGE_MOVE_STOLEN(atop_64(map_data_size) + atop_64(kentry_data_size) + atop_64(map_holes_data_size));
+

        lck_grp_attr_setdefault(&vm_map_lck_grp_attr);
        lck_grp_init(&vm_map_lck_grp, "vm_map", &vm_map_lck_grp_attr);
        lck_attr_setdefault(&vm_map_lck_attr);  
        lck_grp_attr_setdefault(&vm_map_lck_grp_attr);
        lck_grp_init(&vm_map_lck_grp, "vm_map", &vm_map_lck_grp_attr);
        lck_attr_setdefault(&vm_map_lck_attr);  


@@ -714,10 +835,52 @@ vm_map_steal_memory(
 
        kentry_data_size = kentry_initial_pages * PAGE_SIZE;
        kentry_data = pmap_steal_memory(kentry_data_size);
 
        kentry_data_size = kentry_initial_pages * PAGE_SIZE;
        kentry_data = pmap_steal_memory(kentry_data_size);

+
+       map_holes_data_size = kentry_data_size;
+       map_holes_data = pmap_steal_memory(map_holes_data_size);

 }
 
 }
 

-void vm_kernel_reserved_entry_init(void) {
+void
+vm_kernel_reserved_entry_init(void) {

        zone_prio_refill_configure(vm_map_entry_reserved_zone, (6*PAGE_SIZE)/sizeof(struct vm_map_entry));
        zone_prio_refill_configure(vm_map_entry_reserved_zone, (6*PAGE_SIZE)/sizeof(struct vm_map_entry));

+       zone_prio_refill_configure(vm_map_holes_zone, (6*PAGE_SIZE)/sizeof(struct vm_map_links));
+}
+
+void
+vm_map_disable_hole_optimization(vm_map_t map)
+{
+       vm_map_entry_t  head_entry, hole_entry, next_hole_entry;
+
+       if (map->holelistenabled) {
+
+               head_entry = hole_entry = (vm_map_entry_t) map->holes_list;
+
+               while (hole_entry != NULL) {
+
+                       next_hole_entry = hole_entry->vme_next;
+
+                       hole_entry->vme_next = NULL;
+                       hole_entry->vme_prev = NULL;
+                       zfree(vm_map_holes_zone, hole_entry);
+
+                       if (next_hole_entry == head_entry) {
+                               hole_entry = NULL;
+                       } else {
+                               hole_entry = next_hole_entry;
+                       }
+               }
+
+               map->holes_list = NULL;
+               map->holelistenabled = FALSE;
+
+               map->first_free = vm_map_first_entry(map);
+               SAVE_HINT_HOLE_WRITE(map, NULL);
+       }
+}
+
+boolean_t
+vm_kernel_map_is_kernel(vm_map_t map) {
+       return (map->pmap == kernel_pmap);

 }
 
 /*
 }
 
 /*


@@ -727,6 +890,9 @@ void vm_kernel_reserved_entry_init(void) {
  *     the given physical map structure, and having
  *     the given lower and upper address bounds.
  */
  *     the given physical map structure, and having
  *     the given lower and upper address bounds.
  */

+
+boolean_t vm_map_supports_hole_optimization = TRUE;
+

 vm_map_t
 vm_map_create(
        pmap_t                  pmap,
 vm_map_t
 vm_map_create(
        pmap_t                  pmap,


@@ -736,6 +902,7 @@ vm_map_create(
 {
        static int              color_seed = 0;
        register vm_map_t       result;
 {
        static int              color_seed = 0;
        register vm_map_t       result;

+       struct vm_map_links     *hole_entry = NULL;

 
        result = (vm_map_t) zalloc(vm_map_zone);
        if (result == VM_MAP_NULL)
 
        result = (vm_map_t) zalloc(vm_map_zone);
        if (result == VM_MAP_NULL)


@@ -773,6 +940,21 @@ vm_map_create(
        result->hint = vm_map_to_entry(result);
        result->color_rr = (color_seed++) & vm_color_mask;
        result->jit_entry_exists = FALSE;
        result->hint = vm_map_to_entry(result);
        result->color_rr = (color_seed++) & vm_color_mask;
        result->jit_entry_exists = FALSE;

+
+       if (vm_map_supports_hole_optimization && pmap != kernel_pmap) {
+               hole_entry = zalloc(vm_map_holes_zone);
+
+               hole_entry->start = min;
+               hole_entry->end = (max > (vm_map_offset_t)MACH_VM_MAX_ADDRESS) ? max : (vm_map_offset_t)MACH_VM_MAX_ADDRESS;
+               result->holes_list = result->hole_hint = hole_entry;
+               hole_entry->prev = hole_entry->next = (vm_map_entry_t) hole_entry;
+               result->holelistenabled = TRUE;
+
+       } else {
+
+               result->holelistenabled = FALSE;
+       }
+

 #if CONFIG_FREEZE
        result->default_freezer_handle = NULL;
 #endif
 #if CONFIG_FREEZE
        result->default_freezer_handle = NULL;
 #endif


@@ -970,6 +1152,9 @@ vm_map_destroy(
 {      
        vm_map_lock(map);
 
 {      
        vm_map_lock(map);
 

+       /* final cleanup: no need to unnest shared region */
+       flags |= VM_MAP_REMOVE_NO_UNNESTING;
+

        /* clean up regular map entries */
        (void) vm_map_delete(map, map->min_offset, map->max_offset,
                             flags, VM_MAP_NULL);
        /* clean up regular map entries */
        (void) vm_map_delete(map, map->min_offset, map->max_offset,
                             flags, VM_MAP_NULL);


@@ -983,6 +1168,7 @@ vm_map_destroy(
                map->default_freezer_handle = NULL;
        }
 #endif
                map->default_freezer_handle = NULL;
        }
 #endif

+       vm_map_disable_hole_optimization(map);

        vm_map_unlock(map);
 
        assert(map->hdr.nentries == 0);
        vm_map_unlock(map);
 
        assert(map->hdr.nentries == 0);


@@ -1088,14 +1274,14 @@ void vm_map_swapin (vm_map_t map)
        entry = vm_map_first_entry(map);
 
        while (entry != vm_map_to_entry(map)) {
        entry = vm_map_first_entry(map);
 
        while (entry != vm_map_to_entry(map)) {

-               if (entry->object.vm_object != VM_OBJECT_NULL) {
+               if (VME_OBJECT(entry) != VM_OBJECT_NULL) {

                        if (entry->is_sub_map) {
                        if (entry->is_sub_map) {

-                               vm_map_t lmap = entry->object.sub_map;
+                               vm_map_t lmap = VME_SUBMAP(entry);

                                lck_mtx_lock(&lmap->s_lock);
                                vm_map_res_reference(lmap);
                                lck_mtx_unlock(&lmap->s_lock);
                        } else {
                                lck_mtx_lock(&lmap->s_lock);
                                vm_map_res_reference(lmap);
                                lck_mtx_unlock(&lmap->s_lock);
                        } else {

-                               vm_object_t object = entry->object.vm_object;
+                               vm_object_t object = VME_OBEJCT(entry);

                                vm_object_lock(object);
                                /*
                                 * This call may iterate through the
                                vm_object_lock(object);
                                /*
                                 * This call may iterate through the


@@ -1149,14 +1335,14 @@ void vm_map_swapout(vm_map_t map)
        entry = vm_map_first_entry(map);
 
        while (entry != vm_map_to_entry(map)) {
        entry = vm_map_first_entry(map);
 
        while (entry != vm_map_to_entry(map)) {

-               if (entry->object.vm_object != VM_OBJECT_NULL) {
+               if (VME_OBJECT(entry) != VM_OBJECT_NULL) {

                        if (entry->is_sub_map) {
                        if (entry->is_sub_map) {

-                               vm_map_t lmap = entry->object.sub_map;
+                               vm_map_t lmap = VME_SUBMAP(entry);

                                lck_mtx_lock(&lmap->s_lock);
                                vm_map_res_deallocate(lmap);
                                lck_mtx_unlock(&lmap->s_lock);
                        } else {
                                lck_mtx_lock(&lmap->s_lock);
                                vm_map_res_deallocate(lmap);
                                lck_mtx_unlock(&lmap->s_lock);
                        } else {

-                               vm_object_t object = entry->object.vm_object;
+                               vm_object_t object = VME_OBJECT(entry);

                                vm_object_lock(object);
                                /*
                                 * This call may take a long time, 
                                vm_object_lock(object);
                                /*
                                 * This call may take a long time, 


@@ -1217,9 +1403,10 @@ vm_map_find_space(
        int                     flags,
        vm_map_entry_t          *o_entry)       /* OUT */
 {
        int                     flags,
        vm_map_entry_t          *o_entry)       /* OUT */
 {

-       register vm_map_entry_t entry, new_entry;
+       vm_map_entry_t                  entry, new_entry;

        register vm_map_offset_t        start;
        register vm_map_offset_t        end;
        register vm_map_offset_t        start;
        register vm_map_offset_t        end;

+       vm_map_entry_t                  hole_entry;

 
        if (size == 0) {
                *address = 0;
 
        if (size == 0) {
                *address = 0;


@@ -1243,11 +1430,27 @@ vm_map_find_space(
        if( map->disable_vmentry_reuse == TRUE) {
                VM_MAP_HIGHEST_ENTRY(map, entry, start);
        } else {
        if( map->disable_vmentry_reuse == TRUE) {
                VM_MAP_HIGHEST_ENTRY(map, entry, start);
        } else {

-               assert(first_free_is_valid(map));
-               if ((entry = map->first_free) == vm_map_to_entry(map))
-                       start = map->min_offset;
-               else
-                       start = entry->vme_end;
+               if (map->holelistenabled) {
+                       hole_entry = (vm_map_entry_t)map->holes_list;
+
+                       if (hole_entry == NULL) {
+                               /*
+                                * No more space in the map?
+                                */
+                               vm_map_entry_dispose(map, new_entry);
+                               vm_map_unlock(map);
+                               return(KERN_NO_SPACE);
+                       }
+
+                       entry = hole_entry;
+                       start = entry->vme_start;
+               } else {
+                       assert(first_free_is_valid(map));
+                       if ((entry = map->first_free) == vm_map_to_entry(map))
+                               start = map->min_offset;
+                       else
+                               start = entry->vme_end;
+               }

        }
 
        /*
        }
 
        /*


@@ -1284,28 +1487,53 @@ vm_map_find_space(
                        return(KERN_NO_SPACE);
                }
 
                        return(KERN_NO_SPACE);
                }
 

-               /*
-                *      If there are no more entries, we must win.
-                */
-

                next = entry->vme_next;
                next = entry->vme_next;

-               if (next == vm_map_to_entry(map))
-                       break;

 
 

-               /*
-                *      If there is another entry, it must be
-                *      after the end of the potential new region.
-                */
+               if (map->holelistenabled) {
+                       if (entry->vme_end >= end)
+                               break;
+               } else {
+                       /*
+                        *      If there are no more entries, we must win.
+                        *
+                        *      OR
+                        *
+                        *      If there is another entry, it must be
+                        *      after the end of the potential new region.
+                        */

 
 

-               if (next->vme_start >= end)
-                       break;
+                       if (next == vm_map_to_entry(map))
+                               break;
+
+                       if (next->vme_start >= end)
+                               break;
+               }

 
                /*
                 *      Didn't fit -- move to the next entry.
                 */
 
                entry = next;
 
                /*
                 *      Didn't fit -- move to the next entry.
                 */
 
                entry = next;

-               start = entry->vme_end;
+
+               if (map->holelistenabled) {
+                       if (entry == (vm_map_entry_t) map->holes_list) {
+                               /*
+                                * Wrapped around
+                                */
+                               vm_map_entry_dispose(map, new_entry);
+                               vm_map_unlock(map);
+                               return(KERN_NO_SPACE);
+                       }
+                       start = entry->vme_start;
+               } else {
+                       start = entry->vme_end;
+               }
+       }
+
+       if (map->holelistenabled) {
+               if (vm_map_lookup_entry(map, entry->vme_start, &entry)) {
+                       panic("Found an existing entry (%p) instead of potential hole at address: 0x%llx.\n", entry, (unsigned long long)entry->vme_start);
+               }

        }
 
        /*
        }
 
        /*


@@ -1337,8 +1565,8 @@ vm_map_find_space(
        new_entry->is_shared = FALSE;
        new_entry->is_sub_map = FALSE;
        new_entry->use_pmap = TRUE;
        new_entry->is_shared = FALSE;
        new_entry->is_sub_map = FALSE;
        new_entry->use_pmap = TRUE;

-       new_entry->object.vm_object = VM_OBJECT_NULL;
-       new_entry->offset = (vm_object_offset_t) 0;
+       VME_OBJECT_SET(new_entry, VM_OBJECT_NULL);
+       VME_OFFSET_SET(new_entry, (vm_object_offset_t) 0);

 
        new_entry->needs_copy = FALSE;
 
 
        new_entry->needs_copy = FALSE;
 


@@ -1360,13 +1588,15 @@ vm_map_find_space(
                new_entry->map_aligned = FALSE;
        }
 
                new_entry->map_aligned = FALSE;
        }
 

-       new_entry->used_for_jit = 0;
-
-       new_entry->alias = 0;
+       new_entry->used_for_jit = FALSE;

        new_entry->zero_wired_pages = FALSE;
        new_entry->iokit_acct = FALSE;
        new_entry->zero_wired_pages = FALSE;
        new_entry->iokit_acct = FALSE;

+       new_entry->vme_resilient_codesign = FALSE;
+       new_entry->vme_resilient_media = FALSE;

 
 

-       VM_GET_FLAGS_ALIAS(flags, new_entry->alias);
+       int alias;
+       VM_GET_FLAGS_ALIAS(flags, alias);
+       VME_ALIAS_SET(new_entry, alias);

 
        /*
         *      Insert the new entry into the list
 
        /*
         *      Insert the new entry into the list


@@ -1601,11 +1831,14 @@ vm_map_enter(
        boolean_t               permanent = ((flags & VM_FLAGS_PERMANENT) != 0);
        boolean_t               entry_for_jit = ((flags & VM_FLAGS_MAP_JIT) != 0);
        boolean_t               iokit_acct = ((flags & VM_FLAGS_IOKIT_ACCT) != 0);
        boolean_t               permanent = ((flags & VM_FLAGS_PERMANENT) != 0);
        boolean_t               entry_for_jit = ((flags & VM_FLAGS_MAP_JIT) != 0);
        boolean_t               iokit_acct = ((flags & VM_FLAGS_IOKIT_ACCT) != 0);

+       boolean_t               resilient_codesign = ((flags & VM_FLAGS_RESILIENT_CODESIGN) != 0);
+       boolean_t               resilient_media = ((flags & VM_FLAGS_RESILIENT_MEDIA) != 0);

        unsigned int            superpage_size = ((flags & VM_FLAGS_SUPERPAGE_MASK) >> VM_FLAGS_SUPERPAGE_SHIFT);
        unsigned int            superpage_size = ((flags & VM_FLAGS_SUPERPAGE_MASK) >> VM_FLAGS_SUPERPAGE_SHIFT);

-       char                    alias;
+       vm_tag_t                alias, user_alias;

        vm_map_offset_t         effective_min_offset, effective_max_offset;
        kern_return_t           kr;
        boolean_t               clear_map_aligned = FALSE;
        vm_map_offset_t         effective_min_offset, effective_max_offset;
        kern_return_t           kr;
        boolean_t               clear_map_aligned = FALSE;

+       vm_map_entry_t          hole_entry;

 
        if (superpage_size) {
                switch (superpage_size) {
 
        if (superpage_size) {
                switch (superpage_size) {


@@ -1634,6 +1867,13 @@ vm_map_enter(
 
 
 
 
 
 

+       if (resilient_codesign || resilient_media) {
+               if ((cur_protection & (VM_PROT_WRITE | VM_PROT_EXECUTE)) ||
+                   (max_protection & (VM_PROT_WRITE | VM_PROT_EXECUTE))) {
+                       return KERN_PROTECTION_FAILURE;
+               }
+       }
+

        if (is_submap) {
                if (purgable) {
                        /* submaps can not be purgeable */
        if (is_submap) {
                if (purgable) {
                        /* submaps can not be purgeable */


@@ -1679,6 +1919,11 @@ vm_map_enter(
        }
 
        VM_GET_FLAGS_ALIAS(flags, alias);
        }
 
        VM_GET_FLAGS_ALIAS(flags, alias);

+       if (map->pmap == kernel_pmap) {
+               user_alias = VM_KERN_MEMORY_NONE;
+       } else {
+               user_alias = alias;
+       }

 
 #define        RETURN(value)   { result = value; goto BailOut; }
 
 
 #define        RETURN(value)   { result = value; goto BailOut; }
 


@@ -1736,6 +1981,7 @@ vm_map_enter(
                                            *address + size,
                                            map->hdr.entries_pageable);
                vm_map_set_page_shift(zap_old_map, VM_MAP_PAGE_SHIFT(map));
                                            *address + size,
                                            map->hdr.entries_pageable);
                vm_map_set_page_shift(zap_old_map, VM_MAP_PAGE_SHIFT(map));

+               vm_map_disable_hole_optimization(zap_old_map);

        }
 
 StartAgain: ;
        }
 
 StartAgain: ;


@@ -1780,41 +2026,82 @@ StartAgain: ;
                if( map->disable_vmentry_reuse == TRUE) {
                        VM_MAP_HIGHEST_ENTRY(map, entry, start);
                } else {
                if( map->disable_vmentry_reuse == TRUE) {
                        VM_MAP_HIGHEST_ENTRY(map, entry, start);
                } else {

-                       assert(first_free_is_valid(map));

 
 

-                       entry = map->first_free;
+                       if (map->holelistenabled) {
+                               hole_entry = (vm_map_entry_t)map->holes_list;

 
 

-                       if (entry == vm_map_to_entry(map)) {
-                               entry = NULL;
+                               if (hole_entry == NULL) {
+                                       /*
+                                        * No more space in the map?
+                                        */
+                                       result = KERN_NO_SPACE;
+                                       goto BailOut;
+                               } else {
+
+                                       boolean_t found_hole = FALSE;
+
+                                       do {
+                                               if (hole_entry->vme_start >= start) {
+                                                       start = hole_entry->vme_start;
+                                                       found_hole = TRUE;
+                                                       break;
+                                               }
+
+                                               if (hole_entry->vme_end > start) {
+                                                       found_hole = TRUE;
+                                                       break;
+                                               }
+                                               hole_entry = hole_entry->vme_next;
+
+                                       } while (hole_entry != (vm_map_entry_t) map->holes_list);
+
+                                       if (found_hole == FALSE) {
+                                               result = KERN_NO_SPACE;
+                                               goto BailOut;
+                                       }
+
+                                       entry = hole_entry;
+
+                                       if (start == 0)
+                                               start += PAGE_SIZE_64;
+                               }

                        } else {
                        } else {

-                              if (entry->vme_next == vm_map_to_entry(map)){
-                                      /*
-                                       * Hole at the end of the map.
-                                       */
+                               assert(first_free_is_valid(map));
+
+                               entry = map->first_free;
+
+                               if (entry == vm_map_to_entry(map)) {

                                        entry = NULL;
                                        entry = NULL;

-                              } else {
-                                       if (start < (entry->vme_next)->vme_start ) {
-                                               start = entry->vme_end;
+                               } else {
+                                      if (entry->vme_next == vm_map_to_entry(map)){
+                                              /*
+                                               * Hole at the end of the map.
+                                               */
+                                               entry = NULL;
+                                      } else {
+                                               if (start < (entry->vme_next)->vme_start ) {
+                                                       start = entry->vme_end;
+                                                       start = vm_map_round_page(start,
+                                                                                 VM_MAP_PAGE_MASK(map));
+                                               } else {
+                                                       /*
+                                                        * Need to do a lookup.
+                                                        */
+                                                       entry = NULL;
+                                               }
+                                      }
+                               }
+
+                               if (entry == NULL) {
+                                       vm_map_entry_t  tmp_entry;
+                                       if (vm_map_lookup_entry(map, start, &tmp_entry)) {
+                                               assert(!entry_for_jit);
+                                               start = tmp_entry->vme_end;

                                                start = vm_map_round_page(start,
                                                                          VM_MAP_PAGE_MASK(map));
                                                start = vm_map_round_page(start,
                                                                          VM_MAP_PAGE_MASK(map));

-                                       } else {
-                                               /*
-                                                * Need to do a lookup.
-                                                */
-                                               entry = NULL;

                                        }
                                        }

-                              }
-                       }
-
-                       if (entry == NULL) {
-                               vm_map_entry_t  tmp_entry;
-                               if (vm_map_lookup_entry(map, start, &tmp_entry)) {
-                                       assert(!entry_for_jit);
-                                       start = tmp_entry->vme_end;
-                                       start = vm_map_round_page(start,
-                                                                 VM_MAP_PAGE_MASK(map));
+                                       entry = tmp_entry;

                                }
                                }

-                               entry = tmp_entry;

                        }
                }
 
                        }
                }
 


@@ -1859,31 +2146,57 @@ StartAgain: ;
                                RETURN(KERN_NO_SPACE);
                        }
 
                                RETURN(KERN_NO_SPACE);
                        }
 

-                       /*
-                        *      If there are no more entries, we must win.
-                        */
-

                        next = entry->vme_next;
                        next = entry->vme_next;

-                       if (next == vm_map_to_entry(map))
-                               break;

 
 

-                       /*
-                        *      If there is another entry, it must be
-                        *      after the end of the potential new region.
-                        */
+                       if (map->holelistenabled) {
+                               if (entry->vme_end >= end)
+                                       break;
+                       } else {
+                               /*
+                                *      If there are no more entries, we must win.
+                                *
+                                *      OR
+                                *
+                                *      If there is another entry, it must be
+                                *      after the end of the potential new region.
+                                */

 
 

-                       if (next->vme_start >= end)
-                               break;
+                               if (next == vm_map_to_entry(map))
+                                       break;
+
+                               if (next->vme_start >= end)
+                                       break;
+                       }

 
                        /*
                         *      Didn't fit -- move to the next entry.
                         */
 
                        entry = next;
 
                        /*
                         *      Didn't fit -- move to the next entry.
                         */
 
                        entry = next;

-                       start = entry->vme_end;
+
+                       if (map->holelistenabled) {
+                               if (entry == (vm_map_entry_t) map->holes_list) {
+                                       /*
+                                        * Wrapped around
+                                        */
+                                       result = KERN_NO_SPACE;
+                                       goto BailOut;
+                               }
+                               start = entry->vme_start;
+                       } else {
+                               start = entry->vme_end;
+                       }
+

                        start = vm_map_round_page(start,
                                                  VM_MAP_PAGE_MASK(map));
                }
                        start = vm_map_round_page(start,
                                                  VM_MAP_PAGE_MASK(map));
                }

+
+               if (map->holelistenabled) {
+                       if (vm_map_lookup_entry(map, entry->vme_start, &entry)) {
+                               panic("Found an existing entry (%p) instead of potential hole at address: 0x%llx.\n", entry, (unsigned long long)entry->vme_start);
+                       }
+               }
+

                *address = start;
                assert(VM_MAP_PAGE_ALIGNED(*address,
                                           VM_MAP_PAGE_MASK(map)));
                *address = start;
                assert(VM_MAP_PAGE_ALIGNED(*address,
                                           VM_MAP_PAGE_MASK(map)));


@@ -1950,13 +2263,13 @@ StartAgain: ;
                                if (entry == vm_map_to_entry(map) ||
                                    entry->vme_start != tmp_start ||
                                    entry->is_sub_map != is_submap ||
                                if (entry == vm_map_to_entry(map) ||
                                    entry->vme_start != tmp_start ||
                                    entry->is_sub_map != is_submap ||

-                                   entry->offset != tmp_offset ||
+                                   VME_OFFSET(entry) != tmp_offset ||

                                    entry->needs_copy != needs_copy ||
                                    entry->protection != cur_protection ||
                                    entry->max_protection != max_protection ||
                                    entry->inheritance != inheritance ||
                                    entry->iokit_acct != iokit_acct ||
                                    entry->needs_copy != needs_copy ||
                                    entry->protection != cur_protection ||
                                    entry->max_protection != max_protection ||
                                    entry->inheritance != inheritance ||
                                    entry->iokit_acct != iokit_acct ||

-                                   entry->alias != alias) {
+                                   VME_ALIAS(entry) != alias) {

                                        /* not the same mapping ! */
                                        RETURN(KERN_NO_SPACE);
                                }
                                        /* not the same mapping ! */
                                        RETURN(KERN_NO_SPACE);
                                }


@@ -1964,17 +2277,17 @@ StartAgain: ;
                                 * Check if the same object is being mapped.
                                 */
                                if (is_submap) {
                                 * Check if the same object is being mapped.
                                 */
                                if (is_submap) {

-                                       if (entry->object.sub_map !=
+                                       if (VME_SUBMAP(entry) !=

                                            (vm_map_t) object) {
                                                /* not the same submap */
                                                RETURN(KERN_NO_SPACE);
                                        }
                                } else {
                                            (vm_map_t) object) {
                                                /* not the same submap */
                                                RETURN(KERN_NO_SPACE);
                                        }
                                } else {

-                                       if (entry->object.vm_object != object) {
+                                       if (VME_OBJECT(entry) != object) {

                                                /* not the same VM object... */
                                                vm_object_t obj2;
 
                                                /* not the same VM object... */
                                                vm_object_t obj2;
 

-                                               obj2 = entry->object.vm_object;
+                                               obj2 = VME_OBJECT(entry);

                                                if ((obj2 == VM_OBJECT_NULL ||
                                                     obj2->internal) &&
                                                    (object == VM_OBJECT_NULL ||
                                                if ((obj2 == VM_OBJECT_NULL ||
                                                     obj2->internal) &&
                                                    (object == VM_OBJECT_NULL ||


@@ -2032,6 +2345,7 @@ StartAgain: ;
 
        if (purgable || entry_for_jit) {
                if (object == VM_OBJECT_NULL) {
 
        if (purgable || entry_for_jit) {
                if (object == VM_OBJECT_NULL) {

+

                        object = vm_object_allocate(size);
                        object->copy_strategy = MEMORY_OBJECT_COPY_NONE;
                        object->true_share = TRUE;
                        object = vm_object_allocate(size);
                        object->copy_strategy = MEMORY_OBJECT_COPY_NONE;
                        object->true_share = TRUE;


@@ -2073,7 +2387,8 @@ StartAgain: ;
                   (entry->protection == cur_protection) &&
                   (entry->max_protection == max_protection) &&
                   (entry->inheritance == inheritance) &&
                   (entry->protection == cur_protection) &&
                   (entry->max_protection == max_protection) &&
                   (entry->inheritance == inheritance) &&

-                  ((alias == VM_MEMORY_REALLOC) || (entry->alias == alias)) &&
+                  ((user_alias == VM_MEMORY_REALLOC) ||
+                   (VME_ALIAS(entry) == alias)) &&

                   (entry->no_cache == no_cache) &&
                   (entry->permanent == permanent) &&
                   (!entry->superpage_size && !superpage_size) &&
                   (entry->no_cache == no_cache) &&
                   (entry->permanent == permanent) &&
                   (!entry->superpage_size && !superpage_size) &&


@@ -2085,16 +2400,18 @@ StartAgain: ;
                   (!entry->zero_wired_pages) &&
                   (!entry->used_for_jit && !entry_for_jit) &&
                   (entry->iokit_acct == iokit_acct) &&
                   (!entry->zero_wired_pages) &&
                   (!entry->used_for_jit && !entry_for_jit) &&
                   (entry->iokit_acct == iokit_acct) &&

+                  (!entry->vme_resilient_codesign) &&
+                  (!entry->vme_resilient_media) &&

 
                   ((entry->vme_end - entry->vme_start) + size <=
 
                   ((entry->vme_end - entry->vme_start) + size <=

-                   (alias == VM_MEMORY_REALLOC ?
+                   (user_alias == VM_MEMORY_REALLOC ?

                     ANON_CHUNK_SIZE :
                     NO_COALESCE_LIMIT)) &&
 
                   (entry->wired_count == 0)) { /* implies user_wired_count == 0 */
                     ANON_CHUNK_SIZE :
                     NO_COALESCE_LIMIT)) &&
 
                   (entry->wired_count == 0)) { /* implies user_wired_count == 0 */

-               if (vm_object_coalesce(entry->object.vm_object,
+               if (vm_object_coalesce(VME_OBJECT(entry),

                                       VM_OBJECT_NULL,
                                       VM_OBJECT_NULL,

-                                      entry->offset,
+                                      VME_OFFSET(entry),

                                       (vm_object_offset_t) 0,
                                       (vm_map_size_t)(entry->vme_end - entry->vme_start),
                                       (vm_map_size_t)(end - entry->vme_end))) {
                                       (vm_object_offset_t) 0,
                                       (vm_map_size_t)(entry->vme_end - entry->vme_start),
                                       (vm_map_size_t)(end - entry->vme_end))) {


@@ -2108,8 +2425,14 @@ StartAgain: ;
                        assert(entry->vme_start < end);
                        assert(VM_MAP_PAGE_ALIGNED(end,
                                                   VM_MAP_PAGE_MASK(map)));
                        assert(entry->vme_start < end);
                        assert(VM_MAP_PAGE_ALIGNED(end,
                                                   VM_MAP_PAGE_MASK(map)));

+                       if (__improbable(vm_debug_events))
+                               DTRACE_VM5(map_entry_extend, vm_map_t, map, vm_map_entry_t, entry, vm_address_t, entry->vme_start, vm_address_t, entry->vme_end, vm_address_t, end);

                        entry->vme_end = end;
                        entry->vme_end = end;

-                       vm_map_store_update_first_free(map, map->first_free);
+                       if (map->holelistenabled) {
+                               vm_map_store_update_first_free(map, entry, TRUE);
+                       } else {
+                               vm_map_store_update_first_free(map, map->first_free, TRUE);
+                       }

                        new_mapping_established = TRUE;
                        RETURN(KERN_SUCCESS);
                }
                        new_mapping_established = TRUE;
                        RETURN(KERN_SUCCESS);
                }


@@ -2155,7 +2478,10 @@ StartAgain: ;
                                                        superpage_size,
                                                        clear_map_aligned,
                                                        is_submap);
                                                        superpage_size,
                                                        clear_map_aligned,
                                                        is_submap);

-                       new_entry->alias = alias;
+
+                       assert((object != kernel_object) || (VM_KERN_MEMORY_NONE != alias));
+                       VME_ALIAS_SET(new_entry, alias);
+

                        if (entry_for_jit){
                                if (!(map->jit_entry_exists)){
                                        new_entry->used_for_jit = TRUE;
                        if (entry_for_jit){
                                if (!(map->jit_entry_exists)){
                                        new_entry->used_for_jit = TRUE;


@@ -2163,6 +2489,18 @@ StartAgain: ;
                                }
                        }
 
                                }
                        }
 

+                       if (resilient_codesign &&
+                           ! ((cur_protection | max_protection) &
+                              (VM_PROT_WRITE | VM_PROT_EXECUTE))) {
+                               new_entry->vme_resilient_codesign = TRUE;
+                       }
+
+                       if (resilient_media &&
+                           ! ((cur_protection | max_protection) &
+                              (VM_PROT_WRITE | VM_PROT_EXECUTE))) {
+                               new_entry->vme_resilient_media = TRUE;
+                       }
+

                        assert(!new_entry->iokit_acct);
                        if (!is_submap &&
                            object != VM_OBJECT_NULL &&
                        assert(!new_entry->iokit_acct);
                        if (!is_submap &&
                            object != VM_OBJECT_NULL &&


@@ -2176,12 +2514,20 @@ StartAgain: ;
                                 */
                                new_entry->use_pmap = FALSE;
                        } else if (!is_submap &&
                                 */
                                new_entry->use_pmap = FALSE;
                        } else if (!is_submap &&

-                                  iokit_acct) {
+                                  iokit_acct &&
+                                  object != VM_OBJECT_NULL &&
+                                  object->internal) {

                                /* alternate accounting */
                                assert(!new_entry->iokit_acct);
                                assert(new_entry->use_pmap);
                                new_entry->iokit_acct = TRUE;
                                new_entry->use_pmap = FALSE;
                                /* alternate accounting */
                                assert(!new_entry->iokit_acct);
                                assert(new_entry->use_pmap);
                                new_entry->iokit_acct = TRUE;
                                new_entry->use_pmap = FALSE;

+                               DTRACE_VM4(
+                                       vm_map_iokit_mapped_region,
+                                       vm_map_t, map,
+                                       vm_map_offset_t, new_entry->vme_start,
+                                       vm_map_offset_t, new_entry->vme_end,
+                                       int, VME_ALIAS(new_entry));

                                vm_map_iokit_mapped_region(
                                        map,
                                        (new_entry->vme_end -
                                vm_map_iokit_mapped_region(
                                        map,
                                        (new_entry->vme_end -


@@ -2201,7 +2547,7 @@ StartAgain: ;
                                assert(!new_entry->iokit_acct);
                                submap = (vm_map_t) object;
                                submap_is_64bit = vm_map_is_64bit(submap);
                                assert(!new_entry->iokit_acct);
                                submap = (vm_map_t) object;
                                submap_is_64bit = vm_map_is_64bit(submap);

-                               use_pmap = (alias == VM_MEMORY_SHARED_PMAP);
+                               use_pmap = (user_alias == VM_MEMORY_SHARED_PMAP);

 #ifndef NO_NESTED_PMAP
                                if (use_pmap && submap->pmap == NULL) {
                                        ledger_t ledger = map->pmap->ledger;
 #ifndef NO_NESTED_PMAP
                                if (use_pmap && submap->pmap == NULL) {
                                        ledger_t ledger = map->pmap->ledger;


@@ -2239,12 +2585,15 @@ StartAgain: ;
                                vm_page_t pages, m;
                                vm_object_t sp_object;
 
                                vm_page_t pages, m;
                                vm_object_t sp_object;
 

-                               entry->offset = 0;
+                               VME_OFFSET_SET(entry, 0);

 
                                /* allocate one superpage */
                                kr = cpm_allocate(SUPERPAGE_SIZE, &pages, 0, SUPERPAGE_NBASEPAGES-1, TRUE, 0);
                                if (kr != KERN_SUCCESS) {
 
                                /* allocate one superpage */
                                kr = cpm_allocate(SUPERPAGE_SIZE, &pages, 0, SUPERPAGE_NBASEPAGES-1, TRUE, 0);
                                if (kr != KERN_SUCCESS) {

-                                       new_mapping_established = TRUE; /* will cause deallocation of whole range */
+                                       /* deallocate whole range... */
+                                       new_mapping_established = TRUE;
+                                       /* ... but only up to "tmp_end" */
+                                       size -= end - tmp_end;

                                        RETURN(kr);
                                }
 
                                        RETURN(kr);
                                }
 


@@ -2252,7 +2601,7 @@ StartAgain: ;
                                sp_object = vm_object_allocate((vm_map_size_t)(entry->vme_end - entry->vme_start));
                                sp_object->phys_contiguous = TRUE;
                                sp_object->vo_shadow_offset = (vm_object_offset_t)pages->phys_page*PAGE_SIZE;
                                sp_object = vm_object_allocate((vm_map_size_t)(entry->vme_end - entry->vme_start));
                                sp_object->phys_contiguous = TRUE;
                                sp_object->vo_shadow_offset = (vm_object_offset_t)pages->phys_page*PAGE_SIZE;

-                               entry->object.vm_object = sp_object;
+                               VME_OBJECT_SET(entry, sp_object);

                                assert(entry->use_pmap);
 
                                /* enter the base pages into the object */
                                assert(entry->use_pmap);
 
                                /* enter the base pages into the object */


@@ -2262,7 +2611,7 @@ StartAgain: ;
                                        pmap_zero_page(m->phys_page);
                                        pages = NEXT_PAGE(m);
                                        *(NEXT_PAGE_PTR(m)) = VM_PAGE_NULL;
                                        pmap_zero_page(m->phys_page);
                                        pages = NEXT_PAGE(m);
                                        *(NEXT_PAGE_PTR(m)) = VM_PAGE_NULL;

-                                       vm_page_insert(m, sp_object, offset);
+                                       vm_page_insert_wired(m, sp_object, offset, VM_KERN_MEMORY_OSFMK);

                                }
                                vm_object_unlock(sp_object);
                        }
                                }
                                vm_object_unlock(sp_object);
                        }


@@ -2350,7 +2699,8 @@ BailOut:
                        assert(!keep_map_locked);
                        pmap_empty = FALSE; /* pmap won't be empty */
                        kr = vm_map_wire(map, start, end,
                        assert(!keep_map_locked);
                        pmap_empty = FALSE; /* pmap won't be empty */
                        kr = vm_map_wire(map, start, end,

-                                            new_entry->protection, TRUE);
+                                            new_entry->protection | VM_PROT_MEMORY_TAG_MAKE(VM_KERN_MEMORY_MLOCK),
+                                            TRUE);

                        result = kr;
                }
 
                        result = kr;
                }
 


@@ -2370,6 +2720,8 @@ BailOut:
                                                    map->hdr.entries_pageable);
                        vm_map_set_page_shift(zap_new_map,
                                              VM_MAP_PAGE_SHIFT(map));
                                                    map->hdr.entries_pageable);
                        vm_map_set_page_shift(zap_new_map,
                                              VM_MAP_PAGE_SHIFT(map));

+                       vm_map_disable_hole_optimization(zap_new_map);
+

                        if (!map_locked) {
                                vm_map_lock(map);
                                map_locked = TRUE;
                        if (!map_locked) {
                                vm_map_lock(map);
                                map_locked = TRUE;


@@ -2461,6 +2813,7 @@ BailOut:
 #undef RETURN
 }
 
 #undef RETURN
 }
 

+

 /*
  * Counters for the prefault optimization.
  */
 /*
  * Counters for the prefault optimization.
  */


@@ -2490,7 +2843,7 @@ vm_map_enter_mem_object_helper(
        kern_return_t           result;
        boolean_t               mask_cur_protection, mask_max_protection;
        boolean_t               try_prefault = (page_list_count != 0);
        kern_return_t           result;
        boolean_t               mask_cur_protection, mask_max_protection;
        boolean_t               try_prefault = (page_list_count != 0);

-       vm_map_offset_t         offset_in_mapping;
+       vm_map_offset_t         offset_in_mapping = 0;

 
        mask_cur_protection = cur_protection & VM_PROT_IS_MASK;
        mask_max_protection = max_protection & VM_PROT_IS_MASK;
 
        mask_cur_protection = cur_protection & VM_PROT_IS_MASK;
        mask_max_protection = max_protection & VM_PROT_IS_MASK;


@@ -2505,13 +2858,16 @@ vm_map_enter_mem_object_helper(
            (max_protection & ~VM_PROT_ALL) ||
            (inheritance > VM_INHERIT_LAST_VALID) ||
            (try_prefault && (copy || !page_list)) ||
            (max_protection & ~VM_PROT_ALL) ||
            (inheritance > VM_INHERIT_LAST_VALID) ||
            (try_prefault && (copy || !page_list)) ||

-           initial_size == 0)
+           initial_size == 0) {

                return KERN_INVALID_ARGUMENT;
                return KERN_INVALID_ARGUMENT;

+       }

        
        

-       map_addr = vm_map_trunc_page(*address,
-                                    VM_MAP_PAGE_MASK(target_map));
-       map_size = vm_map_round_page(initial_size,
-                                    VM_MAP_PAGE_MASK(target_map));
+       {
+               map_addr = vm_map_trunc_page(*address,
+                                            VM_MAP_PAGE_MASK(target_map));
+               map_size = vm_map_round_page(initial_size,
+                                            VM_MAP_PAGE_MASK(target_map));
+       }

        size = vm_object_round_page(initial_size);
 
        /*
        size = vm_object_round_page(initial_size);
 
        /*


@@ -2526,7 +2882,8 @@ vm_map_enter_mem_object_helper(
 
                named_entry = (vm_named_entry_t) port->ip_kobject;
 
 
                named_entry = (vm_named_entry_t) port->ip_kobject;
 

-               if ((flags & VM_FLAGS_RETURN_DATA_ADDR) != 0) {
+               if (flags & (VM_FLAGS_RETURN_DATA_ADDR |
+                            VM_FLAGS_RETURN_4K_DATA_ADDR)) {

                        offset += named_entry->data_offset;
                }
                
                        offset += named_entry->data_offset;
                }
                


@@ -2552,8 +2909,9 @@ vm_map_enter_mem_object_helper(
                        /* overflow */
                        return KERN_INVALID_ARGUMENT;
                }
                        /* overflow */
                        return KERN_INVALID_ARGUMENT;
                }

-               if (named_entry->size < (offset + size))
+               if (named_entry->size < (offset + initial_size)) {

                        return KERN_INVALID_ARGUMENT;
                        return KERN_INVALID_ARGUMENT;

+               }

 
                if (named_entry->is_copy) {
                        /* for a vm_map_copy, we can only map it whole */
 
                if (named_entry->is_copy) {
                        /* for a vm_map_copy, we can only map it whole */


@@ -2598,7 +2956,8 @@ vm_map_enter_mem_object_helper(
                if (named_entry->is_sub_map) {
                        vm_map_t                submap;
 
                if (named_entry->is_sub_map) {
                        vm_map_t                submap;
 

-                       if ((flags & VM_FLAGS_RETURN_DATA_ADDR) != 0) {
+                       if (flags & (VM_FLAGS_RETURN_DATA_ADDR |
+                                    VM_FLAGS_RETURN_4K_DATA_ADDR)) {

                                panic("VM_FLAGS_RETURN_DATA_ADDR not expected for submap.");
                        }
 
                                panic("VM_FLAGS_RETURN_DATA_ADDR not expected for submap.");
                        }
 


@@ -2656,7 +3015,8 @@ vm_map_enter_mem_object_helper(
                        protections = named_entry->protection & VM_PROT_ALL;
                        access = GET_MAP_MEM(named_entry->protection);
 
                        protections = named_entry->protection & VM_PROT_ALL;
                        access = GET_MAP_MEM(named_entry->protection);
 

-                       if ((flags & VM_FLAGS_RETURN_DATA_ADDR) != 0) {
+                       if (flags & (VM_FLAGS_RETURN_DATA_ADDR|
+                                    VM_FLAGS_RETURN_4K_DATA_ADDR)) {

                                panic("VM_FLAGS_RETURN_DATA_ADDR not expected for submap.");
                        }
 
                                panic("VM_FLAGS_RETURN_DATA_ADDR not expected for submap.");
                        }
 


@@ -2738,13 +3098,17 @@ vm_map_enter_mem_object_helper(
                        if (flags & ~(VM_FLAGS_FIXED |
                                      VM_FLAGS_ANYWHERE |
                                      VM_FLAGS_OVERWRITE |
                        if (flags & ~(VM_FLAGS_FIXED |
                                      VM_FLAGS_ANYWHERE |
                                      VM_FLAGS_OVERWRITE |

+                                     VM_FLAGS_RETURN_4K_DATA_ADDR |

                                      VM_FLAGS_RETURN_DATA_ADDR)) {
                                named_entry_unlock(named_entry);
                                return KERN_INVALID_ARGUMENT;
                        }
 
                                      VM_FLAGS_RETURN_DATA_ADDR)) {
                                named_entry_unlock(named_entry);
                                return KERN_INVALID_ARGUMENT;
                        }
 

-                       if ((flags & VM_FLAGS_RETURN_DATA_ADDR) != 0) {
+                       if (flags & (VM_FLAGS_RETURN_DATA_ADDR |
+                                    VM_FLAGS_RETURN_4K_DATA_ADDR)) {

                                offset_in_mapping = offset - vm_object_trunc_page(offset);
                                offset_in_mapping = offset - vm_object_trunc_page(offset);

+                               if (flags & VM_FLAGS_RETURN_4K_DATA_ADDR)
+                                       offset_in_mapping &= ~((signed)(0xFFF));

                                offset = vm_object_trunc_page(offset);
                                map_size = vm_object_round_page(offset + offset_in_mapping + initial_size) - offset;
                        }
                                offset = vm_object_trunc_page(offset);
                                map_size = vm_object_round_page(offset + offset_in_mapping + initial_size) - offset;
                        }


@@ -2769,6 +3133,7 @@ vm_map_enter_mem_object_helper(
                                          mask,
                                          flags & (VM_FLAGS_ANYWHERE |
                                                   VM_FLAGS_OVERWRITE |
                                          mask,
                                          flags & (VM_FLAGS_ANYWHERE |
                                                   VM_FLAGS_OVERWRITE |

+                                                  VM_FLAGS_RETURN_4K_DATA_ADDR |

                                                   VM_FLAGS_RETURN_DATA_ADDR),
                                          VM_OBJECT_NULL,
                                          0,
                                                   VM_FLAGS_RETURN_DATA_ADDR),
                                          VM_OBJECT_NULL,
                                          0,


@@ -2792,7 +3157,7 @@ vm_map_enter_mem_object_helper(
                                vm_map_size_t           copy_size;
                                vm_object_offset_t      copy_offset;
 
                                vm_map_size_t           copy_size;
                                vm_object_offset_t      copy_offset;
 

-                               copy_offset = copy_entry->offset;
+                               copy_offset = VME_OFFSET(copy_entry);

                                copy_size = (copy_entry->vme_end -
                                             copy_entry->vme_start);
 
                                copy_size = (copy_entry->vme_end -
                                             copy_entry->vme_start);
 


@@ -2809,15 +3174,13 @@ vm_map_enter_mem_object_helper(
                                /* take a reference on the object */
                                if (copy_entry->is_sub_map) {
                                        remap_flags |= VM_FLAGS_SUBMAP;
                                /* take a reference on the object */
                                if (copy_entry->is_sub_map) {
                                        remap_flags |= VM_FLAGS_SUBMAP;

-                                       copy_submap =
-                                               copy_entry->object.sub_map;
+                                       copy_submap = VME_SUBMAP(copy_entry);

                                        vm_map_lock(copy_submap);
                                        vm_map_reference(copy_submap);
                                        vm_map_unlock(copy_submap);
                                        copy_object = (vm_object_t) copy_submap;
                                } else {
                                        vm_map_lock(copy_submap);
                                        vm_map_reference(copy_submap);
                                        vm_map_unlock(copy_submap);
                                        copy_object = (vm_object_t) copy_submap;
                                } else {

-                                       copy_object =
-                                               copy_entry->object.vm_object;
+                                       copy_object = VME_OBJECT(copy_entry);

                                        vm_object_reference(copy_object);
                                }
 
                                        vm_object_reference(copy_object);
                                }
 


@@ -2852,7 +3215,8 @@ vm_map_enter_mem_object_helper(
                        }
                        
                        if (kr == KERN_SUCCESS) {
                        }
                        
                        if (kr == KERN_SUCCESS) {

-                               if ((flags & VM_FLAGS_RETURN_DATA_ADDR) != 0) {
+                               if (flags & (VM_FLAGS_RETURN_DATA_ADDR |
+                                            VM_FLAGS_RETURN_4K_DATA_ADDR)) {

                                        *address = map_addr + offset_in_mapping;
                                } else {
                                        *address = map_addr;
                                        *address = map_addr + offset_in_mapping;
                                } else {
                                        *address = map_addr;


@@ -2902,8 +3266,11 @@ vm_map_enter_mem_object_helper(
                        /* object cannot be mapped until it is ready  */
                        /* we can therefore avoid the ready check     */
                        /* in this case.  */
                        /* object cannot be mapped until it is ready  */
                        /* we can therefore avoid the ready check     */
                        /* in this case.  */

-                       if ((flags & VM_FLAGS_RETURN_DATA_ADDR) != 0) {
+                       if (flags & (VM_FLAGS_RETURN_DATA_ADDR |
+                                    VM_FLAGS_RETURN_4K_DATA_ADDR)) {

                                offset_in_mapping = offset - vm_object_trunc_page(offset);
                                offset_in_mapping = offset - vm_object_trunc_page(offset);

+                               if (flags & VM_FLAGS_RETURN_4K_DATA_ADDR)
+                                       offset_in_mapping &= ~((signed)(0xFFF));

                                offset = vm_object_trunc_page(offset);
                                map_size = vm_object_round_page(offset + offset_in_mapping + initial_size) - offset;
                        } 
                                offset = vm_object_trunc_page(offset);
                                map_size = vm_object_round_page(offset + offset_in_mapping + initial_size) - offset;
                        } 


@@ -2922,7 +3289,8 @@ vm_map_enter_mem_object_helper(
                 * this case, the port isn't really a port at all, but
                 * instead is just a raw memory object.
                 */
                 * this case, the port isn't really a port at all, but
                 * instead is just a raw memory object.
                 */

-               if ((flags & VM_FLAGS_RETURN_DATA_ADDR) != 0) {
+               if (flags & (VM_FLAGS_RETURN_DATA_ADDR |
+                            VM_FLAGS_RETURN_4K_DATA_ADDR)) {

                        panic("VM_FLAGS_RETURN_DATA_ADDR not expected for raw memory object.");
                }
 
                        panic("VM_FLAGS_RETURN_DATA_ADDR not expected for raw memory object.");
                }
 


@@ -3006,7 +3374,8 @@ vm_map_enter_mem_object_helper(
                vm_object_t             new_object;
                vm_object_offset_t      new_offset;
 
                vm_object_t             new_object;
                vm_object_offset_t      new_offset;
 

-               result = vm_object_copy_strategically(object, offset, size,
+               result = vm_object_copy_strategically(object, offset,
+                                                     map_size,

                                                      &new_object, &new_offset,
                                                      &copy);
 
                                                      &new_object, &new_offset,
                                                      &copy);
 


@@ -3028,7 +3397,8 @@ vm_map_enter_mem_object_helper(
                        new_object = object;
                        new_offset = offset;
                        success = vm_object_copy_quickly(&new_object,
                        new_object = object;
                        new_offset = offset;
                        success = vm_object_copy_quickly(&new_object,

-                                                        new_offset, size,
+                                                        new_offset,
+                                                        map_size,

                                                         &src_needs_copy,
                                                         &copy);
                        assert(success);
                                                         &src_needs_copy,
                                                         &copy);
                        assert(success);


@@ -3041,8 +3411,9 @@ vm_map_enter_mem_object_helper(
 
                vm_object_deallocate(object);
 
 
                vm_object_deallocate(object);
 

-               if (result != KERN_SUCCESS)
+               if (result != KERN_SUCCESS) {

                        return result;
                        return result;

+               }

 
                object = new_object;
                offset = new_offset;
 
                object = new_object;
                offset = new_offset;


@@ -3054,13 +3425,17 @@ vm_map_enter_mem_object_helper(
         */
        if (try_prefault)
                flags |= VM_FLAGS_KEEP_MAP_LOCKED;
         */
        if (try_prefault)
                flags |= VM_FLAGS_KEEP_MAP_LOCKED;

-       result = vm_map_enter(target_map,
-                             &map_addr, map_size,
-                             (vm_map_offset_t)mask,
-                             flags,
-                             object, offset,
-                             copy,
-                             cur_protection, max_protection, inheritance);
+
+       {
+               result = vm_map_enter(target_map,
+                                     &map_addr, map_size,
+                                     (vm_map_offset_t)mask,
+                                     flags,
+                                     object, offset,
+                                     copy,
+                                     cur_protection, max_protection,
+                                     inheritance);
+       }

        if (result != KERN_SUCCESS)
                vm_object_deallocate(object);
 
        if (result != KERN_SUCCESS)
                vm_object_deallocate(object);
 


@@ -3089,7 +3464,7 @@ vm_map_enter_mem_object_helper(
                                                        0, TRUE, PMAP_OPTIONS_NOWAIT, NULL);
                                if (kr != KERN_SUCCESS) {
                                        OSIncrementAtomic64(&vm_prefault_nb_bailout);
                                                        0, TRUE, PMAP_OPTIONS_NOWAIT, NULL);
                                if (kr != KERN_SUCCESS) {
                                        OSIncrementAtomic64(&vm_prefault_nb_bailout);

-                                       goto BailOut;
+                                       break;

                                }
                                OSIncrementAtomic64(&vm_prefault_nb_pages);
                        }
                                }
                                OSIncrementAtomic64(&vm_prefault_nb_pages);
                        }


@@ -3097,11 +3472,11 @@ vm_map_enter_mem_object_helper(
                        /* Next virtual address */
                        va += PAGE_SIZE;
                }
                        /* Next virtual address */
                        va += PAGE_SIZE;
                }

-BailOut:

                vm_map_unlock(target_map);
        }
 
                vm_map_unlock(target_map);
        }
 

-       if ((flags & VM_FLAGS_RETURN_DATA_ADDR) != 0) {
+       if (flags & (VM_FLAGS_RETURN_DATA_ADDR |
+                    VM_FLAGS_RETURN_4K_DATA_ADDR)) {

                *address = map_addr + offset_in_mapping;
        } else {
                *address = map_addr;
                *address = map_addr + offset_in_mapping;
        } else {
                *address = map_addr;


@@ -3178,14 +3553,17 @@ vm_map_enter_mem_object_control(
            (cur_protection & ~VM_PROT_ALL) ||
            (max_protection & ~VM_PROT_ALL) ||
            (inheritance > VM_INHERIT_LAST_VALID) ||
            (cur_protection & ~VM_PROT_ALL) ||
            (max_protection & ~VM_PROT_ALL) ||
            (inheritance > VM_INHERIT_LAST_VALID) ||

-           initial_size == 0)
+           initial_size == 0) {

                return KERN_INVALID_ARGUMENT;
                return KERN_INVALID_ARGUMENT;

+       }

 
 

-       map_addr = vm_map_trunc_page(*address,
-                                    VM_MAP_PAGE_MASK(target_map));
-       map_size = vm_map_round_page(initial_size,
-                                    VM_MAP_PAGE_MASK(target_map));
-       size = vm_object_round_page(initial_size);      
+       {
+               map_addr = vm_map_trunc_page(*address,
+                                            VM_MAP_PAGE_MASK(target_map));
+               map_size = vm_map_round_page(initial_size,
+                                            VM_MAP_PAGE_MASK(target_map));
+       }
+       size = vm_object_round_page(initial_size);

 
        object = memory_object_control_to_vm_object(control);
 
 
        object = memory_object_control_to_vm_object(control);
 


@@ -3275,20 +3653,24 @@ vm_map_enter_mem_object_control(
 
                vm_object_deallocate(object);
 
 
                vm_object_deallocate(object);
 

-               if (result != KERN_SUCCESS)
+               if (result != KERN_SUCCESS) {

                        return result;
                        return result;

+               }

 
                object = new_object;
                offset = new_offset;
        }
 
 
                object = new_object;
                offset = new_offset;
        }
 

-       result = vm_map_enter(target_map,
-                             &map_addr, map_size,
-                             (vm_map_offset_t)mask,
-                             flags,
-                             object, offset,
-                             copy,
-                             cur_protection, max_protection, inheritance);
+       {
+               result = vm_map_enter(target_map,
+                                     &map_addr, map_size,
+                                     (vm_map_offset_t)mask,
+                                     flags,
+                                     object, offset,
+                                     copy,
+                                     cur_protection, max_protection,
+                                     inheritance);
+       }

        if (result != KERN_SUCCESS)
                vm_object_deallocate(object);
        *address = map_addr;
        if (result != KERN_SUCCESS)
                vm_object_deallocate(object);
        *address = map_addr;


@@ -3330,6 +3712,9 @@ vm_map_enter_cpm(
 #endif /* MACH_ASSERT */
 
        boolean_t               anywhere = ((VM_FLAGS_ANYWHERE & flags) != 0);
 #endif /* MACH_ASSERT */
 
        boolean_t               anywhere = ((VM_FLAGS_ANYWHERE & flags) != 0);

+       vm_tag_t tag;
+
+       VM_GET_FLAGS_ALIAS(flags, tag);

 
        if (size == 0) {
                *addr = 0;
 
        if (size == 0) {
                *addr = 0;


@@ -3551,7 +3936,7 @@ vm_map_clip_unnest(
        vm_map_offset_t old_end_unnest = end_unnest;
 
        assert(entry->is_sub_map);
        vm_map_offset_t old_end_unnest = end_unnest;
 
        assert(entry->is_sub_map);

-       assert(entry->object.sub_map != NULL);
+       assert(VME_SUBMAP(entry) != NULL);

        assert(entry->use_pmap);
 
        /*
        assert(entry->use_pmap);
 
        /*


@@ -3578,13 +3963,21 @@ vm_map_clip_unnest(
                _vm_map_clip_start(&map->hdr,
                                   entry,
                                   start_unnest);
                _vm_map_clip_start(&map->hdr,
                                   entry,
                                   start_unnest);

-               vm_map_store_update_first_free(map, map->first_free);
+               if (map->holelistenabled) {
+                       vm_map_store_update_first_free(map, NULL, FALSE);
+               } else {
+                       vm_map_store_update_first_free(map, map->first_free, FALSE);
+               }

        }
        if (entry->vme_end > end_unnest) {
                _vm_map_clip_end(&map->hdr,
                                 entry,
                                 end_unnest);
        }
        if (entry->vme_end > end_unnest) {
                _vm_map_clip_end(&map->hdr,
                                 entry,
                                 end_unnest);

-               vm_map_store_update_first_free(map, map->first_free);
+               if (map->holelistenabled) {
+                       vm_map_store_update_first_free(map, NULL, FALSE);
+               } else {
+                       vm_map_store_update_first_free(map, map->first_free, FALSE);
+               }

        }
 
        pmap_unnest(map->pmap,
        }
 
        pmap_unnest(map->pmap,


@@ -3595,12 +3988,13 @@ vm_map_clip_unnest(
                vm_map_submap_pmap_clean(
                        map, entry->vme_start,
                        entry->vme_end,
                vm_map_submap_pmap_clean(
                        map, entry->vme_start,
                        entry->vme_end,

-                       entry->object.sub_map,
-                       entry->offset);
+                       VME_SUBMAP(entry),
+                       VME_OFFSET(entry));

        }
        entry->use_pmap = FALSE;
        }
        entry->use_pmap = FALSE;

-       if (entry->alias == VM_MEMORY_SHARED_PMAP) {
-               entry->alias = VM_MEMORY_UNSHARED_PMAP;
+       if ((map->pmap != kernel_pmap) &&
+           (VME_ALIAS(entry) == VM_MEMORY_SHARED_PMAP)) {
+               VME_ALIAS_SET(entry, VM_MEMORY_UNSHARED_PMAP);

        }
 }
 #endif /* NO_NESTED_PMAP */
        }
 }
 #endif /* NO_NESTED_PMAP */


@@ -3637,15 +4031,19 @@ vm_map_clip_start(
        }
 #endif /* NO_NESTED_PMAP */
        if (startaddr > entry->vme_start) {
        }
 #endif /* NO_NESTED_PMAP */
        if (startaddr > entry->vme_start) {

-               if (entry->object.vm_object &&
+               if (VME_OBJECT(entry) &&

                    !entry->is_sub_map &&
                    !entry->is_sub_map &&

-                   entry->object.vm_object->phys_contiguous) {
+                   VME_OBJECT(entry)->phys_contiguous) {

                        pmap_remove(map->pmap,
                                    (addr64_t)(entry->vme_start),
                                    (addr64_t)(entry->vme_end));
                }
                _vm_map_clip_start(&map->hdr, entry, startaddr);
                        pmap_remove(map->pmap,
                                    (addr64_t)(entry->vme_start),
                                    (addr64_t)(entry->vme_end));
                }
                _vm_map_clip_start(&map->hdr, entry, startaddr);

-               vm_map_store_update_first_free(map, map->first_free);
+               if (map->holelistenabled) {
+                       vm_map_store_update_first_free(map, NULL, FALSE);
+               } else {
+                       vm_map_store_update_first_free(map, map->first_free, FALSE);
+               }

        }
 }
 
        }
 }
 


@@ -3664,7 +4062,7 @@ static void
 _vm_map_clip_start(
        register struct vm_map_header   *map_header,
        register vm_map_entry_t         entry,
 _vm_map_clip_start(
        register struct vm_map_header   *map_header,
        register vm_map_entry_t         entry,

-       register vm_map_offset_t                start)
+       register vm_map_offset_t        start)

 {
        register vm_map_entry_t new_entry;
 
 {
        register vm_map_entry_t new_entry;
 


@@ -3686,16 +4084,16 @@ _vm_map_clip_start(
 
        new_entry->vme_end = start;
        assert(new_entry->vme_start < new_entry->vme_end);
 
        new_entry->vme_end = start;
        assert(new_entry->vme_start < new_entry->vme_end);

-       entry->offset += (start - entry->vme_start);
+       VME_OFFSET_SET(entry, VME_OFFSET(entry) + (start - entry->vme_start));

        assert(start < entry->vme_end);
        entry->vme_start = start;
 
        _vm_map_store_entry_link(map_header, entry->vme_prev, new_entry);
 
        if (entry->is_sub_map)
        assert(start < entry->vme_end);
        entry->vme_start = start;
 
        _vm_map_store_entry_link(map_header, entry->vme_prev, new_entry);
 
        if (entry->is_sub_map)

-               vm_map_reference(new_entry->object.sub_map);
+               vm_map_reference(VME_SUBMAP(new_entry));

        else
        else

-               vm_object_reference(new_entry->object.vm_object);
+               vm_object_reference(VME_OBJECT(new_entry));

 }
 
 
 }
 
 


@@ -3738,15 +4136,19 @@ vm_map_clip_end(
        }
 #endif /* NO_NESTED_PMAP */
        if (endaddr < entry->vme_end) {
        }
 #endif /* NO_NESTED_PMAP */
        if (endaddr < entry->vme_end) {

-               if (entry->object.vm_object &&
+               if (VME_OBJECT(entry) &&

                    !entry->is_sub_map &&
                    !entry->is_sub_map &&

-                   entry->object.vm_object->phys_contiguous) {
+                   VME_OBJECT(entry)->phys_contiguous) {

                        pmap_remove(map->pmap,
                                    (addr64_t)(entry->vme_start),
                                    (addr64_t)(entry->vme_end));
                }
                _vm_map_clip_end(&map->hdr, entry, endaddr);
                        pmap_remove(map->pmap,
                                    (addr64_t)(entry->vme_start),
                                    (addr64_t)(entry->vme_end));
                }
                _vm_map_clip_end(&map->hdr, entry, endaddr);

-               vm_map_store_update_first_free(map, map->first_free);
+               if (map->holelistenabled) {
+                       vm_map_store_update_first_free(map, NULL, FALSE);
+               } else {
+                       vm_map_store_update_first_free(map, map->first_free, FALSE);
+               }

        }
 }
 
        }
 }
 


@@ -3784,15 +4186,16 @@ _vm_map_clip_end(
 
        assert(entry->vme_start < end);
        new_entry->vme_start = entry->vme_end = end;
 
        assert(entry->vme_start < end);
        new_entry->vme_start = entry->vme_end = end;

-       new_entry->offset += (end - entry->vme_start);
+       VME_OFFSET_SET(new_entry,
+                      VME_OFFSET(new_entry) + (end - entry->vme_start));

        assert(new_entry->vme_start < new_entry->vme_end);
 
        _vm_map_store_entry_link(map_header, entry, new_entry);
 
        if (entry->is_sub_map)
        assert(new_entry->vme_start < new_entry->vme_end);
 
        _vm_map_store_entry_link(map_header, entry, new_entry);
 
        if (entry->is_sub_map)

-               vm_map_reference(new_entry->object.sub_map);
+               vm_map_reference(VME_SUBMAP(new_entry));

        else
        else

-               vm_object_reference(new_entry->object.vm_object);
+               vm_object_reference(VME_OBJECT(new_entry));

 }
 
 
 }
 
 


@@ -3924,17 +4327,17 @@ vm_map_submap(
 
        if ((entry->vme_start == start) && (entry->vme_end == end) &&
            (!entry->is_sub_map) &&
 
        if ((entry->vme_start == start) && (entry->vme_end == end) &&
            (!entry->is_sub_map) &&

-           ((object = entry->object.vm_object) == vm_submap_object) &&
+           ((object = VME_OBJECT(entry)) == vm_submap_object) &&

            (object->resident_page_count == 0) &&
            (object->copy == VM_OBJECT_NULL) &&
            (object->shadow == VM_OBJECT_NULL) &&
            (!object->pager_created)) {
            (object->resident_page_count == 0) &&
            (object->copy == VM_OBJECT_NULL) &&
            (object->shadow == VM_OBJECT_NULL) &&
            (!object->pager_created)) {

-               entry->offset = (vm_object_offset_t)offset;
-               entry->object.vm_object = VM_OBJECT_NULL;
+               VME_OFFSET_SET(entry, (vm_object_offset_t)offset);
+               VME_OBJECT_SET(entry, VM_OBJECT_NULL);

                vm_object_deallocate(object);
                entry->is_sub_map = TRUE;
                entry->use_pmap = FALSE;
                vm_object_deallocate(object);
                entry->is_sub_map = TRUE;
                entry->use_pmap = FALSE;

-               entry->object.sub_map = submap;
+               VME_SUBMAP_SET(entry, submap);

                vm_map_reference(submap);
                if (submap->mapped_in_other_pmaps == FALSE &&
                    vm_map_pmap(submap) != PMAP_NULL &&
                vm_map_reference(submap);
                if (submap->mapped_in_other_pmaps == FALSE &&
                    vm_map_pmap(submap) != PMAP_NULL &&


@@ -3963,7 +4366,7 @@ vm_map_submap(
                                }
                        }
                        result = pmap_nest(map->pmap,
                                }
                        }
                        result = pmap_nest(map->pmap,

-                                          (entry->object.sub_map)->pmap, 
+                                          (VME_SUBMAP(entry))->pmap, 

                                           (addr64_t)start,
                                           (addr64_t)start,
                                           (uint64_t)(end - start));
                                           (addr64_t)start,
                                           (addr64_t)start,
                                           (uint64_t)(end - start));


@@ -3981,6 +4384,7 @@ vm_map_submap(
        return(result);
 }
 
        return(result);
 }
 

+

 /*
  *     vm_map_protect:
  *
 /*
  *     vm_map_protect:
  *


@@ -4110,11 +4514,17 @@ vm_map_protect(
                        /* for loss of shared memory communication in the */
                        /* target area after taking this step */
 
                        /* for loss of shared memory communication in the */
                        /* target area after taking this step */
 

-                       if (current->is_sub_map == FALSE && current->object.vm_object == VM_OBJECT_NULL){
-                               current->object.vm_object = vm_object_allocate((vm_map_size_t)(current->vme_end - current->vme_start));
-                               current->offset = 0;
+                       if (current->is_sub_map == FALSE &&
+                           VME_OBJECT(current) == VM_OBJECT_NULL) {
+                               VME_OBJECT_SET(current, 
+                                              vm_object_allocate(
+                                                      (vm_map_size_t)
+                                                      (current->vme_end -
+                                                       current->vme_start)));
+                               VME_OFFSET_SET(current, 0);

                                assert(current->use_pmap);
                        }
                                assert(current->use_pmap);
                        }

+                       assert(current->wired_count == 0);

                        current->needs_copy = TRUE;
                        current->max_protection |= VM_PROT_WRITE;
                }
                        current->needs_copy = TRUE;
                        current->max_protection |= VM_PROT_WRITE;
                }


@@ -4145,11 +4555,12 @@ vm_map_protect(
 
                        prot = current->protection & ~VM_PROT_WRITE;
 
 
                        prot = current->protection & ~VM_PROT_WRITE;
 

-                       if (override_nx(map, current->alias) && prot)
+                       if (override_nx(map, VME_ALIAS(current)) && prot)

                                prot |= VM_PROT_EXECUTE;
 
                                prot |= VM_PROT_EXECUTE;
 

+

                        if (current->is_sub_map && current->use_pmap) {
                        if (current->is_sub_map && current->use_pmap) {

-                               pmap_protect(current->object.sub_map->pmap, 
+                               pmap_protect(VME_SUBMAP(current)->pmap, 

                                             current->vme_start,
                                             current->vme_end,
                                             prot);
                                             current->vme_start,
                                             current->vme_end,
                                             prot);


@@ -4369,13 +4780,14 @@ vm_map_wire_nested(
        register vm_map_t       map,
        register vm_map_offset_t        start,
        register vm_map_offset_t        end,
        register vm_map_t       map,
        register vm_map_offset_t        start,
        register vm_map_offset_t        end,

-       register vm_prot_t      access_type,
+       register vm_prot_t      caller_prot,

        boolean_t               user_wire,
        pmap_t                  map_pmap, 
        vm_map_offset_t         pmap_addr,
        ppnum_t                 *physpage_p)
 {
        register vm_map_entry_t entry;
        boolean_t               user_wire,
        pmap_t                  map_pmap, 
        vm_map_offset_t         pmap_addr,
        ppnum_t                 *physpage_p)
 {
        register vm_map_entry_t entry;

+       register vm_prot_t      access_type;

        struct vm_map_entry     *first_entry, tmp_entry;
        vm_map_t                real_map;
        register vm_map_offset_t        s,e;
        struct vm_map_entry     *first_entry, tmp_entry;
        vm_map_t                real_map;
        register vm_map_offset_t        s,e;


@@ -4388,6 +4800,8 @@ vm_map_wire_nested(
        vm_map_size_t           size;
        boolean_t               wire_and_extract;
 
        vm_map_size_t           size;
        boolean_t               wire_and_extract;
 

+       access_type = (caller_prot & VM_PROT_ALL);
+

        wire_and_extract = FALSE;
        if (physpage_p != NULL) {
                /*
        wire_and_extract = FALSE;
        if (physpage_p != NULL) {
                /*


@@ -4536,9 +4950,9 @@ vm_map_wire_nested(
                        vm_map_clip_start(map, entry, s);
                        vm_map_clip_end(map, entry, end);
 
                        vm_map_clip_start(map, entry, s);
                        vm_map_clip_end(map, entry, end);
 

-                       sub_start = entry->offset;
+                       sub_start = VME_OFFSET(entry);

                        sub_end = entry->vme_end;
                        sub_end = entry->vme_end;

-                       sub_end += entry->offset - entry->vme_start;
+                       sub_end += VME_OFFSET(entry) - entry->vme_start;

                
                        local_end = entry->vme_end;
                        if(map_pmap == NULL) {
                
                        local_end = entry->vme_end;
                        if(map_pmap == NULL) {


@@ -4551,7 +4965,7 @@ vm_map_wire_nested(
                                vm_map_t                lookup_map;
 
                                if(entry->use_pmap) {
                                vm_map_t                lookup_map;
 
                                if(entry->use_pmap) {

-                                       pmap = entry->object.sub_map->pmap;
+                                       pmap = VME_SUBMAP(entry)->pmap;

                                        /* ppc implementation requires that */
                                        /* submaps pmap address ranges line */
                                        /* up with parent map */
                                        /* ppc implementation requires that */
                                        /* submaps pmap address ranges line */
                                        /* up with parent map */


@@ -4595,6 +5009,7 @@ vm_map_wire_nested(
                                           &real_map)) {
 
                                        vm_map_unlock_read(lookup_map);
                                           &real_map)) {
 
                                        vm_map_unlock_read(lookup_map);

+                                       assert(map_pmap == NULL);

                                        vm_map_unwire(map, start,
                                                      s, user_wire);
                                        return(KERN_FAILURE);
                                        vm_map_unwire(map, start,
                                                      s, user_wire);
                                        return(KERN_FAILURE);


@@ -4642,9 +5057,9 @@ vm_map_wire_nested(
                        entry->in_transition = TRUE;
 
                        vm_map_unlock(map);
                        entry->in_transition = TRUE;
 
                        vm_map_unlock(map);

-                       rc = vm_map_wire_nested(entry->object.sub_map, 
+                       rc = vm_map_wire_nested(VME_SUBMAP(entry), 

                                                sub_start, sub_end,
                                                sub_start, sub_end,

-                                               access_type, 
+                                               caller_prot, 

                                                user_wire, pmap, pmap_addr,
                                                NULL);
                        vm_map_lock(map);
                                                user_wire, pmap, pmap_addr,
                                                NULL);
                        vm_map_lock(map);


@@ -4735,18 +5150,18 @@ vm_map_wire_nested(
                                       == PAGE_SIZE);
                                assert(!entry->needs_copy);
                                assert(!entry->is_sub_map);
                                       == PAGE_SIZE);
                                assert(!entry->needs_copy);
                                assert(!entry->is_sub_map);

-                               assert(entry->object.vm_object);
+                               assert(VME_OBJECT(entry));

                                if (((entry->vme_end - entry->vme_start)
                                     != PAGE_SIZE) ||
                                    entry->needs_copy ||
                                    entry->is_sub_map ||
                                if (((entry->vme_end - entry->vme_start)
                                     != PAGE_SIZE) ||
                                    entry->needs_copy ||
                                    entry->is_sub_map ||

-                                   entry->object.vm_object == VM_OBJECT_NULL) {
+                                   VME_OBJECT(entry) == VM_OBJECT_NULL) {

                                        rc = KERN_INVALID_ARGUMENT;
                                        goto done;
                                }
 
                                        rc = KERN_INVALID_ARGUMENT;
                                        goto done;
                                }
 

-                               object = entry->object.vm_object;
-                               offset = entry->offset;
+                               object = VME_OBJECT(entry);
+                               offset = VME_OFFSET(entry);

                                /* need exclusive lock to update m->dirty */
                                if (entry->protection & VM_PROT_WRITE) {
                                        vm_object_lock(object);
                                /* need exclusive lock to update m->dirty */
                                if (entry->protection & VM_PROT_WRITE) {
                                        vm_object_lock(object);


@@ -4802,11 +5217,10 @@ vm_map_wire_nested(
                                rc = KERN_INVALID_ARGUMENT;
                                goto done;
                        }
                                rc = KERN_INVALID_ARGUMENT;
                                goto done;
                        }

-                                
-                       vm_object_shadow(&entry->object.vm_object,
-                                        &entry->offset, size);
+
+                       VME_OBJECT_SHADOW(entry, size);

                        entry->needs_copy = FALSE;
                        entry->needs_copy = FALSE;

-               } else if (entry->object.vm_object == VM_OBJECT_NULL) {
+               } else if (VME_OBJECT(entry) == VM_OBJECT_NULL) {

                        if (wire_and_extract) {
                                /*
                                 * We're supposed to share with the original
                        if (wire_and_extract) {
                                /*
                                 * We're supposed to share with the original


@@ -4815,8 +5229,8 @@ vm_map_wire_nested(
                                rc = KERN_INVALID_ARGUMENT;
                                goto done;
                        }
                                rc = KERN_INVALID_ARGUMENT;
                                goto done;
                        }

-                       entry->object.vm_object = vm_object_allocate(size);
-                       entry->offset = (vm_object_offset_t)0;
+                       VME_OBJECT_SET(entry, vm_object_allocate(size));
+                       VME_OFFSET_SET(entry, (vm_object_offset_t)0);

                        assert(entry->use_pmap);
                }
 
                        assert(entry->use_pmap);
                }
 


@@ -4880,11 +5294,11 @@ vm_map_wire_nested(
 
                if(map_pmap)
                        rc = vm_fault_wire(map, 
 
                if(map_pmap)
                        rc = vm_fault_wire(map, 

-                                          &tmp_entry, map_pmap, pmap_addr,
+                                          &tmp_entry, caller_prot, map_pmap, pmap_addr,

                                           physpage_p);
                else
                        rc = vm_fault_wire(map, 
                                           physpage_p);
                else
                        rc = vm_fault_wire(map, 

-                                          &tmp_entry, map->pmap, 
+                                          &tmp_entry, caller_prot, map->pmap, 

                                           tmp_entry.vme_start,
                                           physpage_p);
 
                                           tmp_entry.vme_start,
                                           physpage_p);
 


@@ -4944,7 +5358,8 @@ done:
 
        if (rc != KERN_SUCCESS) {
                /* undo what has been wired so far */
 
        if (rc != KERN_SUCCESS) {
                /* undo what has been wired so far */

-               vm_map_unwire(map, start, s, user_wire);
+               vm_map_unwire_nested(map, start, s, user_wire,
+                                    map_pmap, pmap_addr);

                if (physpage_p) {
                        *physpage_p = 0;
                }
                if (physpage_p) {
                        *physpage_p = 0;
                }


@@ -4955,36 +5370,78 @@ done:
 }
 
 kern_return_t
 }
 
 kern_return_t

-vm_map_wire(
+vm_map_wire_external(

        register vm_map_t       map,
        register vm_map_offset_t        start,
        register vm_map_offset_t        end,
        register vm_map_t       map,
        register vm_map_offset_t        start,
        register vm_map_offset_t        end,

-       register vm_prot_t      access_type,
+       register vm_prot_t      caller_prot,

        boolean_t               user_wire)
 {
        boolean_t               user_wire)
 {

+       kern_return_t   kret;

 
 

+       caller_prot &= ~VM_PROT_MEMORY_TAG_MASK;
+       caller_prot |= VM_PROT_MEMORY_TAG_MAKE(vm_tag_bt());
+       kret = vm_map_wire_nested(map, start, end, caller_prot, 
+                                 user_wire, (pmap_t)NULL, 0, NULL);
+       return kret;
+}
+
+kern_return_t
+vm_map_wire(
+       register vm_map_t       map,
+       register vm_map_offset_t        start,
+       register vm_map_offset_t        end,
+       register vm_prot_t      caller_prot,
+       boolean_t               user_wire)
+{

        kern_return_t   kret;
 
        kern_return_t   kret;
 

-       kret = vm_map_wire_nested(map, start, end, access_type, 
+       kret = vm_map_wire_nested(map, start, end, caller_prot, 

                                  user_wire, (pmap_t)NULL, 0, NULL);
        return kret;
 }
 
 kern_return_t
                                  user_wire, (pmap_t)NULL, 0, NULL);
        return kret;
 }
 
 kern_return_t

-vm_map_wire_and_extract(
+vm_map_wire_and_extract_external(

        vm_map_t        map,
        vm_map_offset_t start,
        vm_map_t        map,
        vm_map_offset_t start,

-       vm_prot_t       access_type,
+       vm_prot_t       caller_prot,

        boolean_t       user_wire,
        ppnum_t         *physpage_p)
 {
        boolean_t       user_wire,
        ppnum_t         *physpage_p)
 {

+       kern_return_t   kret;
+
+       caller_prot &= ~VM_PROT_MEMORY_TAG_MASK;
+       caller_prot |= VM_PROT_MEMORY_TAG_MAKE(vm_tag_bt());
+       kret = vm_map_wire_nested(map,
+                                 start,
+                                 start+VM_MAP_PAGE_SIZE(map),
+                                 caller_prot, 
+                                 user_wire,
+                                 (pmap_t)NULL,
+                                 0,
+                                 physpage_p);
+       if (kret != KERN_SUCCESS &&
+           physpage_p != NULL) {
+               *physpage_p = 0;
+       }
+       return kret;
+}

 
 

+kern_return_t
+vm_map_wire_and_extract(
+       vm_map_t        map,
+       vm_map_offset_t start,
+       vm_prot_t       caller_prot,
+       boolean_t       user_wire,
+       ppnum_t         *physpage_p)
+{

        kern_return_t   kret;
 
        kret = vm_map_wire_nested(map,
                                  start,
                                  start+VM_MAP_PAGE_SIZE(map),
        kern_return_t   kret;
 
        kret = vm_map_wire_nested(map,
                                  start,
                                  start+VM_MAP_PAGE_SIZE(map),

-                                 access_type, 
+                                 caller_prot, 

                                  user_wire,
                                  (pmap_t)NULL,
                                  0,
                                  user_wire,
                                  (pmap_t)NULL,
                                  0,


@@ -5117,13 +5574,13 @@ vm_map_unwire_nested(
                        vm_map_clip_start(map, entry, start);
                        vm_map_clip_end(map, entry, end);
 
                        vm_map_clip_start(map, entry, start);
                        vm_map_clip_end(map, entry, end);
 

-                       sub_start = entry->offset;
+                       sub_start = VME_OFFSET(entry);

                        sub_end = entry->vme_end - entry->vme_start;
                        sub_end = entry->vme_end - entry->vme_start;

-                       sub_end += entry->offset;
+                       sub_end += VME_OFFSET(entry);

                        local_end = entry->vme_end;
                        if(map_pmap == NULL) {
                                if(entry->use_pmap) {
                        local_end = entry->vme_end;
                        if(map_pmap == NULL) {
                                if(entry->use_pmap) {

-                                       pmap = entry->object.sub_map->pmap;
+                                       pmap = VME_SUBMAP(entry)->pmap;

                                        pmap_addr = sub_start;
                                } else {
                                        pmap = map->pmap;
                                        pmap_addr = sub_start;
                                } else {
                                        pmap = map->pmap;


@@ -5169,7 +5626,7 @@ vm_map_unwire_nested(
                                 * guarantees existance of the entry.
                                 */
                                vm_map_unlock(map);
                                 * guarantees existance of the entry.
                                 */
                                vm_map_unlock(map);

-                               vm_map_unwire_nested(entry->object.sub_map, 
+                               vm_map_unwire_nested(VME_SUBMAP(entry), 

                                                     sub_start, sub_end, user_wire, pmap, pmap_addr);
                                vm_map_lock(map);
 
                                                     sub_start, sub_end, user_wire, pmap, pmap_addr);
                                vm_map_lock(map);
 


@@ -5207,7 +5664,7 @@ vm_map_unwire_nested(
                                continue;
                        } else {
                                vm_map_unlock(map);
                                continue;
                        } else {
                                vm_map_unlock(map);

-                               vm_map_unwire_nested(entry->object.sub_map,
+                               vm_map_unwire_nested(VME_SUBMAP(entry),

                                                     sub_start, sub_end, user_wire, map_pmap,
                                                     pmap_addr);
                                vm_map_lock(map);
                                                     sub_start, sub_end, user_wire, map_pmap,
                                                     pmap_addr);
                                vm_map_lock(map);


@@ -5382,10 +5839,10 @@ vm_map_entry_delete(
 
        if (entry->is_sub_map) {
                object = NULL;
 
        if (entry->is_sub_map) {
                object = NULL;

-               submap = entry->object.sub_map;
+               submap = VME_SUBMAP(entry);

        } else {
                submap = NULL;
        } else {
                submap = NULL;

-               object = entry->object.vm_object;
+               object = VME_OBJECT(entry);

        }
 
        vm_map_store_entry_unlink(map, entry);
        }
 
        vm_map_store_entry_unlink(map, entry);


@@ -5438,19 +5895,22 @@ vm_map_submap_pmap_clean(
                                sub_map,
                                start,
                                start + remove_size,
                                sub_map,
                                start,
                                start + remove_size,

-                               entry->object.sub_map,
-                               entry->offset);
+                               VME_SUBMAP(entry),
+                               VME_OFFSET(entry));

                } else {
 
                        if((map->mapped_in_other_pmaps) && (map->ref_count)
                } else {
 
                        if((map->mapped_in_other_pmaps) && (map->ref_count)

-                          && (entry->object.vm_object != NULL)) {
-                               vm_object_pmap_protect(
-                                       entry->object.vm_object,
-                                       entry->offset+(offset-entry->vme_start),
+                          && (VME_OBJECT(entry) != NULL)) {
+                               vm_object_pmap_protect_options(
+                                       VME_OBJECT(entry),
+                                       (VME_OFFSET(entry) +
+                                        offset -
+                                        entry->vme_start),

                                        remove_size,
                                        PMAP_NULL,
                                        entry->vme_start,
                                        remove_size,
                                        PMAP_NULL,
                                        entry->vme_start,

-                                       VM_PROT_NONE);
+                                       VM_PROT_NONE,
+                                       PMAP_OPTIONS_REMOVE);

                        } else {
                                pmap_remove(map->pmap, 
                                            (addr64_t)start, 
                        } else {
                                pmap_remove(map->pmap, 
                                            (addr64_t)start, 


@@ -5472,18 +5932,19 @@ vm_map_submap_pmap_clean(
                                sub_map,
                                (start + entry->vme_start) - offset,
                                ((start + entry->vme_start) - offset) + remove_size,
                                sub_map,
                                (start + entry->vme_start) - offset,
                                ((start + entry->vme_start) - offset) + remove_size,

-                               entry->object.sub_map,
-                               entry->offset);
+                               VME_SUBMAP(entry),
+                               VME_OFFSET(entry));

                } else {
                        if((map->mapped_in_other_pmaps) && (map->ref_count)
                } else {
                        if((map->mapped_in_other_pmaps) && (map->ref_count)

-                          && (entry->object.vm_object != NULL)) {
-                               vm_object_pmap_protect(
-                                       entry->object.vm_object,
-                                       entry->offset,
+                          && (VME_OBJECT(entry) != NULL)) {
+                               vm_object_pmap_protect_options(
+                                       VME_OBJECT(entry),
+                                       VME_OFFSET(entry),

                                        remove_size,
                                        PMAP_NULL,
                                        entry->vme_start,
                                        remove_size,
                                        PMAP_NULL,
                                        entry->vme_start,

-                                       VM_PROT_NONE);
+                                       VM_PROT_NONE,
+                                       PMAP_OPTIONS_REMOVE);

                        } else {
                                pmap_remove(map->pmap, 
                                            (addr64_t)((start + entry->vme_start) 
                        } else {
                                pmap_remove(map->pmap, 
                                            (addr64_t)((start + entry->vme_start) 


@@ -5735,8 +6196,6 @@ vm_map_delete(
                         * may not exist anymore.  Look it up again.
                         */
                        if (!vm_map_lookup_entry(map, s, &first_entry)) {
                         * may not exist anymore.  Look it up again.
                         */
                        if (!vm_map_lookup_entry(map, s, &first_entry)) {

-                               assert((map != kernel_map) && 
-                                      (!entry->is_sub_map));

                                /*
                                 * User: use the next entry
                                 */
                                /*
                                 * User: use the next entry
                                 */


@@ -5843,8 +6302,8 @@ vm_map_delete(
                                vm_map_offset_t pmap_addr;
                                
 
                                vm_map_offset_t pmap_addr;
                                
 

-                               sub_map = tmp_entry.object.sub_map;
-                               sub_start = tmp_entry.offset;
+                               sub_map = VME_SUBMAP(&tmp_entry);
+                               sub_start = VME_OFFSET(&tmp_entry);

                                sub_end = sub_start + (tmp_entry.vme_end -
                                                       tmp_entry.vme_start);
                                if (tmp_entry.use_pmap) {
                                sub_end = sub_start + (tmp_entry.vme_end -
                                                       tmp_entry.vme_start);
                                if (tmp_entry.use_pmap) {


@@ -5860,7 +6319,7 @@ vm_map_delete(
                                                            pmap, pmap_addr);
                        } else {
 
                                                            pmap, pmap_addr);
                        } else {
 

-                               if (tmp_entry.object.vm_object == kernel_object) {
+                               if (VME_OBJECT(&tmp_entry) == kernel_object) {

                                        pmap_protect_options(
                                                map->pmap,
                                                tmp_entry.vme_start,
                                        pmap_protect_options(
                                                map->pmap,
                                                tmp_entry.vme_start,


@@ -5870,7 +6329,7 @@ vm_map_delete(
                                                NULL);
                                }
                                vm_fault_unwire(map, &tmp_entry,
                                                NULL);
                                }
                                vm_fault_unwire(map, &tmp_entry,

-                                               tmp_entry.object.vm_object == kernel_object,
+                                               VME_OBJECT(&tmp_entry) == kernel_object,

                                                map->pmap, tmp_entry.vme_start);
                        }
 
                                                map->pmap, tmp_entry.vme_start);
                        }
 


@@ -5935,37 +6394,61 @@ vm_map_delete(
                } else if (entry->is_sub_map) {
                        if (entry->use_pmap) {
 #ifndef NO_NESTED_PMAP
                } else if (entry->is_sub_map) {
                        if (entry->use_pmap) {
 #ifndef NO_NESTED_PMAP

-                               pmap_unnest(map->pmap,
-                                           (addr64_t)entry->vme_start,
-                                           entry->vme_end - entry->vme_start);
+                               int pmap_flags;
+
+                               if (flags & VM_MAP_REMOVE_NO_UNNESTING) {
+                                       /*
+                                        * This is the final cleanup of the
+                                        * address space being terminated.
+                                        * No new mappings are expected and
+                                        * we don't really need to unnest the
+                                        * shared region (and lose the "global"
+                                        * pmap mappings, if applicable).
+                                        *
+                                        * Tell the pmap layer that we're
+                                        * "clean" wrt nesting.
+                                        */
+                                       pmap_flags = PMAP_UNNEST_CLEAN;
+                               } else {
+                                       /*
+                                        * We're unmapping part of the nested
+                                        * shared region, so we can't keep the
+                                        * nested pmap.
+                                        */
+                                       pmap_flags = 0;
+                               }
+                               pmap_unnest_options(
+                                       map->pmap,
+                                       (addr64_t)entry->vme_start,
+                                       entry->vme_end - entry->vme_start,
+                                       pmap_flags);

 #endif /* NO_NESTED_PMAP */
                                if ((map->mapped_in_other_pmaps) && (map->ref_count)) {
                                        /* clean up parent map/maps */
                                        vm_map_submap_pmap_clean(
                                                map, entry->vme_start,
                                                entry->vme_end,
 #endif /* NO_NESTED_PMAP */
                                if ((map->mapped_in_other_pmaps) && (map->ref_count)) {
                                        /* clean up parent map/maps */
                                        vm_map_submap_pmap_clean(
                                                map, entry->vme_start,
                                                entry->vme_end,

-                                               entry->object.sub_map,
-                                               entry->offset);
+                                               VME_SUBMAP(entry),
+                                               VME_OFFSET(entry));

                                }
                        } else {
                                vm_map_submap_pmap_clean(
                                        map, entry->vme_start, entry->vme_end,
                                }
                        } else {
                                vm_map_submap_pmap_clean(
                                        map, entry->vme_start, entry->vme_end,

-                                       entry->object.sub_map,
-                                       entry->offset);
+                                       VME_SUBMAP(entry),
+                                       VME_OFFSET(entry));

                        }
                        }

-               } else if (entry->object.vm_object != kernel_object &&
-                          entry->object.vm_object != compressor_object) {
-                       object = entry->object.vm_object;
+               } else if (VME_OBJECT(entry) != kernel_object &&
+                          VME_OBJECT(entry) != compressor_object) {
+                       object = VME_OBJECT(entry);

                        if ((map->mapped_in_other_pmaps) && (map->ref_count)) {
                                vm_object_pmap_protect_options(
                        if ((map->mapped_in_other_pmaps) && (map->ref_count)) {
                                vm_object_pmap_protect_options(

-                                       object, entry->offset,
+                                       object, VME_OFFSET(entry),

                                        entry->vme_end - entry->vme_start,
                                        PMAP_NULL,
                                        entry->vme_start,
                                        VM_PROT_NONE,
                                        PMAP_OPTIONS_REMOVE);
                                        entry->vme_end - entry->vme_start,
                                        PMAP_NULL,
                                        entry->vme_start,
                                        VM_PROT_NONE,
                                        PMAP_OPTIONS_REMOVE);

-                       } else if ((entry->object.vm_object !=
-                                   VM_OBJECT_NULL) ||
+                       } else if ((VME_OBJECT(entry) != VM_OBJECT_NULL) ||

                                   (map->pmap == kernel_pmap)) {
                                /* Remove translations associated
                                 * with this range unless the entry
                                   (map->pmap == kernel_pmap)) {
                                /* Remove translations associated
                                 * with this range unless the entry


@@ -5987,6 +6470,11 @@ vm_map_delete(
 
                if (entry->iokit_acct) {
                        /* alternate accounting */
 
                if (entry->iokit_acct) {
                        /* alternate accounting */

+                       DTRACE_VM4(vm_map_iokit_unmapped_region,
+                                  vm_map_t, map,
+                                  vm_map_offset_t, entry->vme_start,
+                                  vm_map_offset_t, entry->vme_end,
+                                  int, VME_ALIAS(entry));

                        vm_map_iokit_unmapped_region(map,
                                                     (entry->vme_end -
                                                      entry->vme_start));
                        vm_map_iokit_unmapped_region(map,
                                                     (entry->vme_end -
                                                      entry->vme_start));


@@ -6144,9 +6632,9 @@ vm_map_copy_discard(
 
                        vm_map_copy_entry_unlink(copy, entry);
                        if (entry->is_sub_map) {
 
                        vm_map_copy_entry_unlink(copy, entry);
                        if (entry->is_sub_map) {

-                               vm_map_deallocate(entry->object.sub_map);
+                               vm_map_deallocate(VME_SUBMAP(entry));

                        } else {
                        } else {

-                               vm_object_deallocate(entry->object.vm_object);
+                               vm_object_deallocate(VME_OBJECT(entry));

                        }
                        vm_map_copy_entry_dispose(copy, entry);
                }
                        }
                        vm_map_copy_entry_dispose(copy, entry);
                }


@@ -6161,7 +6649,10 @@ vm_map_copy_discard(
                 * allocated by a single call to kalloc(), i.e. the
                 * vm_map_copy_t was not allocated out of the zone.
                 */
                 * allocated by a single call to kalloc(), i.e. the
                 * vm_map_copy_t was not allocated out of the zone.
                 */

-               kfree(copy, copy->cpy_kalloc_size);
+               if (copy->size > msg_ool_size_small || copy->offset)
+                       panic("Invalid vm_map_copy_t sz:%lld, ofst:%lld",
+                             (long long)copy->size, (long long)copy->offset);
+               kfree(copy, copy->size + cpy_kdata_hdr_sz);

                return;
        }
        zfree(vm_map_copy_zone, copy);
                return;
        }
        zfree(vm_map_copy_zone, copy);


@@ -6286,19 +6777,19 @@ start_pass_1:
                        }
 
                        encountered_sub_map = TRUE;
                        }
 
                        encountered_sub_map = TRUE;

-                       sub_start = entry->offset;
+                       sub_start = VME_OFFSET(entry);

 
                        if(entry->vme_end < dst_end)
                                sub_end = entry->vme_end;
                        else 
                                sub_end = dst_end;
                        sub_end -= entry->vme_start;
 
                        if(entry->vme_end < dst_end)
                                sub_end = entry->vme_end;
                        else 
                                sub_end = dst_end;
                        sub_end -= entry->vme_start;

-                       sub_end += entry->offset;
+                       sub_end += VME_OFFSET(entry);

                        local_end = entry->vme_end;
                        vm_map_unlock(dst_map);
                        
                        result = vm_map_overwrite_submap_recurse(
                        local_end = entry->vme_end;
                        vm_map_unlock(dst_map);
                        
                        result = vm_map_overwrite_submap_recurse(

-                               entry->object.sub_map,
+                               VME_SUBMAP(entry),

                                sub_start,
                                sub_end - sub_start);
 
                                sub_start,
                                sub_end - sub_start);
 


@@ -6356,9 +6847,9 @@ start_pass_1:
                /*
                 *      Check for permanent objects in the destination.
                 */
                /*
                 *      Check for permanent objects in the destination.
                 */

-               if ((entry->object.vm_object != VM_OBJECT_NULL) &&
-                   ((!entry->object.vm_object->internal) ||
-                    (entry->object.vm_object->true_share))) {
+               if ((VME_OBJECT(entry) != VM_OBJECT_NULL) &&
+                   ((!VME_OBJECT(entry)->internal) ||
+                    (VME_OBJECT(entry)->true_share))) {

                        if(encountered_sub_map) {
                                vm_map_unlock(dst_map);
                                return(KERN_FAILURE);
                        if(encountered_sub_map) {
                                vm_map_unlock(dst_map);
                                return(KERN_FAILURE);


@@ -6542,18 +7033,18 @@ start_pass_1:
                                /* there is no need for the follow-  */
                                /* ing check.                        */
                                encountered_sub_map = TRUE;
                                /* there is no need for the follow-  */
                                /* ing check.                        */
                                encountered_sub_map = TRUE;

-                               sub_start = entry->offset;
+                               sub_start = VME_OFFSET(entry);

 
                                if(entry->vme_end < dst_end)
                                        sub_end = entry->vme_end;
                                else 
                                        sub_end = dst_end;
                                sub_end -= entry->vme_start;
 
                                if(entry->vme_end < dst_end)
                                        sub_end = entry->vme_end;
                                else 
                                        sub_end = dst_end;
                                sub_end -= entry->vme_start;

-                               sub_end += entry->offset;
+                               sub_end += VME_OFFSET(entry);

                                vm_map_unlock(dst_map);
                        
                                kr = vm_map_overwrite_submap_recurse(
                                vm_map_unlock(dst_map);
                        
                                kr = vm_map_overwrite_submap_recurse(

-                                       entry->object.sub_map,
+                                       VME_SUBMAP(entry),

                                        sub_start,
                                        sub_end - sub_start);
                                if(kr != KERN_SUCCESS)
                                        sub_start,
                                        sub_end - sub_start);
                                if(kr != KERN_SUCCESS)


@@ -6610,9 +7101,9 @@ start_pass_1:
                /*
                 *      Check for permanent objects in the destination.
                 */
                /*
                 *      Check for permanent objects in the destination.
                 */

-               if ((entry->object.vm_object != VM_OBJECT_NULL) &&
-                   ((!entry->object.vm_object->internal) ||
-                    (entry->object.vm_object->true_share))) {
+               if ((VME_OBJECT(entry) != VM_OBJECT_NULL) &&
+                   ((!VME_OBJECT(entry)->internal) ||
+                    (VME_OBJECT(entry)->true_share))) {

                        contains_permanent_objects = TRUE;
                }
 
                        contains_permanent_objects = TRUE;
                }
 


@@ -6717,11 +7208,11 @@ start_overwrite:
                                        assert(!entry->use_pmap);
                                        entry->is_sub_map = FALSE;
                                        vm_map_deallocate(
                                        assert(!entry->use_pmap);
                                        entry->is_sub_map = FALSE;
                                        vm_map_deallocate(

-                                               entry->object.sub_map);
-                                       entry->object.sub_map = NULL;
+                                               VME_SUBMAP(entry));
+                                       VME_SUBMAP_SET(entry, NULL);

                                        entry->is_shared = FALSE;
                                        entry->needs_copy = FALSE;
                                        entry->is_shared = FALSE;
                                        entry->needs_copy = FALSE;

-                                       entry->offset = 0;
+                                       VME_OFFSET_SET(entry, 0);

                                        /*
                                         * XXX FBDP
                                         * We should propagate the protections
                                        /*
                                         * XXX FBDP
                                         * We should propagate the protections


@@ -6747,14 +7238,14 @@ start_overwrite:
                                                entry->vme_start - base_addr;
                                        break;
                                }
                                                entry->vme_start - base_addr;
                                        break;
                                }

-                               sub_start = entry->offset;
+                               sub_start = VME_OFFSET(entry);

 
                                if(entry->vme_end < dst_end)
                                        sub_end = entry->vme_end;
                                else 
                                        sub_end = dst_end;
                                sub_end -= entry->vme_start;
 
                                if(entry->vme_end < dst_end)
                                        sub_end = entry->vme_end;
                                else 
                                        sub_end = dst_end;
                                sub_end -= entry->vme_start;

-                               sub_end += entry->offset;
+                               sub_end += VME_OFFSET(entry);

                                local_end = entry->vme_end;
                                vm_map_unlock(dst_map);
                                copy_size = sub_end - sub_start;
                                local_end = entry->vme_end;
                                vm_map_unlock(dst_map);
                                copy_size = sub_end - sub_start;


@@ -6807,22 +7298,22 @@ start_overwrite:
                        
                                if((entry->use_pmap) && (pmap == NULL)) {
                                        kr = vm_map_copy_overwrite_nested(
                        
                                if((entry->use_pmap) && (pmap == NULL)) {
                                        kr = vm_map_copy_overwrite_nested(

-                                               entry->object.sub_map,
+                                               VME_SUBMAP(entry),

                                                sub_start,
                                                copy,
                                                interruptible, 
                                                sub_start,
                                                copy,
                                                interruptible, 

-                                               entry->object.sub_map->pmap,
+                                               VME_SUBMAP(entry)->pmap,

                                                TRUE);
                                } else if (pmap != NULL) {
                                        kr = vm_map_copy_overwrite_nested(
                                                TRUE);
                                } else if (pmap != NULL) {
                                        kr = vm_map_copy_overwrite_nested(

-                                               entry->object.sub_map,
+                                               VME_SUBMAP(entry),

                                                sub_start,
                                                copy,
                                                interruptible, pmap,
                                                TRUE);
                                } else {
                                        kr = vm_map_copy_overwrite_nested(
                                                sub_start,
                                                copy,
                                                interruptible, pmap,
                                                TRUE);
                                } else {
                                        kr = vm_map_copy_overwrite_nested(

-                                               entry->object.sub_map,
+                                               VME_SUBMAP(entry),

                                                sub_start,
                                                copy,
                                                interruptible,
                                                sub_start,
                                                copy,
                                                interruptible,


@@ -7387,14 +7878,13 @@ vm_map_copy_overwrite_unaligned(
                                vm_map_lock_read(dst_map);
                                goto RetryLookup;
                        }
                                vm_map_lock_read(dst_map);
                                goto RetryLookup;
                        }

-                       vm_object_shadow(&entry->object.vm_object,
-                                        &entry->offset,
-                                        (vm_map_size_t)(entry->vme_end
-                                                        - entry->vme_start));
+                       VME_OBJECT_SHADOW(entry,
+                                         (vm_map_size_t)(entry->vme_end
+                                                         - entry->vme_start));

                        entry->needs_copy = FALSE;
                        vm_map_lock_write_to_read(dst_map);
                }
                        entry->needs_copy = FALSE;
                        vm_map_lock_write_to_read(dst_map);
                }

-               dst_object = entry->object.vm_object;
+               dst_object = VME_OBJECT(entry);

 /*
  *             unlike with the virtual (aligned) copy we're going
  *             to fault on it therefore we need a target object.
 /*
  *             unlike with the virtual (aligned) copy we're going
  *             to fault on it therefore we need a target object.


@@ -7406,8 +7896,8 @@ vm_map_copy_overwrite_unaligned(
                        }
                        dst_object = vm_object_allocate((vm_map_size_t)
                                                        entry->vme_end - entry->vme_start);
                        }
                        dst_object = vm_object_allocate((vm_map_size_t)
                                                        entry->vme_end - entry->vme_start);

-                       entry->object.vm_object = dst_object;
-                       entry->offset = 0;
+                       VME_OBJECT(entry) = dst_object;
+                       VME_OFFSET_SET(entry, 0);

                        assert(entry->use_pmap);
                        vm_map_lock_write_to_read(dst_map);
                }
                        assert(entry->use_pmap);
                        vm_map_lock_write_to_read(dst_map);
                }


@@ -7417,15 +7907,15 @@ vm_map_copy_overwrite_unaligned(
  */
                vm_object_reference(dst_object);
                version.main_timestamp = dst_map->timestamp;
  */
                vm_object_reference(dst_object);
                version.main_timestamp = dst_map->timestamp;

-               entry_offset = entry->offset;
+               entry_offset = VME_OFFSET(entry);

                entry_end = entry->vme_end;
                vm_map_unlock_read(dst_map);
 /*
  *             Copy as much as possible in one pass
  */
                kr = vm_fault_copy(
                entry_end = entry->vme_end;
                vm_map_unlock_read(dst_map);
 /*
  *             Copy as much as possible in one pass
  */
                kr = vm_fault_copy(

-                       copy_entry->object.vm_object,
-                       copy_entry->offset + src_offset,
+                       VME_OBJECT(copy_entry),
+                       VME_OFFSET(copy_entry) + src_offset,

                        &copy_size,
                        dst_object,
                        entry_offset + dst_offset,
                        &copy_size,
                        dst_object,
                        entry_offset + dst_offset,


@@ -7457,8 +7947,7 @@ vm_map_copy_overwrite_unaligned(
                        if (discard_on_success) {
                                vm_map_copy_entry_unlink(copy, copy_entry);
                                assert(!copy_entry->is_sub_map);
                        if (discard_on_success) {
                                vm_map_copy_entry_unlink(copy, copy_entry);
                                assert(!copy_entry->is_sub_map);

-                               vm_object_deallocate(
-                                       copy_entry->object.vm_object);
+                               vm_object_deallocate(VME_OBJECT(copy_entry));

                                vm_map_copy_entry_dispose(copy, copy_entry);
                        }
 
                                vm_map_copy_entry_dispose(copy, copy_entry);
                        }
 


@@ -7624,21 +8113,21 @@ vm_map_copy_overwrite_aligned(
                 *      installing the source data.
                 */
 
                 *      installing the source data.
                 */
 

-               object = entry->object.vm_object;
+               object = VME_OBJECT(entry);

                if ((!entry->is_shared && 
                     ((object == VM_OBJECT_NULL) || 
                      (object->internal && !object->true_share))) ||
                    entry->needs_copy) {
                if ((!entry->is_shared && 
                     ((object == VM_OBJECT_NULL) || 
                      (object->internal && !object->true_share))) ||
                    entry->needs_copy) {

-                       vm_object_t     old_object = entry->object.vm_object;
-                       vm_object_offset_t      old_offset = entry->offset;
+                       vm_object_t     old_object = VME_OBJECT(entry);
+                       vm_object_offset_t      old_offset = VME_OFFSET(entry);

                        vm_object_offset_t      offset;
 
                        /*
                         * Ensure that the source and destination aren't
                         * identical
                         */
                        vm_object_offset_t      offset;
 
                        /*
                         * Ensure that the source and destination aren't
                         * identical
                         */

-                       if (old_object == copy_entry->object.vm_object &&
-                           old_offset == copy_entry->offset) {
+                       if (old_object == VME_OBJECT(copy_entry) &&
+                           old_offset == VME_OFFSET(copy_entry)) {

                                vm_map_copy_entry_unlink(copy, copy_entry);
                                vm_map_copy_entry_dispose(copy, copy_entry);
 
                                vm_map_copy_entry_unlink(copy, copy_entry);
                                vm_map_copy_entry_dispose(copy, copy_entry);
 


@@ -7652,8 +8141,8 @@ vm_map_copy_overwrite_aligned(
 
 #define __TRADEOFF1_OBJ_SIZE (64 * 1024 * 1024)        /* 64 MB */
 #define __TRADEOFF1_COPY_SIZE (128 * 1024)     /* 128 KB */
 
 #define __TRADEOFF1_OBJ_SIZE (64 * 1024 * 1024)        /* 64 MB */
 #define __TRADEOFF1_COPY_SIZE (128 * 1024)     /* 128 KB */

-                       if (copy_entry->object.vm_object != VM_OBJECT_NULL &&
-                           copy_entry->object.vm_object->vo_size >= __TRADEOFF1_OBJ_SIZE &&
+                       if (VME_OBJECT(copy_entry) != VM_OBJECT_NULL &&
+                           VME_OBJECT(copy_entry)->vo_size >= __TRADEOFF1_OBJ_SIZE &&

                            copy_size <= __TRADEOFF1_COPY_SIZE) {
                                /*
                                 * Virtual vs. Physical copy tradeoff #1.
                            copy_size <= __TRADEOFF1_COPY_SIZE) {
                                /*
                                 * Virtual vs. Physical copy tradeoff #1.


@@ -7668,15 +8157,16 @@ vm_map_copy_overwrite_aligned(
                                goto slow_copy;
                        }
 
                                goto slow_copy;
                        }
 

-                       if (entry->alias >= VM_MEMORY_MALLOC &&
-                           entry->alias <= VM_MEMORY_MALLOC_LARGE_REUSED) {
+                       if ((dst_map->pmap != kernel_pmap) &&
+                           (VME_ALIAS(entry) >= VM_MEMORY_MALLOC) &&
+                           (VME_ALIAS(entry) <= VM_MEMORY_MALLOC_LARGE_REUSED)) {

                                vm_object_t new_object, new_shadow;
 
                                /*
                                 * We're about to map something over a mapping
                                 * established by malloc()...
                                 */
                                vm_object_t new_object, new_shadow;
 
                                /*
                                 * We're about to map something over a mapping
                                 * established by malloc()...
                                 */

-                               new_object = copy_entry->object.vm_object;
+                               new_object = VME_OBJECT(copy_entry);

                                if (new_object != VM_OBJECT_NULL) {
                                        vm_object_lock_shared(new_object);
                                }
                                if (new_object != VM_OBJECT_NULL) {
                                        vm_object_lock_shared(new_object);
                                }


@@ -7743,23 +8233,22 @@ vm_map_copy_overwrite_aligned(
                                                        vm_map_submap_pmap_clean(
                                                                dst_map, entry->vme_start,
                                                                entry->vme_end,
                                                        vm_map_submap_pmap_clean(
                                                                dst_map, entry->vme_start,
                                                                entry->vme_end,

-                                                               entry->object.sub_map,
-                                                               entry->offset);
+                                                               VME_SUBMAP(entry),
+                                                               VME_OFFSET(entry));

                                                }
                                        } else {
                                                vm_map_submap_pmap_clean(
                                                        dst_map, entry->vme_start, 
                                                        entry->vme_end,
                                                }
                                        } else {
                                                vm_map_submap_pmap_clean(
                                                        dst_map, entry->vme_start, 
                                                        entry->vme_end,

-                                                       entry->object.sub_map,
-                                                       entry->offset);
+                                                       VME_SUBMAP(entry),
+                                                       VME_OFFSET(entry));

                                        }
                                        }

-                                       vm_map_deallocate(
-                                               entry->object.sub_map);
+                                       vm_map_deallocate(VME_SUBMAP(entry));

                                } else {
                                        if(dst_map->mapped_in_other_pmaps) {
                                                vm_object_pmap_protect_options(
                                } else {
                                        if(dst_map->mapped_in_other_pmaps) {
                                                vm_object_pmap_protect_options(

-                                                       entry->object.vm_object,
-                                                       entry->offset,
+                                                       VME_OBJECT(entry),
+                                                       VME_OFFSET(entry),

                                                        entry->vme_end 
                                                        - entry->vme_start,
                                                        PMAP_NULL,
                                                        entry->vme_end 
                                                        - entry->vme_start,
                                                        PMAP_NULL,


@@ -7778,12 +8267,13 @@ vm_map_copy_overwrite_aligned(
                        }
 
                        entry->is_sub_map = FALSE;
                        }
 
                        entry->is_sub_map = FALSE;

-                       entry->object = copy_entry->object;
-                       object = entry->object.vm_object;
+                       VME_OBJECT_SET(entry, VME_OBJECT(copy_entry));
+                       object = VME_OBJECT(entry);

                        entry->needs_copy = copy_entry->needs_copy;
                        entry->wired_count = 0;
                        entry->user_wired_count = 0;
                        entry->needs_copy = copy_entry->needs_copy;
                        entry->wired_count = 0;
                        entry->user_wired_count = 0;

-                       offset = entry->offset = copy_entry->offset;
+                       offset = VME_OFFSET(copy_entry);
+                       VME_OFFSET_SET(entry, offset); 

 
                        vm_map_copy_entry_unlink(copy, copy_entry);
                        vm_map_copy_entry_dispose(copy, copy_entry);
 
                        vm_map_copy_entry_unlink(copy, copy_entry);
                        vm_map_copy_entry_dispose(copy, copy_entry);


@@ -7815,15 +8305,14 @@ vm_map_copy_overwrite_aligned(
 
                slow_copy:
                        if (entry->needs_copy) {
 
                slow_copy:
                        if (entry->needs_copy) {

-                               vm_object_shadow(&entry->object.vm_object,
-                                                &entry->offset,
-                                                (entry->vme_end -
-                                                 entry->vme_start));
+                               VME_OBJECT_SHADOW(entry,
+                                                 (entry->vme_end -
+                                                  entry->vme_start));

                                entry->needs_copy = FALSE;
                        }
 
                                entry->needs_copy = FALSE;
                        }
 

-                       dst_object = entry->object.vm_object;
-                       dst_offset = entry->offset;
+                       dst_object = VME_OBJECT(entry);
+                       dst_offset = VME_OFFSET(entry);

 
                        /*
                         *      Take an object reference, and record
 
                        /*
                         *      Take an object reference, and record


@@ -7847,8 +8336,8 @@ vm_map_copy_overwrite_aligned(
                                dst_object = vm_object_allocate(
                                        entry->vme_end - entry->vme_start);
                                dst_offset = 0;
                                dst_object = vm_object_allocate(
                                        entry->vme_end - entry->vme_start);
                                dst_offset = 0;

-                               entry->object.vm_object = dst_object;
-                               entry->offset = dst_offset;
+                               VME_OBJECT_SET(entry, dst_object);
+                               VME_OFFSET_SET(entry, dst_offset);

                                assert(entry->use_pmap);
                                
                        }
                                assert(entry->use_pmap);
                                
                        }


@@ -7866,8 +8355,8 @@ vm_map_copy_overwrite_aligned(
 
                        copy_size = size;
                        r = vm_fault_copy(
 
                        copy_size = size;
                        r = vm_fault_copy(

-                               copy_entry->object.vm_object,
-                               copy_entry->offset,
+                               VME_OBJECT(copy_entry),
+                               VME_OFFSET(copy_entry),

                                &copy_size,
                                dst_object,
                                dst_offset,
                                &copy_size,
                                dst_object,
                                dst_offset,


@@ -7896,7 +8385,7 @@ vm_map_copy_overwrite_aligned(
                                vm_map_copy_clip_end(copy, copy_entry,
                                                     copy_entry->vme_start + copy_size);
                                vm_map_copy_entry_unlink(copy, copy_entry);
                                vm_map_copy_clip_end(copy, copy_entry,
                                                     copy_entry->vme_start + copy_size);
                                vm_map_copy_entry_unlink(copy, copy_entry);

-                               vm_object_deallocate(copy_entry->object.vm_object);
+                               vm_object_deallocate(VME_OBJECT(copy_entry));

                                vm_map_copy_entry_dispose(copy, copy_entry);
                        }
 
                                vm_map_copy_entry_dispose(copy, copy_entry);
                        }
 


@@ -7966,24 +8455,19 @@ vm_map_copyin_kernel_buffer(
        vm_map_copy_t copy;
        vm_size_t kalloc_size;
 
        vm_map_copy_t copy;
        vm_size_t kalloc_size;
 

-       if ((vm_size_t) len != len) {
-               /* "len" is too big and doesn't fit in a "vm_size_t" */
-               return KERN_RESOURCE_SHORTAGE;
-       }
-       kalloc_size = (vm_size_t) (sizeof(struct vm_map_copy) + len);
-       assert((vm_map_size_t) kalloc_size == sizeof (struct vm_map_copy) + len);
+       if (len > msg_ool_size_small)
+               return KERN_INVALID_ARGUMENT;

 
 

-       copy = (vm_map_copy_t) kalloc(kalloc_size);
-       if (copy == VM_MAP_COPY_NULL) {
+       kalloc_size = (vm_size_t)(cpy_kdata_hdr_sz + len);
+
+       copy = (vm_map_copy_t)kalloc(kalloc_size);
+       if (copy == VM_MAP_COPY_NULL)

                return KERN_RESOURCE_SHORTAGE;
                return KERN_RESOURCE_SHORTAGE;

-       }

        copy->type = VM_MAP_COPY_KERNEL_BUFFER;
        copy->size = len;
        copy->offset = 0;
        copy->type = VM_MAP_COPY_KERNEL_BUFFER;
        copy->size = len;
        copy->offset = 0;

-       copy->cpy_kdata = (void *) (copy + 1);
-       copy->cpy_kalloc_size = kalloc_size;

 
 

-       kr = copyinmap(src_map, src_addr, copy->cpy_kdata, (vm_size_t) len);
+       kr = copyinmap(src_map, src_addr, copy->cpy_kdata, (vm_size_t)len);

        if (kr != KERN_SUCCESS) {
                kfree(copy, kalloc_size);
                return kr;
        if (kr != KERN_SUCCESS) {
                kfree(copy, kalloc_size);
                return kr;


@@ -8026,6 +8510,13 @@ vm_map_copyout_kernel_buffer(
        kern_return_t kr = KERN_SUCCESS;
        thread_t thread = current_thread();
 
        kern_return_t kr = KERN_SUCCESS;
        thread_t thread = current_thread();
 

+       /*
+        * check for corrupted vm_map_copy structure
+        */
+       if (copy->size > msg_ool_size_small || copy->offset)
+               panic("Invalid vm_map_copy_t sz:%lld, ofst:%lld",
+                     (long long)copy->size, (long long)copy->offset);
+

        if (!overwrite) {
 
                /*
        if (!overwrite) {
 
                /*


@@ -8103,7 +8594,7 @@ vm_map_copyout_kernel_buffer(
        } else {
                /* copy was successful, dicard the copy structure */
                if (consume_on_success) {
        } else {
                /* copy was successful, dicard the copy structure */
                if (consume_on_success) {

-                       kfree(copy, copy->cpy_kalloc_size);
+                       kfree(copy, copy->size + cpy_kdata_hdr_sz);

                }
        }
 
                }
        }
 


@@ -8158,11 +8649,11 @@ vm_map_copy_remap(
                /* take an extra reference on the entry's "object" */
                if (new_entry->is_sub_map) {
                        assert(!new_entry->use_pmap); /* not nested */
                /* take an extra reference on the entry's "object" */
                if (new_entry->is_sub_map) {
                        assert(!new_entry->use_pmap); /* not nested */

-                       vm_map_lock(new_entry->object.sub_map);
-                       vm_map_reference(new_entry->object.sub_map);
-                       vm_map_unlock(new_entry->object.sub_map);
+                       vm_map_lock(VME_SUBMAP(new_entry));
+                       vm_map_reference(VME_SUBMAP(new_entry));
+                       vm_map_unlock(VME_SUBMAP(new_entry));

                } else {
                } else {

-                       vm_object_reference(new_entry->object.vm_object);
+                       vm_object_reference(VME_OBJECT(new_entry));

                }
                /* insert the new entry in the map */
                vm_map_store_entry_link(map, where, new_entry);
                }
                /* insert the new entry in the map */
                vm_map_store_entry_link(map, where, new_entry);


@@ -8171,6 +8662,39 @@ vm_map_copy_remap(
        }
 }
 
        }
 }
 

+
+boolean_t
+vm_map_copy_validate_size(
+       vm_map_t                dst_map,
+       vm_map_copy_t           copy,
+       vm_map_size_t           size)
+{
+       if (copy == VM_MAP_COPY_NULL)
+               return FALSE;
+       switch (copy->type) {
+       case VM_MAP_COPY_OBJECT:
+       case VM_MAP_COPY_KERNEL_BUFFER:
+               if (size == copy->size)
+                       return TRUE;
+               break;
+       case VM_MAP_COPY_ENTRY_LIST:
+               /*
+                * potential page-size rounding prevents us from exactly
+                * validating this flavor of vm_map_copy, but we can at least
+                * assert that it's within a range.
+                */
+               if (copy->size >= size &&
+                   copy->size <= vm_map_round_page(size,
+                                                   VM_MAP_PAGE_MASK(dst_map)))
+                       return TRUE;
+               break;
+       default:
+               break;
+       }
+       return FALSE;
+}
+
+

 /*
  *     Routine:        vm_map_copyout
  *
 /*
  *     Routine:        vm_map_copyout
  *


@@ -8211,6 +8735,7 @@ vm_map_copyout_internal(
        vm_object_offset_t      vm_copy_start;
        vm_map_entry_t          last;
        vm_map_entry_t          entry;
        vm_object_offset_t      vm_copy_start;
        vm_map_entry_t          last;
        vm_map_entry_t          entry;

+       vm_map_entry_t          hole_entry;

 
        /*
         *      Check for null copy object.
 
        /*
         *      Check for null copy object.


@@ -8281,9 +8806,24 @@ StartAgain: ;
                VM_MAP_HIGHEST_ENTRY(dst_map, entry, start);
                last = entry;
        } else {
                VM_MAP_HIGHEST_ENTRY(dst_map, entry, start);
                last = entry;
        } else {

-               assert(first_free_is_valid(dst_map));
-               start = ((last = dst_map->first_free) == vm_map_to_entry(dst_map)) ?
-               vm_map_min(dst_map) : last->vme_end;
+               if (dst_map->holelistenabled) {
+                       hole_entry = (vm_map_entry_t)dst_map->holes_list;
+
+                       if (hole_entry == NULL) {
+                               /*
+                                * No more space in the map?
+                                */
+                               vm_map_unlock(dst_map);
+                               return(KERN_NO_SPACE);
+                       }
+
+                       last = hole_entry;
+                       start = last->vme_start;
+               } else {
+                       assert(first_free_is_valid(dst_map));
+                       start = ((last = dst_map->first_free) == vm_map_to_entry(dst_map)) ?
+                       vm_map_min(dst_map) : last->vme_end;
+               }

                start = vm_map_round_page(start,
                                          VM_MAP_PAGE_MASK(dst_map));
        }
                start = vm_map_round_page(start,
                                          VM_MAP_PAGE_MASK(dst_map));
        }


@@ -8306,16 +8846,51 @@ StartAgain: ;
                        return(KERN_NO_SPACE);
                }
 
                        return(KERN_NO_SPACE);
                }
 

-               if ((next == vm_map_to_entry(dst_map)) ||
-                   (next->vme_start >= end))
-                       break;
+               if (dst_map->holelistenabled) {
+                       if (last->vme_end >= end)
+                               break;
+               } else {
+                       /*
+                        *      If there are no more entries, we must win.
+                        *
+                        *      OR
+                        *
+                        *      If there is another entry, it must be
+                        *      after the end of the potential new region.
+                        */
+
+                       if (next == vm_map_to_entry(dst_map))
+                               break;
+
+                       if (next->vme_start >= end)
+                               break;
+               }

 
                last = next;
 
                last = next;

-               start = last->vme_end;
+
+               if (dst_map->holelistenabled) {
+                       if (last == (vm_map_entry_t) dst_map->holes_list) {
+                               /*
+                                * Wrapped around
+                                */
+                               vm_map_unlock(dst_map);
+                               return(KERN_NO_SPACE);
+                       }
+                       start = last->vme_start;
+               } else {
+                       start = last->vme_end;
+               }

                start = vm_map_round_page(start,
                                          VM_MAP_PAGE_MASK(dst_map));
        }
 
                start = vm_map_round_page(start,
                                          VM_MAP_PAGE_MASK(dst_map));
        }
 

+       if (dst_map->holelistenabled) {
+               if (vm_map_lookup_entry(dst_map, last->vme_start, &last)) {
+                       panic("Found an existing entry (%p) instead of potential hole at address: 0x%llx.\n", last, (unsigned long long)last->vme_start);
+               }
+       }
+
+

        adjustment = start - vm_copy_start;
        if (! consume_on_success) {
                /*
        adjustment = start - vm_copy_start;
        if (! consume_on_success) {
                /*


@@ -8423,8 +8998,8 @@ StartAgain: ;
                        vm_prot_t prot;
                        int     type_of_fault;
 
                        vm_prot_t prot;
                        int     type_of_fault;
 

-                       object = entry->object.vm_object;
-                       offset = entry->offset;
+                       object = VME_OBJECT(entry);
+                       offset = VME_OFFSET(entry);

                        va = entry->vme_start;
 
                        pmap_pageable(dst_map->pmap,
                        va = entry->vme_start;
 
                        pmap_pageable(dst_map->pmap,


@@ -8471,14 +9046,15 @@ StartAgain: ;
 
                                prot = entry->protection;
 
 
                                prot = entry->protection;
 

-                               if (override_nx(dst_map, entry->alias) && prot)
+                               if (override_nx(dst_map, VME_ALIAS(entry)) &&
+                                   prot)

                                        prot |= VM_PROT_EXECUTE;
 
                                type_of_fault = DBG_CACHE_HIT_FAULT;
 
                                vm_fault_enter(m, dst_map->pmap, va, prot, prot,
                                               VM_PAGE_WIRED(m), FALSE, FALSE,
                                        prot |= VM_PROT_EXECUTE;
 
                                type_of_fault = DBG_CACHE_HIT_FAULT;
 
                                vm_fault_enter(m, dst_map->pmap, va, prot, prot,
                                               VM_PAGE_WIRED(m), FALSE, FALSE,

-                                              FALSE, entry->alias,
+                                              FALSE, VME_ALIAS(entry),

                                               ((entry->iokit_acct ||
                                                 (!entry->is_sub_map &&
                                                  !entry->use_pmap))
                                               ((entry->iokit_acct ||
                                                 (!entry->is_sub_map &&
                                                  !entry->use_pmap))


@@ -8593,13 +9169,35 @@ vm_map_copyin_common(
        __unused boolean_t      src_volatile,
        vm_map_copy_t   *copy_result,   /* OUT */
        boolean_t       use_maxprot)
        __unused boolean_t      src_volatile,
        vm_map_copy_t   *copy_result,   /* OUT */
        boolean_t       use_maxprot)

+{
+       int flags;
+
+       flags = 0;
+       if (src_destroy) {
+               flags |= VM_MAP_COPYIN_SRC_DESTROY;
+       }
+       if (use_maxprot) {
+               flags |= VM_MAP_COPYIN_USE_MAXPROT;
+       }
+       return vm_map_copyin_internal(src_map,
+                                     src_addr,
+                                     len,
+                                     flags,
+                                     copy_result);
+}
+kern_return_t
+vm_map_copyin_internal(
+       vm_map_t        src_map,
+       vm_map_address_t src_addr,
+       vm_map_size_t   len,
+       int             flags,
+       vm_map_copy_t   *copy_result)   /* OUT */

 {
        vm_map_entry_t  tmp_entry;      /* Result of last map lookup --
                                         * in multi-level lookup, this
                                         * entry contains the actual
                                         * vm_object/offset.
                                         */
 {
        vm_map_entry_t  tmp_entry;      /* Result of last map lookup --
                                         * in multi-level lookup, this
                                         * entry contains the actual
                                         * vm_object/offset.
                                         */

-       register

        vm_map_entry_t  new_entry = VM_MAP_ENTRY_NULL;  /* Map entry for copy */
 
        vm_map_offset_t src_start;      /* Start of current entry --
        vm_map_entry_t  new_entry = VM_MAP_ENTRY_NULL;  /* Map entry for copy */
 
        vm_map_offset_t src_start;      /* Start of current entry --


@@ -8612,10 +9210,18 @@ vm_map_copyin_common(
        boolean_t       map_share=FALSE;
        submap_map_t    *parent_maps = NULL;
 
        boolean_t       map_share=FALSE;
        submap_map_t    *parent_maps = NULL;
 

-       register

        vm_map_copy_t   copy;           /* Resulting copy */
        vm_map_address_t copy_addr;
        vm_map_size_t   copy_size;
        vm_map_copy_t   copy;           /* Resulting copy */
        vm_map_address_t copy_addr;
        vm_map_size_t   copy_size;

+       boolean_t       src_destroy;
+       boolean_t       use_maxprot;
+
+       if (flags & ~VM_MAP_COPYIN_ALL_FLAGS) {
+               return KERN_INVALID_ARGUMENT;
+       }
+               
+       src_destroy = (flags & VM_MAP_COPYIN_SRC_DESTROY) ? TRUE : FALSE;
+       use_maxprot = (flags & VM_MAP_COPYIN_USE_MAXPROT) ? TRUE : FALSE;

 
        /*
         *      Check for copies of zero bytes.
 
        /*
         *      Check for copies of zero bytes.


@@ -8639,7 +9245,9 @@ vm_map_copyin_common(
         * setting up VM (and taking C-O-W faults) dominates the copy costs
         * for small regions.
         */
         * setting up VM (and taking C-O-W faults) dominates the copy costs
         * for small regions.
         */

-       if ((len < msg_ool_size_small) && !use_maxprot)
+       if ((len < msg_ool_size_small) &&
+           !use_maxprot &&
+           !(flags & VM_MAP_COPYIN_ENTRY_LIST))

                return vm_map_copyin_kernel_buffer(src_map, src_addr, len,
                                                   src_destroy, copy_result);
 
                return vm_map_copyin_kernel_buffer(src_map, src_addr, len,
                                                   src_destroy, copy_result);
 


@@ -8785,9 +9393,9 @@ vm_map_copyin_common(
                        ptr->base_len = submap_len;
        
                        src_start -= tmp_entry->vme_start;
                        ptr->base_len = submap_len;
        
                        src_start -= tmp_entry->vme_start;

-                       src_start += tmp_entry->offset;
+                       src_start += VME_OFFSET(tmp_entry);

                        src_end = src_start + submap_len;
                        src_end = src_start + submap_len;

-                       src_map = tmp_entry->object.sub_map;
+                       src_map = VME_SUBMAP(tmp_entry);

                        vm_map_lock(src_map);
                        /* keep an outstanding reference for all maps in */
                        /* the parents tree except the base map */
                        vm_map_lock(src_map);
                        /* keep an outstanding reference for all maps in */
                        /* the parents tree except the base map */


@@ -8803,8 +9411,8 @@ vm_map_copyin_common(
                }
                /* we are now in the lowest level submap... */
 
                }
                /* we are now in the lowest level submap... */
 

-               if ((tmp_entry->object.vm_object != VM_OBJECT_NULL) && 
-                   (tmp_entry->object.vm_object->phys_contiguous)) {
+               if ((VME_OBJECT(tmp_entry) != VM_OBJECT_NULL) && 
+                   (VME_OBJECT(tmp_entry)->phys_contiguous)) {

                        /* This is not, supported for now.In future */
                        /* we will need to detect the phys_contig   */
                        /* condition and then upgrade copy_slowly   */
                        /* This is not, supported for now.In future */
                        /* we will need to detect the phys_contig   */
                        /* condition and then upgrade copy_slowly   */


@@ -8853,8 +9461,8 @@ vm_map_copyin_common(
                vm_map_clip_end(src_map, src_entry, src_end);
 
                src_size = src_entry->vme_end - src_start;
                vm_map_clip_end(src_map, src_entry, src_end);
 
                src_size = src_entry->vme_end - src_start;

-               src_object = src_entry->object.vm_object;
-               src_offset = src_entry->offset;
+               src_object = VME_OBJECT(src_entry);
+               src_offset = VME_OFFSET(src_entry);

                was_wired = (src_entry->wired_count != 0);
 
                vm_map_entry_copy(new_entry, src_entry);
                was_wired = (src_entry->wired_count != 0);
 
                vm_map_entry_copy(new_entry, src_entry);


@@ -8892,12 +9500,12 @@ vm_map_copyin_common(
 
        RestartCopy:
                XPR(XPR_VM_MAP, "vm_map_copyin_common src_obj 0x%x ent 0x%x obj 0x%x was_wired %d\n",
 
        RestartCopy:
                XPR(XPR_VM_MAP, "vm_map_copyin_common src_obj 0x%x ent 0x%x obj 0x%x was_wired %d\n",

-                   src_object, new_entry, new_entry->object.vm_object,
+                   src_object, new_entry, VME_OBJECT(new_entry),

                    was_wired, 0);
                if ((src_object == VM_OBJECT_NULL ||
                     (!was_wired && !map_share && !tmp_entry->is_shared)) &&
                    vm_object_copy_quickly(
                    was_wired, 0);
                if ((src_object == VM_OBJECT_NULL ||
                     (!was_wired && !map_share && !tmp_entry->is_shared)) &&
                    vm_object_copy_quickly(

-                           &new_entry->object.vm_object,
+                           &VME_OBJECT(new_entry),

                            src_offset,
                            src_size,
                            &src_needs_copy,
                            src_offset,
                            src_size,
                            &src_needs_copy,


@@ -8914,7 +9522,8 @@ vm_map_copyin_common(
 
                                prot = src_entry->protection & ~VM_PROT_WRITE;
 
 
                                prot = src_entry->protection & ~VM_PROT_WRITE;
 

-                               if (override_nx(src_map, src_entry->alias) && prot)
+                               if (override_nx(src_map, VME_ALIAS(src_entry))
+                                   && prot)

                                        prot |= VM_PROT_EXECUTE;
 
                                vm_object_pmap_protect(
                                        prot |= VM_PROT_EXECUTE;
 
                                vm_object_pmap_protect(


@@ -8927,6 +9536,7 @@ vm_map_copyin_common(
                                        src_entry->vme_start,
                                        prot);
 
                                        src_entry->vme_start,
                                        prot);
 

+                               assert(tmp_entry->wired_count == 0);

                                tmp_entry->needs_copy = TRUE;
                        }
 
                                tmp_entry->needs_copy = TRUE;
                        }
 


@@ -8967,8 +9577,8 @@ vm_map_copyin_common(
                                src_offset,
                                src_size,
                                THREAD_UNINT,
                                src_offset,
                                src_size,
                                THREAD_UNINT,

-                               &new_entry->object.vm_object);
-                       new_entry->offset = 0;
+                               &VME_OBJECT(new_entry));
+                       VME_OFFSET_SET(new_entry, 0);

                        new_entry->needs_copy = FALSE;
 
                }
                        new_entry->needs_copy = FALSE;
 
                }


@@ -8985,7 +9595,8 @@ vm_map_copyin_common(
                        if (new_object == VM_OBJECT_NULL)
                                goto CopySlowly;
 
                        if (new_object == VM_OBJECT_NULL)
                                goto CopySlowly;
 

-                       new_entry->object.vm_object = new_object;
+                       VME_OBJECT_SET(new_entry, new_object);
+                       assert(new_entry->wired_count == 0);

                        new_entry->needs_copy = TRUE;
                        assert(!new_entry->iokit_acct);
                        assert(new_object->purgable == VM_PURGABLE_DENY);
                        new_entry->needs_copy = TRUE;
                        assert(!new_entry->iokit_acct);
                        assert(new_object->purgable == VM_PURGABLE_DENY);


@@ -8993,12 +9604,17 @@ vm_map_copyin_common(
                        result = KERN_SUCCESS;
 
                } else {
                        result = KERN_SUCCESS;
 
                } else {

+                       vm_object_offset_t new_offset;
+                       new_offset = VME_OFFSET(new_entry);

                        result = vm_object_copy_strategically(src_object,
                                                              src_offset,
                                                              src_size,
                        result = vm_object_copy_strategically(src_object,
                                                              src_offset,
                                                              src_size,

-                                                             &new_entry->object.vm_object,
-                                                             &new_entry->offset,
+                                                             &VME_OBJECT(new_entry),
+                                                             &new_offset,

                                                              &new_entry_needs_copy);
                                                              &new_entry_needs_copy);

+                       if (new_offset != VME_OFFSET(new_entry)) {
+                               VME_OFFSET_SET(new_entry, new_offset);
+                       }

 
                        new_entry->needs_copy = new_entry_needs_copy;
                }
 
                        new_entry->needs_copy = new_entry_needs_copy;
                }


@@ -9039,8 +9655,8 @@ vm_map_copyin_common(
 
                if (!vm_map_lookup_entry(src_map, src_start, &tmp_entry)) {
                        if (result != KERN_MEMORY_RESTART_COPY) {
 
                if (!vm_map_lookup_entry(src_map, src_start, &tmp_entry)) {
                        if (result != KERN_MEMORY_RESTART_COPY) {

-                               vm_object_deallocate(new_entry->object.vm_object);
-                               new_entry->object.vm_object = VM_OBJECT_NULL;
+                               vm_object_deallocate(VME_OBJECT(new_entry));
+                               VME_OBJECT_SET(new_entry, VM_OBJECT_NULL);

                                assert(!new_entry->iokit_acct);
                                new_entry->use_pmap = TRUE;
                        }
                                assert(!new_entry->iokit_acct);
                                new_entry->use_pmap = TRUE;
                        }


@@ -9062,8 +9678,8 @@ vm_map_copyin_common(
                        src_size = new_entry->vme_end - src_start;
                }
 
                        src_size = new_entry->vme_end - src_start;
                }
 

-               if ((src_entry->object.vm_object != src_object) ||
-                   (src_entry->offset != src_offset) ) {
+               if ((VME_OBJECT(src_entry) != src_object) ||
+                   (VME_OFFSET(src_entry) != src_offset) ) {

 
                        /*
                         *      Verification failed.
 
                        /*
                         *      Verification failed.


@@ -9073,7 +9689,7 @@ vm_map_copyin_common(
 
                VerificationFailed: ;
 
 
                VerificationFailed: ;
 

-                       vm_object_deallocate(new_entry->object.vm_object);
+                       vm_object_deallocate(VME_OBJECT(new_entry));

                        tmp_entry = src_entry;
                        continue;
                }
                        tmp_entry = src_entry;
                        continue;
                }


@@ -9213,7 +9829,7 @@ vm_map_copyin_common(
                        vm_map_offset_t adjustment;
 
                        original_start = tmp_entry->vme_start;
                        vm_map_offset_t adjustment;
 
                        original_start = tmp_entry->vme_start;

-                       original_offset = tmp_entry->offset;
+                       original_offset = VME_OFFSET(tmp_entry);

 
                        /* map-align the start of the first copy entry... */
                        adjustment = (tmp_entry->vme_start -
 
                        /* map-align the start of the first copy entry... */
                        adjustment = (tmp_entry->vme_start -


@@ -9221,7 +9837,8 @@ vm_map_copyin_common(
                                              tmp_entry->vme_start,
                                              VM_MAP_PAGE_MASK(src_map)));
                        tmp_entry->vme_start -= adjustment;
                                              tmp_entry->vme_start,
                                              VM_MAP_PAGE_MASK(src_map)));
                        tmp_entry->vme_start -= adjustment;

-                       tmp_entry->offset -= adjustment;
+                       VME_OFFSET_SET(tmp_entry,
+                                      VME_OFFSET(tmp_entry) - adjustment);

                        copy_addr -= adjustment;
                        assert(tmp_entry->vme_start < tmp_entry->vme_end);
                        /* ... adjust for mis-aligned start of copy range */
                        copy_addr -= adjustment;
                        assert(tmp_entry->vme_start < tmp_entry->vme_end);
                        /* ... adjust for mis-aligned start of copy range */


@@ -9234,7 +9851,9 @@ vm_map_copyin_common(
                                assert(page_aligned(adjustment));
                                assert(adjustment < VM_MAP_PAGE_SIZE(src_map));
                                tmp_entry->vme_start += adjustment;
                                assert(page_aligned(adjustment));
                                assert(adjustment < VM_MAP_PAGE_SIZE(src_map));
                                tmp_entry->vme_start += adjustment;

-                               tmp_entry->offset += adjustment;
+                               VME_OFFSET_SET(tmp_entry,
+                                              (VME_OFFSET(tmp_entry) +
+                                               adjustment));

                                copy_addr += adjustment;
                                assert(tmp_entry->vme_start < tmp_entry->vme_end);
                        }
                                copy_addr += adjustment;
                                assert(tmp_entry->vme_start < tmp_entry->vme_end);
                        }


@@ -9244,7 +9863,7 @@ vm_map_copyin_common(
                         * more than was originally copied...
                         */
                        assert(tmp_entry->vme_start >= original_start);
                         * more than was originally copied...
                         */
                        assert(tmp_entry->vme_start >= original_start);

-                       assert(tmp_entry->offset >= original_offset);
+                       assert(VME_OFFSET(tmp_entry) >= original_offset);

                        /*
                         * ... and that it did not adjust outside of a
                         * a single 16K page.
                        /*
                         * ... and that it did not adjust outside of a
                         * a single 16K page.


@@ -9478,7 +10097,7 @@ vm_map_fork_share(
         *      make a new shadow and share it.
         */
        
         *      make a new shadow and share it.
         */
        

-       object = old_entry->object.vm_object;
+       object = VME_OBJECT(old_entry);

        if (old_entry->is_sub_map) {
                assert(old_entry->wired_count == 0);
 #ifndef NO_NESTED_PMAP
        if (old_entry->is_sub_map) {
                assert(old_entry->wired_count == 0);
 #ifndef NO_NESTED_PMAP


@@ -9486,7 +10105,7 @@ vm_map_fork_share(
                        kern_return_t   result;
 
                        result = pmap_nest(new_map->pmap, 
                        kern_return_t   result;
 
                        result = pmap_nest(new_map->pmap, 

-                                          (old_entry->object.sub_map)->pmap, 
+                                          (VME_SUBMAP(old_entry))->pmap, 

                                           (addr64_t)old_entry->vme_start,
                                           (addr64_t)old_entry->vme_start,
                                           (uint64_t)(old_entry->vme_end - old_entry->vme_start));
                                           (addr64_t)old_entry->vme_start,
                                           (addr64_t)old_entry->vme_start,
                                           (uint64_t)(old_entry->vme_end - old_entry->vme_start));


@@ -9497,8 +10116,8 @@ vm_map_fork_share(
        } else if (object == VM_OBJECT_NULL) {
                object = vm_object_allocate((vm_map_size_t)(old_entry->vme_end -
                                                            old_entry->vme_start));
        } else if (object == VM_OBJECT_NULL) {
                object = vm_object_allocate((vm_map_size_t)(old_entry->vme_end -
                                                            old_entry->vme_start));

-               old_entry->offset = 0;
-               old_entry->object.vm_object = object;
+               VME_OFFSET_SET(old_entry, 0);
+               VME_OBJECT_SET(old_entry, object);

                old_entry->use_pmap = TRUE;
                assert(!old_entry->needs_copy);
        } else if (object->copy_strategy !=
                old_entry->use_pmap = TRUE;
                assert(!old_entry->needs_copy);
        } else if (object->copy_strategy !=


@@ -9596,10 +10215,9 @@ vm_map_fork_share(
                 *      (This is a preemptive version of
                 *      case 2.)
                 */
                 *      (This is a preemptive version of
                 *      case 2.)
                 */

-               vm_object_shadow(&old_entry->object.vm_object,
-                                &old_entry->offset,
-                                (vm_map_size_t) (old_entry->vme_end -
-                                                 old_entry->vme_start));
+               VME_OBJECT_SHADOW(old_entry,
+                                 (vm_map_size_t) (old_entry->vme_end -
+                                                  old_entry->vme_start));

                
                /*
                 *      If we're making a shadow for other than
                
                /*
                 *      If we're making a shadow for other than


@@ -9613,13 +10231,13 @@ vm_map_fork_share(
 
                        prot = old_entry->protection & ~VM_PROT_WRITE;
 
 
                        prot = old_entry->protection & ~VM_PROT_WRITE;
 

-                       if (override_nx(old_map, old_entry->alias) && prot)
+                       if (override_nx(old_map, VME_ALIAS(old_entry)) && prot)

                                prot |= VM_PROT_EXECUTE;
 
                        if (old_map->mapped_in_other_pmaps) {
                                vm_object_pmap_protect(
                                prot |= VM_PROT_EXECUTE;
 
                        if (old_map->mapped_in_other_pmaps) {
                                vm_object_pmap_protect(

-                                       old_entry->object.vm_object,
-                                       old_entry->offset,
+                                       VME_OBJECT(old_entry),
+                                       VME_OFFSET(old_entry),

                                        (old_entry->vme_end -
                                         old_entry->vme_start),
                                        PMAP_NULL,
                                        (old_entry->vme_end -
                                         old_entry->vme_start),
                                        PMAP_NULL,


@@ -9634,7 +10252,7 @@ vm_map_fork_share(
                }
                
                old_entry->needs_copy = FALSE;
                }
                
                old_entry->needs_copy = FALSE;

-               object = old_entry->object.vm_object;
+               object = VME_OBJECT(old_entry);

        }
 
        
        }
 
        


@@ -9648,9 +10266,9 @@ vm_map_fork_share(
         */
        
        if(old_entry->is_sub_map) {
         */
        
        if(old_entry->is_sub_map) {

-               vm_map_lock(old_entry->object.sub_map);
-               vm_map_reference(old_entry->object.sub_map);
-               vm_map_unlock(old_entry->object.sub_map);
+               vm_map_lock(VME_SUBMAP(old_entry));
+               vm_map_reference(VME_SUBMAP(old_entry));
+               vm_map_unlock(VME_SUBMAP(old_entry));

        } else {
                vm_object_lock(object);
                vm_object_reference_locked(object);
        } else {
                vm_object_lock(object);
                vm_object_reference_locked(object);


@@ -9788,14 +10406,16 @@ vm_map_fork(
        vm_map_entry_t  new_entry;
        boolean_t       src_needs_copy;
        boolean_t       new_entry_needs_copy;
        vm_map_entry_t  new_entry;
        boolean_t       src_needs_copy;
        boolean_t       new_entry_needs_copy;

+       boolean_t       pmap_is64bit;

 
 

-       new_pmap = pmap_create(ledger, (vm_map_size_t) 0,
+       pmap_is64bit =

 #if defined(__i386__) || defined(__x86_64__)
 #if defined(__i386__) || defined(__x86_64__)

-                              old_map->pmap->pm_task_map != TASK_MAP_32BIT
+                              old_map->pmap->pm_task_map != TASK_MAP_32BIT;

 #else
 #error Unknown architecture.
 #endif
 #else
 #error Unknown architecture.
 #endif

-                              );
+
+       new_pmap = pmap_create(ledger, (vm_map_size_t) 0, pmap_is64bit);

 
        vm_map_reference_swap(old_map);
        vm_map_lock(old_map);
 
        vm_map_reference_swap(old_map);
        vm_map_lock(old_map);


@@ -9833,8 +10453,8 @@ vm_map_fork(
                        if(old_entry->is_sub_map)
                                break;
                        if ((old_entry->wired_count != 0) ||
                        if(old_entry->is_sub_map)
                                break;
                        if ((old_entry->wired_count != 0) ||

-                           ((old_entry->object.vm_object != NULL) &&
-                            (old_entry->object.vm_object->true_share))) {
+                           ((VME_OBJECT(old_entry) != NULL) &&
+                            (VME_OBJECT(old_entry)->true_share))) {

                                goto slow_vm_map_fork_copy;
                        }
 
                                goto slow_vm_map_fork_copy;
                        }
 


@@ -9846,8 +10466,8 @@ vm_map_fork(
                        }
 
                        if (! vm_object_copy_quickly(
                        }
 
                        if (! vm_object_copy_quickly(

-                                   &new_entry->object.vm_object,
-                                   old_entry->offset,
+                                   &VME_OBJECT(new_entry),
+                                   VME_OFFSET(old_entry),

                                    (old_entry->vme_end -
                                     old_entry->vme_start),
                                    &src_needs_copy,
                                    (old_entry->vme_end -
                                     old_entry->vme_start),
                                    &src_needs_copy,


@@ -9865,12 +10485,13 @@ vm_map_fork(
 
                                prot = old_entry->protection & ~VM_PROT_WRITE;
 
 
                                prot = old_entry->protection & ~VM_PROT_WRITE;
 

-                               if (override_nx(old_map, old_entry->alias) && prot)
+                               if (override_nx(old_map, VME_ALIAS(old_entry))
+                                   && prot)

                                        prot |= VM_PROT_EXECUTE;
 
                                vm_object_pmap_protect(
                                        prot |= VM_PROT_EXECUTE;
 
                                vm_object_pmap_protect(

-                                       old_entry->object.vm_object,
-                                       old_entry->offset,
+                                       VME_OBJECT(old_entry),
+                                       VME_OFFSET(old_entry),

                                        (old_entry->vme_end -
                                         old_entry->vme_start),
                                        ((old_entry->is_shared 
                                        (old_entry->vme_end -
                                         old_entry->vme_start),
                                        ((old_entry->is_shared 


@@ -9880,6 +10501,7 @@ vm_map_fork(
                                        old_entry->vme_start,
                                        prot);
 
                                        old_entry->vme_start,
                                        prot);
 

+                               assert(old_entry->wired_count == 0);

                                old_entry->needs_copy = TRUE;
                        }
                        new_entry->needs_copy = new_entry_needs_copy;
                                old_entry->needs_copy = TRUE;
                        }
                        new_entry->needs_copy = new_entry_needs_copy;


@@ -10059,7 +10681,7 @@ submap_recurse:
                        if ((*real_map != map) && 
                            (*real_map != cow_sub_map_parent))
                                vm_map_unlock(*real_map);
                        if ((*real_map != map) && 
                            (*real_map != cow_sub_map_parent))
                                vm_map_unlock(*real_map);

-                       *real_map = entry->object.sub_map;
+                       *real_map = VME_SUBMAP(entry);

                }
 
                if(entry->needs_copy && (fault_type & VM_PROT_WRITE)) {
                }
 
                if(entry->needs_copy && (fault_type & VM_PROT_WRITE)) {


@@ -10069,8 +10691,8 @@ submap_recurse:
                                        *real_map = map;
                                        goto RetryLookup;
                                }
                                        *real_map = map;
                                        goto RetryLookup;
                                }

-                               vm_map_lock_read(entry->object.sub_map);
-                               *var_map = entry->object.sub_map;
+                               vm_map_lock_read(VME_SUBMAP(entry));
+                               *var_map = VME_SUBMAP(entry);

                                cow_sub_map_parent = map;
                                /* reset base to map before cow object */
                                /* this is the map which will accept   */
                                cow_sub_map_parent = map;
                                /* reset base to map before cow object */
                                /* this is the map which will accept   */


@@ -10080,15 +10702,15 @@ submap_recurse:
                                cow_parent_vaddr = vaddr;
                                mapped_needs_copy = TRUE;
                        } else {
                                cow_parent_vaddr = vaddr;
                                mapped_needs_copy = TRUE;
                        } else {

-                               vm_map_lock_read(entry->object.sub_map);
-                               *var_map = entry->object.sub_map;
+                               vm_map_lock_read(VME_SUBMAP(entry));
+                               *var_map = VME_SUBMAP(entry);

                                if((cow_sub_map_parent != map) &&
                                   (*real_map != map))
                                        vm_map_unlock(map);
                        }
                } else {
                                if((cow_sub_map_parent != map) &&
                                   (*real_map != map))
                                        vm_map_unlock(map);
                        }
                } else {

-                       vm_map_lock_read(entry->object.sub_map);
-                       *var_map = entry->object.sub_map;       
+                       vm_map_lock_read(VME_SUBMAP(entry));
+                       *var_map = VME_SUBMAP(entry);   

                        /* leave map locked if it is a target */
                        /* cow sub_map above otherwise, just  */
                        /* follow the maps down to the object */
                        /* leave map locked if it is a target */
                        /* cow sub_map above otherwise, just  */
                        /* follow the maps down to the object */


@@ -10101,7 +10723,7 @@ submap_recurse:
                map = *var_map;
 
                /* calculate the offset in the submap for vaddr */
                map = *var_map;
 
                /* calculate the offset in the submap for vaddr */

-               local_vaddr = (local_vaddr - entry->vme_start) + entry->offset;
+               local_vaddr = (local_vaddr - entry->vme_start) + VME_OFFSET(entry);

 
        RetrySubMap:
                if(!vm_map_lookup_entry(map, local_vaddr, &submap_entry)) {
 
        RetrySubMap:
                if(!vm_map_lookup_entry(map, local_vaddr, &submap_entry)) {


@@ -10126,13 +10748,13 @@ submap_recurse:
                /* ultimately be clipped in the top map will only need    */
                /* to be as big as the portion of the underlying entry    */
                /* which is mapped */
                /* ultimately be clipped in the top map will only need    */
                /* to be as big as the portion of the underlying entry    */
                /* which is mapped */

-               start_delta = submap_entry->vme_start > entry->offset ?
-                       submap_entry->vme_start - entry->offset : 0;
+               start_delta = submap_entry->vme_start > VME_OFFSET(entry) ?
+                       submap_entry->vme_start - VME_OFFSET(entry) : 0;

 
                end_delta = 
 
                end_delta = 

-                       (entry->offset + start_delta + (old_end - old_start)) <=
+                       (VME_OFFSET(entry) + start_delta + (old_end - old_start)) <=

                        submap_entry->vme_end ?
                        submap_entry->vme_end ?

-                       0 : (entry->offset + 
+                       0 : (VME_OFFSET(entry) + 

                             (old_end - old_start))
                        - submap_entry->vme_end; 
 
                             (old_end - old_start))
                        - submap_entry->vme_end; 
 


@@ -10161,15 +10783,15 @@ submap_recurse:
                        }
 
 
                        }
 
 

-                       sub_object = submap_entry->object.vm_object;
+                       sub_object = VME_OBJECT(submap_entry);

                        if (sub_object == VM_OBJECT_NULL) {
                                sub_object =
                                        vm_object_allocate(
                                                (vm_map_size_t)
                                                (submap_entry->vme_end -
                                                 submap_entry->vme_start));
                        if (sub_object == VM_OBJECT_NULL) {
                                sub_object =
                                        vm_object_allocate(
                                                (vm_map_size_t)
                                                (submap_entry->vme_end -
                                                 submap_entry->vme_start));

-                               submap_entry->object.vm_object = sub_object;
-                               submap_entry->offset = 0;
+                               VME_OBJECT_SET(submap_entry, sub_object);
+                               VME_OFFSET_SET(submap_entry, 0);

                        }
                        local_start =  local_vaddr - 
                                (cow_parent_vaddr - old_start);
                        }
                        local_start =  local_vaddr - 
                                (cow_parent_vaddr - old_start);


@@ -10193,7 +10815,7 @@ submap_recurse:
                            MEMORY_OBJECT_COPY_NONE)) {
                                vm_object_lock(sub_object);
                                vm_object_copy_slowly(sub_object,
                            MEMORY_OBJECT_COPY_NONE)) {
                                vm_object_lock(sub_object);
                                vm_object_copy_slowly(sub_object,

-                                                     submap_entry->offset,
+                                                     VME_OFFSET(submap_entry),

                                                      (submap_entry->vme_end -
                                                       submap_entry->vme_start),
                                                      FALSE,
                                                      (submap_entry->vme_end -
                                                       submap_entry->vme_start),
                                                      FALSE,


@@ -10205,16 +10827,19 @@ submap_recurse:
                                copy_object = sub_object;
                                vm_object_reference(copy_object);
                                sub_object->shadowed = TRUE;
                                copy_object = sub_object;
                                vm_object_reference(copy_object);
                                sub_object->shadowed = TRUE;

+                               assert(submap_entry->wired_count == 0);

                                submap_entry->needs_copy = TRUE;
 
                                prot = submap_entry->protection & ~VM_PROT_WRITE;
 
                                submap_entry->needs_copy = TRUE;
 
                                prot = submap_entry->protection & ~VM_PROT_WRITE;
 

-                               if (override_nx(old_map, submap_entry->alias) && prot)
+                               if (override_nx(old_map,
+                                               VME_ALIAS(submap_entry))
+                                   && prot)

                                        prot |= VM_PROT_EXECUTE;
 
                                vm_object_pmap_protect(
                                        sub_object,
                                        prot |= VM_PROT_EXECUTE;
 
                                vm_object_pmap_protect(
                                        sub_object,

-                                       submap_entry->offset,
+                                       VME_OFFSET(submap_entry),

                                        submap_entry->vme_end - 
                                        submap_entry->vme_start,
                                        (submap_entry->is_shared 
                                        submap_entry->vme_end - 
                                        submap_entry->vme_start,
                                        (submap_entry->is_shared 


@@ -10229,7 +10854,7 @@ submap_recurse:
                         */
                        copy_offset = (local_vaddr -
                                       submap_entry->vme_start +
                         */
                        copy_offset = (local_vaddr -
                                       submap_entry->vme_start +

-                                      submap_entry->offset);
+                                      VME_OFFSET(submap_entry));

 
                        /* This works diffently than the   */
                        /* normal submap case. We go back  */
 
                        /* This works diffently than the   */
                        /* normal submap case. We go back  */


@@ -10290,22 +10915,23 @@ submap_recurse:
 
                        /* substitute copy object for */
                        /* shared map entry           */
 
                        /* substitute copy object for */
                        /* shared map entry           */

-                       vm_map_deallocate(entry->object.sub_map);
+                       vm_map_deallocate(VME_SUBMAP(entry));

                        assert(!entry->iokit_acct);
                        entry->is_sub_map = FALSE;
                        entry->use_pmap = TRUE;
                        assert(!entry->iokit_acct);
                        entry->is_sub_map = FALSE;
                        entry->use_pmap = TRUE;

-                       entry->object.vm_object = copy_object;
+                       VME_OBJECT_SET(entry, copy_object);

 
                        /* propagate the submap entry's protections */
                        entry->protection |= submap_entry->protection;
                        entry->max_protection |= submap_entry->max_protection;
 
                        if(copied_slowly) {
 
                        /* propagate the submap entry's protections */
                        entry->protection |= submap_entry->protection;
                        entry->max_protection |= submap_entry->max_protection;
 
                        if(copied_slowly) {

-                               entry->offset = local_start - old_start;
+                               VME_OFFSET_SET(entry, local_start - old_start);

                                entry->needs_copy = FALSE;
                                entry->is_shared = FALSE;
                        } else {
                                entry->needs_copy = FALSE;
                                entry->is_shared = FALSE;
                        } else {

-                               entry->offset = copy_offset;
+                               VME_OFFSET_SET(entry, copy_offset);
+                               assert(entry->wired_count == 0);

                                entry->needs_copy = TRUE;
                                if(entry->inheritance == VM_INHERIT_SHARE) 
                                        entry->inheritance = VM_INHERIT_COPY;
                                entry->needs_copy = TRUE;
                                if(entry->inheritance == VM_INHERIT_SHARE) 
                                        entry->inheritance = VM_INHERIT_COPY;


@@ -10334,7 +10960,7 @@ submap_recurse:
 
        prot = entry->protection;
 
 
        prot = entry->protection;
 

-       if (override_nx(old_map, entry->alias) && prot) {
+       if (override_nx(old_map, VME_ALIAS(entry)) && prot) {

                /*
                 * HACK -- if not a stack, then allow execution
                 */
                /*
                 * HACK -- if not a stack, then allow execution
                 */


@@ -10395,12 +11021,11 @@ submap_recurse:
                                vm_map_lock_read(map);
                                goto RetryLookup;
                        }
                                vm_map_lock_read(map);
                                goto RetryLookup;
                        }

-                       vm_object_shadow(&entry->object.vm_object,
-                                        &entry->offset,
-                                        (vm_map_size_t) (entry->vme_end -
-                                                         entry->vme_start));
+                       VME_OBJECT_SHADOW(entry,
+                                         (vm_map_size_t) (entry->vme_end -
+                                                          entry->vme_start));

 
 

-                       entry->object.vm_object->shadowed = TRUE;
+                       VME_OBJECT(entry)->shadowed = TRUE;

                        entry->needs_copy = FALSE;
                        vm_map_lock_write_to_read(map);
                }
                        entry->needs_copy = FALSE;
                        vm_map_lock_write_to_read(map);
                }


@@ -10417,16 +11042,18 @@ submap_recurse:
        /*
         *      Create an object if necessary.
         */
        /*
         *      Create an object if necessary.
         */

-       if (entry->object.vm_object == VM_OBJECT_NULL) {
+       if (VME_OBJECT(entry) == VM_OBJECT_NULL) {

 
                if (vm_map_lock_read_to_write(map)) {
                        vm_map_lock_read(map);
                        goto RetryLookup;
                }
 
 
                if (vm_map_lock_read_to_write(map)) {
                        vm_map_lock_read(map);
                        goto RetryLookup;
                }
 

-               entry->object.vm_object = vm_object_allocate(
-                       (vm_map_size_t)(entry->vme_end - entry->vme_start));
-               entry->offset = 0;
+               VME_OBJECT_SET(entry,
+                              vm_object_allocate(
+                                      (vm_map_size_t)(entry->vme_end -
+                                                      entry->vme_start)));
+               VME_OFFSET_SET(entry, 0);

                vm_map_lock_write_to_read(map);
        }
 
                vm_map_lock_write_to_read(map);
        }
 


@@ -10436,27 +11063,33 @@ submap_recurse:
         *      return the protection.
         */
 
         *      return the protection.
         */
 

-        *offset = (vaddr - entry->vme_start) + entry->offset;
-        *object = entry->object.vm_object;
+        *offset = (vaddr - entry->vme_start) + VME_OFFSET(entry);
+        *object = VME_OBJECT(entry);

        *out_prot = prot;
 
        if (fault_info) {
                fault_info->interruptible = THREAD_UNINT; /* for now... */
                /* ... the caller will change "interruptible" if needed */
                fault_info->cluster_size = 0;
        *out_prot = prot;
 
        if (fault_info) {
                fault_info->interruptible = THREAD_UNINT; /* for now... */
                /* ... the caller will change "interruptible" if needed */
                fault_info->cluster_size = 0;

-               fault_info->user_tag = entry->alias;
+               fault_info->user_tag = VME_ALIAS(entry);

                fault_info->pmap_options = 0;
                if (entry->iokit_acct ||
                    (!entry->is_sub_map && !entry->use_pmap)) {
                        fault_info->pmap_options |= PMAP_OPTIONS_ALT_ACCT;
                }
                fault_info->behavior = entry->behavior;
                fault_info->pmap_options = 0;
                if (entry->iokit_acct ||
                    (!entry->is_sub_map && !entry->use_pmap)) {
                        fault_info->pmap_options |= PMAP_OPTIONS_ALT_ACCT;
                }
                fault_info->behavior = entry->behavior;

-               fault_info->lo_offset = entry->offset;
-               fault_info->hi_offset = (entry->vme_end - entry->vme_start) + entry->offset;
+               fault_info->lo_offset = VME_OFFSET(entry);
+               fault_info->hi_offset =
+                       (entry->vme_end - entry->vme_start) + VME_OFFSET(entry);

                fault_info->no_cache  = entry->no_cache;
                fault_info->stealth = FALSE;
                fault_info->io_sync = FALSE;
                fault_info->no_cache  = entry->no_cache;
                fault_info->stealth = FALSE;
                fault_info->io_sync = FALSE;

-               fault_info->cs_bypass = (entry->used_for_jit)? TRUE : FALSE;
+               if (entry->used_for_jit ||
+                   entry->vme_resilient_codesign) {
+                       fault_info->cs_bypass = TRUE;
+               } else {
+                       fault_info->cs_bypass = FALSE;
+               }

                fault_info->mark_zf_absent = FALSE;
                fault_info->batch_pmap_op = FALSE;
        }
                fault_info->mark_zf_absent = FALSE;
                fault_info->batch_pmap_op = FALSE;
        }


@@ -10612,6 +11245,11 @@ vm_map_region_recurse_64(
        user_address = *address;
        user_max_depth = *nesting_depth;
        
        user_address = *address;
        user_max_depth = *nesting_depth;
        

+       if (not_in_kdp) {
+               vm_map_lock_read(map);
+       }
+
+recurse_again:

        curr_entry = NULL;
        curr_map = map;
        curr_address = user_address;
        curr_entry = NULL;
        curr_map = map;
        curr_address = user_address;


@@ -10630,10 +11268,6 @@ vm_map_region_recurse_64(
        next_max_above = (vm_map_offset_t) -1;
        next_max_below = (vm_map_offset_t) -1;
 
        next_max_above = (vm_map_offset_t) -1;
        next_max_below = (vm_map_offset_t) -1;
 

-       if (not_in_kdp) {
-               vm_map_lock_read(curr_map);
-       }
-

        for (;;) {
                if (vm_map_lookup_entry(curr_map,
                                        curr_address,
        for (;;) {
                if (vm_map_lookup_entry(curr_map,
                                        curr_address,


@@ -10658,6 +11292,7 @@ vm_map_region_recurse_64(
                                }
                                curr_entry = NULL;
                                curr_map = NULL;
                                }
                                curr_entry = NULL;
                                curr_map = NULL;

+                               curr_skip = 0;

                                curr_offset = 0;
                                curr_depth = 0;
                                curr_max_above = 0;
                                curr_offset = 0;
                                curr_depth = 0;
                                curr_max_above = 0;


@@ -10668,7 +11303,7 @@ vm_map_region_recurse_64(
                        /* adjust current address and offset */
                        skip = curr_entry->vme_start - curr_address;
                        curr_address = curr_entry->vme_start;
                        /* adjust current address and offset */
                        skip = curr_entry->vme_start - curr_address;
                        curr_address = curr_entry->vme_start;

-                       curr_skip = skip;
+                       curr_skip += skip;

                        curr_offset += skip;
                        curr_max_above -= skip;
                        curr_max_below = 0;
                        curr_offset += skip;
                        curr_max_above -= skip;
                        curr_max_below = 0;


@@ -10707,6 +11342,7 @@ vm_map_region_recurse_64(
                        next_depth = curr_depth;
                        next_address = next_entry->vme_start;
                        next_skip = curr_skip;
                        next_depth = curr_depth;
                        next_address = next_entry->vme_start;
                        next_skip = curr_skip;

+                       next_skip += (next_address - curr_address);

                        next_offset = curr_offset;
                        next_offset += (next_address - curr_address);
                        next_max_above = MIN(next_max_above, curr_max_above);
                        next_offset = curr_offset;
                        next_offset += (next_address - curr_address);
                        next_max_above = MIN(next_max_above, curr_max_above);


@@ -10723,7 +11359,7 @@ vm_map_region_recurse_64(
                 * the rest of that submap is irrelevant to us, since it's not
                 * mapped here.
                 * The relevant portion of the map starts at
                 * the rest of that submap is irrelevant to us, since it's not
                 * mapped here.
                 * The relevant portion of the map starts at

-                * "curr_entry->offset" up to the size of "curr_entry".
+                * "VME_OFFSET(curr_entry)" up to the size of "curr_entry".

                 */
                curr_max_above = MIN(curr_max_above,
                                     curr_entry->vme_end - curr_address);
                 */
                curr_max_above = MIN(curr_max_above,
                                     curr_entry->vme_end - curr_address);


@@ -10750,7 +11386,7 @@ vm_map_region_recurse_64(
                 * later.
                 */
                if (not_in_kdp) {
                 * later.
                 */
                if (not_in_kdp) {

-                       vm_map_lock_read(curr_entry->object.sub_map);
+                       vm_map_lock_read(VME_SUBMAP(curr_entry));

                }
                if (curr_map == next_map) {
                        /* keep "next_map" locked in case we need it */
                }
                if (curr_map == next_map) {
                        /* keep "next_map" locked in case we need it */


@@ -10763,17 +11399,17 @@ vm_map_region_recurse_64(
                /*
                 * Adjust the offset.  "curr_entry" maps the submap
                 * at relative address "curr_entry->vme_start" in the
                /*
                 * Adjust the offset.  "curr_entry" maps the submap
                 * at relative address "curr_entry->vme_start" in the

-                * curr_map but skips the first "curr_entry->offset"
+                * curr_map but skips the first "VME_OFFSET(curr_entry)"

                 * bytes of the submap.
                 * "curr_offset" always represents the offset of a virtual
                 * address in the curr_map relative to the absolute address
                 * space (i.e. the top-level VM map).
                 */
                curr_offset +=
                 * bytes of the submap.
                 * "curr_offset" always represents the offset of a virtual
                 * address in the curr_map relative to the absolute address
                 * space (i.e. the top-level VM map).
                 */
                curr_offset +=

-                       (curr_entry->offset - curr_entry->vme_start);
+                       (VME_OFFSET(curr_entry) - curr_entry->vme_start);

                curr_address = user_address + curr_offset;
                /* switch to the submap */
                curr_address = user_address + curr_offset;
                /* switch to the submap */

-               curr_map = curr_entry->object.sub_map;
+               curr_map = VME_SUBMAP(curr_entry);

                curr_depth++;
                curr_entry = NULL;
        }
                curr_depth++;
                curr_entry = NULL;
        }


@@ -10793,9 +11429,6 @@ vm_map_region_recurse_64(
                curr_depth = next_depth;
                curr_max_above = next_max_above;
                curr_max_below = next_max_below;
                curr_depth = next_depth;
                curr_max_above = next_max_above;
                curr_max_below = next_max_below;

-               if (curr_map == map) {
-                       user_address = curr_address;
-               }

        } else {
                /* we won't need "next_entry" after all */
                if (next_entry != NULL) {
        } else {
                /* we won't need "next_entry" after all */
                if (next_entry != NULL) {


@@ -10813,6 +11446,18 @@ vm_map_region_recurse_64(
        next_max_below = -1;
        next_max_above = -1;
 
        next_max_below = -1;
        next_max_above = -1;
 

+       if (curr_entry->is_sub_map &&
+           curr_depth < user_max_depth) {
+               /*
+                * We're not as deep as we could be:  we must have
+                * gone back up after not finding anything mapped
+                * below the original top-level map entry's.
+                * Let's move "curr_address" forward and recurse again.
+                */
+               user_address = curr_address;
+               goto recurse_again;
+       }
+

        *nesting_depth = curr_depth;
        *size = curr_max_above + curr_max_below;
        *address = user_address + curr_skip - curr_max_below;
        *nesting_depth = curr_depth;
        *size = curr_max_above + curr_max_below;
        *address = user_address + curr_skip - curr_max_below;


@@ -10823,25 +11468,25 @@ vm_map_region_recurse_64(
 #define INFO_MAKE_OBJECT_ID(p) ((uint32_t)(uintptr_t)VM_KERNEL_ADDRPERM(p))
 
        if (look_for_pages) {
 #define INFO_MAKE_OBJECT_ID(p) ((uint32_t)(uintptr_t)VM_KERNEL_ADDRPERM(p))
 
        if (look_for_pages) {

-               submap_info->user_tag = curr_entry->alias;
-               submap_info->offset = curr_entry->offset; 
+               submap_info->user_tag = VME_ALIAS(curr_entry);
+               submap_info->offset = VME_OFFSET(curr_entry); 

                submap_info->protection = curr_entry->protection;
                submap_info->inheritance = curr_entry->inheritance;
                submap_info->max_protection = curr_entry->max_protection;
                submap_info->behavior = curr_entry->behavior;
                submap_info->user_wired_count = curr_entry->user_wired_count;
                submap_info->is_submap = curr_entry->is_sub_map;
                submap_info->protection = curr_entry->protection;
                submap_info->inheritance = curr_entry->inheritance;
                submap_info->max_protection = curr_entry->max_protection;
                submap_info->behavior = curr_entry->behavior;
                submap_info->user_wired_count = curr_entry->user_wired_count;
                submap_info->is_submap = curr_entry->is_sub_map;

-               submap_info->object_id = INFO_MAKE_OBJECT_ID(curr_entry->object.vm_object);
+               submap_info->object_id = INFO_MAKE_OBJECT_ID(VME_OBJECT(curr_entry));

        } else {
        } else {

-               short_info->user_tag = curr_entry->alias;
-               short_info->offset = curr_entry->offset; 
+               short_info->user_tag = VME_ALIAS(curr_entry);
+               short_info->offset = VME_OFFSET(curr_entry); 

                short_info->protection = curr_entry->protection;
                short_info->inheritance = curr_entry->inheritance;
                short_info->max_protection = curr_entry->max_protection;
                short_info->behavior = curr_entry->behavior;
                short_info->user_wired_count = curr_entry->user_wired_count;
                short_info->is_submap = curr_entry->is_sub_map;
                short_info->protection = curr_entry->protection;
                short_info->inheritance = curr_entry->inheritance;
                short_info->max_protection = curr_entry->max_protection;
                short_info->behavior = curr_entry->behavior;
                short_info->user_wired_count = curr_entry->user_wired_count;
                short_info->is_submap = curr_entry->is_sub_map;

-               short_info->object_id = INFO_MAKE_OBJECT_ID(curr_entry->object.vm_object);
+               short_info->object_id = INFO_MAKE_OBJECT_ID(VME_OBJECT(curr_entry));

        }
 
        extended.pages_resident = 0;
        }
 
        extended.pages_resident = 0;


@@ -10851,6 +11496,8 @@ vm_map_region_recurse_64(
        extended.pages_reusable = 0;
        extended.external_pager = 0;
        extended.shadow_depth = 0;
        extended.pages_reusable = 0;
        extended.external_pager = 0;
        extended.shadow_depth = 0;

+       extended.share_mode = SM_EMPTY;
+       extended.ref_count = 0;

 
        if (not_in_kdp) {
                if (!curr_entry->is_sub_map) {
 
        if (not_in_kdp) {
                if (!curr_entry->is_sub_map) {


@@ -10862,7 +11509,7 @@ vm_map_region_recurse_64(
                        vm_map_region_walk(curr_map,
                                           range_start,
                                           curr_entry,
                        vm_map_region_walk(curr_map,
                                           range_start,
                                           curr_entry,

-                                          (curr_entry->offset +
+                                          (VME_OFFSET(curr_entry) +

                                            (range_start -
                                             curr_entry->vme_start)),
                                           range_end - range_start,
                                            (range_start -
                                             curr_entry->vme_start)),
                                           range_end - range_start,


@@ -10879,8 +11526,7 @@ vm_map_region_recurse_64(
                        } else {
                                extended.share_mode = SM_PRIVATE;
                        }
                        } else {
                                extended.share_mode = SM_PRIVATE;
                        }

-                       extended.ref_count =
-                               curr_entry->object.sub_map->ref_count;
+                       extended.ref_count = VME_SUBMAP(curr_entry)->ref_count;

                }
        }
 
                }
        }
 


@@ -10968,7 +11614,7 @@ vm_map_region(
 
                start = entry->vme_start;
 
 
                start = entry->vme_start;
 

-               basic->offset = (uint32_t)entry->offset;
+               basic->offset = (uint32_t)VME_OFFSET(entry);

                basic->protection = entry->protection;
                basic->inheritance = entry->inheritance;
                basic->max_protection = entry->max_protection;
                basic->protection = entry->protection;
                basic->inheritance = entry->inheritance;
                basic->max_protection = entry->max_protection;


@@ -11013,7 +11659,7 @@ vm_map_region(
 
                start = entry->vme_start;
 
 
                start = entry->vme_start;
 

-               basic->offset = entry->offset;
+               basic->offset = VME_OFFSET(entry);

                basic->protection = entry->protection;
                basic->inheritance = entry->inheritance;
                basic->max_protection = entry->max_protection;
                basic->protection = entry->protection;
                basic->inheritance = entry->inheritance;
                basic->max_protection = entry->max_protection;


@@ -11061,7 +11707,7 @@ vm_map_region(
                start = entry->vme_start;
 
                extended->protection = entry->protection;
                start = entry->vme_start;
 
                extended->protection = entry->protection;

-               extended->user_tag = entry->alias;
+               extended->user_tag = VME_ALIAS(entry);

                extended->pages_resident = 0;
                extended->pages_swapped_out = 0;
                extended->pages_shared_now_private = 0;
                extended->pages_resident = 0;
                extended->pages_swapped_out = 0;
                extended->pages_shared_now_private = 0;


@@ -11077,7 +11723,7 @@ vm_map_region(
                        *count = VM_REGION_EXTENDED_INFO_COUNT;
                }
 
                        *count = VM_REGION_EXTENDED_INFO_COUNT;
                }
 

-               vm_map_region_walk(map, start, entry, entry->offset, entry->vme_end - start, extended, TRUE, *count);
+               vm_map_region_walk(map, start, entry, VME_OFFSET(entry), entry->vme_end - start, extended, TRUE, *count);

 
                if (extended->external_pager && extended->ref_count == 2 && extended->share_mode == SM_SHARED)
                        extended->share_mode = SM_PRIVATE;
 
                if (extended->external_pager && extended->ref_count == 2 && extended->share_mode == SM_SHARED)
                        extended->share_mode = SM_PRIVATE;


@@ -11144,7 +11790,7 @@ vm_map_region_top_walk(
        vm_region_top_info_t       top)
 {
 
        vm_region_top_info_t       top)
 {
 

-       if (entry->object.vm_object == 0 || entry->is_sub_map) {
+       if (VME_OBJECT(entry) == 0 || entry->is_sub_map) {

                top->share_mode = SM_EMPTY;
                top->ref_count = 0;
                top->obj_id = 0;
                top->share_mode = SM_EMPTY;
                top->ref_count = 0;
                top->obj_id = 0;


@@ -11158,7 +11804,7 @@ vm_map_region_top_walk(
 
                entry_size = (uint32_t) ((entry->vme_end - entry->vme_start) / PAGE_SIZE_64);
 
 
                entry_size = (uint32_t) ((entry->vme_end - entry->vme_start) / PAGE_SIZE_64);
 

-               obj = entry->object.vm_object;
+               obj = VME_OBJECT(entry);

 
                vm_object_lock(obj);
 
 
                vm_object_lock(obj);
 


@@ -11239,9 +11885,9 @@ vm_map_region_walk(
        struct vm_object        *shadow_object;
        int                     shadow_depth;
 
        struct vm_object        *shadow_object;
        int                     shadow_depth;
 

-       if ((entry->object.vm_object == 0) ||
+       if ((VME_OBJECT(entry) == 0) ||

            (entry->is_sub_map) ||
            (entry->is_sub_map) ||

-           (entry->object.vm_object->phys_contiguous &&
+           (VME_OBJECT(entry)->phys_contiguous &&

             !entry->superpage_size)) {
                extended->share_mode = SM_EMPTY;
                extended->ref_count = 0;
             !entry->superpage_size)) {
                extended->share_mode = SM_EMPTY;
                extended->ref_count = 0;


@@ -11259,7 +11905,7 @@ vm_map_region_walk(
        }
 
        {
        }
 
        {

-               obj = entry->object.vm_object;
+               obj = VME_OBJECT(entry);

 
                vm_object_lock(obj);
 
 
                vm_object_lock(obj);
 


@@ -11336,7 +11982,7 @@ vm_map_region_walk(
                        register vm_map_entry_t      last;
                        int      my_refs;
 
                        register vm_map_entry_t      last;
                        int      my_refs;
 

-                       obj = entry->object.vm_object;
+                       obj = VME_OBJECT(entry);

                        last = vm_map_to_entry(map);
                        my_refs = 0;
 
                        last = vm_map_to_entry(map);
                        my_refs = 0;
 


@@ -11492,7 +12138,7 @@ vm_map_region_count_obj_refs(
        register vm_object_t chk_obj;
        register vm_object_t tmp_obj;
 
        register vm_object_t chk_obj;
        register vm_object_t tmp_obj;
 

-       if (entry->object.vm_object == 0)
+       if (VME_OBJECT(entry) == 0)

                return(0);
 
         if (entry->is_sub_map)
                return(0);
 
         if (entry->is_sub_map)


@@ -11500,7 +12146,7 @@ vm_map_region_count_obj_refs(
        else {
                ref_count = 0;
 
        else {
                ref_count = 0;
 

-               chk_obj = entry->object.vm_object;
+               chk_obj = VME_OBJECT(entry);

                vm_object_lock(chk_obj);
 
                while (chk_obj) {
                vm_object_lock(chk_obj);
 
                while (chk_obj) {


@@ -11548,10 +12194,10 @@ vm_map_simplify_entry(
            (prev_entry->vme_end == this_entry->vme_start) &&
 
            (prev_entry->is_sub_map == this_entry->is_sub_map) &&
            (prev_entry->vme_end == this_entry->vme_start) &&
 
            (prev_entry->is_sub_map == this_entry->is_sub_map) &&

-           (prev_entry->object.vm_object == this_entry->object.vm_object) &&
-           ((prev_entry->offset + (prev_entry->vme_end -
+           (VME_OBJECT(prev_entry) == VME_OBJECT(this_entry)) &&
+           ((VME_OFFSET(prev_entry) + (prev_entry->vme_end -

                                    prev_entry->vme_start))
                                    prev_entry->vme_start))

-            == this_entry->offset) &&
+            == VME_OFFSET(this_entry)) &&

 
            (prev_entry->behavior == this_entry->behavior) &&
            (prev_entry->needs_copy == this_entry->needs_copy) &&
 
            (prev_entry->behavior == this_entry->behavior) &&
            (prev_entry->needs_copy == this_entry->needs_copy) &&


@@ -11559,7 +12205,7 @@ vm_map_simplify_entry(
            (prev_entry->max_protection == this_entry->max_protection) &&
            (prev_entry->inheritance == this_entry->inheritance) &&
            (prev_entry->use_pmap == this_entry->use_pmap) &&
            (prev_entry->max_protection == this_entry->max_protection) &&
            (prev_entry->inheritance == this_entry->inheritance) &&
            (prev_entry->use_pmap == this_entry->use_pmap) &&

-           (prev_entry->alias == this_entry->alias) &&
+           (VME_ALIAS(prev_entry) == VME_ALIAS(this_entry)) &&

            (prev_entry->no_cache == this_entry->no_cache) &&
            (prev_entry->permanent == this_entry->permanent) &&
            (prev_entry->map_aligned == this_entry->map_aligned) &&
            (prev_entry->no_cache == this_entry->no_cache) &&
            (prev_entry->permanent == this_entry->permanent) &&
            (prev_entry->map_aligned == this_entry->map_aligned) &&


@@ -11567,6 +12213,10 @@ vm_map_simplify_entry(
            (prev_entry->used_for_jit == this_entry->used_for_jit) &&
            /* from_reserved_zone: OK if that field doesn't match */
            (prev_entry->iokit_acct == this_entry->iokit_acct) &&
            (prev_entry->used_for_jit == this_entry->used_for_jit) &&
            /* from_reserved_zone: OK if that field doesn't match */
            (prev_entry->iokit_acct == this_entry->iokit_acct) &&

+           (prev_entry->vme_resilient_codesign ==
+            this_entry->vme_resilient_codesign) &&
+           (prev_entry->vme_resilient_media ==
+            this_entry->vme_resilient_media) &&

 
            (prev_entry->wired_count == this_entry->wired_count) &&
            (prev_entry->user_wired_count == this_entry->user_wired_count) &&
 
            (prev_entry->wired_count == this_entry->wired_count) &&
            (prev_entry->user_wired_count == this_entry->user_wired_count) &&


@@ -11586,11 +12236,16 @@ vm_map_simplify_entry(
                        assert(VM_MAP_PAGE_ALIGNED(prev_entry->vme_start,
                                                   VM_MAP_PAGE_MASK(map)));
                this_entry->vme_start = prev_entry->vme_start;
                        assert(VM_MAP_PAGE_ALIGNED(prev_entry->vme_start,
                                                   VM_MAP_PAGE_MASK(map)));
                this_entry->vme_start = prev_entry->vme_start;

-               this_entry->offset = prev_entry->offset;
+               VME_OFFSET_SET(this_entry, VME_OFFSET(prev_entry));
+
+               if (map->holelistenabled) {
+                       vm_map_store_update_first_free(map, this_entry, TRUE);
+               }
+

                if (prev_entry->is_sub_map) {
                if (prev_entry->is_sub_map) {

-                       vm_map_deallocate(prev_entry->object.sub_map);
+                       vm_map_deallocate(VME_SUBMAP(prev_entry));

                } else {
                } else {

-                       vm_object_deallocate(prev_entry->object.vm_object);
+                       vm_object_deallocate(VME_OBJECT(prev_entry));

                }
                vm_map_entry_dispose(map, prev_entry);
                SAVE_HINT_MAP_WRITE(map, this_entry);
                }
                vm_map_entry_dispose(map, prev_entry);
                SAVE_HINT_MAP_WRITE(map, this_entry);


@@ -11716,15 +12371,15 @@ vm_map_machine_attribute(
                                vm_map_offset_t sub_end;
 
                                sub_start = (start - entry->vme_start) 
                                vm_map_offset_t sub_end;
 
                                sub_start = (start - entry->vme_start) 

-                                       + entry->offset;
+                                       + VME_OFFSET(entry);

                                sub_end = sub_start + sub_size;
                                vm_map_machine_attribute(
                                sub_end = sub_start + sub_size;
                                vm_map_machine_attribute(

-                                       entry->object.sub_map, 
+                                       VME_SUBMAP(entry), 

                                        sub_start,
                                        sub_end,
                                        attribute, value);
                        } else {
                                        sub_start,
                                        sub_end,
                                        attribute, value);
                        } else {

-                               if(entry->object.vm_object) {
+                               if (VME_OBJECT(entry)) {

                                        vm_page_t               m;
                                        vm_object_t             object;
                                        vm_object_t             base_object;
                                        vm_page_t               m;
                                        vm_object_t             object;
                                        vm_object_t             base_object;


@@ -11734,9 +12389,9 @@ vm_map_machine_attribute(
                                        vm_map_size_t           range;
                                        range = sub_size;
                                        offset = (start - entry->vme_start)
                                        vm_map_size_t           range;
                                        range = sub_size;
                                        offset = (start - entry->vme_start)

-                                               + entry->offset;
+                                               + VME_OFFSET(entry);

                                        base_offset = offset;
                                        base_offset = offset;

-                                       object = entry->object.vm_object;
+                                       object = VME_OBJECT(entry);

                                        base_object = object;
                                        last_object = NULL;
 
                                        base_object = object;
                                        last_object = NULL;
 


@@ -11888,6 +12543,11 @@ vm_map_behavior_set(
        case VM_BEHAVIOR_CAN_REUSE:
                return vm_map_can_reuse(map, start, end);
 
        case VM_BEHAVIOR_CAN_REUSE:
                return vm_map_can_reuse(map, start, end);
 

+#if MACH_ASSERT
+       case VM_BEHAVIOR_PAGEOUT:
+               return vm_map_pageout(map, start, end);
+#endif /* MACH_ASSERT */
+

        default:
                return(KERN_INVALID_ARGUMENT);
        }
        default:
                return(KERN_INVALID_ARGUMENT);
        }


@@ -11964,7 +12624,7 @@ vm_map_willneed(
                 * correspond.  After that, the offset will always be zero to
                 * correspond to the beginning of the current vm_map_entry.
                 */
                 * correspond.  After that, the offset will always be zero to
                 * correspond to the beginning of the current vm_map_entry.
                 */

-               offset = (start - entry->vme_start) + entry->offset;
+               offset = (start - entry->vme_start) + VME_OFFSET(entry);

 
                /*
                 * Set the length so we don't go beyond the end of the
 
                /*
                 * Set the length so we don't go beyond the end of the


@@ -11985,7 +12645,7 @@ vm_map_willneed(
                fault_info.cluster_size = (vm_size_t) len;
                fault_info.lo_offset    = offset; 
                fault_info.hi_offset    = offset + len;
                fault_info.cluster_size = (vm_size_t) len;
                fault_info.lo_offset    = offset; 
                fault_info.hi_offset    = offset + len;

-               fault_info.user_tag     = entry->alias;
+               fault_info.user_tag     = VME_ALIAS(entry);

                fault_info.pmap_options = 0;
                if (entry->iokit_acct ||
                    (!entry->is_sub_map && !entry->use_pmap)) {
                fault_info.pmap_options = 0;
                if (entry->iokit_acct ||
                    (!entry->is_sub_map && !entry->use_pmap)) {


@@ -12083,9 +12743,15 @@ static boolean_t
 vm_map_entry_is_reusable(
        vm_map_entry_t entry)
 {
 vm_map_entry_is_reusable(
        vm_map_entry_t entry)
 {

+       /* Only user map entries */
+

        vm_object_t object;
 
        vm_object_t object;
 

-       switch (entry->alias) {
+       if (entry->is_sub_map) {
+               return FALSE;
+       }
+
+       switch (VME_ALIAS(entry)) {

        case VM_MEMORY_MALLOC:
        case VM_MEMORY_MALLOC_SMALL:
        case VM_MEMORY_MALLOC_LARGE:
        case VM_MEMORY_MALLOC:
        case VM_MEMORY_MALLOC_SMALL:
        case VM_MEMORY_MALLOC_LARGE:


@@ -12122,7 +12788,7 @@ vm_map_entry_is_reusable(
                return FALSE;
        }
 
                return FALSE;
        }
 

-       object = entry->object.vm_object;
+       object = VME_OBJECT(entry);

        if (object == VM_OBJECT_NULL) {
                return TRUE;
        }
        if (object == VM_OBJECT_NULL) {
                return TRUE;
        }


@@ -12173,6 +12839,7 @@ vm_map_reuse_pages(
         */
 
        vm_map_lock_read(map);
         */
 
        vm_map_lock_read(map);

+       assert(map->pmap != kernel_pmap);       /* protect alias access */

 
        /*
         * The madvise semantics require that the address range be fully
 
        /*
         * The madvise semantics require that the address range be fully


@@ -12211,10 +12878,11 @@ vm_map_reuse_pages(
                        start_offset = 0;
                }
                end_offset = MIN(end, entry->vme_end) - entry->vme_start;
                        start_offset = 0;
                }
                end_offset = MIN(end, entry->vme_end) - entry->vme_start;

-               start_offset += entry->offset;
-               end_offset += entry->offset;
+               start_offset += VME_OFFSET(entry);
+               end_offset += VME_OFFSET(entry);

 
 

-               object = entry->object.vm_object;
+               assert(!entry->is_sub_map);
+               object = VME_OBJECT(entry);

                if (object != VM_OBJECT_NULL) {
                        vm_object_lock(object);
                        vm_object_reuse_pages(object, start_offset, end_offset,
                if (object != VM_OBJECT_NULL) {
                        vm_object_lock(object);
                        vm_object_reuse_pages(object, start_offset, end_offset,


@@ -12222,7 +12890,7 @@ vm_map_reuse_pages(
                        vm_object_unlock(object);
                }
 
                        vm_object_unlock(object);
                }
 

-               if (entry->alias == VM_MEMORY_MALLOC_LARGE_REUSABLE) {
+               if (VME_ALIAS(entry) == VM_MEMORY_MALLOC_LARGE_REUSABLE) {

                        /*
                         * XXX
                         * We do not hold the VM map exclusively here.
                        /*
                         * XXX
                         * We do not hold the VM map exclusively here.


@@ -12231,7 +12899,7 @@ vm_map_reuse_pages(
                         * one that can be modified while holding the VM map
                         * "shared".
                         */
                         * one that can be modified while holding the VM map
                         * "shared".
                         */

-                       entry->alias = VM_MEMORY_MALLOC_LARGE_REUSED;
+                       VME_ALIAS_SET(entry, VM_MEMORY_MALLOC_LARGE_REUSED);

                }
        }
        
                }
        }
        


@@ -12250,6 +12918,7 @@ vm_map_reusable_pages(
        vm_map_entry_t                  entry;
        vm_object_t                     object;
        vm_object_offset_t              start_offset, end_offset;
        vm_map_entry_t                  entry;
        vm_object_t                     object;
        vm_object_offset_t              start_offset, end_offset;

+       vm_map_offset_t                 pmap_offset;

 
        /*
         * The MADV_REUSABLE operation doesn't require any changes to the
 
        /*
         * The MADV_REUSABLE operation doesn't require any changes to the


@@ -12257,6 +12926,7 @@ vm_map_reusable_pages(
         */
 
        vm_map_lock_read(map);
         */
 
        vm_map_lock_read(map);

+       assert(map->pmap != kernel_pmap);       /* protect alias access */

 
        /*
         * The madvise semantics require that the address range be fully
 
        /*
         * The madvise semantics require that the address range be fully


@@ -12293,14 +12963,17 @@ vm_map_reusable_pages(
                 */
                if (entry->vme_start < start) {
                        start_offset = start - entry->vme_start;
                 */
                if (entry->vme_start < start) {
                        start_offset = start - entry->vme_start;

+                       pmap_offset = start;

                } else {
                        start_offset = 0;
                } else {
                        start_offset = 0;

+                       pmap_offset = entry->vme_start;

                }
                end_offset = MIN(end, entry->vme_end) - entry->vme_start;
                }
                end_offset = MIN(end, entry->vme_end) - entry->vme_start;

-               start_offset += entry->offset;
-               end_offset += entry->offset;
+               start_offset += VME_OFFSET(entry);
+               end_offset += VME_OFFSET(entry);

 
 

-               object = entry->object.vm_object;
+               assert(!entry->is_sub_map);
+               object = VME_OBJECT(entry);

                if (object == VM_OBJECT_NULL)
                        continue;
 
                if (object == VM_OBJECT_NULL)
                        continue;
 


@@ -12325,14 +12998,16 @@ vm_map_reusable_pages(
                                                   start_offset,
                                                   end_offset - start_offset,
                                                   kill_pages,
                                                   start_offset,
                                                   end_offset - start_offset,
                                                   kill_pages,

-                                                  TRUE /*reusable_pages*/);
+                                                  TRUE /*reusable_pages*/,
+                                                  map->pmap,
+                                                  pmap_offset);

                } else {
                        vm_page_stats_reusable.reusable_pages_shared++;
                }
                vm_object_unlock(object);
 
                } else {
                        vm_page_stats_reusable.reusable_pages_shared++;
                }
                vm_object_unlock(object);
 

-               if (entry->alias == VM_MEMORY_MALLOC_LARGE ||
-                   entry->alias == VM_MEMORY_MALLOC_LARGE_REUSED) {
+               if (VME_ALIAS(entry) == VM_MEMORY_MALLOC_LARGE ||
+                   VME_ALIAS(entry) == VM_MEMORY_MALLOC_LARGE_REUSED) {

                        /*
                         * XXX
                         * We do not hold the VM map exclusively here.
                        /*
                         * XXX
                         * We do not hold the VM map exclusively here.


@@ -12341,7 +13016,7 @@ vm_map_reusable_pages(
                         * one that can be modified while holding the VM map
                         * "shared".
                         */
                         * one that can be modified while holding the VM map
                         * "shared".
                         */

-                       entry->alias = VM_MEMORY_MALLOC_LARGE_REUSABLE;
+                       VME_ALIAS_SET(entry, VM_MEMORY_MALLOC_LARGE_REUSABLE);

                }
        }
        
                }
        }
        


@@ -12365,6 +13040,7 @@ vm_map_can_reuse(
         */
 
        vm_map_lock_read(map);
         */
 
        vm_map_lock_read(map);

+       assert(map->pmap != kernel_pmap);       /* protect alias access */

 
        /*
         * The madvise semantics require that the address range be fully
 
        /*
         * The madvise semantics require that the address range be fully


@@ -12399,6 +13075,97 @@ vm_map_can_reuse(
 }
 
 
 }
 
 

+#if MACH_ASSERT
+static kern_return_t
+vm_map_pageout(
+       vm_map_t        map,
+       vm_map_offset_t start,
+       vm_map_offset_t end)
+{
+       vm_map_entry_t                  entry;
+
+       /*
+        * The MADV_PAGEOUT operation doesn't require any changes to the
+        * vm_map_entry_t's, so the read lock is sufficient.
+        */
+
+       vm_map_lock_read(map);
+
+       /*
+        * The madvise semantics require that the address range be fully
+        * allocated with no holes.  Otherwise, we're required to return
+        * an error.
+        */
+
+       if (!vm_map_range_check(map, start, end, &entry)) {
+               vm_map_unlock_read(map);
+               return KERN_INVALID_ADDRESS;
+       }
+
+       /*
+        * Examine each vm_map_entry_t in the range.
+        */
+       for (; entry != vm_map_to_entry(map) && entry->vme_start < end;
+            entry = entry->vme_next) {
+               vm_object_t     object;
+
+               /*
+                * Sanity check on the VM map entry.
+                */
+               if (entry->is_sub_map) {
+                       vm_map_t submap;
+                       vm_map_offset_t submap_start;
+                       vm_map_offset_t submap_end;
+                       vm_map_entry_t submap_entry;
+
+                       submap = VME_SUBMAP(entry);
+                       submap_start = VME_OFFSET(entry);
+                       submap_end = submap_start + (entry->vme_end - 
+                                                    entry->vme_start);
+
+                       vm_map_lock_read(submap);
+
+                       if (! vm_map_range_check(submap,
+                                                submap_start,
+                                                submap_end,
+                                                &submap_entry)) {
+                               vm_map_unlock_read(submap);
+                               vm_map_unlock_read(map);
+                               return KERN_INVALID_ADDRESS;
+                       }
+
+                       object = VME_OBJECT(submap_entry);
+                       if (submap_entry->is_sub_map ||
+                           object == VM_OBJECT_NULL ||
+                           !object->internal) {
+                               vm_map_unlock_read(submap);
+                               continue;
+                       }
+
+                       vm_object_pageout(object);
+
+                       vm_map_unlock_read(submap);
+                       submap = VM_MAP_NULL;
+                       submap_entry = VM_MAP_ENTRY_NULL;
+                       continue;
+               }
+
+               object = VME_OBJECT(entry);
+               if (entry->is_sub_map ||
+                   object == VM_OBJECT_NULL ||
+                   !object->internal) {
+                       continue;
+               }
+
+               vm_object_pageout(object);
+       }
+       
+       vm_map_unlock_read(map);
+       return KERN_SUCCESS;
+}
+#endif /* MACH_ASSERT */
+
+

 /*
  *     Routine:        vm_map_entry_insert
  *
 /*
  *     Routine:        vm_map_entry_insert
  *


@@ -12455,8 +13222,8 @@ vm_map_entry_insert(
        }
        assert(new_entry->vme_start < new_entry->vme_end);
 
        }
        assert(new_entry->vme_start < new_entry->vme_end);
 

-       new_entry->object.vm_object = object;
-       new_entry->offset = offset;
+       VME_OBJECT_SET(new_entry, object);
+       VME_OFFSET_SET(new_entry, offset);

        new_entry->is_shared = is_shared;
        new_entry->is_sub_map = is_submap;
        new_entry->needs_copy = needs_copy;
        new_entry->is_shared = is_shared;
        new_entry->is_sub_map = is_submap;
        new_entry->needs_copy = needs_copy;


@@ -12481,7 +13248,7 @@ vm_map_entry_insert(
                 */
                new_entry->use_pmap = TRUE;
        }
                 */
                new_entry->use_pmap = TRUE;
        }

-       new_entry->alias = 0;
+       VME_ALIAS_SET(new_entry, 0);

        new_entry->zero_wired_pages = FALSE;
        new_entry->no_cache = no_cache;
        new_entry->permanent = permanent;
        new_entry->zero_wired_pages = FALSE;
        new_entry->no_cache = no_cache;
        new_entry->permanent = permanent;


@@ -12491,6 +13258,8 @@ vm_map_entry_insert(
                new_entry->superpage_size = FALSE;
        new_entry->used_for_jit = FALSE;
        new_entry->iokit_acct = FALSE;
                new_entry->superpage_size = FALSE;
        new_entry->used_for_jit = FALSE;
        new_entry->iokit_acct = FALSE;

+       new_entry->vme_resilient_codesign = FALSE;
+       new_entry->vme_resilient_media = FALSE;

 
        /*
         *      Insert the new entry into the list.
 
        /*
         *      Insert the new entry into the list.


@@ -12601,10 +13370,10 @@ vm_map_remap_extract(
                                             src_entry->vme_start);
 
                if(src_entry->is_sub_map) {
                                             src_entry->vme_start);
 
                if(src_entry->is_sub_map) {

-                       vm_map_reference(src_entry->object.sub_map);
+                       vm_map_reference(VME_SUBMAP(src_entry));

                        object = VM_OBJECT_NULL;
                } else {
                        object = VM_OBJECT_NULL;
                } else {

-                       object = src_entry->object.vm_object;
+                       object = VME_OBJECT(src_entry);

                        if (src_entry->iokit_acct) {
                                /*
                                 * This entry uses "IOKit accounting".
                        if (src_entry->iokit_acct) {
                                /*
                                 * This entry uses "IOKit accounting".


@@ -12626,8 +13395,8 @@ vm_map_remap_extract(
 
                        if (object == VM_OBJECT_NULL) {
                                object = vm_object_allocate(entry_size);
 
                        if (object == VM_OBJECT_NULL) {
                                object = vm_object_allocate(entry_size);

-                               src_entry->offset = 0;
-                               src_entry->object.vm_object = object;
+                               VME_OFFSET_SET(src_entry, 0);
+                               VME_OBJECT_SET(src_entry, object);

                        } else if (object->copy_strategy !=
                                   MEMORY_OBJECT_COPY_SYMMETRIC) {
                                /*
                        } else if (object->copy_strategy !=
                                   MEMORY_OBJECT_COPY_SYMMETRIC) {
                                /*


@@ -12641,9 +13410,7 @@ vm_map_remap_extract(
                                    !src_entry->is_shared &&
                                    object->vo_size > entry_size)) {
 
                                    !src_entry->is_shared &&
                                    object->vo_size > entry_size)) {
 

-                               vm_object_shadow(&src_entry->object.vm_object,
-                                                &src_entry->offset,
-                                                entry_size);
+                               VME_OBJECT_SHADOW(src_entry, entry_size);

 
                                if (!src_entry->needs_copy &&
                                    (src_entry->protection & VM_PROT_WRITE)) {
 
                                if (!src_entry->needs_copy &&
                                    (src_entry->protection & VM_PROT_WRITE)) {


@@ -12651,13 +13418,15 @@ vm_map_remap_extract(
 
                                        prot = src_entry->protection & ~VM_PROT_WRITE;
 
 
                                        prot = src_entry->protection & ~VM_PROT_WRITE;
 

-                                       if (override_nx(map, src_entry->alias) && prot)
+                                       if (override_nx(map,
+                                                       VME_ALIAS(src_entry))
+                                           && prot)

                                                prot |= VM_PROT_EXECUTE;
 
                                        if(map->mapped_in_other_pmaps) {
                                                vm_object_pmap_protect(
                                                prot |= VM_PROT_EXECUTE;
 
                                        if(map->mapped_in_other_pmaps) {
                                                vm_object_pmap_protect(

-                                                       src_entry->object.vm_object,
-                                                       src_entry->offset,
+                                                       VME_OBJECT(src_entry),
+                                                       VME_OFFSET(src_entry),

                                                        entry_size,
                                                        PMAP_NULL,
                                                        src_entry->vme_start,
                                                        entry_size,
                                                        PMAP_NULL,
                                                        src_entry->vme_start,


@@ -12670,7 +13439,7 @@ vm_map_remap_extract(
                                        }
                                }
 
                                        }
                                }
 

-                               object = src_entry->object.vm_object;
+                               object = VME_OBJECT(src_entry);

                                src_entry->needs_copy = FALSE;
                        }
 
                                src_entry->needs_copy = FALSE;
                        }
 


@@ -12685,7 +13454,8 @@ vm_map_remap_extract(
                        vm_object_unlock(object);
                }
 
                        vm_object_unlock(object);
                }
 

-               offset = src_entry->offset + (src_start - src_entry->vme_start);
+               offset = (VME_OFFSET(src_entry) +
+                         (src_start - src_entry->vme_start));

 
                new_entry = _vm_map_entry_create(map_header, !map_header->entries_pageable);
                vm_map_entry_copy(new_entry, src_entry);
 
                new_entry = _vm_map_entry_create(map_header, !map_header->entries_pageable);
                vm_map_entry_copy(new_entry, src_entry);


@@ -12700,7 +13470,7 @@ vm_map_remap_extract(
                new_entry->vme_end = map_address + tmp_size;
                assert(new_entry->vme_start < new_entry->vme_end);
                new_entry->inheritance = inheritance;
                new_entry->vme_end = map_address + tmp_size;
                assert(new_entry->vme_start < new_entry->vme_end);
                new_entry->inheritance = inheritance;

-               new_entry->offset = offset;
+               VME_OFFSET_SET(new_entry, offset);

 
                /*
                 * The new region has to be copied now if required.
 
                /*
                 * The new region has to be copied now if required.


@@ -12722,11 +13492,12 @@ vm_map_remap_extract(
 
                } else if (src_entry->is_sub_map) {
                        /* make this a COW sub_map if not already */
 
                } else if (src_entry->is_sub_map) {
                        /* make this a COW sub_map if not already */

+                       assert(new_entry->wired_count == 0);

                        new_entry->needs_copy = TRUE;
                        object = VM_OBJECT_NULL;
                } else if (src_entry->wired_count == 0 &&
                        new_entry->needs_copy = TRUE;
                        object = VM_OBJECT_NULL;
                } else if (src_entry->wired_count == 0 &&

-                          vm_object_copy_quickly(&new_entry->object.vm_object,
-                                                 new_entry->offset,
+                          vm_object_copy_quickly(&VME_OBJECT(new_entry),
+                                                 VME_OFFSET(new_entry),

                                                  (new_entry->vme_end -
                                                   new_entry->vme_start),
                                                  &src_needs_copy,
                                                  (new_entry->vme_end -
                                                   new_entry->vme_start),
                                                  &src_needs_copy,


@@ -12743,7 +13514,9 @@ vm_map_remap_extract(
 
                                prot = src_entry->protection & ~VM_PROT_WRITE;
 
 
                                prot = src_entry->protection & ~VM_PROT_WRITE;
 

-                               if (override_nx(map, src_entry->alias) && prot)
+                               if (override_nx(map,
+                                               VME_ALIAS(src_entry))
+                                   && prot)

                                        prot |= VM_PROT_EXECUTE;
 
                                vm_object_pmap_protect(object,
                                        prot |= VM_PROT_EXECUTE;
 
                                vm_object_pmap_protect(object,


@@ -12755,6 +13528,7 @@ vm_map_remap_extract(
                                                       src_entry->vme_start,
                                                       prot);
 
                                                       src_entry->vme_start,
                                                       prot);
 

+                               assert(src_entry->wired_count == 0);

                                src_entry->needs_copy = TRUE;
                        }
                        /*
                                src_entry->needs_copy = TRUE;
                        }
                        /*


@@ -12785,18 +13559,24 @@ vm_map_remap_extract(
                                        offset,
                                        entry_size,
                                        THREAD_UNINT,
                                        offset,
                                        entry_size,
                                        THREAD_UNINT,

-                                       &new_entry->object.vm_object);
+                                       &VME_OBJECT(new_entry));

 
 

-                               new_entry->offset = 0;
+                               VME_OFFSET_SET(new_entry, 0);

                                new_entry->needs_copy = FALSE;
                        } else {
                                new_entry->needs_copy = FALSE;
                        } else {

+                               vm_object_offset_t new_offset;
+
+                               new_offset = VME_OFFSET(new_entry);

                                result = vm_object_copy_strategically(
                                        object,
                                        offset,
                                        entry_size,
                                result = vm_object_copy_strategically(
                                        object,
                                        offset,
                                        entry_size,

-                                       &new_entry->object.vm_object,
-                                       &new_entry->offset,
+                                       &VME_OBJECT(new_entry),
+                                       &new_offset,

                                        &new_entry_needs_copy);
                                        &new_entry_needs_copy);

+                               if (new_offset != VME_OFFSET(new_entry)) {
+                                       VME_OFFSET_SET(new_entry, new_offset);
+                               }

 
                                new_entry->needs_copy = new_entry_needs_copy;
                        }
 
                                new_entry->needs_copy = new_entry_needs_copy;
                        }


@@ -12825,8 +13605,7 @@ vm_map_remap_extract(
                                 * Retry the lookup and verify that the
                                 * same object/offset are still present.
                                 */
                                 * Retry the lookup and verify that the
                                 * same object/offset are still present.
                                 */

-                               vm_object_deallocate(new_entry->
-                                                    object.vm_object);
+                               vm_object_deallocate(VME_OBJECT(new_entry));

                                _vm_map_entry_dispose(map_header, new_entry);
                                if (result == KERN_MEMORY_RESTART_COPY)
                                        result = KERN_SUCCESS;
                                _vm_map_entry_dispose(map_header, new_entry);
                                if (result == KERN_MEMORY_RESTART_COPY)
                                        result = KERN_SUCCESS;


@@ -12864,9 +13643,9 @@ vm_map_remap_extract(
                        new_entry = src_entry->vme_next;
                        _vm_map_store_entry_unlink(map_header, src_entry);
                        if (src_entry->is_sub_map) {
                        new_entry = src_entry->vme_next;
                        _vm_map_store_entry_unlink(map_header, src_entry);
                        if (src_entry->is_sub_map) {

-                               vm_map_deallocate(src_entry->object.sub_map);
+                               vm_map_deallocate(VME_SUBMAP(src_entry));

                        } else {
                        } else {

-                               vm_object_deallocate(src_entry->object.vm_object);
+                               vm_object_deallocate(VME_OBJECT(src_entry));

                        }
                        _vm_map_entry_dispose(map_header, src_entry);
                }
                        }
                        _vm_map_entry_dispose(map_header, src_entry);
                }


@@ -12974,6 +13753,13 @@ vm_map_remap(
                new_entry = entry->vme_next;
                _vm_map_store_entry_unlink(&map_header, entry);
                if (result == KERN_SUCCESS) {
                new_entry = entry->vme_next;
                _vm_map_store_entry_unlink(&map_header, entry);
                if (result == KERN_SUCCESS) {

+                       if (flags & VM_FLAGS_RESILIENT_CODESIGN) {
+                               /* no codesigning -> read-only access */
+                               assert(!entry->used_for_jit);
+                               entry->max_protection = VM_PROT_READ;
+                               entry->protection = VM_PROT_READ;
+                               entry->vme_resilient_codesign = TRUE;
+                       }

                        entry->vme_start += *address;
                        entry->vme_end += *address;
                        assert(!entry->map_aligned);
                        entry->vme_start += *address;
                        entry->vme_end += *address;
                        assert(!entry->map_aligned);


@@ -12981,14 +13767,19 @@ vm_map_remap(
                        insp_entry = entry;
                } else {
                        if (!entry->is_sub_map) {
                        insp_entry = entry;
                } else {
                        if (!entry->is_sub_map) {

-                               vm_object_deallocate(entry->object.vm_object);
+                               vm_object_deallocate(VME_OBJECT(entry));

                        } else {
                        } else {

-                               vm_map_deallocate(entry->object.sub_map);
+                               vm_map_deallocate(VME_SUBMAP(entry));

                        }
                        _vm_map_entry_dispose(&map_header, entry);
                }
        }
 
                        }
                        _vm_map_entry_dispose(&map_header, entry);
                }
        }
 

+       if (flags & VM_FLAGS_RESILIENT_CODESIGN) {
+               *cur_protection = VM_PROT_READ;
+               *max_protection = VM_PROT_READ;
+       }
+

        if( target_map->disable_vmentry_reuse == TRUE) {
                if( target_map->highest_entry_end < insp_entry->vme_end ){
                        target_map->highest_entry_end = insp_entry->vme_end;
        if( target_map->disable_vmentry_reuse == TRUE) {
                if( target_map->highest_entry_end < insp_entry->vme_end ){
                        target_map->highest_entry_end = insp_entry->vme_end;


@@ -13003,7 +13794,8 @@ vm_map_remap(
 
        if (result == KERN_SUCCESS && target_map->wiring_required)
                result = vm_map_wire(target_map, *address,
 
        if (result == KERN_SUCCESS && target_map->wiring_required)
                result = vm_map_wire(target_map, *address,

-                                    *address + size, *cur_protection, TRUE);
+                                    *address + size, *cur_protection | VM_PROT_MEMORY_TAG_MAKE(VM_KERN_MEMORY_MLOCK),
+                                    TRUE);

 
        /* 
         * If requested, return the address of the data pointed to by the 
 
        /* 
         * If requested, return the address of the data pointed to by the 


@@ -13040,6 +13832,7 @@ vm_map_remap_range_allocate(
        vm_map_offset_t start;
        vm_map_offset_t end;
        kern_return_t   kr;
        vm_map_offset_t start;
        vm_map_offset_t end;
        kern_return_t   kr;

+       vm_map_entry_t          hole_entry;

 
 StartAgain: ;
 
 
 StartAgain: ;
 


@@ -13065,15 +13858,51 @@ StartAgain: ;
                if( map->disable_vmentry_reuse == TRUE) {
                        VM_MAP_HIGHEST_ENTRY(map, entry, start);
                } else {
                if( map->disable_vmentry_reuse == TRUE) {
                        VM_MAP_HIGHEST_ENTRY(map, entry, start);
                } else {

-                       assert(first_free_is_valid(map));
-                       if (start == map->min_offset) {
-                               if ((entry = map->first_free) != vm_map_to_entry(map))
-                                       start = entry->vme_end;
+
+                       if (map->holelistenabled) {
+                               hole_entry = (vm_map_entry_t)map->holes_list;
+
+                               if (hole_entry == NULL) {
+                                       /*
+                                        * No more space in the map?
+                                        */
+                                       return(KERN_NO_SPACE);
+                               } else {
+
+                                       boolean_t found_hole = FALSE;
+
+                                       do {
+                                               if (hole_entry->vme_start >= start) {
+                                                       start = hole_entry->vme_start;
+                                                       found_hole = TRUE;
+                                                       break;
+                                               }
+
+                                               if (hole_entry->vme_end > start) {
+                                                       found_hole = TRUE;
+                                                       break;
+                                               }
+                                               hole_entry = hole_entry->vme_next;
+
+                                       } while (hole_entry != (vm_map_entry_t) map->holes_list);
+
+                                       if (found_hole == FALSE) {
+                                               return (KERN_NO_SPACE);
+                                       }
+
+                                       entry = hole_entry;
+                               }

                        } else {
                        } else {

-                               vm_map_entry_t  tmp_entry;
-                               if (vm_map_lookup_entry(map, start, &tmp_entry))
-                                       start = tmp_entry->vme_end;
-                               entry = tmp_entry;
+                               assert(first_free_is_valid(map));
+                               if (start == map->min_offset) {
+                                       if ((entry = map->first_free) != vm_map_to_entry(map))
+                                               start = entry->vme_end;
+                               } else {
+                                       vm_map_entry_t  tmp_entry;
+                                       if (vm_map_lookup_entry(map, start, &tmp_entry))
+                                               start = tmp_entry->vme_end;
+                                       entry = tmp_entry;
+                               }

                        }
                        start = vm_map_round_page(start,
                                                  VM_MAP_PAGE_MASK(map));
                        }
                        start = vm_map_round_page(start,
                                                  VM_MAP_PAGE_MASK(map));


@@ -13117,30 +13946,56 @@ StartAgain: ;
                                return(KERN_NO_SPACE);
                        }
 
                                return(KERN_NO_SPACE);
                        }
 

-                       /*
-                        *      If there are no more entries, we must win.
-                        */
-

                        next = entry->vme_next;
                        next = entry->vme_next;

-                       if (next == vm_map_to_entry(map))
-                               break;

 
 

-                       /*
-                        *      If there is another entry, it must be
-                        *      after the end of the potential new region.
-                        */
+                       if (map->holelistenabled) {
+                               if (entry->vme_end >= end)
+                                       break;
+                       } else {
+                               /*
+                                *      If there are no more entries, we must win.
+                                *
+                                *      OR
+                                *
+                                *      If there is another entry, it must be
+                                *      after the end of the potential new region.
+                                */

 
 

-                       if (next->vme_start >= end)
-                               break;
+                               if (next == vm_map_to_entry(map))
+                                       break;
+
+                               if (next->vme_start >= end)
+                                       break;
+                       }

 
                        /*
                         *      Didn't fit -- move to the next entry.
                         */
 
                        entry = next;
 
                        /*
                         *      Didn't fit -- move to the next entry.
                         */
 
                        entry = next;

-                       start = entry->vme_end;
+
+                       if (map->holelistenabled) {
+                               if (entry == (vm_map_entry_t) map->holes_list) {
+                                       /*
+                                        * Wrapped around
+                                        */
+                                       return(KERN_NO_SPACE);
+                               }
+                               start = entry->vme_start;
+                       } else {
+                               start = entry->vme_end;
+                       }
+               }
+
+               if (map->holelistenabled) {
+
+                       if (vm_map_lookup_entry(map, entry->vme_start, &entry)) {
+                               panic("Found an existing entry (%p) instead of potential hole at address: 0x%llx.\n", entry, (unsigned long long)entry->vme_start);
+                       }

                }
                }

+

                *address = start;
                *address = start;

+

        } else {
                vm_map_entry_t          temp_entry;
        
        } else {
                vm_map_entry_t          temp_entry;
        


@@ -13187,6 +14042,7 @@ StartAgain: ;
                                return KERN_RESOURCE_SHORTAGE;
                        }
                        vm_map_set_page_shift(zap_map, VM_MAP_PAGE_SHIFT(map));
                                return KERN_RESOURCE_SHORTAGE;
                        }
                        vm_map_set_page_shift(zap_map, VM_MAP_PAGE_SHIFT(map));

+                       vm_map_disable_hole_optimization(zap_map);

 
                        kr = vm_map_delete(map, start, end,
                                           (VM_MAP_REMOVE_SAVE_ENTRIES |
 
                        kr = vm_map_delete(map, start, end,
                                           (VM_MAP_REMOVE_SAVE_ENTRIES |


@@ -13448,7 +14304,7 @@ vm_map_purgable_control(
                return(KERN_PROTECTION_FAILURE);
        }
 
                return(KERN_PROTECTION_FAILURE);
        }
 

-       object = entry->object.vm_object;
+       object = VME_OBJECT(entry);

        if (object == VM_OBJECT_NULL ||
            object->purgable == VM_PURGABLE_DENY) {
                /*
        if (object == VM_OBJECT_NULL ||
            object->purgable == VM_PURGABLE_DENY) {
                /*


@@ -13461,7 +14317,7 @@ vm_map_purgable_control(
        vm_object_lock(object);
 
 #if 00
        vm_object_lock(object);
 
 #if 00

-       if (entry->offset != 0 || 
+       if (VME_OFFSET(entry) != 0 || 

            entry->vme_end - entry->vme_start != object->vo_size) {
                /*
                 * Can only apply purgable controls to the whole (existing)
            entry->vme_end - entry->vme_start != object->vo_size) {
                /*
                 * Can only apply purgable controls to the whole (existing)


@@ -13582,12 +14438,12 @@ vm_map_page_info(
                /* compute offset from this map entry's start */
                offset -= map_entry->vme_start;
                /* compute offset into this map entry's object (or submap) */
                /* compute offset from this map entry's start */
                offset -= map_entry->vme_start;
                /* compute offset into this map entry's object (or submap) */

-               offset += map_entry->offset;
+               offset += VME_OFFSET(map_entry);

 
                if (map_entry->is_sub_map) {
                        vm_map_t sub_map;
 
 
                if (map_entry->is_sub_map) {
                        vm_map_t sub_map;
 

-                       sub_map = map_entry->object.sub_map;
+                       sub_map = VME_SUBMAP(map_entry);

                        vm_map_lock_read(sub_map);
                        vm_map_unlock_read(map);
 
                        vm_map_lock_read(sub_map);
                        vm_map_unlock_read(map);
 


@@ -13599,7 +14455,7 @@ vm_map_page_info(
                break;
        }
 
                break;
        }
 

-       object = map_entry->object.vm_object;
+       object = VME_OBJECT(map_entry);

        if (object == VM_OBJECT_NULL) {
                /* no object -> no page */
                vm_map_unlock_read(map);
        if (object == VM_OBJECT_NULL) {
                /* no object -> no page */
                vm_map_unlock_read(map);


@@ -13812,6 +14668,7 @@ vm_map_msync(
        boolean_t               do_sync_req;
        boolean_t               had_hole = FALSE;
        memory_object_t         pager;
        boolean_t               do_sync_req;
        boolean_t               had_hole = FALSE;
        memory_object_t         pager;

+       vm_map_offset_t         pmap_offset;

        
        if ((sync_flags & VM_SYNC_ASYNCHRONOUS) &&
            (sync_flags & VM_SYNC_SYNCHRONOUS))
        
        if ((sync_flags & VM_SYNC_ASYNCHRONOUS) &&
            (sync_flags & VM_SYNC_SYNCHRONOUS))


@@ -13842,9 +14699,7 @@ vm_map_msync(
 
                vm_map_lock(map);
                if (!vm_map_lookup_entry(map,
 
                vm_map_lock(map);
                if (!vm_map_lookup_entry(map,

-                                        vm_map_trunc_page(
-                                                address,
-                                                VM_MAP_PAGE_MASK(map)),
+                                        address,

                                         &entry)) {
 
                        vm_map_size_t   skip;
                                         &entry)) {
 
                        vm_map_size_t   skip;


@@ -13885,6 +14740,7 @@ vm_map_msync(
                }
 
                offset = address - entry->vme_start;
                }
 
                offset = address - entry->vme_start;

+               pmap_offset = address;

 
                /*
                 * do we have more to flush than is contained in this
 
                /*
                 * do we have more to flush than is contained in this


@@ -13903,8 +14759,8 @@ vm_map_msync(
                        vm_map_t        local_map;
                        vm_map_offset_t local_offset;
 
                        vm_map_t        local_map;
                        vm_map_offset_t local_offset;
 

-                       local_map = entry->object.sub_map;
-                       local_offset = entry->offset;
+                       local_map = VME_SUBMAP(entry);
+                       local_offset = VME_OFFSET(entry);

                        vm_map_unlock(map);
                        if (vm_map_msync(
                                    local_map,
                        vm_map_unlock(map);
                        if (vm_map_msync(
                                    local_map,


@@ -13915,7 +14771,7 @@ vm_map_msync(
                        }
                        continue;
                }
                        }
                        continue;
                }

-               object = entry->object.vm_object;
+               object = VME_OBJECT(entry);

 
                /*
                 * We can't sync this object if the object has not been
 
                /*
                 * We can't sync this object if the object has not been


@@ -13925,7 +14781,7 @@ vm_map_msync(
                        vm_map_unlock(map);
                        continue;
                }
                        vm_map_unlock(map);
                        continue;
                }

-               offset += entry->offset;
+               offset += VME_OFFSET(entry);

 
                 vm_object_lock(object);
 
 
                 vm_object_lock(object);
 


@@ -13940,8 +14796,14 @@ vm_map_msync(
                                        kill_pages = -1;
                        }
                        if (kill_pages != -1)
                                        kill_pages = -1;
                        }
                        if (kill_pages != -1)

-                               vm_object_deactivate_pages(object, offset, 
-                                                          (vm_object_size_t)flush_size, kill_pages, reusable_pages);
+                               vm_object_deactivate_pages(
+                                       object,
+                                       offset,
+                                       (vm_object_size_t) flush_size,
+                                       kill_pages,
+                                       reusable_pages,
+                                       map->pmap,
+                                       pmap_offset);

                        vm_object_unlock(object);
                        vm_map_unlock(map);
                        continue;
                        vm_object_unlock(object);
                        vm_map_unlock(map);
                        continue;


@@ -14306,7 +15168,7 @@ vm_map_set_64bit(vm_map_t map)
 }
 
 vm_map_offset_t
 }
 
 vm_map_offset_t

-vm_compute_max_offset(unsigned is64)
+vm_compute_max_offset(boolean_t is64)

 {
        return (is64 ? (vm_map_offset_t)MACH_VM_MAX_ADDRESS : (vm_map_offset_t)VM_MAX_ADDRESS);
 }
 {
        return (is64 ? (vm_map_offset_t)MACH_VM_MAX_ADDRESS : (vm_map_offset_t)VM_MAX_ADDRESS);
 }


@@ -14399,6 +15261,11 @@ vm_map_raise_min_offset(
                vm_map_unlock(map);
                return KERN_INVALID_ADDRESS;
        }
                vm_map_unlock(map);
                return KERN_INVALID_ADDRESS;
        }

+       if (new_min_offset >= map->max_offset) {
+               /* can't go beyond the end of the address space */
+               vm_map_unlock(map);
+               return KERN_INVALID_ADDRESS;
+       }

 
        first_entry = vm_map_first_entry(map);
        if (first_entry != vm_map_to_entry(map) &&
 
        first_entry = vm_map_first_entry(map);
        if (first_entry != vm_map_to_entry(map) &&


@@ -14413,6 +15280,10 @@ vm_map_raise_min_offset(
 
        map->min_offset = new_min_offset;
 
 
        map->min_offset = new_min_offset;
 

+       assert(map->holes_list);
+       map->holes_list->start = new_min_offset;
+       assert(new_min_offset < map->holes_list->end);
+

        vm_map_unlock(map);
 
        return KERN_SUCCESS;
        vm_map_unlock(map);
 
        return KERN_SUCCESS;


@@ -14500,7 +15371,7 @@ kern_return_t vm_map_sign(vm_map_t map,
                return(KERN_INVALID_ARGUMENT);
        }
        
                return(KERN_INVALID_ARGUMENT);
        }
        

-       object = entry->object.vm_object;
+       object = VME_OBJECT(entry);

        if (object == VM_OBJECT_NULL) {
                /*
                 * Object must already be present or we can't sign.
        if (object == VM_OBJECT_NULL) {
                /*
                 * Object must already be present or we can't sign.


@@ -14515,7 +15386,8 @@ kern_return_t vm_map_sign(vm_map_t map,
        while(start < end) {
                uint32_t refmod;
                
        while(start < end) {
                uint32_t refmod;
                

-               m = vm_page_lookup(object, start - entry->vme_start + entry->offset );
+               m = vm_page_lookup(object,
+                                  start - entry->vme_start + VME_OFFSET(entry));

                if (m==VM_PAGE_NULL) {
                        /* shoud we try to fault a page here? we can probably 
                         * demand it exists and is locked for this request */
                if (m==VM_PAGE_NULL) {
                        /* shoud we try to fault a page here? we can probably 
                         * demand it exists and is locked for this request */


@@ -14580,17 +15452,20 @@ kern_return_t vm_map_partial_reap(vm_map_t map, unsigned int *reclaimed_resident
 
        vm_map_set_page_shift(zap_map, 
                              VM_MAP_PAGE_SHIFT(map));
 
        vm_map_set_page_shift(zap_map, 
                              VM_MAP_PAGE_SHIFT(map));

+       vm_map_disable_hole_optimization(zap_map);

 
        for (entry = vm_map_first_entry(map);
             entry != vm_map_to_entry(map);
             entry = next_entry) {
                next_entry = entry->vme_next;
                
 
        for (entry = vm_map_first_entry(map);
             entry != vm_map_to_entry(map);
             entry = next_entry) {
                next_entry = entry->vme_next;
                

-               if (entry->object.vm_object && !entry->is_sub_map && (entry->object.vm_object->internal == TRUE)
-                   && (entry->object.vm_object->ref_count == 1)) {
+               if (VME_OBJECT(entry) &&
+                   !entry->is_sub_map &&
+                   (VME_OBJECT(entry)->internal == TRUE) &&
+                   (VME_OBJECT(entry)->ref_count == 1)) {

 
 

-                       *reclaimed_resident += entry->object.vm_object->resident_page_count;
-                       *reclaimed_compressed += vm_compressor_pager_get_count(entry->object.vm_object->pager);
+                       *reclaimed_resident += VME_OBJECT(entry)->resident_page_count;
+                       *reclaimed_compressed += vm_compressor_pager_get_count(VME_OBJECT(entry)->pager);

 
                        (void)vm_map_delete(map, 
                                            entry->vme_start, 
 
                        (void)vm_map_delete(map, 
                                            entry->vme_start, 


@@ -14638,13 +15513,13 @@ kern_return_t vm_map_freeze_walk(
                unsigned int purgeable, clean, dirty, wired;
                boolean_t shared;
 
                unsigned int purgeable, clean, dirty, wired;
                boolean_t shared;
 

-               if ((entry->object.vm_object == 0) ||
+               if ((VME_OBJECT(entry) == 0) ||

                    (entry->is_sub_map) ||
                    (entry->is_sub_map) ||

-                   (entry->object.vm_object->phys_contiguous)) {
+                   (VME_OBJECT(entry)->phys_contiguous)) {

                        continue;
                }
 
                        continue;
                }
 

-               default_freezer_pack(&purgeable, &wired, &clean, &dirty, dirty_budget, &shared, entry->object.vm_object, NULL);
+               default_freezer_pack(&purgeable, &wired, &clean, &dirty, dirty_budget, &shared, VME_OBJECT(entry), NULL);

                
                *purgeable_count += purgeable;
                *wired_count += wired;
                
                *purgeable_count += purgeable;
                *wired_count += wired;


@@ -14669,6 +15544,10 @@ kern_return_t vm_map_freeze_walk(
        return KERN_SUCCESS;
 }
 
        return KERN_SUCCESS;
 }
 

+int c_freezer_swapout_count;
+int c_freezer_compression_count = 0;
+AbsoluteTime c_freezer_last_yield_ts = 0;
+

 kern_return_t vm_map_freeze(
                vm_map_t map,
                unsigned int *purgeable_count,
 kern_return_t vm_map_freeze(
                vm_map_t map,
                unsigned int *purgeable_count,


@@ -14694,7 +15573,13 @@ kern_return_t vm_map_freeze(
 
        if (COMPRESSED_PAGER_IS_ACTIVE || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_ACTIVE) {
                default_freezer_active = FALSE;
 
        if (COMPRESSED_PAGER_IS_ACTIVE || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_ACTIVE) {
                default_freezer_active = FALSE;

+                               
+               if (vm_compressor_low_on_space() || vm_swap_low_on_space()) {
+                       kr = KERN_NO_SPACE;
+                       goto done;      
+               }

        }
        }

+       assert(default_freezer_active == FALSE);

        
        if (default_freezer_active) {
                if (map->default_freezer_handle == NULL) {      
        
        if (default_freezer_active) {
                if (map->default_freezer_handle == NULL) {      


@@ -14710,14 +15595,18 @@ kern_return_t vm_map_freeze(
                        goto done;
                }
        }
                        goto done;
                }
        }

-       
+       c_freezer_compression_count = 0;
+       clock_get_uptime(&c_freezer_last_yield_ts);
+

        for (entry2 = vm_map_first_entry(map);
             entry2 != vm_map_to_entry(map);
             entry2 = entry2->vme_next) {
        
        for (entry2 = vm_map_first_entry(map);
             entry2 != vm_map_to_entry(map);
             entry2 = entry2->vme_next) {
        

-               vm_object_t     src_object = entry2->object.vm_object;
+               vm_object_t     src_object = VME_OBJECT(entry2);

 
 

-               if (entry2->object.vm_object && !entry2->is_sub_map && !entry2->object.vm_object->phys_contiguous) {
+               if (VME_OBJECT(entry2) &&
+                   !entry2->is_sub_map &&
+                   !VME_OBJECT(entry2)->phys_contiguous) {

                        /* If eligible, scan the entry, moving eligible pages over to our parent object */
                        if (default_freezer_active) {
                                unsigned int purgeable, clean, dirty, wired;
                        /* If eligible, scan the entry, moving eligible pages over to our parent object */
                        if (default_freezer_active) {
                                unsigned int purgeable, clean, dirty, wired;


@@ -14743,11 +15632,24 @@ kern_return_t vm_map_freeze(
                                        *has_shared = TRUE;
                                }
                        } else {
                                        *has_shared = TRUE;
                                }
                        } else {

-                               /*
-                                * To the compressor.
-                                */
-                               if (entry2->object.vm_object->internal == TRUE) {
-                                       vm_object_pageout(entry2->object.vm_object);
+                               if (VME_OBJECT(entry2)->internal == TRUE) {
+                                       
+                                       if (DEFAULT_FREEZER_COMPRESSED_PAGER_IS_SWAPBACKED) {
+                                               /*
+                                                * Pages belonging to this object could be swapped to disk.
+                                                * Make sure it's not a shared object because we could end
+                                                * up just bringing it back in again.
+                                                */
+                                               if (VME_OBJECT(entry2)->ref_count > 1) {
+                                                       continue;
+                                               }
+                                       }
+                                       vm_object_compressed_freezer_pageout(VME_OBJECT(entry2));
+                               }
+
+                               if (vm_compressor_low_on_space() || vm_swap_low_on_space()) {
+                                       kr = KERN_NO_SPACE;
+                                       break;  

                                }
                        }
                }
                                }
                        }
                }


@@ -14761,6 +15663,16 @@ kern_return_t vm_map_freeze(
 done:
        vm_map_unlock(map);
        
 done:
        vm_map_unlock(map);
        

+       if (!default_freezer_active) {
+               vm_object_compressed_freezer_done();
+       }
+       if (DEFAULT_FREEZER_COMPRESSED_PAGER_IS_SWAPBACKED) {
+               /*
+                * reset the counter tracking the # of swapped c_segs
+                * because we are now done with this freeze session and task.
+                */
+               c_freezer_swapout_count = 0;
+       }

        return kr;
 }
 
        return kr;
 }
 


@@ -14812,6 +15724,8 @@ out:
  *     - copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC
  *     - !true_share
  *     - vo_size == ANON_CHUNK_SIZE
  *     - copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC
  *     - !true_share
  *     - vo_size == ANON_CHUNK_SIZE

+ *
+ * Only non-kernel map entries.

  */
 boolean_t
 vm_map_entry_should_cow_for_true_share(
  */
 boolean_t
 vm_map_entry_should_cow_for_true_share(


@@ -14829,8 +15743,8 @@ vm_map_entry_should_cow_for_true_share(
                return FALSE;
        }
 
                return FALSE;
        }
 

-       if (entry->alias != VM_MEMORY_MALLOC &&
-           entry->alias != VM_MEMORY_MALLOC_SMALL) {
+       if (VME_ALIAS(entry) != VM_MEMORY_MALLOC &&
+           VME_ALIAS(entry) != VM_MEMORY_MALLOC_SMALL) {

                /* not a malloc heap or Obj-C Garbage Collector heap */
                return FALSE;
        }
                /* not a malloc heap or Obj-C Garbage Collector heap */
                return FALSE;
        }


@@ -14841,7 +15755,7 @@ vm_map_entry_should_cow_for_true_share(
                return FALSE;
        }
 
                return FALSE;
        }
 

-       object = entry->object.vm_object;
+       object = VME_OBJECT(entry);

 
        if (object == VM_OBJECT_NULL) {
                /* no object yet... */
 
        if (object == VM_OBJECT_NULL) {
                /* no object yet... */


@@ -14863,13 +15777,13 @@ vm_map_entry_should_cow_for_true_share(
                return FALSE;
        }
 
                return FALSE;
        }
 

-       if (entry->alias == VM_MEMORY_MALLOC &&
+       if (VME_ALIAS(entry) == VM_MEMORY_MALLOC &&

            object->vo_size != ANON_CHUNK_SIZE) {
                /* ... not an object created for the ObjC Garbage Collector */
                return FALSE;
        }
 
            object->vo_size != ANON_CHUNK_SIZE) {
                /* ... not an object created for the ObjC Garbage Collector */
                return FALSE;
        }
 

-       if (entry->alias == VM_MEMORY_MALLOC_SMALL &&
+       if (VME_ALIAS(entry) == VM_MEMORY_MALLOC_SMALL &&

            object->vo_size != 2048 * 4096) {
                /* ... not a "MALLOC_SMALL" heap */
                return FALSE;
            object->vo_size != 2048 * 4096) {
                /* ... not a "MALLOC_SMALL" heap */
                return FALSE;


@@ -14900,6 +15814,14 @@ vm_map_trunc_page_mask(
        return VM_MAP_TRUNC_PAGE(offset, mask);
 }
 
        return VM_MAP_TRUNC_PAGE(offset, mask);
 }
 

+boolean_t
+vm_map_page_aligned(
+       vm_map_offset_t offset,
+       vm_map_offset_t mask)
+{
+       return ((offset) & mask) == 0;
+}
+

 int
 vm_map_page_shift(
        vm_map_t map)
 int
 vm_map_page_shift(
        vm_map_t map)


@@ -14914,7 +15836,7 @@ vm_map_page_size(
        return VM_MAP_PAGE_SIZE(map);
 }
 
        return VM_MAP_PAGE_SIZE(map);
 }
 

-int
+vm_map_offset_t

 vm_map_page_mask(
        vm_map_t map)
 {
 vm_map_page_mask(
        vm_map_t map)
 {


@@ -14958,7 +15880,7 @@ vm_map_purge(
                if (! (entry->protection & VM_PROT_WRITE)) {
                        goto next;
                }
                if (! (entry->protection & VM_PROT_WRITE)) {
                        goto next;
                }

-               object = entry->object.vm_object;
+               object = VME_OBJECT(entry);

                if (object == VM_OBJECT_NULL) {
                        goto next;
                }
                if (object == VM_OBJECT_NULL) {
                        goto next;
                }


@@ -14968,7 +15890,7 @@ vm_map_purge(
 
                vm_object_lock(object);
 #if 00
 
                vm_object_lock(object);
 #if 00

-               if (entry->offset != 0 ||
+               if (VME_OFFSET(entry) != 0 ||

                    (entry->vme_end - entry->vme_start) != object->vo_size) {
                        vm_object_unlock(object);
                        goto next;
                    (entry->vme_end - entry->vme_start) != object->vo_size) {
                        vm_object_unlock(object);
                        goto next;


@@ -15002,11 +15924,15 @@ vm_map_query_volatile(
        vm_map_t        map,
        mach_vm_size_t  *volatile_virtual_size_p,
        mach_vm_size_t  *volatile_resident_size_p,
        vm_map_t        map,
        mach_vm_size_t  *volatile_virtual_size_p,
        mach_vm_size_t  *volatile_resident_size_p,

-       mach_vm_size_t  *volatile_pmap_size_p)
+       mach_vm_size_t  *volatile_compressed_size_p,
+       mach_vm_size_t  *volatile_pmap_size_p,
+       mach_vm_size_t  *volatile_compressed_pmap_size_p)

 {
        mach_vm_size_t  volatile_virtual_size;
        mach_vm_size_t  volatile_resident_count;
 {
        mach_vm_size_t  volatile_virtual_size;
        mach_vm_size_t  volatile_resident_count;

+       mach_vm_size_t  volatile_compressed_count;

        mach_vm_size_t  volatile_pmap_count;
        mach_vm_size_t  volatile_pmap_count;

+       mach_vm_size_t  volatile_compressed_pmap_count;

        mach_vm_size_t  resident_count;
        vm_map_entry_t  entry;
        vm_object_t     object;
        mach_vm_size_t  resident_count;
        vm_map_entry_t  entry;
        vm_object_t     object;


@@ -15015,25 +15941,30 @@ vm_map_query_volatile(
 
        volatile_virtual_size = 0;
        volatile_resident_count = 0;
 
        volatile_virtual_size = 0;
        volatile_resident_count = 0;

+       volatile_compressed_count = 0;

        volatile_pmap_count = 0;
        volatile_pmap_count = 0;

+       volatile_compressed_pmap_count = 0;

 
        for (entry = vm_map_first_entry(map);
             entry != vm_map_to_entry(map);
             entry = entry->vme_next) {
 
        for (entry = vm_map_first_entry(map);
             entry != vm_map_to_entry(map);
             entry = entry->vme_next) {

+               mach_vm_size_t  pmap_resident_bytes, pmap_compressed_bytes;
+

                if (entry->is_sub_map) {
                        continue;
                }
                if (! (entry->protection & VM_PROT_WRITE)) {
                        continue;
                }
                if (entry->is_sub_map) {
                        continue;
                }
                if (! (entry->protection & VM_PROT_WRITE)) {
                        continue;
                }

-               object = entry->object.vm_object;
+               object = VME_OBJECT(entry);

                if (object == VM_OBJECT_NULL) {
                        continue;
                }
                if (object == VM_OBJECT_NULL) {
                        continue;
                }

-               if (object->purgable != VM_PURGABLE_VOLATILE) {
+               if (object->purgable != VM_PURGABLE_VOLATILE &&
+                   object->purgable != VM_PURGABLE_EMPTY) {

                        continue;
                }
                        continue;
                }

-               if (entry->offset != 0) {
+               if (VME_OFFSET(entry)) {

                        /*
                         * If the map entry has been split and the object now
                         * appears several times in the VM map, we don't want
                        /*
                         * If the map entry has been split and the object now
                         * appears several times in the VM map, we don't want


@@ -15044,28 +15975,75 @@ vm_map_query_volatile(
                        continue;
                }
                resident_count = object->resident_page_count;
                        continue;
                }
                resident_count = object->resident_page_count;

-               if ((entry->offset / PAGE_SIZE) >= resident_count) {
+               if ((VME_OFFSET(entry) / PAGE_SIZE) >= resident_count) {

                        resident_count = 0;
                } else {
                        resident_count = 0;
                } else {

-                       resident_count -= (entry->offset / PAGE_SIZE);
+                       resident_count -= (VME_OFFSET(entry) / PAGE_SIZE);

                }
 
                volatile_virtual_size += entry->vme_end - entry->vme_start;
                volatile_resident_count += resident_count;
                }
 
                volatile_virtual_size += entry->vme_end - entry->vme_start;
                volatile_resident_count += resident_count;

-               volatile_pmap_count += pmap_query_resident(map->pmap,
-                                                          entry->vme_start,
-                                                          entry->vme_end);
+               if (object->pager) {
+                       volatile_compressed_count +=
+                               vm_compressor_pager_get_count(object->pager);
+               }
+               pmap_compressed_bytes = 0;
+               pmap_resident_bytes =
+                       pmap_query_resident(map->pmap,
+                                           entry->vme_start,
+                                           entry->vme_end,
+                                           &pmap_compressed_bytes);
+               volatile_pmap_count += (pmap_resident_bytes / PAGE_SIZE);
+               volatile_compressed_pmap_count += (pmap_compressed_bytes
+                                                  / PAGE_SIZE);

        }
 
        /* map is still locked on return */
 
        *volatile_virtual_size_p = volatile_virtual_size;
        *volatile_resident_size_p = volatile_resident_count * PAGE_SIZE;
        }
 
        /* map is still locked on return */
 
        *volatile_virtual_size_p = volatile_virtual_size;
        *volatile_resident_size_p = volatile_resident_count * PAGE_SIZE;

+       *volatile_compressed_size_p = volatile_compressed_count * PAGE_SIZE;

        *volatile_pmap_size_p = volatile_pmap_count * PAGE_SIZE;
        *volatile_pmap_size_p = volatile_pmap_count * PAGE_SIZE;

+       *volatile_compressed_pmap_size_p = volatile_compressed_pmap_count * PAGE_SIZE;

 
        return KERN_SUCCESS;
 }
 
 
        return KERN_SUCCESS;
 }
 

+void
+vm_map_sizes(vm_map_t map,
+               vm_map_size_t * psize,
+               vm_map_size_t * pfree,
+               vm_map_size_t * plargest_free)
+{
+    vm_map_entry_t  entry;
+    vm_map_offset_t prev;
+    vm_map_size_t   free, total_free, largest_free;
+    boolean_t       end;
+
+    total_free = largest_free = 0;
+
+    vm_map_lock_read(map);
+    if (psize) *psize = map->max_offset - map->min_offset;
+
+    prev = map->min_offset;
+    for (entry = vm_map_first_entry(map);; entry = entry->vme_next)
+    {
+       end = (entry == vm_map_to_entry(map));
+
+       if (end) free = entry->vme_end   - prev;
+       else     free = entry->vme_start - prev;
+
+       total_free += free;
+       if (free > largest_free) largest_free = free;
+
+       if (end) break;
+       prev = entry->vme_end;
+    }
+    vm_map_unlock_read(map);
+    if (pfree)         *pfree = total_free;
+    if (plargest_free) *plargest_free = largest_free;
+}
+

 #if VM_SCAN_FOR_SHADOW_CHAIN
 int vm_map_shadow_max(vm_map_t map);
 int vm_map_shadow_max(
 #if VM_SCAN_FOR_SHADOW_CHAIN
 int vm_map_shadow_max(vm_map_t map);
 int vm_map_shadow_max(


@@ -15088,7 +16066,7 @@ int vm_map_shadow_max(
                if (entry->is_sub_map) {
                        continue;
                }
                if (entry->is_sub_map) {
                        continue;
                }

-               object = entry->object.vm_object;
+               object = VME_OBJECT(entry);

                if (object == NULL) {
                        continue;
                }
                if (object == NULL) {
                        continue;
                }