xnu-517.11.1.tar.gz

[apple/xnu.git] / osfmk / vm / vm_object.c

diff --git a/osfmk/vm/vm_object.c b/osfmk/vm/vm_object.c
index ac2a0a1f1fa3422e23e5e2dece236007081b396e..7fad2ac73925c26e011a7817dac6ac60a10d0b41 100644 (file)
--- a/osfmk/vm/vm_object.c
+++ b/osfmk/vm/vm_object.c
@@ -1,24 +1,21 @@
 /*
 /*

- * Copyright (c) 2000-2001 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2003 Apple Computer, Inc. All rights reserved.

  *
  * @APPLE_LICENSE_HEADER_START@
  * 
  *
  * @APPLE_LICENSE_HEADER_START@
  * 

- * Copyright (c) 1999-2003 Apple Computer, Inc.  All Rights Reserved.
+ * The contents of this file constitute Original Code as defined in and
+ * are subject to the Apple Public Source License Version 1.1 (the
+ * "License").  You may not use this file except in compliance with the
+ * License.  Please obtain a copy of the License at
+ * http://www.apple.com/publicsource and read it before using this file.

  * 
  * 

- * This file contains Original Code and/or Modifications of Original Code
- * as defined in and that are subject to the Apple Public Source License
- * Version 2.0 (the 'License'). You may not use this file except in
- * compliance with the License. Please obtain a copy of the License at
- * http://www.opensource.apple.com/apsl/ and read it before using this
- * file.
- * 
- * The Original Code and all software distributed under the License are
- * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * This Original Code and all software distributed under the License are
+ * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER

  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,

- * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
- * Please see the License for the specific language governing rights and
- * limitations under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
+ * License for the specific language governing rights and limitations
+ * under the License.

  * 
  * @APPLE_LICENSE_HEADER_END@
  */
  * 
  * @APPLE_LICENSE_HEADER_END@
  */


@@ -89,8 +86,6 @@ extern int    vnode_pager_workaround;
 #include <vm/vm_pageout.h>
 #include <kern/misc_protos.h>
 
 #include <vm/vm_pageout.h>
 #include <kern/misc_protos.h>
 

-
-

 /*
  *     Virtual memory objects maintain the actual data
  *     associated with allocated virtual memory.  A given
 /*
  *     Virtual memory objects maintain the actual data
  *     associated with allocated virtual memory.  A given


@@ -409,8 +404,8 @@ vm_object_bootstrap(void)
        register        i;
 
        vm_object_zone = zinit((vm_size_t) sizeof(struct vm_object),
        register        i;
 
        vm_object_zone = zinit((vm_size_t) sizeof(struct vm_object),

-                               round_page(512*1024),
-                               round_page(12*1024),
+                               round_page_32(512*1024),
+                               round_page_32(12*1024),

                                "vm objects");
 
        queue_init(&vm_object_cached_list);
                                "vm objects");
 
        queue_init(&vm_object_cached_list);


@@ -418,8 +413,8 @@ vm_object_bootstrap(void)
 
        vm_object_hash_zone =
                        zinit((vm_size_t) sizeof (struct vm_object_hash_entry),
 
        vm_object_hash_zone =
                        zinit((vm_size_t) sizeof (struct vm_object_hash_entry),

-                             round_page(512*1024),
-                             round_page(12*1024),
+                             round_page_32(512*1024),
+                             round_page_32(12*1024),

                              "vm object hash entries");
 
        for (i = 0; i < VM_OBJECT_HASH_COUNT; i++)
                              "vm object hash entries");
 
        for (i = 0; i < VM_OBJECT_HASH_COUNT; i++)


@@ -440,7 +435,7 @@ vm_object_bootstrap(void)
        vm_object_template.copy = VM_OBJECT_NULL;
        vm_object_template.shadow = VM_OBJECT_NULL;
        vm_object_template.shadow_offset = (vm_object_offset_t) 0;
        vm_object_template.copy = VM_OBJECT_NULL;
        vm_object_template.shadow = VM_OBJECT_NULL;
        vm_object_template.shadow_offset = (vm_object_offset_t) 0;

-       vm_object_template.cow_hint = 0;
+       vm_object_template.cow_hint = ~(vm_offset_t)0;

        vm_object_template.true_share = FALSE;
 
        vm_object_template.pager = MEMORY_OBJECT_NULL;
        vm_object_template.true_share = FALSE;
 
        vm_object_template.pager = MEMORY_OBJECT_NULL;


@@ -496,10 +491,17 @@ vm_object_bootstrap(void)
 
 /*
  *     Note that in the following size specifications, we need to add 1 because 
 
 /*
  *     Note that in the following size specifications, we need to add 1 because 

- *     VM_MAX_KERNEL_ADDRESS is a maximum address, not a size.
+ *     VM_MAX_KERNEL_ADDRESS (vm_last_addr) is a maximum address, not a size.

  */
  */

+
+#ifdef ppc
+       _vm_object_allocate((vm_last_addr - VM_MIN_KERNEL_ADDRESS) + 1,
+                       kernel_object);
+#else

        _vm_object_allocate((VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS) + 1,
                        kernel_object);
        _vm_object_allocate((VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS) + 1,
                        kernel_object);

+#endif
+       kernel_object->copy_strategy = MEMORY_OBJECT_COPY_NONE;

 
        /*
         *      Initialize the "submap object".  Make it as large as the
 
        /*
         *      Initialize the "submap object".  Make it as large as the


@@ -507,8 +509,15 @@ vm_object_bootstrap(void)
         */
 
        vm_submap_object = &vm_submap_object_store;
         */
 
        vm_submap_object = &vm_submap_object_store;

+#ifdef ppc
+       _vm_object_allocate((vm_last_addr - VM_MIN_KERNEL_ADDRESS) + 1,
+                       vm_submap_object);
+#else

        _vm_object_allocate((VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS) + 1,
                        vm_submap_object);
        _vm_object_allocate((VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS) + 1,
                        vm_submap_object);

+#endif
+       vm_submap_object->copy_strategy = MEMORY_OBJECT_COPY_NONE;
+

        /*
         * Create an "extra" reference to this object so that we never
         * try to deallocate it; zfree doesn't like to be called with
        /*
         * Create an "extra" reference to this object so that we never
         * try to deallocate it; zfree doesn't like to be called with


@@ -571,10 +580,21 @@ vm_object_deallocate(
                 *      the object; we must lock it before removing
                 *      the object.
                 */
                 *      the object; we must lock it before removing
                 *      the object.
                 */

+               for (;;) {
+                       vm_object_cache_lock();

 
 

-               vm_object_cache_lock();
-               vm_object_lock(object);
-
+                       /*
+                        * if we try to take a regular lock here
+                        * we risk deadlocking against someone
+                        * holding a lock on this object while
+                        * trying to vm_object_deallocate a different
+                        * object
+                        */
+                       if (vm_object_lock_try(object))
+                               break;
+                       vm_object_cache_unlock();
+                       mutex_pause();  /* wait a bit */
+               }

                assert(object->ref_count > 0);
 
                /*
                assert(object->ref_count > 0);
 
                /*


@@ -594,8 +614,21 @@ vm_object_deallocate(
                                        
                                memory_object_unmap(pager);
 
                                        
                                memory_object_unmap(pager);
 

-                               vm_object_cache_lock();
-                               vm_object_lock(object);
+                               for (;;) {
+                                       vm_object_cache_lock();
+
+                                       /*
+                                        * if we try to take a regular lock here
+                                        * we risk deadlocking against someone
+                                        * holding a lock on this object while
+                                        * trying to vm_object_deallocate a different
+                                        * object
+                                        */
+                                       if (vm_object_lock_try(object))
+                                               break;
+                                       vm_object_cache_unlock();
+                                       mutex_pause();  /* wait a bit */
+                               }

                                assert(object->ref_count > 0);
                        }
                }
                                assert(object->ref_count > 0);
                        }
                }


@@ -904,6 +937,7 @@ vm_object_terminate(
                }
 
                vm_page_lock_queues();
                }
 
                vm_page_lock_queues();

+               p->busy = TRUE;

                VM_PAGE_QUEUES_REMOVE(p);
                vm_page_unlock_queues();
 
                VM_PAGE_QUEUES_REMOVE(p);
                vm_page_unlock_queues();
 


@@ -924,16 +958,10 @@ vm_object_terminate(
                        panic("vm_object_terminate.4 0x%x 0x%x", object, p);
 
                if (!p->dirty)
                        panic("vm_object_terminate.4 0x%x 0x%x", object, p);
 
                if (!p->dirty)

-                       p->dirty = pmap_is_modified(p->phys_addr);
+                       p->dirty = pmap_is_modified(p->phys_page);

 
                if ((p->dirty || p->precious) && !p->error && object->alive) {
 
                if ((p->dirty || p->precious) && !p->error && object->alive) {

-                       p->busy = TRUE;
-                       vm_object_paging_begin(object);
-                       /* protect the object from re-use/caching while it */
-                       /* is unlocked */
-                       vm_object_unlock(object);

                        vm_pageout_cluster(p); /* flush page */
                        vm_pageout_cluster(p); /* flush page */

-                       vm_object_lock(object);

                        vm_object_paging_wait(object, THREAD_UNINT);
                        XPR(XPR_VM_OBJECT,
                            "vm_object_terminate restart, object 0x%X ref %d\n",
                        vm_object_paging_wait(object, THREAD_UNINT);
                        XPR(XPR_VM_OBJECT,
                            "vm_object_terminate restart, object 0x%X ref %d\n",


@@ -982,14 +1010,14 @@ vm_object_terminate(
 
        /*
         *      Detach the object from its shadow if we are the shadow's
 
        /*
         *      Detach the object from its shadow if we are the shadow's

-        *      copy.
+        *      copy. The reference we hold on the shadow must be dropped
+        *      by our caller.

         */
        if (((shadow_object = object->shadow) != VM_OBJECT_NULL) &&
            !(object->pageout)) {
                vm_object_lock(shadow_object);
         */
        if (((shadow_object = object->shadow) != VM_OBJECT_NULL) &&
            !(object->pageout)) {
                vm_object_lock(shadow_object);

-               assert((shadow_object->copy == object) ||
-                      (shadow_object->copy == VM_OBJECT_NULL));
-               shadow_object->copy = VM_OBJECT_NULL;
+               if (shadow_object->copy == object)
+                       shadow_object->copy = VM_OBJECT_NULL;

                vm_object_unlock(shadow_object);
        }
 
                vm_object_unlock(shadow_object);
        }
 


@@ -1320,12 +1348,12 @@ vm_object_deactivate_pages(
                                if ((m->wire_count == 0) && (!m->private) && (!m->gobbled) && (!m->busy)) {
 
                                        m->reference = FALSE;
                                if ((m->wire_count == 0) && (!m->private) && (!m->gobbled) && (!m->busy)) {
 
                                        m->reference = FALSE;

-                                       pmap_clear_reference(m->phys_addr);
+                                       pmap_clear_reference(m->phys_page);

 
                                        if ((kill_page) && (object->internal)) {
                                                m->precious = FALSE;
                                                m->dirty = FALSE;
 
                                        if ((kill_page) && (object->internal)) {
                                                m->precious = FALSE;
                                                m->dirty = FALSE;

-                                               pmap_clear_modify(m->phys_addr);
+                                               pmap_clear_modify(m->phys_page);

                                                vm_external_state_clr(object->existence_map, offset);
                                        }
                                        VM_PAGE_QUEUES_REMOVE(m);
                                                vm_external_state_clr(object->existence_map, offset);
                                        }
                                        VM_PAGE_QUEUES_REMOVE(m);


@@ -1411,10 +1439,10 @@ vm_object_pmap_protect(
 
        vm_object_lock(object);
 
 
        vm_object_lock(object);
 

-       assert(object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC);
+       assert(object->internal);

 
        while (TRUE) {
 
        while (TRUE) {

-           if (object->resident_page_count > atop(size) / 2 &&
+           if (object->resident_page_count > atop_32(size) / 2 &&

                    pmap != PMAP_NULL) {
                vm_object_unlock(object);
                pmap_protect(pmap, pmap_start, pmap_start + size, prot);
                    pmap != PMAP_NULL) {
                vm_object_unlock(object);
                pmap_protect(pmap, pmap_start, pmap_start + size, prot);


@@ -1424,7 +1452,7 @@ vm_object_pmap_protect(
            /* if we are doing large ranges with respect to resident */
            /* page count then we should interate over pages otherwise */
            /* inverse page look-up will be faster */
            /* if we are doing large ranges with respect to resident */
            /* page count then we should interate over pages otherwise */
            /* inverse page look-up will be faster */

-           if ((object->resident_page_count / 4) <  atop(size)) {
+           if ((object->resident_page_count / 4) <  atop_32(size)) {

                vm_page_t               p;
                vm_object_offset_t      end;
 
                vm_page_t               p;
                vm_object_offset_t      end;
 


@@ -1446,7 +1474,7 @@ vm_object_pmap_protect(
                    if (!p->fictitious &&
                        (offset <= p->offset) && (p->offset < end)) {
 
                    if (!p->fictitious &&
                        (offset <= p->offset) && (p->offset < end)) {
 

-                           pmap_page_protect(p->phys_addr,
+                           pmap_page_protect(p->phys_page,

                                              prot & ~p->page_lock);
                    }
                  }
                                              prot & ~p->page_lock);
                    }
                  }


@@ -1472,7 +1500,7 @@ vm_object_pmap_protect(
                        for(target_off = offset; 
                                target_off < end; target_off += PAGE_SIZE) {
                                if(p = vm_page_lookup(object, target_off)) {
                        for(target_off = offset; 
                                target_off < end; target_off += PAGE_SIZE) {
                                if(p = vm_page_lookup(object, target_off)) {

-                                       pmap_page_protect(p->phys_addr,
+                                       pmap_page_protect(p->phys_page,

                                                      prot & ~p->page_lock);
                                }
                        }
                                                      prot & ~p->page_lock);
                                }
                        }


@@ -1935,7 +1963,8 @@ static int copy_delayed_protect_lookup_wait = 0;
  *             the asymmetric copy-on-write algorithm.
  *
  *     In/out conditions:
  *             the asymmetric copy-on-write algorithm.
  *
  *     In/out conditions:

- *             The object must be unlocked on entry.
+ *             The src_object must be locked on entry.  It will be unlocked
+ *             on exit - so the caller must also hold a reference to it.

  *
  *             This routine will not block waiting for user-generated
  *             events.  It is not interruptible.
  *
  *             This routine will not block waiting for user-generated
  *             events.  It is not interruptible.


@@ -1949,7 +1978,7 @@ vm_object_copy_delayed(
        vm_object_t             new_copy = VM_OBJECT_NULL;
        vm_object_t             old_copy;
        vm_page_t               p;
        vm_object_t             new_copy = VM_OBJECT_NULL;
        vm_object_t             old_copy;
        vm_page_t               p;

-       vm_object_size_t        copy_size;
+       vm_object_size_t        copy_size = src_offset + size;

 
        int collisions = 0;
        /*
 
        int collisions = 0;
        /*


@@ -1992,8 +2021,13 @@ vm_object_copy_delayed(
         */
 
  Retry:
         */
 
  Retry:

-       vm_object_lock(src_object);

  
  

+       /*
+        * Wait for paging in progress.
+        */
+       if (!src_object->true_share)
+               vm_object_paging_wait(src_object, THREAD_UNINT);
+

        /*
         *      See whether we can reuse the result of a previous
         *      copy operation.
        /*
         *      See whether we can reuse the result of a previous
         *      copy operation.


@@ -2016,6 +2050,7 @@ vm_object_copy_delayed(
                        if (collisions > copy_delayed_max_collisions)
                                copy_delayed_max_collisions = collisions;
 
                        if (collisions > copy_delayed_max_collisions)
                                copy_delayed_max_collisions = collisions;
 

+                       vm_object_lock(src_object);

                        goto Retry;
                }
 
                        goto Retry;
                }
 


@@ -2030,52 +2065,66 @@ vm_object_copy_delayed(
                         *      It has not been modified.
                         *
                         *      Return another reference to
                         *      It has not been modified.
                         *
                         *      Return another reference to

-                        *      the existing copy-object.
+                        *      the existing copy-object if
+                        *      we can safely grow it (if
+                        *      needed).

                         */
                         */

-                       assert(old_copy->ref_count > 0);
-                       old_copy->ref_count++;
-
-                       if (old_copy->size < src_offset+size)
-                               old_copy->size = src_offset+size;
-
-#if    TASK_SWAPPER
-                       /*
-                        * We have to reproduce some of the code from
-                        * vm_object_res_reference because we've taken
-                        * the locks out of order here, and deadlock
-                        * would result if we simply called that function.
-                        */
-                       if (++old_copy->res_count == 1) {
-                               assert(old_copy->shadow == src_object);
-                               vm_object_res_reference(src_object);
-                       }
-#endif /* TASK_SWAPPER */
-
-                       vm_object_unlock(old_copy);
-                       vm_object_unlock(src_object);

 
                        if (new_copy != VM_OBJECT_NULL) {
                                vm_object_unlock(new_copy);
                                vm_object_deallocate(new_copy);
                        }
 
 
                        if (new_copy != VM_OBJECT_NULL) {
                                vm_object_unlock(new_copy);
                                vm_object_deallocate(new_copy);
                        }
 

-                       return(old_copy);
-               }
-               if (new_copy == VM_OBJECT_NULL) {
+                       if (old_copy->size < copy_size) {
+                               /*
+                                * We can't perform a delayed copy if any of the
+                                * pages in the extended range are wired (because
+                                * we can't safely take write permission away from
+                                * wired pages).  If the pages aren't wired, then
+                                * go ahead and protect them.
+                                */
+                               copy_delayed_protect_iterate++;
+                               queue_iterate(&src_object->memq, p, vm_page_t, listq) {
+                                       if (!p->fictitious && 
+                                           p->offset >= old_copy->size && 
+                                           p->offset < copy_size) {
+                                               if (p->wire_count > 0) {
+                                                       vm_object_unlock(old_copy);
+                                                       vm_object_unlock(src_object);
+                                                       return VM_OBJECT_NULL;
+                                               } else {
+                                                       pmap_page_protect(p->phys_page, 
+                                                               (VM_PROT_ALL & ~VM_PROT_WRITE &
+                                                                ~p->page_lock));
+                                               }
+                                       }
+                               }
+                               old_copy->size = copy_size;
+                       }
+                               
+                       vm_object_reference_locked(old_copy);

                        vm_object_unlock(old_copy);
                        vm_object_unlock(src_object);
                        vm_object_unlock(old_copy);
                        vm_object_unlock(src_object);

-                       new_copy = vm_object_allocate(src_offset + size);
-                       vm_object_lock(new_copy);
-                       goto Retry;
+                       return(old_copy);

                }
 
                /*
                 * Adjust the size argument so that the newly-created 
                 * copy object will be large enough to back either the
                }
 
                /*
                 * Adjust the size argument so that the newly-created 
                 * copy object will be large enough to back either the

-                * new old copy object or the new mapping.
+                * old copy object or the new mapping.

                 */
                 */

-               if (old_copy->size > src_offset+size)
-                       size =  old_copy->size - src_offset;
+               if (old_copy->size > copy_size)
+                       copy_size = old_copy->size;
+
+               if (new_copy == VM_OBJECT_NULL) {
+                       vm_object_unlock(old_copy);
+                       vm_object_unlock(src_object);
+                       new_copy = vm_object_allocate(copy_size);
+                       vm_object_lock(src_object);
+                       vm_object_lock(new_copy);
+                       goto Retry;
+               }
+               new_copy->size = copy_size;     

 
                /*
                 *      The copy-object is always made large enough to
 
                /*
                 *      The copy-object is always made large enough to


@@ -2087,6 +2136,44 @@ vm_object_copy_delayed(
                assert((old_copy->shadow == src_object) &&
                    (old_copy->shadow_offset == (vm_object_offset_t) 0));
 
                assert((old_copy->shadow == src_object) &&
                    (old_copy->shadow_offset == (vm_object_offset_t) 0));
 

+       } else if (new_copy == VM_OBJECT_NULL) {
+               vm_object_unlock(src_object);
+               new_copy = vm_object_allocate(copy_size);
+               vm_object_lock(src_object);
+               vm_object_lock(new_copy);
+               goto Retry;
+       }
+
+       /*
+        * We now have the src object locked, and the new copy object
+        * allocated and locked (and potentially the old copy locked).
+        * Before we go any further, make sure we can still perform
+        * a delayed copy, as the situation may have changed.
+        *
+        * Specifically, we can't perform a delayed copy if any of the
+        * pages in the range are wired (because we can't safely take
+        * write permission away from wired pages).  If the pages aren't
+        * wired, then go ahead and protect them.
+        */
+       copy_delayed_protect_iterate++;
+       queue_iterate(&src_object->memq, p, vm_page_t, listq) {
+               if (!p->fictitious && p->offset < copy_size) {
+                       if (p->wire_count > 0) {
+                               if (old_copy)
+                                       vm_object_unlock(old_copy);
+                               vm_object_unlock(src_object);
+                               vm_object_unlock(new_copy);
+                               vm_object_deallocate(new_copy);
+                               return VM_OBJECT_NULL;
+                       } else {
+                               pmap_page_protect(p->phys_page, 
+                                       (VM_PROT_ALL & ~VM_PROT_WRITE &
+                                        ~p->page_lock));
+                       }
+               }
+       }
+
+       if (old_copy != VM_OBJECT_NULL) {

                /*
                 *      Make the old copy-object shadow the new one.
                 *      It will receive no more pages from the original
                /*
                 *      Make the old copy-object shadow the new one.
                 *      It will receive no more pages from the original


@@ -2107,26 +2194,11 @@ vm_object_copy_delayed(
 #endif
 
                vm_object_unlock(old_copy);     /* done with old_copy */
 #endif
 
                vm_object_unlock(old_copy);     /* done with old_copy */

-       } else if (new_copy == VM_OBJECT_NULL) {
-               vm_object_unlock(src_object);
-               new_copy = vm_object_allocate(src_offset + size);
-               vm_object_lock(new_copy);
-               goto Retry;
-       }
-
-       /*
-        * Readjust the copy-object size if necessary.
-        */
-       copy_size = new_copy->size;
-       if (copy_size < src_offset+size) {
-               copy_size = src_offset+size;
-               new_copy->size = copy_size;

        }
 
        /*
         *      Point the new copy at the existing object.
         */
        }
 
        /*
         *      Point the new copy at the existing object.
         */

-

        new_copy->shadow = src_object;
        new_copy->shadow_offset = 0;
        new_copy->shadowed = TRUE;      /* caller must set needs_copy */
        new_copy->shadow = src_object;
        new_copy->shadow_offset = 0;
        new_copy->shadowed = TRUE;      /* caller must set needs_copy */


@@ -2134,23 +2206,9 @@ vm_object_copy_delayed(
        src_object->ref_count++;
        VM_OBJ_RES_INCR(src_object);
        src_object->copy = new_copy;
        src_object->ref_count++;
        VM_OBJ_RES_INCR(src_object);
        src_object->copy = new_copy;

+       vm_object_unlock(src_object);

        vm_object_unlock(new_copy);
 
        vm_object_unlock(new_copy);
 

-       /*
-        *      Mark all (current) pages of the existing object copy-on-write.
-        *      This object may have a shadow chain below it, but
-        *      those pages will already be marked copy-on-write.
-        */
-
-       vm_object_paging_wait(src_object, THREAD_UNINT);
-       copy_delayed_protect_iterate++;
-       queue_iterate(&src_object->memq, p, vm_page_t, listq) {
-           if (!p->fictitious)
-               pmap_page_protect(p->phys_addr, 
-                                 (VM_PROT_ALL & ~VM_PROT_WRITE &
-                                  ~p->page_lock));
-       }
-       vm_object_unlock(src_object);

        XPR(XPR_VM_OBJECT,
                "vm_object_copy_delayed: used copy object %X for source %X\n",
                (integer_t)new_copy, (integer_t)src_object, 0, 0, 0);
        XPR(XPR_VM_OBJECT,
                "vm_object_copy_delayed: used copy object %X for source %X\n",
                (integer_t)new_copy, (integer_t)src_object, 0, 0, 0);


@@ -2210,6 +2268,18 @@ vm_object_copy_strategically(
         */
 
        switch (copy_strategy) {
         */
 
        switch (copy_strategy) {

+           case MEMORY_OBJECT_COPY_DELAY:
+               *dst_object = vm_object_copy_delayed(src_object,
+                                                    src_offset, size);
+               if (*dst_object != VM_OBJECT_NULL) {
+                       *dst_offset = src_offset;
+                       *dst_needs_copy = TRUE;
+                       result = KERN_SUCCESS;
+                       break;
+               }
+               vm_object_lock(src_object);
+               /* fall thru when delayed copy not allowed */
+

            case MEMORY_OBJECT_COPY_NONE:
                result = vm_object_copy_slowly(src_object, src_offset, size,
                                               interruptible, dst_object);
            case MEMORY_OBJECT_COPY_NONE:
                result = vm_object_copy_slowly(src_object, src_offset, size,
                                               interruptible, dst_object);


@@ -2228,15 +2298,6 @@ vm_object_copy_strategically(
                }
                break;
 
                }
                break;
 

-           case MEMORY_OBJECT_COPY_DELAY:
-               vm_object_unlock(src_object);
-               *dst_object = vm_object_copy_delayed(src_object,
-                                                    src_offset, size);
-               *dst_offset = src_offset;
-               *dst_needs_copy = TRUE;
-               result = KERN_SUCCESS;
-               break;
-

            case MEMORY_OBJECT_COPY_SYMMETRIC:
                XPR(XPR_VM_OBJECT, "v_o_c_strategically obj 0x%x off 0x%x size 0x%x\n",(natural_t)src_object, src_offset, size, 0, 0);
                vm_object_unlock(src_object);
            case MEMORY_OBJECT_COPY_SYMMETRIC:
                XPR(XPR_VM_OBJECT, "v_o_c_strategically obj 0x%x off 0x%x size 0x%x\n",(natural_t)src_object, src_offset, size, 0, 0);
                vm_object_unlock(src_object);


@@ -2525,69 +2586,53 @@ vm_object_enter(
         *      Look for an object associated with this port.
         */
 
         *      Look for an object associated with this port.
         */
 

-restart:

        vm_object_cache_lock();
        vm_object_cache_lock();

-       for (;;) {
+       do {

                entry = vm_object_hash_lookup(pager, FALSE);
 
                entry = vm_object_hash_lookup(pager, FALSE);
 

-               /*
-                *      If a previous object is being terminated,
-                *      we must wait for the termination message
-                *      to be queued.
-                *
-                *      We set kobject to a non-null value to let the
-                *      terminator know that someone is waiting.
-                *      Among the possibilities is that the port
-                *      could die while we're waiting.  Must restart
-                *      instead of continuing the loop.
-                */
-
-               if (entry != VM_OBJECT_HASH_ENTRY_NULL) {
-                       if (entry->object != VM_OBJECT_NULL)
-                               break;
-
+               if (entry == VM_OBJECT_HASH_ENTRY_NULL) {
+                       if (new_object == VM_OBJECT_NULL) {
+                               /*
+                                *      We must unlock to create a new object;
+                                *      if we do so, we must try the lookup again.
+                                */
+                               vm_object_cache_unlock();
+                               assert(new_entry == VM_OBJECT_HASH_ENTRY_NULL);
+                               new_entry = vm_object_hash_entry_alloc(pager);
+                               new_object = vm_object_allocate(size);
+                               vm_object_cache_lock();
+                       } else {
+                               /*
+                                *      Lookup failed twice, and we have something
+                                *      to insert; set the object.
+                                */
+                               vm_object_hash_insert(new_entry);
+                               entry = new_entry;
+                               entry->object = new_object;
+                               new_entry = VM_OBJECT_HASH_ENTRY_NULL;
+                               new_object = VM_OBJECT_NULL;
+                               must_init = TRUE;
+                       }
+               } else if (entry->object == VM_OBJECT_NULL) {
+                       /*
+                        *      If a previous object is being terminated,
+                        *      we must wait for the termination message
+                        *      to be queued (and lookup the entry again).
+                        */

                        entry->waiting = TRUE;
                        entry->waiting = TRUE;

+                       entry = VM_OBJECT_HASH_ENTRY_NULL;

                        assert_wait((event_t) pager, THREAD_UNINT);
                        vm_object_cache_unlock();
                        thread_block((void (*)(void))0);
                        assert_wait((event_t) pager, THREAD_UNINT);
                        vm_object_cache_unlock();
                        thread_block((void (*)(void))0);

-                       goto restart;
-               }
-
-               /*
-                *      We must unlock to create a new object;
-                *      if we do so, we must try the lookup again.
-                */
-
-               if (new_object == VM_OBJECT_NULL) {
-                       vm_object_cache_unlock();
-                       assert(new_entry == VM_OBJECT_HASH_ENTRY_NULL);
-                       new_entry = vm_object_hash_entry_alloc(pager);
-                       new_object = vm_object_allocate(size);

                        vm_object_cache_lock();
                        vm_object_cache_lock();

-               } else {
-                       /*
-                        *      Lookup failed twice, and we have something
-                        *      to insert; set the object.
-                        */
-
-                       if (entry == VM_OBJECT_HASH_ENTRY_NULL) {
-                               vm_object_hash_insert(new_entry);
-                               entry = new_entry;
-                               new_entry = VM_OBJECT_HASH_ENTRY_NULL;
-                       }
-
-                       entry->object = new_object;
-                       new_object = VM_OBJECT_NULL;
-                       must_init = TRUE;

                }
                }

-       }
+       } while (entry == VM_OBJECT_HASH_ENTRY_NULL);

 
        object = entry->object;
        assert(object != VM_OBJECT_NULL);
 
        if (!must_init) {
                vm_object_lock(object);
 
        object = entry->object;
        assert(object != VM_OBJECT_NULL);
 
        if (!must_init) {
                vm_object_lock(object);

-               assert(object->pager_created);

                assert(!internal || object->internal);
                if (named) {
                        assert(!object->named);
                assert(!internal || object->internal);
                if (named) {
                        assert(!object->named);


@@ -2958,8 +3003,13 @@ vm_object_do_collapse(
                }
        }
        
                }
        }
        

-       assert(object->pager == MEMORY_OBJECT_NULL ||
-              backing_object->pager == MEMORY_OBJECT_NULL);
+#if    !MACH_PAGEMAP
+       assert(!object->pager_created && object->pager == MEMORY_OBJECT_NULL
+               || (!backing_object->pager_created
+               &&  backing_object->pager == MEMORY_OBJECT_NULL));
+#else 
+        assert(!object->pager_created && object->pager == MEMORY_OBJECT_NULL);
+#endif /* !MACH_PAGEMAP */

 
        if (backing_object->pager != MEMORY_OBJECT_NULL) {
                vm_object_hash_entry_t  entry;
 
        if (backing_object->pager != MEMORY_OBJECT_NULL) {
                vm_object_hash_entry_t  entry;


@@ -2972,6 +3022,7 @@ vm_object_do_collapse(
                 *      unused portion.
                 */
 
                 *      unused portion.
                 */
 

+               assert(!object->paging_in_progress);

                object->pager = backing_object->pager;
                entry = vm_object_hash_lookup(object->pager, FALSE);
                assert(entry != VM_OBJECT_HASH_ENTRY_NULL);
                object->pager = backing_object->pager;
                entry = vm_object_hash_lookup(object->pager, FALSE);
                assert(entry != VM_OBJECT_HASH_ENTRY_NULL);


@@ -2991,8 +3042,6 @@ vm_object_do_collapse(
 
        vm_object_cache_unlock();
 
 
        vm_object_cache_unlock();
 

-       object->paging_offset = backing_object->paging_offset + backing_offset;
-

 #if    MACH_PAGEMAP
        /*
         *      If the shadow offset is 0, the use the existence map from
 #if    MACH_PAGEMAP
        /*
         *      If the shadow offset is 0, the use the existence map from


@@ -3028,7 +3077,7 @@ vm_object_do_collapse(
        object->shadow = backing_object->shadow;
        object->shadow_offset += backing_object->shadow_offset;
        assert((object->shadow == VM_OBJECT_NULL) ||
        object->shadow = backing_object->shadow;
        object->shadow_offset += backing_object->shadow_offset;
        assert((object->shadow == VM_OBJECT_NULL) ||

-              (object->shadow->copy == VM_OBJECT_NULL));
+              (object->shadow->copy != backing_object));

 
        /*
         *      Discard backing_object.
 
        /*
         *      Discard backing_object.


@@ -3168,15 +3217,12 @@ vm_object_do_bypass(
  */
 __private_extern__ void
 vm_object_collapse(
  */
 __private_extern__ void
 vm_object_collapse(

-       register vm_object_t    object)
+       register vm_object_t                    object,
+       register vm_object_offset_t             hint_offset)

 {
        register vm_object_t                    backing_object;
 {
        register vm_object_t                    backing_object;

-       register vm_object_offset_t             backing_offset;
-       register vm_object_size_t               size;
-       register vm_object_offset_t             new_offset;
-       register vm_page_t                      p;
-
-       vm_offset_t                     current_offset; 
+       register unsigned int                   rcount;
+       register unsigned int                   size;

 
        if (! vm_object_collapse_allowed && ! vm_object_bypass_allowed) {
                return;
 
        if (! vm_object_collapse_allowed && ! vm_object_bypass_allowed) {
                return;


@@ -3233,7 +3279,7 @@ vm_object_collapse(
                 *      parent object.
                 */
                if (backing_object->shadow != VM_OBJECT_NULL &&
                 *      parent object.
                 */
                if (backing_object->shadow != VM_OBJECT_NULL &&

-                   backing_object->shadow->copy != VM_OBJECT_NULL) {
+                   backing_object->shadow->copy == backing_object) {

                        vm_object_unlock(backing_object);
                        return;
                }
                        vm_object_unlock(backing_object);
                        return;
                }


@@ -3248,16 +3294,22 @@ vm_object_collapse(
                 *      object, we may be able to collapse it into the
                 *      parent.
                 *
                 *      object, we may be able to collapse it into the
                 *      parent.
                 *

-                *      The backing object must not have a pager
-                *      created for it, since collapsing an object
-                *      into a backing_object dumps new pages into
-                *      the backing_object that its pager doesn't
-                *      know about.
+                *      If MACH_PAGEMAP is defined:
+                *      The parent must not have a pager created for it,
+                *      since collapsing a backing_object dumps new pages
+                *      into the parent that its pager doesn't know about
+                *      (and the collapse code can't merge the existence
+                *      maps).
+                *      Otherwise:
+                *      As long as one of the objects is still not known
+                *      to the pager, we can collapse them.

                 */
                 */

-

                if (backing_object->ref_count == 1 &&
                if (backing_object->ref_count == 1 &&

-                   ! object->pager_created &&
-                   vm_object_collapse_allowed) {
+                   (!object->pager_created 
+#if    !MACH_PAGEMAP
+                       || !backing_object->pager_created
+#endif /*!MACH_PAGEMAP */
+                   ) && vm_object_collapse_allowed) {

 
                        XPR(XPR_VM_OBJECT, 
                   "vm_object_collapse: %x to %x, pager %x, pager_request %x\n",
 
                        XPR(XPR_VM_OBJECT, 
                   "vm_object_collapse: %x to %x, pager %x, pager_request %x\n",


@@ -3298,96 +3350,161 @@ vm_object_collapse(
 
 
                /*
 
 
                /*

-                *      If the backing object has a pager but no pagemap,
-                *      then we cannot bypass it, because we don't know
-                *      what pages it has.
+                *      If the object doesn't have all its pages present,
+                *      we have to make sure no pages in the backing object
+                *      "show through" before bypassing it.

                 */
                 */

-               if (backing_object->pager_created
+               size = atop(object->size);
+               rcount = object->resident_page_count;
+               if (rcount != size) {
+                       vm_object_size_t        size;
+                       vm_object_offset_t      offset;
+                       vm_object_offset_t      backing_offset;
+                       unsigned int            backing_rcount;
+                       unsigned int            lookups = 0;
+
+                       /*
+                        *      If the backing object has a pager but no pagemap,
+                        *      then we cannot bypass it, because we don't know
+                        *      what pages it has.
+                        */
+                       if (backing_object->pager_created

 #if    MACH_PAGEMAP
 #if    MACH_PAGEMAP

-                   && (backing_object->existence_map == VM_EXTERNAL_NULL)
+                               && (backing_object->existence_map == VM_EXTERNAL_NULL)

 #endif /* MACH_PAGEMAP */
 #endif /* MACH_PAGEMAP */

-                   ) {
-                       vm_object_unlock(backing_object);
-                       return;
-               }
+                               ) {
+                               vm_object_unlock(backing_object);
+                               return;
+                       }

 
 

-               /*
-                *      If the object has a pager but no pagemap,
-                *      then we cannot bypass it, because we don't know
-                *      what pages it has.
-                */
-               if (object->pager_created
+                       /*
+                        *      If the object has a pager but no pagemap,
+                        *      then we cannot bypass it, because we don't know
+                        *      what pages it has.
+                        */
+                       if (object->pager_created

 #if    MACH_PAGEMAP
 #if    MACH_PAGEMAP

-                   && (object->existence_map == VM_EXTERNAL_NULL)
+                               && (object->existence_map == VM_EXTERNAL_NULL)

 #endif /* MACH_PAGEMAP */
 #endif /* MACH_PAGEMAP */

-                   ) {
-                       vm_object_unlock(backing_object);
-                       return;
-               }
+                               ) {
+                               vm_object_unlock(backing_object);
+                               return;
+                       }

 
 

-               backing_offset = object->shadow_offset;
-               size = object->size;
+                       /*
+                        *      If all of the pages in the backing object are
+                        *      shadowed by the parent object, the parent
+                        *      object no longer has to shadow the backing
+                        *      object; it can shadow the next one in the
+                        *      chain.
+                        *
+                        *      If the backing object has existence info,
+                        *      we must check examine its existence info
+                        *      as well.
+                        *
+                        */

 
 

-               /*
-                *      If all of the pages in the backing object are
-                *      shadowed by the parent object, the parent
-                *      object no longer has to shadow the backing
-                *      object; it can shadow the next one in the
-                *      chain.
-                *
-                *      If the backing object has existence info,
-                *      we must check examine its existence info
-                *      as well.
-                *
-                */
+                       backing_offset = object->shadow_offset;
+                       backing_rcount = backing_object->resident_page_count;

 
 

-               if(object->cow_hint >= size)
-                       object->cow_hint = 0;
-               current_offset = object->cow_hint;
-               while(TRUE) {
-                       if (vm_page_lookup(object, 
-                               (vm_object_offset_t)current_offset) 
-                                                       != VM_PAGE_NULL) {
-                               current_offset+=PAGE_SIZE;
-                       } else if ((object->pager_created) &&
-                               (object->existence_map != NULL) &&
-                               (vm_external_state_get(object->existence_map, 
-                               current_offset) 
-                                       != VM_EXTERNAL_STATE_ABSENT)) {
-                               current_offset+=PAGE_SIZE;
-                       } else if (vm_page_lookup(backing_object,
-                               (vm_object_offset_t)current_offset
-                               + backing_offset)!= VM_PAGE_NULL) {
-                               /* found a dependency */
-                               object->cow_hint = current_offset;
-                               vm_object_unlock(backing_object);
-                               return;
-                       } else if ((backing_object->pager_created) &&
-                               (backing_object->existence_map != NULL) &&
-                                       (vm_external_state_get(
-                                       backing_object->existence_map, 
-                                       current_offset + backing_offset) 
-                                               != VM_EXTERNAL_STATE_ABSENT)) {
-                               /* found a dependency */
-                               object->cow_hint = current_offset;
+#define EXISTS_IN_OBJECT(obj, off, rc) \
+       (vm_external_state_get((obj)->existence_map, \
+        (vm_offset_t)(off)) == VM_EXTERNAL_STATE_EXISTS || \
+        ((rc) && ++lookups && vm_page_lookup((obj), (off)) != VM_PAGE_NULL && (rc)--))
+
+                       /*
+                        * Check the hint location first
+                        * (since it is often the quickest way out of here).
+                        */
+                       if (object->cow_hint != ~(vm_offset_t)0)
+                               hint_offset = (vm_object_offset_t)object->cow_hint;
+                       else
+                               hint_offset = (hint_offset > 8 * PAGE_SIZE_64) ?
+                                             (hint_offset - 8 * PAGE_SIZE_64) : 0;
+
+                       if (EXISTS_IN_OBJECT(backing_object, hint_offset +
+                                            backing_offset, backing_rcount) &&
+                           !EXISTS_IN_OBJECT(object, hint_offset, rcount)) {
+                               /* dependency right at the hint */
+                               object->cow_hint = (vm_offset_t)hint_offset;

                                vm_object_unlock(backing_object);
                                return;
                                vm_object_unlock(backing_object);
                                return;

-                       } else {
-                               current_offset+=PAGE_SIZE;

                        }
                        }

-                       if(current_offset >= size) {
-                               /* wrap at end of object */
-                               current_offset = 0;
+
+                       /*
+                        * If the object's window onto the backing_object
+                        * is large compared to the number of resident
+                        * pages in the backing object, it makes sense to
+                        * walk the backing_object's resident pages first.
+                        *
+                        * NOTE: Pages may be in both the existence map and 
+                        * resident.  So, we can't permanently decrement
+                        * the rcount here because the second loop may
+                        * find the same pages in the backing object'
+                        * existence map that we found here and we would
+                        * double-decrement the rcount.  We also may or
+                        * may not have found the 
+                        */
+                       if (backing_rcount && size >
+                           ((backing_object->existence_map) ?
+                            backing_rcount : (backing_rcount >> 1))) {
+                               unsigned int rc = rcount;
+                               vm_page_t p;
+
+                               backing_rcount = backing_object->resident_page_count;
+                               p = (vm_page_t)queue_first(&backing_object->memq);
+                               do {
+                                       /* Until we get more than one lookup lock */
+                                       if (lookups > 256) {
+                                               lookups = 0;
+                                               delay(1);
+                                       }
+
+                                       offset = (p->offset - backing_offset);
+                                       if (offset < object->size &&
+                                           offset != hint_offset &&
+                                           !EXISTS_IN_OBJECT(object, offset, rc)) {
+                                               /* found a dependency */
+                                               object->cow_hint = (vm_offset_t)offset;
+                                               vm_object_unlock(backing_object);
+                                               return;
+                                       }
+                                       p = queue_next(p);
+
+                               } while (--backing_rcount);

                        }
                        }

-                       if(current_offset == object->cow_hint) {
-                               /* we are free of shadow influence */
-                               break;
+
+                       /*
+                        * Walk through the offsets looking for pages in the
+                        * backing object that show through to the object.
+                        */
+                       if (backing_rcount || backing_object->existence_map) {
+                               offset = hint_offset;
+                               
+                               while((offset =
+                                     (offset + PAGE_SIZE_64 < object->size) ?
+                                     (offset + PAGE_SIZE_64) : 0) != hint_offset) {
+
+                                       /* Until we get more than one lookup lock */
+                                       if (lookups > 256) {
+                                               lookups = 0;
+                                               delay(1);
+                                       }
+
+                                       if (EXISTS_IN_OBJECT(backing_object, offset +
+                                           backing_offset, backing_rcount) &&
+                                           !EXISTS_IN_OBJECT(object, offset, rcount)) {
+                                               /* found a dependency */
+                                               object->cow_hint = (vm_offset_t)offset;
+                                               vm_object_unlock(backing_object);
+                                               return;
+                                       }
+                               }

                        }
                }
                        }
                }

-               /* reset the cow_hint for any objects deeper in the chain */
-               object->cow_hint = 0;
-

 
 

+               /* reset the offset hint for any objects deeper in the chain */
+               object->cow_hint = (vm_offset_t)0;

 
                /*
                 *      All interesting pages in the backing object
 
                /*
                 *      All interesting pages in the backing object


@@ -3433,7 +3550,7 @@ vm_object_page_remove(
         *      It balances vm_object_lookup vs iteration.
         */
 
         *      It balances vm_object_lookup vs iteration.
         */
 

-       if (atop(end - start) < (unsigned)object->resident_page_count/16) {
+       if (atop_64(end - start) < (unsigned)object->resident_page_count/16) {

                vm_object_page_remove_lookup++;
 
                for (; start < end; start += PAGE_SIZE_64) {
                vm_object_page_remove_lookup++;
 
                for (; start < end; start += PAGE_SIZE_64) {


@@ -3441,7 +3558,7 @@ vm_object_page_remove(
                        if (p != VM_PAGE_NULL) {
                                assert(!p->cleaning && !p->pageout);
                                if (!p->fictitious)
                        if (p != VM_PAGE_NULL) {
                                assert(!p->cleaning && !p->pageout);
                                if (!p->fictitious)

-                                       pmap_page_protect(p->phys_addr,
+                                       pmap_page_protect(p->phys_page,

                                                          VM_PROT_NONE);
                                VM_PAGE_FREE(p);
                        }
                                                          VM_PROT_NONE);
                                VM_PAGE_FREE(p);
                        }


@@ -3455,7 +3572,7 @@ vm_object_page_remove(
                        if ((start <= p->offset) && (p->offset < end)) {
                                assert(!p->cleaning && !p->pageout);
                                if (!p->fictitious)
                        if ((start <= p->offset) && (p->offset < end)) {
                                assert(!p->cleaning && !p->pageout);
                                if (!p->fictitious)

-                                   pmap_page_protect(p->phys_addr,
+                                   pmap_page_protect(p->phys_page,

                                                      VM_PROT_NONE);
                                VM_PAGE_FREE(p);
                        }
                                                      VM_PROT_NONE);
                                VM_PAGE_FREE(p);
                        }


@@ -3522,7 +3639,7 @@ vm_object_coalesce(
        /*
         *      Try to collapse the object first
         */
        /*
         *      Try to collapse the object first
         */

-       vm_object_collapse(prev_object);
+       vm_object_collapse(prev_object, prev_offset);

 
        /*
         *      Can't coalesce if pages not mapped to
 
        /*
         *      Can't coalesce if pages not mapped to


@@ -3600,7 +3717,7 @@ vm_object_page_map(
        vm_page_t       old_page;
        vm_object_offset_t      addr;
 
        vm_page_t       old_page;
        vm_object_offset_t      addr;
 

-       num_pages = atop(size);
+       num_pages = atop_64(size);

 
        for (i = 0; i < num_pages; i++, offset += PAGE_SIZE_64) {
 
 
        for (i = 0; i < num_pages; i++, offset += PAGE_SIZE_64) {
 


@@ -3965,27 +4082,24 @@ vm_object_find(
 
 kern_return_t
 vm_object_populate_with_private(
 
 kern_return_t
 vm_object_populate_with_private(

-               vm_object_t     object,
+               vm_object_t             object,

                vm_object_offset_t      offset,
                vm_object_offset_t      offset,

-               vm_offset_t     phys_addr,
-               vm_size_t       size)
+               ppnum_t                 phys_page,
+               vm_size_t               size)

 {
 {

-       vm_offset_t             base_addr;
+       ppnum_t                 base_page;

        vm_object_offset_t      base_offset;
 
 
        if(!object->private)
                return KERN_FAILURE;
 
        vm_object_offset_t      base_offset;
 
 
        if(!object->private)
                return KERN_FAILURE;
 

-       if((base_addr = trunc_page(phys_addr)) != phys_addr) {
-               return KERN_FAILURE;
-       }
-
+       base_page = phys_page;

 
        vm_object_lock(object);
        if(!object->phys_contiguous) {
                vm_page_t       m;
 
        vm_object_lock(object);
        if(!object->phys_contiguous) {
                vm_page_t       m;

-               if((base_offset = trunc_page(offset)) != offset) {
+               if((base_offset = trunc_page_64(offset)) != offset) {

                        vm_object_unlock(object);
                        return KERN_FAILURE;
                }
                        vm_object_unlock(object);
                        return KERN_FAILURE;
                }


@@ -3997,7 +4111,7 @@ vm_object_populate_with_private(
                                        vm_page_lock_queues();
                                        m->fictitious = FALSE;
                                        m->private = TRUE;
                                        vm_page_lock_queues();
                                        m->fictitious = FALSE;
                                        m->private = TRUE;

-                                       m->phys_addr = base_addr;
+                                       m->phys_page = base_page;

                                        if(!m->busy) {
                                                m->busy = TRUE;
                                        }
                                        if(!m->busy) {
                                                m->busy = TRUE;
                                        }


@@ -4007,11 +4121,11 @@ vm_object_populate_with_private(
                                        }
                                        m->list_req_pending = TRUE;
                                        vm_page_unlock_queues();
                                        }
                                        m->list_req_pending = TRUE;
                                        vm_page_unlock_queues();

-                               } else if (m->phys_addr != base_addr) {
+                               } else if (m->phys_page != base_page) {

                                        /* pmap call to clear old mapping */
                                        /* pmap call to clear old mapping */

-                                       pmap_page_protect(m->phys_addr,
+                                       pmap_page_protect(m->phys_page,

                                                                VM_PROT_NONE);
                                                                VM_PROT_NONE);

-                                       m->phys_addr = base_addr;
+                                       m->phys_page = base_page;

                                }
                        } else {
                                while ((m = vm_page_grab_fictitious()) 
                                }
                        } else {
                                while ((m = vm_page_grab_fictitious()) 


@@ -4020,7 +4134,7 @@ vm_object_populate_with_private(
                                vm_page_lock_queues();
                                m->fictitious = FALSE;
                                m->private = TRUE;
                                vm_page_lock_queues();
                                m->fictitious = FALSE;
                                m->private = TRUE;

-                               m->phys_addr = base_addr;
+                               m->phys_page = base_page;

                                m->list_req_pending = TRUE;
                                m->absent = TRUE;
                                m->unusual = TRUE;
                                m->list_req_pending = TRUE;
                                m->absent = TRUE;
                                m->unusual = TRUE;


@@ -4028,7 +4142,7 @@ vm_object_populate_with_private(
                                vm_page_unlock_queues();
                                vm_page_insert(m, object, base_offset);
                        }
                                vm_page_unlock_queues();
                                vm_page_insert(m, object, base_offset);
                        }

-                       base_addr += PAGE_SIZE;
+                       base_page++;                                                                    /* Go to the next physical page */

                        base_offset += PAGE_SIZE;
                        size -= PAGE_SIZE;
                }
                        base_offset += PAGE_SIZE;
                        size -= PAGE_SIZE;
                }


@@ -4041,7 +4155,7 @@ vm_object_populate_with_private(
                
                /* shadows on contiguous memory are not allowed */
                /* we therefore can use the offset field */
                
                /* shadows on contiguous memory are not allowed */
                /* we therefore can use the offset field */

-               object->shadow_offset = (vm_object_offset_t)phys_addr;
+               object->shadow_offset = (vm_object_offset_t)(phys_page << 12);

                object->size = size;
        }
        vm_object_unlock(object);
                object->size = size;
        }
        vm_object_unlock(object);


@@ -4382,7 +4496,7 @@ vm_object_lock_request(
        if ((prot & ~VM_PROT_ALL) != 0 && prot != VM_PROT_NO_CHANGE)
                return (KERN_INVALID_ARGUMENT);
 
        if ((prot & ~VM_PROT_ALL) != 0 && prot != VM_PROT_NO_CHANGE)
                return (KERN_INVALID_ARGUMENT);
 

-       size = round_page(size);
+       size = round_page_64(size);

 
        /*
         *      Lock the object, and acquire a paging reference to
 
        /*
         *      Lock the object, and acquire a paging reference to


@@ -4390,7 +4504,6 @@ vm_object_lock_request(
         */
        vm_object_lock(object);
        vm_object_paging_begin(object);
         */
        vm_object_lock(object);
        vm_object_paging_begin(object);

-       offset -= object->paging_offset;

 
        (void)vm_object_update(object,
                offset, size, should_return, flags, prot);
 
        (void)vm_object_update(object,
                offset, size, should_return, flags, prot);