xnu-7195.50.7.100.1.tar.gz

[apple/xnu.git] / osfmk / vm / memory_object.c

diff --git a/osfmk/vm/memory_object.c b/osfmk/vm/memory_object.c
index 7f2ebbd3b5838a7a6610c1f0f00696e16274704b..e4013fd8248c433b7617d10acd65c0bb21c33410 100644 (file)
--- a/osfmk/vm/memory_object.c
+++ b/osfmk/vm/memory_object.c
@@ -1,8 +1,8 @@
 /*
 /*

- * Copyright (c) 2000-2008 Apple Inc. All rights reserved.
+ * Copyright (c) 2000-2020 Apple Inc. All rights reserved.

  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@

- * 
+ *

  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in
  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in


@@ -11,10 +11,10 @@
  * unlawful or unlicensed copies of an Apple operating system, or to
  * circumvent, violate, or enable the circumvention or violation of, any
  * terms of an Apple operating system software license agreement.
  * unlawful or unlicensed copies of an Apple operating system, or to
  * circumvent, violate, or enable the circumvention or violation of, any
  * terms of an Apple operating system software license agreement.

- * 
+ *

  * Please obtain a copy of the License at
  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  * Please obtain a copy of the License at
  * http://www.opensource.apple.com/apsl/ and read it before using this file.

- * 
+ *

  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,


@@ -22,34 +22,34 @@
  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  * Please see the License for the specific language governing rights and
  * limitations under the License.
  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  * Please see the License for the specific language governing rights and
  * limitations under the License.

- * 
+ *

  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
  */
  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
  */

-/* 
+/*

  * Mach Operating System
  * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
  * All Rights Reserved.
  * Mach Operating System
  * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
  * All Rights Reserved.

- * 
+ *

  * Permission to use, copy, modify and distribute this software and its
  * documentation is hereby granted, provided that both the copyright
  * notice and this permission notice appear in all copies of the
  * software, derivative works or modified versions, and any portions
  * thereof, and that both notices appear in supporting documentation.
  * Permission to use, copy, modify and distribute this software and its
  * documentation is hereby granted, provided that both the copyright
  * notice and this permission notice appear in all copies of the
  * software, derivative works or modified versions, and any portions
  * thereof, and that both notices appear in supporting documentation.

- * 
+ *

  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
  * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
  * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.

- * 
+ *

  * Carnegie Mellon requests users of this software to return to
  * Carnegie Mellon requests users of this software to return to

- * 
+ *

  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  *  School of Computer Science
  *  Carnegie Mellon University
  *  Pittsburgh PA 15213-3890
  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  *  School of Computer Science
  *  Carnegie Mellon University
  *  Pittsburgh PA 15213-3890

- * 
+ *

  * any improvements or extensions that they make and grant Carnegie Mellon
  * the rights to redistribute these changes.
  */
  * any improvements or extensions that they make and grant Carnegie Mellon
  * the rights to redistribute these changes.
  */


@@ -62,13 +62,11 @@
  *     External memory management interface control functions.
  */
 
  *     External memory management interface control functions.
  */
 

-#include <advisory_pageout.h>
-

 /*
  *     Interface dependencies:
  */
 
 /*
  *     Interface dependencies:
  */
 

-#include <mach/std_types.h>    /* For pointer_t */
+#include <mach/std_types.h>     /* For pointer_t */

 #include <mach/mach_types.h>
 
 #include <mach/mig.h>
 #include <mach/mach_types.h>
 
 #include <mach/mig.h>


@@ -84,11 +82,10 @@
 /*
  *     Implementation dependencies:
  */
 /*
  *     Implementation dependencies:
  */

-#include <string.h>            /* For memcpy() */
+#include <string.h>             /* For memcpy() */

 
 

-#include <kern/xpr.h>          

 #include <kern/host.h>
 #include <kern/host.h>

-#include <kern/thread.h>       /* For current_thread() */
+#include <kern/thread.h>        /* For current_thread() */

 #include <kern/ipc_mig.h>
 #include <kern/misc_protos.h>
 
 #include <kern/ipc_mig.h>
 #include <kern/misc_protos.h>
 


@@ -97,19 +94,18 @@
 #include <vm/memory_object.h>
 #include <vm/vm_page.h>
 #include <vm/vm_pageout.h>
 #include <vm/memory_object.h>
 #include <vm/vm_page.h>
 #include <vm/vm_pageout.h>

-#include <vm/pmap.h>           /* For pmap_clear_modify */
-#include <vm/vm_kern.h>                /* For kernel_map, vm_move */
-#include <vm/vm_map.h>         /* For vm_map_pageable */
-#include <vm/vm_purgeable_internal.h>  /* Needed by some vm_page.h macros */
+#include <vm/pmap.h>            /* For pmap_clear_modify */
+#include <vm/vm_kern.h>         /* For kernel_map, vm_move */
+#include <vm/vm_map.h>          /* For vm_map_pageable */
+#include <vm/vm_purgeable_internal.h>   /* Needed by some vm_page.h macros */

 #include <vm/vm_shared_region.h>
 
 #include <vm/vm_external.h>
 
 #include <vm/vm_protos.h>
 
 #include <vm/vm_shared_region.h>
 
 #include <vm/vm_external.h>
 
 #include <vm/vm_protos.h>
 

-
-memory_object_default_t        memory_manager_default = MEMORY_OBJECT_DEFAULT_NULL;
-decl_lck_mtx_data(,    memory_manager_default_lock)
+memory_object_default_t memory_manager_default = MEMORY_OBJECT_DEFAULT_NULL;
+LCK_MTX_EARLY_DECLARE(memory_manager_default_lock, &vm_object_lck_grp);

 
 
 /*
 
 
 /*


@@ -125,29 +121,29 @@ decl_lck_mtx_data(,       memory_manager_default_lock)
  *             2. Page is precious and should_return is RETURN_ALL.
  *             3. Should_return is RETURN_ANYTHING.
  *
  *             2. Page is precious and should_return is RETURN_ALL.
  *             3. Should_return is RETURN_ANYTHING.
  *

- *             As a side effect, m->dirty will be made consistent
+ *             As a side effect, m->vmp_dirty will be made consistent

  *             with pmap_is_modified(m), if should_return is not
  *             MEMORY_OBJECT_RETURN_NONE.
  */
 
  *             with pmap_is_modified(m), if should_return is not
  *             MEMORY_OBJECT_RETURN_NONE.
  */
 

-#define        memory_object_should_return_page(m, should_return) \
+#define memory_object_should_return_page(m, should_return) \

     (should_return != MEMORY_OBJECT_RETURN_NONE && \
     (should_return != MEMORY_OBJECT_RETURN_NONE && \

-     (((m)->dirty || ((m)->dirty = pmap_is_modified((m)->phys_page))) || \
-      ((m)->precious && (should_return) == MEMORY_OBJECT_RETURN_ALL) || \
+     (((m)->vmp_dirty || ((m)->vmp_dirty = pmap_is_modified(VM_PAGE_GET_PHYS_PAGE(m)))) || \
+      ((m)->vmp_precious && (should_return) == MEMORY_OBJECT_RETURN_ALL) || \

       (should_return) == MEMORY_OBJECT_RETURN_ANYTHING))
 
       (should_return) == MEMORY_OBJECT_RETURN_ANYTHING))
 

-typedef        int     memory_object_lock_result_t;
+typedef int     memory_object_lock_result_t;

 
 

-#define MEMORY_OBJECT_LOCK_RESULT_DONE                 0
-#define MEMORY_OBJECT_LOCK_RESULT_MUST_BLOCK           1
-#define MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN          2
-#define MEMORY_OBJECT_LOCK_RESULT_MUST_FREE            3
+#define MEMORY_OBJECT_LOCK_RESULT_DONE                  0
+#define MEMORY_OBJECT_LOCK_RESULT_MUST_BLOCK            1
+#define MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN           2
+#define MEMORY_OBJECT_LOCK_RESULT_MUST_FREE             3

 
 memory_object_lock_result_t memory_object_lock_page(
 
 memory_object_lock_result_t memory_object_lock_page(

-                               vm_page_t               m,
-                               memory_object_return_t  should_return,
-                               boolean_t               should_flush,
-                               vm_prot_t               prot);
+       vm_page_t               m,
+       memory_object_return_t  should_return,
+       boolean_t               should_flush,
+       vm_prot_t               prot);

 
 /*
  *     Routine:        memory_object_lock_page
 
 /*
  *     Routine:        memory_object_lock_page


@@ -164,58 +160,56 @@ memory_object_lock_result_t memory_object_lock_page(
  */
 memory_object_lock_result_t
 memory_object_lock_page(
  */
 memory_object_lock_result_t
 memory_object_lock_page(

-       vm_page_t               m,
-       memory_object_return_t  should_return,
-       boolean_t               should_flush,
-       vm_prot_t               prot)
+       vm_page_t               m,
+       memory_object_return_t  should_return,
+       boolean_t               should_flush,
+       vm_prot_t               prot)

 {
 {

-        XPR(XPR_MEMORY_OBJECT,
-            "m_o_lock_page, page 0x%X rtn %d flush %d prot %d\n",
-            m, should_return, should_flush, prot, 0);
-
-
-       if (m->busy || m->cleaning)
-               return (MEMORY_OBJECT_LOCK_RESULT_MUST_BLOCK);
+       if (m->vmp_busy || m->vmp_cleaning) {
+               return MEMORY_OBJECT_LOCK_RESULT_MUST_BLOCK;
+       }

 
 

-       if (m->laundry)
+       if (m->vmp_laundry) {

                vm_pageout_steal_laundry(m, FALSE);
                vm_pageout_steal_laundry(m, FALSE);

+       }

 
        /*
         *      Don't worry about pages for which the kernel
         *      does not have any data.
         */
 
        /*
         *      Don't worry about pages for which the kernel
         *      does not have any data.
         */

-       if (m->absent || m->error || m->restart) {
-               if (m->error && should_flush && !VM_PAGE_WIRED(m)) {
+       if (m->vmp_absent || m->vmp_error || m->vmp_restart) {
+               if (m->vmp_error && should_flush && !VM_PAGE_WIRED(m)) {

                        /*
                         * dump the page, pager wants us to
                         * clean it up and there is no
                         * relevant data to return
                         */
                        /*
                         * dump the page, pager wants us to
                         * clean it up and there is no
                         * relevant data to return
                         */

-                       return (MEMORY_OBJECT_LOCK_RESULT_MUST_FREE);
+                       return MEMORY_OBJECT_LOCK_RESULT_MUST_FREE;

                }
                }

-               return (MEMORY_OBJECT_LOCK_RESULT_DONE);
+               return MEMORY_OBJECT_LOCK_RESULT_DONE;

        }
        }

-       assert(!m->fictitious);
+       assert(!m->vmp_fictitious);

 
        if (VM_PAGE_WIRED(m)) {
                /*
                 * The page is wired... just clean or return the page if needed.
                 * Wired pages don't get flushed or disconnected from the pmap.
                 */
 
        if (VM_PAGE_WIRED(m)) {
                /*
                 * The page is wired... just clean or return the page if needed.
                 * Wired pages don't get flushed or disconnected from the pmap.
                 */

-               if (memory_object_should_return_page(m, should_return))
-                       return (MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN);
+               if (memory_object_should_return_page(m, should_return)) {
+                       return MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN;
+               }

 
 

-               return (MEMORY_OBJECT_LOCK_RESULT_DONE);
-       }               
+               return MEMORY_OBJECT_LOCK_RESULT_DONE;
+       }

 
        if (should_flush) {
                /*
 
        if (should_flush) {
                /*

-                * must do the pmap_disconnect before determining the 
+                * must do the pmap_disconnect before determining the

                 * need to return the page... otherwise it's possible
                 * for the page to go from the clean to the dirty state
                 * after we've made our decision
                 */
                 * need to return the page... otherwise it's possible
                 * for the page to go from the clean to the dirty state
                 * after we've made our decision
                 */

-               if (pmap_disconnect(m->phys_page) & VM_MEM_MODIFIED) {
+               if (pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m)) & VM_MEM_MODIFIED) {

                        SET_PAGE_DIRTY(m, FALSE);
                }
        } else {
                        SET_PAGE_DIRTY(m, FALSE);
                }
        } else {


@@ -224,8 +218,9 @@ memory_object_lock_page(
                 * let the fault handler take care of increases
                 * (pmap_page_protect may not increase protection).
                 */
                 * let the fault handler take care of increases
                 * (pmap_page_protect may not increase protection).
                 */

-               if (prot != VM_PROT_NO_CHANGE)
-                       pmap_page_protect(m->phys_page, VM_PROT_ALL & ~prot);
+               if (prot != VM_PROT_NO_CHANGE) {
+                       pmap_page_protect(VM_PAGE_GET_PHYS_PAGE(m), VM_PROT_ALL & ~prot);
+               }

        }
        /*
         *      Handle returning dirty or precious pages
        }
        /*
         *      Handle returning dirty or precious pages


@@ -241,16 +236,17 @@ memory_object_lock_page(
                 * faulted back into an address space
                 *
                 *      if (!should_flush)
                 * faulted back into an address space
                 *
                 *      if (!should_flush)

-                *              pmap_disconnect(m->phys_page);
+                *              pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m));

                 */
                 */

-               return (MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN);
+               return MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN;

        }
 
        /*
         *      Handle flushing clean pages
         */
        }
 
        /*
         *      Handle flushing clean pages
         */

-       if (should_flush)
-               return (MEMORY_OBJECT_LOCK_RESULT_MUST_FREE);
+       if (should_flush) {
+               return MEMORY_OBJECT_LOCK_RESULT_MUST_FREE;
+       }

 
        /*
         * we use to deactivate clean pages at this point,
 
        /*
         * we use to deactivate clean pages at this point,


@@ -268,7 +264,7 @@ memory_object_lock_page(
         *      return (MEMORY_OBJECT_LOCK_RESULT_MUST_DEACTIVATE);
         */
 
         *      return (MEMORY_OBJECT_LOCK_RESULT_MUST_DEACTIVATE);
         */
 

-       return (MEMORY_OBJECT_LOCK_RESULT_DONE);
+       return MEMORY_OBJECT_LOCK_RESULT_DONE;

 }
 
 
 }
 
 


@@ -284,7 +280,7 @@ memory_object_lock_page(
  *                         forms specified by "prot");
  *                     2)  return data to the manager (if "should_return"
  *                         is RETURN_DIRTY and the page is dirty, or
  *                         forms specified by "prot");
  *                     2)  return data to the manager (if "should_return"
  *                         is RETURN_DIRTY and the page is dirty, or

- *                         "should_return" is RETURN_ALL and the page
+ *                          "should_return" is RETURN_ALL and the page

  *                         is either dirty or precious); and,
  *                     3)  flush the cached copy (if "should_flush"
  *                         is asserted).
  *                         is either dirty or precious); and,
  *                     3)  flush the cached copy (if "should_flush"
  *                         is asserted).


@@ -300,26 +296,28 @@ memory_object_lock_page(
 
 kern_return_t
 memory_object_lock_request(
 
 kern_return_t
 memory_object_lock_request(

-       memory_object_control_t         control,
-       memory_object_offset_t          offset,
-       memory_object_size_t            size,
-       memory_object_offset_t  *       resid_offset,
-       int                     *       io_errno,
-       memory_object_return_t          should_return,
-       int                             flags,
-       vm_prot_t                       prot)
+       memory_object_control_t         control,
+       memory_object_offset_t          offset,
+       memory_object_size_t            size,
+       memory_object_offset_t  *       resid_offset,
+       int                     *       io_errno,
+       memory_object_return_t          should_return,
+       int                             flags,
+       vm_prot_t                       prot)

 {
 {

-       vm_object_t     object;
+       vm_object_t     object;

 
 

-        /*
+       /*

         *      Check for bogus arguments.
         */
        object = memory_object_control_to_vm_object(control);
         *      Check for bogus arguments.
         */
        object = memory_object_control_to_vm_object(control);

-       if (object == VM_OBJECT_NULL)
-               return (KERN_INVALID_ARGUMENT);
+       if (object == VM_OBJECT_NULL) {
+               return KERN_INVALID_ARGUMENT;
+       }

 
 

-       if ((prot & ~VM_PROT_ALL) != 0 && prot != VM_PROT_NO_CHANGE)
-               return (KERN_INVALID_ARGUMENT);
+       if ((prot & ~VM_PROT_ALL) != 0 && prot != VM_PROT_NO_CHANGE) {
+               return KERN_INVALID_ARGUMENT;
+       }

 
        size = round_page_64(size);
 
 
        size = round_page_64(size);
 


@@ -338,16 +336,17 @@ memory_object_lock_request(
        }
        offset -= object->paging_offset;
 
        }
        offset -= object->paging_offset;
 

-       if (flags & MEMORY_OBJECT_DATA_FLUSH_ALL)
+       if (flags & MEMORY_OBJECT_DATA_FLUSH_ALL) {

                vm_object_reap_pages(object, REAP_DATA_FLUSH);
                vm_object_reap_pages(object, REAP_DATA_FLUSH);

-       else
+       } else {

                (void)vm_object_update(object, offset, size, resid_offset,
                (void)vm_object_update(object, offset, size, resid_offset,

-                                      io_errno, should_return, flags, prot);
+                   io_errno, should_return, flags, prot);
+       }

 
        vm_object_paging_end(object);
        vm_object_unlock(object);
 
 
        vm_object_paging_end(object);
        vm_object_unlock(object);
 

-       return (KERN_SUCCESS);
+       return KERN_SUCCESS;

 }
 
 /*
 }
 
 /*


@@ -362,21 +361,22 @@ memory_object_lock_request(
  *     being the name.
  *     If the decision is made to proceed the name field flag is set to
  *     false and the reference count is decremented.  If the RESPECT_CACHE
  *     being the name.
  *     If the decision is made to proceed the name field flag is set to
  *     false and the reference count is decremented.  If the RESPECT_CACHE

- *     flag is set and the reference count has gone to zero, the 
+ *     flag is set and the reference count has gone to zero, the

  *     memory_object is checked to see if it is cacheable otherwise when
  *     the reference count is zero, it is simply terminated.
  */
 
 kern_return_t
 memory_object_release_name(
  *     memory_object is checked to see if it is cacheable otherwise when
  *     the reference count is zero, it is simply terminated.
  */
 
 kern_return_t
 memory_object_release_name(

-       memory_object_control_t control,
-       int                             flags)
+       memory_object_control_t control,
+       int                             flags)

 {
 {

-       vm_object_t     object;
+       vm_object_t     object;

 
        object = memory_object_control_to_vm_object(control);
 
        object = memory_object_control_to_vm_object(control);

-       if (object == VM_OBJECT_NULL)
-               return (KERN_INVALID_ARGUMENT);
+       if (object == VM_OBJECT_NULL) {
+               return KERN_INVALID_ARGUMENT;
+       }

 
        return vm_object_release_name(object, flags);
 }
 
        return vm_object_release_name(object, flags);
 }


@@ -392,16 +392,17 @@ memory_object_release_name(
  */
 kern_return_t
 memory_object_destroy(
  */
 kern_return_t
 memory_object_destroy(

-       memory_object_control_t control,
-       kern_return_t           reason)
+       memory_object_control_t control,
+       kern_return_t           reason)

 {
 {

-       vm_object_t             object;
+       vm_object_t             object;

 
        object = memory_object_control_to_vm_object(control);
 
        object = memory_object_control_to_vm_object(control);

-       if (object == VM_OBJECT_NULL)
-               return (KERN_INVALID_ARGUMENT);
+       if (object == VM_OBJECT_NULL) {
+               return KERN_INVALID_ARGUMENT;
+       }

 
 

-       return (vm_object_destroy(object, reason));
+       return vm_object_destroy(object, reason);

 }
 
 /*
 }
 
 /*


@@ -430,20 +431,16 @@ memory_object_destroy(
 
 boolean_t
 vm_object_sync(
 
 boolean_t
 vm_object_sync(

-       vm_object_t             object,
-       vm_object_offset_t      offset,
-       vm_object_size_t        size,
-       boolean_t               should_flush,
-       boolean_t               should_return,
-       boolean_t               should_iosync)
+       vm_object_t             object,
+       vm_object_offset_t      offset,
+       vm_object_size_t        size,
+       boolean_t               should_flush,
+       boolean_t               should_return,
+       boolean_t               should_iosync)

 {
 {

-       boolean_t       rv;
+       boolean_t       rv;

        int             flags;
 
        int             flags;
 

-        XPR(XPR_VM_OBJECT,
-            "vm_o_sync, object 0x%X, offset 0x%X size 0x%x flush %d rtn %d\n",
-            object, offset, size, should_flush, should_return);
-

        /*
         * Lock the object, and acquire a paging reference to
         * prevent the memory_object and control ports from
        /*
         * Lock the object, and acquire a paging reference to
         * prevent the memory_object and control ports from


@@ -453,7 +450,7 @@ vm_object_sync(
        vm_object_paging_begin(object);
 
        if (should_flush) {
        vm_object_paging_begin(object);
 
        if (should_flush) {

-               flags = MEMORY_OBJECT_DATA_FLUSH;
+               flags = MEMORY_OBJECT_DATA_FLUSH;

                /*
                 * This flush is from an msync(), not a truncate(), so the
                 * contents of the file are not affected.
                /*
                 * This flush is from an msync(), not a truncate(), so the
                 * contents of the file are not affected.


@@ -462,18 +459,20 @@ vm_object_sync(
                 * push the old contents to a copy object.
                 */
                flags |= MEMORY_OBJECT_DATA_NO_CHANGE;
                 * push the old contents to a copy object.
                 */
                flags |= MEMORY_OBJECT_DATA_NO_CHANGE;

-       } else
-               flags = 0;
+       } else {
+               flags = 0;
+       }

 
 

-       if (should_iosync)
-               flags |= MEMORY_OBJECT_IO_SYNC;
+       if (should_iosync) {
+               flags |= MEMORY_OBJECT_IO_SYNC;
+       }

 
        rv = vm_object_update(object, offset, (vm_object_size_t)size, NULL, NULL,
 
        rv = vm_object_update(object, offset, (vm_object_size_t)size, NULL, NULL,

-               (should_return) ?
-                       MEMORY_OBJECT_RETURN_ALL :
-                       MEMORY_OBJECT_RETURN_NONE,
-               flags,
-               VM_PROT_NO_CHANGE);
+           (should_return) ?
+           MEMORY_OBJECT_RETURN_ALL :
+           MEMORY_OBJECT_RETURN_NONE,
+           flags,
+           VM_PROT_NO_CHANGE);

 
 
        vm_object_paging_end(object);
 
 
        vm_object_paging_end(object);


@@ -484,92 +483,97 @@ vm_object_sync(
 
 
 #define LIST_REQ_PAGEOUT_PAGES(object, data_cnt, po, ro, ioerr, iosync)    \
 
 
 #define LIST_REQ_PAGEOUT_PAGES(object, data_cnt, po, ro, ioerr, iosync)    \

-MACRO_BEGIN                                                            \
-                                                                       \
-        int                    upl_flags;                              \
-       memory_object_t         pager;                                  \
-                                                                       \
-       if (object->object_slid) {                                      \
-               panic("Objects with slid pages not allowed\n");         \
-       }                                                               \
-                                                                       \
-       if ((pager = (object)->pager) != MEMORY_OBJECT_NULL) {          \
-               vm_object_paging_begin(object);                         \
-               vm_object_unlock(object);                               \
-                                                                       \
-                if (iosync)                                            \
-                        upl_flags = UPL_MSYNC | UPL_IOSYNC;            \
-                else                                                   \
-                        upl_flags = UPL_MSYNC;                         \
-                                                                       \
-               (void) memory_object_data_return(pager,                 \
-                       po,                                             \
-                       (memory_object_cluster_size_t)data_cnt,         \
+MACRO_BEGIN                                                             \
+                                                                        \
+       int                     upl_flags;                              \
+       memory_object_t         pager;                                  \
+                                                                        \
+       if ((pager = (object)->pager) != MEMORY_OBJECT_NULL) {          \
+               vm_object_paging_begin(object);                         \
+               vm_object_unlock(object);                               \
+                                                                        \
+               if (iosync)                                             \
+                       upl_flags = UPL_MSYNC | UPL_IOSYNC;             \
+               else                                                    \
+                       upl_flags = UPL_MSYNC;                          \
+                                                                        \
+               (void) memory_object_data_return(pager,                 \
+                       po,                                             \
+                       (memory_object_cluster_size_t)data_cnt,         \

                        ro,                                             \
                        ioerr,                                          \
                        ro,                                             \
                        ioerr,                                          \

-                       FALSE,                                          \
-                       FALSE,                                          \
-                       upl_flags);                                     \
-                                                                       \
-               vm_object_lock(object);                                 \
-               vm_object_paging_end(object);                           \
-       }                                                               \
+                       FALSE,                                          \
+                       FALSE,                                          \
+                       upl_flags);                                     \
+                                                                        \
+               vm_object_lock(object);                                 \
+               vm_object_paging_end(object);                           \
+       }                                                               \

 MACRO_END
 
 MACRO_END
 

-
+extern struct vnode *
+vnode_pager_lookup_vnode(memory_object_t);

 
 static int
 vm_object_update_extent(
 
 static int
 vm_object_update_extent(

-        vm_object_t            object,
-        vm_object_offset_t     offset,
-       vm_object_offset_t      offset_end,
-       vm_object_offset_t      *offset_resid,
-       int                     *io_errno,
-        boolean_t              should_flush,
-       memory_object_return_t  should_return,
-        boolean_t              should_iosync, 
-        vm_prot_t              prot)
+       vm_object_t             object,
+       vm_object_offset_t      offset,
+       vm_object_offset_t      offset_end,
+       vm_object_offset_t      *offset_resid,
+       int                     *io_errno,
+       boolean_t               should_flush,
+       memory_object_return_t  should_return,
+       boolean_t               should_iosync,
+       vm_prot_t               prot)

 {
 {

-        vm_page_t      m;
-        int            retval = 0;
-       vm_object_offset_t      paging_offset = 0;
-       vm_object_offset_t      next_offset = offset;
-        memory_object_lock_result_t    page_lock_result;
-       memory_object_cluster_size_t    data_cnt = 0;
-       struct vm_page_delayed_work     dw_array[DEFAULT_DELAYED_WORK_LIMIT];
-       struct vm_page_delayed_work     *dwp;
-       int             dw_count;
-       int             dw_limit;
-
-        dwp = &dw_array[0];
-        dw_count = 0;
+       vm_page_t       m;
+       int             retval = 0;
+       vm_object_offset_t      paging_offset = 0;
+       vm_object_offset_t      next_offset = offset;
+       memory_object_lock_result_t     page_lock_result;
+       memory_object_cluster_size_t    data_cnt = 0;
+       struct  vm_page_delayed_work    dw_array;
+       struct  vm_page_delayed_work    *dwp, *dwp_start;
+       bool            dwp_finish_ctx = TRUE;
+       int             dw_count;
+       int             dw_limit;
+       int             dirty_count;
+
+       dwp_start = dwp = NULL;
+       dw_count = 0;

        dw_limit = DELAYED_WORK_LIMIT(DEFAULT_DELAYED_WORK_LIMIT);
        dw_limit = DELAYED_WORK_LIMIT(DEFAULT_DELAYED_WORK_LIMIT);

+       dwp_start = vm_page_delayed_work_get_ctx();
+       if (dwp_start == NULL) {
+               dwp_start = &dw_array;
+               dw_limit = 1;
+               dwp_finish_ctx = FALSE;
+       }
+       dwp = dwp_start;

 
 

-       for (;
-            offset < offset_end && object->resident_page_count;
-            offset += PAGE_SIZE_64) {
+       dirty_count = 0;

 
 

-               /*
+       for (;
+           offset < offset_end && object->resident_page_count;
+           offset += PAGE_SIZE_64) {
+               /*

                 * Limit the number of pages to be cleaned at once to a contiguous
                 * Limit the number of pages to be cleaned at once to a contiguous

-                * run, or at most MAX_UPL_TRANSFER size
+                * run, or at most MAX_UPL_TRANSFER_BYTES

                 */
                if (data_cnt) {
                 */
                if (data_cnt) {

-                       if ((data_cnt >= PAGE_SIZE * MAX_UPL_TRANSFER) || (next_offset != offset)) {
-
+                       if ((data_cnt >= MAX_UPL_TRANSFER_BYTES) || (next_offset != offset)) {

                                if (dw_count) {
                                if (dw_count) {

-                                       vm_page_do_delayed_work(object, &dw_array[0], dw_count);
-                                       dwp = &dw_array[0];
+                                       vm_page_do_delayed_work(object, VM_KERN_MEMORY_NONE, dwp_start, dw_count);
+                                       dwp = dwp_start;

                                        dw_count = 0;
                                }
                                LIST_REQ_PAGEOUT_PAGES(object, data_cnt,
                                        dw_count = 0;
                                }
                                LIST_REQ_PAGEOUT_PAGES(object, data_cnt,

-                                                      paging_offset, offset_resid, io_errno, should_iosync);
+                                   paging_offset, offset_resid, io_errno, should_iosync);

                                data_cnt = 0;
                        }
                }
                while ((m = vm_page_lookup(object, offset)) != VM_PAGE_NULL) {
                                data_cnt = 0;
                        }
                }
                while ((m = vm_page_lookup(object, offset)) != VM_PAGE_NULL) {

-

                        dwp->dw_mask = 0;
                        dwp->dw_mask = 0;

-                       
+

                        page_lock_result = memory_object_lock_page(m, should_return, should_flush, prot);
 
                        if (data_cnt && page_lock_result != MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN) {
                        page_lock_result = memory_object_lock_page(m, should_return, should_flush, prot);
 
                        if (data_cnt && page_lock_result != MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN) {


@@ -577,12 +581,12 @@ vm_object_update_extent(
                                 *      End of a run of dirty/precious pages.
                                 */
                                if (dw_count) {
                                 *      End of a run of dirty/precious pages.
                                 */
                                if (dw_count) {

-                                       vm_page_do_delayed_work(object, &dw_array[0], dw_count);
-                                       dwp = &dw_array[0];
+                                       vm_page_do_delayed_work(object, VM_KERN_MEMORY_NONE, dwp_start, dw_count);
+                                       dwp = dwp_start;

                                        dw_count = 0;
                                }
                                LIST_REQ_PAGEOUT_PAGES(object, data_cnt,
                                        dw_count = 0;
                                }
                                LIST_REQ_PAGEOUT_PAGES(object, data_cnt,

-                                                      paging_offset, offset_resid, io_errno, should_iosync);
+                                   paging_offset, offset_resid, io_errno, should_iosync);

                                /*
                                 * LIST_REQ_PAGEOUT_PAGES will drop the object lock which will
                                 * allow the state of page 'm' to change... we need to re-lookup
                                /*
                                 * LIST_REQ_PAGEOUT_PAGES will drop the object lock which will
                                 * allow the state of page 'm' to change... we need to re-lookup


@@ -593,11 +597,13 @@ vm_object_update_extent(
                        }
 
                        switch (page_lock_result) {
                        }
 
                        switch (page_lock_result) {

-

                        case MEMORY_OBJECT_LOCK_RESULT_DONE:
                                break;
 
                        case MEMORY_OBJECT_LOCK_RESULT_MUST_FREE:
                        case MEMORY_OBJECT_LOCK_RESULT_DONE:
                                break;
 
                        case MEMORY_OBJECT_LOCK_RESULT_MUST_FREE:

+                               if (m->vmp_dirty == TRUE) {
+                                       dirty_count++;
+                               }

                                dwp->dw_mask |= DW_vm_page_free;
                                break;
 
                                dwp->dw_mask |= DW_vm_page_free;
                                break;
 


@@ -606,8 +612,9 @@ vm_object_update_extent(
                                continue;
 
                        case MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN:
                                continue;
 
                        case MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN:

-                               if (data_cnt == 0)
+                               if (data_cnt == 0) {

                                        paging_offset = offset;
                                        paging_offset = offset;

+                               }

 
                                data_cnt += PAGE_SIZE;
                                next_offset = offset + PAGE_SIZE_64;
 
                                data_cnt += PAGE_SIZE;
                                next_offset = offset + PAGE_SIZE_64;


@@ -618,12 +625,11 @@ vm_object_update_extent(
                                 * no need to remove them
                                 */
                                if (!VM_PAGE_WIRED(m)) {
                                 * no need to remove them
                                 */
                                if (!VM_PAGE_WIRED(m)) {

-

                                        if (should_flush) {
                                                /*
                                                 * add additional state for the flush
                                                 */
                                        if (should_flush) {
                                                /*
                                                 * add additional state for the flush
                                                 */

-                                               m->pageout = TRUE;
+                                               m->vmp_free_when_done = TRUE;

                                        }
                                        /*
                                         * we use to remove the page from the queues at this
                                        }
                                        /*
                                         * we use to remove the page from the queues at this


@@ -641,26 +647,37 @@ vm_object_update_extent(
                                VM_PAGE_ADD_DELAYED_WORK(dwp, m, dw_count);
 
                                if (dw_count >= dw_limit) {
                                VM_PAGE_ADD_DELAYED_WORK(dwp, m, dw_count);
 
                                if (dw_count >= dw_limit) {

-                                       vm_page_do_delayed_work(object, &dw_array[0], dw_count);
-                                       dwp = &dw_array[0];
+                                       vm_page_do_delayed_work(object, VM_KERN_MEMORY_NONE, dwp_start, dw_count);
+                                       dwp = dwp_start;

                                        dw_count = 0;
                                }
                        }
                        break;
                }
        }
                                        dw_count = 0;
                                }
                        }
                        break;
                }
        }

+
+       if (object->pager) {
+               task_update_logical_writes(current_task(), (dirty_count * PAGE_SIZE), TASK_WRITE_INVALIDATED, vnode_pager_lookup_vnode(object->pager));
+       }

        /*
         *      We have completed the scan for applicable pages.
         *      Clean any pages that have been saved.
         */
        /*
         *      We have completed the scan for applicable pages.
         *      Clean any pages that have been saved.
         */

-       if (dw_count)
-               vm_page_do_delayed_work(object, &dw_array[0], dw_count);
+       if (dw_count) {
+               vm_page_do_delayed_work(object, VM_KERN_MEMORY_NONE, dwp_start, dw_count);
+       }

 
        if (data_cnt) {
 
        if (data_cnt) {

-               LIST_REQ_PAGEOUT_PAGES(object, data_cnt,
-                                      paging_offset, offset_resid, io_errno, should_iosync);
+               LIST_REQ_PAGEOUT_PAGES(object, data_cnt,
+                   paging_offset, offset_resid, io_errno, should_iosync);

        }
        }

-       return (retval);
+
+       if (dwp_start && dwp_finish_ctx) {
+               vm_page_delayed_work_finish_ctx(dwp_start);
+               dwp_start = dwp = NULL;
+       }
+
+       return retval;

 }
 
 
 }
 
 


@@ -674,30 +691,30 @@ vm_object_update_extent(
  */
 kern_return_t
 vm_object_update(
  */
 kern_return_t
 vm_object_update(

-       vm_object_t             object,
-       vm_object_offset_t      offset,
-       vm_object_size_t        size,
-       vm_object_offset_t      *resid_offset,
-       int                     *io_errno,
-       memory_object_return_t  should_return,
-       int                     flags,
-       vm_prot_t               protection)
+       vm_object_t             object,
+       vm_object_offset_t      offset,
+       vm_object_size_t        size,
+       vm_object_offset_t      *resid_offset,
+       int                     *io_errno,
+       memory_object_return_t  should_return,
+       int                     flags,
+       vm_prot_t               protection)

 {
 {

-        vm_object_t            copy_object = VM_OBJECT_NULL;
-       boolean_t               data_returned = FALSE;
-       boolean_t               update_cow;
-       boolean_t               should_flush = (flags & MEMORY_OBJECT_DATA_FLUSH) ? TRUE : FALSE;
-       boolean_t               should_iosync = (flags & MEMORY_OBJECT_IO_SYNC) ? TRUE : FALSE;
-       vm_fault_return_t       result;
-       int                     num_of_extents;
-       int                     n;
-#define MAX_EXTENTS    8
-#define EXTENT_SIZE    (1024 * 1024 * 256)
-#define RESIDENT_LIMIT (1024 * 32)
+       vm_object_t             copy_object = VM_OBJECT_NULL;
+       boolean_t               data_returned = FALSE;
+       boolean_t               update_cow;
+       boolean_t               should_flush = (flags & MEMORY_OBJECT_DATA_FLUSH) ? TRUE : FALSE;
+       boolean_t               should_iosync = (flags & MEMORY_OBJECT_IO_SYNC) ? TRUE : FALSE;
+       vm_fault_return_t       result;
+       int                     num_of_extents;
+       int                     n;
+#define MAX_EXTENTS     8
+#define EXTENT_SIZE     (1024 * 1024 * 256)
+#define RESIDENT_LIMIT  (1024 * 32)

        struct extent {
        struct extent {

-               vm_object_offset_t e_base;
-               vm_object_offset_t e_min;
-               vm_object_offset_t e_max;
+               vm_object_offset_t e_base;
+               vm_object_offset_t e_min;
+               vm_object_offset_t e_max;

        } extents[MAX_EXTENTS];
 
        /*
        } extents[MAX_EXTENTS];
 
        /*


@@ -717,34 +734,34 @@ vm_object_update(
         * XXX  coalescing implications before doing so.
         */
 
         * XXX  coalescing implications before doing so.
         */
 

-       update_cow = ((flags & MEMORY_OBJECT_DATA_FLUSH) 
-                       && (!(flags & MEMORY_OBJECT_DATA_NO_CHANGE) &&
-                                       !(flags & MEMORY_OBJECT_DATA_PURGE)))
-                               || (flags & MEMORY_OBJECT_COPY_SYNC);
-                       
+       update_cow = ((flags & MEMORY_OBJECT_DATA_FLUSH)
+           && (!(flags & MEMORY_OBJECT_DATA_NO_CHANGE) &&
+           !(flags & MEMORY_OBJECT_DATA_PURGE)))
+           || (flags & MEMORY_OBJECT_COPY_SYNC);
+

        if (update_cow || (flags & (MEMORY_OBJECT_DATA_PURGE | MEMORY_OBJECT_DATA_SYNC))) {
        if (update_cow || (flags & (MEMORY_OBJECT_DATA_PURGE | MEMORY_OBJECT_DATA_SYNC))) {

-               int collisions = 0;
+               int collisions = 0;

 
 

-               while ((copy_object = object->copy) != VM_OBJECT_NULL) {
-                       /*
+               while ((copy_object = object->copy) != VM_OBJECT_NULL) {
+                       /*

                         * need to do a try here since we're swimming upstream
                         * against the normal lock ordering... however, we need
                         * to hold the object stable until we gain control of the
                         * copy object so we have to be careful how we approach this
                         */
                         * need to do a try here since we're swimming upstream
                         * against the normal lock ordering... however, we need
                         * to hold the object stable until we gain control of the
                         * copy object so we have to be careful how we approach this
                         */

-                       if (vm_object_lock_try(copy_object)) {
-                              /*
-                               * we 'won' the lock on the copy object...
-                               * no need to hold the object lock any longer...
-                               * take a real reference on the copy object because
-                               * we're going to call vm_fault_page on it which may
-                               * under certain conditions drop the lock and the paging
-                               * reference we're about to take... the reference
-                               * will keep the copy object from going away if that happens
-                               */
-                              vm_object_unlock(object);
-                              vm_object_reference_locked(copy_object);
-                              break;
+                       if (vm_object_lock_try(copy_object)) {
+                               /*
+                                * we 'won' the lock on the copy object...
+                                * no need to hold the object lock any longer...
+                                * take a real reference on the copy object because
+                                * we're going to call vm_fault_page on it which may
+                                * under certain conditions drop the lock and the paging
+                                * reference we're about to take... the reference
+                                * will keep the copy object from going away if that happens
+                                */
+                               vm_object_unlock(object);
+                               vm_object_reference_locked(copy_object);
+                               break;

                        }
                        vm_object_unlock(object);
 
                        }
                        vm_object_unlock(object);
 


@@ -755,46 +772,45 @@ vm_object_update(
                }
        }
        if ((copy_object != VM_OBJECT_NULL && update_cow) || (flags & MEMORY_OBJECT_DATA_SYNC)) {
                }
        }
        if ((copy_object != VM_OBJECT_NULL && update_cow) || (flags & MEMORY_OBJECT_DATA_SYNC)) {

-               vm_map_size_t           i;
-               vm_map_size_t           copy_size;
-               vm_map_offset_t         copy_offset;
-               vm_prot_t               prot;
-               vm_page_t               page;
-               vm_page_t               top_page;
-               kern_return_t           error = 0;
-               struct vm_object_fault_info fault_info;
+               vm_object_offset_t      i;
+               vm_object_size_t        copy_size;
+               vm_object_offset_t      copy_offset;
+               vm_prot_t               prot;
+               vm_page_t               page;
+               vm_page_t               top_page;
+               kern_return_t           error = 0;
+               struct vm_object_fault_info fault_info = {};

 
                if (copy_object != VM_OBJECT_NULL) {
 
                if (copy_object != VM_OBJECT_NULL) {

-                       /*
+                       /*

                         * translate offset with respect to shadow's offset
                         */
                         * translate offset with respect to shadow's offset
                         */

-                       copy_offset = (offset >= copy_object->vo_shadow_offset) ?
-                         (vm_map_offset_t)(offset - copy_object->vo_shadow_offset) :
-                         (vm_map_offset_t) 0;
+                       copy_offset = (offset >= copy_object->vo_shadow_offset) ?
+                           (offset - copy_object->vo_shadow_offset) : 0;

 
 

-                       if (copy_offset > copy_object->vo_size)
-                               copy_offset = copy_object->vo_size;
+                       if (copy_offset > copy_object->vo_size) {
+                               copy_offset = copy_object->vo_size;
+                       }

 
                        /*
                         * clip size with respect to shadow offset
                         */
                        if (offset >= copy_object->vo_shadow_offset) {
 
                        /*
                         * clip size with respect to shadow offset
                         */
                        if (offset >= copy_object->vo_shadow_offset) {

-                               copy_size = size;
+                               copy_size = size;

                        } else if (size >= copy_object->vo_shadow_offset - offset) {
                        } else if (size >= copy_object->vo_shadow_offset - offset) {

-                               copy_size = size - (copy_object->vo_shadow_offset - offset);
+                               copy_size = (size - (copy_object->vo_shadow_offset - offset));

                        } else {
                        } else {

-                               copy_size = 0;
+                               copy_size = 0;

                        }
                        }

-                       
+

                        if (copy_offset + copy_size > copy_object->vo_size) {
                        if (copy_offset + copy_size > copy_object->vo_size) {

-                               if (copy_object->vo_size >= copy_offset) {
-                                       copy_size = copy_object->vo_size - copy_offset;
+                               if (copy_object->vo_size >= copy_offset) {
+                                       copy_size = copy_object->vo_size - copy_offset;

                                } else {
                                } else {

-                                       copy_size = 0;
+                                       copy_size = 0;

                                }
                        }
                                }
                        }

-                       copy_size+=copy_offset;
-
+                       copy_size += copy_offset;

                } else {
                        copy_object = object;
 
                } else {
                        copy_object = object;
 


@@ -803,70 +819,64 @@ vm_object_update(
                }
                fault_info.interruptible = THREAD_UNINT;
                fault_info.behavior  = VM_BEHAVIOR_SEQUENTIAL;
                }
                fault_info.interruptible = THREAD_UNINT;
                fault_info.behavior  = VM_BEHAVIOR_SEQUENTIAL;

-               fault_info.user_tag  = 0;

                fault_info.lo_offset = copy_offset;
                fault_info.hi_offset = copy_size;
                fault_info.lo_offset = copy_offset;
                fault_info.hi_offset = copy_size;

-               fault_info.no_cache   = FALSE;

                fault_info.stealth = TRUE;
                fault_info.stealth = TRUE;

-               fault_info.io_sync = FALSE;
-               fault_info.cs_bypass = FALSE;
-               fault_info.mark_zf_absent = FALSE;
-               fault_info.batch_pmap_op = FALSE;
+               assert(fault_info.cs_bypass == FALSE);
+               assert(fault_info.pmap_cs_associated == FALSE);

 
                vm_object_paging_begin(copy_object);
 
                for (i = copy_offset; i < copy_size; i += PAGE_SIZE) {
 
                vm_object_paging_begin(copy_object);
 
                for (i = copy_offset; i < copy_size; i += PAGE_SIZE) {

-       RETRY_COW_OF_LOCK_REQUEST:
+RETRY_COW_OF_LOCK_REQUEST:

                        fault_info.cluster_size = (vm_size_t) (copy_size - i);
                        assert(fault_info.cluster_size == copy_size - i);
 
                        fault_info.cluster_size = (vm_size_t) (copy_size - i);
                        assert(fault_info.cluster_size == copy_size - i);
 

-                       prot =  VM_PROT_WRITE|VM_PROT_READ;
+                       prot =  VM_PROT_WRITE | VM_PROT_READ;

                        page = VM_PAGE_NULL;
                        page = VM_PAGE_NULL;

-                       result = vm_fault_page(copy_object, i, 
-                                              VM_PROT_WRITE|VM_PROT_READ,
-                                              FALSE,
-                                              FALSE, /* page not looked up */
-                                              &prot,
-                                              &page,
-                                              &top_page,
-                                              (int *)0,
-                                              &error,
-                                              FALSE,
-                                              FALSE, &fault_info);
+                       result = vm_fault_page(copy_object, i,
+                           VM_PROT_WRITE | VM_PROT_READ,
+                           FALSE,
+                           FALSE,                    /* page not looked up */
+                           &prot,
+                           &page,
+                           &top_page,
+                           (int *)0,
+                           &error,
+                           FALSE,
+                           FALSE, &fault_info);

 
                        switch (result) {
                        case VM_FAULT_SUCCESS:
                                if (top_page) {
                                        vm_fault_cleanup(
 
                        switch (result) {
                        case VM_FAULT_SUCCESS:
                                if (top_page) {
                                        vm_fault_cleanup(

-                                               page->object, top_page);
+                                               VM_PAGE_OBJECT(page), top_page);

                                        vm_object_lock(copy_object);
                                        vm_object_paging_begin(copy_object);
                                }
                                        vm_object_lock(copy_object);
                                        vm_object_paging_begin(copy_object);
                                }

-                               if (!page->active &&
-                                   !page->inactive &&
-                                   !page->throttled) {
+                               if ((!VM_PAGE_NON_SPECULATIVE_PAGEABLE(page))) {

                                        vm_page_lockspin_queues();
                                        vm_page_lockspin_queues();

-                                       if (!page->active &&
-                                           !page->inactive &&
-                                           !page->throttled)
+
+                                       if ((!VM_PAGE_NON_SPECULATIVE_PAGEABLE(page))) {

                                                vm_page_deactivate(page);
                                                vm_page_deactivate(page);

+                                       }

                                        vm_page_unlock_queues();
                                }
                                PAGE_WAKEUP_DONE(page);
                                break;
                        case VM_FAULT_RETRY:
                                        vm_page_unlock_queues();
                                }
                                PAGE_WAKEUP_DONE(page);
                                break;
                        case VM_FAULT_RETRY:

-                               prot =  VM_PROT_WRITE|VM_PROT_READ;
+                               prot =  VM_PROT_WRITE | VM_PROT_READ;

                                vm_object_lock(copy_object);
                                vm_object_paging_begin(copy_object);
                                goto RETRY_COW_OF_LOCK_REQUEST;
                        case VM_FAULT_INTERRUPTED:
                                vm_object_lock(copy_object);
                                vm_object_paging_begin(copy_object);
                                goto RETRY_COW_OF_LOCK_REQUEST;
                        case VM_FAULT_INTERRUPTED:

-                               prot =  VM_PROT_WRITE|VM_PROT_READ;
+                               prot =  VM_PROT_WRITE | VM_PROT_READ;

                                vm_object_lock(copy_object);
                                vm_object_paging_begin(copy_object);
                                goto RETRY_COW_OF_LOCK_REQUEST;
                        case VM_FAULT_MEMORY_SHORTAGE:
                                VM_PAGE_WAIT();
                                vm_object_lock(copy_object);
                                vm_object_paging_begin(copy_object);
                                goto RETRY_COW_OF_LOCK_REQUEST;
                        case VM_FAULT_MEMORY_SHORTAGE:
                                VM_PAGE_WAIT();

-                               prot =  VM_PROT_WRITE|VM_PROT_READ;
+                               prot =  VM_PROT_WRITE | VM_PROT_READ;

                                vm_object_lock(copy_object);
                                vm_object_paging_begin(copy_object);
                                goto RETRY_COW_OF_LOCK_REQUEST;
                                vm_object_lock(copy_object);
                                vm_object_paging_begin(copy_object);
                                goto RETRY_COW_OF_LOCK_REQUEST;


@@ -874,31 +884,32 @@ vm_object_update(
                                /* success but no VM page: fail */
                                vm_object_paging_end(copy_object);
                                vm_object_unlock(copy_object);
                                /* success but no VM page: fail */
                                vm_object_paging_end(copy_object);
                                vm_object_unlock(copy_object);

-                               /*FALLTHROUGH*/
+                               OS_FALLTHROUGH;

                        case VM_FAULT_MEMORY_ERROR:
                        case VM_FAULT_MEMORY_ERROR:

-                               if (object != copy_object)
-                                       vm_object_deallocate(copy_object);
+                               if (object != copy_object) {
+                                       vm_object_deallocate(copy_object);
+                               }

                                vm_object_lock(object);
                                goto BYPASS_COW_COPYIN;
                        default:
                                panic("vm_object_update: unexpected error 0x%x"
                                vm_object_lock(object);
                                goto BYPASS_COW_COPYIN;
                        default:
                                panic("vm_object_update: unexpected error 0x%x"

-                                     " from vm_fault_page()\n", result);
+                                   " from vm_fault_page()\n", result);

                        }
                        }

-

                }
                vm_object_paging_end(copy_object);
        }
        if ((flags & (MEMORY_OBJECT_DATA_SYNC | MEMORY_OBJECT_COPY_SYNC))) {
                }
                vm_object_paging_end(copy_object);
        }
        if ((flags & (MEMORY_OBJECT_DATA_SYNC | MEMORY_OBJECT_COPY_SYNC))) {

-               if (copy_object != VM_OBJECT_NULL && copy_object != object) {
+               if (copy_object != VM_OBJECT_NULL && copy_object != object) {

                        vm_object_unlock(copy_object);
                        vm_object_unlock(copy_object);

-                       vm_object_deallocate(copy_object);
+                       vm_object_deallocate(copy_object);

                        vm_object_lock(object);
                }
                return KERN_SUCCESS;
        }
        if (copy_object != VM_OBJECT_NULL && copy_object != object) {
                        vm_object_lock(object);
                }
                return KERN_SUCCESS;
        }
        if (copy_object != VM_OBJECT_NULL && copy_object != object) {

-               if ((flags & MEMORY_OBJECT_DATA_PURGE)) {
-                       copy_object->shadow_severed = TRUE;
+               if ((flags & MEMORY_OBJECT_DATA_PURGE)) {
+                       vm_object_lock_assert_exclusive(copy_object);
+                       copy_object->shadow_severed = TRUE;

                        copy_object->shadowed = FALSE;
                        copy_object->shadow = NULL;
                        /*
                        copy_object->shadowed = FALSE;
                        copy_object->shadow = NULL;
                        /*


@@ -907,7 +918,7 @@ vm_object_update(
                        vm_object_deallocate(object);
                }
                vm_object_unlock(copy_object);
                        vm_object_deallocate(object);
                }
                vm_object_unlock(copy_object);

-               vm_object_deallocate(copy_object);
+               vm_object_deallocate(copy_object);

                vm_object_lock(object);
        }
 BYPASS_COW_COPYIN:
                vm_object_lock(object);
        }
 BYPASS_COW_COPYIN:


@@ -920,9 +931,9 @@ BYPASS_COW_COPYIN:
         * the page which means the resident queue can change which
         * means we can't walk the queue as we process the pages
         * we also want to do the processing in offset order to allow
         * the page which means the resident queue can change which
         * means we can't walk the queue as we process the pages
         * we also want to do the processing in offset order to allow

-        * 'runs' of pages to be collected if we're being told to 
+        * 'runs' of pages to be collected if we're being told to

         * flush to disk... the resident page queue is NOT ordered.
         * flush to disk... the resident page queue is NOT ordered.

-        * 
+        *

         * a temporary solution (until we figure out how to deal with
         * large address spaces more generically) is to pre-flight
         * the resident page queue (if it's small enough) and develop
         * a temporary solution (until we figure out how to deal with
         * large address spaces more generically) is to pre-flight
         * the resident page queue (if it's small enough) and develop


@@ -936,12 +947,12 @@ BYPASS_COW_COPYIN:
         * is not a theoretical problem
         */
 
         * is not a theoretical problem
         */
 

-       if ((object->resident_page_count < RESIDENT_LIMIT) && 
-           (atop_64(size) > (unsigned)(object->resident_page_count/(8 * MAX_EXTENTS)))) {
-               vm_page_t               next;
-               vm_object_offset_t      start;
-               vm_object_offset_t      end;
-               vm_object_size_t        e_mask;
+       if ((object->resident_page_count < RESIDENT_LIMIT) &&
+           (atop_64(size) > (unsigned)(object->resident_page_count / (8 * MAX_EXTENTS)))) {
+               vm_page_t               next;
+               vm_object_offset_t      start;
+               vm_object_offset_t      end;
+               vm_object_size_t        e_mask;

                vm_page_t               m;
 
                start = offset;
                vm_page_t               m;
 
                start = offset;


@@ -949,60 +960,63 @@ BYPASS_COW_COPYIN:
                num_of_extents = 0;
                e_mask = ~((vm_object_size_t)(EXTENT_SIZE - 1));
 
                num_of_extents = 0;
                e_mask = ~((vm_object_size_t)(EXTENT_SIZE - 1));
 

-               m = (vm_page_t) queue_first(&object->memq);
+               m = (vm_page_t) vm_page_queue_first(&object->memq);

 
 

-               while (!queue_end(&object->memq, (queue_entry_t) m)) {
-                       next = (vm_page_t) queue_next(&m->listq);
+               while (!vm_page_queue_end(&object->memq, (vm_page_queue_entry_t) m)) {
+                       next = (vm_page_t) vm_page_queue_next(&m->vmp_listq);

 
 

-                       if ((m->offset >= start) && (m->offset < end)) {
-                               /*
+                       if ((m->vmp_offset >= start) && (m->vmp_offset < end)) {
+                               /*

                                 * this is a page we're interested in
                                 * try to fit it into a current extent
                                 */
                                 * this is a page we're interested in
                                 * try to fit it into a current extent
                                 */

-                               for (n = 0; n < num_of_extents; n++) {
-                                       if ((m->offset & e_mask) == extents[n].e_base) {
-                                               /*
+                               for (n = 0; n < num_of_extents; n++) {
+                                       if ((m->vmp_offset & e_mask) == extents[n].e_base) {
+                                               /*

                                                 * use (PAGE_SIZE - 1) to determine the
                                                 * max offset so that we don't wrap if
                                                 * we're at the last page of the space
                                                 */
                                                 * use (PAGE_SIZE - 1) to determine the
                                                 * max offset so that we don't wrap if
                                                 * we're at the last page of the space
                                                 */

-                                               if (m->offset < extents[n].e_min)
-                                                       extents[n].e_min = m->offset;
-                                               else if ((m->offset + (PAGE_SIZE - 1)) > extents[n].e_max)
-                                                       extents[n].e_max = m->offset + (PAGE_SIZE - 1);
-                                               break;
+                                               if (m->vmp_offset < extents[n].e_min) {
+                                                       extents[n].e_min = m->vmp_offset;
+                                               } else if ((m->vmp_offset + (PAGE_SIZE - 1)) > extents[n].e_max) {
+                                                       extents[n].e_max = m->vmp_offset + (PAGE_SIZE - 1);
+                                               }
+                                               break;

                                        }
                                }
                                if (n == num_of_extents) {
                                        }
                                }
                                if (n == num_of_extents) {

-                                       /*
+                                       /*

                                         * didn't find a current extent that can encompass
                                         * this page
                                         */
                                         * didn't find a current extent that can encompass
                                         * this page
                                         */

-                                       if (n < MAX_EXTENTS) {
-                                               /*
-                                                * if we still have room, 
+                                       if (n < MAX_EXTENTS) {
+                                               /*
+                                                * if we still have room,

                                                 * create a new extent
                                                 */
                                                 * create a new extent
                                                 */

-                                               extents[n].e_base = m->offset & e_mask;
-                                               extents[n].e_min  = m->offset;
-                                               extents[n].e_max  = m->offset + (PAGE_SIZE - 1);
+                                               extents[n].e_base = m->vmp_offset & e_mask;
+                                               extents[n].e_min  = m->vmp_offset;
+                                               extents[n].e_max  = m->vmp_offset + (PAGE_SIZE - 1);

 
                                                num_of_extents++;
                                        } else {
                                                /*
                                                 * no room to create a new extent...
                                                 * fall back to a single extent based
 
                                                num_of_extents++;
                                        } else {
                                                /*
                                                 * no room to create a new extent...
                                                 * fall back to a single extent based

-                                                * on the min and max page offsets 
+                                                * on the min and max page offsets

                                                 * we find in the range we're interested in...
                                                 * first, look through the extent list and
                                                 * develop the overall min and max for the
                                                 * pages we've looked at up to this point
                                                 * we find in the range we're interested in...
                                                 * first, look through the extent list and
                                                 * develop the overall min and max for the
                                                 * pages we've looked at up to this point

-                                                */                                             
-                                               for (n = 1; n < num_of_extents; n++) {
-                                                       if (extents[n].e_min < extents[0].e_min)
-                                                               extents[0].e_min = extents[n].e_min;
-                                                       if (extents[n].e_max > extents[0].e_max)
-                                                               extents[0].e_max = extents[n].e_max;
+                                                */
+                                               for (n = 1; n < num_of_extents; n++) {
+                                                       if (extents[n].e_min < extents[0].e_min) {
+                                                               extents[0].e_min = extents[n].e_min;
+                                                       }
+                                                       if (extents[n].e_max > extents[0].e_max) {
+                                                               extents[0].e_max = extents[n].e_max;
+                                                       }

                                                }
                                                /*
                                                 * now setup to run through the remaining pages
                                                }
                                                /*
                                                 * now setup to run through the remaining pages


@@ -1025,118 +1039,48 @@ BYPASS_COW_COPYIN:
                        m = next;
                }
        } else {
                        m = next;
                }
        } else {

-               extents[0].e_min = offset;
+               extents[0].e_min = offset;

                extents[0].e_max = offset + (size - 1);
 
                num_of_extents = 1;
        }
        for (n = 0; n < num_of_extents; n++) {
                extents[0].e_max = offset + (size - 1);
 
                num_of_extents = 1;
        }
        for (n = 0; n < num_of_extents; n++) {

-               if (vm_object_update_extent(object, extents[n].e_min, extents[n].e_max, resid_offset, io_errno,
-                                           should_flush, should_return, should_iosync, protection))
-                       data_returned = TRUE;
-       }
-       return (data_returned);
-}
-
-
-/*
- *     Routine:        memory_object_synchronize_completed [user interface]
- *
- *     Tell kernel that previously synchronized data
- *     (memory_object_synchronize) has been queue or placed on the
- *     backing storage.
- *
- *     Note: there may be multiple synchronize requests for a given
- *     memory object outstanding but they will not overlap.
- */
-
-kern_return_t
-memory_object_synchronize_completed(
-       memory_object_control_t control,
-       memory_object_offset_t  offset,
-       memory_object_size_t    length)
-{
-       vm_object_t                     object;
-       msync_req_t                     msr;
-
-       object = memory_object_control_to_vm_object(control);
-
-        XPR(XPR_MEMORY_OBJECT,
-           "m_o_sync_completed, object 0x%X, offset 0x%X length 0x%X\n",
-           object, offset, length, 0, 0);
-
-       /*
-        *      Look for bogus arguments
-        */
-
-       if (object == VM_OBJECT_NULL)
-               return (KERN_INVALID_ARGUMENT);
-
-       vm_object_lock(object);
-
-/*
- *     search for sync request structure
- */
-       queue_iterate(&object->msr_q, msr, msync_req_t, msr_q) {
-               if (msr->offset == offset && msr->length == length) {
-                       queue_remove(&object->msr_q, msr, msync_req_t, msr_q);
-                       break;
+               if (vm_object_update_extent(object, extents[n].e_min, extents[n].e_max, resid_offset, io_errno,
+                   should_flush, should_return, should_iosync, protection)) {
+                       data_returned = TRUE;

                }
                }

-        }/* queue_iterate */
-
-       if (queue_end(&object->msr_q, (queue_entry_t)msr)) {
-               vm_object_unlock(object);
-               return KERN_INVALID_ARGUMENT;

        }
        }

+       return data_returned;
+}

 
 

-       msr_lock(msr);
-       vm_object_unlock(object);
-       msr->flag = VM_MSYNC_DONE;
-       msr_unlock(msr);
-       thread_wakeup((event_t) msr);
-
-       return KERN_SUCCESS;
-}/* memory_object_synchronize_completed */

 
 static kern_return_t
 vm_object_set_attributes_common(
 
 static kern_return_t
 vm_object_set_attributes_common(

-       vm_object_t     object,
-       boolean_t       may_cache,
-       memory_object_copy_strategy_t copy_strategy,
-       boolean_t       temporary,
-       __unused boolean_t      silent_overwrite,
-       boolean_t       advisory_pageout)
+       vm_object_t     object,
+       boolean_t       may_cache,
+       memory_object_copy_strategy_t copy_strategy)

 {
 {

-       boolean_t       object_became_ready;
-
-        XPR(XPR_MEMORY_OBJECT,
-           "m_o_set_attr_com, object 0x%X flg %x strat %d\n",
-           object, (may_cache&1)|((temporary&1)<1), copy_strategy, 0, 0);
+       boolean_t       object_became_ready;

 
 

-       if (object == VM_OBJECT_NULL)
-               return(KERN_INVALID_ARGUMENT);
+       if (object == VM_OBJECT_NULL) {
+               return KERN_INVALID_ARGUMENT;
+       }

 
        /*
         *      Verify the attributes of importance
         */
 
 
        /*
         *      Verify the attributes of importance
         */
 

-       switch(copy_strategy) {
-               case MEMORY_OBJECT_COPY_NONE:
-               case MEMORY_OBJECT_COPY_DELAY:
-                       break;
-               default:
-                       return(KERN_INVALID_ARGUMENT);
+       switch (copy_strategy) {
+       case MEMORY_OBJECT_COPY_NONE:
+       case MEMORY_OBJECT_COPY_DELAY:
+               break;
+       default:
+               return KERN_INVALID_ARGUMENT;

        }
 
        }
 

-#if    !ADVISORY_PAGEOUT
-       if (silent_overwrite || advisory_pageout)
-               return(KERN_INVALID_ARGUMENT);
-
-#endif /* !ADVISORY_PAGEOUT */
-       if (may_cache)
+       if (may_cache) {

                may_cache = TRUE;
                may_cache = TRUE;

-       if (temporary)
-               temporary = TRUE;
+       }

 
        vm_object_lock(object);
 
 
        vm_object_lock(object);
 


@@ -1147,9 +1091,6 @@ vm_object_set_attributes_common(
        object_became_ready = !object->pager_ready;
        object->copy_strategy = copy_strategy;
        object->can_persist = may_cache;
        object_became_ready = !object->pager_ready;
        object->copy_strategy = copy_strategy;
        object->can_persist = may_cache;

-       object->temporary = temporary;
-//     object->silent_overwrite = silent_overwrite;
-       object->advisory_pageout = advisory_pageout;

 
        /*
         *      Wake up anyone waiting for the ready attribute
 
        /*
         *      Wake up anyone waiting for the ready attribute


@@ -1163,186 +1104,183 @@ vm_object_set_attributes_common(
 
        vm_object_unlock(object);
 
 
        vm_object_unlock(object);
 

-       return(KERN_SUCCESS);
+       return KERN_SUCCESS;
+}
+
+
+kern_return_t
+memory_object_synchronize_completed(
+       __unused    memory_object_control_t control,
+       __unused    memory_object_offset_t  offset,
+       __unused    memory_object_size_t    length)
+{
+       panic("memory_object_synchronize_completed no longer supported\n");
+       return KERN_FAILURE;

 }
 
 }
 

+

 /*
  *     Set the memory object attribute as provided.
  *
 /*
  *     Set the memory object attribute as provided.
  *

- *     XXX This routine cannot be completed until the vm_msync, clean 
+ *     XXX This routine cannot be completed until the vm_msync, clean

  *          in place, and cluster work is completed. See ifdef notyet
  *          below and note that vm_object_set_attributes_common()
  *          may have to be expanded.
  */
 kern_return_t
 memory_object_change_attributes(
  *          in place, and cluster work is completed. See ifdef notyet
  *          below and note that vm_object_set_attributes_common()
  *          may have to be expanded.
  */
 kern_return_t
 memory_object_change_attributes(

-       memory_object_control_t         control,
-       memory_object_flavor_t          flavor,
-       memory_object_info_t            attributes,
-       mach_msg_type_number_t          count)
+       memory_object_control_t         control,
+       memory_object_flavor_t          flavor,
+       memory_object_info_t            attributes,
+       mach_msg_type_number_t          count)

 {
 {

-       vm_object_t                     object;
-       kern_return_t                   result = KERN_SUCCESS;
-       boolean_t                       temporary;
-       boolean_t                       may_cache;
-       boolean_t                       invalidate;
-       memory_object_copy_strategy_t   copy_strategy;
-       boolean_t                       silent_overwrite;
-       boolean_t                       advisory_pageout;
+       vm_object_t                     object;
+       kern_return_t                   result = KERN_SUCCESS;
+       boolean_t                       may_cache;
+       boolean_t                       invalidate;
+       memory_object_copy_strategy_t   copy_strategy;

 
        object = memory_object_control_to_vm_object(control);
 
        object = memory_object_control_to_vm_object(control);

-       if (object == VM_OBJECT_NULL)
-               return (KERN_INVALID_ARGUMENT);
+       if (object == VM_OBJECT_NULL) {
+               return KERN_INVALID_ARGUMENT;
+       }

 
        vm_object_lock(object);
 
 
        vm_object_lock(object);
 

-       temporary = object->temporary;

        may_cache = object->can_persist;
        copy_strategy = object->copy_strategy;
        may_cache = object->can_persist;
        copy_strategy = object->copy_strategy;

-//     silent_overwrite = object->silent_overwrite;
-       silent_overwrite = FALSE;
-       advisory_pageout = object->advisory_pageout;

 #if notyet
        invalidate = object->invalidate;
 #endif
 #if notyet
        invalidate = object->invalidate;
 #endif

-       vm_object_unlock(object);       
+       vm_object_unlock(object);

 
        switch (flavor) {
 
        switch (flavor) {

-           case OLD_MEMORY_OBJECT_BEHAVIOR_INFO:
-           {
-                old_memory_object_behave_info_t     behave;
+       case OLD_MEMORY_OBJECT_BEHAVIOR_INFO:
+       {
+               old_memory_object_behave_info_t     behave;

 
 

-                if (count != OLD_MEMORY_OBJECT_BEHAVE_INFO_COUNT) {
-                        result = KERN_INVALID_ARGUMENT;
-                        break;
-                }
+               if (count != OLD_MEMORY_OBJECT_BEHAVE_INFO_COUNT) {
+                       result = KERN_INVALID_ARGUMENT;
+                       break;
+               }

 
 

-                behave = (old_memory_object_behave_info_t) attributes;
+               behave = (old_memory_object_behave_info_t) attributes;

 
 

-               temporary = behave->temporary;

                invalidate = behave->invalidate;
                copy_strategy = behave->copy_strategy;
 
                break;
                invalidate = behave->invalidate;
                copy_strategy = behave->copy_strategy;
 
                break;

-           }
+       }

 
 

-           case MEMORY_OBJECT_BEHAVIOR_INFO:
-           {
-                memory_object_behave_info_t     behave;
+       case MEMORY_OBJECT_BEHAVIOR_INFO:
+       {
+               memory_object_behave_info_t     behave;

 
 

-                if (count != MEMORY_OBJECT_BEHAVE_INFO_COUNT) {
-                        result = KERN_INVALID_ARGUMENT;
-                        break;
-                }
+               if (count != MEMORY_OBJECT_BEHAVE_INFO_COUNT) {
+                       result = KERN_INVALID_ARGUMENT;
+                       break;
+               }

 
 

-                behave = (memory_object_behave_info_t) attributes;
+               behave = (memory_object_behave_info_t) attributes;

 
 

-               temporary = behave->temporary;

                invalidate = behave->invalidate;
                copy_strategy = behave->copy_strategy;
                invalidate = behave->invalidate;
                copy_strategy = behave->copy_strategy;

-               silent_overwrite = behave->silent_overwrite;
-               advisory_pageout = behave->advisory_pageout;

                break;
                break;

-           }
+       }

 
 

-           case MEMORY_OBJECT_PERFORMANCE_INFO:
-           {
-               memory_object_perf_info_t       perf;
+       case MEMORY_OBJECT_PERFORMANCE_INFO:
+       {
+               memory_object_perf_info_t       perf;

 
 

-                if (count != MEMORY_OBJECT_PERF_INFO_COUNT) {
-                        result = KERN_INVALID_ARGUMENT;
-                        break;
-                }
+               if (count != MEMORY_OBJECT_PERF_INFO_COUNT) {
+                       result = KERN_INVALID_ARGUMENT;
+                       break;
+               }

 
 

-                perf = (memory_object_perf_info_t) attributes;
+               perf = (memory_object_perf_info_t) attributes;

 
                may_cache = perf->may_cache;
 
                break;
 
                may_cache = perf->may_cache;
 
                break;

-           }
+       }

 
 

-           case OLD_MEMORY_OBJECT_ATTRIBUTE_INFO:
-           {
-               old_memory_object_attr_info_t   attr;
+       case OLD_MEMORY_OBJECT_ATTRIBUTE_INFO:
+       {
+               old_memory_object_attr_info_t   attr;

 
 

-                if (count != OLD_MEMORY_OBJECT_ATTR_INFO_COUNT) {
-                        result = KERN_INVALID_ARGUMENT;
-                        break;
-                }
+               if (count != OLD_MEMORY_OBJECT_ATTR_INFO_COUNT) {
+                       result = KERN_INVALID_ARGUMENT;
+                       break;
+               }

 
                attr = (old_memory_object_attr_info_t) attributes;
 
 
                attr = (old_memory_object_attr_info_t) attributes;
 

-                may_cache = attr->may_cache;
-                copy_strategy = attr->copy_strategy;
+               may_cache = attr->may_cache;
+               copy_strategy = attr->copy_strategy;

 
                break;
 
                break;

-           }
+       }

 
 

-           case MEMORY_OBJECT_ATTRIBUTE_INFO:
-           {
-               memory_object_attr_info_t       attr;
+       case MEMORY_OBJECT_ATTRIBUTE_INFO:
+       {
+               memory_object_attr_info_t       attr;

 
 

-                if (count != MEMORY_OBJECT_ATTR_INFO_COUNT) {
-                        result = KERN_INVALID_ARGUMENT;
-                        break;
-                }
+               if (count != MEMORY_OBJECT_ATTR_INFO_COUNT) {
+                       result = KERN_INVALID_ARGUMENT;
+                       break;
+               }

 
                attr = (memory_object_attr_info_t) attributes;
 
                copy_strategy = attr->copy_strategy;
 
                attr = (memory_object_attr_info_t) attributes;
 
                copy_strategy = attr->copy_strategy;

-                may_cache = attr->may_cache_object;
-               temporary = attr->temporary;
+               may_cache = attr->may_cache_object;

 
                break;
 
                break;

-           }
+       }

 
 

-           default:
+       default:

                result = KERN_INVALID_ARGUMENT;
                break;
        }
 
                result = KERN_INVALID_ARGUMENT;
                break;
        }
 

-       if (result != KERN_SUCCESS)
-               return(result);
+       if (result != KERN_SUCCESS) {
+               return result;
+       }

 
        if (copy_strategy == MEMORY_OBJECT_COPY_TEMPORARY) {
                copy_strategy = MEMORY_OBJECT_COPY_DELAY;
 
        if (copy_strategy == MEMORY_OBJECT_COPY_TEMPORARY) {
                copy_strategy = MEMORY_OBJECT_COPY_DELAY;

-               temporary = TRUE;
-       } else {
-               temporary = FALSE;

        }
 
        /*
         * XXX  may_cache may become a tri-valued variable to handle
         * XXX  uncache if not in use.
         */
        }
 
        /*
         * XXX  may_cache may become a tri-valued variable to handle
         * XXX  uncache if not in use.
         */

-       return (vm_object_set_attributes_common(object,
-                                                    may_cache,
-                                                    copy_strategy,
-                                                    temporary,
-                                                    silent_overwrite,
-                                                    advisory_pageout));
+       return vm_object_set_attributes_common(object,
+                  may_cache,
+                  copy_strategy);

 }
 
 kern_return_t
 memory_object_get_attributes(
 }
 
 kern_return_t
 memory_object_get_attributes(

-        memory_object_control_t        control,
-        memory_object_flavor_t         flavor,
-       memory_object_info_t    attributes,     /* pointer to OUT array */
-       mach_msg_type_number_t  *count)         /* IN/OUT */
+       memory_object_control_t control,
+       memory_object_flavor_t  flavor,
+       memory_object_info_t    attributes,     /* pointer to OUT array */
+       mach_msg_type_number_t  *count)         /* IN/OUT */

 {
 {

-       kern_return_t           ret = KERN_SUCCESS;
-       vm_object_t             object;
+       kern_return_t           ret = KERN_SUCCESS;
+       vm_object_t             object;

 
        object = memory_object_control_to_vm_object(control);
 
        object = memory_object_control_to_vm_object(control);

-       if (object == VM_OBJECT_NULL)
-               return (KERN_INVALID_ARGUMENT);
+       if (object == VM_OBJECT_NULL) {
+               return KERN_INVALID_ARGUMENT;
+       }

 
 

-        vm_object_lock(object);
+       vm_object_lock(object);

 
        switch (flavor) {
 
        switch (flavor) {

-           case OLD_MEMORY_OBJECT_BEHAVIOR_INFO:
-           {
-               old_memory_object_behave_info_t behave;
+       case OLD_MEMORY_OBJECT_BEHAVIOR_INFO:
+       {
+               old_memory_object_behave_info_t behave;

 
                if (*count < OLD_MEMORY_OBJECT_BEHAVE_INFO_COUNT) {
                        ret = KERN_INVALID_ARGUMENT;
 
                if (*count < OLD_MEMORY_OBJECT_BEHAVE_INFO_COUNT) {
                        ret = KERN_INVALID_ARGUMENT;


@@ -1351,44 +1289,43 @@ memory_object_get_attributes(
 
                behave = (old_memory_object_behave_info_t) attributes;
                behave->copy_strategy = object->copy_strategy;
 
                behave = (old_memory_object_behave_info_t) attributes;
                behave->copy_strategy = object->copy_strategy;

-               behave->temporary = object->temporary;
-#if notyet     /* remove when vm_msync complies and clean in place fini */
-                behave->invalidate = object->invalidate;
+               behave->temporary = FALSE;
+#if notyet      /* remove when vm_msync complies and clean in place fini */
+               behave->invalidate = object->invalidate;

 #else
                behave->invalidate = FALSE;
 #endif
 
                *count = OLD_MEMORY_OBJECT_BEHAVE_INFO_COUNT;
                break;
 #else
                behave->invalidate = FALSE;
 #endif
 
                *count = OLD_MEMORY_OBJECT_BEHAVE_INFO_COUNT;
                break;

-           }
+       }

 
 

-           case MEMORY_OBJECT_BEHAVIOR_INFO:
-           {
-               memory_object_behave_info_t     behave;
+       case MEMORY_OBJECT_BEHAVIOR_INFO:
+       {
+               memory_object_behave_info_t     behave;

 
                if (*count < MEMORY_OBJECT_BEHAVE_INFO_COUNT) {
 
                if (*count < MEMORY_OBJECT_BEHAVE_INFO_COUNT) {

-                        ret = KERN_INVALID_ARGUMENT;
-                        break;
-                }
-
-                behave = (memory_object_behave_info_t) attributes;
-                behave->copy_strategy = object->copy_strategy;
-               behave->temporary = object->temporary;
-#if notyet     /* remove when vm_msync complies and clean in place fini */
-                behave->invalidate = object->invalidate;
+                       ret = KERN_INVALID_ARGUMENT;
+                       break;
+               }
+
+               behave = (memory_object_behave_info_t) attributes;
+               behave->copy_strategy = object->copy_strategy;
+               behave->temporary = FALSE;
+#if notyet      /* remove when vm_msync complies and clean in place fini */
+               behave->invalidate = object->invalidate;

 #else
                behave->invalidate = FALSE;
 #endif
 #else
                behave->invalidate = FALSE;
 #endif

-               behave->advisory_pageout = object->advisory_pageout;
-//             behave->silent_overwrite = object->silent_overwrite;
+               behave->advisory_pageout = FALSE;

                behave->silent_overwrite = FALSE;
                behave->silent_overwrite = FALSE;

-                *count = MEMORY_OBJECT_BEHAVE_INFO_COUNT;
+               *count = MEMORY_OBJECT_BEHAVE_INFO_COUNT;

                break;
                break;

-           }
+       }

 
 

-           case MEMORY_OBJECT_PERFORMANCE_INFO:
-           {
-               memory_object_perf_info_t       perf;
+       case MEMORY_OBJECT_PERFORMANCE_INFO:
+       {
+               memory_object_perf_info_t       perf;

 
                if (*count < MEMORY_OBJECT_PERF_INFO_COUNT) {
                        ret = KERN_INVALID_ARGUMENT;
 
                if (*count < MEMORY_OBJECT_PERF_INFO_COUNT) {
                        ret = KERN_INVALID_ARGUMENT;


@@ -1401,68 +1338,69 @@ memory_object_get_attributes(
 
                *count = MEMORY_OBJECT_PERF_INFO_COUNT;
                break;
 
                *count = MEMORY_OBJECT_PERF_INFO_COUNT;
                break;

-           }
+       }

 
 

-            case OLD_MEMORY_OBJECT_ATTRIBUTE_INFO:
-            {
-                old_memory_object_attr_info_t       attr;
+       case OLD_MEMORY_OBJECT_ATTRIBUTE_INFO:
+       {
+               old_memory_object_attr_info_t       attr;

 
 

-                if (*count < OLD_MEMORY_OBJECT_ATTR_INFO_COUNT) {
-                        ret = KERN_INVALID_ARGUMENT;
-                        break;
-                }
+               if (*count < OLD_MEMORY_OBJECT_ATTR_INFO_COUNT) {
+                       ret = KERN_INVALID_ARGUMENT;
+                       break;
+               }

 
 

-                attr = (old_memory_object_attr_info_t) attributes;
-               attr->may_cache = object->can_persist;
-               attr->copy_strategy = object->copy_strategy;
+               attr = (old_memory_object_attr_info_t) attributes;
+               attr->may_cache = object->can_persist;
+               attr->copy_strategy = object->copy_strategy;

 
 

-                *count = OLD_MEMORY_OBJECT_ATTR_INFO_COUNT;
-                break;
-            }
+               *count = OLD_MEMORY_OBJECT_ATTR_INFO_COUNT;
+               break;
+       }

 
 

-            case MEMORY_OBJECT_ATTRIBUTE_INFO:
-            {
-                memory_object_attr_info_t       attr;
+       case MEMORY_OBJECT_ATTRIBUTE_INFO:
+       {
+               memory_object_attr_info_t       attr;

 
 

-                if (*count < MEMORY_OBJECT_ATTR_INFO_COUNT) {
-                        ret = KERN_INVALID_ARGUMENT;
-                        break;
-                }
+               if (*count < MEMORY_OBJECT_ATTR_INFO_COUNT) {
+                       ret = KERN_INVALID_ARGUMENT;
+                       break;
+               }

 
 

-                attr = (memory_object_attr_info_t) attributes;
-               attr->copy_strategy = object->copy_strategy;
+               attr = (memory_object_attr_info_t) attributes;
+               attr->copy_strategy = object->copy_strategy;

                attr->cluster_size = PAGE_SIZE;
                attr->cluster_size = PAGE_SIZE;

-               attr->may_cache_object = object->can_persist;
-               attr->temporary = object->temporary;
+               attr->may_cache_object = object->can_persist;
+               attr->temporary = FALSE;

 
 

-                *count = MEMORY_OBJECT_ATTR_INFO_COUNT;
-                break;
-            }
+               *count = MEMORY_OBJECT_ATTR_INFO_COUNT;
+               break;
+       }

 
 

-           default:
+       default:

                ret = KERN_INVALID_ARGUMENT;
                break;
        }
 
                ret = KERN_INVALID_ARGUMENT;
                break;
        }
 

-        vm_object_unlock(object);
+       vm_object_unlock(object);

 
 

-        return(ret);
+       return ret;

 }
 
 
 kern_return_t
 memory_object_iopl_request(
 }
 
 
 kern_return_t
 memory_object_iopl_request(

-       ipc_port_t              port,
-       memory_object_offset_t  offset,
-       upl_size_t              *upl_size,
-       upl_t                   *upl_ptr,
-       upl_page_info_array_t   user_page_list,
-       unsigned int            *page_list_count,
-       int                     *flags)
+       ipc_port_t              port,
+       memory_object_offset_t  offset,
+       upl_size_t              *upl_size,
+       upl_t                   *upl_ptr,
+       upl_page_info_array_t   user_page_list,
+       unsigned int            *page_list_count,
+       upl_control_flags_t     *flags,
+       vm_tag_t                tag)

 {
 {

-       vm_object_t             object;
-       kern_return_t           ret;
-       int                     caller_flags;
+       vm_object_t             object;
+       kern_return_t           ret;
+       upl_control_flags_t     caller_flags;

 
        caller_flags = *flags;
 
 
        caller_flags = *flags;
 


@@ -1475,99 +1413,73 @@ memory_object_iopl_request(
        }
 
        if (ip_kotype(port) == IKOT_NAMED_ENTRY) {
        }
 
        if (ip_kotype(port) == IKOT_NAMED_ENTRY) {

-               vm_named_entry_t        named_entry;
+               vm_named_entry_t        named_entry;

 
 

-               named_entry = (vm_named_entry_t)port->ip_kobject;
+               named_entry = (vm_named_entry_t) ip_get_kobject(port);

                /* a few checks to make sure user is obeying rules */
                /* a few checks to make sure user is obeying rules */

-               if(*upl_size == 0) {
-                       if(offset >= named_entry->size)
-                               return(KERN_INVALID_RIGHT);
+               if (*upl_size == 0) {
+                       if (offset >= named_entry->size) {
+                               return KERN_INVALID_RIGHT;
+                       }

                        *upl_size = (upl_size_t)(named_entry->size - offset);
                        *upl_size = (upl_size_t)(named_entry->size - offset);

-                       if (*upl_size != named_entry->size - offset)
+                       if (*upl_size != named_entry->size - offset) {

                                return KERN_INVALID_ARGUMENT;
                                return KERN_INVALID_ARGUMENT;

+                       }

                }
                }

-               if(caller_flags & UPL_COPYOUT_FROM) {
-                       if((named_entry->protection & VM_PROT_READ) 
-                                               != VM_PROT_READ) {
-                               return(KERN_INVALID_RIGHT);
+               if (caller_flags & UPL_COPYOUT_FROM) {
+                       if ((named_entry->protection & VM_PROT_READ)
+                           != VM_PROT_READ) {
+                               return KERN_INVALID_RIGHT;

                        }
                } else {
                        }
                } else {

-                       if((named_entry->protection & 
-                               (VM_PROT_READ | VM_PROT_WRITE)) 
-                               != (VM_PROT_READ | VM_PROT_WRITE)) {
-                               return(KERN_INVALID_RIGHT);
+                       if ((named_entry->protection &
+                           (VM_PROT_READ | VM_PROT_WRITE))
+                           != (VM_PROT_READ | VM_PROT_WRITE)) {
+                               return KERN_INVALID_RIGHT;

                        }
                }
                        }
                }

-               if(named_entry->size < (offset + *upl_size))
-                       return(KERN_INVALID_ARGUMENT);
+               if (named_entry->size < (offset + *upl_size)) {
+                       return KERN_INVALID_ARGUMENT;
+               }

 
                /* the callers parameter offset is defined to be the */
                /* offset from beginning of named entry offset in object */
                offset = offset + named_entry->offset;
 
                /* the callers parameter offset is defined to be the */
                /* offset from beginning of named entry offset in object */
                offset = offset + named_entry->offset;

+               offset += named_entry->data_offset;

 
                if (named_entry->is_sub_map ||
 
                if (named_entry->is_sub_map ||

-                   named_entry->is_copy)
+                   named_entry->is_copy) {

                        return KERN_INVALID_ARGUMENT;
                        return KERN_INVALID_ARGUMENT;

-               
-               named_entry_lock(named_entry);
-
-               if (named_entry->is_pager) {
-                       object = vm_object_enter(named_entry->backing.pager, 
-                                       named_entry->offset + named_entry->size, 
-                                       named_entry->internal, 
-                                       FALSE,
-                                       FALSE);
-                       if (object == VM_OBJECT_NULL) {
-                               named_entry_unlock(named_entry);
-                               return(KERN_INVALID_OBJECT);
-                       }
+               }
+               if (!named_entry->is_object) {
+                       return KERN_INVALID_ARGUMENT;
+               }

 
 

-                       /* JMM - drop reference on pager here? */
+               named_entry_lock(named_entry);

 
 

-                       /* create an extra reference for the named entry */
-                       vm_object_lock(object);
-                       vm_object_reference_locked(object);
-                       named_entry->backing.object = object;
-                       named_entry->is_pager = FALSE;
-                       named_entry_unlock(named_entry);
-
-                       /* wait for object to be ready */
-                       while (!object->pager_ready) {
-                               vm_object_wait(object,
-                                               VM_OBJECT_EVENT_PAGER_READY,
-                                               THREAD_UNINT);
-                               vm_object_lock(object);
-                       }
-                       vm_object_unlock(object);
-               } else {
-                       /* This is the case where we are going to map */
-                       /* an already mapped object.  If the object is */
-                       /* not ready it is internal.  An external     */
-                       /* object cannot be mapped until it is ready  */
-                       /* we can therefore avoid the ready check     */
-                       /* in this case.  */
-                       object = named_entry->backing.object;
-                       vm_object_reference(object);
-                       named_entry_unlock(named_entry);
-               }
+               object = vm_named_entry_to_vm_object(named_entry);
+               assert(object != VM_OBJECT_NULL);
+               vm_object_reference(object);
+               named_entry_unlock(named_entry);

        } else if (ip_kotype(port) == IKOT_MEM_OBJ_CONTROL) {
        } else if (ip_kotype(port) == IKOT_MEM_OBJ_CONTROL) {

-               memory_object_control_t control;
+               memory_object_control_t control;

                control = (memory_object_control_t) port;
                control = (memory_object_control_t) port;

-               if (control == NULL)
-                       return (KERN_INVALID_ARGUMENT);
+               if (control == NULL) {
+                       return KERN_INVALID_ARGUMENT;
+               }

                object = memory_object_control_to_vm_object(control);
                object = memory_object_control_to_vm_object(control);

-               if (object == VM_OBJECT_NULL)
-                       return (KERN_INVALID_ARGUMENT);
+               if (object == VM_OBJECT_NULL) {
+                       return KERN_INVALID_ARGUMENT;
+               }

                vm_object_reference(object);
        } else {
                return KERN_INVALID_ARGUMENT;
        }
                vm_object_reference(object);
        } else {
                return KERN_INVALID_ARGUMENT;
        }

-       if (object == VM_OBJECT_NULL)
-               return (KERN_INVALID_ARGUMENT);
+       if (object == VM_OBJECT_NULL) {
+               return KERN_INVALID_ARGUMENT;
+       }

 
        if (!object->private) {
 
        if (!object->private) {

-               if (*upl_size > (MAX_UPL_TRANSFER*PAGE_SIZE))
-                       *upl_size = (MAX_UPL_TRANSFER*PAGE_SIZE);

                if (object->phys_contiguous) {
                        *flags = UPL_PHYS_CONTIG;
                } else {
                if (object->phys_contiguous) {
                        *flags = UPL_PHYS_CONTIG;
                } else {


@@ -1578,17 +1490,18 @@ memory_object_iopl_request(
        }
 
        ret = vm_object_iopl_request(object,
        }
 
        ret = vm_object_iopl_request(object,

-                                    offset,
-                                    *upl_size,
-                                    upl_ptr,
-                                    user_page_list,
-                                    page_list_count,
-                                    caller_flags);
+           offset,
+           *upl_size,
+           upl_ptr,
+           user_page_list,
+           page_list_count,
+           caller_flags,
+           tag);

        vm_object_deallocate(object);
        return ret;
 }
 
        vm_object_deallocate(object);
        return ret;
 }
 

-/*  
+/*

  *     Routine:        memory_object_upl_request [interface]
  *     Purpose:
  *             Cause the population of a portion of a vm_object.
  *     Routine:        memory_object_upl_request [interface]
  *     Purpose:
  *             Cause the population of a portion of a vm_object.


@@ -1599,30 +1512,35 @@ memory_object_iopl_request(
 
 kern_return_t
 memory_object_upl_request(
 
 kern_return_t
 memory_object_upl_request(

-       memory_object_control_t control,
-       memory_object_offset_t  offset,
-       upl_size_t              size,
-       upl_t                   *upl_ptr,
-       upl_page_info_array_t   user_page_list,
-       unsigned int            *page_list_count,
-       int                     cntrl_flags)
+       memory_object_control_t control,
+       memory_object_offset_t  offset,
+       upl_size_t              size,
+       upl_t                   *upl_ptr,
+       upl_page_info_array_t   user_page_list,
+       unsigned int            *page_list_count,
+       int                     cntrl_flags,
+       int                     tag)

 {
 {

-       vm_object_t             object;
+       vm_object_t             object;
+       vm_tag_t                vmtag = (vm_tag_t)tag;
+       assert(vmtag == tag);

 
        object = memory_object_control_to_vm_object(control);
 
        object = memory_object_control_to_vm_object(control);

-       if (object == VM_OBJECT_NULL)
-               return (KERN_TERMINATED);
+       if (object == VM_OBJECT_NULL) {
+               return KERN_TERMINATED;
+       }

 
        return vm_object_upl_request(object,
 
        return vm_object_upl_request(object,

-                                    offset,
-                                    size,
-                                    upl_ptr,
-                                    user_page_list,
-                                    page_list_count,
-                                    cntrl_flags);
+                  offset,
+                  size,
+                  upl_ptr,
+                  user_page_list,
+                  page_list_count,
+                  (upl_control_flags_t)(unsigned int) cntrl_flags,
+                  vmtag);

 }
 
 }
 

-/*  
+/*

  *     Routine:        memory_object_super_upl_request [interface]
  *     Purpose:
  *             Cause the population of a portion of a vm_object
  *     Routine:        memory_object_super_upl_request [interface]
  *     Purpose:
  *             Cause the population of a portion of a vm_object


@@ -1636,57 +1554,67 @@ memory_object_upl_request(
 kern_return_t
 memory_object_super_upl_request(
        memory_object_control_t control,
 kern_return_t
 memory_object_super_upl_request(
        memory_object_control_t control,

-       memory_object_offset_t  offset,
-       upl_size_t              size,
-       upl_size_t              super_cluster,
-       upl_t                   *upl,
-       upl_page_info_t         *user_page_list,
-       unsigned int            *page_list_count,
-       int                     cntrl_flags)
+       memory_object_offset_t  offset,
+       upl_size_t              size,
+       upl_size_t              super_cluster,
+       upl_t                   *upl,
+       upl_page_info_t         *user_page_list,
+       unsigned int            *page_list_count,
+       int                     cntrl_flags,
+       int                     tag)

 {
 {

-       vm_object_t             object;
+       vm_object_t             object;
+       vm_tag_t                vmtag = (vm_tag_t)tag;
+       assert(vmtag == tag);

 
        object = memory_object_control_to_vm_object(control);
 
        object = memory_object_control_to_vm_object(control);

-       if (object == VM_OBJECT_NULL)
-               return (KERN_INVALID_ARGUMENT);
+       if (object == VM_OBJECT_NULL) {
+               return KERN_INVALID_ARGUMENT;
+       }

 
        return vm_object_super_upl_request(object,
 
        return vm_object_super_upl_request(object,

-                                          offset,
-                                          size,
-                                          super_cluster,
-                                          upl,
-                                          user_page_list,
-                                          page_list_count,
-                                          cntrl_flags);
+                  offset,
+                  size,
+                  super_cluster,
+                  upl,
+                  user_page_list,
+                  page_list_count,
+                  (upl_control_flags_t)(unsigned int) cntrl_flags,
+                  vmtag);

 }
 
 kern_return_t
 }
 
 kern_return_t

-memory_object_cluster_size(memory_object_control_t control, memory_object_offset_t *start,
-                          vm_size_t *length, uint32_t *io_streaming, memory_object_fault_info_t fault_info)
+memory_object_cluster_size(
+       memory_object_control_t control,
+       memory_object_offset_t  *start,
+       vm_size_t               *length,
+       uint32_t                *io_streaming,
+       memory_object_fault_info_t mo_fault_info)

 {
 {

-       vm_object_t             object;
+       vm_object_t             object;
+       vm_object_fault_info_t  fault_info;

 
        object = memory_object_control_to_vm_object(control);
 
 
        object = memory_object_control_to_vm_object(control);
 

-       if (object == VM_OBJECT_NULL || object->paging_offset > *start)
-               return (KERN_INVALID_ARGUMENT);
+       if (object == VM_OBJECT_NULL || object->paging_offset > *start) {
+               return KERN_INVALID_ARGUMENT;
+       }

 
        *start -= object->paging_offset;
 
 
        *start -= object->paging_offset;
 

-       vm_object_cluster_size(object, (vm_object_offset_t *)start, length, (vm_object_fault_info_t)fault_info, io_streaming);
+       fault_info = (vm_object_fault_info_t)(uintptr_t) mo_fault_info;
+       vm_object_cluster_size(object,
+           (vm_object_offset_t *)start,
+           length,
+           fault_info,
+           io_streaming);

 
        *start += object->paging_offset;
 
 
        *start += object->paging_offset;
 

-       return (KERN_SUCCESS);
+       return KERN_SUCCESS;

 }
 
 
 }
 
 

-int vm_stat_discard_cleared_reply = 0;
-int vm_stat_discard_cleared_unset = 0;
-int vm_stat_discard_cleared_too_late = 0;
-
-
-

 /*
  *     Routine:        host_default_memory_manager [interface]
  *     Purpose:
 /*
  *     Routine:        host_default_memory_manager [interface]
  *     Purpose:


@@ -1697,8 +1625,8 @@ int vm_stat_discard_cleared_too_late = 0;
  */
 kern_return_t
 host_default_memory_manager(
  */
 kern_return_t
 host_default_memory_manager(

-       host_priv_t             host_priv,
-       memory_object_default_t *default_manager,
+       host_priv_t             host_priv,
+       memory_object_default_t *default_manager,

        __unused memory_object_cluster_size_t cluster_size)
 {
        memory_object_default_t current_manager;
        __unused memory_object_cluster_size_t cluster_size)
 {
        memory_object_default_t current_manager;


@@ -1706,8 +1634,9 @@ host_default_memory_manager(
        memory_object_default_t returned_manager;
        kern_return_t result = KERN_SUCCESS;
 
        memory_object_default_t returned_manager;
        kern_return_t result = KERN_SUCCESS;
 

-       if (host_priv == HOST_PRIV_NULL)
-               return(KERN_INVALID_HOST);
+       if (host_priv == HOST_PRIV_NULL) {
+               return KERN_INVALID_HOST;
+       }

 
        assert(host_priv == &realhost);
 
 
        assert(host_priv == &realhost);
 


@@ -1723,6 +1652,14 @@ host_default_memory_manager(
                returned_manager = current_manager;
                memory_object_default_reference(returned_manager);
        } else {
                returned_manager = current_manager;
                memory_object_default_reference(returned_manager);
        } else {

+               /*
+                *      Only allow the kernel to change the value.
+                */
+               extern task_t kernel_task;
+               if (current_task() != kernel_task) {
+                       result = KERN_NO_ACCESS;
+                       goto out;
+               }

 
                /*
                 *      If this is the first non-null manager, start
 
                /*
                 *      If this is the first non-null manager, start


@@ -1730,8 +1667,9 @@ host_default_memory_manager(
                 */
                if (current_manager == MEMORY_OBJECT_DEFAULT_NULL) {
                        result = vm_pageout_internal_start();
                 */
                if (current_manager == MEMORY_OBJECT_DEFAULT_NULL) {
                        result = vm_pageout_internal_start();

-                       if (result != KERN_SUCCESS)
+                       if (result != KERN_SUCCESS) {

                                goto out;
                                goto out;

+                       }

                }
 
                /*
                }
 
                /*


@@ -1757,16 +1695,15 @@ host_default_memory_manager(
                 * reactivate all the throttled pages (i.e. dirty pages with
                 * no pager).
                 */
                 * reactivate all the throttled pages (i.e. dirty pages with
                 * no pager).
                 */

-               if (current_manager == MEMORY_OBJECT_DEFAULT_NULL)
-               {
+               if (current_manager == MEMORY_OBJECT_DEFAULT_NULL) {

                        vm_page_reactivate_all_throttled();
                }
        }
                        vm_page_reactivate_all_throttled();
                }
        }

- out:
+out:

        lck_mtx_unlock(&memory_manager_default_lock);
 
        *default_manager = returned_manager;
        lck_mtx_unlock(&memory_manager_default_lock);
 
        *default_manager = returned_manager;

-       return(result);
+       return result;

 }
 
 /*
 }
 
 /*


@@ -1788,9 +1725,9 @@ memory_manager_default_reference(void)
                wait_result_t res;
 
                res = lck_mtx_sleep(&memory_manager_default_lock,
                wait_result_t res;
 
                res = lck_mtx_sleep(&memory_manager_default_lock,

-                                       LCK_SLEEP_DEFAULT,
-                                       (event_t) &memory_manager_default,
-                                       THREAD_UNINT);
+                   LCK_SLEEP_DEFAULT,
+                   (event_t) &memory_manager_default,
+                   THREAD_UNINT);

                assert(res == THREAD_AWAKENED);
                current_manager = memory_manager_default;
        }
                assert(res == THREAD_AWAKENED);
                current_manager = memory_manager_default;
        }


@@ -1820,142 +1757,214 @@ memory_manager_default_check(void)
        lck_mtx_lock(&memory_manager_default_lock);
        current = memory_manager_default;
        if (current == MEMORY_OBJECT_DEFAULT_NULL) {
        lck_mtx_lock(&memory_manager_default_lock);
        current = memory_manager_default;
        if (current == MEMORY_OBJECT_DEFAULT_NULL) {

-               static boolean_t logged;        /* initialized to 0 */
-               boolean_t       complain = !logged;
+               static boolean_t logged;        /* initialized to 0 */
+               boolean_t       complain = !logged;

                logged = TRUE;
                lck_mtx_unlock(&memory_manager_default_lock);
                logged = TRUE;
                lck_mtx_unlock(&memory_manager_default_lock);

-               if (complain)
+               if (complain) {

                        printf("Warning: No default memory manager\n");
                        printf("Warning: No default memory manager\n");

-               return(KERN_FAILURE);
+               }
+               return KERN_FAILURE;

        } else {
                lck_mtx_unlock(&memory_manager_default_lock);
        } else {
                lck_mtx_unlock(&memory_manager_default_lock);

-               return(KERN_SUCCESS);
+               return KERN_SUCCESS;

        }
 }
 
        }
 }
 

-__private_extern__ void
-memory_manager_default_init(void)
-{
-       memory_manager_default = MEMORY_OBJECT_DEFAULT_NULL;
-       lck_mtx_init(&memory_manager_default_lock, &vm_object_lck_grp, &vm_object_lck_attr);
-}
-
-
-

 /* Allow manipulation of individual page state.  This is actually part of */
 /* the UPL regimen but takes place on the object rather than on a UPL */
 
 kern_return_t
 memory_object_page_op(
 /* Allow manipulation of individual page state.  This is actually part of */
 /* the UPL regimen but takes place on the object rather than on a UPL */
 
 kern_return_t
 memory_object_page_op(

-       memory_object_control_t control,
-       memory_object_offset_t  offset,
-       int                     ops,
-       ppnum_t                 *phys_entry,
-       int                     *flags)
+       memory_object_control_t control,
+       memory_object_offset_t  offset,
+       int                     ops,
+       ppnum_t                 *phys_entry,
+       int                     *flags)

 {
 {

-       vm_object_t             object;
+       vm_object_t             object;

 
        object = memory_object_control_to_vm_object(control);
 
        object = memory_object_control_to_vm_object(control);

-       if (object == VM_OBJECT_NULL)
-               return (KERN_INVALID_ARGUMENT);
+       if (object == VM_OBJECT_NULL) {
+               return KERN_INVALID_ARGUMENT;
+       }

 
        return vm_object_page_op(object, offset, ops, phys_entry, flags);
 }
 
 /*
 
        return vm_object_page_op(object, offset, ops, phys_entry, flags);
 }
 
 /*

- * memory_object_range_op offers performance enhancement over 
- * memory_object_page_op for page_op functions which do not require page 
- * level state to be returned from the call.  Page_op was created to provide 
- * a low-cost alternative to page manipulation via UPLs when only a single 
- * page was involved.  The range_op call establishes the ability in the _op 
+ * memory_object_range_op offers performance enhancement over
+ * memory_object_page_op for page_op functions which do not require page
+ * level state to be returned from the call.  Page_op was created to provide
+ * a low-cost alternative to page manipulation via UPLs when only a single
+ * page was involved.  The range_op call establishes the ability in the _op

  * family of functions to work on multiple pages where the lack of page level
  * state handling allows the caller to avoid the overhead of the upl structures.
  */
 
 kern_return_t
 memory_object_range_op(
  * family of functions to work on multiple pages where the lack of page level
  * state handling allows the caller to avoid the overhead of the upl structures.
  */
 
 kern_return_t
 memory_object_range_op(

-       memory_object_control_t control,
-       memory_object_offset_t  offset_beg,
-       memory_object_offset_t  offset_end,
+       memory_object_control_t control,
+       memory_object_offset_t  offset_beg,
+       memory_object_offset_t  offset_end,

        int                     ops,
        int                     *range)
 {
        int                     ops,
        int                     *range)
 {

-       vm_object_t             object;
+       vm_object_t             object;

 
        object = memory_object_control_to_vm_object(control);
 
        object = memory_object_control_to_vm_object(control);

-       if (object == VM_OBJECT_NULL)
-               return (KERN_INVALID_ARGUMENT);
+       if (object == VM_OBJECT_NULL) {
+               return KERN_INVALID_ARGUMENT;
+       }

 
        return vm_object_range_op(object,
 
        return vm_object_range_op(object,

-                                 offset_beg,
-                                 offset_end,
-                                 ops,
-                                 (uint32_t *) range);
+                  offset_beg,
+                  offset_end,
+                  ops,
+                  (uint32_t *) range);

 }
 
 
 void
 memory_object_mark_used(
 }
 
 
 void
 memory_object_mark_used(

-        memory_object_control_t        control)
+       memory_object_control_t control)

 {
 {

-       vm_object_t             object;
+       vm_object_t             object;

 
 

-       if (control == NULL)
+       if (control == NULL) {

                return;
                return;

+       }

 
        object = memory_object_control_to_vm_object(control);
 
 
        object = memory_object_control_to_vm_object(control);
 

-       if (object != VM_OBJECT_NULL)
+       if (object != VM_OBJECT_NULL) {

                vm_object_cache_remove(object);
                vm_object_cache_remove(object);

+       }

 }
 
 
 void
 memory_object_mark_unused(
 }
 
 
 void
 memory_object_mark_unused(

-       memory_object_control_t control,
-       __unused boolean_t      rage)
+       memory_object_control_t control,
+       __unused boolean_t      rage)

 {
 {

-       vm_object_t             object;
+       vm_object_t             object;

 
 

-       if (control == NULL)
+       if (control == NULL) {

                return;
                return;

+       }

 
        object = memory_object_control_to_vm_object(control);
 
 
        object = memory_object_control_to_vm_object(control);
 

-       if (object != VM_OBJECT_NULL)
+       if (object != VM_OBJECT_NULL) {

                vm_object_cache_add(object);
                vm_object_cache_add(object);

+       }
+}
+
+void
+memory_object_mark_io_tracking(
+       memory_object_control_t control)
+{
+       vm_object_t             object;
+
+       if (control == NULL) {
+               return;
+       }
+       object = memory_object_control_to_vm_object(control);
+
+       if (object != VM_OBJECT_NULL) {
+               vm_object_lock(object);
+               object->io_tracking = TRUE;
+               vm_object_unlock(object);
+       }

 }
 
 }
 

+void
+memory_object_mark_trusted(
+       memory_object_control_t control)
+{
+       vm_object_t             object;
+
+       if (control == NULL) {
+               return;
+       }
+       object = memory_object_control_to_vm_object(control);
+
+       if (object != VM_OBJECT_NULL) {
+               vm_object_lock(object);
+               object->pager_trusted = TRUE;
+               vm_object_unlock(object);
+       }
+}
+
+#if CONFIG_SECLUDED_MEMORY
+void
+memory_object_mark_eligible_for_secluded(
+       memory_object_control_t control,
+       boolean_t               eligible_for_secluded)
+{
+       vm_object_t             object;
+
+       if (control == NULL) {
+               return;
+       }
+       object = memory_object_control_to_vm_object(control);
+
+       if (object == VM_OBJECT_NULL) {
+               return;
+       }
+
+       vm_object_lock(object);
+       if (eligible_for_secluded &&
+           secluded_for_filecache && /* global boot-arg */
+           !object->eligible_for_secluded) {
+               object->eligible_for_secluded = TRUE;
+               vm_page_secluded.eligible_for_secluded += object->resident_page_count;
+       } else if (!eligible_for_secluded &&
+           object->eligible_for_secluded) {
+               object->eligible_for_secluded = FALSE;
+               vm_page_secluded.eligible_for_secluded -= object->resident_page_count;
+               if (object->resident_page_count) {
+                       /* XXX FBDP TODO: flush pages from secluded queue? */
+                       // printf("FBDP TODO: flush %d pages from %p from secluded queue\n", object->resident_page_count, object);
+               }
+       }
+       vm_object_unlock(object);
+}
+#endif /* CONFIG_SECLUDED_MEMORY */

 
 kern_return_t
 memory_object_pages_resident(
 
 kern_return_t
 memory_object_pages_resident(

-       memory_object_control_t control,
-       boolean_t                       *       has_pages_resident)
+       memory_object_control_t control,
+       boolean_t                       *       has_pages_resident)

 {
 {

-       vm_object_t             object;
+       vm_object_t             object;

 
        *has_pages_resident = FALSE;
 
        object = memory_object_control_to_vm_object(control);
 
        *has_pages_resident = FALSE;
 
        object = memory_object_control_to_vm_object(control);

-       if (object == VM_OBJECT_NULL)
-               return (KERN_INVALID_ARGUMENT);
+       if (object == VM_OBJECT_NULL) {
+               return KERN_INVALID_ARGUMENT;
+       }

 
 

-       if (object->resident_page_count)
+       if (object->resident_page_count) {

                *has_pages_resident = TRUE;
                *has_pages_resident = TRUE;

-       
-       return (KERN_SUCCESS);
+       }
+
+       return KERN_SUCCESS;

 }
 
 kern_return_t
 memory_object_signed(
 }
 
 kern_return_t
 memory_object_signed(

-       memory_object_control_t control,
-       boolean_t               is_signed)
+       memory_object_control_t control,
+       boolean_t               is_signed)

 {
 {

-       vm_object_t     object;
+       vm_object_t     object;

 
        object = memory_object_control_to_vm_object(control);
 
        object = memory_object_control_to_vm_object(control);

-       if (object == VM_OBJECT_NULL)
+       if (object == VM_OBJECT_NULL) {

                return KERN_INVALID_ARGUMENT;
                return KERN_INVALID_ARGUMENT;

+       }

 
        vm_object_lock(object);
        object->code_signed = is_signed;
 
        vm_object_lock(object);
        object->code_signed = is_signed;


@@ -1966,14 +1975,15 @@ memory_object_signed(
 
 boolean_t
 memory_object_is_signed(
 
 boolean_t
 memory_object_is_signed(

-       memory_object_control_t control)
+       memory_object_control_t control)

 {
 {

-       boolean_t       is_signed;
-       vm_object_t     object;
+       boolean_t       is_signed;
+       vm_object_t     object;

 
        object = memory_object_control_to_vm_object(control);
 
        object = memory_object_control_to_vm_object(control);

-       if (object == VM_OBJECT_NULL)
+       if (object == VM_OBJECT_NULL) {

                return FALSE;
                return FALSE;

+       }

 
        vm_object_lock_shared(object);
        is_signed = object->code_signed;
 
        vm_object_lock_shared(object);
        is_signed = object->code_signed;


@@ -1983,36 +1993,26 @@ memory_object_is_signed(
 }
 
 boolean_t
 }
 
 boolean_t

-memory_object_is_slid(
-       memory_object_control_t control)
+memory_object_is_shared_cache(
+       memory_object_control_t control)

 {
 {

-       vm_object_t     object = VM_OBJECT_NULL;
+       vm_object_t     object = VM_OBJECT_NULL;

 
        object = memory_object_control_to_vm_object(control);
 
        object = memory_object_control_to_vm_object(control);

-       if (object == VM_OBJECT_NULL)
+       if (object == VM_OBJECT_NULL) {

                return FALSE;
                return FALSE;

+       }

 
 

-       return object->object_slid;
+       return object->object_is_shared_cache;

 }
 
 }
 

-static zone_t mem_obj_control_zone;
-
-__private_extern__ void
-memory_object_control_bootstrap(void)
-{
-       int     i;
-
-       i = (vm_size_t) sizeof (struct memory_object_control);
-       mem_obj_control_zone = zinit (i, 8192*i, 4096, "mem_obj_control");
-       zone_change(mem_obj_control_zone, Z_CALLERACCT, FALSE);
-       zone_change(mem_obj_control_zone, Z_NOENCRYPT, TRUE);
-       return;
-}
+static ZONE_DECLARE(mem_obj_control_zone, "mem_obj_control",
+    sizeof(struct memory_object_control), ZC_NOENCRYPT);

 
 __private_extern__ memory_object_control_t
 memory_object_control_allocate(
 
 __private_extern__ memory_object_control_t
 memory_object_control_allocate(

-       vm_object_t             object)
-{                     
+       vm_object_t             object)
+{

        memory_object_control_t control;
 
        control = (memory_object_control_t)zalloc(mem_obj_control_zone);
        memory_object_control_t control;
 
        control = (memory_object_control_t)zalloc(mem_obj_control_zone);


@@ -2020,33 +2020,50 @@ memory_object_control_allocate(
                control->moc_object = object;
                control->moc_ikot = IKOT_MEM_OBJ_CONTROL; /* fake ip_kotype */
        }
                control->moc_object = object;
                control->moc_ikot = IKOT_MEM_OBJ_CONTROL; /* fake ip_kotype */
        }

-       return (control);
+       return control;

 }
 
 __private_extern__ void
 memory_object_control_collapse(
 }
 
 __private_extern__ void
 memory_object_control_collapse(

-       memory_object_control_t control,                       
-       vm_object_t             object)
-{                     
+       memory_object_control_t control,
+       vm_object_t             object)
+{

        assert((control->moc_object != VM_OBJECT_NULL) &&
        assert((control->moc_object != VM_OBJECT_NULL) &&

-              (control->moc_object != object));
+           (control->moc_object != object));

        control->moc_object = object;
 }
 
 __private_extern__ vm_object_t
 memory_object_control_to_vm_object(
        control->moc_object = object;
 }
 
 __private_extern__ vm_object_t
 memory_object_control_to_vm_object(

-       memory_object_control_t control)
+       memory_object_control_t control)

 {
        if (control == MEMORY_OBJECT_CONTROL_NULL ||
 {
        if (control == MEMORY_OBJECT_CONTROL_NULL ||

-           control->moc_ikot != IKOT_MEM_OBJ_CONTROL)
+           control->moc_ikot != IKOT_MEM_OBJ_CONTROL) {

                return VM_OBJECT_NULL;
                return VM_OBJECT_NULL;

+       }

 
 

-       return (control->moc_object);
+       return control->moc_object;
+}
+
+__private_extern__ vm_object_t
+memory_object_to_vm_object(
+       memory_object_t mem_obj)
+{
+       memory_object_control_t mo_control;
+
+       if (mem_obj == MEMORY_OBJECT_NULL) {
+               return VM_OBJECT_NULL;
+       }
+       mo_control = mem_obj->mo_control;
+       if (mo_control == NULL) {
+               return VM_OBJECT_NULL;
+       }
+       return memory_object_control_to_vm_object(mo_control);

 }
 
 memory_object_control_t
 convert_port_to_mo_control(
 }
 
 memory_object_control_t
 convert_port_to_mo_control(

-       __unused mach_port_t    port)
+       __unused mach_port_t    port)

 {
        return MEMORY_OBJECT_CONTROL_NULL;
 }
 {
        return MEMORY_OBJECT_CONTROL_NULL;
 }


@@ -2054,14 +2071,14 @@ convert_port_to_mo_control(
 
 mach_port_t
 convert_mo_control_to_port(
 
 mach_port_t
 convert_mo_control_to_port(

-       __unused memory_object_control_t        control)
+       __unused memory_object_control_t        control)

 {
        return MACH_PORT_NULL;
 }
 
 void
 memory_object_control_reference(
 {
        return MACH_PORT_NULL;
 }
 
 void
 memory_object_control_reference(

-       __unused memory_object_control_t        control)
+       __unused memory_object_control_t        control)

 {
        return;
 }
 {
        return;
 }


@@ -2073,14 +2090,14 @@ memory_object_control_reference(
  */
 void
 memory_object_control_deallocate(
  */
 void
 memory_object_control_deallocate(

-       memory_object_control_t control)
+       memory_object_control_t control)

 {
        zfree(mem_obj_control_zone, control);
 }
 
 void
 memory_object_control_disable(
 {
        zfree(mem_obj_control_zone, control);
 }
 
 void
 memory_object_control_disable(

-       memory_object_control_t control)
+       memory_object_control_t control)

 {
        assert(control->moc_object != VM_OBJECT_NULL);
        control->moc_object = VM_OBJECT_NULL;
 {
        assert(control->moc_object != VM_OBJECT_NULL);
        control->moc_object = VM_OBJECT_NULL;


@@ -2102,22 +2119,23 @@ memory_object_default_deallocate(
 
 memory_object_t
 convert_port_to_memory_object(
 
 memory_object_t
 convert_port_to_memory_object(

-       __unused mach_port_t    port)
+       __unused mach_port_t    port)

 {
 {

-       return (MEMORY_OBJECT_NULL);
+       return MEMORY_OBJECT_NULL;

 }
 
 
 mach_port_t
 convert_memory_object_to_port(
 }
 
 
 mach_port_t
 convert_memory_object_to_port(

-       __unused memory_object_t        object)
+       __unused memory_object_t        object)

 {
 {

-       return (MACH_PORT_NULL);
+       return MACH_PORT_NULL;

 }
 
 
 /* Routine memory_object_reference */
 }
 
 
 /* Routine memory_object_reference */

-void memory_object_reference(
+void
+memory_object_reference(

        memory_object_t memory_object)
 {
        (memory_object->mo_pager_ops->memory_object_reference)(
        memory_object_t memory_object)
 {
        (memory_object->mo_pager_ops->memory_object_reference)(


@@ -2125,16 +2143,18 @@ void memory_object_reference(
 }
 
 /* Routine memory_object_deallocate */
 }
 
 /* Routine memory_object_deallocate */

-void memory_object_deallocate(
+void
+memory_object_deallocate(

        memory_object_t memory_object)
 {
        (memory_object->mo_pager_ops->memory_object_deallocate)(
        memory_object_t memory_object)
 {
        (memory_object->mo_pager_ops->memory_object_deallocate)(

-                memory_object);
+               memory_object);

 }
 
 
 /* Routine memory_object_init */
 }
 
 
 /* Routine memory_object_init */

-kern_return_t memory_object_init
+kern_return_t
+memory_object_init

 (
        memory_object_t memory_object,
        memory_object_control_t memory_control,
 (
        memory_object_t memory_object,
        memory_object_control_t memory_control,


@@ -2148,7 +2168,8 @@ kern_return_t memory_object_init
 }
 
 /* Routine memory_object_terminate */
 }
 
 /* Routine memory_object_terminate */

-kern_return_t memory_object_terminate
+kern_return_t
+memory_object_terminate

 (
        memory_object_t memory_object
 )
 (
        memory_object_t memory_object
 )


@@ -2158,7 +2179,8 @@ kern_return_t memory_object_terminate
 }
 
 /* Routine memory_object_data_request */
 }
 
 /* Routine memory_object_data_request */

-kern_return_t memory_object_data_request
+kern_return_t
+memory_object_data_request

 (
        memory_object_t memory_object,
        memory_object_offset_t offset,
 (
        memory_object_t memory_object,
        memory_object_offset_t offset,


@@ -2169,23 +2191,24 @@ kern_return_t memory_object_data_request
 {
        return (memory_object->mo_pager_ops->memory_object_data_request)(
                memory_object,
 {
        return (memory_object->mo_pager_ops->memory_object_data_request)(
                memory_object,

-               offset, 
+               offset,

                length,
                desired_access,
                fault_info);
 }
 
 /* Routine memory_object_data_return */
                length,
                desired_access,
                fault_info);
 }
 
 /* Routine memory_object_data_return */

-kern_return_t memory_object_data_return
+kern_return_t
+memory_object_data_return

 (
        memory_object_t memory_object,
        memory_object_offset_t offset,
        memory_object_cluster_size_t size,
        memory_object_offset_t *resid_offset,
 (
        memory_object_t memory_object,
        memory_object_offset_t offset,
        memory_object_cluster_size_t size,
        memory_object_offset_t *resid_offset,

-       int     *io_error,
+       int     *io_error,

        boolean_t dirty,
        boolean_t kernel_copy,
        boolean_t dirty,
        boolean_t kernel_copy,

-       int     upl_flags
+       int     upl_flags

 )
 {
        return (memory_object->mo_pager_ops->memory_object_data_return)(
 )
 {
        return (memory_object->mo_pager_ops->memory_object_data_return)(


@@ -2200,7 +2223,8 @@ kern_return_t memory_object_data_return
 }
 
 /* Routine memory_object_data_initialize */
 }
 
 /* Routine memory_object_data_initialize */

-kern_return_t memory_object_data_initialize
+kern_return_t
+memory_object_data_initialize

 (
        memory_object_t memory_object,
        memory_object_offset_t offset,
 (
        memory_object_t memory_object,
        memory_object_offset_t offset,


@@ -2214,7 +2238,8 @@ kern_return_t memory_object_data_initialize
 }
 
 /* Routine memory_object_data_unlock */
 }
 
 /* Routine memory_object_data_unlock */

-kern_return_t memory_object_data_unlock
+kern_return_t
+memory_object_data_unlock

 (
        memory_object_t memory_object,
        memory_object_offset_t offset,
 (
        memory_object_t memory_object,
        memory_object_offset_t offset,


@@ -2230,7 +2255,8 @@ kern_return_t memory_object_data_unlock
 }
 
 /* Routine memory_object_synchronize */
 }
 
 /* Routine memory_object_synchronize */

-kern_return_t memory_object_synchronize
+kern_return_t
+memory_object_synchronize

 (
        memory_object_t memory_object,
        memory_object_offset_t offset,
 (
        memory_object_t memory_object,
        memory_object_offset_t offset,


@@ -2238,6 +2264,8 @@ kern_return_t memory_object_synchronize
        vm_sync_t sync_flags
 )
 {
        vm_sync_t sync_flags
 )
 {

+       panic("memory_object_syncrhonize no longer supported\n");
+

        return (memory_object->mo_pager_ops->memory_object_synchronize)(
                memory_object,
                offset,
        return (memory_object->mo_pager_ops->memory_object_synchronize)(
                memory_object,
                offset,


@@ -2250,7 +2278,7 @@ kern_return_t memory_object_synchronize
  * memory_object_map() is called by VM (in vm_map_enter() and its variants)
  * each time a "named" VM object gets mapped directly or indirectly
  * (copy-on-write mapping).  A "named" VM object has an extra reference held
  * memory_object_map() is called by VM (in vm_map_enter() and its variants)
  * each time a "named" VM object gets mapped directly or indirectly
  * (copy-on-write mapping).  A "named" VM object has an extra reference held

- * by the pager to keep it alive until the pager decides that the 
+ * by the pager to keep it alive until the pager decides that the

  * memory object (and its VM object) can be reclaimed.
  * VM calls memory_object_last_unmap() (in vm_object_deallocate()) when all
  * the mappings of that memory object have been removed.
  * memory object (and its VM object) can be reclaimed.
  * VM calls memory_object_last_unmap() (in vm_object_deallocate()) when all
  * the mappings of that memory object have been removed.


@@ -2265,7 +2293,8 @@ kern_return_t memory_object_synchronize
  */
 
 /* Routine memory_object_map */
  */
 
 /* Routine memory_object_map */

-kern_return_t memory_object_map
+kern_return_t
+memory_object_map

 (
        memory_object_t memory_object,
        vm_prot_t prot
 (
        memory_object_t memory_object,
        vm_prot_t prot


@@ -2277,7 +2306,8 @@ kern_return_t memory_object_map
 }
 
 /* Routine memory_object_last_unmap */
 }
 
 /* Routine memory_object_last_unmap */

-kern_return_t memory_object_last_unmap
+kern_return_t
+memory_object_last_unmap

 (
        memory_object_t memory_object
 )
 (
        memory_object_t memory_object
 )


@@ -2287,62 +2317,51 @@ kern_return_t memory_object_last_unmap
 }
 
 /* Routine memory_object_data_reclaim */
 }
 
 /* Routine memory_object_data_reclaim */

-kern_return_t memory_object_data_reclaim
+kern_return_t
+memory_object_data_reclaim

 (
        memory_object_t memory_object,
 (
        memory_object_t memory_object,

-       boolean_t       reclaim_backing_store
+       boolean_t       reclaim_backing_store

 )
 {
 )
 {

-       if (memory_object->mo_pager_ops->memory_object_data_reclaim == NULL)
+       if (memory_object->mo_pager_ops->memory_object_data_reclaim == NULL) {

                return KERN_NOT_SUPPORTED;
                return KERN_NOT_SUPPORTED;

+       }

        return (memory_object->mo_pager_ops->memory_object_data_reclaim)(
                memory_object,
                reclaim_backing_store);
 }
 
        return (memory_object->mo_pager_ops->memory_object_data_reclaim)(
                memory_object,
                reclaim_backing_store);
 }
 

-/* Routine memory_object_create */
-kern_return_t memory_object_create
-(
-       memory_object_default_t default_memory_manager,
-       vm_size_t new_memory_object_size,
-       memory_object_t *new_memory_object
-)
-{
-       return default_pager_memory_object_create(default_memory_manager,
-                                                 new_memory_object_size,
-                                                 new_memory_object);
-}
-

 upl_t
 convert_port_to_upl(
 upl_t
 convert_port_to_upl(

-       ipc_port_t      port)
+       ipc_port_t      port)

 {
        upl_t upl;
 
        ip_lock(port);
        if (!ip_active(port) || (ip_kotype(port) != IKOT_UPL)) {
 {
        upl_t upl;
 
        ip_lock(port);
        if (!ip_active(port) || (ip_kotype(port) != IKOT_UPL)) {

-                       ip_unlock(port);
-                       return (upl_t)NULL;
+               ip_unlock(port);
+               return (upl_t)NULL;

        }
        }

-       upl = (upl_t) port->ip_kobject;
+       upl = (upl_t) ip_get_kobject(port);

        ip_unlock(port);
        upl_lock(upl);
        ip_unlock(port);
        upl_lock(upl);

-       upl->ref_count+=1;
+       upl->ref_count += 1;

        upl_unlock(upl);
        return upl;
 }
 
 mach_port_t
 convert_upl_to_port(
        upl_unlock(upl);
        return upl;
 }
 
 mach_port_t
 convert_upl_to_port(

-       __unused upl_t          upl)
+       __unused upl_t          upl)

 {
        return MACH_PORT_NULL;
 }
 
 __private_extern__ void
 upl_no_senders(
 {
        return MACH_PORT_NULL;
 }
 
 __private_extern__ void
 upl_no_senders(

-       __unused ipc_port_t                             port,
-       __unused mach_port_mscount_t    mscount)
+       __unused ipc_port_t                             port,
+       __unused mach_port_mscount_t    mscount)

 {
        return;
 }
 {
        return;
 }