xnu-6153.101.6.tar.gz

[apple/xnu.git] / osfmk / vm / vm_object.h

diff --git a/osfmk/vm/vm_object.h b/osfmk/vm/vm_object.h
index 81a303fb440f58e0db0809ec5b8bfdf145b71085..c4fde23b120a512c4ef8548be6f9400ef293bf15 100644 (file)
--- a/osfmk/vm/vm_object.h
+++ b/osfmk/vm/vm_object.h
@@ -1,49 +1,55 @@
 /*
 /*

- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.

  *
  *

- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ *
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ *
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ *
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER

  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,

- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@

  */
 /*
  * @OSF_COPYRIGHT@
  */
  */
 /*
  * @OSF_COPYRIGHT@
  */

-/* 
+/*

  * Mach Operating System
  * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
  * All Rights Reserved.
  * Mach Operating System
  * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
  * All Rights Reserved.

- * 
+ *

  * Permission to use, copy, modify and distribute this software and its
  * documentation is hereby granted, provided that both the copyright
  * notice and this permission notice appear in all copies of the
  * software, derivative works or modified versions, and any portions
  * thereof, and that both notices appear in supporting documentation.
  * Permission to use, copy, modify and distribute this software and its
  * documentation is hereby granted, provided that both the copyright
  * notice and this permission notice appear in all copies of the
  * software, derivative works or modified versions, and any portions
  * thereof, and that both notices appear in supporting documentation.

- * 
+ *

  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
  * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
  * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.

- * 
+ *

  * Carnegie Mellon requests users of this software to return to
  * Carnegie Mellon requests users of this software to return to

- * 
+ *

  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  *  School of Computer Science
  *  Carnegie Mellon University
  *  Pittsburgh PA 15213-3890
  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  *  School of Computer Science
  *  Carnegie Mellon University
  *  Pittsburgh PA 15213-3890

- * 
+ *

  * any improvements or extensions that they make and grant Carnegie Mellon
  * the rights to redistribute these changes.
  */
  * any improvements or extensions that they make and grant Carnegie Mellon
  * the rights to redistribute these changes.
  */


@@ -57,9 +63,11 @@
  *     Virtual memory object module definitions.
  */
 
  *     Virtual memory object module definitions.
  */
 

-#ifndef        _VM_VM_OBJECT_H_
+#ifndef _VM_VM_OBJECT_H_

 #define _VM_VM_OBJECT_H_
 
 #define _VM_VM_OBJECT_H_
 

+#include <debug.h>
+#include <mach_assert.h>

 #include <mach_pagemap.h>
 #include <task_swapper.h>
 
 #include <mach_pagemap.h>
 #include <task_swapper.h>
 


@@ -68,482 +76,1132 @@
 #include <mach/memory_object_types.h>
 #include <mach/port.h>
 #include <mach/vm_prot.h>
 #include <mach/memory_object_types.h>
 #include <mach/port.h>
 #include <mach/vm_prot.h>

+#include <mach/vm_param.h>

 #include <mach/machine/vm_types.h>
 #include <kern/queue.h>
 #include <mach/machine/vm_types.h>
 #include <kern/queue.h>

-#include <kern/lock.h>
+#include <kern/locks.h>

 #include <kern/assert.h>
 #include <kern/assert.h>

+#include <kern/misc_protos.h>

 #include <kern/macro_help.h>
 #include <ipc/ipc_types.h>
 #include <vm/pmap.h>
 #include <kern/macro_help.h>
 #include <ipc/ipc_types.h>
 #include <vm/pmap.h>

-#include <kern/misc_protos.h>

 
 

-#if    MACH_PAGEMAP

 #include <vm/vm_external.h>
 #include <vm/vm_external.h>

-#endif /* MACH_PAGEMAP */

 
 

-typedef struct ipc_port *      pager_request_t;
-#define        PAGER_REQUEST_NULL      ((pager_request_t) 0)
+#include <vm/vm_options.h>
+#include <vm/vm_page.h>
+
+#if VM_OBJECT_TRACKING
+#include <libkern/OSDebug.h>
+#include <kern/btlog.h>
+extern void vm_object_tracking_init(void);
+extern boolean_t vm_object_tracking_inited;
+extern btlog_t *vm_object_tracking_btlog;
+#define VM_OBJECT_TRACKING_NUM_RECORDS  50000
+#define VM_OBJECT_TRACKING_BTDEPTH 7
+#define VM_OBJECT_TRACKING_OP_CREATED   1
+#define VM_OBJECT_TRACKING_OP_MODIFIED  2
+#define VM_OBJECT_TRACKING_OP_TRUESHARE 3
+#endif /* VM_OBJECT_TRACKING */
+
+struct vm_page;

 
 /*
  *     Types defined:
  *
  *     vm_object_t             Virtual memory object.
 
 /*
  *     Types defined:
  *
  *     vm_object_t             Virtual memory object.

- *
- *     We use "struct ipc_port *" instead of "ipc_port_t"
- *     to avoid include file circularities.
+ *     vm_object_fault_info_t  Used to determine cluster size.

  */
 
  */
 

-typedef unsigned long long vm_object_size_t;
+struct vm_object_fault_info {
+       int             interruptible;
+       uint32_t        user_tag;
+       vm_size_t       cluster_size;
+       vm_behavior_t   behavior;
+       vm_map_offset_t lo_offset;
+       vm_map_offset_t hi_offset;
+       unsigned int
+       /* boolean_t */ no_cache:1,
+       /* boolean_t */ stealth:1,
+       /* boolean_t */ io_sync:1,
+       /* boolean_t */ cs_bypass:1,
+       /* boolean_t */ pmap_cs_associated:1,
+       /* boolean_t */ mark_zf_absent:1,
+       /* boolean_t */ batch_pmap_op:1,
+       /* boolean_t */ resilient_media:1,
+       /* boolean_t */ no_copy_on_read:1,
+           __vm_object_fault_info_unused_bits:23;
+       int             pmap_options;
+};
+

 
 

+#define vo_size                         vo_un1.vou_size
+#define vo_cache_pages_to_scan          vo_un1.vou_cache_pages_to_scan
+#define vo_shadow_offset                vo_un2.vou_shadow_offset
+#define vo_cache_ts                     vo_un2.vou_cache_ts
+#define vo_owner                        vo_un2.vou_owner

 
 struct vm_object {
 
 struct vm_object {

-       queue_head_t            memq;           /* Resident memory */
-       decl_mutex_data(,       Lock)           /* Synchronization */
-
-       vm_object_size_t        size;           /* Object size (only valid
-                                                * if internal)
-                                                */
-       vm_object_size_t        frozen_size;    /* How much has been marked
-                                                * copy-on-write (only
-                                                * valid if copy_symmetric)
-                                                */
-       int                     ref_count;      /* Number of references */
-#if    TASK_SWAPPER
-       int                     res_count;      /* Residency references (swap)*/
-#endif /* TASK_SWAPPER */
-       unsigned int            resident_page_count;
-                                               /* number of resident pages */
-
-       struct vm_object        *copy;          /* Object that should receive
-                                                * a copy of my changed pages,
-                                                * for copy_delay, or just the
-                                                * temporary object that
-                                                * shadows this object, for
-                                                * copy_call.
-                                                */
-       struct vm_object        *shadow;        /* My shadow */
-       vm_object_offset_t      shadow_offset;  /* Offset into shadow */
-
-       struct ipc_port         *pager;         /* Where to get data */
-       vm_object_offset_t      paging_offset;  /* Offset into memory object */
-       pager_request_t         pager_request;  /* Where data comes back */
+       /*
+        * on 64 bit systems we pack the pointers hung off the memq.
+        * those pointers have to be able to point back to the memq.
+        * the packed pointers are required to be on a 64 byte boundary
+        * which means 2 things for the vm_object...  (1) the memq
+        * struct has to be the first element of the structure so that
+        * we can control it's alignment... (2) the vm_object must be
+        * aligned on a 64 byte boundary... for static vm_object's
+        * this is accomplished via the 'aligned' attribute... for
+        * vm_object's in the zone pool, this is accomplished by
+        * rounding the size of the vm_object element to the nearest
+        * 64 byte size before creating the zone.
+        */
+       vm_page_queue_head_t    memq;           /* Resident memory - must be first */
+       lck_rw_t                Lock;           /* Synchronization */
+
+#if DEVELOPMENT || DEBUG
+       thread_t                Lock_owner;
+#endif
+       union {
+               vm_object_size_t  vou_size;     /* Object size (only valid if internal) */
+               int               vou_cache_pages_to_scan;      /* pages yet to be visited in an
+                                                                * external object in cache
+                                                                */
+       } vo_un1;
+
+       struct vm_page          *memq_hint;
+       int                     ref_count;      /* Number of references */
+       unsigned int            resident_page_count;
+       /* number of resident pages */
+       unsigned int            wired_page_count; /* number of wired pages
+                                                  *  use VM_OBJECT_WIRED_PAGE_UPDATE macros to update */
+       unsigned int            reusable_page_count;
+
+       struct vm_object        *copy;          /* Object that should receive
+                                                * a copy of my changed pages,
+                                                * for copy_delay, or just the
+                                                * temporary object that
+                                                * shadows this object, for
+                                                * copy_call.
+                                                */
+       struct vm_object        *shadow;        /* My shadow */
+       memory_object_t         pager;          /* Where to get data */
+
+       union {
+               vm_object_offset_t vou_shadow_offset;   /* Offset into shadow */
+               clock_sec_t     vou_cache_ts;   /* age of an external object
+                                                * present in cache
+                                                */
+               task_t          vou_owner;      /* If the object is purgeable
+                                                * or has a "ledger_tag", this
+                                                * is the task that owns it.
+                                                */
+       } vo_un2;
+
+       vm_object_offset_t      paging_offset;  /* Offset into memory object */
+       memory_object_control_t pager_control;  /* Where data comes back */

 
        memory_object_copy_strategy_t
 
        memory_object_copy_strategy_t

-                               copy_strategy;  /* How to handle data copy */
-
-       unsigned int            absent_count;   /* The number of pages that
-                                                * have been requested but
-                                                * not filled.  That is, the
-                                                * number of pages for which
-                                                * the "absent" attribute is
-                                                * asserted.
-                                                */
-
-       unsigned int            paging_in_progress;
-                                               /* The memory object ports are
-                                                * being used (e.g., for pagein
-                                                * or pageout) -- don't change
-                                                * any of these fields (i.e.,
-                                                * don't collapse, destroy or
-                                                * terminate)
-                                                */
+           copy_strategy;                      /* How to handle data copy */
+
+#if __LP64__
+       /*
+        * Some user processes (mostly VirtualMachine software) take a large
+        * number of UPLs (via IOMemoryDescriptors) to wire pages in large
+        * VM objects and overflow the 16-bit "activity_in_progress" counter.
+        * Since we never enforced any limit there, let's give them 32 bits
+        * for backwards compatibility's sake.
+        */
+       unsigned int            paging_in_progress:16,
+           __object1_unused_bits:16;
+       unsigned int            activity_in_progress;
+#else /* __LP64__ */
+       /*
+        * On 32-bit platforms, enlarging "activity_in_progress" would increase
+        * the size of "struct vm_object".  Since we don't know of any actual
+        * overflow of these counters on these platforms, let's keep the
+        * counters as 16-bit integers.
+        */
+       unsigned short          paging_in_progress;
+       unsigned short          activity_in_progress;
+#endif /* __LP64__ */
+       /* The memory object ports are
+        * being used (e.g., for pagein
+        * or pageout) -- don't change
+        * any of these fields (i.e.,
+        * don't collapse, destroy or
+        * terminate)
+        */
+

        unsigned int
        unsigned int

-       /* boolean_t array */   all_wanted:11,  /* Bit array of "want to be
-                                                * awakened" notations.  See
-                                                * VM_OBJECT_EVENT_* items
-                                                * below */
-       /* boolean_t */ pager_created:1,        /* Has pager been created? */
-       /* boolean_t */ pager_initialized:1,    /* Are fields ready to use? */
-       /* boolean_t */ pager_ready:1,          /* Will pager take requests? */
-
-       /* boolean_t */         pager_trusted:1,/* The pager for this object
-                                                * is trusted. This is true for
-                                                * all internal objects (backed
-                                                * by the default pager)
-                                                */
-       /* boolean_t */         can_persist:1,  /* The kernel may keep the data
-                                                * for this object (and rights
-                                                * to the memory object) after
-                                                * all address map references 
-                                                * are deallocated?
-                                                */
-       /* boolean_t */         internal:1,     /* Created by the kernel (and
-                                                * therefore, managed by the
-                                                * default memory manger)
-                                                */
-       /* boolean_t */         temporary:1,    /* Permanent objects may be
-                                                * changed externally by the 
-                                                * memory manager, and changes
-                                                * made in memory must be
-                                                * reflected back to the memory
-                                                * manager.  Temporary objects
-                                                * lack both of these
-                                                * characteristics.
-                                                */
-       /* boolean_t */         private:1,      /* magic device_pager object,
-                                                * holds private pages only */
-       /* boolean_t */         pageout:1,      /* pageout object. contains
-                                                * private pages that refer to
-                                                * a real memory object. */
-       /* boolean_t */         alive:1,        /* Not yet terminated */
-
-       /* boolean_t */         lock_in_progress:1,
-                                               /* Is a multi-page lock
-                                                * request in progress?
-                                                */
-       /* boolean_t */         lock_restart:1,
-                                               /* Should lock request in
-                                                * progress restart search?
-                                                */
-       /* boolean_t */         shadowed:1,     /* Shadow may exist */
-       /* boolean_t */         silent_overwrite:1,
-                                               /* Allow full page overwrite
-                                                * without data_request if
-                                                * page is absent */
-       /* boolean_t */         advisory_pageout:1,
-                                               /* Instead of sending page
-                                                * via OOL, just notify
-                                                * pager that the kernel
-                                                * wants to discard it, page
-                                                * remains in object */
-       /* boolean_t */         true_share:1,
-                                               /* This object is mapped
-                                                * in more than one place
-                                                * and hence cannot be 
-                                                * coalesced */
-       /* boolean_t */         terminating:1,
-                                               /* Allows vm_object_lookup
-                                                * and vm_object_deallocate
-                                                * to special case their
-                                                * behavior when they are
-                                                * called as a result of
-                                                * page cleaning during
-                                                * object termination
-                                                */
-       /* boolean_t */         named:1,        /* An enforces an internal
-                                                * naming convention, by
-                                                * calling the right routines
-                                                * for allocation and 
-                                                * destruction, UBC references
-                                                * against the vm_object are
-                                                * checked.
-                                                */
-       /* boolean_t */         shadow_severed:1,
-                                               /* When a permanent object 
-                                                * backing a COW goes away
-                                                * unexpectedly.  This bit
-                                                * allows vm_fault to return
-                                                * an error rather than a
-                                                * zero filled page.
-                                                */
-       /* boolean_t */         phys_contiguous:1;
-                                               /* Memory is wired and
-                                                * guaranteed physically 
-                                                * contiguous.  However
-                                                * it is not device memory
-                                                * and obeys normal virtual
-                                                * memory rules w.r.t pmap
-                                                * access bits.
-                                                */
-                                               
-
-
-       queue_chain_t           cached_list;    /* Attachment point for the
-                                                * list of objects cached as a
-                                                * result of their can_persist
-                                                * value
-                                                */
-
-       queue_head_t            msr_q;          /* memory object synchronise
-                                                  request queue */
-
-       vm_object_offset_t      last_alloc;     /* last allocation offset */
-       vm_size_t               cluster_size;   /* size of paging cluster */
-#if    MACH_PAGEMAP
-       vm_external_map_t       existence_map;  /* bitmap of pages written to
-                                                * backing storage */
-#endif /* MACH_PAGEMAP */
-#if    MACH_ASSERT
-       struct vm_object        *paging_object; /* object which pages to be
-                                                * swapped out are temporary
-                                                * put in current object
-                                                */
+       /* boolean_t array */ all_wanted:11,    /* Bit array of "want to be
+                                                * awakened" notations.  See
+                                                * VM_OBJECT_EVENT_* items
+                                                * below */
+       /* boolean_t */ pager_created:1,        /* Has pager been created? */
+       /* boolean_t */ pager_initialized:1,    /* Are fields ready to use? */
+       /* boolean_t */ pager_ready:1,          /* Will pager take requests? */
+
+       /* boolean_t */ pager_trusted:1,        /* The pager for this object
+                                                * is trusted. This is true for
+                                                * all internal objects (backed
+                                                * by the default pager)
+                                                */
+       /* boolean_t */ can_persist:1,          /* The kernel may keep the data
+                                                * for this object (and rights
+                                                * to the memory object) after
+                                                * all address map references
+                                                * are deallocated?
+                                                */
+       /* boolean_t */ internal:1,             /* Created by the kernel (and
+                                                * therefore, managed by the
+                                                * default memory manger)
+                                                */
+       /* boolean_t */ private:1,              /* magic device_pager object,
+                                               * holds private pages only */
+       /* boolean_t */ pageout:1,              /* pageout object. contains
+                                                * private pages that refer to
+                                                * a real memory object. */
+       /* boolean_t */ alive:1,                /* Not yet terminated */
+
+       /* boolean_t */ purgable:2,             /* Purgable state.  See
+                                                * VM_PURGABLE_*
+                                                */
+       /* boolean_t */ purgeable_only_by_kernel:1,
+       /* boolean_t */ purgeable_when_ripe:1,         /* Purgeable when a token
+                                                       * becomes ripe.
+                                                       */
+       /* boolean_t */ shadowed:1,             /* Shadow may exist */
+       /* boolean_t */ true_share:1,
+       /* This object is mapped
+        * in more than one place
+        * and hence cannot be
+        * coalesced */
+       /* boolean_t */ terminating:1,
+       /* Allows vm_object_lookup
+        * and vm_object_deallocate
+        * to special case their
+        * behavior when they are
+        * called as a result of
+        * page cleaning during
+        * object termination
+        */
+       /* boolean_t */ named:1,                /* An enforces an internal
+                                                * naming convention, by
+                                                * calling the right routines
+                                                * for allocation and
+                                                * destruction, UBC references
+                                                * against the vm_object are
+                                                * checked.
+                                                */
+       /* boolean_t */ shadow_severed:1,
+       /* When a permanent object
+        * backing a COW goes away
+        * unexpectedly.  This bit
+        * allows vm_fault to return
+        * an error rather than a
+        * zero filled page.
+        */
+       /* boolean_t */ phys_contiguous:1,
+       /* Memory is wired and
+        * guaranteed physically
+        * contiguous.  However
+        * it is not device memory
+        * and obeys normal virtual
+        * memory rules w.r.t pmap
+        * access bits.
+        */
+       /* boolean_t */ nophyscache:1,
+       /* When mapped at the
+        * pmap level, don't allow
+        * primary caching. (for
+        * I/O)
+        */
+       /* boolean_t */ _object5_unused_bits:1;
+
+       queue_chain_t           cached_list;    /* Attachment point for the
+                                                * list of objects cached as a
+                                                * result of their can_persist
+                                                * value
+                                                */
+       /*
+        * the following fields are not protected by any locks
+        * they are updated via atomic compare and swap
+        */
+       vm_object_offset_t      last_alloc;     /* last allocation offset */
+       vm_offset_t             cow_hint;       /* last page present in     */
+                                               /* shadow but not in object */
+       int                     sequential;     /* sequential access size */
+
+       uint32_t                pages_created;
+       uint32_t                pages_used;
+       /* hold object lock when altering */
+       unsigned        int
+           wimg_bits:8,                /* cache WIMG bits         */
+           code_signed:1,              /* pages are signed and should be
+                                        *  validated; the signatures are stored
+                                        *  with the pager */
+           transposed:1,               /* object was transposed with another */
+           mapping_in_progress:1,      /* pager being mapped/unmapped */
+           phantom_isssd:1,
+           volatile_empty:1,
+           volatile_fault:1,
+           all_reusable:1,
+           blocked_access:1,
+           set_cache_attr:1,
+           object_is_shared_cache:1,
+           purgeable_queue_type:2,
+           purgeable_queue_group:3,
+           io_tracking:1,
+           no_tag_update:1,            /*  */
+#if CONFIG_SECLUDED_MEMORY
+           eligible_for_secluded:1,
+           can_grab_secluded:1,
+#else /* CONFIG_SECLUDED_MEMORY */
+       __object3_unused_bits:2,
+#endif /* CONFIG_SECLUDED_MEMORY */
+#if VM_OBJECT_ACCESS_TRACKING
+           access_tracking:1,
+#else /* VM_OBJECT_ACCESS_TRACKING */
+       __unused_access_tracking:1,
+#endif /* VM_OBJECT_ACCESS_TRACKING */
+       vo_ledger_tag:3,
+           vo_no_footprint:1;
+
+#if VM_OBJECT_ACCESS_TRACKING
+       uint32_t        access_tracking_reads;
+       uint32_t        access_tracking_writes;
+#endif /* VM_OBJECT_ACCESS_TRACKING */
+
+       uint8_t                 scan_collisions;
+       uint8_t                 __object4_unused_bits[1];
+       vm_tag_t                wire_tag;
+
+#if CONFIG_PHANTOM_CACHE
+       uint32_t                phantom_object_id;

 #endif
 #endif

-#ifdef UBC_DEBUG
-       queue_head_t            uplq;           /* List of outstanding upls */
-#endif /* UBC_DEBUG */
+#if CONFIG_IOSCHED || UPL_DEBUG
+       queue_head_t            uplq;           /* List of outstanding upls */
+#endif
+
+#ifdef  VM_PIP_DEBUG
+/*
+ * Keep track of the stack traces for the first holders
+ * of a "paging_in_progress" reference for this VM object.
+ */
+#define VM_PIP_DEBUG_STACK_FRAMES       25      /* depth of each stack trace */
+#define VM_PIP_DEBUG_MAX_REFS           10      /* track that many references */
+       struct __pip_backtrace {
+               void *pip_retaddr[VM_PIP_DEBUG_STACK_FRAMES];
+       } pip_holders[VM_PIP_DEBUG_MAX_REFS];
+#endif  /* VM_PIP_DEBUG  */
+
+       queue_chain_t           objq;      /* object queue - currently used for purgable queues */
+       queue_chain_t           task_objq; /* objects owned by task - protected by task lock */
+
+#if !VM_TAG_ACTIVE_UPDATE
+       queue_chain_t           wired_objq;
+#endif /* !VM_TAG_ACTIVE_UPDATE */
+
+#if DEBUG
+       void *purgeable_owner_bt[16];
+       task_t vo_purgeable_volatilizer; /* who made it volatile? */
+       void *purgeable_volatilizer_bt[16];
+#endif /* DEBUG */

 };
 
 };
 

+#define VM_OBJECT_PURGEABLE_FAULT_ERROR(object)                         \
+       ((object)->volatile_fault &&                                    \
+        ((object)->purgable == VM_PURGABLE_VOLATILE ||                 \
+         (object)->purgable == VM_PURGABLE_EMPTY))
+
+#if VM_OBJECT_ACCESS_TRACKING
+extern uint64_t vm_object_access_tracking_reads;
+extern uint64_t vm_object_access_tracking_writes;
+extern void vm_object_access_tracking(vm_object_t object,
+    int *access_tracking,
+    uint32_t *access_tracking_reads,
+    uint32_t *acess_tracking_writes);
+#endif /* VM_OBJECT_ACCESS_TRACKING */
+

 extern
 extern

-vm_object_t    kernel_object;          /* the single kernel object */
-
-int            vm_object_absent_max;   /* maximum number of absent pages
-                                          at a time for each object */
-
-# define       VM_MSYNC_INITIALIZED                    0
-# define       VM_MSYNC_SYNCHRONIZING                  1
-# define       VM_MSYNC_DONE                           2
-
-struct msync_req {
-       queue_chain_t           msr_q;          /* object request queue */
-       queue_chain_t           req_q;          /* vm_msync request queue */
-       unsigned int            flag;
-       vm_object_offset_t      offset;
-       vm_object_size_t        length;
-       vm_object_t             object;         /* back pointer */
-       decl_mutex_data(,       msync_req_lock) /* Lock for this structure */
-};
+vm_object_t     kernel_object;          /* the single kernel object */
+
+extern
+vm_object_t     compressor_object;      /* the single compressor object */
+
+extern
+unsigned int    vm_object_absent_max;   /* maximum number of absent pages
+                                         *  at a time for each object */
+
+# define        VM_MSYNC_INITIALIZED                    0
+# define        VM_MSYNC_SYNCHRONIZING                  1
+# define        VM_MSYNC_DONE                           2
+
+
+extern lck_grp_t                vm_map_lck_grp;
+extern lck_attr_t               vm_map_lck_attr;
+
+#ifndef VM_TAG_ACTIVE_UPDATE
+#error VM_TAG_ACTIVE_UPDATE
+#endif

 
 

-typedef struct msync_req       *msync_req_t;
-#define MSYNC_REQ_NULL         ((msync_req_t) 0)
+#if VM_TAG_ACTIVE_UPDATE
+#define VM_OBJECT_WIRED_ENQUEUE(object) panic("VM_OBJECT_WIRED_ENQUEUE")
+#define VM_OBJECT_WIRED_DEQUEUE(object) panic("VM_OBJECT_WIRED_DEQUEUE")
+#else /* VM_TAG_ACTIVE_UPDATE */
+#define VM_OBJECT_WIRED_ENQUEUE(object)                                 \
+       MACRO_BEGIN                                                     \
+       lck_spin_lock_grp(&vm_objects_wired_lock, &vm_page_lck_grp_bucket);                          \
+       assert(!(object)->wired_objq.next);                             \
+       assert(!(object)->wired_objq.prev);                             \
+       queue_enter(&vm_objects_wired, (object),                        \
+                   vm_object_t, wired_objq);                           \
+       lck_spin_unlock(&vm_objects_wired_lock);                        \
+       MACRO_END
+#define VM_OBJECT_WIRED_DEQUEUE(object)                                 \
+       MACRO_BEGIN                                                     \
+       if ((object)->wired_objq.next) {                                \
+               lck_spin_lock_grp(&vm_objects_wired_lock, &vm_page_lck_grp_bucket);                  \
+               queue_remove(&vm_objects_wired, (object),               \
+                            vm_object_t, wired_objq);                  \
+               lck_spin_unlock(&vm_objects_wired_lock);                \
+       }                                                               \
+       MACRO_END
+#endif /* VM_TAG_ACTIVE_UPDATE */
+
+#define VM_OBJECT_WIRED(object, tag)                                    \
+    MACRO_BEGIN                                                         \
+    assert(VM_KERN_MEMORY_NONE != (tag));                               \
+    assert(VM_KERN_MEMORY_NONE == (object)->wire_tag);                  \
+    (object)->wire_tag = (tag);                                         \
+    if (!VM_TAG_ACTIVE_UPDATE) {                                        \
+       VM_OBJECT_WIRED_ENQUEUE((object));                              \
+    }                                                                   \
+    MACRO_END
+
+#define VM_OBJECT_UNWIRED(object)                                                       \
+    MACRO_BEGIN                                                                         \
+    if (!VM_TAG_ACTIVE_UPDATE) {                                                        \
+           VM_OBJECT_WIRED_DEQUEUE((object));                                          \
+    }                                                                                   \
+    if (VM_KERN_MEMORY_NONE != (object)->wire_tag) {                                    \
+       vm_tag_update_size((object)->wire_tag, -ptoa_64((object)->wired_page_count));   \
+       (object)->wire_tag = VM_KERN_MEMORY_NONE;                                       \
+    }                                                                                   \
+    MACRO_END
+
+// These two macros start & end a C block
+#define VM_OBJECT_WIRED_PAGE_UPDATE_START(object)                                       \
+    MACRO_BEGIN                                                                         \
+    {                                                                                   \
+       int64_t __wireddelta = 0; vm_tag_t __waswired = (object)->wire_tag;
+
+#define VM_OBJECT_WIRED_PAGE_UPDATE_END(object, tag)                                    \
+       if (__wireddelta) {                                                             \
+           boolean_t __overflow __assert_only =                                        \
+           os_add_overflow((object)->wired_page_count, __wireddelta,                   \
+                           &(object)->wired_page_count);                               \
+           assert(!__overflow);                                                        \
+           if (!(object)->pageout && !(object)->no_tag_update) {                       \
+               if (__wireddelta > 0) {                                                 \
+                   assert (VM_KERN_MEMORY_NONE != (tag));                              \
+                   if (VM_KERN_MEMORY_NONE == __waswired) {                            \
+                       VM_OBJECT_WIRED((object), (tag));                               \
+                   }                                                                   \
+                   vm_tag_update_size((object)->wire_tag, ptoa_64(__wireddelta));      \
+               } else if (VM_KERN_MEMORY_NONE != __waswired) {                         \
+                   assert (VM_KERN_MEMORY_NONE != (object)->wire_tag);                 \
+                   vm_tag_update_size((object)->wire_tag, ptoa_64(__wireddelta));      \
+                   if (!(object)->wired_page_count) {                                  \
+                       VM_OBJECT_UNWIRED((object));                                    \
+                   }                                                                   \
+               }                                                                       \
+           }                                                                           \
+       }                                                                               \
+    }                                                                                   \
+    MACRO_END
+
+#define VM_OBJECT_WIRED_PAGE_COUNT(object, delta)               \
+    __wireddelta += delta; \
+
+#define VM_OBJECT_WIRED_PAGE_ADD(object, m)                     \
+    if (!(m)->vmp_private && !(m)->vmp_fictitious) __wireddelta++;
+
+#define VM_OBJECT_WIRED_PAGE_REMOVE(object, m)                  \
+    if (!(m)->vmp_private && !(m)->vmp_fictitious) __wireddelta--;
+
+
+
+#define OBJECT_LOCK_SHARED      0
+#define OBJECT_LOCK_EXCLUSIVE   1
+
+extern lck_grp_t        vm_object_lck_grp;
+extern lck_grp_attr_t   vm_object_lck_grp_attr;
+extern lck_attr_t       vm_object_lck_attr;
+extern lck_attr_t       kernel_object_lck_attr;
+extern lck_attr_t       compressor_object_lck_attr;
+
+extern vm_object_t      vm_pageout_scan_wants_object;
+
+extern void             vm_object_lock(vm_object_t);
+extern boolean_t        vm_object_lock_try(vm_object_t);
+extern boolean_t        _vm_object_lock_try(vm_object_t);
+extern boolean_t        vm_object_lock_avoid(vm_object_t);
+extern void             vm_object_lock_shared(vm_object_t);
+extern boolean_t        vm_object_lock_yield_shared(vm_object_t);
+extern boolean_t        vm_object_lock_try_shared(vm_object_t);
+extern void             vm_object_unlock(vm_object_t);
+extern boolean_t        vm_object_lock_upgrade(vm_object_t);

 
 /*
 
 /*

- * Macros to allocate and free msync_reqs
+ *     Object locking macros

  */
  */

-#define msync_req_alloc(msr)                                           \
-       MACRO_BEGIN                                                     \
-        (msr) = (msync_req_t)kalloc(sizeof(struct msync_req));         \
-        mutex_init(&(msr)->msync_req_lock, ETAP_VM_MSYNC);             \
-       msr->flag = VM_MSYNC_INITIALIZED;                               \
-        MACRO_END

 
 

-#define msync_req_free(msr)                                            \
-       (kfree((vm_offset_t)(msr), sizeof(struct msync_req)))
+#define vm_object_lock_init(object)                                     \
+       lck_rw_init(&(object)->Lock, &vm_object_lck_grp,                \
+                   (((object) == kernel_object ||                      \
+                     (object) == vm_submap_object) ?                   \
+                    &kernel_object_lck_attr :                          \
+                    (((object) == compressor_object) ?                 \
+                    &compressor_object_lck_attr :                      \
+                     &vm_object_lck_attr)))
+#define vm_object_lock_destroy(object)  lck_rw_destroy(&(object)->Lock, &vm_object_lck_grp)
+
+#define vm_object_lock_try_scan(object) _vm_object_lock_try(object)
+
+/*
+ * CAUTION: the following vm_object_lock_assert_held*() macros merely
+ * check if anyone is holding the lock, but the holder may not necessarily
+ * be the caller...
+ */
+#if MACH_ASSERT || DEBUG
+#define vm_object_lock_assert_held(object) \
+       lck_rw_assert(&(object)->Lock, LCK_RW_ASSERT_HELD)
+#define vm_object_lock_assert_shared(object) \
+       lck_rw_assert(&(object)->Lock, LCK_RW_ASSERT_SHARED)
+#define vm_object_lock_assert_exclusive(object) \
+       lck_rw_assert(&(object)->Lock, LCK_RW_ASSERT_EXCLUSIVE)
+#define vm_object_lock_assert_notheld(object) \
+       lck_rw_assert(&(object)->Lock, LCK_RW_ASSERT_NOTHELD)
+#else  /* MACH_ASSERT || DEBUG */
+#define vm_object_lock_assert_held(object)
+#define vm_object_lock_assert_shared(object)
+#define vm_object_lock_assert_exclusive(object)
+#define vm_object_lock_assert_notheld(object)
+#endif /* MACH_ASSERT || DEBUG */

 
 

-#define msr_lock(msr)   mutex_lock(&(msr)->msync_req_lock)
-#define msr_unlock(msr) mutex_unlock(&(msr)->msync_req_lock)

 
 /*
  *     Declare procedures that operate on VM objects.
  */
 
 
 /*
  *     Declare procedures that operate on VM objects.
  */
 

-extern void            vm_object_bootstrap(void);
-
-extern void            vm_object_init(void);
-
-extern vm_object_t     vm_object_allocate(
-                                       vm_object_size_t        size);
-
-#if    MACH_ASSERT
-extern void            vm_object_reference(
-                                       vm_object_t     object);
-#else  /* MACH_ASSERT */
-#define        vm_object_reference(object)                     \
-MACRO_BEGIN                                            \
-       vm_object_t Object = (object);                  \
-       if (Object) {                                   \
-               vm_object_lock(Object);                 \
-               Object->ref_count++;                    \
-               vm_object_res_reference(Object);        \
-               vm_object_unlock(Object);               \
-       }                                               \
-MACRO_END
-#endif /* MACH_ASSERT */
-
-extern void            vm_object_deallocate(
-                                       vm_object_t     object);
-
-extern void            vm_object_pmap_protect(
-                                       vm_object_t             object,
-                                       vm_object_offset_t      offset,
-                                       vm_size_t               size,
-                                       pmap_t                  pmap,
-                                       vm_offset_t             pmap_start,
-                                       vm_prot_t               prot);
-
-extern void            vm_object_page_remove(
-                                       vm_object_t             object,
-                                       vm_object_offset_t      start,
-                                       vm_object_offset_t      end);
-
-extern boolean_t       vm_object_coalesce(
-                                       vm_object_t             prev_object,
-                                       vm_object_t             next_object,
-                                       vm_object_offset_t      prev_offset,
-                                       vm_object_offset_t      next_offset,
-                                       vm_object_size_t        prev_size,
-                                       vm_object_size_t        next_size);
-
-extern boolean_t       vm_object_shadow(
-                                       vm_object_t             *object,
-                                       vm_object_offset_t      *offset,
-                                       vm_object_size_t        length);
-
-extern void            vm_object_collapse(
-                                       vm_object_t     object);
-
-extern vm_object_t     vm_object_lookup(
-                                       ipc_port_t      port);
-
-extern ipc_port_t      vm_object_name(
-                                       vm_object_t     object);
-
-extern boolean_t       vm_object_copy_quickly(
-                               vm_object_t             *_object,
-                               vm_object_offset_t      src_offset,
-                               vm_object_size_t        size,
-                               boolean_t               *_src_needs_copy,
-                               boolean_t               *_dst_needs_copy);
-
-extern kern_return_t   vm_object_copy_strategically(
-                               vm_object_t             src_object,
-                               vm_object_offset_t      src_offset,
-                               vm_object_size_t        size,
-                               vm_object_t             *dst_object,
-                               vm_object_offset_t      *dst_offset,
-                               boolean_t               *dst_needs_copy);
-
-extern kern_return_t   vm_object_copy_slowly(
-                               vm_object_t             src_object,
-                               vm_object_offset_t      src_offset,
-                               vm_object_size_t        size,
-                               int                     interruptible,
-                               vm_object_t             *_result_object);
-
-extern void            vm_object_pager_create(
-                                       vm_object_t     object);
-
-extern void            vm_object_destroy(
-                                       ipc_port_t      pager);
-
-extern void            vm_object_pager_wakeup(
-                                       ipc_port_t      pager);
-
-extern void            vm_object_page_map(
-                               vm_object_t     object,
-                               vm_object_offset_t      offset,
-                               vm_object_size_t        size,
-                               vm_object_offset_t      (*map_fn)
-                                       (void *, vm_object_offset_t),
-                                       void            *map_fn_data);
-
-#if    TASK_SWAPPER
-
-extern void            vm_object_res_reference(
-                                       vm_object_t object);
-extern void            vm_object_res_deallocate(
-                                       vm_object_t object);
-#define        VM_OBJ_RES_INCR(object) (object)->res_count++
-#define        VM_OBJ_RES_DECR(object) (object)->res_count--
-
-#else  /* TASK_SWAPPER */
-
-#define        VM_OBJ_RES_INCR(object)
-#define        VM_OBJ_RES_DECR(object)
+__private_extern__ void         vm_object_bootstrap(void);
+
+__private_extern__ void         vm_object_init(void);
+
+__private_extern__ void         vm_object_init_lck_grp(void);
+
+__private_extern__ void         vm_object_reaper_init(void);
+
+__private_extern__ vm_object_t  vm_object_allocate(vm_object_size_t size);
+
+__private_extern__ void    _vm_object_allocate(vm_object_size_t size,
+    vm_object_t object);
+
+#if     TASK_SWAPPER
+
+__private_extern__ void vm_object_res_reference(
+       vm_object_t             object);
+__private_extern__ void vm_object_res_deallocate(
+       vm_object_t             object);
+#define VM_OBJ_RES_INCR(object) (object)->res_count++
+#define VM_OBJ_RES_DECR(object) (object)->res_count--
+
+#else   /* TASK_SWAPPER */
+
+#define VM_OBJ_RES_INCR(object)
+#define VM_OBJ_RES_DECR(object)

 #define vm_object_res_reference(object)
 #define vm_object_res_deallocate(object)
 
 #define vm_object_res_reference(object)
 #define vm_object_res_deallocate(object)
 

-#endif /* TASK_SWAPPER */
+#endif  /* TASK_SWAPPER */
+
+#define vm_object_reference_locked(object)              \
+       MACRO_BEGIN                                     \
+       vm_object_t RLObject = (object);                \
+       vm_object_lock_assert_exclusive(object);        \
+       assert((RLObject)->ref_count > 0);              \
+       (RLObject)->ref_count++;                        \
+       assert((RLObject)->ref_count > 1);              \
+       vm_object_res_reference(RLObject);              \
+       MACRO_END
+
+
+#define vm_object_reference_shared(object)                              \
+       MACRO_BEGIN                                                     \
+       vm_object_t RLObject = (object);                                \
+       vm_object_lock_assert_shared(object);                           \
+       assert((RLObject)->ref_count > 0);                              \
+       OSAddAtomic(1, &(RLObject)->ref_count);         \
+       assert((RLObject)->ref_count > 0);                              \
+       /* XXX we would need an atomic version of the following ... */  \
+       vm_object_res_reference(RLObject);                              \
+       MACRO_END
+
+
+__private_extern__ void         vm_object_reference(
+       vm_object_t     object);

 
 

-extern vm_object_t     vm_object_enter(
-                                       ipc_port_t              pager,
-                                       vm_object_size_t        size,
-                                       boolean_t               internal,
-                                       boolean_t               init,
-                                       boolean_t               check_named);
+#if     !MACH_ASSERT

 
 

+#define vm_object_reference(object)                     \
+MACRO_BEGIN                                             \
+       vm_object_t RObject = (object);                 \
+       if (RObject) {                                  \
+               vm_object_lock_shared(RObject);         \
+               vm_object_reference_shared(RObject);    \
+               vm_object_unlock(RObject);              \
+       }                                               \
+MACRO_END

 
 

-extern vm_object_t     vm_object_copy_delayed(
-                               vm_object_t             src_object,
-                               vm_object_offset_t      src_offset,
-                               vm_object_size_t        size);
+#endif  /* MACH_ASSERT */
+
+__private_extern__ void         vm_object_deallocate(
+       vm_object_t     object);
+
+__private_extern__ kern_return_t vm_object_release_name(
+       vm_object_t     object,
+       int             flags);
+
+__private_extern__ void         vm_object_pmap_protect(
+       vm_object_t             object,
+       vm_object_offset_t      offset,
+       vm_object_size_t        size,
+       pmap_t                  pmap,
+       vm_map_offset_t         pmap_start,
+       vm_prot_t               prot);
+
+__private_extern__ void         vm_object_pmap_protect_options(
+       vm_object_t             object,
+       vm_object_offset_t      offset,
+       vm_object_size_t        size,
+       pmap_t                  pmap,
+       vm_map_offset_t         pmap_start,
+       vm_prot_t               prot,
+       int                     options);
+
+__private_extern__ void         vm_object_page_remove(
+       vm_object_t             object,
+       vm_object_offset_t      start,
+       vm_object_offset_t      end);
+
+__private_extern__ void         vm_object_deactivate_pages(
+       vm_object_t             object,
+       vm_object_offset_t      offset,
+       vm_object_size_t        size,
+       boolean_t               kill_page,
+       boolean_t               reusable_page,
+       struct pmap             *pmap,
+       vm_map_offset_t         pmap_offset);
+
+__private_extern__ void vm_object_reuse_pages(
+       vm_object_t             object,
+       vm_object_offset_t      start_offset,
+       vm_object_offset_t      end_offset,
+       boolean_t               allow_partial_reuse);
+
+__private_extern__ uint64_t     vm_object_purge(
+       vm_object_t              object,
+       int                      flags);
+
+__private_extern__ kern_return_t vm_object_purgable_control(
+       vm_object_t     object,
+       vm_purgable_t   control,
+       int             *state);
+
+__private_extern__ kern_return_t vm_object_get_page_counts(
+       vm_object_t             object,
+       vm_object_offset_t      offset,
+       vm_object_size_t        size,
+       unsigned int            *resident_page_count,
+       unsigned int            *dirty_page_count);
+
+__private_extern__ boolean_t    vm_object_coalesce(
+       vm_object_t             prev_object,
+       vm_object_t             next_object,
+       vm_object_offset_t      prev_offset,
+       vm_object_offset_t      next_offset,
+       vm_object_size_t        prev_size,
+       vm_object_size_t        next_size);
+
+__private_extern__ boolean_t    vm_object_shadow(
+       vm_object_t             *object,
+       vm_object_offset_t      *offset,
+       vm_object_size_t        length);
+
+__private_extern__ void         vm_object_collapse(
+       vm_object_t             object,
+       vm_object_offset_t      offset,
+       boolean_t               can_bypass);
+
+__private_extern__ boolean_t    vm_object_copy_quickly(
+       vm_object_t             *_object,
+       vm_object_offset_t      src_offset,
+       vm_object_size_t        size,
+       boolean_t               *_src_needs_copy,
+       boolean_t               *_dst_needs_copy);
+
+__private_extern__ kern_return_t        vm_object_copy_strategically(
+       vm_object_t             src_object,
+       vm_object_offset_t      src_offset,
+       vm_object_size_t        size,
+       vm_object_t             *dst_object,
+       vm_object_offset_t      *dst_offset,
+       boolean_t               *dst_needs_copy);
+
+__private_extern__ kern_return_t        vm_object_copy_slowly(
+       vm_object_t             src_object,
+       vm_object_offset_t      src_offset,
+       vm_object_size_t        size,
+       boolean_t               interruptible,
+       vm_object_t             *_result_object);
+
+__private_extern__ vm_object_t  vm_object_copy_delayed(
+       vm_object_t             src_object,
+       vm_object_offset_t      src_offset,
+       vm_object_size_t        size,
+       boolean_t               src_object_shared);
+
+
+
+__private_extern__ kern_return_t        vm_object_destroy(
+       vm_object_t     object,
+       kern_return_t   reason);
+
+__private_extern__ void         vm_object_pager_create(
+       vm_object_t     object);
+
+__private_extern__ void         vm_object_compressor_pager_create(
+       vm_object_t     object);
+
+__private_extern__ void         vm_object_page_map(
+       vm_object_t     object,
+       vm_object_offset_t      offset,
+       vm_object_size_t        size,
+       vm_object_offset_t      (*map_fn)
+       (void *, vm_object_offset_t),
+       void            *map_fn_data);
+
+__private_extern__ kern_return_t vm_object_upl_request(
+       vm_object_t             object,
+       vm_object_offset_t      offset,
+       upl_size_t              size,
+       upl_t                   *upl,
+       upl_page_info_t         *page_info,
+       unsigned int            *count,
+       upl_control_flags_t     flags,
+       vm_tag_t            tag);
+
+__private_extern__ kern_return_t vm_object_transpose(
+       vm_object_t             object1,
+       vm_object_t             object2,
+       vm_object_size_t        transpose_size);
+
+__private_extern__ boolean_t vm_object_sync(
+       vm_object_t             object,
+       vm_object_offset_t      offset,
+       vm_object_size_t        size,
+       boolean_t               should_flush,
+       boolean_t               should_return,
+       boolean_t               should_iosync);
+
+__private_extern__ kern_return_t vm_object_update(
+       vm_object_t             object,
+       vm_object_offset_t      offset,
+       vm_object_size_t        size,
+       vm_object_offset_t      *error_offset,
+       int                     *io_errno,
+       memory_object_return_t  should_return,
+       int                     flags,
+       vm_prot_t               prot);
+
+__private_extern__ kern_return_t vm_object_lock_request(
+       vm_object_t             object,
+       vm_object_offset_t      offset,
+       vm_object_size_t        size,
+       memory_object_return_t  should_return,
+       int                     flags,
+       vm_prot_t               prot);
+
+
+
+__private_extern__ vm_object_t  vm_object_memory_object_associate(
+       memory_object_t         pager,
+       vm_object_t             object,
+       vm_object_size_t        size,
+       boolean_t               check_named);
+
+
+__private_extern__ void vm_object_cluster_size(
+       vm_object_t             object,
+       vm_object_offset_t      *start,
+       vm_size_t               *length,
+       vm_object_fault_info_t  fault_info,
+       uint32_t                *io_streaming);
+
+__private_extern__ kern_return_t vm_object_populate_with_private(
+       vm_object_t             object,
+       vm_object_offset_t      offset,
+       ppnum_t                 phys_page,
+       vm_size_t               size);
+
+__private_extern__ void vm_object_change_wimg_mode(
+       vm_object_t             object,
+       unsigned int            wimg_mode);
+
+extern kern_return_t adjust_vm_object_cache(
+       vm_size_t oval,
+       vm_size_t nval);
+
+extern kern_return_t vm_object_page_op(
+       vm_object_t             object,
+       vm_object_offset_t      offset,
+       int                     ops,
+       ppnum_t                 *phys_entry,
+       int                     *flags);
+
+extern kern_return_t vm_object_range_op(
+       vm_object_t             object,
+       vm_object_offset_t      offset_beg,
+       vm_object_offset_t      offset_end,
+       int                     ops,
+       uint32_t                *range);
+
+
+__private_extern__ void         vm_object_reap_pages(
+       vm_object_t object,
+       int     reap_type);
+#define REAP_REAP       0
+#define REAP_TERMINATE  1
+#define REAP_PURGEABLE  2
+#define REAP_DATA_FLUSH 3
+
+#if CONFIG_FREEZE
+
+__private_extern__ uint32_t
+vm_object_compressed_freezer_pageout(
+       vm_object_t     object, uint32_t dirty_budget);
+
+__private_extern__ void
+vm_object_compressed_freezer_done(
+       void);
+
+#endif /* CONFIG_FREEZE */
+
+__private_extern__ void
+vm_object_pageout(
+       vm_object_t     object);
+
+#if CONFIG_IOSCHED
+struct io_reprioritize_req {
+       uint64_t        blkno;
+       uint32_t        len;
+       int             priority;
+       struct vnode    *devvp;
+       queue_chain_t   io_reprioritize_list;
+};
+typedef struct io_reprioritize_req *io_reprioritize_req_t;

 
 

+extern void vm_io_reprioritize_init(void);
+#endif

 
 /*
  *     Event waiting handling
  */
 
 
 /*
  *     Event waiting handling
  */
 

-#define        VM_OBJECT_EVENT_INITIALIZED             0
-#define        VM_OBJECT_EVENT_PAGER_READY             1
-#define        VM_OBJECT_EVENT_PAGING_IN_PROGRESS      2
-#define        VM_OBJECT_EVENT_ABSENT_COUNT            3
-#define        VM_OBJECT_EVENT_LOCK_IN_PROGRESS        4
-#define        VM_OBJECT_EVENT_UNCACHING               5
-#define        VM_OBJECT_EVENT_COPY_CALL               6
-#define        VM_OBJECT_EVENT_CACHING                 7
-
-#define        vm_object_assert_wait(object, event, interruptible)             \
-       MACRO_BEGIN                                                     \
-       (object)->all_wanted |= 1 << (event);                           \
-       assert_wait((event_t)((vm_offset_t)(object)+(event)),(interruptible)); \
-       MACRO_END
+#define VM_OBJECT_EVENT_INITIALIZED             0
+#define VM_OBJECT_EVENT_PAGER_READY             1
+#define VM_OBJECT_EVENT_PAGING_IN_PROGRESS      2
+#define VM_OBJECT_EVENT_MAPPING_IN_PROGRESS     3
+#define VM_OBJECT_EVENT_LOCK_IN_PROGRESS        4
+#define VM_OBJECT_EVENT_UNCACHING               5
+#define VM_OBJECT_EVENT_COPY_CALL               6
+#define VM_OBJECT_EVENT_CACHING                 7
+#define VM_OBJECT_EVENT_UNBLOCKED               8
+#define VM_OBJECT_EVENT_PAGING_ONLY_IN_PROGRESS 9
+
+#define VM_OBJECT_EVENT_MAX 10 /* 11 bits in "all_wanted", so 0->10 */
+
+static __inline__ wait_result_t
+vm_object_assert_wait(
+       vm_object_t             object,
+       int                     event,
+       wait_interrupt_t        interruptible)
+{
+       wait_result_t wr;
+
+       vm_object_lock_assert_exclusive(object);
+       assert(event >= 0 && event <= VM_OBJECT_EVENT_MAX);
+
+       object->all_wanted |= 1 << event;
+       wr = assert_wait((event_t)((vm_offset_t)object + event),
+           interruptible);
+       return wr;
+}
+
+static __inline__ wait_result_t
+vm_object_wait(
+       vm_object_t             object,
+       int                     event,
+       wait_interrupt_t        interruptible)
+{
+       wait_result_t wr;
+
+       vm_object_assert_wait(object, event, interruptible);
+       vm_object_unlock(object);
+       wr = thread_block(THREAD_CONTINUE_NULL);
+       return wr;
+}
+
+static __inline__ wait_result_t
+thread_sleep_vm_object(
+       vm_object_t             object,
+       event_t                 event,
+       wait_interrupt_t        interruptible)
+{
+       wait_result_t wr;
+
+#if DEVELOPMENT || DEBUG
+       if (object->Lock_owner != current_thread()) {
+               panic("thread_sleep_vm_object: now owner - %p\n", object);
+       }
+       object->Lock_owner = 0;
+#endif
+       wr = lck_rw_sleep(&object->Lock,
+           LCK_SLEEP_PROMOTED_PRI,
+           event,
+           interruptible);
+#if DEVELOPMENT || DEBUG
+       object->Lock_owner = current_thread();
+#endif
+       return wr;
+}
+
+static __inline__ wait_result_t
+vm_object_sleep(
+       vm_object_t             object,
+       int                     event,
+       wait_interrupt_t        interruptible)
+{
+       wait_result_t wr;
+
+       vm_object_lock_assert_exclusive(object);
+       assert(event >= 0 && event <= VM_OBJECT_EVENT_MAX);
+
+       object->all_wanted |= 1 << event;
+       wr = thread_sleep_vm_object(object,
+           (event_t)((vm_offset_t)object + event),
+           interruptible);
+       return wr;
+}
+
+static __inline__ void
+vm_object_wakeup(
+       vm_object_t             object,
+       int                     event)
+{
+       vm_object_lock_assert_exclusive(object);
+       assert(event >= 0 && event <= VM_OBJECT_EVENT_MAX);
+
+       if (object->all_wanted & (1 << event)) {
+               thread_wakeup((event_t)((vm_offset_t)object + event));
+       }
+       object->all_wanted &= ~(1 << event);
+}
+
+static __inline__ void
+vm_object_set_wanted(
+       vm_object_t             object,
+       int                     event)
+{
+       vm_object_lock_assert_exclusive(object);
+       assert(event >= 0 && event <= VM_OBJECT_EVENT_MAX);
+
+       object->all_wanted |= (1 << event);
+}
+
+static __inline__ int
+vm_object_wanted(
+       vm_object_t             object,
+       int                     event)
+{
+       vm_object_lock_assert_held(object);
+       assert(event >= 0 && event <= VM_OBJECT_EVENT_MAX);
+
+       return object->all_wanted & (1 << event);
+}

 
 

-#define        vm_object_wait(object, event, interruptible)                    \
-       MACRO_BEGIN                                                     \
-       vm_object_assert_wait((object),(event),(interruptible));        \
-       vm_object_unlock(object);                                       \
-       thread_block((void (*)(void)) 0);                               \
+/*
+ *     Routines implemented as macros
+ */
+#ifdef VM_PIP_DEBUG
+#include <libkern/OSDebug.h>
+#define VM_PIP_DEBUG_BEGIN(object)                                      \
+       MACRO_BEGIN                                                     \
+       int pip = ((object)->paging_in_progress +                       \
+                  (object)->activity_in_progress);                     \
+       if (pip < VM_PIP_DEBUG_MAX_REFS) {                              \
+               (void) OSBacktrace(&(object)->pip_holders[pip].pip_retaddr[0], \
+                                  VM_PIP_DEBUG_STACK_FRAMES);          \
+       }                                                               \

        MACRO_END
        MACRO_END

-
-#define        vm_object_wakeup(object, event)                                 \
-       MACRO_BEGIN                                                     \
-       if ((object)->all_wanted & (1 << (event)))                      \
-               thread_wakeup((event_t)((vm_offset_t)(object) + (event))); \
-       (object)->all_wanted &= ~(1 << (event));                        \
+#else   /* VM_PIP_DEBUG */
+#define VM_PIP_DEBUG_BEGIN(object)
+#endif  /* VM_PIP_DEBUG */
+
+#define         vm_object_activity_begin(object)                        \
+       MACRO_BEGIN                                                     \
+       vm_object_lock_assert_exclusive((object));                      \
+       VM_PIP_DEBUG_BEGIN((object));                                   \
+       (object)->activity_in_progress++;                               \
+       if ((object)->activity_in_progress == 0) {                      \
+               panic("vm_object_activity_begin(%p): overflow\n", (object));\
+       }                                                               \

        MACRO_END
 
        MACRO_END
 

-#define        vm_object_set_wanted(object, event)                             \
-       MACRO_BEGIN                                                     \
-       ((object)->all_wanted |= (1 << (event)));                       \
+#define         vm_object_activity_end(object)                          \
+       MACRO_BEGIN                                                     \
+       vm_object_lock_assert_exclusive((object));                      \
+       if ((object)->activity_in_progress == 0) {                      \
+               panic("vm_object_activity_end(%p): underflow\n", (object));\
+       }                                                               \
+       (object)->activity_in_progress--;                               \
+       if ((object)->paging_in_progress == 0 &&                        \
+           (object)->activity_in_progress == 0)                        \
+               vm_object_wakeup((object),                              \
+                                VM_OBJECT_EVENT_PAGING_IN_PROGRESS);   \

        MACRO_END
 
        MACRO_END
 

-#define        vm_object_wanted(object, event)                                 \
-       ((object)->all_wanted & (1 << (event)))
-
-/*
- *     Routines implemented as macros
- */
+#define         vm_object_paging_begin(object)                          \
+       MACRO_BEGIN                                                     \
+       vm_object_lock_assert_exclusive((object));                      \
+       VM_PIP_DEBUG_BEGIN((object));                                   \
+       (object)->paging_in_progress++;                                 \
+       if ((object)->paging_in_progress == 0) {                        \
+               panic("vm_object_paging_begin(%p): overflow\n", (object));\
+       }                                                               \
+       MACRO_END

 
 

-#define                vm_object_paging_begin(object)                          \
-       MACRO_BEGIN                                                     \
-       (object)->paging_in_progress++;                                 \
+#define         vm_object_paging_end(object)                            \
+       MACRO_BEGIN                                                     \
+       vm_object_lock_assert_exclusive((object));                      \
+       if ((object)->paging_in_progress == 0) {                        \
+               panic("vm_object_paging_end(%p): underflow\n", (object));\
+       }                                                               \
+       (object)->paging_in_progress--;                                 \
+       if ((object)->paging_in_progress == 0) {                        \
+               vm_object_wakeup((object),                              \
+                                VM_OBJECT_EVENT_PAGING_ONLY_IN_PROGRESS); \
+               if ((object)->activity_in_progress == 0)                \
+                       vm_object_wakeup((object),                      \
+                                        VM_OBJECT_EVENT_PAGING_IN_PROGRESS); \
+       }                                                               \

        MACRO_END
 
        MACRO_END
 

-#define                vm_object_paging_end(object)                            \
-       MACRO_BEGIN                                                     \
-       assert((object)->paging_in_progress != 0);                      \
-       if (--(object)->paging_in_progress == 0) {                      \
-               vm_object_wakeup(object,                                \
-                       VM_OBJECT_EVENT_PAGING_IN_PROGRESS);            \
-       }                                                               \
+#define         vm_object_paging_wait(object, interruptible)            \
+       MACRO_BEGIN                                                     \
+       vm_object_lock_assert_exclusive((object));                      \
+       while ((object)->paging_in_progress != 0 ||                     \
+              (object)->activity_in_progress != 0) {                   \
+               wait_result_t  _wr;                                     \
+                                                                        \
+               _wr = vm_object_sleep((object),                         \
+                               VM_OBJECT_EVENT_PAGING_IN_PROGRESS,     \
+                               (interruptible));                       \
+                                                                        \
+       /*XXX if ((interruptible) && (_wr != THREAD_AWAKENED))*/ \
+       /*XXX break; */                                 \
+       }                                                               \

        MACRO_END
 
        MACRO_END
 

-#define                vm_object_paging_wait(object, interruptible)            \
-       MACRO_BEGIN                                                     \
-       while ((object)->paging_in_progress != 0) {                     \
-               vm_object_wait( (object),                               \
-                               VM_OBJECT_EVENT_PAGING_IN_PROGRESS,     \
-                               (interruptible));                       \
-               vm_object_lock(object);                                 \
-                                                                       \
-               /*XXX if ((interruptible) &&    */                      \
-                   /*XXX (current_thread()->wait_result != THREAD_AWAKENED))*/ \
-                       /*XXX break; */                                 \
-       }                                                               \
+#define vm_object_paging_only_wait(object, interruptible)               \
+       MACRO_BEGIN                                                     \
+       vm_object_lock_assert_exclusive((object));                      \
+       while ((object)->paging_in_progress != 0) {                     \
+               wait_result_t  _wr;                                     \
+                                                                        \
+               _wr = vm_object_sleep((object),                         \
+                               VM_OBJECT_EVENT_PAGING_ONLY_IN_PROGRESS,\
+                               (interruptible));                       \
+                                                                        \
+       /*XXX if ((interruptible) && (_wr != THREAD_AWAKENED))*/ \
+       /*XXX break; */                                 \
+       }                                                               \

        MACRO_END
 
        MACRO_END
 

-#define        vm_object_absent_assert_wait(object, interruptible)             \
-       MACRO_BEGIN                                                     \
-       vm_object_assert_wait(  (object),                               \
-                       VM_OBJECT_EVENT_ABSENT_COUNT,                   \
-                       (interruptible));                               \
+
+#define vm_object_mapping_begin(object)                                 \
+       MACRO_BEGIN                                                     \
+       vm_object_lock_assert_exclusive((object));                      \
+       assert(! (object)->mapping_in_progress);                        \
+       (object)->mapping_in_progress = TRUE;                           \

        MACRO_END
 
        MACRO_END
 

+#define vm_object_mapping_end(object)                                   \
+       MACRO_BEGIN                                                     \
+       vm_object_lock_assert_exclusive((object));                      \
+       assert((object)->mapping_in_progress);                          \
+       (object)->mapping_in_progress = FALSE;                          \
+       vm_object_wakeup((object),                                      \
+                        VM_OBJECT_EVENT_MAPPING_IN_PROGRESS);          \
+       MACRO_END

 
 

-#define        vm_object_absent_release(object)                                \
-       MACRO_BEGIN                                                     \
-       (object)->absent_count--;                                       \
-       vm_object_wakeup((object),                                      \
-                        VM_OBJECT_EVENT_ABSENT_COUNT);                 \
+#define vm_object_mapping_wait(object, interruptible)                   \
+       MACRO_BEGIN                                                     \
+       vm_object_lock_assert_exclusive((object));                      \
+       while ((object)->mapping_in_progress) {                         \
+               wait_result_t   _wr;                                    \
+                                                                        \
+               _wr = vm_object_sleep((object),                         \
+                                     VM_OBJECT_EVENT_MAPPING_IN_PROGRESS, \
+                                     (interruptible));                 \
+       /*XXX if ((interruptible) && (_wr != THREAD_AWAKENED))*/ \
+       /*XXX break; */                                 \
+       }                                                               \
+       assert(!(object)->mapping_in_progress);                         \

        MACRO_END
 
        MACRO_END
 

-/*
- *     Object locking macros
- */

 
 

-#define vm_object_lock_init(object)    mutex_init(&(object)->Lock, ETAP_VM_OBJ)
-#define vm_object_lock(object)         mutex_lock(&(object)->Lock)
-#define vm_object_unlock(object)       mutex_unlock(&(object)->Lock)
-#define vm_object_lock_try(object)     mutex_try(&(object)->Lock)

 
 

-#endif /* _VM_VM_OBJECT_H_ */
+#define vm_object_round_page(x) (((vm_object_offset_t)(x) + PAGE_MASK) & ~((signed)PAGE_MASK))
+#define vm_object_trunc_page(x) ((vm_object_offset_t)(x) & ~((signed)PAGE_MASK))
+
+extern void     vm_object_cache_add(vm_object_t);
+extern void     vm_object_cache_remove(vm_object_t);
+extern int      vm_object_cache_evict(int, int);
+
+#define VM_OBJECT_OWNER_DISOWNED ((task_t) -1)
+#define VM_OBJECT_OWNER(object)                                         \
+       ((((object)->purgable == VM_PURGABLE_DENY &&                    \
+          (object)->vo_ledger_tag == 0) ||                             \
+         (object)->vo_owner == TASK_NULL)                              \
+        ? TASK_NULL    /* not owned */                                 \
+        : (((object)->vo_owner == VM_OBJECT_OWNER_DISOWNED)            \
+           ? kernel_task /* disowned -> kernel */                      \
+           : (object)->vo_owner)) /* explicit owner */                 \
+
+extern void     vm_object_ledger_tag_ledgers(
+       vm_object_t object,
+       int *ledger_idx_volatile,
+       int *ledger_idx_nonvolatile,
+       int *ledger_idx_volatile_compressed,
+       int *ledger_idx_nonvolatile_compressed,
+       boolean_t *do_footprint);
+extern kern_return_t vm_object_ownership_change(
+       vm_object_t object,
+       int new_ledger_tag,
+       task_t new_owner,
+       int new_ledger_flags,
+       boolean_t task_objq_locked);
+
+#endif  /* _VM_VM_OBJECT_H_ */