xnu-4570.71.2.tar.gz

[apple/xnu.git] / osfmk / vm / vm_object.h

diff --git a/osfmk/vm/vm_object.h b/osfmk/vm/vm_object.h
index 0d21734af26937855460fd500efb708e467cf783..2474375718d03d4ba05a4e6b8243427f6b3e2262 100644 (file)
--- a/osfmk/vm/vm_object.h
+++ b/osfmk/vm/vm_object.h
@@ -79,7 +79,6 @@
 #include <mach/vm_param.h>
 #include <mach/machine/vm_types.h>
 #include <kern/queue.h>
 #include <mach/vm_param.h>
 #include <mach/machine/vm_types.h>
 #include <kern/queue.h>

-#include <kern/lock.h>

 #include <kern/locks.h>
 #include <kern/assert.h>
 #include <kern/misc_protos.h>
 #include <kern/locks.h>
 #include <kern/assert.h>
 #include <kern/misc_protos.h>


@@ -87,13 +86,26 @@
 #include <ipc/ipc_types.h>
 #include <vm/pmap.h>
 
 #include <ipc/ipc_types.h>
 #include <vm/pmap.h>
 

-#if    MACH_PAGEMAP

 #include <vm/vm_external.h>
 #include <vm/vm_external.h>

-#endif /* MACH_PAGEMAP */

 
 #include <vm/vm_options.h>
 
 #include <vm/vm_options.h>

+#include <vm/vm_page.h>
+
+#if VM_OBJECT_TRACKING
+#include <libkern/OSDebug.h>
+#include <kern/btlog.h>
+extern void vm_object_tracking_init(void);
+extern boolean_t vm_object_tracking_inited;
+extern btlog_t *vm_object_tracking_btlog;
+#define VM_OBJECT_TRACKING_NUM_RECORDS 50000
+#define VM_OBJECT_TRACKING_BTDEPTH 7
+#define VM_OBJECT_TRACKING_OP_CREATED  1
+#define VM_OBJECT_TRACKING_OP_MODIFIED 2
+#define VM_OBJECT_TRACKING_OP_TRUESHARE        3
+#endif /* VM_OBJECT_TRACKING */

 
 struct vm_page;
 
 struct vm_page;

+struct vm_shared_region_slide_info;

 
 /*
  *     Types defined:
 
 /*
  *     Types defined:


@@ -115,19 +127,39 @@ struct vm_object_fault_info {
        /* boolean_t */ io_sync:1,
        /* boolean_t */ cs_bypass:1,
        /* boolean_t */ mark_zf_absent:1,
        /* boolean_t */ io_sync:1,
        /* boolean_t */ cs_bypass:1,
        /* boolean_t */ mark_zf_absent:1,

-               __vm_object_fault_info_unused_bits:27;
+       /* boolean_t */ batch_pmap_op:1,
+               __vm_object_fault_info_unused_bits:26;
+       int             pmap_options;

 };
 
 
 };
 
 

-#define        vo_size                 vo_un1.vou_size
-#define vo_cache_pages_to_scan vo_un1.vou_cache_pages_to_scan
-#define vo_shadow_offset       vo_un2.vou_shadow_offset
-#define vo_cache_ts            vo_un2.vou_cache_ts
+#define        vo_size                         vo_un1.vou_size
+#define vo_cache_pages_to_scan         vo_un1.vou_cache_pages_to_scan
+#define vo_shadow_offset               vo_un2.vou_shadow_offset
+#define vo_cache_ts                    vo_un2.vou_cache_ts
+#define vo_purgeable_owner             vo_un2.vou_purgeable_owner
+#define vo_slide_info                  vo_un2.vou_slide_info

 
 struct vm_object {
 
 struct vm_object {

-       queue_head_t            memq;           /* Resident memory */
+       /*
+        * on 64 bit systems we pack the pointers hung off the memq.
+        * those pointers have to be able to point back to the memq.
+        * the packed pointers are required to be on a 64 byte boundary
+        * which means 2 things for the vm_object...  (1) the memq
+        * struct has to be the first element of the structure so that
+        * we can control it's alignment... (2) the vm_object must be
+        * aligned on a 64 byte boundary... for static vm_object's
+        * this is accomplished via the 'aligned' attribute... for
+        * vm_object's in the zone pool, this is accomplished by
+        * rounding the size of the vm_object element to the nearest
+        * 64 byte size before creating the zone.
+        */
+       vm_page_queue_head_t    memq;           /* Resident memory - must be first */

         lck_rw_t               Lock;           /* Synchronization */
 
         lck_rw_t               Lock;           /* Synchronization */
 

+#if DEVELOPMENT || DEBUG
+       thread_t                Lock_owner;
+#endif

        union {
                vm_object_size_t  vou_size;     /* Object size (only valid if internal) */
                int               vou_cache_pages_to_scan;      /* pages yet to be visited in an
        union {
                vm_object_size_t  vou_size;     /* Object size (only valid if internal) */
                int               vou_cache_pages_to_scan;      /* pages yet to be visited in an


@@ -137,12 +169,10 @@ struct vm_object {
 
        struct vm_page          *memq_hint;
        int                     ref_count;      /* Number of references */
 
        struct vm_page          *memq_hint;
        int                     ref_count;      /* Number of references */

-#if    TASK_SWAPPER
-       int                     res_count;      /* Residency references (swap)*/
-#endif /* TASK_SWAPPER */

        unsigned int            resident_page_count;
                                                /* number of resident pages */
        unsigned int            resident_page_count;
                                                /* number of resident pages */

-       unsigned int            wired_page_count; /* number of wired pages */
+       const unsigned int      wired_page_count; /* number of wired pages
+                                                    use VM_OBJECT_WIRED_PAGE_UPDATE macros to update */

        unsigned int            reusable_page_count;
 
        struct vm_object        *copy;          /* Object that should receive
        unsigned int            reusable_page_count;
 
        struct vm_object        *copy;          /* Object that should receive


@@ -156,9 +186,14 @@ struct vm_object {
 
        union {
                vm_object_offset_t vou_shadow_offset;   /* Offset into shadow */
 
        union {
                vm_object_offset_t vou_shadow_offset;   /* Offset into shadow */

-               clock_sec_t        vou_cache_ts;        /* age of an external object
-                                                        * present in cache
+               clock_sec_t     vou_cache_ts;   /* age of an external object
+                                                * present in cache
+                                                */
+               task_t          vou_purgeable_owner;    /* If the purg'a'ble bits below are set 
+                                                        * to volatile/emtpy, this is the task 
+                                                        * that owns this purgeable object.

                                                         */
                                                         */

+               struct vm_shared_region_slide_info *vou_slide_info;

        } vo_un2;
 
        memory_object_t         pager;          /* Where to get data */
        } vo_un2;
 
        memory_object_t         pager;          /* Where to get data */


@@ -168,7 +203,27 @@ struct vm_object {
        memory_object_copy_strategy_t
                                copy_strategy;  /* How to handle data copy */
 
        memory_object_copy_strategy_t
                                copy_strategy;  /* How to handle data copy */
 

-       short                   paging_in_progress;
+#if __LP64__
+       /*
+        * Some user processes (mostly VirtualMachine software) take a large
+        * number of UPLs (via IOMemoryDescriptors) to wire pages in large
+        * VM objects and overflow the 16-bit "activity_in_progress" counter.
+        * Since we never enforced any limit there, let's give them 32 bits
+        * for backwards compatibility's sake.
+        */
+       unsigned int            paging_in_progress:16,
+                               __object1_unused_bits:16;
+       unsigned int            activity_in_progress;
+#else /* __LP64__ */
+       /*
+        * On 32-bit platforms, enlarging "activity_in_progress" would increase
+        * the size of "struct vm_object".  Since we don't know of any actual
+        * overflow of these counters on these platforms, let's keep the
+        * counters as 16-bit integers.
+        */
+       unsigned short          paging_in_progress;
+       unsigned short          activity_in_progress;
+#endif /* __LP64__ */

                                                /* The memory object ports are
                                                 * being used (e.g., for pagein
                                                 * or pageout) -- don't change
                                                /* The memory object ports are
                                                 * being used (e.g., for pagein
                                                 * or pageout) -- don't change


@@ -176,7 +231,6 @@ struct vm_object {
                                                 * don't collapse, destroy or
                                                 * terminate)
                                                 */
                                                 * don't collapse, destroy or
                                                 * terminate)
                                                 */

-       short                   activity_in_progress;

 
        unsigned int
        /* boolean_t array */   all_wanted:11,  /* Bit array of "want to be
 
        unsigned int
        /* boolean_t array */   all_wanted:11,  /* Bit array of "want to be


@@ -202,15 +256,6 @@ struct vm_object {
                                                 * therefore, managed by the
                                                 * default memory manger)
                                                 */
                                                 * therefore, managed by the
                                                 * default memory manger)
                                                 */

-       /* boolean_t */         temporary:1,    /* Permanent objects may be
-                                                * changed externally by the 
-                                                * memory manager, and changes
-                                                * made in memory must be
-                                                * reflected back to the memory
-                                                * manager.  Temporary objects
-                                                * lack both of these
-                                                * characteristics.
-                                                */

        /* boolean_t */         private:1,      /* magic device_pager object,
                                                 * holds private pages only */
        /* boolean_t */         pageout:1,      /* pageout object. contains
        /* boolean_t */         private:1,      /* magic device_pager object,
                                                 * holds private pages only */
        /* boolean_t */         pageout:1,      /* pageout object. contains


@@ -221,17 +266,11 @@ struct vm_object {
        /* boolean_t */         purgable:2,     /* Purgable state.  See
                                                 * VM_PURGABLE_* 
                                                 */
        /* boolean_t */         purgable:2,     /* Purgable state.  See
                                                 * VM_PURGABLE_* 
                                                 */

+       /* boolean_t */         purgeable_only_by_kernel:1,
+       /* boolean_t */         purgeable_when_ripe:1, /* Purgeable when a token
+                                                       * becomes ripe.
+                                                       */

        /* boolean_t */         shadowed:1,     /* Shadow may exist */
        /* boolean_t */         shadowed:1,     /* Shadow may exist */

-       /* boolean_t */         silent_overwrite:1,
-                                               /* Allow full page overwrite
-                                                * without data_request if
-                                                * page is absent */
-       /* boolean_t */         advisory_pageout:1,
-                                               /* Instead of sending page
-                                                * via OOL, just notify
-                                                * pager that the kernel
-                                                * wants to discard it, page
-                                                * remains in object */

        /* boolean_t */         true_share:1,
                                                /* This object is mapped
                                                 * in more than one place
        /* boolean_t */         true_share:1,
                                                /* This object is mapped
                                                 * in more than one place


@@ -271,24 +310,19 @@ struct vm_object {
                                                 * memory rules w.r.t pmap
                                                 * access bits.
                                                 */
                                                 * memory rules w.r.t pmap
                                                 * access bits.
                                                 */

-       /* boolean_t */         nophyscache:1;
+       /* boolean_t */         nophyscache:1,

                                                /* When mapped at the 
                                                 * pmap level, don't allow
                                                 * primary caching. (for
                                                 * I/O)
                                                 */
                                                /* When mapped at the 
                                                 * pmap level, don't allow
                                                 * primary caching. (for
                                                 * I/O)
                                                 */

-                                               
-
+       /* boolean_t */         _object5_unused_bits:1;

 
        queue_chain_t           cached_list;    /* Attachment point for the
                                                 * list of objects cached as a
                                                 * result of their can_persist
                                                 * value
                                                 */
 
        queue_chain_t           cached_list;    /* Attachment point for the
                                                 * list of objects cached as a
                                                 * result of their can_persist
                                                 * value
                                                 */

-
-       queue_head_t            msr_q;          /* memory object synchronise
-                                                  request queue */
-

   /*
    * the following fields are not protected by any locks
    * they are updated via atomic compare and swap
   /*
    * the following fields are not protected by any locks
    * they are updated via atomic compare and swap


@@ -298,39 +332,45 @@ struct vm_object {
 
         uint32_t               pages_created;
         uint32_t               pages_used;
 
         uint32_t               pages_created;
         uint32_t               pages_used;

-#if    MACH_PAGEMAP
-       vm_external_map_t       existence_map;  /* bitmap of pages written to
-                                                * backing storage */
-#endif /* MACH_PAGEMAP */

        vm_offset_t             cow_hint;       /* last page present in     */
                                                /* shadow but not in object */
        vm_offset_t             cow_hint;       /* last page present in     */
                                                /* shadow but not in object */

-#if    MACH_ASSERT
-       struct vm_object        *paging_object; /* object which pages to be
-                                                * swapped out are temporary
-                                                * put in current object
-                                                */
-#endif

        /* hold object lock when altering */
        unsigned        int
                wimg_bits:8,            /* cache WIMG bits         */           
                code_signed:1,          /* pages are signed and should be
                                           validated; the signatures are stored
                                           with the pager */
        /* hold object lock when altering */
        unsigned        int
                wimg_bits:8,            /* cache WIMG bits         */           
                code_signed:1,          /* pages are signed and should be
                                           validated; the signatures are stored
                                           with the pager */

-               hashed:1,               /* object/pager entered in hash */

                transposed:1,           /* object was transposed with another */
                mapping_in_progress:1,  /* pager being mapped/unmapped */
                transposed:1,           /* object was transposed with another */
                mapping_in_progress:1,  /* pager being mapped/unmapped */

+               phantom_isssd:1,

                volatile_empty:1,
                volatile_fault:1,
                all_reusable:1,
                blocked_access:1,
                set_cache_attr:1,
                volatile_empty:1,
                volatile_fault:1,
                all_reusable:1,
                blocked_access:1,
                set_cache_attr:1,

-               __object2_unused_bits:15;       /* for expansion */
-
-       uint32_t                scan_collisions;
-
-#if    UPL_DEBUG
+               object_slid:1,
+               purgeable_queue_type:2,
+               purgeable_queue_group:3,
+               io_tracking:1,
+               no_tag_update:1,        /*  */
+#if CONFIG_SECLUDED_MEMORY
+               eligible_for_secluded:1,
+               can_grab_secluded:1,
+#else /* CONFIG_SECLUDED_MEMORY */
+               __object3_unused_bits:2,
+#endif /* CONFIG_SECLUDED_MEMORY */
+               __object2_unused_bits:5;        /* for expansion */
+
+       uint8_t                 scan_collisions;
+        vm_tag_t               wire_tag;
+       uint8_t                 __object4_unused_bits[2];
+
+#if CONFIG_PHANTOM_CACHE
+       uint32_t                phantom_object_id;
+#endif
+#if CONFIG_IOSCHED || UPL_DEBUG

        queue_head_t            uplq;           /* List of outstanding upls */
        queue_head_t            uplq;           /* List of outstanding upls */

-#endif /* UPL_DEBUG */
+#endif

 
 #ifdef VM_PIP_DEBUG
 /*
 
 #ifdef VM_PIP_DEBUG
 /*


@@ -345,6 +385,13 @@ struct vm_object {
 #endif /* VM_PIP_DEBUG  */
 
         queue_chain_t          objq;      /* object queue - currently used for purgable queues */
 #endif /* VM_PIP_DEBUG  */
 
         queue_chain_t          objq;      /* object queue - currently used for purgable queues */

+       queue_chain_t           task_objq; /* objects owned by task - protected by task lock */
+
+#if DEBUG
+       void *purgeable_owner_bt[16];
+       task_t vo_purgeable_volatilizer; /* who made it volatile? */
+       void *purgeable_volatilizer_bt[16];
+#endif /* DEBUG */

 };
 
 #define VM_OBJECT_PURGEABLE_FAULT_ERROR(object)                                \
 };
 
 #define VM_OBJECT_PURGEABLE_FAULT_ERROR(object)                                \


@@ -352,38 +399,13 @@ struct vm_object {
         ((object)->purgable == VM_PURGABLE_VOLATILE ||                 \
          (object)->purgable == VM_PURGABLE_EMPTY))
 
         ((object)->purgable == VM_PURGABLE_VOLATILE ||                 \
          (object)->purgable == VM_PURGABLE_EMPTY))
 

-#define VM_PAGE_REMOVE(page)                                           \
-       MACRO_BEGIN                                                     \
-       vm_page_t __page = (page);                                      \
-       vm_object_t __object = __page->object;                          \
-       if (__page == __object->memq_hint) {                            \
-               vm_page_t       __new_hint;                             \
-               queue_entry_t   __qe;                                   \
-               __qe = queue_next(&__page->listq);                      \
-               if (queue_end(&__object->memq, __qe)) {                 \
-                       __qe = queue_prev(&__page->listq);              \
-                       if (queue_end(&__object->memq, __qe)) {         \
-                               __qe = NULL;                            \
-                       }                                               \
-               }                                                       \
-               __new_hint = (vm_page_t) __qe;                          \
-               __object->memq_hint = __new_hint;                       \
-       }                                                               \
-       queue_remove(&__object->memq, __page, vm_page_t, listq);        \
-       MACRO_END
-
-#define VM_PAGE_INSERT(page, object)                           \
-       MACRO_BEGIN                                             \
-       vm_page_t __page = (page);                              \
-       vm_object_t __object = (object);                        \
-       queue_enter(&__object->memq, __page, vm_page_t, listq); \
-       __object->memq_hint = __page;                           \
-       MACRO_END
-
-__private_extern__
+extern

 vm_object_t    kernel_object;          /* the single kernel object */
 
 vm_object_t    kernel_object;          /* the single kernel object */
 

-__private_extern__
+extern
+vm_object_t    compressor_object;      /* the single compressor object */
+
+extern

 unsigned int   vm_object_absent_max;   /* maximum number of absent pages
                                           at a time for each object */
 
 unsigned int   vm_object_absent_max;   /* maximum number of absent pages
                                           at a time for each object */
 


@@ -391,44 +413,151 @@ unsigned int     vm_object_absent_max;   /* maximum number of absent pages
 # define       VM_MSYNC_SYNCHRONIZING                  1
 # define       VM_MSYNC_DONE                           2
 
 # define       VM_MSYNC_SYNCHRONIZING                  1
 # define       VM_MSYNC_DONE                           2
 

-struct msync_req {
-       queue_chain_t           msr_q;          /* object request queue */
-       queue_chain_t           req_q;          /* vm_msync request queue */
-       unsigned int            flag;
-       vm_object_offset_t      offset;
-       vm_object_size_t        length;
-       vm_object_t             object;         /* back pointer */
-       decl_lck_mtx_data(,     msync_req_lock) /* Lock for this structure */
-};
-
-typedef struct msync_req       *msync_req_t;
-#define MSYNC_REQ_NULL         ((msync_req_t) 0)
-

 
 extern lck_grp_t               vm_map_lck_grp;
 extern lck_attr_t              vm_map_lck_attr;
 
 
 extern lck_grp_t               vm_map_lck_grp;
 extern lck_attr_t              vm_map_lck_attr;
 

+#ifndef VM_TAG_ACTIVE_UPDATE
+#error VM_TAG_ACTIVE_UPDATE
+#endif
+
+#define VM_OBJECT_WIRED(object, tag)                                   \
+    MACRO_BEGIN                                                                \
+    assert(VM_KERN_MEMORY_NONE != (tag));                              \
+    assert(VM_KERN_MEMORY_NONE == (object)->wire_tag);                 \
+    (object)->wire_tag = (tag);                                        \
+    if (!VM_TAG_ACTIVE_UPDATE                                          \
+       && ((object)->purgable == VM_PURGABLE_DENY))                    \
+    {                                                                  \
+       lck_spin_lock(&vm_objects_wired_lock);                          \
+       assert(!(object)->objq.next);                                   \
+       assert(!(object)->objq.prev);                                   \
+       queue_enter(&vm_objects_wired, (object), vm_object_t, objq);    \
+       lck_spin_unlock(&vm_objects_wired_lock);                        \
+    }                                                                  \
+    MACRO_END
+
+#define VM_OBJECT_UNWIRED(object)                                                      \
+    MACRO_BEGIN                                                                                \
+    if (!VM_TAG_ACTIVE_UPDATE                                                          \
+       && ((object)->purgable == VM_PURGABLE_DENY) && (object)->objq.next)             \
+    {                                                                                  \
+       lck_spin_lock(&vm_objects_wired_lock);                                          \
+       queue_remove(&vm_objects_wired, (object), vm_object_t, objq);                   \
+       lck_spin_unlock(&vm_objects_wired_lock);                                        \
+    }                                                                                  \
+    if (VM_KERN_MEMORY_NONE != (object)->wire_tag) {                                   \
+       vm_tag_update_size((object)->wire_tag, -ptoa_64((object)->wired_page_count));   \
+       (object)->wire_tag = VM_KERN_MEMORY_NONE;                                       \
+    }                                                                                  \
+    MACRO_END
+
+// These two macros start & end a C block
+#define VM_OBJECT_WIRED_PAGE_UPDATE_START(object)                                      \
+    MACRO_BEGIN                                                                                \
+    {                                                                                  \
+       int64_t __wireddelta = 0; vm_tag_t __waswired = (object)->wire_tag;
+
+#define VM_OBJECT_WIRED_PAGE_UPDATE_END(object, tag)                                   \
+       if (__wireddelta) {                                                             \
+           boolean_t __overflow __assert_only =                                        \
+           os_add_overflow((object)->wired_page_count, __wireddelta,                   \
+                           (unsigned int *)(uintptr_t)&(object)->wired_page_count);    \
+           assert(!__overflow);                                                        \
+           if (!(object)->pageout && !(object)->no_tag_update) {                       \
+               if (__wireddelta > 0) {                                                 \
+                   assert (VM_KERN_MEMORY_NONE != (tag));                              \
+                   if (VM_KERN_MEMORY_NONE == __waswired) {                            \
+                       VM_OBJECT_WIRED((object), (tag));                               \
+                   }                                                                   \
+                   vm_tag_update_size((object)->wire_tag, ptoa_64(__wireddelta));      \
+               } else if (VM_KERN_MEMORY_NONE != __waswired) {                         \
+                   assert (VM_KERN_MEMORY_NONE != (object)->wire_tag);                 \
+                   vm_tag_update_size((object)->wire_tag, ptoa_64(__wireddelta));      \
+                   if (!(object)->wired_page_count) {                                  \
+                       VM_OBJECT_UNWIRED((object));                                    \
+                   }                                                                   \
+               }                                                                       \
+           }                                                                           \
+       }                                                                               \
+    }                                                                                  \
+    MACRO_END
+
+#define VM_OBJECT_WIRED_PAGE_COUNT(object, delta)               \
+    __wireddelta += delta; \
+
+#define VM_OBJECT_WIRED_PAGE_ADD(object, m)                     \
+    if (!m->private && !m->fictitious) __wireddelta++;
+
+#define VM_OBJECT_WIRED_PAGE_REMOVE(object, m)                  \
+    if (!m->private && !m->fictitious) __wireddelta--;
+
+
+
+#define OBJECT_LOCK_SHARED     0
+#define OBJECT_LOCK_EXCLUSIVE  1
+
+extern lck_grp_t       vm_object_lck_grp;
+extern lck_grp_attr_t  vm_object_lck_grp_attr;
+extern lck_attr_t      vm_object_lck_attr;
+extern lck_attr_t      kernel_object_lck_attr;
+extern lck_attr_t      compressor_object_lck_attr;
+
+extern vm_object_t     vm_pageout_scan_wants_object;
+
+extern void            vm_object_lock(vm_object_t);
+extern boolean_t       vm_object_lock_try(vm_object_t);
+extern boolean_t       _vm_object_lock_try(vm_object_t);
+extern boolean_t       vm_object_lock_avoid(vm_object_t);
+extern void            vm_object_lock_shared(vm_object_t);
+extern boolean_t       vm_object_lock_yield_shared(vm_object_t);
+extern boolean_t       vm_object_lock_try_shared(vm_object_t);
+extern void            vm_object_unlock(vm_object_t);
+extern boolean_t       vm_object_lock_upgrade(vm_object_t);
+

 /*
 /*

- * Macros to allocate and free msync_reqs
+ *     Object locking macros

  */
  */

-#define msync_req_alloc(msr)                                           \
-    MACRO_BEGIN                                                        \
-        (msr) = (msync_req_t)kalloc(sizeof(struct msync_req));         \
-        lck_mtx_init(&(msr)->msync_req_lock, &vm_map_lck_grp, &vm_map_lck_attr);               \
-        msr->flag = VM_MSYNC_INITIALIZED;                              \
-    MACRO_END

 
 

-#define msync_req_free(msr)                                            \
-       (kfree((msr), sizeof(struct msync_req)))
+#define vm_object_lock_init(object)                                    \
+       lck_rw_init(&(object)->Lock, &vm_object_lck_grp,                \
+                   (((object) == kernel_object ||                      \
+                     (object) == vm_submap_object) ?                   \
+                    &kernel_object_lck_attr :                          \
+                    (((object) == compressor_object) ?                 \
+                    &compressor_object_lck_attr :                      \
+                     &vm_object_lck_attr)))
+#define vm_object_lock_destroy(object) lck_rw_destroy(&(object)->Lock, &vm_object_lck_grp)
+
+#define vm_object_lock_try_scan(object)        _vm_object_lock_try(object)
+
+/*
+ * CAUTION: the following vm_object_lock_assert_held*() macros merely
+ * check if anyone is holding the lock, but the holder may not necessarily
+ * be the caller...
+ */
+#if MACH_ASSERT || DEBUG
+#define vm_object_lock_assert_held(object) \
+       lck_rw_assert(&(object)->Lock, LCK_RW_ASSERT_HELD)
+#define vm_object_lock_assert_shared(object) \
+       lck_rw_assert(&(object)->Lock, LCK_RW_ASSERT_SHARED)
+#define vm_object_lock_assert_exclusive(object) \
+       lck_rw_assert(&(object)->Lock, LCK_RW_ASSERT_EXCLUSIVE)
+#define vm_object_lock_assert_notheld(object) \
+       lck_rw_assert(&(object)->Lock, LCK_RW_ASSERT_NOTHELD)
+#else  /* MACH_ASSERT || DEBUG */ 
+#define vm_object_lock_assert_held(object)
+#define vm_object_lock_assert_shared(object)
+#define vm_object_lock_assert_exclusive(object)
+#define vm_object_lock_assert_notheld(object)
+#endif /* MACH_ASSERT || DEBUG */

 
 

-#define msr_lock(msr)   lck_mtx_lock(&(msr)->msync_req_lock)
-#define msr_unlock(msr) lck_mtx_unlock(&(msr)->msync_req_lock)

 
 /*
  *     Declare procedures that operate on VM objects.
  */
 
 
 /*
  *     Declare procedures that operate on VM objects.
  */
 

-__private_extern__ void                vm_object_bootstrap(void) __attribute__((section("__TEXT, initcode")));
+__private_extern__ void                vm_object_bootstrap(void);

 
 __private_extern__ void                vm_object_init(void);
 
 
 __private_extern__ void                vm_object_init(void);
 


@@ -436,8 +565,7 @@ __private_extern__ void             vm_object_init_lck_grp(void);
 
 __private_extern__ void                vm_object_reaper_init(void);
 
 
 __private_extern__ void                vm_object_reaper_init(void);
 

-__private_extern__ vm_object_t vm_object_allocate(
-                                       vm_object_size_t        size);
+__private_extern__ vm_object_t vm_object_allocate(vm_object_size_t size);

 
 __private_extern__ void    _vm_object_allocate(vm_object_size_t size,
                            vm_object_t object);
 
 __private_extern__ void    _vm_object_allocate(vm_object_size_t size,
                            vm_object_t object);


@@ -477,7 +605,7 @@ __private_extern__ void     vm_object_res_deallocate(
        vm_object_lock_assert_shared(object);                           \
        assert((RLObject)->ref_count > 0);                              \
        OSAddAtomic(1, &(RLObject)->ref_count);         \
        vm_object_lock_assert_shared(object);                           \
        assert((RLObject)->ref_count > 0);                              \
        OSAddAtomic(1, &(RLObject)->ref_count);         \

-       assert((RLObject)->ref_count > 1);                              \
+       assert((RLObject)->ref_count > 0);                              \

        /* XXX we would need an atomic version of the following ... */  \
        vm_object_res_reference(RLObject);                              \
        MACRO_END
        /* XXX we would need an atomic version of the following ... */  \
        vm_object_res_reference(RLObject);                              \
        MACRO_END


@@ -515,6 +643,15 @@ __private_extern__ void            vm_object_pmap_protect(
                                        vm_map_offset_t         pmap_start,
                                        vm_prot_t               prot);
 
                                        vm_map_offset_t         pmap_start,
                                        vm_prot_t               prot);
 

+__private_extern__ void                vm_object_pmap_protect_options(
+                                       vm_object_t             object,
+                                       vm_object_offset_t      offset,
+                                       vm_object_size_t        size,
+                                       pmap_t                  pmap,
+                                       vm_map_offset_t         pmap_start,
+                                       vm_prot_t               prot,
+                                       int                     options);
+

 __private_extern__ void                vm_object_page_remove(
                                        vm_object_t             object,
                                        vm_object_offset_t      start,
 __private_extern__ void                vm_object_page_remove(
                                        vm_object_t             object,
                                        vm_object_offset_t      start,


@@ -525,7 +662,9 @@ __private_extern__ void             vm_object_deactivate_pages(
                                        vm_object_offset_t      offset,
                                        vm_object_size_t        size,
                                        boolean_t               kill_page,
                                        vm_object_offset_t      offset,
                                        vm_object_size_t        size,
                                        boolean_t               kill_page,

-                                       boolean_t               reusable_page);
+                                       boolean_t               reusable_page,
+                                       struct pmap             *pmap,
+                                       vm_map_offset_t         pmap_offset);

 
 __private_extern__ void        vm_object_reuse_pages(
        vm_object_t             object,
 
 __private_extern__ void        vm_object_reuse_pages(
        vm_object_t             object,


@@ -533,14 +672,22 @@ __private_extern__ void   vm_object_reuse_pages(
        vm_object_offset_t      end_offset,
        boolean_t               allow_partial_reuse);
 
        vm_object_offset_t      end_offset,
        boolean_t               allow_partial_reuse);
 

-__private_extern__ void                vm_object_purge(
-                                       vm_object_t             object);
+__private_extern__ uint64_t    vm_object_purge(
+                                      vm_object_t              object,
+                                      int                      flags);

 
 __private_extern__ kern_return_t vm_object_purgable_control(
        vm_object_t     object,
        vm_purgable_t   control,
        int             *state);
 
 
 __private_extern__ kern_return_t vm_object_purgable_control(
        vm_object_t     object,
        vm_purgable_t   control,
        int             *state);
 

+__private_extern__ kern_return_t vm_object_get_page_counts(
+       vm_object_t             object,
+       vm_object_offset_t      offset,
+       vm_object_size_t        size,
+       unsigned int            *resident_page_count,
+       unsigned int            *dirty_page_count);
+

 __private_extern__ boolean_t   vm_object_coalesce(
                                        vm_object_t             prev_object,
                                        vm_object_t             next_object,
 __private_extern__ boolean_t   vm_object_coalesce(
                                        vm_object_t             prev_object,
                                        vm_object_t             next_object,


@@ -596,6 +743,9 @@ __private_extern__ kern_return_t    vm_object_destroy(
 __private_extern__ void                vm_object_pager_create(
                                        vm_object_t     object);
 
 __private_extern__ void                vm_object_pager_create(
                                        vm_object_t     object);
 

+__private_extern__ void                vm_object_compressor_pager_create(
+                                       vm_object_t     object);
+

 __private_extern__ void                vm_object_page_map(
                                vm_object_t     object,
                                vm_object_offset_t      offset,
 __private_extern__ void                vm_object_page_map(
                                vm_object_t     object,
                                vm_object_offset_t      offset,


@@ -611,7 +761,8 @@ __private_extern__ kern_return_t vm_object_upl_request(
                                upl_t                   *upl,
                                upl_page_info_t         *page_info,
                                unsigned int            *count,
                                upl_t                   *upl,
                                upl_page_info_t         *page_info,
                                unsigned int            *count,

-                               int                     flags);
+                               upl_control_flags_t     flags,
+                               vm_tag_t            tag);

 
 __private_extern__ kern_return_t vm_object_transpose(
                                vm_object_t             object1,
 
 __private_extern__ kern_return_t vm_object_transpose(
                                vm_object_t             object1,


@@ -646,11 +797,10 @@ __private_extern__ kern_return_t vm_object_lock_request(
 
 
 
 
 
 

-__private_extern__ vm_object_t vm_object_enter(
+__private_extern__ vm_object_t vm_object_memory_object_associate(

                                        memory_object_t         pager,
                                        memory_object_t         pager,

+                                       vm_object_t             object,

                                        vm_object_size_t        size,
                                        vm_object_size_t        size,

-                                       boolean_t               internal,
-                                       boolean_t               init,

                                        boolean_t               check_named);
 
 
                                        boolean_t               check_named);
 
 


@@ -700,40 +850,33 @@ __private_extern__ void           vm_object_reap_pages(
 
 #if CONFIG_FREEZE
 
 
 #if CONFIG_FREEZE
 

-__private_extern__ kern_return_t 
-vm_object_pack(
-       unsigned int       *purgeable_count,
-       unsigned int       *wired_count,
-       unsigned int       *clean_count,
-       unsigned int       *dirty_count,
-       boolean_t          *shared,
-       vm_object_t         src_object,
-       vm_object_t         dst_object,
-       void                  **table,
-       vm_object_offset_t *offset);
-

 __private_extern__ void
 __private_extern__ void

-vm_object_pack_pages(
-       unsigned int       *wired_count,
-       unsigned int       *clean_count,
-       unsigned int       *dirty_count,
-       vm_object_t         src_object,
-       vm_object_t         dst_object,
-       void                  **table,
-       vm_object_offset_t *offset);
-
-__private_extern__ void vm_object_pageout(
-    vm_object_t     object);
-
-__private_extern__  kern_return_t vm_object_pagein(
+vm_object_compressed_freezer_pageout(

        vm_object_t     object);
 
        vm_object_t     object);
 

-__private_extern__ void vm_object_unpack(
-       vm_object_t     object,
-       void          **table);
+__private_extern__ void
+vm_object_compressed_freezer_done(
+       void);

 
 #endif /* CONFIG_FREEZE */
 
 
 #endif /* CONFIG_FREEZE */
 

+__private_extern__ void
+vm_object_pageout(
+       vm_object_t     object);
+
+#if CONFIG_IOSCHED
+struct io_reprioritize_req {
+       uint64_t        blkno;
+       uint32_t        len;
+       int             priority;
+       struct vnode    *devvp;
+       queue_chain_t   io_reprioritize_list;
+};
+typedef struct io_reprioritize_req *io_reprioritize_req_t;
+
+extern void vm_io_reprioritize_init(void);
+#endif
+

 /*
  *     Event waiting handling
  */
 /*
  *     Event waiting handling
  */


@@ -749,37 +892,114 @@ __private_extern__ void vm_object_unpack(
 #define VM_OBJECT_EVENT_UNBLOCKED              8
 #define VM_OBJECT_EVENT_PAGING_ONLY_IN_PROGRESS        9
 
 #define VM_OBJECT_EVENT_UNBLOCKED              8
 #define VM_OBJECT_EVENT_PAGING_ONLY_IN_PROGRESS        9
 

-#define        vm_object_assert_wait(object, event, interruptible)             \
-       (((object)->all_wanted |= 1 << (event)),                        \
-        assert_wait((event_t)((vm_offset_t)(object)+(event)),(interruptible)))
-
-#define        vm_object_wait(object, event, interruptible)                    \
-       (vm_object_assert_wait((object),(event),(interruptible)),       \
-       vm_object_unlock(object),                                       \
-       thread_block(THREAD_CONTINUE_NULL))                             \
+#define VM_OBJECT_EVENT_MAX 10 /* 11 bits in "all_wanted", so 0->10 */

 
 

-#define thread_sleep_vm_object(object, event, interruptible)           \
-        lck_rw_sleep(&(object)->Lock, LCK_SLEEP_DEFAULT, (event_t)(event), (interruptible))
+static __inline__ wait_result_t
+vm_object_assert_wait(
+       vm_object_t             object,
+       int                     event,
+       wait_interrupt_t        interruptible)
+{
+       wait_result_t wr;
+
+       vm_object_lock_assert_exclusive(object);
+       assert(event >= 0 && event <= VM_OBJECT_EVENT_MAX);
+
+       object->all_wanted |= 1 << event;
+       wr = assert_wait((event_t)((vm_offset_t)object + event),
+                        interruptible);
+       return wr;
+}
+
+static __inline__ wait_result_t
+vm_object_wait(
+       vm_object_t             object,
+       int                     event,
+       wait_interrupt_t        interruptible)
+{
+       wait_result_t wr;
+
+       vm_object_assert_wait(object, event, interruptible);
+       vm_object_unlock(object);
+       wr = thread_block(THREAD_CONTINUE_NULL);
+       return wr;
+}
+
+static __inline__ wait_result_t
+thread_sleep_vm_object(
+       vm_object_t             object,
+       event_t                 event,
+       wait_interrupt_t        interruptible)
+{
+       wait_result_t wr;
+
+#if DEVELOPMENT || DEBUG
+       if (object->Lock_owner != current_thread())
+               panic("thread_sleep_vm_object: now owner - %p\n", object);
+       object->Lock_owner = 0;
+#endif
+        wr = lck_rw_sleep(&object->Lock,
+                         LCK_SLEEP_PROMOTED_PRI,
+                         event,
+                         interruptible);
+#if DEVELOPMENT || DEBUG
+       object->Lock_owner = current_thread();
+#endif
+       return wr;
+}

 
 

-#define vm_object_sleep(object, event, interruptible)                  \
-       (((object)->all_wanted |= 1 << (event)),                        \
-        thread_sleep_vm_object((object),                               \
-               ((vm_offset_t)(object)+(event)), (interruptible)))
+static __inline__ wait_result_t
+vm_object_sleep(
+       vm_object_t             object,
+       int                     event,
+       wait_interrupt_t        interruptible)
+{
+       wait_result_t wr;
+
+       vm_object_lock_assert_exclusive(object);
+       assert(event >= 0 && event <= VM_OBJECT_EVENT_MAX);
+
+       object->all_wanted |= 1 << event;
+       wr = thread_sleep_vm_object(object,
+                                   (event_t)((vm_offset_t)object + event),
+                                   interruptible);
+       return wr;
+}
+
+static __inline__ void
+vm_object_wakeup(
+       vm_object_t             object,
+       int                     event)
+{
+       vm_object_lock_assert_exclusive(object);
+       assert(event >= 0 && event <= VM_OBJECT_EVENT_MAX);
+
+       if (object->all_wanted & (1 << event))
+               thread_wakeup((event_t)((vm_offset_t)object + event));
+       object->all_wanted &= ~(1 << event);
+}
+
+static __inline__ void
+vm_object_set_wanted(
+       vm_object_t             object,
+       int                     event)
+{
+       vm_object_lock_assert_exclusive(object);
+       assert(event >= 0 && event <= VM_OBJECT_EVENT_MAX);

 
 

-#define        vm_object_wakeup(object, event)                                 \
-       MACRO_BEGIN                                                     \
-       if ((object)->all_wanted & (1 << (event)))                      \
-               thread_wakeup((event_t)((vm_offset_t)(object) + (event))); \
-       (object)->all_wanted &= ~(1 << (event));                        \
-       MACRO_END
+       object->all_wanted |= (1 << event);
+}

 
 

-#define        vm_object_set_wanted(object, event)                             \
-       MACRO_BEGIN                                                     \
-       ((object)->all_wanted |= (1 << (event)));                       \
-       MACRO_END
+static __inline__ int
+vm_object_wanted(
+       vm_object_t             object,
+       int                     event)
+{
+       vm_object_lock_assert_held(object);
+       assert(event >= 0 && event <= VM_OBJECT_EVENT_MAX);

 
 

-#define        vm_object_wanted(object, event)                                 \
-       ((object)->all_wanted & (1 << (event)))
+       return object->all_wanted & (1 << event);
+}

 
 /*
  *     Routines implemented as macros
 
 /*
  *     Routines implemented as macros


@@ -802,15 +1022,19 @@ __private_extern__ void vm_object_unpack(
 #define                vm_object_activity_begin(object)                        \
        MACRO_BEGIN                                                     \
        vm_object_lock_assert_exclusive((object));                      \
 #define                vm_object_activity_begin(object)                        \
        MACRO_BEGIN                                                     \
        vm_object_lock_assert_exclusive((object));                      \

-       assert((object)->paging_in_progress >= 0);                      \

        VM_PIP_DEBUG_BEGIN((object));                                   \
        (object)->activity_in_progress++;                               \
        VM_PIP_DEBUG_BEGIN((object));                                   \
        (object)->activity_in_progress++;                               \

+       if ((object)->activity_in_progress == 0) {                      \
+               panic("vm_object_activity_begin(%p): overflow\n", (object));\
+       }                                                               \

        MACRO_END
 
 #define                vm_object_activity_end(object)                          \
        MACRO_BEGIN                                                     \
        vm_object_lock_assert_exclusive((object));                      \
        MACRO_END
 
 #define                vm_object_activity_end(object)                          \
        MACRO_BEGIN                                                     \
        vm_object_lock_assert_exclusive((object));                      \

-       assert((object)->activity_in_progress > 0);                     \
+       if ((object)->activity_in_progress == 0) {                      \
+               panic("vm_object_activity_end(%p): underflow\n", (object));\
+       }                                                               \

        (object)->activity_in_progress--;                               \
        if ((object)->paging_in_progress == 0 &&                        \
            (object)->activity_in_progress == 0)                        \
        (object)->activity_in_progress--;                               \
        if ((object)->paging_in_progress == 0 &&                        \
            (object)->activity_in_progress == 0)                        \


@@ -821,15 +1045,19 @@ __private_extern__ void vm_object_unpack(
 #define                vm_object_paging_begin(object)                          \
        MACRO_BEGIN                                                     \
        vm_object_lock_assert_exclusive((object));                      \
 #define                vm_object_paging_begin(object)                          \
        MACRO_BEGIN                                                     \
        vm_object_lock_assert_exclusive((object));                      \

-       assert((object)->paging_in_progress >= 0);                      \

        VM_PIP_DEBUG_BEGIN((object));                                   \
        (object)->paging_in_progress++;                                 \
        VM_PIP_DEBUG_BEGIN((object));                                   \
        (object)->paging_in_progress++;                                 \

+       if ((object)->paging_in_progress == 0) {                        \
+               panic("vm_object_paging_begin(%p): overflow\n", (object));\
+       }                                                               \

        MACRO_END
 
 #define                vm_object_paging_end(object)                            \
        MACRO_BEGIN                                                     \
        vm_object_lock_assert_exclusive((object));                      \
        MACRO_END
 
 #define                vm_object_paging_end(object)                            \
        MACRO_BEGIN                                                     \
        vm_object_lock_assert_exclusive((object));                      \

-       assert((object)->paging_in_progress > 0);                       \
+       if ((object)->paging_in_progress == 0) {                        \
+               panic("vm_object_paging_end(%p): underflow\n", (object));\
+       }                                                               \

        (object)->paging_in_progress--;                                 \
        if ((object)->paging_in_progress == 0) {                        \
                vm_object_wakeup((object),                              \
        (object)->paging_in_progress--;                                 \
        if ((object)->paging_in_progress == 0) {                        \
                vm_object_wakeup((object),                              \


@@ -905,57 +1133,6 @@ __private_extern__ void vm_object_unpack(
 
 
 
 
 
 

-#define OBJECT_LOCK_SHARED     0
-#define OBJECT_LOCK_EXCLUSIVE  1
-
-extern lck_grp_t       vm_object_lck_grp;
-extern lck_grp_attr_t  vm_object_lck_grp_attr;
-extern lck_attr_t      vm_object_lck_attr;
-extern lck_attr_t      kernel_object_lck_attr;
-
-extern vm_object_t     vm_pageout_scan_wants_object;
-
-extern void            vm_object_lock(vm_object_t);
-extern boolean_t       vm_object_lock_try(vm_object_t);
-extern boolean_t       _vm_object_lock_try(vm_object_t);
-extern boolean_t       vm_object_lock_avoid(vm_object_t);
-extern void            vm_object_lock_shared(vm_object_t);
-extern boolean_t       vm_object_lock_try_shared(vm_object_t);
-
-/*
- *     Object locking macros
- */
-
-#define vm_object_lock_init(object)                                    \
-       lck_rw_init(&(object)->Lock, &vm_object_lck_grp,                \
-                   (((object) == kernel_object ||                      \
-                     (object) == vm_submap_object) ?                   \
-                    &kernel_object_lck_attr :                          \
-                    &vm_object_lck_attr))
-#define vm_object_lock_destroy(object) lck_rw_destroy(&(object)->Lock, &vm_object_lck_grp)
-
-#define vm_object_unlock(object)       lck_rw_done(&(object)->Lock)
-#define vm_object_lock_upgrade(object) lck_rw_lock_shared_to_exclusive(&(object)->Lock)
-#define vm_object_lock_try_scan(object)        _vm_object_lock_try(object)
-
-/*
- * CAUTION: the following vm_object_lock_assert_held*() macros merely
- * check if anyone is holding the lock, but the holder may not necessarily
- * be the caller...
- */
-#if MACH_ASSERT || DEBUG
-#define vm_object_lock_assert_held(object) \
-       lck_rw_assert(&(object)->Lock, LCK_RW_ASSERT_HELD)
-#define vm_object_lock_assert_shared(object)   \
-       lck_rw_assert(&(object)->Lock, LCK_RW_ASSERT_SHARED)
-#define vm_object_lock_assert_exclusive(object) \
-       lck_rw_assert(&(object)->Lock, LCK_RW_ASSERT_EXCLUSIVE)
-#else  /* MACH_ASSERT || DEBUG */ 
-#define vm_object_lock_assert_held(object)
-#define vm_object_lock_assert_shared(object)
-#define vm_object_lock_assert_exclusive(object)
-#endif /* MACH_ASSERT || DEBUG */
-

 #define vm_object_round_page(x) (((vm_object_offset_t)(x) + PAGE_MASK) & ~((signed)PAGE_MASK))
 #define vm_object_trunc_page(x) ((vm_object_offset_t)(x) & ~((signed)PAGE_MASK))
 
 #define vm_object_round_page(x) (((vm_object_offset_t)(x) + PAGE_MASK) & ~((signed)PAGE_MASK))
 #define vm_object_trunc_page(x) ((vm_object_offset_t)(x) & ~((signed)PAGE_MASK))