xnu-792.13.8.tar.gz

[apple/xnu.git] / osfmk / vm / vm_object.h

diff --git a/osfmk/vm/vm_object.h b/osfmk/vm/vm_object.h
index dbfc2a85ec3a91b82acbe68be64bb57b0e8b16a8..c5507c29c303242786db0f4bcbbf765a102fe666 100644 (file)
--- a/osfmk/vm/vm_object.h
+++ b/osfmk/vm/vm_object.h
@@ -1,26 +1,31 @@
 /*
 /*

- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.

  *
  *

- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_LICENSE_OSREFERENCE_HEADER_START@

  * 
  * 

- * Copyright (c) 1999-2003 Apple Computer, Inc.  All Rights Reserved.
- * 
- * This file contains Original Code and/or Modifications of Original Code
- * as defined in and that are subject to the Apple Public Source License
- * Version 2.0 (the 'License'). You may not use this file except in
- * compliance with the License. Please obtain a copy of the License at
- * http://www.opensource.apple.com/apsl/ and read it before using this
+ * This file contains Original Code and/or Modifications of Original Code 
+ * as defined in and that are subject to the Apple Public Source License 
+ * Version 2.0 (the 'License'). You may not use this file except in 
+ * compliance with the License.  The rights granted to you under the 
+ * License may not be used to create, or enable the creation or 
+ * redistribution of, unlawful or unlicensed copies of an Apple operating 
+ * system, or to circumvent, violate, or enable the circumvention or 
+ * violation of, any terms of an Apple operating system software license 
+ * agreement.
+ *
+ * Please obtain a copy of the License at 
+ * http://www.opensource.apple.com/apsl/ and read it before using this 

  * file.
  * file.

- * 
- * The Original Code and all software distributed under the License are
- * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
- * Please see the License for the specific language governing rights and
+ *
+ * The Original Code and all software distributed under the License are 
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 
+ * Please see the License for the specific language governing rights and 

  * limitations under the License.
  * limitations under the License.

- * 
- * @APPLE_LICENSE_HEADER_END@
+ *
+ * @APPLE_LICENSE_OSREFERENCE_HEADER_END@

  */
 /*
  * @OSF_COPYRIGHT@
  */
 /*
  * @OSF_COPYRIGHT@


@@ -71,11 +76,11 @@
 #include <mach/memory_object_types.h>
 #include <mach/port.h>
 #include <mach/vm_prot.h>
 #include <mach/memory_object_types.h>
 #include <mach/port.h>
 #include <mach/vm_prot.h>

+#include <mach/vm_param.h>

 #include <mach/machine/vm_types.h>
 #include <kern/queue.h>
 #include <kern/lock.h>
 #include <kern/assert.h>
 #include <mach/machine/vm_types.h>
 #include <kern/queue.h>
 #include <kern/lock.h>
 #include <kern/assert.h>

-#include <kern/ipc_mig.h>

 #include <kern/misc_protos.h>
 #include <kern/macro_help.h>
 #include <ipc/ipc_types.h>
 #include <kern/misc_protos.h>
 #include <kern/macro_help.h>
 #include <ipc/ipc_types.h>


@@ -85,8 +90,7 @@
 #include <vm/vm_external.h>
 #endif /* MACH_PAGEMAP */
 
 #include <vm/vm_external.h>
 #endif /* MACH_PAGEMAP */
 

-typedef memory_object_control_t        pager_request_t;
-#define        PAGER_REQUEST_NULL      ((pager_request_t) 0)
+struct vm_page;

 
 /*
  *     Types defined:
 
 /*
  *     Types defined:


@@ -94,9 +98,6 @@ typedef memory_object_control_t       pager_request_t;
  *     vm_object_t             Virtual memory object.
  */
 
  *     vm_object_t             Virtual memory object.
  */
 

-typedef unsigned long long vm_object_size_t;
-
-

 struct vm_object {
        queue_head_t            memq;           /* Resident memory */
        decl_mutex_data(,       Lock)           /* Synchronization */
 struct vm_object {
        queue_head_t            memq;           /* Resident memory */
        decl_mutex_data(,       Lock)           /* Synchronization */


@@ -104,10 +105,7 @@ struct vm_object {
        vm_object_size_t        size;           /* Object size (only valid
                                                 * if internal)
                                                 */
        vm_object_size_t        size;           /* Object size (only valid
                                                 * if internal)
                                                 */

-       vm_object_size_t        frozen_size;    /* How much has been marked
-                                                * copy-on-write (only
-                                                * valid if copy_symmetric)
-                                                */
+       struct vm_page          *memq_hint;

        int                     ref_count;      /* Number of references */
 #if    TASK_SWAPPER
        int                     res_count;      /* Residency references (swap)*/
        int                     ref_count;      /* Number of references */
 #if    TASK_SWAPPER
        int                     res_count;      /* Residency references (swap)*/


@@ -127,7 +125,7 @@ struct vm_object {
 
        memory_object_t         pager;          /* Where to get data */
        vm_object_offset_t      paging_offset;  /* Offset into memory object */
 
        memory_object_t         pager;          /* Where to get data */
        vm_object_offset_t      paging_offset;  /* Offset into memory object */

-       pager_request_t         pager_request;  /* Where data comes back */
+       memory_object_control_t pager_control;  /* Where data comes back */

 
        memory_object_copy_strategy_t
                                copy_strategy;  /* How to handle data copy */
 
        memory_object_copy_strategy_t
                                copy_strategy;  /* How to handle data copy */


@@ -140,7 +138,7 @@ struct vm_object {
                                                 * asserted.
                                                 */
 
                                                 * asserted.
                                                 */
 

-       unsigned int            paging_in_progress;
+       int                     paging_in_progress;

                                                /* The memory object ports are
                                                 * being used (e.g., for pagein
                                                 * or pageout) -- don't change
                                                /* The memory object ports are
                                                 * being used (e.g., for pagein
                                                 * or pageout) -- don't change


@@ -188,13 +186,9 @@ struct vm_object {
                                                 * a real memory object. */
        /* boolean_t */         alive:1,        /* Not yet terminated */
 
                                                 * a real memory object. */
        /* boolean_t */         alive:1,        /* Not yet terminated */
 

-       /* boolean_t */         lock_in_progress:1,
-                                               /* Is a multi-page lock
-                                                * request in progress?
-                                                */
-       /* boolean_t */         lock_restart:1,
-                                               /* Should lock request in
-                                                * progress restart search?
+       /* boolean_t */         purgable:2,     /* Purgable state.  See
+                                                * VM_OBJECT_PURGABLE_*
+                                                * items below.

                                                 */
        /* boolean_t */         shadowed:1,     /* Shadow may exist */
        /* boolean_t */         silent_overwrite:1,
                                                 */
        /* boolean_t */         shadowed:1,     /* Shadow may exist */
        /* boolean_t */         silent_overwrite:1,


@@ -265,12 +259,13 @@ struct vm_object {
                                                   request queue */
 
        vm_object_offset_t      last_alloc;     /* last allocation offset */
                                                   request queue */
 
        vm_object_offset_t      last_alloc;     /* last allocation offset */

+       vm_object_offset_t      sequential;     /* sequential access size */

        vm_size_t               cluster_size;   /* size of paging cluster */
 #if    MACH_PAGEMAP
        vm_external_map_t       existence_map;  /* bitmap of pages written to
                                                 * backing storage */
 #endif /* MACH_PAGEMAP */
        vm_size_t               cluster_size;   /* size of paging cluster */
 #if    MACH_PAGEMAP
        vm_external_map_t       existence_map;  /* bitmap of pages written to
                                                 * backing storage */
 #endif /* MACH_PAGEMAP */

-       int                     cow_hint;       /* last page present in     */
+       vm_offset_t             cow_hint;       /* last page present in     */

                                                /* shadow but not in object */
 #if    MACH_ASSERT
        struct vm_object        *paging_object; /* object which pages to be
                                                /* shadow but not in object */
 #if    MACH_ASSERT
        struct vm_object        *paging_object; /* object which pages to be


@@ -282,16 +277,55 @@ struct vm_object {
        unsigned        int                     /* cache WIMG bits         */           
                        wimg_bits:8,            /* wimg plus some expansion*/
                        not_in_use:24;
        unsigned        int                     /* cache WIMG bits         */           
                        wimg_bits:8,            /* wimg plus some expansion*/
                        not_in_use:24;

-#ifdef UBC_DEBUG
+#ifdef UPL_DEBUG

        queue_head_t            uplq;           /* List of outstanding upls */
        queue_head_t            uplq;           /* List of outstanding upls */

-#endif /* UBC_DEBUG */
+#ifdef VM_PIP_DEBUG
+/*
+ * Keep track of the stack traces for the first holders
+ * of a "paging_in_progress" reference for this VM object.
+ */
+#define VM_PIP_DEBUG_STACK_FRAMES      25      /* depth of each stack trace */
+#define VM_PIP_DEBUG_MAX_REFS          10      /* track that many references */
+       struct __pip_backtrace {
+               void *pip_retaddr[VM_PIP_DEBUG_STACK_FRAMES];
+       } pip_holders[VM_PIP_DEBUG_MAX_REFS];
+#endif /* VM_PIP_DEBUG  */
+#endif /* UPL_DEBUG */

 };
 
 };
 

+#define VM_PAGE_REMOVE(page)                                           \
+       MACRO_BEGIN                                                     \
+       vm_page_t __page = (page);                                      \
+       vm_object_t __object = __page->object;                          \
+       if (__page == __object->memq_hint) {                            \
+               vm_page_t       __new_hint;                             \
+               queue_entry_t   __qe;                                   \
+               __qe = queue_next(&__page->listq);                      \
+               if (queue_end(&__object->memq, __qe)) {                 \
+                       __qe = queue_prev(&__page->listq);              \
+                       if (queue_end(&__object->memq, __qe)) {         \
+                               __qe = NULL;                            \
+                       }                                               \
+               }                                                       \
+               __new_hint = (vm_page_t) __qe;                          \
+               __object->memq_hint = __new_hint;                       \
+       }                                                               \
+       queue_remove(&__object->memq, __page, vm_page_t, listq);        \
+       MACRO_END
+
+#define VM_PAGE_INSERT(page, object)                           \
+       MACRO_BEGIN                                             \
+       vm_page_t __page = (page);                              \
+       vm_object_t __object = (object);                        \
+       queue_enter(&__object->memq, __page, vm_page_t, listq); \
+       __object->memq_hint = __page;                           \
+       MACRO_END
+

 __private_extern__
 vm_object_t    kernel_object;          /* the single kernel object */
 
 __private_extern__
 __private_extern__
 vm_object_t    kernel_object;          /* the single kernel object */
 
 __private_extern__

-int            vm_object_absent_max;   /* maximum number of absent pages
+unsigned int   vm_object_absent_max;   /* maximum number of absent pages

                                           at a time for each object */
 
 # define       VM_MSYNC_INITIALIZED                    0
                                           at a time for each object */
 
 # define       VM_MSYNC_INITIALIZED                    0


@@ -317,12 +351,12 @@ typedef struct msync_req  *msync_req_t;
 #define msync_req_alloc(msr)                                           \
        MACRO_BEGIN                                                     \
         (msr) = (msync_req_t)kalloc(sizeof(struct msync_req));         \
 #define msync_req_alloc(msr)                                           \
        MACRO_BEGIN                                                     \
         (msr) = (msync_req_t)kalloc(sizeof(struct msync_req));         \

-        mutex_init(&(msr)->msync_req_lock, ETAP_VM_MSYNC);             \
+        mutex_init(&(msr)->msync_req_lock, 0);         \

        msr->flag = VM_MSYNC_INITIALIZED;                               \
         MACRO_END
 
 #define msync_req_free(msr)                                            \
        msr->flag = VM_MSYNC_INITIALIZED;                               \
         MACRO_END
 
 #define msync_req_free(msr)                                            \

-       (kfree((vm_offset_t)(msr), sizeof(struct msync_req)))
+       (kfree((msr), sizeof(struct msync_req)))

 
 #define msr_lock(msr)   mutex_lock(&(msr)->msync_req_lock)
 #define msr_unlock(msr) mutex_unlock(&(msr)->msync_req_lock)
 
 #define msr_lock(msr)   mutex_lock(&(msr)->msync_req_lock)
 #define msr_unlock(msr) mutex_unlock(&(msr)->msync_req_lock)


@@ -338,6 +372,9 @@ __private_extern__ void             vm_object_init(void);
 __private_extern__ vm_object_t vm_object_allocate(
                                        vm_object_size_t        size);
 
 __private_extern__ vm_object_t vm_object_allocate(
                                        vm_object_size_t        size);
 

+__private_extern__ void    _vm_object_allocate(vm_object_size_t size,
+                           vm_object_t object);
+

 #if    TASK_SWAPPER
 
 __private_extern__ void        vm_object_res_reference(
 #if    TASK_SWAPPER
 
 __private_extern__ void        vm_object_res_reference(


@@ -365,12 +402,10 @@ MACRO_BEGIN                                               \
 MACRO_END
 
 
 MACRO_END
 
 

-#if    MACH_ASSERT
-

 __private_extern__ void                vm_object_reference(
                                        vm_object_t     object);
 
 __private_extern__ void                vm_object_reference(
                                        vm_object_t     object);
 

-#else  /* MACH_ASSERT */
+#if    !MACH_ASSERT

 
 #define        vm_object_reference(object)                     \
 MACRO_BEGIN                                            \
 
 #define        vm_object_reference(object)                     \
 MACRO_BEGIN                                            \


@@ -394,9 +429,9 @@ __private_extern__ kern_return_t vm_object_release_name(
 __private_extern__ void                vm_object_pmap_protect(
                                        vm_object_t             object,
                                        vm_object_offset_t      offset,
 __private_extern__ void                vm_object_pmap_protect(
                                        vm_object_t             object,
                                        vm_object_offset_t      offset,

-                                       vm_size_t               size,
+                                       vm_object_size_t        size,

                                        pmap_t                  pmap,
                                        pmap_t                  pmap,

-                                       vm_offset_t             pmap_start,
+                                       vm_map_offset_t         pmap_start,

                                        vm_prot_t               prot);
 
 __private_extern__ void                vm_object_page_remove(
                                        vm_prot_t               prot);
 
 __private_extern__ void                vm_object_page_remove(


@@ -410,6 +445,14 @@ __private_extern__ void            vm_object_deactivate_pages(
                                        vm_object_size_t        size,
                                        boolean_t               kill_page);
 
                                        vm_object_size_t        size,
                                        boolean_t               kill_page);
 

+__private_extern__ unsigned int        vm_object_purge(
+                                       vm_object_t             object);
+
+__private_extern__ kern_return_t vm_object_purgable_control(
+       vm_object_t     object,
+       vm_purgable_t   control,
+       int             *state);
+

 __private_extern__ boolean_t   vm_object_coalesce(
                                        vm_object_t             prev_object,
                                        vm_object_t             next_object,
 __private_extern__ boolean_t   vm_object_coalesce(
                                        vm_object_t             prev_object,
                                        vm_object_t             next_object,


@@ -424,7 +467,9 @@ __private_extern__ boolean_t        vm_object_shadow(
                                        vm_object_size_t        length);
 
 __private_extern__ void                vm_object_collapse(
                                        vm_object_size_t        length);
 
 __private_extern__ void                vm_object_collapse(

-                                       vm_object_t     object);
+                                       vm_object_t             object,
+                                       vm_object_offset_t      offset,
+                                       boolean_t               can_bypass);

 
 __private_extern__ boolean_t   vm_object_copy_quickly(
                                vm_object_t             *_object,
 
 __private_extern__ boolean_t   vm_object_copy_quickly(
                                vm_object_t             *_object,


@@ -473,23 +518,31 @@ __private_extern__ void           vm_object_page_map(
 __private_extern__ kern_return_t vm_object_upl_request(
                                vm_object_t             object, 
                                vm_object_offset_t      offset,
 __private_extern__ kern_return_t vm_object_upl_request(
                                vm_object_t             object, 
                                vm_object_offset_t      offset,

-                               vm_size_t               size,
+                               upl_size_t              size,

                                upl_t                   *upl,
                                upl_page_info_t         *page_info,
                                unsigned int            *count,
                                int                     flags);
 
                                upl_t                   *upl,
                                upl_page_info_t         *page_info,
                                unsigned int            *count,
                                int                     flags);
 

+__private_extern__ kern_return_t vm_object_transpose(
+                               vm_object_t             object1,
+                               vm_object_t             object2,
+                               vm_object_size_t        transpose_size);
+

 __private_extern__ boolean_t vm_object_sync(
                                vm_object_t             object,
                                vm_object_offset_t      offset,
 __private_extern__ boolean_t vm_object_sync(
                                vm_object_t             object,
                                vm_object_offset_t      offset,

-                               vm_size_t               size,
+                               vm_object_size_t        size,

                                boolean_t               should_flush,
                                boolean_t               should_flush,

-                               boolean_t               should_return);
+                               boolean_t               should_return,
+                               boolean_t               should_iosync);

 
 __private_extern__ kern_return_t vm_object_update(
                                vm_object_t             object,
                                vm_object_offset_t      offset,
 
 __private_extern__ kern_return_t vm_object_update(
                                vm_object_t             object,
                                vm_object_offset_t      offset,

-                               vm_size_t               size, /* should be 64 */
+                               vm_object_size_t        size,
+                               vm_object_offset_t      *error_offset,
+                               int                     *io_errno,

                                memory_object_return_t  should_return,
                                int                     flags,
                                vm_prot_t               prot);
                                memory_object_return_t  should_return,
                                int                     flags,
                                vm_prot_t               prot);


@@ -512,6 +565,39 @@ __private_extern__ vm_object_t     vm_object_enter(
                                        boolean_t               check_named);
 
 
                                        boolean_t               check_named);
 
 

+/*
+ * Purgable object state.
+ */
+
+#define VM_OBJECT_NONPURGABLE          0       /* not a purgable object */
+#define VM_OBJECT_PURGABLE_NONVOLATILE 1       /* non-volatile purgable object */
+#define VM_OBJECT_PURGABLE_VOLATILE    2       /* volatile (but intact) purgable object */
+#define VM_OBJECT_PURGABLE_EMPTY       3       /* volatile purgable object that has been emptied */
+
+__private_extern__ kern_return_t vm_object_populate_with_private(
+       vm_object_t             object,
+       vm_object_offset_t      offset,
+       ppnum_t                 phys_page,
+       vm_size_t               size);
+
+extern kern_return_t adjust_vm_object_cache(
+       vm_size_t oval,
+       vm_size_t nval);
+
+extern kern_return_t vm_object_page_op(
+       vm_object_t             object,
+       vm_object_offset_t      offset,
+       int                     ops,
+       ppnum_t                 *phys_entry,
+       int                     *flags);
+
+extern kern_return_t vm_object_range_op(
+       vm_object_t             object,
+       vm_object_offset_t      offset_beg,
+       vm_object_offset_t      offset_end,
+       int                     ops,
+       int                     *range);
+

 /*
  *     Event waiting handling
  */
 /*
  *     Event waiting handling
  */


@@ -560,15 +646,30 @@ __private_extern__ vm_object_t    vm_object_enter(
 /*
  *     Routines implemented as macros
  */
 /*
  *     Routines implemented as macros
  */

+#ifdef VM_PIP_DEBUG
+extern unsigned OSBacktrace(void **bt, unsigned maxAddrs);
+#define VM_PIP_DEBUG_BEGIN(object)                                     \
+       MACRO_BEGIN                                                     \
+       if ((object)->paging_in_progress < VM_PIP_DEBUG_MAX_REFS) {     \
+               int pip = (object)->paging_in_progress;                 \
+               (void) OSBacktrace(&(object)->pip_holders[pip].retaddr[0], \
+                                  VM_PIP_DEBUG_STACK_FRAMES);          \
+       }                                                               \
+       MACRO_END
+#else  /* VM_PIP_DEBUG */
+#define VM_PIP_DEBUG_BEGIN(object)
+#endif /* VM_PIP_DEBUG */

 
 #define                vm_object_paging_begin(object)                          \
        MACRO_BEGIN                                                     \
 
 #define                vm_object_paging_begin(object)                          \
        MACRO_BEGIN                                                     \

+       assert((object)->paging_in_progress >= 0);                      \
+       VM_PIP_DEBUG_BEGIN((object));                                   \

        (object)->paging_in_progress++;                                 \
        MACRO_END
 
 #define                vm_object_paging_end(object)                            \
        MACRO_BEGIN                                                     \
        (object)->paging_in_progress++;                                 \
        MACRO_END
 
 #define                vm_object_paging_end(object)                            \
        MACRO_BEGIN                                                     \

-       assert((object)->paging_in_progress != 0);                      \
+       assert((object)->paging_in_progress > 0);                       \

        if (--(object)->paging_in_progress == 0) {                      \
                vm_object_wakeup(object,                                \
                        VM_OBJECT_EVENT_PAGING_IN_PROGRESS);            \
        if (--(object)->paging_in_progress == 0) {                      \
                vm_object_wakeup(object,                                \
                        VM_OBJECT_EVENT_PAGING_IN_PROGRESS);            \


@@ -608,9 +709,12 @@ __private_extern__ vm_object_t     vm_object_enter(
  *     Object locking macros
  */
 
  *     Object locking macros
  */
 

-#define vm_object_lock_init(object)    mutex_init(&(object)->Lock, ETAP_VM_OBJ)
+#define vm_object_lock_init(object)    mutex_init(&(object)->Lock, 0)

 #define vm_object_lock(object)         mutex_lock(&(object)->Lock)
 #define vm_object_unlock(object)       mutex_unlock(&(object)->Lock)
 #define vm_object_lock_try(object)     mutex_try(&(object)->Lock)
 
 #define vm_object_lock(object)         mutex_lock(&(object)->Lock)
 #define vm_object_unlock(object)       mutex_unlock(&(object)->Lock)
 #define vm_object_lock_try(object)     mutex_try(&(object)->Lock)
 

+#define vm_object_round_page(x) (((vm_object_offset_t)(x) + PAGE_MASK) & ~((signed)PAGE_MASK))
+#define vm_object_trunc_page(x) ((vm_object_offset_t)(x) & ~((signed)PAGE_MASK))
+

 #endif /* _VM_VM_OBJECT_H_ */
 #endif /* _VM_VM_OBJECT_H_ */