xnu-6153.101.6.tar.gz

[apple/xnu.git] / osfmk / vm / pmap.h

diff --git a/osfmk/vm/pmap.h b/osfmk/vm/pmap.h
index d30299778fee450467eec6a5066869fe4e0a23d1..873bae99888fc081f9cf0f3c0afb94ddd5fa005f 100644 (file)
--- a/osfmk/vm/pmap.h
+++ b/osfmk/vm/pmap.h
@@ -1,8 +1,8 @@
 /*
 /*

- * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
+ * Copyright (c) 2000-2012 Apple Inc. All rights reserved.

  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@

- * 
+ *

  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in
  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in


@@ -11,10 +11,10 @@
  * unlawful or unlicensed copies of an Apple operating system, or to
  * circumvent, violate, or enable the circumvention or violation of, any
  * terms of an Apple operating system software license agreement.
  * unlawful or unlicensed copies of an Apple operating system, or to
  * circumvent, violate, or enable the circumvention or violation of, any
  * terms of an Apple operating system software license agreement.

- * 
+ *

  * Please obtain a copy of the License at
  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  * Please obtain a copy of the License at
  * http://www.opensource.apple.com/apsl/ and read it before using this file.

- * 
+ *

  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,


@@ -22,34 +22,34 @@
  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  * Please see the License for the specific language governing rights and
  * limitations under the License.
  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  * Please see the License for the specific language governing rights and
  * limitations under the License.

- * 
+ *

  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
  */
  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
  */

-/* 
+/*

  * Mach Operating System
  * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
  * All Rights Reserved.
  * Mach Operating System
  * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
  * All Rights Reserved.

- * 
+ *

  * Permission to use, copy, modify and distribute this software and its
  * documentation is hereby granted, provided that both the copyright
  * notice and this permission notice appear in all copies of the
  * software, derivative works or modified versions, and any portions
  * thereof, and that both notices appear in supporting documentation.
  * Permission to use, copy, modify and distribute this software and its
  * documentation is hereby granted, provided that both the copyright
  * notice and this permission notice appear in all copies of the
  * software, derivative works or modified versions, and any portions
  * thereof, and that both notices appear in supporting documentation.

- * 
+ *

  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
  * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
  * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.

- * 
+ *

  * Carnegie Mellon requests users of this software to return to
  * Carnegie Mellon requests users of this software to return to

- * 
+ *

  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  *  School of Computer Science
  *  Carnegie Mellon University
  *  Pittsburgh PA 15213-3890
  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  *  School of Computer Science
  *  Carnegie Mellon University
  *  Pittsburgh PA 15213-3890

- * 
+ *

  * any improvements or extensions that they make and grant Carnegie Mellon
  * the rights to redistribute these changes.
  */
  * any improvements or extensions that they make and grant Carnegie Mellon
  * the rights to redistribute these changes.
  */


@@ -64,7 +64,7 @@
  *     section.  [For machine-dependent section, see "machine/pmap.h".]
  */
 
  *     section.  [For machine-dependent section, see "machine/pmap.h".]
  */
 

-#ifndef        _VM_PMAP_H_
+#ifndef _VM_PMAP_H_

 #define _VM_PMAP_H_
 
 #include <mach/kern_return.h>
 #define _VM_PMAP_H_
 
 #include <mach/kern_return.h>


@@ -74,7 +74,9 @@
 #include <mach/boolean.h>
 #include <mach/vm_prot.h>
 
 #include <mach/boolean.h>
 #include <mach/vm_prot.h>
 

-#ifdef KERNEL_PRIVATE
+#include <kern/trustcache.h>
+
+#ifdef  KERNEL_PRIVATE

 
 /*
  *     The following is a description of the interface to the
 
 /*
  *     The following is a description of the interface to the


@@ -90,11 +92,11 @@
 
 /* Copy between a physical page and a virtual address */
 /* LP64todo - switch to vm_map_offset_t when it grows */
 
 /* Copy between a physical page and a virtual address */
 /* LP64todo - switch to vm_map_offset_t when it grows */

-extern kern_return_t   copypv(
-                               addr64_t source, 
-                               addr64_t sink, 
-                               unsigned int size, 
-                               int which);     
+extern kern_return_t    copypv(
+       addr64_t source,
+       addr64_t sink,
+       unsigned int size,
+       int which);

 #define cppvPsnk        1
 #define cppvPsnkb      31
 #define cppvPsrc        2
 #define cppvPsnk        1
 #define cppvPsnkb      31
 #define cppvPsrc        2


@@ -107,12 +109,17 @@ extern kern_return_t      copypv(
 #define cppvNoModSnkb  27
 #define cppvNoRefSrc   32
 #define cppvNoRefSrcb  26
 #define cppvNoModSnkb  27
 #define cppvNoRefSrc   32
 #define cppvNoRefSrcb  26

-#define cppvKmap       64      /* Use the kernel's vm_map */
+#define cppvKmap       64       /* Use the kernel's vm_map */

 #define cppvKmapb      25
 
 #define cppvKmapb      25
 

-#ifdef MACH_KERNEL_PRIVATE
+extern boolean_t pmap_has_managed_page(ppnum_t first, ppnum_t last);
+
+#ifdef  MACH_KERNEL_PRIVATE
+
+#include <mach_assert.h>

 
 #include <machine/pmap.h>
 
 #include <machine/pmap.h>

+#include <vm/memory_types.h>

 
 /*
  *     Routines used for initialization.
 
 /*
  *     Routines used for initialization.


@@ -126,36 +133,24 @@ extern kern_return_t      copypv(
  *     vm_offset_t, etc... types.
  */
 
  *     vm_offset_t, etc... types.
  */
 

-extern void            *pmap_steal_memory(vm_size_t size);
-                                               /* During VM initialization,
-                                                * steal a chunk of memory.
-                                                */
-extern unsigned int    pmap_free_pages(void);  /* During VM initialization,
-                                                * report remaining unused
-                                                * physical pages.
-                                                */
-extern void            pmap_startup(
-                               vm_offset_t *startp,
-                               vm_offset_t *endp);
-                                               /* During VM initialization,
-                                                * use remaining physical pages
-                                                * to allocate page frames.
-                                                */
-extern void            pmap_init(void) __attribute__((section("__TEXT, initcode")));
-                                               /* Initialization,
-                                                * after kernel runs
-                                                * in virtual memory.
-                                                */
-
-extern void            mapping_adjust(void);   /* Adjust free mapping count */
-
-extern void            mapping_free_prime(void); /* Primes the mapping block release list */
-
-#ifndef        MACHINE_PAGES
+extern void *pmap_steal_memory(vm_size_t size); /* Early memory allocation */
+extern void *pmap_steal_freeable_memory(vm_size_t size); /* Early memory allocation */
+
+extern uint_t pmap_free_pages(void); /* report remaining unused physical pages */
+
+extern void pmap_startup(vm_offset_t *startp, vm_offset_t *endp); /* allocate vm_page structs */
+
+extern void pmap_init(void); /* Initialization, once we have kernel virtual memory.  */
+
+extern void mapping_adjust(void); /* Adjust free mapping count */
+
+extern void mapping_free_prime(void); /* Primes the mapping block release list */
+
+#ifndef MACHINE_PAGES

 /*
  *     If machine/pmap.h defines MACHINE_PAGES, it must implement
  *     the above functions.  The pmap module has complete control.
 /*
  *     If machine/pmap.h defines MACHINE_PAGES, it must implement
  *     the above functions.  The pmap module has complete control.

- *     Otherwise, it must implement
+ *     Otherwise, it must implement the following functions:

  *             pmap_free_pages
  *             pmap_virtual_space
  *             pmap_next_page
  *             pmap_free_pages
  *             pmap_virtual_space
  *             pmap_next_page


@@ -168,156 +163,221 @@ extern void             mapping_free_prime(void); /* Primes the mapping block release list
  *     However, for best performance pmap_free_pages should be accurate.
  */
 
  *     However, for best performance pmap_free_pages should be accurate.
  */
 

-extern boolean_t       pmap_next_page(ppnum_t *pnum);
-                                               /* During VM initialization,
-                                                * return the next unused
-                                                * physical page.
-                                                */
-extern void            pmap_virtual_space(
-                                       vm_offset_t     *virtual_start,
-                                       vm_offset_t     *virtual_end);
-                                               /* During VM initialization,
-                                                * report virtual space
-                                                * available for the kernel.
-                                                */
-#endif /* MACHINE_PAGES */
-

 /*
 /*

- *     Routines to manage the physical map data structure.
+ * Routines to return the next unused physical page.

  */
  */

-extern pmap_t          pmap_create(    /* Create a pmap_t. */
-                               vm_map_size_t   size,
-#ifdef __i386__
-                               boolean_t       is_64bit);
-#else
-                               __unused boolean_t      is_64bit);
+extern boolean_t pmap_next_page(ppnum_t *pnum);
+extern boolean_t pmap_next_page_hi(ppnum_t *pnum, boolean_t might_free);
+#ifdef __x86_64__
+extern kern_return_t pmap_next_page_large(ppnum_t *pnum);
+extern void pmap_hi_pages_done(void);

 #endif
 #endif

-extern pmap_t          (pmap_kernel)(void);    /* Return the kernel's pmap */
-extern void            pmap_reference(pmap_t pmap);    /* Gain a reference. */
-extern void            pmap_destroy(pmap_t pmap); /* Release a reference. */
-extern void            pmap_switch(pmap_t);
-

 
 

-extern void            pmap_enter(     /* Enter a mapping */
-                               pmap_t          pmap,
-                               vm_map_offset_t v,
-                               ppnum_t         pn,
-                               vm_prot_t       prot,
-                               unsigned int    flags,
-                               boolean_t       wired);
+/*
+ * Report virtual space available for the kernel.
+ */
+extern void pmap_virtual_space(
+       vm_offset_t     *virtual_start,
+       vm_offset_t     *virtual_end);
+#endif  /* MACHINE_PAGES */

 
 

-extern void            pmap_remove_some_phys(
-                               pmap_t          pmap,
-                               ppnum_t         pn);
+/*
+ * Routines to manage the physical map data structure.
+ */
+extern pmap_t           pmap_create_options(    /* Create a pmap_t. */
+       ledger_t        ledger,
+       vm_map_size_t   size,
+       unsigned int    flags);
+
+extern pmap_t(pmap_kernel)(void);               /* Return the kernel's pmap */
+extern void             pmap_reference(pmap_t pmap);    /* Gain a reference. */
+extern void             pmap_destroy(pmap_t pmap); /* Release a reference. */
+extern void             pmap_switch(pmap_t);
+
+#if MACH_ASSERT
+extern void pmap_set_process(pmap_t pmap,
+    int pid,
+    char *procname);
+#endif /* MACH_ASSERT */
+
+extern kern_return_t    pmap_enter(     /* Enter a mapping */
+       pmap_t          pmap,
+       vm_map_offset_t v,
+       ppnum_t         pn,
+       vm_prot_t       prot,
+       vm_prot_t       fault_type,
+       unsigned int    flags,
+       boolean_t       wired);
+
+extern kern_return_t    pmap_enter_options(
+       pmap_t pmap,
+       vm_map_offset_t v,
+       ppnum_t pn,
+       vm_prot_t prot,
+       vm_prot_t fault_type,
+       unsigned int flags,
+       boolean_t wired,
+       unsigned int options,
+       void *arg);
+
+extern void             pmap_remove_some_phys(
+       pmap_t          pmap,
+       ppnum_t         pn);
+
+extern void             pmap_lock_phys_page(
+       ppnum_t         pn);
+
+extern void             pmap_unlock_phys_page(
+       ppnum_t         pn);

 
 
 /*
  *     Routines that operate on physical addresses.
  */
 
 
 
 /*
  *     Routines that operate on physical addresses.
  */
 

-extern void            pmap_page_protect(      /* Restrict access to page. */
-                               ppnum_t phys,
-                               vm_prot_t       prot);
-
-extern void            (pmap_zero_page)(
-                               ppnum_t         pn);
-
-extern void            (pmap_zero_part_page)(
-                               ppnum_t         pn,
-                               vm_offset_t     offset,
-                               vm_size_t       len);
-
-extern void            (pmap_copy_page)(
-                               ppnum_t         src,
-                               ppnum_t         dest);
-
-extern void            (pmap_copy_part_page)(
-                               ppnum_t         src,
-                               vm_offset_t     src_offset,
-                               ppnum_t         dst,
-                               vm_offset_t     dst_offset,
-                               vm_size_t       len);
-
-extern void            (pmap_copy_part_lpage)(
-                               vm_offset_t     src,
-                               ppnum_t         dst,
-                               vm_offset_t     dst_offset,
-                               vm_size_t       len);
-
-extern void            (pmap_copy_part_rpage)(
-                               ppnum_t         src,
-                               vm_offset_t     src_offset,
-                               vm_offset_t     dst,
-                               vm_size_t       len);
-                               
-extern unsigned int (pmap_disconnect)( /* disconnect mappings and return reference and change */
-                               ppnum_t         phys);
-
-extern kern_return_t   (pmap_attribute_cache_sync)(  /* Flush appropriate 
-                                                      * cache based on
-                                                      * page number sent */
-                               ppnum_t         pn, 
-                               vm_size_t       size, 
-                               vm_machine_attribute_t attribute, 
-                               vm_machine_attribute_val_t* value);
-
-extern unsigned int    (pmap_cache_attributes)(
-                               ppnum_t         pn);
+extern void             pmap_page_protect(      /* Restrict access to page. */
+       ppnum_t phys,
+       vm_prot_t       prot);
+
+extern void             pmap_page_protect_options(      /* Restrict access to page. */
+       ppnum_t phys,
+       vm_prot_t       prot,
+       unsigned int    options,
+       void            *arg);
+
+extern void(pmap_zero_page)(
+       ppnum_t         pn);
+
+extern void(pmap_zero_part_page)(
+       ppnum_t         pn,
+       vm_offset_t     offset,
+       vm_size_t       len);
+
+extern void(pmap_copy_page)(
+       ppnum_t         src,
+       ppnum_t         dest);
+
+extern void(pmap_copy_part_page)(
+       ppnum_t         src,
+       vm_offset_t     src_offset,
+       ppnum_t         dst,
+       vm_offset_t     dst_offset,
+       vm_size_t       len);
+
+extern void(pmap_copy_part_lpage)(
+       vm_offset_t     src,
+       ppnum_t         dst,
+       vm_offset_t     dst_offset,
+       vm_size_t       len);
+
+extern void(pmap_copy_part_rpage)(
+       ppnum_t         src,
+       vm_offset_t     src_offset,
+       vm_offset_t     dst,
+       vm_size_t       len);
+
+extern unsigned int(pmap_disconnect)(   /* disconnect mappings and return reference and change */
+       ppnum_t         phys);
+
+extern unsigned int(pmap_disconnect_options)(   /* disconnect mappings and return reference and change */
+       ppnum_t         phys,
+       unsigned int    options,
+       void            *arg);
+
+extern kern_return_t(pmap_attribute_cache_sync)(      /* Flush appropriate
+                                                       * cache based on
+                                                       * page number sent */
+       ppnum_t         pn,
+       vm_size_t       size,
+       vm_machine_attribute_t attribute,
+       vm_machine_attribute_val_t* value);
+
+extern unsigned int(pmap_cache_attributes)(
+       ppnum_t         pn);

 
 

+/*
+ * Set (override) cache attributes for the specified physical page
+ */
+extern  void            pmap_set_cache_attributes(
+       ppnum_t,
+       unsigned int);
+
+extern void            *pmap_map_compressor_page(
+       ppnum_t);
+
+extern void             pmap_unmap_compressor_page(
+       ppnum_t,
+       void*);
+
+#if defined(__arm__) || defined(__arm64__)
+/* ARM64_TODO */
+extern  boolean_t       pmap_batch_set_cache_attributes(
+       ppnum_t,
+       unsigned int,
+       unsigned int,
+       unsigned int,
+       boolean_t,
+       unsigned int*);
+#endif

 extern void pmap_sync_page_data_phys(ppnum_t pa);
 extern void pmap_sync_page_attributes_phys(ppnum_t pa);
 
 /*
  * debug/assertions. pmap_verify_free returns true iff
  * the given physical page is mapped into no pmap.
 extern void pmap_sync_page_data_phys(ppnum_t pa);
 extern void pmap_sync_page_attributes_phys(ppnum_t pa);
 
 /*
  * debug/assertions. pmap_verify_free returns true iff
  * the given physical page is mapped into no pmap.

+ * pmap_assert_free() will panic() if pn is not free.

  */
  */

-extern boolean_t       pmap_verify_free(ppnum_t pn);
+extern boolean_t        pmap_verify_free(ppnum_t pn);
+#if MACH_ASSERT
+extern void pmap_assert_free(ppnum_t pn);
+#endif

 
 /*
  *     Statistics routines
  */
 
 /*
  *     Statistics routines
  */

-extern int             (pmap_resident_count)(pmap_t pmap);
-extern int             (pmap_resident_max)(pmap_t pmap);
+extern int(pmap_compressed)(pmap_t pmap);
+extern int(pmap_resident_count)(pmap_t pmap);
+extern int(pmap_resident_max)(pmap_t pmap);

 
 /*
  *     Sundry required (internal) routines
  */
 #ifdef CURRENTLY_UNUSED_AND_UNTESTED
 
 /*
  *     Sundry required (internal) routines
  */
 #ifdef CURRENTLY_UNUSED_AND_UNTESTED

-extern void            pmap_collect(pmap_t pmap);/* Perform garbage
-                                                * collection, if any */
+extern void             pmap_collect(pmap_t pmap);/* Perform garbage
+                                                   * collection, if any */

 #endif
 /*
  *     Optional routines
  */
 #endif
 /*
  *     Optional routines
  */

-extern void            (pmap_copy)(            /* Copy range of mappings,
-                                                * if desired. */
-                               pmap_t          dest,
-                               pmap_t          source,
-                               vm_map_offset_t dest_va,
-                               vm_map_size_t   size,
-                               vm_map_offset_t source_va);
-
-extern kern_return_t   (pmap_attribute)(       /* Get/Set special memory
-                                                * attributes */
-                               pmap_t          pmap,
-                               vm_map_offset_t va,
-                               vm_map_size_t   size,
-                               vm_machine_attribute_t  attribute,
-                               vm_machine_attribute_val_t* value);
+extern void(pmap_copy)(                         /* Copy range of mappings,
+                                                 * if desired. */
+       pmap_t          dest,
+       pmap_t          source,
+       vm_map_offset_t dest_va,
+       vm_map_size_t   size,
+       vm_map_offset_t source_va);
+
+extern kern_return_t(pmap_attribute)(           /* Get/Set special memory
+                                                 * attributes */
+       pmap_t          pmap,
+       vm_map_offset_t va,
+       vm_map_size_t   size,
+       vm_machine_attribute_t  attribute,
+       vm_machine_attribute_val_t* value);

 
 /*
  * Routines defined as macros.
  */
 #ifndef PMAP_ACTIVATE_USER
 
 /*
  * Routines defined as macros.
  */
 #ifndef PMAP_ACTIVATE_USER

-#ifndef        PMAP_ACTIVATE
+#ifndef PMAP_ACTIVATE

 #define PMAP_ACTIVATE_USER(thr, cpu)
 #define PMAP_ACTIVATE_USER(thr, cpu)

-#else  /* PMAP_ACTIVATE */
-#define PMAP_ACTIVATE_USER(thr, cpu) {                 \
-       pmap_t  pmap;                                           \
-                                                               \
-       pmap = (thr)->map->pmap;                                \
-       if (pmap != pmap_kernel())                              \
-               PMAP_ACTIVATE(pmap, (thr), (cpu));              \
+#else   /* PMAP_ACTIVATE */
+#define PMAP_ACTIVATE_USER(thr, cpu) {                  \
+       pmap_t  pmap;                                           \
+                                                                \
+       pmap = (thr)->map->pmap;                                \
+       if (pmap != pmap_kernel())                              \
+               PMAP_ACTIVATE(pmap, (thr), (cpu));              \

 }
 #endif  /* PMAP_ACTIVATE */
 #endif  /* PMAP_ACTIVATE_USER */
 }
 #endif  /* PMAP_ACTIVATE */
 #endif  /* PMAP_ACTIVATE_USER */


@@ -325,111 +385,257 @@ extern kern_return_t    (pmap_attribute)(       /* Get/Set special memory
 #ifndef PMAP_DEACTIVATE_USER
 #ifndef PMAP_DEACTIVATE
 #define PMAP_DEACTIVATE_USER(thr, cpu)
 #ifndef PMAP_DEACTIVATE_USER
 #ifndef PMAP_DEACTIVATE
 #define PMAP_DEACTIVATE_USER(thr, cpu)

-#else  /* PMAP_DEACTIVATE */
-#define PMAP_DEACTIVATE_USER(thr, cpu) {                       \
-       pmap_t  pmap;                                           \
-                                                               \
-       pmap = (thr)->map->pmap;                                \
-       if ((pmap) != pmap_kernel())                    \
-               PMAP_DEACTIVATE(pmap, (thr), (cpu));    \
+#else   /* PMAP_DEACTIVATE */
+#define PMAP_DEACTIVATE_USER(thr, cpu) {                        \
+       pmap_t  pmap;                                           \
+                                                                \
+       pmap = (thr)->map->pmap;                                \
+       if ((pmap) != pmap_kernel())                    \
+               PMAP_DEACTIVATE(pmap, (thr), (cpu));    \

 }
 }

-#endif /* PMAP_DEACTIVATE */
+#endif  /* PMAP_DEACTIVATE */

 #endif  /* PMAP_DEACTIVATE_USER */
 
 #endif  /* PMAP_DEACTIVATE_USER */
 

-#ifndef        PMAP_ACTIVATE_KERNEL
+#ifndef PMAP_ACTIVATE_KERNEL

 #ifndef PMAP_ACTIVATE
 #ifndef PMAP_ACTIVATE

-#define        PMAP_ACTIVATE_KERNEL(cpu)
-#else  /* PMAP_ACTIVATE */
-#define        PMAP_ACTIVATE_KERNEL(cpu)                       \
-               PMAP_ACTIVATE(pmap_kernel(), THREAD_NULL, cpu)
-#endif /* PMAP_ACTIVATE */
-#endif /* PMAP_ACTIVATE_KERNEL */
-
-#ifndef        PMAP_DEACTIVATE_KERNEL
+#define PMAP_ACTIVATE_KERNEL(cpu)
+#else   /* PMAP_ACTIVATE */
+#define PMAP_ACTIVATE_KERNEL(cpu)                       \
+               PMAP_ACTIVATE(pmap_kernel(), THREAD_NULL, cpu)
+#endif  /* PMAP_ACTIVATE */
+#endif  /* PMAP_ACTIVATE_KERNEL */
+
+#ifndef PMAP_DEACTIVATE_KERNEL

 #ifndef PMAP_DEACTIVATE
 #ifndef PMAP_DEACTIVATE

-#define        PMAP_DEACTIVATE_KERNEL(cpu)
-#else  /* PMAP_DEACTIVATE */
-#define        PMAP_DEACTIVATE_KERNEL(cpu)                     \
-               PMAP_DEACTIVATE(pmap_kernel(), THREAD_NULL, cpu)
-#endif /* PMAP_DEACTIVATE */
-#endif /* PMAP_DEACTIVATE_KERNEL */
-
-#ifndef        PMAP_ENTER
+#define PMAP_DEACTIVATE_KERNEL(cpu)
+#else   /* PMAP_DEACTIVATE */
+#define PMAP_DEACTIVATE_KERNEL(cpu)                     \
+               PMAP_DEACTIVATE(pmap_kernel(), THREAD_NULL, cpu)
+#endif  /* PMAP_DEACTIVATE */
+#endif  /* PMAP_DEACTIVATE_KERNEL */
+
+#ifndef PMAP_ENTER

 /*
  *     Macro to be used in place of pmap_enter()
  */
 /*
  *     Macro to be used in place of pmap_enter()
  */

-#define PMAP_ENTER(pmap, virtual_address, page, protection, flags, wired) \
-       MACRO_BEGIN                                                     \
-       pmap_t          __pmap = (pmap);                                \
-       vm_page_t       __page = (page);                                \
-                                                                       \
-       if (__pmap != kernel_pmap) {                                    \
-               ASSERT_PAGE_DECRYPTED(__page);                          \
-       }                                                               \
-       if (__page->error) {                                            \
-               panic("VM page %p should not have an error\n",          \
-                       __page);                                        \
-       }                                                               \
-       pmap_enter(__pmap,                                              \
-                  (virtual_address),                                   \
-                  __page->phys_page,                                   \
-                  (protection),                                        \
-                  (flags),                                             \
-                  (wired));                                            \
+#define PMAP_ENTER(pmap, virtual_address, page, protection, fault_type, \
+           flags, wired, result)                                \
+       MACRO_BEGIN                                                     \
+       pmap_t          __pmap = (pmap);                                \
+       vm_page_t       __page = (page);                                \
+       int             __options = 0;                                  \
+       vm_object_t     __obj;                                          \
+                                                                        \
+       PMAP_ENTER_CHECK(__pmap, __page)                                \
+       __obj = VM_PAGE_OBJECT(__page);                                 \
+       if (__obj->internal) {                                          \
+               __options |= PMAP_OPTIONS_INTERNAL;                     \
+       }                                                               \
+       if (__page->vmp_reusable || __obj->all_reusable) {              \
+               __options |= PMAP_OPTIONS_REUSABLE;                     \
+       }                                                               \
+       result = pmap_enter_options(__pmap,                             \
+                                   (virtual_address),                  \
+                                   VM_PAGE_GET_PHYS_PAGE(__page),      \
+                                   (protection),                               \
+                                   (fault_type),                               \
+                                   (flags),                            \
+                                   (wired),                            \
+                                   __options,                          \
+                                   NULL);                              \
+       MACRO_END
+#endif  /* !PMAP_ENTER */
+
+#ifndef PMAP_ENTER_OPTIONS
+#define PMAP_ENTER_OPTIONS(pmap, virtual_address, page, protection,     \
+           fault_type, flags, wired, options, result)   \
+       MACRO_BEGIN                                                     \
+       pmap_t          __pmap = (pmap);                                \
+       vm_page_t       __page = (page);                                \
+       int             __extra_options = 0;                            \
+       vm_object_t     __obj;                                          \
+                                                                        \
+       PMAP_ENTER_CHECK(__pmap, __page)                                \
+       __obj = VM_PAGE_OBJECT(__page);                                 \
+       if (__obj->internal) {                                          \
+               __extra_options |= PMAP_OPTIONS_INTERNAL;               \
+       }                                                               \
+       if (__page->vmp_reusable || __obj->all_reusable) {              \
+               __extra_options |= PMAP_OPTIONS_REUSABLE;               \
+       }                                                               \
+       result = pmap_enter_options(__pmap,                             \
+                                   (virtual_address),                  \
+                                   VM_PAGE_GET_PHYS_PAGE(__page),      \
+                                   (protection),                       \
+                                   (fault_type),                       \
+                                   (flags),                            \
+                                   (wired),                            \
+                                   (options) | __extra_options,        \
+                                   NULL);                              \

        MACRO_END
        MACRO_END

-#endif /* !PMAP_ENTER */
+#endif  /* !PMAP_ENTER_OPTIONS */
+
+#ifndef PMAP_SET_CACHE_ATTR
+#define PMAP_SET_CACHE_ATTR(mem, object, cache_attr, batch_pmap_op)             \
+       MACRO_BEGIN                                                             \
+               if (!batch_pmap_op) {                                           \
+                       pmap_set_cache_attributes(VM_PAGE_GET_PHYS_PAGE(mem), cache_attr); \
+                       object->set_cache_attr = TRUE;                          \
+               }                                                               \
+       MACRO_END
+#endif  /* PMAP_SET_CACHE_ATTR */
+
+#ifndef PMAP_BATCH_SET_CACHE_ATTR
+#if     defined(__arm__) || defined(__arm64__)
+#define PMAP_BATCH_SET_CACHE_ATTR(object, user_page_list,                       \
+           cache_attr, num_pages, batch_pmap_op)   \
+       MACRO_BEGIN                                                             \
+               if ((batch_pmap_op)) {                                          \
+                       unsigned int __page_idx=0;                              \
+                       unsigned int res=0;                                     \
+                       boolean_t batch=TRUE;                                   \
+                       while (__page_idx < (num_pages)) {                      \
+                               if (!pmap_batch_set_cache_attributes(           \
+                                       user_page_list[__page_idx].phys_addr,   \
+                                       (cache_attr),                           \
+                                       (num_pages),                            \
+                                       (__page_idx),                           \
+                                       FALSE,                                  \
+                                       (&res))) {                              \
+                                       batch = FALSE;                          \
+                                       break;                                  \
+                               }                                               \
+                               __page_idx++;                                   \
+                       }                                                       \
+                       __page_idx=0;                                           \
+                       res=0;                                                  \
+                       while (__page_idx < (num_pages)) {                      \
+                               if (batch)                                      \
+                                       (void)pmap_batch_set_cache_attributes(  \
+                                       user_page_list[__page_idx].phys_addr,   \
+                                       (cache_attr),                           \
+                                       (num_pages),                            \
+                                       (__page_idx),                           \
+                                       TRUE,                                   \
+                                       (&res));                                \
+                               else                                            \
+                                       pmap_set_cache_attributes(              \
+                                       user_page_list[__page_idx].phys_addr,   \
+                                       (cache_attr));                          \
+                                       __page_idx++;                           \
+                       }                                                       \
+                       (object)->set_cache_attr = TRUE;                        \
+               }                                                               \
+       MACRO_END
+#else
+#define PMAP_BATCH_SET_CACHE_ATTR(object, user_page_list,                       \
+           cache_attr, num_pages, batch_pmap_op)   \
+       MACRO_BEGIN                                                             \
+               if ((batch_pmap_op)) {                                          \
+                       unsigned int __page_idx=0;                              \
+                       while (__page_idx < (num_pages)) {                      \
+                               pmap_set_cache_attributes(                      \
+                                       user_page_list[__page_idx].phys_addr,   \
+                                       (cache_attr));                          \
+                               __page_idx++;                                   \
+                       }                                                       \
+                       (object)->set_cache_attr = TRUE;                        \
+               }                                                               \
+       MACRO_END
+#endif
+#endif  /* PMAP_BATCH_SET_CACHE_ATTR */
+
+#define PMAP_ENTER_CHECK(pmap, page)                                    \
+{                                                                       \
+       if ((page)->vmp_error) {                                        \
+               panic("VM page %p should not have an error\n",          \
+                       (page));                                        \
+       }                                                               \
+}

 
 /*
  *     Routines to manage reference/modify bits based on
  *     physical addresses, simulating them if not provided
  *     by the hardware.
  */
 
 /*
  *     Routines to manage reference/modify bits based on
  *     physical addresses, simulating them if not provided
  *     by the hardware.
  */

-                               /* Clear reference bit */
-extern void            pmap_clear_reference(ppnum_t     pn);
-                               /* Return reference bit */
-extern boolean_t       (pmap_is_referenced)(ppnum_t     pn);
-                               /* Set modify bit */
-extern void             pmap_set_modify(ppnum_t         pn);
-                               /* Clear modify bit */
-extern void            pmap_clear_modify(ppnum_t pn);
-                               /* Return modify bit */
-extern boolean_t       pmap_is_modified(ppnum_t pn);
-                               /* Return modified and referenced bits */
+struct pfc {
+       long    pfc_cpus;
+       long    pfc_invalid_global;
+};
+
+typedef struct pfc      pmap_flush_context;
+
+/* Clear reference bit */
+extern void             pmap_clear_reference(ppnum_t     pn);
+/* Return reference bit */
+extern boolean_t(pmap_is_referenced)(ppnum_t     pn);
+/* Set modify bit */
+extern void             pmap_set_modify(ppnum_t  pn);
+/* Clear modify bit */
+extern void             pmap_clear_modify(ppnum_t pn);
+/* Return modify bit */
+extern boolean_t        pmap_is_modified(ppnum_t pn);
+/* Return modified and referenced bits */

 extern unsigned int pmap_get_refmod(ppnum_t pn);
 extern unsigned int pmap_get_refmod(ppnum_t pn);

-                               /* Clear modified and referenced bits */
-extern void                    pmap_clear_refmod(ppnum_t pn, unsigned int mask);
-#define VM_MEM_MODIFIED                0x01    /* Modified bit */
-#define VM_MEM_REFERENCED      0x02    /* Referenced bit */
+/* Clear modified and referenced bits */
+extern void                     pmap_clear_refmod(ppnum_t pn, unsigned int mask);
+#define VM_MEM_MODIFIED         0x01    /* Modified bit */
+#define VM_MEM_REFERENCED       0x02    /* Referenced bit */
+extern void                     pmap_clear_refmod_options(ppnum_t pn, unsigned int mask, unsigned int options, void *);
+
+
+extern void pmap_flush_context_init(pmap_flush_context *);
+extern void pmap_flush(pmap_flush_context *);

 
 /*
  *     Routines that operate on ranges of virtual addresses.
  */
 
 /*
  *     Routines that operate on ranges of virtual addresses.
  */

-extern void            pmap_protect(   /* Change protections. */
-                               pmap_t          map,
-                               vm_map_offset_t s,
-                               vm_map_offset_t e,
-                               vm_prot_t       prot);
-
-extern void            (pmap_pageable)(
-                               pmap_t          pmap,
-                               vm_map_offset_t start,
-                               vm_map_offset_t end,
-                               boolean_t       pageable);
-
-#ifndef NO_NESTED_PMAP
+extern void             pmap_protect(   /* Change protections. */
+       pmap_t          map,
+       vm_map_offset_t s,
+       vm_map_offset_t e,
+       vm_prot_t       prot);
+
+extern void             pmap_protect_options(   /* Change protections. */
+       pmap_t          map,
+       vm_map_offset_t s,
+       vm_map_offset_t e,
+       vm_prot_t       prot,
+       unsigned int    options,
+       void            *arg);
+
+extern void(pmap_pageable)(
+       pmap_t          pmap,
+       vm_map_offset_t start,
+       vm_map_offset_t end,
+       boolean_t       pageable);
+
+

 extern uint64_t pmap_nesting_size_min;
 extern uint64_t pmap_nesting_size_max;
 extern uint64_t pmap_nesting_size_min;
 extern uint64_t pmap_nesting_size_max;

-extern kern_return_t pmap_nest(pmap_t grand,
-                              pmap_t subord,
-                              addr64_t vstart,
-                              addr64_t nstart,
-                              uint64_t size);
-extern kern_return_t pmap_unnest(pmap_t grand,
-                                addr64_t vaddr,
-                                uint64_t size);
-#endif /* NO_NESTED_PMAP */
-
-#endif /* MACH_KERNEL_PRIVATE */
+
+extern kern_return_t pmap_nest(pmap_t,
+    pmap_t,
+    addr64_t,
+    addr64_t,
+    uint64_t);
+extern kern_return_t pmap_unnest(pmap_t,
+    addr64_t,
+    uint64_t);
+
+#define PMAP_UNNEST_CLEAN       1
+
+extern kern_return_t pmap_unnest_options(pmap_t,
+    addr64_t,
+    uint64_t,
+    unsigned int);
+extern boolean_t pmap_adjust_unnest_parameters(pmap_t, vm_map_offset_t *, vm_map_offset_t *);
+extern void             pmap_advise_pagezero_range(pmap_t, uint64_t);
+#endif  /* MACH_KERNEL_PRIVATE */
+
+extern boolean_t        pmap_is_noencrypt(ppnum_t);
+extern void             pmap_set_noencrypt(ppnum_t pn);
+extern void             pmap_clear_noencrypt(ppnum_t pn);

 
 /*
  * JMM - This portion is exported to other kernel components right now,
 
 /*
  * JMM - This portion is exported to other kernel components right now,


@@ -437,38 +643,184 @@ extern kern_return_t pmap_unnest(pmap_t grand,
  * is provided in a cleaner manner.
  */
 
  * is provided in a cleaner manner.
  */
 

-extern pmap_t  kernel_pmap;                    /* The kernel's map */
-#define                pmap_kernel()   (kernel_pmap)
+extern pmap_t   kernel_pmap;                    /* The kernel's map */
+#define         pmap_kernel()   (kernel_pmap)

 
 

-/* machine independent WIMG bits */
+#define VM_MEM_SUPERPAGE        0x100           /* map a superpage instead of a base page */
+#define VM_MEM_STACK            0x200

 
 

-#define VM_MEM_GUARDED                 0x1             /* (G) Guarded Storage */
-#define VM_MEM_COHERENT                0x2             /* (M) Memory Coherency */
-#define VM_MEM_NOT_CACHEABLE   0x4             /* (I) Cache Inhibit */
-#define VM_MEM_WRITE_THROUGH   0x8             /* (W) Write-Through */
+/* N.B. These use the same numerical space as the PMAP_EXPAND_OPTIONS
+ * definitions in i386/pmap_internal.h
+ */
+#define PMAP_CREATE_64BIT       0x1
+#if __x86_64__
+#define PMAP_CREATE_EPT         0x2
+#define PMAP_CREATE_KNOWN_FLAGS (PMAP_CREATE_64BIT | PMAP_CREATE_EPT)
+#endif

 
 

-#define VM_WIMG_MASK           0xFF
-#define VM_WIMG_USE_DEFAULT    0x80000000
+#define PMAP_OPTIONS_NOWAIT     0x1             /* don't block, return
+                                                * KERN_RESOURCE_SHORTAGE
+                                                * instead */
+#define PMAP_OPTIONS_NOENTER    0x2             /* expand pmap if needed
+                                                * but don't enter mapping
+                                                */
+#define PMAP_OPTIONS_COMPRESSOR 0x4             /* credit the compressor for
+                                                * this operation */
+#define PMAP_OPTIONS_INTERNAL   0x8             /* page from internal object */
+#define PMAP_OPTIONS_REUSABLE   0x10            /* page is "reusable" */
+#define PMAP_OPTIONS_NOFLUSH    0x20            /* delay flushing of pmap */
+#define PMAP_OPTIONS_NOREFMOD   0x40            /* don't need ref/mod on disconnect */
+#define PMAP_OPTIONS_ALT_ACCT   0x80            /* use alternate accounting scheme for page */
+#define PMAP_OPTIONS_REMOVE     0x100           /* removing a mapping */
+#define PMAP_OPTIONS_SET_REUSABLE   0x200       /* page is now "reusable" */
+#define PMAP_OPTIONS_CLEAR_REUSABLE 0x400       /* page no longer "reusable" */
+#define PMAP_OPTIONS_COMPRESSOR_IFF_MODIFIED 0x800 /* credit the compressor
+                                                   * iff page was modified */
+#define PMAP_OPTIONS_PROTECT_IMMEDIATE 0x1000   /* allow protections to be
+                                                * be upgraded */
+#define PMAP_OPTIONS_CLEAR_WRITE 0x2000
+
+
+#if     !defined(__LP64__)
+extern vm_offset_t      pmap_extract(pmap_t pmap,
+    vm_map_offset_t va);
+#endif
+extern void             pmap_change_wiring(     /* Specify pageability */
+       pmap_t          pmap,
+       vm_map_offset_t va,
+       boolean_t       wired);

 
 

-extern vm_offset_t     pmap_extract(pmap_t pmap,
-                               vm_map_offset_t va);
+/* LP64todo - switch to vm_map_offset_t when it grows */
+extern void             pmap_remove(    /* Remove mappings. */
+       pmap_t          map,
+       vm_map_offset_t s,
+       vm_map_offset_t e);
+
+extern void             pmap_remove_options(    /* Remove mappings. */
+       pmap_t          map,
+       vm_map_offset_t s,
+       vm_map_offset_t e,
+       int             options);
+
+extern void             fillPage(ppnum_t pa, unsigned int fill);
+
+#if defined(__LP64__)
+extern void pmap_pre_expand(pmap_t pmap, vm_map_offset_t vaddr);
+extern kern_return_t pmap_pre_expand_large(pmap_t pmap, vm_map_offset_t vaddr);
+extern vm_size_t pmap_query_pagesize(pmap_t map, vm_map_offset_t vaddr);
+#endif

 
 

-extern void            pmap_change_wiring(     /* Specify pageability */
-                               pmap_t          pmap,
-                               vm_map_offset_t va,
-                               boolean_t       wired);
+mach_vm_size_t pmap_query_resident(pmap_t pmap,
+    vm_map_offset_t s,
+    vm_map_offset_t e,
+    mach_vm_size_t *compressed_bytes_p);

 
 

-/* LP64todo - switch to vm_map_offset_t when it grows */
-extern void            pmap_remove(    /* Remove mappings. */
-                               pmap_t          map,
-                               vm_map_offset_t s,
-                               vm_map_offset_t e);
+/* Inform the pmap layer that there is a JIT entry in this map. */
+extern void pmap_set_jit_entitled(pmap_t pmap);

 
 

-extern void            fillPage(ppnum_t pa, unsigned int fill);
+/*
+ * Tell the pmap layer what range within the nested region the VM intends to
+ * use.
+ */
+extern void pmap_trim(pmap_t grand, pmap_t subord, addr64_t vstart, addr64_t nstart, uint64_t size);
+
+/*
+ * Dump page table contents into the specified buffer.  Returns the number of
+ * bytes copied, 0 if insufficient space, (size_t)-1 if unsupported.
+ * This is expected to only be called from kernel debugger context,
+ * so synchronization is not required.
+ */
+
+extern size_t pmap_dump_page_tables(pmap_t pmap, void *bufp, void *buf_end);
+
+/*
+ * Indicates if any special policy is applied to this protection by the pmap
+ * layer.
+ */
+bool pmap_has_prot_policy(vm_prot_t prot);
+
+/*
+ * Causes the pmap to return any available pages that it can return cheaply to
+ * the VM.
+ */
+uint64_t pmap_release_pages_fast(void);
+
+#define PMAP_QUERY_PAGE_PRESENT                 0x01
+#define PMAP_QUERY_PAGE_REUSABLE                0x02
+#define PMAP_QUERY_PAGE_INTERNAL                0x04
+#define PMAP_QUERY_PAGE_ALTACCT                 0x08
+#define PMAP_QUERY_PAGE_COMPRESSED              0x10
+#define PMAP_QUERY_PAGE_COMPRESSED_ALTACCT      0x20
+extern kern_return_t pmap_query_page_info(
+       pmap_t          pmap,
+       vm_map_offset_t va,
+       int             *disp);
+
+#if CONFIG_PGTRACE
+int pmap_pgtrace_add_page(pmap_t pmap, vm_map_offset_t start, vm_map_offset_t end);
+int pmap_pgtrace_delete_page(pmap_t pmap, vm_map_offset_t start, vm_map_offset_t end);
+kern_return_t pmap_pgtrace_fault(pmap_t pmap, vm_map_offset_t va, arm_saved_state_t *ss);
+#endif
+
+
+#ifdef PLATFORM_BridgeOS
+struct pmap_legacy_trust_cache {
+       struct pmap_legacy_trust_cache *next;
+       uuid_t uuid;
+       uint32_t num_hashes;
+       uint8_t hashes[][CS_CDHASH_LEN];
+};
+#else
+struct pmap_legacy_trust_cache;
+#endif

 
 

-extern void pmap_map_sharedpage(task_t task, pmap_t pmap);
-extern void pmap_unmap_sharedpage(pmap_t pmap);
+extern kern_return_t pmap_load_legacy_trust_cache(struct pmap_legacy_trust_cache *trust_cache,
+    const vm_size_t trust_cache_len);
+
+struct pmap_image4_trust_cache {
+       // Filled by pmap layer.
+       struct pmap_image4_trust_cache const *next;             // linked list linkage
+       struct trust_cache_module1 const *module;                       // pointer into module (within data below)
+
+       // Filled by caller.
+       // data is either an image4,
+       // or just the trust cache payload itself if the image4 manifest is external.
+       size_t data_len;
+       uint8_t const data[];
+};
+
+typedef enum {
+       PMAP_TC_SUCCESS = 0,
+       PMAP_TC_UNKNOWN_FORMAT = -1,
+       PMAP_TC_TOO_SMALL_FOR_HEADER = -2,
+       PMAP_TC_TOO_SMALL_FOR_ENTRIES = -3,
+       PMAP_TC_UNKNOWN_VERSION = -4,
+       PMAP_TC_ALREADY_LOADED = -5,
+       PMAP_TC_TOO_BIG = -6,
+       PMAP_TC_RESOURCE_SHORTAGE = -7,
+       PMAP_TC_MANIFEST_TOO_BIG = -8,
+} pmap_tc_ret_t;
+
+extern pmap_tc_ret_t pmap_load_image4_trust_cache(
+       struct pmap_image4_trust_cache *trust_cache, vm_size_t trust_cache_len,
+       uint8_t const *img4_manifest,
+       vm_size_t img4_manifest_buffer_len,
+       vm_size_t img4_manifest_actual_len,
+       bool dry_run);
+
+extern bool pmap_is_trust_cache_loaded(const uuid_t uuid);
+extern uint32_t pmap_lookup_in_static_trust_cache(const uint8_t cdhash[CS_CDHASH_LEN]);
+extern bool pmap_lookup_in_loaded_trust_caches(const uint8_t cdhash[CS_CDHASH_LEN]);
+
+extern bool pmap_in_ppl(void);
+
+extern void *pmap_claim_reserved_ppl_page(void);
+extern void pmap_free_reserved_ppl_page(void *kva);
+
+extern void pmap_ledger_alloc_init(size_t);
+extern ledger_t pmap_ledger_alloc(void);
+extern void pmap_ledger_free(ledger_t);

 
 #endif  /* KERNEL_PRIVATE */
 
 
 #endif  /* KERNEL_PRIVATE */
 

-#endif /* _VM_PMAP_H_ */
+#endif  /* _VM_PMAP_H_ */