osfmk/vm/pmap.h

   1 /*
   2  * Copyright (c) 2000-2020 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 /*
  29  * @OSF_COPYRIGHT@
  30  */
  31 /*
  32  * Mach Operating System
  33  * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
  34  * All Rights Reserved.
  35  *
  36  * Permission to use, copy, modify and distribute this software and its
  37  * documentation is hereby granted, provided that both the copyright
  38  * notice and this permission notice appear in all copies of the
  39  * software, derivative works or modified versions, and any portions
  40  * thereof, and that both notices appear in supporting documentation.
  41  *
  42  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  43  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
  44  * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
  45  *
  46  * Carnegie Mellon requests users of this software to return to
  47  *
  48  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  49  *  School of Computer Science
  50  *  Carnegie Mellon University
  51  *  Pittsburgh PA 15213-3890
  52  *
  53  * any improvements or extensions that they make and grant Carnegie Mellon
  54  * the rights to redistribute these changes.
  55  */
  56 /*
  57  */
  58 /*
  59  *      File:   vm/pmap.h
  60  *      Author: Avadis Tevanian, Jr.
  61  *      Date:   1985
  62  *
  63  *      Machine address mapping definitions -- machine-independent
  64  *      section.  [For machine-dependent section, see "machine/pmap.h".]
  65  */
  66
  67 #ifndef _VM_PMAP_H_
  68 #define _VM_PMAP_H_
  69
  70 #include <mach/kern_return.h>
  71 #include <mach/vm_param.h>
  72 #include <mach/vm_types.h>
  73 #include <mach/vm_attributes.h>
  74 #include <mach/boolean.h>
  75 #include <mach/vm_prot.h>
  76
  77 #include <kern/trustcache.h>
  78
  79
  80 #ifdef  KERNEL_PRIVATE
  81
  82 /*
  83  *      The following is a description of the interface to the
  84  *      machine-dependent "physical map" data structure.  The module
  85  *      must provide a "pmap_t" data type that represents the
  86  *      set of valid virtual-to-physical addresses for one user
  87  *      address space.  [The kernel address space is represented
  88  *      by a distinguished "pmap_t".]  The routines described manage
  89  *      this type, install and update virtual-to-physical mappings,
  90  *      and perform operations on physical addresses common to
  91  *      many address spaces.
  92  */
  93
  94 /* Copy between a physical page and a virtual address */
  95 /* LP64todo - switch to vm_map_offset_t when it grows */
  96 extern kern_return_t    copypv(
  97         addr64_t source,
  98         addr64_t sink,
  99         unsigned int size,
 100         int which);
 101 #define cppvPsnk        1
 102 #define cppvPsnkb      31
 103 #define cppvPsrc        2
 104 #define cppvPsrcb      30
 105 #define cppvFsnk        4
 106 #define cppvFsnkb      29
 107 #define cppvFsrc        8
 108 #define cppvFsrcb      28
 109 #define cppvNoModSnk   16
 110 #define cppvNoModSnkb  27
 111 #define cppvNoRefSrc   32
 112 #define cppvNoRefSrcb  26
 113 #define cppvKmap       64       /* Use the kernel's vm_map */
 114 #define cppvKmapb      25
 115
 116 extern boolean_t pmap_has_managed_page(ppnum_t first, ppnum_t last);
 117
 118 #ifdef  MACH_KERNEL_PRIVATE
 119
 120 #include <mach_assert.h>
 121
 122 #include <machine/pmap.h>
 123 #include <vm/memory_types.h>
 124
 125 /*
 126  *      Routines used for initialization.
 127  *      There is traditionally also a pmap_bootstrap,
 128  *      used very early by machine-dependent code,
 129  *      but it is not part of the interface.
 130  *
 131  *      LP64todo -
 132  *      These interfaces are tied to the size of the
 133  *      kernel pmap - and therefore use the "local"
 134  *      vm_offset_t, etc... types.
 135  */
 136
 137 extern void *pmap_steal_memory(vm_size_t size); /* Early memory allocation */
 138 extern void *pmap_steal_freeable_memory(vm_size_t size); /* Early memory allocation */
 139
 140 extern uint_t pmap_free_pages(void); /* report remaining unused physical pages */
 141
 142 extern void pmap_startup(vm_offset_t *startp, vm_offset_t *endp); /* allocate vm_page structs */
 143
 144 extern void pmap_init(void); /* Initialization, once we have kernel virtual memory.  */
 145
 146 extern void mapping_adjust(void); /* Adjust free mapping count */
 147
 148 extern void mapping_free_prime(void); /* Primes the mapping block release list */
 149
 150 #ifndef MACHINE_PAGES
 151 /*
 152  *      If machine/pmap.h defines MACHINE_PAGES, it must implement
 153  *      the above functions.  The pmap module has complete control.
 154  *      Otherwise, it must implement the following functions:
 155  *              pmap_free_pages
 156  *              pmap_virtual_space
 157  *              pmap_next_page
 158  *              pmap_init
 159  *      and vm/vm_resident.c implements pmap_steal_memory and pmap_startup
 160  *      using pmap_free_pages, pmap_next_page, pmap_virtual_space,
 161  *      and pmap_enter.  pmap_free_pages may over-estimate the number
 162  *      of unused physical pages, and pmap_next_page may return FALSE
 163  *      to indicate that there are no more unused pages to return.
 164  *      However, for best performance pmap_free_pages should be accurate.
 165  */
 166
 167 /*
 168  * Routines to return the next unused physical page.
 169  */
 170 extern boolean_t pmap_next_page(ppnum_t *pnum);
 171 extern boolean_t pmap_next_page_hi(ppnum_t *pnum, boolean_t might_free);
 172 #ifdef __x86_64__
 173 extern kern_return_t pmap_next_page_large(ppnum_t *pnum);
 174 extern void pmap_hi_pages_done(void);
 175 #endif
 176
 177 /*
 178  * Report virtual space available for the kernel.
 179  */
 180 extern void pmap_virtual_space(
 181         vm_offset_t     *virtual_start,
 182         vm_offset_t     *virtual_end);
 183 #endif  /* MACHINE_PAGES */
 184
 185 /*
 186  * Routines to manage the physical map data structure.
 187  */
 188 extern pmap_t           pmap_create_options(    /* Create a pmap_t. */
 189         ledger_t        ledger,
 190         vm_map_size_t   size,
 191         unsigned int    flags);
 192
 193 extern pmap_t(pmap_kernel)(void);               /* Return the kernel's pmap */
 194 extern void             pmap_reference(pmap_t pmap);    /* Gain a reference. */
 195 extern void             pmap_destroy(pmap_t pmap); /* Release a reference. */
 196 extern void             pmap_switch(pmap_t);
 197 extern void             pmap_require(pmap_t pmap);
 198
 199 #if MACH_ASSERT
 200 extern void pmap_set_process(pmap_t pmap,
 201     int pid,
 202     char *procname);
 203 #endif /* MACH_ASSERT */
 204
 205 extern kern_return_t    pmap_enter(     /* Enter a mapping */
 206         pmap_t          pmap,
 207         vm_map_offset_t v,
 208         ppnum_t         pn,
 209         vm_prot_t       prot,
 210         vm_prot_t       fault_type,
 211         unsigned int    flags,
 212         boolean_t       wired);
 213
 214 extern kern_return_t    pmap_enter_options(
 215         pmap_t pmap,
 216         vm_map_offset_t v,
 217         ppnum_t pn,
 218         vm_prot_t prot,
 219         vm_prot_t fault_type,
 220         unsigned int flags,
 221         boolean_t wired,
 222         unsigned int options,
 223         void *arg);
 224 extern kern_return_t    pmap_enter_options_addr(
 225         pmap_t pmap,
 226         vm_map_offset_t v,
 227         pmap_paddr_t pa,
 228         vm_prot_t prot,
 229         vm_prot_t fault_type,
 230         unsigned int flags,
 231         boolean_t wired,
 232         unsigned int options,
 233         void *arg);
 234
 235 extern void             pmap_remove_some_phys(
 236         pmap_t          pmap,
 237         ppnum_t         pn);
 238
 239 extern void             pmap_lock_phys_page(
 240         ppnum_t         pn);
 241
 242 extern void             pmap_unlock_phys_page(
 243         ppnum_t         pn);
 244
 245
 246 /*
 247  *      Routines that operate on physical addresses.
 248  */
 249
 250 extern void             pmap_page_protect(      /* Restrict access to page. */
 251         ppnum_t phys,
 252         vm_prot_t       prot);
 253
 254 extern void             pmap_page_protect_options(      /* Restrict access to page. */
 255         ppnum_t phys,
 256         vm_prot_t       prot,
 257         unsigned int    options,
 258         void            *arg);
 259
 260 extern void(pmap_zero_page)(
 261         ppnum_t         pn);
 262
 263 extern void(pmap_zero_part_page)(
 264         ppnum_t         pn,
 265         vm_offset_t     offset,
 266         vm_size_t       len);
 267
 268 extern void(pmap_copy_page)(
 269         ppnum_t         src,
 270         ppnum_t         dest);
 271
 272 extern void(pmap_copy_part_page)(
 273         ppnum_t         src,
 274         vm_offset_t     src_offset,
 275         ppnum_t         dst,
 276         vm_offset_t     dst_offset,
 277         vm_size_t       len);
 278
 279 extern void(pmap_copy_part_lpage)(
 280         vm_offset_t     src,
 281         ppnum_t         dst,
 282         vm_offset_t     dst_offset,
 283         vm_size_t       len);
 284
 285 extern void(pmap_copy_part_rpage)(
 286         ppnum_t         src,
 287         vm_offset_t     src_offset,
 288         vm_offset_t     dst,
 289         vm_size_t       len);
 290
 291 extern unsigned int(pmap_disconnect)(   /* disconnect mappings and return reference and change */
 292         ppnum_t         phys);
 293
 294 extern unsigned int(pmap_disconnect_options)(   /* disconnect mappings and return reference and change */
 295         ppnum_t         phys,
 296         unsigned int    options,
 297         void            *arg);
 298
 299 extern kern_return_t(pmap_attribute_cache_sync)(      /* Flush appropriate
 300                                                        * cache based on
 301                                                        * page number sent */
 302         ppnum_t         pn,
 303         vm_size_t       size,
 304         vm_machine_attribute_t attribute,
 305         vm_machine_attribute_val_t* value);
 306
 307 extern unsigned int(pmap_cache_attributes)(
 308         ppnum_t         pn);
 309
 310 /*
 311  * Set (override) cache attributes for the specified physical page
 312  */
 313 extern  void            pmap_set_cache_attributes(
 314         ppnum_t,
 315         unsigned int);
 316
 317 extern void            *pmap_map_compressor_page(
 318         ppnum_t);
 319
 320 extern void             pmap_unmap_compressor_page(
 321         ppnum_t,
 322         void*);
 323
 324 #if defined(__arm__) || defined(__arm64__)
 325 /* ARM64_TODO */
 326 extern  boolean_t       pmap_batch_set_cache_attributes(
 327         ppnum_t,
 328         unsigned int,
 329         unsigned int,
 330         unsigned int,
 331         boolean_t,
 332         unsigned int*);
 333 #endif
 334 extern void pmap_sync_page_data_phys(ppnum_t pa);
 335 extern void pmap_sync_page_attributes_phys(ppnum_t pa);
 336
 337 /*
 338  * debug/assertions. pmap_verify_free returns true iff
 339  * the given physical page is mapped into no pmap.
 340  * pmap_assert_free() will panic() if pn is not free.
 341  */
 342 extern boolean_t        pmap_verify_free(ppnum_t pn);
 343 #if MACH_ASSERT
 344 extern void pmap_assert_free(ppnum_t pn);
 345 #endif
 346
 347 /*
 348  *      Statistics routines
 349  */
 350 extern int(pmap_compressed)(pmap_t pmap);
 351 extern int(pmap_resident_count)(pmap_t pmap);
 352 extern int(pmap_resident_max)(pmap_t pmap);
 353
 354 /*
 355  *      Sundry required (internal) routines
 356  */
 357 #ifdef CURRENTLY_UNUSED_AND_UNTESTED
 358 extern void             pmap_collect(pmap_t pmap);/* Perform garbage
 359                                                    * collection, if any */
 360 #endif
 361 /*
 362  *      Optional routines
 363  */
 364 extern void(pmap_copy)(                         /* Copy range of mappings,
 365                                                  * if desired. */
 366         pmap_t          dest,
 367         pmap_t          source,
 368         vm_map_offset_t dest_va,
 369         vm_map_size_t   size,
 370         vm_map_offset_t source_va);
 371
 372 extern kern_return_t(pmap_attribute)(           /* Get/Set special memory
 373                                                  * attributes */
 374         pmap_t          pmap,
 375         vm_map_offset_t va,
 376         vm_map_size_t   size,
 377         vm_machine_attribute_t  attribute,
 378         vm_machine_attribute_val_t* value);
 379
 380 /*
 381  * Routines defined as macros.
 382  */
 383 #ifndef PMAP_ACTIVATE_USER
 384 #ifndef PMAP_ACTIVATE
 385 #define PMAP_ACTIVATE_USER(thr, cpu)
 386 #else   /* PMAP_ACTIVATE */
 387 #define PMAP_ACTIVATE_USER(thr, cpu) {                  \
 388         pmap_t  pmap;                                           \
 389                                                                 \
 390         pmap = (thr)->map->pmap;                                \
 391         if (pmap != pmap_kernel())                              \
 392                 PMAP_ACTIVATE(pmap, (thr), (cpu));              \
 393 }
 394 #endif  /* PMAP_ACTIVATE */
 395 #endif  /* PMAP_ACTIVATE_USER */
 396
 397 #ifndef PMAP_DEACTIVATE_USER
 398 #ifndef PMAP_DEACTIVATE
 399 #define PMAP_DEACTIVATE_USER(thr, cpu)
 400 #else   /* PMAP_DEACTIVATE */
 401 #define PMAP_DEACTIVATE_USER(thr, cpu) {                        \
 402         pmap_t  pmap;                                           \
 403                                                                 \
 404         pmap = (thr)->map->pmap;                                \
 405         if ((pmap) != pmap_kernel())                    \
 406                 PMAP_DEACTIVATE(pmap, (thr), (cpu));    \
 407 }
 408 #endif  /* PMAP_DEACTIVATE */
 409 #endif  /* PMAP_DEACTIVATE_USER */
 410
 411 #ifndef PMAP_ACTIVATE_KERNEL
 412 #ifndef PMAP_ACTIVATE
 413 #define PMAP_ACTIVATE_KERNEL(cpu)
 414 #else   /* PMAP_ACTIVATE */
 415 #define PMAP_ACTIVATE_KERNEL(cpu)                       \
 416                 PMAP_ACTIVATE(pmap_kernel(), THREAD_NULL, cpu)
 417 #endif  /* PMAP_ACTIVATE */
 418 #endif  /* PMAP_ACTIVATE_KERNEL */
 419
 420 #ifndef PMAP_DEACTIVATE_KERNEL
 421 #ifndef PMAP_DEACTIVATE
 422 #define PMAP_DEACTIVATE_KERNEL(cpu)
 423 #else   /* PMAP_DEACTIVATE */
 424 #define PMAP_DEACTIVATE_KERNEL(cpu)                     \
 425                 PMAP_DEACTIVATE(pmap_kernel(), THREAD_NULL, cpu)
 426 #endif  /* PMAP_DEACTIVATE */
 427 #endif  /* PMAP_DEACTIVATE_KERNEL */
 428
 429 #ifndef PMAP_ENTER
 430 /*
 431  *      Macro to be used in place of pmap_enter()
 432  */
 433 #define PMAP_ENTER(pmap, virtual_address, page, protection, fault_type, \
 434             flags, wired, result)                                \
 435         MACRO_BEGIN                                                     \
 436         pmap_t          __pmap = (pmap);                                \
 437         vm_page_t       __page = (page);                                \
 438         int             __options = 0;                                  \
 439         vm_object_t     __obj;                                          \
 440                                                                         \
 441         PMAP_ENTER_CHECK(__pmap, __page)                                \
 442         __obj = VM_PAGE_OBJECT(__page);                                 \
 443         if (__obj->internal) {                                          \
 444                 __options |= PMAP_OPTIONS_INTERNAL;                     \
 445         }                                                               \
 446         if (__page->vmp_reusable || __obj->all_reusable) {              \
 447                 __options |= PMAP_OPTIONS_REUSABLE;                     \
 448         }                                                               \
 449         result = pmap_enter_options(__pmap,                             \
 450                                     (virtual_address),                  \
 451                                     VM_PAGE_GET_PHYS_PAGE(__page),      \
 452                                     (protection),                               \
 453                                     (fault_type),                               \
 454                                     (flags),                            \
 455                                     (wired),                            \
 456                                     __options,                          \
 457                                     NULL);                              \
 458         MACRO_END
 459 #endif  /* !PMAP_ENTER */
 460
 461 #ifndef PMAP_ENTER_OPTIONS
 462 #define PMAP_ENTER_OPTIONS(pmap, virtual_address, fault_phys_offset,   \
 463             page, protection,                                           \
 464             fault_type, flags, wired, options, result)                  \
 465         MACRO_BEGIN                                                     \
 466         pmap_t          __pmap = (pmap);                                \
 467         vm_page_t       __page = (page);                                \
 468         int             __extra_options = 0;                            \
 469         vm_object_t     __obj;                                          \
 470                                                                         \
 471         PMAP_ENTER_CHECK(__pmap, __page)                                \
 472         __obj = VM_PAGE_OBJECT(__page);                                 \
 473         if (__obj->internal) {                                          \
 474                 __extra_options |= PMAP_OPTIONS_INTERNAL;               \
 475         }                                                               \
 476         if (__page->vmp_reusable || __obj->all_reusable) {              \
 477                 __extra_options |= PMAP_OPTIONS_REUSABLE;               \
 478         }                                                               \
 479         result = pmap_enter_options_addr(__pmap,                        \
 480                                     (virtual_address),                  \
 481                                     (((pmap_paddr_t)                    \
 482                                       VM_PAGE_GET_PHYS_PAGE(__page)     \
 483                                       << PAGE_SHIFT)                    \
 484                                      + fault_phys_offset),             \
 485                                     (protection),                       \
 486                                     (fault_type),                       \
 487                                     (flags),                            \
 488                                     (wired),                            \
 489                                     (options) | __extra_options,        \
 490                                     NULL);                              \
 491         MACRO_END
 492 #endif  /* !PMAP_ENTER_OPTIONS */
 493
 494 #ifndef PMAP_SET_CACHE_ATTR
 495 #define PMAP_SET_CACHE_ATTR(mem, object, cache_attr, batch_pmap_op)             \
 496         MACRO_BEGIN                                                             \
 497                 if (!batch_pmap_op) {                                           \
 498                         pmap_set_cache_attributes(VM_PAGE_GET_PHYS_PAGE(mem), cache_attr); \
 499                         object->set_cache_attr = TRUE;                          \
 500                 }                                                               \
 501         MACRO_END
 502 #endif  /* PMAP_SET_CACHE_ATTR */
 503
 504 #ifndef PMAP_BATCH_SET_CACHE_ATTR
 505 #if     defined(__arm__) || defined(__arm64__)
 506 #define PMAP_BATCH_SET_CACHE_ATTR(object, user_page_list,                       \
 507             cache_attr, num_pages, batch_pmap_op)   \
 508         MACRO_BEGIN                                                             \
 509                 if ((batch_pmap_op)) {                                          \
 510                         unsigned int __page_idx=0;                              \
 511                         unsigned int res=0;                                     \
 512                         boolean_t batch=TRUE;                                   \
 513                         while (__page_idx < (num_pages)) {                      \
 514                                 if (!pmap_batch_set_cache_attributes(           \
 515                                         user_page_list[__page_idx].phys_addr,   \
 516                                         (cache_attr),                           \
 517                                         (num_pages),                            \
 518                                         (__page_idx),                           \
 519                                         FALSE,                                  \
 520                                         (&res))) {                              \
 521                                         batch = FALSE;                          \
 522                                         break;                                  \
 523                                 }                                               \
 524                                 __page_idx++;                                   \
 525                         }                                                       \
 526                         __page_idx=0;                                           \
 527                         res=0;                                                  \
 528                         while (__page_idx < (num_pages)) {                      \
 529                                 if (batch)                                      \
 530                                         (void)pmap_batch_set_cache_attributes(  \
 531                                         user_page_list[__page_idx].phys_addr,   \
 532                                         (cache_attr),                           \
 533                                         (num_pages),                            \
 534                                         (__page_idx),                           \
 535                                         TRUE,                                   \
 536                                         (&res));                                \
 537                                 else                                            \
 538                                         pmap_set_cache_attributes(              \
 539                                         user_page_list[__page_idx].phys_addr,   \
 540                                         (cache_attr));                          \
 541                                         __page_idx++;                           \
 542                         }                                                       \
 543                         (object)->set_cache_attr = TRUE;                        \
 544                 }                                                               \
 545         MACRO_END
 546 #else
 547 #define PMAP_BATCH_SET_CACHE_ATTR(object, user_page_list,                       \
 548             cache_attr, num_pages, batch_pmap_op)   \
 549         MACRO_BEGIN                                                             \
 550                 if ((batch_pmap_op)) {                                          \
 551                         unsigned int __page_idx=0;                              \
 552                         while (__page_idx < (num_pages)) {                      \
 553                                 pmap_set_cache_attributes(                      \
 554                                         user_page_list[__page_idx].phys_addr,   \
 555                                         (cache_attr));                          \
 556                                 __page_idx++;                                   \
 557                         }                                                       \
 558                         (object)->set_cache_attr = TRUE;                        \
 559                 }                                                               \
 560         MACRO_END
 561 #endif
 562 #endif  /* PMAP_BATCH_SET_CACHE_ATTR */
 563
 564 #define PMAP_ENTER_CHECK(pmap, page)                                    \
 565 {                                                                       \
 566         if ((page)->vmp_error) {                                        \
 567                 panic("VM page %p should not have an error\n",          \
 568                         (page));                                        \
 569         }                                                               \
 570 }
 571
 572 /*
 573  *      Routines to manage reference/modify bits based on
 574  *      physical addresses, simulating them if not provided
 575  *      by the hardware.
 576  */
 577 struct pfc {
 578         long    pfc_cpus;
 579         long    pfc_invalid_global;
 580 };
 581
 582 typedef struct pfc      pmap_flush_context;
 583
 584 /* Clear reference bit */
 585 extern void             pmap_clear_reference(ppnum_t     pn);
 586 /* Return reference bit */
 587 extern boolean_t(pmap_is_referenced)(ppnum_t     pn);
 588 /* Set modify bit */
 589 extern void             pmap_set_modify(ppnum_t  pn);
 590 /* Clear modify bit */
 591 extern void             pmap_clear_modify(ppnum_t pn);
 592 /* Return modify bit */
 593 extern boolean_t        pmap_is_modified(ppnum_t pn);
 594 /* Return modified and referenced bits */
 595 extern unsigned int pmap_get_refmod(ppnum_t pn);
 596 /* Clear modified and referenced bits */
 597 extern void                     pmap_clear_refmod(ppnum_t pn, unsigned int mask);
 598 #define VM_MEM_MODIFIED         0x01    /* Modified bit */
 599 #define VM_MEM_REFERENCED       0x02    /* Referenced bit */
 600 extern void                     pmap_clear_refmod_options(ppnum_t pn, unsigned int mask, unsigned int options, void *);
 601
 602 /*
 603  * Clears the reference and/or modified bits on a range of virtually
 604  * contiguous pages.
 605  * It returns true if the operation succeeded. If it returns false,
 606  * nothing has been modified.
 607  * This operation is only supported on some platforms, so callers MUST
 608  * handle the case where it returns false.
 609  */
 610 extern bool
 611 pmap_clear_refmod_range_options(
 612         pmap_t pmap,
 613         vm_map_address_t start,
 614         vm_map_address_t end,
 615         unsigned int mask,
 616         unsigned int options);
 617
 618
 619 extern void pmap_flush_context_init(pmap_flush_context *);
 620 extern void pmap_flush(pmap_flush_context *);
 621
 622 /*
 623  *      Routines that operate on ranges of virtual addresses.
 624  */
 625 extern void             pmap_protect(   /* Change protections. */
 626         pmap_t          map,
 627         vm_map_offset_t s,
 628         vm_map_offset_t e,
 629         vm_prot_t       prot);
 630
 631 extern void             pmap_protect_options(   /* Change protections. */
 632         pmap_t          map,
 633         vm_map_offset_t s,
 634         vm_map_offset_t e,
 635         vm_prot_t       prot,
 636         unsigned int    options,
 637         void            *arg);
 638
 639 extern void(pmap_pageable)(
 640         pmap_t          pmap,
 641         vm_map_offset_t start,
 642         vm_map_offset_t end,
 643         boolean_t       pageable);
 644
 645 extern uint64_t pmap_shared_region_size_min(pmap_t map);
 646
 647 /* TODO: <rdar://problem/65247502> Completely remove pmap_nesting_size_max() */
 648 extern uint64_t pmap_nesting_size_max(pmap_t map);
 649
 650 extern kern_return_t pmap_nest(pmap_t,
 651     pmap_t,
 652     addr64_t,
 653     uint64_t);
 654 extern kern_return_t pmap_unnest(pmap_t,
 655     addr64_t,
 656     uint64_t);
 657
 658 #define PMAP_UNNEST_CLEAN       1
 659
 660 extern kern_return_t pmap_unnest_options(pmap_t,
 661     addr64_t,
 662     uint64_t,
 663     unsigned int);
 664 extern boolean_t pmap_adjust_unnest_parameters(pmap_t, vm_map_offset_t *, vm_map_offset_t *);
 665 extern void             pmap_advise_pagezero_range(pmap_t, uint64_t);
 666 #endif  /* MACH_KERNEL_PRIVATE */
 667
 668 extern boolean_t        pmap_is_noencrypt(ppnum_t);
 669 extern void             pmap_set_noencrypt(ppnum_t pn);
 670 extern void             pmap_clear_noencrypt(ppnum_t pn);
 671
 672 /*
 673  * JMM - This portion is exported to other kernel components right now,
 674  * but will be pulled back in the future when the needed functionality
 675  * is provided in a cleaner manner.
 676  */
 677
 678 extern pmap_t   kernel_pmap;                    /* The kernel's map */
 679 #define         pmap_kernel()   (kernel_pmap)
 680
 681 #define VM_MEM_SUPERPAGE        0x100           /* map a superpage instead of a base page */
 682 #define VM_MEM_STACK            0x200
 683
 684 /* N.B. These use the same numerical space as the PMAP_EXPAND_OPTIONS
 685  * definitions in i386/pmap_internal.h
 686  */
 687 #define PMAP_CREATE_64BIT          0x1
 688
 689 #if __x86_64__
 690
 691 #define PMAP_CREATE_EPT            0x2
 692 #define PMAP_CREATE_KNOWN_FLAGS (PMAP_CREATE_64BIT | PMAP_CREATE_EPT)
 693
 694 #else
 695
 696 #define PMAP_CREATE_STAGE2         0
 697 #define PMAP_CREATE_DISABLE_JOP    0
 698 #if __ARM_MIXED_PAGE_SIZE__
 699 #define PMAP_CREATE_FORCE_4K_PAGES 0x8
 700 #else
 701 #define PMAP_CREATE_FORCE_4K_PAGES 0
 702 #endif /* __ARM_MIXED_PAGE_SIZE__ */
 703 #if __arm64__
 704 #define PMAP_CREATE_X86_64         0
 705 #else
 706 #define PMAP_CREATE_X86_64         0
 707 #endif
 708
 709 /* Define PMAP_CREATE_KNOWN_FLAGS in terms of optional flags */
 710 #define PMAP_CREATE_KNOWN_FLAGS (PMAP_CREATE_64BIT | PMAP_CREATE_STAGE2 | PMAP_CREATE_DISABLE_JOP | PMAP_CREATE_FORCE_4K_PAGES | PMAP_CREATE_X86_64)
 711
 712 #endif /* __x86_64__ */
 713
 714 #define PMAP_OPTIONS_NOWAIT     0x1             /* don't block, return
 715                                                  * KERN_RESOURCE_SHORTAGE
 716                                                  * instead */
 717 #define PMAP_OPTIONS_NOENTER    0x2             /* expand pmap if needed
 718                                                  * but don't enter mapping
 719                                                  */
 720 #define PMAP_OPTIONS_COMPRESSOR 0x4             /* credit the compressor for
 721                                                  * this operation */
 722 #define PMAP_OPTIONS_INTERNAL   0x8             /* page from internal object */
 723 #define PMAP_OPTIONS_REUSABLE   0x10            /* page is "reusable" */
 724 #define PMAP_OPTIONS_NOFLUSH    0x20            /* delay flushing of pmap */
 725 #define PMAP_OPTIONS_NOREFMOD   0x40            /* don't need ref/mod on disconnect */
 726 #define PMAP_OPTIONS_ALT_ACCT   0x80            /* use alternate accounting scheme for page */
 727 #define PMAP_OPTIONS_REMOVE     0x100           /* removing a mapping */
 728 #define PMAP_OPTIONS_SET_REUSABLE   0x200       /* page is now "reusable" */
 729 #define PMAP_OPTIONS_CLEAR_REUSABLE 0x400       /* page no longer "reusable" */
 730 #define PMAP_OPTIONS_COMPRESSOR_IFF_MODIFIED 0x800 /* credit the compressor
 731                                                     * iff page was modified */
 732 #define PMAP_OPTIONS_PROTECT_IMMEDIATE 0x1000   /* allow protections to be
 733                                                  * be upgraded */
 734 #define PMAP_OPTIONS_CLEAR_WRITE 0x2000
 735 #define PMAP_OPTIONS_TRANSLATED_ALLOW_EXECUTE 0x4000 /* Honor execute for translated processes */
 736 #if defined(__arm__) || defined(__arm64__)
 737 #define PMAP_OPTIONS_FF_LOCKED  0x8000
 738 #define PMAP_OPTIONS_FF_WIRED   0x10000
 739 #endif
 740
 741 #if     !defined(__LP64__)
 742 extern vm_offset_t      pmap_extract(pmap_t pmap,
 743     vm_map_offset_t va);
 744 #endif
 745 extern void             pmap_change_wiring(     /* Specify pageability */
 746         pmap_t          pmap,
 747         vm_map_offset_t va,
 748         boolean_t       wired);
 749
 750 /* LP64todo - switch to vm_map_offset_t when it grows */
 751 extern void             pmap_remove(    /* Remove mappings. */
 752         pmap_t          map,
 753         vm_map_offset_t s,
 754         vm_map_offset_t e);
 755
 756 extern void             pmap_remove_options(    /* Remove mappings. */
 757         pmap_t          map,
 758         vm_map_offset_t s,
 759         vm_map_offset_t e,
 760         int             options);
 761
 762 extern void             fillPage(ppnum_t pa, unsigned int fill);
 763
 764 #if defined(__LP64__)
 765 extern void pmap_pre_expand(pmap_t pmap, vm_map_offset_t vaddr);
 766 extern kern_return_t pmap_pre_expand_large(pmap_t pmap, vm_map_offset_t vaddr);
 767 extern vm_size_t pmap_query_pagesize(pmap_t map, vm_map_offset_t vaddr);
 768 #endif
 769
 770 mach_vm_size_t pmap_query_resident(pmap_t pmap,
 771     vm_map_offset_t s,
 772     vm_map_offset_t e,
 773     mach_vm_size_t *compressed_bytes_p);
 774
 775 extern void pmap_set_vm_map_cs_enforced(pmap_t pmap, bool new_value);
 776 extern bool pmap_get_vm_map_cs_enforced(pmap_t pmap);
 777
 778 /* Inform the pmap layer that there is a JIT entry in this map. */
 779 extern void pmap_set_jit_entitled(pmap_t pmap);
 780
 781 /* Ask the pmap layer if there is a JIT entry in this map. */
 782 extern bool pmap_get_jit_entitled(pmap_t pmap);
 783
 784 /*
 785  * Tell the pmap layer what range within the nested region the VM intends to
 786  * use.
 787  */
 788 extern void pmap_trim(pmap_t grand, pmap_t subord, addr64_t vstart, uint64_t size);
 789
 790 /*
 791  * Dump page table contents into the specified buffer.  Returns KERN_INSUFFICIENT_BUFFER_SIZE
 792  * if insufficient space, KERN_NOT_SUPPORTED if unsupported in the current configuration.
 793  * This is expected to only be called from kernel debugger context,
 794  * so synchronization is not required.
 795  */
 796
 797 extern kern_return_t pmap_dump_page_tables(pmap_t pmap, void *bufp, void *buf_end, unsigned int level_mask, size_t *bytes_copied);
 798
 799 /*
 800  * Indicates if any special policy is applied to this protection by the pmap
 801  * layer.
 802  */
 803 bool pmap_has_prot_policy(pmap_t pmap, bool translated_allow_execute, vm_prot_t prot);
 804
 805 /*
 806  * Causes the pmap to return any available pages that it can return cheaply to
 807  * the VM.
 808  */
 809 uint64_t pmap_release_pages_fast(void);
 810
 811 #define PMAP_QUERY_PAGE_PRESENT                 0x01
 812 #define PMAP_QUERY_PAGE_REUSABLE                0x02
 813 #define PMAP_QUERY_PAGE_INTERNAL                0x04
 814 #define PMAP_QUERY_PAGE_ALTACCT                 0x08
 815 #define PMAP_QUERY_PAGE_COMPRESSED              0x10
 816 #define PMAP_QUERY_PAGE_COMPRESSED_ALTACCT      0x20
 817 extern kern_return_t pmap_query_page_info(
 818         pmap_t          pmap,
 819         vm_map_offset_t va,
 820         int             *disp);
 821
 822 #if CONFIG_PGTRACE
 823 int pmap_pgtrace_add_page(pmap_t pmap, vm_map_offset_t start, vm_map_offset_t end);
 824 int pmap_pgtrace_delete_page(pmap_t pmap, vm_map_offset_t start, vm_map_offset_t end);
 825 kern_return_t pmap_pgtrace_fault(pmap_t pmap, vm_map_offset_t va, arm_saved_state_t *ss);
 826 #endif
 827
 828
 829 #ifdef PLATFORM_BridgeOS
 830 struct pmap_legacy_trust_cache {
 831         struct pmap_legacy_trust_cache *next;
 832         uuid_t uuid;
 833         uint32_t num_hashes;
 834         uint8_t hashes[][CS_CDHASH_LEN];
 835 };
 836 #else
 837 struct pmap_legacy_trust_cache;
 838 #endif
 839
 840 extern kern_return_t pmap_load_legacy_trust_cache(struct pmap_legacy_trust_cache *trust_cache,
 841     const vm_size_t trust_cache_len);
 842
 843 typedef enum {
 844         PMAP_TC_TYPE_PERSONALIZED,
 845         PMAP_TC_TYPE_PDI,
 846         PMAP_TC_TYPE_CRYPTEX,
 847         PMAP_TC_TYPE_ENGINEERING,
 848         PMAP_TC_TYPE_GLOBAL_FF00,
 849         PMAP_TC_TYPE_GLOBAL_FF01,
 850 } pmap_tc_type_t;
 851
 852 #define PMAP_IMAGE4_TRUST_CACHE_HAS_TYPE 1
 853 struct pmap_image4_trust_cache {
 854         // Filled by pmap layer.
 855         struct pmap_image4_trust_cache const *next;             // linked list linkage
 856         struct trust_cache_module1 const *module;                       // pointer into module (within data below)
 857
 858         // Filled by caller.
 859         // data is either an image4,
 860         // or just the trust cache payload itself if the image4 manifest is external.
 861         pmap_tc_type_t type;
 862         size_t bnch_len;
 863         uint8_t const bnch[48];
 864         size_t data_len;
 865         uint8_t const data[];
 866 };
 867
 868 typedef enum {
 869         PMAP_TC_SUCCESS = 0,
 870         PMAP_TC_UNKNOWN_FORMAT = -1,
 871         PMAP_TC_TOO_SMALL_FOR_HEADER = -2,
 872         PMAP_TC_TOO_SMALL_FOR_ENTRIES = -3,
 873         PMAP_TC_UNKNOWN_VERSION = -4,
 874         PMAP_TC_ALREADY_LOADED = -5,
 875         PMAP_TC_TOO_BIG = -6,
 876         PMAP_TC_RESOURCE_SHORTAGE = -7,
 877         PMAP_TC_MANIFEST_TOO_BIG = -8,
 878         PMAP_TC_MANIFEST_VIOLATION = -9,
 879         PMAP_TC_PAYLOAD_VIOLATION = -10,
 880         PMAP_TC_EXPIRED = -11,
 881         PMAP_TC_CRYPTO_WRONG = -12,
 882         PMAP_TC_OBJECT_WRONG = -13,
 883         PMAP_TC_UNKNOWN_CALLER = -14,
 884         PMAP_TC_UNKNOWN_FAILURE = -15,
 885 } pmap_tc_ret_t;
 886
 887 #define PMAP_HAS_LOCKDOWN_IMAGE4_SLAB 1
 888 extern void pmap_lockdown_image4_slab(vm_offset_t slab, vm_size_t slab_len, uint64_t flags);
 889
 890 extern pmap_tc_ret_t pmap_load_image4_trust_cache(
 891         struct pmap_image4_trust_cache *trust_cache, vm_size_t trust_cache_len,
 892         uint8_t const *img4_manifest,
 893         vm_size_t img4_manifest_buffer_len,
 894         vm_size_t img4_manifest_actual_len,
 895         bool dry_run);
 896
 897 extern bool pmap_is_trust_cache_loaded(const uuid_t uuid);
 898 extern uint32_t pmap_lookup_in_static_trust_cache(const uint8_t cdhash[CS_CDHASH_LEN]);
 899 extern bool pmap_lookup_in_loaded_trust_caches(const uint8_t cdhash[CS_CDHASH_LEN]);
 900
 901 extern bool pmap_in_ppl(void);
 902
 903 extern void *pmap_claim_reserved_ppl_page(void);
 904 extern void pmap_free_reserved_ppl_page(void *kva);
 905
 906 extern void pmap_ledger_alloc_init(size_t);
 907 extern ledger_t pmap_ledger_alloc(void);
 908 extern void pmap_ledger_free(ledger_t);
 909
 910 #if __arm64__
 911 extern bool pmap_is_exotic(pmap_t pmap);
 912 #else /* __arm64__ */
 913 #define pmap_is_exotic(pmap) false
 914 #endif /* __arm64__ */
 915
 916 #endif  /* KERNEL_PRIVATE */
 917
 918 #endif  /* _VM_PMAP_H_ */