]> git.saurik.com Git - apple/xnu.git/blame - osfmk/arm/pmap.h
projects / apple / xnu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
xnu-7195.60.75.tar.gz
[apple/xnu.git] / osfmk / arm / pmap.h
CommitLineData
5ba3f43e 1/*
cb323159 2 * Copyright (c) 2007-2019 Apple Inc. All rights reserved.
5ba3f43e
A		 3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28#ifndef _ARM_PMAP_H_
0a7de745 29#define _ARM_PMAP_H_ 1
5ba3f43e
A		 30
31#include <mach_assert.h>
32
33#include <arm/proc_reg.h>
34#if defined(__arm64__)
35#include <arm64/proc_reg.h>
36#endif
37
38/*
39 * Machine-dependent structures for the physical map module.
40 */
41
42#ifndef ASSEMBLER
43
d9a64523 44#include <stdatomic.h>
cb323159 45#include <stdbool.h>
d9a64523 46#include <libkern/section_keywords.h>
5ba3f43e
A		 47#include <mach/kern_return.h>
48#include <mach/machine/vm_types.h>
d9a64523
A		 49#include <arm/pmap_public.h>
50#include <mach/arm/thread_status.h>
cb323159
A		 51#if defined(__arm64__)
52#include <arm64/tlb.h>
53#else
54#include <arm/tlb.h>
55#endif
56
57
58#define ASID_SHIFT (11) /* Shift for 2048 max virtual ASIDs (2048 pmaps) */
f427ee49 59#define MAX_ASIDS (1 << ASID_SHIFT) /* Max supported ASIDs (can be virtual) */
cb323159
A		 60#ifndef ARM_ASID_SHIFT
61#define ARM_ASID_SHIFT (8) /* Shift for the maximum ARM ASID value (256) */
62#endif
f427ee49 63#define ARM_MAX_ASIDS (1 << ARM_ASID_SHIFT) /* Max ASIDs supported by the hardware */
cb323159 64#define NBBY 8
5c9f4661
A		 65
66#if __ARM_KERNEL_PROTECT__
f427ee49 67#define MAX_HW_ASIDS ((ARM_MAX_ASIDS >> 1) - 1)
cb323159 68#else
f427ee49 69#define MAX_HW_ASIDS (ARM_MAX_ASIDS - 1)
cb323159
A		 70#endif
71
72#ifndef ARM_VMID_SHIFT
73#define ARM_VMID_SHIFT (8)
74#endif
f427ee49 75#define ARM_MAX_VMIDS (1 << ARM_VMID_SHIFT)
cb323159
A		 76
77/* XPRR virtual register map */
78
79#define CPUWINDOWS_MAX 4
5c9f4661 80
f427ee49
A		 81#if defined(__arm64__)
82
83#if defined(ARM_LARGE_MEMORY)
84/*
85 * 2 L1 tables (Linear KVA and V=P), plus 2*16 L2 tables map up to (16*64GB) 1TB of DRAM
86 * Upper limit on how many pages can be consumed by bootstrap page tables
87 */
88#define BOOTSTRAP_TABLE_SIZE (ARM_PGBYTES * 34)
89#else // ARM_LARGE_MEMORY
90#define BOOTSTRAP_TABLE_SIZE (ARM_PGBYTES * 8)
91#endif
92
93typedef uint64_t tt_entry_t; /* translation table entry type */
94#define TT_ENTRY_NULL ((tt_entry_t *) 0)
95
96typedef uint64_t pt_entry_t; /* page table entry type */
97#define PT_ENTRY_NULL ((pt_entry_t *) 0)
98
99#elif defined(__arm__)
100
101typedef uint32_t tt_entry_t; /* translation table entry type */
102#define PT_ENTRY_NULL ((pt_entry_t *) 0)
103
104typedef uint32_t pt_entry_t; /* page table entry type */
105#define TT_ENTRY_NULL ((tt_entry_t *) 0)
106
107#else
108#error unknown arch
109#endif
110
111/* Forward declaration of the structure that controls page table
112 * geometry and TTE/PTE format. */
113struct page_table_attr;
114
115/*
116 * pv_entry_t - structure to track the active mappings for a given page
117 */
118typedef struct pv_entry {
119 struct pv_entry *pve_next; /* next alias */
120 pt_entry_t *pve_ptep; /* page table entry */
121}
122#if __arm__ && (__BIGGEST_ALIGNMENT__ > 4)
123/* For the newer ARMv7k ABI where 64-bit types are 64-bit aligned, but pointers
124 * are 32-bit:
125 * Since pt_desc is 64-bit aligned and we cast often from pv_entry to
126 * pt_desc.
127 */
128__attribute__ ((aligned(8))) pv_entry_t;
129#else
130pv_entry_t;
131#endif
132
133typedef struct {
134 pv_entry_t *list;
135 uint32_t count;
136} pv_free_list_t;
137
5c9f4661 138struct pmap_cpu_data {
c6bf4f31 139#if XNU_MONITOR
c6bf4f31
A		 140 void * ppl_kern_saved_sp;
141 void * ppl_stack;
142 arm_context_t * save_area;
143 unsigned int ppl_state;
144#endif
d9a64523
A		 145#if defined(__arm64__)
146 pmap_t cpu_nested_pmap;
f427ee49
A		 147 const struct page_table_attr *cpu_nested_pmap_attr;
148 vm_map_address_t cpu_nested_region_addr;
149 vm_map_offset_t cpu_nested_region_size;
d9a64523 150#else
5c9f4661 151 pmap_t cpu_user_pmap;
5c9f4661 152 unsigned int cpu_user_pmap_stamp;
d9a64523
A		 153#endif
154 unsigned int cpu_number;
cb323159 155 bool copywindow_strong_sync[CPUWINDOWS_MAX];
f427ee49
A		 156 pv_free_list_t pv_free;
157 pv_entry_t *pv_free_tail;
5c9f4661
A		 158
159 /*
160 * This supports overloading of ARM ASIDs by the pmap. The field needs
161 * to be wide enough to cover all the virtual bits in a virtual ASID.
162 * With 256 physical ASIDs, 8-bit fields let us support up to 65536
163 * Virtual ASIDs, minus all that would map on to 0 (as 0 is a global
164 * ASID).
165 *
166 * If we were to use bitfield shenanigans here, we could save a bit of
f427ee49 167 * memory by only having enough bits to support MAX_ASIDS. However, such
5c9f4661
A		 168 * an implementation would be more error prone.
169 */
f427ee49 170 uint8_t cpu_sw_asids[MAX_HW_ASIDS];
5c9f4661
A		 171};
172typedef struct pmap_cpu_data pmap_cpu_data_t;
173
5ba3f43e
A		 174#include <mach/vm_prot.h>
175#include <mach/vm_statistics.h>
176#include <mach/machine/vm_param.h>
177#include <kern/kern_types.h>
178#include <kern/thread.h>
179#include <kern/queue.h>
180
d9a64523
A		 181
182#include <sys/cdefs.h>
183
5ba3f43e 184/* Base address for low globals. */
f427ee49
A		 185#if defined(ARM_LARGE_MEMORY)
186#define LOW_GLOBAL_BASE_ADDRESS 0xfffffe0000000000ULL
187#else
5ba3f43e 188#define LOW_GLOBAL_BASE_ADDRESS 0xfffffff000000000ULL
f427ee49 189#endif
5ba3f43e
A		 190
191/*
192 * This indicates (roughly) where there is free space for the VM
193 * to use for the heap; this does not need to be precise.
194 */
f427ee49
A		 195#if defined(KERNEL_INTEGRITY_KTRR) || defined(KERNEL_INTEGRITY_CTRR)
196#if defined(ARM_LARGE_MEMORY)
197#define KERNEL_PMAP_HEAP_RANGE_START (VM_MIN_KERNEL_AND_KEXT_ADDRESS+ARM_TT_L1_SIZE)
198#else // ARM_LARGE_MEMORY
5ba3f43e 199#define KERNEL_PMAP_HEAP_RANGE_START VM_MIN_KERNEL_AND_KEXT_ADDRESS
f427ee49 200#endif // ARM_LARGE_MEMORY
5ba3f43e
A		 201#else
202#define KERNEL_PMAP_HEAP_RANGE_START LOW_GLOBAL_BASE_ADDRESS
203#endif
204
cb323159
A		 205struct page_table_level_info {
206 const uint64_t size;
207 const uint64_t offmask;
208 const uint64_t shift;
209 const uint64_t index_mask;
210 const uint64_t valid_mask;
211 const uint64_t type_mask;
212 const uint64_t type_block;
213};
214
f427ee49
A		 215/*
216 * For setups where the kernel page size does not match the hardware
217 * page size (assumably, the kernel page size must be a multiple of
218 * the hardware page size), we will need to determine what the page
219 * ratio is.
220 */
221#define PAGE_RATIO ((1 << PAGE_SHIFT) >> ARM_PGSHIFT)
222#define TEST_PAGE_RATIO_4 (PAGE_RATIO == 4)
223
5ba3f43e
A		 224
225/* superpages */
0a7de745 226#define SUPERPAGE_NBASEPAGES 1 /* No superpages support */
5ba3f43e
A		 227
228/*
229 * Convert addresses to pages and vice versa.
230 * No rounding is used.
231 */
232#define arm_atop(x) (((vm_map_address_t)(x)) >> ARM_PGSHIFT)
233#define arm_ptoa(x) (((vm_map_address_t)(x)) << ARM_PGSHIFT)
234
235/*
236 * Round off or truncate to the nearest page. These will work
237 * for either addresses or counts. (i.e. 1 byte rounds to 1 page
238 * bytes.
239 */
240#define arm_round_page(x) \
241 ((((vm_map_address_t)(x)) + ARM_PGMASK) & ~ARM_PGMASK)
242#define arm_trunc_page(x) (((vm_map_address_t)(x)) & ~ARM_PGMASK)
243
f427ee49 244#if __arm__
5ba3f43e
A		 245/* Convert address offset to page table index */
246#define ptenum(a) ((((a) & ARM_TT_LEAF_INDEX_MASK) >> ARM_TT_LEAF_SHIFT))
f427ee49 247#endif
5ba3f43e
A		 248
249#if (__ARM_VMSA__ <= 7)
0a7de745
A		 250#define NTTES (ARM_PGBYTES / sizeof(tt_entry_t))
251#define NPTES ((ARM_PGBYTES/4) /sizeof(pt_entry_t))
5ba3f43e 252#else
0a7de745
A		 253#define NTTES (ARM_PGBYTES / sizeof(tt_entry_t))
254#define NPTES (ARM_PGBYTES / sizeof(pt_entry_t))
5ba3f43e
A		 255#endif
256
5ba3f43e
A		 257extern void flush_mmu_tlb_region(vm_offset_t va, unsigned length);
258
259#if defined(__arm64__)
260extern uint64_t get_mmu_control(void);
5ba3f43e
A		 261extern uint64_t get_aux_control(void);
262extern void set_aux_control(uint64_t);
263extern void set_mmu_ttb(uint64_t);
264extern void set_mmu_ttb_alternate(uint64_t);
265extern uint64_t get_tcr(void);
266extern void set_tcr(uint64_t);
d9a64523 267extern uint64_t pmap_get_arm64_prot(pmap_t, vm_offset_t);
c6bf4f31
A		 268#if defined(HAS_VMSA_LOCK)
269extern void vmsa_lock(void);
270#endif
5ba3f43e
A		 271#else
272extern uint32_t get_mmu_control(void);
273extern void set_mmu_control(uint32_t);
274extern uint32_t get_aux_control(void);
275extern void set_aux_control(uint32_t);
276extern void set_mmu_ttb(pmap_paddr_t);
277extern void set_mmu_ttb_alternate(pmap_paddr_t);
278extern void set_context_id(uint32_t);
279#endif
280
281extern pmap_paddr_t get_mmu_ttb(void);
0a7de745
A		 282extern pmap_paddr_t mmu_kvtop(vm_offset_t va);
283extern pmap_paddr_t mmu_kvtop_wpreflight(vm_offset_t va);
284extern pmap_paddr_t mmu_uvtop(vm_offset_t va);
5ba3f43e
A		 285
286#if (__ARM_VMSA__ <= 7)
287/* Convert address offset to translation table index */
0a7de745 288#define ttenum(a) ((a) >> ARM_TT_L1_SHIFT)
5ba3f43e
A		 289
290/* Convert translation table index to user virtual address */
0a7de745 291#define tteitova(a) ((a) << ARM_TT_L1_SHIFT)
5ba3f43e 292
0a7de745
A		 293#define pa_to_suptte(a) ((a) & ARM_TTE_SUPER_L1_MASK)
294#define suptte_to_pa(p) ((p) & ARM_TTE_SUPER_L1_MASK)
5ba3f43e 295
0a7de745
A		 296#define pa_to_sectte(a) ((a) & ARM_TTE_BLOCK_L1_MASK)
297#define sectte_to_pa(p) ((p) & ARM_TTE_BLOCK_L1_MASK)
5ba3f43e 298
0a7de745
A		 299#define pa_to_tte(a) ((a) & ARM_TTE_TABLE_MASK)
300#define tte_to_pa(p) ((p) & ARM_TTE_TABLE_MASK)
5ba3f43e 301
0a7de745
A		 302#define pa_to_pte(a) ((a) & ARM_PTE_PAGE_MASK)
303#define pte_to_pa(p) ((p) & ARM_PTE_PAGE_MASK)
304#define pte_increment_pa(p) ((p) += ptoa(1))
5ba3f43e 305
0a7de745
A		 306#define ARM_NESTING_SIZE_MIN ((PAGE_SIZE/0x1000)*4*ARM_TT_L1_SIZE)
307#define ARM_NESTING_SIZE_MAX ((256*ARM_TT_L1_SIZE))
5ba3f43e
A		 308
309#else
310
311/* Convert address offset to translation table index */
0a7de745
A		 312#define ttel0num(a) ((a & ARM_TTE_L0_MASK) >> ARM_TT_L0_SHIFT)
313#define ttel1num(a) ((a & ARM_TTE_L1_MASK) >> ARM_TT_L1_SHIFT)
314#define ttel2num(a) ((a & ARM_TTE_L2_MASK) >> ARM_TT_L2_SHIFT)
5ba3f43e 315
0a7de745
A		 316#define pa_to_tte(a) ((a) & ARM_TTE_TABLE_MASK)
317#define tte_to_pa(p) ((p) & ARM_TTE_TABLE_MASK)
5ba3f43e 318
f427ee49
A		 319#define pa_to_pte(a) ((a) & ARM_PTE_PAGE_MASK)
320#define pte_to_pa(p) ((p) & ARM_PTE_PAGE_MASK)
0a7de745
A		 321#define pte_to_ap(p) (((p) & ARM_PTE_APMASK) >> ARM_PTE_APSHIFT)
322#define pte_increment_pa(p) ((p) += ptoa(1))
5ba3f43e 323
0a7de745 324#define ARM_NESTING_SIZE_MAX (0x0000000010000000ULL)
5ba3f43e 325
0a7de745 326#define TLBFLUSH_SIZE (ARM_TTE_MAX/((sizeof(unsigned int))*BYTE_SIZE))
5ba3f43e 327
0a7de745 328#endif /* __ARM_VMSA__ <= 7 */
5ba3f43e 329
0a7de745
A		 330#define PMAP_GC_INFLIGHT 1
331#define PMAP_GC_WAIT 2
5ba3f43e 332
d9a64523 333#if DEVELOPMENT || DEBUG
d9a64523 334#define pmap_cs_log_h(msg, args...) { if(pmap_cs_log_hacks) printf("PMAP_CS: " msg "\n", args); }
94ff46dc 335#define pmap_cs_log pmap_cs_log_h
d9a64523 336
d9a64523
A		 337#else
338#define pmap_cs_log(msg, args...)
339#define pmap_cs_log_h(msg, args...)
340#endif /* DEVELOPMENT || DEBUG */
341
342
cb323159 343
5ba3f43e
A		 344/*
345 * Convert translation/page table entry to kernel virtual address
346 */
347#define ttetokv(a) (phystokv(tte_to_pa(a)))
348#define ptetokv(a) (phystokv(pte_to_pa(a)))
349
350struct pmap {
f427ee49 351 tt_entry_t *XNU_PTRAUTH_SIGNED_PTR("pmap.tte") tte; /* translation table entries */
0a7de745
A		 352 pmap_paddr_t ttep; /* translation table physical */
353 vm_map_address_t min; /* min address in pmap */
354 vm_map_address_t max; /* max address in pmap */
cb323159
A		 355#if ARM_PARAMETERIZED_PMAP
356 const struct page_table_attr * pmap_pt_attr; /* details about page table layout */
357#endif /* ARM_PARAMETERIZED_PMAP */
0a7de745 358 ledger_t ledger; /* ledger tracking phys mappings */
f427ee49
A		 359
360 decl_lck_rw_data(, rwlock);
361
0a7de745
A		 362 struct pmap_statistics stats; /* map statistics */
363 queue_chain_t pmaps; /* global list of pmaps */
364 tt_entry_t *tt_entry_free; /* free translation table entries */
f427ee49
A		 365 struct pmap *XNU_PTRAUTH_SIGNED_PTR("pmap.nested_pmap") nested_pmap; /* nested pmap */
366 vm_map_address_t nested_region_addr;
0a7de745
A		 367 vm_map_offset_t nested_region_size;
368 vm_map_offset_t nested_region_true_start;
369 vm_map_offset_t nested_region_true_end;
370 unsigned int *nested_region_asid_bitmap;
5ba3f43e
A		 371
372#if (__ARM_VMSA__ <= 7)
cb323159 373 unsigned int tte_index_max; /* max tte index in translation table entries */
5ba3f43e 374#endif
d9a64523 375
2a1bd2d3
A		 376 void * reserved0;
377 void * reserved1;
378 uint64_t reserved2;
379 uint64_t reserved3;
d9a64523 380
0a7de745
A		 381 unsigned int stamp; /* creation stamp */
382 _Atomic int32_t ref_count; /* pmap reference count */
383 unsigned int gc_status; /* gc status */
384 unsigned int nested_region_asid_bitmap_size;
0a7de745 385 uint32_t nested_no_bounds_refcnt;/* number of pmaps that nested this pmap without bounds set */
cb323159
A		 386 uint16_t hw_asid;
387 uint8_t sw_asid;
d9a64523 388
5ba3f43e 389#if MACH_ASSERT
0a7de745
A		 390 int pmap_pid;
391 char pmap_procname[17];
392 bool pmap_stats_assert;
5ba3f43e 393#endif /* MACH_ASSERT */
2a1bd2d3 394 bool reserved4;
f427ee49 395 bool pmap_vm_map_cs_enforced;
2a1bd2d3
A		 396 boolean_t reserved5;
397 uint64_t reserved6;
398 uint64_t reserved7;
399 bool reserved8;
400 bool reserved9;
5ba3f43e 401#if DEVELOPMENT || DEBUG
0a7de745
A		 402 bool footprint_suspended;
403 bool footprint_was_suspended;
5ba3f43e 404#endif /* DEVELOPMENT || DEBUG */
cb323159
A		 405 bool nx_enabled; /* no execute */
406 bool nested; /* is nested */
407 bool is_64bit; /* is 64bit */
0a7de745
A		 408 bool nested_has_no_bounds_ref; /* nested a pmap when the bounds were not set */
409 bool nested_bounds_set; /* The nesting bounds have been set */
cb323159
A		 410#if HAS_APPLE_PAC
411 bool disable_jop;
2a1bd2d3
A		 412#else
413 bool reserved10;
cb323159 414#endif /* HAS_APPLE_PAC */
5ba3f43e
A		 415};
416
cb323159 417#define PMAP_VASID(pmap) (((uint32_t)((pmap)->sw_asid) << 16) | pmap->hw_asid)
5ba3f43e
A		 418
419#if VM_DEBUG
420extern int pmap_list_resident_pages(
0a7de745
A		 421 pmap_t pmap,
422 vm_offset_t *listp,
423 int space
424 );
5ba3f43e
A		 425#else /* #if VM_DEBUG */
426#define pmap_list_resident_pages(pmap, listp, space) (0)
427#endif /* #if VM_DEBUG */
428
429extern int copysafe(vm_map_address_t from, vm_map_address_t to, uint32_t cnt, int type, uint32_t *bytes_copied);
430
431/* globals shared between arm_vm_init and pmap */
0a7de745 432extern tt_entry_t *cpu_tte; /* first CPUs translation table (shared with kernel pmap) */
5ba3f43e
A		 433extern pmap_paddr_t cpu_ttep; /* physical translation table addr */
434
435#if __arm64__
436extern void *ropagetable_begin;
437extern void *ropagetable_end;
438#endif
439
440#if __arm64__
0a7de745 441extern tt_entry_t *invalid_tte; /* global invalid translation table */
5ba3f43e
A		 442extern pmap_paddr_t invalid_ttep; /* physical invalid translation table addr */
443#endif
444
445#define PMAP_CONTEXT(pmap, thread)
446
447/*
448 * platform dependent Prototypes
449 */
450extern void pmap_switch_user_ttb(pmap_t pmap);
d9a64523 451extern void pmap_clear_user_ttb(void);
5ba3f43e 452extern void pmap_bootstrap(vm_offset_t);
0a7de745 453extern vm_map_address_t pmap_ptov(pmap_t, ppnum_t);
f427ee49
A		 454extern pmap_paddr_t pmap_find_pa(pmap_t map, addr64_t va);
455extern pmap_paddr_t pmap_find_pa_nofault(pmap_t map, addr64_t va);
5ba3f43e 456extern ppnum_t pmap_find_phys(pmap_t map, addr64_t va);
f427ee49 457extern ppnum_t pmap_find_phys_nofault(pmap_t map, addr64_t va);
5ba3f43e
A		 458extern void pmap_set_pmap(pmap_t pmap, thread_t thread);
459extern void pmap_collect(pmap_t pmap);
0a7de745 460extern void pmap_gc(void);
2a1bd2d3 461#if HAS_APPLE_PAC
f427ee49
A		 462extern void * pmap_sign_user_ptr(void *value, ptrauth_key key, uint64_t data, uint64_t jop_key);
463extern void * pmap_auth_user_ptr(void *value, ptrauth_key key, uint64_t data, uint64_t jop_key);
c6bf4f31 464#endif /* HAS_APPLE_PAC && XNU_MONITOR */
5ba3f43e
A		 465
466/*
467 * Interfaces implemented as macros.
468 */
469
0a7de745
A		 470#define PMAP_SWITCH_USER(th, new_map, my_cpu) { \
471 th->map = new_map; \
472 pmap_set_pmap(vm_map_pmap(new_map), th); \
5ba3f43e
A		 473}
474
0a7de745 475#define pmap_kernel() \
5ba3f43e
A		 476 (kernel_pmap)
477
0a7de745 478#define pmap_compressed(pmap) \
5ba3f43e
A		 479 ((pmap)->stats.compressed)
480
0a7de745 481#define pmap_resident_count(pmap) \
5ba3f43e
A		 482 ((pmap)->stats.resident_count)
483
0a7de745 484#define pmap_resident_max(pmap) \
5ba3f43e
A		 485 ((pmap)->stats.resident_max)
486
487#define MACRO_NOOP
488
0a7de745 489#define pmap_copy(dst_pmap, src_pmap, dst_addr, len, src_addr) \
5ba3f43e
A		 490 MACRO_NOOP
491
0a7de745 492#define pmap_pageable(pmap, start, end, pageable) \
5ba3f43e
A		 493 MACRO_NOOP
494
0a7de745 495#define pmap_kernel_va(VA) \
5ba3f43e
A		 496 (((VA) >= VM_MIN_KERNEL_ADDRESS) && ((VA) <= VM_MAX_KERNEL_ADDRESS))
497
0a7de745 498#define pmap_attribute(pmap, addr, size, attr, value) \
5ba3f43e
A		 499 (KERN_INVALID_ADDRESS)
500
0a7de745 501#define copyinmsg(from, to, cnt) \
5ba3f43e
A		 502 copyin(from, to, cnt)
503
0a7de745 504#define copyoutmsg(from, to, cnt) \
5ba3f43e
A		 505 copyout(from, to, cnt)
506
0a7de745 507extern pmap_paddr_t kvtophys(vm_offset_t va);
d9a64523
A		 508extern vm_map_address_t phystokv(pmap_paddr_t pa);
509extern vm_map_address_t phystokv_range(pmap_paddr_t pa, vm_size_t *max_len);
5ba3f43e
A		 510
511extern vm_map_address_t pmap_map(vm_map_address_t va, vm_offset_t sa, vm_offset_t ea, vm_prot_t prot, unsigned int flags);
512extern vm_map_address_t pmap_map_high_window_bd( vm_offset_t pa, vm_size_t len, vm_prot_t prot);
513extern kern_return_t pmap_map_block(pmap_t pmap, addr64_t va, ppnum_t pa, uint32_t size, vm_prot_t prot, int attr, unsigned int flags);
514extern void pmap_map_globals(void);
515
cb323159
A		 516#define PMAP_MAP_BD_DEVICE 0x0
517#define PMAP_MAP_BD_WCOMB 0x1
518#define PMAP_MAP_BD_POSTED 0x2
519#define PMAP_MAP_BD_POSTED_REORDERED 0x3
520#define PMAP_MAP_BD_POSTED_COMBINED_REORDERED 0x4
521#define PMAP_MAP_BD_MASK 0x7
5ba3f43e
A		 522
523extern vm_map_address_t pmap_map_bd_with_options(vm_map_address_t va, vm_offset_t sa, vm_offset_t ea, vm_prot_t prot, int32_t options);
524extern vm_map_address_t pmap_map_bd(vm_map_address_t va, vm_offset_t sa, vm_offset_t ea, vm_prot_t prot);
525
f427ee49 526extern void pmap_init_pte_page(pmap_t, pt_entry_t *, vm_offset_t, unsigned int ttlevel, boolean_t alloc_ptd);
5ba3f43e
A		 527
528extern boolean_t pmap_valid_address(pmap_paddr_t addr);
529extern void pmap_disable_NX(pmap_t pmap);
530extern void pmap_set_nested(pmap_t pmap);
f427ee49 531extern void pmap_create_sharedpages(vm_map_address_t *kernel_data_addr, vm_map_address_t *kernel_text_addr, vm_map_address_t *user_text_addr);
5ba3f43e
A		 532extern void pmap_insert_sharedpage(pmap_t pmap);
533extern void pmap_protect_sharedpage(void);
534
535extern vm_offset_t pmap_cpu_windows_copy_addr(int cpu_num, unsigned int index);
536extern unsigned int pmap_map_cpu_windows_copy(ppnum_t pn, vm_prot_t prot, unsigned int wimg_bits);
537extern void pmap_unmap_cpu_windows_copy(unsigned int index);
538
f427ee49
A		 539#if XNU_MONITOR
540/* exposed for use by the HMAC SHA driver */
541extern void pmap_invoke_with_page(ppnum_t page_number, void *ctx,
542 void (*callback)(void *ctx, ppnum_t page_number, const void *page));
543extern void pmap_hibernate_invoke(void *ctx, void (*callback)(void *ctx, uint64_t addr, uint64_t len));
544extern void pmap_set_ppl_hashed_flag(const pmap_paddr_t addr);
545extern void pmap_clear_ppl_hashed_flag_all(void);
546extern void pmap_check_ppl_hashed_flag_all(void);
547#endif /* XNU_MONITOR */
5ba3f43e
A		 548
549extern boolean_t pmap_valid_page(ppnum_t pn);
4ba76501 550extern boolean_t pmap_bootloader_page(ppnum_t pn);
5ba3f43e
A		 551
552#define MACHINE_PMAP_IS_EMPTY 1
553extern boolean_t pmap_is_empty(pmap_t pmap, vm_map_offset_t start, vm_map_offset_t end);
554
0a7de745
A		 555#define ARM_PMAP_MAX_OFFSET_DEFAULT 0x01
556#define ARM_PMAP_MAX_OFFSET_MIN 0x02
557#define ARM_PMAP_MAX_OFFSET_MAX 0x04
558#define ARM_PMAP_MAX_OFFSET_DEVICE 0x08
559#define ARM_PMAP_MAX_OFFSET_JUMBO 0x10
5ba3f43e 560
5ba3f43e
A		 561
562extern vm_map_offset_t pmap_max_offset(boolean_t is64, unsigned int option);
d9a64523
A		 563extern vm_map_offset_t pmap_max_64bit_offset(unsigned int option);
564extern vm_map_offset_t pmap_max_32bit_offset(unsigned int option);
5ba3f43e
A		 565
566boolean_t pmap_virtual_region(unsigned int region_select, vm_map_offset_t *startp, vm_map_size_t *size);
567
568boolean_t pmap_enforces_execute_only(pmap_t pmap);
569
f427ee49
A		 570
571
572#if __has_feature(ptrauth_calls) && defined(XNU_TARGET_OS_OSX)
573extern void
574pmap_disable_user_jop(pmap_t pmap);
575#endif /* __has_feature(ptrauth_calls) && defined(XNU_TARGET_OS_OSX) */
576
5ba3f43e
A		 577/* pmap dispatch indices */
578#define ARM_FAST_FAULT_INDEX 0
579#define ARM_FORCE_FAST_FAULT_INDEX 1
580#define MAPPING_FREE_PRIME_INDEX 2
581#define MAPPING_REPLENISH_INDEX 3
582#define PHYS_ATTRIBUTE_CLEAR_INDEX 4
583#define PHYS_ATTRIBUTE_SET_INDEX 5
584#define PMAP_BATCH_SET_CACHE_ATTRIBUTES_INDEX 6
585#define PMAP_CHANGE_WIRING_INDEX 7
586#define PMAP_CREATE_INDEX 8
587#define PMAP_DESTROY_INDEX 9
588#define PMAP_ENTER_OPTIONS_INDEX 10
f427ee49
A		 589/* #define PMAP_EXTRACT_INDEX 11 -- Not used*/
590#define PMAP_FIND_PA_INDEX 12
5ba3f43e
A		 591#define PMAP_INSERT_SHAREDPAGE_INDEX 13
592#define PMAP_IS_EMPTY_INDEX 14
593#define PMAP_MAP_CPU_WINDOWS_COPY_INDEX 15
594#define PMAP_MARK_PAGE_AS_PMAP_PAGE_INDEX 16
595#define PMAP_NEST_INDEX 17
596#define PMAP_PAGE_PROTECT_OPTIONS_INDEX 18
597#define PMAP_PROTECT_OPTIONS_INDEX 19
598#define PMAP_QUERY_PAGE_INFO_INDEX 20
599#define PMAP_QUERY_RESIDENT_INDEX 21
600#define PMAP_REFERENCE_INDEX 22
601#define PMAP_REMOVE_OPTIONS_INDEX 23
602#define PMAP_RETURN_INDEX 24
603#define PMAP_SET_CACHE_ATTRIBUTES_INDEX 25
604#define PMAP_SET_NESTED_INDEX 26
605#define PMAP_SET_PROCESS_INDEX 27
606#define PMAP_SWITCH_INDEX 28
607#define PMAP_SWITCH_USER_TTB_INDEX 29
d9a64523 608#define PMAP_CLEAR_USER_TTB_INDEX 30
5ba3f43e
A		 609#define PMAP_UNMAP_CPU_WINDOWS_COPY_INDEX 31
610#define PMAP_UNNEST_OPTIONS_INDEX 32
611#define PMAP_FOOTPRINT_SUSPEND_INDEX 33
612#define PMAP_CPU_DATA_INIT_INDEX 34
613#define PMAP_RELEASE_PAGES_TO_KERNEL_INDEX 35
d9a64523
A		 614#define PMAP_SET_JIT_ENTITLED_INDEX 36
615
616
f427ee49
A		 617#define PMAP_UPDATE_COMPRESSOR_PAGE_INDEX 55
618#define PMAP_TRIM_INDEX 56
619#define PMAP_LEDGER_ALLOC_INIT_INDEX 57
620#define PMAP_LEDGER_ALLOC_INDEX 58
621#define PMAP_LEDGER_FREE_INDEX 59
5ba3f43e 622
c6bf4f31 623#if HAS_APPLE_PAC && XNU_MONITOR
f427ee49
A		 624#define PMAP_SIGN_USER_PTR 60
625#define PMAP_AUTH_USER_PTR 61
c6bf4f31 626#endif /* HAS_APPLE_PAC && XNU_MONITOR */
cb323159 627
f427ee49
A		 628#define PHYS_ATTRIBUTE_CLEAR_RANGE_INDEX 66
629
630
631#if __has_feature(ptrauth_calls) && defined(XNU_TARGET_OS_OSX)
632#define PMAP_DISABLE_USER_JOP_INDEX 69
633#endif /* __has_feature(ptrauth_calls) && defined(XNU_TARGET_OS_OSX) */
634
635
636
637#define PMAP_SET_VM_MAP_CS_ENFORCED_INDEX 72
cb323159 638
f427ee49 639#define PMAP_COUNT 73
5ba3f43e
A		 640
641#define PMAP_INVALID_CPU_NUM (~0U)
642
d9a64523
A		 643struct pmap_cpu_data_array_entry {
644 pmap_cpu_data_t cpu_data;
f427ee49 645} __attribute__((aligned(1 << MAX_L2_CLINE)));
d9a64523 646
5ba3f43e
A		 647/* Initialize the pmap per-CPU data for the current CPU. */
648extern void pmap_cpu_data_init(void);
649
650/* Get the pmap per-CPU data for the current CPU. */
651extern pmap_cpu_data_t * pmap_get_cpu_data(void);
652
c6bf4f31
A		 653#if XNU_MONITOR
654extern boolean_t pmap_ppl_locked_down;
655
656/*
657 * Denotes the bounds of the PPL stacks. These are visible so that other code
658 * can check if addresses are part of the PPL stacks.
659 */
660extern void * pmap_stacks_start;
661extern void * pmap_stacks_end;
662
663/* Asks if a page belongs to the monitor. */
664extern boolean_t pmap_is_monitor(ppnum_t pn);
665
666/*
667 * Indicates that we are done with our static bootstrap
668 * allocations, so the monitor may now mark the pages
669 * that it owns.
670 */
671extern void pmap_static_allocations_done(void);
672
673/*
674 * Indicates that we are done mutating sensitive state in the system, and that
675 * the PPL may now restict write access to PPL owned mappings.
676 */
677extern void pmap_lockdown_ppl(void);
678
679
680#ifdef KASAN
681#define PPL_STACK_SIZE (PAGE_SIZE << 2)
682#else
683#define PPL_STACK_SIZE PAGE_SIZE
684#endif
685
686/* One stack for each CPU, plus a guard page below each stack and above the last stack */
687#define PPL_STACK_REGION_SIZE ((MAX_CPUS * (PPL_STACK_SIZE + ARM_PGBYTES)) + ARM_PGBYTES)
688
689#define PPL_DATA_SEGMENT_SECTION_NAME "__PPLDATA,__data"
690#define PPL_TEXT_SEGMENT_SECTION_NAME "__PPLTEXT,__text,regular,pure_instructions"
691#define PPL_DATACONST_SEGMENT_SECTION_NAME "__PPLDATA,__const"
692
693#define MARK_AS_PMAP_DATA \
694 __PLACE_IN_SECTION(PPL_DATA_SEGMENT_SECTION_NAME)
695#define MARK_AS_PMAP_TEXT \
696 __attribute__((used, section(PPL_TEXT_SEGMENT_SECTION_NAME), noinline))
697#define MARK_AS_PMAP_RODATA \
698 __PLACE_IN_SECTION(PPL_DATACONST_SEGMENT_SECTION_NAME)
699
700#else /* XNU_MONITOR */
d9a64523 701
5ba3f43e
A		 702#define MARK_AS_PMAP_TEXT
703#define MARK_AS_PMAP_DATA
d9a64523
A		 704#define MARK_AS_PMAP_RODATA
705
c6bf4f31 706#endif /* !XNU_MONITOR */
d9a64523 707
5ba3f43e
A		 708
709extern kern_return_t pmap_return(boolean_t do_panic, boolean_t do_recurse);
710
0a7de745
A		 711extern lck_grp_t pmap_lck_grp;
712
c6bf4f31 713#if XNU_MONITOR
f427ee49 714extern void CleanPoC_DcacheRegion_Force_nopreempt(vm_offset_t va, size_t length);
c6bf4f31
A		 715#define pmap_force_dcache_clean(va, sz) CleanPoC_DcacheRegion_Force_nopreempt(va, sz)
716#define pmap_simple_lock(l) simple_lock_nopreempt(l, &pmap_lck_grp)
717#define pmap_simple_unlock(l) simple_unlock_nopreempt(l)
718#define pmap_simple_lock_try(l) simple_lock_try_nopreempt(l, &pmap_lck_grp)
719#define pmap_lock_bit(l, i) hw_lock_bit_nopreempt(l, i, &pmap_lck_grp)
720#define pmap_unlock_bit(l, i) hw_unlock_bit_nopreempt(l, i)
721#else
0a7de745
A		 722#define pmap_force_dcache_clean(va, sz) CleanPoC_DcacheRegion_Force(va, sz)
723#define pmap_simple_lock(l) simple_lock(l, &pmap_lck_grp)
724#define pmap_simple_unlock(l) simple_unlock(l)
725#define pmap_simple_lock_try(l) simple_lock_try(l, &pmap_lck_grp)
726#define pmap_lock_bit(l, i) hw_lock_bit(l, i, &pmap_lck_grp)
727#define pmap_unlock_bit(l, i) hw_unlock_bit(l, i)
c6bf4f31 728#endif
d9a64523 729
5ba3f43e
A		 730#endif /* #ifndef ASSEMBLER */
731
5c9f4661
A		 732#if __ARM_KERNEL_PROTECT__
733/*
734 * The exception vector mappings start at the middle of the kernel page table
735 * range (so that the EL0 mapping can be located at the base of the range).
736 */
737#define ARM_KERNEL_PROTECT_EXCEPTION_START ((~((ARM_TT_ROOT_SIZE + ARM_TT_ROOT_INDEX_MASK) / 2ULL)) + 1ULL)
738#endif /* __ARM_KERNEL_PROTECT__ */
739
5ba3f43e 740#endif /* #ifndef _ARM_PMAP_H_ */
mirror of XNU