]> git.saurik.com Git - apple/xnu.git/blob - osfmk/arm/pmap.c
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-4570.20.62.tar.gz
[apple/xnu.git] / osfmk / arm / pmap.c
1 /*
2 * Copyright (c) 2011-2016 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 #include <string.h>
29 #include <mach_assert.h>
30 #include <mach_ldebug.h>
31
32 #include <mach/shared_region.h>
33 #include <mach/vm_param.h>
34 #include <mach/vm_prot.h>
35 #include <mach/vm_map.h>
36 #include <mach/machine/vm_param.h>
37 #include <mach/machine/vm_types.h>
38
39 #include <mach/boolean.h>
40 #include <kern/thread.h>
41 #include <kern/sched.h>
42 #include <kern/zalloc.h>
43 #include <kern/kalloc.h>
44 #include <kern/ledger.h>
45 #include <kern/misc_protos.h>
46 #include <kern/spl.h>
47 #include <kern/xpr.h>
48
49 #include <vm/pmap.h>
50 #include <vm/vm_map.h>
51 #include <vm/vm_kern.h>
52 #include <vm/vm_protos.h>
53 #include <vm/vm_object.h>
54 #include <vm/vm_page.h>
55 #include <vm/vm_pageout.h>
56 #include <vm/cpm.h>
57
58 #include <libkern/section_keywords.h>
59
60 #include <machine/atomic.h>
61 #include <machine/thread.h>
62 #include <machine/lowglobals.h>
63
64 #include <arm/caches_internal.h>
65 #include <arm/cpu_data.h>
66 #include <arm/cpu_data_internal.h>
67 #include <arm/cpu_capabilities.h>
68 #include <arm/cpu_number.h>
69 #include <arm/machine_cpu.h>
70 #include <arm/misc_protos.h>
71 #include <arm/trap.h>
72
73 #include <libkern/section_keywords.h>
74
75 #if (__ARM_VMSA__ > 7)
76 #include <arm64/proc_reg.h>
77 #include <pexpert/arm64/boot.h>
78 #if CONFIG_PGTRACE
79 #include <stdint.h>
80 #include <arm64/pgtrace.h>
81 #if CONFIG_PGTRACE_NONKEXT
82 #include <arm64/pgtrace_decoder.h>
83 #endif // CONFIG_PGTRACE_NONKEXT
84 #endif
85 #endif
86
87 #include <pexpert/device_tree.h>
88
89 #include <san/kasan.h>
90
91 #if DEVELOPMENT || DEBUG
92 #define PMAP_FOOTPRINT_SUSPENDED(pmap) ((pmap)->footprint_suspended)
93 #else /* DEVELOPMENT || DEBUG */
94 #define PMAP_FOOTPRINT_SUSPENDED(pmap) (FALSE)
95 #endif /* DEVELOPMENT || DEBUG */
96
97
98
99 #if DEVELOPMENT || DEBUG
100 int panic_on_unsigned_execute = 0;
101 #endif /* DEVELOPMENT || DEBUG */
102
103
104 /* Virtual memory region for early allocation */
105 #if (__ARM_VMSA__ == 7)
106 #define VREGION1_START (VM_HIGH_KERNEL_WINDOW & ~ARM_TT_L1_PT_OFFMASK)
107 #else
108 #define VREGION1_HIGH_WINDOW (PE_EARLY_BOOT_VA)
109 #define VREGION1_START ((VM_MAX_KERNEL_ADDRESS & CPUWINDOWS_BASE_MASK) - VREGION1_HIGH_WINDOW)
110 #endif
111 #define VREGION1_SIZE (trunc_page(VM_MAX_KERNEL_ADDRESS - (VREGION1_START)))
112
113 extern unsigned int not_in_kdp;
114
115 extern vm_offset_t first_avail;
116
117 extern pmap_paddr_t avail_start;
118 extern pmap_paddr_t avail_end;
119
120 extern vm_offset_t virtual_space_start; /* Next available kernel VA */
121 extern vm_offset_t virtual_space_end; /* End of kernel address space */
122
123 extern int hard_maxproc;
124
125 #if (__ARM_VMSA__ > 7)
126 /* The number of address bits one TTBR can cover. */
127 #define PGTABLE_ADDR_BITS (64ULL - T0SZ_BOOT)
128
129 /*
130 * The bounds on our TTBRs. These are for sanity checking that
131 * an address is accessible by a TTBR before we attempt to map it.
132 */
133 #define ARM64_TTBR0_MIN_ADDR (0ULL)
134 #define ARM64_TTBR0_MAX_ADDR (0ULL + (1ULL << PGTABLE_ADDR_BITS) - 1)
135 #define ARM64_TTBR1_MIN_ADDR (0ULL - (1ULL << PGTABLE_ADDR_BITS))
136 #define ARM64_TTBR1_MAX_ADDR (~0ULL)
137
138 /* The level of the root of a page table. */
139 const uint64_t arm64_root_pgtable_level = (3 - ((PGTABLE_ADDR_BITS - 1 - ARM_PGSHIFT) / (ARM_PGSHIFT - TTE_SHIFT)));
140
141 /* The number of entries in the root TT of a page table. */
142 const uint64_t arm64_root_pgtable_num_ttes = (2 << ((PGTABLE_ADDR_BITS - 1 - ARM_PGSHIFT) % (ARM_PGSHIFT - TTE_SHIFT)));
143 #else
144 const uint64_t arm64_root_pgtable_level = 0;
145 const uint64_t arm64_root_pgtable_num_ttes = 0;
146 #endif
147
148 struct pmap kernel_pmap_store MARK_AS_PMAP_DATA;
149 SECURITY_READ_ONLY_LATE(pmap_t) kernel_pmap = &kernel_pmap_store;
150
151 struct vm_object pmap_object_store __attribute__((aligned(VM_PACKED_POINTER_ALIGNMENT))); /* store pt pages */
152 vm_object_t pmap_object = &pmap_object_store;
153
154 static struct zone *pmap_zone; /* zone of pmap structures */
155
156 decl_simple_lock_data(, pmaps_lock MARK_AS_PMAP_DATA)
157 unsigned int pmap_stamp MARK_AS_PMAP_DATA;
158 queue_head_t map_pmap_list MARK_AS_PMAP_DATA;
159
160 queue_head_t tt_pmap_list MARK_AS_PMAP_DATA;
161 unsigned int tt_pmap_count MARK_AS_PMAP_DATA;
162 unsigned int tt_pmap_max MARK_AS_PMAP_DATA;
163
164 decl_simple_lock_data(, pt_pages_lock MARK_AS_PMAP_DATA)
165 queue_head_t pt_page_list MARK_AS_PMAP_DATA; /* pt page ptd entries list */
166
167 decl_simple_lock_data(, pmap_pages_lock MARK_AS_PMAP_DATA)
168
169 typedef struct page_free_entry {
170 struct page_free_entry *next;
171 } page_free_entry_t;
172
173 #define PAGE_FREE_ENTRY_NULL ((page_free_entry_t *) 0)
174
175 page_free_entry_t *pmap_pages_reclaim_list MARK_AS_PMAP_DATA; /* Reclaimed pt page list */
176 unsigned int pmap_pages_request_count MARK_AS_PMAP_DATA; /* Pending requests to reclaim pt page */
177 unsigned long long pmap_pages_request_acum MARK_AS_PMAP_DATA;
178
179
180 typedef struct tt_free_entry {
181 struct tt_free_entry *next;
182 } tt_free_entry_t;
183
184 #define TT_FREE_ENTRY_NULL ((tt_free_entry_t *) 0)
185
186 tt_free_entry_t *free_page_size_tt_list MARK_AS_PMAP_DATA;
187 unsigned int free_page_size_tt_count MARK_AS_PMAP_DATA;
188 unsigned int free_page_size_tt_max MARK_AS_PMAP_DATA;
189 #define FREE_PAGE_SIZE_TT_MAX 4
190 tt_free_entry_t *free_two_page_size_tt_list MARK_AS_PMAP_DATA;
191 unsigned int free_two_page_size_tt_count MARK_AS_PMAP_DATA;
192 unsigned int free_two_page_size_tt_max MARK_AS_PMAP_DATA;
193 #define FREE_TWO_PAGE_SIZE_TT_MAX 4
194 tt_free_entry_t *free_tt_list MARK_AS_PMAP_DATA;
195 unsigned int free_tt_count MARK_AS_PMAP_DATA;
196 unsigned int free_tt_max MARK_AS_PMAP_DATA;
197
198 #define TT_FREE_ENTRY_NULL ((tt_free_entry_t *) 0)
199
200 boolean_t pmap_gc_allowed MARK_AS_PMAP_DATA = TRUE;
201 boolean_t pmap_gc_forced MARK_AS_PMAP_DATA = FALSE;
202 boolean_t pmap_gc_allowed_by_time_throttle = TRUE;
203
204 unsigned int inuse_user_ttepages_count MARK_AS_PMAP_DATA = 0; /* non-root, non-leaf user pagetable pages, in units of PAGE_SIZE */
205 unsigned int inuse_user_ptepages_count MARK_AS_PMAP_DATA = 0; /* leaf user pagetable pages, in units of PAGE_SIZE */
206 unsigned int inuse_user_tteroot_count MARK_AS_PMAP_DATA = 0; /* root user pagetables, in units of PMAP_ROOT_ALLOC_SIZE */
207 unsigned int inuse_kernel_ttepages_count MARK_AS_PMAP_DATA = 0; /* non-root, non-leaf kernel pagetable pages, in units of PAGE_SIZE */
208 unsigned int inuse_kernel_ptepages_count MARK_AS_PMAP_DATA = 0; /* leaf kernel pagetable pages, in units of PAGE_SIZE */
209 unsigned int inuse_kernel_tteroot_count MARK_AS_PMAP_DATA = 0; /* root kernel pagetables, in units of PMAP_ROOT_ALLOC_SIZE */
210 unsigned int inuse_pmap_pages_count = 0; /* debugging */
211
212 SECURITY_READ_ONLY_LATE(tt_entry_t *) invalid_tte = 0;
213 SECURITY_READ_ONLY_LATE(pmap_paddr_t) invalid_ttep = 0;
214
215 SECURITY_READ_ONLY_LATE(tt_entry_t *) cpu_tte = 0; /* set by arm_vm_init() - keep out of bss */
216 SECURITY_READ_ONLY_LATE(pmap_paddr_t) cpu_ttep = 0; /* set by arm_vm_init() - phys tte addr */
217
218 #if DEVELOPMENT || DEBUG
219 int nx_enabled = 1; /* enable no-execute protection */
220 int allow_data_exec = 0; /* No apps may execute data */
221 int allow_stack_exec = 0; /* No apps may execute from the stack */
222 #else /* DEVELOPMENT || DEBUG */
223 const int nx_enabled = 1; /* enable no-execute protection */
224 const int allow_data_exec = 0; /* No apps may execute data */
225 const int allow_stack_exec = 0; /* No apps may execute from the stack */
226 #endif /* DEVELOPMENT || DEBUG */
227
228 /*
229 * pv_entry_t - structure to track the active mappings for a given page
230 */
231 typedef struct pv_entry {
232 struct pv_entry *pve_next; /* next alias */
233 pt_entry_t *pve_ptep; /* page table entry */
234 #if __arm__ && (__BIGGEST_ALIGNMENT__ > 4)
235 /* For the newer ARMv7k ABI where 64-bit types are 64-bit aligned, but pointers
236 * are 32-bit:
237 * Since pt_desc is 64-bit aligned and we cast often from pv_entry to
238 * pt_desc.
239 */
240 } __attribute__ ((aligned(8))) pv_entry_t;
241 #else
242 } pv_entry_t;
243 #endif
244
245 #define PV_ENTRY_NULL ((pv_entry_t *) 0)
246
247 /*
248 * PMAP LEDGERS:
249 * We use the least significant bit of the "pve_next" pointer in a "pv_entry"
250 * as a marker for pages mapped through an "alternate accounting" mapping.
251 * These macros set, clear and test for this marker and extract the actual
252 * value of the "pve_next" pointer.
253 */
254 #define PVE_NEXT_ALTACCT ((uintptr_t) 0x1)
255 #define PVE_NEXT_SET_ALTACCT(pve_next_p) \
256 *(pve_next_p) = (struct pv_entry *) (((uintptr_t) *(pve_next_p)) | \
257 PVE_NEXT_ALTACCT)
258 #define PVE_NEXT_CLR_ALTACCT(pve_next_p) \
259 *(pve_next_p) = (struct pv_entry *) (((uintptr_t) *(pve_next_p)) & \
260 ~PVE_NEXT_ALTACCT)
261 #define PVE_NEXT_IS_ALTACCT(pve_next) \
262 ((((uintptr_t) (pve_next)) & PVE_NEXT_ALTACCT) ? TRUE : FALSE)
263 #define PVE_NEXT_PTR(pve_next) \
264 ((struct pv_entry *)(((uintptr_t) (pve_next)) & \
265 ~PVE_NEXT_ALTACCT))
266 #if MACH_ASSERT
267 static void pmap_check_ledgers(pmap_t pmap);
268 #else
269 static inline void pmap_check_ledgers(__unused pmap_t pmap) {}
270 #endif /* MACH_ASSERT */
271
272 SECURITY_READ_ONLY_LATE(pv_entry_t **) pv_head_table; /* array of pv entry pointers */
273
274 pv_entry_t *pv_free_list MARK_AS_PMAP_DATA;
275 pv_entry_t *pv_kern_free_list MARK_AS_PMAP_DATA;
276 decl_simple_lock_data(,pv_free_list_lock MARK_AS_PMAP_DATA)
277 decl_simple_lock_data(,pv_kern_free_list_lock MARK_AS_PMAP_DATA)
278
279 decl_simple_lock_data(,phys_backup_lock)
280
281 /*
282 * pt_desc - structure to keep info on page assigned to page tables
283 */
284 #if (__ARM_VMSA__ == 7)
285 #define PT_INDEX_MAX 1
286 #else
287 #if (ARM_PGSHIFT == 14)
288 #define PT_INDEX_MAX 1
289 #else
290 #define PT_INDEX_MAX 4
291 #endif
292 #endif
293
294 #define PT_DESC_REFCOUNT 0x4000U
295
296 typedef struct pt_desc {
297 queue_chain_t pt_page;
298 struct {
299 unsigned short refcnt;
300 unsigned short wiredcnt;
301 } pt_cnt[PT_INDEX_MAX];
302 struct pmap *pmap;
303 struct {
304 vm_offset_t va;
305 } pt_map[PT_INDEX_MAX];
306 } pt_desc_t;
307
308
309 #define PTD_ENTRY_NULL ((pt_desc_t *) 0)
310
311 SECURITY_READ_ONLY_LATE(pt_desc_t *) ptd_root_table;
312
313 pt_desc_t *ptd_free_list MARK_AS_PMAP_DATA = PTD_ENTRY_NULL;
314 SECURITY_READ_ONLY_LATE(boolean_t) ptd_preboot = TRUE;
315 unsigned int ptd_free_count MARK_AS_PMAP_DATA = 0;
316 decl_simple_lock_data(,ptd_free_list_lock MARK_AS_PMAP_DATA)
317
318 /*
319 * physical page attribute
320 */
321 typedef u_int16_t pp_attr_t;
322
323 #define PP_ATTR_WIMG_MASK 0x003F
324 #define PP_ATTR_WIMG(x) ((x) & PP_ATTR_WIMG_MASK)
325
326 #define PP_ATTR_REFERENCED 0x0040
327 #define PP_ATTR_MODIFIED 0x0080
328
329 #define PP_ATTR_INTERNAL 0x0100
330 #define PP_ATTR_REUSABLE 0x0200
331 #define PP_ATTR_ALTACCT 0x0400
332 #define PP_ATTR_NOENCRYPT 0x0800
333
334 #define PP_ATTR_REFFAULT 0x1000
335 #define PP_ATTR_MODFAULT 0x2000
336
337
338 SECURITY_READ_ONLY_LATE(pp_attr_t*) pp_attr_table;
339
340
341 typedef uint8_t io_attr_t;
342
343 #define IO_ATTR_WIMG_MASK 0x3F
344 #define IO_ATTR_WIMG(x) ((x) & IO_ATTR_WIMG_MASK)
345
346 SECURITY_READ_ONLY_LATE(io_attr_t*) io_attr_table;
347
348 SECURITY_READ_ONLY_LATE(pmap_paddr_t) vm_first_phys = (pmap_paddr_t) 0;
349 SECURITY_READ_ONLY_LATE(pmap_paddr_t) vm_last_phys = (pmap_paddr_t) 0;
350
351 SECURITY_READ_ONLY_LATE(pmap_paddr_t) io_rgn_start = 0;
352 SECURITY_READ_ONLY_LATE(pmap_paddr_t) io_rgn_end = 0;
353 SECURITY_READ_ONLY_LATE(uint32_t) io_rgn_granule = 0;
354
355 SECURITY_READ_ONLY_LATE(boolean_t) pmap_initialized = FALSE; /* Has pmap_init completed? */
356
357 SECURITY_READ_ONLY_LATE(uint64_t) pmap_nesting_size_min;
358 SECURITY_READ_ONLY_LATE(uint64_t) pmap_nesting_size_max;
359
360 SECURITY_READ_ONLY_LATE(vm_map_offset_t) arm_pmap_max_offset_default = 0x0;
361 #if defined(__arm64__)
362 SECURITY_READ_ONLY_LATE(vm_map_offset_t) arm64_pmap_max_offset_default = 0x0;
363 #endif
364
365 /* free address spaces (1 means free) */
366 static uint32_t asid_bitmap[MAX_ASID / (sizeof(uint32_t) * NBBY)] MARK_AS_PMAP_DATA;
367
368 #if (__ARM_VMSA__ > 7)
369 SECURITY_READ_ONLY_LATE(pmap_t) u32_sharedpage_pmap;
370 #endif
371
372
373 #define pa_index(pa) \
374 (atop((pa) - vm_first_phys))
375
376 #define pai_to_pvh(pai) \
377 (&pv_head_table[pai])
378
379 #define pa_valid(x) \
380 ((x) >= vm_first_phys && (x) < vm_last_phys)
381
382 /* PTE Define Macros */
383
384 #define pte_is_wired(pte) \
385 (((pte) & ARM_PTE_WIRED_MASK) == ARM_PTE_WIRED)
386
387 #define pte_set_wired(ptep, wired) \
388 do { \
389 SInt16 *ptd_wiredcnt_ptr; \
390 ptd_wiredcnt_ptr = (SInt16 *)&(ptep_get_ptd(ptep)->pt_cnt[ARM_PT_DESC_INDEX(ptep)].wiredcnt); \
391 if (wired) { \
392 *ptep |= ARM_PTE_WIRED; \
393 OSAddAtomic16(1, ptd_wiredcnt_ptr); \
394 } else { \
395 *ptep &= ~ARM_PTE_WIRED; \
396 OSAddAtomic16(-1, ptd_wiredcnt_ptr); \
397 } \
398 } while(0)
399
400 #define pte_is_ffr(pte) \
401 (((pte) & ARM_PTE_WRITEABLE) == ARM_PTE_WRITEABLE)
402
403 #define pte_set_ffr(pte, ffr) \
404 do { \
405 if (ffr) { \
406 pte |= ARM_PTE_WRITEABLE; \
407 } else { \
408 pte &= ~ARM_PTE_WRITEABLE; \
409 } \
410 } while(0)
411
412 /* PVE Define Macros */
413
414 #define pve_next(pve) \
415 ((pve)->pve_next)
416
417 #define pve_link_field(pve) \
418 (&pve_next(pve))
419
420 #define pve_link(pp, e) \
421 ((pve_next(e) = pve_next(pp)), (pve_next(pp) = (e)))
422
423 #define pve_unlink(pp, e) \
424 (pve_next(pp) = pve_next(e))
425
426 /* bits held in the ptep pointer field */
427
428 #define pve_get_ptep(pve) \
429 ((pve)->pve_ptep)
430
431 #define pve_set_ptep(pve, ptep_new) \
432 do { \
433 (pve)->pve_ptep = (ptep_new); \
434 } while (0)
435
436 /* PTEP Define Macros */
437
438 #if (__ARM_VMSA__ == 7)
439
440 #define ARM_PT_DESC_INDEX_MASK 0x00000
441 #define ARM_PT_DESC_INDEX_SHIFT 0
442
443 /*
444 * mask for page descriptor index: 4MB per page table
445 */
446 #define ARM_TT_PT_INDEX_MASK 0xfffU /* mask for page descriptor index: 4MB per page table */
447
448 /*
449 * Shift value used for reconstructing the virtual address for a PTE.
450 */
451 #define ARM_TT_PT_ADDR_SHIFT (10U)
452
453 #define ARM_PT_DESC_INDEX(ptep) \
454 (((unsigned)(ptep) & ARM_PT_DESC_INDEX_MASK) >> ARM_PT_DESC_INDEX_SHIFT)
455
456 #define ptep_get_ptd(ptep) \
457 ((struct pt_desc *)((*((vm_offset_t *)(pai_to_pvh(pa_index((vm_offset_t)(ptep) - gVirtBase + gPhysBase))))) & PVH_LIST_MASK))
458
459 #define ptep_get_va(ptep) \
460 ((((pt_desc_t *) (pvh_list(pai_to_pvh(pa_index((((vm_offset_t)(ptep) & ~0xFFF) - gVirtBase + gPhysBase))))))->pt_map[ARM_PT_DESC_INDEX(ptep)].va)+ ((((unsigned)(ptep)) & ARM_TT_PT_INDEX_MASK)<<ARM_TT_PT_ADDR_SHIFT))
461
462 #define ptep_get_pmap(ptep) \
463 ((((pt_desc_t *) (pvh_list(pai_to_pvh(pa_index((((vm_offset_t)(ptep) & ~0xFFF) - gVirtBase + gPhysBase))))))->pmap))
464
465
466 #else
467
468 #if (ARM_PGSHIFT == 12)
469 #define ARM_PT_DESC_INDEX_MASK ((PAGE_SHIFT_CONST == ARM_PGSHIFT )? 0x00000ULL : 0x03000ULL)
470 #define ARM_PT_DESC_INDEX_SHIFT ((PAGE_SHIFT_CONST == ARM_PGSHIFT )? 0 : 12)
471 /*
472 * mask for page descriptor index: 2MB per page table
473 */
474 #define ARM_TT_PT_INDEX_MASK (0x0fffULL)
475 /*
476 * Shift value used for reconstructing the virtual address for a PTE.
477 */
478 #define ARM_TT_PT_ADDR_SHIFT (9ULL)
479
480 /* TODO: Give this a better name/documentation than "other" */
481 #define ARM_TT_PT_OTHER_MASK (0x0fffULL)
482
483 #else
484
485 #define ARM_PT_DESC_INDEX_MASK (0x00000)
486 #define ARM_PT_DESC_INDEX_SHIFT (0)
487 /*
488 * mask for page descriptor index: 32MB per page table
489 */
490 #define ARM_TT_PT_INDEX_MASK (0x3fffULL)
491 /*
492 * Shift value used for reconstructing the virtual address for a PTE.
493 */
494 #define ARM_TT_PT_ADDR_SHIFT (11ULL)
495
496 /* TODO: Give this a better name/documentation than "other" */
497 #define ARM_TT_PT_OTHER_MASK (0x3fffULL)
498 #endif
499
500 #define ARM_PT_DESC_INDEX(ptep) \
501 (((unsigned)(ptep) & ARM_PT_DESC_INDEX_MASK) >> ARM_PT_DESC_INDEX_SHIFT)
502
503
504 #define ptep_get_ptd(ptep) \
505 ((struct pt_desc *)((*((vm_offset_t *)(pai_to_pvh(pa_index((vm_offset_t)(ptep) - gVirtBase + gPhysBase))))) & PVH_LIST_MASK))
506
507 #define ptep_get_va(ptep) \
508 ((((pt_desc_t *) (pvh_list(pai_to_pvh(pa_index((((vm_offset_t)(ptep) & ~ARM_TT_PT_OTHER_MASK) - gVirtBase + gPhysBase))))))->pt_map[ARM_PT_DESC_INDEX(ptep)].va)+ ((((unsigned)(ptep)) & ARM_TT_PT_INDEX_MASK)<<ARM_TT_PT_ADDR_SHIFT))
509
510 #define ptep_get_pmap(ptep) \
511 ((((pt_desc_t *) (pvh_list(pai_to_pvh(pa_index((((vm_offset_t)(ptep) & ~ARM_TT_PT_OTHER_MASK) - gVirtBase + gPhysBase))))))->pmap))
512
513 #endif
514
515
516 /* PVH Define Macros */
517
518 /* pvhead type */
519 #define PVH_TYPE_NULL 0x0UL
520 #define PVH_TYPE_PVEP 0x1UL
521 #define PVH_TYPE_PTEP 0x2UL
522 #define PVH_TYPE_PTDP 0x3UL
523
524 #define PVH_TYPE_MASK (0x3UL)
525 #define PVH_LIST_MASK (~PVH_TYPE_MASK)
526
527 #if (__ARM_VMSA__ == 7)
528 #define pvh_set_bits(h, b) \
529 do { \
530 while (!OSCompareAndSwap(*(vm_offset_t *)(h), *(vm_offset_t *)(h) | (b), (vm_offset_t *)(h))); \
531 } while (0)
532
533 #define pvh_clear_bits(h, b) \
534 do { \
535 while (!OSCompareAndSwap(*(vm_offset_t *)(h), *(vm_offset_t *)(h) & ~(b), (vm_offset_t *)(h))); \
536 } while (0)
537 #else
538 #define pvh_set_bits(h, b) \
539 do { \
540 while (!OSCompareAndSwap64(*(vm_offset_t *)(h), *(vm_offset_t *)(h) | ((int64_t)b), (vm_offset_t *)(h))); \
541 } while (0)
542
543 #define pvh_clear_bits(h, b) \
544 do { \
545 while (!OSCompareAndSwap64(*(vm_offset_t *)(h), *(vm_offset_t *)(h) & ~((int64_t)b), (vm_offset_t *)(h))); \
546 } while (0)
547 #endif
548
549 #define pvh_test_type(h, b) \
550 ((*(vm_offset_t *)(h) & (PVH_TYPE_MASK)) == (b))
551
552 #define pvh_ptep(h) \
553 ((pt_entry_t *)(*(vm_offset_t *)(h) & PVH_LIST_MASK))
554
555 #define pvh_list(h) \
556 ((pv_entry_t *)(*(vm_offset_t *)(h) & PVH_LIST_MASK))
557
558 #define pvh_bits(h) \
559 (*(vm_offset_t *)(h) & PVH_TYPE_MASK)
560
561 #if (__ARM_VMSA__ == 7)
562 #define pvh_update_head(h, e, t) \
563 do { \
564 while (!OSCompareAndSwap(*(vm_offset_t *)(h), (vm_offset_t)(e) | (t), (vm_offset_t *)(h))); \
565 } while (0)
566 #else
567 #define pvh_update_head(h, e, t) \
568 do { \
569 while (!OSCompareAndSwap64(*(vm_offset_t *)(h), (vm_offset_t)(e) | (t), (vm_offset_t *)(h))); \
570 } while (0)
571 #endif
572
573 #define pvh_add(h, e) \
574 do { \
575 assert(!pvh_test_type((h), PVH_TYPE_PTEP)); \
576 pve_next(e) = pvh_list(h); \
577 pvh_update_head((h), (e), PVH_TYPE_PVEP); \
578 } while (0)
579
580 #define pvh_remove(h, p, e) \
581 do { \
582 assert(!PVE_NEXT_IS_ALTACCT(pve_next((e)))); \
583 if ((p) == (h)) { \
584 if (PVE_NEXT_PTR(pve_next((e))) == PV_ENTRY_NULL) { \
585 pvh_update_head((h), PV_ENTRY_NULL, PVH_TYPE_NULL); \
586 } else { \
587 pvh_update_head((h), PVE_NEXT_PTR(pve_next((e))), PVH_TYPE_PVEP); \
588 } \
589 } else { \
590 /* \
591 * PMAP LEDGERS: \
592 * preserve the "alternate accounting" bit \
593 * when updating "p" (the previous entry's \
594 * "pve_next"). \
595 */ \
596 boolean_t __is_altacct; \
597 __is_altacct = PVE_NEXT_IS_ALTACCT(*(p)); \
598 *(p) = PVE_NEXT_PTR(pve_next((e))); \
599 if (__is_altacct) { \
600 PVE_NEXT_SET_ALTACCT((p)); \
601 } else { \
602 PVE_NEXT_CLR_ALTACCT((p)); \
603 } \
604 } \
605 } while (0)
606
607
608 /* PPATTR Define Macros */
609
610 #define ppattr_set_bits(h, b) \
611 do { \
612 while (!OSCompareAndSwap16(*(pp_attr_t *)(h), *(pp_attr_t *)(h) | (b), (pp_attr_t *)(h))); \
613 } while (0)
614
615 #define ppattr_clear_bits(h, b) \
616 do { \
617 while (!OSCompareAndSwap16(*(pp_attr_t *)(h), *(pp_attr_t *)(h) & ~(b), (pp_attr_t *)(h))); \
618 } while (0)
619
620 #define ppattr_test_bits(h, b) \
621 ((*(pp_attr_t *)(h) & (b)) == (b))
622
623 #define pa_set_bits(x, b) \
624 do { \
625 if (pa_valid(x)) \
626 ppattr_set_bits(&pp_attr_table[pa_index(x)], \
627 (b)); \
628 } while (0)
629
630 #define pa_test_bits(x, b) \
631 (pa_valid(x) ? ppattr_test_bits(&pp_attr_table[pa_index(x)],\
632 (b)) : FALSE)
633
634 #define pa_clear_bits(x, b) \
635 do { \
636 if (pa_valid(x)) \
637 ppattr_clear_bits(&pp_attr_table[pa_index(x)], \
638 (b)); \
639 } while (0)
640
641 #define pa_set_modify(x) \
642 pa_set_bits(x, PP_ATTR_MODIFIED)
643
644 #define pa_clear_modify(x) \
645 pa_clear_bits(x, PP_ATTR_MODIFIED)
646
647 #define pa_set_reference(x) \
648 pa_set_bits(x, PP_ATTR_REFERENCED)
649
650 #define pa_clear_reference(x) \
651 pa_clear_bits(x, PP_ATTR_REFERENCED)
652
653
654 #define IS_INTERNAL_PAGE(pai) \
655 ppattr_test_bits(&pp_attr_table[pai], PP_ATTR_INTERNAL)
656 #define SET_INTERNAL_PAGE(pai) \
657 ppattr_set_bits(&pp_attr_table[pai], PP_ATTR_INTERNAL)
658 #define CLR_INTERNAL_PAGE(pai) \
659 ppattr_clear_bits(&pp_attr_table[pai], PP_ATTR_INTERNAL)
660
661 #define IS_REUSABLE_PAGE(pai) \
662 ppattr_test_bits(&pp_attr_table[pai], PP_ATTR_REUSABLE)
663 #define SET_REUSABLE_PAGE(pai) \
664 ppattr_set_bits(&pp_attr_table[pai], PP_ATTR_REUSABLE)
665 #define CLR_REUSABLE_PAGE(pai) \
666 ppattr_clear_bits(&pp_attr_table[pai], PP_ATTR_REUSABLE)
667
668 #define IS_ALTACCT_PAGE(pai, pve_p) \
669 (((pve_p) == NULL) \
670 ? ppattr_test_bits(&pp_attr_table[pai], PP_ATTR_ALTACCT) \
671 : PVE_NEXT_IS_ALTACCT(pve_next((pve_p))))
672 #define SET_ALTACCT_PAGE(pai, pve_p) \
673 if ((pve_p) == NULL) { \
674 ppattr_set_bits(&pp_attr_table[pai], PP_ATTR_ALTACCT); \
675 } else { \
676 PVE_NEXT_SET_ALTACCT(&pve_next((pve_p))); \
677 }
678 #define CLR_ALTACCT_PAGE(pai, pve_p) \
679 if ((pve_p) == NULL) { \
680 ppattr_clear_bits(&pp_attr_table[pai], PP_ATTR_ALTACCT);\
681 } else { \
682 PVE_NEXT_CLR_ALTACCT(&pve_next((pve_p))); \
683 }
684
685 #define IS_REFFAULT_PAGE(pai) \
686 ppattr_test_bits(&pp_attr_table[pai], PP_ATTR_REFFAULT)
687 #define SET_REFFAULT_PAGE(pai) \
688 ppattr_set_bits(&pp_attr_table[pai], PP_ATTR_REFFAULT)
689 #define CLR_REFFAULT_PAGE(pai) \
690 ppattr_clear_bits(&pp_attr_table[pai], PP_ATTR_REFFAULT)
691
692 #define IS_MODFAULT_PAGE(pai) \
693 ppattr_test_bits(&pp_attr_table[pai], PP_ATTR_MODFAULT)
694 #define SET_MODFAULT_PAGE(pai) \
695 ppattr_set_bits(&pp_attr_table[pai], PP_ATTR_MODFAULT)
696 #define CLR_MODFAULT_PAGE(pai) \
697 ppattr_clear_bits(&pp_attr_table[pai], PP_ATTR_MODFAULT)
698
699
700 #if (__ARM_VMSA__ == 7)
701
702 #define tte_index(pmap, addr) \
703 ttenum((addr))
704
705 #define tte_get_ptd(tte) \
706 ((struct pt_desc *)((*((vm_offset_t *)(pai_to_pvh(pa_index((vm_offset_t)((tte) & ~PAGE_MASK)))))) & PVH_LIST_MASK))
707
708 #else
709
710 #define tt0_index(pmap, addr) \
711 (((addr) & ARM_TT_L0_INDEX_MASK) >> ARM_TT_L0_SHIFT)
712
713 #define tt1_index(pmap, addr) \
714 (((addr) & ARM_TT_L1_INDEX_MASK) >> ARM_TT_L1_SHIFT)
715
716 #define tt2_index(pmap, addr) \
717 (((addr) & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT)
718
719 #define tt3_index(pmap, addr) \
720 (((addr) & ARM_TT_L3_INDEX_MASK) >> ARM_TT_L3_SHIFT)
721
722 #define tte_index(pmap, addr) \
723 (((addr) & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT)
724
725 #define tte_get_ptd(tte) \
726 ((struct pt_desc *)((*((vm_offset_t *)(pai_to_pvh(pa_index((vm_offset_t)((tte) & ~PAGE_MASK)))))) & PVH_LIST_MASK))
727
728 #endif
729
730 /*
731 * Lock on pmap system
732 */
733
734 #define PMAP_LOCK_INIT(pmap) { \
735 simple_lock_init(&(pmap)->lock, 0); \
736 }
737
738 #define PMAP_LOCK(pmap) { \
739 simple_lock(&(pmap)->lock); \
740 }
741
742 #define PMAP_UNLOCK(pmap) { \
743 simple_unlock(&(pmap)->lock); \
744 }
745
746 #if MACH_ASSERT
747 #define PMAP_ASSERT_LOCKED(pmap) { \
748 simple_lock_assert(&(pmap)->lock, LCK_ASSERT_OWNED); \
749 }
750 #else
751 #define PMAP_ASSERT_LOCKED(pmap)
752 #endif
753
754 /*
755 * Each entry in the pv_head_table is locked by a bit in the
756 * pv lock array, which is stored in the region preceding pv_head_table.
757 * The lock bits are accessed by the physical address of the page they lock.
758 */
759 #define LOCK_PVH(index) { \
760 hw_lock_bit((hw_lock_bit_t *) \
761 ((unsigned int*)pv_head_table)-1-(index>>5), \
762 (index&0x1F)); \
763 }
764
765 #define UNLOCK_PVH(index) { \
766 hw_unlock_bit((hw_lock_bit_t *) \
767 ((unsigned int*)pv_head_table)-1-(index>>5), \
768 (index&0x1F)); \
769 }
770
771 #define ASSERT_PVH_LOCKED(index) { \
772 assert(*(((unsigned int*)pv_head_table)-1-(index>>5)) & (1 << (index & 0x1F))); \
773 }
774
775 #define PMAP_UPDATE_TLBS(pmap, s, e) { \
776 flush_mmu_tlb_region_asid(s, (unsigned)(e - s), pmap); \
777 }
778
779 #ifdef __ARM_L1_PTW__
780
781 #define FLUSH_PTE_RANGE(spte, epte) \
782 __asm__ volatile("dsb ish");
783
784 #define FLUSH_PTE(pte_p) \
785 __asm__ volatile("dsb ish");
786
787 #else
788
789 #define FLUSH_PTE_RANGE(spte, epte) \
790 CleanPoU_DcacheRegion((vm_offset_t)spte, \
791 (vm_offset_t)epte - (vm_offset_t)spte);
792
793 #define FLUSH_PTE(pte_p) \
794 CleanPoU_DcacheRegion((vm_offset_t)pte_p, sizeof(pt_entry_t));
795 #endif
796
797 #define WRITE_PTE(pte_p, pte_entry) \
798 __unreachable_ok_push \
799 if (TEST_PAGE_RATIO_4) { \
800 do { \
801 if (((unsigned)(pte_p)) & 0x1f) panic("WRITE_PTE\n"); \
802 if (((pte_entry) & ~ARM_PTE_COMPRESSED_MASK) == ARM_PTE_EMPTY) { \
803 *(pte_p) = (pte_entry); \
804 *((pte_p)+1) = (pte_entry); \
805 *((pte_p)+2) = (pte_entry); \
806 *((pte_p)+3) = (pte_entry); \
807 } else { \
808 *(pte_p) = (pte_entry); \
809 *((pte_p)+1) = (pte_entry) | 0x1000; \
810 *((pte_p)+2) = (pte_entry) | 0x2000; \
811 *((pte_p)+3) = (pte_entry) | 0x3000; \
812 } \
813 FLUSH_PTE_RANGE((pte_p),((pte_p)+4)); \
814 } while(0); \
815 } else { \
816 do { \
817 *(pte_p) = (pte_entry); \
818 FLUSH_PTE(pte_p); \
819 } while(0); \
820 } \
821 __unreachable_ok_pop
822
823 #define WRITE_PTE_FAST(pte_p, pte_entry) \
824 __unreachable_ok_push \
825 if (TEST_PAGE_RATIO_4) { \
826 if (((unsigned)(pte_p)) & 0x1f) panic("WRITE_PTE\n"); \
827 if (((pte_entry) & ~ARM_PTE_COMPRESSED_MASK) == ARM_PTE_EMPTY) { \
828 *(pte_p) = (pte_entry); \
829 *((pte_p)+1) = (pte_entry); \
830 *((pte_p)+2) = (pte_entry); \
831 *((pte_p)+3) = (pte_entry); \
832 } else { \
833 *(pte_p) = (pte_entry); \
834 *((pte_p)+1) = (pte_entry) | 0x1000; \
835 *((pte_p)+2) = (pte_entry) | 0x2000; \
836 *((pte_p)+3) = (pte_entry) | 0x3000; \
837 } \
838 } else { \
839 *(pte_p) = (pte_entry); \
840 } \
841 __unreachable_ok_pop
842
843
844 /*
845 * Other useful macros.
846 */
847 #define current_pmap() \
848 (vm_map_pmap(current_thread()->map))
849
850 #define PMAP_IS_VALID(x) (TRUE)
851
852 #ifdef PMAP_TRACES
853 unsigned int pmap_trace = 0;
854
855 #define PMAP_TRACE(...) \
856 if (pmap_trace) { \
857 KDBG_RELEASE(__VA_ARGS__); \
858 }
859 #else
860 #define PMAP_TRACE(...) KDBG_DEBUG(__VA_ARGS__)
861 #endif
862
863 #define PMAP_TRACE_CONSTANT(...) KDBG_RELEASE(__VA_ARGS__)
864
865 /*
866 * Internal function prototypes (forward declarations).
867 */
868
869 static void pv_init(
870 void);
871
872 static boolean_t pv_alloc(
873 pmap_t pmap,
874 unsigned int pai,
875 pv_entry_t **pvepp);
876
877 static void pv_free(
878 pv_entry_t *pvep);
879
880 static void pv_list_free(
881 pv_entry_t *pvehp,
882 pv_entry_t *pvetp,
883 unsigned int cnt);
884
885 static void ptd_bootstrap(
886 pt_desc_t *ptdp, unsigned int ptd_cnt);
887
888 static pt_desc_t *ptd_alloc(
889 pmap_t pmap);
890
891 static void ptd_deallocate(
892 pt_desc_t *ptdp);
893
894 static void ptd_init(
895 pt_desc_t *ptdp, pmap_t pmap, vm_map_address_t va, unsigned int ttlevel, pt_entry_t * pte_p);
896
897 static void pmap_zone_init(
898 void);
899
900 static void pmap_set_reference(
901 ppnum_t pn);
902
903 ppnum_t pmap_vtophys(
904 pmap_t pmap, addr64_t va);
905
906 void pmap_switch_user_ttb(
907 pmap_t pmap);
908
909 static void flush_mmu_tlb_region_asid(
910 vm_offset_t va, unsigned length, pmap_t pmap);
911
912 static kern_return_t pmap_expand(
913 pmap_t, vm_map_address_t, unsigned int options, unsigned int level);
914
915 static int pmap_remove_range(
916 pmap_t, vm_map_address_t, pt_entry_t *, pt_entry_t *, uint32_t *);
917
918 static int pmap_remove_range_options(
919 pmap_t, vm_map_address_t, pt_entry_t *, pt_entry_t *, uint32_t *, int);
920
921 static tt_entry_t *pmap_tt1_allocate(
922 pmap_t, vm_size_t, unsigned int);
923
924 #define PMAP_TT_ALLOCATE_NOWAIT 0x1
925
926 static void pmap_tt1_deallocate(
927 pmap_t, tt_entry_t *, vm_size_t, unsigned int);
928
929 #define PMAP_TT_DEALLOCATE_NOBLOCK 0x1
930
931 static kern_return_t pmap_tt_allocate(
932 pmap_t, tt_entry_t **, unsigned int, unsigned int);
933
934 #define PMAP_TT_ALLOCATE_NOWAIT 0x1
935
936 static void pmap_tte_deallocate(
937 pmap_t, tt_entry_t *, unsigned int);
938
939 #define PMAP_TT_L1_LEVEL 0x1
940 #define PMAP_TT_L2_LEVEL 0x2
941 #define PMAP_TT_L3_LEVEL 0x3
942 #if (__ARM_VMSA__ == 7)
943 #define PMAP_TT_MAX_LEVEL PMAP_TT_L2_LEVEL
944 #else
945 #define PMAP_TT_MAX_LEVEL PMAP_TT_L3_LEVEL
946 #endif
947
948 #ifdef __ARM64_PMAP_SUBPAGE_L1__
949 #if (__ARM_VMSA__ <= 7)
950 #error This is not supported for old-style page tables
951 #endif
952 #define PMAP_ROOT_ALLOC_SIZE (((ARM_TT_L1_INDEX_MASK >> ARM_TT_L1_SHIFT) + 1) * sizeof(tt_entry_t))
953 #else
954 #define PMAP_ROOT_ALLOC_SIZE (ARM_PGBYTES)
955 #endif
956
957 const unsigned int arm_hardware_page_size = ARM_PGBYTES;
958 const unsigned int arm_pt_desc_size = sizeof(pt_desc_t);
959 const unsigned int arm_pt_root_size = PMAP_ROOT_ALLOC_SIZE;
960
961 #define PMAP_TT_DEALLOCATE_NOBLOCK 0x1
962
963 void pmap_init_pte_page_internal(
964 pmap_t, pt_entry_t *, vm_offset_t, unsigned int , pt_desc_t **);
965
966
967 #if (__ARM_VMSA__ > 7)
968
969 static inline tt_entry_t *pmap_tt1e(
970 pmap_t, vm_map_address_t);
971
972 static inline tt_entry_t *pmap_tt2e(
973 pmap_t, vm_map_address_t);
974
975 static inline pt_entry_t *pmap_tt3e(
976 pmap_t, vm_map_address_t);
977
978 static void pmap_unmap_sharedpage32(
979 pmap_t pmap);
980
981 static void pmap_sharedpage_flush_32_to_64(
982 void);
983
984 static boolean_t
985 pmap_is_64bit(pmap_t);
986
987
988 #endif
989 static inline tt_entry_t *pmap_tte(
990 pmap_t, vm_map_address_t);
991
992 static inline pt_entry_t *pmap_pte(
993 pmap_t, vm_map_address_t);
994
995 static void pmap_update_cache_attributes_locked(
996 ppnum_t, unsigned);
997
998 boolean_t arm_clear_fast_fault(
999 ppnum_t ppnum,
1000 vm_prot_t fault_type);
1001
1002 static pmap_paddr_t pmap_pages_reclaim(
1003 void);
1004
1005 static kern_return_t pmap_pages_alloc(
1006 pmap_paddr_t *pa,
1007 unsigned size,
1008 unsigned option);
1009
1010 #define PMAP_PAGES_ALLOCATE_NOWAIT 0x1
1011 #define PMAP_PAGES_RECLAIM_NOWAIT 0x2
1012
1013 static void pmap_pages_free(
1014 pmap_paddr_t pa,
1015 unsigned size);
1016
1017
1018 #define PMAP_SUPPORT_PROTOTYPES(__return_type, __function_name, __function_args, __function_index) \
1019 static __return_type __function_name##_internal __function_args;
1020
1021 PMAP_SUPPORT_PROTOTYPES(
1022 kern_return_t,
1023 arm_fast_fault, (pmap_t pmap,
1024 vm_map_address_t va,
1025 vm_prot_t fault_type,
1026 boolean_t from_user), ARM_FAST_FAULT_INDEX);
1027
1028
1029 PMAP_SUPPORT_PROTOTYPES(
1030 boolean_t,
1031 arm_force_fast_fault, (ppnum_t ppnum,
1032 vm_prot_t allow_mode,
1033 int options), ARM_FORCE_FAST_FAULT_INDEX);
1034
1035 PMAP_SUPPORT_PROTOTYPES(
1036 kern_return_t,
1037 mapping_free_prime, (void), MAPPING_FREE_PRIME_INDEX);
1038
1039 PMAP_SUPPORT_PROTOTYPES(
1040 kern_return_t,
1041 mapping_replenish, (void), MAPPING_REPLENISH_INDEX);
1042
1043 PMAP_SUPPORT_PROTOTYPES(
1044 boolean_t,
1045 pmap_batch_set_cache_attributes, (ppnum_t pn,
1046 unsigned int cacheattr,
1047 unsigned int page_cnt,
1048 unsigned int page_index,
1049 boolean_t doit,
1050 unsigned int *res), PMAP_BATCH_SET_CACHE_ATTRIBUTES_INDEX);
1051
1052 PMAP_SUPPORT_PROTOTYPES(
1053 void,
1054 pmap_change_wiring, (pmap_t pmap,
1055 vm_map_address_t v,
1056 boolean_t wired), PMAP_CHANGE_WIRING_INDEX);
1057
1058 PMAP_SUPPORT_PROTOTYPES(
1059 pmap_t,
1060 pmap_create, (ledger_t ledger,
1061 vm_map_size_t size,
1062 boolean_t is_64bit), PMAP_CREATE_INDEX);
1063
1064 PMAP_SUPPORT_PROTOTYPES(
1065 void,
1066 pmap_destroy, (pmap_t pmap), PMAP_DESTROY_INDEX);
1067
1068
1069
1070 PMAP_SUPPORT_PROTOTYPES(
1071 kern_return_t,
1072 pmap_enter_options, (pmap_t pmap,
1073 vm_map_address_t v,
1074 ppnum_t pn,
1075 vm_prot_t prot,
1076 vm_prot_t fault_type,
1077 unsigned int flags,
1078 boolean_t wired,
1079 unsigned int options), PMAP_ENTER_OPTIONS_INDEX);
1080
1081 PMAP_SUPPORT_PROTOTYPES(
1082 vm_offset_t,
1083 pmap_extract, (pmap_t pmap,
1084 vm_map_address_t va), PMAP_EXTRACT_INDEX);
1085
1086 PMAP_SUPPORT_PROTOTYPES(
1087 ppnum_t,
1088 pmap_find_phys, (pmap_t pmap,
1089 addr64_t va), PMAP_FIND_PHYS_INDEX);
1090
1091 #if (__ARM_VMSA__ > 7)
1092 PMAP_SUPPORT_PROTOTYPES(
1093 void,
1094 pmap_insert_sharedpage, (pmap_t pmap), PMAP_INSERT_SHAREDPAGE_INDEX);
1095 #endif
1096
1097
1098 PMAP_SUPPORT_PROTOTYPES(
1099 boolean_t,
1100 pmap_is_empty, (pmap_t pmap,
1101 vm_map_offset_t va_start,
1102 vm_map_offset_t va_end), PMAP_IS_EMPTY_INDEX);
1103
1104
1105 PMAP_SUPPORT_PROTOTYPES(
1106 unsigned int,
1107 pmap_map_cpu_windows_copy, (ppnum_t pn,
1108 vm_prot_t prot,
1109 unsigned int wimg_bits), PMAP_MAP_CPU_WINDOWS_COPY_INDEX);
1110
1111 PMAP_SUPPORT_PROTOTYPES(
1112 kern_return_t,
1113 pmap_nest, (pmap_t grand,
1114 pmap_t subord,
1115 addr64_t vstart,
1116 addr64_t nstart,
1117 uint64_t size), PMAP_NEST_INDEX);
1118
1119 PMAP_SUPPORT_PROTOTYPES(
1120 void,
1121 pmap_page_protect_options, (ppnum_t ppnum,
1122 vm_prot_t prot,
1123 unsigned int options), PMAP_PAGE_PROTECT_OPTIONS_INDEX);
1124
1125 PMAP_SUPPORT_PROTOTYPES(
1126 void,
1127 pmap_protect_options, (pmap_t pmap,
1128 vm_map_address_t start,
1129 vm_map_address_t end,
1130 vm_prot_t prot,
1131 unsigned int options,
1132 void *args), PMAP_PROTECT_OPTIONS_INDEX);
1133
1134 PMAP_SUPPORT_PROTOTYPES(
1135 kern_return_t,
1136 pmap_query_page_info, (pmap_t pmap,
1137 vm_map_offset_t va,
1138 int *disp_p), PMAP_QUERY_PAGE_INFO_INDEX);
1139
1140 PMAP_SUPPORT_PROTOTYPES(
1141 boolean_t,
1142 pmap_query_resident, (pmap_t pmap,
1143 vm_map_address_t start,
1144 vm_map_address_t end,
1145 mach_vm_size_t *resident_bytes_p,
1146 mach_vm_size_t *compressed_bytes_p), PMAP_QUERY_RESIDENT_INDEX);
1147
1148 PMAP_SUPPORT_PROTOTYPES(
1149 void,
1150 pmap_reference, (pmap_t pmap), PMAP_REFERENCE_INDEX);
1151
1152 PMAP_SUPPORT_PROTOTYPES(
1153 int,
1154 pmap_remove_options, (pmap_t pmap,
1155 vm_map_address_t start,
1156 vm_map_address_t end,
1157 int options), PMAP_REMOVE_OPTIONS_INDEX);
1158
1159 PMAP_SUPPORT_PROTOTYPES(
1160 kern_return_t,
1161 pmap_return, (boolean_t do_panic,
1162 boolean_t do_recurse), PMAP_RETURN_INDEX);
1163
1164 PMAP_SUPPORT_PROTOTYPES(
1165 void,
1166 pmap_set_cache_attributes, (ppnum_t pn,
1167 unsigned int cacheattr), PMAP_SET_CACHE_ATTRIBUTES_INDEX);
1168
1169 PMAP_SUPPORT_PROTOTYPES(
1170 void,
1171 pmap_set_nested, (pmap_t pmap), PMAP_SET_NESTED_INDEX);
1172
1173 #if MACH_ASSERT
1174 PMAP_SUPPORT_PROTOTYPES(
1175 void,
1176 pmap_set_process, (pmap_t pmap,
1177 int pid,
1178 char *procname), PMAP_SET_PROCESS_INDEX);
1179 #endif
1180
1181
1182 PMAP_SUPPORT_PROTOTYPES(
1183 void,
1184 pmap_unmap_cpu_windows_copy, (unsigned int index), PMAP_UNMAP_CPU_WINDOWS_COPY_INDEX);
1185
1186 PMAP_SUPPORT_PROTOTYPES(
1187 kern_return_t,
1188 pmap_unnest_options, (pmap_t grand,
1189 addr64_t vaddr,
1190 uint64_t size,
1191 unsigned int option), PMAP_UNNEST_OPTIONS_INDEX);
1192
1193
1194 PMAP_SUPPORT_PROTOTYPES(
1195 void,
1196 phys_attribute_set, (ppnum_t pn,
1197 unsigned int bits), PHYS_ATTRIBUTE_SET_INDEX);
1198
1199
1200 PMAP_SUPPORT_PROTOTYPES(
1201 void,
1202 phys_attribute_clear, (ppnum_t pn,
1203 unsigned int bits,
1204 int options,
1205 void *arg), PHYS_ATTRIBUTE_CLEAR_INDEX);
1206
1207 PMAP_SUPPORT_PROTOTYPES(
1208 void,
1209 pmap_switch, (pmap_t pmap), PMAP_SWITCH_INDEX);
1210
1211 PMAP_SUPPORT_PROTOTYPES(
1212 void,
1213 pmap_switch_user_ttb, (pmap_t pmap), PMAP_SWITCH_USER_TTB_INDEX);
1214
1215
1216
1217 void pmap_footprint_suspend(vm_map_t map,
1218 boolean_t suspend);
1219 PMAP_SUPPORT_PROTOTYPES(
1220 void,
1221 pmap_footprint_suspend, (vm_map_t map,
1222 boolean_t suspend),
1223 PMAP_FOOTPRINT_SUSPEND_INDEX);
1224
1225 #if CONFIG_PGTRACE
1226 boolean_t pgtrace_enabled = 0;
1227
1228 typedef struct {
1229 queue_chain_t chain;
1230
1231 /*
1232 pmap - pmap for below addresses
1233 ova - original va page address
1234 cva - clone va addresses for pre, target and post pages
1235 cva_spte - clone saved ptes
1236 range - trace range in this map
1237 cloned - has been cloned or not
1238 */
1239 pmap_t pmap;
1240 vm_map_offset_t ova;
1241 vm_map_offset_t cva[3];
1242 pt_entry_t cva_spte[3];
1243 struct {
1244 pmap_paddr_t start;
1245 pmap_paddr_t end;
1246 } range;
1247 bool cloned;
1248 } pmap_pgtrace_map_t;
1249
1250 static void pmap_pgtrace_init(void);
1251 static bool pmap_pgtrace_enter_clone(pmap_t pmap, vm_map_offset_t va_page, vm_map_offset_t start, vm_map_offset_t end);
1252 static void pmap_pgtrace_remove_clone(pmap_t pmap, pmap_paddr_t pa_page, vm_map_offset_t va_page);
1253 static void pmap_pgtrace_remove_all_clone(pmap_paddr_t pa);
1254 #endif
1255
1256 #if (__ARM_VMSA__ > 7)
1257 /*
1258 * The low global vector page is mapped at a fixed alias.
1259 * Since the page size is 16k for H8 and newer we map the globals to a 16k
1260 * aligned address. Readers of the globals (e.g. lldb, panic server) need
1261 * to check both addresses anyway for backward compatibility. So for now
1262 * we leave H6 and H7 where they were.
1263 */
1264 #if (ARM_PGSHIFT == 14)
1265 #define LOWGLOBAL_ALIAS (LOW_GLOBAL_BASE_ADDRESS + 0x4000)
1266 #else
1267 #define LOWGLOBAL_ALIAS (LOW_GLOBAL_BASE_ADDRESS + 0x2000)
1268 #endif
1269
1270 #else
1271 #define LOWGLOBAL_ALIAS (0xFFFF1000)
1272 #endif
1273
1274 long long alloc_tteroot_count __attribute__((aligned(8))) MARK_AS_PMAP_DATA = 0LL;
1275 long long alloc_ttepages_count __attribute__((aligned(8))) MARK_AS_PMAP_DATA = 0LL;
1276 long long alloc_ptepages_count __attribute__((aligned(8))) MARK_AS_PMAP_DATA = 0LL;
1277 long long alloc_pmap_pages_count __attribute__((aligned(8))) = 0LL;
1278
1279 int pt_fake_zone_index = -1; /* index of pmap fake zone */
1280
1281
1282
1283 /*
1284 * Allocates and initializes a per-CPU data structure for the pmap.
1285 */
1286 static void
1287 pmap_cpu_data_init_internal(unsigned int cpu_number)
1288 {
1289 pmap_cpu_data_t * pmap_cpu_data = NULL;
1290
1291 pmap_cpu_data = pmap_get_cpu_data();
1292 pmap_cpu_data->cpu_number = cpu_number;
1293 }
1294
1295 void
1296 pmap_cpu_data_init(void)
1297 {
1298 pmap_cpu_data_init_internal(cpu_number());
1299 }
1300
1301 static void
1302 pmap_cpu_data_array_init(void)
1303 {
1304
1305 pmap_cpu_data_init();
1306 }
1307
1308 pmap_cpu_data_t *
1309 pmap_get_cpu_data(void)
1310 {
1311 pmap_cpu_data_t * pmap_cpu_data = NULL;
1312
1313 pmap_cpu_data = &getCpuDatap()->cpu_pmap_cpu_data;
1314
1315 return pmap_cpu_data;
1316 }
1317
1318
1319 /* TODO */
1320 pmap_paddr_t
1321 pmap_pages_reclaim(
1322 void)
1323 {
1324 boolean_t found_page;
1325 unsigned i;
1326 pt_desc_t *ptdp;
1327
1328
1329 /*
1330 * pmap_pages_reclaim() is returning a page by freeing an active pt page.
1331 * To be eligible, a pt page is assigned to a user pmap. It doesn't have any wired pte
1332 * entry and it contains at least one valid pte entry.
1333 *
1334 * In a loop, check for a page in the reclaimed pt page list.
1335 * if one is present, unlink that page and return the physical page address.
1336 * Otherwise, scan the pt page list for an eligible pt page to reclaim.
1337 * If found, invoke pmap_remove_range() on its pmap and address range then
1338 * deallocates that pt page. This will end up adding the pt page to the
1339 * reclaimed pt page list.
1340 * If no eligible page were found in the pt page list, panic.
1341 */
1342
1343 simple_lock(&pmap_pages_lock);
1344 pmap_pages_request_count++;
1345 pmap_pages_request_acum++;
1346
1347 while (1) {
1348
1349 if (pmap_pages_reclaim_list != (page_free_entry_t *)NULL) {
1350 page_free_entry_t *page_entry;
1351
1352 page_entry = pmap_pages_reclaim_list;
1353 pmap_pages_reclaim_list = pmap_pages_reclaim_list->next;
1354 simple_unlock(&pmap_pages_lock);
1355
1356 return((pmap_paddr_t)ml_static_vtop((vm_offset_t)page_entry));
1357 }
1358
1359 simple_unlock(&pmap_pages_lock);
1360
1361 simple_lock(&pt_pages_lock);
1362 ptdp = (pt_desc_t *)queue_first(&pt_page_list);
1363 found_page = FALSE;
1364
1365 while (!queue_end(&pt_page_list, (queue_entry_t)ptdp)) {
1366 if ((ptdp->pmap != kernel_pmap)
1367 && (ptdp->pmap->nested == FALSE)
1368 && (simple_lock_try(&ptdp->pmap->lock))) {
1369
1370 unsigned refcnt_acc = 0;
1371 unsigned wiredcnt_acc = 0;
1372
1373 for (i = 0 ; i < PT_INDEX_MAX ; i++) {
1374 if (ptdp->pt_cnt[i].refcnt & PT_DESC_REFCOUNT) {
1375 /* Do not attempt to free a page that contains an L2 table
1376 * or is currently being operated on by pmap_enter(),
1377 * which can drop the pmap lock. */
1378 refcnt_acc = 0;
1379 break;
1380 }
1381 refcnt_acc += ptdp->pt_cnt[i].refcnt;
1382 wiredcnt_acc += ptdp->pt_cnt[i].wiredcnt;
1383 }
1384 if ((wiredcnt_acc == 0) && (refcnt_acc != 0)) {
1385 found_page = TRUE;
1386 /* Leave ptdp->pmap locked here. We're about to reclaim
1387 * a tt page from it, so we don't want anyone else messing
1388 * with it while we do that. */
1389 break;
1390 }
1391 simple_unlock(&ptdp->pmap->lock);
1392 }
1393 ptdp = (pt_desc_t *)queue_next((queue_t)ptdp);
1394 }
1395 if (!found_page) {
1396 panic("pmap_pages_reclaim(): No eligible page in pt_page_list\n");
1397 } else {
1398 int remove_count = 0;
1399 vm_map_address_t va;
1400 pmap_t pmap;
1401 pt_entry_t *bpte, *epte;
1402 pt_entry_t *pte_p;
1403 tt_entry_t *tte_p;
1404 uint32_t rmv_spte=0;
1405
1406 simple_unlock(&pt_pages_lock);
1407 pmap = ptdp->pmap;
1408 PMAP_ASSERT_LOCKED(pmap); // pmap lock should be held from loop above
1409 for (i = 0 ; i < PT_INDEX_MAX ; i++) {
1410 va = ptdp->pt_map[i].va;
1411
1412 tte_p = pmap_tte(pmap, va);
1413 if ((tte_p != (tt_entry_t *) NULL)
1414 && ((*tte_p & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_TABLE)) {
1415
1416 #if (__ARM_VMSA__ == 7)
1417 pte_p = (pt_entry_t *) ttetokv(*tte_p);
1418 bpte = &pte_p[ptenum(va)];
1419 epte = bpte + PAGE_SIZE/sizeof(pt_entry_t);
1420 #else
1421 pte_p = (pt_entry_t *) ttetokv(*tte_p);
1422 bpte = &pte_p[tt3_index(pmap, va)];
1423 epte = bpte + PAGE_SIZE/sizeof(pt_entry_t);
1424 #endif
1425 /*
1426 * Use PMAP_OPTIONS_REMOVE to clear any
1427 * "compressed" markers and update the
1428 * "compressed" counter in pmap->stats.
1429 * This means that we lose accounting for
1430 * any compressed pages in this range
1431 * but the alternative is to not be able
1432 * to account for their future decompression,
1433 * which could cause the counter to drift
1434 * more and more.
1435 */
1436 remove_count += pmap_remove_range_options(
1437 pmap, va, bpte, epte,
1438 &rmv_spte, PMAP_OPTIONS_REMOVE);
1439 if (ptdp->pt_cnt[ARM_PT_DESC_INDEX(pte_p)].refcnt != 0)
1440 panic("pmap_pages_reclaim(): ptdp %p, count %d\n", ptdp, ptdp->pt_cnt[ARM_PT_DESC_INDEX(pte_p)].refcnt);
1441 #if (__ARM_VMSA__ == 7)
1442 pmap_tte_deallocate(pmap, tte_p, PMAP_TT_L1_LEVEL);
1443 flush_mmu_tlb_entry((va & ~ARM_TT_L1_PT_OFFMASK) | (pmap->asid & 0xff));
1444 flush_mmu_tlb_entry(((va & ~ARM_TT_L1_PT_OFFMASK) + ARM_TT_L1_SIZE) | (pmap->asid & 0xff));
1445 flush_mmu_tlb_entry(((va & ~ARM_TT_L1_PT_OFFMASK) + 2*ARM_TT_L1_SIZE)| (pmap->asid & 0xff));
1446 flush_mmu_tlb_entry(((va & ~ARM_TT_L1_PT_OFFMASK) + 3*ARM_TT_L1_SIZE)| (pmap->asid & 0xff));
1447 #else
1448 pmap_tte_deallocate(pmap, tte_p, PMAP_TT_L2_LEVEL);
1449 flush_mmu_tlb_entry(tlbi_addr(va & ~ARM_TT_L2_OFFMASK) | tlbi_asid(pmap->asid));
1450 #endif
1451
1452 if (remove_count > 0) {
1453 #if (__ARM_VMSA__ == 7)
1454 PMAP_UPDATE_TLBS(pmap, va, va+4*ARM_TT_L1_SIZE);
1455 #else
1456 PMAP_UPDATE_TLBS(pmap, va, va+ARM_TT_L2_SIZE);
1457 #endif
1458 }
1459 }
1460 }
1461 // Undo the lock we grabbed when we found ptdp above
1462 PMAP_UNLOCK(pmap);
1463 }
1464 simple_lock(&pmap_pages_lock);
1465 }
1466 }
1467
1468
1469 static kern_return_t
1470 pmap_pages_alloc(
1471 pmap_paddr_t *pa,
1472 unsigned size,
1473 unsigned option)
1474 {
1475 vm_page_t m = VM_PAGE_NULL, m_prev;
1476
1477 if(option & PMAP_PAGES_RECLAIM_NOWAIT) {
1478 assert(size == PAGE_SIZE);
1479 *pa = pmap_pages_reclaim();
1480 return KERN_SUCCESS;
1481 }
1482 if (size == PAGE_SIZE) {
1483 while ((m = vm_page_grab()) == VM_PAGE_NULL) {
1484 if(option & PMAP_PAGES_ALLOCATE_NOWAIT) {
1485 return KERN_RESOURCE_SHORTAGE;
1486 }
1487
1488 VM_PAGE_WAIT();
1489 }
1490 vm_page_lock_queues();
1491 vm_page_wire(m, VM_KERN_MEMORY_PTE, TRUE);
1492 vm_page_unlock_queues();
1493 }
1494 if (size == 2*PAGE_SIZE) {
1495 while (cpm_allocate(size, &m, 0, 1, TRUE, 0) != KERN_SUCCESS) {
1496 if(option & PMAP_PAGES_ALLOCATE_NOWAIT)
1497 return KERN_RESOURCE_SHORTAGE;
1498
1499 VM_PAGE_WAIT();
1500 }
1501 }
1502
1503 *pa = (pmap_paddr_t)ptoa(VM_PAGE_GET_PHYS_PAGE(m));
1504
1505 vm_object_lock(pmap_object);
1506 while (m != VM_PAGE_NULL) {
1507 vm_page_insert_wired(m, pmap_object, (vm_object_offset_t) ((ptoa(VM_PAGE_GET_PHYS_PAGE(m))) - gPhysBase), VM_KERN_MEMORY_PTE);
1508 m_prev = m;
1509 m = NEXT_PAGE(m_prev);
1510 *(NEXT_PAGE_PTR(m_prev)) = VM_PAGE_NULL;
1511 }
1512 vm_object_unlock(pmap_object);
1513
1514 OSAddAtomic(size>>PAGE_SHIFT, &inuse_pmap_pages_count);
1515 OSAddAtomic64(size>>PAGE_SHIFT, &alloc_pmap_pages_count);
1516
1517 return KERN_SUCCESS;
1518 }
1519
1520
1521 static void
1522 pmap_pages_free(
1523 pmap_paddr_t pa,
1524 unsigned size)
1525 {
1526 simple_lock(&pmap_pages_lock);
1527
1528 if (pmap_pages_request_count != 0) {
1529 page_free_entry_t *page_entry;
1530
1531 pmap_pages_request_count--;
1532 page_entry = (page_free_entry_t *)phystokv(pa);
1533 page_entry->next = pmap_pages_reclaim_list;
1534 pmap_pages_reclaim_list = page_entry;
1535 simple_unlock(&pmap_pages_lock);
1536
1537 return;
1538 }
1539
1540 simple_unlock(&pmap_pages_lock);
1541
1542 vm_page_t m;
1543 pmap_paddr_t pa_max;
1544
1545 OSAddAtomic(-(size>>PAGE_SHIFT), &inuse_pmap_pages_count);
1546
1547 for (pa_max = pa + size; pa < pa_max; pa = pa + PAGE_SIZE) {
1548 vm_object_lock(pmap_object);
1549 m = vm_page_lookup(pmap_object, (pa - gPhysBase));
1550 assert(m != VM_PAGE_NULL);
1551 assert(VM_PAGE_WIRED(m));
1552 vm_page_lock_queues();
1553 vm_page_free(m);
1554 vm_page_unlock_queues();
1555 vm_object_unlock(pmap_object);
1556 }
1557 }
1558
1559 static inline void
1560 PMAP_ZINFO_PALLOC(
1561 pmap_t pmap, int bytes)
1562 {
1563 pmap_ledger_credit(pmap, task_ledgers.tkm_private, bytes);
1564 }
1565
1566 static inline void
1567 PMAP_ZINFO_PFREE(
1568 pmap_t pmap,
1569 int bytes)
1570 {
1571 pmap_ledger_debit(pmap, task_ledgers.tkm_private, bytes);
1572 }
1573
1574 static inline void
1575 pmap_tt_ledger_credit(
1576 pmap_t pmap,
1577 vm_size_t size)
1578 {
1579 if (pmap != kernel_pmap) {
1580 pmap_ledger_credit(pmap, task_ledgers.phys_footprint, size);
1581 pmap_ledger_credit(pmap, task_ledgers.page_table, size);
1582 }
1583 }
1584
1585 static inline void
1586 pmap_tt_ledger_debit(
1587 pmap_t pmap,
1588 vm_size_t size)
1589 {
1590 if (pmap != kernel_pmap) {
1591 pmap_ledger_debit(pmap, task_ledgers.phys_footprint, size);
1592 pmap_ledger_debit(pmap, task_ledgers.page_table, size);
1593 }
1594 }
1595
1596 static unsigned int
1597 alloc_asid(
1598 void)
1599 {
1600 unsigned int asid_bitmap_index;
1601
1602 simple_lock(&pmaps_lock);
1603 for (asid_bitmap_index = 0; asid_bitmap_index < (MAX_ASID / (sizeof(uint32_t) * NBBY)); asid_bitmap_index++) {
1604 unsigned int temp = ffs(asid_bitmap[asid_bitmap_index]);
1605 if (temp > 0) {
1606 temp -= 1;
1607 asid_bitmap[asid_bitmap_index] &= ~(1 << temp);
1608 simple_unlock(&pmaps_lock);
1609
1610 /* We should never vend out physical ASID 0 through this method. */
1611 assert(((asid_bitmap_index * sizeof(uint32_t) * NBBY + temp) % ARM_MAX_ASID) != 0);
1612
1613 return (asid_bitmap_index * sizeof(uint32_t) * NBBY + temp);
1614 }
1615 }
1616 simple_unlock(&pmaps_lock);
1617 /*
1618 * ToDo: Add code to deal with pmap with no asid panic for now. Not
1619 * an issue with the small config process hard limit
1620 */
1621 panic("alloc_asid(): out of ASID number");
1622 return MAX_ASID;
1623 }
1624
1625 static void
1626 free_asid(
1627 int asid)
1628 {
1629 /* Don't free up any alias of physical ASID 0. */
1630 assert((asid % ARM_MAX_ASID) != 0);
1631
1632 simple_lock(&pmaps_lock);
1633 setbit(asid, (int *) asid_bitmap);
1634 simple_unlock(&pmaps_lock);
1635 }
1636
1637 #define PV_LOW_WATER_MARK_DEFAULT 0x200
1638 #define PV_KERN_LOW_WATER_MARK_DEFAULT 0x200
1639 #define PV_ALLOC_CHUNK_INITIAL 0x200
1640 #define PV_KERN_ALLOC_CHUNK_INITIAL 0x200
1641 #define PV_ALLOC_INITIAL_TARGET (PV_ALLOC_CHUNK_INITIAL * 5)
1642 #define PV_KERN_ALLOC_INITIAL_TARGET (PV_KERN_ALLOC_CHUNK_INITIAL)
1643
1644
1645 uint32_t pv_free_count MARK_AS_PMAP_DATA = 0;
1646 uint32_t pv_page_count MARK_AS_PMAP_DATA = 0;
1647 uint32_t pv_kern_free_count MARK_AS_PMAP_DATA = 0;
1648
1649 uint32_t pv_low_water_mark MARK_AS_PMAP_DATA;
1650 uint32_t pv_kern_low_water_mark MARK_AS_PMAP_DATA;
1651 uint32_t pv_alloc_chunk MARK_AS_PMAP_DATA;
1652 uint32_t pv_kern_alloc_chunk MARK_AS_PMAP_DATA;
1653
1654 thread_t mapping_replenish_thread;
1655 event_t mapping_replenish_event;
1656 event_t pmap_user_pv_throttle_event;
1657 volatile uint32_t mappingrecurse = 0;
1658
1659 uint64_t pmap_pv_throttle_stat;
1660 uint64_t pmap_pv_throttled_waiters;
1661
1662 unsigned pmap_mapping_thread_wakeups;
1663 unsigned pmap_kernel_reserve_replenish_stat MARK_AS_PMAP_DATA;
1664 unsigned pmap_user_reserve_replenish_stat MARK_AS_PMAP_DATA;
1665 unsigned pmap_kern_reserve_alloc_stat MARK_AS_PMAP_DATA;
1666
1667
1668 static void
1669 pv_init(
1670 void)
1671 {
1672 simple_lock_init(&pv_free_list_lock, 0);
1673 simple_lock_init(&pv_kern_free_list_lock, 0);
1674 pv_free_list = PV_ENTRY_NULL;
1675 pv_free_count = 0x0U;
1676 pv_kern_free_list = PV_ENTRY_NULL;
1677 pv_kern_free_count = 0x0U;
1678 }
1679
1680 static inline void PV_ALLOC(pv_entry_t **pv_ep);
1681 static inline void PV_KERN_ALLOC(pv_entry_t **pv_e);
1682 static inline void PV_FREE_LIST(pv_entry_t *pv_eh, pv_entry_t *pv_et, int pv_cnt);
1683 static inline void PV_KERN_FREE_LIST(pv_entry_t *pv_eh, pv_entry_t *pv_et, int pv_cnt);
1684
1685 static inline void pmap_pv_throttle(pmap_t p);
1686
1687 static boolean_t
1688 pv_alloc(
1689 pmap_t pmap,
1690 unsigned int pai,
1691 pv_entry_t **pvepp)
1692 {
1693 PMAP_ASSERT_LOCKED(pmap);
1694 ASSERT_PVH_LOCKED(pai);
1695 PV_ALLOC(pvepp);
1696 if (PV_ENTRY_NULL == *pvepp) {
1697
1698 if (kernel_pmap == pmap) {
1699
1700 PV_KERN_ALLOC(pvepp);
1701
1702 if (PV_ENTRY_NULL == *pvepp) {
1703 pv_entry_t *pv_e;
1704 pv_entry_t *pv_eh;
1705 pv_entry_t *pv_et;
1706 int pv_cnt;
1707 unsigned j;
1708 pmap_paddr_t pa;
1709 kern_return_t ret;
1710
1711 UNLOCK_PVH(pai);
1712 PMAP_UNLOCK(pmap);
1713
1714 ret = pmap_pages_alloc(&pa, PAGE_SIZE, PMAP_PAGES_ALLOCATE_NOWAIT);
1715
1716 if (ret == KERN_RESOURCE_SHORTAGE) {
1717 ret = pmap_pages_alloc(&pa, PAGE_SIZE, PMAP_PAGES_RECLAIM_NOWAIT);
1718 }
1719
1720 if (ret != KERN_SUCCESS) {
1721 panic("%s: failed to alloc page for kernel, ret=%d, "
1722 "pmap=%p, pai=%u, pvepp=%p",
1723 __FUNCTION__, ret,
1724 pmap, pai, pvepp);
1725 }
1726
1727 pv_page_count++;
1728
1729 pv_e = (pv_entry_t *)phystokv(pa);
1730 pv_cnt = 0;
1731 pv_eh = pv_et = PV_ENTRY_NULL;
1732 *pvepp = pv_e;
1733 pv_e++;
1734
1735 for (j = 1; j < (PAGE_SIZE/sizeof(pv_entry_t)) ; j++) {
1736 pv_e->pve_next = pv_eh;
1737 pv_eh = pv_e;
1738
1739 if (pv_et == PV_ENTRY_NULL)
1740 pv_et = pv_e;
1741 pv_cnt++;
1742 pv_e++;
1743 }
1744 PV_KERN_FREE_LIST(pv_eh, pv_et, pv_cnt);
1745 PMAP_LOCK(pmap);
1746 LOCK_PVH(pai);
1747 return FALSE;
1748 }
1749 } else {
1750 UNLOCK_PVH(pai);
1751 PMAP_UNLOCK(pmap);
1752 pmap_pv_throttle(pmap);
1753 {
1754 pv_entry_t *pv_e;
1755 pv_entry_t *pv_eh;
1756 pv_entry_t *pv_et;
1757 int pv_cnt;
1758 unsigned j;
1759 pmap_paddr_t pa;
1760 kern_return_t ret;
1761
1762 ret = pmap_pages_alloc(&pa, PAGE_SIZE, 0);
1763
1764 if (ret != KERN_SUCCESS) {
1765 panic("%s: failed to alloc page, ret=%d, "
1766 "pmap=%p, pai=%u, pvepp=%p",
1767 __FUNCTION__, ret,
1768 pmap, pai, pvepp);
1769 }
1770
1771 pv_page_count++;
1772
1773 pv_e = (pv_entry_t *)phystokv(pa);
1774 pv_cnt = 0;
1775 pv_eh = pv_et = PV_ENTRY_NULL;
1776 *pvepp = pv_e;
1777 pv_e++;
1778
1779 for (j = 1; j < (PAGE_SIZE/sizeof(pv_entry_t)) ; j++) {
1780 pv_e->pve_next = pv_eh;
1781 pv_eh = pv_e;
1782
1783 if (pv_et == PV_ENTRY_NULL)
1784 pv_et = pv_e;
1785 pv_cnt++;
1786 pv_e++;
1787 }
1788 PV_FREE_LIST(pv_eh, pv_et, pv_cnt);
1789 }
1790 PMAP_LOCK(pmap);
1791 LOCK_PVH(pai);
1792 return FALSE;
1793 }
1794 }
1795 assert(PV_ENTRY_NULL != *pvepp);
1796 return TRUE;
1797 }
1798
1799 static void
1800 pv_free(
1801 pv_entry_t *pvep)
1802 {
1803 PV_FREE_LIST(pvep, pvep, 1);
1804 }
1805
1806 static void
1807 pv_list_free(
1808 pv_entry_t *pvehp,
1809 pv_entry_t *pvetp,
1810 unsigned int cnt)
1811 {
1812 PV_FREE_LIST(pvehp, pvetp, cnt);
1813 }
1814
1815
1816
1817 static inline void PV_ALLOC(pv_entry_t **pv_ep) {
1818 assert(*pv_ep == PV_ENTRY_NULL);
1819 simple_lock(&pv_free_list_lock);
1820 /*
1821 * If the kernel reserved pool is low, let non-kernel mappings allocate
1822 * synchronously, possibly subject to a throttle.
1823 */
1824 if ((pv_kern_free_count >= pv_kern_low_water_mark) && ((*pv_ep = pv_free_list) != 0)) {
1825 pv_free_list = (pv_entry_t *)(*pv_ep)->pve_next;
1826 (*pv_ep)->pve_next = PV_ENTRY_NULL;
1827 pv_free_count--;
1828 }
1829
1830 simple_unlock(&pv_free_list_lock);
1831
1832 if ((pv_free_count < pv_low_water_mark) || (pv_kern_free_count < pv_kern_low_water_mark)) {
1833 if (!mappingrecurse && hw_compare_and_store(0,1, &mappingrecurse))
1834 thread_wakeup(&mapping_replenish_event);
1835 }
1836 }
1837
1838 static inline void PV_FREE_LIST(pv_entry_t *pv_eh, pv_entry_t *pv_et, int pv_cnt) {
1839 simple_lock(&pv_free_list_lock);
1840 pv_et->pve_next = (pv_entry_t *)pv_free_list;
1841 pv_free_list = pv_eh;
1842 pv_free_count += pv_cnt;
1843 simple_unlock(&pv_free_list_lock);
1844 }
1845
1846 static inline void PV_KERN_ALLOC(pv_entry_t **pv_e) {
1847 assert(*pv_e == PV_ENTRY_NULL);
1848 simple_lock(&pv_kern_free_list_lock);
1849
1850 if ((*pv_e = pv_kern_free_list) != 0) {
1851 pv_kern_free_list = (pv_entry_t *)(*pv_e)->pve_next;
1852 (*pv_e)->pve_next = PV_ENTRY_NULL;
1853 pv_kern_free_count--;
1854 pmap_kern_reserve_alloc_stat++;
1855 }
1856
1857 simple_unlock(&pv_kern_free_list_lock);
1858
1859 if (pv_kern_free_count < pv_kern_low_water_mark) {
1860 if (!mappingrecurse && hw_compare_and_store(0,1, &mappingrecurse)) {
1861 thread_wakeup(&mapping_replenish_event);
1862 }
1863 }
1864 }
1865
1866 static inline void PV_KERN_FREE_LIST(pv_entry_t *pv_eh, pv_entry_t *pv_et, int pv_cnt) {
1867 simple_lock(&pv_kern_free_list_lock);
1868 pv_et->pve_next = pv_kern_free_list;
1869 pv_kern_free_list = pv_eh;
1870 pv_kern_free_count += pv_cnt;
1871 simple_unlock(&pv_kern_free_list_lock);
1872 }
1873
1874 static inline void pmap_pv_throttle(__unused pmap_t p) {
1875 assert(p != kernel_pmap);
1876 /* Apply throttle on non-kernel mappings */
1877 if (pv_kern_free_count < (pv_kern_low_water_mark / 2)) {
1878 pmap_pv_throttle_stat++;
1879 /* This doesn't need to be strictly accurate, merely a hint
1880 * to eliminate the timeout when the reserve is replenished.
1881 */
1882 pmap_pv_throttled_waiters++;
1883 assert_wait_timeout(&pmap_user_pv_throttle_event, THREAD_UNINT, 1, 1000 * NSEC_PER_USEC);
1884 thread_block(THREAD_CONTINUE_NULL);
1885 }
1886 }
1887
1888 /*
1889 * Creates a target number of free pv_entry_t objects for the kernel free list
1890 * and the general free list.
1891 */
1892 static kern_return_t
1893 mapping_free_prime_internal(void)
1894 {
1895 unsigned j;
1896 pmap_paddr_t pa;
1897 kern_return_t ret;
1898 pv_entry_t *pv_e;
1899 pv_entry_t *pv_eh;
1900 pv_entry_t *pv_et;
1901 int pv_cnt;
1902 int alloc_options = 0;
1903 int needed_pv_cnt = 0;
1904 int target_pv_free_cnt = 0;
1905
1906 SECURITY_READ_ONLY_LATE(static boolean_t) mapping_free_prime_internal_called = FALSE;
1907 SECURITY_READ_ONLY_LATE(static boolean_t) mapping_free_prime_internal_done = FALSE;
1908
1909 if (mapping_free_prime_internal_done) {
1910 return KERN_FAILURE;
1911 }
1912
1913 if (!mapping_free_prime_internal_called) {
1914 mapping_free_prime_internal_called = TRUE;
1915
1916 pv_low_water_mark = PV_LOW_WATER_MARK_DEFAULT;
1917
1918 /* Alterable via sysctl */
1919 pv_kern_low_water_mark = PV_KERN_LOW_WATER_MARK_DEFAULT;
1920
1921 pv_kern_alloc_chunk = PV_KERN_ALLOC_CHUNK_INITIAL;
1922 pv_alloc_chunk = PV_ALLOC_CHUNK_INITIAL;
1923 }
1924
1925 pv_cnt = 0;
1926 pv_eh = pv_et = PV_ENTRY_NULL;
1927 target_pv_free_cnt = PV_ALLOC_INITIAL_TARGET;
1928
1929 /*
1930 * We don't take the lock to read pv_free_count, as we should not be
1931 * invoking this from a multithreaded context.
1932 */
1933 needed_pv_cnt = target_pv_free_cnt - pv_free_count;
1934
1935 if (needed_pv_cnt > target_pv_free_cnt) {
1936 needed_pv_cnt = 0;
1937 }
1938
1939 while (pv_cnt < needed_pv_cnt) {
1940 ret = pmap_pages_alloc(&pa, PAGE_SIZE, alloc_options);
1941
1942 assert(ret == KERN_SUCCESS);
1943
1944 pv_page_count++;
1945
1946 pv_e = (pv_entry_t *)phystokv(pa);
1947
1948 for (j = 0; j < (PAGE_SIZE/sizeof(pv_entry_t)) ; j++) {
1949 pv_e->pve_next = pv_eh;
1950 pv_eh = pv_e;
1951
1952 if (pv_et == PV_ENTRY_NULL)
1953 pv_et = pv_e;
1954 pv_cnt++;
1955 pv_e++;
1956 }
1957 }
1958
1959 if (pv_cnt) {
1960 PV_FREE_LIST(pv_eh, pv_et, pv_cnt);
1961 }
1962
1963 pv_cnt = 0;
1964 pv_eh = pv_et = PV_ENTRY_NULL;
1965 target_pv_free_cnt = PV_KERN_ALLOC_INITIAL_TARGET;
1966
1967 /*
1968 * We don't take the lock to read pv_kern_free_count, as we should not
1969 * be invoking this from a multithreaded context.
1970 */
1971 needed_pv_cnt = target_pv_free_cnt - pv_kern_free_count;
1972
1973 if (needed_pv_cnt > target_pv_free_cnt) {
1974 needed_pv_cnt = 0;
1975 }
1976
1977 while (pv_cnt < needed_pv_cnt) {
1978
1979 ret = pmap_pages_alloc(&pa, PAGE_SIZE, alloc_options);
1980
1981 assert(ret == KERN_SUCCESS);
1982 pv_page_count++;
1983
1984 pv_e = (pv_entry_t *)phystokv(pa);
1985
1986 for (j = 0; j < (PAGE_SIZE/sizeof(pv_entry_t)) ; j++) {
1987 pv_e->pve_next = pv_eh;
1988 pv_eh = pv_e;
1989
1990 if (pv_et == PV_ENTRY_NULL)
1991 pv_et = pv_e;
1992 pv_cnt++;
1993 pv_e++;
1994 }
1995 }
1996
1997 if (pv_cnt) {
1998 PV_KERN_FREE_LIST(pv_eh, pv_et, pv_cnt);
1999 }
2000
2001 mapping_free_prime_internal_done = TRUE;
2002 return KERN_SUCCESS;
2003 }
2004
2005 void
2006 mapping_free_prime(void)
2007 {
2008 kern_return_t kr = KERN_FAILURE;
2009
2010 kr = mapping_free_prime_internal();
2011
2012 if (kr != KERN_SUCCESS) {
2013 panic("%s: failed, kr=%d", __FUNCTION__, kr);
2014 }
2015 }
2016
2017 void mapping_replenish(void);
2018
2019 void mapping_adjust(void) {
2020 kern_return_t mres;
2021
2022 mres = kernel_thread_start_priority((thread_continue_t)mapping_replenish, NULL, MAXPRI_KERNEL, &mapping_replenish_thread);
2023 if (mres != KERN_SUCCESS) {
2024 panic("pmap: mapping_replenish thread creation failed");
2025 }
2026 thread_deallocate(mapping_replenish_thread);
2027 }
2028
2029 /*
2030 * Fills the kernel and general PV free lists back up to their low watermarks.
2031 */
2032 static kern_return_t
2033 mapping_replenish_internal(void)
2034 {
2035 pv_entry_t *pv_e;
2036 pv_entry_t *pv_eh;
2037 pv_entry_t *pv_et;
2038 int pv_cnt;
2039 unsigned j;
2040 pmap_paddr_t pa;
2041 kern_return_t ret = KERN_SUCCESS;
2042
2043 while (pv_kern_free_count < pv_kern_low_water_mark) {
2044 pv_cnt = 0;
2045 pv_eh = pv_et = PV_ENTRY_NULL;
2046
2047 ret = pmap_pages_alloc(&pa, PAGE_SIZE, 0);
2048 assert(ret == KERN_SUCCESS);
2049
2050 pv_page_count++;
2051
2052 pv_e = (pv_entry_t *)phystokv(pa);
2053
2054 for (j = 0; j < (PAGE_SIZE/sizeof(pv_entry_t)) ; j++) {
2055 pv_e->pve_next = pv_eh;
2056 pv_eh = pv_e;
2057
2058 if (pv_et == PV_ENTRY_NULL)
2059 pv_et = pv_e;
2060 pv_cnt++;
2061 pv_e++;
2062 }
2063 pmap_kernel_reserve_replenish_stat += pv_cnt;
2064 PV_KERN_FREE_LIST(pv_eh, pv_et, pv_cnt);
2065 }
2066
2067 while (pv_free_count < pv_low_water_mark) {
2068 pv_cnt = 0;
2069 pv_eh = pv_et = PV_ENTRY_NULL;
2070
2071 ret = pmap_pages_alloc(&pa, PAGE_SIZE, 0);
2072 assert(ret == KERN_SUCCESS);
2073
2074 pv_page_count++;
2075
2076 pv_e = (pv_entry_t *)phystokv(pa);
2077
2078 for (j = 0; j < (PAGE_SIZE/sizeof(pv_entry_t)) ; j++) {
2079 pv_e->pve_next = pv_eh;
2080 pv_eh = pv_e;
2081
2082 if (pv_et == PV_ENTRY_NULL)
2083 pv_et = pv_e;
2084 pv_cnt++;
2085 pv_e++;
2086 }
2087 pmap_user_reserve_replenish_stat += pv_cnt;
2088 PV_FREE_LIST(pv_eh, pv_et, pv_cnt);
2089 }
2090
2091 return ret;
2092 }
2093
2094 /*
2095 * Continuation function that keeps the PV free lists from running out of free
2096 * elements.
2097 */
2098 __attribute__((noreturn))
2099 void
2100 mapping_replenish(void)
2101 {
2102 kern_return_t kr;
2103
2104 /* We qualify for VM privileges...*/
2105 current_thread()->options |= TH_OPT_VMPRIV;
2106
2107 for (;;) {
2108 kr = mapping_replenish_internal();
2109
2110 if (kr != KERN_SUCCESS) {
2111 panic("%s: failed, kr=%d", __FUNCTION__, kr);
2112 }
2113
2114 /*
2115 * Wake threads throttled while the kernel reserve was being replenished.
2116 */
2117 if (pmap_pv_throttled_waiters) {
2118 pmap_pv_throttled_waiters = 0;
2119 thread_wakeup(&pmap_user_pv_throttle_event);
2120 }
2121
2122 /* Check if the kernel pool has been depleted since the
2123 * first pass, to reduce refill latency.
2124 */
2125 if (pv_kern_free_count < pv_kern_low_water_mark)
2126 continue;
2127 /* Block sans continuation to avoid yielding kernel stack */
2128 assert_wait(&mapping_replenish_event, THREAD_UNINT);
2129 mappingrecurse = 0;
2130 thread_block(THREAD_CONTINUE_NULL);
2131 pmap_mapping_thread_wakeups++;
2132 }
2133 }
2134
2135
2136 static void
2137 ptd_bootstrap(
2138 pt_desc_t *ptdp,
2139 unsigned int ptd_cnt)
2140 {
2141 simple_lock_init(&ptd_free_list_lock, 0);
2142 while (ptd_cnt != 0) {
2143 (*(void **)ptdp) = (void *)ptd_free_list;
2144 ptd_free_list = ptdp;
2145 ptdp++;
2146 ptd_cnt--;
2147 ptd_free_count++;
2148 }
2149 ptd_preboot = FALSE;
2150 }
2151
2152 static pt_desc_t
2153 *ptd_alloc(
2154 pmap_t pmap)
2155 {
2156 pt_desc_t *ptdp;
2157 unsigned i;
2158
2159 if (!ptd_preboot)
2160 simple_lock(&ptd_free_list_lock);
2161
2162 if (ptd_free_count == 0) {
2163 unsigned int ptd_cnt;
2164 pt_desc_t *ptdp_next;
2165
2166 if (ptd_preboot) {
2167 ptdp = (pt_desc_t *)avail_start;
2168 avail_start += ARM_PGBYTES;
2169 ptdp_next = ptdp;
2170 ptd_cnt = ARM_PGBYTES/sizeof(pt_desc_t);
2171 } else {
2172 pmap_paddr_t pa;
2173 kern_return_t ret;
2174
2175 simple_unlock(&ptd_free_list_lock);
2176
2177 if (pmap_pages_alloc(&pa, PAGE_SIZE, PMAP_PAGES_ALLOCATE_NOWAIT) != KERN_SUCCESS) {
2178 ret = pmap_pages_alloc(&pa, PAGE_SIZE, PMAP_PAGES_RECLAIM_NOWAIT);
2179 assert(ret == KERN_SUCCESS);
2180 }
2181 ptdp = (pt_desc_t *)phystokv(pa);
2182
2183 simple_lock(&ptd_free_list_lock);
2184 ptdp_next = ptdp;
2185 ptd_cnt = PAGE_SIZE/sizeof(pt_desc_t);
2186 }
2187
2188 while (ptd_cnt != 0) {
2189 (*(void **)ptdp_next) = (void *)ptd_free_list;
2190 ptd_free_list = ptdp_next;
2191 ptdp_next++;
2192 ptd_cnt--;
2193 ptd_free_count++;
2194 }
2195 }
2196
2197 if ((ptdp = ptd_free_list) != PTD_ENTRY_NULL) {
2198 ptd_free_list = (pt_desc_t *)(*(void **)ptdp);
2199 ptd_free_count--;
2200 } else {
2201 panic("out of ptd entry\n");
2202 }
2203
2204 if (!ptd_preboot)
2205 simple_unlock(&ptd_free_list_lock);
2206
2207 ptdp->pt_page.next = NULL;
2208 ptdp->pt_page.prev = NULL;
2209 ptdp->pmap = pmap;
2210
2211 for (i = 0 ; i < PT_INDEX_MAX ; i++) {
2212 ptdp->pt_map[i].va = 0;
2213 ptdp->pt_cnt[i].refcnt = 0;
2214 ptdp->pt_cnt[i].wiredcnt = 0;
2215 }
2216 simple_lock(&pt_pages_lock);
2217 queue_enter(&pt_page_list, ptdp, pt_desc_t *, pt_page);
2218 simple_unlock(&pt_pages_lock);
2219
2220 pmap_tt_ledger_credit(pmap, sizeof(*ptdp));
2221
2222 return(ptdp);
2223 }
2224
2225 static void
2226 ptd_deallocate(
2227 pt_desc_t *ptdp)
2228 {
2229 unsigned i;
2230 pmap_t pmap = ptdp->pmap;
2231
2232 if (ptd_preboot) {
2233 panic("ptd_deallocate(): early boot\n");
2234 }
2235 for (i = 0 ; i < PT_INDEX_MAX ; i++) {
2236 if (ptdp->pt_cnt[i].refcnt != 0)
2237 panic("ptd_deallocate(): ptdp=%p refcnt=0x%x \n", ptdp, ptdp->pt_cnt[i].refcnt);
2238 }
2239
2240 if (ptdp->pt_page.next != NULL) {
2241 simple_lock(&pt_pages_lock);
2242 queue_remove(&pt_page_list, ptdp, pt_desc_t *, pt_page);
2243 simple_unlock(&pt_pages_lock);
2244 }
2245 simple_lock(&ptd_free_list_lock);
2246 (*(void **)ptdp) = (void *)ptd_free_list;
2247 ptd_free_list = (pt_desc_t *)ptdp;
2248 ptd_free_count++;
2249 simple_unlock(&ptd_free_list_lock);
2250 pmap_tt_ledger_debit(pmap, sizeof(*ptdp));
2251 }
2252
2253 static void
2254 ptd_init(
2255 pt_desc_t *ptdp,
2256 pmap_t pmap,
2257 vm_map_address_t va,
2258 unsigned int level,
2259 pt_entry_t *pte_p)
2260 {
2261 if (ptdp->pmap != pmap)
2262 panic("ptd_init(): pmap mismatch\n");
2263
2264 #if (__ARM_VMSA__ == 7)
2265 assert(level == 2);
2266 ptdp->pt_map[ARM_PT_DESC_INDEX(pte_p)].va = (vm_offset_t) va & ~(ARM_TT_L1_PT_OFFMASK);
2267 #else
2268 if (level == 3) {
2269 ptdp->pt_map[ARM_PT_DESC_INDEX(pte_p)].va = (vm_offset_t) va & ~ARM_TT_L2_OFFMASK ;
2270 } else if (level == 2)
2271 ptdp->pt_map[ARM_PT_DESC_INDEX(pte_p)].va = (vm_offset_t) va & ~ARM_TT_L1_OFFMASK ;
2272 #endif
2273 if (level < PMAP_TT_MAX_LEVEL)
2274 ptdp->pt_cnt[ARM_PT_DESC_INDEX(pte_p)].refcnt = PT_DESC_REFCOUNT;
2275
2276 }
2277
2278
2279 boolean_t
2280 pmap_valid_address(
2281 pmap_paddr_t addr)
2282 {
2283 return pa_valid(addr);
2284 }
2285
2286 #if (__ARM_VMSA__ == 7)
2287
2288 /*
2289 * Given an offset and a map, compute the address of the
2290 * corresponding translation table entry.
2291 */
2292 static inline tt_entry_t *
2293 pmap_tte(pmap_t pmap,
2294 vm_map_address_t addr)
2295 {
2296 if (!(tte_index(pmap, addr) < pmap->tte_index_max))
2297 return (tt_entry_t *)NULL;
2298 return (&pmap->tte[tte_index(pmap, addr)]);
2299 }
2300
2301
2302 /*
2303 * Given an offset and a map, compute the address of the
2304 * pte. If the address is invalid with respect to the map
2305 * then PT_ENTRY_NULL is returned (and the map may need to grow).
2306 *
2307 * This is only used internally.
2308 */
2309 static inline pt_entry_t *
2310 pmap_pte(
2311 pmap_t pmap,
2312 vm_map_address_t addr)
2313 {
2314 pt_entry_t *ptp;
2315 tt_entry_t *ttp;
2316 tt_entry_t tte;
2317
2318 ttp = pmap_tte(pmap, addr);
2319 if (ttp == (tt_entry_t *)NULL)
2320 return (PT_ENTRY_NULL);
2321 tte = *ttp;
2322 #if MACH_ASSERT
2323 if ((tte & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_BLOCK)
2324 panic("Attempt to demote L1 block: pmap=%p, va=0x%llx, tte=0x%llx\n", pmap, (uint64_t)addr, (uint64_t)tte);
2325 #endif
2326 if ((tte & ARM_TTE_TYPE_MASK) != ARM_TTE_TYPE_TABLE)
2327 return (PT_ENTRY_NULL);
2328 ptp = (pt_entry_t *) ttetokv(tte) + ptenum(addr);
2329 return (ptp);
2330 }
2331
2332 #else
2333
2334 /*
2335 * Given an offset and a map, compute the address of level 1 translation table entry.
2336 * If the tranlation is invalid then PT_ENTRY_NULL is returned.
2337 */
2338 static inline tt_entry_t *
2339 pmap_tt1e(pmap_t pmap,
2340 vm_map_address_t addr)
2341 {
2342 #if __ARM64_TWO_LEVEL_PMAP__
2343 #pragma unused(pmap, addr)
2344 panic("pmap_tt1e called on a two level pmap");
2345 return (NULL);
2346 #else
2347 return (&pmap->tte[tt1_index(pmap, addr)]);
2348 #endif
2349 }
2350
2351 /*
2352 * Given an offset and a map, compute the address of level 2 translation table entry.
2353 * If the tranlation is invalid then PT_ENTRY_NULL is returned.
2354 */
2355 static inline tt_entry_t *
2356 pmap_tt2e(pmap_t pmap,
2357 vm_map_address_t addr)
2358 {
2359 #if __ARM64_TWO_LEVEL_PMAP__
2360 return (&pmap->tte[tt2_index(pmap, addr)]);
2361 #else
2362 tt_entry_t *ttp;
2363 tt_entry_t tte;
2364
2365 ttp = pmap_tt1e(pmap, addr);
2366 tte = *ttp;
2367 #if MACH_ASSERT
2368 if ((tte & (ARM_TTE_TYPE_MASK | ARM_TTE_VALID)) == (ARM_TTE_TYPE_BLOCK | ARM_TTE_VALID))
2369 panic("Attempt to demote L1 block (?!): pmap=%p, va=0x%llx, tte=0x%llx\n", pmap, (uint64_t)addr, (uint64_t)tte);
2370 #endif
2371 if ((tte & (ARM_TTE_TYPE_MASK | ARM_TTE_VALID)) != (ARM_TTE_TYPE_TABLE | ARM_TTE_VALID))
2372 return (PT_ENTRY_NULL);
2373
2374 ttp = &((tt_entry_t*) phystokv(tte & ARM_TTE_TABLE_MASK))[tt2_index(pmap, addr)];
2375 return ((tt_entry_t *)ttp);
2376 #endif
2377 }
2378
2379
2380 /*
2381 * Given an offset and a map, compute the address of level 3 translation table entry.
2382 * If the tranlation is invalid then PT_ENTRY_NULL is returned.
2383 */
2384 static inline pt_entry_t *
2385 pmap_tt3e(
2386 pmap_t pmap,
2387 vm_map_address_t addr)
2388 {
2389 pt_entry_t *ptp;
2390 tt_entry_t *ttp;
2391 tt_entry_t tte;
2392
2393 /* Level 0 currently unused */
2394 #if __ARM64_TWO_LEVEL_PMAP__
2395 ttp = pmap_tt2e(pmap, addr);
2396 tte = *ttp;
2397 #else
2398 /* Get first-level (1GB) entry */
2399 ttp = pmap_tt1e(pmap, addr);
2400 tte = *ttp;
2401 #if MACH_ASSERT
2402 if ((tte & (ARM_TTE_TYPE_MASK | ARM_TTE_VALID)) == (ARM_TTE_TYPE_BLOCK | ARM_TTE_VALID))
2403 panic("Attempt to demote L1 block (?!): pmap=%p, va=0x%llx, tte=0x%llx\n", pmap, (uint64_t)addr, (uint64_t)tte);
2404 #endif
2405 if ((tte & (ARM_TTE_TYPE_MASK | ARM_TTE_VALID)) != (ARM_TTE_TYPE_TABLE | ARM_TTE_VALID))
2406 return (PT_ENTRY_NULL);
2407
2408 tte = ((tt_entry_t*) phystokv(tte & ARM_TTE_TABLE_MASK))[tt2_index(pmap, addr)];
2409 #endif
2410 #if MACH_ASSERT
2411 if ((tte & (ARM_TTE_TYPE_MASK | ARM_TTE_VALID)) == (ARM_TTE_TYPE_BLOCK | ARM_TTE_VALID))
2412 panic("Attempt to demote L2 block: pmap=%p, va=0x%llx, tte=0x%llx\n", pmap, (uint64_t)addr, (uint64_t)tte);
2413 #endif
2414 if ((tte & (ARM_TTE_TYPE_MASK | ARM_TTE_VALID)) != (ARM_TTE_TYPE_TABLE | ARM_TTE_VALID)) {
2415 return (PT_ENTRY_NULL);
2416 }
2417
2418 /* Get third-level (4KB) entry */
2419 ptp = &(((pt_entry_t*) phystokv(tte & ARM_TTE_TABLE_MASK))[tt3_index(pmap, addr)]);
2420 return (ptp);
2421 }
2422
2423
2424 static inline tt_entry_t *
2425 pmap_tte(
2426 pmap_t pmap,
2427 vm_map_address_t addr)
2428 {
2429 return(pmap_tt2e(pmap, addr));
2430 }
2431
2432
2433 static inline pt_entry_t *
2434 pmap_pte(
2435 pmap_t pmap,
2436 vm_map_address_t addr)
2437 {
2438 return(pmap_tt3e(pmap, addr));
2439 }
2440
2441 #endif
2442
2443
2444 /*
2445 * Map memory at initialization. The physical addresses being
2446 * mapped are not managed and are never unmapped.
2447 *
2448 * For now, VM is already on, we only need to map the
2449 * specified memory.
2450 */
2451 vm_map_address_t
2452 pmap_map(
2453 vm_map_address_t virt,
2454 vm_offset_t start,
2455 vm_offset_t end,
2456 vm_prot_t prot,
2457 unsigned int flags)
2458 {
2459 kern_return_t kr;
2460 vm_size_t ps;
2461
2462 ps = PAGE_SIZE;
2463 while (start < end) {
2464 kr = pmap_enter(kernel_pmap, virt, (ppnum_t)atop(start),
2465 prot, VM_PROT_NONE, flags, FALSE);
2466
2467 if (kr != KERN_SUCCESS) {
2468 panic("%s: failed pmap_enter, "
2469 "virt=%p, start_addr=%p, end_addr=%p, prot=%#x, flags=%#x",
2470 __FUNCTION__,
2471 (void *) virt, (void *) start, (void *) end, prot, flags);
2472 }
2473
2474 virt += ps;
2475 start += ps;
2476 }
2477 return (virt);
2478 }
2479
2480 vm_map_address_t
2481 pmap_map_bd_with_options(
2482 vm_map_address_t virt,
2483 vm_offset_t start,
2484 vm_offset_t end,
2485 vm_prot_t prot,
2486 int32_t options)
2487 {
2488 pt_entry_t tmplate;
2489 pt_entry_t *ptep;
2490 vm_map_address_t vaddr;
2491 vm_offset_t paddr;
2492 pt_entry_t mem_attr;
2493
2494 switch (options & PMAP_MAP_BD_MASK) {
2495 case PMAP_MAP_BD_WCOMB:
2496 mem_attr = ARM_PTE_ATTRINDX(CACHE_ATTRINDX_WRITECOMB);
2497 #if (__ARM_VMSA__ > 7)
2498 mem_attr |= ARM_PTE_SH(SH_OUTER_MEMORY);
2499 #else
2500 mem_attr |= ARM_PTE_SH;
2501 #endif
2502 break;
2503 case PMAP_MAP_BD_POSTED:
2504 mem_attr = ARM_PTE_ATTRINDX(CACHE_ATTRINDX_POSTED);
2505 break;
2506 default:
2507 mem_attr = ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DISABLE);
2508 break;
2509 }
2510
2511 tmplate = pa_to_pte(start) | ARM_PTE_AP((prot & VM_PROT_WRITE) ? AP_RWNA : AP_RONA) |
2512 mem_attr | ARM_PTE_TYPE | ARM_PTE_NX | ARM_PTE_PNX | ARM_PTE_AF;
2513
2514 vaddr = virt;
2515 paddr = start;
2516 while (paddr < end) {
2517
2518 ptep = pmap_pte(kernel_pmap, vaddr);
2519 if (ptep == PT_ENTRY_NULL) {
2520 panic("pmap_map_bd");
2521 }
2522 assert(!ARM_PTE_IS_COMPRESSED(*ptep));
2523 WRITE_PTE(ptep, tmplate);
2524
2525 pte_increment_pa(tmplate);
2526 vaddr += PAGE_SIZE;
2527 paddr += PAGE_SIZE;
2528 }
2529
2530 if (end >= start)
2531 flush_mmu_tlb_region(virt, (unsigned)(end - start));
2532
2533 return (vaddr);
2534 }
2535
2536 /*
2537 * Back-door routine for mapping kernel VM at initialization.
2538 * Useful for mapping memory outside the range
2539 * [vm_first_phys, vm_last_phys] (i.e., devices).
2540 * Otherwise like pmap_map.
2541 */
2542 vm_map_address_t
2543 pmap_map_bd(
2544 vm_map_address_t virt,
2545 vm_offset_t start,
2546 vm_offset_t end,
2547 vm_prot_t prot)
2548 {
2549 pt_entry_t tmplate;
2550 pt_entry_t *ptep;
2551 vm_map_address_t vaddr;
2552 vm_offset_t paddr;
2553
2554 /* not cacheable and not buffered */
2555 tmplate = pa_to_pte(start)
2556 | ARM_PTE_TYPE | ARM_PTE_AF | ARM_PTE_NX | ARM_PTE_PNX
2557 | ARM_PTE_AP((prot & VM_PROT_WRITE) ? AP_RWNA : AP_RONA)
2558 | ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DISABLE);
2559
2560 vaddr = virt;
2561 paddr = start;
2562 while (paddr < end) {
2563
2564 ptep = pmap_pte(kernel_pmap, vaddr);
2565 if (ptep == PT_ENTRY_NULL) {
2566 panic("pmap_map_bd");
2567 }
2568 assert(!ARM_PTE_IS_COMPRESSED(*ptep));
2569 WRITE_PTE(ptep, tmplate);
2570
2571 pte_increment_pa(tmplate);
2572 vaddr += PAGE_SIZE;
2573 paddr += PAGE_SIZE;
2574 }
2575
2576 if (end >= start)
2577 flush_mmu_tlb_region(virt, (unsigned)(end - start));
2578
2579 return (vaddr);
2580 }
2581
2582 /*
2583 * Back-door routine for mapping kernel VM at initialization.
2584 * Useful for mapping memory specific physical addresses in early
2585 * boot (i.e., before kernel_map is initialized).
2586 *
2587 * Maps are in the VM_HIGH_KERNEL_WINDOW area.
2588 */
2589
2590 vm_map_address_t
2591 pmap_map_high_window_bd(
2592 vm_offset_t pa_start,
2593 vm_size_t len,
2594 vm_prot_t prot)
2595 {
2596 pt_entry_t *ptep, pte;
2597 #if (__ARM_VMSA__ == 7)
2598 vm_map_address_t va_start = VM_HIGH_KERNEL_WINDOW;
2599 vm_map_address_t va_max = VM_MAX_KERNEL_ADDRESS;
2600 #else
2601 vm_map_address_t va_start = VREGION1_START;
2602 vm_map_address_t va_max = VREGION1_START + VREGION1_SIZE;
2603 #endif
2604 vm_map_address_t va_end;
2605 vm_map_address_t va;
2606 vm_size_t offset;
2607
2608 offset = pa_start & PAGE_MASK;
2609 pa_start -= offset;
2610 len += offset;
2611
2612 if (len > (va_max - va_start)) {
2613 panic("pmap_map_high_window_bd: area too large\n");
2614 }
2615
2616 scan:
2617 for ( ; va_start < va_max; va_start += PAGE_SIZE) {
2618 ptep = pmap_pte(kernel_pmap, va_start);
2619 assert(!ARM_PTE_IS_COMPRESSED(*ptep));
2620 if (*ptep == ARM_PTE_TYPE_FAULT)
2621 break;
2622 }
2623 if (va_start > va_max) {
2624 panic("pmap_map_high_window_bd: insufficient pages\n");
2625 }
2626
2627 for (va_end = va_start + PAGE_SIZE; va_end < va_start + len; va_end += PAGE_SIZE) {
2628 ptep = pmap_pte(kernel_pmap, va_end);
2629 assert(!ARM_PTE_IS_COMPRESSED(*ptep));
2630 if (*ptep != ARM_PTE_TYPE_FAULT) {
2631 va_start = va_end + PAGE_SIZE;
2632 goto scan;
2633 }
2634 }
2635
2636 for (va = va_start; va < va_end; va += PAGE_SIZE, pa_start += PAGE_SIZE) {
2637 ptep = pmap_pte(kernel_pmap, va);
2638 pte = pa_to_pte(pa_start)
2639 | ARM_PTE_TYPE | ARM_PTE_AF | ARM_PTE_NX | ARM_PTE_PNX
2640 | ARM_PTE_AP((prot & VM_PROT_WRITE) ? AP_RWNA : AP_RONA)
2641 | ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DEFAULT);
2642 #if (__ARM_VMSA__ > 7)
2643 pte |= ARM_PTE_SH(SH_OUTER_MEMORY);
2644 #else
2645 pte |= ARM_PTE_SH;
2646 #endif
2647 WRITE_PTE(ptep, pte);
2648 }
2649 PMAP_UPDATE_TLBS(kernel_pmap, va_start, va_start + len);
2650 #if KASAN
2651 kasan_notify_address(va_start, len);
2652 #endif
2653 return va_start;
2654 }
2655
2656 #define PMAP_ALIGN(addr, align) ((addr) + ((align) - 1) & ~((align) - 1))
2657
2658 typedef struct pmap_io_range
2659 {
2660 uint64_t addr;
2661 uint32_t len;
2662 uint32_t wimg;
2663 } __attribute__((packed)) pmap_io_range_t;
2664
2665 static unsigned int
2666 pmap_compute_io_rgns(void)
2667 {
2668 DTEntry entry;
2669 pmap_io_range_t *ranges;
2670 void *prop = NULL;
2671 int err;
2672 unsigned int prop_size;
2673
2674 err = DTLookupEntry(NULL, "/defaults", &entry);
2675 assert(err == kSuccess);
2676
2677 if (kSuccess != DTGetProperty(entry, "pmap-io-granule", &prop, &prop_size))
2678 return 0;
2679
2680 io_rgn_granule = *((uint32_t*)prop);
2681
2682 if (kSuccess != DTGetProperty(entry, "pmap-io-ranges", &prop, &prop_size))
2683 return 0;
2684
2685 if ((io_rgn_granule == 0) || (io_rgn_granule & PAGE_MASK))
2686 panic("pmap I/O region granularity is not page-aligned!\n");
2687
2688 ranges = prop;
2689 for (unsigned int i = 0; i < (prop_size / sizeof(*ranges)); ++i) {
2690 if ((i == 0) || (ranges[i].addr < io_rgn_start))
2691 io_rgn_start = ranges[i].addr;
2692 if ((i == 0) || ((ranges[i].addr + ranges[i].len) > io_rgn_end))
2693 io_rgn_end = ranges[i].addr + ranges[i].len;
2694 }
2695
2696 if (io_rgn_start & PAGE_MASK)
2697 panic("pmap I/O region start is not page-aligned!\n");
2698
2699 if (io_rgn_end & PAGE_MASK)
2700 panic("pmap I/O region end is not page-aligned!\n");
2701
2702 if (((io_rgn_start < gPhysBase) && (io_rgn_end >= gPhysBase)) ||
2703 ((io_rgn_start < avail_end) && (io_rgn_end >= avail_end)))
2704 panic("pmap I/O region overlaps physical memory!\n");
2705
2706 return (unsigned int)((io_rgn_end - io_rgn_start) / io_rgn_granule);
2707 }
2708
2709 static void
2710 pmap_load_io_rgns(void)
2711 {
2712 DTEntry entry;
2713 pmap_io_range_t *ranges;
2714 void *prop = NULL;
2715 int err;
2716 unsigned int prop_size;
2717
2718 if (io_rgn_granule == 0)
2719 return;
2720
2721 err = DTLookupEntry(NULL, "/defaults", &entry);
2722 assert(err == kSuccess);
2723
2724 err = DTGetProperty(entry, "pmap-io-ranges", &prop, &prop_size);
2725 assert(err == kSuccess);
2726
2727 ranges = prop;
2728 for (unsigned int i = 0; i < (prop_size / sizeof(*ranges)); ++i) {
2729 if ((ranges[i].addr - io_rgn_start) % io_rgn_granule)
2730 panic("pmap I/O region %d is not aligned to I/O granularity!\n", i);
2731 if (ranges[i].len % io_rgn_granule)
2732 panic("pmap I/O region %d size is not a multiple of I/O granularity!\n", i);
2733 for (uint32_t offs = 0; offs < ranges[i].len; offs += io_rgn_granule) {
2734 io_attr_table[(ranges[i].addr + offs - io_rgn_start) / io_rgn_granule] =
2735 IO_ATTR_WIMG(ranges[i].wimg);
2736 }
2737 }
2738 }
2739
2740
2741 /*
2742 * Bootstrap the system enough to run with virtual memory.
2743 *
2744 * The early VM initialization code has already allocated
2745 * the first CPU's translation table and made entries for
2746 * all the one-to-one mappings to be found there.
2747 *
2748 * We must set up the kernel pmap structures, the
2749 * physical-to-virtual translation lookup tables for the
2750 * physical memory to be managed (between avail_start and
2751 * avail_end).
2752
2753 * Map the kernel's code and data, and allocate the system page table.
2754 * Page_size must already be set.
2755 *
2756 * Parameters:
2757 * first_avail first available physical page -
2758 * after kernel page tables
2759 * avail_start PA of first managed physical page
2760 * avail_end PA of last managed physical page
2761 */
2762
2763 void
2764 pmap_bootstrap(
2765 vm_offset_t vstart)
2766 {
2767 pmap_paddr_t pmap_struct_start;
2768 vm_size_t pv_head_size;
2769 vm_size_t pv_lock_table_size;
2770 vm_size_t ptd_root_table_size;
2771 vm_size_t pp_attr_table_size;
2772 vm_size_t io_attr_table_size;
2773 unsigned int niorgns;
2774 unsigned int npages;
2775 unsigned int i;
2776 vm_map_offset_t maxoffset;
2777
2778
2779 #ifdef PMAP_TRACES
2780 if (PE_parse_boot_argn("-pmap_trace", &pmap_trace, sizeof (pmap_trace))) {
2781 kprintf("Kernel traces for pmap operations enabled\n");
2782 }
2783 #endif
2784
2785 /*
2786 * Initialize the kernel pmap.
2787 */
2788 pmap_stamp = 1;
2789 kernel_pmap->tte = cpu_tte;
2790 kernel_pmap->ttep = cpu_ttep;
2791 #if (__ARM_VMSA__ > 7)
2792 kernel_pmap->min = ARM64_TTBR1_MIN_ADDR;
2793 #else
2794 kernel_pmap->min = VM_MIN_KERNEL_AND_KEXT_ADDRESS;
2795 #endif
2796 kernel_pmap->max = VM_MAX_KERNEL_ADDRESS;
2797 kernel_pmap->wired = 0;
2798 kernel_pmap->ref_count = 1;
2799 kernel_pmap->gc_status = 0;
2800 kernel_pmap->nx_enabled = TRUE;
2801 #ifdef __arm64__
2802 kernel_pmap->is_64bit = TRUE;
2803 #else
2804 kernel_pmap->is_64bit = FALSE;
2805 #endif
2806 kernel_pmap->stamp = hw_atomic_add(&pmap_stamp, 1);
2807
2808 kernel_pmap->nested_region_grand_addr = 0x0ULL;
2809 kernel_pmap->nested_region_subord_addr = 0x0ULL;
2810 kernel_pmap->nested_region_size = 0x0ULL;
2811 kernel_pmap->nested_region_asid_bitmap = NULL;
2812 kernel_pmap->nested_region_asid_bitmap_size = 0x0UL;
2813
2814 #if (__ARM_VMSA__ == 7)
2815 kernel_pmap->tte_index_max = 4*NTTES;
2816 #else
2817 kernel_pmap->tte_index_max = (ARM_PGBYTES / sizeof(tt_entry_t));
2818 #endif
2819 kernel_pmap->prev_tte = (tt_entry_t *) NULL;
2820 kernel_pmap->cpu_ref = 0;
2821
2822 PMAP_LOCK_INIT(kernel_pmap);
2823 #if (__ARM_VMSA__ == 7)
2824 simple_lock_init(&kernel_pmap->tt1_lock, 0);
2825 #endif
2826 memset((void *) &kernel_pmap->stats, 0, sizeof(kernel_pmap->stats));
2827
2828 /* allocate space for and initialize the bookkeeping structures */
2829 niorgns = pmap_compute_io_rgns();
2830 npages = (unsigned int)atop(mem_size);
2831 pp_attr_table_size = npages * sizeof(pp_attr_t);
2832 io_attr_table_size = niorgns * sizeof(io_attr_t);
2833 pv_lock_table_size = npages;
2834 pv_head_size = round_page(sizeof(pv_entry_t *) * npages);
2835 #if (__ARM_VMSA__ == 7)
2836 ptd_root_table_size = sizeof(pt_desc_t) * (1<<((mem_size>>30)+12));
2837 #else
2838 ptd_root_table_size = sizeof(pt_desc_t) * (1<<((mem_size>>30)+13));
2839 #endif
2840
2841 pmap_struct_start = avail_start;
2842
2843 pp_attr_table = (pp_attr_t *) phystokv(avail_start);
2844 avail_start = PMAP_ALIGN(avail_start + pp_attr_table_size, __alignof(pp_attr_t));
2845 io_attr_table = (io_attr_t *) phystokv(avail_start);
2846 avail_start = PMAP_ALIGN(avail_start + io_attr_table_size + pv_lock_table_size, __alignof(pv_entry_t*));
2847 pv_head_table = (pv_entry_t **) phystokv(avail_start);
2848 avail_start = PMAP_ALIGN(avail_start + pv_head_size, __alignof(pt_desc_t));
2849 ptd_root_table = (pt_desc_t *)phystokv(avail_start);
2850 avail_start = round_page(avail_start + ptd_root_table_size);
2851
2852 memset((char *)phystokv(pmap_struct_start), 0, avail_start - pmap_struct_start);
2853
2854 pmap_load_io_rgns();
2855 ptd_bootstrap(ptd_root_table, (unsigned int)(ptd_root_table_size/sizeof(pt_desc_t)));
2856
2857 pmap_cpu_data_array_init();
2858
2859 vm_first_phys = gPhysBase;
2860 vm_last_phys = trunc_page(avail_end);
2861
2862 simple_lock_init(&pmaps_lock, 0);
2863 queue_init(&map_pmap_list);
2864 queue_enter(&map_pmap_list, kernel_pmap, pmap_t, pmaps);
2865 queue_init(&tt_pmap_list);
2866 tt_pmap_count = 0;
2867 tt_pmap_max = 0;
2868 free_page_size_tt_list = TT_FREE_ENTRY_NULL;
2869 free_page_size_tt_count = 0;
2870 free_page_size_tt_max = 0;
2871 free_two_page_size_tt_list = TT_FREE_ENTRY_NULL;
2872 free_two_page_size_tt_count = 0;
2873 free_two_page_size_tt_max = 0;
2874 free_tt_list = TT_FREE_ENTRY_NULL;
2875 free_tt_count = 0;
2876 free_tt_max = 0;
2877
2878 simple_lock_init(&pt_pages_lock, 0);
2879 queue_init(&pt_page_list);
2880
2881 simple_lock_init(&pmap_pages_lock, 0);
2882 pmap_pages_request_count = 0;
2883 pmap_pages_request_acum = 0;
2884 pmap_pages_reclaim_list = PAGE_FREE_ENTRY_NULL;
2885
2886 virtual_space_start = vstart;
2887 virtual_space_end = VM_MAX_KERNEL_ADDRESS;
2888
2889 /* mark all the address spaces in use */
2890 for (i = 0; i < MAX_ASID / (sizeof(uint32_t) * NBBY); i++)
2891 asid_bitmap[i] = 0xffffffff;
2892
2893 /*
2894 * The kernel gets ASID 0, and all aliases of it. This is
2895 * important because ASID 0 is global; if we vend ASID 0
2896 * out to a user pmap, those translations will show up in
2897 * other processes through the TLB.
2898 */
2899 for (i = 0; i < MAX_ASID; i += ARM_MAX_ASID) {
2900 asid_bitmap[i / (sizeof(uint32_t) * NBBY)] &= ~(1 << (i % (sizeof(uint32_t) * NBBY)));
2901 }
2902
2903 kernel_pmap->asid = 0;
2904 kernel_pmap->vasid = 0;
2905
2906 if (PE_parse_boot_argn("arm_maxoffset", &maxoffset, sizeof (maxoffset))) {
2907 maxoffset = trunc_page(maxoffset);
2908 if ((maxoffset >= pmap_max_offset(FALSE, ARM_PMAP_MAX_OFFSET_MIN))
2909 && (maxoffset <= pmap_max_offset(FALSE, ARM_PMAP_MAX_OFFSET_MAX))) {
2910 arm_pmap_max_offset_default = maxoffset;
2911 }
2912 }
2913 #if defined(__arm64__)
2914 if (PE_parse_boot_argn("arm64_maxoffset", &maxoffset, sizeof (maxoffset))) {
2915 maxoffset = trunc_page(maxoffset);
2916 if ((maxoffset >= pmap_max_offset(TRUE, ARM_PMAP_MAX_OFFSET_MIN))
2917 && (maxoffset <= pmap_max_offset(TRUE, ARM_PMAP_MAX_OFFSET_MAX))) {
2918 arm64_pmap_max_offset_default = maxoffset;
2919 }
2920 }
2921 #endif
2922
2923 #if DEVELOPMENT || DEBUG
2924 PE_parse_boot_argn("panic_on_unsigned_execute", &panic_on_unsigned_execute, sizeof (panic_on_unsigned_execute));
2925 #endif /* DEVELOPMENT || DEBUG */
2926
2927 pmap_nesting_size_min = ARM_NESTING_SIZE_MIN;
2928 pmap_nesting_size_max = ARM_NESTING_SIZE_MAX;
2929
2930 simple_lock_init(&phys_backup_lock, 0);
2931 }
2932
2933
2934 void
2935 pmap_virtual_space(
2936 vm_offset_t *startp,
2937 vm_offset_t *endp
2938 )
2939 {
2940 *startp = virtual_space_start;
2941 *endp = virtual_space_end;
2942 }
2943
2944
2945 boolean_t
2946 pmap_virtual_region(
2947 unsigned int region_select,
2948 vm_map_offset_t *startp,
2949 vm_map_size_t *size
2950 )
2951 {
2952 boolean_t ret = FALSE;
2953 #if __ARM64_PMAP_SUBPAGE_L1__ && __ARM_16K_PG__
2954 if (region_select == 0) {
2955 /*
2956 * In this config, the bootstrap mappings should occupy their own L2
2957 * TTs, as they should be immutable after boot. Having the associated
2958 * TTEs and PTEs in their own pages allows us to lock down those pages,
2959 * while allowing the rest of the kernel address range to be remapped.
2960 */
2961 #if (__ARM_VMSA__ > 7)
2962 *startp = LOW_GLOBAL_BASE_ADDRESS & ~ARM_TT_L2_OFFMASK;
2963 #else
2964 #error Unsupported configuration
2965 #endif
2966 *size = ((VM_MAX_KERNEL_ADDRESS - *startp) & ~PAGE_MASK);
2967 ret = TRUE;
2968 }
2969 #else
2970 #if (__ARM_VMSA__ > 7)
2971 unsigned long low_global_vr_mask = 0;
2972 vm_map_size_t low_global_vr_size = 0;
2973 #endif
2974
2975 if (region_select == 0) {
2976 #if (__ARM_VMSA__ == 7)
2977 *startp = gVirtBase & 0xFFC00000;
2978 *size = ((virtual_space_start-(gVirtBase & 0xFFC00000)) + ~0xFFC00000) & 0xFFC00000;
2979 #else
2980 /* Round to avoid overlapping with the V=P area; round to at least the L2 block size. */
2981 if (!TEST_PAGE_SIZE_4K) {
2982 *startp = gVirtBase & 0xFFFFFFFFFE000000;
2983 *size = ((virtual_space_start-(gVirtBase & 0xFFFFFFFFFE000000)) + ~0xFFFFFFFFFE000000) & 0xFFFFFFFFFE000000;
2984 } else {
2985 *startp = gVirtBase & 0xFFFFFFFFFF800000;
2986 *size = ((virtual_space_start-(gVirtBase & 0xFFFFFFFFFF800000)) + ~0xFFFFFFFFFF800000) & 0xFFFFFFFFFF800000;
2987 }
2988 #endif
2989 ret = TRUE;
2990 }
2991 if (region_select == 1) {
2992 *startp = VREGION1_START;
2993 *size = VREGION1_SIZE;
2994 ret = TRUE;
2995 }
2996 #if (__ARM_VMSA__ > 7)
2997 /* We need to reserve a range that is at least the size of an L2 block mapping for the low globals */
2998 if (!TEST_PAGE_SIZE_4K) {
2999 low_global_vr_mask = 0xFFFFFFFFFE000000;
3000 low_global_vr_size = 0x2000000;
3001 } else {
3002 low_global_vr_mask = 0xFFFFFFFFFF800000;
3003 low_global_vr_size = 0x800000;
3004 }
3005
3006 if (((gVirtBase & low_global_vr_mask) != LOW_GLOBAL_BASE_ADDRESS) && (region_select == 2)) {
3007 *startp = LOW_GLOBAL_BASE_ADDRESS;
3008 *size = low_global_vr_size;
3009 ret = TRUE;
3010 }
3011
3012 if (region_select == 3) {
3013 /* In this config, we allow the bootstrap mappings to occupy the same
3014 * page table pages as the heap.
3015 */
3016 *startp = VM_MIN_KERNEL_ADDRESS;
3017 *size = LOW_GLOBAL_BASE_ADDRESS - *startp;
3018 ret = TRUE;
3019 }
3020 #endif
3021 #endif
3022 return ret;
3023 }
3024
3025 unsigned int
3026 pmap_free_pages(
3027 void)
3028 {
3029 return (unsigned int)atop(avail_end - first_avail);
3030 }
3031
3032
3033 boolean_t
3034 pmap_next_page_hi(
3035 ppnum_t * pnum)
3036 {
3037 return pmap_next_page(pnum);
3038 }
3039
3040
3041 boolean_t
3042 pmap_next_page(
3043 ppnum_t *pnum)
3044 {
3045 if (first_avail != avail_end) {
3046 *pnum = (ppnum_t)atop(first_avail);
3047 first_avail += PAGE_SIZE;
3048 return TRUE;
3049 }
3050 return FALSE;
3051 }
3052
3053
3054 /*
3055 * Initialize the pmap module.
3056 * Called by vm_init, to initialize any structures that the pmap
3057 * system needs to map virtual memory.
3058 */
3059 void
3060 pmap_init(
3061 void)
3062 {
3063 /*
3064 * Protect page zero in the kernel map.
3065 * (can be overruled by permanent transltion
3066 * table entries at page zero - see arm_vm_init).
3067 */
3068 vm_protect(kernel_map, 0, PAGE_SIZE, TRUE, VM_PROT_NONE);
3069
3070 pmap_initialized = TRUE;
3071
3072 pmap_zone_init();
3073
3074
3075 /*
3076 * Initialize the pmap object (for tracking the vm_page_t
3077 * structures for pages we allocate to be page tables in
3078 * pmap_expand().
3079 */
3080 _vm_object_allocate(mem_size, pmap_object);
3081 pmap_object->copy_strategy = MEMORY_OBJECT_COPY_NONE;
3082
3083 pv_init();
3084
3085 /*
3086 * The value of hard_maxproc may have been scaled, make sure
3087 * it is still less than the value of MAX_ASID.
3088 */
3089 assert(hard_maxproc < MAX_ASID);
3090
3091 #if CONFIG_PGTRACE
3092 pmap_pgtrace_init();
3093 #endif
3094 }
3095
3096 boolean_t
3097 pmap_verify_free(
3098 ppnum_t ppnum)
3099 {
3100 pv_entry_t **pv_h;
3101 int pai;
3102 boolean_t result = TRUE;
3103 pmap_paddr_t phys = ptoa(ppnum);
3104
3105 assert(phys != vm_page_fictitious_addr);
3106
3107 if (!pa_valid(phys))
3108 return (FALSE);
3109
3110 pai = (int)pa_index(phys);
3111 pv_h = pai_to_pvh(pai);
3112
3113 result = (pvh_list(pv_h) == PV_ENTRY_NULL);
3114
3115 return (result);
3116 }
3117
3118
3119 /*
3120 * Initialize zones used by pmap.
3121 */
3122 static void
3123 pmap_zone_init(
3124 void)
3125 {
3126 /*
3127 * Create the zone of physical maps
3128 * and the physical-to-virtual entries.
3129 */
3130 pmap_zone = zinit((vm_size_t) sizeof(struct pmap), (vm_size_t) sizeof(struct pmap)*256,
3131 PAGE_SIZE, "pmap");
3132 }
3133
3134
3135 /*
3136 * Create and return a physical map.
3137 *
3138 * If the size specified for the map
3139 * is zero, the map is an actual physical
3140 * map, and may be referenced by the
3141 * hardware.
3142 *
3143 * If the size specified is non-zero,
3144 * the map will be used in software only, and
3145 * is bounded by that size.
3146 */
3147 static pmap_t
3148 pmap_create_internal(
3149 ledger_t ledger,
3150 vm_map_size_t size,
3151 boolean_t is_64bit)
3152 {
3153 unsigned i;
3154 pmap_t p;
3155
3156 /*
3157 * A software use-only map doesn't even need a pmap.
3158 */
3159 if (size != 0) {
3160 return (PMAP_NULL);
3161 }
3162
3163
3164 /*
3165 * Allocate a pmap struct from the pmap_zone. Then allocate
3166 * the translation table of the right size for the pmap.
3167 */
3168 if ((p = (pmap_t) zalloc(pmap_zone)) == PMAP_NULL)
3169 return (PMAP_NULL);
3170
3171 if (is_64bit) {
3172 p->min = MACH_VM_MIN_ADDRESS;
3173 p->max = MACH_VM_MAX_ADDRESS;
3174 } else {
3175 p->min = VM_MIN_ADDRESS;
3176 p->max = VM_MAX_ADDRESS;
3177 }
3178
3179 p->wired = 0;
3180 p->ref_count = 1;
3181 p->gc_status = 0;
3182 p->stamp = hw_atomic_add(&pmap_stamp, 1);
3183 p->nx_enabled = TRUE;
3184 p->is_64bit = is_64bit;
3185 p->nested = FALSE;
3186 p->nested_pmap = PMAP_NULL;
3187
3188
3189 ledger_reference(ledger);
3190 p->ledger = ledger;
3191
3192 PMAP_LOCK_INIT(p);
3193 #if (__ARM_VMSA__ == 7)
3194 simple_lock_init(&p->tt1_lock, 0);
3195 #endif
3196 memset((void *) &p->stats, 0, sizeof(p->stats));
3197
3198 p->tt_entry_free = (tt_entry_t *)0;
3199
3200 p->tte = pmap_tt1_allocate(p, PMAP_ROOT_ALLOC_SIZE, 0);
3201 p->ttep = ml_static_vtop((vm_offset_t)p->tte);
3202
3203 #if (__ARM_VMSA__ == 7)
3204 p->tte_index_max = NTTES;
3205 #else
3206 p->tte_index_max = (PMAP_ROOT_ALLOC_SIZE / sizeof(tt_entry_t));
3207 #endif
3208 p->prev_tte = (tt_entry_t *) NULL;
3209 p->cpu_ref = 0;
3210
3211 /* nullify the translation table */
3212 for (i = 0; i < p->tte_index_max; i++)
3213 p->tte[i] = ARM_TTE_TYPE_FAULT;
3214
3215 #ifndef __ARM_L1_PTW__
3216 CleanPoU_DcacheRegion((vm_offset_t) (p->tte), PMAP_ROOT_ALLOC_SIZE);
3217 #else
3218 __asm__ volatile("dsb ish");
3219 #endif
3220 /* assign a asid */
3221 p->vasid = alloc_asid();
3222 p->asid = p->vasid % ARM_MAX_ASID;
3223
3224 /*
3225 * initialize the rest of the structure
3226 */
3227 p->nested_region_grand_addr = 0x0ULL;
3228 p->nested_region_subord_addr = 0x0ULL;
3229 p->nested_region_size = 0x0ULL;
3230 p->nested_region_asid_bitmap = NULL;
3231 p->nested_region_asid_bitmap_size = 0x0UL;
3232
3233 #if MACH_ASSERT
3234 p->pmap_pid = 0;
3235 strlcpy(p->pmap_procname, "<nil>", sizeof (p->pmap_procname));
3236 #endif /* MACH_ASSERT */
3237 #if DEVELOPMENT || DEBUG
3238 p->footprint_suspended = FALSE;
3239 p->footprint_was_suspended = FALSE;
3240 #endif /* DEVELOPMENT || DEBUG */
3241
3242 simple_lock(&pmaps_lock);
3243 queue_enter(&map_pmap_list, p, pmap_t, pmaps);
3244 simple_unlock(&pmaps_lock);
3245
3246 return (p);
3247 }
3248
3249 pmap_t
3250 pmap_create(
3251 ledger_t ledger,
3252 vm_map_size_t size,
3253 boolean_t is_64bit)
3254 {
3255 pmap_t pmap;
3256
3257 PMAP_TRACE(PMAP_CODE(PMAP__CREATE) | DBG_FUNC_START, size, is_64bit);
3258
3259 pmap = pmap_create_internal(ledger, size, is_64bit);
3260
3261 PMAP_TRACE(PMAP_CODE(PMAP__CREATE) | DBG_FUNC_END,
3262 VM_KERNEL_ADDRHIDE(pmap));
3263
3264 return pmap;
3265 }
3266
3267 #if MACH_ASSERT
3268 static void
3269 pmap_set_process_internal(
3270 __unused pmap_t pmap,
3271 __unused int pid,
3272 __unused char *procname)
3273 {
3274 #if MACH_ASSERT
3275 if (pmap == NULL) {
3276 return;
3277 }
3278
3279 pmap->pmap_pid = pid;
3280 strlcpy(pmap->pmap_procname, procname, sizeof (pmap->pmap_procname));
3281 #endif
3282 }
3283 #endif
3284
3285 #if MACH_ASSERT
3286 void
3287 pmap_set_process(
3288 pmap_t pmap,
3289 int pid,
3290 char *procname)
3291 {
3292 pmap_set_process_internal(pmap, pid, procname);
3293 }
3294
3295 /*
3296 * We maintain stats and ledgers so that a task's physical footprint is:
3297 * phys_footprint = ((internal - alternate_accounting)
3298 * + (internal_compressed - alternate_accounting_compressed)
3299 * + iokit_mapped
3300 * + purgeable_nonvolatile
3301 * + purgeable_nonvolatile_compressed
3302 * + page_table)
3303 * where "alternate_accounting" includes "iokit" and "purgeable" memory.
3304 */
3305
3306 struct {
3307 uint64_t num_pmaps_checked;
3308
3309 int phys_footprint_over;
3310 ledger_amount_t phys_footprint_over_total;
3311 ledger_amount_t phys_footprint_over_max;
3312 int phys_footprint_under;
3313 ledger_amount_t phys_footprint_under_total;
3314 ledger_amount_t phys_footprint_under_max;
3315
3316 int internal_over;
3317 ledger_amount_t internal_over_total;
3318 ledger_amount_t internal_over_max;
3319 int internal_under;
3320 ledger_amount_t internal_under_total;
3321 ledger_amount_t internal_under_max;
3322
3323 int internal_compressed_over;
3324 ledger_amount_t internal_compressed_over_total;
3325 ledger_amount_t internal_compressed_over_max;
3326 int internal_compressed_under;
3327 ledger_amount_t internal_compressed_under_total;
3328 ledger_amount_t internal_compressed_under_max;
3329
3330 int iokit_mapped_over;
3331 ledger_amount_t iokit_mapped_over_total;
3332 ledger_amount_t iokit_mapped_over_max;
3333 int iokit_mapped_under;
3334 ledger_amount_t iokit_mapped_under_total;
3335 ledger_amount_t iokit_mapped_under_max;
3336
3337 int alternate_accounting_over;
3338 ledger_amount_t alternate_accounting_over_total;
3339 ledger_amount_t alternate_accounting_over_max;
3340 int alternate_accounting_under;
3341 ledger_amount_t alternate_accounting_under_total;
3342 ledger_amount_t alternate_accounting_under_max;
3343
3344 int alternate_accounting_compressed_over;
3345 ledger_amount_t alternate_accounting_compressed_over_total;
3346 ledger_amount_t alternate_accounting_compressed_over_max;
3347 int alternate_accounting_compressed_under;
3348 ledger_amount_t alternate_accounting_compressed_under_total;
3349 ledger_amount_t alternate_accounting_compressed_under_max;
3350
3351 int page_table_over;
3352 ledger_amount_t page_table_over_total;
3353 ledger_amount_t page_table_over_max;
3354 int page_table_under;
3355 ledger_amount_t page_table_under_total;
3356 ledger_amount_t page_table_under_max;
3357
3358 int purgeable_volatile_over;
3359 ledger_amount_t purgeable_volatile_over_total;
3360 ledger_amount_t purgeable_volatile_over_max;
3361 int purgeable_volatile_under;
3362 ledger_amount_t purgeable_volatile_under_total;
3363 ledger_amount_t purgeable_volatile_under_max;
3364
3365 int purgeable_nonvolatile_over;
3366 ledger_amount_t purgeable_nonvolatile_over_total;
3367 ledger_amount_t purgeable_nonvolatile_over_max;
3368 int purgeable_nonvolatile_under;
3369 ledger_amount_t purgeable_nonvolatile_under_total;
3370 ledger_amount_t purgeable_nonvolatile_under_max;
3371
3372 int purgeable_volatile_compressed_over;
3373 ledger_amount_t purgeable_volatile_compressed_over_total;
3374 ledger_amount_t purgeable_volatile_compressed_over_max;
3375 int purgeable_volatile_compressed_under;
3376 ledger_amount_t purgeable_volatile_compressed_under_total;
3377 ledger_amount_t purgeable_volatile_compressed_under_max;
3378
3379 int purgeable_nonvolatile_compressed_over;
3380 ledger_amount_t purgeable_nonvolatile_compressed_over_total;
3381 ledger_amount_t purgeable_nonvolatile_compressed_over_max;
3382 int purgeable_nonvolatile_compressed_under;
3383 ledger_amount_t purgeable_nonvolatile_compressed_under_total;
3384 ledger_amount_t purgeable_nonvolatile_compressed_under_max;
3385 } pmap_ledgers_drift;
3386 #endif /* MACH_ASSERT */
3387
3388 /*
3389 * Retire the given physical map from service.
3390 * Should only be called if the map contains
3391 * no valid mappings.
3392 */
3393 static void
3394 pmap_destroy_internal(
3395 pmap_t pmap)
3396 {
3397 #if (__ARM_VMSA__ == 7)
3398 pt_entry_t *ttep;
3399 unsigned int i;
3400 pmap_t tmp_pmap, tt_pmap;
3401 queue_head_t tmp_pmap_list;
3402
3403 queue_init(&tmp_pmap_list);
3404 simple_lock(&pmaps_lock);
3405 tt_pmap = CAST_DOWN_EXPLICIT(pmap_t, queue_first(&tt_pmap_list));
3406 while (!queue_end(&tt_pmap_list, (queue_entry_t)tt_pmap)) {
3407 if (tt_pmap->cpu_ref == 0 ) {
3408 tmp_pmap = tt_pmap;
3409 tt_pmap = CAST_DOWN_EXPLICIT(pmap_t, queue_next(&tmp_pmap->pmaps));
3410 queue_remove(&tt_pmap_list, tmp_pmap, pmap_t, pmaps);
3411 tt_pmap_count--;
3412 queue_enter(&tmp_pmap_list, tmp_pmap, pmap_t, pmaps);
3413 } else {
3414 tmp_pmap = tt_pmap;
3415 tt_pmap = CAST_DOWN_EXPLICIT(pmap_t, queue_next(&tmp_pmap->pmaps));
3416 }
3417 }
3418 simple_unlock(&pmaps_lock);
3419
3420 tmp_pmap = CAST_DOWN_EXPLICIT(pmap_t, queue_first(&tmp_pmap_list));
3421 while (!queue_end(&tmp_pmap_list, (queue_entry_t)tmp_pmap)) {
3422 tt_pmap = tmp_pmap;
3423 tmp_pmap = CAST_DOWN_EXPLICIT(pmap_t, queue_next(&tt_pmap->pmaps));
3424 queue_remove(&tmp_pmap_list, tt_pmap, pmap_t, pmaps);
3425 if (tt_pmap->tte) {
3426 pmap_tt1_deallocate(pmap, tt_pmap->tte, tt_pmap->tte_index_max*sizeof(tt_entry_t), 0);
3427 tt_pmap->tte = (tt_entry_t *) NULL;
3428 tt_pmap->ttep = 0;
3429 tt_pmap->tte_index_max = 0;
3430 }
3431 if (tt_pmap->prev_tte) {
3432 pmap_tt1_deallocate(pmap, tt_pmap->prev_tte, PMAP_ROOT_ALLOC_SIZE, 0);
3433 tt_pmap->prev_tte = (tt_entry_t *) NULL;
3434 }
3435 assert((tt_free_entry_t*)pmap->tt_entry_free == NULL);
3436 free_asid(tt_pmap->vasid);
3437
3438 pmap_check_ledgers(tt_pmap);
3439 ledger_dereference(tt_pmap->ledger);
3440
3441 zfree(pmap_zone, tt_pmap);
3442 }
3443
3444 if (pmap == PMAP_NULL)
3445 return;
3446
3447 if (hw_atomic_sub(&pmap->ref_count, 1) != 0)
3448 return;
3449
3450 simple_lock(&pmaps_lock);
3451
3452 while (pmap->gc_status & PMAP_GC_INFLIGHT) {
3453 pmap->gc_status |= PMAP_GC_WAIT;
3454 assert_wait((event_t) & pmap->gc_status, THREAD_UNINT);
3455 simple_unlock(&pmaps_lock);
3456 (void) thread_block(THREAD_CONTINUE_NULL);
3457 simple_lock(&pmaps_lock);
3458
3459 }
3460
3461 queue_remove(&map_pmap_list, pmap, pmap_t, pmaps);
3462 simple_unlock(&pmaps_lock);
3463
3464 /*
3465 * Free the memory maps, then the
3466 * pmap structure.
3467 */
3468 PMAP_LOCK(pmap);
3469 for (i = 0; i < pmap->tte_index_max; i++) {
3470 ttep = &pmap->tte[i];
3471 if ((*ttep & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_TABLE) {
3472 pmap_tte_deallocate(pmap, ttep, PMAP_TT_L1_LEVEL);
3473 flush_mmu_tlb_entry((i<<ARM_TT_L1_SHIFT) | (pmap->asid & 0xff));
3474 }
3475 }
3476 PMAP_UNLOCK(pmap);
3477
3478 if (pmap->cpu_ref == 0) {
3479 if (pmap->tte) {
3480 pmap_tt1_deallocate(pmap, pmap->tte, pmap->tte_index_max*sizeof(tt_entry_t), 0);
3481 pmap->tte = (tt_entry_t *) NULL;
3482 pmap->ttep = 0;
3483 pmap->tte_index_max = 0;
3484 }
3485 if (pmap->prev_tte) {
3486 pmap_tt1_deallocate(pmap, pmap->prev_tte, PMAP_ROOT_ALLOC_SIZE, 0);
3487 pmap->prev_tte = (tt_entry_t *) NULL;
3488 }
3489 assert((tt_free_entry_t*)pmap->tt_entry_free == NULL);
3490
3491 /* return its asid to the pool */
3492 free_asid(pmap->vasid);
3493 pmap_check_ledgers(pmap);
3494
3495 ledger_dereference(pmap->ledger);
3496 if (pmap->nested_region_asid_bitmap)
3497 kfree(pmap->nested_region_asid_bitmap, pmap->nested_region_asid_bitmap_size*sizeof(unsigned int));
3498 zfree(pmap_zone, pmap);
3499 } else {
3500 simple_lock(&pmaps_lock);
3501 queue_enter(&tt_pmap_list, pmap, pmap_t, pmaps);
3502 tt_pmap_count++;
3503 if (tt_pmap_count > tt_pmap_max)
3504 tt_pmap_max = tt_pmap_count;
3505 simple_unlock(&pmaps_lock);
3506 }
3507 #else
3508 pt_entry_t *ttep;
3509 pmap_paddr_t pa;
3510 vm_map_address_t c;
3511
3512 if (pmap == PMAP_NULL) {
3513 return;
3514 }
3515
3516 if (!pmap->is_64bit)
3517 pmap_unmap_sharedpage32(pmap);
3518
3519 if (hw_atomic_sub(&pmap->ref_count, 1) == 0) {
3520
3521 simple_lock(&pmaps_lock);
3522 while (pmap->gc_status & PMAP_GC_INFLIGHT) {
3523 pmap->gc_status |= PMAP_GC_WAIT;
3524 assert_wait((event_t) & pmap->gc_status, THREAD_UNINT);
3525 simple_unlock(&pmaps_lock);
3526 (void) thread_block(THREAD_CONTINUE_NULL);
3527 simple_lock(&pmaps_lock);
3528 }
3529 queue_remove(&map_pmap_list, pmap, pmap_t, pmaps);
3530 simple_unlock(&pmaps_lock);
3531
3532 /*
3533 * Free the memory maps, then the
3534 * pmap structure.
3535 */
3536 for (c = pmap->min; c < pmap->max; c += ARM_TT_L2_SIZE) {
3537 ttep = pmap_tt2e(pmap, c);
3538 if ((ttep != PT_ENTRY_NULL) && (*ttep & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_TABLE) {
3539 PMAP_LOCK(pmap);
3540 pmap_tte_deallocate(pmap, ttep, PMAP_TT_L2_LEVEL);
3541 PMAP_UNLOCK(pmap);
3542 flush_mmu_tlb_entry(tlbi_addr(c) | tlbi_asid(pmap->asid));
3543 }
3544 }
3545 #if !__ARM64_TWO_LEVEL_PMAP__
3546 for (c = pmap->min; c < pmap->max; c += ARM_TT_L1_SIZE) {
3547 ttep = pmap_tt1e(pmap, c);
3548 if ((ttep != PT_ENTRY_NULL) && (*ttep & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_TABLE) {
3549 PMAP_LOCK(pmap);
3550 pmap_tte_deallocate(pmap, ttep, PMAP_TT_L1_LEVEL);
3551 PMAP_UNLOCK(pmap);
3552 }
3553 }
3554 #endif
3555
3556 if (pmap->tte) {
3557 pa = pmap->ttep;
3558 pmap_tt1_deallocate(pmap, (tt_entry_t *)phystokv(pa), PMAP_ROOT_ALLOC_SIZE, 0);
3559 }
3560
3561
3562 assert((tt_free_entry_t*)pmap->tt_entry_free == NULL);
3563
3564 flush_mmu_tlb_asid((uint64_t)(pmap->asid) << TLBI_ASID_SHIFT);
3565 free_asid(pmap->vasid);
3566
3567 if (pmap->nested_region_asid_bitmap) {
3568 kfree(pmap->nested_region_asid_bitmap, pmap->nested_region_asid_bitmap_size*sizeof(unsigned int));
3569 }
3570
3571 pmap_check_ledgers(pmap);
3572 ledger_dereference(pmap->ledger);
3573
3574 zfree(pmap_zone, pmap);
3575 }
3576
3577 #endif
3578 }
3579
3580 void
3581 pmap_destroy(
3582 pmap_t pmap)
3583 {
3584 PMAP_TRACE(PMAP_CODE(PMAP__DESTROY) | DBG_FUNC_START,
3585 VM_KERNEL_ADDRHIDE(pmap));
3586
3587 pmap_destroy_internal(pmap);
3588
3589 PMAP_TRACE(PMAP_CODE(PMAP__DESTROY) | DBG_FUNC_END);
3590 }
3591
3592
3593 /*
3594 * Add a reference to the specified pmap.
3595 */
3596 static void
3597 pmap_reference_internal(
3598 pmap_t pmap)
3599 {
3600 if (pmap != PMAP_NULL) {
3601 (void) hw_atomic_add(&pmap->ref_count, 1);
3602 }
3603 }
3604
3605 void
3606 pmap_reference(
3607 pmap_t pmap)
3608 {
3609 pmap_reference_internal(pmap);
3610 }
3611
3612 static tt_entry_t *
3613 pmap_tt1_allocate(
3614 pmap_t pmap,
3615 vm_size_t size,
3616 unsigned option)
3617 {
3618 tt_entry_t *tt1;
3619 tt_free_entry_t *tt1_free;
3620 pmap_paddr_t pa;
3621 vm_address_t va;
3622 vm_address_t va_end;
3623 kern_return_t ret;
3624
3625 simple_lock(&pmaps_lock);
3626 if ((size == PAGE_SIZE) && (free_page_size_tt_count != 0)) {
3627 free_page_size_tt_count--;
3628 tt1 = (tt_entry_t *)free_page_size_tt_list;
3629 free_page_size_tt_list = ((tt_free_entry_t *)tt1)->next;
3630 simple_unlock(&pmaps_lock);
3631 pmap_tt_ledger_credit(pmap, size);
3632 return (tt_entry_t *)tt1;
3633 };
3634 if ((size == 2*PAGE_SIZE) && (free_two_page_size_tt_count != 0)) {
3635 free_two_page_size_tt_count--;
3636 tt1 = (tt_entry_t *)free_two_page_size_tt_list;
3637 free_two_page_size_tt_list = ((tt_free_entry_t *)tt1)->next;
3638 simple_unlock(&pmaps_lock);
3639 pmap_tt_ledger_credit(pmap, size);
3640 return (tt_entry_t *)tt1;
3641 };
3642 if (free_tt_count != 0) {
3643 free_tt_count--;
3644 tt1 = (tt_entry_t *)free_tt_list;
3645 free_tt_list = (tt_free_entry_t *)((tt_free_entry_t *)tt1)->next;
3646 simple_unlock(&pmaps_lock);
3647 pmap_tt_ledger_credit(pmap, size);
3648 return (tt_entry_t *)tt1;
3649 }
3650
3651 simple_unlock(&pmaps_lock);
3652
3653 ret = pmap_pages_alloc(&pa, (unsigned)((size < PAGE_SIZE)? PAGE_SIZE : size), ((option & PMAP_TT_ALLOCATE_NOWAIT)? PMAP_PAGES_ALLOCATE_NOWAIT : 0));
3654
3655 if(ret == KERN_RESOURCE_SHORTAGE)
3656 return (tt_entry_t *)0;
3657
3658
3659 if (size < PAGE_SIZE) {
3660 simple_lock(&pmaps_lock);
3661
3662 for (va_end = phystokv(pa) + PAGE_SIZE, va = phystokv(pa) + size; va < va_end; va = va+size) {
3663 tt1_free = (tt_free_entry_t *)va;
3664 tt1_free->next = free_tt_list;
3665 free_tt_list = tt1_free;
3666 free_tt_count++;
3667 }
3668 if (free_tt_count > free_tt_max)
3669 free_tt_max = free_tt_count;
3670
3671 simple_unlock(&pmaps_lock);
3672 }
3673
3674 /* Always report root allocations in units of PMAP_ROOT_ALLOC_SIZE, which can be obtained by sysctl arm_pt_root_size.
3675 * Depending on the device, this can vary between 512b and 16K. */
3676 OSAddAtomic((uint32_t)(size / PMAP_ROOT_ALLOC_SIZE), (pmap == kernel_pmap ? &inuse_kernel_tteroot_count : &inuse_user_tteroot_count));
3677 OSAddAtomic64(size / PMAP_ROOT_ALLOC_SIZE, &alloc_tteroot_count);
3678 pmap_tt_ledger_credit(pmap, size);
3679
3680 return (tt_entry_t *) phystokv(pa);
3681 }
3682
3683 static void
3684 pmap_tt1_deallocate(
3685 pmap_t pmap,
3686 tt_entry_t *tt,
3687 vm_size_t size,
3688 unsigned option)
3689 {
3690 tt_free_entry_t *tt_entry;
3691
3692 tt_entry = (tt_free_entry_t *)tt;
3693 if (not_in_kdp)
3694 simple_lock(&pmaps_lock);
3695
3696 if (size < PAGE_SIZE) {
3697 free_tt_count++;
3698 if (free_tt_count > free_tt_max)
3699 free_tt_max = free_tt_count;
3700 tt_entry->next = free_tt_list;
3701 free_tt_list = tt_entry;
3702 }
3703
3704 if (size == PAGE_SIZE) {
3705 free_page_size_tt_count++;
3706 if (free_page_size_tt_count > free_page_size_tt_max)
3707 free_page_size_tt_max = free_page_size_tt_count;
3708 tt_entry->next = free_page_size_tt_list;
3709 free_page_size_tt_list = tt_entry;
3710 }
3711
3712 if (size == 2*PAGE_SIZE) {
3713 free_two_page_size_tt_count++;
3714 if (free_two_page_size_tt_count > free_two_page_size_tt_max)
3715 free_two_page_size_tt_max = free_two_page_size_tt_count;
3716 tt_entry->next = free_two_page_size_tt_list;
3717 free_two_page_size_tt_list = tt_entry;
3718 }
3719
3720 if ((option & PMAP_TT_DEALLOCATE_NOBLOCK) || (!not_in_kdp)) {
3721 if (not_in_kdp)
3722 simple_unlock(&pmaps_lock);
3723 pmap_tt_ledger_debit(pmap, size);
3724 return;
3725 }
3726
3727 while (free_page_size_tt_count > FREE_PAGE_SIZE_TT_MAX) {
3728
3729 free_page_size_tt_count--;
3730 tt = (tt_entry_t *)free_page_size_tt_list;
3731 free_page_size_tt_list = ((tt_free_entry_t *)tt)->next;
3732
3733 simple_unlock(&pmaps_lock);
3734
3735 pmap_pages_free(ml_static_vtop((vm_offset_t)tt), PAGE_SIZE);
3736
3737 OSAddAtomic(-(int32_t)(PAGE_SIZE / PMAP_ROOT_ALLOC_SIZE), (pmap == kernel_pmap ? &inuse_kernel_tteroot_count : &inuse_user_tteroot_count));
3738
3739 simple_lock(&pmaps_lock);
3740 }
3741
3742 while (free_two_page_size_tt_count > FREE_TWO_PAGE_SIZE_TT_MAX) {
3743 free_two_page_size_tt_count--;
3744 tt = (tt_entry_t *)free_two_page_size_tt_list;
3745 free_two_page_size_tt_list = ((tt_free_entry_t *)tt)->next;
3746
3747 simple_unlock(&pmaps_lock);
3748
3749 pmap_pages_free(ml_static_vtop((vm_offset_t)tt), 2*PAGE_SIZE);
3750
3751 OSAddAtomic(-2 * (int32_t)(PAGE_SIZE / PMAP_ROOT_ALLOC_SIZE), (pmap == kernel_pmap ? &inuse_kernel_tteroot_count : &inuse_user_tteroot_count));
3752
3753 simple_lock(&pmaps_lock);
3754 }
3755 simple_unlock(&pmaps_lock);
3756 pmap_tt_ledger_debit(pmap, size);
3757 }
3758
3759 static kern_return_t
3760 pmap_tt_allocate(
3761 pmap_t pmap,
3762 tt_entry_t **ttp,
3763 unsigned int level,
3764 unsigned int options)
3765 {
3766 pmap_paddr_t pa;
3767 *ttp = NULL;
3768
3769 PMAP_LOCK(pmap);
3770 if ((tt_free_entry_t *)pmap->tt_entry_free != NULL) {
3771 tt_free_entry_t *tt_free_next;
3772
3773 tt_free_next = ((tt_free_entry_t *)pmap->tt_entry_free)->next;
3774 *ttp = (tt_entry_t *)pmap->tt_entry_free;
3775 pmap->tt_entry_free = (tt_entry_t *)tt_free_next;
3776 }
3777 PMAP_UNLOCK(pmap);
3778
3779 if (*ttp == NULL) {
3780 pt_desc_t *ptdp;
3781
3782 /*
3783 * Allocate a VM page for the level x page table entries.
3784 */
3785 while (pmap_pages_alloc(&pa, PAGE_SIZE, ((options & PMAP_TT_ALLOCATE_NOWAIT)? PMAP_PAGES_ALLOCATE_NOWAIT : 0)) != KERN_SUCCESS) {
3786 if(options & PMAP_OPTIONS_NOWAIT) {
3787 return KERN_RESOURCE_SHORTAGE;
3788 }
3789 VM_PAGE_WAIT();
3790 }
3791
3792 if (level < PMAP_TT_MAX_LEVEL) {
3793 OSAddAtomic64(1, &alloc_ttepages_count);
3794 OSAddAtomic(1, (pmap == kernel_pmap ? &inuse_kernel_ttepages_count : &inuse_user_ttepages_count));
3795 } else {
3796 OSAddAtomic64(1, &alloc_ptepages_count);
3797 OSAddAtomic(1, (pmap == kernel_pmap ? &inuse_kernel_ptepages_count : &inuse_user_ptepages_count));
3798 }
3799
3800 pmap_tt_ledger_credit(pmap, PAGE_SIZE);
3801
3802 PMAP_ZINFO_PALLOC(pmap, PAGE_SIZE);
3803
3804 ptdp = ptd_alloc(pmap);
3805 *(pt_desc_t **)pai_to_pvh(pa_index(pa)) = ptdp;
3806
3807 __unreachable_ok_push
3808 if (TEST_PAGE_RATIO_4) {
3809 vm_address_t va;
3810 vm_address_t va_end;
3811
3812 PMAP_LOCK(pmap);
3813
3814 for (va_end = phystokv(pa) + PAGE_SIZE, va = phystokv(pa) + ARM_PGBYTES; va < va_end; va = va+ARM_PGBYTES) {
3815 ((tt_free_entry_t *)va)->next = (tt_free_entry_t *)pmap->tt_entry_free;
3816 pmap->tt_entry_free = (tt_entry_t *)va;
3817 }
3818 PMAP_UNLOCK(pmap);
3819 }
3820 __unreachable_ok_pop
3821
3822 *ttp = (tt_entry_t *)phystokv(pa);
3823 }
3824
3825
3826 return KERN_SUCCESS;
3827 }
3828
3829
3830 static void
3831 pmap_tt_deallocate(
3832 pmap_t pmap,
3833 tt_entry_t *ttp,
3834 unsigned int level)
3835 {
3836 pt_desc_t *ptdp;
3837 unsigned pt_acc_cnt;
3838 unsigned i, max_pt_index = PAGE_RATIO;
3839 vm_offset_t free_page=0;
3840
3841 PMAP_LOCK(pmap);
3842
3843 ptdp = ptep_get_ptd((vm_offset_t)ttp);
3844
3845 if (level < PMAP_TT_MAX_LEVEL) {
3846
3847 if (ptdp->pt_cnt[ARM_PT_DESC_INDEX(ttp)].refcnt == PT_DESC_REFCOUNT)
3848 ptdp->pt_cnt[ARM_PT_DESC_INDEX(ttp)].refcnt = 0;
3849 }
3850
3851 ptdp->pt_map[ARM_PT_DESC_INDEX(ttp)].va = 0;
3852
3853 if (ptdp->pt_cnt[ARM_PT_DESC_INDEX(ttp)].refcnt != 0)
3854 panic("pmap_tt_deallocate(): ptdp %p, count %d\n", ptdp, ptdp->pt_cnt[ARM_PT_DESC_INDEX(ttp)].refcnt);
3855
3856 for (i = 0, pt_acc_cnt = 0 ; i < max_pt_index ; i++)
3857 pt_acc_cnt += ptdp->pt_cnt[i].refcnt;
3858
3859 if (pt_acc_cnt == 0) {
3860 tt_free_entry_t *tt_free_list = (tt_free_entry_t *)&pmap->tt_entry_free;
3861 unsigned pt_free_entry_cnt = 1;
3862
3863 while (pt_free_entry_cnt < max_pt_index && tt_free_list) {
3864 tt_free_entry_t *tt_free_list_next;
3865
3866 tt_free_list_next = tt_free_list->next;
3867 if ((((vm_offset_t)tt_free_list_next) - ((vm_offset_t)ttp & ~PAGE_MASK)) < PAGE_SIZE) {
3868 pt_free_entry_cnt++;
3869 }
3870 tt_free_list = tt_free_list_next;
3871 }
3872 if (pt_free_entry_cnt == max_pt_index) {
3873 tt_free_entry_t *tt_free_list_cur;
3874
3875 free_page = (vm_offset_t)ttp & ~PAGE_MASK;
3876 tt_free_list = (tt_free_entry_t *)&pmap->tt_entry_free;
3877 tt_free_list_cur = (tt_free_entry_t *)&pmap->tt_entry_free;
3878
3879 while (tt_free_list_cur) {
3880 tt_free_entry_t *tt_free_list_next;
3881
3882 tt_free_list_next = tt_free_list_cur->next;
3883 if ((((vm_offset_t)tt_free_list_next) - free_page) < PAGE_SIZE) {
3884 tt_free_list->next = tt_free_list_next->next;
3885 } else {
3886 tt_free_list = tt_free_list_next;
3887 }
3888 tt_free_list_cur = tt_free_list_next;
3889 }
3890 } else {
3891 ((tt_free_entry_t *)ttp)->next = (tt_free_entry_t *)pmap->tt_entry_free;
3892 pmap->tt_entry_free = ttp;
3893 }
3894 } else {
3895 ((tt_free_entry_t *)ttp)->next = (tt_free_entry_t *)pmap->tt_entry_free;
3896 pmap->tt_entry_free = ttp;
3897 }
3898
3899 PMAP_UNLOCK(pmap);
3900
3901 if (free_page != 0) {
3902
3903 ptd_deallocate(ptep_get_ptd((vm_offset_t)free_page));
3904 *(pt_desc_t **)pai_to_pvh(pa_index(ml_static_vtop(free_page))) = NULL;
3905 pmap_pages_free(ml_static_vtop(free_page), PAGE_SIZE);
3906 if (level < PMAP_TT_MAX_LEVEL)
3907 OSAddAtomic(-1, (pmap == kernel_pmap ? &inuse_kernel_ttepages_count : &inuse_user_ttepages_count));
3908 else
3909 OSAddAtomic(-1, (pmap == kernel_pmap ? &inuse_kernel_ptepages_count : &inuse_user_ptepages_count));
3910 PMAP_ZINFO_PFREE(pmap, PAGE_SIZE);
3911 pmap_tt_ledger_debit(pmap, PAGE_SIZE);
3912 }
3913 }
3914
3915 static void
3916 pmap_tte_deallocate(
3917 pmap_t pmap,
3918 tt_entry_t *ttep,
3919 unsigned int level)
3920 {
3921 pmap_paddr_t pa;
3922 tt_entry_t tte;
3923
3924 PMAP_ASSERT_LOCKED(pmap);
3925
3926 tte = *ttep;
3927
3928 if (tte == 0) {
3929 panic("pmap_tte_deallocate(): null tt_entry ttep==%p\n", ttep);
3930 }
3931
3932 #if MACH_ASSERT
3933 if (tte_get_ptd(tte)->pmap != pmap) {
3934 panic("pmap_tte_deallocate(): ptd=%p ptd->pmap=%p pmap=%p \n",
3935 tte_get_ptd(tte), tte_get_ptd(tte)->pmap, pmap);
3936 }
3937 #endif
3938 if (((level+1) == PMAP_TT_MAX_LEVEL) && (tte_get_ptd(tte)->pt_cnt[ARM_PT_DESC_INDEX(ttetokv(*ttep))].refcnt != 0)) {
3939 panic("pmap_tte_deallocate(): pmap=%p ttep=%p ptd=%p refcnt=0x%x \n", pmap, ttep,
3940 tte_get_ptd(tte), (tte_get_ptd(tte)->pt_cnt[ARM_PT_DESC_INDEX(ttetokv(*ttep))].refcnt));
3941 }
3942
3943 #if (__ARM_VMSA__ == 7)
3944 {
3945 tt_entry_t *ttep_4M = (tt_entry_t *) ((vm_offset_t)ttep & 0xFFFFFFF0);
3946 unsigned i;
3947
3948 for (i = 0; i<4; i++, ttep_4M++)
3949 *ttep_4M = (tt_entry_t) 0;
3950 }
3951 #else
3952 *ttep = (tt_entry_t) 0;
3953 #endif
3954
3955 #ifndef __ARM_L1_PTW__
3956 CleanPoU_DcacheRegion((vm_offset_t) ttep, sizeof(tt_entry_t));
3957 #else
3958 __asm__ volatile("dsb ish");
3959 #endif
3960 if ((tte & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_TABLE) {
3961 #if MACH_ASSERT
3962 {
3963 pt_entry_t *pte_p = ((pt_entry_t *) (ttetokv(tte) & ~ARM_PGMASK));
3964 unsigned i;
3965
3966 for (i = 0; i < (ARM_PGBYTES / sizeof(*pte_p)); i++,pte_p++) {
3967 if (ARM_PTE_IS_COMPRESSED(*pte_p)) {
3968 panic("pmap_tte_deallocate: tte=0x%llx pmap=%p, pte_p=%p, pte=0x%llx compressed\n",
3969 (uint64_t)tte, pmap, pte_p, (uint64_t)(*pte_p));
3970 } else if (((*pte_p) & ARM_PTE_TYPE_MASK) != ARM_PTE_TYPE_FAULT) {
3971 panic("pmap_tte_deallocate: tte=0x%llx pmap=%p, pte_p=%p, pte=0x%llx\n",
3972 (uint64_t)tte, pmap, pte_p, (uint64_t)(*pte_p));
3973 }
3974 }
3975 }
3976 #endif
3977 PMAP_UNLOCK(pmap);
3978
3979 /* Clear any page offset: we mean to free the whole page, but armv7 TTEs may only be
3980 * aligned on 1K boundaries. We clear the surrounding "chunk" of 4 TTEs above. */
3981 pa = tte_to_pa(tte) & ~ARM_PGMASK;
3982 pmap_tt_deallocate(pmap, (tt_entry_t *) phystokv(pa), level+1);
3983 PMAP_LOCK(pmap);
3984 }
3985 }
3986
3987 /*
3988 * Remove a range of hardware page-table entries.
3989 * The entries given are the first (inclusive)
3990 * and last (exclusive) entries for the VM pages.
3991 * The virtual address is the va for the first pte.
3992 *
3993 * The pmap must be locked.
3994 * If the pmap is not the kernel pmap, the range must lie
3995 * entirely within one pte-page. This is NOT checked.
3996 * Assumes that the pte-page exists.
3997 *
3998 * Returns the number of PTE changed, and sets *rmv_cnt
3999 * to the number of SPTE changed.
4000 */
4001 static int
4002 pmap_remove_range(
4003 pmap_t pmap,
4004 vm_map_address_t va,
4005 pt_entry_t *bpte,
4006 pt_entry_t *epte,
4007 uint32_t *rmv_cnt)
4008 {
4009 return pmap_remove_range_options(pmap, va, bpte, epte, rmv_cnt,
4010 PMAP_OPTIONS_REMOVE);
4011 }
4012
4013 #if MACH_ASSERT
4014 int num_reusable_mismatch = 0;
4015 #endif /* MACH_ASSERT */
4016
4017 static int
4018 pmap_remove_range_options(
4019 pmap_t pmap,
4020 vm_map_address_t va,
4021 pt_entry_t *bpte,
4022 pt_entry_t *epte,
4023 uint32_t *rmv_cnt,
4024 int options)
4025 {
4026 pt_entry_t *cpte;
4027 int num_removed, num_unwired;
4028 int num_pte_changed;
4029 int pai = 0;
4030 pmap_paddr_t pa;
4031 int num_external, num_internal, num_reusable;
4032 int num_alt_internal;
4033 uint64_t num_compressed, num_alt_compressed;
4034
4035 PMAP_ASSERT_LOCKED(pmap);
4036
4037 num_removed = 0;
4038 num_unwired = 0;
4039 num_pte_changed = 0;
4040 num_external = 0;
4041 num_internal = 0;
4042 num_reusable = 0;
4043 num_compressed = 0;
4044 num_alt_internal = 0;
4045 num_alt_compressed = 0;
4046
4047 for (cpte = bpte; cpte < epte;
4048 cpte += PAGE_SIZE/ARM_PGBYTES, va += PAGE_SIZE) {
4049 pv_entry_t **pv_h, **pve_pp;
4050 pv_entry_t *pve_p;
4051 pt_entry_t spte;
4052 boolean_t managed=FALSE;
4053
4054 spte = *cpte;
4055
4056 #if CONFIG_PGTRACE
4057 if (pgtrace_enabled) {
4058 pmap_pgtrace_remove_clone(pmap, pte_to_pa(spte), va);
4059 }
4060 #endif
4061
4062 while (!managed) {
4063 if (pmap != kernel_pmap &&
4064 (options & PMAP_OPTIONS_REMOVE) &&
4065 (ARM_PTE_IS_COMPRESSED(spte))) {
4066 /*
4067 * "pmap" must be locked at this point,
4068 * so this should not race with another
4069 * pmap_remove_range() or pmap_enter().
4070 */
4071
4072 /* one less "compressed"... */
4073 num_compressed++;
4074 if (spte & ARM_PTE_COMPRESSED_ALT) {
4075 /* ... but it used to be "ALTACCT" */
4076 num_alt_compressed++;
4077 }
4078
4079 /* clear marker */
4080 WRITE_PTE_FAST(cpte, ARM_PTE_TYPE_FAULT);
4081 /*
4082 * "refcnt" also accounts for
4083 * our "compressed" markers,
4084 * so let's update it here.
4085 */
4086 if (OSAddAtomic16(-1, (SInt16 *) &(ptep_get_ptd(cpte)->pt_cnt[ARM_PT_DESC_INDEX(cpte)].refcnt)) <= 0)
4087 panic("pmap_remove_range_options: over-release of ptdp %p for pte %p\n", ptep_get_ptd(cpte), cpte);
4088 spte = *cpte;
4089 }
4090 /*
4091 * It may be possible for the pte to transition from managed
4092 * to unmanaged in this timeframe; for now, elide the assert.
4093 * We should break out as a consequence of checking pa_valid.
4094 */
4095 //assert(!ARM_PTE_IS_COMPRESSED(spte));
4096 pa = pte_to_pa(spte);
4097 if (!pa_valid(pa)) {
4098 break;
4099 }
4100 pai = (int)pa_index(pa);
4101 LOCK_PVH(pai);
4102 spte = *cpte;
4103 pa = pte_to_pa(spte);
4104 if (pai == (int)pa_index(pa)) {
4105 managed =TRUE;
4106 break; // Leave pai locked as we will unlock it after we free the PV entry
4107 }
4108 UNLOCK_PVH(pai);
4109 }
4110
4111 if (ARM_PTE_IS_COMPRESSED(*cpte)) {
4112 /*
4113 * There used to be a valid mapping here but it
4114 * has already been removed when the page was
4115 * sent to the VM compressor, so nothing left to
4116 * remove now...
4117 */
4118 continue;
4119 }
4120
4121 /* remove the translation, do not flush the TLB */
4122 if (*cpte != ARM_PTE_TYPE_FAULT) {
4123 assert(!ARM_PTE_IS_COMPRESSED(*cpte));
4124 #if MACH_ASSERT
4125 if (managed && (pmap != kernel_pmap) && (ptep_get_va(cpte) != va)) {
4126 panic("pmap_remove_range_options(): cpte=%p ptd=%p pte=0x%llx va=0x%llx\n",
4127 cpte, ptep_get_ptd(cpte), (uint64_t)*cpte, (uint64_t)va);
4128 }
4129 #endif
4130 WRITE_PTE_FAST(cpte, ARM_PTE_TYPE_FAULT);
4131 num_pte_changed++;
4132 }
4133
4134 if ((spte != ARM_PTE_TYPE_FAULT) &&
4135 (pmap != kernel_pmap)) {
4136 assert(!ARM_PTE_IS_COMPRESSED(spte));
4137 if (OSAddAtomic16(-1, (SInt16 *) &(ptep_get_ptd(cpte)->pt_cnt[ARM_PT_DESC_INDEX(cpte)].refcnt)) <= 0)
4138 panic("pmap_remove_range_options: over-release of ptdp %p for pte %p\n", ptep_get_ptd(cpte), cpte);
4139 if(rmv_cnt) (*rmv_cnt)++;
4140 }
4141
4142 if (pte_is_wired(spte)) {
4143 pte_set_wired(cpte, 0);
4144 num_unwired++;
4145 }
4146 /*
4147 * if not managed, we're done
4148 */
4149 if (!managed)
4150 continue;
4151 /*
4152 * find and remove the mapping from the chain for this
4153 * physical address.
4154 */
4155 ASSERT_PVH_LOCKED(pai); // Should have been locked when we found the managed PTE above
4156 pv_h = pai_to_pvh(pai);
4157
4158 if (pvh_test_type(pv_h, PVH_TYPE_PTEP)) {
4159 if (__builtin_expect((cpte != pvh_ptep(pv_h)), 0))
4160 panic("pmap_remove_range(): cpte=%p (0x%llx) does not match pv_h=%p (%p)\n", cpte, (uint64_t)spte, pv_h, pvh_ptep(pv_h));
4161 if (IS_ALTACCT_PAGE(pai, PV_ENTRY_NULL)) {
4162 assert(IS_INTERNAL_PAGE(pai));
4163 num_internal++;
4164 num_alt_internal++;
4165 CLR_ALTACCT_PAGE(pai, PV_ENTRY_NULL);
4166 } else if (IS_INTERNAL_PAGE(pai)) {
4167 if (IS_REUSABLE_PAGE(pai)) {
4168 num_reusable++;
4169 } else {
4170 num_internal++;
4171 }
4172 } else {
4173 num_external++;
4174 }
4175 pvh_update_head(pv_h, PV_ENTRY_NULL, PVH_TYPE_NULL);
4176 } else if (pvh_test_type(pv_h, PVH_TYPE_PVEP)) {
4177
4178 pve_pp = pv_h;
4179 pve_p = pvh_list(pv_h);
4180
4181 while (pve_p != PV_ENTRY_NULL &&
4182 (pve_get_ptep(pve_p) != cpte)) {
4183 pve_pp = pve_link_field(pve_p);
4184 pve_p = PVE_NEXT_PTR(pve_next(pve_p));
4185 }
4186
4187 if (__builtin_expect((pve_p == PV_ENTRY_NULL), 0)) {
4188 UNLOCK_PVH(pai);
4189 panic("pmap_remove_range(): cpte=%p (0x%llx) not in pv_h=%p\n", cpte, (uint64_t)spte, pv_h);
4190 }
4191
4192 #if MACH_ASSERT
4193 if (kern_feature_override(KF_PMAPV_OVRD) == FALSE) {
4194 pv_entry_t *check_pve_p = PVE_NEXT_PTR(pve_next(pve_p));
4195 while (check_pve_p != PV_ENTRY_NULL) {
4196 if (pve_get_ptep(check_pve_p) == cpte) {
4197 panic("pmap_remove_range(): duplicate pve entry cpte=%p pmap=%p, pv_h=%p, pve_p=%p, pte=0x%llx, va=0x%llx\n",
4198 cpte, pmap, pv_h, pve_p, (uint64_t)spte, (uint64_t)va);
4199 }
4200 check_pve_p = PVE_NEXT_PTR(pve_next(check_pve_p));
4201 }
4202 }
4203 #endif
4204
4205 if (IS_ALTACCT_PAGE(pai, pve_p)) {
4206 assert(IS_INTERNAL_PAGE(pai));
4207 num_internal++;
4208 num_alt_internal++;
4209 CLR_ALTACCT_PAGE(pai, pve_p);
4210 } else if (IS_INTERNAL_PAGE(pai)) {
4211 if (IS_REUSABLE_PAGE(pai)) {
4212 num_reusable++;
4213 } else {
4214 num_internal++;
4215 }
4216 } else {
4217 num_external++;
4218 }
4219
4220 pvh_remove(pv_h, pve_pp, pve_p) ;
4221 pv_free(pve_p);
4222 } else {
4223 panic("pmap_remove_range(): unexpected PV head %p, cpte=%p pmap=%p pv_h=%p pte=0x%llx va=0x%llx\n",
4224 *pv_h, cpte, pmap, pv_h, (uint64_t)spte, (uint64_t)va);
4225 }
4226
4227 UNLOCK_PVH(pai);
4228 num_removed++;
4229 }
4230
4231 /*
4232 * Update the counts
4233 */
4234 OSAddAtomic(-num_removed, (SInt32 *) &pmap->stats.resident_count);
4235 pmap_ledger_debit(pmap, task_ledgers.phys_mem, machine_ptob(num_removed));
4236
4237 if (pmap != kernel_pmap) {
4238 /* sanity checks... */
4239 #if MACH_ASSERT
4240 if (pmap->stats.internal < num_internal) {
4241 if ((pmap->stats.internal + pmap->stats.reusable) ==
4242 (num_internal + num_reusable)) {
4243 num_reusable_mismatch++;
4244 printf("pmap_remove_range_options(%p,0x%llx,%p,%p,0x%x): num_internal=%d num_removed=%d num_unwired=%d num_external=%d num_reusable=%d num_compressed=%lld num_alt_internal=%d num_alt_compressed=%lld num_pte_changed=%d stats.internal=%d stats.reusable=%d\n",
4245 pmap,
4246 (uint64_t) va,
4247 bpte,
4248 epte,
4249 options,
4250 num_internal,
4251 num_removed,
4252 num_unwired,
4253 num_external,
4254 num_reusable,
4255 num_compressed,
4256 num_alt_internal,
4257 num_alt_compressed,
4258 num_pte_changed,
4259 pmap->stats.internal,
4260 pmap->stats.reusable);
4261 /* slight mismatch: fix it... */
4262 num_internal = pmap->stats.internal;
4263 num_reusable = pmap->stats.reusable;
4264 } else {
4265 panic("pmap_remove_range_options(%p,0x%llx,%p,%p,0x%x): num_internal=%d num_removed=%d num_unwired=%d num_external=%d num_reusable=%d num_compressed=%lld num_alt_internal=%d num_alt_compressed=%lld num_pte_changed=%d stats.internal=%d stats.reusable=%d",
4266 pmap,
4267 (uint64_t) va,
4268 bpte,
4269 epte,
4270 options,
4271 num_internal,
4272 num_removed,
4273 num_unwired,
4274 num_external,
4275 num_reusable,
4276 num_compressed,
4277 num_alt_internal,
4278 num_alt_compressed,
4279 num_pte_changed,
4280 pmap->stats.internal,
4281 pmap->stats.reusable);
4282 }
4283 }
4284 #endif /* MACH_ASSERT */
4285 assertf(pmap->stats.external >= num_external,
4286 "pmap=%p num_external=%d stats.external=%d",
4287 pmap, num_external, pmap->stats.external);
4288 assertf(pmap->stats.internal >= num_internal,
4289 "pmap=%p num_internal=%d stats.internal=%d num_reusable=%d stats.reusable=%d",
4290 pmap,
4291 num_internal, pmap->stats.internal,
4292 num_reusable, pmap->stats.reusable);
4293 assertf(pmap->stats.reusable >= num_reusable,
4294 "pmap=%p num_internal=%d stats.internal=%d num_reusable=%d stats.reusable=%d",
4295 pmap,
4296 num_internal, pmap->stats.internal,
4297 num_reusable, pmap->stats.reusable);
4298 assertf(pmap->stats.compressed >= num_compressed,
4299 "pmap=%p num_compressed=%lld num_alt_compressed=%lld stats.compressed=%lld",
4300 pmap, num_compressed, num_alt_compressed,
4301 pmap->stats.compressed);
4302
4303 /* update pmap stats... */
4304 OSAddAtomic(-num_unwired, (SInt32 *) &pmap->stats.wired_count);
4305 if (num_external)
4306 OSAddAtomic(-num_external, &pmap->stats.external);
4307 if (num_internal)
4308 OSAddAtomic(-num_internal, &pmap->stats.internal);
4309 if (num_reusable)
4310 OSAddAtomic(-num_reusable, &pmap->stats.reusable);
4311 if (num_compressed)
4312 OSAddAtomic64(-num_compressed, &pmap->stats.compressed);
4313 /* ... and ledgers */
4314 pmap_ledger_debit(pmap, task_ledgers.wired_mem, machine_ptob(num_unwired));
4315 pmap_ledger_debit(pmap, task_ledgers.internal, machine_ptob(num_internal));
4316 pmap_ledger_debit(pmap, task_ledgers.alternate_accounting, machine_ptob(num_alt_internal));
4317 pmap_ledger_debit(pmap, task_ledgers.alternate_accounting_compressed, machine_ptob(num_alt_compressed));
4318 pmap_ledger_debit(pmap, task_ledgers.internal_compressed, machine_ptob(num_compressed));
4319 /* make needed adjustments to phys_footprint */
4320 pmap_ledger_debit(pmap, task_ledgers.phys_footprint,
4321 machine_ptob((num_internal -
4322 num_alt_internal) +
4323 (num_compressed -
4324 num_alt_compressed)));
4325 }
4326
4327 /* flush the ptable entries we have written */
4328 if (num_pte_changed > 0)
4329 FLUSH_PTE_RANGE(bpte, epte);
4330
4331 return num_pte_changed;
4332 }
4333
4334
4335 /*
4336 * Remove the given range of addresses
4337 * from the specified map.
4338 *
4339 * It is assumed that the start and end are properly
4340 * rounded to the hardware page size.
4341 */
4342 void
4343 pmap_remove(
4344 pmap_t pmap,
4345 vm_map_address_t start,
4346 vm_map_address_t end)
4347 {
4348 pmap_remove_options(pmap, start, end, PMAP_OPTIONS_REMOVE);
4349 }
4350
4351 static int
4352 pmap_remove_options_internal(pmap_t pmap,
4353 vm_map_address_t start,
4354 vm_map_address_t end,
4355 int options)
4356 {
4357 int remove_count = 0;
4358 pt_entry_t *bpte, *epte;
4359 pt_entry_t *pte_p;
4360 tt_entry_t *tte_p;
4361 uint32_t rmv_spte=0;
4362
4363 PMAP_LOCK(pmap);
4364
4365 tte_p = pmap_tte(pmap, start);
4366
4367 if (tte_p == (tt_entry_t *) NULL) {
4368 goto done;
4369 }
4370
4371 if ((*tte_p & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_TABLE) {
4372 pte_p = (pt_entry_t *) ttetokv(*tte_p);
4373 bpte = &pte_p[ptenum(start)];
4374 epte = bpte + ((end - start) >> ARM_TT_LEAF_SHIFT);
4375
4376 remove_count += pmap_remove_range_options(pmap, start, bpte, epte,
4377 &rmv_spte, options);
4378
4379 #if (__ARM_VMSA__ == 7)
4380 if (rmv_spte && (ptep_get_ptd(pte_p)->pt_cnt[ARM_PT_DESC_INDEX(pte_p)].refcnt == 0) &&
4381 (pmap != kernel_pmap) && (pmap->nested == FALSE)) {
4382 pmap_tte_deallocate(pmap, tte_p, PMAP_TT_L1_LEVEL);
4383 flush_mmu_tlb_entry((start & ~ARM_TT_L1_OFFMASK) | (pmap->asid & 0xff));
4384 }
4385 #else
4386 if (rmv_spte && (ptep_get_ptd(pte_p)->pt_cnt[ARM_PT_DESC_INDEX(pte_p)].refcnt == 0) &&
4387 (pmap != kernel_pmap) && (pmap->nested == FALSE)) {
4388 pmap_tte_deallocate(pmap, tte_p, PMAP_TT_L2_LEVEL);
4389 flush_mmu_tlb_entry(tlbi_addr(start & ~ARM_TT_L2_OFFMASK) | tlbi_asid(pmap->asid));
4390 }
4391 #endif
4392 }
4393
4394 done:
4395 PMAP_UNLOCK(pmap);
4396
4397 return remove_count;
4398 }
4399
4400 void
4401 pmap_remove_options(
4402 pmap_t pmap,
4403 vm_map_address_t start,
4404 vm_map_address_t end,
4405 int options)
4406 {
4407 int remove_count = 0;
4408 vm_map_address_t va;
4409
4410 if (pmap == PMAP_NULL)
4411 return;
4412
4413 PMAP_TRACE(PMAP_CODE(PMAP__REMOVE) | DBG_FUNC_START,
4414 VM_KERNEL_ADDRHIDE(pmap), VM_KERNEL_ADDRHIDE(start),
4415 VM_KERNEL_ADDRHIDE(end));
4416
4417 #if MACH_ASSERT
4418 if ((start|end) & PAGE_MASK) {
4419 panic("pmap_remove_options() pmap %p start 0x%llx end 0x%llx\n",
4420 pmap, (uint64_t)start, (uint64_t)end);
4421 }
4422 if ((end < start) || (start < pmap->min) || (end > pmap->max)) {
4423 panic("pmap_remove_options(): invalid address range, pmap=%p, start=0x%llx, end=0x%llx\n",
4424 pmap, (uint64_t)start, (uint64_t)end);
4425 }
4426 #endif
4427
4428 /*
4429 * Invalidate the translation buffer first
4430 */
4431 va = start;
4432 while (va < end) {
4433 vm_map_address_t l;
4434
4435 #if (__ARM_VMSA__ == 7)
4436 l = ((va + ARM_TT_L1_SIZE) & ~ARM_TT_L1_OFFMASK);
4437 #else
4438 l = ((va + ARM_TT_L2_SIZE) & ~ARM_TT_L2_OFFMASK);
4439 #endif
4440 if (l > end)
4441 l = end;
4442
4443 remove_count += pmap_remove_options_internal(pmap, va, l, options);
4444
4445 va = l;
4446 }
4447
4448
4449 if (remove_count > 0)
4450 PMAP_UPDATE_TLBS(pmap, start, end);
4451
4452 PMAP_TRACE(PMAP_CODE(PMAP__REMOVE) | DBG_FUNC_END);
4453 }
4454
4455
4456 /*
4457 * Remove phys addr if mapped in specified map
4458 */
4459 void
4460 pmap_remove_some_phys(
4461 __unused pmap_t map,
4462 __unused ppnum_t pn)
4463 {
4464 /* Implement to support working set code */
4465 }
4466
4467
4468 void
4469 pmap_set_pmap(
4470 pmap_t pmap,
4471 #if !__ARM_USER_PROTECT__
4472 __unused
4473 #endif
4474 thread_t thread)
4475 {
4476 pmap_switch(pmap);
4477 #if __ARM_USER_PROTECT__
4478 if (pmap->tte_index_max == NTTES) {
4479 thread->machine.uptw_ttc = 2;
4480 thread->machine.uptw_ttb = ((unsigned int) pmap->ttep) | TTBR_SETUP;
4481 } else {
4482 thread->machine.uptw_ttc = 1; \
4483 thread->machine.uptw_ttb = ((unsigned int) pmap->ttep ) | TTBR_SETUP;
4484 }
4485 thread->machine.asid = pmap->asid;
4486 #endif
4487 }
4488
4489 static void
4490 pmap_flush_core_tlb_asid(pmap_t pmap)
4491 {
4492 #if (__ARM_VMSA__ == 7)
4493 flush_core_tlb_asid(pmap->asid);
4494 #else
4495 flush_core_tlb_asid(((uint64_t) pmap->asid) << TLBI_ASID_SHIFT);
4496 #endif
4497 }
4498
4499 static void
4500 pmap_switch_internal(
4501 pmap_t pmap)
4502 {
4503 pmap_cpu_data_t *cpu_data_ptr = pmap_get_cpu_data();
4504 uint32_t last_asid_high_bits, asid_high_bits;
4505 pmap_t cur_pmap;
4506 pmap_t cur_user_pmap;
4507 boolean_t do_asid_flush = FALSE;
4508
4509 #if (__ARM_VMSA__ == 7)
4510 if (not_in_kdp)
4511 simple_lock(&pmap->tt1_lock);
4512 #endif
4513
4514 cur_pmap = current_pmap();
4515 cur_user_pmap = cpu_data_ptr->cpu_user_pmap;
4516
4517 /* Paranoia. */
4518 assert(pmap->asid < (sizeof(cpu_data_ptr->cpu_asid_high_bits) / sizeof(*cpu_data_ptr->cpu_asid_high_bits)));
4519
4520 /* Extract the "virtual" bits of the ASIDs (which could cause us to alias). */
4521 asid_high_bits = pmap->vasid >> ARM_ASID_SHIFT;
4522 last_asid_high_bits = (uint32_t) cpu_data_ptr->cpu_asid_high_bits[pmap->asid];
4523
4524 if (asid_high_bits != last_asid_high_bits) {
4525 /*
4526 * If the virtual ASID of the new pmap does not match the virtual ASID
4527 * last seen on this CPU for the physical ASID (that was a mouthful),
4528 * then this switch runs the risk of aliasing. We need to flush the
4529 * TLB for this phyiscal ASID in this case.
4530 */
4531 cpu_data_ptr->cpu_asid_high_bits[pmap->asid] = (uint8_t) asid_high_bits;
4532 do_asid_flush = TRUE;
4533 }
4534
4535 if ((cur_user_pmap == cur_pmap) && (cur_pmap == pmap)) {
4536 if (cpu_data_ptr->cpu_user_pmap_stamp == pmap->stamp) {
4537 pmap_switch_user_ttb_internal(pmap);
4538
4539 #if (__ARM_VMSA__ == 7)
4540 if (not_in_kdp)
4541 simple_unlock(&pmap->tt1_lock);
4542 #endif
4543
4544 if (do_asid_flush) {
4545 pmap_flush_core_tlb_asid(pmap);
4546 }
4547
4548 return;
4549 } else
4550 cur_user_pmap = NULL;
4551 } else if ((cur_user_pmap == pmap) && (cpu_data_ptr->cpu_user_pmap_stamp != pmap->stamp))
4552 cur_user_pmap = NULL;
4553
4554 pmap_switch_user_ttb_internal(pmap);
4555
4556 if (do_asid_flush) {
4557 pmap_flush_core_tlb_asid(pmap);
4558 }
4559
4560 #if (__ARM_VMSA__ == 7)
4561 if (not_in_kdp)
4562 simple_unlock(&pmap->tt1_lock);
4563 #else
4564 if (pmap != kernel_pmap) {
4565
4566 if (cur_user_pmap != PMAP_NULL) {
4567 /*
4568 * We have a low-address global mapping for the commpage
4569 * for 32-bit processes; flush it if we switch to a 64-bot
4570 * process.
4571 */
4572 if (pmap_is_64bit(pmap) && !pmap_is_64bit(cur_user_pmap)) {
4573 pmap_sharedpage_flush_32_to_64();
4574 }
4575
4576 } else
4577 flush_core_tlb();
4578 }
4579 #endif
4580 }
4581
4582 void
4583 pmap_switch(
4584 pmap_t pmap)
4585 {
4586 pmap_switch_internal(pmap);
4587 }
4588
4589 void
4590 pmap_page_protect(
4591 ppnum_t ppnum,
4592 vm_prot_t prot)
4593 {
4594 pmap_page_protect_options(ppnum, prot, 0, NULL);
4595 }
4596
4597 /*
4598 * Routine: pmap_page_protect_options
4599 *
4600 * Function:
4601 * Lower the permission for all mappings to a given
4602 * page.
4603 */
4604 static void
4605 pmap_page_protect_options_internal(
4606 ppnum_t ppnum,
4607 vm_prot_t prot,
4608 unsigned int options)
4609 {
4610 pmap_paddr_t phys = ptoa(ppnum);
4611 pv_entry_t **pv_h;
4612 pv_entry_t *pve_p;
4613 pv_entry_t *pveh_p;
4614 pv_entry_t *pvet_p;
4615 pt_entry_t *pte_p;
4616 int pai;
4617 boolean_t remove;
4618 boolean_t set_NX;
4619 unsigned int pvh_cnt = 0;
4620
4621 assert(ppnum != vm_page_fictitious_addr);
4622
4623 /* Only work with managed pages. */
4624 if (!pa_valid(phys)) {
4625 return;
4626 }
4627
4628 /*
4629 * Determine the new protection.
4630 */
4631 switch (prot) {
4632 case VM_PROT_ALL:
4633 return; /* nothing to do */
4634 case VM_PROT_READ:
4635 case VM_PROT_READ | VM_PROT_EXECUTE:
4636 remove = FALSE;
4637 break;
4638 default:
4639 remove = TRUE;
4640 break;
4641 }
4642
4643 pai = (int)pa_index(phys);
4644 LOCK_PVH(pai);
4645 pv_h = pai_to_pvh(pai);
4646
4647 pte_p = PT_ENTRY_NULL;
4648 pve_p = PV_ENTRY_NULL;
4649 pveh_p = PV_ENTRY_NULL;
4650 pvet_p = PV_ENTRY_NULL;
4651 if (pvh_test_type(pv_h, PVH_TYPE_PTEP)) {
4652 pte_p = pvh_ptep(pv_h);
4653 } else if (pvh_test_type(pv_h, PVH_TYPE_PVEP)) {
4654 pve_p = pvh_list(pv_h);
4655 pveh_p = pve_p;
4656 }
4657
4658 while ((pve_p != PV_ENTRY_NULL) || (pte_p != PT_ENTRY_NULL)) {
4659 vm_map_address_t va;
4660 pmap_t pmap;
4661 pt_entry_t tmplate;
4662 boolean_t update = FALSE;
4663
4664 if (pve_p != PV_ENTRY_NULL)
4665 pte_p = pve_get_ptep(pve_p);
4666
4667 pmap = ptep_get_pmap(pte_p);
4668 va = ptep_get_va(pte_p);
4669
4670 if (pte_p == PT_ENTRY_NULL) {
4671 panic("pmap_page_protect: pmap=%p prot=%d options=%u, pv_h=%p, pveh_p=%p, pve_p=%p, va=0x%llx ppnum: 0x%x\n",
4672 pmap, prot, options, pv_h, pveh_p, pve_p, (uint64_t)va, ppnum);
4673 } else if ((pmap == NULL) || (atop(pte_to_pa(*pte_p)) != ppnum)) {
4674 #if MACH_ASSERT
4675 if (kern_feature_override(KF_PMAPV_OVRD) == FALSE) {
4676
4677 pv_entry_t *check_pve_p = pveh_p;
4678 while (check_pve_p != PV_ENTRY_NULL) {
4679 if ((check_pve_p != pve_p) && (pve_get_ptep(check_pve_p) == pte_p)) {
4680 panic("pmap_page_protect: duplicate pve entry pte_p=%p pmap=%p prot=%d options=%u, pv_h=%p, pveh_p=%p, pve_p=%p, pte=0x%llx, va=0x%llx ppnum: 0x%x\n",
4681 pte_p, pmap, prot, options, pv_h, pveh_p, pve_p, (uint64_t)*pte_p, (uint64_t)va, ppnum);
4682 }
4683 check_pve_p = PVE_NEXT_PTR(pve_next(check_pve_p));
4684 }
4685 }
4686 #endif
4687 panic("pmap_page_protect: bad pve entry pte_p=%p pmap=%p prot=%d options=%u, pv_h=%p, pveh_p=%p, pve_p=%p, pte=0x%llx, va=0x%llx ppnum: 0x%x\n",
4688 pte_p, pmap, prot, options, pv_h, pveh_p, pve_p, (uint64_t)*pte_p, (uint64_t)va, ppnum);
4689 }
4690
4691 #if DEVELOPMENT || DEBUG
4692 if ((prot & VM_PROT_EXECUTE) || !nx_enabled || !pmap->nx_enabled)
4693 #else
4694 if ((prot & VM_PROT_EXECUTE))
4695 #endif
4696 set_NX = FALSE;
4697 else
4698 set_NX = TRUE;
4699
4700 /* Remove the mapping if new protection is NONE */
4701 if (remove) {
4702 boolean_t is_altacct = FALSE;
4703
4704 if (IS_ALTACCT_PAGE(pai, pve_p)) {
4705 is_altacct = TRUE;
4706 } else {
4707 is_altacct = FALSE;
4708 }
4709
4710 if (pte_is_wired(*pte_p)) {
4711 pte_set_wired(pte_p, 0);
4712 if (pmap != kernel_pmap) {
4713 pmap_ledger_debit(pmap, task_ledgers.wired_mem, PAGE_SIZE);
4714 OSAddAtomic(-1, (SInt32 *) &pmap->stats.wired_count);
4715 }
4716 }
4717
4718 if (*pte_p != ARM_PTE_TYPE_FAULT &&
4719 pmap != kernel_pmap &&
4720 (options & PMAP_OPTIONS_COMPRESSOR) &&
4721 IS_INTERNAL_PAGE(pai)) {
4722 assert(!ARM_PTE_IS_COMPRESSED(*pte_p));
4723 /* mark this PTE as having been "compressed" */
4724 tmplate = ARM_PTE_COMPRESSED;
4725 if (is_altacct) {
4726 tmplate |= ARM_PTE_COMPRESSED_ALT;
4727 is_altacct = TRUE;
4728 }
4729 } else {
4730 tmplate = ARM_PTE_TYPE_FAULT;
4731 }
4732
4733 if ((*pte_p != ARM_PTE_TYPE_FAULT) &&
4734 tmplate == ARM_PTE_TYPE_FAULT &&
4735 (pmap != kernel_pmap)) {
4736 if (OSAddAtomic16(-1, (SInt16 *) &(ptep_get_ptd(pte_p)->pt_cnt[ARM_PT_DESC_INDEX(pte_p)].refcnt)) <= 0)
4737 panic("pmap_page_protect_options(): over-release of ptdp %p for pte %p\n", ptep_get_ptd(pte_p), pte_p);
4738 }
4739
4740 if (*pte_p != tmplate) {
4741 WRITE_PTE(pte_p, tmplate);
4742 update = TRUE;
4743 }
4744 pvh_cnt++;
4745 pmap_ledger_debit(pmap, task_ledgers.phys_mem, PAGE_SIZE);
4746 OSAddAtomic(-1, (SInt32 *) &pmap->stats.resident_count);
4747
4748 #if MACH_ASSERT
4749 /*
4750 * We only ever compress internal pages.
4751 */
4752 if (options & PMAP_OPTIONS_COMPRESSOR) {
4753 assert(IS_INTERNAL_PAGE(pai));
4754 }
4755 #endif
4756
4757 if (pmap != kernel_pmap) {
4758 if (IS_REUSABLE_PAGE(pai) &&
4759 IS_INTERNAL_PAGE(pai) &&
4760 !is_altacct) {
4761 assert(pmap->stats.reusable > 0);
4762 OSAddAtomic(-1, &pmap->stats.reusable);
4763 } else if (IS_INTERNAL_PAGE(pai)) {
4764 assert(pmap->stats.internal > 0);
4765 OSAddAtomic(-1, &pmap->stats.internal);
4766 } else {
4767 assert(pmap->stats.external > 0);
4768 OSAddAtomic(-1, &pmap->stats.external);
4769 }
4770 if ((options & PMAP_OPTIONS_COMPRESSOR) &&
4771 IS_INTERNAL_PAGE(pai)) {
4772 /* adjust "compressed" stats */
4773 OSAddAtomic64(+1, &pmap->stats.compressed);
4774 PMAP_STATS_PEAK(pmap->stats.compressed);
4775 pmap->stats.compressed_lifetime++;
4776 }
4777
4778 if (IS_ALTACCT_PAGE(pai, pve_p)) {
4779 assert(IS_INTERNAL_PAGE(pai));
4780 pmap_ledger_debit(pmap, task_ledgers.internal, PAGE_SIZE);
4781 pmap_ledger_debit(pmap, task_ledgers.alternate_accounting, PAGE_SIZE);
4782 if (options & PMAP_OPTIONS_COMPRESSOR) {
4783 pmap_ledger_credit(pmap, task_ledgers.internal_compressed, PAGE_SIZE);
4784 pmap_ledger_credit(pmap, task_ledgers.alternate_accounting_compressed, PAGE_SIZE);
4785 }
4786
4787 /*
4788 * Cleanup our marker before
4789 * we free this pv_entry.
4790 */
4791 CLR_ALTACCT_PAGE(pai, pve_p);
4792
4793 } else if (IS_REUSABLE_PAGE(pai)) {
4794 assert(IS_INTERNAL_PAGE(pai));
4795 if (options & PMAP_OPTIONS_COMPRESSOR) {
4796 pmap_ledger_credit(pmap, task_ledgers.internal_compressed, PAGE_SIZE);
4797 /* was not in footprint, but is now */
4798 pmap_ledger_credit(pmap, task_ledgers.phys_footprint, PAGE_SIZE);
4799 }
4800
4801 } else if (IS_INTERNAL_PAGE(pai)) {
4802 pmap_ledger_debit(pmap, task_ledgers.internal, PAGE_SIZE);
4803
4804 /*
4805 * Update all stats related to physical footprint, which only
4806 * deals with internal pages.
4807 */
4808 if (options & PMAP_OPTIONS_COMPRESSOR) {
4809 /*
4810 * This removal is only being done so we can send this page to
4811 * the compressor; therefore it mustn't affect total task footprint.
4812 */
4813 pmap_ledger_credit(pmap, task_ledgers.internal_compressed, PAGE_SIZE);
4814 } else {
4815 /*
4816 * This internal page isn't going to the compressor, so adjust stats to keep
4817 * phys_footprint up to date.
4818 */
4819 pmap_ledger_debit(pmap, task_ledgers.phys_footprint, PAGE_SIZE);
4820 }
4821 } else {
4822 /* external page: no impact on ledgers */
4823 }
4824 }
4825
4826 if (pve_p != PV_ENTRY_NULL) {
4827 assert(pve_next(pve_p) == PVE_NEXT_PTR(pve_next(pve_p)));
4828 }
4829
4830 } else {
4831 pt_entry_t spte;
4832
4833 spte = *pte_p;
4834
4835 if (pmap == kernel_pmap)
4836 tmplate = ((spte & ~ARM_PTE_APMASK) | ARM_PTE_AP(AP_RONA));
4837 else
4838 tmplate = ((spte & ~ARM_PTE_APMASK) | ARM_PTE_AP(AP_RORO));
4839
4840 pte_set_ffr(tmplate, 0);
4841
4842 #if (__ARM_VMSA__ == 7)
4843 if (set_NX) {
4844 tmplate |= ARM_PTE_NX;
4845 } else {
4846 /*
4847 * While the naive implementation of this would serve to add execute
4848 * permission, this is not how the VM uses this interface, or how
4849 * x86_64 implements it. So ignore requests to add execute permissions.
4850 */
4851 #if 0
4852 tmplate &= ~ARM_PTE_NX;
4853 #else
4854 ;
4855 #endif
4856 }
4857 #else
4858 if (set_NX)
4859 tmplate |= ARM_PTE_NX | ARM_PTE_PNX;
4860 else {
4861 /*
4862 * While the naive implementation of this would serve to add execute
4863 * permission, this is not how the VM uses this interface, or how
4864 * x86_64 implements it. So ignore requests to add execute permissions.
4865 */
4866 #if 0
4867 if (pmap == kernel_pmap) {
4868 tmplate &= ~ARM_PTE_PNX;
4869 tmplate |= ARM_PTE_NX;
4870 } else {
4871 tmplate &= ~ARM_PTE_NX;
4872 tmplate |= ARM_PTE_PNX;
4873 }
4874 #else
4875 ;
4876 #endif
4877 }
4878 #endif
4879
4880
4881 if (*pte_p != ARM_PTE_TYPE_FAULT &&
4882 !ARM_PTE_IS_COMPRESSED(*pte_p) &&
4883 *pte_p != tmplate) {
4884 WRITE_PTE(pte_p, tmplate);
4885 update = TRUE;
4886 }
4887 }
4888
4889 /* Invalidate TLBs for all CPUs using it */
4890 if (update)
4891 PMAP_UPDATE_TLBS(pmap, va, va + PAGE_SIZE);
4892
4893 pte_p = PT_ENTRY_NULL;
4894 pvet_p = pve_p;
4895 if (pve_p != PV_ENTRY_NULL) {
4896 pvet_p = pve_p;
4897 if (remove) {
4898 assert(pve_next(pve_p) == PVE_NEXT_PTR(pve_next(pve_p)));
4899 }
4900 pve_p = PVE_NEXT_PTR(pve_next(pve_p));
4901 }
4902 }
4903
4904 /* if we removed a bunch of entries, take care of them now */
4905 if (remove) {
4906 pvh_update_head(pv_h, PV_ENTRY_NULL, PVH_TYPE_NULL);
4907 }
4908
4909 UNLOCK_PVH(pai);
4910
4911 if (remove && (pveh_p != PV_ENTRY_NULL)) {
4912 pv_list_free(pveh_p, pvet_p, pvh_cnt);
4913 }
4914 }
4915
4916 void
4917 pmap_page_protect_options(
4918 ppnum_t ppnum,
4919 vm_prot_t prot,
4920 unsigned int options,
4921 __unused void *arg)
4922 {
4923 pmap_paddr_t phys = ptoa(ppnum);
4924
4925 assert(ppnum != vm_page_fictitious_addr);
4926
4927 /* Only work with managed pages. */
4928 if (!pa_valid(phys))
4929 return;
4930
4931 /*
4932 * Determine the new protection.
4933 */
4934 if (prot == VM_PROT_ALL) {
4935 return; /* nothing to do */
4936 }
4937
4938 PMAP_TRACE(PMAP_CODE(PMAP__PAGE_PROTECT) | DBG_FUNC_START, ppnum, prot);
4939
4940 pmap_page_protect_options_internal(ppnum, prot, options);
4941
4942 PMAP_TRACE(PMAP_CODE(PMAP__PAGE_PROTECT) | DBG_FUNC_END);
4943 }
4944
4945 /*
4946 * Indicates if the pmap layer enforces some additional restrictions on the
4947 * given set of protections.
4948 */
4949 bool pmap_has_prot_policy(__unused vm_prot_t prot)
4950 {
4951 return FALSE;
4952 }
4953
4954 /*
4955 * Set the physical protection on the
4956 * specified range of this map as requested.
4957 * VERY IMPORTANT: Will not increase permissions.
4958 * VERY IMPORTANT: Only pmap_enter() is allowed to grant permissions.
4959 */
4960 void
4961 pmap_protect(
4962 pmap_t pmap,
4963 vm_map_address_t b,
4964 vm_map_address_t e,
4965 vm_prot_t prot)
4966 {
4967 pmap_protect_options(pmap, b, e, prot, 0, NULL);
4968 }
4969
4970 static void
4971 pmap_protect_options_internal(pmap_t pmap,
4972 vm_map_address_t start,
4973 vm_map_address_t end,
4974 vm_prot_t prot,
4975 unsigned int options,
4976 __unused void *args)
4977 {
4978 tt_entry_t *tte_p;
4979 pt_entry_t *bpte_p, *epte_p;
4980 pt_entry_t *pte_p;
4981 boolean_t set_NX = TRUE;
4982 #if (__ARM_VMSA__ > 7)
4983 boolean_t set_XO = FALSE;
4984 #endif
4985 boolean_t should_have_removed = FALSE;
4986
4987 #ifndef __ARM_IC_NOALIAS_ICACHE__
4988 boolean_t InvalidatePoU_Icache_Done = FALSE;
4989 #endif
4990
4991 #if DEVELOPMENT || DEBUG
4992 if (options & PMAP_OPTIONS_PROTECT_IMMEDIATE) {
4993 if ((prot & VM_PROT_ALL) == VM_PROT_NONE) {
4994 should_have_removed = TRUE;
4995 }
4996 } else
4997 #endif
4998 {
4999 /* Determine the new protection. */
5000 switch (prot) {
5001 #if (__ARM_VMSA__ > 7)
5002 case VM_PROT_EXECUTE:
5003 set_XO = TRUE;
5004 /* fall through */
5005 #endif
5006 case VM_PROT_READ:
5007 case VM_PROT_READ | VM_PROT_EXECUTE:
5008 break;
5009 case VM_PROT_READ | VM_PROT_WRITE:
5010 case VM_PROT_ALL:
5011 return; /* nothing to do */
5012 default:
5013 should_have_removed = TRUE;
5014 }
5015 }
5016
5017 if (should_have_removed) {
5018 panic("%s: should have been a remove operation, "
5019 "pmap=%p, start=%p, end=%p, prot=%#x, options=%#x, args=%p",
5020 __FUNCTION__,
5021 pmap, (void *)start, (void *)end, prot, options, args);
5022 }
5023
5024 #if DEVELOPMENT || DEBUG
5025 if ((prot & VM_PROT_EXECUTE) || !nx_enabled || !pmap->nx_enabled)
5026 #else
5027 if ((prot & VM_PROT_EXECUTE))
5028 #endif
5029 {
5030 set_NX = FALSE;
5031 } else {
5032 set_NX = TRUE;
5033 }
5034
5035 PMAP_LOCK(pmap);
5036 tte_p = pmap_tte(pmap, start);
5037
5038 if ((tte_p != (tt_entry_t *) NULL) && (*tte_p & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_TABLE) {
5039 bpte_p = (pt_entry_t *) ttetokv(*tte_p);
5040 bpte_p = &bpte_p[ptenum(start)];
5041 epte_p = bpte_p + arm_atop(end - start);
5042 pte_p = bpte_p;
5043
5044 for (pte_p = bpte_p;
5045 pte_p < epte_p;
5046 pte_p += PAGE_SIZE/ARM_PGBYTES) {
5047 pt_entry_t spte;
5048 #if DEVELOPMENT || DEBUG
5049 boolean_t force_write = FALSE;
5050 #endif
5051
5052 spte = *pte_p;
5053
5054 if ((spte == ARM_PTE_TYPE_FAULT) ||
5055 ARM_PTE_IS_COMPRESSED(spte)) {
5056 continue;
5057 }
5058
5059 pmap_paddr_t pa;
5060 int pai=0;
5061 boolean_t managed=FALSE;
5062
5063 while (!managed) {
5064 /*
5065 * It may be possible for the pte to transition from managed
5066 * to unmanaged in this timeframe; for now, elide the assert.
5067 * We should break out as a consequence of checking pa_valid.
5068 */
5069 // assert(!ARM_PTE_IS_COMPRESSED(spte));
5070 pa = pte_to_pa(spte);
5071 if (!pa_valid(pa))
5072 break;
5073 pai = (int)pa_index(pa);
5074 LOCK_PVH(pai);
5075 spte = *pte_p;
5076 pa = pte_to_pa(spte);
5077 if (pai == (int)pa_index(pa)) {
5078 managed =TRUE;
5079 break; // Leave the PVH locked as we will unlock it after we free the PTE
5080 }
5081 UNLOCK_PVH(pai);
5082 }
5083
5084 if ((spte == ARM_PTE_TYPE_FAULT) ||
5085 ARM_PTE_IS_COMPRESSED(spte)) {
5086 continue;
5087 }
5088
5089 pt_entry_t tmplate;
5090
5091 if (pmap == kernel_pmap) {
5092 #if DEVELOPMENT || DEBUG
5093 if ((options & PMAP_OPTIONS_PROTECT_IMMEDIATE) && (prot & VM_PROT_WRITE)) {
5094 force_write = TRUE;
5095 tmplate = ((spte & ~ARM_PTE_APMASK) | ARM_PTE_AP(AP_RWNA));
5096 } else
5097 #endif
5098 {
5099 tmplate = ((spte & ~ARM_PTE_APMASK) | ARM_PTE_AP(AP_RONA));
5100 }
5101 } else {
5102 #if DEVELOPMENT || DEBUG
5103 if ((options & PMAP_OPTIONS_PROTECT_IMMEDIATE) && (prot & VM_PROT_WRITE)) {
5104 force_write = TRUE;
5105 tmplate = ((spte & ~ARM_PTE_APMASK) | ARM_PTE_AP(AP_RWRW));
5106 } else
5107 #endif
5108 {
5109 tmplate = ((spte & ~ARM_PTE_APMASK) | ARM_PTE_AP(AP_RORO));
5110 }
5111 }
5112
5113 /*
5114 * XXX Removing "NX" would
5115 * grant "execute" access
5116 * immediately, bypassing any
5117 * checks VM might want to do
5118 * in its soft fault path.
5119 * pmap_protect() and co. are
5120 * not allowed to increase
5121 * access permissions.
5122 */
5123 #if (__ARM_VMSA__ == 7)
5124 if (set_NX)
5125 tmplate |= ARM_PTE_NX;
5126 else {
5127 /* do NOT clear "NX"! */
5128 }
5129 #else
5130 if (set_NX)
5131 tmplate |= ARM_PTE_NX | ARM_PTE_PNX;
5132 else {
5133 if (pmap == kernel_pmap) {
5134 /*
5135 * TODO: Run CS/Monitor checks here;
5136 * should we be clearing PNX here? Is
5137 * this just for dtrace?
5138 */
5139 tmplate &= ~ARM_PTE_PNX;
5140 tmplate |= ARM_PTE_NX;
5141 } else {
5142 /* do NOT clear "NX"! */
5143 tmplate |= ARM_PTE_PNX;
5144 if (set_XO) {
5145 tmplate &= ~ARM_PTE_APMASK;
5146 tmplate |= ARM_PTE_AP(AP_RONA);
5147 }
5148 }
5149 }
5150 #endif
5151
5152 #if DEVELOPMENT || DEBUG
5153 if (force_write) {
5154 /*
5155 * TODO: Run CS/Monitor checks here.
5156 */
5157 if (managed) {
5158 /*
5159 * We are marking the page as writable,
5160 * so we consider it to be modified and
5161 * referenced.
5162 */
5163 pa_set_bits(pa, PP_ATTR_REFERENCED | PP_ATTR_MODIFIED);
5164 tmplate |= ARM_PTE_AF;
5165
5166 if (IS_REFFAULT_PAGE(pai)) {
5167 CLR_REFFAULT_PAGE(pai);
5168 }
5169
5170 if (IS_MODFAULT_PAGE(pai)) {
5171 CLR_MODFAULT_PAGE(pai);
5172 }
5173 }
5174 } else if (options & PMAP_OPTIONS_PROTECT_IMMEDIATE) {
5175 /*
5176 * An immediate request for anything other than
5177 * write should still mark the page as
5178 * referenced if managed.
5179 */
5180 if (managed) {
5181 pa_set_bits(pa, PP_ATTR_REFERENCED);
5182 tmplate |= ARM_PTE_AF;
5183
5184 if (IS_REFFAULT_PAGE(pai)) {
5185 CLR_REFFAULT_PAGE(pai);
5186 }
5187 }
5188 }
5189 #endif
5190
5191 /* We do not expect to write fast fault the entry. */
5192 pte_set_ffr(tmplate, 0);
5193
5194 /* TODO: Doesn't this need to worry about PNX? */
5195 if (((spte & ARM_PTE_NX) == ARM_PTE_NX) && (prot & VM_PROT_EXECUTE)) {
5196 CleanPoU_DcacheRegion((vm_offset_t) phystokv(pa), PAGE_SIZE);
5197 #ifdef __ARM_IC_NOALIAS_ICACHE__
5198 InvalidatePoU_IcacheRegion((vm_offset_t) phystokv(pa), PAGE_SIZE);
5199 #else
5200 if (!InvalidatePoU_Icache_Done) {
5201 InvalidatePoU_Icache();
5202 InvalidatePoU_Icache_Done = TRUE;
5203 }
5204 #endif
5205 }
5206
5207 WRITE_PTE_FAST(pte_p, tmplate);
5208
5209 if (managed) {
5210 ASSERT_PVH_LOCKED(pai);
5211 UNLOCK_PVH(pai);
5212 }
5213 }
5214
5215 FLUSH_PTE_RANGE(bpte_p, epte_p);
5216 PMAP_UPDATE_TLBS(pmap, start, end);
5217 }
5218
5219 PMAP_UNLOCK(pmap);
5220 }
5221
5222 void
5223 pmap_protect_options(
5224 pmap_t pmap,
5225 vm_map_address_t b,
5226 vm_map_address_t e,
5227 vm_prot_t prot,
5228 unsigned int options,
5229 __unused void *args)
5230 {
5231 vm_map_address_t l, beg;
5232
5233 if ((b|e) & PAGE_MASK) {
5234 panic("pmap_protect_options() pmap %p start 0x%llx end 0x%llx\n",
5235 pmap, (uint64_t)b, (uint64_t)e);
5236 }
5237
5238 #if DEVELOPMENT || DEBUG
5239 if (options & PMAP_OPTIONS_PROTECT_IMMEDIATE) {
5240 if ((prot & VM_PROT_ALL) == VM_PROT_NONE) {
5241 pmap_remove_options(pmap, b, e, options);
5242 return;
5243 }
5244 } else
5245 #endif
5246 {
5247 /* Determine the new protection. */
5248 switch (prot) {
5249 case VM_PROT_EXECUTE:
5250 case VM_PROT_READ:
5251 case VM_PROT_READ | VM_PROT_EXECUTE:
5252 break;
5253 case VM_PROT_READ | VM_PROT_WRITE:
5254 case VM_PROT_ALL:
5255 return; /* nothing to do */
5256 default:
5257 pmap_remove_options(pmap, b, e, options);
5258 return;
5259 }
5260 }
5261
5262 PMAP_TRACE(PMAP_CODE(PMAP__PROTECT) | DBG_FUNC_START,
5263 VM_KERNEL_ADDRHIDE(pmap), VM_KERNEL_ADDRHIDE(b),
5264 VM_KERNEL_ADDRHIDE(e));
5265
5266 beg = b;
5267
5268 while (beg < e) {
5269 l = ((beg + ARM_TT_TWIG_SIZE) & ~ARM_TT_TWIG_OFFMASK);
5270
5271 if (l > e)
5272 l = e;
5273
5274 pmap_protect_options_internal(pmap, beg, l, prot, options, args);
5275
5276 beg = l;
5277 }
5278
5279 PMAP_TRACE(PMAP_CODE(PMAP__PROTECT) | DBG_FUNC_END);
5280 }
5281
5282 /* Map a (possibly) autogenned block */
5283 kern_return_t
5284 pmap_map_block(
5285 pmap_t pmap,
5286 addr64_t va,
5287 ppnum_t pa,
5288 uint32_t size,
5289 vm_prot_t prot,
5290 int attr,
5291 __unused unsigned int flags)
5292 {
5293 kern_return_t kr;
5294 addr64_t original_va = va;
5295 uint32_t page;
5296
5297 for (page = 0; page < size; page++) {
5298 kr = pmap_enter(pmap, va, pa, prot, VM_PROT_NONE, attr, TRUE);
5299
5300 if (kr != KERN_SUCCESS) {
5301 /*
5302 * This will panic for now, as it is unclear that
5303 * removing the mappings is correct.
5304 */
5305 panic("%s: failed pmap_enter, "
5306 "pmap=%p, va=%#llx, pa=%u, size=%u, prot=%#x, flags=%#x",
5307 __FUNCTION__,
5308 pmap, va, pa, size, prot, flags);
5309
5310 pmap_remove(pmap, original_va, va - original_va);
5311 return kr;
5312 }
5313
5314 va += PAGE_SIZE;
5315 pa++;
5316 }
5317
5318 return KERN_SUCCESS;
5319 }
5320
5321 /*
5322 * Insert the given physical page (p) at
5323 * the specified virtual address (v) in the
5324 * target physical map with the protection requested.
5325 *
5326 * If specified, the page will be wired down, meaning
5327 * that the related pte can not be reclaimed.
5328 *
5329 * NB: This is the only routine which MAY NOT lazy-evaluate
5330 * or lose information. That is, this routine must actually
5331 * insert this page into the given map eventually (must make
5332 * forward progress eventually.
5333 */
5334 kern_return_t
5335 pmap_enter(
5336 pmap_t pmap,
5337 vm_map_address_t v,
5338 ppnum_t pn,
5339 vm_prot_t prot,
5340 vm_prot_t fault_type,
5341 unsigned int flags,
5342 boolean_t wired)
5343 {
5344 return pmap_enter_options(pmap, v, pn, prot, fault_type, flags, wired, 0, NULL);
5345 }
5346
5347
5348 static inline void pmap_enter_pte(pmap_t pmap, pt_entry_t *pte_p, pt_entry_t pte, vm_map_address_t v)
5349 {
5350 if (pmap != kernel_pmap && ((pte & ARM_PTE_WIRED) != (*pte_p & ARM_PTE_WIRED)))
5351 {
5352 SInt16 *ptd_wiredcnt_ptr = (SInt16 *)&(ptep_get_ptd(pte_p)->pt_cnt[ARM_PT_DESC_INDEX(pte_p)].wiredcnt);
5353 if (pte & ARM_PTE_WIRED) {
5354 OSAddAtomic16(1, ptd_wiredcnt_ptr);
5355 pmap_ledger_credit(pmap, task_ledgers.wired_mem, PAGE_SIZE);
5356 OSAddAtomic(1, (SInt32 *) &pmap->stats.wired_count);
5357 } else {
5358 OSAddAtomic16(-1, ptd_wiredcnt_ptr);
5359 pmap_ledger_debit(pmap, task_ledgers.wired_mem, PAGE_SIZE);
5360 OSAddAtomic(-1, (SInt32 *) &pmap->stats.wired_count);
5361 }
5362 }
5363 if (*pte_p != ARM_PTE_TYPE_FAULT &&
5364 !ARM_PTE_IS_COMPRESSED(*pte_p)) {
5365 WRITE_PTE(pte_p, pte);
5366 PMAP_UPDATE_TLBS(pmap, v, v + PAGE_SIZE);
5367 } else {
5368 WRITE_PTE(pte_p, pte);
5369 __asm__ volatile("isb");
5370 }
5371 }
5372
5373 static pt_entry_t
5374 wimg_to_pte(unsigned int wimg)
5375 {
5376 pt_entry_t pte;
5377
5378 switch (wimg & (VM_WIMG_MASK)) {
5379 case VM_WIMG_IO:
5380 pte = ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DISABLE);
5381 pte |= ARM_PTE_NX | ARM_PTE_PNX;
5382 break;
5383 case VM_WIMG_POSTED:
5384 pte = ARM_PTE_ATTRINDX(CACHE_ATTRINDX_POSTED);
5385 pte |= ARM_PTE_NX | ARM_PTE_PNX;
5386 break;
5387 case VM_WIMG_WCOMB:
5388 pte = ARM_PTE_ATTRINDX(CACHE_ATTRINDX_WRITECOMB);
5389 pte |= ARM_PTE_NX | ARM_PTE_PNX;
5390 break;
5391 case VM_WIMG_WTHRU:
5392 pte = ARM_PTE_ATTRINDX(CACHE_ATTRINDX_WRITETHRU);
5393 #if (__ARM_VMSA__ > 7)
5394 pte |= ARM_PTE_SH(SH_OUTER_MEMORY);
5395 #else
5396 pte |= ARM_PTE_SH;
5397 #endif
5398 break;
5399 case VM_WIMG_COPYBACK:
5400 pte = ARM_PTE_ATTRINDX(CACHE_ATTRINDX_WRITEBACK);
5401 #if (__ARM_VMSA__ > 7)
5402 pte |= ARM_PTE_SH(SH_OUTER_MEMORY);
5403 #else
5404 pte |= ARM_PTE_SH;
5405 #endif
5406 break;
5407 case VM_WIMG_INNERWBACK:
5408 pte = ARM_PTE_ATTRINDX(CACHE_ATTRINDX_INNERWRITEBACK);
5409 #if (__ARM_VMSA__ > 7)
5410 pte |= ARM_PTE_SH(SH_INNER_MEMORY);
5411 #else
5412 pte |= ARM_PTE_SH;
5413 #endif
5414 break;
5415 default:
5416 pte = ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DEFAULT);
5417 #if (__ARM_VMSA__ > 7)
5418 pte |= ARM_PTE_SH(SH_OUTER_MEMORY);
5419 #else
5420 pte |= ARM_PTE_SH;
5421 #endif
5422 }
5423
5424 return pte;
5425 }
5426
5427 static kern_return_t
5428 pmap_enter_options_internal(
5429 pmap_t pmap,
5430 vm_map_address_t v,
5431 ppnum_t pn,
5432 vm_prot_t prot,
5433 vm_prot_t fault_type,
5434 unsigned int flags,
5435 boolean_t wired,
5436 unsigned int options)
5437 {
5438 pmap_paddr_t pa = ptoa(pn);
5439 pt_entry_t pte;
5440 pt_entry_t spte;
5441 pt_entry_t *pte_p;
5442 pv_entry_t *pve_p;
5443 boolean_t set_NX;
5444 boolean_t set_XO = FALSE;
5445 boolean_t refcnt_updated;
5446 unsigned int wimg_bits;
5447 boolean_t was_compressed, was_alt_compressed;
5448
5449 if ((v) & PAGE_MASK) {
5450 panic("pmap_enter_options() pmap %p v 0x%llx\n",
5451 pmap, (uint64_t)v);
5452 }
5453
5454 if ((prot & VM_PROT_EXECUTE) && (prot & VM_PROT_WRITE) && (pmap == kernel_pmap)) {
5455 panic("pmap_enter_options(): WX request on kernel_pmap");
5456 }
5457
5458 #if DEVELOPMENT || DEBUG
5459 if ((prot & VM_PROT_EXECUTE) || !nx_enabled || !pmap->nx_enabled)
5460 #else
5461 if ((prot & VM_PROT_EXECUTE))
5462 #endif
5463 set_NX = FALSE;
5464 else
5465 set_NX = TRUE;
5466
5467 #if (__ARM_VMSA__ > 7)
5468 if (prot == VM_PROT_EXECUTE) {
5469 set_XO = TRUE;
5470 }
5471 #endif
5472
5473 assert(pn != vm_page_fictitious_addr);
5474
5475 refcnt_updated = FALSE;
5476 pve_p = PV_ENTRY_NULL;
5477 was_compressed = FALSE;
5478 was_alt_compressed = FALSE;
5479
5480 PMAP_LOCK(pmap);
5481
5482 /*
5483 * Expand pmap to include this pte. Assume that
5484 * pmap is always expanded to include enough hardware
5485 * pages to map one VM page.
5486 */
5487 while ((pte_p = pmap_pte(pmap, v)) == PT_ENTRY_NULL) {
5488 /* Must unlock to expand the pmap. */
5489 PMAP_UNLOCK(pmap);
5490
5491 kern_return_t kr=pmap_expand(pmap, v, options, PMAP_TT_MAX_LEVEL);
5492
5493 if(kr) {
5494 return kr;
5495 }
5496
5497 PMAP_LOCK(pmap);
5498 }
5499
5500 if (options & PMAP_OPTIONS_NOENTER) {
5501 PMAP_UNLOCK(pmap);
5502 return KERN_SUCCESS;
5503 }
5504
5505 Pmap_enter_retry:
5506
5507 spte = *pte_p;
5508
5509 if (ARM_PTE_IS_COMPRESSED(spte)) {
5510 /*
5511 * "pmap" should be locked at this point, so this should
5512 * not race with another pmap_enter() or pmap_remove_range().
5513 */
5514 assert(pmap != kernel_pmap);
5515
5516 /* one less "compressed" */
5517 OSAddAtomic64(-1, &pmap->stats.compressed);
5518 pmap_ledger_debit(pmap, task_ledgers.internal_compressed,
5519 PAGE_SIZE);
5520
5521 was_compressed = TRUE;
5522 if (spte & ARM_PTE_COMPRESSED_ALT) {
5523 was_alt_compressed = TRUE;
5524 pmap_ledger_debit(
5525 pmap,
5526 task_ledgers.alternate_accounting_compressed,
5527 PAGE_SIZE);
5528 } else {
5529 /* was part of the footprint */
5530 pmap_ledger_debit(pmap, task_ledgers.phys_footprint, PAGE_SIZE);
5531 }
5532
5533 /* clear "compressed" marker */
5534 /* XXX is it necessary since we're about to overwrite it ? */
5535 WRITE_PTE_FAST(pte_p, ARM_PTE_TYPE_FAULT);
5536 spte = ARM_PTE_TYPE_FAULT;
5537
5538 /*
5539 * We're replacing a "compressed" marker with a valid PTE,
5540 * so no change for "refcnt".
5541 */
5542 refcnt_updated = TRUE;
5543 }
5544
5545 if ((spte != ARM_PTE_TYPE_FAULT) && (pte_to_pa(spte) != pa)) {
5546 pmap_remove_range(pmap, v, pte_p, pte_p + 1, 0);
5547 PMAP_UPDATE_TLBS(pmap, v, v + PAGE_SIZE);
5548 }
5549
5550 pte = pa_to_pte(pa) | ARM_PTE_TYPE;
5551
5552 /* Don't bother tracking wiring for kernel PTEs. We use ARM_PTE_WIRED to track
5553 * wired memory statistics for user pmaps, but kernel PTEs are assumed
5554 * to be wired in nearly all cases. For VM layer functionality, the wired
5555 * count in vm_page_t is sufficient. */
5556 if (wired && pmap != kernel_pmap)
5557 pte |= ARM_PTE_WIRED;
5558
5559 #if (__ARM_VMSA__ == 7)
5560 if (set_NX)
5561 pte |= ARM_PTE_NX;
5562 #else
5563 if (set_NX)
5564 pte |= ARM_PTE_NX | ARM_PTE_PNX;
5565 else {
5566 if (pmap == kernel_pmap) {
5567 pte |= ARM_PTE_NX;
5568 } else {
5569 pte |= ARM_PTE_PNX;
5570 }
5571 }
5572 #endif
5573
5574 if ((flags & (VM_WIMG_MASK | VM_WIMG_USE_DEFAULT)))
5575 wimg_bits = (flags & (VM_WIMG_MASK | VM_WIMG_USE_DEFAULT));
5576 else
5577 wimg_bits = pmap_cache_attributes(pn);
5578
5579 pte |= wimg_to_pte(wimg_bits);
5580
5581 if (pmap == kernel_pmap) {
5582 if (prot & VM_PROT_WRITE) {
5583 pte |= ARM_PTE_AP(AP_RWNA);
5584 pa_set_bits(pa, PP_ATTR_MODIFIED | PP_ATTR_REFERENCED);
5585 } else {
5586 pte |= ARM_PTE_AP(AP_RONA);
5587 pa_set_bits(pa, PP_ATTR_REFERENCED);
5588 }
5589 #if (__ARM_VMSA__ == 7)
5590 if ((_COMM_PAGE_BASE_ADDRESS <= v) && (v < _COMM_PAGE_BASE_ADDRESS + _COMM_PAGE_AREA_LENGTH))
5591 pte = (pte & ~(ARM_PTE_APMASK)) | ARM_PTE_AP(AP_RORO);
5592 #endif
5593 } else {
5594 if (!(pmap->nested)) {
5595 pte |= ARM_PTE_NG;
5596 } else if ((pmap->nested_region_asid_bitmap)
5597 && (v >= pmap->nested_region_subord_addr)
5598 && (v < (pmap->nested_region_subord_addr+pmap->nested_region_size))) {
5599
5600 unsigned int index = (unsigned int)((v - pmap->nested_region_subord_addr) >> ARM_TT_TWIG_SHIFT);
5601
5602 if ((pmap->nested_region_asid_bitmap)
5603 && testbit(index, (int *)pmap->nested_region_asid_bitmap))
5604 pte |= ARM_PTE_NG;
5605 }
5606 #if MACH_ASSERT
5607 if (pmap->nested_pmap != NULL) {
5608 vm_map_address_t nest_vaddr;
5609 pt_entry_t *nest_pte_p;
5610
5611 nest_vaddr = v - pmap->nested_region_grand_addr + pmap->nested_region_subord_addr;
5612
5613 if ((nest_vaddr >= pmap->nested_region_subord_addr)
5614 && (nest_vaddr < (pmap->nested_region_subord_addr+pmap->nested_region_size))
5615 && ((nest_pte_p = pmap_pte(pmap->nested_pmap, nest_vaddr)) != PT_ENTRY_NULL)
5616 && (*nest_pte_p != ARM_PTE_TYPE_FAULT)
5617 && (!ARM_PTE_IS_COMPRESSED(*nest_pte_p))
5618 && (((*nest_pte_p) & ARM_PTE_NG) != ARM_PTE_NG)) {
5619 unsigned int index = (unsigned int)((v - pmap->nested_region_subord_addr) >> ARM_TT_TWIG_SHIFT);
5620
5621 if ((pmap->nested_pmap->nested_region_asid_bitmap)
5622 && !testbit(index, (int *)pmap->nested_pmap->nested_region_asid_bitmap)) {
5623
5624 panic("pmap_enter(): Global attribute conflict nest_pte_p=%p pmap=%p v=0x%llx spte=0x%llx \n",
5625 nest_pte_p, pmap, (uint64_t)v, (uint64_t)*nest_pte_p);
5626 }
5627 }
5628
5629 }
5630 #endif
5631 if (prot & VM_PROT_WRITE) {
5632
5633 if (pa_valid(pa) && (!pa_test_bits(pa, PP_ATTR_MODIFIED))) {
5634 if (fault_type & VM_PROT_WRITE) {
5635 if (set_XO)
5636 pte |= ARM_PTE_AP(AP_RWNA);
5637 else
5638 pte |= ARM_PTE_AP(AP_RWRW);
5639 pa_set_bits(pa, PP_ATTR_REFERENCED | PP_ATTR_MODIFIED);
5640 } else {
5641 if (set_XO)
5642 pte |= ARM_PTE_AP(AP_RONA);
5643 else
5644 pte |= ARM_PTE_AP(AP_RORO);
5645 pa_set_bits(pa, PP_ATTR_REFERENCED);
5646 pte_set_ffr(pte, 1);
5647 }
5648 } else {
5649 if (set_XO)
5650 pte |= ARM_PTE_AP(AP_RWNA);
5651 else
5652 pte |= ARM_PTE_AP(AP_RWRW);
5653 pa_set_bits(pa, PP_ATTR_REFERENCED);
5654 }
5655 } else {
5656
5657 if (set_XO)
5658 pte |= ARM_PTE_AP(AP_RONA);
5659 else
5660 pte |= ARM_PTE_AP(AP_RORO);
5661 pa_set_bits(pa, PP_ATTR_REFERENCED);
5662 }
5663 }
5664
5665 pte |= ARM_PTE_AF;
5666
5667 volatile uint16_t *refcnt = NULL;
5668 if (pmap != kernel_pmap) {
5669 refcnt = &(ptep_get_ptd(pte_p)->pt_cnt[ARM_PT_DESC_INDEX(pte_p)].refcnt);
5670 /* Mark the PT page active to keep it from being reclaimed. We need this because
5671 * we may drop the PVH and pmap locks later in pmap_enter() if we need to allocate
5672 * a new PV entry. Note that setting this high bit (0x4000) can temporarily
5673 * prevent the refcount underflow checks in pmap_page_protect() and pmap_remove() from
5674 * working. If an underflow should happen during this window, we'll instead get a
5675 * refcount along the lines of 0x3FFF, which will produce a later panic on non-zero
5676 * refcount in pmap_pages_reclaim() or pmap_tt_deallocate(). */
5677 OSBitOrAtomic16(PT_DESC_REFCOUNT, refcnt);
5678 if (!refcnt_updated) {
5679 OSAddAtomic16(1, (volatile int16_t*)refcnt);
5680 refcnt_updated = TRUE;
5681 }
5682 }
5683
5684 if (pa_valid(pa)) {
5685 pv_entry_t **pv_h;
5686 int pai;
5687 boolean_t is_altacct, is_internal;
5688
5689 is_internal = FALSE;
5690 is_altacct = FALSE;
5691
5692 pai = (int)pa_index(pa);
5693 pv_h = pai_to_pvh(pai);
5694
5695 LOCK_PVH(pai);
5696 Pmap_enter_loop:
5697
5698 if (pte == *pte_p) {
5699 /*
5700 * This pmap_enter operation has been completed by another thread
5701 * undo refcnt on pt and return
5702 */
5703 if (refcnt != NULL) {
5704 assert(refcnt_updated);
5705 if (OSAddAtomic16(-1, (volatile int16_t*)refcnt) <= (int16_t)PT_DESC_REFCOUNT)
5706 panic("pmap_enter(): over-release of ptdp %p for pte %p\n", ptep_get_ptd(pte_p), pte_p);
5707 }
5708 UNLOCK_PVH(pai);
5709 goto Pmap_enter_return;
5710 } else if (pte_to_pa(*pte_p) == pa) {
5711 if (refcnt != NULL) {
5712 assert(refcnt_updated);
5713 if (OSAddAtomic16(-1, (volatile int16_t*)refcnt) <= (int16_t)PT_DESC_REFCOUNT)
5714 panic("pmap_enter(): over-release of ptdp %p for pte %p\n", ptep_get_ptd(pte_p), pte_p);
5715 }
5716 pmap_enter_pte(pmap, pte_p, pte, v);
5717 UNLOCK_PVH(pai);
5718 goto Pmap_enter_return;
5719 } else if (*pte_p != ARM_PTE_TYPE_FAULT) {
5720 /*
5721 * pte has been modified by another thread
5722 * hold refcnt on pt and retry pmap_enter operation
5723 */
5724 UNLOCK_PVH(pai);
5725 goto Pmap_enter_retry;
5726 }
5727 if (pvh_test_type(pv_h, PVH_TYPE_NULL)) {
5728 pvh_update_head(pv_h, pte_p, PVH_TYPE_PTEP);
5729 /* 1st mapping: see what kind of page it is */
5730 if (options & PMAP_OPTIONS_INTERNAL) {
5731 SET_INTERNAL_PAGE(pai);
5732 } else {
5733 CLR_INTERNAL_PAGE(pai);
5734 }
5735 if ((options & PMAP_OPTIONS_INTERNAL) &&
5736 (options & PMAP_OPTIONS_REUSABLE)) {
5737 SET_REUSABLE_PAGE(pai);
5738 } else {
5739 CLR_REUSABLE_PAGE(pai);
5740 }
5741 if (pmap != kernel_pmap &&
5742 ((options & PMAP_OPTIONS_ALT_ACCT) ||
5743 PMAP_FOOTPRINT_SUSPENDED(pmap)) &&
5744 IS_INTERNAL_PAGE(pai)) {
5745 /*
5746 * Make a note to ourselves that this mapping is using alternative
5747 * accounting. We'll need this in order to know which ledger to
5748 * debit when the mapping is removed.
5749 *
5750 * The altacct bit must be set while the pv head is locked. Defer
5751 * the ledger accounting until after we've dropped the lock.
5752 */
5753 SET_ALTACCT_PAGE(pai, PV_ENTRY_NULL);
5754 is_altacct = TRUE;
5755 } else {
5756 CLR_ALTACCT_PAGE(pai, PV_ENTRY_NULL);
5757 }
5758 } else {
5759 if (pvh_test_type(pv_h, PVH_TYPE_PTEP)) {
5760 pt_entry_t *pte1_p;
5761
5762 /*
5763 * convert pvh list from PVH_TYPE_PTEP to PVH_TYPE_PVEP
5764 */
5765 pte1_p = pvh_ptep(pv_h);
5766 if((pve_p == PV_ENTRY_NULL) && (!pv_alloc(pmap, pai, &pve_p))) {
5767 goto Pmap_enter_loop;
5768 }
5769 pve_set_ptep(pve_p, pte1_p);
5770 pve_p->pve_next = PV_ENTRY_NULL;
5771
5772 if (IS_ALTACCT_PAGE(pai, PV_ENTRY_NULL)) {
5773 /*
5774 * transfer "altacct" from
5775 * pp_attr to this pve
5776 */
5777 CLR_ALTACCT_PAGE(pai, PV_ENTRY_NULL);
5778 SET_ALTACCT_PAGE(pai, pve_p);
5779 }
5780 pvh_update_head(pv_h, pve_p, PVH_TYPE_PVEP);
5781 pve_p = PV_ENTRY_NULL;
5782 }
5783 /*
5784 * Set up pv_entry for this new mapping and then
5785 * add it to the list for this physical page.
5786 */
5787 if((pve_p == PV_ENTRY_NULL) && (!pv_alloc(pmap, pai, &pve_p))) {
5788 goto Pmap_enter_loop;
5789 }
5790 pve_set_ptep(pve_p, pte_p);
5791 pve_p->pve_next = PV_ENTRY_NULL;
5792
5793 pvh_add(pv_h, pve_p);
5794
5795 if (pmap != kernel_pmap &&
5796 ((options & PMAP_OPTIONS_ALT_ACCT) ||
5797 PMAP_FOOTPRINT_SUSPENDED(pmap)) &&
5798 IS_INTERNAL_PAGE(pai)) {
5799 /*
5800 * Make a note to ourselves that this
5801 * mapping is using alternative
5802 * accounting. We'll need this in order
5803 * to know which ledger to debit when
5804 * the mapping is removed.
5805 *
5806 * The altacct bit must be set while
5807 * the pv head is locked. Defer the
5808 * ledger accounting until after we've
5809 * dropped the lock.
5810 */
5811 SET_ALTACCT_PAGE(pai, pve_p);
5812 is_altacct = TRUE;
5813 }
5814
5815 pve_p = PV_ENTRY_NULL;
5816 }
5817
5818 pmap_enter_pte(pmap, pte_p, pte, v);
5819
5820 if (pmap != kernel_pmap) {
5821 if (IS_REUSABLE_PAGE(pai) &&
5822 !is_altacct) {
5823 assert(IS_INTERNAL_PAGE(pai));
5824 OSAddAtomic(+1, &pmap->stats.reusable);
5825 PMAP_STATS_PEAK(pmap->stats.reusable);
5826 } else if (IS_INTERNAL_PAGE(pai)) {
5827 OSAddAtomic(+1, &pmap->stats.internal);
5828 PMAP_STATS_PEAK(pmap->stats.internal);
5829 is_internal = TRUE;
5830 } else {
5831 OSAddAtomic(+1, &pmap->stats.external);
5832 PMAP_STATS_PEAK(pmap->stats.external);
5833 }
5834 }
5835
5836 UNLOCK_PVH(pai);
5837
5838 if (pmap != kernel_pmap) {
5839 pmap_ledger_credit(pmap, task_ledgers.phys_mem, PAGE_SIZE);
5840
5841 if (is_internal) {
5842 /*
5843 * Make corresponding adjustments to
5844 * phys_footprint statistics.
5845 */
5846 pmap_ledger_credit(pmap, task_ledgers.internal, PAGE_SIZE);
5847 if (is_altacct) {
5848 /*
5849 * If this page is internal and
5850 * in an IOKit region, credit
5851 * the task's total count of
5852 * dirty, internal IOKit pages.
5853 * It should *not* count towards
5854 * the task's total physical
5855 * memory footprint, because
5856 * this entire region was
5857 * already billed to the task
5858 * at the time the mapping was
5859 * created.
5860 *
5861 * Put another way, this is
5862 * internal++ and
5863 * alternate_accounting++, so
5864 * net effect on phys_footprint
5865 * is 0. That means: don't
5866 * touch phys_footprint here.
5867 */
5868 pmap_ledger_credit(pmap, task_ledgers.alternate_accounting, PAGE_SIZE);
5869 } else {
5870 pmap_ledger_credit(pmap, task_ledgers.phys_footprint, PAGE_SIZE);
5871 }
5872 }
5873 }
5874
5875 OSAddAtomic(1, (SInt32 *) &pmap->stats.resident_count);
5876 if (pmap->stats.resident_count > pmap->stats.resident_max)
5877 pmap->stats.resident_max = pmap->stats.resident_count;
5878 } else {
5879 pmap_enter_pte(pmap, pte_p, pte, v);
5880 }
5881
5882 Pmap_enter_return:
5883
5884 #if CONFIG_PGTRACE
5885 if (pgtrace_enabled) {
5886 // Clone and invalidate original mapping if eligible
5887 for (int i = 0; i < PAGE_RATIO; i++) {
5888 pmap_pgtrace_enter_clone(pmap, v + ARM_PGBYTES*i, 0, 0);
5889 }
5890 }
5891 #endif
5892
5893 if (pve_p != PV_ENTRY_NULL)
5894 pv_free(pve_p);
5895
5896 if (refcnt != NULL)
5897 OSBitAndAtomic16(~PT_DESC_REFCOUNT, refcnt); // clear active marker
5898 PMAP_UNLOCK(pmap);
5899
5900 return KERN_SUCCESS;
5901 }
5902
5903 kern_return_t
5904 pmap_enter_options(
5905 pmap_t pmap,
5906 vm_map_address_t v,
5907 ppnum_t pn,
5908 vm_prot_t prot,
5909 vm_prot_t fault_type,
5910 unsigned int flags,
5911 boolean_t wired,
5912 unsigned int options,
5913 __unused void *arg)
5914 {
5915 kern_return_t kr = KERN_FAILURE;
5916
5917 PMAP_TRACE(PMAP_CODE(PMAP__ENTER) | DBG_FUNC_START,
5918 VM_KERNEL_ADDRHIDE(pmap), VM_KERNEL_ADDRHIDE(v), pn, prot);
5919
5920 kr = pmap_enter_options_internal(pmap, v, pn, prot, fault_type, flags, wired, options);
5921
5922 PMAP_TRACE(PMAP_CODE(PMAP__ENTER) | DBG_FUNC_END, kr);
5923
5924 return kr;
5925 }
5926
5927 /*
5928 * Routine: pmap_change_wiring
5929 * Function: Change the wiring attribute for a map/virtual-address
5930 * pair.
5931 * In/out conditions:
5932 * The mapping must already exist in the pmap.
5933 */
5934 static void
5935 pmap_change_wiring_internal(
5936 pmap_t pmap,
5937 vm_map_address_t v,
5938 boolean_t wired)
5939 {
5940 pt_entry_t *pte_p;
5941 pmap_paddr_t pa;
5942
5943 /* Don't bother tracking wiring for kernel PTEs. We use ARM_PTE_WIRED to track
5944 * wired memory statistics for user pmaps, but kernel PTEs are assumed
5945 * to be wired in nearly all cases. For VM layer functionality, the wired
5946 * count in vm_page_t is sufficient. */
5947 if (pmap == kernel_pmap) {
5948 return;
5949 }
5950
5951 PMAP_LOCK(pmap);
5952 pte_p = pmap_pte(pmap, v);
5953 assert(pte_p != PT_ENTRY_NULL);
5954 pa = pte_to_pa(*pte_p);
5955 if (pa_valid(pa))
5956 LOCK_PVH((int)pa_index(pa));
5957
5958 if (wired && !pte_is_wired(*pte_p)) {
5959 pte_set_wired(pte_p, wired);
5960 OSAddAtomic(+1, (SInt32 *) &pmap->stats.wired_count);
5961 pmap_ledger_credit(pmap, task_ledgers.wired_mem, PAGE_SIZE);
5962 } else if (!wired && pte_is_wired(*pte_p)) {
5963 assert(pmap->stats.wired_count >= 1);
5964 pte_set_wired(pte_p, wired);
5965 OSAddAtomic(-1, (SInt32 *) &pmap->stats.wired_count);
5966 pmap_ledger_debit(pmap, task_ledgers.wired_mem, PAGE_SIZE);
5967 }
5968
5969 if (pa_valid(pa))
5970 UNLOCK_PVH((int)pa_index(pa));
5971
5972 PMAP_UNLOCK(pmap);
5973 }
5974
5975 void
5976 pmap_change_wiring(
5977 pmap_t pmap,
5978 vm_map_address_t v,
5979 boolean_t wired)
5980 {
5981 pmap_change_wiring_internal(pmap, v, wired);
5982 }
5983
5984 static ppnum_t
5985 pmap_find_phys_internal(
5986 pmap_t pmap,
5987 addr64_t va)
5988 {
5989 ppnum_t ppn=0;
5990
5991 if (pmap != kernel_pmap) {
5992 PMAP_LOCK(pmap);
5993 }
5994
5995 ppn = pmap_vtophys(pmap, va);
5996
5997 if (pmap != kernel_pmap) {
5998 PMAP_UNLOCK(pmap);
5999 }
6000
6001 return ppn;
6002 }
6003
6004 ppnum_t
6005 pmap_find_phys(
6006 pmap_t pmap,
6007 addr64_t va)
6008 {
6009 pmap_paddr_t pa=0;
6010
6011 if (pmap == kernel_pmap)
6012 pa = mmu_kvtop(va);
6013 else if ((current_thread()->map) && (pmap == vm_map_pmap(current_thread()->map)))
6014 pa = mmu_uvtop(va);
6015
6016 if (pa) return (ppnum_t)(pa >> PAGE_SHIFT);
6017
6018 if (not_in_kdp) {
6019 return pmap_find_phys_internal(pmap, va);
6020 } else {
6021 return pmap_vtophys(pmap, va);
6022 }
6023 }
6024
6025 pmap_paddr_t
6026 kvtophys(
6027 vm_offset_t va)
6028 {
6029 pmap_paddr_t pa;
6030
6031 pa = mmu_kvtop(va);
6032 if (pa) return pa;
6033 pa = ((pmap_paddr_t)pmap_vtophys(kernel_pmap, va)) << PAGE_SHIFT;
6034 if (pa)
6035 pa |= (va & PAGE_MASK);
6036
6037 return ((pmap_paddr_t)pa);
6038 }
6039
6040 ppnum_t
6041 pmap_vtophys(
6042 pmap_t pmap,
6043 addr64_t va)
6044 {
6045 if ((va < pmap->min) || (va >= pmap->max)) {
6046 return 0;
6047 }
6048
6049 #if (__ARM_VMSA__ == 7)
6050 tt_entry_t *tte_p, tte;
6051 pt_entry_t *pte_p;
6052 ppnum_t ppn;
6053
6054 tte_p = pmap_tte(pmap, va);
6055 if (tte_p == (tt_entry_t *) NULL)
6056 return (ppnum_t) 0;
6057
6058 tte = *tte_p;
6059 if ((tte & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_TABLE) {
6060 pte_p = (pt_entry_t *) ttetokv(tte) + ptenum(va);
6061 ppn = (ppnum_t) atop(pte_to_pa(*pte_p) | (va & ARM_PGMASK));
6062 #if DEVELOPMENT || DEBUG
6063 if (ppn != 0 &&
6064 ARM_PTE_IS_COMPRESSED(*pte_p)) {
6065 panic("pmap_vtophys(%p,0x%llx): compressed pte_p=%p 0x%llx with ppn=0x%x\n",
6066 pmap, va, pte_p, (uint64_t) (*pte_p), ppn);
6067 }
6068 #endif /* DEVELOPMENT || DEBUG */
6069 } else if ((tte & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_BLOCK)
6070 if ((tte & ARM_TTE_BLOCK_SUPER) == ARM_TTE_BLOCK_SUPER)
6071 ppn = (ppnum_t) atop(suptte_to_pa(tte) | (va & ARM_TT_L1_SUPER_OFFMASK));
6072 else
6073 ppn = (ppnum_t) atop(sectte_to_pa(tte) | (va & ARM_TT_L1_BLOCK_OFFMASK));
6074 else
6075 ppn = 0;
6076 #else
6077 tt_entry_t *ttp;
6078 tt_entry_t tte;
6079 ppnum_t ppn=0;
6080
6081 /* Level 0 currently unused */
6082
6083 #if __ARM64_TWO_LEVEL_PMAP__
6084 /* We have no L1 entry; go straight to the L2 entry */
6085 ttp = pmap_tt2e(pmap, va);
6086 tte = *ttp;
6087 #else
6088 /* Get first-level (1GB) entry */
6089 ttp = pmap_tt1e(pmap, va);
6090 tte = *ttp;
6091 if ((tte & (ARM_TTE_TYPE_MASK | ARM_TTE_VALID)) != (ARM_TTE_TYPE_TABLE | ARM_TTE_VALID))
6092 return (ppn);
6093
6094 tte = ((tt_entry_t*) phystokv(tte & ARM_TTE_TABLE_MASK))[tt2_index(pmap, va)];
6095 #endif
6096 if ((tte & ARM_TTE_VALID) != (ARM_TTE_VALID))
6097 return (ppn);
6098
6099 if ((tte & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_BLOCK) {
6100 ppn = (ppnum_t) atop((tte & ARM_TTE_BLOCK_L2_MASK)| (va & ARM_TT_L2_OFFMASK));
6101 return(ppn);
6102 }
6103 tte = ((tt_entry_t*) phystokv(tte & ARM_TTE_TABLE_MASK))[tt3_index(pmap, va)];
6104 ppn = (ppnum_t) atop((tte & ARM_PTE_MASK)| (va & ARM_TT_L3_OFFMASK));
6105 #endif
6106
6107 return ppn;
6108 }
6109
6110 static vm_offset_t
6111 pmap_extract_internal(
6112 pmap_t pmap,
6113 vm_map_address_t va)
6114 {
6115 pmap_paddr_t pa=0;
6116 ppnum_t ppn=0;
6117
6118 if (pmap == NULL) {
6119 return 0;
6120 }
6121
6122 PMAP_LOCK(pmap);
6123
6124 ppn = pmap_vtophys(pmap, va);
6125
6126 if (ppn != 0)
6127 pa = ptoa(ppn)| ((va) & PAGE_MASK);
6128
6129 PMAP_UNLOCK(pmap);
6130
6131 return pa;
6132 }
6133
6134 /*
6135 * Routine: pmap_extract
6136 * Function:
6137 * Extract the physical page address associated
6138 * with the given map/virtual_address pair.
6139 *
6140 */
6141 vm_offset_t
6142 pmap_extract(
6143 pmap_t pmap,
6144 vm_map_address_t va)
6145 {
6146 pmap_paddr_t pa=0;
6147
6148 if (pmap == kernel_pmap)
6149 pa = mmu_kvtop(va);
6150 else if (pmap == vm_map_pmap(current_thread()->map))
6151 pa = mmu_uvtop(va);
6152
6153 if (pa) return pa;
6154
6155 return pmap_extract_internal(pmap, va);
6156 }
6157
6158 /*
6159 * pmap_init_pte_page - Initialize a page table page.
6160 */
6161 void
6162 pmap_init_pte_page(
6163 pmap_t pmap,
6164 pt_entry_t *pte_p,
6165 vm_offset_t va,
6166 unsigned int ttlevel,
6167 boolean_t alloc_ptd)
6168 {
6169 pt_desc_t *ptdp;
6170
6171 ptdp = *(pt_desc_t **)pai_to_pvh(pa_index((((vm_offset_t)pte_p) - gVirtBase + gPhysBase)));
6172
6173 if (ptdp == NULL) {
6174 if (alloc_ptd) {
6175 /*
6176 * This path should only be invoked from arm_vm_init. If we are emulating 16KB pages
6177 * on 4KB hardware, we may already have allocated a page table descriptor for a
6178 * bootstrap request, so we check for an existing PTD here.
6179 */
6180 ptdp = ptd_alloc(pmap);
6181 *(pt_desc_t **)pai_to_pvh(pa_index((((vm_offset_t)pte_p) - gVirtBase + gPhysBase))) = ptdp;
6182 } else {
6183 panic("pmap_init_pte_page(): pte_p %p\n", pte_p);
6184 }
6185 }
6186
6187 pmap_init_pte_page_internal(pmap, pte_p, va, ttlevel, &ptdp);
6188 }
6189
6190 /*
6191 * pmap_init_pte_page_internal - Initialize page table page and page table descriptor
6192 */
6193 void
6194 pmap_init_pte_page_internal(
6195 pmap_t pmap,
6196 pt_entry_t *pte_p,
6197 vm_offset_t va,
6198 unsigned int ttlevel,
6199 pt_desc_t **ptdp)
6200 {
6201 bzero(pte_p, ARM_PGBYTES);
6202 // below barrier ensures the page zeroing is visible to PTW before
6203 // it is linked to the PTE of previous level
6204 __asm__ volatile("DMB ST" : : : "memory");
6205 ptd_init(*ptdp, pmap, va, ttlevel, pte_p);
6206 }
6207
6208 /*
6209 * pmap_init_pte_static_page - for static mappings to a known contiguous range of pa's
6210 * Called from arm_vm_init().
6211 */
6212 void
6213 pmap_init_pte_static_page(
6214 __unused pmap_t pmap,
6215 pt_entry_t * pte_p,
6216 pmap_paddr_t pa)
6217 {
6218 #if (__ARM_VMSA__ == 7)
6219 unsigned int i;
6220 pt_entry_t *pte_cur;
6221
6222 for (i = 0, pte_cur = pte_p;
6223 i < (ARM_PGBYTES / sizeof(*pte_p));
6224 i++, pa += PAGE_SIZE) {
6225 if (pa >= avail_end) {
6226 /* We don't want to map memory xnu does not own through this routine. */
6227 break;
6228 }
6229
6230 *pte_cur = pa_to_pte(pa)
6231 | ARM_PTE_TYPE | ARM_PTE_AF | ARM_PTE_SH | ARM_PTE_AP(AP_RONA)
6232 | ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DEFAULT);
6233 pte_cur++;
6234 }
6235 #else
6236 unsigned int i;
6237 pt_entry_t *pte_cur;
6238 pt_entry_t template;
6239
6240 template = ARM_PTE_TYPE | ARM_PTE_AF | ARM_PTE_SH(SH_OUTER_MEMORY) | ARM_PTE_AP(AP_RONA) | ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DEFAULT) | ARM_PTE_NX;
6241
6242 for (i = 0, pte_cur = pte_p;
6243 i < (ARM_PGBYTES / sizeof(*pte_p));
6244 i++, pa += PAGE_SIZE) {
6245 if (pa >= avail_end) {
6246 /* We don't want to map memory xnu does not own through this routine. */
6247 break;
6248 }
6249
6250 /* TEST_PAGE_RATIO_4 may be pre-processor defined to 0 */
6251 __unreachable_ok_push
6252 if (TEST_PAGE_RATIO_4) {
6253 *pte_cur = pa_to_pte(pa) | template;
6254 *(pte_cur+1) = pa_to_pte(pa+0x1000) | template;
6255 *(pte_cur+2) = pa_to_pte(pa+0x2000) | template;
6256 *(pte_cur+3) = pa_to_pte(pa+0x3000) | template;
6257 pte_cur += 4;
6258 } else {
6259 *pte_cur = pa_to_pte(pa) | template;
6260 pte_cur++;
6261 }
6262 __unreachable_ok_pop
6263 }
6264 #endif
6265 bzero(pte_cur, ARM_PGBYTES - ((vm_offset_t)pte_cur - (vm_offset_t)pte_p));
6266 }
6267
6268
6269 /*
6270 * Routine: pmap_expand
6271 *
6272 * Expands a pmap to be able to map the specified virtual address.
6273 *
6274 * Allocates new memory for the default (COARSE) translation table
6275 * entry, initializes all the pte entries to ARM_PTE_TYPE_FAULT and
6276 * also allocates space for the corresponding pv entries.
6277 *
6278 * Nothing should be locked.
6279 */
6280 static kern_return_t
6281 pmap_expand(
6282 pmap_t pmap,
6283 vm_map_address_t v,
6284 unsigned int options,
6285 unsigned int level)
6286 {
6287 #if (__ARM_VMSA__ == 7)
6288 vm_offset_t pa;
6289 tt_entry_t *tte_p;
6290 tt_entry_t *tt_p;
6291 unsigned int i;
6292
6293
6294 while (tte_index(pmap, v) >= pmap->tte_index_max) {
6295 tte_p = pmap_tt1_allocate(pmap, 2*ARM_PGBYTES, ((options & PMAP_OPTIONS_NOWAIT)? PMAP_TT_ALLOCATE_NOWAIT : 0));
6296 if (tte_p == (tt_entry_t *)0)
6297 return KERN_RESOURCE_SHORTAGE;
6298
6299 PMAP_LOCK(pmap);
6300 if (pmap->tte_index_max > NTTES) {
6301 pmap_tt1_deallocate(pmap, tte_p, 2*ARM_PGBYTES, PMAP_TT_DEALLOCATE_NOBLOCK);
6302 PMAP_UNLOCK(pmap);
6303 break;
6304 }
6305
6306 simple_lock(&pmap->tt1_lock);
6307 for (i = 0; i < pmap->tte_index_max; i++)
6308 tte_p[i] = pmap->tte[i];
6309 for (i = NTTES; i < 2*NTTES; i++)
6310 tte_p[i] = ARM_TTE_TYPE_FAULT;
6311
6312 pmap->prev_tte = pmap->tte;
6313 pmap->tte = tte_p;
6314 pmap->ttep = ml_static_vtop((vm_offset_t)pmap->tte);
6315 #ifndef __ARM_L1_PTW__
6316 CleanPoU_DcacheRegion((vm_offset_t) pmap->tte, 2*NTTES * sizeof(tt_entry_t));
6317 #else
6318 __builtin_arm_dsb(DSB_ISH);
6319 #endif
6320 pmap->tte_index_max = 2*NTTES;
6321 pmap->stamp = hw_atomic_add(&pmap_stamp, 1);
6322
6323 for (i = 0; i < NTTES; i++)
6324 pmap->prev_tte[i] = ARM_TTE_TYPE_FAULT;
6325 #ifndef __ARM_L1_PTW__
6326 CleanPoU_DcacheRegion((vm_offset_t) pmap->prev_tte, NTTES * sizeof(tt_entry_t));
6327 #else
6328 __builtin_arm_dsb(DSB_ISH);
6329 #endif
6330
6331 simple_unlock(&pmap->tt1_lock);
6332 PMAP_UNLOCK(pmap);
6333 pmap_set_pmap(pmap, current_thread());
6334
6335 }
6336
6337 if (level == 1)
6338 return (KERN_SUCCESS);
6339
6340 {
6341 tt_entry_t *tte_next_p;
6342
6343 PMAP_LOCK(pmap);
6344 pa = 0;
6345 if (pmap_pte(pmap, v) != PT_ENTRY_NULL) {
6346 PMAP_UNLOCK(pmap);
6347 return (KERN_SUCCESS);
6348 }
6349 tte_p = &pmap->tte[ttenum(v & ~ARM_TT_L1_PT_OFFMASK)];
6350 for (i = 0, tte_next_p = tte_p; i<4; i++) {
6351 if (tte_to_pa(*tte_next_p)) {
6352 pa = tte_to_pa(*tte_next_p);
6353 break;
6354 }
6355 tte_next_p++;
6356 }
6357 pa = pa & ~PAGE_MASK;
6358 if (pa) {
6359 tte_p = &pmap->tte[ttenum(v)];
6360 *tte_p = pa_to_tte(pa) | (((v >> ARM_TT_L1_SHIFT) & 0x3) << 10) | ARM_TTE_TYPE_TABLE;
6361 #ifndef __ARM_L1_PTW__
6362 CleanPoU_DcacheRegion((vm_offset_t) tte_p, sizeof(tt_entry_t));
6363 #endif
6364 PMAP_UNLOCK(pmap);
6365 return (KERN_SUCCESS);
6366 }
6367 PMAP_UNLOCK(pmap);
6368 }
6369 v = v & ~ARM_TT_L1_PT_OFFMASK;
6370
6371
6372 while (pmap_pte(pmap, v) == PT_ENTRY_NULL) {
6373 /*
6374 * Allocate a VM page for the level 2 page table entries.
6375 */
6376 while (pmap_tt_allocate(pmap, &tt_p, PMAP_TT_L2_LEVEL, ((options & PMAP_TT_ALLOCATE_NOWAIT)? PMAP_PAGES_ALLOCATE_NOWAIT : 0)) != KERN_SUCCESS) {
6377 if(options & PMAP_OPTIONS_NOWAIT) {
6378 return KERN_RESOURCE_SHORTAGE;
6379 }
6380 VM_PAGE_WAIT();
6381 }
6382
6383 PMAP_LOCK(pmap);
6384 /*
6385 * See if someone else expanded us first
6386 */
6387 if (pmap_pte(pmap, v) == PT_ENTRY_NULL) {
6388 tt_entry_t *tte_next_p;
6389
6390 pmap_init_pte_page(pmap, (pt_entry_t *) tt_p, v, PMAP_TT_L2_LEVEL, FALSE);
6391 pa = kvtophys((vm_offset_t)tt_p);
6392 #ifndef __ARM_L1_PTW__
6393 CleanPoU_DcacheRegion((vm_offset_t) phystokv(pa), PAGE_SIZE);
6394 #endif
6395 tte_p = &pmap->tte[ttenum(v)];
6396 for (i = 0, tte_next_p = tte_p; i<4; i++) {
6397 *tte_next_p = pa_to_tte(pa) | ARM_TTE_TYPE_TABLE;
6398 tte_next_p++;
6399 pa = pa +0x400;
6400 }
6401 #ifndef __ARM_L1_PTW__
6402 CleanPoU_DcacheRegion((vm_offset_t) tte_p, 4*sizeof(tt_entry_t));
6403 #endif
6404 pa = 0x0ULL;
6405 tt_p = (tt_entry_t *)NULL;
6406 }
6407 PMAP_UNLOCK(pmap);
6408 if (tt_p != (tt_entry_t *)NULL) {
6409 pmap_tt_deallocate(pmap, tt_p, PMAP_TT_L2_LEVEL);
6410 tt_p = (tt_entry_t *)NULL;
6411 }
6412 }
6413 return (KERN_SUCCESS);
6414 #else
6415 pmap_paddr_t pa;
6416 #if __ARM64_TWO_LEVEL_PMAP__
6417 /* If we are using a two level page table, we'll start at L2. */
6418 unsigned int ttlevel = 2;
6419 #else
6420 /* Otherwise, we start at L1 (we use 3 levels by default). */
6421 unsigned int ttlevel = 1;
6422 #endif
6423 tt_entry_t *tte_p;
6424 tt_entry_t *tt_p;
6425
6426 pa = 0x0ULL;
6427 tt_p = (tt_entry_t *)NULL;
6428
6429 for (; ttlevel < level; ttlevel++) {
6430
6431 PMAP_LOCK(pmap);
6432
6433 if (ttlevel == 1) {
6434 if ((pmap_tt2e(pmap, v) == PT_ENTRY_NULL)) {
6435 PMAP_UNLOCK(pmap);
6436 while (pmap_tt_allocate(pmap, &tt_p, PMAP_TT_L2_LEVEL, ((options & PMAP_TT_ALLOCATE_NOWAIT)? PMAP_PAGES_ALLOCATE_NOWAIT : 0)) != KERN_SUCCESS) {
6437 if(options & PMAP_OPTIONS_NOWAIT) {
6438 return KERN_RESOURCE_SHORTAGE;
6439 }
6440 VM_PAGE_WAIT();
6441 }
6442 PMAP_LOCK(pmap);
6443 if ((pmap_tt2e(pmap, v) == PT_ENTRY_NULL)) {
6444 pmap_init_pte_page(pmap, (pt_entry_t *) tt_p, v, PMAP_TT_L2_LEVEL, FALSE);
6445 pa = kvtophys((vm_offset_t)tt_p);
6446 tte_p = pmap_tt1e( pmap, v);
6447 *tte_p = (pa & ARM_TTE_TABLE_MASK) | ARM_TTE_TYPE_TABLE | ARM_TTE_VALID;
6448 pa = 0x0ULL;
6449 tt_p = (tt_entry_t *)NULL;
6450 if ((pmap == kernel_pmap) && (VM_MIN_KERNEL_ADDRESS < 0x00000000FFFFFFFFULL))
6451 current_pmap()->tte[v>>ARM_TT_L1_SHIFT] = kernel_pmap->tte[v>>ARM_TT_L1_SHIFT];
6452 }
6453
6454 }
6455 } else if (ttlevel == 2) {
6456 if (pmap_tt3e(pmap, v) == PT_ENTRY_NULL) {
6457 PMAP_UNLOCK(pmap);
6458 while (pmap_tt_allocate(pmap, &tt_p, PMAP_TT_L3_LEVEL, ((options & PMAP_TT_ALLOCATE_NOWAIT)? PMAP_PAGES_ALLOCATE_NOWAIT : 0)) != KERN_SUCCESS) {
6459 if(options & PMAP_OPTIONS_NOWAIT) {
6460 return KERN_RESOURCE_SHORTAGE;
6461 }
6462 VM_PAGE_WAIT();
6463 }
6464 PMAP_LOCK(pmap);
6465 if ((pmap_tt3e(pmap, v) == PT_ENTRY_NULL)) {
6466 pmap_init_pte_page(pmap, (pt_entry_t *) tt_p, v , PMAP_TT_L3_LEVEL, FALSE);
6467 pa = kvtophys((vm_offset_t)tt_p);
6468 tte_p = pmap_tt2e( pmap, v);
6469 *tte_p = (pa & ARM_TTE_TABLE_MASK) | ARM_TTE_TYPE_TABLE | ARM_TTE_VALID;
6470 pa = 0x0ULL;
6471 tt_p = (tt_entry_t *)NULL;
6472 }
6473 }
6474 }
6475
6476 PMAP_UNLOCK(pmap);
6477
6478 if (tt_p != (tt_entry_t *)NULL) {
6479 pmap_tt_deallocate(pmap, tt_p, ttlevel+1);
6480 tt_p = (tt_entry_t *)NULL;
6481 }
6482 }
6483
6484 return (KERN_SUCCESS);
6485 #endif
6486 }
6487
6488 /*
6489 * Routine: pmap_collect
6490 * Function:
6491 * Garbage collects the physical map system for
6492 * pages which are no longer used.
6493 * Success need not be guaranteed -- that is, there
6494 * may well be pages which are not referenced, but
6495 * others may be collected.
6496 */
6497 void
6498 pmap_collect(pmap_t pmap)
6499 {
6500 if (pmap == PMAP_NULL)
6501 return;
6502
6503 #if 0
6504 PMAP_LOCK(pmap);
6505 if ((pmap->nested == FALSE) && (pmap != kernel_pmap)) {
6506 /* TODO: Scan for vm page assigned to top level page tables with no reference */
6507 }
6508 PMAP_UNLOCK(pmap);
6509 #endif
6510
6511 return;
6512 }
6513
6514 /*
6515 * Routine: pmap_gc
6516 * Function:
6517 * Pmap garbage collection
6518 * Called by the pageout daemon when pages are scarce.
6519 *
6520 */
6521 void
6522 pmap_gc(
6523 void)
6524 {
6525 pmap_t pmap, pmap_next;
6526 boolean_t gc_wait;
6527
6528 if (pmap_gc_allowed &&
6529 (pmap_gc_allowed_by_time_throttle ||
6530 pmap_gc_forced)) {
6531 pmap_gc_forced = FALSE;
6532 pmap_gc_allowed_by_time_throttle = FALSE;
6533 simple_lock(&pmaps_lock);
6534 pmap = CAST_DOWN_EXPLICIT(pmap_t, queue_first(&map_pmap_list));
6535 while (!queue_end(&map_pmap_list, (queue_entry_t)pmap)) {
6536 if (!(pmap->gc_status & PMAP_GC_INFLIGHT))
6537 pmap->gc_status |= PMAP_GC_INFLIGHT;
6538 simple_unlock(&pmaps_lock);
6539
6540 pmap_collect(pmap);
6541
6542 simple_lock(&pmaps_lock);
6543 gc_wait = (pmap->gc_status & PMAP_GC_WAIT);
6544 pmap->gc_status &= ~(PMAP_GC_INFLIGHT|PMAP_GC_WAIT);
6545 pmap_next = CAST_DOWN_EXPLICIT(pmap_t, queue_next(&pmap->pmaps));
6546 if (gc_wait) {
6547 if (!queue_end(&map_pmap_list, (queue_entry_t)pmap_next))
6548 pmap_next->gc_status |= PMAP_GC_INFLIGHT;
6549 simple_unlock(&pmaps_lock);
6550 thread_wakeup((event_t) & pmap->gc_status);
6551 simple_lock(&pmaps_lock);
6552 }
6553 pmap = pmap_next;
6554 }
6555 simple_unlock(&pmaps_lock);
6556 }
6557 }
6558
6559 /*
6560 * Called by the VM to reclaim pages that we can reclaim quickly and cheaply.
6561 */
6562 void
6563 pmap_release_pages_fast(void)
6564 {
6565 }
6566
6567 /*
6568 * By default, don't attempt pmap GC more frequently
6569 * than once / 1 minutes.
6570 */
6571
6572 void
6573 compute_pmap_gc_throttle(
6574 void *arg __unused)
6575 {
6576 pmap_gc_allowed_by_time_throttle = TRUE;
6577 }
6578
6579 /*
6580 * pmap_attribute_cache_sync(vm_offset_t pa)
6581 *
6582 * Invalidates all of the instruction cache on a physical page and
6583 * pushes any dirty data from the data cache for the same physical page
6584 */
6585
6586 kern_return_t
6587 pmap_attribute_cache_sync(
6588 ppnum_t pp,
6589 vm_size_t size,
6590 __unused vm_machine_attribute_t attribute,
6591 __unused vm_machine_attribute_val_t * value)
6592 {
6593 if (size > PAGE_SIZE) {
6594 panic("pmap_attribute_cache_sync size: 0x%llx\n", (uint64_t)size);
6595 } else
6596 cache_sync_page(pp);
6597
6598 return KERN_SUCCESS;
6599 }
6600
6601 /*
6602 * pmap_sync_page_data_phys(ppnum_t pp)
6603 *
6604 * Invalidates all of the instruction cache on a physical page and
6605 * pushes any dirty data from the data cache for the same physical page
6606 */
6607 void
6608 pmap_sync_page_data_phys(
6609 ppnum_t pp)
6610 {
6611 cache_sync_page(pp);
6612 }
6613
6614 /*
6615 * pmap_sync_page_attributes_phys(ppnum_t pp)
6616 *
6617 * Write back and invalidate all cachelines on a physical page.
6618 */
6619 void
6620 pmap_sync_page_attributes_phys(
6621 ppnum_t pp)
6622 {
6623 flush_dcache((vm_offset_t) (pp << PAGE_SHIFT), PAGE_SIZE, TRUE);
6624 }
6625
6626 #if CONFIG_COREDUMP
6627 /* temporary workaround */
6628 boolean_t
6629 coredumpok(
6630 vm_map_t map,
6631 vm_offset_t va)
6632 {
6633 pt_entry_t *pte_p;
6634 pt_entry_t spte;
6635
6636 pte_p = pmap_pte(map->pmap, va);
6637 if (0 == pte_p)
6638 return FALSE;
6639 spte = *pte_p;
6640 return ((spte & ARM_PTE_ATTRINDXMASK) == ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DEFAULT));
6641 }
6642 #endif
6643
6644 void
6645 fillPage(
6646 ppnum_t pn,
6647 unsigned int fill)
6648 {
6649 unsigned int *addr;
6650 int count;
6651
6652 addr = (unsigned int *) phystokv(ptoa(pn));
6653 count = PAGE_SIZE / sizeof(unsigned int);
6654 while (count--)
6655 *addr++ = fill;
6656 }
6657
6658 extern void mapping_set_mod(ppnum_t pn);
6659
6660 void
6661 mapping_set_mod(
6662 ppnum_t pn)
6663 {
6664 pmap_set_modify(pn);
6665 }
6666
6667 extern void mapping_set_ref(ppnum_t pn);
6668
6669 void
6670 mapping_set_ref(
6671 ppnum_t pn)
6672 {
6673 pmap_set_reference(pn);
6674 }
6675
6676 /*
6677 * Clear specified attribute bits.
6678 *
6679 * Try to force an arm_fast_fault() for all mappings of
6680 * the page - to force attributes to be set again at fault time.
6681 * If the forcing succeeds, clear the cached bits at the head.
6682 * Otherwise, something must have been wired, so leave the cached
6683 * attributes alone.
6684 */
6685 static void
6686 phys_attribute_clear_internal(
6687 ppnum_t pn,
6688 unsigned int bits,
6689 int options,
6690 void *arg)
6691 {
6692 pmap_paddr_t pa = ptoa(pn);
6693 vm_prot_t allow_mode = VM_PROT_ALL;
6694
6695
6696 if ((bits & PP_ATTR_MODIFIED) &&
6697 (options & PMAP_OPTIONS_NOFLUSH) &&
6698 (arg == NULL)) {
6699 panic("phys_attribute_clear(0x%x,0x%x,0x%x,%p): "
6700 "should not clear 'modified' without flushing TLBs\n",
6701 pn, bits, options, arg);
6702 }
6703
6704 assert(pn != vm_page_fictitious_addr);
6705 if (bits & PP_ATTR_REFERENCED)
6706 allow_mode &= ~(VM_PROT_READ | VM_PROT_EXECUTE);
6707 if (bits & PP_ATTR_MODIFIED)
6708 allow_mode &= ~VM_PROT_WRITE;
6709
6710 if (bits == PP_ATTR_NOENCRYPT) {
6711 /*
6712 * We short circuit this case; it should not need to
6713 * invoke arm_force_fast_fault, so just clear and
6714 * return. On ARM, this bit is just a debugging aid.
6715 */
6716 pa_clear_bits(pa, bits);
6717 return;
6718 }
6719
6720 if (arm_force_fast_fault_internal(pn, allow_mode, options))
6721 pa_clear_bits(pa, bits);
6722 return;
6723 }
6724
6725 static void
6726 phys_attribute_clear(
6727 ppnum_t pn,
6728 unsigned int bits,
6729 int options,
6730 void *arg)
6731 {
6732 /*
6733 * Do we really want this tracepoint? It will be extremely chatty.
6734 * Also, should we have a corresponding trace point for the set path?
6735 */
6736 PMAP_TRACE(PMAP_CODE(PMAP__ATTRIBUTE_CLEAR) | DBG_FUNC_START, pn, bits);
6737
6738 phys_attribute_clear_internal(pn, bits, options, arg);
6739
6740 PMAP_TRACE(PMAP_CODE(PMAP__ATTRIBUTE_CLEAR) | DBG_FUNC_END);
6741 }
6742
6743 /*
6744 * Set specified attribute bits.
6745 *
6746 * Set cached value in the pv head because we have
6747 * no per-mapping hardware support for referenced and
6748 * modify bits.
6749 */
6750 static void
6751 phys_attribute_set_internal(
6752 ppnum_t pn,
6753 unsigned int bits)
6754 {
6755 pmap_paddr_t pa = ptoa(pn);
6756 assert(pn != vm_page_fictitious_addr);
6757
6758
6759 pa_set_bits(pa, bits);
6760
6761 return;
6762 }
6763
6764 static void
6765 phys_attribute_set(
6766 ppnum_t pn,
6767 unsigned int bits)
6768 {
6769 phys_attribute_set_internal(pn, bits);
6770 }
6771
6772
6773 /*
6774 * Check specified attribute bits.
6775 *
6776 * use the software cached bits (since no hw support).
6777 */
6778 static boolean_t
6779 phys_attribute_test(
6780 ppnum_t pn,
6781 unsigned int bits)
6782 {
6783 pmap_paddr_t pa = ptoa(pn);
6784 assert(pn != vm_page_fictitious_addr);
6785 return pa_test_bits(pa, bits);
6786 }
6787
6788
6789 /*
6790 * Set the modify/reference bits on the specified physical page.
6791 */
6792 void
6793 pmap_set_modify(ppnum_t pn)
6794 {
6795 phys_attribute_set(pn, PP_ATTR_MODIFIED);
6796 }
6797
6798
6799 /*
6800 * Clear the modify bits on the specified physical page.
6801 */
6802 void
6803 pmap_clear_modify(
6804 ppnum_t pn)
6805 {
6806 phys_attribute_clear(pn, PP_ATTR_MODIFIED, 0, NULL);
6807 }
6808
6809
6810 /*
6811 * pmap_is_modified:
6812 *
6813 * Return whether or not the specified physical page is modified
6814 * by any physical maps.
6815 */
6816 boolean_t
6817 pmap_is_modified(
6818 ppnum_t pn)
6819 {
6820 return phys_attribute_test(pn, PP_ATTR_MODIFIED);
6821 }
6822
6823
6824 /*
6825 * Set the reference bit on the specified physical page.
6826 */
6827 static void
6828 pmap_set_reference(
6829 ppnum_t pn)
6830 {
6831 phys_attribute_set(pn, PP_ATTR_REFERENCED);
6832 }
6833
6834 /*
6835 * Clear the reference bits on the specified physical page.
6836 */
6837 void
6838 pmap_clear_reference(
6839 ppnum_t pn)
6840 {
6841 phys_attribute_clear(pn, PP_ATTR_REFERENCED, 0, NULL);
6842 }
6843
6844
6845 /*
6846 * pmap_is_referenced:
6847 *
6848 * Return whether or not the specified physical page is referenced
6849 * by any physical maps.
6850 */
6851 boolean_t
6852 pmap_is_referenced(
6853 ppnum_t pn)
6854 {
6855 return phys_attribute_test(pn, PP_ATTR_REFERENCED);
6856 }
6857
6858 /*
6859 * pmap_get_refmod(phys)
6860 * returns the referenced and modified bits of the specified
6861 * physical page.
6862 */
6863 unsigned int
6864 pmap_get_refmod(
6865 ppnum_t pn)
6866 {
6867 return (((phys_attribute_test(pn, PP_ATTR_MODIFIED)) ? VM_MEM_MODIFIED : 0)
6868 | ((phys_attribute_test(pn, PP_ATTR_REFERENCED)) ? VM_MEM_REFERENCED : 0));
6869 }
6870
6871 /*
6872 * pmap_clear_refmod(phys, mask)
6873 * clears the referenced and modified bits as specified by the mask
6874 * of the specified physical page.
6875 */
6876 void
6877 pmap_clear_refmod_options(
6878 ppnum_t pn,
6879 unsigned int mask,
6880 unsigned int options,
6881 void *arg)
6882 {
6883 unsigned int bits;
6884
6885 bits = ((mask & VM_MEM_MODIFIED) ? PP_ATTR_MODIFIED : 0) |
6886 ((mask & VM_MEM_REFERENCED) ? PP_ATTR_REFERENCED : 0);
6887 phys_attribute_clear(pn, bits, options, arg);
6888 }
6889
6890 void
6891 pmap_clear_refmod(
6892 ppnum_t pn,
6893 unsigned int mask)
6894 {
6895 pmap_clear_refmod_options(pn, mask, 0, NULL);
6896 }
6897
6898 unsigned int
6899 pmap_disconnect_options(
6900 ppnum_t pn,
6901 unsigned int options,
6902 void *arg)
6903 {
6904 if ((options & PMAP_OPTIONS_COMPRESSOR_IFF_MODIFIED)) {
6905 /*
6906 * On ARM, the "modified" bit is managed by software, so
6907 * we know up-front if the physical page is "modified",
6908 * without having to scan all the PTEs pointing to it.
6909 * The caller should have made the VM page "busy" so noone
6910 * should be able to establish any new mapping and "modify"
6911 * the page behind us.
6912 */
6913 if (pmap_is_modified(pn)) {
6914 /*
6915 * The page has been modified and will be sent to
6916 * the VM compressor.
6917 */
6918 options |= PMAP_OPTIONS_COMPRESSOR;
6919 } else {
6920 /*
6921 * The page hasn't been modified and will be freed
6922 * instead of compressed.
6923 */
6924 }
6925 }
6926
6927 /* disconnect the page */
6928 pmap_page_protect_options(pn, 0, options, arg);
6929
6930 /* return ref/chg status */
6931 return (pmap_get_refmod(pn));
6932 }
6933
6934 /*
6935 * Routine:
6936 * pmap_disconnect
6937 *
6938 * Function:
6939 * Disconnect all mappings for this page and return reference and change status
6940 * in generic format.
6941 *
6942 */
6943 unsigned int
6944 pmap_disconnect(
6945 ppnum_t pn)
6946 {
6947 pmap_page_protect(pn, 0); /* disconnect the page */
6948 return (pmap_get_refmod(pn)); /* return ref/chg status */
6949 }
6950
6951 boolean_t
6952 pmap_has_managed_page(ppnum_t first, ppnum_t last)
6953 {
6954 if (ptoa(first) >= vm_last_phys) return (FALSE);
6955 if (ptoa(last) < vm_first_phys) return (FALSE);
6956
6957 return (TRUE);
6958 }
6959
6960 /*
6961 * The state maintained by the noencrypt functions is used as a
6962 * debugging aid on ARM. This incurs some overhead on the part
6963 * of the caller. A special case check in phys_attribute_clear
6964 * (the most expensive path) currently minimizes this overhead,
6965 * but stubbing these functions out on RELEASE kernels yields
6966 * further wins.
6967 */
6968 boolean_t
6969 pmap_is_noencrypt(
6970 ppnum_t pn)
6971 {
6972 #if DEVELOPMENT || DEBUG
6973 boolean_t result = FALSE;
6974
6975 if (!pa_valid(ptoa(pn))) return FALSE;
6976
6977 result = (phys_attribute_test(pn, PP_ATTR_NOENCRYPT));
6978
6979 return result;
6980 #else
6981 #pragma unused(pn)
6982 return FALSE;
6983 #endif
6984 }
6985
6986 void
6987 pmap_set_noencrypt(
6988 ppnum_t pn)
6989 {
6990 #if DEVELOPMENT || DEBUG
6991 if (!pa_valid(ptoa(pn))) return;
6992
6993 phys_attribute_set(pn, PP_ATTR_NOENCRYPT);
6994 #else
6995 #pragma unused(pn)
6996 #endif
6997 }
6998
6999 void
7000 pmap_clear_noencrypt(
7001 ppnum_t pn)
7002 {
7003 #if DEVELOPMENT || DEBUG
7004 if (!pa_valid(ptoa(pn))) return;
7005
7006 phys_attribute_clear(pn, PP_ATTR_NOENCRYPT, 0, NULL);
7007 #else
7008 #pragma unused(pn)
7009 #endif
7010 }
7011
7012
7013 void
7014 pmap_lock_phys_page(ppnum_t pn)
7015 {
7016 int pai;
7017 pmap_paddr_t phys = ptoa(pn);
7018
7019 if (pa_valid(phys)) {
7020 pai = (int)pa_index(phys);
7021 LOCK_PVH(pai);
7022 } else
7023 simple_lock(&phys_backup_lock);
7024 }
7025
7026
7027 void
7028 pmap_unlock_phys_page(ppnum_t pn)
7029 {
7030 int pai;
7031 pmap_paddr_t phys = ptoa(pn);
7032
7033 if (pa_valid(phys)) {
7034 pai = (int)pa_index(phys);
7035 UNLOCK_PVH(pai);
7036 } else
7037 simple_unlock(&phys_backup_lock);
7038 }
7039
7040 static void
7041 pmap_switch_user_ttb_internal(
7042 pmap_t pmap)
7043 {
7044 #if (__ARM_VMSA__ == 7)
7045 pmap_cpu_data_t *cpu_data_ptr;
7046
7047 cpu_data_ptr = pmap_get_cpu_data();
7048
7049 if ((cpu_data_ptr->cpu_user_pmap != PMAP_NULL)
7050 && (cpu_data_ptr->cpu_user_pmap != kernel_pmap)) {
7051 unsigned int c;
7052
7053 c = hw_atomic_sub((volatile uint32_t *)&cpu_data_ptr->cpu_user_pmap->cpu_ref, 1);
7054 if ((c == 0) && (cpu_data_ptr->cpu_user_pmap->prev_tte != 0)) {
7055 /* We saved off the old 1-page tt1 in pmap_expand() in case other cores were still using it.
7056 * Now that the user pmap's cpu_ref is 0, we should be able to safely free it.*/
7057 tt_entry_t *tt_entry;
7058
7059 tt_entry = cpu_data_ptr->cpu_user_pmap->prev_tte;
7060 cpu_data_ptr->cpu_user_pmap->prev_tte = (tt_entry_t *) NULL;
7061 pmap_tt1_deallocate(cpu_data_ptr->cpu_user_pmap, tt_entry, ARM_PGBYTES, PMAP_TT_DEALLOCATE_NOBLOCK);
7062 }
7063 }
7064 cpu_data_ptr->cpu_user_pmap = pmap;
7065 cpu_data_ptr->cpu_user_pmap_stamp = pmap->stamp;
7066 (void) hw_atomic_add((volatile uint32_t *)&pmap->cpu_ref, 1);
7067
7068 #if MACH_ASSERT && __ARM_USER_PROTECT__
7069 {
7070 unsigned int ttbr0_val, ttbr1_val;
7071 __asm__ volatile("mrc p15,0,%0,c2,c0,0\n" : "=r"(ttbr0_val));
7072 __asm__ volatile("mrc p15,0,%0,c2,c0,1\n" : "=r"(ttbr1_val));
7073 if (ttbr0_val != ttbr1_val) {
7074 panic("Misaligned ttbr0 %08X\n", ttbr0_val);
7075 }
7076 }
7077 #endif
7078 if (pmap->tte_index_max == NTTES) {
7079 /* Setting TTBCR.N for TTBR0 TTBR1 boundary at 0x40000000 */
7080 __asm__ volatile("mcr p15,0,%0,c2,c0,2" : : "r"(2));
7081 __asm__ volatile("isb");
7082 #if !__ARM_USER_PROTECT__
7083 set_mmu_ttb(pmap->ttep);
7084 #endif
7085 } else {
7086 #if !__ARM_USER_PROTECT__
7087 set_mmu_ttb(pmap->ttep);
7088 #endif
7089 /* Setting TTBCR.N for TTBR0 TTBR1 boundary at 0x80000000 */
7090 __asm__ volatile("mcr p15,0,%0,c2,c0,2" : : "r"(1));
7091 __asm__ volatile("isb");
7092 #if MACH_ASSERT && __ARM_USER_PROTECT__
7093 if (pmap->ttep & 0x1000) {
7094 panic("Misaligned ttbr0 %08X\n", pmap->ttep);
7095 }
7096 #endif
7097 }
7098
7099 #if !__ARM_USER_PROTECT__
7100 set_context_id(pmap->asid);
7101 #endif
7102 #else
7103
7104 pmap_get_cpu_data()->cpu_user_pmap = pmap;
7105 pmap_get_cpu_data()->cpu_user_pmap_stamp = pmap->stamp;
7106
7107 #if !__arm64__
7108 set_context_id(pmap->asid); /* Not required */
7109 #endif
7110 if (pmap == kernel_pmap) {
7111 set_mmu_ttb(invalid_ttep & TTBR_BADDR_MASK);
7112 } else {
7113 set_mmu_ttb((pmap->ttep & TTBR_BADDR_MASK)|(((uint64_t)pmap->asid) << TTBR_ASID_SHIFT));
7114 }
7115 #endif
7116 }
7117
7118 void
7119 pmap_switch_user_ttb(
7120 pmap_t pmap)
7121 {
7122 pmap_switch_user_ttb_internal(pmap);
7123 }
7124
7125 /*
7126 * Try to "intuit" whether we need to raise a VM_PROT_WRITE fault
7127 * for the given address when a "swp" instruction raised the fault.
7128 * We have to look at the existing pte for the address to see
7129 * if it needs to get bumped, or just added. If just added, do it
7130 * as a read-only mapping first (this could result in extra faults -
7131 * but better that than extra copy-on-write evaluations).
7132 */
7133
7134 #if (__ARM_VMSA__ == 7)
7135 boolean_t
7136 arm_swap_readable_type(
7137 vm_map_address_t addr,
7138 unsigned int spsr)
7139 {
7140 int ap;
7141 pt_entry_t spte;
7142 pt_entry_t *ptep;
7143
7144 ptep = pmap_pte(current_pmap(), addr);
7145 if (ptep == PT_ENTRY_NULL)
7146 return (FALSE);
7147
7148 spte = *ptep;
7149 if (spte == ARM_PTE_TYPE_FAULT ||
7150 ARM_PTE_IS_COMPRESSED(spte))
7151 return (FALSE);
7152
7153 /* get the access permission bitmaps */
7154 /* (all subpages should be the same) */
7155 ap = (spte & ARM_PTE_APMASK);
7156
7157 if (spsr & 0xf) { /* Supervisor mode */
7158 panic("arm_swap_readable_type supv");
7159 return TRUE;
7160 } else { /* User mode */
7161 if ((ap == ARM_PTE_AP(AP_RWRW)) || (ap == ARM_PTE_AP(AP_RORO)))
7162 return (FALSE);
7163 else
7164 return (TRUE);
7165 }
7166 }
7167 #endif
7168
7169 /*
7170 * Routine: arm_force_fast_fault
7171 *
7172 * Function:
7173 * Force all mappings for this page to fault according
7174 * to the access modes allowed, so we can gather ref/modify
7175 * bits again.
7176 */
7177 static boolean_t
7178 arm_force_fast_fault_internal(
7179 ppnum_t ppnum,
7180 vm_prot_t allow_mode,
7181 int options)
7182 {
7183 pmap_paddr_t phys = ptoa(ppnum);
7184 pv_entry_t *pve_p;
7185 pt_entry_t *pte_p;
7186 int pai;
7187 boolean_t result;
7188 pv_entry_t **pv_h;
7189 boolean_t is_reusable, is_internal;
7190 boolean_t ref_fault;
7191 boolean_t mod_fault;
7192
7193 assert(ppnum != vm_page_fictitious_addr);
7194
7195 if (!pa_valid(phys)) {
7196 return FALSE; /* Not a managed page. */
7197 }
7198
7199 result = TRUE;
7200 ref_fault = FALSE;
7201 mod_fault = FALSE;
7202 pai = (int)pa_index(phys);
7203 LOCK_PVH(pai);
7204 pv_h = pai_to_pvh(pai);
7205
7206 pte_p = PT_ENTRY_NULL;
7207 pve_p = PV_ENTRY_NULL;
7208 if (pvh_test_type(pv_h, PVH_TYPE_PTEP)) {
7209 pte_p = pvh_ptep(pv_h);
7210 } else if (pvh_test_type(pv_h, PVH_TYPE_PVEP)) {
7211 pve_p = pvh_list(pv_h);
7212 }
7213
7214 is_reusable = IS_REUSABLE_PAGE(pai);
7215 is_internal = IS_INTERNAL_PAGE(pai);
7216
7217 while ((pve_p != PV_ENTRY_NULL) || (pte_p != PT_ENTRY_NULL)) {
7218 vm_map_address_t va;
7219 pt_entry_t spte;
7220 pt_entry_t tmplate;
7221 pmap_t pmap;
7222 boolean_t update_pte;
7223
7224 if (pve_p != PV_ENTRY_NULL)
7225 pte_p = pve_get_ptep(pve_p);
7226
7227 if (pte_p == PT_ENTRY_NULL) {
7228 panic("pte_p is NULL: pve_p=%p ppnum=0x%x\n", pve_p, ppnum);
7229 }
7230 if (*pte_p == ARM_PTE_EMPTY) {
7231 panic("pte is NULL: pte_p=%p ppnum=0x%x\n", pte_p, ppnum);
7232 }
7233 if (ARM_PTE_IS_COMPRESSED(*pte_p)) {
7234 panic("pte is COMPRESSED: pte_p=%p ppnum=0x%x\n", pte_p, ppnum);
7235 }
7236
7237 pmap = ptep_get_pmap(pte_p);
7238 va = ptep_get_va(pte_p);
7239
7240 assert(va >= pmap->min && va < pmap->max);
7241
7242 if (pte_is_wired(*pte_p) || pmap == kernel_pmap) {
7243 result = FALSE;
7244 break;
7245 }
7246
7247 spte = *pte_p;
7248 tmplate = spte;
7249 update_pte = FALSE;
7250
7251 if ((allow_mode & VM_PROT_READ) != VM_PROT_READ) {
7252 /* read protection sets the pte to fault */
7253 tmplate = tmplate & ~ARM_PTE_AF;
7254 update_pte = TRUE;
7255 ref_fault = TRUE;
7256 }
7257 if ((allow_mode & VM_PROT_WRITE) != VM_PROT_WRITE) {
7258 /* take away write permission if set */
7259 if (pmap == kernel_pmap) {
7260 if ((tmplate & ARM_PTE_APMASK) == ARM_PTE_AP(AP_RWNA)) {
7261 tmplate = ((tmplate & ~ARM_PTE_APMASK) | ARM_PTE_AP(AP_RONA));
7262 }
7263 } else {
7264 if ((tmplate & ARM_PTE_APMASK) == ARM_PTE_AP(AP_RWRW)) {
7265 tmplate = ((tmplate & ~ARM_PTE_APMASK) | ARM_PTE_AP(AP_RORO));
7266 }
7267 }
7268
7269 pte_set_ffr(tmplate, 1);
7270 update_pte = TRUE;
7271 mod_fault = TRUE;
7272 }
7273
7274
7275 if (update_pte) {
7276 if (*pte_p != ARM_PTE_TYPE_FAULT &&
7277 !ARM_PTE_IS_COMPRESSED(*pte_p)) {
7278 WRITE_PTE(pte_p, tmplate);
7279 PMAP_UPDATE_TLBS(pmap, va, va + PAGE_SIZE);
7280 } else {
7281 WRITE_PTE(pte_p, tmplate);
7282 __asm__ volatile("isb");
7283 }
7284 }
7285
7286 /* update pmap stats and ledgers */
7287 if (IS_ALTACCT_PAGE(pai, pve_p)) {
7288 /*
7289 * We do not track "reusable" status for
7290 * "alternate accounting" mappings.
7291 */
7292 } else if ((options & PMAP_OPTIONS_CLEAR_REUSABLE) &&
7293 is_reusable &&
7294 is_internal &&
7295 pmap != kernel_pmap) {
7296 /* one less "reusable" */
7297 assert(pmap->stats.reusable > 0);
7298 OSAddAtomic(-1, &pmap->stats.reusable);
7299 /* one more "internal" */
7300 OSAddAtomic(+1, &pmap->stats.internal);
7301 PMAP_STATS_PEAK(pmap->stats.internal);
7302 assert(pmap->stats.internal > 0);
7303 pmap_ledger_credit(pmap,
7304 task_ledgers.internal,
7305 machine_ptob(1));
7306 assert(!IS_ALTACCT_PAGE(pai, pve_p));
7307 assert(IS_INTERNAL_PAGE(pai));
7308 pmap_ledger_credit(pmap,
7309 task_ledgers.phys_footprint,
7310 machine_ptob(1));
7311
7312 /*
7313 * Avoid the cost of another trap to handle the fast
7314 * fault when we next write to this page: let's just
7315 * handle that now since we already have all the
7316 * necessary information.
7317 */
7318 {
7319 arm_clear_fast_fault(ppnum, VM_PROT_WRITE);
7320 }
7321 } else if ((options & PMAP_OPTIONS_SET_REUSABLE) &&
7322 !is_reusable &&
7323 is_internal &&
7324 pmap != kernel_pmap) {
7325 /* one more "reusable" */
7326 OSAddAtomic(+1, &pmap->stats.reusable);
7327 PMAP_STATS_PEAK(pmap->stats.reusable);
7328 assert(pmap->stats.reusable > 0);
7329 /* one less "internal" */
7330 assert(pmap->stats.internal > 0);
7331 OSAddAtomic(-1, &pmap->stats.internal);
7332 pmap_ledger_debit(pmap,
7333 task_ledgers.internal,
7334 machine_ptob(1));
7335 assert(!IS_ALTACCT_PAGE(pai, pve_p));
7336 assert(IS_INTERNAL_PAGE(pai));
7337 pmap_ledger_debit(pmap,
7338 task_ledgers.phys_footprint,
7339 machine_ptob(1));
7340 }
7341
7342 pte_p = PT_ENTRY_NULL;
7343 if (pve_p != PV_ENTRY_NULL)
7344 pve_p = PVE_NEXT_PTR(pve_next(pve_p));
7345 }
7346
7347 /* update global "reusable" status for this page */
7348 if (is_internal) {
7349 if ((options & PMAP_OPTIONS_CLEAR_REUSABLE) &&
7350 is_reusable) {
7351 CLR_REUSABLE_PAGE(pai);
7352 } else if ((options & PMAP_OPTIONS_SET_REUSABLE) &&
7353 !is_reusable) {
7354 SET_REUSABLE_PAGE(pai);
7355 }
7356 }
7357
7358 if (mod_fault) {
7359 SET_MODFAULT_PAGE(pai);
7360 }
7361 if (ref_fault) {
7362 SET_REFFAULT_PAGE(pai);
7363 }
7364
7365 UNLOCK_PVH(pai);
7366 return result;
7367 }
7368
7369 boolean_t
7370 arm_force_fast_fault(
7371 ppnum_t ppnum,
7372 vm_prot_t allow_mode,
7373 int options,
7374 __unused void *arg)
7375 {
7376 pmap_paddr_t phys = ptoa(ppnum);
7377
7378 assert(ppnum != vm_page_fictitious_addr);
7379
7380 if (!pa_valid(phys)) {
7381 return FALSE; /* Not a managed page. */
7382 }
7383
7384 return arm_force_fast_fault_internal(ppnum, allow_mode, options);
7385 }
7386
7387 /*
7388 * Routine: arm_clear_fast_fault
7389 *
7390 * Function:
7391 * Clear pending force fault for all mappings for this page based on
7392 * the observed fault type, update ref/modify bits.
7393 */
7394 boolean_t
7395 arm_clear_fast_fault(
7396 ppnum_t ppnum,
7397 vm_prot_t fault_type)
7398 {
7399 pmap_paddr_t pa = ptoa(ppnum);
7400 pv_entry_t *pve_p;
7401 pt_entry_t *pte_p;
7402 int pai;
7403 boolean_t result;
7404 pv_entry_t **pv_h;
7405
7406 assert(ppnum != vm_page_fictitious_addr);
7407
7408 if (!pa_valid(pa)) {
7409 return FALSE; /* Not a managed page. */
7410 }
7411
7412 result = FALSE;
7413 pai = (int)pa_index(pa);
7414 ASSERT_PVH_LOCKED(pai);
7415 pv_h = pai_to_pvh(pai);
7416
7417 pte_p = PT_ENTRY_NULL;
7418 pve_p = PV_ENTRY_NULL;
7419 if (pvh_test_type(pv_h, PVH_TYPE_PTEP)) {
7420 pte_p = pvh_ptep(pv_h);
7421 } else if (pvh_test_type(pv_h, PVH_TYPE_PVEP)) {
7422 pve_p = pvh_list(pv_h);
7423 }
7424
7425 while ((pve_p != PV_ENTRY_NULL) || (pte_p != PT_ENTRY_NULL)) {
7426 vm_map_address_t va;
7427 pt_entry_t spte;
7428 pt_entry_t tmplate;
7429 pmap_t pmap;
7430
7431 if (pve_p != PV_ENTRY_NULL)
7432 pte_p = pve_get_ptep(pve_p);
7433
7434 if (pte_p == PT_ENTRY_NULL) {
7435 panic("pte_p is NULL: pve_p=%p ppnum=0x%x\n", pve_p, ppnum);
7436 }
7437 if (*pte_p == ARM_PTE_EMPTY) {
7438 panic("pte is NULL: pte_p=%p ppnum=0x%x\n", pte_p, ppnum);
7439 }
7440
7441 pmap = ptep_get_pmap(pte_p);
7442 va = ptep_get_va(pte_p);
7443
7444 assert(va >= pmap->min && va < pmap->max);
7445
7446 spte = *pte_p;
7447 tmplate = spte;
7448
7449 if ((fault_type & VM_PROT_WRITE) && (pte_is_ffr(spte))) {
7450 {
7451 if (pmap == kernel_pmap)
7452 tmplate = ((spte & ~ARM_PTE_APMASK) | ARM_PTE_AP(AP_RWNA));
7453 else
7454 tmplate = ((spte & ~ARM_PTE_APMASK) | ARM_PTE_AP(AP_RWRW));
7455 }
7456
7457 tmplate |= ARM_PTE_AF;
7458
7459 pte_set_ffr(tmplate, 0);
7460 pa_set_bits(pa, PP_ATTR_REFERENCED | PP_ATTR_MODIFIED);
7461
7462 } else if ((fault_type & VM_PROT_READ) && ((spte & ARM_PTE_AF) != ARM_PTE_AF)) {
7463 tmplate = spte | ARM_PTE_AF;
7464
7465 {
7466 pa_set_bits(pa, PP_ATTR_REFERENCED);
7467 }
7468 }
7469
7470
7471 if (spte != tmplate) {
7472 if (spte != ARM_PTE_TYPE_FAULT) {
7473 WRITE_PTE(pte_p, tmplate);
7474 PMAP_UPDATE_TLBS(pmap, va, va + PAGE_SIZE);
7475 } else {
7476 WRITE_PTE(pte_p, tmplate);
7477 __asm__ volatile("isb");
7478 }
7479 result = TRUE;
7480 }
7481
7482 pte_p = PT_ENTRY_NULL;
7483 if (pve_p != PV_ENTRY_NULL)
7484 pve_p = PVE_NEXT_PTR(pve_next(pve_p));
7485 }
7486 return result;
7487 }
7488
7489 /*
7490 * Determine if the fault was induced by software tracking of
7491 * modify/reference bits. If so, re-enable the mapping (and set
7492 * the appropriate bits).
7493 *
7494 * Returns KERN_SUCCESS if the fault was induced and was
7495 * successfully handled.
7496 *
7497 * Returns KERN_FAILURE if the fault was not induced and
7498 * the function was unable to deal with it.
7499 *
7500 * Returns KERN_PROTECTION_FAILURE if the pmap layer explictly
7501 * disallows this type of access.
7502 */
7503 static kern_return_t
7504 arm_fast_fault_internal(
7505 pmap_t pmap,
7506 vm_map_address_t va,
7507 vm_prot_t fault_type,
7508 __unused boolean_t from_user)
7509 {
7510 kern_return_t result = KERN_FAILURE;
7511 pt_entry_t *ptep;
7512 pt_entry_t spte = ARM_PTE_TYPE_FAULT;
7513 int pai;
7514 pmap_paddr_t pa;
7515
7516 PMAP_LOCK(pmap);
7517
7518 /*
7519 * If the entry doesn't exist, is completely invalid, or is already
7520 * valid, we can't fix it here.
7521 */
7522
7523 ptep = pmap_pte(pmap, va);
7524 if (ptep != PT_ENTRY_NULL) {
7525 spte = *ptep;
7526
7527 pa = pte_to_pa(spte);
7528
7529 if ((spte == ARM_PTE_TYPE_FAULT) ||
7530 ARM_PTE_IS_COMPRESSED(spte) ||
7531 (!pa_valid(pa))) {
7532 PMAP_UNLOCK(pmap);
7533 return result;
7534 }
7535
7536 pai = (int)pa_index(pa);
7537 LOCK_PVH(pai);
7538 } else {
7539 PMAP_UNLOCK(pmap);
7540 return result;
7541 }
7542
7543
7544 if ((IS_REFFAULT_PAGE(pai)) ||
7545 ((fault_type & VM_PROT_WRITE) && IS_MODFAULT_PAGE(pai))) {
7546 /*
7547 * An attempted access will always clear ref/mod fault state, as
7548 * appropriate for the fault type. arm_clear_fast_fault will
7549 * update the associated PTEs for the page as appropriate; if
7550 * any PTEs are updated, we redrive the access. If the mapping
7551 * does not actually allow for the attempted access, the
7552 * following fault will (hopefully) fail to update any PTEs, and
7553 * thus cause arm_fast_fault to decide that it failed to handle
7554 * the fault.
7555 */
7556 if (IS_REFFAULT_PAGE(pai)) {
7557 CLR_REFFAULT_PAGE(pai);
7558 }
7559 if ( (fault_type & VM_PROT_WRITE) && IS_MODFAULT_PAGE(pai)) {
7560 CLR_MODFAULT_PAGE(pai);
7561 }
7562
7563 if (arm_clear_fast_fault((ppnum_t)atop(pa),fault_type)) {
7564 /*
7565 * Should this preserve KERN_PROTECTION_FAILURE? The
7566 * cost of not doing so is a another fault in a case
7567 * that should already result in an exception.
7568 */
7569 result = KERN_SUCCESS;
7570 }
7571 }
7572
7573 UNLOCK_PVH(pai);
7574 PMAP_UNLOCK(pmap);
7575 return result;
7576 }
7577
7578 kern_return_t
7579 arm_fast_fault(
7580 pmap_t pmap,
7581 vm_map_address_t va,
7582 vm_prot_t fault_type,
7583 __unused boolean_t from_user)
7584 {
7585 kern_return_t result = KERN_FAILURE;
7586
7587 if (va < pmap->min || va >= pmap->max)
7588 return result;
7589
7590 PMAP_TRACE(PMAP_CODE(PMAP__FAST_FAULT) | DBG_FUNC_START,
7591 VM_KERNEL_ADDRHIDE(pmap), VM_KERNEL_ADDRHIDE(va), fault_type,
7592 from_user);
7593
7594 #if (__ARM_VMSA__ == 7)
7595 if (pmap != kernel_pmap) {
7596 pmap_cpu_data_t *cpu_data_ptr = pmap_get_cpu_data();
7597 pmap_t cur_pmap;
7598 pmap_t cur_user_pmap;
7599
7600 cur_pmap = current_pmap();
7601 cur_user_pmap = cpu_data_ptr->cpu_user_pmap;
7602
7603 if ((cur_user_pmap == cur_pmap) && (cur_pmap == pmap)) {
7604 if (cpu_data_ptr->cpu_user_pmap_stamp != pmap->stamp) {
7605 pmap_set_pmap(pmap, current_thread());
7606 result = KERN_SUCCESS;
7607 goto done;
7608 }
7609 }
7610 }
7611 #endif
7612
7613 result = arm_fast_fault_internal(pmap, va, fault_type, from_user);
7614
7615 #if (__ARM_VMSA__ == 7)
7616 done:
7617 #endif
7618
7619 PMAP_TRACE(PMAP_CODE(PMAP__FAST_FAULT) | DBG_FUNC_END, result);
7620
7621 return result;
7622 }
7623
7624 void
7625 pmap_copy_page(
7626 ppnum_t psrc,
7627 ppnum_t pdst)
7628 {
7629 bcopy_phys((addr64_t) (ptoa(psrc)),
7630 (addr64_t) (ptoa(pdst)),
7631 PAGE_SIZE);
7632 }
7633
7634
7635 /*
7636 * pmap_copy_page copies the specified (machine independent) pages.
7637 */
7638 void
7639 pmap_copy_part_page(
7640 ppnum_t psrc,
7641 vm_offset_t src_offset,
7642 ppnum_t pdst,
7643 vm_offset_t dst_offset,
7644 vm_size_t len)
7645 {
7646 bcopy_phys((addr64_t) (ptoa(psrc) + src_offset),
7647 (addr64_t) (ptoa(pdst) + dst_offset),
7648 len);
7649 }
7650
7651
7652 /*
7653 * pmap_zero_page zeros the specified (machine independent) page.
7654 */
7655 void
7656 pmap_zero_page(
7657 ppnum_t pn)
7658 {
7659 assert(pn != vm_page_fictitious_addr);
7660 bzero_phys((addr64_t) ptoa(pn), PAGE_SIZE);
7661 }
7662
7663 /*
7664 * pmap_zero_part_page
7665 * zeros the specified (machine independent) part of a page.
7666 */
7667 void
7668 pmap_zero_part_page(
7669 ppnum_t pn,
7670 vm_offset_t offset,
7671 vm_size_t len)
7672 {
7673 assert(pn != vm_page_fictitious_addr);
7674 assert(offset + len <= PAGE_SIZE);
7675 bzero_phys((addr64_t) (ptoa(pn) + offset), len);
7676 }
7677
7678
7679 /*
7680 * nop in current arm implementation
7681 */
7682 void
7683 inval_copy_windows(
7684 __unused thread_t t)
7685 {
7686 }
7687
7688 void
7689 pmap_map_globals(
7690 void)
7691 {
7692 pt_entry_t *ptep, pte;
7693
7694 ptep = pmap_pte(kernel_pmap, LOWGLOBAL_ALIAS);
7695 assert(ptep != PT_ENTRY_NULL);
7696 assert(*ptep == ARM_PTE_EMPTY);
7697
7698 pte = pa_to_pte(ml_static_vtop((vm_offset_t)&lowGlo)) | AP_RONA | ARM_PTE_NX | ARM_PTE_PNX | ARM_PTE_AF | ARM_PTE_TYPE;
7699 pte |= ARM_PTE_ATTRINDX(CACHE_ATTRINDX_WRITEBACK);
7700 #if (__ARM_VMSA__ > 7)
7701 pte |= ARM_PTE_SH(SH_OUTER_MEMORY);
7702 #else
7703 pte |= ARM_PTE_SH;
7704 #endif
7705 *ptep = pte;
7706 FLUSH_PTE_RANGE(ptep,(ptep+1));
7707 PMAP_UPDATE_TLBS(kernel_pmap, LOWGLOBAL_ALIAS, LOWGLOBAL_ALIAS + PAGE_SIZE);
7708 }
7709
7710 vm_offset_t
7711 pmap_cpu_windows_copy_addr(int cpu_num, unsigned int index)
7712 {
7713 return (vm_offset_t)(CPUWINDOWS_BASE + (PAGE_SIZE * ((CPUWINDOWS_MAX * cpu_num) + index)));
7714 }
7715
7716 static unsigned int
7717 pmap_map_cpu_windows_copy_internal(
7718 ppnum_t pn,
7719 vm_prot_t prot,
7720 unsigned int wimg_bits)
7721 {
7722 pt_entry_t *ptep = NULL, pte;
7723 unsigned int cpu_num;
7724 unsigned int i;
7725 vm_offset_t cpu_copywindow_vaddr = 0;
7726
7727 cpu_num = pmap_get_cpu_data()->cpu_number;
7728
7729 for (i = 0; i<CPUWINDOWS_MAX; i++) {
7730 cpu_copywindow_vaddr = pmap_cpu_windows_copy_addr(cpu_num, i);
7731 ptep = pmap_pte(kernel_pmap, cpu_copywindow_vaddr);
7732 assert(!ARM_PTE_IS_COMPRESSED(*ptep));
7733 if (*ptep == ARM_PTE_TYPE_FAULT)
7734 break;
7735 }
7736 if (i == CPUWINDOWS_MAX) {
7737 panic("pmap_map_cpu_windows_copy: out of window\n");
7738 }
7739
7740 pte = pa_to_pte(ptoa(pn)) | ARM_PTE_TYPE | ARM_PTE_AF | ARM_PTE_NX | ARM_PTE_PNX;
7741
7742 pte |= wimg_to_pte(wimg_bits);
7743
7744 if (prot & VM_PROT_WRITE) {
7745 pte |= ARM_PTE_AP(AP_RWNA);
7746 } else {
7747 pte |= ARM_PTE_AP(AP_RONA);
7748 }
7749
7750 WRITE_PTE(ptep, pte);
7751 /*
7752 * Invalidate tlb. Cover nested cpu_copywindow_vaddr usage with the interrupted context
7753 * in pmap_unmap_cpu_windows_copy() after clearing the pte and before tlb invalidate.
7754 */
7755 PMAP_UPDATE_TLBS(kernel_pmap, cpu_copywindow_vaddr, cpu_copywindow_vaddr + PAGE_SIZE);
7756
7757 return(i);
7758 }
7759
7760 unsigned int
7761 pmap_map_cpu_windows_copy(
7762 ppnum_t pn,
7763 vm_prot_t prot,
7764 unsigned int wimg_bits)
7765 {
7766 return pmap_map_cpu_windows_copy_internal(pn, prot, wimg_bits);
7767 }
7768
7769 static void
7770 pmap_unmap_cpu_windows_copy_internal(
7771 unsigned int index)
7772 {
7773 pt_entry_t *ptep;
7774 unsigned int cpu_num;
7775 vm_offset_t cpu_copywindow_vaddr = 0;
7776
7777 cpu_num = pmap_get_cpu_data()->cpu_number;
7778
7779 cpu_copywindow_vaddr = pmap_cpu_windows_copy_addr(cpu_num, index);
7780 __asm__ volatile("dsb sy");
7781 ptep = pmap_pte(kernel_pmap, cpu_copywindow_vaddr);
7782 WRITE_PTE(ptep, ARM_PTE_TYPE_FAULT);
7783 PMAP_UPDATE_TLBS(kernel_pmap, cpu_copywindow_vaddr, cpu_copywindow_vaddr + PAGE_SIZE);
7784 }
7785
7786 void
7787 pmap_unmap_cpu_windows_copy(
7788 unsigned int index)
7789 {
7790 return pmap_unmap_cpu_windows_copy_internal(index);
7791 }
7792
7793 /*
7794 * Marked a pmap has nested
7795 */
7796 static void
7797 pmap_set_nested_internal(
7798 pmap_t pmap)
7799 {
7800 pmap->nested = TRUE;
7801 }
7802
7803 void
7804 pmap_set_nested(
7805 pmap_t pmap)
7806 {
7807 pmap_set_nested_internal(pmap);
7808 }
7809
7810 /*
7811 * kern_return_t pmap_nest(grand, subord, vstart, size)
7812 *
7813 * grand = the pmap that we will nest subord into
7814 * subord = the pmap that goes into the grand
7815 * vstart = start of range in pmap to be inserted
7816 * nstart = start of range in pmap nested pmap
7817 * size = Size of nest area (up to 16TB)
7818 *
7819 * Inserts a pmap into another. This is used to implement shared segments.
7820 *
7821 */
7822
7823 static kern_return_t
7824 pmap_nest_internal(
7825 pmap_t grand,
7826 pmap_t subord,
7827 addr64_t vstart,
7828 addr64_t nstart,
7829 uint64_t size)
7830 {
7831 kern_return_t kr = KERN_FAILURE;
7832 vm_map_offset_t vaddr, nvaddr;
7833 tt_entry_t *stte_p;
7834 tt_entry_t *gtte_p;
7835 unsigned int i;
7836 unsigned int num_tte;
7837 unsigned int nested_region_asid_bitmap_size;
7838 unsigned int* nested_region_asid_bitmap;
7839 int expand_options = 0;
7840
7841
7842 #if (__ARM_VMSA__ == 7)
7843 if (((size|vstart|nstart) & ARM_TT_L1_PT_OFFMASK) != 0x0ULL) {
7844 return KERN_INVALID_VALUE; /* Nest 4MB region */
7845 }
7846 #else
7847 if (((size|vstart|nstart) & (ARM_TT_L2_OFFMASK)) != 0x0ULL) {
7848 panic("pmap_nest() pmap %p has a nested pmap 0x%llx, 0x%llx, 0x%llx\n", grand, vstart, nstart, size);
7849 }
7850 #endif
7851
7852 if ((grand->nested_pmap != PMAP_NULL) && (grand->nested_pmap != subord)) {
7853 panic("pmap_nest() pmap %p has a nested pmap\n", grand);
7854 }
7855
7856 if (subord->nested_region_asid_bitmap == NULL) {
7857 nested_region_asid_bitmap_size = (unsigned int)(size>>ARM_TT_TWIG_SHIFT)/(sizeof(unsigned int)*NBBY);
7858
7859 nested_region_asid_bitmap = kalloc(nested_region_asid_bitmap_size*sizeof(unsigned int));
7860 bzero(nested_region_asid_bitmap, nested_region_asid_bitmap_size*sizeof(unsigned int));
7861
7862 PMAP_LOCK(subord);
7863 if (subord->nested_region_asid_bitmap == NULL) {
7864 subord->nested_region_asid_bitmap = nested_region_asid_bitmap;
7865 subord->nested_region_asid_bitmap_size = nested_region_asid_bitmap_size;
7866 subord->nested_region_subord_addr = nstart;
7867 subord->nested_region_size = (mach_vm_offset_t) size;
7868 nested_region_asid_bitmap = NULL;
7869 }
7870 PMAP_UNLOCK(subord);
7871 if (nested_region_asid_bitmap != NULL) {
7872 kfree(nested_region_asid_bitmap, nested_region_asid_bitmap_size*sizeof(unsigned int));
7873 }
7874 }
7875 if ((subord->nested_region_subord_addr + subord->nested_region_size) < (nstart+size)) {
7876 uint64_t new_size;
7877 unsigned int new_nested_region_asid_bitmap_size;
7878 unsigned int* new_nested_region_asid_bitmap;
7879
7880 nested_region_asid_bitmap = NULL;
7881 nested_region_asid_bitmap_size = 0;
7882 new_size = nstart + size - subord->nested_region_subord_addr;
7883
7884 /* We explicitly add 1 to the bitmap allocation size in order to avoid issues with truncation. */
7885 new_nested_region_asid_bitmap_size = (unsigned int)((new_size>>ARM_TT_TWIG_SHIFT)/(sizeof(unsigned int)*NBBY)) + 1;
7886
7887 new_nested_region_asid_bitmap = kalloc(new_nested_region_asid_bitmap_size*sizeof(unsigned int));
7888 PMAP_LOCK(subord);
7889 if (subord->nested_region_size < new_size) {
7890 bzero(new_nested_region_asid_bitmap, new_nested_region_asid_bitmap_size*sizeof(unsigned int));
7891 bcopy(subord->nested_region_asid_bitmap, new_nested_region_asid_bitmap, subord->nested_region_asid_bitmap_size);
7892 nested_region_asid_bitmap_size = subord->nested_region_asid_bitmap_size;
7893 nested_region_asid_bitmap = subord->nested_region_asid_bitmap;
7894 subord->nested_region_asid_bitmap = new_nested_region_asid_bitmap;
7895 subord->nested_region_asid_bitmap_size = new_nested_region_asid_bitmap_size;
7896 subord->nested_region_size = new_size;
7897 new_nested_region_asid_bitmap = NULL;
7898 }
7899 PMAP_UNLOCK(subord);
7900 if (nested_region_asid_bitmap != NULL)
7901 kfree(nested_region_asid_bitmap, nested_region_asid_bitmap_size*sizeof(unsigned int));
7902 if (new_nested_region_asid_bitmap != NULL)
7903 kfree(new_nested_region_asid_bitmap, new_nested_region_asid_bitmap_size*sizeof(unsigned int));
7904 }
7905
7906 PMAP_LOCK(subord);
7907 if (grand->nested_pmap == PMAP_NULL) {
7908 grand->nested_pmap = subord;
7909 grand->nested_region_grand_addr = vstart;
7910 grand->nested_region_subord_addr = nstart;
7911 grand->nested_region_size = (mach_vm_offset_t) size;
7912 } else {
7913 if ((grand->nested_region_grand_addr > vstart)) {
7914 panic("pmap_nest() pmap %p : attempt to nest outside the nested region\n", grand);
7915 }
7916 else if ((grand->nested_region_grand_addr + grand->nested_region_size) < (vstart+size)) {
7917 grand->nested_region_size = (mach_vm_offset_t)(vstart - grand->nested_region_grand_addr + size);
7918 }
7919 }
7920
7921 #if (__ARM_VMSA__ == 7)
7922 nvaddr = (vm_map_offset_t) nstart;
7923 vaddr = (vm_map_offset_t) vstart;
7924 num_tte = size >> ARM_TT_L1_SHIFT;
7925
7926 for (i = 0; i < num_tte; i++) {
7927 stte_p = pmap_tte(subord, nvaddr);
7928 if ((stte_p == (tt_entry_t *)NULL) || (((*stte_p) & ARM_TTE_TYPE_MASK) != ARM_TTE_TYPE_TABLE)) {
7929 PMAP_UNLOCK(subord);
7930 kr = pmap_expand(subord, nvaddr, expand_options, PMAP_TT_L2_LEVEL);
7931
7932 if (kr != KERN_SUCCESS) {
7933 PMAP_LOCK(grand);
7934 goto done;
7935 }
7936
7937 PMAP_LOCK(subord);
7938 }
7939 PMAP_UNLOCK(subord);
7940 PMAP_LOCK(grand);
7941 stte_p = pmap_tte(grand, vaddr);
7942 if (stte_p == (tt_entry_t *)NULL) {
7943 PMAP_UNLOCK(grand);
7944 kr = pmap_expand(grand, vaddr, expand_options, PMAP_TT_L1_LEVEL);
7945
7946 if (kr != KERN_SUCCESS) {
7947 PMAP_LOCK(grand);
7948 goto done;
7949 }
7950 } else {
7951 PMAP_UNLOCK(grand);
7952 kr = KERN_SUCCESS;
7953 }
7954 PMAP_LOCK(subord);
7955
7956
7957 nvaddr += ARM_TT_L1_SIZE;
7958 vaddr += ARM_TT_L1_SIZE;
7959 }
7960
7961 #else
7962 nvaddr = (vm_map_offset_t) nstart;
7963 num_tte = (unsigned int)(size >> ARM_TT_L2_SHIFT);
7964
7965 for (i = 0; i < num_tte; i++) {
7966 stte_p = pmap_tt2e(subord, nvaddr);
7967 if (stte_p == PT_ENTRY_NULL || *stte_p == ARM_TTE_EMPTY) {
7968 PMAP_UNLOCK(subord);
7969 kr = pmap_expand(subord, nvaddr, expand_options, PMAP_TT_L3_LEVEL);
7970
7971 if (kr != KERN_SUCCESS) {
7972 PMAP_LOCK(grand);
7973 goto done;
7974 }
7975
7976 PMAP_LOCK(subord);
7977 }
7978 nvaddr += ARM_TT_L2_SIZE;
7979 }
7980 #endif
7981 PMAP_UNLOCK(subord);
7982
7983 /*
7984 * copy tte's from subord pmap into grand pmap
7985 */
7986
7987 PMAP_LOCK(grand);
7988 nvaddr = (vm_map_offset_t) nstart;
7989 vaddr = (vm_map_offset_t) vstart;
7990
7991
7992 #if (__ARM_VMSA__ == 7)
7993 for (i = 0; i < num_tte; i++) {
7994
7995 stte_p = pmap_tte(subord, nvaddr);
7996 gtte_p = pmap_tte(grand, vaddr);
7997 *gtte_p = *stte_p;
7998
7999 nvaddr += ARM_TT_L1_SIZE;
8000 vaddr += ARM_TT_L1_SIZE;
8001 }
8002 #else
8003 for (i = 0; i < num_tte; i++) {
8004
8005 stte_p = pmap_tt2e(subord, nstart);
8006 gtte_p = pmap_tt2e(grand, vaddr);
8007 if (gtte_p == PT_ENTRY_NULL) {
8008 PMAP_UNLOCK(grand);
8009 kr = pmap_expand(grand, vaddr, expand_options, PMAP_TT_L2_LEVEL);
8010 PMAP_LOCK(grand);
8011
8012 if (kr != KERN_SUCCESS) {
8013 goto done;
8014 }
8015
8016 gtte_p = pmap_tt2e(grand, vaddr);
8017 }
8018 *gtte_p = *stte_p;
8019 vaddr += ARM_TT_L2_SIZE;
8020 nstart += ARM_TT_L2_SIZE;
8021 }
8022 #endif
8023
8024 kr = KERN_SUCCESS;
8025 done:
8026
8027 #ifndef __ARM_L1_PTW__
8028 CleanPoU_DcacheRegion((vm_offset_t) pmap_tte(grand, vstart), num_tte * sizeof(tt_entry_t));
8029 #endif
8030
8031 #if (__ARM_VMSA__ > 7)
8032 /*
8033 * check for overflow on LP64 arch
8034 */
8035 assert((size & 0xFFFFFFFF00000000ULL) == 0);
8036 #endif
8037 PMAP_UPDATE_TLBS(grand, vstart, vstart + size);
8038
8039 PMAP_UNLOCK(grand);
8040 return kr;
8041 }
8042
8043 kern_return_t pmap_nest(
8044 pmap_t grand,
8045 pmap_t subord,
8046 addr64_t vstart,
8047 addr64_t nstart,
8048 uint64_t size)
8049 {
8050 kern_return_t kr = KERN_FAILURE;
8051
8052 PMAP_TRACE(PMAP_CODE(PMAP__NEST) | DBG_FUNC_START,
8053 VM_KERNEL_ADDRHIDE(grand), VM_KERNEL_ADDRHIDE(subord),
8054 VM_KERNEL_ADDRHIDE(vstart));
8055
8056 kr = pmap_nest_internal(grand, subord, vstart, nstart, size);
8057
8058 PMAP_TRACE(PMAP_CODE(PMAP__NEST) | DBG_FUNC_END, kr);
8059
8060 return kr;
8061 }
8062
8063 /*
8064 * kern_return_t pmap_unnest(grand, vaddr)
8065 *
8066 * grand = the pmap that we will nest subord into
8067 * vaddr = start of range in pmap to be unnested
8068 * size = size of range in pmap to be unnested
8069 *
8070 */
8071
8072 kern_return_t
8073 pmap_unnest(
8074 pmap_t grand,
8075 addr64_t vaddr,
8076 uint64_t size)
8077 {
8078 return(pmap_unnest_options(grand, vaddr, size, 0));
8079 }
8080
8081 static kern_return_t
8082 pmap_unnest_options_internal(
8083 pmap_t grand,
8084 addr64_t vaddr,
8085 uint64_t size,
8086 unsigned int option)
8087 {
8088 vm_map_offset_t start;
8089 vm_map_offset_t addr;
8090 tt_entry_t *tte_p;
8091 unsigned int current_index;
8092 unsigned int start_index;
8093 unsigned int max_index;
8094 unsigned int num_tte;
8095 unsigned int i;
8096
8097 #if (__ARM_VMSA__ == 7)
8098 if (((size|vaddr) & ARM_TT_L1_PT_OFFMASK) != 0x0ULL) {
8099 panic("pmap_unnest(): unaligned request\n");
8100 }
8101 #else
8102 if (((size|vaddr) & ARM_TT_L2_OFFMASK) != 0x0ULL) {
8103 panic("pmap_unnest(): unaligned request\n");
8104 }
8105 #endif
8106
8107 if ((option & PMAP_UNNEST_CLEAN) == 0)
8108 {
8109 PMAP_LOCK(grand->nested_pmap);
8110
8111 start = vaddr - grand->nested_region_grand_addr + grand->nested_region_subord_addr ;
8112 start_index = (unsigned int)((vaddr - grand->nested_region_grand_addr) >> ARM_TT_TWIG_SHIFT);
8113 max_index = (unsigned int)(start_index + (size >> ARM_TT_TWIG_SHIFT));
8114 num_tte = (unsigned int)(size >> ARM_TT_TWIG_SHIFT);
8115
8116 if (size > grand->nested_region_size) {
8117 panic("pmap_unnest() pmap %p %llu, %llu\n", grand, size, (uint64_t)grand->nested_region_size);
8118 }
8119
8120 for (current_index = start_index, addr = start; current_index < max_index; current_index++) {
8121 pt_entry_t *bpte, *epte, *cpte;
8122
8123
8124 if(!testbit(current_index, (int *)grand->nested_pmap->nested_region_asid_bitmap)) {
8125
8126 setbit(current_index, (int *)grand->nested_pmap->nested_region_asid_bitmap);
8127 bpte = pmap_pte(grand->nested_pmap, addr);
8128 epte = bpte + (ARM_TT_LEAF_INDEX_MASK>>ARM_TT_LEAF_SHIFT);
8129
8130 for (cpte = bpte; cpte <= epte; cpte++) {
8131 pmap_paddr_t pa;
8132 int pai=0;
8133 boolean_t managed=FALSE;
8134 pt_entry_t spte;
8135
8136 if ((*cpte != ARM_PTE_TYPE_FAULT)
8137 && (!ARM_PTE_IS_COMPRESSED(*cpte))) {
8138
8139 spte = *cpte;
8140 while (!managed) {
8141 pa = pte_to_pa(spte);
8142 if (!pa_valid(pa))
8143 break;
8144 pai = (int)pa_index(pa);
8145 LOCK_PVH(pai);
8146 spte = *cpte;
8147 pa = pte_to_pa(spte);
8148 if (pai == (int)pa_index(pa)) {
8149 managed =TRUE;
8150 break; // Leave the PVH locked as we'll unlock it after we update the PTE
8151 }
8152 UNLOCK_PVH(pai);
8153 }
8154
8155 if (((spte & ARM_PTE_NG) != ARM_PTE_NG)) {
8156
8157 WRITE_PTE(cpte, (spte | ARM_PTE_NG));
8158 }
8159
8160 if (managed)
8161 {
8162 ASSERT_PVH_LOCKED(pai);
8163 UNLOCK_PVH(pai);
8164 }
8165 }
8166 }
8167 }
8168
8169 addr += ARM_TT_TWIG_SIZE;
8170
8171 #ifndef __ARM_L1_PTW__
8172 CleanPoU_DcacheRegion((vm_offset_t) pmap_pte(grand->nested_pmap, start), num_tte * sizeof(tt_entry_t));
8173 #endif
8174 PMAP_UPDATE_TLBS(grand->nested_pmap, start, start + size);
8175 }
8176
8177 PMAP_UNLOCK(grand->nested_pmap);
8178 }
8179
8180 PMAP_LOCK(grand);
8181
8182 /*
8183 * invalidate all pdes for segment at vaddr in pmap grand
8184 */
8185 start = vaddr;
8186 addr = vaddr;
8187
8188 num_tte = (unsigned int)(size >> ARM_TT_TWIG_SHIFT);
8189
8190 for (i = 0; i < num_tte; i++) {
8191 tte_p = pmap_tte(grand, addr);
8192 *tte_p = ARM_TTE_TYPE_FAULT;
8193
8194 addr += ARM_TT_TWIG_SIZE;
8195 }
8196
8197 #ifndef __ARM_L1_PTW__
8198 CleanPoU_DcacheRegion((vm_offset_t) pmap_tte(grand, start), num_tte * sizeof(tt_entry_t));
8199 #endif
8200 PMAP_UPDATE_TLBS(grand, start, start + size);
8201
8202 PMAP_UNLOCK(grand);
8203
8204 return KERN_SUCCESS;
8205 }
8206
8207 kern_return_t
8208 pmap_unnest_options(
8209 pmap_t grand,
8210 addr64_t vaddr,
8211 uint64_t size,
8212 unsigned int option)
8213 {
8214 kern_return_t kr = KERN_FAILURE;
8215
8216 PMAP_TRACE(PMAP_CODE(PMAP__UNNEST) | DBG_FUNC_START,
8217 VM_KERNEL_ADDRHIDE(grand), VM_KERNEL_ADDRHIDE(vaddr));
8218
8219 kr = pmap_unnest_options_internal(grand, vaddr, size, option);
8220
8221 PMAP_TRACE(PMAP_CODE(PMAP__UNNEST) | DBG_FUNC_END, kr);
8222
8223 return kr;
8224 }
8225
8226 boolean_t
8227 pmap_adjust_unnest_parameters(
8228 __unused pmap_t p,
8229 __unused vm_map_offset_t *s,
8230 __unused vm_map_offset_t *e)
8231 {
8232 return TRUE; /* to get to log_unnest_badness()... */
8233 }
8234
8235 /*
8236 * disable no-execute capability on
8237 * the specified pmap
8238 */
8239 #if DEVELOPMENT || DEBUG
8240 void
8241 pmap_disable_NX(
8242 pmap_t pmap)
8243 {
8244 pmap->nx_enabled = FALSE;
8245 }
8246 #else
8247 void
8248 pmap_disable_NX(
8249 __unused pmap_t pmap)
8250 {
8251 }
8252 #endif
8253
8254 void
8255 pt_fake_zone_init(
8256 int zone_index)
8257 {
8258 pt_fake_zone_index = zone_index;
8259 }
8260
8261 void
8262 pt_fake_zone_info(
8263 int *count,
8264 vm_size_t *cur_size, vm_size_t *max_size, vm_size_t *elem_size, vm_size_t *alloc_size,
8265 uint64_t *sum_size, int *collectable, int *exhaustable, int *caller_acct)
8266 {
8267 *count = inuse_pmap_pages_count;
8268 *cur_size = PAGE_SIZE * (inuse_pmap_pages_count);
8269 *max_size = PAGE_SIZE * (inuse_pmap_pages_count + vm_page_inactive_count + vm_page_active_count + vm_page_free_count);
8270 *elem_size = PAGE_SIZE;
8271 *alloc_size = PAGE_SIZE;
8272 *sum_size = (alloc_pmap_pages_count) * PAGE_SIZE;
8273
8274 *collectable = 1;
8275 *exhaustable = 0;
8276 *caller_acct = 1;
8277 }
8278
8279 /*
8280 * flush a range of hardware TLB entries.
8281 * NOTE: assumes the smallest TLB entry in use will be for
8282 * an ARM small page (4K).
8283 */
8284
8285 #define ARM_FULL_TLB_FLUSH_THRESHOLD 64
8286 #define ARM64_FULL_TLB_FLUSH_THRESHOLD 256
8287
8288 static void
8289 flush_mmu_tlb_region_asid(
8290 vm_offset_t va,
8291 unsigned length,
8292 pmap_t pmap)
8293 {
8294 #if (__ARM_VMSA__ == 7)
8295 vm_offset_t end = va + length;
8296 uint32_t asid;
8297
8298 asid = pmap->asid;
8299
8300 if (length / ARM_SMALL_PAGE_SIZE > ARM_FULL_TLB_FLUSH_THRESHOLD) {
8301 boolean_t flush_all = FALSE;
8302
8303 if ((asid == 0) || (pmap->nested == TRUE))
8304 flush_all = TRUE;
8305 if (flush_all)
8306 flush_mmu_tlb();
8307 else
8308 flush_mmu_tlb_asid(asid);
8309
8310 return;
8311 }
8312 if (pmap->nested == TRUE) {
8313 #if !__ARM_MP_EXT__
8314 flush_mmu_tlb();
8315 #else
8316 va = arm_trunc_page(va);
8317 while (va < end) {
8318 flush_mmu_tlb_mva_entries(va);
8319 va += ARM_SMALL_PAGE_SIZE;
8320 }
8321 #endif
8322 return;
8323 }
8324 va = arm_trunc_page(va) | (asid & 0xff);
8325 flush_mmu_tlb_entries(va, end);
8326
8327 #else
8328 vm_offset_t end = va + length;
8329 uint32_t asid;
8330
8331 asid = pmap->asid;
8332
8333 if ((length >> ARM_TT_L3_SHIFT) > ARM64_FULL_TLB_FLUSH_THRESHOLD) {
8334 boolean_t flush_all = FALSE;
8335
8336 if ((asid == 0) || (pmap->nested == TRUE))
8337 flush_all = TRUE;
8338 if (flush_all)
8339 flush_mmu_tlb();
8340 else
8341 flush_mmu_tlb_asid((uint64_t)asid << TLBI_ASID_SHIFT);
8342 return;
8343 }
8344 va = tlbi_asid(asid) | tlbi_addr(va);
8345 end = tlbi_asid(asid) | tlbi_addr(end);
8346 if (pmap->nested == TRUE) {
8347 flush_mmu_tlb_allentries(va, end);
8348 } else {
8349 flush_mmu_tlb_entries(va, end);
8350 }
8351
8352 #endif
8353 }
8354
8355 void
8356 flush_mmu_tlb_region(
8357 vm_offset_t va,
8358 unsigned length)
8359 {
8360 flush_mmu_tlb_region_asid(va, length, kernel_pmap);
8361 }
8362
8363 unsigned int
8364 pmap_cache_attributes(
8365 ppnum_t pn)
8366 {
8367 pmap_paddr_t paddr;
8368 int pai;
8369 unsigned int result;
8370 pp_attr_t pp_attr_current;
8371
8372 paddr = ptoa(pn);
8373
8374 if ((paddr >= io_rgn_start) && (paddr < io_rgn_end)) {
8375 unsigned int attr = IO_ATTR_WIMG(io_attr_table[(paddr - io_rgn_start) / io_rgn_granule]);
8376 if (attr)
8377 return attr;
8378 else
8379 return (VM_WIMG_IO);
8380 }
8381
8382
8383 if (!pmap_initialized) {
8384 if ((paddr >= gPhysBase) && (paddr < gPhysBase+gPhysSize))
8385 return (VM_WIMG_DEFAULT);
8386 else
8387 return (VM_WIMG_IO);
8388 }
8389
8390
8391 if (!pa_valid(paddr))
8392 return (VM_WIMG_IO);
8393
8394 result = VM_WIMG_DEFAULT;
8395
8396 pai = (int)pa_index(paddr);
8397
8398 pp_attr_current = pp_attr_table[pai];
8399 if (pp_attr_current & PP_ATTR_WIMG_MASK)
8400 result = pp_attr_current & PP_ATTR_WIMG_MASK;
8401 return result;
8402 }
8403
8404 static boolean_t
8405 pmap_batch_set_cache_attributes_internal(
8406 ppnum_t pn,
8407 unsigned int cacheattr,
8408 unsigned int page_cnt,
8409 unsigned int page_index,
8410 boolean_t doit,
8411 unsigned int *res)
8412 {
8413 pmap_paddr_t paddr;
8414 int pai;
8415 pp_attr_t pp_attr_current;
8416 pp_attr_t pp_attr_template;
8417 unsigned int wimg_bits_prev, wimg_bits_new;
8418
8419 if (cacheattr & VM_WIMG_USE_DEFAULT)
8420 cacheattr = VM_WIMG_DEFAULT;
8421
8422 if ((doit == FALSE) && (*res == 0)) {
8423 *res = page_cnt;
8424 if (platform_cache_batch_wimg(cacheattr & (VM_WIMG_MASK), page_cnt<<PAGE_SHIFT) == FALSE) {
8425 return FALSE;
8426 }
8427 }
8428
8429 paddr = ptoa(pn);
8430
8431 if (!pa_valid(paddr)) {
8432 panic("pmap_batch_set_cache_attributes(): pn 0x%08x not managed\n", pn);
8433 }
8434
8435 pai = (int)pa_index(paddr);
8436
8437 if (doit)
8438 LOCK_PVH(pai);
8439
8440 pp_attr_current = pp_attr_table[pai];
8441 wimg_bits_prev = VM_WIMG_DEFAULT;
8442 if (pp_attr_current & PP_ATTR_WIMG_MASK)
8443 wimg_bits_prev = pp_attr_current & PP_ATTR_WIMG_MASK;
8444
8445 pp_attr_template = (pp_attr_current & ~PP_ATTR_WIMG_MASK) | PP_ATTR_WIMG(cacheattr & (VM_WIMG_MASK));
8446
8447 if (doit)
8448 pp_attr_table[pai] = pp_attr_template;
8449
8450 wimg_bits_new = VM_WIMG_DEFAULT;
8451 if (pp_attr_template & PP_ATTR_WIMG_MASK)
8452 wimg_bits_new = pp_attr_template & PP_ATTR_WIMG_MASK;
8453
8454 if (doit) {
8455 if (wimg_bits_new != wimg_bits_prev)
8456 pmap_update_cache_attributes_locked(pn, cacheattr);
8457 UNLOCK_PVH(pai);
8458 } else {
8459 if (wimg_bits_new == VM_WIMG_COPYBACK) {
8460 return FALSE;
8461 }
8462 if (wimg_bits_prev == wimg_bits_new) {
8463 *res = *res-1;
8464 if (!platform_cache_batch_wimg(wimg_bits_new, (*res)<<PAGE_SHIFT)) {
8465 return FALSE;
8466 }
8467 }
8468 return TRUE;
8469 }
8470
8471 if (page_cnt == (page_index+1)) {
8472 wimg_bits_prev = VM_WIMG_COPYBACK;
8473 if (((page_cnt == (page_index+1)) && (wimg_bits_prev != wimg_bits_new))
8474 && ((wimg_bits_prev == VM_WIMG_COPYBACK)
8475 || ((wimg_bits_prev == VM_WIMG_INNERWBACK)
8476 && (wimg_bits_new != VM_WIMG_COPYBACK))
8477 || ((wimg_bits_prev == VM_WIMG_WTHRU)
8478 && ((wimg_bits_new != VM_WIMG_COPYBACK) || (wimg_bits_new != VM_WIMG_INNERWBACK))))) {
8479 platform_cache_flush_wimg(wimg_bits_new);
8480 }
8481 }
8482
8483 return TRUE;
8484 };
8485
8486 boolean_t
8487 pmap_batch_set_cache_attributes(
8488 ppnum_t pn,
8489 unsigned int cacheattr,
8490 unsigned int page_cnt,
8491 unsigned int page_index,
8492 boolean_t doit,
8493 unsigned int *res)
8494 {
8495 return pmap_batch_set_cache_attributes_internal(pn, cacheattr, page_cnt, page_index, doit, res);
8496 }
8497
8498 static void
8499 pmap_set_cache_attributes_internal(
8500 ppnum_t pn,
8501 unsigned int cacheattr)
8502 {
8503 pmap_paddr_t paddr;
8504 int pai;
8505 pp_attr_t pp_attr_current;
8506 pp_attr_t pp_attr_template;
8507 unsigned int wimg_bits_prev, wimg_bits_new;
8508
8509 paddr = ptoa(pn);
8510
8511 if (!pa_valid(paddr)) {
8512 return; /* Not a managed page. */
8513 }
8514
8515 if (cacheattr & VM_WIMG_USE_DEFAULT)
8516 cacheattr = VM_WIMG_DEFAULT;
8517
8518 pai = (int)pa_index(paddr);
8519
8520 LOCK_PVH(pai);
8521
8522 pp_attr_current = pp_attr_table[pai];
8523 wimg_bits_prev = VM_WIMG_DEFAULT;
8524 if (pp_attr_current & PP_ATTR_WIMG_MASK)
8525 wimg_bits_prev = pp_attr_current & PP_ATTR_WIMG_MASK;
8526
8527 pp_attr_template = (pp_attr_current & ~PP_ATTR_WIMG_MASK) | PP_ATTR_WIMG(cacheattr & (VM_WIMG_MASK)) ;
8528
8529 pp_attr_table[pai] = pp_attr_template;
8530 wimg_bits_new = VM_WIMG_DEFAULT;
8531 if (pp_attr_template & PP_ATTR_WIMG_MASK)
8532 wimg_bits_new = pp_attr_template & PP_ATTR_WIMG_MASK;
8533
8534 if (wimg_bits_new != wimg_bits_prev)
8535 pmap_update_cache_attributes_locked(pn, cacheattr);
8536
8537 UNLOCK_PVH(pai);
8538
8539 if ((wimg_bits_prev != wimg_bits_new)
8540 && ((wimg_bits_prev == VM_WIMG_COPYBACK)
8541 || ((wimg_bits_prev == VM_WIMG_INNERWBACK)
8542 && (wimg_bits_new != VM_WIMG_COPYBACK))
8543 || ((wimg_bits_prev == VM_WIMG_WTHRU)
8544 && ((wimg_bits_new != VM_WIMG_COPYBACK) || (wimg_bits_new != VM_WIMG_INNERWBACK)))))
8545 pmap_sync_page_attributes_phys(pn);
8546
8547 }
8548
8549 void
8550 pmap_set_cache_attributes(
8551 ppnum_t pn,
8552 unsigned int cacheattr)
8553 {
8554 pmap_set_cache_attributes_internal(pn, cacheattr);
8555 }
8556
8557 void
8558 pmap_update_cache_attributes_locked(
8559 ppnum_t ppnum,
8560 unsigned attributes)
8561 {
8562 pmap_paddr_t phys = ptoa(ppnum);
8563 pv_entry_t *pve_p;
8564 pt_entry_t *pte_p;
8565 pv_entry_t **pv_h;
8566 pt_entry_t tmplate;
8567 unsigned int pai;
8568
8569 #if (__ARM_VMSA__ == 7)
8570 #define ARM_PTE_SHMASK ARM_PTE_SH
8571 #endif
8572
8573 #if __ARM_PTE_PHYSMAP__
8574 vm_offset_t kva = phystokv(phys);
8575 pte_p = pmap_pte(kernel_pmap, kva);
8576
8577 tmplate = *pte_p;
8578 tmplate &= ~(ARM_PTE_ATTRINDXMASK | ARM_PTE_SHMASK);
8579 tmplate |= wimg_to_pte(attributes);
8580
8581 WRITE_PTE(pte_p, tmplate);
8582 PMAP_UPDATE_TLBS(kernel_pmap, kva, kva + PAGE_SIZE);
8583 #endif
8584
8585 pai = (unsigned int)pa_index(phys);
8586
8587 pv_h = pai_to_pvh(pai);
8588
8589 pte_p = PT_ENTRY_NULL;
8590 pve_p = PV_ENTRY_NULL;
8591 if (pvh_test_type(pv_h, PVH_TYPE_PTEP)) {
8592 pte_p = pvh_ptep(pv_h);
8593 } else if (pvh_test_type(pv_h, PVH_TYPE_PVEP)) {
8594 pve_p = pvh_list(pv_h);
8595 pte_p = PT_ENTRY_NULL;
8596 }
8597
8598 while ((pve_p != PV_ENTRY_NULL) || (pte_p != PT_ENTRY_NULL)) {
8599 vm_map_address_t va;
8600 pmap_t pmap;
8601
8602 if (pve_p != PV_ENTRY_NULL)
8603 pte_p = pve_get_ptep(pve_p);
8604
8605 pmap = ptep_get_pmap(pte_p);
8606 va = ptep_get_va(pte_p);
8607
8608 tmplate = *pte_p;
8609 tmplate &= ~(ARM_PTE_ATTRINDXMASK | ARM_PTE_NX | ARM_PTE_PNX | ARM_PTE_SHMASK);
8610 tmplate |= wimg_to_pte(attributes);
8611
8612 WRITE_PTE(pte_p, tmplate);
8613 PMAP_UPDATE_TLBS(pmap, va, va + PAGE_SIZE);
8614
8615 pte_p = PT_ENTRY_NULL;
8616 if (pve_p != PV_ENTRY_NULL)
8617 pve_p = PVE_NEXT_PTR(pve_next(pve_p));
8618
8619 }
8620 }
8621
8622 #if (__ARM_VMSA__ == 7)
8623 vm_map_address_t
8624 pmap_create_sharedpage(
8625 void)
8626 {
8627 pmap_paddr_t pa;
8628 kern_return_t kr;
8629
8630 (void) pmap_pages_alloc(&pa, PAGE_SIZE, 0);
8631 memset((char *) phystokv(pa), 0, PAGE_SIZE);
8632
8633 kr = pmap_enter(kernel_pmap, _COMM_PAGE_BASE_ADDRESS, atop(pa), VM_PROT_READ | VM_PROT_WRITE, VM_PROT_NONE, VM_WIMG_USE_DEFAULT, TRUE);
8634 assert(kr == KERN_SUCCESS);
8635
8636 return((vm_map_address_t)phystokv(pa));
8637
8638 }
8639 #else
8640 static void
8641 pmap_update_tt3e(
8642 pmap_t pmap,
8643 vm_address_t address,
8644 tt_entry_t template)
8645 {
8646 tt_entry_t *ptep, pte;
8647
8648 ptep = pmap_tt3e(pmap, address);
8649 if (ptep == NULL) {
8650 panic("%s: no ptep?\n", __FUNCTION__);
8651 }
8652
8653 pte = *ptep;
8654 pte = tte_to_pa(pte) | template;
8655 WRITE_PTE(ptep, pte);
8656 }
8657
8658 /* Note absence of non-global bit */
8659 #define PMAP_COMM_PAGE_PTE_TEMPLATE (ARM_PTE_TYPE_VALID \
8660 | ARM_PTE_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) \
8661 | ARM_PTE_SH(SH_INNER_MEMORY) | ARM_PTE_NX \
8662 | ARM_PTE_PNX | ARM_PTE_AP(AP_RORO) | ARM_PTE_AF)
8663
8664 vm_map_address_t
8665 pmap_create_sharedpage(
8666 void
8667 )
8668 {
8669 kern_return_t kr;
8670 pmap_paddr_t pa = 0;
8671
8672
8673 kr = pmap_expand(kernel_pmap, _COMM_PAGE64_BASE_ADDRESS, 0, PMAP_TT_L3_LEVEL);
8674 assert(kr == KERN_SUCCESS);
8675
8676 (void) pmap_pages_alloc(&pa, PAGE_SIZE, 0);
8677
8678 memset((char *) phystokv(pa), 0, PAGE_SIZE);
8679
8680 /*
8681 * This is the mapping which U64 will refer to.
8682 * Create with common path, update to be non-global and user-readable.
8683 */
8684 kr = pmap_enter(kernel_pmap, _COMM_PAGE64_BASE_ADDRESS, (ppnum_t)atop(pa), VM_PROT_READ, VM_PROT_NONE, VM_WIMG_USE_DEFAULT, TRUE);
8685 assert(kr == KERN_SUCCESS);
8686 pmap_update_tt3e(kernel_pmap, _COMM_PAGE64_BASE_ADDRESS, PMAP_COMM_PAGE_PTE_TEMPLATE);
8687
8688 /*
8689 * With PAN enabled kernel drivers can no longer use the previous mapping which is user readable
8690 * They should use the following mapping instead
8691 */
8692 kr = pmap_expand(kernel_pmap, _COMM_PRIV_PAGE64_BASE_ADDRESS, 0, PMAP_TT_L3_LEVEL);
8693 assert(kr == KERN_SUCCESS);
8694 kr = pmap_enter(kernel_pmap, _COMM_PRIV_PAGE64_BASE_ADDRESS, (ppnum_t)atop(pa), VM_PROT_READ, VM_PROT_NONE, VM_WIMG_USE_DEFAULT, TRUE);
8695 assert(kr == KERN_SUCCESS);
8696
8697 /*
8698 * Preferrable to just use a single entry,but we consume
8699 * the full entry 0 of the L1 page table for U32 (i.e.
8700 * for the 1GB of user address space), so we have no choice
8701 * but to burn another L1 entry for the shared page... unless
8702 * something clever comes to us. For the short term (i.e.
8703 * bringup) do a kind of forced nesting (we don't have a
8704 * notion of nesting more than one pmap, and the shared cache
8705 * wins). In effect, just allocate a pmap, fill it out
8706 * to include the one entry we care about, and then
8707 * use its L1 pointer in U32 TTBR0 page tables.
8708 *
8709 * Note that we update parameters of entry for our unique
8710 * needs (NG entry, etc.).
8711 */
8712 u32_sharedpage_pmap = pmap_create(NULL, 0x0, FALSE);
8713 assert(u32_sharedpage_pmap != NULL);
8714 kr = pmap_enter(u32_sharedpage_pmap, _COMM_PAGE32_BASE_ADDRESS, (ppnum_t)atop(pa), VM_PROT_READ, VM_PROT_NONE, VM_WIMG_USE_DEFAULT, TRUE);
8715 assert(kr == KERN_SUCCESS);
8716 pmap_update_tt3e(u32_sharedpage_pmap, _COMM_PAGE32_BASE_ADDRESS, PMAP_COMM_PAGE_PTE_TEMPLATE);
8717
8718 /* For manipulation in kernel, go straight to physical page */
8719 sharedpage_rw_addr = phystokv(pa);
8720 return((vm_map_address_t)sharedpage_rw_addr);
8721 }
8722
8723 static void
8724 pmap_insert_sharedpage_internal(
8725 pmap_t pmap)
8726 {
8727 #if (ARM_PGSHIFT == 14) && !__ARM64_TWO_LEVEL_PMAP__
8728 kern_return_t kr;
8729 #endif
8730 pt_entry_t *ttep, *src_ttep;
8731 #if _COMM_PAGE_AREA_LENGTH != PAGE_SIZE
8732 #error We assume a single page.
8733 #endif
8734
8735 if (pmap_is_64bit(pmap)) {
8736 /* Already in kernel pmap */
8737 return;
8738 }
8739
8740 PMAP_LOCK(pmap);
8741
8742 /*
8743 * For 4KB pages, we can force the commpage to nest at the level one
8744 * page table, as each entry is 1GB (i.e, there will be no overlap
8745 * with regular userspace mappings). For 16KB pages, each level one
8746 * entry is 64GB, so we must go to the second level entry (32MB) in
8747 * order to nest.
8748 */
8749 #if (ARM_PGSHIFT == 12)
8750 #if __ARM64_TWO_LEVEL_PMAP__
8751 #error A two level page table with a page shift of 12 is not currently supported
8752 #endif
8753 /* Just slam in the L1 entry. */
8754 ttep = pmap_tt1e(pmap, _COMM_PAGE32_BASE_ADDRESS);
8755 if (*ttep != ARM_PTE_EMPTY) {
8756 panic("%s: Found something mapped at the commpage address?!", __FUNCTION__);
8757 }
8758
8759 src_ttep = pmap_tt1e(u32_sharedpage_pmap, _COMM_PAGE32_BASE_ADDRESS);
8760 #elif (ARM_PGSHIFT == 14)
8761 #if !__ARM64_TWO_LEVEL_PMAP__
8762 /* Allocate for the L2 entry if necessary, and slam it into place. */
8763 /*
8764 * As long as we are use a three level page table, the first level
8765 * should always exist, so we don't need to check for it.
8766 */
8767 while (*pmap_tt1e(pmap, _COMM_PAGE32_BASE_ADDRESS) == ARM_PTE_EMPTY) {
8768 PMAP_UNLOCK(pmap);
8769
8770 kr = pmap_expand(pmap, _COMM_PAGE32_BASE_ADDRESS, 0, PMAP_TT_L2_LEVEL);
8771
8772 if (kr != KERN_SUCCESS) {
8773 panic("Failed to pmap_expand for 32-bit commpage, pmap=%p", pmap);
8774 }
8775
8776 PMAP_LOCK(pmap);
8777 }
8778 #endif
8779
8780 ttep = pmap_tt2e(pmap, _COMM_PAGE32_BASE_ADDRESS);
8781 if (*ttep != ARM_PTE_EMPTY) {
8782 panic("%s: Found something mapped at the commpage address?!", __FUNCTION__);
8783 }
8784
8785 src_ttep = pmap_tt2e(u32_sharedpage_pmap, _COMM_PAGE32_BASE_ADDRESS);
8786 #endif
8787
8788 *ttep = *src_ttep;
8789 #ifndef __ARM_L1_PTW__
8790 CleanPoU_DcacheRegion((vm_offset_t) ttep, sizeof(tt_entry_t));
8791 #endif
8792 flush_mmu_tlb_region(_COMM_PAGE32_BASE_ADDRESS, PAGE_SIZE);
8793 #if (ARM_PGSHIFT == 12) && !__ARM64_TWO_LEVEL_PMAP__
8794 flush_mmu_tlb_entry(tlbi_addr(_COMM_PAGE32_BASE_ADDRESS & ~ARM_TT_L1_OFFMASK) | tlbi_asid(pmap->asid));
8795 #elif (ARM_PGSHIFT == 14)
8796 flush_mmu_tlb_entry(tlbi_addr(_COMM_PAGE32_BASE_ADDRESS & ~ARM_TT_L2_OFFMASK) | tlbi_asid(pmap->asid));
8797 #endif
8798
8799 PMAP_UNLOCK(pmap);
8800 }
8801
8802 static void
8803 pmap_sharedpage_flush_32_to_64(
8804 void)
8805 {
8806 flush_mmu_tlb_region(_COMM_PAGE32_BASE_ADDRESS, PAGE_SIZE);
8807 }
8808
8809 static void
8810 pmap_unmap_sharedpage32(
8811 pmap_t pmap)
8812 {
8813 pt_entry_t *ttep;
8814
8815 #if _COMM_PAGE_AREA_LENGTH != PAGE_SIZE
8816 #error We assume a single page.
8817 #endif
8818
8819 #if (ARM_PGSHIFT == 12)
8820 #if __ARM64_TWO_LEVEL_PMAP__
8821 #error A two level page table with a page shift of 12 is not currently supported
8822 #endif
8823 ttep = pmap_tt1e(pmap, _COMM_PAGE32_BASE_ADDRESS);
8824 if (ttep == NULL) {
8825 return;
8826 }
8827
8828 /* It had better be mapped to the shared page */
8829 if (*ttep != ARM_TTE_EMPTY && *ttep != *pmap_tt1e(u32_sharedpage_pmap, _COMM_PAGE32_BASE_ADDRESS)) {
8830 panic("%s: Something other than commpage mapped in shared page slot?", __FUNCTION__);
8831 }
8832 #elif (ARM_PGSHIFT == 14)
8833 ttep = pmap_tt2e(pmap, _COMM_PAGE32_BASE_ADDRESS);
8834 if (ttep == NULL) {
8835 return;
8836 }
8837
8838 /* It had better be mapped to the shared page */
8839 if (*ttep != ARM_TTE_EMPTY && *ttep != *pmap_tt2e(u32_sharedpage_pmap, _COMM_PAGE32_BASE_ADDRESS)) {
8840 panic("%s: Something other than commpage mapped in shared page slot?", __FUNCTION__);
8841 }
8842 #endif
8843
8844 *ttep = ARM_TTE_EMPTY;
8845 flush_mmu_tlb_region(_COMM_PAGE32_BASE_ADDRESS, PAGE_SIZE);
8846
8847 #if (ARM_PGSHIFT == 12)
8848 #if __ARM64_TWO_LEVEL_PMAP__
8849 #error A two level page table with a page shift of 12 is not currently supported
8850 #endif
8851 flush_mmu_tlb_entry(tlbi_addr(_COMM_PAGE32_BASE_ADDRESS & ~ARM_TT_L1_OFFMASK) | tlbi_asid(pmap->asid));
8852 #elif (ARM_PGSHIFT == 14)
8853 flush_mmu_tlb_entry(tlbi_addr(_COMM_PAGE32_BASE_ADDRESS & ~ARM_TT_L2_OFFMASK) | tlbi_asid(pmap->asid));
8854 #endif
8855 }
8856
8857 void
8858 pmap_insert_sharedpage(
8859 pmap_t pmap)
8860 {
8861 pmap_insert_sharedpage_internal(pmap);
8862 }
8863
8864 static boolean_t
8865 pmap_is_64bit(
8866 pmap_t pmap)
8867 {
8868 return (pmap->is_64bit);
8869 }
8870
8871 #endif
8872
8873 /* ARMTODO -- an implementation that accounts for
8874 * holes in the physical map, if any.
8875 */
8876 boolean_t
8877 pmap_valid_page(
8878 ppnum_t pn) {
8879 return pa_valid(ptoa(pn));
8880 }
8881
8882 static boolean_t
8883 pmap_is_empty_internal(
8884 pmap_t pmap,
8885 vm_map_offset_t va_start,
8886 vm_map_offset_t va_end)
8887 {
8888 vm_map_offset_t block_start, block_end;
8889 tt_entry_t *tte_p;
8890
8891 if (pmap == NULL) {
8892 return TRUE;
8893 }
8894
8895 if ((pmap != kernel_pmap) && (not_in_kdp)) {
8896 PMAP_LOCK(pmap);
8897 }
8898
8899 #if (__ARM_VMSA__ == 7)
8900 if (tte_index(pmap, va_end) >= pmap->tte_index_max) {
8901 if ((pmap != kernel_pmap) && (not_in_kdp)) {
8902 PMAP_UNLOCK(pmap);
8903 }
8904 return TRUE;
8905 }
8906
8907 block_start = va_start;
8908 tte_p = pmap_tte(pmap, block_start);
8909 while (block_start < va_end) {
8910 block_end = (block_start + ARM_TT_L1_SIZE) & ~(ARM_TT_L1_OFFMASK);
8911 if (block_end > va_end)
8912 block_end = va_end;
8913
8914 if ((*tte_p & ARM_TTE_TYPE_MASK) != 0) {
8915 vm_map_offset_t offset;
8916 ppnum_t phys_page = 0;
8917
8918 for (offset = block_start;
8919 offset < block_end;
8920 offset += ARM_PGBYTES) {
8921 // This does a pmap_find_phys() lookup but assumes lock is held
8922 phys_page = pmap_vtophys(pmap, offset);
8923 if (phys_page) {
8924 if ((pmap != kernel_pmap) && (not_in_kdp)) {
8925 PMAP_UNLOCK(pmap);
8926 }
8927 return FALSE;
8928 }
8929 }
8930 }
8931
8932 block_start = block_end;
8933 tte_p++;
8934 }
8935 #else
8936 block_start = va_start;
8937
8938 while (block_start < va_end) {
8939 pt_entry_t *bpte_p, *epte_p;
8940 pt_entry_t *pte_p;
8941
8942 block_end = (block_start + ARM_TT_L2_SIZE) & ~ARM_TT_L2_OFFMASK;
8943 if (block_end > va_end)
8944 block_end = va_end;
8945
8946 tte_p = pmap_tt2e(pmap, block_start);
8947 if ((tte_p != PT_ENTRY_NULL)
8948 && ((*tte_p & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_TABLE)) {
8949
8950 pte_p = (pt_entry_t *) ttetokv(*tte_p);
8951 bpte_p = &pte_p[tt3_index(pmap, block_start)];
8952 epte_p = bpte_p + (((block_end - block_start) & ARM_TT_L3_INDEX_MASK) >> ARM_TT_L3_SHIFT);
8953
8954 for (pte_p = bpte_p; pte_p < epte_p; pte_p++) {
8955 if (*pte_p != ARM_PTE_EMPTY) {
8956 if ((pmap != kernel_pmap) && (not_in_kdp)) {
8957 PMAP_UNLOCK(pmap);
8958 }
8959 return FALSE;
8960 }
8961 }
8962 }
8963 block_start = block_end;
8964 }
8965 #endif
8966
8967 if ((pmap != kernel_pmap) && (not_in_kdp)) {
8968 PMAP_UNLOCK(pmap);
8969 }
8970
8971 return TRUE;
8972 }
8973
8974 boolean_t
8975 pmap_is_empty(
8976 pmap_t pmap,
8977 vm_map_offset_t va_start,
8978 vm_map_offset_t va_end)
8979 {
8980 return pmap_is_empty_internal(pmap, va_start, va_end);
8981 }
8982
8983 vm_map_offset_t pmap_max_offset(
8984 boolean_t is64 __unused,
8985 unsigned int option)
8986 {
8987 vm_map_offset_t max_offset_ret = 0;
8988
8989 #if defined(__arm64__)
8990 assert (is64);
8991 vm_map_offset_t min_max_offset = SHARED_REGION_BASE_ARM64 + SHARED_REGION_SIZE_ARM64 + 0x20000000; // end of shared region + 512MB for various purposes
8992 if (option == ARM_PMAP_MAX_OFFSET_DEFAULT) {
8993 max_offset_ret = arm64_pmap_max_offset_default;
8994 } else if (option == ARM_PMAP_MAX_OFFSET_MIN) {
8995 max_offset_ret = min_max_offset;
8996 } else if (option == ARM_PMAP_MAX_OFFSET_MAX) {
8997 max_offset_ret = MACH_VM_MAX_ADDRESS;
8998 } else if (option == ARM_PMAP_MAX_OFFSET_DEVICE) {
8999 if (arm64_pmap_max_offset_default) {
9000 max_offset_ret = arm64_pmap_max_offset_default;
9001 } else if (max_mem > 0xC0000000) {
9002 max_offset_ret = 0x0000000318000000ULL; // Max offset is 12.375GB for devices with > 3GB of memory
9003 } else if (max_mem > 0x40000000) {
9004 max_offset_ret = 0x0000000218000000ULL; // Max offset is 8.375GB for devices with > 1GB and <= 3GB of memory
9005 } else {
9006 max_offset_ret = min_max_offset;
9007 }
9008 } else if (option == ARM_PMAP_MAX_OFFSET_JUMBO) {
9009 max_offset_ret = 0x0000000518000000ULL; // Max offset is 20.375GB for pmaps with special "jumbo" blessing
9010 } else {
9011 panic("pmap_max_offset illegal option 0x%x\n", option);
9012 }
9013
9014 assert(max_offset_ret >= min_max_offset);
9015 return max_offset_ret;
9016 #else
9017 if (option == ARM_PMAP_MAX_OFFSET_DEFAULT) {
9018 max_offset_ret = arm_pmap_max_offset_default;
9019 } else if (option == ARM_PMAP_MAX_OFFSET_MIN) {
9020 max_offset_ret = 0x66000000;
9021 } else if (option == ARM_PMAP_MAX_OFFSET_MAX) {
9022 max_offset_ret = VM_MAX_ADDRESS;
9023 } else if (option == ARM_PMAP_MAX_OFFSET_DEVICE) {
9024 if (arm_pmap_max_offset_default) {
9025 max_offset_ret = arm_pmap_max_offset_default;
9026 } else if (max_mem > 0x20000000) {
9027 max_offset_ret = 0x80000000;
9028 } else {
9029 max_offset_ret = 0x66000000;
9030 }
9031 } else {
9032 panic("pmap_max_offset illegal option 0x%x\n", option);
9033 }
9034
9035 return max_offset_ret;
9036 #endif
9037 }
9038
9039 #if CONFIG_DTRACE
9040 /*
9041 * Constrain DTrace copyin/copyout actions
9042 */
9043 extern kern_return_t dtrace_copyio_preflight(addr64_t);
9044 extern kern_return_t dtrace_copyio_postflight(addr64_t);
9045
9046 kern_return_t dtrace_copyio_preflight(
9047 __unused addr64_t va)
9048 {
9049 if (current_map() == kernel_map)
9050 return KERN_FAILURE;
9051 else
9052 return KERN_SUCCESS;
9053 }
9054
9055 kern_return_t dtrace_copyio_postflight(
9056 __unused addr64_t va)
9057 {
9058 return KERN_SUCCESS;
9059 }
9060 #endif /* CONFIG_DTRACE */
9061
9062
9063 void
9064 pmap_flush_context_init(__unused pmap_flush_context *pfc)
9065 {
9066 }
9067
9068
9069 void
9070 pmap_flush(
9071 __unused pmap_flush_context *cpus_to_flush)
9072 {
9073 /* not implemented yet */
9074 return;
9075 }
9076
9077 static boolean_t
9078 pmap_query_resident_internal(
9079 pmap_t pmap,
9080 vm_map_address_t start,
9081 vm_map_address_t end,
9082 mach_vm_size_t *resident_bytes_p,
9083 mach_vm_size_t *compressed_bytes_p)
9084 {
9085 mach_vm_size_t resident_bytes = 0;
9086 mach_vm_size_t compressed_bytes = 0;
9087
9088 pt_entry_t *bpte, *epte;
9089 pt_entry_t *pte_p;
9090 tt_entry_t *tte_p;
9091
9092 if (pmap == NULL) {
9093 return FALSE;
9094 }
9095
9096 /* Ensure that this request is valid, and addresses exactly one TTE. */
9097 assert(!(start % ARM_PGBYTES));
9098 assert(!(end % ARM_PGBYTES));
9099 assert(end >= start);
9100 assert((end - start) <= (PTE_PGENTRIES * ARM_PGBYTES));
9101
9102 PMAP_LOCK(pmap);
9103 tte_p = pmap_tte(pmap, start);
9104 if (tte_p == (tt_entry_t *) NULL) {
9105 PMAP_UNLOCK(pmap);
9106 return FALSE;
9107 }
9108 if ((*tte_p & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_TABLE) {
9109
9110 #if (__ARM_VMSA__ == 7)
9111 pte_p = (pt_entry_t *) ttetokv(*tte_p);
9112 bpte = &pte_p[ptenum(start)];
9113 epte = bpte + atop(end - start);
9114 #else
9115 pte_p = (pt_entry_t *) ttetokv(*tte_p);
9116 bpte = &pte_p[tt3_index(pmap, start)];
9117 epte = bpte + ((end - start) >> ARM_TT_L3_SHIFT);
9118 #endif
9119
9120 for (; bpte < epte; bpte++) {
9121 if (ARM_PTE_IS_COMPRESSED(*bpte)) {
9122 compressed_bytes += ARM_PGBYTES;
9123 } else if (pa_valid(pte_to_pa(*bpte))) {
9124 resident_bytes += ARM_PGBYTES;
9125 }
9126 }
9127 }
9128 PMAP_UNLOCK(pmap);
9129
9130 if (compressed_bytes_p) {
9131 *compressed_bytes_p += compressed_bytes;
9132 }
9133
9134 if (resident_bytes_p) {
9135 *resident_bytes_p += resident_bytes;
9136 }
9137
9138 return TRUE;
9139 }
9140
9141 mach_vm_size_t
9142 pmap_query_resident(
9143 pmap_t pmap,
9144 vm_map_address_t start,
9145 vm_map_address_t end,
9146 mach_vm_size_t *compressed_bytes_p)
9147 {
9148 mach_vm_size_t resident_bytes;
9149 mach_vm_size_t compressed_bytes;
9150 vm_map_address_t va;
9151
9152
9153 if (pmap == PMAP_NULL) {
9154 if (compressed_bytes_p) {
9155 *compressed_bytes_p = 0;
9156 }
9157 return 0;
9158 }
9159
9160 resident_bytes = 0;
9161 compressed_bytes = 0;
9162
9163 PMAP_TRACE(PMAP_CODE(PMAP__QUERY_RESIDENT) | DBG_FUNC_START,
9164 VM_KERNEL_ADDRHIDE(pmap), VM_KERNEL_ADDRHIDE(start),
9165 VM_KERNEL_ADDRHIDE(end));
9166
9167 va = start;
9168 while (va < end) {
9169 vm_map_address_t l;
9170
9171 l = ((va + ARM_TT_TWIG_SIZE) & ~ARM_TT_TWIG_OFFMASK);
9172
9173 if (l > end)
9174 l = end;
9175 if (!pmap_query_resident_internal(pmap, va, l, &resident_bytes, compressed_bytes_p)) {
9176 break;
9177 }
9178
9179 va = l;
9180 }
9181
9182 if (compressed_bytes_p) {
9183 *compressed_bytes_p = compressed_bytes;
9184 }
9185
9186 PMAP_TRACE(PMAP_CODE(PMAP__QUERY_RESIDENT) | DBG_FUNC_END,
9187 resident_bytes);
9188
9189 return resident_bytes;
9190 }
9191
9192 #if MACH_ASSERT
9193 extern int pmap_ledgers_panic;
9194 static void
9195 pmap_check_ledgers(
9196 pmap_t pmap)
9197 {
9198 ledger_amount_t bal;
9199 int pid;
9200 char *procname;
9201 boolean_t do_panic;
9202
9203 if (pmap->pmap_pid == 0) {
9204 /*
9205 * This pmap was not or is no longer fully associated
9206 * with a task (e.g. the old pmap after a fork()/exec() or
9207 * spawn()). Its "ledger" still points at a task that is
9208 * now using a different (and active) address space, so
9209 * we can't check that all the pmap ledgers are balanced here.
9210 *
9211 * If the "pid" is set, that means that we went through
9212 * pmap_set_process() in task_terminate_internal(), so
9213 * this task's ledger should not have been re-used and
9214 * all the pmap ledgers should be back to 0.
9215 */
9216 return;
9217 }
9218
9219 do_panic = FALSE;
9220 pid = pmap->pmap_pid;
9221 procname = pmap->pmap_procname;
9222
9223 pmap_ledgers_drift.num_pmaps_checked++;
9224
9225 ledger_get_balance(pmap->ledger,
9226 task_ledgers.phys_footprint,
9227 &bal);
9228 if (bal != 0) {
9229 #if DEVELOPMENT || DEBUG
9230 // if (!pmap->footprint_was_suspended)
9231 #endif /* DEVELOPMENT || DEBUG */
9232 do_panic = TRUE;
9233 printf("LEDGER BALANCE proc %d (%s) "
9234 "\"phys_footprint\" = %lld\n",
9235 pid, procname, bal);
9236 if (bal > 0) {
9237 pmap_ledgers_drift.phys_footprint_over++;
9238 pmap_ledgers_drift.phys_footprint_over_total += bal;
9239 if (bal > pmap_ledgers_drift.phys_footprint_over_max) {
9240 pmap_ledgers_drift.phys_footprint_over_max = bal;
9241 }
9242 } else {
9243 pmap_ledgers_drift.phys_footprint_under++;
9244 pmap_ledgers_drift.phys_footprint_under_total += bal;
9245 if (bal < pmap_ledgers_drift.phys_footprint_under_max) {
9246 pmap_ledgers_drift.phys_footprint_under_max = bal;
9247 }
9248 }
9249 }
9250 ledger_get_balance(pmap->ledger,
9251 task_ledgers.internal,
9252 &bal);
9253 if (bal != 0) {
9254 do_panic = TRUE;
9255 printf("LEDGER BALANCE proc %d (%s) "
9256 "\"internal\" = %lld\n",
9257 pid, procname, bal);
9258 if (bal > 0) {
9259 pmap_ledgers_drift.internal_over++;
9260 pmap_ledgers_drift.internal_over_total += bal;
9261 if (bal > pmap_ledgers_drift.internal_over_max) {
9262 pmap_ledgers_drift.internal_over_max = bal;
9263 }
9264 } else {
9265 pmap_ledgers_drift.internal_under++;
9266 pmap_ledgers_drift.internal_under_total += bal;
9267 if (bal < pmap_ledgers_drift.internal_under_max) {
9268 pmap_ledgers_drift.internal_under_max = bal;
9269 }
9270 }
9271 }
9272 ledger_get_balance(pmap->ledger,
9273 task_ledgers.internal_compressed,
9274 &bal);
9275 if (bal != 0) {
9276 do_panic = TRUE;
9277 printf("LEDGER BALANCE proc %d (%s) "
9278 "\"internal_compressed\" = %lld\n",
9279 pid, procname, bal);
9280 if (bal > 0) {
9281 pmap_ledgers_drift.internal_compressed_over++;
9282 pmap_ledgers_drift.internal_compressed_over_total += bal;
9283 if (bal > pmap_ledgers_drift.internal_compressed_over_max) {
9284 pmap_ledgers_drift.internal_compressed_over_max = bal;
9285 }
9286 } else {
9287 pmap_ledgers_drift.internal_compressed_under++;
9288 pmap_ledgers_drift.internal_compressed_under_total += bal;
9289 if (bal < pmap_ledgers_drift.internal_compressed_under_max) {
9290 pmap_ledgers_drift.internal_compressed_under_max = bal;
9291 }
9292 }
9293 }
9294 ledger_get_balance(pmap->ledger,
9295 task_ledgers.iokit_mapped,
9296 &bal);
9297 if (bal != 0) {
9298 do_panic = TRUE;
9299 printf("LEDGER BALANCE proc %d (%s) "
9300 "\"iokit_mapped\" = %lld\n",
9301 pid, procname, bal);
9302 if (bal > 0) {
9303 pmap_ledgers_drift.iokit_mapped_over++;
9304 pmap_ledgers_drift.iokit_mapped_over_total += bal;
9305 if (bal > pmap_ledgers_drift.iokit_mapped_over_max) {
9306 pmap_ledgers_drift.iokit_mapped_over_max = bal;
9307 }
9308 } else {
9309 pmap_ledgers_drift.iokit_mapped_under++;
9310 pmap_ledgers_drift.iokit_mapped_under_total += bal;
9311 if (bal < pmap_ledgers_drift.iokit_mapped_under_max) {
9312 pmap_ledgers_drift.iokit_mapped_under_max = bal;
9313 }
9314 }
9315 }
9316 ledger_get_balance(pmap->ledger,
9317 task_ledgers.alternate_accounting,
9318 &bal);
9319 if (bal != 0) {
9320 do_panic = TRUE;
9321 printf("LEDGER BALANCE proc %d (%s) "
9322 "\"alternate_accounting\" = %lld\n",
9323 pid, procname, bal);
9324 if (bal > 0) {
9325 pmap_ledgers_drift.alternate_accounting_over++;
9326 pmap_ledgers_drift.alternate_accounting_over_total += bal;
9327 if (bal > pmap_ledgers_drift.alternate_accounting_over_max) {
9328 pmap_ledgers_drift.alternate_accounting_over_max = bal;
9329 }
9330 } else {
9331 pmap_ledgers_drift.alternate_accounting_under++;
9332 pmap_ledgers_drift.alternate_accounting_under_total += bal;
9333 if (bal < pmap_ledgers_drift.alternate_accounting_under_max) {
9334 pmap_ledgers_drift.alternate_accounting_under_max = bal;
9335 }
9336 }
9337 }
9338 ledger_get_balance(pmap->ledger,
9339 task_ledgers.alternate_accounting_compressed,
9340 &bal);
9341 if (bal != 0) {
9342 do_panic = TRUE;
9343 printf("LEDGER BALANCE proc %d (%s) "
9344 "\"alternate_accounting_compressed\" = %lld\n",
9345 pid, procname, bal);
9346 if (bal > 0) {
9347 pmap_ledgers_drift.alternate_accounting_compressed_over++;
9348 pmap_ledgers_drift.alternate_accounting_compressed_over_total += bal;
9349 if (bal > pmap_ledgers_drift.alternate_accounting_compressed_over_max) {
9350 pmap_ledgers_drift.alternate_accounting_compressed_over_max = bal;
9351 }
9352 } else {
9353 pmap_ledgers_drift.alternate_accounting_compressed_under++;
9354 pmap_ledgers_drift.alternate_accounting_compressed_under_total += bal;
9355 if (bal < pmap_ledgers_drift.alternate_accounting_compressed_under_max) {
9356 pmap_ledgers_drift.alternate_accounting_compressed_under_max = bal;
9357 }
9358 }
9359 }
9360 ledger_get_balance(pmap->ledger,
9361 task_ledgers.page_table,
9362 &bal);
9363 if (bal != 0) {
9364 do_panic = TRUE;
9365 printf("LEDGER BALANCE proc %d (%s) "
9366 "\"page_table\" = %lld\n",
9367 pid, procname, bal);
9368 if (bal > 0) {
9369 pmap_ledgers_drift.page_table_over++;
9370 pmap_ledgers_drift.page_table_over_total += bal;
9371 if (bal > pmap_ledgers_drift.page_table_over_max) {
9372 pmap_ledgers_drift.page_table_over_max = bal;
9373 }
9374 } else {
9375 pmap_ledgers_drift.page_table_under++;
9376 pmap_ledgers_drift.page_table_under_total += bal;
9377 if (bal < pmap_ledgers_drift.page_table_under_max) {
9378 pmap_ledgers_drift.page_table_under_max = bal;
9379 }
9380 }
9381 }
9382 ledger_get_balance(pmap->ledger,
9383 task_ledgers.purgeable_volatile,
9384 &bal);
9385 if (bal != 0) {
9386 do_panic = TRUE;
9387 printf("LEDGER BALANCE proc %d (%s) "
9388 "\"purgeable_volatile\" = %lld\n",
9389 pid, procname, bal);
9390 if (bal > 0) {
9391 pmap_ledgers_drift.purgeable_volatile_over++;
9392 pmap_ledgers_drift.purgeable_volatile_over_total += bal;
9393 if (bal > pmap_ledgers_drift.purgeable_volatile_over_max) {
9394 pmap_ledgers_drift.purgeable_volatile_over_max = bal;
9395 }
9396 } else {
9397 pmap_ledgers_drift.purgeable_volatile_under++;
9398 pmap_ledgers_drift.purgeable_volatile_under_total += bal;
9399 if (bal < pmap_ledgers_drift.purgeable_volatile_under_max) {
9400 pmap_ledgers_drift.purgeable_volatile_under_max = bal;
9401 }
9402 }
9403 }
9404 ledger_get_balance(pmap->ledger,
9405 task_ledgers.purgeable_nonvolatile,
9406 &bal);
9407 if (bal != 0) {
9408 do_panic = TRUE;
9409 printf("LEDGER BALANCE proc %d (%s) "
9410 "\"purgeable_nonvolatile\" = %lld\n",
9411 pid, procname, bal);
9412 if (bal > 0) {
9413 pmap_ledgers_drift.purgeable_nonvolatile_over++;
9414 pmap_ledgers_drift.purgeable_nonvolatile_over_total += bal;
9415 if (bal > pmap_ledgers_drift.purgeable_nonvolatile_over_max) {
9416 pmap_ledgers_drift.purgeable_nonvolatile_over_max = bal;
9417 }
9418 } else {
9419 pmap_ledgers_drift.purgeable_nonvolatile_under++;
9420 pmap_ledgers_drift.purgeable_nonvolatile_under_total += bal;
9421 if (bal < pmap_ledgers_drift.purgeable_nonvolatile_under_max) {
9422 pmap_ledgers_drift.purgeable_nonvolatile_under_max = bal;
9423 }
9424 }
9425 }
9426 ledger_get_balance(pmap->ledger,
9427 task_ledgers.purgeable_volatile_compressed,
9428 &bal);
9429 if (bal != 0) {
9430 do_panic = TRUE;
9431 printf("LEDGER BALANCE proc %d (%s) "
9432 "\"purgeable_volatile_compressed\" = %lld\n",
9433 pid, procname, bal);
9434 if (bal > 0) {
9435 pmap_ledgers_drift.purgeable_volatile_compressed_over++;
9436 pmap_ledgers_drift.purgeable_volatile_compressed_over_total += bal;
9437 if (bal > pmap_ledgers_drift.purgeable_volatile_compressed_over_max) {
9438 pmap_ledgers_drift.purgeable_volatile_compressed_over_max = bal;
9439 }
9440 } else {
9441 pmap_ledgers_drift.purgeable_volatile_compressed_under++;
9442 pmap_ledgers_drift.purgeable_volatile_compressed_under_total += bal;
9443 if (bal < pmap_ledgers_drift.purgeable_volatile_compressed_under_max) {
9444 pmap_ledgers_drift.purgeable_volatile_compressed_under_max = bal;
9445 }
9446 }
9447 }
9448 ledger_get_balance(pmap->ledger,
9449 task_ledgers.purgeable_nonvolatile_compressed,
9450 &bal);
9451 if (bal != 0) {
9452 do_panic = TRUE;
9453 printf("LEDGER BALANCE proc %d (%s) "
9454 "\"purgeable_nonvolatile_compressed\" = %lld\n",
9455 pid, procname, bal);
9456 if (bal > 0) {
9457 pmap_ledgers_drift.purgeable_nonvolatile_compressed_over++;
9458 pmap_ledgers_drift.purgeable_nonvolatile_compressed_over_total += bal;
9459 if (bal > pmap_ledgers_drift.purgeable_nonvolatile_compressed_over_max) {
9460 pmap_ledgers_drift.purgeable_nonvolatile_compressed_over_max = bal;
9461 }
9462 } else {
9463 pmap_ledgers_drift.purgeable_nonvolatile_compressed_under++;
9464 pmap_ledgers_drift.purgeable_nonvolatile_compressed_under_total += bal;
9465 if (bal < pmap_ledgers_drift.purgeable_nonvolatile_compressed_under_max) {
9466 pmap_ledgers_drift.purgeable_nonvolatile_compressed_under_max = bal;
9467 }
9468 }
9469 }
9470
9471 if (do_panic) {
9472 if (pmap_ledgers_panic) {
9473 panic("pmap_destroy(%p) %d[%s] has imbalanced ledgers\n",
9474 pmap, pid, procname);
9475 } else {
9476 printf("pmap_destroy(%p) %d[%s] has imbalanced ledgers\n",
9477 pmap, pid, procname);
9478 }
9479 }
9480
9481 assert(pmap->stats.resident_count == 0);
9482 #if 00
9483 assert(pmap->stats.wired_count == 0);
9484 #endif
9485 assert(pmap->stats.device == 0);
9486 assert(pmap->stats.internal == 0);
9487 assert(pmap->stats.external == 0);
9488 assert(pmap->stats.reusable == 0);
9489 assert(pmap->stats.compressed == 0);
9490 }
9491 #endif /* MACH_ASSERT */
9492
9493 void pmap_advise_pagezero_range(__unused pmap_t p, __unused uint64_t a) {
9494 }
9495
9496
9497 #if CONFIG_PGTRACE
9498 #define PROF_START uint64_t t, nanot;\
9499 t = mach_absolute_time();
9500
9501 #define PROF_END absolutetime_to_nanoseconds(mach_absolute_time()-t, &nanot);\
9502 kprintf("%s: took %llu ns\n", __func__, nanot);
9503
9504 #define PMAP_PGTRACE_LOCK(p) \
9505 do { \
9506 *(p) = ml_set_interrupts_enabled(false); \
9507 if (simple_lock_try(&(pmap_pgtrace.lock))) break; \
9508 ml_set_interrupts_enabled(*(p)); \
9509 } while (true)
9510
9511 #define PMAP_PGTRACE_UNLOCK(p) \
9512 do { \
9513 simple_unlock(&(pmap_pgtrace.lock)); \
9514 ml_set_interrupts_enabled(*(p)); \
9515 } while (0)
9516
9517 #define PGTRACE_WRITE_PTE(pte_p, pte_entry) \
9518 do { \
9519 *(pte_p) = (pte_entry); \
9520 FLUSH_PTE(pte_p); \
9521 } while (0)
9522
9523 #define PGTRACE_MAX_MAP 16 // maximum supported va to same pa
9524
9525 typedef enum {
9526 UNDEFINED,
9527 PA_UNDEFINED,
9528 VA_UNDEFINED,
9529 DEFINED
9530 } pmap_pgtrace_page_state_t;
9531
9532 typedef struct {
9533 queue_chain_t chain;
9534
9535 /*
9536 pa - pa
9537 maps - list of va maps to upper pa
9538 map_pool - map pool
9539 map_waste - waste can
9540 state - state
9541 */
9542 pmap_paddr_t pa;
9543 queue_head_t maps;
9544 queue_head_t map_pool;
9545 queue_head_t map_waste;
9546 pmap_pgtrace_page_state_t state;
9547 } pmap_pgtrace_page_t;
9548
9549 static struct {
9550 /*
9551 pages - list of tracing page info
9552 */
9553 queue_head_t pages;
9554 decl_simple_lock_data(, lock);
9555 } pmap_pgtrace = {};
9556
9557 static void pmap_pgtrace_init(void)
9558 {
9559 queue_init(&(pmap_pgtrace.pages));
9560 simple_lock_init(&(pmap_pgtrace.lock), 0);
9561
9562 boolean_t enabled;
9563
9564 if (PE_parse_boot_argn("pgtrace", &enabled, sizeof(enabled))) {
9565 pgtrace_enabled = enabled;
9566 }
9567 }
9568
9569 // find a page with given pa - pmap_pgtrace should be locked
9570 inline static pmap_pgtrace_page_t *pmap_pgtrace_find_page(pmap_paddr_t pa)
9571 {
9572 queue_head_t *q = &(pmap_pgtrace.pages);
9573 pmap_pgtrace_page_t *p;
9574
9575 queue_iterate(q, p, pmap_pgtrace_page_t *, chain) {
9576 if (p->state == UNDEFINED) {
9577 continue;
9578 }
9579 if (p->state == PA_UNDEFINED) {
9580 continue;
9581 }
9582 if (p->pa == pa) {
9583 return p;
9584 }
9585 }
9586
9587 return NULL;
9588 }
9589
9590 // enter clone of given pmap, va page and range - pmap should be locked
9591 static bool pmap_pgtrace_enter_clone(pmap_t pmap, vm_map_offset_t va_page, vm_map_offset_t start, vm_map_offset_t end)
9592 {
9593 bool ints;
9594 queue_head_t *q = &(pmap_pgtrace.pages);
9595 pmap_paddr_t pa_page;
9596 pt_entry_t *ptep, *cptep;
9597 pmap_pgtrace_page_t *p;
9598 bool found = false;
9599
9600 PMAP_ASSERT_LOCKED(pmap);
9601 assert(va_page == arm_trunc_page(va_page));
9602
9603 PMAP_PGTRACE_LOCK(&ints);
9604
9605 ptep = pmap_pte(pmap, va_page);
9606
9607 // target pte should exist
9608 if (!ptep || !(*ptep & ARM_PTE_TYPE_VALID)) {
9609 PMAP_PGTRACE_UNLOCK(&ints);
9610 return false;
9611 }
9612
9613 queue_head_t *mapq;
9614 queue_head_t *mappool;
9615 pmap_pgtrace_map_t *map = NULL;
9616
9617 pa_page = pte_to_pa(*ptep);
9618
9619 // find if we have a page info defined for this
9620 queue_iterate(q, p, pmap_pgtrace_page_t *, chain) {
9621 mapq = &(p->maps);
9622 mappool = &(p->map_pool);
9623
9624 switch (p->state) {
9625 case PA_UNDEFINED:
9626 queue_iterate(mapq, map, pmap_pgtrace_map_t *, chain) {
9627 if (map->cloned == false && map->pmap == pmap && map->ova == va_page) {
9628 p->pa = pa_page;
9629 map->range.start = start;
9630 map->range.end = end;
9631 found = true;
9632 break;
9633 }
9634 }
9635 break;
9636
9637 case VA_UNDEFINED:
9638 if (p->pa != pa_page) {
9639 break;
9640 }
9641 queue_iterate(mapq, map, pmap_pgtrace_map_t *, chain) {
9642 if (map->cloned == false) {
9643 map->pmap = pmap;
9644 map->ova = va_page;
9645 map->range.start = start;
9646 map->range.end = end;
9647 found = true;
9648 break;
9649 }
9650 }
9651 break;
9652
9653 case DEFINED:
9654 if (p->pa != pa_page) {
9655 break;
9656 }
9657 queue_iterate(mapq, map, pmap_pgtrace_map_t *, chain) {
9658 if (map->cloned == true && map->pmap == pmap && map->ova == va_page) {
9659 kprintf("%s: skip existing mapping at va=%llx\n", __func__, va_page);
9660 break;
9661 } else if (map->cloned == true && map->pmap == kernel_pmap && map->cva[1] == va_page) {
9662 kprintf("%s: skip clone mapping at va=%llx\n", __func__, va_page);
9663 break;
9664 } else if (map->cloned == false && map->pmap == pmap && map->ova == va_page) {
9665 // range should be already defined as well
9666 found = true;
9667 break;
9668 }
9669 }
9670 break;
9671
9672 default:
9673 panic("invalid state p->state=%x\n", p->state);
9674 }
9675
9676 if (found == true) {
9677 break;
9678 }
9679 }
9680
9681 // do not clone if no page info found
9682 if (found == false) {
9683 PMAP_PGTRACE_UNLOCK(&ints);
9684 return false;
9685 }
9686
9687 // copy pre, target and post ptes to clone ptes
9688 for (int i = 0; i < 3; i++) {
9689 ptep = pmap_pte(pmap, va_page + (i-1)*ARM_PGBYTES);
9690 cptep = pmap_pte(kernel_pmap, map->cva[i]);
9691 assert(cptep != NULL);
9692 if (ptep == NULL) {
9693 PGTRACE_WRITE_PTE(cptep, (pt_entry_t)NULL);
9694 } else {
9695 PGTRACE_WRITE_PTE(cptep, *ptep);
9696 }
9697 PMAP_UPDATE_TLBS(kernel_pmap, map->cva[i], map->cva[i]+ARM_PGBYTES);
9698 }
9699
9700 // get ptes for original and clone
9701 ptep = pmap_pte(pmap, va_page);
9702 cptep = pmap_pte(kernel_pmap, map->cva[1]);
9703
9704 // invalidate original pte and mark it as a pgtrace page
9705 PGTRACE_WRITE_PTE(ptep, (*ptep | ARM_PTE_PGTRACE) & ~ARM_PTE_TYPE_VALID);
9706 PMAP_UPDATE_TLBS(pmap, map->ova, map->ova+ARM_PGBYTES);
9707
9708 map->cloned = true;
9709 p->state = DEFINED;
9710
9711 kprintf("%s: pa_page=%llx va_page=%llx cva[1]=%llx pmap=%p ptep=%p cptep=%p\n", __func__, pa_page, va_page, map->cva[1], pmap, ptep, cptep);
9712
9713 PMAP_PGTRACE_UNLOCK(&ints);
9714
9715 return true;
9716 }
9717
9718 // This function removes trace bit and validate pte if applicable. Pmap must be locked.
9719 static void pmap_pgtrace_remove_clone(pmap_t pmap, pmap_paddr_t pa, vm_map_offset_t va)
9720 {
9721 bool ints, found = false;
9722 pmap_pgtrace_page_t *p;
9723 pt_entry_t *ptep;
9724
9725 PMAP_PGTRACE_LOCK(&ints);
9726
9727 // we must have this page info
9728 p = pmap_pgtrace_find_page(pa);
9729 if (p == NULL) {
9730 goto unlock_exit;
9731 }
9732
9733 // find matching map
9734 queue_head_t *mapq = &(p->maps);
9735 queue_head_t *mappool = &(p->map_pool);
9736 pmap_pgtrace_map_t *map;
9737
9738 queue_iterate(mapq, map, pmap_pgtrace_map_t *, chain) {
9739 if (map->pmap == pmap && map->ova == va) {
9740 found = true;
9741 break;
9742 }
9743 }
9744
9745 if (!found) {
9746 goto unlock_exit;
9747 }
9748
9749 if (map->cloned == true) {
9750 // Restore back the pte to original state
9751 ptep = pmap_pte(pmap, map->ova);
9752 assert(ptep);
9753 PGTRACE_WRITE_PTE(ptep, *ptep | ARM_PTE_TYPE_VALID);
9754 PMAP_UPDATE_TLBS(pmap, va, va+ARM_PGBYTES);
9755
9756 // revert clone pages
9757 for (int i = 0; i < 3; i++) {
9758 ptep = pmap_pte(kernel_pmap, map->cva[i]);
9759 assert(ptep != NULL);
9760 PGTRACE_WRITE_PTE(ptep, map->cva_spte[i]);
9761 PMAP_UPDATE_TLBS(kernel_pmap, map->cva[i], map->cva[i]+ARM_PGBYTES);
9762 }
9763 }
9764
9765 queue_remove(mapq, map, pmap_pgtrace_map_t *, chain);
9766 map->pmap = NULL;
9767 map->ova = (vm_map_offset_t)NULL;
9768 map->cloned = false;
9769 queue_enter_first(mappool, map, pmap_pgtrace_map_t *, chain);
9770
9771 kprintf("%s: p=%p pa=%llx va=%llx\n", __func__, p, pa, va);
9772
9773 unlock_exit:
9774 PMAP_PGTRACE_UNLOCK(&ints);
9775 }
9776
9777 // remove all clones of given pa - pmap must be locked
9778 static void pmap_pgtrace_remove_all_clone(pmap_paddr_t pa)
9779 {
9780 bool ints;
9781 pmap_pgtrace_page_t *p;
9782 pt_entry_t *ptep;
9783
9784 PMAP_PGTRACE_LOCK(&ints);
9785
9786 // we must have this page info
9787 p = pmap_pgtrace_find_page(pa);
9788 if (p == NULL) {
9789 PMAP_PGTRACE_UNLOCK(&ints);
9790 return;
9791 }
9792
9793 queue_head_t *mapq = &(p->maps);
9794 queue_head_t *mappool = &(p->map_pool);
9795 queue_head_t *mapwaste = &(p->map_waste);
9796 pmap_pgtrace_map_t *map;
9797
9798 // move maps to waste
9799 while (!queue_empty(mapq)) {
9800 queue_remove_first(mapq, map, pmap_pgtrace_map_t *, chain);
9801 queue_enter_first(mapwaste, map, pmap_pgtrace_map_t*, chain);
9802 }
9803
9804 PMAP_PGTRACE_UNLOCK(&ints);
9805
9806 // sanitize maps in waste
9807 queue_iterate(mapwaste, map, pmap_pgtrace_map_t *, chain) {
9808 if (map->cloned == true) {
9809 PMAP_LOCK(map->pmap);
9810
9811 // restore back original pte
9812 ptep = pmap_pte(map->pmap, map->ova);
9813 assert(ptep);
9814 PGTRACE_WRITE_PTE(ptep, *ptep | ARM_PTE_TYPE_VALID);
9815 PMAP_UPDATE_TLBS(map->pmap, map->ova, map->ova+ARM_PGBYTES);
9816
9817 // revert clone ptes
9818 for (int i = 0; i < 3; i++) {
9819 ptep = pmap_pte(kernel_pmap, map->cva[i]);
9820 assert(ptep != NULL);
9821 PGTRACE_WRITE_PTE(ptep, map->cva_spte[i]);
9822 PMAP_UPDATE_TLBS(kernel_pmap, map->cva[i], map->cva[i]+ARM_PGBYTES);
9823 }
9824
9825 PMAP_UNLOCK(map->pmap);
9826 }
9827
9828 map->pmap = NULL;
9829 map->ova = (vm_map_offset_t)NULL;
9830 map->cloned = false;
9831 }
9832
9833 PMAP_PGTRACE_LOCK(&ints);
9834
9835 // recycle maps back to map_pool
9836 while (!queue_empty(mapwaste)) {
9837 queue_remove_first(mapwaste, map, pmap_pgtrace_map_t *, chain);
9838 queue_enter_first(mappool, map, pmap_pgtrace_map_t*, chain);
9839 }
9840
9841 PMAP_PGTRACE_UNLOCK(&ints);
9842 }
9843
9844 inline static void pmap_pgtrace_get_search_space(pmap_t pmap, vm_map_offset_t *startp, vm_map_offset_t *endp)
9845 {
9846 uint64_t tsz;
9847 vm_map_offset_t end;
9848
9849 if (pmap == kernel_pmap) {
9850 tsz = (get_tcr() >> TCR_T1SZ_SHIFT) & TCR_TSZ_MASK;
9851 *startp = MAX(VM_MIN_KERNEL_ADDRESS, (UINT64_MAX >> (64-tsz)) << (64-tsz));
9852 *endp = VM_MAX_KERNEL_ADDRESS;
9853 } else {
9854 tsz = (get_tcr() >> TCR_T0SZ_SHIFT) & TCR_TSZ_MASK;
9855 if (tsz == 64) {
9856 end = 0;
9857 } else {
9858 end = ((uint64_t)1 << (64-tsz)) - 1;
9859 }
9860
9861 *startp = 0;
9862 *endp = end;
9863 }
9864
9865 assert(*endp > *startp);
9866
9867 return;
9868 }
9869
9870 // has pa mapped in given pmap? then clone it
9871 static uint64_t pmap_pgtrace_clone_from_pa(pmap_t pmap, pmap_paddr_t pa, vm_map_offset_t start_offset, vm_map_offset_t end_offset) {
9872 uint64_t ret = 0;
9873 vm_map_offset_t min, max;
9874 vm_map_offset_t cur_page, end_page;
9875 pt_entry_t *ptep;
9876 tt_entry_t *ttep;
9877 tt_entry_t tte;
9878
9879 pmap_pgtrace_get_search_space(pmap, &min, &max);
9880
9881 cur_page = arm_trunc_page(min);
9882 end_page = arm_trunc_page(max);
9883 while (cur_page <= end_page) {
9884 vm_map_offset_t add = 0;
9885
9886 PMAP_LOCK(pmap);
9887
9888 // skip uninterested space
9889 if (pmap == kernel_pmap &&
9890 ((vm_kernel_base <= cur_page && cur_page < vm_kernel_top) ||
9891 (vm_kext_base <= cur_page && cur_page < vm_kext_top))) {
9892 add = ARM_PGBYTES;
9893 goto unlock_continue;
9894 }
9895
9896 #if __ARM64_TWO_LEVEL_PMAP__
9897 // check whether we can skip l2
9898 ttep = pmap_tt2e(pmap, cur_page);
9899 assert(ttep);
9900 tte = *ttep;
9901 #else
9902 // check whether we can skip l1
9903 ttep = pmap_tt1e(pmap, cur_page);
9904 assert(ttep);
9905 tte = *ttep;
9906 if ((tte & (ARM_TTE_TYPE_MASK | ARM_TTE_VALID)) != (ARM_TTE_TYPE_TABLE | ARM_TTE_VALID)) {
9907 add = ARM_TT_L1_SIZE;
9908 goto unlock_continue;
9909 }
9910
9911 // how about l2
9912 tte = ((tt_entry_t*) phystokv(tte & ARM_TTE_TABLE_MASK))[tt2_index(pmap, cur_page)];
9913 #endif
9914 if ((tte & (ARM_TTE_TYPE_MASK | ARM_TTE_VALID)) != (ARM_TTE_TYPE_TABLE | ARM_TTE_VALID)) {
9915 add = ARM_TT_L2_SIZE;
9916 goto unlock_continue;
9917 }
9918
9919 // ptep finally
9920 ptep = &(((pt_entry_t*) phystokv(tte & ARM_TTE_TABLE_MASK))[tt3_index(pmap, cur_page)]);
9921 if (ptep == PT_ENTRY_NULL) {
9922 add = ARM_TT_L3_SIZE;
9923 goto unlock_continue;
9924 }
9925
9926 if (arm_trunc_page(pa) == pte_to_pa(*ptep)) {
9927 if (pmap_pgtrace_enter_clone(pmap, cur_page, start_offset, end_offset) == true) {
9928 ret++;
9929 }
9930 }
9931
9932 add = ARM_PGBYTES;
9933
9934 unlock_continue:
9935 PMAP_UNLOCK(pmap);
9936
9937 //overflow
9938 if (cur_page + add < cur_page) {
9939 break;
9940 }
9941
9942 cur_page += add;
9943 }
9944
9945
9946 return ret;
9947 }
9948
9949 // search pv table and clone vas of given pa
9950 static uint64_t pmap_pgtrace_clone_from_pvtable(pmap_paddr_t pa, vm_map_offset_t start_offset, vm_map_offset_t end_offset)
9951 {
9952 uint64_t ret = 0;
9953 unsigned long pai;
9954 pv_entry_t **pvh;
9955 pt_entry_t *ptep;
9956 pmap_t pmap;
9957
9958 typedef struct {
9959 queue_chain_t chain;
9960 pmap_t pmap;
9961 vm_map_offset_t va;
9962 } pmap_va_t;
9963
9964 queue_head_t pmapvaq;
9965 pmap_va_t *pmapva;
9966
9967 queue_init(&pmapvaq);
9968
9969 pai = pa_index(pa);
9970 LOCK_PVH(pai);
9971 pvh = pai_to_pvh(pai);
9972
9973 // collect pmap/va pair from pvh
9974 if (pvh_test_type(pvh, PVH_TYPE_PTEP)) {
9975 ptep = pvh_ptep(pvh);
9976 pmap = ptep_get_pmap(ptep);
9977
9978 pmapva = (pmap_va_t *)kalloc(sizeof(pmap_va_t));
9979 pmapva->pmap = pmap;
9980 pmapva->va = ptep_get_va(ptep);
9981
9982 queue_enter_first(&pmapvaq, pmapva, pmap_va_t *, chain);
9983
9984 } else if (pvh_test_type(pvh, PVH_TYPE_PVEP)) {
9985 pv_entry_t *pvep;
9986
9987 pvep = pvh_list(pvh);
9988 while (pvep) {
9989 ptep = pve_get_ptep(pvep);
9990 pmap = ptep_get_pmap(ptep);
9991
9992 pmapva = (pmap_va_t *)kalloc(sizeof(pmap_va_t));
9993 pmapva->pmap = pmap;
9994 pmapva->va = ptep_get_va(ptep);
9995
9996 queue_enter_first(&pmapvaq, pmapva, pmap_va_t *, chain);
9997
9998 pvep = PVE_NEXT_PTR(pve_next(pvep));
9999 }
10000 }
10001
10002 UNLOCK_PVH(pai);
10003
10004 // clone them while making sure mapping still exists
10005 queue_iterate(&pmapvaq, pmapva, pmap_va_t *, chain) {
10006 PMAP_LOCK(pmapva->pmap);
10007 ptep = pmap_pte(pmapva->pmap, pmapva->va);
10008 if (pte_to_pa(*ptep) == pa) {
10009 if (pmap_pgtrace_enter_clone(pmapva->pmap, pmapva->va, start_offset, end_offset) == true) {
10010 ret++;
10011 }
10012 }
10013 PMAP_UNLOCK(pmapva->pmap);
10014
10015 kfree(pmapva, sizeof(pmap_va_t));
10016 }
10017
10018 return ret;
10019 }
10020
10021 // allocate a page info
10022 static pmap_pgtrace_page_t *pmap_pgtrace_alloc_page(void)
10023 {
10024 pmap_pgtrace_page_t *p;
10025 queue_head_t *mapq;
10026 queue_head_t *mappool;
10027 queue_head_t *mapwaste;
10028 pmap_pgtrace_map_t *map;
10029
10030 p = kalloc(sizeof(pmap_pgtrace_page_t));
10031 assert(p);
10032
10033 p->state = UNDEFINED;
10034
10035 mapq = &(p->maps);
10036 mappool = &(p->map_pool);
10037 mapwaste = &(p->map_waste);
10038 queue_init(mapq);
10039 queue_init(mappool);
10040 queue_init(mapwaste);
10041
10042 for (int i = 0; i < PGTRACE_MAX_MAP; i++) {
10043 vm_map_offset_t newcva;
10044 pt_entry_t *cptep;
10045 kern_return_t kr;
10046 vm_map_entry_t entry;
10047
10048 // get a clone va
10049 vm_object_reference(kernel_object);
10050 kr = vm_map_find_space(kernel_map, &newcva, vm_map_round_page(3*ARM_PGBYTES, PAGE_MASK), 0, 0, VM_MAP_KERNEL_FLAGS_NONE, VM_KERN_MEMORY_DIAG, &entry);
10051 if (kr != KERN_SUCCESS) {
10052 panic("%s VM couldn't find any space kr=%d\n", __func__, kr);
10053 }
10054 VME_OBJECT_SET(entry, kernel_object);
10055 VME_OFFSET_SET(entry, newcva);
10056 vm_map_unlock(kernel_map);
10057
10058 // fill default clone page info and add to pool
10059 map = kalloc(sizeof(pmap_pgtrace_map_t));
10060 for (int j = 0; j < 3; j ++) {
10061 vm_map_offset_t addr = newcva + j * ARM_PGBYTES;
10062
10063 // pre-expand pmap while preemption enabled
10064 kr = pmap_expand(kernel_pmap, addr, 0, PMAP_TT_MAX_LEVEL);
10065 if (kr != KERN_SUCCESS) {
10066 panic("%s: pmap_expand(kernel_pmap, addr=%llx) returns kr=%d\n", __func__, addr, kr);
10067 }
10068
10069 cptep = pmap_pte(kernel_pmap, addr);
10070 assert(cptep != NULL);
10071
10072 map->cva[j] = addr;
10073 map->cva_spte[j] = *cptep;
10074 }
10075 map->range.start = map->range.end = 0;
10076 map->cloned = false;
10077 queue_enter_first(mappool, map, pmap_pgtrace_map_t *, chain);
10078 }
10079
10080 return p;
10081 }
10082
10083 // free a page info
10084 static void pmap_pgtrace_free_page(pmap_pgtrace_page_t *p)
10085 {
10086 queue_head_t *mapq;
10087 queue_head_t *mappool;
10088 queue_head_t *mapwaste;
10089 pmap_pgtrace_map_t *map;
10090
10091 assert(p);
10092
10093 mapq = &(p->maps);
10094 mappool = &(p->map_pool);
10095 mapwaste = &(p->map_waste);
10096
10097 while (!queue_empty(mapq)) {
10098 queue_remove_first(mapq, map, pmap_pgtrace_map_t *, chain);
10099 kfree(map, sizeof(pmap_pgtrace_map_t));
10100 }
10101
10102 while (!queue_empty(mappool)) {
10103 queue_remove_first(mappool, map, pmap_pgtrace_map_t *, chain);
10104 kfree(map, sizeof(pmap_pgtrace_map_t));
10105 }
10106
10107 while (!queue_empty(mapwaste)) {
10108 queue_remove_first(mapwaste, map, pmap_pgtrace_map_t *, chain);
10109 kfree(map, sizeof(pmap_pgtrace_map_t));
10110 }
10111
10112 kfree(p, sizeof(pmap_pgtrace_page_t));
10113 }
10114
10115 // construct page infos with the given address range
10116 int pmap_pgtrace_add_page(pmap_t pmap, vm_map_offset_t start, vm_map_offset_t end)
10117 {
10118 int ret = 0;
10119 pt_entry_t *ptep;
10120 queue_head_t *q = &(pmap_pgtrace.pages);
10121 bool ints;
10122 vm_map_offset_t cur_page, end_page;
10123
10124 if (start > end) {
10125 kprintf("%s: invalid start=%llx > end=%llx\n", __func__, start, end);
10126 return -1;
10127 }
10128
10129 PROF_START
10130
10131 // add each page in given range
10132 cur_page = arm_trunc_page(start);
10133 end_page = arm_trunc_page(end);
10134 while (cur_page <= end_page) {
10135 pmap_paddr_t pa_page = 0;
10136 uint64_t num_cloned = 0;
10137 pmap_pgtrace_page_t *p = NULL, *newp;
10138 bool free_newp = true;
10139 pmap_pgtrace_page_state_t state;
10140
10141 // do all allocations outside of spinlocks
10142 newp = pmap_pgtrace_alloc_page();
10143
10144 // keep lock orders in pmap, kernel_pmap and pgtrace lock
10145 if (pmap != NULL) {
10146 PMAP_LOCK(pmap);
10147 }
10148 if (pmap != kernel_pmap) {
10149 PMAP_LOCK(kernel_pmap);
10150 }
10151
10152 // addresses are physical if pmap is null
10153 if (pmap == NULL) {
10154 ptep = NULL;
10155 pa_page = cur_page;
10156 state = VA_UNDEFINED;
10157 } else {
10158 ptep = pmap_pte(pmap, cur_page);
10159 if (ptep != NULL) {
10160 pa_page = pte_to_pa(*ptep);
10161 state = DEFINED;
10162 } else {
10163 state = PA_UNDEFINED;
10164 }
10165 }
10166
10167 // search if we have a page info already
10168 PMAP_PGTRACE_LOCK(&ints);
10169 if (state != PA_UNDEFINED) {
10170 p = pmap_pgtrace_find_page(pa_page);
10171 }
10172
10173 // add pre-allocated page info if nothing found
10174 if (p == NULL) {
10175 queue_enter_first(q, newp, pmap_pgtrace_page_t *, chain);
10176 p = newp;
10177 free_newp = false;
10178 }
10179
10180 // now p points what we want
10181 p->state = state;
10182
10183 queue_head_t *mapq = &(p->maps);
10184 queue_head_t *mappool = &(p->map_pool);
10185 pmap_pgtrace_map_t *map;
10186 vm_map_offset_t start_offset, end_offset;
10187
10188 // calculate trace offsets in the page
10189 if (cur_page > start) {
10190 start_offset = 0;
10191 } else {
10192 start_offset = start-cur_page;
10193 }
10194 if (cur_page == end_page) {
10195 end_offset = end-end_page;
10196 } else {
10197 end_offset = ARM_PGBYTES-1;
10198 }
10199
10200 kprintf("%s: pmap=%p cur_page=%llx ptep=%p state=%d start_offset=%llx end_offset=%llx\n", __func__, pmap, cur_page, ptep, state, start_offset, end_offset);
10201
10202 // fill map info
10203 assert(!queue_empty(mappool));
10204 queue_remove_first(mappool, map, pmap_pgtrace_map_t *, chain);
10205 if (p->state == PA_UNDEFINED) {
10206 map->pmap = pmap;
10207 map->ova = cur_page;
10208 map->range.start = start_offset;
10209 map->range.end = end_offset;
10210 } else if (p->state == VA_UNDEFINED) {
10211 p->pa = pa_page;
10212 map->range.start = start_offset;
10213 map->range.end = end_offset;
10214 } else if (p->state == DEFINED) {
10215 p->pa = pa_page;
10216 map->pmap = pmap;
10217 map->ova = cur_page;
10218 map->range.start = start_offset;
10219 map->range.end = end_offset;
10220 } else {
10221 panic("invalid p->state=%d\n", p->state);
10222 }
10223
10224 // not cloned yet
10225 map->cloned = false;
10226 queue_enter(mapq, map, pmap_pgtrace_map_t *, chain);
10227
10228 // unlock locks
10229 PMAP_PGTRACE_UNLOCK(&ints);
10230 if (pmap != kernel_pmap) {
10231 PMAP_UNLOCK(kernel_pmap);
10232 }
10233 if (pmap != NULL) {
10234 PMAP_UNLOCK(pmap);
10235 }
10236
10237 // now clone it
10238 if (pa_valid(pa_page)) {
10239 num_cloned = pmap_pgtrace_clone_from_pvtable(pa_page, start_offset, end_offset);
10240 }
10241 if (pmap == NULL) {
10242 num_cloned += pmap_pgtrace_clone_from_pa(kernel_pmap, pa_page, start_offset, end_offset);
10243 } else {
10244 num_cloned += pmap_pgtrace_clone_from_pa(pmap, pa_page, start_offset, end_offset);
10245 }
10246
10247 // free pre-allocations if we didn't add it to the q
10248 if (free_newp) {
10249 pmap_pgtrace_free_page(newp);
10250 }
10251
10252 if (num_cloned == 0) {
10253 kprintf("%s: no mapping found for pa_page=%llx but will be added when a page entered\n", __func__, pa_page);
10254 }
10255
10256 ret += num_cloned;
10257
10258 // overflow
10259 if (cur_page + ARM_PGBYTES < cur_page) {
10260 break;
10261 } else {
10262 cur_page += ARM_PGBYTES;
10263 }
10264 }
10265
10266 PROF_END
10267
10268 return ret;
10269 }
10270
10271 // delete page infos for given address range
10272 int pmap_pgtrace_delete_page(pmap_t pmap, vm_map_offset_t start, vm_map_offset_t end)
10273 {
10274 int ret = 0;
10275 bool ints;
10276 queue_head_t *q = &(pmap_pgtrace.pages);
10277 pmap_pgtrace_page_t *p;
10278 vm_map_offset_t cur_page, end_page;
10279
10280 kprintf("%s start=%llx end=%llx\n", __func__, start, end);
10281
10282 PROF_START
10283
10284 pt_entry_t *ptep;
10285 pmap_paddr_t pa_page;
10286
10287 // remove page info from start to end
10288 cur_page = arm_trunc_page(start);
10289 end_page = arm_trunc_page(end);
10290 while (cur_page <= end_page) {
10291 p = NULL;
10292
10293 if (pmap == NULL) {
10294 pa_page = cur_page;
10295 } else {
10296 PMAP_LOCK(pmap);
10297 ptep = pmap_pte(pmap, cur_page);
10298 if (ptep == NULL) {
10299 PMAP_UNLOCK(pmap);
10300 goto cont;
10301 }
10302 pa_page = pte_to_pa(*ptep);
10303 PMAP_UNLOCK(pmap);
10304 }
10305
10306 // remove all clones and validate
10307 pmap_pgtrace_remove_all_clone(pa_page);
10308
10309 // find page info and delete
10310 PMAP_PGTRACE_LOCK(&ints);
10311 p = pmap_pgtrace_find_page(pa_page);
10312 if (p != NULL) {
10313 queue_remove(q, p, pmap_pgtrace_page_t *, chain);
10314 ret++;
10315 }
10316 PMAP_PGTRACE_UNLOCK(&ints);
10317
10318 // free outside of locks
10319 if (p != NULL) {
10320 pmap_pgtrace_free_page(p);
10321 }
10322
10323 cont:
10324 // overflow
10325 if (cur_page + ARM_PGBYTES < cur_page) {
10326 break;
10327 } else {
10328 cur_page += ARM_PGBYTES;
10329 }
10330 }
10331
10332 PROF_END
10333
10334 return ret;
10335 }
10336
10337 kern_return_t pmap_pgtrace_fault(pmap_t pmap, vm_map_offset_t va, arm_saved_state_t *ss)
10338 {
10339 pt_entry_t *ptep;
10340 pgtrace_run_result_t res;
10341 pmap_pgtrace_page_t *p;
10342 bool ints, found = false;
10343 pmap_paddr_t pa;
10344
10345 // Quick check if we are interested
10346 ptep = pmap_pte(pmap, va);
10347 if (!ptep || !(*ptep & ARM_PTE_PGTRACE)) {
10348 return KERN_FAILURE;
10349 }
10350
10351 PMAP_PGTRACE_LOCK(&ints);
10352
10353 // Check again since access is serialized
10354 ptep = pmap_pte(pmap, va);
10355 if (!ptep || !(*ptep & ARM_PTE_PGTRACE)) {
10356 PMAP_PGTRACE_UNLOCK(&ints);
10357 return KERN_FAILURE;
10358
10359 } else if ((*ptep & ARM_PTE_TYPE_VALID) == ARM_PTE_TYPE_VALID) {
10360 // Somehow this cpu's tlb has not updated
10361 kprintf("%s Somehow this cpu's tlb has not updated?\n", __func__);
10362 PMAP_UPDATE_TLBS(pmap, va, va+ARM_PGBYTES);
10363
10364 PMAP_PGTRACE_UNLOCK(&ints);
10365 return KERN_SUCCESS;
10366 }
10367
10368 // Find if this pa is what we are tracing
10369 pa = pte_to_pa(*ptep);
10370
10371 p = pmap_pgtrace_find_page(arm_trunc_page(pa));
10372 if (p == NULL) {
10373 panic("%s Can't find va=%llx pa=%llx from tracing pages\n", __func__, va, pa);
10374 }
10375
10376 // find if pmap and va are also matching
10377 queue_head_t *mapq = &(p->maps);
10378 queue_head_t *mapwaste = &(p->map_waste);
10379 pmap_pgtrace_map_t *map;
10380
10381 queue_iterate(mapq, map, pmap_pgtrace_map_t *, chain) {
10382 if (map->pmap == pmap && map->ova == arm_trunc_page(va)) {
10383 found = true;
10384 break;
10385 }
10386 }
10387
10388 // if not found, search map waste as they are still valid
10389 if (!found) {
10390 queue_iterate(mapwaste, map, pmap_pgtrace_map_t *, chain) {
10391 if (map->pmap == pmap && map->ova == arm_trunc_page(va)) {
10392 found = true;
10393 break;
10394 }
10395 }
10396 }
10397
10398 if (!found) {
10399 panic("%s Can't find va=%llx pa=%llx from tracing pages\n", __func__, va, pa);
10400 }
10401
10402 // Decode and run it on the clone map
10403 bzero(&res, sizeof(res));
10404 pgtrace_decode_and_run(*(uint32_t *)get_saved_state_pc(ss), // instruction
10405 va, map->cva, // fault va and clone page vas
10406 ss, &res);
10407
10408 // write a log if in range
10409 vm_map_offset_t offset = va - map->ova;
10410 if (map->range.start <= offset && offset <= map->range.end) {
10411 pgtrace_write_log(res);
10412 }
10413
10414 PMAP_PGTRACE_UNLOCK(&ints);
10415
10416 // Return to next instruction
10417 set_saved_state_pc(ss, get_saved_state_pc(ss) + sizeof(uint32_t));
10418
10419 return KERN_SUCCESS;
10420 }
10421 #endif
10422
10423 boolean_t
10424 pmap_enforces_execute_only(
10425 #if (__ARM_VMSA__ == 7)
10426 __unused
10427 #endif
10428 pmap_t pmap)
10429 {
10430 #if (__ARM_VMSA__ > 7)
10431 return (pmap != kernel_pmap);
10432 #else
10433 return FALSE;
10434 #endif
10435 }
10436
10437 void
10438 pmap_set_jit_entitled(
10439 __unused pmap_t pmap)
10440 {
10441 return;
10442 }
10443
10444 static kern_return_t
10445 pmap_query_page_info_internal(
10446 pmap_t pmap,
10447 vm_map_offset_t va,
10448 int *disp_p)
10449 {
10450 int disp;
10451 pmap_paddr_t pa;
10452 int pai;
10453 pt_entry_t *pte;
10454 pv_entry_t **pv_h, *pve_p;
10455
10456 if (pmap == PMAP_NULL || pmap == kernel_pmap) {
10457 *disp_p = 0;
10458 return KERN_INVALID_ARGUMENT;
10459 }
10460
10461 disp = 0;
10462
10463 PMAP_LOCK(pmap);
10464
10465 pte = pmap_pte(pmap, va);
10466 if (pte == PT_ENTRY_NULL) {
10467 goto done;
10468 }
10469
10470 pa = pte_to_pa(*pte);
10471 if (pa == 0) {
10472 if (ARM_PTE_IS_COMPRESSED(*pte)) {
10473 disp |= PMAP_QUERY_PAGE_COMPRESSED;
10474 if (*pte & ARM_PTE_COMPRESSED_ALT) {
10475 disp |= PMAP_QUERY_PAGE_COMPRESSED_ALTACCT;
10476 }
10477 }
10478 } else {
10479 disp |= PMAP_QUERY_PAGE_PRESENT;
10480 pai = (int) pa_index(pa);
10481 if (!pa_valid(pa)) {
10482 goto done;
10483 }
10484 LOCK_PVH(pai);
10485 pv_h = pai_to_pvh(pai);
10486 pve_p = PV_ENTRY_NULL;
10487 if (pvh_test_type(pv_h, PVH_TYPE_PVEP)) {
10488 pve_p = pvh_list(pv_h);
10489 while (pve_p != PV_ENTRY_NULL &&
10490 pve_get_ptep(pve_p) != pte) {
10491 pve_p = pvh_list(pv_h);
10492 }
10493 }
10494 if (IS_ALTACCT_PAGE(pai, pve_p)) {
10495 disp |= PMAP_QUERY_PAGE_ALTACCT;
10496 } else if (IS_REUSABLE_PAGE(pai)) {
10497 disp |= PMAP_QUERY_PAGE_REUSABLE;
10498 } else if (IS_INTERNAL_PAGE(pai)) {
10499 disp |= PMAP_QUERY_PAGE_INTERNAL;
10500 }
10501 UNLOCK_PVH(pai);
10502 }
10503
10504 done:
10505 PMAP_UNLOCK(pmap);
10506 *disp_p = disp;
10507 return KERN_SUCCESS;
10508 }
10509
10510 kern_return_t
10511 pmap_query_page_info(
10512 pmap_t pmap,
10513 vm_map_offset_t va,
10514 int *disp_p)
10515 {
10516 return pmap_query_page_info_internal(pmap, va, disp_p);
10517 }
10518
10519 kern_return_t
10520 pmap_return_internal(__unused boolean_t do_panic, __unused boolean_t do_recurse)
10521 {
10522
10523 return KERN_SUCCESS;
10524 }
10525
10526 kern_return_t
10527 pmap_return(boolean_t do_panic, boolean_t do_recurse)
10528 {
10529 return pmap_return_internal(do_panic, do_recurse);
10530 }
10531
10532 static void
10533 pmap_footprint_suspend_internal(
10534 vm_map_t map,
10535 boolean_t suspend)
10536 {
10537 #if DEVELOPMENT || DEBUG
10538 if (suspend) {
10539 map->pmap->footprint_suspended = TRUE;
10540 map->pmap->footprint_was_suspended = TRUE;
10541 } else {
10542 map->pmap->footprint_suspended = FALSE;
10543 }
10544 #else /* DEVELOPMENT || DEBUG */
10545 (void) map;
10546 (void) suspend;
10547 #endif /* DEVELOPMENT || DEBUG */
10548 }
10549 void
10550 pmap_footprint_suspend(
10551 vm_map_t map,
10552 boolean_t suspend)
10553 {
10554 pmap_footprint_suspend_internal(map, suspend);
10555 }
mirror of XNU