]> git.saurik.com Git - apple/xnu.git/blob - osfmk/i386/pmap.h
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-4570.41.2.tar.gz
[apple/xnu.git] / osfmk / i386 / pmap.h
1 /*
2 * Copyright (c) 2000-2017 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56 /*
57 */
58
59 /*
60 * File: pmap.h
61 *
62 * Authors: Avadis Tevanian, Jr., Michael Wayne Young
63 * Date: 1985
64 *
65 * Machine-dependent structures for the physical map module.
66 */
67 #ifdef KERNEL_PRIVATE
68 #ifndef _PMAP_MACHINE_
69 #define _PMAP_MACHINE_ 1
70
71 #ifndef ASSEMBLER
72
73 #include <mach/kern_return.h>
74 #include <mach/machine/vm_types.h>
75 #include <mach/vm_prot.h>
76 #include <mach/vm_statistics.h>
77 #include <mach/machine/vm_param.h>
78 #include <kern/kern_types.h>
79 #include <kern/thread.h>
80 #include <kern/simple_lock.h>
81 #include <mach/branch_predicates.h>
82
83 #include <i386/mp.h>
84 #include <i386/proc_reg.h>
85
86 #include <i386/pal_routines.h>
87
88 /*
89 * Define the generic in terms of the specific
90 */
91
92 #define INTEL_PGBYTES I386_PGBYTES
93 #define INTEL_PGSHIFT I386_PGSHIFT
94 #define intel_btop(x) i386_btop(x)
95 #define intel_ptob(x) i386_ptob(x)
96 #define intel_round_page(x) i386_round_page(x)
97 #define intel_trunc_page(x) i386_trunc_page(x)
98
99 /*
100 * i386/i486/i860 Page Table Entry
101 */
102
103 #endif /* ASSEMBLER */
104
105 #define NPGPTD 4ULL
106 #define PDESHIFT 21ULL
107 #define PTEMASK 0x1ffULL
108 #define PTEINDX 3ULL
109
110 #define PTESHIFT 12ULL
111
112 #define LOW_4GB_MASK ((vm_offset_t)0x00000000FFFFFFFFUL)
113
114 #define PDESIZE sizeof(pd_entry_t) /* for assembly files */
115 #define PTESIZE sizeof(pt_entry_t) /* for assembly files */
116
117 #define INTEL_OFFMASK (I386_PGBYTES - 1)
118 #define INTEL_LOFFMASK (I386_LPGBYTES - 1)
119 #define PG_FRAME 0x000FFFFFFFFFF000ULL
120 #define NPTEPG (PAGE_SIZE/(sizeof (pt_entry_t)))
121 #define NPTDPG (PAGE_SIZE/(sizeof (pd_entry_t)))
122
123 #define NBPTD (NPGPTD << PAGE_SHIFT)
124 #define NPDEPTD (NBPTD / (sizeof (pd_entry_t)))
125 #define NPDEPG (PAGE_SIZE/(sizeof (pd_entry_t)))
126 #define NBPDE (1ULL << PDESHIFT)
127 #define PDEMASK (NBPDE - 1)
128
129 #define PTE_PER_PAGE 512 /* number of PTE's per page on any level */
130
131 /* cleanly define parameters for all the page table levels */
132 typedef uint64_t pml4_entry_t;
133 #define NPML4PG (PAGE_SIZE/(sizeof (pml4_entry_t)))
134 #define PML4SHIFT 39
135 #define PML4PGSHIFT 9
136 #define NBPML4 (1ULL << PML4SHIFT)
137 #define PML4MASK (NBPML4-1)
138 #define PML4_ENTRY_NULL ((pml4_entry_t *) 0)
139
140 typedef uint64_t pdpt_entry_t;
141 #define NPDPTPG (PAGE_SIZE/(sizeof (pdpt_entry_t)))
142 #define PDPTSHIFT 30
143 #define PDPTPGSHIFT 9
144 #define NBPDPT (1ULL << PDPTSHIFT)
145 #define PDPTMASK (NBPDPT-1)
146 #define PDPT_ENTRY_NULL ((pdpt_entry_t *) 0)
147
148 typedef uint64_t pd_entry_t;
149 #define NPDPG (PAGE_SIZE/(sizeof (pd_entry_t)))
150 #define PDSHIFT 21
151 #define PDPGSHIFT 9
152 #define NBPD (1ULL << PDSHIFT)
153 #define PDMASK (NBPD-1)
154 #define PD_ENTRY_NULL ((pd_entry_t *) 0)
155
156 typedef uint64_t pt_entry_t;
157 #define NPTPG (PAGE_SIZE/(sizeof (pt_entry_t)))
158 #define PTSHIFT 12
159 #define PTPGSHIFT 9
160 #define NBPT (1ULL << PTSHIFT)
161 #define PTMASK (NBPT-1)
162 #define PT_ENTRY_NULL ((pt_entry_t *) 0)
163
164 typedef uint64_t pmap_paddr_t;
165
166 #if DEVELOPMENT || DEBUG
167 #define PMAP_ASSERT 1
168 extern int pmap_asserts_enabled;
169 extern int pmap_asserts_traced;
170 #endif
171
172 #if PMAP_ASSERT
173 #define pmap_assert(ex) (pmap_asserts_enabled ? ((ex) ? (void)0 : Assert(__FILE__, __LINE__, # ex)) : (void)0)
174
175 #define pmap_assert2(ex, fmt, args...) \
176 do { \
177 if (__improbable(pmap_asserts_enabled && !(ex))) { \
178 if (pmap_asserts_traced) { \
179 KERNEL_DEBUG_CONSTANT(0xDEAD1000, __builtin_return_address(0), __LINE__, 0, 0, 0); \
180 kdebug_enable = 0; \
181 } else { \
182 kprintf("Assertion %s failed (%s:%d, caller %p) " fmt , #ex, __FILE__, __LINE__, __builtin_return_address(0), ##args); \
183 panic("Assertion %s failed (%s:%d, caller %p) " fmt , #ex, __FILE__, __LINE__, __builtin_return_address(0), ##args); \
184 } \
185 } \
186 } while(0)
187 #else
188 #define pmap_assert(ex)
189 #define pmap_assert2(ex, fmt, args...)
190 #endif
191
192 /* superpages */
193 #define SUPERPAGE_NBASEPAGES 512
194
195 /*
196 * Atomic 64-bit store of a page table entry.
197 */
198 static inline void
199 pmap_store_pte(pt_entry_t *entryp, pt_entry_t value)
200 {
201 /*
202 * In the 32-bit kernel a compare-and-exchange loop was
203 * required to provide atomicity. For K64, life is easier:
204 */
205 *entryp = value;
206 }
207
208 /* in 64 bit spaces, the number of each type of page in the page tables */
209 #define NPML4PGS (1ULL * (PAGE_SIZE/(sizeof (pml4_entry_t))))
210 #define NPDPTPGS (NPML4PGS * (PAGE_SIZE/(sizeof (pdpt_entry_t))))
211 #define NPDEPGS (NPDPTPGS * (PAGE_SIZE/(sizeof (pd_entry_t))))
212 #define NPTEPGS (NPDEPGS * (PAGE_SIZE/(sizeof (pt_entry_t))))
213
214 #define KERNEL_PML4_INDEX 511
215 #define KERNEL_KEXTS_INDEX 510 /* Home of KEXTs - the basement */
216 #define KERNEL_PHYSMAP_PML4_INDEX 509 /* virtual to physical map */
217 #define KERNEL_KASAN_PML4_INDEX0 508
218 #define KERNEL_KASAN_PML4_INDEX1 507
219 #define KERNEL_DBLMAP_PML4_INDEX (506)
220 #define KERNEL_BASE (0ULL - NBPML4)
221 #define KERNEL_BASEMENT (KERNEL_BASE - NBPML4)
222
223 #define VM_WIMG_COPYBACK VM_MEM_COHERENT
224 #define VM_WIMG_COPYBACKLW VM_WIMG_COPYBACK
225 #define VM_WIMG_DEFAULT VM_MEM_COHERENT
226 /* ?? intel ?? */
227 #define VM_WIMG_IO (VM_MEM_COHERENT | \
228 VM_MEM_NOT_CACHEABLE | VM_MEM_GUARDED)
229 #define VM_WIMG_POSTED VM_WIMG_IO
230 #define VM_WIMG_WTHRU (VM_MEM_WRITE_THROUGH | VM_MEM_COHERENT | VM_MEM_GUARDED)
231 /* write combining mode, aka store gather */
232 #define VM_WIMG_WCOMB (VM_MEM_NOT_CACHEABLE | VM_MEM_COHERENT)
233 #define VM_WIMG_INNERWBACK VM_MEM_COHERENT
234 /*
235 * Pte related macros
236 */
237 #define KVADDR(pmi, pdpi, pdi, pti) \
238 ((vm_offset_t) \
239 ((uint64_t) -1 << 47) | \
240 ((uint64_t)(pmi) << PML4SHIFT) | \
241 ((uint64_t)(pdpi) << PDPTSHIFT) | \
242 ((uint64_t)(pdi) << PDESHIFT) | \
243 ((uint64_t)(pti) << PTESHIFT))
244
245 /*
246 * Size of Kernel address space. This is the number of page table pages
247 * (4MB each) to use for the kernel. 256 pages == 1 Gigabyte.
248 * This **MUST** be a multiple of 4 (eg: 252, 256, 260, etc).
249 */
250 #ifndef KVA_PAGES
251 #define KVA_PAGES 1024
252 #endif
253
254 #ifndef NKPT
255 #define NKPT 500 /* actual number of kernel page tables */
256 #endif
257 #ifndef NKPDE
258 #define NKPDE (KVA_PAGES - 1) /* addressable number of page tables/pde's */
259 #endif
260
261
262
263 /*
264 * Convert address offset to page descriptor index
265 */
266 #define pdptnum(pmap, a) (((vm_offset_t)(a) >> PDPTSHIFT) & PDPTMASK)
267 #define pdenum(pmap, a) (((vm_offset_t)(a) >> PDESHIFT) & PDEMASK)
268 #define PMAP_INVALID_PDPTNUM (~0ULL)
269
270 #define pdeidx(pmap, a) (((a) >> PDSHIFT) & ((1ULL<<(48 - PDSHIFT)) -1))
271 #define pdptidx(pmap, a) (((a) >> PDPTSHIFT) & ((1ULL<<(48 - PDPTSHIFT)) -1))
272 #define pml4idx(pmap, a) (((a) >> PML4SHIFT) & ((1ULL<<(48 - PML4SHIFT)) -1))
273
274
275 /*
276 * Convert page descriptor index to user virtual address
277 */
278 #define pdetova(a) ((vm_offset_t)(a) << PDESHIFT)
279
280 /*
281 * Convert address offset to page table index
282 */
283 #define ptenum(a) (((vm_offset_t)(a) >> PTESHIFT) & PTEMASK)
284
285 /*
286 * Hardware pte bit definitions (to be used directly on the ptes
287 * without using the bit fields).
288 */
289
290 #define INTEL_PTE_VALID 0x00000001ULL
291 #define INTEL_PTE_WRITE 0x00000002ULL
292 #define INTEL_PTE_RW 0x00000002ULL
293 #define INTEL_PTE_USER 0x00000004ULL
294 #define INTEL_PTE_WTHRU 0x00000008ULL
295 #define INTEL_PTE_NCACHE 0x00000010ULL
296 #define INTEL_PTE_REF 0x00000020ULL
297 #define INTEL_PTE_MOD 0x00000040ULL
298 #define INTEL_PTE_PS 0x00000080ULL
299 #define INTEL_PTE_PTA 0x00000080ULL
300 #define INTEL_PTE_GLOBAL 0x00000100ULL
301 #define INTEL_PTE_WIRED 0x00000400ULL
302 #define INTEL_PDPTE_NESTED 0x00000800ULL
303 #define INTEL_PTE_PFN PG_FRAME
304
305 #define INTEL_PTE_NX (1ULL << 63)
306
307 #define INTEL_PTE_INVALID 0
308 /* This is conservative, but suffices */
309 #define INTEL_PTE_RSVD ((1ULL << 10) | (1ULL << 11) | (0x1FFULL << 54))
310
311 #define INTEL_PTE_COMPRESSED (1ULL << 62) /* marker, for invalid PTE only -- ignored by hardware for both regular/EPT entries*/
312 #define INTEL_PTE_COMPRESSED_ALT (1ULL << 61) /* compressed but with "alternate accounting" */
313
314 #define INTEL_PTE_COMPRESSED_MASK (INTEL_PTE_COMPRESSED | \
315 INTEL_PTE_COMPRESSED_ALT)
316 #define PTE_IS_COMPRESSED(x) \
317 ((((x) & INTEL_PTE_VALID) == 0) && /* PTE is not valid... */ \
318 ((x) & INTEL_PTE_COMPRESSED) && /* ...has "compressed" marker" */ \
319 ((!((x) & ~INTEL_PTE_COMPRESSED_MASK)) || /* ...no other bits */ \
320 (panic("compressed PTE %p 0x%llx has extra bits 0x%llx: corrupted?", \
321 &(x), (x), (x) & ~INTEL_PTE_COMPRESSED_MASK), FALSE)))
322
323 #define pa_to_pte(a) ((a) & INTEL_PTE_PFN) /* XXX */
324 #define pte_to_pa(p) ((p) & INTEL_PTE_PFN) /* XXX */
325 #define pte_increment_pa(p) ((p) += INTEL_OFFMASK+1)
326
327 #define pte_kernel_rw(p) ((pt_entry_t)(pa_to_pte(p) | INTEL_PTE_VALID|INTEL_PTE_RW))
328 #define pte_kernel_ro(p) ((pt_entry_t)(pa_to_pte(p) | INTEL_PTE_VALID))
329 #define pte_user_rw(p) ((pt_entry_t)(pa_to_pte(p) | INTEL_PTE_VALID|INTEL_PTE_USER|INTEL_PTE_RW))
330 #define pte_user_ro(p) ((pt_entry_t)(pa_to_pte(p) | INTEL_PTE_VALID|INTEL_PTE_USER))
331
332 #define PMAP_INVEPT_SINGLE_CONTEXT 1
333
334
335 #define INTEL_EPTP_AD 0x00000040ULL
336
337 #define INTEL_EPT_READ 0x00000001ULL
338 #define INTEL_EPT_WRITE 0x00000002ULL
339 #define INTEL_EPT_EX 0x00000004ULL
340 #define INTEL_EPT_IPTA 0x00000040ULL
341 #define INTEL_EPT_PS 0x00000080ULL
342 #define INTEL_EPT_REF 0x00000100ULL
343 #define INTEL_EPT_MOD 0x00000200ULL
344
345 #define INTEL_EPT_CACHE_MASK 0x00000038ULL
346 #define INTEL_EPT_NCACHE 0x00000000ULL
347 #define INTEL_EPT_WC 0x00000008ULL
348 #define INTEL_EPT_WTHRU 0x00000020ULL
349 #define INTEL_EPT_WP 0x00000028ULL
350 #define INTEL_EPT_WB 0x00000030ULL
351
352 /*
353 * Routines to filter correct bits depending on the pmap type
354 */
355
356 static inline pt_entry_t
357 pte_remove_ex(pt_entry_t pte, boolean_t is_ept)
358 {
359 if (__probable(!is_ept)) {
360 return (pte | INTEL_PTE_NX);
361 }
362
363 return (pte & (~INTEL_EPT_EX));
364 }
365
366 static inline pt_entry_t
367 pte_set_ex(pt_entry_t pte, boolean_t is_ept)
368 {
369 if (__probable(!is_ept)) {
370 return (pte & (~INTEL_PTE_NX));
371 }
372
373 return (pte | INTEL_EPT_EX);
374 }
375
376 static inline pt_entry_t
377 physmap_refmod_to_ept(pt_entry_t physmap_pte)
378 {
379 pt_entry_t ept_pte = 0;
380
381 if (physmap_pte & INTEL_PTE_MOD) {
382 ept_pte |= INTEL_EPT_MOD;
383 }
384
385 if (physmap_pte & INTEL_PTE_REF) {
386 ept_pte |= INTEL_EPT_REF;
387 }
388
389 return ept_pte;
390 }
391
392 static inline pt_entry_t
393 ept_refmod_to_physmap(pt_entry_t ept_pte)
394 {
395 pt_entry_t physmap_pte = 0;
396
397 assert((ept_pte & ~(INTEL_EPT_REF | INTEL_EPT_MOD)) == 0);
398
399 if (ept_pte & INTEL_EPT_REF) {
400 physmap_pte |= INTEL_PTE_REF;
401 }
402
403 if (ept_pte & INTEL_EPT_MOD) {
404 physmap_pte |= INTEL_PTE_MOD;
405 }
406
407 return physmap_pte;
408 }
409
410 /*
411 * Note: Not all Intel processors support EPT referenced access and dirty bits.
412 * During pmap_init() we check the VMX capability for the current hardware
413 * and update this variable accordingly.
414 */
415 extern boolean_t pmap_ept_support_ad;
416
417 #define PTE_VALID_MASK(is_ept) ((is_ept) ? (INTEL_EPT_READ | INTEL_EPT_WRITE | INTEL_EPT_EX) : INTEL_PTE_VALID)
418 #define PTE_READ(is_ept) ((is_ept) ? INTEL_EPT_READ : INTEL_PTE_VALID)
419 #define PTE_WRITE(is_ept) ((is_ept) ? INTEL_EPT_WRITE : INTEL_PTE_WRITE)
420 #define PTE_PS INTEL_PTE_PS
421 #define PTE_COMPRESSED INTEL_PTE_COMPRESSED
422 #define PTE_COMPRESSED_ALT INTEL_PTE_COMPRESSED_ALT
423 #define PTE_NCACHE(is_ept) ((is_ept) ? INTEL_EPT_NCACHE : INTEL_PTE_NCACHE)
424 #define PTE_WTHRU(is_ept) ((is_ept) ? INTEL_EPT_WTHRU : INTEL_PTE_WTHRU)
425 #define PTE_REF(is_ept) ((is_ept) ? INTEL_EPT_REF : INTEL_PTE_REF)
426 #define PTE_MOD(is_ept) ((is_ept) ? INTEL_EPT_MOD : INTEL_PTE_MOD)
427 #define PTE_WIRED INTEL_PTE_WIRED
428
429
430 #define PMAP_DEFAULT_CACHE 0
431 #define PMAP_INHIBIT_CACHE 1
432 #define PMAP_GUARDED_CACHE 2
433 #define PMAP_ACTIVATE_CACHE 4
434 #define PMAP_NO_GUARD_CACHE 8
435
436 #ifndef ASSEMBLER
437
438 #include <sys/queue.h>
439
440 /*
441 * Address of current and alternate address space page table maps
442 * and directories.
443 */
444
445 extern pt_entry_t *PTmap;
446 extern pdpt_entry_t *IdlePDPT;
447 extern pml4_entry_t *IdlePML4;
448 extern boolean_t no_shared_cr3;
449 extern pd_entry_t *IdlePTD; /* physical addr of "Idle" state PTD */
450
451 extern uint64_t pmap_pv_hashlist_walks;
452 extern uint64_t pmap_pv_hashlist_cnts;
453 extern uint32_t pmap_pv_hashlist_max;
454 extern uint32_t pmap_kernel_text_ps;
455
456 #define ID_MAP_VTOP(x) ((void *)(((uint64_t)(x)) & LOW_4GB_MASK))
457
458 extern uint64_t physmap_base, physmap_max;
459
460 #define NPHYSMAP (MAX(K64_MAXMEM/GB + 4, 4))
461
462 static inline boolean_t physmap_enclosed(addr64_t a) {
463 return (a < (NPHYSMAP * GB));
464 }
465
466 static inline void * PHYSMAP_PTOV_check(void *paddr) {
467 uint64_t pvaddr = (uint64_t)paddr + physmap_base;
468
469 if (__improbable(pvaddr >= physmap_max))
470 panic("PHYSMAP_PTOV bounds exceeded, 0x%qx, 0x%qx, 0x%qx",
471 pvaddr, physmap_base, physmap_max);
472
473 return (void *)pvaddr;
474 }
475
476 #define PHYSMAP_PTOV(x) (PHYSMAP_PTOV_check((void*) (x)))
477 #if MACH_KERNEL_PRIVATE
478 extern uint64_t dblmap_base, dblmap_max, dblmap_dist;
479
480 static inline uint64_t DBLMAP_CHECK(uintptr_t x) {
481 uint64_t dbladdr = (uint64_t)x + dblmap_dist;
482 if (__improbable((dbladdr >= dblmap_max) || (dbladdr < dblmap_base))) {
483 panic("DBLMAP bounds exceeded, 0x%qx, 0x%qx 0x%qx, 0x%qx",
484 (uint64_t)x, dbladdr, dblmap_base, dblmap_max);
485 }
486 return dbladdr;
487
488 }
489 #define DBLMAP(x) (DBLMAP_CHECK((uint64_t) x))
490 extern uint64_t ldt_alias_offset;
491 static inline uint64_t LDTALIAS_CHECK(uintptr_t x) {
492 uint64_t dbladdr = (uint64_t)x + ldt_alias_offset;
493 if (__improbable((dbladdr >= dblmap_max) || (dbladdr < dblmap_base))) {
494 panic("LDTALIAS: bounds exceeded, 0x%qx, 0x%qx 0x%qx, 0x%qx",
495 (uint64_t)x, dbladdr, dblmap_base, dblmap_max);
496 }
497 return dbladdr;
498 }
499 #define LDTALIAS(x) (LDTALIAS_CHECK((uint64_t) x))
500 #endif
501
502 /*
503 * For KASLR, we alias the master processor's IDT and GDT at fixed
504 * virtual addresses to defeat SIDT/SGDT address leakage.
505 * And non-boot processor's GDT aliases likewise (skipping LOWGLOBAL_ALIAS)
506 * The low global vector page is mapped at a fixed alias also.
507 */
508 #define LOWGLOBAL_ALIAS (VM_MIN_KERNEL_ADDRESS + 0x2000)
509
510 /*
511 * This indicates (roughly) where there is free space for the VM
512 * to use for the heap; this does not need to be precise.
513 */
514 #define KERNEL_PMAP_HEAP_RANGE_START VM_MIN_KERNEL_AND_KEXT_ADDRESS
515
516 #include <vm/vm_page.h>
517
518 /*
519 * For each vm_page_t, there is a list of all currently
520 * valid virtual mappings of that page. An entry is
521 * a pv_entry_t; the list is the pv_table.
522 */
523
524 struct pmap {
525 decl_simple_lock_data(,lock) /* lock on map */
526 pmap_paddr_t pm_cr3; /* Kernel+user shared PML4 physical*/
527 pmap_paddr_t pm_ucr3; /* Mirrored user PML4 physical */
528 task_map_t pm_task_map;
529 boolean_t pm_shared;
530 boolean_t pagezero_accessible;
531 #define PMAP_PCID_MAX_CPUS MAX_CPUS /* Must be a multiple of 8 */
532 pcid_t pmap_pcid_cpus[PMAP_PCID_MAX_CPUS];
533 volatile uint8_t pmap_pcid_coherency_vector[PMAP_PCID_MAX_CPUS];
534 struct pmap_statistics stats; /* map statistics */
535 int ref_count; /* reference count */
536 int nx_enabled;
537 pml4_entry_t *pm_pml4; /* VKA of top level */
538 pml4_entry_t *pm_upml4; /* Shadow VKA of top level */
539 vm_object_t pm_obj; /* object to hold pde's */
540 vm_object_t pm_obj_pdpt; /* holds pdpt pages */
541 vm_object_t pm_obj_pml4; /* holds pml4 pages */
542 pmap_paddr_t pm_eptp; /* EPTP */
543 ledger_t ledger; /* ledger tracking phys mappings */
544 #if MACH_ASSERT
545 int pmap_pid;
546 char pmap_procname[17];
547 #endif /* MACH_ASSERT */
548 };
549
550 static inline boolean_t
551 is_ept_pmap(pmap_t p)
552 {
553 if (__probable(p->pm_cr3 != 0)) {
554 assert(p->pm_eptp == 0);
555 return FALSE;
556 }
557
558 assert(p->pm_eptp != 0);
559
560 return TRUE;
561 }
562
563 void hv_ept_pmap_create(void **ept_pmap, void **eptp);
564
565 #if NCOPY_WINDOWS > 0
566 #define PMAP_PDPT_FIRST_WINDOW 0
567 #define PMAP_PDPT_NWINDOWS 4
568 #define PMAP_PDE_FIRST_WINDOW (PMAP_PDPT_NWINDOWS)
569 #define PMAP_PDE_NWINDOWS 4
570 #define PMAP_PTE_FIRST_WINDOW (PMAP_PDE_FIRST_WINDOW + PMAP_PDE_NWINDOWS)
571 #define PMAP_PTE_NWINDOWS 4
572
573 #define PMAP_NWINDOWS_FIRSTFREE (PMAP_PTE_FIRST_WINDOW + PMAP_PTE_NWINDOWS)
574 #define PMAP_WINDOW_SIZE 8
575 #define PMAP_NWINDOWS (PMAP_NWINDOWS_FIRSTFREE + PMAP_WINDOW_SIZE)
576
577 typedef struct {
578 pt_entry_t *prv_CMAP;
579 caddr_t prv_CADDR;
580 } mapwindow_t;
581
582 typedef struct cpu_pmap {
583 int pdpt_window_index;
584 int pde_window_index;
585 int pte_window_index;
586 mapwindow_t mapwindow[PMAP_NWINDOWS];
587 } cpu_pmap_t;
588
589
590 extern mapwindow_t *pmap_get_mapwindow(pt_entry_t pentry);
591 extern void pmap_put_mapwindow(mapwindow_t *map);
592 #endif
593
594 typedef struct pmap_memory_regions {
595 ppnum_t base; /* first page of this region */
596 ppnum_t alloc_up; /* pages below this one have been "stolen" */
597 ppnum_t alloc_down; /* pages above this one have been "stolen" */
598 ppnum_t end; /* last page of this region */
599 uint32_t type;
600 uint64_t attribute;
601 } pmap_memory_region_t;
602
603 extern unsigned pmap_memory_region_count;
604 extern unsigned pmap_memory_region_current;
605
606 #define PMAP_MEMORY_REGIONS_SIZE 128
607
608 extern pmap_memory_region_t pmap_memory_regions[];
609 #include <i386/pmap_pcid.h>
610
611 static inline void
612 set_dirbase(pmap_t tpmap, thread_t thread, int my_cpu) {
613 int ccpu = my_cpu;
614 uint64_t pcr3 = tpmap->pm_cr3, ucr3 = tpmap->pm_ucr3;
615 cpu_datap(ccpu)->cpu_task_cr3 = pcr3;
616 cpu_shadowp(ccpu)->cpu_task_cr3 = pcr3;
617
618 cpu_datap(ccpu)->cpu_ucr3 = ucr3;
619 cpu_shadowp(ccpu)->cpu_ucr3 = ucr3;
620
621 cpu_datap(ccpu)->cpu_task_map = tpmap->pm_task_map;
622
623 assert((get_preemption_level() > 0) || (ml_get_interrupts_enabled() == FALSE));
624 assert(ccpu == cpu_number());
625 /*
626 * Switch cr3 if necessary
627 * - unless running with no_shared_cr3 debugging mode
628 * and we're not on the kernel's cr3 (after pre-empted copyio)
629 */
630 boolean_t nopagezero = tpmap->pagezero_accessible;
631 boolean_t priorpagezero = cpu_datap(ccpu)->cpu_pagezero_mapped;
632 cpu_datap(ccpu)->cpu_pagezero_mapped = nopagezero;
633
634 if (__probable(!no_shared_cr3)) {
635 if (__improbable(nopagezero)) {
636 boolean_t copyio_active = ((thread->machine.specFlags & CopyIOActive) != 0);
637 if (pmap_pcid_ncpus) {
638 pmap_pcid_activate(tpmap, ccpu, TRUE, copyio_active);
639 } else {
640 if (copyio_active) {
641 if (get_cr3_base() != tpmap->pm_cr3) {
642 set_cr3_raw(tpmap->pm_cr3);
643 }
644 } else if (get_cr3_base() != cpu_datap(ccpu)->cpu_kernel_cr3) {
645 set_cr3_raw(cpu_datap(ccpu)->cpu_kernel_cr3);
646 }
647 }
648 } else if ((get_cr3_base() != tpmap->pm_cr3) || priorpagezero) {
649 if (pmap_pcid_ncpus) {
650 pmap_pcid_activate(tpmap, ccpu, FALSE, FALSE);
651 } else {
652 set_cr3_raw(tpmap->pm_cr3);
653 }
654 }
655 } else {
656 if (get_cr3_base() != cpu_datap(ccpu)->cpu_kernel_cr3)
657 set_cr3_raw(cpu_datap(ccpu)->cpu_kernel_cr3);
658 }
659 }
660
661 /*
662 * External declarations for PMAP_ACTIVATE.
663 */
664
665 extern void process_pmap_updates(void);
666 extern void pmap_update_interrupt(void);
667
668 extern addr64_t (kvtophys)(
669 vm_offset_t addr);
670
671 extern kern_return_t pmap_expand(
672 pmap_t pmap,
673 vm_map_offset_t addr,
674 unsigned int options);
675 extern vm_offset_t pmap_map(
676 vm_offset_t virt,
677 vm_map_offset_t start,
678 vm_map_offset_t end,
679 vm_prot_t prot,
680 unsigned int flags);
681
682 extern vm_offset_t pmap_map_bd(
683 vm_offset_t virt,
684 vm_map_offset_t start,
685 vm_map_offset_t end,
686 vm_prot_t prot,
687 unsigned int flags);
688 extern void pmap_bootstrap(
689 vm_offset_t load_start,
690 boolean_t IA32e);
691
692 extern boolean_t pmap_valid_page(
693 ppnum_t pn);
694
695 extern int pmap_list_resident_pages(
696 struct pmap *pmap,
697 vm_offset_t *listp,
698 int space);
699 extern void x86_filter_TLB_coherency_interrupts(boolean_t);
700 /*
701 * Get cache attributes (as pagetable bits) for the specified phys page
702 */
703 extern unsigned pmap_get_cache_attributes(ppnum_t, boolean_t is_ept);
704 #if NCOPY_WINDOWS > 0
705 extern struct cpu_pmap *pmap_cpu_alloc(
706 boolean_t is_boot_cpu);
707 extern void pmap_cpu_free(
708 struct cpu_pmap *cp);
709 #endif
710
711 extern kern_return_t pmap_map_block(
712 pmap_t pmap,
713 addr64_t va,
714 ppnum_t pa,
715 uint32_t size,
716 vm_prot_t prot,
717 int attr,
718 unsigned int flags);
719
720 extern void invalidate_icache(vm_offset_t addr, unsigned cnt, int phys);
721 extern void flush_dcache(vm_offset_t addr, unsigned count, int phys);
722 extern ppnum_t pmap_find_phys(pmap_t map, addr64_t va);
723
724 extern void pmap_cpu_init(void);
725 extern void pmap_disable_NX(pmap_t pmap);
726
727 extern void pt_fake_zone_init(int);
728 extern void pt_fake_zone_info(int *, vm_size_t *, vm_size_t *, vm_size_t *, vm_size_t *,
729 uint64_t *, int *, int *, int *);
730 extern void pmap_pagetable_corruption_msg_log(int (*)(const char * fmt, ...)__printflike(1,2));
731
732 /*
733 * Macros for speed.
734 */
735
736
737 #include <kern/spl.h>
738
739
740 #define PMAP_ACTIVATE_MAP(map, thread, my_cpu) { \
741 pmap_t tpmap; \
742 \
743 tpmap = vm_map_pmap(map); \
744 set_dirbase(tpmap, thread, my_cpu); \
745 }
746
747 #if defined(__x86_64__)
748 #define PMAP_DEACTIVATE_MAP(map, thread, ccpu) \
749 pmap_assert2((pmap_pcid_ncpus ? (pcid_for_pmap_cpu_tuple(map->pmap, thread, ccpu) == (get_cr3_raw() & 0xFFF)) : TRUE),"PCIDs: 0x%x, active PCID: 0x%x, CR3: 0x%lx, pmap_cr3: 0x%llx, kernel_cr3: 0x%llx, kernel pmap cr3: 0x%llx, CPU active PCID: 0x%x, CPU kernel PCID: 0x%x, specflags: 0x%x, pagezero: 0x%x", pmap_pcid_ncpus, pcid_for_pmap_cpu_tuple(map->pmap, thread, ccpu), get_cr3_raw(), map->pmap->pm_cr3, cpu_datap(ccpu)->cpu_kernel_cr3, kernel_pmap->pm_cr3, cpu_datap(ccpu)->cpu_active_pcid, cpu_datap(ccpu)->cpu_kernel_pcid, thread->machine.specFlags, map->pmap->pagezero_accessible);
750 #else
751 #define PMAP_DEACTIVATE_MAP(map, thread)
752 #endif
753
754 #if NCOPY_WINDOWS > 0
755 #define PMAP_SWITCH_USER(th, new_map, my_cpu) { \
756 spl_t spl; \
757 \
758 spl = splhigh(); \
759 PMAP_DEACTIVATE_MAP(th->map, th); \
760 th->map = new_map; \
761 PMAP_ACTIVATE_MAP(th->map, th); \
762 splx(spl); \
763 inval_copy_windows(th); \
764 }
765 #else
766 #define PMAP_SWITCH_USER(th, new_map, my_cpu) { \
767 spl_t spl; \
768 \
769 spl = splhigh(); \
770 PMAP_DEACTIVATE_MAP(th->map, th, my_cpu); \
771 th->map = new_map; \
772 PMAP_ACTIVATE_MAP(th->map, th, my_cpu); \
773 splx(spl); \
774 }
775 #endif
776
777 /*
778 * Marking the current cpu's cr3 inactive is achieved by setting its lsb.
779 * Marking the current cpu's cr3 active once more involves clearng this bit.
780 * Note that valid page tables are page-aligned and so the bottom 12 bits
781 * are normally zero, modulo PCID.
782 * We can only mark the current cpu active/inactive but we can test any cpu.
783 */
784 #define CPU_CR3_MARK_INACTIVE() \
785 current_cpu_datap()->cpu_active_cr3 |= 1
786
787 #define CPU_CR3_MARK_ACTIVE() \
788 current_cpu_datap()->cpu_active_cr3 &= ~1
789
790 #define CPU_CR3_IS_ACTIVE(cpu) \
791 ((cpu_datap(cpu)->cpu_active_cr3 & 1) == 0)
792
793 #define CPU_GET_ACTIVE_CR3(cpu) \
794 (cpu_datap(cpu)->cpu_active_cr3 & ~1)
795
796 #define CPU_GET_TASK_CR3(cpu) \
797 (cpu_datap(cpu)->cpu_task_cr3)
798
799 /*
800 * Mark this cpu idle, and remove it from the active set,
801 * since it is not actively using any pmap. Signal_cpus
802 * will notice that it is idle, and avoid signaling it,
803 * but will queue the update request for when the cpu
804 * becomes active.
805 */
806 #define MARK_CPU_IDLE(my_cpu) { \
807 assert(ml_get_interrupts_enabled() == FALSE); \
808 CPU_CR3_MARK_INACTIVE(); \
809 mfence(); \
810 }
811
812 #define MARK_CPU_ACTIVE(my_cpu) { \
813 assert(ml_get_interrupts_enabled() == FALSE); \
814 /* \
815 * If a kernel_pmap update was requested while this cpu \
816 * was idle, process it as if we got the interrupt. \
817 * Before doing so, remove this cpu from the idle set. \
818 * Since we do not grab any pmap locks while we flush \
819 * our TLB, another cpu may start an update operation \
820 * before we finish. Removing this cpu from the idle \
821 * set assures that we will receive another update \
822 * interrupt if this happens. \
823 */ \
824 CPU_CR3_MARK_ACTIVE(); \
825 mfence(); \
826 \
827 if (current_cpu_datap()->cpu_tlb_invalid) \
828 process_pmap_updates(); \
829 }
830
831 #define PMAP_CONTEXT(pmap, thread)
832
833 #define pmap_kernel_va(VA) \
834 ((((vm_offset_t) (VA)) >= vm_min_kernel_address) && \
835 (((vm_offset_t) (VA)) <= vm_max_kernel_address))
836
837
838 #define pmap_compressed(pmap) ((pmap)->stats.compressed)
839 #define pmap_resident_count(pmap) ((pmap)->stats.resident_count)
840 #define pmap_resident_max(pmap) ((pmap)->stats.resident_max)
841 #define pmap_copy(dst_pmap,src_pmap,dst_addr,len,src_addr)
842 #define pmap_attribute(pmap,addr,size,attr,value) \
843 (KERN_INVALID_ADDRESS)
844 #define pmap_attribute_cache_sync(addr,size,attr,value) \
845 (KERN_INVALID_ADDRESS)
846
847 #define MACHINE_PMAP_IS_EMPTY 1
848 extern boolean_t pmap_is_empty(pmap_t pmap,
849 vm_map_offset_t start,
850 vm_map_offset_t end);
851
852 #define MACHINE_BOOTSTRAPPTD 1 /* Static bootstrap page-tables */
853
854 kern_return_t
855 pmap_permissions_verify(pmap_t, vm_map_t, vm_offset_t, vm_offset_t);
856
857 #if MACH_ASSERT
858 extern int pmap_stats_assert;
859 #define PMAP_STATS_ASSERTF(args) \
860 MACRO_BEGIN \
861 if (pmap_stats_assert) assertf args; \
862 MACRO_END
863 #else /* MACH_ASSERT */
864 #define PMAP_STATS_ASSERTF(args)
865 #endif /* MACH_ASSERT */
866 #endif /* ASSEMBLER */
867 #endif /* _PMAP_MACHINE_ */
868 #endif /* KERNEL_PRIVATE */
mirror of XNU