]> git.saurik.com Git - apple/xnu.git/blob - osfmk/i386/pmap.h
projects / apple / xnu.git / blob
? search:


ccad03ea7b859a11ea36da830ebeffc66c99fdfd
[apple/xnu.git] / osfmk / i386 / pmap.h
1 /*
2 * Copyright (c) 2000-2012 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56 /*
57 */
58
59 /*
60 * File: pmap.h
61 *
62 * Authors: Avadis Tevanian, Jr., Michael Wayne Young
63 * Date: 1985
64 *
65 * Machine-dependent structures for the physical map module.
66 */
67 #ifdef KERNEL_PRIVATE
68 #ifndef _PMAP_MACHINE_
69 #define _PMAP_MACHINE_ 1
70
71 #ifndef ASSEMBLER
72
73
74 #include <mach/kern_return.h>
75 #include <mach/machine/vm_types.h>
76 #include <mach/vm_prot.h>
77 #include <mach/vm_statistics.h>
78 #include <mach/machine/vm_param.h>
79 #include <kern/kern_types.h>
80 #include <kern/thread.h>
81 #include <kern/simple_lock.h>
82 #include <mach/branch_predicates.h>
83
84 #include <i386/mp.h>
85 #include <i386/proc_reg.h>
86
87 #include <i386/pal_routines.h>
88
89 /*
90 * Define the generic in terms of the specific
91 */
92
93 #define INTEL_PGBYTES I386_PGBYTES
94 #define INTEL_PGSHIFT I386_PGSHIFT
95 #define intel_btop(x) i386_btop(x)
96 #define intel_ptob(x) i386_ptob(x)
97 #define intel_round_page(x) i386_round_page(x)
98 #define intel_trunc_page(x) i386_trunc_page(x)
99
100 /*
101 * i386/i486/i860 Page Table Entry
102 */
103
104 #endif /* ASSEMBLER */
105
106 #define NPGPTD 4ULL
107 #define PDESHIFT 21ULL
108 #define PTEMASK 0x1ffULL
109 #define PTEINDX 3ULL
110
111 #define PTESHIFT 12ULL
112
113
114 #ifdef __x86_64__
115 #define LOW_4GB_MASK ((vm_offset_t)0x00000000FFFFFFFFUL)
116 #endif
117
118 #define PDESIZE sizeof(pd_entry_t) /* for assembly files */
119 #define PTESIZE sizeof(pt_entry_t) /* for assembly files */
120
121 #define INTEL_OFFMASK (I386_PGBYTES - 1)
122 #define INTEL_LOFFMASK (I386_LPGBYTES - 1)
123 #define PG_FRAME 0x000FFFFFFFFFF000ULL
124 #define NPTEPG (PAGE_SIZE/(sizeof (pt_entry_t)))
125 #define NPTDPG (PAGE_SIZE/(sizeof (pd_entry_t)))
126
127 #define NBPTD (NPGPTD << PAGE_SHIFT)
128 #define NPDEPTD (NBPTD / (sizeof (pd_entry_t)))
129 #define NPDEPG (PAGE_SIZE/(sizeof (pd_entry_t)))
130 #define NBPDE (1ULL << PDESHIFT)
131 #define PDEMASK (NBPDE - 1)
132
133 #define PTE_PER_PAGE 512 /* number of PTE's per page on any level */
134
135 /* cleanly define parameters for all the page table levels */
136 typedef uint64_t pml4_entry_t;
137 #define NPML4PG (PAGE_SIZE/(sizeof (pml4_entry_t)))
138 #define PML4SHIFT 39
139 #define PML4PGSHIFT 9
140 #define NBPML4 (1ULL << PML4SHIFT)
141 #define PML4MASK (NBPML4-1)
142 #define PML4_ENTRY_NULL ((pml4_entry_t *) 0)
143
144 typedef uint64_t pdpt_entry_t;
145 #define NPDPTPG (PAGE_SIZE/(sizeof (pdpt_entry_t)))
146 #define PDPTSHIFT 30
147 #define PDPTPGSHIFT 9
148 #define NBPDPT (1ULL << PDPTSHIFT)
149 #define PDPTMASK (NBPDPT-1)
150 #define PDPT_ENTRY_NULL ((pdpt_entry_t *) 0)
151
152 typedef uint64_t pd_entry_t;
153 #define NPDPG (PAGE_SIZE/(sizeof (pd_entry_t)))
154 #define PDSHIFT 21
155 #define PDPGSHIFT 9
156 #define NBPD (1ULL << PDSHIFT)
157 #define PDMASK (NBPD-1)
158 #define PD_ENTRY_NULL ((pd_entry_t *) 0)
159
160 typedef uint64_t pt_entry_t;
161 #define NPTPG (PAGE_SIZE/(sizeof (pt_entry_t)))
162 #define PTSHIFT 12
163 #define PTPGSHIFT 9
164 #define NBPT (1ULL << PTSHIFT)
165 #define PTMASK (NBPT-1)
166 #define PT_ENTRY_NULL ((pt_entry_t *) 0)
167
168 typedef uint64_t pmap_paddr_t;
169
170 #if DEVELOPMENT || DEBUG
171 #define PMAP_ASSERT 1
172 extern int pmap_asserts_enabled;
173 extern int pmap_asserts_traced;
174 #endif
175
176 #if PMAP_ASSERT
177 #define pmap_assert(ex) (pmap_asserts_enabled ? ((ex) ? (void)0 : Assert(__FILE__, __LINE__, # ex)) : (void)0)
178
179 #define pmap_assert2(ex, fmt, args...) \
180 do { \
181 if (__improbable(pmap_asserts_enabled && !(ex))) { \
182 if (pmap_asserts_traced) { \
183 KERNEL_DEBUG_CONSTANT(0xDEAD1000, __builtin_return_address(0), __LINE__, 0, 0, 0); \
184 kdebug_enable = 0; \
185 } else { \
186 kprintf("Assertion %s failed (%s:%d, caller %p) " fmt , #ex, __FILE__, __LINE__, __builtin_return_address(0), ##args); \
187 panic("Assertion %s failed (%s:%d, caller %p) " fmt , #ex, __FILE__, __LINE__, __builtin_return_address(0), ##args); \
188 } \
189 } \
190 } while(0)
191 #else
192 #define pmap_assert(ex)
193 #define pmap_assert2(ex, fmt, args...)
194 #endif
195
196 /* superpages */
197 #ifdef __x86_64__
198 #define SUPERPAGE_NBASEPAGES 512
199 #else
200 #define SUPERPAGE_NBASEPAGES 1 /* we don't support superpages on i386 */
201 #endif
202
203 /*
204 * Atomic 64-bit store of a page table entry.
205 */
206 static inline void
207 pmap_store_pte(pt_entry_t *entryp, pt_entry_t value)
208 {
209 /*
210 * In the 32-bit kernel a compare-and-exchange loop was
211 * required to provide atomicity. For K64, life is easier:
212 */
213 *entryp = value;
214 }
215
216 /* in 64 bit spaces, the number of each type of page in the page tables */
217 #define NPML4PGS (1ULL * (PAGE_SIZE/(sizeof (pml4_entry_t))))
218 #define NPDPTPGS (NPML4PGS * (PAGE_SIZE/(sizeof (pdpt_entry_t))))
219 #define NPDEPGS (NPDPTPGS * (PAGE_SIZE/(sizeof (pd_entry_t))))
220 #define NPTEPGS (NPDEPGS * (PAGE_SIZE/(sizeof (pt_entry_t))))
221
222 #define KERNEL_PML4_INDEX 511
223 #define KERNEL_KEXTS_INDEX 510 /* Home of KEXTs - the basement */
224 #define KERNEL_PHYSMAP_PML4_INDEX 509 /* virtual to physical map */
225 #define KERNEL_BASE (0ULL - NBPML4)
226 #define KERNEL_BASEMENT (KERNEL_BASE - NBPML4)
227
228 #define VM_WIMG_COPYBACK VM_MEM_COHERENT
229 #define VM_WIMG_COPYBACKLW VM_WIMG_COPYBACK
230 #define VM_WIMG_DEFAULT VM_MEM_COHERENT
231 /* ?? intel ?? */
232 #define VM_WIMG_IO (VM_MEM_COHERENT | \
233 VM_MEM_NOT_CACHEABLE | VM_MEM_GUARDED)
234 #define VM_WIMG_WTHRU (VM_MEM_WRITE_THROUGH | VM_MEM_COHERENT | VM_MEM_GUARDED)
235 /* write combining mode, aka store gather */
236 #define VM_WIMG_WCOMB (VM_MEM_NOT_CACHEABLE | VM_MEM_COHERENT)
237 #define VM_WIMG_INNERWBACK VM_MEM_COHERENT
238 /*
239 * Pte related macros
240 */
241 #define KVADDR(pmi, pdpi, pdi, pti) \
242 ((vm_offset_t) \
243 ((uint64_t) -1 << 47) | \
244 ((uint64_t)(pmi) << PML4SHIFT) | \
245 ((uint64_t)(pdpi) << PDPTSHIFT) | \
246 ((uint64_t)(pdi) << PDESHIFT) | \
247 ((uint64_t)(pti) << PTESHIFT))
248
249 /*
250 * Size of Kernel address space. This is the number of page table pages
251 * (4MB each) to use for the kernel. 256 pages == 1 Gigabyte.
252 * This **MUST** be a multiple of 4 (eg: 252, 256, 260, etc).
253 */
254 #ifndef KVA_PAGES
255 #define KVA_PAGES 1024
256 #endif
257
258 #ifndef NKPT
259 #define NKPT 500 /* actual number of kernel page tables */
260 #endif
261 #ifndef NKPDE
262 #define NKPDE (KVA_PAGES - 1) /* addressable number of page tables/pde's */
263 #endif
264
265
266
267 /*
268 * Convert address offset to page descriptor index
269 */
270 #define pdptnum(pmap, a) (((vm_offset_t)(a) >> PDPTSHIFT) & PDPTMASK)
271 #define pdenum(pmap, a) (((vm_offset_t)(a) >> PDESHIFT) & PDEMASK)
272 #define PMAP_INVALID_PDPTNUM (~0ULL)
273
274 #define pdeidx(pmap, a) (((a) >> PDSHIFT) & ((1ULL<<(48 - PDSHIFT)) -1))
275 #define pdptidx(pmap, a) (((a) >> PDPTSHIFT) & ((1ULL<<(48 - PDPTSHIFT)) -1))
276 #define pml4idx(pmap, a) (((a) >> PML4SHIFT) & ((1ULL<<(48 - PML4SHIFT)) -1))
277
278
279 /*
280 * Convert page descriptor index to user virtual address
281 */
282 #define pdetova(a) ((vm_offset_t)(a) << PDESHIFT)
283
284 /*
285 * Convert address offset to page table index
286 */
287 #define ptenum(a) (((vm_offset_t)(a) >> PTESHIFT) & PTEMASK)
288
289 /*
290 * Hardware pte bit definitions (to be used directly on the ptes
291 * without using the bit fields).
292 */
293
294 #define INTEL_PTE_VALID 0x00000001ULL
295 #define INTEL_PTE_WRITE 0x00000002ULL
296 #define INTEL_PTE_RW 0x00000002ULL
297 #define INTEL_PTE_USER 0x00000004ULL
298 #define INTEL_PTE_WTHRU 0x00000008ULL
299 #define INTEL_PTE_NCACHE 0x00000010ULL
300 #define INTEL_PTE_REF 0x00000020ULL
301 #define INTEL_PTE_MOD 0x00000040ULL
302 #define INTEL_PTE_PS 0x00000080ULL
303 #define INTEL_PTE_PTA 0x00000080ULL
304 #define INTEL_PTE_GLOBAL 0x00000100ULL
305 #define INTEL_PTE_WIRED 0x00000400ULL
306 #define INTEL_PDPTE_NESTED 0x00000800ULL
307 #define INTEL_PTE_PFN PG_FRAME
308
309 #define INTEL_PTE_NX (1ULL << 63)
310
311 #define INTEL_PTE_INVALID 0
312 /* This is conservative, but suffices */
313 #define INTEL_PTE_RSVD ((1ULL << 10) | (1ULL << 11) | (0x1FFULL << 54))
314
315 #define INTEL_PTE_COMPRESSED (1ULL << 62) /* marker, for invalid PTE only -- ignored by hardware for both regular/EPT entries*/
316 #define INTEL_PTE_COMPRESSED_ALT (1ULL << 61) /* compressed but with "alternate accounting" */
317
318 #define INTEL_PTE_COMPRESSED_MASK (INTEL_PTE_COMPRESSED | \
319 INTEL_PTE_COMPRESSED_ALT)
320 #define PTE_IS_COMPRESSED(x) \
321 ((((x) & INTEL_PTE_VALID) == 0) && /* PTE is not valid... */ \
322 ((x) & INTEL_PTE_COMPRESSED) && /* ...has "compressed" marker" */ \
323 ((!((x) & ~INTEL_PTE_COMPRESSED_MASK)) || /* ...no other bits */ \
324 (panic("compressed PTE %p 0x%llx has extra bits 0x%llx: corrupted?", \
325 &(x), (x), (x) & ~INTEL_PTE_COMPRESSED_MASK), FALSE)))
326
327 #define pa_to_pte(a) ((a) & INTEL_PTE_PFN) /* XXX */
328 #define pte_to_pa(p) ((p) & INTEL_PTE_PFN) /* XXX */
329 #define pte_increment_pa(p) ((p) += INTEL_OFFMASK+1)
330
331 #define pte_kernel_rw(p) ((pt_entry_t)(pa_to_pte(p) | INTEL_PTE_VALID|INTEL_PTE_RW))
332 #define pte_kernel_ro(p) ((pt_entry_t)(pa_to_pte(p) | INTEL_PTE_VALID))
333 #define pte_user_rw(p) ((pt_entry_t)(pa_to_pte(p) | INTEL_PTE_VALID|INTEL_PTE_USER|INTEL_PTE_RW))
334 #define pte_user_ro(p) ((pt_entry_t)(pa_to_pte(p) | INTEL_PTE_VALID|INTEL_PTE_USER))
335
336 #define PMAP_INVEPT_SINGLE_CONTEXT 1
337
338
339 #define INTEL_EPTP_AD 0x00000040ULL
340
341 #define INTEL_EPT_READ 0x00000001ULL
342 #define INTEL_EPT_WRITE 0x00000002ULL
343 #define INTEL_EPT_EX 0x00000004ULL
344 #define INTEL_EPT_IPTA 0x00000040ULL
345 #define INTEL_EPT_PS 0x00000080ULL
346 #define INTEL_EPT_REF 0x00000100ULL
347 #define INTEL_EPT_MOD 0x00000200ULL
348
349 #define INTEL_EPT_CACHE_MASK 0x00000038ULL
350 #define INTEL_EPT_NCACHE 0x00000000ULL
351 #define INTEL_EPT_WC 0x00000008ULL
352 #define INTEL_EPT_WTHRU 0x00000020ULL
353 #define INTEL_EPT_WP 0x00000028ULL
354 #define INTEL_EPT_WB 0x00000030ULL
355
356 /*
357 * Routines to filter correct bits depending on the pmap type
358 */
359
360 static inline pt_entry_t
361 pte_remove_ex(pt_entry_t pte, boolean_t is_ept)
362 {
363 if (__probable(!is_ept)) {
364 return (pte | INTEL_PTE_NX);
365 }
366
367 return (pte & (~INTEL_EPT_EX));
368 }
369
370 static inline pt_entry_t
371 pte_set_ex(pt_entry_t pte, boolean_t is_ept)
372 {
373 if (__probable(!is_ept)) {
374 return (pte & (~INTEL_PTE_NX));
375 }
376
377 return (pte | INTEL_EPT_EX);
378 }
379
380 static inline pt_entry_t
381 physmap_refmod_to_ept(pt_entry_t physmap_pte)
382 {
383 pt_entry_t ept_pte = 0;
384
385 if (physmap_pte & INTEL_PTE_MOD) {
386 ept_pte |= INTEL_EPT_MOD;
387 }
388
389 if (physmap_pte & INTEL_PTE_REF) {
390 ept_pte |= INTEL_EPT_REF;
391 }
392
393 return ept_pte;
394 }
395
396 static inline pt_entry_t
397 ept_refmod_to_physmap(pt_entry_t ept_pte)
398 {
399 pt_entry_t physmap_pte = 0;
400
401 assert((ept_pte & ~(INTEL_EPT_REF | INTEL_EPT_MOD)) == 0);
402
403 if (ept_pte & INTEL_EPT_REF) {
404 physmap_pte |= INTEL_PTE_REF;
405 }
406
407 if (ept_pte & INTEL_EPT_MOD) {
408 physmap_pte |= INTEL_PTE_MOD;
409 }
410
411 return physmap_pte;
412 }
413
414 /*
415 * Note: Not all Intel processors support EPT referenced access and dirty bits.
416 * During pmap_init() we check the VMX capability for the current hardware
417 * and update this variable accordingly.
418 */
419 extern boolean_t pmap_ept_support_ad;
420
421 #define PTE_VALID_MASK(is_ept) ((is_ept) ? (INTEL_EPT_READ | INTEL_EPT_WRITE | INTEL_EPT_EX) : INTEL_PTE_VALID)
422 #define PTE_READ(is_ept) ((is_ept) ? INTEL_EPT_READ : INTEL_PTE_VALID)
423 #define PTE_WRITE(is_ept) ((is_ept) ? INTEL_EPT_WRITE : INTEL_PTE_WRITE)
424 #define PTE_PS INTEL_PTE_PS
425 #define PTE_COMPRESSED INTEL_PTE_COMPRESSED
426 #define PTE_COMPRESSED_ALT INTEL_PTE_COMPRESSED_ALT
427 #define PTE_NCACHE(is_ept) ((is_ept) ? INTEL_EPT_NCACHE : INTEL_PTE_NCACHE)
428 #define PTE_WTHRU(is_ept) ((is_ept) ? INTEL_EPT_WTHRU : INTEL_PTE_WTHRU)
429 #define PTE_REF(is_ept) ((is_ept) ? INTEL_EPT_REF : INTEL_PTE_REF)
430 #define PTE_MOD(is_ept) ((is_ept) ? INTEL_EPT_MOD : INTEL_PTE_MOD)
431 #define PTE_WIRED INTEL_PTE_WIRED
432
433
434 #define PMAP_DEFAULT_CACHE 0
435 #define PMAP_INHIBIT_CACHE 1
436 #define PMAP_GUARDED_CACHE 2
437 #define PMAP_ACTIVATE_CACHE 4
438 #define PMAP_NO_GUARD_CACHE 8
439
440 #ifndef ASSEMBLER
441
442 #include <sys/queue.h>
443
444 /*
445 * Address of current and alternate address space page table maps
446 * and directories.
447 */
448
449 extern pt_entry_t *PTmap;
450 extern pdpt_entry_t *IdlePDPT;
451 extern pml4_entry_t *IdlePML4;
452 extern boolean_t no_shared_cr3;
453 extern pd_entry_t *IdlePTD; /* physical addr of "Idle" state PTD */
454
455 extern uint64_t pmap_pv_hashlist_walks;
456 extern uint64_t pmap_pv_hashlist_cnts;
457 extern uint32_t pmap_pv_hashlist_max;
458 extern uint32_t pmap_kernel_text_ps;
459
460
461
462 #ifdef __x86_64__
463 #define ID_MAP_VTOP(x) ((void *)(((uint64_t)(x)) & LOW_4GB_MASK))
464
465 extern uint64_t physmap_base, physmap_max;
466
467 #define NPHYSMAP (MAX(K64_MAXMEM/GB + 4, 4))
468
469 static inline boolean_t physmap_enclosed(addr64_t a) {
470 return (a < (NPHYSMAP * GB));
471 }
472
473 static inline void * PHYSMAP_PTOV_check(void *paddr) {
474 uint64_t pvaddr = (uint64_t)paddr + physmap_base;
475
476 if (__improbable(pvaddr >= physmap_max))
477 panic("PHYSMAP_PTOV bounds exceeded, 0x%qx, 0x%qx, 0x%qx",
478 pvaddr, physmap_base, physmap_max);
479
480 return (void *)pvaddr;
481 }
482
483 #define PHYSMAP_PTOV(x) (PHYSMAP_PTOV_check((void*) (x)))
484
485 /*
486 * For KASLR, we alias the master processor's IDT and GDT at fixed
487 * virtual addresses to defeat SIDT/SGDT address leakage.
488 * And non-boot processor's GDT aliases likewise (skipping LOWGLOBAL_ALIAS)
489 * The low global vector page is mapped at a fixed alias also.
490 */
491 #define MASTER_IDT_ALIAS (VM_MIN_KERNEL_ADDRESS + 0x0000)
492 #define MASTER_GDT_ALIAS (VM_MIN_KERNEL_ADDRESS + 0x1000)
493 #define LOWGLOBAL_ALIAS (VM_MIN_KERNEL_ADDRESS + 0x2000)
494 #define CPU_GDT_ALIAS(_cpu) (LOWGLOBAL_ALIAS + (0x1000*(_cpu)))
495
496 /*
497 * This indicates (roughly) where there is free space for the VM
498 * to use for the heap; this does not need to be precise.
499 */
500 #define KERNEL_PMAP_HEAP_RANGE_START VM_MIN_KERNEL_AND_KEXT_ADDRESS
501
502 #endif /*__x86_64__ */
503
504 #include <vm/vm_page.h>
505
506 /*
507 * For each vm_page_t, there is a list of all currently
508 * valid virtual mappings of that page. An entry is
509 * a pv_entry_t; the list is the pv_table.
510 */
511
512 struct pmap {
513 decl_simple_lock_data(,lock) /* lock on map */
514 pmap_paddr_t pm_cr3; /* physical addr */
515 task_map_t pm_task_map;
516 boolean_t pm_shared;
517 boolean_t pagezero_accessible;
518 #define PMAP_PCID_MAX_CPUS MAX_CPUS /* Must be a multiple of 8 */
519 pcid_t pmap_pcid_cpus[PMAP_PCID_MAX_CPUS];
520 volatile uint8_t pmap_pcid_coherency_vector[PMAP_PCID_MAX_CPUS];
521 struct pmap_statistics stats; /* map statistics */
522 int ref_count; /* reference count */
523 int nx_enabled;
524 pdpt_entry_t *pm_pdpt; /* KVA of 3rd level page */
525 pml4_entry_t *pm_pml4; /* VKA of top level */
526 vm_object_t pm_obj; /* object to hold pde's */
527 vm_object_t pm_obj_pdpt; /* holds pdpt pages */
528 vm_object_t pm_obj_pml4; /* holds pml4 pages */
529 pmap_paddr_t pm_eptp; /* EPTP */
530 pd_entry_t *dirbase; /* page directory pointer */
531 ledger_t ledger; /* ledger tracking phys mappings */
532 #if MACH_ASSERT
533 int pmap_pid;
534 char pmap_procname[17];
535 #endif /* MACH_ASSERT */
536 };
537
538 static inline boolean_t
539 is_ept_pmap(pmap_t p)
540 {
541 if (__probable(p->pm_cr3 != 0)) {
542 assert(p->pm_eptp == 0);
543 return FALSE;
544 }
545
546 assert(p->pm_eptp != 0);
547
548 return TRUE;
549 }
550
551 void hv_ept_pmap_create(void **ept_pmap, void **eptp);
552
553 #if NCOPY_WINDOWS > 0
554 #define PMAP_PDPT_FIRST_WINDOW 0
555 #define PMAP_PDPT_NWINDOWS 4
556 #define PMAP_PDE_FIRST_WINDOW (PMAP_PDPT_NWINDOWS)
557 #define PMAP_PDE_NWINDOWS 4
558 #define PMAP_PTE_FIRST_WINDOW (PMAP_PDE_FIRST_WINDOW + PMAP_PDE_NWINDOWS)
559 #define PMAP_PTE_NWINDOWS 4
560
561 #define PMAP_NWINDOWS_FIRSTFREE (PMAP_PTE_FIRST_WINDOW + PMAP_PTE_NWINDOWS)
562 #define PMAP_WINDOW_SIZE 8
563 #define PMAP_NWINDOWS (PMAP_NWINDOWS_FIRSTFREE + PMAP_WINDOW_SIZE)
564
565 typedef struct {
566 pt_entry_t *prv_CMAP;
567 caddr_t prv_CADDR;
568 } mapwindow_t;
569
570 typedef struct cpu_pmap {
571 int pdpt_window_index;
572 int pde_window_index;
573 int pte_window_index;
574 mapwindow_t mapwindow[PMAP_NWINDOWS];
575 } cpu_pmap_t;
576
577
578 extern mapwindow_t *pmap_get_mapwindow(pt_entry_t pentry);
579 extern void pmap_put_mapwindow(mapwindow_t *map);
580 #endif
581
582 typedef struct pmap_memory_regions {
583 ppnum_t base; /* first page of this region */
584 ppnum_t alloc_up; /* pages below this one have been "stolen" */
585 ppnum_t alloc_down; /* pages above this one have been "stolen" */
586 ppnum_t end; /* last page of this region */
587 uint32_t type;
588 uint64_t attribute;
589 } pmap_memory_region_t;
590
591 extern unsigned pmap_memory_region_count;
592 extern unsigned pmap_memory_region_current;
593
594 #define PMAP_MEMORY_REGIONS_SIZE 128
595
596 extern pmap_memory_region_t pmap_memory_regions[];
597 #include <i386/pmap_pcid.h>
598 #include <vm/vm_map.h>
599
600 static inline void
601 set_dirbase(pmap_t tpmap, thread_t thread, int my_cpu) {
602 int ccpu = my_cpu;
603 cpu_datap(ccpu)->cpu_task_cr3 = tpmap->pm_cr3;
604 cpu_datap(ccpu)->cpu_task_map = tpmap->pm_task_map;
605
606 assert((get_preemption_level() > 0) || (ml_get_interrupts_enabled() == FALSE));
607 assert(ccpu == cpu_number());
608 /*
609 * Switch cr3 if necessary
610 * - unless running with no_shared_cr3 debugging mode
611 * and we're not on the kernel's cr3 (after pre-empted copyio)
612 */
613 boolean_t nopagezero = tpmap->pagezero_accessible;
614 boolean_t priorpagezero = cpu_datap(ccpu)->cpu_pagezero_mapped;
615 cpu_datap(ccpu)->cpu_pagezero_mapped = nopagezero;
616
617 if (__probable(!no_shared_cr3)) {
618 if (__improbable(nopagezero)) {
619 boolean_t copyio_active = ((thread->machine.specFlags & CopyIOActive) != 0);
620 if (pmap_pcid_ncpus) {
621 pmap_pcid_activate(tpmap, ccpu, TRUE, copyio_active);
622 } else {
623 if (copyio_active) {
624 if (get_cr3_base() != tpmap->pm_cr3) {
625 set_cr3_raw(tpmap->pm_cr3);
626 }
627 } else if (get_cr3_base() != cpu_datap(ccpu)->cpu_kernel_cr3) {
628 set_cr3_raw(cpu_datap(ccpu)->cpu_kernel_cr3);
629 }
630 }
631 } else if ((get_cr3_base() != tpmap->pm_cr3) || priorpagezero) {
632 if (pmap_pcid_ncpus) {
633 pmap_pcid_activate(tpmap, ccpu, FALSE, FALSE);
634 } else {
635 set_cr3_raw(tpmap->pm_cr3);
636 }
637 }
638 } else {
639 if (get_cr3_base() != cpu_datap(ccpu)->cpu_kernel_cr3)
640 set_cr3_raw(cpu_datap(ccpu)->cpu_kernel_cr3);
641 }
642 }
643
644 /*
645 * External declarations for PMAP_ACTIVATE.
646 */
647
648 extern void process_pmap_updates(void);
649 extern void pmap_update_interrupt(void);
650
651 /*
652 * Machine dependent routines that are used only for i386/i486/i860.
653 */
654
655 extern addr64_t (kvtophys)(
656 vm_offset_t addr);
657
658 extern kern_return_t pmap_expand(
659 pmap_t pmap,
660 vm_map_offset_t addr,
661 unsigned int options);
662 #if !defined(__x86_64__)
663 extern pt_entry_t *pmap_pte(
664 struct pmap *pmap,
665 vm_map_offset_t addr);
666
667 extern pd_entry_t *pmap_pde(
668 struct pmap *pmap,
669 vm_map_offset_t addr);
670
671 extern pd_entry_t *pmap64_pde(
672 struct pmap *pmap,
673 vm_map_offset_t addr);
674
675 extern pdpt_entry_t *pmap64_pdpt(
676 struct pmap *pmap,
677 vm_map_offset_t addr);
678 #endif
679 extern vm_offset_t pmap_map(
680 vm_offset_t virt,
681 vm_map_offset_t start,
682 vm_map_offset_t end,
683 vm_prot_t prot,
684 unsigned int flags);
685
686 extern vm_offset_t pmap_map_bd(
687 vm_offset_t virt,
688 vm_map_offset_t start,
689 vm_map_offset_t end,
690 vm_prot_t prot,
691 unsigned int flags);
692
693 extern void pmap_bootstrap(
694 vm_offset_t load_start,
695 boolean_t IA32e);
696
697 extern boolean_t pmap_valid_page(
698 ppnum_t pn);
699
700 extern int pmap_list_resident_pages(
701 struct pmap *pmap,
702 vm_offset_t *listp,
703 int space);
704 extern void x86_filter_TLB_coherency_interrupts(boolean_t);
705 /*
706 * Get cache attributes (as pagetable bits) for the specified phys page
707 */
708 extern unsigned pmap_get_cache_attributes(ppnum_t, boolean_t is_ept);
709 #if NCOPY_WINDOWS > 0
710 extern struct cpu_pmap *pmap_cpu_alloc(
711 boolean_t is_boot_cpu);
712 extern void pmap_cpu_free(
713 struct cpu_pmap *cp);
714 #endif
715
716 extern void pmap_map_block(
717 pmap_t pmap,
718 addr64_t va,
719 ppnum_t pa,
720 uint32_t size,
721 vm_prot_t prot,
722 int attr,
723 unsigned int flags);
724
725 extern void invalidate_icache(vm_offset_t addr, unsigned cnt, int phys);
726 extern void flush_dcache(vm_offset_t addr, unsigned count, int phys);
727 extern ppnum_t pmap_find_phys(pmap_t map, addr64_t va);
728
729 extern void pmap_cpu_init(void);
730 extern void pmap_disable_NX(pmap_t pmap);
731
732 extern void pt_fake_zone_init(int);
733 extern void pt_fake_zone_info(int *, vm_size_t *, vm_size_t *, vm_size_t *, vm_size_t *,
734 uint64_t *, int *, int *, int *);
735 extern void pmap_pagetable_corruption_msg_log(int (*)(const char * fmt, ...)__printflike(1,2));
736
737 /*
738 * Macros for speed.
739 */
740
741
742 #include <kern/spl.h>
743
744
745 #define PMAP_ACTIVATE_MAP(map, thread, my_cpu) { \
746 pmap_t tpmap; \
747 \
748 tpmap = vm_map_pmap(map); \
749 set_dirbase(tpmap, thread, my_cpu); \
750 }
751
752 #if defined(__x86_64__)
753 #define PMAP_DEACTIVATE_MAP(map, thread, ccpu) \
754 pmap_assert2((pmap_pcid_ncpus ? (pcid_for_pmap_cpu_tuple(map->pmap, thread, ccpu) == (get_cr3_raw() & 0xFFF)) : TRUE),"PCIDs: 0x%x, active PCID: 0x%x, CR3: 0x%lx, pmap_cr3: 0x%llx, kernel_cr3: 0x%llx, kernel pmap cr3: 0x%llx, CPU active PCID: 0x%x, CPU kernel PCID: 0x%x, specflags: 0x%x, pagezero: 0x%x", pmap_pcid_ncpus, pcid_for_pmap_cpu_tuple(map->pmap, thread, ccpu), get_cr3_raw(), map->pmap->pm_cr3, cpu_datap(ccpu)->cpu_kernel_cr3, kernel_pmap->pm_cr3, cpu_datap(ccpu)->cpu_active_pcid, cpu_datap(ccpu)->cpu_kernel_pcid, thread->machine.specFlags, map->pmap->pagezero_accessible);
755 #else
756 #define PMAP_DEACTIVATE_MAP(map, thread)
757 #endif
758
759 #if NCOPY_WINDOWS > 0
760 #define PMAP_SWITCH_USER(th, new_map, my_cpu) { \
761 spl_t spl; \
762 \
763 spl = splhigh(); \
764 PMAP_DEACTIVATE_MAP(th->map, th); \
765 th->map = new_map; \
766 PMAP_ACTIVATE_MAP(th->map, th); \
767 splx(spl); \
768 inval_copy_windows(th); \
769 }
770 #else
771 #define PMAP_SWITCH_USER(th, new_map, my_cpu) { \
772 spl_t spl; \
773 \
774 spl = splhigh(); \
775 PMAP_DEACTIVATE_MAP(th->map, th, my_cpu); \
776 th->map = new_map; \
777 PMAP_ACTIVATE_MAP(th->map, th, my_cpu); \
778 splx(spl); \
779 }
780 #endif
781
782 /*
783 * Marking the current cpu's cr3 inactive is achieved by setting its lsb.
784 * Marking the current cpu's cr3 active once more involves clearng this bit.
785 * Note that valid page tables are page-aligned and so the bottom 12 bits
786 * are normally zero, modulo PCID.
787 * We can only mark the current cpu active/inactive but we can test any cpu.
788 */
789 #define CPU_CR3_MARK_INACTIVE() \
790 current_cpu_datap()->cpu_active_cr3 |= 1
791
792 #define CPU_CR3_MARK_ACTIVE() \
793 current_cpu_datap()->cpu_active_cr3 &= ~1
794
795 #define CPU_CR3_IS_ACTIVE(cpu) \
796 ((cpu_datap(cpu)->cpu_active_cr3 & 1) == 0)
797
798 #define CPU_GET_ACTIVE_CR3(cpu) \
799 (cpu_datap(cpu)->cpu_active_cr3 & ~1)
800
801 #define CPU_GET_TASK_CR3(cpu) \
802 (cpu_datap(cpu)->cpu_task_cr3)
803
804 /*
805 * Mark this cpu idle, and remove it from the active set,
806 * since it is not actively using any pmap. Signal_cpus
807 * will notice that it is idle, and avoid signaling it,
808 * but will queue the update request for when the cpu
809 * becomes active.
810 */
811 #define MARK_CPU_IDLE(my_cpu) { \
812 assert(ml_get_interrupts_enabled() == FALSE); \
813 CPU_CR3_MARK_INACTIVE(); \
814 mfence(); \
815 }
816
817 #define MARK_CPU_ACTIVE(my_cpu) { \
818 assert(ml_get_interrupts_enabled() == FALSE); \
819 /* \
820 * If a kernel_pmap update was requested while this cpu \
821 * was idle, process it as if we got the interrupt. \
822 * Before doing so, remove this cpu from the idle set. \
823 * Since we do not grab any pmap locks while we flush \
824 * our TLB, another cpu may start an update operation \
825 * before we finish. Removing this cpu from the idle \
826 * set assures that we will receive another update \
827 * interrupt if this happens. \
828 */ \
829 CPU_CR3_MARK_ACTIVE(); \
830 mfence(); \
831 \
832 if (current_cpu_datap()->cpu_tlb_invalid) \
833 process_pmap_updates(); \
834 }
835
836 #define PMAP_CONTEXT(pmap, thread)
837
838 #define pmap_kernel_va(VA) \
839 ((((vm_offset_t) (VA)) >= vm_min_kernel_address) && \
840 (((vm_offset_t) (VA)) <= vm_max_kernel_address))
841
842
843 #define pmap_compressed(pmap) ((pmap)->stats.compressed)
844 #define pmap_resident_count(pmap) ((pmap)->stats.resident_count)
845 #define pmap_resident_max(pmap) ((pmap)->stats.resident_max)
846 #define pmap_copy(dst_pmap,src_pmap,dst_addr,len,src_addr)
847 #define pmap_attribute(pmap,addr,size,attr,value) \
848 (KERN_INVALID_ADDRESS)
849 #define pmap_attribute_cache_sync(addr,size,attr,value) \
850 (KERN_INVALID_ADDRESS)
851
852 #define MACHINE_PMAP_IS_EMPTY 1
853 extern boolean_t pmap_is_empty(pmap_t pmap,
854 vm_map_offset_t start,
855 vm_map_offset_t end);
856
857 #define MACHINE_BOOTSTRAPPTD 1 /* Static bootstrap page-tables */
858
859 kern_return_t
860 pmap_permissions_verify(pmap_t, vm_map_t, vm_offset_t, vm_offset_t);
861
862 #if MACH_ASSERT
863 extern int pmap_stats_assert;
864 #define PMAP_STATS_ASSERTF(args) \
865 MACRO_BEGIN \
866 if (pmap_stats_assert) assertf args; \
867 MACRO_END
868 #else /* MACH_ASSERT */
869 #define PMAP_STATS_ASSERTF(args)
870 #endif /* MACH_ASSERT */
871
872 #endif /* ASSEMBLER */
873
874
875 #endif /* _PMAP_MACHINE_ */
876
877
878 #endif /* KERNEL_PRIVATE */
mirror of XNU