]> git.saurik.com Git - apple/xnu.git/blob - osfmk/i386/pmap.h
projects / apple / xnu.git / blob
? search:


939e47174605d98a377a3a33963f02104a7b63e4
[apple/xnu.git] / osfmk / i386 / pmap.h
1 /*
2 * Copyright (c) 2000-2012 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56 /*
57 */
58
59 /*
60 * File: pmap.h
61 *
62 * Authors: Avadis Tevanian, Jr., Michael Wayne Young
63 * Date: 1985
64 *
65 * Machine-dependent structures for the physical map module.
66 */
67 #ifdef KERNEL_PRIVATE
68 #ifndef _PMAP_MACHINE_
69 #define _PMAP_MACHINE_ 1
70
71 #ifndef ASSEMBLER
72
73
74 #include <mach/kern_return.h>
75 #include <mach/machine/vm_types.h>
76 #include <mach/vm_prot.h>
77 #include <mach/vm_statistics.h>
78 #include <mach/machine/vm_param.h>
79 #include <kern/kern_types.h>
80 #include <kern/thread.h>
81 #include <kern/simple_lock.h>
82 #include <mach/branch_predicates.h>
83
84 #include <i386/mp.h>
85 #include <i386/proc_reg.h>
86
87 #include <i386/pal_routines.h>
88
89 /*
90 * Define the generic in terms of the specific
91 */
92
93 #define INTEL_PGBYTES I386_PGBYTES
94 #define INTEL_PGSHIFT I386_PGSHIFT
95 #define intel_btop(x) i386_btop(x)
96 #define intel_ptob(x) i386_ptob(x)
97 #define intel_round_page(x) i386_round_page(x)
98 #define intel_trunc_page(x) i386_trunc_page(x)
99 #define trunc_intel_to_vm(x) trunc_i386_to_vm(x)
100 #define round_intel_to_vm(x) round_i386_to_vm(x)
101 #define vm_to_intel(x) vm_to_i386(x)
102
103 /*
104 * i386/i486/i860 Page Table Entry
105 */
106
107 #endif /* ASSEMBLER */
108
109 #define NPGPTD 4ULL
110 #define PDESHIFT 21ULL
111 #define PTEMASK 0x1ffULL
112 #define PTEINDX 3ULL
113
114 #define PTESHIFT 12ULL
115
116
117 #ifdef __x86_64__
118 #define LOW_4GB_MASK ((vm_offset_t)0x00000000FFFFFFFFUL)
119 #endif
120
121 #define PDESIZE sizeof(pd_entry_t) /* for assembly files */
122 #define PTESIZE sizeof(pt_entry_t) /* for assembly files */
123
124 #define INTEL_OFFMASK (I386_PGBYTES - 1)
125 #define INTEL_LOFFMASK (I386_LPGBYTES - 1)
126 #define PG_FRAME 0x000FFFFFFFFFF000ULL
127 #define NPTEPG (PAGE_SIZE/(sizeof (pt_entry_t)))
128 #define NPTDPG (PAGE_SIZE/(sizeof (pd_entry_t)))
129
130 #define NBPTD (NPGPTD << PAGE_SHIFT)
131 #define NPDEPTD (NBPTD / (sizeof (pd_entry_t)))
132 #define NPDEPG (PAGE_SIZE/(sizeof (pd_entry_t)))
133 #define NBPDE (1ULL << PDESHIFT)
134 #define PDEMASK (NBPDE - 1)
135
136 #define PTE_PER_PAGE 512 /* number of PTE's per page on any level */
137
138 /* cleanly define parameters for all the page table levels */
139 typedef uint64_t pml4_entry_t;
140 #define NPML4PG (PAGE_SIZE/(sizeof (pml4_entry_t)))
141 #define PML4SHIFT 39
142 #define PML4PGSHIFT 9
143 #define NBPML4 (1ULL << PML4SHIFT)
144 #define PML4MASK (NBPML4-1)
145 #define PML4_ENTRY_NULL ((pml4_entry_t *) 0)
146
147 typedef uint64_t pdpt_entry_t;
148 #define NPDPTPG (PAGE_SIZE/(sizeof (pdpt_entry_t)))
149 #define PDPTSHIFT 30
150 #define PDPTPGSHIFT 9
151 #define NBPDPT (1ULL << PDPTSHIFT)
152 #define PDPTMASK (NBPDPT-1)
153 #define PDPT_ENTRY_NULL ((pdpt_entry_t *) 0)
154
155 typedef uint64_t pd_entry_t;
156 #define NPDPG (PAGE_SIZE/(sizeof (pd_entry_t)))
157 #define PDSHIFT 21
158 #define PDPGSHIFT 9
159 #define NBPD (1ULL << PDSHIFT)
160 #define PDMASK (NBPD-1)
161 #define PD_ENTRY_NULL ((pd_entry_t *) 0)
162
163 typedef uint64_t pt_entry_t;
164 #define NPTPG (PAGE_SIZE/(sizeof (pt_entry_t)))
165 #define PTSHIFT 12
166 #define PTPGSHIFT 9
167 #define NBPT (1ULL << PTSHIFT)
168 #define PTMASK (NBPT-1)
169 #define PT_ENTRY_NULL ((pt_entry_t *) 0)
170
171 typedef uint64_t pmap_paddr_t;
172
173 #if DEBUG
174 #define PMAP_ASSERT 1
175 #endif
176 #if PMAP_ASSERT
177 #define pmap_assert(ex) ((ex) ? (void)0 : Assert(__FILE__, __LINE__, # ex))
178
179 #define pmap_assert2(ex, fmt, args...) \
180 do { \
181 if (!(ex)) { \
182 kprintf("Assertion %s failed (%s:%d, caller %p) " fmt , #ex, __FILE__, __LINE__, __builtin_return_address(0), ##args); \
183 panic("Assertion %s failed (%s:%d, caller %p) " fmt , #ex, __FILE__, __LINE__, __builtin_return_address(0), ##args); \
184 } \
185 } while(0)
186 #else
187 #define pmap_assert(ex)
188 #define pmap_assert2(ex, fmt, args...)
189 #endif
190
191 /* superpages */
192 #ifdef __x86_64__
193 #define SUPERPAGE_NBASEPAGES 512
194 #else
195 #define SUPERPAGE_NBASEPAGES 1 /* we don't support superpages on i386 */
196 #endif
197
198 /*
199 * Atomic 64-bit store of a page table entry.
200 */
201 static inline void
202 pmap_store_pte(pt_entry_t *entryp, pt_entry_t value)
203 {
204 /*
205 * In the 32-bit kernel a compare-and-exchange loop was
206 * required to provide atomicity. For K64, life is easier:
207 */
208 *entryp = value;
209 }
210
211 /* in 64 bit spaces, the number of each type of page in the page tables */
212 #define NPML4PGS (1ULL * (PAGE_SIZE/(sizeof (pml4_entry_t))))
213 #define NPDPTPGS (NPML4PGS * (PAGE_SIZE/(sizeof (pdpt_entry_t))))
214 #define NPDEPGS (NPDPTPGS * (PAGE_SIZE/(sizeof (pd_entry_t))))
215 #define NPTEPGS (NPDEPGS * (PAGE_SIZE/(sizeof (pt_entry_t))))
216
217 #define KERNEL_PML4_INDEX 511
218 #define KERNEL_KEXTS_INDEX 510 /* Home of KEXTs - the basement */
219 #define KERNEL_PHYSMAP_PML4_INDEX 509 /* virtual to physical map */
220 #define KERNEL_BASE (0ULL - NBPML4)
221 #define KERNEL_BASEMENT (KERNEL_BASE - NBPML4)
222
223 #define VM_WIMG_COPYBACK VM_MEM_COHERENT
224 #define VM_WIMG_COPYBACKLW VM_WIMG_COPYBACK
225 #define VM_WIMG_DEFAULT VM_MEM_COHERENT
226 /* ?? intel ?? */
227 #define VM_WIMG_IO (VM_MEM_COHERENT | \
228 VM_MEM_NOT_CACHEABLE | VM_MEM_GUARDED)
229 #define VM_WIMG_WTHRU (VM_MEM_WRITE_THROUGH | VM_MEM_COHERENT | VM_MEM_GUARDED)
230 /* write combining mode, aka store gather */
231 #define VM_WIMG_WCOMB (VM_MEM_NOT_CACHEABLE | VM_MEM_COHERENT)
232 #define VM_WIMG_INNERWBACK VM_MEM_COHERENT
233 /*
234 * Pte related macros
235 */
236 #define KVADDR(pmi, pdpi, pdi, pti) \
237 ((vm_offset_t) \
238 ((uint64_t) -1 << 47) | \
239 ((uint64_t)(pmi) << PML4SHIFT) | \
240 ((uint64_t)(pdpi) << PDPTSHIFT) | \
241 ((uint64_t)(pdi) << PDESHIFT) | \
242 ((uint64_t)(pti) << PTESHIFT))
243
244 /*
245 * Size of Kernel address space. This is the number of page table pages
246 * (4MB each) to use for the kernel. 256 pages == 1 Gigabyte.
247 * This **MUST** be a multiple of 4 (eg: 252, 256, 260, etc).
248 */
249 #ifndef KVA_PAGES
250 #define KVA_PAGES 1024
251 #endif
252
253 #ifndef NKPT
254 #define NKPT 500 /* actual number of kernel page tables */
255 #endif
256 #ifndef NKPDE
257 #define NKPDE (KVA_PAGES - 1) /* addressable number of page tables/pde's */
258 #endif
259
260
261
262 /*
263 * Convert address offset to page descriptor index
264 */
265 #define pdptnum(pmap, a) (((vm_offset_t)(a) >> PDPTSHIFT) & PDPTMASK)
266 #define pdenum(pmap, a) (((vm_offset_t)(a) >> PDESHIFT) & PDEMASK)
267 #define PMAP_INVALID_PDPTNUM (~0ULL)
268
269 #define pdeidx(pmap, a) (((a) >> PDSHIFT) & ((1ULL<<(48 - PDSHIFT)) -1))
270 #define pdptidx(pmap, a) (((a) >> PDPTSHIFT) & ((1ULL<<(48 - PDPTSHIFT)) -1))
271 #define pml4idx(pmap, a) (((a) >> PML4SHIFT) & ((1ULL<<(48 - PML4SHIFT)) -1))
272
273
274 /*
275 * Convert page descriptor index to user virtual address
276 */
277 #define pdetova(a) ((vm_offset_t)(a) << PDESHIFT)
278
279 /*
280 * Convert address offset to page table index
281 */
282 #define ptenum(a) (((vm_offset_t)(a) >> PTESHIFT) & PTEMASK)
283
284 /*
285 * Hardware pte bit definitions (to be used directly on the ptes
286 * without using the bit fields).
287 */
288
289 #define INTEL_PTE_VALID 0x00000001ULL
290 #define INTEL_PTE_WRITE 0x00000002ULL
291 #define INTEL_PTE_RW 0x00000002ULL
292 #define INTEL_PTE_USER 0x00000004ULL
293 #define INTEL_PTE_WTHRU 0x00000008ULL
294 #define INTEL_PTE_NCACHE 0x00000010ULL
295 #define INTEL_PTE_REF 0x00000020ULL
296 #define INTEL_PTE_MOD 0x00000040ULL
297 #define INTEL_PTE_PS 0x00000080ULL
298 #define INTEL_PTE_PTA 0x00000080ULL
299 #define INTEL_PTE_GLOBAL 0x00000100ULL
300 #define INTEL_PTE_WIRED 0x00000400ULL
301 #define INTEL_PDPTE_NESTED 0x00000800ULL
302 #define INTEL_PTE_PFN PG_FRAME
303
304 #define INTEL_PTE_NX (1ULL << 63)
305
306 #define INTEL_PTE_INVALID 0
307 /* This is conservative, but suffices */
308 #define INTEL_PTE_RSVD ((1ULL << 10) | (1ULL << 11) | (0x1FFULL << 54))
309
310 #define INTEL_COMPRESSED (1ULL << 62) /* marker, for invalid PTE only -- ignored by hardware for both regular/EPT entries*/
311
312 #define pa_to_pte(a) ((a) & INTEL_PTE_PFN) /* XXX */
313 #define pte_to_pa(p) ((p) & INTEL_PTE_PFN) /* XXX */
314 #define pte_increment_pa(p) ((p) += INTEL_OFFMASK+1)
315
316 #define pte_kernel_rw(p) ((pt_entry_t)(pa_to_pte(p) | INTEL_PTE_VALID|INTEL_PTE_RW))
317 #define pte_kernel_ro(p) ((pt_entry_t)(pa_to_pte(p) | INTEL_PTE_VALID))
318 #define pte_user_rw(p) ((pt_entry_t)(pa_to_pte(p) | INTEL_PTE_VALID|INTEL_PTE_USER|INTEL_PTE_RW))
319 #define pte_user_ro(p) ((pt_entry_t)(pa_to_pte(p) | INTEL_PTE_VALID|INTEL_PTE_USER))
320
321 #define PMAP_INVEPT_SINGLE_CONTEXT 1
322
323
324 #define INTEL_EPTP_AD 0x00000040ULL
325
326 #define INTEL_EPT_READ 0x00000001ULL
327 #define INTEL_EPT_WRITE 0x00000002ULL
328 #define INTEL_EPT_EX 0x00000004ULL
329 #define INTEL_EPT_IPTA 0x00000040ULL
330 #define INTEL_EPT_PS 0x00000080ULL
331 #define INTEL_EPT_REF 0x00000100ULL
332 #define INTEL_EPT_MOD 0x00000200ULL
333
334 #define INTEL_EPT_CACHE_MASK 0x00000038ULL
335 #define INTEL_EPT_NCACHE 0x00000000ULL
336 #define INTEL_EPT_WC 0x00000008ULL
337 #define INTEL_EPT_WTHRU 0x00000020ULL
338 #define INTEL_EPT_WP 0x00000028ULL
339 #define INTEL_EPT_WB 0x00000030ULL
340
341 /*
342 * Routines to filter correct bits depending on the pmap type
343 */
344
345 static inline pt_entry_t
346 pte_remove_ex(pt_entry_t pte, boolean_t is_ept)
347 {
348 if (__probable(!is_ept)) {
349 return (pte | INTEL_PTE_NX);
350 }
351
352 return (pte & (~INTEL_EPT_EX));
353 }
354
355 static inline pt_entry_t
356 pte_set_ex(pt_entry_t pte, boolean_t is_ept)
357 {
358 if (__probable(!is_ept)) {
359 return (pte & (~INTEL_PTE_NX));
360 }
361
362 return (pte | INTEL_EPT_EX);
363 }
364
365 static inline pt_entry_t
366 physmap_refmod_to_ept(pt_entry_t physmap_pte)
367 {
368 pt_entry_t ept_pte = 0;
369
370 if (physmap_pte & INTEL_PTE_MOD) {
371 ept_pte |= INTEL_EPT_MOD;
372 }
373
374 if (physmap_pte & INTEL_PTE_REF) {
375 ept_pte |= INTEL_EPT_REF;
376 }
377
378 return ept_pte;
379 }
380
381 static inline pt_entry_t
382 ept_refmod_to_physmap(pt_entry_t ept_pte)
383 {
384 pt_entry_t physmap_pte = 0;
385
386 assert((ept_pte & ~(INTEL_EPT_REF | INTEL_EPT_MOD)) == 0);
387
388 if (ept_pte & INTEL_EPT_REF) {
389 physmap_pte |= INTEL_PTE_REF;
390 }
391
392 if (ept_pte & INTEL_EPT_MOD) {
393 physmap_pte |= INTEL_PTE_MOD;
394 }
395
396 return physmap_pte;
397 }
398
399 /*
400 * Note: Not all Intel processors support EPT referenced access and dirty bits.
401 * During pmap_init() we check the VMX capability for the current hardware
402 * and update this variable accordingly.
403 */
404 extern boolean_t pmap_ept_support_ad;
405
406 #define PTE_VALID_MASK(is_ept) ((is_ept) ? (INTEL_EPT_READ | INTEL_EPT_WRITE | INTEL_EPT_EX) : INTEL_PTE_VALID)
407 #define PTE_READ(is_ept) ((is_ept) ? INTEL_EPT_READ : INTEL_PTE_VALID)
408 #define PTE_WRITE(is_ept) ((is_ept) ? INTEL_EPT_WRITE : INTEL_PTE_WRITE)
409 #define PTE_PS INTEL_PTE_PS
410 #define PTE_COMPRESSED INTEL_COMPRESSED
411 #define PTE_NCACHE(is_ept) ((is_ept) ? INTEL_EPT_NCACHE : INTEL_PTE_NCACHE)
412 #define PTE_WTHRU(is_ept) ((is_ept) ? INTEL_EPT_WTHRU : INTEL_PTE_WTHRU)
413 #define PTE_REF(is_ept) ((is_ept) ? INTEL_EPT_REF : INTEL_PTE_REF)
414 #define PTE_MOD(is_ept) ((is_ept) ? INTEL_EPT_MOD : INTEL_PTE_MOD)
415 #define PTE_WIRED INTEL_PTE_WIRED
416
417
418 #define PMAP_DEFAULT_CACHE 0
419 #define PMAP_INHIBIT_CACHE 1
420 #define PMAP_GUARDED_CACHE 2
421 #define PMAP_ACTIVATE_CACHE 4
422 #define PMAP_NO_GUARD_CACHE 8
423
424 #ifndef ASSEMBLER
425
426 #include <sys/queue.h>
427
428 /*
429 * Address of current and alternate address space page table maps
430 * and directories.
431 */
432
433 extern pt_entry_t *PTmap;
434 extern pdpt_entry_t *IdlePDPT;
435 extern pml4_entry_t *IdlePML4;
436 extern boolean_t no_shared_cr3;
437 extern addr64_t kernel64_cr3;
438 extern pd_entry_t *IdlePTD; /* physical addr of "Idle" state PTD */
439
440 extern uint64_t pmap_pv_hashlist_walks;
441 extern uint64_t pmap_pv_hashlist_cnts;
442 extern uint32_t pmap_pv_hashlist_max;
443 extern uint32_t pmap_kernel_text_ps;
444
445
446
447 #ifdef __x86_64__
448 #define ID_MAP_VTOP(x) ((void *)(((uint64_t)(x)) & LOW_4GB_MASK))
449
450 extern uint64_t physmap_base, physmap_max;
451
452 #define NPHYSMAP (MAX(K64_MAXMEM/GB + 4, 4))
453
454 static inline boolean_t physmap_enclosed(addr64_t a) {
455 return (a < (NPHYSMAP * GB));
456 }
457
458 static inline void * PHYSMAP_PTOV_check(void *paddr) {
459 uint64_t pvaddr = (uint64_t)paddr + physmap_base;
460
461 if (__improbable(pvaddr >= physmap_max))
462 panic("PHYSMAP_PTOV bounds exceeded, 0x%qx, 0x%qx, 0x%qx",
463 pvaddr, physmap_base, physmap_max);
464
465 return (void *)pvaddr;
466 }
467
468 #define PHYSMAP_PTOV(x) (PHYSMAP_PTOV_check((void*) (x)))
469
470 /*
471 * For KASLR, we alias the master processor's IDT and GDT at fixed
472 * virtual addresses to defeat SIDT/SGDT address leakage.
473 * And non-boot processor's GDT aliases likewise (skipping LOWGLOBAL_ALIAS)
474 * The low global vector page is mapped at a fixed alias also.
475 */
476 #define MASTER_IDT_ALIAS (VM_MIN_KERNEL_ADDRESS + 0x0000)
477 #define MASTER_GDT_ALIAS (VM_MIN_KERNEL_ADDRESS + 0x1000)
478 #define LOWGLOBAL_ALIAS (VM_MIN_KERNEL_ADDRESS + 0x2000)
479 #define CPU_GDT_ALIAS(_cpu) (LOWGLOBAL_ALIAS + (0x1000*(_cpu)))
480
481 #endif /*__x86_64__ */
482
483 #include <vm/vm_page.h>
484
485 /*
486 * For each vm_page_t, there is a list of all currently
487 * valid virtual mappings of that page. An entry is
488 * a pv_entry_t; the list is the pv_table.
489 */
490
491 struct pmap {
492 decl_simple_lock_data(,lock) /* lock on map */
493 pmap_paddr_t pm_cr3; /* physical addr */
494 pmap_paddr_t pm_eptp; /* EPTP */
495 boolean_t pm_shared;
496 pd_entry_t *dirbase; /* page directory pointer */
497 vm_object_t pm_obj; /* object to hold pde's */
498 task_map_t pm_task_map;
499 pdpt_entry_t *pm_pdpt; /* KVA of 3rd level page */
500 pml4_entry_t *pm_pml4; /* VKA of top level */
501 vm_object_t pm_obj_pdpt; /* holds pdpt pages */
502 vm_object_t pm_obj_pml4; /* holds pml4 pages */
503 #define PMAP_PCID_MAX_CPUS MAX_CPUS /* Must be a multiple of 8 */
504 pcid_t pmap_pcid_cpus[PMAP_PCID_MAX_CPUS];
505 volatile uint8_t pmap_pcid_coherency_vector[PMAP_PCID_MAX_CPUS];
506 struct pmap_statistics stats; /* map statistics */
507 int ref_count; /* reference count */
508 int nx_enabled;
509 ledger_t ledger; /* ledger tracking phys mappings */
510 };
511
512 static inline boolean_t
513 is_ept_pmap(pmap_t p)
514 {
515 if (__probable(p->pm_cr3 != 0)) {
516 assert(p->pm_eptp == 0);
517 return FALSE;
518 }
519
520 assert(p->pm_eptp != 0);
521
522 return TRUE;
523 }
524
525 void hv_ept_pmap_create(void **ept_pmap, void **eptp);
526
527 #if NCOPY_WINDOWS > 0
528 #define PMAP_PDPT_FIRST_WINDOW 0
529 #define PMAP_PDPT_NWINDOWS 4
530 #define PMAP_PDE_FIRST_WINDOW (PMAP_PDPT_NWINDOWS)
531 #define PMAP_PDE_NWINDOWS 4
532 #define PMAP_PTE_FIRST_WINDOW (PMAP_PDE_FIRST_WINDOW + PMAP_PDE_NWINDOWS)
533 #define PMAP_PTE_NWINDOWS 4
534
535 #define PMAP_NWINDOWS_FIRSTFREE (PMAP_PTE_FIRST_WINDOW + PMAP_PTE_NWINDOWS)
536 #define PMAP_WINDOW_SIZE 8
537 #define PMAP_NWINDOWS (PMAP_NWINDOWS_FIRSTFREE + PMAP_WINDOW_SIZE)
538
539 typedef struct {
540 pt_entry_t *prv_CMAP;
541 caddr_t prv_CADDR;
542 } mapwindow_t;
543
544 typedef struct cpu_pmap {
545 int pdpt_window_index;
546 int pde_window_index;
547 int pte_window_index;
548 mapwindow_t mapwindow[PMAP_NWINDOWS];
549 } cpu_pmap_t;
550
551
552 extern mapwindow_t *pmap_get_mapwindow(pt_entry_t pentry);
553 extern void pmap_put_mapwindow(mapwindow_t *map);
554 #endif
555
556 typedef struct pmap_memory_regions {
557 ppnum_t base; /* first page of this region */
558 ppnum_t alloc_up; /* pages below this one have been "stolen" */
559 ppnum_t alloc_down; /* pages above this one have been "stolen" */
560 ppnum_t end; /* last page of this region */
561 uint32_t type;
562 uint64_t attribute;
563 } pmap_memory_region_t;
564
565 extern unsigned pmap_memory_region_count;
566 extern unsigned pmap_memory_region_current;
567
568 #define PMAP_MEMORY_REGIONS_SIZE 128
569
570 extern pmap_memory_region_t pmap_memory_regions[];
571 #include <i386/pmap_pcid.h>
572
573 static inline void
574 set_dirbase(pmap_t tpmap, __unused thread_t thread, int my_cpu) {
575 int ccpu = my_cpu;
576 cpu_datap(ccpu)->cpu_task_cr3 = tpmap->pm_cr3;
577 cpu_datap(ccpu)->cpu_task_map = tpmap->pm_task_map;
578 /*
579 * Switch cr3 if necessary
580 * - unless running with no_shared_cr3 debugging mode
581 * and we're not on the kernel's cr3 (after pre-empted copyio)
582 */
583 if (__probable(!no_shared_cr3)) {
584 if (get_cr3_base() != tpmap->pm_cr3) {
585 if (pmap_pcid_ncpus) {
586 pmap_pcid_activate(tpmap, ccpu);
587 }
588 else
589 set_cr3_raw(tpmap->pm_cr3);
590 }
591 } else {
592 if (get_cr3_base() != cpu_datap(ccpu)->cpu_kernel_cr3)
593 set_cr3_raw(cpu_datap(ccpu)->cpu_kernel_cr3);
594 }
595 }
596
597 /*
598 * External declarations for PMAP_ACTIVATE.
599 */
600
601 extern void process_pmap_updates(void);
602 extern void pmap_update_interrupt(void);
603
604 /*
605 * Machine dependent routines that are used only for i386/i486/i860.
606 */
607
608 extern addr64_t (kvtophys)(
609 vm_offset_t addr);
610
611 extern kern_return_t pmap_expand(
612 pmap_t pmap,
613 vm_map_offset_t addr,
614 unsigned int options);
615 #if !defined(__x86_64__)
616 extern pt_entry_t *pmap_pte(
617 struct pmap *pmap,
618 vm_map_offset_t addr);
619
620 extern pd_entry_t *pmap_pde(
621 struct pmap *pmap,
622 vm_map_offset_t addr);
623
624 extern pd_entry_t *pmap64_pde(
625 struct pmap *pmap,
626 vm_map_offset_t addr);
627
628 extern pdpt_entry_t *pmap64_pdpt(
629 struct pmap *pmap,
630 vm_map_offset_t addr);
631 #endif
632 extern vm_offset_t pmap_map(
633 vm_offset_t virt,
634 vm_map_offset_t start,
635 vm_map_offset_t end,
636 vm_prot_t prot,
637 unsigned int flags);
638
639 extern vm_offset_t pmap_map_bd(
640 vm_offset_t virt,
641 vm_map_offset_t start,
642 vm_map_offset_t end,
643 vm_prot_t prot,
644 unsigned int flags);
645
646 extern void pmap_bootstrap(
647 vm_offset_t load_start,
648 boolean_t IA32e);
649
650 extern boolean_t pmap_valid_page(
651 ppnum_t pn);
652
653 extern int pmap_list_resident_pages(
654 struct pmap *pmap,
655 vm_offset_t *listp,
656 int space);
657 extern void x86_filter_TLB_coherency_interrupts(boolean_t);
658 /*
659 * Get cache attributes (as pagetable bits) for the specified phys page
660 */
661 extern unsigned pmap_get_cache_attributes(ppnum_t, boolean_t is_ept);
662 #if NCOPY_WINDOWS > 0
663 extern struct cpu_pmap *pmap_cpu_alloc(
664 boolean_t is_boot_cpu);
665 extern void pmap_cpu_free(
666 struct cpu_pmap *cp);
667 #endif
668
669 extern void pmap_map_block(
670 pmap_t pmap,
671 addr64_t va,
672 ppnum_t pa,
673 uint32_t size,
674 vm_prot_t prot,
675 int attr,
676 unsigned int flags);
677
678 extern void invalidate_icache(vm_offset_t addr, unsigned cnt, int phys);
679 extern void flush_dcache(vm_offset_t addr, unsigned count, int phys);
680 extern ppnum_t pmap_find_phys(pmap_t map, addr64_t va);
681
682 extern void pmap_cpu_init(void);
683 extern void pmap_disable_NX(pmap_t pmap);
684
685 extern void pt_fake_zone_init(int);
686 extern void pt_fake_zone_info(int *, vm_size_t *, vm_size_t *, vm_size_t *, vm_size_t *,
687 uint64_t *, int *, int *, int *);
688 extern void pmap_pagetable_corruption_msg_log(int (*)(const char * fmt, ...)__printflike(1,2));
689
690 /*
691 * Macros for speed.
692 */
693
694
695 #include <kern/spl.h>
696
697
698 #define PMAP_ACTIVATE_MAP(map, thread, my_cpu) { \
699 register pmap_t tpmap; \
700 \
701 tpmap = vm_map_pmap(map); \
702 set_dirbase(tpmap, thread, my_cpu); \
703 }
704
705 #if defined(__x86_64__)
706 #define PMAP_DEACTIVATE_MAP(map, thread, ccpu) \
707 pmap_assert(pmap_pcid_ncpus ? (pcid_for_pmap_cpu_tuple(map->pmap, ccpu) == (get_cr3_raw() & 0xFFF)) : TRUE);
708 #else
709 #define PMAP_DEACTIVATE_MAP(map, thread)
710 #endif
711
712 #define PMAP_SWITCH_CONTEXT(old_th, new_th, my_cpu) { \
713 \
714 pmap_assert(ml_get_interrupts_enabled() == FALSE); \
715 if (old_th->map != new_th->map) { \
716 PMAP_DEACTIVATE_MAP(old_th->map, old_th, my_cpu); \
717 PMAP_ACTIVATE_MAP(new_th->map, new_th, my_cpu); \
718 } \
719 }
720
721 #if NCOPY_WINDOWS > 0
722 #define PMAP_SWITCH_USER(th, new_map, my_cpu) { \
723 spl_t spl; \
724 \
725 spl = splhigh(); \
726 PMAP_DEACTIVATE_MAP(th->map, th); \
727 th->map = new_map; \
728 PMAP_ACTIVATE_MAP(th->map, th); \
729 splx(spl); \
730 inval_copy_windows(th); \
731 }
732 #else
733 #define PMAP_SWITCH_USER(th, new_map, my_cpu) { \
734 spl_t spl; \
735 \
736 spl = splhigh(); \
737 PMAP_DEACTIVATE_MAP(th->map, th, my_cpu); \
738 th->map = new_map; \
739 PMAP_ACTIVATE_MAP(th->map, th, my_cpu); \
740 splx(spl); \
741 }
742 #endif
743
744 /*
745 * Marking the current cpu's cr3 inactive is achieved by setting its lsb.
746 * Marking the current cpu's cr3 active once more involves clearng this bit.
747 * Note that valid page tables are page-aligned and so the bottom 12 bits
748 * are normally zero, modulo PCID.
749 * We can only mark the current cpu active/inactive but we can test any cpu.
750 */
751 #define CPU_CR3_MARK_INACTIVE() \
752 current_cpu_datap()->cpu_active_cr3 |= 1
753
754 #define CPU_CR3_MARK_ACTIVE() \
755 current_cpu_datap()->cpu_active_cr3 &= ~1
756
757 #define CPU_CR3_IS_ACTIVE(cpu) \
758 ((cpu_datap(cpu)->cpu_active_cr3 & 1) == 0)
759
760 #define CPU_GET_ACTIVE_CR3(cpu) \
761 (cpu_datap(cpu)->cpu_active_cr3 & ~1)
762
763 #define CPU_GET_TASK_CR3(cpu) \
764 (cpu_datap(cpu)->cpu_task_cr3)
765
766 /*
767 * Mark this cpu idle, and remove it from the active set,
768 * since it is not actively using any pmap. Signal_cpus
769 * will notice that it is idle, and avoid signaling it,
770 * but will queue the update request for when the cpu
771 * becomes active.
772 */
773 #define MARK_CPU_IDLE(my_cpu) { \
774 assert(ml_get_interrupts_enabled() == FALSE); \
775 CPU_CR3_MARK_INACTIVE(); \
776 mfence(); \
777 }
778
779 #define MARK_CPU_ACTIVE(my_cpu) { \
780 assert(ml_get_interrupts_enabled() == FALSE); \
781 /* \
782 * If a kernel_pmap update was requested while this cpu \
783 * was idle, process it as if we got the interrupt. \
784 * Before doing so, remove this cpu from the idle set. \
785 * Since we do not grab any pmap locks while we flush \
786 * our TLB, another cpu may start an update operation \
787 * before we finish. Removing this cpu from the idle \
788 * set assures that we will receive another update \
789 * interrupt if this happens. \
790 */ \
791 CPU_CR3_MARK_ACTIVE(); \
792 mfence(); \
793 \
794 if (current_cpu_datap()->cpu_tlb_invalid) \
795 process_pmap_updates(); \
796 }
797
798 #define PMAP_CONTEXT(pmap, thread)
799
800 #define pmap_kernel_va(VA) \
801 ((((vm_offset_t) (VA)) >= vm_min_kernel_address) && \
802 (((vm_offset_t) (VA)) <= vm_max_kernel_address))
803
804
805 #define pmap_compressed(pmap) ((pmap)->stats.compressed)
806 #define pmap_resident_count(pmap) ((pmap)->stats.resident_count)
807 #define pmap_resident_max(pmap) ((pmap)->stats.resident_max)
808 #define pmap_copy(dst_pmap,src_pmap,dst_addr,len,src_addr)
809 #define pmap_attribute(pmap,addr,size,attr,value) \
810 (KERN_INVALID_ADDRESS)
811 #define pmap_attribute_cache_sync(addr,size,attr,value) \
812 (KERN_INVALID_ADDRESS)
813
814 #define MACHINE_PMAP_IS_EMPTY 1
815 extern boolean_t pmap_is_empty(pmap_t pmap,
816 vm_map_offset_t start,
817 vm_map_offset_t end);
818
819 #define MACHINE_BOOTSTRAPPTD 1 /* Static bootstrap page-tables */
820
821 kern_return_t
822 pmap_permissions_verify(pmap_t, vm_map_t, vm_offset_t, vm_offset_t);
823
824 #endif /* ASSEMBLER */
825
826
827 #endif /* _PMAP_MACHINE_ */
828
829
830 #endif /* KERNEL_PRIVATE */
mirror of XNU