]> git.saurik.com Git - apple/xnu.git/blob - osfmk/i386/pmap.h
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-2050.22.13.tar.gz
[apple/xnu.git] / osfmk / i386 / pmap.h
1 /*
2 * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56 /*
57 */
58
59 /*
60 * File: pmap.h
61 *
62 * Authors: Avadis Tevanian, Jr., Michael Wayne Young
63 * Date: 1985
64 *
65 * Machine-dependent structures for the physical map module.
66 */
67 #ifdef KERNEL_PRIVATE
68 #ifndef _PMAP_MACHINE_
69 #define _PMAP_MACHINE_ 1
70
71 #ifndef ASSEMBLER
72
73 #include <platforms.h>
74
75 #include <mach/kern_return.h>
76 #include <mach/machine/vm_types.h>
77 #include <mach/vm_prot.h>
78 #include <mach/vm_statistics.h>
79 #include <mach/machine/vm_param.h>
80 #include <kern/kern_types.h>
81 #include <kern/thread.h>
82 #include <kern/lock.h>
83 #include <mach/branch_predicates.h>
84
85 #include <i386/mp.h>
86 #include <i386/proc_reg.h>
87
88 #include <i386/pal_routines.h>
89
90 /*
91 * Define the generic in terms of the specific
92 */
93
94 #define INTEL_PGBYTES I386_PGBYTES
95 #define INTEL_PGSHIFT I386_PGSHIFT
96 #define intel_btop(x) i386_btop(x)
97 #define intel_ptob(x) i386_ptob(x)
98 #define intel_round_page(x) i386_round_page(x)
99 #define intel_trunc_page(x) i386_trunc_page(x)
100 #define trunc_intel_to_vm(x) trunc_i386_to_vm(x)
101 #define round_intel_to_vm(x) round_i386_to_vm(x)
102 #define vm_to_intel(x) vm_to_i386(x)
103
104 /*
105 * i386/i486/i860 Page Table Entry
106 */
107
108 #endif /* ASSEMBLER */
109
110 #define NPGPTD 4ULL
111 #define PDESHIFT 21ULL
112 #define PTEMASK 0x1ffULL
113 #define PTEINDX 3ULL
114
115 #define PTESHIFT 12ULL
116
117 #ifdef __i386__
118 #define INITPT_SEG_BASE 0x100000
119 #endif
120
121 #ifdef __x86_64__
122 #define LOW_4GB_MASK ((vm_offset_t)0x00000000FFFFFFFFUL)
123 #endif
124
125 #define PDESIZE sizeof(pd_entry_t) /* for assembly files */
126 #define PTESIZE sizeof(pt_entry_t) /* for assembly files */
127
128 #define INTEL_OFFMASK (I386_PGBYTES - 1)
129 #define INTEL_LOFFMASK (I386_LPGBYTES - 1)
130 #define PG_FRAME 0x000FFFFFFFFFF000ULL
131 #define NPTEPG (PAGE_SIZE/(sizeof (pt_entry_t)))
132 #define NPTDPG (PAGE_SIZE/(sizeof (pd_entry_t)))
133
134 #define NBPTD (NPGPTD << PAGE_SHIFT)
135 #define NPDEPTD (NBPTD / (sizeof (pd_entry_t)))
136 #define NPDEPG (PAGE_SIZE/(sizeof (pd_entry_t)))
137 #define NBPDE (1ULL << PDESHIFT)
138 #define PDEMASK (NBPDE - 1)
139
140 #define PTE_PER_PAGE 512 /* number of PTE's per page on any level */
141
142 /* cleanly define parameters for all the page table levels */
143 typedef uint64_t pml4_entry_t;
144 #define NPML4PG (PAGE_SIZE/(sizeof (pml4_entry_t)))
145 #define PML4SHIFT 39
146 #define PML4PGSHIFT 9
147 #define NBPML4 (1ULL << PML4SHIFT)
148 #define PML4MASK (NBPML4-1)
149 #define PML4_ENTRY_NULL ((pml4_entry_t *) 0)
150
151 typedef uint64_t pdpt_entry_t;
152 #define NPDPTPG (PAGE_SIZE/(sizeof (pdpt_entry_t)))
153 #define PDPTSHIFT 30
154 #define PDPTPGSHIFT 9
155 #define NBPDPT (1ULL << PDPTSHIFT)
156 #define PDPTMASK (NBPDPT-1)
157 #define PDPT_ENTRY_NULL ((pdpt_entry_t *) 0)
158
159 typedef uint64_t pd_entry_t;
160 #define NPDPG (PAGE_SIZE/(sizeof (pd_entry_t)))
161 #define PDSHIFT 21
162 #define PDPGSHIFT 9
163 #define NBPD (1ULL << PDSHIFT)
164 #define PDMASK (NBPD-1)
165 #define PD_ENTRY_NULL ((pd_entry_t *) 0)
166
167 typedef uint64_t pt_entry_t;
168 #define NPTPG (PAGE_SIZE/(sizeof (pt_entry_t)))
169 #define PTSHIFT 12
170 #define PTPGSHIFT 9
171 #define NBPT (1ULL << PTSHIFT)
172 #define PTMASK (NBPT-1)
173 #define PT_ENTRY_NULL ((pt_entry_t *) 0)
174
175 typedef uint64_t pmap_paddr_t;
176
177 #if DEBUG
178 #define PMAP_ASSERT 1
179 #endif
180 #if PMAP_ASSERT
181 #define pmap_assert(ex) ((ex) ? (void)0 : Assert(__FILE__, __LINE__, # ex))
182
183 #define pmap_assert2(ex, fmt, args...) \
184 do { \
185 if (!(ex)) { \
186 kprintf("Assertion %s failed (%s:%d, caller %p) " fmt , #ex, __FILE__, __LINE__, __builtin_return_address(0), ##args); \
187 panic("Assertion %s failed (%s:%d, caller %p) " fmt , #ex, __FILE__, __LINE__, __builtin_return_address(0), ##args); \
188 } \
189 } while(0)
190 #else
191 #define pmap_assert(ex)
192 #define pmap_assert2(ex, fmt, args...)
193 #endif
194
195 /* superpages */
196 #ifdef __x86_64__
197 #define SUPERPAGE_NBASEPAGES 512
198 #else
199 #define SUPERPAGE_NBASEPAGES 1 /* we don't support superpages on i386 */
200 #endif
201
202 /*
203 * Atomic 64-bit store of a page table entry.
204 */
205 static inline void
206 pmap_store_pte(pt_entry_t *entryp, pt_entry_t value)
207 {
208 #ifdef __i386__
209 /*
210 * Load the new value into %ecx:%ebx
211 * Load the old value into %edx:%eax
212 * Compare-exchange-8bytes at address entryp (loaded in %edi)
213 * If the compare succeeds, the new value will have been stored.
214 * Otherwise, the old value changed and reloaded, so try again.
215 */
216 __asm__ volatile(
217 " movl (%0), %%eax \n\t"
218 " movl 4(%0), %%edx \n\t"
219 "1: \n\t"
220 " cmpxchg8b (%0) \n\t"
221 " jnz 1b"
222 :
223 : "D" (entryp),
224 "b" ((uint32_t)value),
225 "c" ((uint32_t)(value >> 32))
226 : "eax", "edx", "memory");
227 #else
228 /*
229 * In the 32-bit kernel a compare-and-exchange loop was
230 * required to provide atomicity. For K64, life is easier:
231 */
232 *entryp = value;
233 #endif
234 }
235
236 /* in 64 bit spaces, the number of each type of page in the page tables */
237 #define NPML4PGS (1ULL * (PAGE_SIZE/(sizeof (pml4_entry_t))))
238 #define NPDPTPGS (NPML4PGS * (PAGE_SIZE/(sizeof (pdpt_entry_t))))
239 #define NPDEPGS (NPDPTPGS * (PAGE_SIZE/(sizeof (pd_entry_t))))
240 #define NPTEPGS (NPDEPGS * (PAGE_SIZE/(sizeof (pt_entry_t))))
241
242 #ifdef __i386__
243 /*
244 * The 64-bit kernel is remapped in uber-space which is at the base
245 * the highest 4th-level directory (KERNEL_UBER_PML4_INDEX). That is,
246 * 512GB from the top of virtual space (or zero).
247 */
248 #define KERNEL_UBER_PML4_INDEX 511
249 #define KERNEL_UBER_BASE (0ULL - NBPML4)
250 #define KERNEL_UBER_BASE_HI32 ((uint32_t)(KERNEL_UBER_BASE >> 32))
251 #else
252 #define KERNEL_PML4_INDEX 511
253 #define KERNEL_KEXTS_INDEX 510 /* Home of KEXTs - the basement */
254 #define KERNEL_PHYSMAP_PML4_INDEX 509 /* virtual to physical map */
255 #define KERNEL_BASE (0ULL - NBPML4)
256 #define KERNEL_BASEMENT (KERNEL_BASE - NBPML4)
257 #endif
258
259 #define VM_WIMG_COPYBACK VM_MEM_COHERENT
260 #define VM_WIMG_COPYBACKLW VM_WIMG_COPYBACK
261 #define VM_WIMG_DEFAULT VM_MEM_COHERENT
262 /* ?? intel ?? */
263 #define VM_WIMG_IO (VM_MEM_COHERENT | \
264 VM_MEM_NOT_CACHEABLE | VM_MEM_GUARDED)
265 #define VM_WIMG_WTHRU (VM_MEM_WRITE_THROUGH | VM_MEM_COHERENT | VM_MEM_GUARDED)
266 /* write combining mode, aka store gather */
267 #define VM_WIMG_WCOMB (VM_MEM_NOT_CACHEABLE | VM_MEM_COHERENT)
268 #define VM_WIMG_INNERWBACK VM_MEM_COHERENT
269 /*
270 * Pte related macros
271 */
272 #ifdef __i386__
273 #define VADDR(pdi, pti) ((vm_offset_t)(((pdi)<<PDESHIFT)|((pti)<<PTESHIFT)))
274 #define VADDR64(pmi, pdi, pti) ((vm_offset_t)(((pmi)<<PLM4SHIFT))((pdi)<<PDESHIFT)|((pti)<<PTESHIFT))
275 #else
276 #define KVADDR(pmi, pdpi, pdi, pti) \
277 ((vm_offset_t) \
278 ((uint64_t) -1 << 47) | \
279 ((uint64_t)(pmi) << PML4SHIFT) | \
280 ((uint64_t)(pdpi) << PDPTSHIFT) | \
281 ((uint64_t)(pdi) << PDESHIFT) | \
282 ((uint64_t)(pti) << PTESHIFT))
283 #endif
284
285 /*
286 * Size of Kernel address space. This is the number of page table pages
287 * (4MB each) to use for the kernel. 256 pages == 1 Gigabyte.
288 * This **MUST** be a multiple of 4 (eg: 252, 256, 260, etc).
289 */
290 #ifndef KVA_PAGES
291 #define KVA_PAGES 1024
292 #endif
293
294 #ifndef NKPT
295 #define NKPT 500 /* actual number of kernel page tables */
296 #endif
297 #ifndef NKPDE
298 #define NKPDE (KVA_PAGES - 1) /* addressable number of page tables/pde's */
299 #endif
300
301
302 #ifdef __i386__
303 enum high_cpu_types {
304 HIGH_CPU_ISS0,
305 HIGH_CPU_ISS1,
306 HIGH_CPU_DESC,
307 HIGH_CPU_LDT_BEGIN,
308 HIGH_CPU_LDT_END = HIGH_CPU_LDT_BEGIN + (LDTSZ / 512) - 1,
309 HIGH_CPU_END
310 };
311
312 enum high_fixed_addresses {
313 HIGH_FIXED_TRAMPS, /* must be first */
314 HIGH_FIXED_TRAMPS_END,
315 HIGH_FIXED_GDT,
316 HIGH_FIXED_IDT,
317 HIGH_FIXED_LDT_BEGIN,
318 HIGH_FIXED_LDT_END = HIGH_FIXED_LDT_BEGIN + (LDTSZ / 512) - 1,
319 HIGH_FIXED_KTSS,
320 HIGH_FIXED_DFTSS,
321 HIGH_FIXED_DBTSS,
322 HIGH_FIXED_CPUS_BEGIN,
323 HIGH_FIXED_CPUS_END = HIGH_FIXED_CPUS_BEGIN + (HIGH_CPU_END * MAX_CPUS) - 1,
324 };
325
326
327 /* XXX64 below PTDI values need cleanup */
328 /*
329 * The *PTDI values control the layout of virtual memory
330 *
331 */
332 #define KPTDI (0x000)/* start of kernel virtual pde's */
333 #define PTDPTDI (0x7F4) /* ptd entry that points to ptd! */
334 #define APTDPTDI (0x7F8) /* alt ptd entry that points to APTD */
335 #define UMAXPTDI (0x7F8) /* ptd entry for user space end */
336 #define UMAXPTEOFF (NPTEPG) /* pte entry for user space end */
337
338 #define KERNBASE VADDR(KPTDI,0)
339
340 /*
341 * Convert address offset to directory address
342 * containing the page table pointer - legacy
343 */
344 /*#define pmap_pde(m,v) (&((m)->dirbase[(vm_offset_t)(v) >> PDESHIFT]))*/
345
346 #define HIGH_MEM_BASE ((uint32_t)( -NBPDE) ) /* shared gdt etc seg addr */ /* XXX64 ?? */
347 #define pmap_index_to_virt(x) (HIGH_MEM_BASE | ((unsigned)(x) << PAGE_SHIFT))
348 #endif
349
350 /*
351 * Convert address offset to page descriptor index
352 */
353 #define pdptnum(pmap, a) (((vm_offset_t)(a) >> PDPTSHIFT) & PDPTMASK)
354 #define pdenum(pmap, a) (((vm_offset_t)(a) >> PDESHIFT) & PDEMASK)
355 #define PMAP_INVALID_PDPTNUM (~0ULL)
356
357 #define pdeidx(pmap, a) (((a) >> PDSHIFT) & ((1ULL<<(48 - PDSHIFT)) -1))
358 #define pdptidx(pmap, a) (((a) >> PDPTSHIFT) & ((1ULL<<(48 - PDPTSHIFT)) -1))
359 #define pml4idx(pmap, a) (((a) >> PML4SHIFT) & ((1ULL<<(48 - PML4SHIFT)) -1))
360
361
362 /*
363 * Convert page descriptor index to user virtual address
364 */
365 #define pdetova(a) ((vm_offset_t)(a) << PDESHIFT)
366
367 /*
368 * Convert address offset to page table index
369 */
370 #define ptenum(a) (((vm_offset_t)(a) >> PTESHIFT) & PTEMASK)
371
372 /*
373 * Hardware pte bit definitions (to be used directly on the ptes
374 * without using the bit fields).
375 */
376
377 #define INTEL_PTE_VALID 0x00000001ULL
378 #define INTEL_PTE_WRITE 0x00000002ULL
379 #define INTEL_PTE_RW 0x00000002ULL
380 #define INTEL_PTE_USER 0x00000004ULL
381 #define INTEL_PTE_WTHRU 0x00000008ULL
382 #define INTEL_PTE_NCACHE 0x00000010ULL
383 #define INTEL_PTE_REF 0x00000020ULL
384 #define INTEL_PTE_MOD 0x00000040ULL
385 #define INTEL_PTE_PS 0x00000080ULL
386 #define INTEL_PTE_PTA 0x00000080ULL
387 #define INTEL_PTE_GLOBAL 0x00000100ULL
388 #define INTEL_PTE_WIRED 0x00000200ULL
389 #define INTEL_PDPTE_NESTED 0x00000400ULL
390 #define INTEL_PTE_PFN PG_FRAME
391
392 #define INTEL_PTE_NX (1ULL << 63)
393
394 #define INTEL_PTE_INVALID 0
395 /* This is conservative, but suffices */
396 #define INTEL_PTE_RSVD ((1ULL << 10) | (1ULL << 11) | (0x1FFULL << 54))
397
398 #define pa_to_pte(a) ((a) & INTEL_PTE_PFN) /* XXX */
399 #define pte_to_pa(p) ((p) & INTEL_PTE_PFN) /* XXX */
400 #define pte_increment_pa(p) ((p) += INTEL_OFFMASK+1)
401
402 #define pte_kernel_rw(p) ((pt_entry_t)(pa_to_pte(p) | INTEL_PTE_VALID|INTEL_PTE_RW))
403 #define pte_kernel_ro(p) ((pt_entry_t)(pa_to_pte(p) | INTEL_PTE_VALID))
404 #define pte_user_rw(p) ((pt_entry)t)(pa_to_pte(p) | INTEL_PTE_VALID|INTEL_PTE_USER|INTEL_PTE_RW))
405 #define pte_user_ro(p) ((pt_entry_t)(pa_to_pte(p) | INTEL_PTE_VALID|INTEL_PTE_USER))
406
407 #define PMAP_DEFAULT_CACHE 0
408 #define PMAP_INHIBIT_CACHE 1
409 #define PMAP_GUARDED_CACHE 2
410 #define PMAP_ACTIVATE_CACHE 4
411 #define PMAP_NO_GUARD_CACHE 8
412
413
414 #ifndef ASSEMBLER
415
416 #include <sys/queue.h>
417
418 /*
419 * Address of current and alternate address space page table maps
420 * and directories.
421 */
422
423 #ifdef __i386__
424 extern pt_entry_t PTmap[], APTmap[], Upte;
425 extern pd_entry_t PTD[], APTD[], PTDpde[], APTDpde[], Upde;
426 extern pmap_paddr_t lo_kernel_cr3;
427 extern pdpt_entry_t *IdlePDPT64;
428 extern pdpt_entry_t IdlePDPT[];
429 extern pml4_entry_t IdlePML4[];
430 #else
431 extern pt_entry_t *PTmap;
432 extern pdpt_entry_t *IdlePDPT;
433 extern pml4_entry_t *IdlePML4;
434 #endif
435 extern boolean_t no_shared_cr3;
436 extern addr64_t kernel64_cr3;
437 extern pd_entry_t *IdlePTD; /* physical addr of "Idle" state PTD */
438
439 extern uint64_t pmap_pv_hashlist_walks;
440 extern uint64_t pmap_pv_hashlist_cnts;
441 extern uint32_t pmap_pv_hashlist_max;
442 extern uint32_t pmap_kernel_text_ps;
443
444 #ifdef __i386__
445 /*
446 * ** i386 **
447 * virtual address to page table entry and
448 * to physical address. Likewise for alternate address space.
449 * Note: these work recursively, thus vtopte of a pte will give
450 * the corresponding pde that in turn maps it.
451 */
452
453 #define vtopte(va) (PTmap + i386_btop((vm_offset_t)va))
454 #endif
455
456
457 #ifdef __x86_64__
458 #define ID_MAP_VTOP(x) ((void *)(((uint64_t)(x)) & LOW_4GB_MASK))
459
460 extern uint64_t physmap_base, physmap_max;
461
462 #define NPHYSMAP (MAX(K64_MAXMEM/GB + 4, 4))
463
464 static inline boolean_t physmap_enclosed(addr64_t a) {
465 return (a < (NPHYSMAP * GB));
466 }
467
468 static inline void * PHYSMAP_PTOV_check(void *paddr) {
469 uint64_t pvaddr = (uint64_t)paddr + physmap_base;
470
471 if (__improbable(pvaddr >= physmap_max))
472 panic("PHYSMAP_PTOV bounds exceeded, 0x%qx, 0x%qx, 0x%qx",
473 pvaddr, physmap_base, physmap_max);
474
475 return (void *)pvaddr;
476 }
477
478 #define PHYSMAP_PTOV(x) (PHYSMAP_PTOV_check((void*) (x)))
479
480 /*
481 * For KASLR, we alias the master processor's IDT and GDT at fixed
482 * virtual addresses to defeat SIDT/SGDT address leakage.
483 */
484 #define MASTER_IDT_ALIAS (VM_MIN_KERNEL_ADDRESS + 0x0000)
485 #define MASTER_GDT_ALIAS (VM_MIN_KERNEL_ADDRESS + 0x1000)
486
487 /*
488 * The low global vector page is mapped at a fixed alias also.
489 */
490 #define LOWGLOBAL_ALIAS (VM_MIN_KERNEL_ADDRESS + 0x2000)
491
492 #endif /*__x86_64__ */
493
494 typedef volatile long cpu_set; /* set of CPUs - must be <= 32 */
495 /* changed by other processors */
496 #include <vm/vm_page.h>
497
498 /*
499 * For each vm_page_t, there is a list of all currently
500 * valid virtual mappings of that page. An entry is
501 * a pv_entry_t; the list is the pv_table.
502 */
503
504 struct pmap {
505 decl_simple_lock_data(,lock) /* lock on map */
506 pmap_paddr_t pm_cr3; /* physical addr */
507 boolean_t pm_shared;
508 pd_entry_t *dirbase; /* page directory pointer */
509 #ifdef __i386__
510 pmap_paddr_t pdirbase; /* phys. address of dirbase */
511 vm_offset_t pm_hold; /* true pdpt zalloc addr */
512 #endif
513 vm_object_t pm_obj; /* object to hold pde's */
514 task_map_t pm_task_map;
515 pdpt_entry_t *pm_pdpt; /* KVA of 3rd level page */
516 pml4_entry_t *pm_pml4; /* VKA of top level */
517 vm_object_t pm_obj_pdpt; /* holds pdpt pages */
518 vm_object_t pm_obj_pml4; /* holds pml4 pages */
519 #define PMAP_PCID_MAX_CPUS (48) /* Must be a multiple of 8 */
520 pcid_t pmap_pcid_cpus[PMAP_PCID_MAX_CPUS];
521 volatile uint8_t pmap_pcid_coherency_vector[PMAP_PCID_MAX_CPUS];
522 struct pmap_statistics stats; /* map statistics */
523 int ref_count; /* reference count */
524 int nx_enabled;
525 ledger_t ledger; /* ledger tracking phys mappings */
526 };
527
528
529 #if NCOPY_WINDOWS > 0
530 #define PMAP_PDPT_FIRST_WINDOW 0
531 #define PMAP_PDPT_NWINDOWS 4
532 #define PMAP_PDE_FIRST_WINDOW (PMAP_PDPT_NWINDOWS)
533 #define PMAP_PDE_NWINDOWS 4
534 #define PMAP_PTE_FIRST_WINDOW (PMAP_PDE_FIRST_WINDOW + PMAP_PDE_NWINDOWS)
535 #define PMAP_PTE_NWINDOWS 4
536
537 #define PMAP_NWINDOWS_FIRSTFREE (PMAP_PTE_FIRST_WINDOW + PMAP_PTE_NWINDOWS)
538 #define PMAP_WINDOW_SIZE 8
539 #define PMAP_NWINDOWS (PMAP_NWINDOWS_FIRSTFREE + PMAP_WINDOW_SIZE)
540
541 typedef struct {
542 pt_entry_t *prv_CMAP;
543 caddr_t prv_CADDR;
544 } mapwindow_t;
545
546 typedef struct cpu_pmap {
547 int pdpt_window_index;
548 int pde_window_index;
549 int pte_window_index;
550 mapwindow_t mapwindow[PMAP_NWINDOWS];
551 } cpu_pmap_t;
552
553
554 extern mapwindow_t *pmap_get_mapwindow(pt_entry_t pentry);
555 extern void pmap_put_mapwindow(mapwindow_t *map);
556 #endif
557
558 typedef struct pmap_memory_regions {
559 ppnum_t base;
560 ppnum_t end;
561 ppnum_t alloc;
562 uint32_t type;
563 uint64_t attribute;
564 } pmap_memory_region_t;
565
566 extern unsigned pmap_memory_region_count;
567 extern unsigned pmap_memory_region_current;
568
569 #define PMAP_MEMORY_REGIONS_SIZE 128
570
571 extern pmap_memory_region_t pmap_memory_regions[];
572 #include <i386/pmap_pcid.h>
573
574 static inline void
575 set_dirbase(pmap_t tpmap, __unused thread_t thread) {
576 int ccpu = cpu_number();
577 cpu_datap(ccpu)->cpu_task_cr3 = tpmap->pm_cr3;
578 cpu_datap(ccpu)->cpu_task_map = tpmap->pm_task_map;
579 #ifndef __i386__
580 /*
581 * Switch cr3 if necessary
582 * - unless running with no_shared_cr3 debugging mode
583 * and we're not on the kernel's cr3 (after pre-empted copyio)
584 */
585 if (__probable(!no_shared_cr3)) {
586 if (get_cr3_base() != tpmap->pm_cr3) {
587 if (pmap_pcid_ncpus) {
588 pmap_pcid_activate(tpmap, ccpu);
589 }
590 else
591 set_cr3_raw(tpmap->pm_cr3);
592 }
593 } else {
594 if (get_cr3_base() != cpu_datap(ccpu)->cpu_kernel_cr3)
595 set_cr3_raw(cpu_datap(ccpu)->cpu_kernel_cr3);
596 }
597 #endif
598 }
599
600 /*
601 * External declarations for PMAP_ACTIVATE.
602 */
603
604 extern void process_pmap_updates(void);
605 extern void pmap_update_interrupt(void);
606
607 /*
608 * Machine dependent routines that are used only for i386/i486/i860.
609 */
610
611 extern addr64_t (kvtophys)(
612 vm_offset_t addr);
613
614 extern kern_return_t pmap_expand(
615 pmap_t pmap,
616 vm_map_offset_t addr,
617 unsigned int options);
618 #if !defined(__x86_64__)
619 extern pt_entry_t *pmap_pte(
620 struct pmap *pmap,
621 vm_map_offset_t addr);
622
623 extern pd_entry_t *pmap_pde(
624 struct pmap *pmap,
625 vm_map_offset_t addr);
626
627 extern pd_entry_t *pmap64_pde(
628 struct pmap *pmap,
629 vm_map_offset_t addr);
630
631 extern pdpt_entry_t *pmap64_pdpt(
632 struct pmap *pmap,
633 vm_map_offset_t addr);
634 #endif
635 extern vm_offset_t pmap_map(
636 vm_offset_t virt,
637 vm_map_offset_t start,
638 vm_map_offset_t end,
639 vm_prot_t prot,
640 unsigned int flags);
641
642 extern vm_offset_t pmap_map_bd(
643 vm_offset_t virt,
644 vm_map_offset_t start,
645 vm_map_offset_t end,
646 vm_prot_t prot,
647 unsigned int flags);
648
649 extern void pmap_bootstrap(
650 vm_offset_t load_start,
651 boolean_t IA32e);
652
653 extern boolean_t pmap_valid_page(
654 ppnum_t pn);
655
656 extern int pmap_list_resident_pages(
657 struct pmap *pmap,
658 vm_offset_t *listp,
659 int space);
660 extern void x86_filter_TLB_coherency_interrupts(boolean_t);
661 #ifdef __i386__
662 extern void pmap_commpage32_init(
663 vm_offset_t kernel,
664 vm_offset_t user,
665 int count);
666 extern void pmap_commpage64_init(
667 vm_offset_t kernel,
668 vm_map_offset_t user,
669 int count);
670
671 #endif
672 /*
673 * Get cache attributes (as pagetable bits) for the specified phys page
674 */
675 extern unsigned pmap_get_cache_attributes(ppnum_t);
676 #if NCOPY_WINDOWS > 0
677 extern struct cpu_pmap *pmap_cpu_alloc(
678 boolean_t is_boot_cpu);
679 extern void pmap_cpu_free(
680 struct cpu_pmap *cp);
681 #endif
682
683 extern void pmap_map_block(
684 pmap_t pmap,
685 addr64_t va,
686 ppnum_t pa,
687 uint32_t size,
688 vm_prot_t prot,
689 int attr,
690 unsigned int flags);
691
692 extern void invalidate_icache(vm_offset_t addr, unsigned cnt, int phys);
693 extern void flush_dcache(vm_offset_t addr, unsigned count, int phys);
694 extern ppnum_t pmap_find_phys(pmap_t map, addr64_t va);
695
696 extern void pmap_cpu_init(void);
697 extern void pmap_disable_NX(pmap_t pmap);
698 #ifdef __i386__
699 extern void pmap_set_4GB_pagezero(pmap_t pmap);
700 extern void pmap_clear_4GB_pagezero(pmap_t pmap);
701 extern void pmap_load_kernel_cr3(void);
702 extern vm_offset_t pmap_cpu_high_map_vaddr(int, enum high_cpu_types);
703 extern vm_offset_t pmap_high_map_vaddr(enum high_cpu_types);
704 extern vm_offset_t pmap_high_map(pt_entry_t, enum high_cpu_types);
705 extern vm_offset_t pmap_cpu_high_shared_remap(int, enum high_cpu_types, vm_offset_t, int);
706 extern vm_offset_t pmap_high_shared_remap(enum high_fixed_addresses, vm_offset_t, int);
707 #endif
708
709 extern void pt_fake_zone_init(int);
710 extern void pt_fake_zone_info(int *, vm_size_t *, vm_size_t *, vm_size_t *, vm_size_t *,
711 uint64_t *, int *, int *, int *);
712 extern void pmap_pagetable_corruption_msg_log(int (*)(const char * fmt, ...)__printflike(1,2));
713
714 /*
715 * Macros for speed.
716 */
717
718
719 #include <kern/spl.h>
720
721
722 #define PMAP_ACTIVATE_MAP(map, thread) { \
723 register pmap_t tpmap; \
724 \
725 tpmap = vm_map_pmap(map); \
726 set_dirbase(tpmap, thread); \
727 }
728
729 #ifdef __i386__
730 #define PMAP_DEACTIVATE_MAP(map, thread) \
731 if (vm_map_pmap(map)->pm_task_map == TASK_MAP_64BIT_SHARED) \
732 pmap_load_kernel_cr3();
733 #elif defined(__x86_64__)
734 #define PMAP_DEACTIVATE_MAP(map, thread) \
735 pmap_assert(pmap_pcid_ncpus ? (pcid_for_pmap_cpu_tuple(map->pmap, cpu_number()) == (get_cr3_raw() & 0xFFF)) : TRUE);
736 #else
737 #define PMAP_DEACTIVATE_MAP(map, thread)
738 #endif
739
740 #if defined(__i386__)
741
742 #define PMAP_SWITCH_CONTEXT(old_th, new_th, my_cpu) { \
743 spl_t spl; \
744 pt_entry_t *kpdp; \
745 pt_entry_t *updp; \
746 int i; \
747 int need_flush; \
748 \
749 need_flush = 0; \
750 spl = splhigh(); \
751 if ((old_th->map != new_th->map) || (new_th->task != old_th->task)) { \
752 PMAP_DEACTIVATE_MAP(old_th->map, old_th); \
753 PMAP_ACTIVATE_MAP(new_th->map, new_th); \
754 } \
755 kpdp = current_cpu_datap()->cpu_copywindow_pdp; \
756 for (i = 0; i < NCOPY_WINDOWS; i++) { \
757 if (new_th->machine.copy_window[i].user_base != (user_addr_t)-1) { \
758 updp = pmap_pde(new_th->map->pmap, \
759 new_th->machine.copy_window[i].user_base);\
760 pmap_store_pte(kpdp, updp ? *updp : 0); \
761 } \
762 kpdp++; \
763 } \
764 splx(spl); \
765 if (new_th->machine.copyio_state == WINDOWS_OPENED) \
766 need_flush = 1; \
767 else \
768 new_th->machine.copyio_state = WINDOWS_DIRTY; \
769 if (new_th->machine.physwindow_pte) { \
770 pmap_store_pte((current_cpu_datap()->cpu_physwindow_ptep), \
771 new_th->machine.physwindow_pte); \
772 if (need_flush == 0) \
773 invlpg((uintptr_t)current_cpu_datap()->cpu_physwindow_base);\
774 } \
775 if (need_flush) \
776 flush_tlb(); \
777 }
778
779 #else /* __x86_64__ */
780 #define PMAP_SWITCH_CONTEXT(old_th, new_th, my_cpu) { \
781 \
782 pmap_assert(ml_get_interrupts_enabled() == FALSE); \
783 if (old_th->map != new_th->map) { \
784 PMAP_DEACTIVATE_MAP(old_th->map, old_th); \
785 PMAP_ACTIVATE_MAP(new_th->map, new_th); \
786 } \
787 }
788 #endif /* __i386__ */
789
790 #if NCOPY_WINDOWS > 0
791 #define PMAP_SWITCH_USER(th, new_map, my_cpu) { \
792 spl_t spl; \
793 \
794 spl = splhigh(); \
795 PMAP_DEACTIVATE_MAP(th->map, th); \
796 th->map = new_map; \
797 PMAP_ACTIVATE_MAP(th->map, th); \
798 splx(spl); \
799 inval_copy_windows(th); \
800 }
801 #else
802 #define PMAP_SWITCH_USER(th, new_map, my_cpu) { \
803 spl_t spl; \
804 \
805 spl = splhigh(); \
806 PMAP_DEACTIVATE_MAP(th->map, th); \
807 th->map = new_map; \
808 PMAP_ACTIVATE_MAP(th->map, th); \
809 splx(spl); \
810 }
811 #endif
812
813 /*
814 * Marking the current cpu's cr3 inactive is achieved by setting its lsb.
815 * Marking the current cpu's cr3 active once more involves clearng this bit.
816 * Note that valid page tables are page-aligned and so the bottom 12 bits
817 * are normally zero, modulo PCID.
818 * We can only mark the current cpu active/inactive but we can test any cpu.
819 */
820 #define CPU_CR3_MARK_INACTIVE() \
821 current_cpu_datap()->cpu_active_cr3 |= 1
822
823 #define CPU_CR3_MARK_ACTIVE() \
824 current_cpu_datap()->cpu_active_cr3 &= ~1
825
826 #define CPU_CR3_IS_ACTIVE(cpu) \
827 ((cpu_datap(cpu)->cpu_active_cr3 & 1) == 0)
828
829 #define CPU_GET_ACTIVE_CR3(cpu) \
830 (cpu_datap(cpu)->cpu_active_cr3 & ~1)
831
832 #define CPU_GET_TASK_CR3(cpu) \
833 (cpu_datap(cpu)->cpu_task_cr3)
834
835 /*
836 * Mark this cpu idle, and remove it from the active set,
837 * since it is not actively using any pmap. Signal_cpus
838 * will notice that it is idle, and avoid signaling it,
839 * but will queue the update request for when the cpu
840 * becomes active.
841 */
842 #if defined(__x86_64__)
843 #define MARK_CPU_IDLE(my_cpu) { \
844 assert(ml_get_interrupts_enabled() == FALSE); \
845 CPU_CR3_MARK_INACTIVE(); \
846 __asm__ volatile("mfence"); \
847 }
848 #else /* __i386__ native */
849 #define MARK_CPU_IDLE(my_cpu) { \
850 assert(ml_get_interrupts_enabled() == FALSE); \
851 /* \
852 * Mark this cpu idle, and remove it from the active set, \
853 * since it is not actively using any pmap. Signal_cpus \
854 * will notice that it is idle, and avoid signaling it, \
855 * but will queue the update request for when the cpu \
856 * becomes active. \
857 */ \
858 if (!cpu_mode_is64bit() || no_shared_cr3) \
859 process_pmap_updates(); \
860 else \
861 pmap_load_kernel_cr3(); \
862 CPU_CR3_MARK_INACTIVE(); \
863 __asm__ volatile("mfence"); \
864 }
865 #endif /* __i386__ */
866
867 #define MARK_CPU_ACTIVE(my_cpu) { \
868 assert(ml_get_interrupts_enabled() == FALSE); \
869 /* \
870 * If a kernel_pmap update was requested while this cpu \
871 * was idle, process it as if we got the interrupt. \
872 * Before doing so, remove this cpu from the idle set. \
873 * Since we do not grab any pmap locks while we flush \
874 * our TLB, another cpu may start an update operation \
875 * before we finish. Removing this cpu from the idle \
876 * set assures that we will receive another update \
877 * interrupt if this happens. \
878 */ \
879 CPU_CR3_MARK_ACTIVE(); \
880 __asm__ volatile("mfence"); \
881 \
882 if (current_cpu_datap()->cpu_tlb_invalid) \
883 process_pmap_updates(); \
884 }
885
886 #define PMAP_CONTEXT(pmap, thread)
887
888 #define pmap_kernel_va(VA) \
889 ((((vm_offset_t) (VA)) >= vm_min_kernel_address) && \
890 (((vm_offset_t) (VA)) <= vm_max_kernel_address))
891
892
893 #define pmap_resident_count(pmap) ((pmap)->stats.resident_count)
894 #define pmap_resident_max(pmap) ((pmap)->stats.resident_max)
895 #define pmap_copy(dst_pmap,src_pmap,dst_addr,len,src_addr)
896 #define pmap_attribute(pmap,addr,size,attr,value) \
897 (KERN_INVALID_ADDRESS)
898 #define pmap_attribute_cache_sync(addr,size,attr,value) \
899 (KERN_INVALID_ADDRESS)
900
901 #define MACHINE_PMAP_IS_EMPTY 1
902 extern boolean_t pmap_is_empty(pmap_t pmap,
903 vm_map_offset_t start,
904 vm_map_offset_t end);
905
906 #define MACHINE_BOOTSTRAPPTD 1 /* Static bootstrap page-tables */
907
908 kern_return_t
909 pmap_permissions_verify(pmap_t, vm_map_t, vm_offset_t, vm_offset_t);
910
911 #endif /* ASSEMBLER */
912
913
914 #endif /* _PMAP_MACHINE_ */
915
916
917 #endif /* KERNEL_PRIVATE */
mirror of XNU