]> git.saurik.com Git - apple/xnu.git/blame - osfmk/i386/pmap.h
projects / apple / xnu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
xnu-1228.15.4.tar.gz
[apple/xnu.git] / osfmk / i386 / pmap.h
CommitLineData
1c79356b 1/*
0c530ab8 2 * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
1c79356b 3 *
2d21ac55 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
8f6c56a5 5 *
2d21ac55
A		 6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
8f6c56a5 14 *
2d21ac55
A		 15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
8f6c56a5
A		 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
2d21ac55
A		 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
8f6c56a5 25 *
2d21ac55 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
1c79356b
A		 27 */
28/*
29 * @OSF_COPYRIGHT@
30 */
31/*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56/*
57 */
58
59/*
60 * File: pmap.h
61 *
62 * Authors: Avadis Tevanian, Jr., Michael Wayne Young
63 * Date: 1985
64 *
65 * Machine-dependent structures for the physical map module.
66 */
0c530ab8 67#ifdef KERNEL_PRIVATE
1c79356b
A		 68#ifndef _PMAP_MACHINE_
69#define _PMAP_MACHINE_ 1
70
71#ifndef ASSEMBLER
72
73#include <platforms.h>
1c79356b
A		 74
75#include <mach/kern_return.h>
76#include <mach/machine/vm_types.h>
77#include <mach/vm_prot.h>
78#include <mach/vm_statistics.h>
79#include <mach/machine/vm_param.h>
80#include <kern/kern_types.h>
91447636 81#include <kern/thread.h>
1c79356b 82#include <kern/lock.h>
0c530ab8
A		 83
84#include <i386/mp.h>
85#include <i386/proc_reg.h>
1c79356b
A		 86
87/*
88 * Define the generic in terms of the specific
89 */
90
91#define INTEL_PGBYTES I386_PGBYTES
92#define INTEL_PGSHIFT I386_PGSHIFT
93#define intel_btop(x) i386_btop(x)
94#define intel_ptob(x) i386_ptob(x)
95#define intel_round_page(x) i386_round_page(x)
96#define intel_trunc_page(x) i386_trunc_page(x)
97#define trunc_intel_to_vm(x) trunc_i386_to_vm(x)
98#define round_intel_to_vm(x) round_i386_to_vm(x)
99#define vm_to_intel(x) vm_to_i386(x)
100
101/*
102 * i386/i486/i860 Page Table Entry
103 */
104
1c79356b
A		 105#endif /* ASSEMBLER */
106
91447636
A		 107#define NPGPTD 4
108#define PDESHIFT 21
109#define PTEMASK 0x1ff
110#define PTEINDX 3
0c530ab8 111
91447636
A		 112#define PTESHIFT 12
113
114#define PDESIZE sizeof(pd_entry_t) /* for assembly files */
115#define PTESIZE sizeof(pt_entry_t) /* for assembly files */
116
117#define INTEL_OFFMASK (I386_PGBYTES - 1)
0c530ab8 118#define PG_FRAME 0x000FFFFFFFFFF000ULL
91447636 119#define NPTEPG (PAGE_SIZE/(sizeof (pt_entry_t)))
0c530ab8 120#define NPTDPG (PAGE_SIZE/(sizeof (pd_entry_t)))
1c79356b 121
91447636
A		 122#define NBPTD (NPGPTD << PAGE_SHIFT)
123#define NPDEPTD (NBPTD / (sizeof (pd_entry_t)))
124#define NPDEPG (PAGE_SIZE/(sizeof (pd_entry_t)))
125#define NBPDE (1 << PDESHIFT)
126#define PDEMASK (NBPDE - 1)
9bccf70c 127
0c530ab8
A		 128 /* cleanly define parameters for all the page table levels */
129typedef uint64_t pml4_entry_t;
130#define NPML4PG (PAGE_SIZE/(sizeof (pml4_entry_t)))
131#define PML4SHIFT 39
132#define PML4PGSHIFT 9
133#define NBPML4 (1ULL << PML4SHIFT)
134#define PML4MASK (NBPML4-1)
135#define PML4_ENTRY_NULL ((pml4_entry_t *) 0)
136
137typedef uint64_t pdpt_entry_t;
138#define NPDPTPG (PAGE_SIZE/(sizeof (pdpt_entry_t)))
139#define PDPTSHIFT 30
140#define PDPTPGSHIFT 9
141#define NBPDPT (1 << PDPTSHIFT)
142#define PDPTMASK (NBPDPT-1)
143#define PDPT_ENTRY_NULL ((pdpt_entry_t *) 0)
144
145typedef uint64_t pd_entry_t;
146#define NPDPG (PAGE_SIZE/(sizeof (pd_entry_t)))
147#define PDSHIFT 21
148#define PDPGSHIFT 9
149#define NBPD (1 << PDSHIFT)
150#define PDMASK (NBPD-1)
151#define PD_ENTRY_NULL ((pd_entry_t *) 0)
152
153typedef uint64_t pt_entry_t;
154#define NPTPG (PAGE_SIZE/(sizeof (pt_entry_t)))
155#define PTSHIFT 12
156#define PTPGSHIFT 9
157#define NBPT (1 << PTSHIFT)
158#define PTMASK (NBPT-1)
159#define PT_ENTRY_NULL ((pt_entry_t *) 0)
160
161typedef uint64_t pmap_paddr_t;
162
163/*
164 * Atomic 64-bit store of a page table entry.
165 */
166static inline void
167pmap_store_pte(pt_entry_t *entryp, pt_entry_t value)
168{
169 /*
170 * Load the new value into %ecx:%ebx
171 * Load the old value into %edx:%eax
172 * Compare-exchange-8bytes at address entryp (loaded in %edi)
173 * If the compare succeeds, the new value will have been stored.
174 * Otherwise, the old value changed and reloaded, so try again.
175 */
2d21ac55 176 __asm__ volatile(
0c530ab8
A		 177 " movl (%0), %%eax \n\t"
178 " movl 4(%0), %%edx \n\t"
179 "1: \n\t"
180 " cmpxchg8b (%0) \n\t"
181 " jnz 1b"
182 :
183 : "D" (entryp),
184 "b" ((uint32_t)value),
185 "c" ((uint32_t)(value >> 32))
186 : "eax", "edx", "memory");
187}
188
189/*
190 * Atomic 64-bit compare and exchange of a page table entry.
191 */
192static inline boolean_t
193pmap_cmpx_pte(pt_entry_t *entryp, pt_entry_t old, pt_entry_t new)
194{
195 boolean_t ret;
196
197 /*
198 * Load the old value into %edx:%eax
199 * Load the new value into %ecx:%ebx
200 * Compare-exchange-8bytes at address entryp (loaded in %edi)
201 * If the compare succeeds, the new value is stored, return TRUE.
202 * Otherwise, no swap is made, return FALSE.
203 */
204 asm volatile(
205 " lock; cmpxchg8b (%1) \n\t"
206 " setz %%al \n\t"
207 " movzbl %%al,%0"
208 : "=a" (ret)
209 : "D" (entryp),
210 "a" ((uint32_t)old),
211 "d" ((uint32_t)(old >> 32)),
212 "b" ((uint32_t)new),
213 "c" ((uint32_t)(new >> 32))
214 : "memory");
215 return ret;
216}
217
218#define pmap_update_pte(entryp, old, new) \
219 while (!pmap_cmpx_pte((entryp), (old), (new)))
220
2d21ac55 221
0c530ab8
A		 222/* in 64 bit spaces, the number of each type of page in the page tables */
223#define NPML4PGS (1ULL * (PAGE_SIZE/(sizeof (pml4_entry_t))))
224#define NPDPTPGS (NPML4PGS * (PAGE_SIZE/(sizeof (pdpt_entry_t))))
225#define NPDEPGS (NPDPTPGS * (PAGE_SIZE/(sizeof (pd_entry_t))))
226#define NPTEPGS (NPDEPGS * (PAGE_SIZE/(sizeof (pt_entry_t))))
227
228/*
229 * The 64-bit kernel is remapped in uber-space which is at the base
230 * the highest 4th-level directory (KERNEL_UBER_PML4_INDEX). That is,
231 * 512GB from the top of virtual space (or zero).
232 */
233#define KERNEL_UBER_PML4_INDEX 511
234#define KERNEL_UBER_BASE (0ULL - NBPML4)
235#define KERNEL_UBER_BASE_HI32 ((uint32_t)(KERNEL_UBER_BASE >> 32))
236
55e303ae 237#define VM_WIMG_COPYBACK VM_MEM_COHERENT
9bccf70c 238#define VM_WIMG_DEFAULT VM_MEM_COHERENT
55e303ae
A		 239/* ?? intel ?? */
240#define VM_WIMG_IO (VM_MEM_COHERENT | \
241 VM_MEM_NOT_CACHEABLE | VM_MEM_GUARDED)
242#define VM_WIMG_WTHRU (VM_MEM_WRITE_THROUGH | VM_MEM_COHERENT | VM_MEM_GUARDED)
243/* write combining mode, aka store gather */
244#define VM_WIMG_WCOMB (VM_MEM_NOT_CACHEABLE | VM_MEM_COHERENT)
9bccf70c 245
0c530ab8
A		 246/*
247 * Pte related macros
248 */
249#define VADDR(pdi, pti) ((vm_offset_t)(((pdi)<<PDESHIFT)|((pti)<<PTESHIFT)))
250#define VADDR64(pmi, pdi, pti) ((vm_offset_t)(((pmi)<<PLM4SHIFT))((pdi)<<PDESHIFT)|((pti)<<PTESHIFT))
251
1c79356b 252/*
91447636
A		 253 * Size of Kernel address space. This is the number of page table pages
254 * (4MB each) to use for the kernel. 256 pages == 1 Gigabyte.
255 * This **MUST** be a multiple of 4 (eg: 252, 256, 260, etc).
1c79356b 256 */
91447636 257#ifndef KVA_PAGES
0c530ab8 258#define KVA_PAGES 1024
91447636 259#endif
1c79356b 260
91447636 261#ifndef NKPT
91447636 262#define NKPT 500 /* actual number of kernel page tables */
91447636
A		 263#endif
264#ifndef NKPDE
265#define NKPDE (KVA_PAGES - 1) /* addressable number of page tables/pde's */
266#endif
267
0c530ab8
A		 268
269enum high_cpu_types {
270 HIGH_CPU_ISS0,
271 HIGH_CPU_ISS1,
272 HIGH_CPU_DESC,
273 HIGH_CPU_LDT_BEGIN,
274 HIGH_CPU_LDT_END = HIGH_CPU_LDT_BEGIN + (LDTSZ / 512) - 1,
275 HIGH_CPU_END
276};
277
278enum high_fixed_addresses {
279 HIGH_FIXED_TRAMPS, /* must be first */
280 HIGH_FIXED_TRAMPS_END,
281 HIGH_FIXED_GDT,
282 HIGH_FIXED_IDT,
283 HIGH_FIXED_LDT_BEGIN,
284 HIGH_FIXED_LDT_END = HIGH_FIXED_LDT_BEGIN + (LDTSZ / 512) - 1,
285 HIGH_FIXED_KTSS,
286 HIGH_FIXED_DFTSS,
287 HIGH_FIXED_DBTSS,
288 HIGH_FIXED_CPUS_BEGIN,
289 HIGH_FIXED_CPUS_END = HIGH_FIXED_CPUS_BEGIN + (HIGH_CPU_END * MAX_CPUS) - 1,
290};
291
292
293/* XXX64 below PTDI values need cleanup */
91447636
A		 294/*
295 * The *PTDI values control the layout of virtual memory
296 *
297 */
0c530ab8 298#define KPTDI (0x000)/* start of kernel virtual pde's */
91447636
A		 299#define PTDPTDI (0x7F4) /* ptd entry that points to ptd! */
300#define APTDPTDI (0x7F8) /* alt ptd entry that points to APTD */
0c530ab8 301#define UMAXPTDI (0x7F8) /* ptd entry for user space end */
6601e61a 302#define UMAXPTEOFF (NPTEPG) /* pte entry for user space end */
91447636
A		 303
304#define KERNBASE VADDR(KPTDI,0)
1c79356b 305
0c530ab8
A		 306/*
307 * Convert address offset to directory address
308 * containing the page table pointer - legacy
309 */
310/*#define pmap_pde(m,v) (&((m)->dirbase[(vm_offset_t)(v) >> PDESHIFT]))*/
311
312#define HIGH_MEM_BASE ((uint32_t)( -NBPDE) ) /* shared gdt etc seg addr */ /* XXX64 ?? */
313#define pmap_index_to_virt(x) (HIGH_MEM_BASE | ((unsigned)(x) << PAGE_SHIFT))
314
1c79356b
A		 315/*
316 * Convert address offset to page descriptor index
317 */
0c530ab8 318#define pdenum(pmap, a) (((vm_offset_t)(a) >> PDESHIFT) & PDEMASK)
91447636 319
0c530ab8
A		 320#define pdeidx(pmap, a) (((a) >> PDSHIFT) & ((1ULL<<(48 - PDSHIFT)) -1))
321#define pdptidx(pmap, a) (((a) >> PDPTSHIFT) & ((1ULL<<(48 - PDPTSHIFT)) -1))
322#define pml4idx(pmap, a) (((a) >> PML4SHIFT) & ((1ULL<<(48 - PML4SHIFT)) -1))
1c79356b
A		 323
324/*
325 * Convert page descriptor index to user virtual address
326 */
327#define pdetova(a) ((vm_offset_t)(a) << PDESHIFT)
328
329/*
330 * Convert address offset to page table index
331 */
0c530ab8 332#define ptenum(a) (((vm_offset_t)(a) >> PTESHIFT) & PTEMASK)
1c79356b 333
1c79356b
A		 334/*
335 * Hardware pte bit definitions (to be used directly on the ptes
336 * without using the bit fields).
337 */
338
339#define INTEL_PTE_VALID 0x00000001
340#define INTEL_PTE_WRITE 0x00000002
91447636 341#define INTEL_PTE_RW 0x00000002
1c79356b
A		 342#define INTEL_PTE_USER 0x00000004
343#define INTEL_PTE_WTHRU 0x00000008
344#define INTEL_PTE_NCACHE 0x00000010
345#define INTEL_PTE_REF 0x00000020
346#define INTEL_PTE_MOD 0x00000040
91447636
A		 347#define INTEL_PTE_PS 0x00000080
348#define INTEL_PTE_GLOBAL 0x00000100
1c79356b 349#define INTEL_PTE_WIRED 0x00000200
0c530ab8 350#define INTEL_PTE_PFN PG_FRAME
55e303ae 351#define INTEL_PTE_PTA 0x00000080
1c79356b 352
0c530ab8
A		 353#define INTEL_PTE_NX (1ULL << 63)
354
355#define INTEL_PTE_INVALID 0
356
91447636
A		 357#define pa_to_pte(a) ((a) & INTEL_PTE_PFN) /* XXX */
358#define pte_to_pa(p) ((p) & INTEL_PTE_PFN) /* XXX */
1c79356b
A		 359#define pte_increment_pa(p) ((p) += INTEL_OFFMASK+1)
360
0c530ab8
A		 361#define pte_kernel_rw(p) ((pt_entry_t)(pa_to_pte(p) | INTEL_PTE_VALID|INTEL_PTE_RW))
362#define pte_kernel_ro(p) ((pt_entry_t)(pa_to_pte(p) | INTEL_PTE_VALID))
363#define pte_user_rw(p) ((pt_entry)t)(pa_to_pte(p) | INTEL_PTE_VALID|INTEL_PTE_USER|INTEL_PTE_RW))
364#define pte_user_ro(p) ((pt_entry_t)(pa_to_pte(p) | INTEL_PTE_VALID|INTEL_PTE_USER))
365
9bccf70c
A		 366#define PMAP_DEFAULT_CACHE 0
367#define PMAP_INHIBIT_CACHE 1
368#define PMAP_GUARDED_CACHE 2
369#define PMAP_ACTIVATE_CACHE 4
370#define PMAP_NO_GUARD_CACHE 8
371
372
91447636
A		 373#ifndef ASSEMBLER
374
375#include <sys/queue.h>
376
1c79356b 377/*
91447636
A		 378 * Address of current and alternate address space page table maps
379 * and directories.
1c79356b 380 */
1c79356b 381
91447636
A		 382extern pt_entry_t PTmap[], APTmap[], Upte;
383extern pd_entry_t PTD[], APTD[], PTDpde[], APTDpde[], Upde;
384
385extern pd_entry_t *IdlePTD; /* physical address of "Idle" state directory */
91447636 386extern pdpt_entry_t *IdlePDPT;
0c530ab8
A		 387
388extern pmap_paddr_t lo_kernel_cr3;
389
390extern pml4_entry_t *IdlePML4;
391extern pdpt_entry_t *IdlePDPT64;
392extern addr64_t kernel64_cr3;
393extern boolean_t no_shared_cr3;
91447636 394
2d21ac55
A		 395extern uint64_t pmap_pv_hashlist_walks;
396extern uint64_t pmap_pv_hashlist_cnts;
397extern uint32_t pmap_pv_hashlist_max;
398
91447636
A		 399/*
400 * virtual address to page table entry and
401 * to physical address. Likewise for alternate address space.
402 * Note: these work recursively, thus vtopte of a pte will give
403 * the corresponding pde that in turn maps it.
404 */
0c530ab8 405#define vtopte(va) (PTmap + i386_btop((vm_offset_t)va))
91447636
A		 406
407
1c79356b
A		 408typedef volatile long cpu_set; /* set of CPUs - must be <= 32 */
409 /* changed by other processors */
91447636
A		 410struct md_page {
411 int pv_list_count;
412 TAILQ_HEAD(,pv_entry) pv_list;
413};
414
415#include <vm/vm_page.h>
416
417/*
418 * For each vm_page_t, there is a list of all currently
419 * valid virtual mappings of that page. An entry is
420 * a pv_entry_t; the list is the pv_table.
421 */
1c79356b
A		 422
423struct pmap {
0c530ab8
A		 424 pd_entry_t *dirbase; /* page directory pointer */
425 pmap_paddr_t pdirbase; /* phys. address of dirbase */
426 vm_object_t pm_obj; /* object to hold pde's */
1c79356b 427 int ref_count; /* reference count */
0c530ab8 428 int nx_enabled;
2d21ac55 429 task_map_t pm_task_map;
1c79356b
A		 430 decl_simple_lock_data(,lock) /* lock on map */
431 struct pmap_statistics stats; /* map statistics */
91447636 432 vm_offset_t pm_hold; /* true pdpt zalloc addr */
0c530ab8
A		 433 pmap_paddr_t pm_cr3; /* physical addr */
434 pdpt_entry_t *pm_pdpt; /* KVA of 3rd level page */
435 pml4_entry_t *pm_pml4; /* VKA of top level */
436 vm_object_t pm_obj_pdpt; /* holds pdpt pages */
437 vm_object_t pm_obj_pml4; /* holds pml4 pages */
438 vm_object_t pm_obj_top; /* holds single top level page */
2d21ac55 439 boolean_t pm_shared;
1c79356b
A		 440};
441
0c530ab8
A		 442
443#define PMAP_PDPT_FIRST_WINDOW 0
444#define PMAP_PDPT_NWINDOWS 4
445#define PMAP_PDE_FIRST_WINDOW (PMAP_PDPT_NWINDOWS)
446#define PMAP_PDE_NWINDOWS 4
447#define PMAP_PTE_FIRST_WINDOW (PMAP_PDE_FIRST_WINDOW + PMAP_PDE_NWINDOWS)
448#define PMAP_PTE_NWINDOWS 4
449
450#define PMAP_NWINDOWS_FIRSTFREE (PMAP_PTE_FIRST_WINDOW + PMAP_PTE_NWINDOWS)
451#define PMAP_WINDOW_SIZE 8
452#define PMAP_NWINDOWS (PMAP_NWINDOWS_FIRSTFREE + PMAP_WINDOW_SIZE)
453
91447636
A		 454typedef struct {
455 pt_entry_t *prv_CMAP;
456 caddr_t prv_CADDR;
457} mapwindow_t;
458
459typedef struct cpu_pmap {
0c530ab8
A		 460 int pdpt_window_index;
461 int pde_window_index;
462 int pte_window_index;
91447636 463 mapwindow_t mapwindow[PMAP_NWINDOWS];
91447636
A		 464} cpu_pmap_t;
465
0c530ab8
A		 466
467extern mapwindow_t *pmap_get_mapwindow(pt_entry_t pentry);
2d21ac55
A		 468extern void pmap_put_mapwindow(mapwindow_t *map);
469
91447636
A		 470
471typedef struct pmap_memory_regions {
472 ppnum_t base;
473 ppnum_t end;
474 ppnum_t alloc;
475 uint32_t type;
476} pmap_memory_region_t;
477
478unsigned pmap_memory_region_count;
479unsigned pmap_memory_region_current;
480
0c530ab8 481#define PMAP_MEMORY_REGIONS_SIZE 128
91447636
A		 482
483extern pmap_memory_region_t pmap_memory_regions[];
484
0c530ab8
A		 485static inline void set_dirbase(pmap_t tpmap, __unused int tcpu) {
486 current_cpu_datap()->cpu_task_cr3 = (pmap_paddr_t)((tpmap)->pm_cr3);
2d21ac55 487 current_cpu_datap()->cpu_task_map = tpmap->pm_task_map;
1c79356b
A		 488}
489
1c79356b
A		 490/*
491 * External declarations for PMAP_ACTIVATE.
492 */
493
0c530ab8 494extern void process_pmap_updates(void);
1c79356b 495extern void pmap_update_interrupt(void);
1c79356b
A		 496
497/*
498 * Machine dependent routines that are used only for i386/i486/i860.
499 */
1c79356b 500
0c530ab8 501extern addr64_t (kvtophys)(
1c79356b
A		 502 vm_offset_t addr);
503
2d21ac55
A		 504extern void pmap_expand(
505 pmap_t pmap,
506 vm_map_offset_t addr);
507
1c79356b
A		 508extern pt_entry_t *pmap_pte(
509 struct pmap *pmap,
0c530ab8
A		 510 vm_map_offset_t addr);
511
512extern pd_entry_t *pmap_pde(
513 struct pmap *pmap,
514 vm_map_offset_t addr);
515
516extern pd_entry_t *pmap64_pde(
517 struct pmap *pmap,
518 vm_map_offset_t addr);
519
520extern pdpt_entry_t *pmap64_pdpt(
521 struct pmap *pmap,
522 vm_map_offset_t addr);
1c79356b
A		 523
524extern vm_offset_t pmap_map(
525 vm_offset_t virt,
0c530ab8
A		 526 vm_map_offset_t start,
527 vm_map_offset_t end,
528 vm_prot_t prot,
529 unsigned int flags);
1c79356b
A		 530
531extern vm_offset_t pmap_map_bd(
532 vm_offset_t virt,
0c530ab8
A		 533 vm_map_offset_t start,
534 vm_map_offset_t end,
535 vm_prot_t prot,
536 unsigned int flags);
1c79356b
A		 537
538extern void pmap_bootstrap(
0c530ab8
A		 539 vm_offset_t load_start,
540 boolean_t IA32e);
1c79356b
A		 541
542extern boolean_t pmap_valid_page(
91447636 543 ppnum_t pn);
1c79356b
A		 544
545extern int pmap_list_resident_pages(
546 struct pmap *pmap,
547 vm_offset_t *listp,
548 int space);
549
0c530ab8 550extern void pmap_commpage32_init(
91447636
A		 551 vm_offset_t kernel,
552 vm_offset_t user,
553 int count);
0c530ab8
A		 554extern void pmap_commpage64_init(
555 vm_offset_t kernel,
556 vm_map_offset_t user,
557 int count);
558
91447636
A		 559extern struct cpu_pmap *pmap_cpu_alloc(
560 boolean_t is_boot_cpu);
561extern void pmap_cpu_free(
562 struct cpu_pmap *cp);
0c530ab8
A		 563
564extern void pmap_map_block(
565 pmap_t pmap,
566 addr64_t va,
567 ppnum_t pa,
568 uint32_t size,
569 vm_prot_t prot,
570 int attr,
571 unsigned int flags);
91447636 572
1c79356b
A		 573extern void invalidate_icache(vm_offset_t addr, unsigned cnt, int phys);
574extern void flush_dcache(vm_offset_t addr, unsigned count, int phys);
55e303ae 575extern ppnum_t pmap_find_phys(pmap_t map, addr64_t va);
1c79356b 576
2d21ac55 577extern void pmap_cpu_init(void);
0c530ab8
A		 578extern void pmap_disable_NX(pmap_t pmap);
579extern void pmap_set_4GB_pagezero(pmap_t pmap);
580extern void pmap_clear_4GB_pagezero(pmap_t pmap);
581extern void pmap_load_kernel_cr3(void);
582extern vm_offset_t pmap_cpu_high_map_vaddr(int, enum high_cpu_types);
583extern vm_offset_t pmap_high_map_vaddr(enum high_cpu_types);
584extern vm_offset_t pmap_high_map(pt_entry_t, enum high_cpu_types);
585extern vm_offset_t pmap_cpu_high_shared_remap(int, enum high_cpu_types, vm_offset_t, int);
586extern vm_offset_t pmap_high_shared_remap(enum high_fixed_addresses, vm_offset_t, int);
587
588extern void pt_fake_zone_info(int *, vm_size_t *, vm_size_t *, vm_size_t *, vm_size_t *, int *, int *);
589
590
591
1c79356b
A		 592/*
593 * Macros for speed.
594 */
595
1c79356b
A		 596
597#include <kern/spl.h>
598
55e303ae
A		 599#if defined(PMAP_ACTIVATE_KERNEL)
600#undef PMAP_ACTIVATE_KERNEL
601#undef PMAP_DEACTIVATE_KERNEL
602#undef PMAP_ACTIVATE_USER
603#undef PMAP_DEACTIVATE_USER
604#endif
605
1c79356b 606
0c530ab8
A		 607#define PMAP_ACTIVATE_KERNEL(my_cpu) { \
608 spl_t spl; \
609 \
610 spl = splhigh(); \
611 if (current_cpu_datap()->cpu_tlb_invalid) \
612 process_pmap_updates(); \
613 splx(spl); \
1c79356b
A		 614}
615
0c530ab8
A		 616#define PMAP_DEACTIVATE_KERNEL(my_cpu) { \
617 spl_t spl; \
618 \
619 spl = splhigh(); \
620 process_pmap_updates(); \
621 splx(spl); \
1c79356b
A		 622}
623
0c530ab8 624
1c79356b 625#define PMAP_ACTIVATE_MAP(map, my_cpu) { \
55e303ae 626 register pmap_t tpmap; \
0c530ab8
A		 627 \
628 tpmap = vm_map_pmap(map); \
629 set_dirbase(tpmap, my_cpu); \
1c79356b
A		 630}
631
2d21ac55
A		 632#define PMAP_DEACTIVATE_MAP(map, my_cpu) \
633 if (vm_map_pmap(map)->pm_task_map == TASK_MAP_64BIT_SHARED) \
634 pmap_load_kernel_cr3();
635
1c79356b 636
0c530ab8
A		 637#define PMAP_ACTIVATE_USER(th, my_cpu) { \
638 spl_t spl; \
639 \
640 spl = splhigh(); \
641 PMAP_ACTIVATE_MAP(th->map, my_cpu) \
642 splx(spl); \
1c79356b
A		 643}
644
91447636 645#define PMAP_DEACTIVATE_USER(th, my_cpu)
1c79356b 646
0c530ab8 647
1c79356b
A		 648#define PMAP_SWITCH_CONTEXT(old_th, new_th, my_cpu) { \
649 spl_t spl; \
0c530ab8
A		 650 pt_entry_t *kpdp; \
651 pt_entry_t *updp; \
652 int i; \
653 int need_flush; \
654 \
655 need_flush = 0; \
656 spl = splhigh(); \
1c79356b 657 if (old_th->map != new_th->map) { \
1c79356b
A		 658 PMAP_DEACTIVATE_MAP(old_th->map, my_cpu); \
659 PMAP_ACTIVATE_MAP(new_th->map, my_cpu); \
1c79356b 660 } \
0c530ab8
A		 661 kpdp = current_cpu_datap()->cpu_copywindow_pdp; \
662 for (i = 0; i < NCOPY_WINDOWS; i++) { \
663 if (new_th->machine.copy_window[i].user_base != (user_addr_t)-1) { \
664 updp = pmap_pde(new_th->map->pmap, \
665 new_th->machine.copy_window[i].user_base);\
2d21ac55 666 pmap_store_pte(kpdp, updp ? *updp : 0); \
0c530ab8
A		 667 } \
668 kpdp++; \
669 } \
670 splx(spl); \
671 if (new_th->machine.copyio_state == WINDOWS_OPENED) \
672 need_flush = 1; \
673 else \
674 new_th->machine.copyio_state = WINDOWS_DIRTY; \
675 if (new_th->machine.physwindow_pte) { \
2d21ac55
A		 676 pmap_store_pte((current_cpu_datap()->cpu_physwindow_ptep), \
677 new_th->machine.physwindow_pte); \
0c530ab8
A		 678 if (need_flush == 0) \
679 invlpg((uintptr_t)current_cpu_datap()->cpu_physwindow_base);\
680 } \
681 if (need_flush) \
682 flush_tlb(); \
1c79356b
A		 683}
684
685#define PMAP_SWITCH_USER(th, new_map, my_cpu) { \
686 spl_t spl; \
687 \
0c530ab8 688 spl = splhigh(); \
1c79356b
A		 689 PMAP_DEACTIVATE_MAP(th->map, my_cpu); \
690 th->map = new_map; \
691 PMAP_ACTIVATE_MAP(th->map, my_cpu); \
692 splx(spl); \
0c530ab8 693 inval_copy_windows(th); \
1c79356b
A		 694}
695
0c530ab8
A		 696/*
697 * Marking the current cpu's cr3 inactive is achieved by setting its lsb.
698 * Marking the current cpu's cr3 active once more involves clearng this bit.
699 * Note that valid page tables are page-aligned and so the bottom 12 bits
700 * are noramlly zero.
701 * We can only mark the current cpu active/inactive but we can test any cpu.
702 */
703#define CPU_CR3_MARK_INACTIVE() \
704 current_cpu_datap()->cpu_active_cr3 |= 1
705
706#define CPU_CR3_MARK_ACTIVE() \
707 current_cpu_datap()->cpu_active_cr3 &= ~1
708
709#define CPU_CR3_IS_ACTIVE(cpu) \
710 ((cpu_datap(cpu)->cpu_active_cr3 & 1) == 0)
711
2d21ac55
A		 712#define CPU_GET_ACTIVE_CR3(cpu) \
713 (cpu_datap(cpu)->cpu_active_cr3 & ~1)
0c530ab8 714
1c79356b
A		 715#define MARK_CPU_IDLE(my_cpu) { \
716 /* \
717 * Mark this cpu idle, and remove it from the active set, \
718 * since it is not actively using any pmap. Signal_cpus \
719 * will notice that it is idle, and avoid signaling it, \
720 * but will queue the update request for when the cpu \
721 * becomes active. \
722 */ \
723 int s = splhigh(); \
0c530ab8
A		 724 if (!cpu_mode_is64bit() || no_shared_cr3) \
725 process_pmap_updates(); \
726 else \
727 pmap_load_kernel_cr3(); \
728 CPU_CR3_MARK_INACTIVE(); \
729 __asm__ volatile("mfence"); \
1c79356b 730 splx(s); \
1c79356b
A		 731}
732
0c530ab8 733#define MARK_CPU_ACTIVE(my_cpu) { \
1c79356b
A		 734 \
735 int s = splhigh(); \
736 /* \
737 * If a kernel_pmap update was requested while this cpu \
738 * was idle, process it as if we got the interrupt. \
739 * Before doing so, remove this cpu from the idle set. \
740 * Since we do not grab any pmap locks while we flush \
741 * our TLB, another cpu may start an update operation \
742 * before we finish. Removing this cpu from the idle \
743 * set assures that we will receive another update \
744 * interrupt if this happens. \
745 */ \
0c530ab8
A		 746 CPU_CR3_MARK_ACTIVE(); \
747 __asm__ volatile("mfence"); \
55e303ae 748 \
0c530ab8
A		 749 if (current_cpu_datap()->cpu_tlb_invalid) \
750 process_pmap_updates(); \
1c79356b 751 splx(s); \
1c79356b
A		 752}
753
1c79356b
A		 754#define PMAP_CONTEXT(pmap, thread)
755
756#define pmap_kernel_va(VA) \
0c530ab8
A		 757 ((((vm_offset_t) (VA)) >= vm_min_kernel_address) && \
758 (((vm_offset_t) (VA)) <= vm_max_kernel_address))
759
1c79356b
A		 760
761#define pmap_resident_count(pmap) ((pmap)->stats.resident_count)
2d21ac55 762#define pmap_resident_max(pmap) ((pmap)->stats.resident_max)
1c79356b
A		 763#define pmap_copy(dst_pmap,src_pmap,dst_addr,len,src_addr)
764#define pmap_attribute(pmap,addr,size,attr,value) \
765 (KERN_INVALID_ADDRESS)
9bccf70c
A		 766#define pmap_attribute_cache_sync(addr,size,attr,value) \
767 (KERN_INVALID_ADDRESS)
765c9de3 768
2d21ac55
A		 769#define MACHINE_PMAP_IS_EMPTY 1
770extern boolean_t pmap_is_empty(pmap_t pmap,
771 vm_map_offset_t start,
772 vm_map_offset_t end);
773
1c79356b
A		 774#endif /* ASSEMBLER */
775
0c530ab8 776
1c79356b 777#endif /* _PMAP_MACHINE_ */
0c530ab8
A		 778
779
780#endif /* KERNEL_PRIVATE */
mirror of XNU