3 * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
5 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
7 * This file contains Original Code and/or Modifications of Original Code
8 * as defined in and that are subject to the Apple Public Source License
9 * Version 2.0 (the 'License'). You may not use this file except in
10 * compliance with the License. The rights granted to you under the License
11 * may not be used to create, or enable the creation or redistribution of,
12 * unlawful or unlicensed copies of an Apple operating system, or to
13 * circumvent, violate, or enable the circumvention or violation of, any
14 * terms of an Apple operating system software license agreement.
16 * Please obtain a copy of the License at
17 * http://www.opensource.apple.com/apsl/ and read it before using this file.
19 * The Original Code and all software distributed under the License are
20 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
21 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
22 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
23 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
24 * Please see the License for the specific language governing rights and
25 * limitations under the License.
27 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
33 * Mach Operating System
34 * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University
35 * All Rights Reserved.
37 * Permission to use, copy, modify and distribute this software and its
38 * documentation is hereby granted, provided that both the copyright
39 * notice and this permission notice appear in all copies of the
40 * software, derivative works or modified versions, and any portions
41 * thereof, and that both notices appear in supporting documentation.
43 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
44 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
45 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
47 * Carnegie Mellon requests users of this software to return to
49 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
50 * School of Computer Science
51 * Carnegie Mellon University
52 * Pittsburgh PA 15213-3890
54 * any improvements or extensions that they make and grant Carnegie Mellon
55 * the rights to redistribute these changes.
62 * Author: Avadis Tevanian, Jr., Michael Wayne Young
63 * (These guys wrote the Vax version)
65 * Physical Map management code for Intel i386, i486, and i860.
67 * Manages physical address maps.
69 * In addition to hardware address maps, this
70 * module is called upon to provide software-use-only
71 * maps which may or may not be stored in the same
72 * form as hardware maps. These pseudo-maps are
73 * used to store intermediate results from copy
74 * operations to and from address spaces.
76 * Since the information managed by this module is
77 * also stored by the logical address mapping module,
78 * this module may throw away valid virtual-to-physical
79 * mappings at almost any time. However, invalidations
80 * of virtual-to-physical mappings must be done as
83 * In order to cope with hardware architectures which
84 * make virtual-to-physical map invalidates expensive,
85 * this module may delay invalidate or reduced protection
86 * operations until such time as they are actually
87 * necessary. This module is given full information as
88 * to which processors are currently using which maps,
89 * and to when physical maps must be made correct.
94 #include <mach_ldebug.h>
96 #include <libkern/OSAtomic.h>
98 #include <mach/machine/vm_types.h>
100 #include <mach/boolean.h>
101 #include <kern/thread.h>
102 #include <kern/zalloc.h>
103 #include <kern/queue.h>
105 #include <kern/lock.h>
106 #include <kern/kalloc.h>
107 #include <kern/spl.h>
110 #include <vm/vm_map.h>
111 #include <vm/vm_kern.h>
112 #include <mach/vm_param.h>
113 #include <mach/vm_prot.h>
114 #include <vm/vm_object.h>
115 #include <vm/vm_page.h>
117 #include <mach/machine/vm_param.h>
118 #include <machine/thread.h>
120 #include <kern/misc_protos.h> /* prototyping */
121 #include <i386/misc_protos.h>
122 #include <x86_64/lowglobals.h>
124 #include <i386/cpuid.h>
125 #include <i386/cpu_data.h>
126 #include <i386/cpu_number.h>
127 #include <i386/machine_cpu.h>
128 #include <i386/seg.h>
129 #include <i386/serial_io.h>
130 #include <i386/cpu_capabilities.h>
131 #include <i386/machine_routines.h>
132 #include <i386/proc_reg.h>
133 #include <i386/tsc.h>
134 #include <i386/pmap_internal.h>
137 #include <ddb/db_command.h>
138 #include <ddb/db_output.h>
139 #include <ddb/db_sym.h>
140 #include <ddb/db_print.h>
141 #endif /* MACH_KDB */
143 #include <vm/vm_protos.h>
146 #include <i386/mp_desc.h>
153 #define POSTCODE_DELAY 1
154 #include <i386/postcode.h>
155 #endif /* IWANTTODEBUG */
157 boolean_t pmap_trace
= FALSE
;
160 #define DBG(x...) kprintf("DBG: " x)
165 boolean_t no_shared_cr3
= DEBUG
; /* TRUE for DEBUG by default */
168 * Forward declarations for internal functions.
172 void phys_attribute_clear(
176 int phys_attribute_test(
180 void phys_attribute_set(
184 void pmap_set_reference(
187 boolean_t
phys_page_exists(
191 int nx_enabled
= 1; /* enable no-execute protection */
192 int allow_data_exec
= VM_ABI_32
; /* 32-bit apps may execute data by default, 64-bit apps may not */
193 int allow_stack_exec
= 0; /* No apps may execute from the stack by default */
195 const boolean_t cpu_64bit
= TRUE
; /* Mais oui! */
197 uint64_t max_preemption_latency_tsc
= 0;
199 pv_hashed_entry_t
*pv_hash_table
; /* hash lists */
201 uint32_t npvhash
= 0;
203 pv_hashed_entry_t pv_hashed_free_list
= PV_HASHED_ENTRY_NULL
;
204 pv_hashed_entry_t pv_hashed_kern_free_list
= PV_HASHED_ENTRY_NULL
;
205 decl_simple_lock_data(,pv_hashed_free_list_lock
)
206 decl_simple_lock_data(,pv_hashed_kern_free_list_lock
)
207 decl_simple_lock_data(,pv_hash_table_lock
)
209 int pv_hashed_free_count
= 0;
210 int pv_hashed_kern_free_count
= 0;
213 zone_t pv_hashed_list_zone
; /* zone of pv_hashed_entry structures */
215 static zone_t pdpt_zone
;
218 * Each entry in the pv_head_table is locked by a bit in the
219 * pv_lock_table. The lock bits are accessed by the physical
220 * address of the page they lock.
223 char *pv_lock_table
; /* pointer to array of bits */
226 char *pv_hash_lock_table
;
230 * First and last physical addresses that we maintain any information
231 * for. Initialized to zero so that pmap operations done before
232 * pmap_init won't touch any non-existent structures.
234 boolean_t pmap_initialized
= FALSE
;/* Has pmap_init completed? */
236 static struct vm_object kptobj_object_store
;
237 static struct vm_object kpml4obj_object_store
;
238 static struct vm_object kpdptobj_object_store
;
241 * Array of physical page attributes for managed pages.
242 * One byte per physical page.
244 char *pmap_phys_attributes
;
245 unsigned int last_managed_page
= 0;
246 uint64_t pde_mapped_size
= PDE_MAPPED_SIZE
;
248 unsigned pmap_memory_region_count
;
249 unsigned pmap_memory_region_current
;
251 pmap_memory_region_t pmap_memory_regions
[PMAP_MEMORY_REGIONS_SIZE
];
254 * Other useful macros.
256 #define current_pmap() (vm_map_pmap(current_thread()->map))
258 struct pmap kernel_pmap_store
;
261 pd_entry_t high_shared_pde
;
262 pd_entry_t commpage64_pde
;
264 struct zone
*pmap_zone
; /* zone of pmap structures */
266 unsigned int inuse_ptepages_count
= 0;
268 addr64_t kernel64_cr3
;
271 * Pmap cache. Cache is threaded through ref_count field of pmap.
272 * Max will eventually be constant -- variable for experimentation.
274 int pmap_cache_max
= 32;
275 int pmap_alloc_chunk
= 8;
276 pmap_t pmap_cache_list
;
277 int pmap_cache_count
;
278 decl_simple_lock_data(,pmap_cache_lock
)
284 pt_entry_t
*DMAP1
, *DMAP2
;
288 * for legacy, returns the address of the pde entry.
289 * for 64 bit, causes the pdpt page containing the pde entry to be mapped,
290 * then returns the mapped address of the pde entry in that page
293 pmap_pde(pmap_t m
, vm_map_offset_t v
)
299 if (m
== kernel_pmap
)
300 pde
= (&((m
)->dirbase
[(vm_offset_t
)(v
) >> PDESHIFT
]));
303 pde
= pmap64_pde(m
, v
);
309 * the single pml4 page per pmap is allocated at pmap create time and exists
310 * for the duration of the pmap. we allocate this page in kernel vm.
311 * this returns the address of the requested pml4 entry in the top level page.
315 pmap64_pml4(pmap_t pmap
, vm_map_offset_t vaddr
)
317 return &pmap
->pm_pml4
[(vaddr
>> PML4SHIFT
) & (NPML4PG
-1)];
321 * maps in the pml4 page, if any, containing the pdpt entry requested
322 * and returns the address of the pdpt entry in that mapped page
325 pmap64_pdpt(pmap_t pmap
, vm_map_offset_t vaddr
)
331 if ((vaddr
> 0x00007FFFFFFFFFFFULL
) &&
332 (vaddr
< 0xFFFF800000000000ULL
)) {
336 pml4
= pmap64_pml4(pmap
, vaddr
);
337 if (pml4
&& ((*pml4
& INTEL_PTE_VALID
))) {
338 newpf
= *pml4
& PG_FRAME
;
339 return &((pdpt_entry_t
*) PHYSMAP_PTOV(newpf
))
340 [(vaddr
>> PDPTSHIFT
) & (NPDPTPG
-1)];
345 * maps in the pdpt page, if any, containing the pde entry requested
346 * and returns the address of the pde entry in that mapped page
349 pmap64_pde(pmap_t pmap
, vm_map_offset_t vaddr
)
355 if ((vaddr
> 0x00007FFFFFFFFFFFULL
) &&
356 (vaddr
< 0xFFFF800000000000ULL
)) {
360 pdpt
= pmap64_pdpt(pmap
, vaddr
);
362 if (pdpt
&& ((*pdpt
& INTEL_PTE_VALID
))) {
363 newpf
= *pdpt
& PG_FRAME
;
364 return &((pd_entry_t
*) PHYSMAP_PTOV(newpf
))
365 [(vaddr
>> PDSHIFT
) & (NPDPG
-1)];
371 * return address of mapped pte for vaddr va in pmap pmap.
373 * physically maps the pde page, if any, containing the pte in and returns
374 * the address of the pte in that mapped page
376 * In case the pde maps a superpage, return the pde, which, in this case
377 * is the actual page table entry.
380 pmap_pte(pmap_t pmap
, vm_map_offset_t vaddr
)
386 pde
= pmap_pde(pmap
, vaddr
);
388 if (pde
&& ((*pde
& INTEL_PTE_VALID
))) {
389 if (*pde
& INTEL_PTE_PS
)
391 newpf
= *pde
& PG_FRAME
;
392 return &((pt_entry_t
*)PHYSMAP_PTOV(newpf
))
393 [i386_btop(vaddr
) & (ppnum_t
)(NPTEPG
-1)];
399 * Map memory at initialization. The physical addresses being
400 * mapped are not managed and are never unmapped.
402 * For now, VM is already on, we only need to map the
408 vm_map_offset_t start_addr
,
409 vm_map_offset_t end_addr
,
416 while (start_addr
< end_addr
) {
417 pmap_enter(kernel_pmap
, (vm_map_offset_t
)virt
,
418 (ppnum_t
) i386_btop(start_addr
), prot
, flags
, FALSE
);
426 * Back-door routine for mapping kernel VM at initialization.
427 * Useful for mapping memory outside the range
428 * Sets no-cache, A, D.
429 * Otherwise like pmap_map.
434 vm_map_offset_t start_addr
,
435 vm_map_offset_t end_addr
,
443 template = pa_to_pte(start_addr
)
449 if (flags
& (VM_MEM_NOT_CACHEABLE
| VM_WIMG_USE_DEFAULT
)) {
450 template |= INTEL_PTE_NCACHE
;
451 if (!(flags
& (VM_MEM_GUARDED
| VM_WIMG_USE_DEFAULT
)))
452 template |= INTEL_PTE_PTA
;
454 if (prot
& VM_PROT_WRITE
)
455 template |= INTEL_PTE_WRITE
;
458 while (start_addr
< end_addr
) {
460 pte
= pmap_pte(kernel_pmap
, (vm_map_offset_t
)virt
);
461 if (pte
== PT_ENTRY_NULL
) {
462 panic("pmap_map_bd: Invalid kernel address\n");
464 pmap_store_pte(pte
, template);
466 pte_increment_pa(template);
468 start_addr
+= PAGE_SIZE
;
476 extern char *first_avail
;
477 extern vm_offset_t virtual_avail
, virtual_end
;
478 extern pmap_paddr_t avail_start
, avail_end
;
479 extern vm_offset_t sHIB
;
480 extern vm_offset_t eHIB
;
481 extern vm_offset_t stext
;
482 extern vm_offset_t etext
;
483 extern vm_offset_t sdata
;
489 * Here early in the life of a processor (from cpu_mode_init()).
490 * Ensure global page feature is disabled.
492 set_cr4(get_cr4() &~ CR4_PGE
);
495 * Initialize the per-cpu, TLB-related fields.
497 current_cpu_datap()->cpu_kernel_cr3
= kernel_pmap
->pm_cr3
;
498 current_cpu_datap()->cpu_active_cr3
= kernel_pmap
->pm_cr3
;
499 current_cpu_datap()->cpu_tlb_invalid
= FALSE
;
505 * Bootstrap the system enough to run with virtual memory.
506 * Map the kernel's code and data, and allocate the system page table.
507 * Called with mapping OFF. Page_size must already be set.
512 __unused vm_offset_t load_start
,
513 __unused boolean_t IA32e
)
515 #if NCOPY_WINDOWS > 0
522 vm_last_addr
= VM_MAX_KERNEL_ADDRESS
; /* Set the highest address
525 * The kernel's pmap is statically allocated so we don't
526 * have to use pmap_create, which is unlikely to work
527 * correctly at this part of the boot sequence.
530 kernel_pmap
= &kernel_pmap_store
;
531 kernel_pmap
->ref_count
= 1;
532 kernel_pmap
->nx_enabled
= FALSE
;
533 kernel_pmap
->pm_task_map
= TASK_MAP_64BIT
;
534 kernel_pmap
->pm_obj
= (vm_object_t
) NULL
;
535 kernel_pmap
->dirbase
= (pd_entry_t
*)((uintptr_t)IdlePTD
);
536 kernel_pmap
->pm_pdpt
= (pd_entry_t
*) ((uintptr_t)IdlePDPT
);
537 kernel_pmap
->pm_pml4
= IdlePML4
;
538 kernel_pmap
->pm_cr3
= (uintptr_t)ID_MAP_VTOP(IdlePML4
);
541 current_cpu_datap()->cpu_kernel_cr3
= (addr64_t
) kernel_pmap
->pm_cr3
;
544 OSAddAtomic(NKPT
, &inuse_ptepages_count
);
546 virtual_avail
= (vm_offset_t
)(VM_MIN_KERNEL_ADDRESS
) + (vm_offset_t
)first_avail
;
547 virtual_end
= (vm_offset_t
)(VM_MAX_KERNEL_ADDRESS
);
549 #if NCOPY_WINDOWS > 0
551 * Reserve some special page table entries/VA space for temporary
554 #define SYSMAP(c, p, v, n) \
555 v = (c)va; va += ((n)*INTEL_PGBYTES);
559 for (i
=0; i
<PMAP_NWINDOWS
; i
++) {
561 kprintf("trying to do SYSMAP idx %d %p\n", i
,
562 current_cpu_datap());
563 kprintf("cpu_pmap %p\n", current_cpu_datap()->cpu_pmap
);
564 kprintf("mapwindow %p\n", current_cpu_datap()->cpu_pmap
->mapwindow
);
565 kprintf("two stuff %p %p\n",
566 (void *)(current_cpu_datap()->cpu_pmap
->mapwindow
[i
].prv_CMAP
),
567 (void *)(current_cpu_datap()->cpu_pmap
->mapwindow
[i
].prv_CADDR
));
570 (current_cpu_datap()->cpu_pmap
->mapwindow
[i
].prv_CMAP
),
571 (current_cpu_datap()->cpu_pmap
->mapwindow
[i
].prv_CADDR
),
573 current_cpu_datap()->cpu_pmap
->mapwindow
[i
].prv_CMAP
=
574 &(current_cpu_datap()->cpu_pmap
->mapwindow
[i
].prv_CMAP_store
);
575 *current_cpu_datap()->cpu_pmap
->mapwindow
[i
].prv_CMAP
= 0;
578 /* DMAP user for debugger */
579 SYSMAP(caddr_t
, DMAP1
, DADDR1
, 1);
580 SYSMAP(caddr_t
, DMAP2
, DADDR2
, 1); /* XXX temporary - can remove */
585 if (PE_parse_boot_argn("npvhash", &npvhash
, sizeof (npvhash
))) {
586 if (0 != ((npvhash
+ 1) & npvhash
)) {
587 kprintf("invalid hash %d, must be ((2^N)-1), "
588 "using default %d\n", npvhash
, NPVHASH
);
595 printf("npvhash=%d\n", npvhash
);
597 simple_lock_init(&kernel_pmap
->lock
, 0);
598 simple_lock_init(&pv_hashed_free_list_lock
, 0);
599 simple_lock_init(&pv_hashed_kern_free_list_lock
, 0);
600 simple_lock_init(&pv_hash_table_lock
,0);
604 kprintf("Kernel virtual space from 0x%lx to 0x%lx.\n",
605 (long)KERNEL_BASE
, (long)virtual_end
);
606 kprintf("Available physical space from 0x%llx to 0x%llx\n",
607 avail_start
, avail_end
);
610 * The -no_shared_cr3 boot-arg is a debugging feature (set by default
611 * in the DEBUG kernel) to force the kernel to switch to its own map
612 * (and cr3) when control is in kernelspace. The kernel's map does not
613 * include (i.e. share) userspace so wild references will cause
614 * a panic. Only copyin and copyout are exempt from this.
616 (void) PE_parse_boot_argn("-no_shared_cr3",
617 &no_shared_cr3
, sizeof (no_shared_cr3
));
619 kprintf("Kernel not sharing user map\n");
622 if (PE_parse_boot_argn("-pmap_trace", &pmap_trace
, sizeof (pmap_trace
))) {
623 kprintf("Kernel traces for pmap operations enabled\n");
625 #endif /* PMAP_TRACES */
633 *startp
= virtual_avail
;
638 * Initialize the pmap module.
639 * Called by vm_init, to initialize any structures that the pmap
640 * system needs to map virtual memory.
648 vm_map_offset_t vaddr
;
652 kernel_pmap
->pm_obj_pml4
= &kpml4obj_object_store
;
653 _vm_object_allocate((vm_object_size_t
)NPML4PGS
, &kpml4obj_object_store
);
655 kernel_pmap
->pm_obj_pdpt
= &kpdptobj_object_store
;
656 _vm_object_allocate((vm_object_size_t
)NPDPTPGS
, &kpdptobj_object_store
);
658 kernel_pmap
->pm_obj
= &kptobj_object_store
;
659 _vm_object_allocate((vm_object_size_t
)NPDEPGS
, &kptobj_object_store
);
662 * Allocate memory for the pv_head_table and its lock bits,
663 * the modify bit array, and the pte_page table.
667 * zero bias all these arrays now instead of off avail_start
668 * so we cover all memory
671 npages
= i386_btop(avail_end
);
672 s
= (vm_size_t
) (sizeof(struct pv_rooted_entry
) * npages
673 + (sizeof (struct pv_hashed_entry_t
*) * (npvhash
+1))
674 + pv_lock_table_size(npages
)
675 + pv_hash_lock_table_size((npvhash
+1))
679 if (kernel_memory_allocate(kernel_map
, &addr
, s
, 0,
680 KMA_KOBJECT
| KMA_PERMANENT
)
684 memset((char *)addr
, 0, s
);
687 if (0 == npvhash
) panic("npvhash not initialized");
691 * Allocate the structures first to preserve word-alignment.
693 pv_head_table
= (pv_rooted_entry_t
) addr
;
694 addr
= (vm_offset_t
) (pv_head_table
+ npages
);
696 pv_hash_table
= (pv_hashed_entry_t
*)addr
;
697 addr
= (vm_offset_t
) (pv_hash_table
+ (npvhash
+ 1));
699 pv_lock_table
= (char *) addr
;
700 addr
= (vm_offset_t
) (pv_lock_table
+ pv_lock_table_size(npages
));
702 pv_hash_lock_table
= (char *) addr
;
703 addr
= (vm_offset_t
) (pv_hash_lock_table
+ pv_hash_lock_table_size((npvhash
+1)));
705 pmap_phys_attributes
= (char *) addr
;
707 ppnum_t last_pn
= i386_btop(avail_end
);
709 pmap_memory_region_t
*pmptr
= pmap_memory_regions
;
710 for (i
= 0; i
< pmap_memory_region_count
; i
++, pmptr
++) {
711 if (pmptr
->type
!= kEfiConventionalMemory
)
714 for (pn
= pmptr
->base
; pn
<= pmptr
->end
; pn
++) {
716 pmap_phys_attributes
[pn
] |= PHYS_MANAGED
;
717 if (pn
> last_managed_page
)
718 last_managed_page
= pn
;
724 * Create the zone of physical maps,
725 * and of the physical-to-virtual entries.
727 s
= (vm_size_t
) sizeof(struct pmap
);
728 pmap_zone
= zinit(s
, 400*s
, 4096, "pmap"); /* XXX */
729 s
= (vm_size_t
) sizeof(struct pv_hashed_entry
);
730 pv_hashed_list_zone
= zinit(s
, 10000*s
, 4096, "pv_list"); /* XXX */
732 pdpt_zone
= zinit(s
, 400*s
, 4096, "pdpt"); /* XXX */
735 /* create pv entries for kernel pages mapped by low level
736 startup code. these have to exist so we can pmap_remove()
737 e.g. kext pages from the middle of our addr space */
739 vaddr
= (vm_map_offset_t
) VM_MIN_KERNEL_ADDRESS
;
740 for (ppn
= 0; ppn
< i386_btop(avail_start
); ppn
++) {
741 pv_rooted_entry_t pv_e
;
743 pv_e
= pai_to_pvh(ppn
);
746 pv_e
->pmap
= kernel_pmap
;
747 queue_init(&pv_e
->qlink
);
749 pmap_initialized
= TRUE
;
752 * Initialize pmap cache.
754 pmap_cache_list
= PMAP_NULL
;
755 pmap_cache_count
= 0;
756 simple_lock_init(&pmap_cache_lock
, 0);
758 max_preemption_latency_tsc
= tmrCvt((uint64_t)MAX_PREEMPTION_LATENCY_NS
, tscFCvtn2t
);
761 * Ensure the kernel's PML4 entry exists for the basement
762 * before this is shared with any user.
764 pmap_expand_pml4(kernel_pmap
, KERNEL_BASEMENT
);
769 * this function is only used for debugging fron the vm layer
775 pv_rooted_entry_t pv_h
;
779 assert(pn
!= vm_page_fictitious_addr
);
781 if (!pmap_initialized
)
784 if (pn
== vm_page_guard_addr
)
787 pai
= ppn_to_pai(pn
);
788 if (!IS_MANAGED_PAGE(pai
))
790 pv_h
= pai_to_pvh(pn
);
791 result
= (pv_h
->pmap
== PMAP_NULL
);
798 vm_map_offset_t va_start
,
799 vm_map_offset_t va_end
)
801 vm_map_offset_t offset
;
804 if (pmap
== PMAP_NULL
) {
809 * Check the resident page count
810 * - if it's zero, the pmap is completely empty.
811 * This short-circuit test prevents a virtual address scan which is
812 * painfully slow for 64-bit spaces.
813 * This assumes the count is correct
814 * .. the debug kernel ought to be checking perhaps by page table walk.
816 if (pmap
->stats
.resident_count
== 0)
819 for (offset
= va_start
;
821 offset
+= PAGE_SIZE_64
) {
822 phys_page
= pmap_find_phys(pmap
, offset
);
824 kprintf("pmap_is_empty(%p,0x%llx,0x%llx): "
825 "page %d at 0x%llx\n",
826 pmap
, va_start
, va_end
, phys_page
, offset
);
836 * Create and return a physical map.
838 * If the size specified for the map
839 * is zero, the map is an actual physical
840 * map, and may be referenced by the
843 * If the size specified is non-zero,
844 * the map will be used in software only, and
845 * is bounded by that size.
857 PMAP_TRACE(PMAP_CODE(PMAP__CREATE
) | DBG_FUNC_START
,
858 (uint32_t) (sz
>>32), (uint32_t) sz
, is_64bit
, 0, 0);
860 size
= (vm_size_t
) sz
;
863 * A software use-only map doesn't even need a map.
870 p
= (pmap_t
) zalloc(pmap_zone
);
872 panic("pmap_create zalloc");
874 /* init counts now since we'll be bumping some */
875 simple_lock_init(&p
->lock
, 0);
876 p
->stats
.resident_count
= 0;
877 p
->stats
.resident_max
= 0;
878 p
->stats
.wired_count
= 0;
881 p
->pm_shared
= FALSE
;
883 p
->pm_task_map
= is_64bit
? TASK_MAP_64BIT
: TASK_MAP_32BIT
;;
885 /* alloc the pml4 page in kernel vm */
886 if (KERN_SUCCESS
!= kmem_alloc_kobject(kernel_map
, (vm_offset_t
*)(&p
->pm_pml4
), PAGE_SIZE
))
887 panic("pmap_create kmem_alloc_kobject pml4");
889 memset((char *)p
->pm_pml4
, 0, PAGE_SIZE
);
890 p
->pm_cr3
= (pmap_paddr_t
)kvtophys((vm_offset_t
)p
->pm_pml4
);
892 OSAddAtomic(1, &inuse_ptepages_count
);
894 /* allocate the vm_objs to hold the pdpt, pde and pte pages */
896 p
->pm_obj_pml4
= vm_object_allocate((vm_object_size_t
)(NPML4PGS
));
897 if (NULL
== p
->pm_obj_pml4
)
898 panic("pmap_create pdpt obj");
900 p
->pm_obj_pdpt
= vm_object_allocate((vm_object_size_t
)(NPDPTPGS
));
901 if (NULL
== p
->pm_obj_pdpt
)
902 panic("pmap_create pdpt obj");
904 p
->pm_obj
= vm_object_allocate((vm_object_size_t
)(NPDEPGS
));
905 if (NULL
== p
->pm_obj
)
906 panic("pmap_create pte obj");
908 /* All pmaps share the kennel's pml4 */
909 pml4
= pmap64_pml4(p
, 0ULL);
910 kpml4
= kernel_pmap
->pm_pml4
;
911 pml4
[KERNEL_PML4_INDEX
] = kpml4
[KERNEL_PML4_INDEX
];
912 pml4
[KERNEL_KEXTS_INDEX
] = kpml4
[KERNEL_KEXTS_INDEX
];
913 pml4
[KERNEL_PHYSMAP_INDEX
] = kpml4
[KERNEL_PHYSMAP_INDEX
];
915 PMAP_TRACE(PMAP_CODE(PMAP__CREATE
) | DBG_FUNC_START
,
916 p
, is_64bit
, 0, 0, 0);
922 * Retire the given physical map from service.
923 * Should only be called if the map contains
936 PMAP_TRACE(PMAP_CODE(PMAP__DESTROY
) | DBG_FUNC_START
,
945 * If some cpu is not using the physical pmap pointer that it
946 * is supposed to be (see set_dirbase), we might be using the
947 * pmap that is being destroyed! Make sure we are
948 * physically on the right pmap:
950 PMAP_UPDATE_TLBS(p
, 0x0ULL
, 0xFFFFFFFFFFFFF000ULL
);
956 PMAP_TRACE(PMAP_CODE(PMAP__DESTROY
) | DBG_FUNC_END
,
958 return; /* still in use */
962 * Free the memory maps, then the
965 int inuse_ptepages
= 0;
968 kmem_free(kernel_map
, (vm_offset_t
)p
->pm_pml4
, PAGE_SIZE
);
970 inuse_ptepages
+= p
->pm_obj_pml4
->resident_page_count
;
971 vm_object_deallocate(p
->pm_obj_pml4
);
973 inuse_ptepages
+= p
->pm_obj_pdpt
->resident_page_count
;
974 vm_object_deallocate(p
->pm_obj_pdpt
);
976 inuse_ptepages
+= p
->pm_obj
->resident_page_count
;
977 vm_object_deallocate(p
->pm_obj
);
979 OSAddAtomic(-inuse_ptepages
, &inuse_ptepages_count
);
983 PMAP_TRACE(PMAP_CODE(PMAP__DESTROY
) | DBG_FUNC_END
,
988 * Add a reference to the specified pmap.
992 pmap_reference(pmap_t p
)
994 if (p
!= PMAP_NULL
) {
1002 * Remove phys addr if mapped in specified map
1006 pmap_remove_some_phys(
1007 __unused pmap_t map
,
1008 __unused ppnum_t pn
)
1011 /* Implement to support working set code */
1021 * Disconnect all mappings for this page and return reference and change status
1022 * in generic format.
1025 unsigned int pmap_disconnect(
1028 pmap_page_protect(pa
, 0); /* disconnect the page */
1029 return (pmap_get_refmod(pa
)); /* return ref/chg status */
1033 * Set the physical protection on the
1034 * specified range of this map as requested.
1035 * Will not increase permissions.
1040 vm_map_offset_t sva
,
1041 vm_map_offset_t eva
,
1045 pt_entry_t
*spte
, *epte
;
1046 vm_map_offset_t lva
;
1047 vm_map_offset_t orig_sva
;
1053 if (map
== PMAP_NULL
)
1056 if (prot
== VM_PROT_NONE
) {
1057 pmap_remove(map
, sva
, eva
);
1060 PMAP_TRACE(PMAP_CODE(PMAP__PROTECT
) | DBG_FUNC_START
,
1062 (uint32_t) (sva
>> 32), (uint32_t) sva
,
1063 (uint32_t) (eva
>> 32), (uint32_t) eva
);
1065 if ((prot
& VM_PROT_EXECUTE
) || !nx_enabled
|| !map
->nx_enabled
)
1074 lva
= (sva
+ pde_mapped_size
) & ~(pde_mapped_size
- 1);
1077 pde
= pmap_pde(map
, sva
);
1078 if (pde
&& (*pde
& INTEL_PTE_VALID
)) {
1079 if (*pde
& INTEL_PTE_PS
) {
1082 epte
= spte
+1; /* excluded */
1084 spte
= pmap_pte(map
, (sva
& ~(pde_mapped_size
- 1)));
1085 spte
= &spte
[ptenum(sva
)];
1086 epte
= &spte
[intel_btop(lva
- sva
)];
1089 for (; spte
< epte
; spte
++) {
1090 if (!(*spte
& INTEL_PTE_VALID
))
1093 if (prot
& VM_PROT_WRITE
)
1094 pmap_update_pte(spte
, *spte
,
1095 *spte
| INTEL_PTE_WRITE
);
1097 pmap_update_pte(spte
, *spte
,
1098 *spte
& ~INTEL_PTE_WRITE
);
1101 pmap_update_pte(spte
, *spte
,
1102 *spte
| INTEL_PTE_NX
);
1104 pmap_update_pte(spte
, *spte
,
1105 *spte
& ~INTEL_PTE_NX
);
1113 PMAP_UPDATE_TLBS(map
, orig_sva
, eva
);
1117 PMAP_TRACE(PMAP_CODE(PMAP__PROTECT
) | DBG_FUNC_END
,
1122 /* Map a (possibly) autogenned block */
1131 __unused
unsigned int flags
)
1136 if (attr
& VM_MEM_SUPERPAGE
)
1137 cur_page_size
= SUPERPAGE_SIZE
;
1139 cur_page_size
= PAGE_SIZE
;
1141 for (page
= 0; page
< size
; page
+=cur_page_size
/PAGE_SIZE
) {
1142 pmap_enter(pmap
, va
, pa
, prot
, attr
, TRUE
);
1143 va
+= cur_page_size
;
1144 pa
+=cur_page_size
/PAGE_SIZE
;
1149 * Routine: pmap_change_wiring
1150 * Function: Change the wiring attribute for a map/virtual-address
1152 * In/out conditions:
1153 * The mapping must already exist in the pmap.
1158 vm_map_offset_t vaddr
,
1165 if ((pte
= pmap_pte(map
, vaddr
)) == PT_ENTRY_NULL
)
1166 panic("pmap_change_wiring: pte missing");
1168 if (wired
&& !iswired(*pte
)) {
1170 * wiring down mapping
1172 OSAddAtomic(+1, &map
->stats
.wired_count
);
1173 pmap_update_pte(pte
, *pte
, (*pte
| INTEL_PTE_WIRED
));
1175 else if (!wired
&& iswired(*pte
)) {
1179 assert(map
->stats
.wired_count
>= 1);
1180 OSAddAtomic(-1, &map
->stats
.wired_count
);
1181 pmap_update_pte(pte
, *pte
, (*pte
& ~INTEL_PTE_WIRED
));
1190 vm_map_offset_t vaddr
)
1196 pml4_entry_t
*pml4p
;
1198 DBG("pmap_expand_pml4(%p,%p)\n", map
, (void *)vaddr
);
1201 * Allocate a VM page for the pml4 page
1203 while ((m
= vm_page_grab()) == VM_PAGE_NULL
)
1207 * put the page into the pmap's obj list so it
1208 * can be found later.
1212 i
= pml4idx(map
, vaddr
);
1219 vm_page_lockspin_queues();
1221 vm_page_unlock_queues();
1223 OSAddAtomic(1, &inuse_ptepages_count
);
1225 /* Take the oject lock (mutex) before the PMAP_LOCK (spinlock) */
1226 vm_object_lock(map
->pm_obj_pml4
);
1230 * See if someone else expanded us first
1232 if (pmap64_pdpt(map
, vaddr
) != PDPT_ENTRY_NULL
) {
1234 vm_object_unlock(map
->pm_obj_pml4
);
1238 OSAddAtomic(-1, &inuse_ptepages_count
);
1243 if (0 != vm_page_lookup(map
->pm_obj_pml4
, (vm_object_offset_t
)i
)) {
1244 panic("pmap_expand_pml4: obj not empty, pmap %p pm_obj %p vaddr 0x%llx i 0x%llx\n",
1245 map
, map
->pm_obj_pml4
, vaddr
, i
);
1248 vm_page_insert(m
, map
->pm_obj_pml4
, (vm_object_offset_t
)i
);
1249 vm_object_unlock(map
->pm_obj_pml4
);
1252 * Set the page directory entry for this page table.
1254 pml4p
= pmap64_pml4(map
, vaddr
); /* refetch under lock */
1256 pmap_store_pte(pml4p
, pa_to_pte(pa
)
1269 vm_map_offset_t vaddr
)
1275 pdpt_entry_t
*pdptp
;
1277 DBG("pmap_expand_pdpt(%p,%p)\n", map
, (void *)vaddr
);
1279 while ((pdptp
= pmap64_pdpt(map
, vaddr
)) == PDPT_ENTRY_NULL
) {
1280 pmap_expand_pml4(map
, vaddr
);
1284 * Allocate a VM page for the pdpt page
1286 while ((m
= vm_page_grab()) == VM_PAGE_NULL
)
1290 * put the page into the pmap's obj list so it
1291 * can be found later.
1295 i
= pdptidx(map
, vaddr
);
1302 vm_page_lockspin_queues();
1304 vm_page_unlock_queues();
1306 OSAddAtomic(1, &inuse_ptepages_count
);
1308 /* Take the oject lock (mutex) before the PMAP_LOCK (spinlock) */
1309 vm_object_lock(map
->pm_obj_pdpt
);
1313 * See if someone else expanded us first
1315 if (pmap64_pde(map
, vaddr
) != PD_ENTRY_NULL
) {
1317 vm_object_unlock(map
->pm_obj_pdpt
);
1321 OSAddAtomic(-1, &inuse_ptepages_count
);
1326 if (0 != vm_page_lookup(map
->pm_obj_pdpt
, (vm_object_offset_t
)i
)) {
1327 panic("pmap_expand_pdpt: obj not empty, pmap %p pm_obj %p vaddr 0x%llx i 0x%llx\n",
1328 map
, map
->pm_obj_pdpt
, vaddr
, i
);
1331 vm_page_insert(m
, map
->pm_obj_pdpt
, (vm_object_offset_t
)i
);
1332 vm_object_unlock(map
->pm_obj_pdpt
);
1335 * Set the page directory entry for this page table.
1337 pdptp
= pmap64_pdpt(map
, vaddr
); /* refetch under lock */
1339 pmap_store_pte(pdptp
, pa_to_pte(pa
)
1353 * Routine: pmap_expand
1355 * Expands a pmap to be able to map the specified virtual address.
1357 * Allocates new virtual memory for the P0 or P1 portion of the
1358 * pmap, then re-maps the physical pages that were in the old
1359 * pmap to be in the new pmap.
1361 * Must be called with the pmap system and the pmap unlocked,
1362 * since these must be unlocked to use vm_allocate or vm_deallocate.
1363 * Thus it must be called in a loop that checks whether the map
1364 * has been expanded enough.
1365 * (We won't loop forever, since page tables aren't shrunk.)
1370 vm_map_offset_t vaddr
)
1373 register vm_page_t m
;
1374 register pmap_paddr_t pa
;
1380 * For the kernel, the virtual address must be in or above the basement
1381 * which is for kexts and is in the 512GB immediately below the kernel..
1382 * XXX - should use VM_MIN_KERNEL_AND_KEXT_ADDRESS not KERNEL_BASEMENT
1384 if (map
== kernel_pmap
&&
1385 !(vaddr
>= KERNEL_BASEMENT
&& vaddr
<= VM_MAX_KERNEL_ADDRESS
))
1386 panic("pmap_expand: bad vaddr 0x%llx for kernel pmap", vaddr
);
1389 while ((pdp
= pmap64_pde(map
, vaddr
)) == PD_ENTRY_NULL
) {
1390 /* need room for another pde entry */
1391 pmap_expand_pdpt(map
, vaddr
);
1395 * Allocate a VM page for the pde entries.
1397 while ((m
= vm_page_grab()) == VM_PAGE_NULL
)
1401 * put the page into the pmap's obj list so it
1402 * can be found later.
1406 i
= pdeidx(map
, vaddr
);
1413 vm_page_lockspin_queues();
1415 vm_page_unlock_queues();
1417 OSAddAtomic(1, &inuse_ptepages_count
);
1419 /* Take the oject lock (mutex) before the PMAP_LOCK (spinlock) */
1420 vm_object_lock(map
->pm_obj
);
1425 * See if someone else expanded us first
1427 if (pmap_pte(map
, vaddr
) != PT_ENTRY_NULL
) {
1429 vm_object_unlock(map
->pm_obj
);
1433 OSAddAtomic(-1, &inuse_ptepages_count
);
1438 if (0 != vm_page_lookup(map
->pm_obj
, (vm_object_offset_t
)i
)) {
1439 panic("pmap_expand: obj not empty, pmap 0x%x pm_obj 0x%x vaddr 0x%llx i 0x%llx\n",
1440 map
, map
->pm_obj
, vaddr
, i
);
1443 vm_page_insert(m
, map
->pm_obj
, (vm_object_offset_t
)i
);
1444 vm_object_unlock(map
->pm_obj
);
1447 * Set the page directory entry for this page table.
1449 pdp
= pmap_pde(map
, vaddr
);
1450 pmap_store_pte(pdp
, pa_to_pte(pa
)
1460 /* On K64 machines with more than 32GB of memory, pmap_steal_memory
1461 * will allocate past the 1GB of pre-expanded virtual kernel area. This
1462 * function allocates all the page tables using memory from the same pool
1463 * that pmap_steal_memory uses, rather than calling vm_page_grab (which
1464 * isn't available yet). */
1466 pmap_pre_expand(pmap_t pmap
, vm_map_offset_t vaddr
) {
1472 if(pmap64_pdpt(pmap
, vaddr
) == PDPT_ENTRY_NULL
) {
1473 if (!pmap_next_page_k64(&pn
))
1474 panic("pmap_pre_expand");
1478 pte
= pmap64_pml4(pmap
, vaddr
);
1480 pmap_store_pte(pte
, pa_to_pte(i386_ptob(pn
))
1486 if(pmap64_pde(pmap
, vaddr
) == PD_ENTRY_NULL
) {
1487 if (!pmap_next_page_k64(&pn
))
1488 panic("pmap_pre_expand");
1492 pte
= pmap64_pdpt(pmap
, vaddr
);
1494 pmap_store_pte(pte
, pa_to_pte(i386_ptob(pn
))
1500 if(pmap_pte(pmap
, vaddr
) == PT_ENTRY_NULL
) {
1501 if (!pmap_next_page_k64(&pn
))
1502 panic("pmap_pre_expand");
1506 pte
= pmap64_pde(pmap
, vaddr
);
1508 pmap_store_pte(pte
, pa_to_pte(i386_ptob(pn
))
1518 * pmap_sync_page_data_phys(ppnum_t pa)
1520 * Invalidates all of the instruction cache on a physical page and
1521 * pushes any dirty data from the data cache for the same physical page
1522 * Not required in i386.
1525 pmap_sync_page_data_phys(__unused ppnum_t pa
)
1531 * pmap_sync_page_attributes_phys(ppnum_t pa)
1533 * Write back and invalidate all cachelines on a physical page.
1536 pmap_sync_page_attributes_phys(ppnum_t pa
)
1538 cache_flush_page_phys(pa
);
1543 #ifdef CURRENTLY_UNUSED_AND_UNTESTED
1549 * Routine: pmap_collect
1551 * Garbage collects the physical map system for
1552 * pages which are no longer used.
1553 * Success need not be guaranteed -- that is, there
1554 * may well be pages which are not referenced, but
1555 * others may be collected.
1557 * Called by the pageout daemon when pages are scarce.
1563 register pt_entry_t
*pdp
, *ptp
;
1570 if (p
== kernel_pmap
)
1574 * Garbage collect map.
1578 for (pdp
= (pt_entry_t
*)p
->dirbase
;
1579 pdp
< (pt_entry_t
*)&p
->dirbase
[(UMAXPTDI
+1)];
1582 if (*pdp
& INTEL_PTE_VALID
) {
1583 if(*pdp
& INTEL_PTE_REF
) {
1584 pmap_store_pte(pdp
, *pdp
& ~INTEL_PTE_REF
);
1588 ptp
= pmap_pte(p
, pdetova(pdp
- (pt_entry_t
*)p
->dirbase
));
1589 eptp
= ptp
+ NPTEPG
;
1592 * If the pte page has any wired mappings, we cannot
1597 register pt_entry_t
*ptep
;
1598 for (ptep
= ptp
; ptep
< eptp
; ptep
++) {
1599 if (iswired(*ptep
)) {
1607 * Remove the virtual addresses mapped by this pte page.
1609 pmap_remove_range(p
,
1610 pdetova(pdp
- (pt_entry_t
*)p
->dirbase
),
1615 * Invalidate the page directory pointer.
1617 pmap_store_pte(pdp
, 0x0);
1622 * And free the pte page itself.
1625 register vm_page_t m
;
1627 vm_object_lock(p
->pm_obj
);
1629 m
= vm_page_lookup(p
->pm_obj
,(vm_object_offset_t
)(pdp
- (pt_entry_t
*)&p
->dirbase
[0]));
1630 if (m
== VM_PAGE_NULL
)
1631 panic("pmap_collect: pte page not in object");
1635 OSAddAtomic(-1, &inuse_ptepages_count
);
1637 vm_object_unlock(p
->pm_obj
);
1646 PMAP_UPDATE_TLBS(p
, 0x0, 0xFFFFFFFFFFFFF000ULL
);
1655 pmap_copy_page(ppnum_t src
, ppnum_t dst
)
1657 bcopy_phys((addr64_t
)i386_ptob(src
),
1658 (addr64_t
)i386_ptob(dst
),
1664 * Routine: pmap_pageable
1666 * Make the specified pages (by pmap, offset)
1667 * pageable (or not) as requested.
1669 * A page which is not pageable may not take
1670 * a fault; therefore, its page table entry
1671 * must remain valid for the duration.
1673 * This routine is merely advisory; pmap_enter
1674 * will specify that these pages are to be wired
1675 * down (or not) as appropriate.
1679 __unused pmap_t pmap
,
1680 __unused vm_map_offset_t start_addr
,
1681 __unused vm_map_offset_t end_addr
,
1682 __unused boolean_t pageable
)
1685 pmap
++; start_addr
++; end_addr
++; pageable
++;
1690 * Clear specified attribute bits.
1693 phys_attribute_clear(
1697 pv_rooted_entry_t pv_h
;
1698 pv_hashed_entry_t pv_e
;
1704 assert(pn
!= vm_page_fictitious_addr
);
1705 if (pn
== vm_page_guard_addr
)
1708 pai
= ppn_to_pai(pn
);
1710 if (!IS_MANAGED_PAGE(pai
)) {
1712 * Not a managed page.
1718 PMAP_TRACE(PMAP_CODE(PMAP__ATTRIBUTE_CLEAR
) | DBG_FUNC_START
,
1721 pv_h
= pai_to_pvh(pai
);
1726 * Walk down PV list, clearing all modify or reference bits.
1727 * We do not have to lock the pv_list because we have
1728 * the entire pmap system locked.
1730 if (pv_h
->pmap
!= PMAP_NULL
) {
1732 * There are some mappings.
1735 pv_e
= (pv_hashed_entry_t
)pv_h
;
1744 * Clear modify and/or reference bits.
1746 pte
= pmap_pte(pmap
, va
);
1747 pmap_update_pte(pte
, *pte
, (*pte
& ~bits
));
1748 /* Ensure all processors using this translation
1749 * invalidate this TLB entry. The invalidation *must*
1750 * follow the PTE update, to ensure that the TLB
1751 * shadow of the 'D' bit (in particular) is
1752 * synchronized with the updated PTE.
1754 PMAP_UPDATE_TLBS(pmap
, va
, va
+ PAGE_SIZE
);
1756 pv_e
= (pv_hashed_entry_t
)queue_next(&pv_e
->qlink
);
1758 } while (pv_e
!= (pv_hashed_entry_t
)pv_h
);
1760 pmap_phys_attributes
[pai
] &= ~bits
;
1764 PMAP_TRACE(PMAP_CODE(PMAP__ATTRIBUTE_CLEAR
) | DBG_FUNC_END
,
1769 * Check specified attribute bits.
1772 phys_attribute_test(
1776 pv_rooted_entry_t pv_h
;
1777 pv_hashed_entry_t pv_e
;
1784 assert(pn
!= vm_page_fictitious_addr
);
1785 if (pn
== vm_page_guard_addr
)
1788 pai
= ppn_to_pai(pn
);
1790 if (!IS_MANAGED_PAGE(pai
)) {
1792 * Not a managed page.
1798 * super fast check... if bits already collected
1799 * no need to take any locks...
1800 * if not set, we need to recheck after taking
1801 * the lock in case they got pulled in while
1802 * we were waiting for the lock
1804 if ((pmap_phys_attributes
[pai
] & bits
) == bits
)
1807 pv_h
= pai_to_pvh(pai
);
1811 attributes
= pmap_phys_attributes
[pai
] & bits
;
1815 * Walk down PV list, checking the mappings until we
1816 * reach the end or we've found the attributes we've asked for
1817 * We do not have to lock the pv_list because we have
1818 * the entire pmap system locked.
1820 if (attributes
!= bits
&&
1821 pv_h
->pmap
!= PMAP_NULL
) {
1823 * There are some mappings.
1825 pv_e
= (pv_hashed_entry_t
)pv_h
;
1832 * first make sure any processor actively
1833 * using this pmap, flushes its TLB state
1835 PMAP_UPDATE_TLBS(pmap
, va
, va
+ PAGE_SIZE
);
1838 * pick up modify and/or reference bits from mapping
1841 pte
= pmap_pte(pmap
, va
);
1842 attributes
|= (int)(*pte
& bits
);
1844 pv_e
= (pv_hashed_entry_t
)queue_next(&pv_e
->qlink
);
1846 } while ((attributes
!= bits
) &&
1847 (pv_e
!= (pv_hashed_entry_t
)pv_h
));
1851 return (attributes
);
1855 * Set specified attribute bits.
1865 assert(pn
!= vm_page_fictitious_addr
);
1866 if (pn
== vm_page_guard_addr
)
1869 pai
= ppn_to_pai(pn
);
1871 if (!IS_MANAGED_PAGE(pai
)) {
1872 /* Not a managed page. */
1877 pmap_phys_attributes
[pai
] |= bits
;
1882 * Set the modify bit on the specified physical page.
1886 pmap_set_modify(ppnum_t pn
)
1888 phys_attribute_set(pn
, PHYS_MODIFIED
);
1892 * Clear the modify bits on the specified physical page.
1896 pmap_clear_modify(ppnum_t pn
)
1898 phys_attribute_clear(pn
, PHYS_MODIFIED
);
1904 * Return whether or not the specified physical page is modified
1905 * by any physical maps.
1909 pmap_is_modified(ppnum_t pn
)
1911 if (phys_attribute_test(pn
, PHYS_MODIFIED
))
1917 * pmap_clear_reference:
1919 * Clear the reference bit on the specified physical page.
1923 pmap_clear_reference(ppnum_t pn
)
1925 phys_attribute_clear(pn
, PHYS_REFERENCED
);
1929 pmap_set_reference(ppnum_t pn
)
1931 phys_attribute_set(pn
, PHYS_REFERENCED
);
1935 * pmap_is_referenced:
1937 * Return whether or not the specified physical page is referenced
1938 * by any physical maps.
1942 pmap_is_referenced(ppnum_t pn
)
1944 if (phys_attribute_test(pn
, PHYS_REFERENCED
))
1950 * pmap_get_refmod(phys)
1951 * returns the referenced and modified bits of the specified
1955 pmap_get_refmod(ppnum_t pn
)
1958 unsigned int retval
= 0;
1960 refmod
= phys_attribute_test(pn
, PHYS_MODIFIED
| PHYS_REFERENCED
);
1962 if (refmod
& PHYS_MODIFIED
)
1963 retval
|= VM_MEM_MODIFIED
;
1964 if (refmod
& PHYS_REFERENCED
)
1965 retval
|= VM_MEM_REFERENCED
;
1971 * pmap_clear_refmod(phys, mask)
1972 * clears the referenced and modified bits as specified by the mask
1973 * of the specified physical page.
1976 pmap_clear_refmod(ppnum_t pn
, unsigned int mask
)
1978 unsigned int x86Mask
;
1980 x86Mask
= ( ((mask
& VM_MEM_MODIFIED
)? PHYS_MODIFIED
: 0)
1981 | ((mask
& VM_MEM_REFERENCED
)? PHYS_REFERENCED
: 0));
1982 phys_attribute_clear(pn
, x86Mask
);
1986 invalidate_icache(__unused vm_offset_t addr
,
1987 __unused
unsigned cnt
,
1994 flush_dcache(__unused vm_offset_t addr
,
1995 __unused
unsigned count
,
2003 * Constrain DTrace copyin/copyout actions
2005 extern kern_return_t
dtrace_copyio_preflight(addr64_t
);
2006 extern kern_return_t
dtrace_copyio_postflight(addr64_t
);
2008 kern_return_t
dtrace_copyio_preflight(__unused addr64_t va
)
2010 thread_t thread
= current_thread();
2012 if (current_map() == kernel_map
)
2013 return KERN_FAILURE
;
2014 else if (get_cr3() != thread
->map
->pmap
->pm_cr3
)
2015 return KERN_FAILURE
;
2016 else if (thread
->machine
.specFlags
& CopyIOActive
)
2017 return KERN_FAILURE
;
2019 return KERN_SUCCESS
;
2022 kern_return_t
dtrace_copyio_postflight(__unused addr64_t va
)
2024 return KERN_SUCCESS
;
2026 #endif /* CONFIG_DTRACE */
2028 #include <mach_vm_debug.h>
2030 #include <vm/vm_debug.h>
2033 pmap_list_resident_pages(
2034 __unused pmap_t pmap
,
2035 __unused vm_offset_t
*listp
,
2040 #endif /* MACH_VM_DEBUG */
2044 /* temporary workaround */
2046 coredumpok(__unused vm_map_t map
, __unused vm_offset_t va
)
2051 ptep
= pmap_pte(map
->pmap
, va
);
2054 return ((*ptep
& (INTEL_PTE_NCACHE
| INTEL_PTE_WIRED
)) != (INTEL_PTE_NCACHE
| INTEL_PTE_WIRED
));
2062 phys_page_exists(ppnum_t pn
)
2064 assert(pn
!= vm_page_fictitious_addr
);
2066 if (!pmap_initialized
)
2069 if (pn
== vm_page_guard_addr
)
2072 if (!IS_MANAGED_PAGE(ppn_to_pai(pn
)))
2079 pmap_switch(pmap_t tpmap
)
2083 s
= splhigh(); /* Make sure interruptions are disabled */
2084 set_dirbase(tpmap
, current_thread());
2090 * disable no-execute capability on
2091 * the specified pmap
2094 pmap_disable_NX(pmap_t pmap
)
2096 pmap
->nx_enabled
= 0;
2102 vm_size_t
*cur_size
,
2103 vm_size_t
*max_size
,
2104 vm_size_t
*elem_size
,
2105 vm_size_t
*alloc_size
,
2109 *count
= inuse_ptepages_count
;
2110 *cur_size
= PAGE_SIZE
* inuse_ptepages_count
;
2111 *max_size
= PAGE_SIZE
* (inuse_ptepages_count
+
2112 vm_page_inactive_count
+
2113 vm_page_active_count
+
2114 vm_page_free_count
);
2115 *elem_size
= PAGE_SIZE
;
2116 *alloc_size
= PAGE_SIZE
;
2123 pmap_cpuset_NMIPI(cpu_set cpu_mask
) {
2124 unsigned int cpu
, cpu_bit
;
2127 for (cpu
= 0, cpu_bit
= 1; cpu
< real_ncpus
; cpu
++, cpu_bit
<<= 1) {
2128 if (cpu_mask
& cpu_bit
)
2129 cpu_NMI_interrupt(cpu
);
2131 deadline
= mach_absolute_time() + (LockTimeOut
);
2132 while (mach_absolute_time() < deadline
)
2137 * Called with pmap locked, we:
2138 * - scan through per-cpu data to see which other cpus need to flush
2139 * - send an IPI to each non-idle cpu to be flushed
2140 * - wait for all to signal back that they are inactive or we see that
2141 * they are at a safe point (idle).
2142 * - flush the local tlb if active for this pmap
2143 * - return ... the caller will unlock the pmap
2146 pmap_flush_tlbs(pmap_t pmap
)
2149 unsigned int cpu_bit
;
2150 cpu_set cpus_to_signal
;
2151 unsigned int my_cpu
= cpu_number();
2152 pmap_paddr_t pmap_cr3
= pmap
->pm_cr3
;
2153 boolean_t flush_self
= FALSE
;
2156 assert((processor_avail_count
< 2) ||
2157 (ml_get_interrupts_enabled() && get_preemption_level() != 0));
2160 * Scan other cpus for matching active or task CR3.
2161 * For idle cpus (with no active map) we mark them invalid but
2162 * don't signal -- they'll check as they go busy.
2165 for (cpu
= 0, cpu_bit
= 1; cpu
< real_ncpus
; cpu
++, cpu_bit
<<= 1) {
2166 if (!cpu_datap(cpu
)->cpu_running
)
2168 uint64_t cpu_active_cr3
= CPU_GET_ACTIVE_CR3(cpu
);
2169 uint64_t cpu_task_cr3
= CPU_GET_TASK_CR3(cpu
);
2171 if ((pmap_cr3
== cpu_task_cr3
) ||
2172 (pmap_cr3
== cpu_active_cr3
) ||
2173 (pmap
->pm_shared
) ||
2174 (pmap
== kernel_pmap
)) {
2175 if (cpu
== my_cpu
) {
2179 cpu_datap(cpu
)->cpu_tlb_invalid
= TRUE
;
2180 __asm__
volatile("mfence");
2183 * We don't need to signal processors which will flush
2184 * lazily at the idle state or kernel boundary.
2185 * For example, if we're invalidating the kernel pmap,
2186 * processors currently in userspace don't need to flush
2187 * their TLBs until the next time they enter the kernel.
2188 * Alterations to the address space of a task active
2189 * on a remote processor result in a signal, to
2190 * account for copy operations. (There may be room
2191 * for optimization in such cases).
2192 * The order of the loads below with respect
2193 * to the store to the "cpu_tlb_invalid" field above
2194 * is important--hence the barrier.
2196 if (CPU_CR3_IS_ACTIVE(cpu
) &&
2197 (pmap_cr3
== CPU_GET_ACTIVE_CR3(cpu
) ||
2199 (pmap_cr3
== CPU_GET_TASK_CR3(cpu
)))) {
2200 cpus_to_signal
|= cpu_bit
;
2201 i386_signal_cpu(cpu
, MP_TLB_FLUSH
, ASYNC
);
2206 PMAP_TRACE(PMAP_CODE(PMAP__FLUSH_TLBS
) | DBG_FUNC_START
,
2207 pmap
, cpus_to_signal
, flush_self
, 0, 0);
2210 * Flush local tlb if required.
2211 * Do this now to overlap with other processors responding.
2216 if (cpus_to_signal
) {
2217 cpu_set cpus_to_respond
= cpus_to_signal
;
2219 deadline
= mach_absolute_time() + LockTimeOut
;
2221 * Wait for those other cpus to acknowledge
2223 while (cpus_to_respond
!= 0) {
2224 if (mach_absolute_time() > deadline
) {
2225 if (mp_recent_debugger_activity())
2227 if (!panic_active()) {
2228 pmap_tlb_flush_timeout
= TRUE
;
2229 pmap_cpuset_NMIPI(cpus_to_respond
);
2231 panic("pmap_flush_tlbs() timeout: "
2232 "cpu(s) failing to respond to interrupts, pmap=%p cpus_to_respond=0x%lx",
2233 pmap
, cpus_to_respond
);
2236 for (cpu
= 0, cpu_bit
= 1; cpu
< real_ncpus
; cpu
++, cpu_bit
<<= 1) {
2237 if ((cpus_to_respond
& cpu_bit
) != 0) {
2238 if (!cpu_datap(cpu
)->cpu_running
||
2239 cpu_datap(cpu
)->cpu_tlb_invalid
== FALSE
||
2240 !CPU_CR3_IS_ACTIVE(cpu
)) {
2241 cpus_to_respond
&= ~cpu_bit
;
2245 if (cpus_to_respond
== 0)
2251 PMAP_TRACE(PMAP_CODE(PMAP__FLUSH_TLBS
) | DBG_FUNC_END
,
2252 pmap
, cpus_to_signal
, flush_self
, 0, 0);
2256 process_pmap_updates(void)
2258 assert(ml_get_interrupts_enabled() == 0 || get_preemption_level() != 0);
2262 current_cpu_datap()->cpu_tlb_invalid
= FALSE
;
2263 __asm__
volatile("mfence");
2267 pmap_update_interrupt(void)
2269 PMAP_TRACE(PMAP_CODE(PMAP__UPDATE_INTERRUPT
) | DBG_FUNC_START
,
2272 process_pmap_updates();
2274 PMAP_TRACE(PMAP_CODE(PMAP__UPDATE_INTERRUPT
) | DBG_FUNC_END
,
2280 pmap_cache_attributes(ppnum_t pn
)
2282 return IS_MANAGED_PAGE(ppn_to_pai(pn
)) ? VM_WIMG_COPYBACK