]> git.saurik.com Git - apple/xnu.git/blob - osfmk/x86_64/pmap.c
xnu-1504.7.4.tar.gz
[apple/xnu.git] / osfmk / x86_64 / pmap.c
1
2 /*
3 * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
4 *
5 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
6 *
7 * This file contains Original Code and/or Modifications of Original Code
8 * as defined in and that are subject to the Apple Public Source License
9 * Version 2.0 (the 'License'). You may not use this file except in
10 * compliance with the License. The rights granted to you under the License
11 * may not be used to create, or enable the creation or redistribution of,
12 * unlawful or unlicensed copies of an Apple operating system, or to
13 * circumvent, violate, or enable the circumvention or violation of, any
14 * terms of an Apple operating system software license agreement.
15 *
16 * Please obtain a copy of the License at
17 * http://www.opensource.apple.com/apsl/ and read it before using this file.
18 *
19 * The Original Code and all software distributed under the License are
20 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
21 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
22 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
23 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
24 * Please see the License for the specific language governing rights and
25 * limitations under the License.
26 *
27 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
28 */
29 /*
30 * @OSF_COPYRIGHT@
31 */
32 /*
33 * Mach Operating System
34 * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University
35 * All Rights Reserved.
36 *
37 * Permission to use, copy, modify and distribute this software and its
38 * documentation is hereby granted, provided that both the copyright
39 * notice and this permission notice appear in all copies of the
40 * software, derivative works or modified versions, and any portions
41 * thereof, and that both notices appear in supporting documentation.
42 *
43 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
44 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
45 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
46 *
47 * Carnegie Mellon requests users of this software to return to
48 *
49 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
50 * School of Computer Science
51 * Carnegie Mellon University
52 * Pittsburgh PA 15213-3890
53 *
54 * any improvements or extensions that they make and grant Carnegie Mellon
55 * the rights to redistribute these changes.
56 */
57 /*
58 */
59
60 /*
61 * File: pmap.c
62 * Author: Avadis Tevanian, Jr., Michael Wayne Young
63 * (These guys wrote the Vax version)
64 *
65 * Physical Map management code for Intel i386, i486, and i860.
66 *
67 * Manages physical address maps.
68 *
69 * In addition to hardware address maps, this
70 * module is called upon to provide software-use-only
71 * maps which may or may not be stored in the same
72 * form as hardware maps. These pseudo-maps are
73 * used to store intermediate results from copy
74 * operations to and from address spaces.
75 *
76 * Since the information managed by this module is
77 * also stored by the logical address mapping module,
78 * this module may throw away valid virtual-to-physical
79 * mappings at almost any time. However, invalidations
80 * of virtual-to-physical mappings must be done as
81 * requested.
82 *
83 * In order to cope with hardware architectures which
84 * make virtual-to-physical map invalidates expensive,
85 * this module may delay invalidate or reduced protection
86 * operations until such time as they are actually
87 * necessary. This module is given full information as
88 * to which processors are currently using which maps,
89 * and to when physical maps must be made correct.
90 */
91
92 #include <string.h>
93 #include <mach_kdb.h>
94 #include <mach_ldebug.h>
95
96 #include <libkern/OSAtomic.h>
97
98 #include <mach/machine/vm_types.h>
99
100 #include <mach/boolean.h>
101 #include <kern/thread.h>
102 #include <kern/zalloc.h>
103 #include <kern/queue.h>
104
105 #include <kern/lock.h>
106 #include <kern/kalloc.h>
107 #include <kern/spl.h>
108
109 #include <vm/pmap.h>
110 #include <vm/vm_map.h>
111 #include <vm/vm_kern.h>
112 #include <mach/vm_param.h>
113 #include <mach/vm_prot.h>
114 #include <vm/vm_object.h>
115 #include <vm/vm_page.h>
116
117 #include <mach/machine/vm_param.h>
118 #include <machine/thread.h>
119
120 #include <kern/misc_protos.h> /* prototyping */
121 #include <i386/misc_protos.h>
122 #include <x86_64/lowglobals.h>
123
124 #include <i386/cpuid.h>
125 #include <i386/cpu_data.h>
126 #include <i386/cpu_number.h>
127 #include <i386/machine_cpu.h>
128 #include <i386/seg.h>
129 #include <i386/serial_io.h>
130 #include <i386/cpu_capabilities.h>
131 #include <i386/machine_routines.h>
132 #include <i386/proc_reg.h>
133 #include <i386/tsc.h>
134 #include <i386/pmap_internal.h>
135
136 #if MACH_KDB
137 #include <ddb/db_command.h>
138 #include <ddb/db_output.h>
139 #include <ddb/db_sym.h>
140 #include <ddb/db_print.h>
141 #endif /* MACH_KDB */
142
143 #include <vm/vm_protos.h>
144
145 #include <i386/mp.h>
146 #include <i386/mp_desc.h>
147
148
149
150 #ifdef IWANTTODEBUG
151 #undef DEBUG
152 #define DEBUG 1
153 #define POSTCODE_DELAY 1
154 #include <i386/postcode.h>
155 #endif /* IWANTTODEBUG */
156
157 boolean_t pmap_trace = FALSE;
158
159 #if PMAP_DBG
160 #define DBG(x...) kprintf("DBG: " x)
161 #else
162 #define DBG(x...)
163 #endif
164
165 boolean_t no_shared_cr3 = DEBUG; /* TRUE for DEBUG by default */
166
167 /*
168 * Forward declarations for internal functions.
169 */
170
171
172 void phys_attribute_clear(
173 ppnum_t phys,
174 int bits);
175
176 int phys_attribute_test(
177 ppnum_t phys,
178 int bits);
179
180 void phys_attribute_set(
181 ppnum_t phys,
182 int bits);
183
184 void pmap_set_reference(
185 ppnum_t pn);
186
187 boolean_t phys_page_exists(
188 ppnum_t pn);
189
190
191 int nx_enabled = 1; /* enable no-execute protection */
192 int allow_data_exec = VM_ABI_32; /* 32-bit apps may execute data by default, 64-bit apps may not */
193 int allow_stack_exec = 0; /* No apps may execute from the stack by default */
194
195 const boolean_t cpu_64bit = TRUE; /* Mais oui! */
196
197 uint64_t max_preemption_latency_tsc = 0;
198
199 pv_hashed_entry_t *pv_hash_table; /* hash lists */
200
201 uint32_t npvhash = 0;
202
203 pv_hashed_entry_t pv_hashed_free_list = PV_HASHED_ENTRY_NULL;
204 pv_hashed_entry_t pv_hashed_kern_free_list = PV_HASHED_ENTRY_NULL;
205 decl_simple_lock_data(,pv_hashed_free_list_lock)
206 decl_simple_lock_data(,pv_hashed_kern_free_list_lock)
207 decl_simple_lock_data(,pv_hash_table_lock)
208
209 int pv_hashed_free_count = 0;
210 int pv_hashed_kern_free_count = 0;
211
212
213 zone_t pv_hashed_list_zone; /* zone of pv_hashed_entry structures */
214
215 static zone_t pdpt_zone;
216
217 /*
218 * Each entry in the pv_head_table is locked by a bit in the
219 * pv_lock_table. The lock bits are accessed by the physical
220 * address of the page they lock.
221 */
222
223 char *pv_lock_table; /* pointer to array of bits */
224
225
226 char *pv_hash_lock_table;
227
228
229 /*
230 * First and last physical addresses that we maintain any information
231 * for. Initialized to zero so that pmap operations done before
232 * pmap_init won't touch any non-existent structures.
233 */
234 boolean_t pmap_initialized = FALSE;/* Has pmap_init completed? */
235
236 static struct vm_object kptobj_object_store;
237 static struct vm_object kpml4obj_object_store;
238 static struct vm_object kpdptobj_object_store;
239
240 /*
241 * Array of physical page attributes for managed pages.
242 * One byte per physical page.
243 */
244 char *pmap_phys_attributes;
245 unsigned int last_managed_page = 0;
246 uint64_t pde_mapped_size = PDE_MAPPED_SIZE;
247
248 unsigned pmap_memory_region_count;
249 unsigned pmap_memory_region_current;
250
251 pmap_memory_region_t pmap_memory_regions[PMAP_MEMORY_REGIONS_SIZE];
252
253 /*
254 * Other useful macros.
255 */
256 #define current_pmap() (vm_map_pmap(current_thread()->map))
257
258 struct pmap kernel_pmap_store;
259 pmap_t kernel_pmap;
260
261 pd_entry_t high_shared_pde;
262 pd_entry_t commpage64_pde;
263
264 struct zone *pmap_zone; /* zone of pmap structures */
265
266 unsigned int inuse_ptepages_count = 0;
267
268 addr64_t kernel64_cr3;
269
270 /*
271 * Pmap cache. Cache is threaded through ref_count field of pmap.
272 * Max will eventually be constant -- variable for experimentation.
273 */
274 int pmap_cache_max = 32;
275 int pmap_alloc_chunk = 8;
276 pmap_t pmap_cache_list;
277 int pmap_cache_count;
278 decl_simple_lock_data(,pmap_cache_lock)
279
280 extern char end;
281
282 static int nkpt;
283
284 pt_entry_t *DMAP1, *DMAP2;
285 caddr_t DADDR1;
286 caddr_t DADDR2;
287 /*
288 * for legacy, returns the address of the pde entry.
289 * for 64 bit, causes the pdpt page containing the pde entry to be mapped,
290 * then returns the mapped address of the pde entry in that page
291 */
292 pd_entry_t *
293 pmap_pde(pmap_t m, vm_map_offset_t v)
294 {
295 pd_entry_t *pde;
296
297 assert(m);
298 #if 0
299 if (m == kernel_pmap)
300 pde = (&((m)->dirbase[(vm_offset_t)(v) >> PDESHIFT]));
301 else
302 #endif
303 pde = pmap64_pde(m, v);
304
305 return pde;
306 }
307
308 /*
309 * the single pml4 page per pmap is allocated at pmap create time and exists
310 * for the duration of the pmap. we allocate this page in kernel vm.
311 * this returns the address of the requested pml4 entry in the top level page.
312 */
313 static inline
314 pml4_entry_t *
315 pmap64_pml4(pmap_t pmap, vm_map_offset_t vaddr)
316 {
317 return &pmap->pm_pml4[(vaddr >> PML4SHIFT) & (NPML4PG-1)];
318 }
319
320 /*
321 * maps in the pml4 page, if any, containing the pdpt entry requested
322 * and returns the address of the pdpt entry in that mapped page
323 */
324 pdpt_entry_t *
325 pmap64_pdpt(pmap_t pmap, vm_map_offset_t vaddr)
326 {
327 pml4_entry_t newpf;
328 pml4_entry_t *pml4;
329
330 assert(pmap);
331 if ((vaddr > 0x00007FFFFFFFFFFFULL) &&
332 (vaddr < 0xFFFF800000000000ULL)) {
333 return (0);
334 }
335
336 pml4 = pmap64_pml4(pmap, vaddr);
337 if (pml4 && ((*pml4 & INTEL_PTE_VALID))) {
338 newpf = *pml4 & PG_FRAME;
339 return &((pdpt_entry_t *) PHYSMAP_PTOV(newpf))
340 [(vaddr >> PDPTSHIFT) & (NPDPTPG-1)];
341 }
342 return (NULL);
343 }
344 /*
345 * maps in the pdpt page, if any, containing the pde entry requested
346 * and returns the address of the pde entry in that mapped page
347 */
348 pd_entry_t *
349 pmap64_pde(pmap_t pmap, vm_map_offset_t vaddr)
350 {
351 pdpt_entry_t newpf;
352 pdpt_entry_t *pdpt;
353
354 assert(pmap);
355 if ((vaddr > 0x00007FFFFFFFFFFFULL) &&
356 (vaddr < 0xFFFF800000000000ULL)) {
357 return (0);
358 }
359
360 pdpt = pmap64_pdpt(pmap, vaddr);
361
362 if (pdpt && ((*pdpt & INTEL_PTE_VALID))) {
363 newpf = *pdpt & PG_FRAME;
364 return &((pd_entry_t *) PHYSMAP_PTOV(newpf))
365 [(vaddr >> PDSHIFT) & (NPDPG-1)];
366 }
367 return (NULL);
368 }
369
370 /*
371 * return address of mapped pte for vaddr va in pmap pmap.
372 *
373 * physically maps the pde page, if any, containing the pte in and returns
374 * the address of the pte in that mapped page
375 *
376 * In case the pde maps a superpage, return the pde, which, in this case
377 * is the actual page table entry.
378 */
379 pt_entry_t *
380 pmap_pte(pmap_t pmap, vm_map_offset_t vaddr)
381 {
382 pd_entry_t *pde;
383 pd_entry_t newpf;
384
385 assert(pmap);
386 pde = pmap_pde(pmap, vaddr);
387
388 if (pde && ((*pde & INTEL_PTE_VALID))) {
389 if (*pde & INTEL_PTE_PS)
390 return pde;
391 newpf = *pde & PG_FRAME;
392 return &((pt_entry_t *)PHYSMAP_PTOV(newpf))
393 [i386_btop(vaddr) & (ppnum_t)(NPTEPG-1)];
394 }
395 return (NULL);
396 }
397
398 /*
399 * Map memory at initialization. The physical addresses being
400 * mapped are not managed and are never unmapped.
401 *
402 * For now, VM is already on, we only need to map the
403 * specified memory.
404 */
405 vm_offset_t
406 pmap_map(
407 vm_offset_t virt,
408 vm_map_offset_t start_addr,
409 vm_map_offset_t end_addr,
410 vm_prot_t prot,
411 unsigned int flags)
412 {
413 int ps;
414
415 ps = PAGE_SIZE;
416 while (start_addr < end_addr) {
417 pmap_enter(kernel_pmap, (vm_map_offset_t)virt,
418 (ppnum_t) i386_btop(start_addr), prot, flags, FALSE);
419 virt += ps;
420 start_addr += ps;
421 }
422 return(virt);
423 }
424
425 /*
426 * Back-door routine for mapping kernel VM at initialization.
427 * Useful for mapping memory outside the range
428 * Sets no-cache, A, D.
429 * Otherwise like pmap_map.
430 */
431 vm_offset_t
432 pmap_map_bd(
433 vm_offset_t virt,
434 vm_map_offset_t start_addr,
435 vm_map_offset_t end_addr,
436 vm_prot_t prot,
437 unsigned int flags)
438 {
439 pt_entry_t template;
440 pt_entry_t *pte;
441 spl_t spl;
442
443 template = pa_to_pte(start_addr)
444 | INTEL_PTE_REF
445 | INTEL_PTE_MOD
446 | INTEL_PTE_WIRED
447 | INTEL_PTE_VALID;
448
449 if (flags & (VM_MEM_NOT_CACHEABLE | VM_WIMG_USE_DEFAULT)) {
450 template |= INTEL_PTE_NCACHE;
451 if (!(flags & (VM_MEM_GUARDED | VM_WIMG_USE_DEFAULT)))
452 template |= INTEL_PTE_PTA;
453 }
454 if (prot & VM_PROT_WRITE)
455 template |= INTEL_PTE_WRITE;
456
457
458 while (start_addr < end_addr) {
459 spl = splhigh();
460 pte = pmap_pte(kernel_pmap, (vm_map_offset_t)virt);
461 if (pte == PT_ENTRY_NULL) {
462 panic("pmap_map_bd: Invalid kernel address\n");
463 }
464 pmap_store_pte(pte, template);
465 splx(spl);
466 pte_increment_pa(template);
467 virt += PAGE_SIZE;
468 start_addr += PAGE_SIZE;
469 }
470
471
472 flush_tlb();
473 return(virt);
474 }
475
476 extern char *first_avail;
477 extern vm_offset_t virtual_avail, virtual_end;
478 extern pmap_paddr_t avail_start, avail_end;
479 extern vm_offset_t sHIB;
480 extern vm_offset_t eHIB;
481 extern vm_offset_t stext;
482 extern vm_offset_t etext;
483 extern vm_offset_t sdata;
484
485 void
486 pmap_cpu_init(void)
487 {
488 /*
489 * Here early in the life of a processor (from cpu_mode_init()).
490 * Ensure global page feature is disabled.
491 */
492 set_cr4(get_cr4() &~ CR4_PGE);
493
494 /*
495 * Initialize the per-cpu, TLB-related fields.
496 */
497 current_cpu_datap()->cpu_kernel_cr3 = kernel_pmap->pm_cr3;
498 current_cpu_datap()->cpu_active_cr3 = kernel_pmap->pm_cr3;
499 current_cpu_datap()->cpu_tlb_invalid = FALSE;
500 }
501
502
503
504 /*
505 * Bootstrap the system enough to run with virtual memory.
506 * Map the kernel's code and data, and allocate the system page table.
507 * Called with mapping OFF. Page_size must already be set.
508 */
509
510 void
511 pmap_bootstrap(
512 __unused vm_offset_t load_start,
513 __unused boolean_t IA32e)
514 {
515 #if NCOPY_WINDOWS > 0
516 vm_offset_t va;
517 int i;
518 #endif
519
520 assert(IA32e);
521
522 vm_last_addr = VM_MAX_KERNEL_ADDRESS; /* Set the highest address
523 * known to VM */
524 /*
525 * The kernel's pmap is statically allocated so we don't
526 * have to use pmap_create, which is unlikely to work
527 * correctly at this part of the boot sequence.
528 */
529
530 kernel_pmap = &kernel_pmap_store;
531 kernel_pmap->ref_count = 1;
532 kernel_pmap->nx_enabled = FALSE;
533 kernel_pmap->pm_task_map = TASK_MAP_64BIT;
534 kernel_pmap->pm_obj = (vm_object_t) NULL;
535 kernel_pmap->dirbase = (pd_entry_t *)((uintptr_t)IdlePTD);
536 kernel_pmap->pm_pdpt = (pd_entry_t *) ((uintptr_t)IdlePDPT);
537 kernel_pmap->pm_pml4 = IdlePML4;
538 kernel_pmap->pm_cr3 = (uintptr_t)ID_MAP_VTOP(IdlePML4);
539
540
541 current_cpu_datap()->cpu_kernel_cr3 = (addr64_t) kernel_pmap->pm_cr3;
542
543 nkpt = NKPT;
544 OSAddAtomic(NKPT, &inuse_ptepages_count);
545
546 virtual_avail = (vm_offset_t)(VM_MIN_KERNEL_ADDRESS) + (vm_offset_t)first_avail;
547 virtual_end = (vm_offset_t)(VM_MAX_KERNEL_ADDRESS);
548
549 #if NCOPY_WINDOWS > 0
550 /*
551 * Reserve some special page table entries/VA space for temporary
552 * mapping of pages.
553 */
554 #define SYSMAP(c, p, v, n) \
555 v = (c)va; va += ((n)*INTEL_PGBYTES);
556
557 va = virtual_avail;
558
559 for (i=0; i<PMAP_NWINDOWS; i++) {
560 #if 1
561 kprintf("trying to do SYSMAP idx %d %p\n", i,
562 current_cpu_datap());
563 kprintf("cpu_pmap %p\n", current_cpu_datap()->cpu_pmap);
564 kprintf("mapwindow %p\n", current_cpu_datap()->cpu_pmap->mapwindow);
565 kprintf("two stuff %p %p\n",
566 (void *)(current_cpu_datap()->cpu_pmap->mapwindow[i].prv_CMAP),
567 (void *)(current_cpu_datap()->cpu_pmap->mapwindow[i].prv_CADDR));
568 #endif
569 SYSMAP(caddr_t,
570 (current_cpu_datap()->cpu_pmap->mapwindow[i].prv_CMAP),
571 (current_cpu_datap()->cpu_pmap->mapwindow[i].prv_CADDR),
572 1);
573 current_cpu_datap()->cpu_pmap->mapwindow[i].prv_CMAP =
574 &(current_cpu_datap()->cpu_pmap->mapwindow[i].prv_CMAP_store);
575 *current_cpu_datap()->cpu_pmap->mapwindow[i].prv_CMAP = 0;
576 }
577
578 /* DMAP user for debugger */
579 SYSMAP(caddr_t, DMAP1, DADDR1, 1);
580 SYSMAP(caddr_t, DMAP2, DADDR2, 1); /* XXX temporary - can remove */
581
582 virtual_avail = va;
583 #endif
584
585 if (PE_parse_boot_argn("npvhash", &npvhash, sizeof (npvhash))) {
586 if (0 != ((npvhash + 1) & npvhash)) {
587 kprintf("invalid hash %d, must be ((2^N)-1), "
588 "using default %d\n", npvhash, NPVHASH);
589 npvhash = NPVHASH;
590 }
591 } else {
592 npvhash = NPVHASH;
593 }
594
595 printf("npvhash=%d\n", npvhash);
596
597 simple_lock_init(&kernel_pmap->lock, 0);
598 simple_lock_init(&pv_hashed_free_list_lock, 0);
599 simple_lock_init(&pv_hashed_kern_free_list_lock, 0);
600 simple_lock_init(&pv_hash_table_lock,0);
601
602 pmap_cpu_init();
603
604 kprintf("Kernel virtual space from 0x%lx to 0x%lx.\n",
605 (long)KERNEL_BASE, (long)virtual_end);
606 kprintf("Available physical space from 0x%llx to 0x%llx\n",
607 avail_start, avail_end);
608
609 /*
610 * The -no_shared_cr3 boot-arg is a debugging feature (set by default
611 * in the DEBUG kernel) to force the kernel to switch to its own map
612 * (and cr3) when control is in kernelspace. The kernel's map does not
613 * include (i.e. share) userspace so wild references will cause
614 * a panic. Only copyin and copyout are exempt from this.
615 */
616 (void) PE_parse_boot_argn("-no_shared_cr3",
617 &no_shared_cr3, sizeof (no_shared_cr3));
618 if (no_shared_cr3)
619 kprintf("Kernel not sharing user map\n");
620
621 #ifdef PMAP_TRACES
622 if (PE_parse_boot_argn("-pmap_trace", &pmap_trace, sizeof (pmap_trace))) {
623 kprintf("Kernel traces for pmap operations enabled\n");
624 }
625 #endif /* PMAP_TRACES */
626 }
627
628 void
629 pmap_virtual_space(
630 vm_offset_t *startp,
631 vm_offset_t *endp)
632 {
633 *startp = virtual_avail;
634 *endp = virtual_end;
635 }
636
637 /*
638 * Initialize the pmap module.
639 * Called by vm_init, to initialize any structures that the pmap
640 * system needs to map virtual memory.
641 */
642 void
643 pmap_init(void)
644 {
645 long npages;
646 vm_offset_t addr;
647 vm_size_t s;
648 vm_map_offset_t vaddr;
649 ppnum_t ppn;
650
651
652 kernel_pmap->pm_obj_pml4 = &kpml4obj_object_store;
653 _vm_object_allocate((vm_object_size_t)NPML4PGS, &kpml4obj_object_store);
654
655 kernel_pmap->pm_obj_pdpt = &kpdptobj_object_store;
656 _vm_object_allocate((vm_object_size_t)NPDPTPGS, &kpdptobj_object_store);
657
658 kernel_pmap->pm_obj = &kptobj_object_store;
659 _vm_object_allocate((vm_object_size_t)NPDEPGS, &kptobj_object_store);
660
661 /*
662 * Allocate memory for the pv_head_table and its lock bits,
663 * the modify bit array, and the pte_page table.
664 */
665
666 /*
667 * zero bias all these arrays now instead of off avail_start
668 * so we cover all memory
669 */
670
671 npages = i386_btop(avail_end);
672 s = (vm_size_t) (sizeof(struct pv_rooted_entry) * npages
673 + (sizeof (struct pv_hashed_entry_t *) * (npvhash+1))
674 + pv_lock_table_size(npages)
675 + pv_hash_lock_table_size((npvhash+1))
676 + npages);
677
678 s = round_page(s);
679 if (kernel_memory_allocate(kernel_map, &addr, s, 0,
680 KMA_KOBJECT | KMA_PERMANENT)
681 != KERN_SUCCESS)
682 panic("pmap_init");
683
684 memset((char *)addr, 0, s);
685
686 #if PV_DEBUG
687 if (0 == npvhash) panic("npvhash not initialized");
688 #endif
689
690 /*
691 * Allocate the structures first to preserve word-alignment.
692 */
693 pv_head_table = (pv_rooted_entry_t) addr;
694 addr = (vm_offset_t) (pv_head_table + npages);
695
696 pv_hash_table = (pv_hashed_entry_t *)addr;
697 addr = (vm_offset_t) (pv_hash_table + (npvhash + 1));
698
699 pv_lock_table = (char *) addr;
700 addr = (vm_offset_t) (pv_lock_table + pv_lock_table_size(npages));
701
702 pv_hash_lock_table = (char *) addr;
703 addr = (vm_offset_t) (pv_hash_lock_table + pv_hash_lock_table_size((npvhash+1)));
704
705 pmap_phys_attributes = (char *) addr;
706
707 ppnum_t last_pn = i386_btop(avail_end);
708 unsigned int i;
709 pmap_memory_region_t *pmptr = pmap_memory_regions;
710 for (i = 0; i < pmap_memory_region_count; i++, pmptr++) {
711 if (pmptr->type != kEfiConventionalMemory)
712 continue;
713 unsigned int pn;
714 for (pn = pmptr->base; pn <= pmptr->end; pn++) {
715 if (pn < last_pn) {
716 pmap_phys_attributes[pn] |= PHYS_MANAGED;
717 if (pn > last_managed_page)
718 last_managed_page = pn;
719 }
720 }
721 }
722
723 /*
724 * Create the zone of physical maps,
725 * and of the physical-to-virtual entries.
726 */
727 s = (vm_size_t) sizeof(struct pmap);
728 pmap_zone = zinit(s, 400*s, 4096, "pmap"); /* XXX */
729 s = (vm_size_t) sizeof(struct pv_hashed_entry);
730 pv_hashed_list_zone = zinit(s, 10000*s, 4096, "pv_list"); /* XXX */
731 s = 63;
732 pdpt_zone = zinit(s, 400*s, 4096, "pdpt"); /* XXX */
733
734
735 /* create pv entries for kernel pages mapped by low level
736 startup code. these have to exist so we can pmap_remove()
737 e.g. kext pages from the middle of our addr space */
738
739 vaddr = (vm_map_offset_t) VM_MIN_KERNEL_ADDRESS;
740 for (ppn = 0; ppn < i386_btop(avail_start); ppn++) {
741 pv_rooted_entry_t pv_e;
742
743 pv_e = pai_to_pvh(ppn);
744 pv_e->va = vaddr;
745 vaddr += PAGE_SIZE;
746 pv_e->pmap = kernel_pmap;
747 queue_init(&pv_e->qlink);
748 }
749 pmap_initialized = TRUE;
750
751 /*
752 * Initialize pmap cache.
753 */
754 pmap_cache_list = PMAP_NULL;
755 pmap_cache_count = 0;
756 simple_lock_init(&pmap_cache_lock, 0);
757
758 max_preemption_latency_tsc = tmrCvt((uint64_t)MAX_PREEMPTION_LATENCY_NS, tscFCvtn2t);
759
760 /*
761 * Ensure the kernel's PML4 entry exists for the basement
762 * before this is shared with any user.
763 */
764 pmap_expand_pml4(kernel_pmap, KERNEL_BASEMENT);
765 }
766
767
768 /*
769 * this function is only used for debugging fron the vm layer
770 */
771 boolean_t
772 pmap_verify_free(
773 ppnum_t pn)
774 {
775 pv_rooted_entry_t pv_h;
776 int pai;
777 boolean_t result;
778
779 assert(pn != vm_page_fictitious_addr);
780
781 if (!pmap_initialized)
782 return(TRUE);
783
784 if (pn == vm_page_guard_addr)
785 return TRUE;
786
787 pai = ppn_to_pai(pn);
788 if (!IS_MANAGED_PAGE(pai))
789 return(FALSE);
790 pv_h = pai_to_pvh(pn);
791 result = (pv_h->pmap == PMAP_NULL);
792 return(result);
793 }
794
795 boolean_t
796 pmap_is_empty(
797 pmap_t pmap,
798 vm_map_offset_t va_start,
799 vm_map_offset_t va_end)
800 {
801 vm_map_offset_t offset;
802 ppnum_t phys_page;
803
804 if (pmap == PMAP_NULL) {
805 return TRUE;
806 }
807
808 /*
809 * Check the resident page count
810 * - if it's zero, the pmap is completely empty.
811 * This short-circuit test prevents a virtual address scan which is
812 * painfully slow for 64-bit spaces.
813 * This assumes the count is correct
814 * .. the debug kernel ought to be checking perhaps by page table walk.
815 */
816 if (pmap->stats.resident_count == 0)
817 return TRUE;
818
819 for (offset = va_start;
820 offset < va_end;
821 offset += PAGE_SIZE_64) {
822 phys_page = pmap_find_phys(pmap, offset);
823 if (phys_page) {
824 kprintf("pmap_is_empty(%p,0x%llx,0x%llx): "
825 "page %d at 0x%llx\n",
826 pmap, va_start, va_end, phys_page, offset);
827 return FALSE;
828 }
829 }
830
831 return TRUE;
832 }
833
834
835 /*
836 * Create and return a physical map.
837 *
838 * If the size specified for the map
839 * is zero, the map is an actual physical
840 * map, and may be referenced by the
841 * hardware.
842 *
843 * If the size specified is non-zero,
844 * the map will be used in software only, and
845 * is bounded by that size.
846 */
847 pmap_t
848 pmap_create(
849 vm_map_size_t sz,
850 boolean_t is_64bit)
851 {
852 pmap_t p;
853 vm_size_t size;
854 pml4_entry_t *pml4;
855 pml4_entry_t *kpml4;
856
857 PMAP_TRACE(PMAP_CODE(PMAP__CREATE) | DBG_FUNC_START,
858 (uint32_t) (sz>>32), (uint32_t) sz, is_64bit, 0, 0);
859
860 size = (vm_size_t) sz;
861
862 /*
863 * A software use-only map doesn't even need a map.
864 */
865
866 if (size != 0) {
867 return(PMAP_NULL);
868 }
869
870 p = (pmap_t) zalloc(pmap_zone);
871 if (PMAP_NULL == p)
872 panic("pmap_create zalloc");
873
874 /* init counts now since we'll be bumping some */
875 simple_lock_init(&p->lock, 0);
876 p->stats.resident_count = 0;
877 p->stats.resident_max = 0;
878 p->stats.wired_count = 0;
879 p->ref_count = 1;
880 p->nx_enabled = 1;
881 p->pm_shared = FALSE;
882
883 p->pm_task_map = is_64bit ? TASK_MAP_64BIT : TASK_MAP_32BIT;;
884
885 /* alloc the pml4 page in kernel vm */
886 if (KERN_SUCCESS != kmem_alloc_kobject(kernel_map, (vm_offset_t *)(&p->pm_pml4), PAGE_SIZE))
887 panic("pmap_create kmem_alloc_kobject pml4");
888
889 memset((char *)p->pm_pml4, 0, PAGE_SIZE);
890 p->pm_cr3 = (pmap_paddr_t)kvtophys((vm_offset_t)p->pm_pml4);
891
892 OSAddAtomic(1, &inuse_ptepages_count);
893
894 /* allocate the vm_objs to hold the pdpt, pde and pte pages */
895
896 p->pm_obj_pml4 = vm_object_allocate((vm_object_size_t)(NPML4PGS));
897 if (NULL == p->pm_obj_pml4)
898 panic("pmap_create pdpt obj");
899
900 p->pm_obj_pdpt = vm_object_allocate((vm_object_size_t)(NPDPTPGS));
901 if (NULL == p->pm_obj_pdpt)
902 panic("pmap_create pdpt obj");
903
904 p->pm_obj = vm_object_allocate((vm_object_size_t)(NPDEPGS));
905 if (NULL == p->pm_obj)
906 panic("pmap_create pte obj");
907
908 /* All pmaps share the kennel's pml4 */
909 pml4 = pmap64_pml4(p, 0ULL);
910 kpml4 = kernel_pmap->pm_pml4;
911 pml4[KERNEL_PML4_INDEX] = kpml4[KERNEL_PML4_INDEX];
912 pml4[KERNEL_KEXTS_INDEX] = kpml4[KERNEL_KEXTS_INDEX];
913 pml4[KERNEL_PHYSMAP_INDEX] = kpml4[KERNEL_PHYSMAP_INDEX];
914
915 PMAP_TRACE(PMAP_CODE(PMAP__CREATE) | DBG_FUNC_START,
916 p, is_64bit, 0, 0, 0);
917
918 return(p);
919 }
920
921 /*
922 * Retire the given physical map from service.
923 * Should only be called if the map contains
924 * no valid mappings.
925 */
926
927 void
928 pmap_destroy(
929 register pmap_t p)
930 {
931 register int c;
932
933 if (p == PMAP_NULL)
934 return;
935
936 PMAP_TRACE(PMAP_CODE(PMAP__DESTROY) | DBG_FUNC_START,
937 p, 0, 0, 0, 0);
938
939 PMAP_LOCK(p);
940
941 c = --p->ref_count;
942
943 if (c == 0) {
944 /*
945 * If some cpu is not using the physical pmap pointer that it
946 * is supposed to be (see set_dirbase), we might be using the
947 * pmap that is being destroyed! Make sure we are
948 * physically on the right pmap:
949 */
950 PMAP_UPDATE_TLBS(p, 0x0ULL, 0xFFFFFFFFFFFFF000ULL);
951 }
952
953 PMAP_UNLOCK(p);
954
955 if (c != 0) {
956 PMAP_TRACE(PMAP_CODE(PMAP__DESTROY) | DBG_FUNC_END,
957 p, 1, 0, 0, 0);
958 return; /* still in use */
959 }
960
961 /*
962 * Free the memory maps, then the
963 * pmap structure.
964 */
965 int inuse_ptepages = 0;
966
967 inuse_ptepages++;
968 kmem_free(kernel_map, (vm_offset_t)p->pm_pml4, PAGE_SIZE);
969
970 inuse_ptepages += p->pm_obj_pml4->resident_page_count;
971 vm_object_deallocate(p->pm_obj_pml4);
972
973 inuse_ptepages += p->pm_obj_pdpt->resident_page_count;
974 vm_object_deallocate(p->pm_obj_pdpt);
975
976 inuse_ptepages += p->pm_obj->resident_page_count;
977 vm_object_deallocate(p->pm_obj);
978
979 OSAddAtomic(-inuse_ptepages, &inuse_ptepages_count);
980
981 zfree(pmap_zone, p);
982
983 PMAP_TRACE(PMAP_CODE(PMAP__DESTROY) | DBG_FUNC_END,
984 0, 0, 0, 0, 0);
985 }
986
987 /*
988 * Add a reference to the specified pmap.
989 */
990
991 void
992 pmap_reference(pmap_t p)
993 {
994 if (p != PMAP_NULL) {
995 PMAP_LOCK(p);
996 p->ref_count++;
997 PMAP_UNLOCK(p);;
998 }
999 }
1000
1001 /*
1002 * Remove phys addr if mapped in specified map
1003 *
1004 */
1005 void
1006 pmap_remove_some_phys(
1007 __unused pmap_t map,
1008 __unused ppnum_t pn)
1009 {
1010
1011 /* Implement to support working set code */
1012
1013 }
1014
1015
1016 /*
1017 * Routine:
1018 * pmap_disconnect
1019 *
1020 * Function:
1021 * Disconnect all mappings for this page and return reference and change status
1022 * in generic format.
1023 *
1024 */
1025 unsigned int pmap_disconnect(
1026 ppnum_t pa)
1027 {
1028 pmap_page_protect(pa, 0); /* disconnect the page */
1029 return (pmap_get_refmod(pa)); /* return ref/chg status */
1030 }
1031
1032 /*
1033 * Set the physical protection on the
1034 * specified range of this map as requested.
1035 * Will not increase permissions.
1036 */
1037 void
1038 pmap_protect(
1039 pmap_t map,
1040 vm_map_offset_t sva,
1041 vm_map_offset_t eva,
1042 vm_prot_t prot)
1043 {
1044 pt_entry_t *pde;
1045 pt_entry_t *spte, *epte;
1046 vm_map_offset_t lva;
1047 vm_map_offset_t orig_sva;
1048 boolean_t set_NX;
1049 int num_found = 0;
1050
1051 pmap_intr_assert();
1052
1053 if (map == PMAP_NULL)
1054 return;
1055
1056 if (prot == VM_PROT_NONE) {
1057 pmap_remove(map, sva, eva);
1058 return;
1059 }
1060 PMAP_TRACE(PMAP_CODE(PMAP__PROTECT) | DBG_FUNC_START,
1061 map,
1062 (uint32_t) (sva >> 32), (uint32_t) sva,
1063 (uint32_t) (eva >> 32), (uint32_t) eva);
1064
1065 if ((prot & VM_PROT_EXECUTE) || !nx_enabled || !map->nx_enabled)
1066 set_NX = FALSE;
1067 else
1068 set_NX = TRUE;
1069
1070 PMAP_LOCK(map);
1071
1072 orig_sva = sva;
1073 while (sva < eva) {
1074 lva = (sva + pde_mapped_size) & ~(pde_mapped_size - 1);
1075 if (lva > eva)
1076 lva = eva;
1077 pde = pmap_pde(map, sva);
1078 if (pde && (*pde & INTEL_PTE_VALID)) {
1079 if (*pde & INTEL_PTE_PS) {
1080 /* superpage */
1081 spte = pde;
1082 epte = spte+1; /* excluded */
1083 } else {
1084 spte = pmap_pte(map, (sva & ~(pde_mapped_size - 1)));
1085 spte = &spte[ptenum(sva)];
1086 epte = &spte[intel_btop(lva - sva)];
1087 }
1088
1089 for (; spte < epte; spte++) {
1090 if (!(*spte & INTEL_PTE_VALID))
1091 continue;
1092
1093 if (prot & VM_PROT_WRITE)
1094 pmap_update_pte(spte, *spte,
1095 *spte | INTEL_PTE_WRITE);
1096 else
1097 pmap_update_pte(spte, *spte,
1098 *spte & ~INTEL_PTE_WRITE);
1099
1100 if (set_NX)
1101 pmap_update_pte(spte, *spte,
1102 *spte | INTEL_PTE_NX);
1103 else
1104 pmap_update_pte(spte, *spte,
1105 *spte & ~INTEL_PTE_NX);
1106
1107 num_found++;
1108 }
1109 }
1110 sva = lva;
1111 }
1112 if (num_found)
1113 PMAP_UPDATE_TLBS(map, orig_sva, eva);
1114
1115 PMAP_UNLOCK(map);
1116
1117 PMAP_TRACE(PMAP_CODE(PMAP__PROTECT) | DBG_FUNC_END,
1118 0, 0, 0, 0, 0);
1119
1120 }
1121
1122 /* Map a (possibly) autogenned block */
1123 void
1124 pmap_map_block(
1125 pmap_t pmap,
1126 addr64_t va,
1127 ppnum_t pa,
1128 uint32_t size,
1129 vm_prot_t prot,
1130 int attr,
1131 __unused unsigned int flags)
1132 {
1133 uint32_t page;
1134 int cur_page_size;
1135
1136 if (attr & VM_MEM_SUPERPAGE)
1137 cur_page_size = SUPERPAGE_SIZE;
1138 else
1139 cur_page_size = PAGE_SIZE;
1140
1141 for (page = 0; page < size; page+=cur_page_size/PAGE_SIZE) {
1142 pmap_enter(pmap, va, pa, prot, attr, TRUE);
1143 va += cur_page_size;
1144 pa+=cur_page_size/PAGE_SIZE;
1145 }
1146 }
1147
1148 /*
1149 * Routine: pmap_change_wiring
1150 * Function: Change the wiring attribute for a map/virtual-address
1151 * pair.
1152 * In/out conditions:
1153 * The mapping must already exist in the pmap.
1154 */
1155 void
1156 pmap_change_wiring(
1157 pmap_t map,
1158 vm_map_offset_t vaddr,
1159 boolean_t wired)
1160 {
1161 pt_entry_t *pte;
1162
1163 PMAP_LOCK(map);
1164
1165 if ((pte = pmap_pte(map, vaddr)) == PT_ENTRY_NULL)
1166 panic("pmap_change_wiring: pte missing");
1167
1168 if (wired && !iswired(*pte)) {
1169 /*
1170 * wiring down mapping
1171 */
1172 OSAddAtomic(+1, &map->stats.wired_count);
1173 pmap_update_pte(pte, *pte, (*pte | INTEL_PTE_WIRED));
1174 }
1175 else if (!wired && iswired(*pte)) {
1176 /*
1177 * unwiring mapping
1178 */
1179 assert(map->stats.wired_count >= 1);
1180 OSAddAtomic(-1, &map->stats.wired_count);
1181 pmap_update_pte(pte, *pte, (*pte & ~INTEL_PTE_WIRED));
1182 }
1183
1184 PMAP_UNLOCK(map);
1185 }
1186
1187 void
1188 pmap_expand_pml4(
1189 pmap_t map,
1190 vm_map_offset_t vaddr)
1191 {
1192 vm_page_t m;
1193 pmap_paddr_t pa;
1194 uint64_t i;
1195 ppnum_t pn;
1196 pml4_entry_t *pml4p;
1197
1198 DBG("pmap_expand_pml4(%p,%p)\n", map, (void *)vaddr);
1199
1200 /*
1201 * Allocate a VM page for the pml4 page
1202 */
1203 while ((m = vm_page_grab()) == VM_PAGE_NULL)
1204 VM_PAGE_WAIT();
1205
1206 /*
1207 * put the page into the pmap's obj list so it
1208 * can be found later.
1209 */
1210 pn = m->phys_page;
1211 pa = i386_ptob(pn);
1212 i = pml4idx(map, vaddr);
1213
1214 /*
1215 * Zero the page.
1216 */
1217 pmap_zero_page(pn);
1218
1219 vm_page_lockspin_queues();
1220 vm_page_wire(m);
1221 vm_page_unlock_queues();
1222
1223 OSAddAtomic(1, &inuse_ptepages_count);
1224
1225 /* Take the oject lock (mutex) before the PMAP_LOCK (spinlock) */
1226 vm_object_lock(map->pm_obj_pml4);
1227
1228 PMAP_LOCK(map);
1229 /*
1230 * See if someone else expanded us first
1231 */
1232 if (pmap64_pdpt(map, vaddr) != PDPT_ENTRY_NULL) {
1233 PMAP_UNLOCK(map);
1234 vm_object_unlock(map->pm_obj_pml4);
1235
1236 VM_PAGE_FREE(m);
1237
1238 OSAddAtomic(-1, &inuse_ptepages_count);
1239 return;
1240 }
1241
1242 #if 0 /* DEBUG */
1243 if (0 != vm_page_lookup(map->pm_obj_pml4, (vm_object_offset_t)i)) {
1244 panic("pmap_expand_pml4: obj not empty, pmap %p pm_obj %p vaddr 0x%llx i 0x%llx\n",
1245 map, map->pm_obj_pml4, vaddr, i);
1246 }
1247 #endif
1248 vm_page_insert(m, map->pm_obj_pml4, (vm_object_offset_t)i);
1249 vm_object_unlock(map->pm_obj_pml4);
1250
1251 /*
1252 * Set the page directory entry for this page table.
1253 */
1254 pml4p = pmap64_pml4(map, vaddr); /* refetch under lock */
1255
1256 pmap_store_pte(pml4p, pa_to_pte(pa)
1257 | INTEL_PTE_VALID
1258 | INTEL_PTE_USER
1259 | INTEL_PTE_WRITE);
1260
1261 PMAP_UNLOCK(map);
1262
1263 return;
1264 }
1265
1266 void
1267 pmap_expand_pdpt(
1268 pmap_t map,
1269 vm_map_offset_t vaddr)
1270 {
1271 vm_page_t m;
1272 pmap_paddr_t pa;
1273 uint64_t i;
1274 ppnum_t pn;
1275 pdpt_entry_t *pdptp;
1276
1277 DBG("pmap_expand_pdpt(%p,%p)\n", map, (void *)vaddr);
1278
1279 while ((pdptp = pmap64_pdpt(map, vaddr)) == PDPT_ENTRY_NULL) {
1280 pmap_expand_pml4(map, vaddr);
1281 }
1282
1283 /*
1284 * Allocate a VM page for the pdpt page
1285 */
1286 while ((m = vm_page_grab()) == VM_PAGE_NULL)
1287 VM_PAGE_WAIT();
1288
1289 /*
1290 * put the page into the pmap's obj list so it
1291 * can be found later.
1292 */
1293 pn = m->phys_page;
1294 pa = i386_ptob(pn);
1295 i = pdptidx(map, vaddr);
1296
1297 /*
1298 * Zero the page.
1299 */
1300 pmap_zero_page(pn);
1301
1302 vm_page_lockspin_queues();
1303 vm_page_wire(m);
1304 vm_page_unlock_queues();
1305
1306 OSAddAtomic(1, &inuse_ptepages_count);
1307
1308 /* Take the oject lock (mutex) before the PMAP_LOCK (spinlock) */
1309 vm_object_lock(map->pm_obj_pdpt);
1310
1311 PMAP_LOCK(map);
1312 /*
1313 * See if someone else expanded us first
1314 */
1315 if (pmap64_pde(map, vaddr) != PD_ENTRY_NULL) {
1316 PMAP_UNLOCK(map);
1317 vm_object_unlock(map->pm_obj_pdpt);
1318
1319 VM_PAGE_FREE(m);
1320
1321 OSAddAtomic(-1, &inuse_ptepages_count);
1322 return;
1323 }
1324
1325 #if 0 /* DEBUG */
1326 if (0 != vm_page_lookup(map->pm_obj_pdpt, (vm_object_offset_t)i)) {
1327 panic("pmap_expand_pdpt: obj not empty, pmap %p pm_obj %p vaddr 0x%llx i 0x%llx\n",
1328 map, map->pm_obj_pdpt, vaddr, i);
1329 }
1330 #endif
1331 vm_page_insert(m, map->pm_obj_pdpt, (vm_object_offset_t)i);
1332 vm_object_unlock(map->pm_obj_pdpt);
1333
1334 /*
1335 * Set the page directory entry for this page table.
1336 */
1337 pdptp = pmap64_pdpt(map, vaddr); /* refetch under lock */
1338
1339 pmap_store_pte(pdptp, pa_to_pte(pa)
1340 | INTEL_PTE_VALID
1341 | INTEL_PTE_USER
1342 | INTEL_PTE_WRITE);
1343
1344 PMAP_UNLOCK(map);
1345
1346 return;
1347
1348 }
1349
1350
1351
1352 /*
1353 * Routine: pmap_expand
1354 *
1355 * Expands a pmap to be able to map the specified virtual address.
1356 *
1357 * Allocates new virtual memory for the P0 or P1 portion of the
1358 * pmap, then re-maps the physical pages that were in the old
1359 * pmap to be in the new pmap.
1360 *
1361 * Must be called with the pmap system and the pmap unlocked,
1362 * since these must be unlocked to use vm_allocate or vm_deallocate.
1363 * Thus it must be called in a loop that checks whether the map
1364 * has been expanded enough.
1365 * (We won't loop forever, since page tables aren't shrunk.)
1366 */
1367 void
1368 pmap_expand(
1369 pmap_t map,
1370 vm_map_offset_t vaddr)
1371 {
1372 pt_entry_t *pdp;
1373 register vm_page_t m;
1374 register pmap_paddr_t pa;
1375 uint64_t i;
1376 ppnum_t pn;
1377
1378
1379 /*
1380 * For the kernel, the virtual address must be in or above the basement
1381 * which is for kexts and is in the 512GB immediately below the kernel..
1382 * XXX - should use VM_MIN_KERNEL_AND_KEXT_ADDRESS not KERNEL_BASEMENT
1383 */
1384 if (map == kernel_pmap &&
1385 !(vaddr >= KERNEL_BASEMENT && vaddr <= VM_MAX_KERNEL_ADDRESS))
1386 panic("pmap_expand: bad vaddr 0x%llx for kernel pmap", vaddr);
1387
1388
1389 while ((pdp = pmap64_pde(map, vaddr)) == PD_ENTRY_NULL) {
1390 /* need room for another pde entry */
1391 pmap_expand_pdpt(map, vaddr);
1392 }
1393
1394 /*
1395 * Allocate a VM page for the pde entries.
1396 */
1397 while ((m = vm_page_grab()) == VM_PAGE_NULL)
1398 VM_PAGE_WAIT();
1399
1400 /*
1401 * put the page into the pmap's obj list so it
1402 * can be found later.
1403 */
1404 pn = m->phys_page;
1405 pa = i386_ptob(pn);
1406 i = pdeidx(map, vaddr);
1407
1408 /*
1409 * Zero the page.
1410 */
1411 pmap_zero_page(pn);
1412
1413 vm_page_lockspin_queues();
1414 vm_page_wire(m);
1415 vm_page_unlock_queues();
1416
1417 OSAddAtomic(1, &inuse_ptepages_count);
1418
1419 /* Take the oject lock (mutex) before the PMAP_LOCK (spinlock) */
1420 vm_object_lock(map->pm_obj);
1421
1422 PMAP_LOCK(map);
1423
1424 /*
1425 * See if someone else expanded us first
1426 */
1427 if (pmap_pte(map, vaddr) != PT_ENTRY_NULL) {
1428 PMAP_UNLOCK(map);
1429 vm_object_unlock(map->pm_obj);
1430
1431 VM_PAGE_FREE(m);
1432
1433 OSAddAtomic(-1, &inuse_ptepages_count);
1434 return;
1435 }
1436
1437 #if 0 /* DEBUG */
1438 if (0 != vm_page_lookup(map->pm_obj, (vm_object_offset_t)i)) {
1439 panic("pmap_expand: obj not empty, pmap 0x%x pm_obj 0x%x vaddr 0x%llx i 0x%llx\n",
1440 map, map->pm_obj, vaddr, i);
1441 }
1442 #endif
1443 vm_page_insert(m, map->pm_obj, (vm_object_offset_t)i);
1444 vm_object_unlock(map->pm_obj);
1445
1446 /*
1447 * Set the page directory entry for this page table.
1448 */
1449 pdp = pmap_pde(map, vaddr);
1450 pmap_store_pte(pdp, pa_to_pte(pa)
1451 | INTEL_PTE_VALID
1452 | INTEL_PTE_USER
1453 | INTEL_PTE_WRITE);
1454
1455 PMAP_UNLOCK(map);
1456
1457 return;
1458 }
1459
1460 /* On K64 machines with more than 32GB of memory, pmap_steal_memory
1461 * will allocate past the 1GB of pre-expanded virtual kernel area. This
1462 * function allocates all the page tables using memory from the same pool
1463 * that pmap_steal_memory uses, rather than calling vm_page_grab (which
1464 * isn't available yet). */
1465 void
1466 pmap_pre_expand(pmap_t pmap, vm_map_offset_t vaddr) {
1467 ppnum_t pn;
1468 pt_entry_t *pte;
1469
1470 PMAP_LOCK(pmap);
1471
1472 if(pmap64_pdpt(pmap, vaddr) == PDPT_ENTRY_NULL) {
1473 if (!pmap_next_page_k64(&pn))
1474 panic("pmap_pre_expand");
1475
1476 pmap_zero_page(pn);
1477
1478 pte = pmap64_pml4(pmap, vaddr);
1479
1480 pmap_store_pte(pte, pa_to_pte(i386_ptob(pn))
1481 | INTEL_PTE_VALID
1482 | INTEL_PTE_USER
1483 | INTEL_PTE_WRITE);
1484 }
1485
1486 if(pmap64_pde(pmap, vaddr) == PD_ENTRY_NULL) {
1487 if (!pmap_next_page_k64(&pn))
1488 panic("pmap_pre_expand");
1489
1490 pmap_zero_page(pn);
1491
1492 pte = pmap64_pdpt(pmap, vaddr);
1493
1494 pmap_store_pte(pte, pa_to_pte(i386_ptob(pn))
1495 | INTEL_PTE_VALID
1496 | INTEL_PTE_USER
1497 | INTEL_PTE_WRITE);
1498 }
1499
1500 if(pmap_pte(pmap, vaddr) == PT_ENTRY_NULL) {
1501 if (!pmap_next_page_k64(&pn))
1502 panic("pmap_pre_expand");
1503
1504 pmap_zero_page(pn);
1505
1506 pte = pmap64_pde(pmap, vaddr);
1507
1508 pmap_store_pte(pte, pa_to_pte(i386_ptob(pn))
1509 | INTEL_PTE_VALID
1510 | INTEL_PTE_USER
1511 | INTEL_PTE_WRITE);
1512 }
1513
1514 PMAP_UNLOCK(pmap);
1515 }
1516
1517 /*
1518 * pmap_sync_page_data_phys(ppnum_t pa)
1519 *
1520 * Invalidates all of the instruction cache on a physical page and
1521 * pushes any dirty data from the data cache for the same physical page
1522 * Not required in i386.
1523 */
1524 void
1525 pmap_sync_page_data_phys(__unused ppnum_t pa)
1526 {
1527 return;
1528 }
1529
1530 /*
1531 * pmap_sync_page_attributes_phys(ppnum_t pa)
1532 *
1533 * Write back and invalidate all cachelines on a physical page.
1534 */
1535 void
1536 pmap_sync_page_attributes_phys(ppnum_t pa)
1537 {
1538 cache_flush_page_phys(pa);
1539 }
1540
1541
1542
1543 #ifdef CURRENTLY_UNUSED_AND_UNTESTED
1544
1545 int collect_ref;
1546 int collect_unref;
1547
1548 /*
1549 * Routine: pmap_collect
1550 * Function:
1551 * Garbage collects the physical map system for
1552 * pages which are no longer used.
1553 * Success need not be guaranteed -- that is, there
1554 * may well be pages which are not referenced, but
1555 * others may be collected.
1556 * Usage:
1557 * Called by the pageout daemon when pages are scarce.
1558 */
1559 void
1560 pmap_collect(
1561 pmap_t p)
1562 {
1563 register pt_entry_t *pdp, *ptp;
1564 pt_entry_t *eptp;
1565 int wired;
1566
1567 if (p == PMAP_NULL)
1568 return;
1569
1570 if (p == kernel_pmap)
1571 return;
1572
1573 /*
1574 * Garbage collect map.
1575 */
1576 PMAP_LOCK(p);
1577
1578 for (pdp = (pt_entry_t *)p->dirbase;
1579 pdp < (pt_entry_t *)&p->dirbase[(UMAXPTDI+1)];
1580 pdp++)
1581 {
1582 if (*pdp & INTEL_PTE_VALID) {
1583 if(*pdp & INTEL_PTE_REF) {
1584 pmap_store_pte(pdp, *pdp & ~INTEL_PTE_REF);
1585 collect_ref++;
1586 } else {
1587 collect_unref++;
1588 ptp = pmap_pte(p, pdetova(pdp - (pt_entry_t *)p->dirbase));
1589 eptp = ptp + NPTEPG;
1590
1591 /*
1592 * If the pte page has any wired mappings, we cannot
1593 * free it.
1594 */
1595 wired = 0;
1596 {
1597 register pt_entry_t *ptep;
1598 for (ptep = ptp; ptep < eptp; ptep++) {
1599 if (iswired(*ptep)) {
1600 wired = 1;
1601 break;
1602 }
1603 }
1604 }
1605 if (!wired) {
1606 /*
1607 * Remove the virtual addresses mapped by this pte page.
1608 */
1609 pmap_remove_range(p,
1610 pdetova(pdp - (pt_entry_t *)p->dirbase),
1611 ptp,
1612 eptp);
1613
1614 /*
1615 * Invalidate the page directory pointer.
1616 */
1617 pmap_store_pte(pdp, 0x0);
1618
1619 PMAP_UNLOCK(p);
1620
1621 /*
1622 * And free the pte page itself.
1623 */
1624 {
1625 register vm_page_t m;
1626
1627 vm_object_lock(p->pm_obj);
1628
1629 m = vm_page_lookup(p->pm_obj,(vm_object_offset_t)(pdp - (pt_entry_t *)&p->dirbase[0]));
1630 if (m == VM_PAGE_NULL)
1631 panic("pmap_collect: pte page not in object");
1632
1633 VM_PAGE_FREE(m);
1634
1635 OSAddAtomic(-1, &inuse_ptepages_count);
1636
1637 vm_object_unlock(p->pm_obj);
1638 }
1639
1640 PMAP_LOCK(p);
1641 }
1642 }
1643 }
1644 }
1645
1646 PMAP_UPDATE_TLBS(p, 0x0, 0xFFFFFFFFFFFFF000ULL);
1647 PMAP_UNLOCK(p);
1648 return;
1649
1650 }
1651 #endif
1652
1653
1654 void
1655 pmap_copy_page(ppnum_t src, ppnum_t dst)
1656 {
1657 bcopy_phys((addr64_t)i386_ptob(src),
1658 (addr64_t)i386_ptob(dst),
1659 PAGE_SIZE);
1660 }
1661
1662
1663 /*
1664 * Routine: pmap_pageable
1665 * Function:
1666 * Make the specified pages (by pmap, offset)
1667 * pageable (or not) as requested.
1668 *
1669 * A page which is not pageable may not take
1670 * a fault; therefore, its page table entry
1671 * must remain valid for the duration.
1672 *
1673 * This routine is merely advisory; pmap_enter
1674 * will specify that these pages are to be wired
1675 * down (or not) as appropriate.
1676 */
1677 void
1678 pmap_pageable(
1679 __unused pmap_t pmap,
1680 __unused vm_map_offset_t start_addr,
1681 __unused vm_map_offset_t end_addr,
1682 __unused boolean_t pageable)
1683 {
1684 #ifdef lint
1685 pmap++; start_addr++; end_addr++; pageable++;
1686 #endif /* lint */
1687 }
1688
1689 /*
1690 * Clear specified attribute bits.
1691 */
1692 void
1693 phys_attribute_clear(
1694 ppnum_t pn,
1695 int bits)
1696 {
1697 pv_rooted_entry_t pv_h;
1698 pv_hashed_entry_t pv_e;
1699 pt_entry_t *pte;
1700 int pai;
1701 pmap_t pmap;
1702
1703 pmap_intr_assert();
1704 assert(pn != vm_page_fictitious_addr);
1705 if (pn == vm_page_guard_addr)
1706 return;
1707
1708 pai = ppn_to_pai(pn);
1709
1710 if (!IS_MANAGED_PAGE(pai)) {
1711 /*
1712 * Not a managed page.
1713 */
1714 return;
1715 }
1716
1717
1718 PMAP_TRACE(PMAP_CODE(PMAP__ATTRIBUTE_CLEAR) | DBG_FUNC_START,
1719 pn, bits, 0, 0, 0);
1720
1721 pv_h = pai_to_pvh(pai);
1722
1723 LOCK_PVH(pai);
1724
1725 /*
1726 * Walk down PV list, clearing all modify or reference bits.
1727 * We do not have to lock the pv_list because we have
1728 * the entire pmap system locked.
1729 */
1730 if (pv_h->pmap != PMAP_NULL) {
1731 /*
1732 * There are some mappings.
1733 */
1734
1735 pv_e = (pv_hashed_entry_t)pv_h;
1736
1737 do {
1738 vm_map_offset_t va;
1739
1740 pmap = pv_e->pmap;
1741 va = pv_e->va;
1742
1743 /*
1744 * Clear modify and/or reference bits.
1745 */
1746 pte = pmap_pte(pmap, va);
1747 pmap_update_pte(pte, *pte, (*pte & ~bits));
1748 /* Ensure all processors using this translation
1749 * invalidate this TLB entry. The invalidation *must*
1750 * follow the PTE update, to ensure that the TLB
1751 * shadow of the 'D' bit (in particular) is
1752 * synchronized with the updated PTE.
1753 */
1754 PMAP_UPDATE_TLBS(pmap, va, va + PAGE_SIZE);
1755
1756 pv_e = (pv_hashed_entry_t)queue_next(&pv_e->qlink);
1757
1758 } while (pv_e != (pv_hashed_entry_t)pv_h);
1759 }
1760 pmap_phys_attributes[pai] &= ~bits;
1761
1762 UNLOCK_PVH(pai);
1763
1764 PMAP_TRACE(PMAP_CODE(PMAP__ATTRIBUTE_CLEAR) | DBG_FUNC_END,
1765 0, 0, 0, 0, 0);
1766 }
1767
1768 /*
1769 * Check specified attribute bits.
1770 */
1771 int
1772 phys_attribute_test(
1773 ppnum_t pn,
1774 int bits)
1775 {
1776 pv_rooted_entry_t pv_h;
1777 pv_hashed_entry_t pv_e;
1778 pt_entry_t *pte;
1779 int pai;
1780 pmap_t pmap;
1781 int attributes = 0;
1782
1783 pmap_intr_assert();
1784 assert(pn != vm_page_fictitious_addr);
1785 if (pn == vm_page_guard_addr)
1786 return 0;
1787
1788 pai = ppn_to_pai(pn);
1789
1790 if (!IS_MANAGED_PAGE(pai)) {
1791 /*
1792 * Not a managed page.
1793 */
1794 return 0;
1795 }
1796
1797 /*
1798 * super fast check... if bits already collected
1799 * no need to take any locks...
1800 * if not set, we need to recheck after taking
1801 * the lock in case they got pulled in while
1802 * we were waiting for the lock
1803 */
1804 if ((pmap_phys_attributes[pai] & bits) == bits)
1805 return bits;
1806
1807 pv_h = pai_to_pvh(pai);
1808
1809 LOCK_PVH(pai);
1810
1811 attributes = pmap_phys_attributes[pai] & bits;
1812
1813
1814 /*
1815 * Walk down PV list, checking the mappings until we
1816 * reach the end or we've found the attributes we've asked for
1817 * We do not have to lock the pv_list because we have
1818 * the entire pmap system locked.
1819 */
1820 if (attributes != bits &&
1821 pv_h->pmap != PMAP_NULL) {
1822 /*
1823 * There are some mappings.
1824 */
1825 pv_e = (pv_hashed_entry_t)pv_h;
1826 do {
1827 vm_map_offset_t va;
1828
1829 pmap = pv_e->pmap;
1830 va = pv_e->va;
1831 /*
1832 * first make sure any processor actively
1833 * using this pmap, flushes its TLB state
1834 */
1835 PMAP_UPDATE_TLBS(pmap, va, va + PAGE_SIZE);
1836
1837 /*
1838 * pick up modify and/or reference bits from mapping
1839 */
1840
1841 pte = pmap_pte(pmap, va);
1842 attributes |= (int)(*pte & bits);
1843
1844 pv_e = (pv_hashed_entry_t)queue_next(&pv_e->qlink);
1845
1846 } while ((attributes != bits) &&
1847 (pv_e != (pv_hashed_entry_t)pv_h));
1848 }
1849
1850 UNLOCK_PVH(pai);
1851 return (attributes);
1852 }
1853
1854 /*
1855 * Set specified attribute bits.
1856 */
1857 void
1858 phys_attribute_set(
1859 ppnum_t pn,
1860 int bits)
1861 {
1862 int pai;
1863
1864 pmap_intr_assert();
1865 assert(pn != vm_page_fictitious_addr);
1866 if (pn == vm_page_guard_addr)
1867 return;
1868
1869 pai = ppn_to_pai(pn);
1870
1871 if (!IS_MANAGED_PAGE(pai)) {
1872 /* Not a managed page. */
1873 return;
1874 }
1875
1876 LOCK_PVH(pai);
1877 pmap_phys_attributes[pai] |= bits;
1878 UNLOCK_PVH(pai);
1879 }
1880
1881 /*
1882 * Set the modify bit on the specified physical page.
1883 */
1884
1885 void
1886 pmap_set_modify(ppnum_t pn)
1887 {
1888 phys_attribute_set(pn, PHYS_MODIFIED);
1889 }
1890
1891 /*
1892 * Clear the modify bits on the specified physical page.
1893 */
1894
1895 void
1896 pmap_clear_modify(ppnum_t pn)
1897 {
1898 phys_attribute_clear(pn, PHYS_MODIFIED);
1899 }
1900
1901 /*
1902 * pmap_is_modified:
1903 *
1904 * Return whether or not the specified physical page is modified
1905 * by any physical maps.
1906 */
1907
1908 boolean_t
1909 pmap_is_modified(ppnum_t pn)
1910 {
1911 if (phys_attribute_test(pn, PHYS_MODIFIED))
1912 return TRUE;
1913 return FALSE;
1914 }
1915
1916 /*
1917 * pmap_clear_reference:
1918 *
1919 * Clear the reference bit on the specified physical page.
1920 */
1921
1922 void
1923 pmap_clear_reference(ppnum_t pn)
1924 {
1925 phys_attribute_clear(pn, PHYS_REFERENCED);
1926 }
1927
1928 void
1929 pmap_set_reference(ppnum_t pn)
1930 {
1931 phys_attribute_set(pn, PHYS_REFERENCED);
1932 }
1933
1934 /*
1935 * pmap_is_referenced:
1936 *
1937 * Return whether or not the specified physical page is referenced
1938 * by any physical maps.
1939 */
1940
1941 boolean_t
1942 pmap_is_referenced(ppnum_t pn)
1943 {
1944 if (phys_attribute_test(pn, PHYS_REFERENCED))
1945 return TRUE;
1946 return FALSE;
1947 }
1948
1949 /*
1950 * pmap_get_refmod(phys)
1951 * returns the referenced and modified bits of the specified
1952 * physical page.
1953 */
1954 unsigned int
1955 pmap_get_refmod(ppnum_t pn)
1956 {
1957 int refmod;
1958 unsigned int retval = 0;
1959
1960 refmod = phys_attribute_test(pn, PHYS_MODIFIED | PHYS_REFERENCED);
1961
1962 if (refmod & PHYS_MODIFIED)
1963 retval |= VM_MEM_MODIFIED;
1964 if (refmod & PHYS_REFERENCED)
1965 retval |= VM_MEM_REFERENCED;
1966
1967 return (retval);
1968 }
1969
1970 /*
1971 * pmap_clear_refmod(phys, mask)
1972 * clears the referenced and modified bits as specified by the mask
1973 * of the specified physical page.
1974 */
1975 void
1976 pmap_clear_refmod(ppnum_t pn, unsigned int mask)
1977 {
1978 unsigned int x86Mask;
1979
1980 x86Mask = ( ((mask & VM_MEM_MODIFIED)? PHYS_MODIFIED : 0)
1981 | ((mask & VM_MEM_REFERENCED)? PHYS_REFERENCED : 0));
1982 phys_attribute_clear(pn, x86Mask);
1983 }
1984
1985 void
1986 invalidate_icache(__unused vm_offset_t addr,
1987 __unused unsigned cnt,
1988 __unused int phys)
1989 {
1990 return;
1991 }
1992
1993 void
1994 flush_dcache(__unused vm_offset_t addr,
1995 __unused unsigned count,
1996 __unused int phys)
1997 {
1998 return;
1999 }
2000
2001 #if CONFIG_DTRACE
2002 /*
2003 * Constrain DTrace copyin/copyout actions
2004 */
2005 extern kern_return_t dtrace_copyio_preflight(addr64_t);
2006 extern kern_return_t dtrace_copyio_postflight(addr64_t);
2007
2008 kern_return_t dtrace_copyio_preflight(__unused addr64_t va)
2009 {
2010 thread_t thread = current_thread();
2011
2012 if (current_map() == kernel_map)
2013 return KERN_FAILURE;
2014 else if (get_cr3() != thread->map->pmap->pm_cr3)
2015 return KERN_FAILURE;
2016 else if (thread->machine.specFlags & CopyIOActive)
2017 return KERN_FAILURE;
2018 else
2019 return KERN_SUCCESS;
2020 }
2021
2022 kern_return_t dtrace_copyio_postflight(__unused addr64_t va)
2023 {
2024 return KERN_SUCCESS;
2025 }
2026 #endif /* CONFIG_DTRACE */
2027
2028 #include <mach_vm_debug.h>
2029 #if MACH_VM_DEBUG
2030 #include <vm/vm_debug.h>
2031
2032 int
2033 pmap_list_resident_pages(
2034 __unused pmap_t pmap,
2035 __unused vm_offset_t *listp,
2036 __unused int space)
2037 {
2038 return 0;
2039 }
2040 #endif /* MACH_VM_DEBUG */
2041
2042
2043
2044 /* temporary workaround */
2045 boolean_t
2046 coredumpok(__unused vm_map_t map, __unused vm_offset_t va)
2047 {
2048 #if 0
2049 pt_entry_t *ptep;
2050
2051 ptep = pmap_pte(map->pmap, va);
2052 if (0 == ptep)
2053 return FALSE;
2054 return ((*ptep & (INTEL_PTE_NCACHE | INTEL_PTE_WIRED)) != (INTEL_PTE_NCACHE | INTEL_PTE_WIRED));
2055 #else
2056 return TRUE;
2057 #endif
2058 }
2059
2060
2061 boolean_t
2062 phys_page_exists(ppnum_t pn)
2063 {
2064 assert(pn != vm_page_fictitious_addr);
2065
2066 if (!pmap_initialized)
2067 return TRUE;
2068
2069 if (pn == vm_page_guard_addr)
2070 return FALSE;
2071
2072 if (!IS_MANAGED_PAGE(ppn_to_pai(pn)))
2073 return FALSE;
2074
2075 return TRUE;
2076 }
2077
2078 void
2079 pmap_switch(pmap_t tpmap)
2080 {
2081 spl_t s;
2082
2083 s = splhigh(); /* Make sure interruptions are disabled */
2084 set_dirbase(tpmap, current_thread());
2085 splx(s);
2086 }
2087
2088
2089 /*
2090 * disable no-execute capability on
2091 * the specified pmap
2092 */
2093 void
2094 pmap_disable_NX(pmap_t pmap)
2095 {
2096 pmap->nx_enabled = 0;
2097 }
2098
2099 void
2100 pt_fake_zone_info(
2101 int *count,
2102 vm_size_t *cur_size,
2103 vm_size_t *max_size,
2104 vm_size_t *elem_size,
2105 vm_size_t *alloc_size,
2106 int *collectable,
2107 int *exhaustable)
2108 {
2109 *count = inuse_ptepages_count;
2110 *cur_size = PAGE_SIZE * inuse_ptepages_count;
2111 *max_size = PAGE_SIZE * (inuse_ptepages_count +
2112 vm_page_inactive_count +
2113 vm_page_active_count +
2114 vm_page_free_count);
2115 *elem_size = PAGE_SIZE;
2116 *alloc_size = PAGE_SIZE;
2117
2118 *collectable = 1;
2119 *exhaustable = 0;
2120 }
2121
2122 static inline void
2123 pmap_cpuset_NMIPI(cpu_set cpu_mask) {
2124 unsigned int cpu, cpu_bit;
2125 uint64_t deadline;
2126
2127 for (cpu = 0, cpu_bit = 1; cpu < real_ncpus; cpu++, cpu_bit <<= 1) {
2128 if (cpu_mask & cpu_bit)
2129 cpu_NMI_interrupt(cpu);
2130 }
2131 deadline = mach_absolute_time() + (LockTimeOut);
2132 while (mach_absolute_time() < deadline)
2133 cpu_pause();
2134 }
2135
2136 /*
2137 * Called with pmap locked, we:
2138 * - scan through per-cpu data to see which other cpus need to flush
2139 * - send an IPI to each non-idle cpu to be flushed
2140 * - wait for all to signal back that they are inactive or we see that
2141 * they are at a safe point (idle).
2142 * - flush the local tlb if active for this pmap
2143 * - return ... the caller will unlock the pmap
2144 */
2145 void
2146 pmap_flush_tlbs(pmap_t pmap)
2147 {
2148 unsigned int cpu;
2149 unsigned int cpu_bit;
2150 cpu_set cpus_to_signal;
2151 unsigned int my_cpu = cpu_number();
2152 pmap_paddr_t pmap_cr3 = pmap->pm_cr3;
2153 boolean_t flush_self = FALSE;
2154 uint64_t deadline;
2155
2156 assert((processor_avail_count < 2) ||
2157 (ml_get_interrupts_enabled() && get_preemption_level() != 0));
2158
2159 /*
2160 * Scan other cpus for matching active or task CR3.
2161 * For idle cpus (with no active map) we mark them invalid but
2162 * don't signal -- they'll check as they go busy.
2163 */
2164 cpus_to_signal = 0;
2165 for (cpu = 0, cpu_bit = 1; cpu < real_ncpus; cpu++, cpu_bit <<= 1) {
2166 if (!cpu_datap(cpu)->cpu_running)
2167 continue;
2168 uint64_t cpu_active_cr3 = CPU_GET_ACTIVE_CR3(cpu);
2169 uint64_t cpu_task_cr3 = CPU_GET_TASK_CR3(cpu);
2170
2171 if ((pmap_cr3 == cpu_task_cr3) ||
2172 (pmap_cr3 == cpu_active_cr3) ||
2173 (pmap->pm_shared) ||
2174 (pmap == kernel_pmap)) {
2175 if (cpu == my_cpu) {
2176 flush_self = TRUE;
2177 continue;
2178 }
2179 cpu_datap(cpu)->cpu_tlb_invalid = TRUE;
2180 __asm__ volatile("mfence");
2181
2182 /*
2183 * We don't need to signal processors which will flush
2184 * lazily at the idle state or kernel boundary.
2185 * For example, if we're invalidating the kernel pmap,
2186 * processors currently in userspace don't need to flush
2187 * their TLBs until the next time they enter the kernel.
2188 * Alterations to the address space of a task active
2189 * on a remote processor result in a signal, to
2190 * account for copy operations. (There may be room
2191 * for optimization in such cases).
2192 * The order of the loads below with respect
2193 * to the store to the "cpu_tlb_invalid" field above
2194 * is important--hence the barrier.
2195 */
2196 if (CPU_CR3_IS_ACTIVE(cpu) &&
2197 (pmap_cr3 == CPU_GET_ACTIVE_CR3(cpu) ||
2198 pmap->pm_shared ||
2199 (pmap_cr3 == CPU_GET_TASK_CR3(cpu)))) {
2200 cpus_to_signal |= cpu_bit;
2201 i386_signal_cpu(cpu, MP_TLB_FLUSH, ASYNC);
2202 }
2203 }
2204 }
2205
2206 PMAP_TRACE(PMAP_CODE(PMAP__FLUSH_TLBS) | DBG_FUNC_START,
2207 pmap, cpus_to_signal, flush_self, 0, 0);
2208
2209 /*
2210 * Flush local tlb if required.
2211 * Do this now to overlap with other processors responding.
2212 */
2213 if (flush_self)
2214 flush_tlb();
2215
2216 if (cpus_to_signal) {
2217 cpu_set cpus_to_respond = cpus_to_signal;
2218
2219 deadline = mach_absolute_time() + LockTimeOut;
2220 /*
2221 * Wait for those other cpus to acknowledge
2222 */
2223 while (cpus_to_respond != 0) {
2224 if (mach_absolute_time() > deadline) {
2225 if (mp_recent_debugger_activity())
2226 continue;
2227 if (!panic_active()) {
2228 pmap_tlb_flush_timeout = TRUE;
2229 pmap_cpuset_NMIPI(cpus_to_respond);
2230 }
2231 panic("pmap_flush_tlbs() timeout: "
2232 "cpu(s) failing to respond to interrupts, pmap=%p cpus_to_respond=0x%lx",
2233 pmap, cpus_to_respond);
2234 }
2235
2236 for (cpu = 0, cpu_bit = 1; cpu < real_ncpus; cpu++, cpu_bit <<= 1) {
2237 if ((cpus_to_respond & cpu_bit) != 0) {
2238 if (!cpu_datap(cpu)->cpu_running ||
2239 cpu_datap(cpu)->cpu_tlb_invalid == FALSE ||
2240 !CPU_CR3_IS_ACTIVE(cpu)) {
2241 cpus_to_respond &= ~cpu_bit;
2242 }
2243 cpu_pause();
2244 }
2245 if (cpus_to_respond == 0)
2246 break;
2247 }
2248 }
2249 }
2250
2251 PMAP_TRACE(PMAP_CODE(PMAP__FLUSH_TLBS) | DBG_FUNC_END,
2252 pmap, cpus_to_signal, flush_self, 0, 0);
2253 }
2254
2255 void
2256 process_pmap_updates(void)
2257 {
2258 assert(ml_get_interrupts_enabled() == 0 || get_preemption_level() != 0);
2259
2260 flush_tlb();
2261
2262 current_cpu_datap()->cpu_tlb_invalid = FALSE;
2263 __asm__ volatile("mfence");
2264 }
2265
2266 void
2267 pmap_update_interrupt(void)
2268 {
2269 PMAP_TRACE(PMAP_CODE(PMAP__UPDATE_INTERRUPT) | DBG_FUNC_START,
2270 0, 0, 0, 0, 0);
2271
2272 process_pmap_updates();
2273
2274 PMAP_TRACE(PMAP_CODE(PMAP__UPDATE_INTERRUPT) | DBG_FUNC_END,
2275 0, 0, 0, 0, 0);
2276 }
2277
2278
2279 unsigned int
2280 pmap_cache_attributes(ppnum_t pn)
2281 {
2282 return IS_MANAGED_PAGE(ppn_to_pai(pn)) ? VM_WIMG_COPYBACK
2283 : VM_WIMG_IO;
2284 }
2285
2286