2 * Copyright (c) 2000-2010 Apple Inc. All rights reserved.
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
29 #include <kern/ledger.h>
30 #include <i386/pmap_internal.h>
34 * Each entry in the pv_head_table is locked by a bit in the
35 * pv_lock_table. The lock bits are accessed by the physical
36 * address of the page they lock.
39 char *pv_lock_table
; /* pointer to array of bits */
40 char *pv_hash_lock_table
;
42 pv_rooted_entry_t pv_head_table
; /* array of entries, one per
44 uint32_t pv_hashed_free_count
= 0;
45 uint32_t pv_hashed_kern_free_count
= 0;
47 pmap_pagetable_corruption_record_t pmap_pagetable_corruption_records
[PMAP_PAGETABLE_CORRUPTION_MAX_LOG
];
48 uint32_t pmap_pagetable_corruption_incidents
;
49 uint64_t pmap_pagetable_corruption_last_abstime
= (~(0ULL) >> 1);
50 uint64_t pmap_pagetable_corruption_interval_abstime
;
51 thread_call_t pmap_pagetable_corruption_log_call
;
52 static thread_call_data_t pmap_pagetable_corruption_log_call_data
;
53 boolean_t pmap_pagetable_corruption_timeout
= FALSE
;
55 volatile uint32_t mappingrecurse
= 0;
57 uint32_t pv_hashed_low_water_mark
, pv_hashed_kern_low_water_mark
, pv_hashed_alloc_chunk
, pv_hashed_kern_alloc_chunk
;
59 thread_t mapping_replenish_thread
;
60 event_t mapping_replenish_event
, pmap_user_pv_throttle_event
;
62 uint64_t pmap_pv_throttle_stat
, pmap_pv_throttled_waiters
;
64 unsigned int pmap_cache_attributes(ppnum_t pn
) {
65 if (pmap_get_cache_attributes(pn
) & INTEL_PTE_NCACHE
)
68 return (VM_WIMG_COPYBACK
);
71 void pmap_set_cache_attributes(ppnum_t pn
, unsigned int cacheattr
) {
72 unsigned int current
, template = 0;
75 if (cacheattr
& VM_MEM_NOT_CACHEABLE
) {
76 if(!(cacheattr
& VM_MEM_GUARDED
))
78 template |= PHYS_NCACHE
;
83 assert((pn
!= vm_page_fictitious_addr
) && (pn
!= vm_page_guard_addr
));
87 if (!IS_MANAGED_PAGE(pai
)) {
91 /* override cache attributes for this phys page
92 * Does not walk through existing mappings to adjust,
93 * assumes page is disconnected
98 pmap_update_cache_attributes_locked(pn
, template);
100 current
= pmap_phys_attributes
[pai
] & PHYS_CACHEABILITY_MASK
;
101 pmap_phys_attributes
[pai
] &= ~PHYS_CACHEABILITY_MASK
;
102 pmap_phys_attributes
[pai
] |= template;
106 if ((template & PHYS_NCACHE
) && !(current
& PHYS_NCACHE
)) {
107 pmap_sync_page_attributes_phys(pn
);
111 unsigned pmap_get_cache_attributes(ppnum_t pn
) {
112 if (last_managed_page
== 0)
115 if (!IS_MANAGED_PAGE(ppn_to_pai(pn
))) {
116 return INTEL_PTE_NCACHE
;
120 * The cache attributes are read locklessly for efficiency.
122 unsigned int attr
= pmap_phys_attributes
[ppn_to_pai(pn
)];
123 unsigned int template = 0;
126 template |= INTEL_PTE_PTA
;
127 if (attr
& PHYS_NCACHE
)
128 template |= INTEL_PTE_NCACHE
;
135 pmap_is_noencrypt(ppnum_t pn
)
139 pai
= ppn_to_pai(pn
);
141 if (!IS_MANAGED_PAGE(pai
))
144 if (pmap_phys_attributes
[pai
] & PHYS_NOENCRYPT
)
152 pmap_set_noencrypt(ppnum_t pn
)
156 pai
= ppn_to_pai(pn
);
158 if (IS_MANAGED_PAGE(pai
)) {
161 pmap_phys_attributes
[pai
] |= PHYS_NOENCRYPT
;
169 pmap_clear_noencrypt(ppnum_t pn
)
173 pai
= ppn_to_pai(pn
);
175 if (IS_MANAGED_PAGE(pai
)) {
177 * synchronization at VM layer prevents PHYS_NOENCRYPT
178 * from changing state, so we don't need the lock to inspect
180 if (pmap_phys_attributes
[pai
] & PHYS_NOENCRYPT
) {
183 pmap_phys_attributes
[pai
] &= ~PHYS_NOENCRYPT
;
191 compute_pmap_gc_throttle(void *arg __unused
)
197 __private_extern__
void
198 pmap_pagetable_corruption_msg_log(int (*log_func
)(const char * fmt
, ...)__printflike(1,2)) {
199 if (pmap_pagetable_corruption_incidents
> 0) {
200 int i
, e
= MIN(pmap_pagetable_corruption_incidents
, PMAP_PAGETABLE_CORRUPTION_MAX_LOG
);
201 (*log_func
)("%u pagetable corruption incident(s) detected, timeout: %u\n", pmap_pagetable_corruption_incidents
, pmap_pagetable_corruption_timeout
);
202 for (i
= 0; i
< e
; i
++) {
203 (*log_func
)("Incident 0x%x, reason: 0x%x, action: 0x%x, time: 0x%llx\n", pmap_pagetable_corruption_records
[i
].incident
, pmap_pagetable_corruption_records
[i
].reason
, pmap_pagetable_corruption_records
[i
].action
, pmap_pagetable_corruption_records
[i
].abstime
);
209 pmap_pagetable_corruption_log_setup(void) {
210 if (pmap_pagetable_corruption_log_call
== NULL
) {
211 nanotime_to_absolutetime(PMAP_PAGETABLE_CORRUPTION_INTERVAL
, 0, &pmap_pagetable_corruption_interval_abstime
);
212 thread_call_setup(&pmap_pagetable_corruption_log_call_data
,
213 (thread_call_func_t
) pmap_pagetable_corruption_msg_log
,
214 (thread_call_param_t
) &printf
);
215 pmap_pagetable_corruption_log_call
= &pmap_pagetable_corruption_log_call_data
;
220 mapping_free_prime(void)
223 pv_hashed_entry_t pvh_e
;
224 pv_hashed_entry_t pvh_eh
;
225 pv_hashed_entry_t pvh_et
;
228 /* Scale based on DRAM size */
229 pv_hashed_low_water_mark
= MAX(PV_HASHED_LOW_WATER_MARK_DEFAULT
, ((uint32_t)(sane_size
>> 30)) * 2000);
230 pv_hashed_low_water_mark
= MIN(pv_hashed_low_water_mark
, 16000);
231 /* Alterable via sysctl */
232 pv_hashed_kern_low_water_mark
= MAX(PV_HASHED_KERN_LOW_WATER_MARK_DEFAULT
, ((uint32_t)(sane_size
>> 30)) * 1000);
233 pv_hashed_kern_low_water_mark
= MIN(pv_hashed_kern_low_water_mark
, 16000);
234 pv_hashed_kern_alloc_chunk
= PV_HASHED_KERN_ALLOC_CHUNK_INITIAL
;
235 pv_hashed_alloc_chunk
= PV_HASHED_ALLOC_CHUNK_INITIAL
;
238 pvh_eh
= pvh_et
= PV_HASHED_ENTRY_NULL
;
240 for (i
= 0; i
< (5 * PV_HASHED_ALLOC_CHUNK_INITIAL
); i
++) {
241 pvh_e
= (pv_hashed_entry_t
) zalloc(pv_hashed_list_zone
);
243 pvh_e
->qlink
.next
= (queue_entry_t
)pvh_eh
;
246 if (pvh_et
== PV_HASHED_ENTRY_NULL
)
250 PV_HASHED_FREE_LIST(pvh_eh
, pvh_et
, pv_cnt
);
253 pvh_eh
= pvh_et
= PV_HASHED_ENTRY_NULL
;
254 for (i
= 0; i
< PV_HASHED_KERN_ALLOC_CHUNK_INITIAL
; i
++) {
255 pvh_e
= (pv_hashed_entry_t
) zalloc(pv_hashed_list_zone
);
257 pvh_e
->qlink
.next
= (queue_entry_t
)pvh_eh
;
260 if (pvh_et
== PV_HASHED_ENTRY_NULL
)
264 PV_HASHED_KERN_FREE_LIST(pvh_eh
, pvh_et
, pv_cnt
);
267 void mapping_replenish(void);
269 void mapping_adjust(void) {
272 pmap_pagetable_corruption_log_setup();
274 mres
= kernel_thread_start_priority((thread_continue_t
)mapping_replenish
, NULL
, MAXPRI_KERNEL
, &mapping_replenish_thread
);
275 if (mres
!= KERN_SUCCESS
) {
276 panic("pmap: mapping_replenish_thread creation failed");
278 thread_deallocate(mapping_replenish_thread
);
281 unsigned pmap_mapping_thread_wakeups
;
282 unsigned pmap_kernel_reserve_replenish_stat
;
283 unsigned pmap_user_reserve_replenish_stat
;
284 unsigned pmap_kern_reserve_alloc_stat
;
286 void mapping_replenish(void)
288 pv_hashed_entry_t pvh_e
;
289 pv_hashed_entry_t pvh_eh
;
290 pv_hashed_entry_t pvh_et
;
294 /* We qualify for VM privileges...*/
295 current_thread()->options
|= TH_OPT_VMPRIV
;
299 while (pv_hashed_kern_free_count
< pv_hashed_kern_low_water_mark
) {
301 pvh_eh
= pvh_et
= PV_HASHED_ENTRY_NULL
;
303 for (i
= 0; i
< pv_hashed_kern_alloc_chunk
; i
++) {
304 pvh_e
= (pv_hashed_entry_t
) zalloc(pv_hashed_list_zone
);
305 pvh_e
->qlink
.next
= (queue_entry_t
)pvh_eh
;
308 if (pvh_et
== PV_HASHED_ENTRY_NULL
)
312 pmap_kernel_reserve_replenish_stat
+= pv_cnt
;
313 PV_HASHED_KERN_FREE_LIST(pvh_eh
, pvh_et
, pv_cnt
);
317 pvh_eh
= pvh_et
= PV_HASHED_ENTRY_NULL
;
319 if (pv_hashed_free_count
< pv_hashed_low_water_mark
) {
320 for (i
= 0; i
< pv_hashed_alloc_chunk
; i
++) {
321 pvh_e
= (pv_hashed_entry_t
) zalloc(pv_hashed_list_zone
);
323 pvh_e
->qlink
.next
= (queue_entry_t
)pvh_eh
;
326 if (pvh_et
== PV_HASHED_ENTRY_NULL
)
330 pmap_user_reserve_replenish_stat
+= pv_cnt
;
331 PV_HASHED_FREE_LIST(pvh_eh
, pvh_et
, pv_cnt
);
333 /* Wake threads throttled while the kernel reserve was being replenished.
335 if (pmap_pv_throttled_waiters
) {
336 pmap_pv_throttled_waiters
= 0;
337 thread_wakeup(&pmap_user_pv_throttle_event
);
339 /* Check if the kernel pool has been depleted since the
340 * first pass, to reduce refill latency.
342 if (pv_hashed_kern_free_count
< pv_hashed_kern_low_water_mark
)
344 /* Block sans continuation to avoid yielding kernel stack */
345 assert_wait(&mapping_replenish_event
, THREAD_UNINT
);
347 thread_block(THREAD_CONTINUE_NULL
);
348 pmap_mapping_thread_wakeups
++;
353 * Set specified attribute bits.
364 assert(pn
!= vm_page_fictitious_addr
);
365 if (pn
== vm_page_guard_addr
)
368 pai
= ppn_to_pai(pn
);
370 if (!IS_MANAGED_PAGE(pai
)) {
371 /* Not a managed page. */
376 pmap_phys_attributes
[pai
] |= bits
;
381 * Set the modify bit on the specified physical page.
385 pmap_set_modify(ppnum_t pn
)
387 phys_attribute_set(pn
, PHYS_MODIFIED
);
391 * Clear the modify bits on the specified physical page.
395 pmap_clear_modify(ppnum_t pn
)
397 phys_attribute_clear(pn
, PHYS_MODIFIED
);
403 * Return whether or not the specified physical page is modified
404 * by any physical maps.
408 pmap_is_modified(ppnum_t pn
)
410 if (phys_attribute_test(pn
, PHYS_MODIFIED
))
417 * pmap_clear_reference:
419 * Clear the reference bit on the specified physical page.
423 pmap_clear_reference(ppnum_t pn
)
425 phys_attribute_clear(pn
, PHYS_REFERENCED
);
429 pmap_set_reference(ppnum_t pn
)
431 phys_attribute_set(pn
, PHYS_REFERENCED
);
435 * pmap_is_referenced:
437 * Return whether or not the specified physical page is referenced
438 * by any physical maps.
442 pmap_is_referenced(ppnum_t pn
)
444 if (phys_attribute_test(pn
, PHYS_REFERENCED
))
451 * pmap_get_refmod(phys)
452 * returns the referenced and modified bits of the specified
456 pmap_get_refmod(ppnum_t pn
)
459 unsigned int retval
= 0;
461 refmod
= phys_attribute_test(pn
, PHYS_MODIFIED
| PHYS_REFERENCED
);
463 if (refmod
& PHYS_MODIFIED
)
464 retval
|= VM_MEM_MODIFIED
;
465 if (refmod
& PHYS_REFERENCED
)
466 retval
|= VM_MEM_REFERENCED
;
472 * pmap_clear_refmod(phys, mask)
473 * clears the referenced and modified bits as specified by the mask
474 * of the specified physical page.
477 pmap_clear_refmod(ppnum_t pn
, unsigned int mask
)
479 unsigned int x86Mask
;
481 x86Mask
= ( ((mask
& VM_MEM_MODIFIED
)? PHYS_MODIFIED
: 0)
482 | ((mask
& VM_MEM_REFERENCED
)? PHYS_REFERENCED
: 0));
483 phys_attribute_clear(pn
, x86Mask
);
491 * Disconnect all mappings for this page and return reference and change status
496 pmap_disconnect(ppnum_t pa
)
498 unsigned refmod
, vmrefmod
= 0;
500 pmap_page_protect(pa
, 0); /* disconnect the page */
502 pmap_assert(pa
!= vm_page_fictitious_addr
);
503 if ((pa
== vm_page_guard_addr
) || !IS_MANAGED_PAGE(pa
))
505 refmod
= pmap_phys_attributes
[pa
] & (PHYS_MODIFIED
| PHYS_REFERENCED
);
507 if (refmod
& PHYS_MODIFIED
)
508 vmrefmod
|= VM_MEM_MODIFIED
;
509 if (refmod
& PHYS_REFERENCED
)
510 vmrefmod
|= VM_MEM_REFERENCED
;