osfmk/i386/pmap_common.c

   1 /*
   2  * Copyright (c) 2000-2010 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 #include <vm/pmap.h>
  29 #include <i386/pmap_internal.h>
  30
  31 /*
  32  *      Each entry in the pv_head_table is locked by a bit in the
  33  *      pv_lock_table.  The lock bits are accessed by the physical
  34  *      address of the page they lock.
  35  */
  36
  37 char    *pv_lock_table;         /* pointer to array of bits */
  38 char    *pv_hash_lock_table;
  39
  40 pv_rooted_entry_t       pv_head_table;          /* array of entries, one per
  41                                                  * page */
  42 uint32_t                        pv_hashed_free_count = 0;
  43 uint32_t                        pv_hashed_kern_free_count = 0;
  44
  45 pmap_pagetable_corruption_record_t pmap_pagetable_corruption_records[PMAP_PAGETABLE_CORRUPTION_MAX_LOG];
  46 uint32_t pmap_pagetable_corruption_incidents;
  47 uint64_t pmap_pagetable_corruption_last_abstime = (~(0ULL) >> 1);
  48 uint64_t pmap_pagetable_corruption_interval_abstime;
  49 thread_call_t   pmap_pagetable_corruption_log_call;
  50 static thread_call_data_t       pmap_pagetable_corruption_log_call_data;
  51 boolean_t pmap_pagetable_corruption_timeout = FALSE;
  52
  53 volatile uint32_t       mappingrecurse = 0;
  54
  55 uint32_t  pv_hashed_low_water_mark, pv_hashed_kern_low_water_mark, pv_hashed_alloc_chunk, pv_hashed_kern_alloc_chunk;
  56
  57 thread_t mapping_replenish_thread;
  58 event_t mapping_replenish_event, pmap_user_pv_throttle_event;
  59
  60 uint64_t pmap_pv_throttle_stat, pmap_pv_throttled_waiters;
  61
  62 unsigned int pmap_cache_attributes(ppnum_t pn) {
  63         if (pmap_get_cache_attributes(pn) & INTEL_PTE_NCACHE)
  64                 return (VM_WIMG_IO);
  65         else
  66                 return (VM_WIMG_COPYBACK);
  67 }
  68
  69 void    pmap_set_cache_attributes(ppnum_t pn, unsigned int cacheattr) {
  70         unsigned int current, template = 0;
  71         int pai;
  72
  73         if (cacheattr & VM_MEM_NOT_CACHEABLE) {
  74                 if(!(cacheattr & VM_MEM_GUARDED))
  75                         template |= PHYS_PTA;
  76                 template |= PHYS_NCACHE;
  77         }
  78
  79         pmap_intr_assert();
  80
  81         assert((pn != vm_page_fictitious_addr) && (pn != vm_page_guard_addr));
  82
  83         pai = ppn_to_pai(pn);
  84
  85         if (!IS_MANAGED_PAGE(pai)) {
  86                 return;
  87         }
  88
  89         /* override cache attributes for this phys page
  90          * Does not walk through existing mappings to adjust,
  91          * assumes page is disconnected
  92          */
  93
  94         LOCK_PVH(pai);
  95
  96         pmap_update_cache_attributes_locked(pn, template);
  97
  98         current = pmap_phys_attributes[pai] & PHYS_CACHEABILITY_MASK;
  99         pmap_phys_attributes[pai] &= ~PHYS_CACHEABILITY_MASK;
 100         pmap_phys_attributes[pai] |= template;
 101
 102         UNLOCK_PVH(pai);
 103
 104         if ((template & PHYS_NCACHE) && !(current & PHYS_NCACHE)) {
 105                 pmap_sync_page_attributes_phys(pn);
 106         }
 107 }
 108
 109 unsigned        pmap_get_cache_attributes(ppnum_t pn) {
 110         if (last_managed_page == 0)
 111                 return 0;
 112
 113         if (!IS_MANAGED_PAGE(ppn_to_pai(pn))) {
 114             return INTEL_PTE_NCACHE;
 115         }
 116
 117         /*
 118          * The cache attributes are read locklessly for efficiency.
 119          */
 120         unsigned int attr = pmap_phys_attributes[ppn_to_pai(pn)];
 121         unsigned int template = 0;
 122
 123         if (attr & PHYS_PTA)
 124                 template |= INTEL_PTE_PTA;
 125         if (attr & PHYS_NCACHE)
 126                 template |= INTEL_PTE_NCACHE;
 127         return template;
 128 }
 129
 130
 131
 132 boolean_t
 133 pmap_is_noencrypt(ppnum_t pn)
 134 {
 135         int             pai;
 136
 137         pai = ppn_to_pai(pn);
 138
 139         if (!IS_MANAGED_PAGE(pai))
 140                 return (TRUE);
 141
 142         if (pmap_phys_attributes[pai] & PHYS_NOENCRYPT)
 143                 return (TRUE);
 144
 145         return (FALSE);
 146 }
 147
 148
 149 void
 150 pmap_set_noencrypt(ppnum_t pn)
 151 {
 152         int             pai;
 153
 154         pai = ppn_to_pai(pn);
 155
 156         if (IS_MANAGED_PAGE(pai)) {
 157                 LOCK_PVH(pai);
 158
 159                 pmap_phys_attributes[pai] |= PHYS_NOENCRYPT;
 160
 161                 UNLOCK_PVH(pai);
 162         }
 163 }
 164
 165
 166 void
 167 pmap_clear_noencrypt(ppnum_t pn)
 168 {
 169         int             pai;
 170
 171         pai = ppn_to_pai(pn);
 172
 173         if (IS_MANAGED_PAGE(pai)) {
 174                 LOCK_PVH(pai);
 175
 176                 pmap_phys_attributes[pai] &= ~PHYS_NOENCRYPT;
 177
 178                 UNLOCK_PVH(pai);
 179         }
 180 }
 181
 182 void
 183 compute_pmap_gc_throttle(void *arg __unused)
 184 {
 185
 186 }
 187
 188
 189 __private_extern__ void
 190 pmap_pagetable_corruption_msg_log(int (*log_func)(const char * fmt, ...)__printflike(1,2)) {
 191         if (pmap_pagetable_corruption_incidents > 0) {
 192                 int i, e = MIN(pmap_pagetable_corruption_incidents, PMAP_PAGETABLE_CORRUPTION_MAX_LOG);
 193                 (*log_func)("%u pagetable corruption incident(s) detected, timeout: %u\n", pmap_pagetable_corruption_incidents, pmap_pagetable_corruption_timeout);
 194                 for (i = 0; i < e; i++) {
 195                         (*log_func)("Incident 0x%x, reason: 0x%x, action: 0x%x, time: 0x%llx\n", pmap_pagetable_corruption_records[i].incident,  pmap_pagetable_corruption_records[i].reason, pmap_pagetable_corruption_records[i].action, pmap_pagetable_corruption_records[i].abstime);
 196                 }
 197         }
 198 }
 199
 200 static inline void
 201 pmap_pagetable_corruption_log_setup(void) {
 202         if (pmap_pagetable_corruption_log_call == NULL) {
 203                 nanotime_to_absolutetime(PMAP_PAGETABLE_CORRUPTION_INTERVAL, 0, &pmap_pagetable_corruption_interval_abstime);
 204                 thread_call_setup(&pmap_pagetable_corruption_log_call_data,
 205                     (thread_call_func_t) pmap_pagetable_corruption_msg_log,
 206                     (thread_call_param_t) &printf);
 207                 pmap_pagetable_corruption_log_call = &pmap_pagetable_corruption_log_call_data;
 208         }
 209 }
 210
 211 void
 212 mapping_free_prime(void)
 213 {
 214         unsigned                i;
 215         pv_hashed_entry_t       pvh_e;
 216         pv_hashed_entry_t       pvh_eh;
 217         pv_hashed_entry_t       pvh_et;
 218         int                     pv_cnt;
 219
 220         /* Scale based on DRAM size */
 221         pv_hashed_low_water_mark = MAX(PV_HASHED_LOW_WATER_MARK_DEFAULT, ((uint32_t)(sane_size >> 30)) * 2000);
 222         pv_hashed_low_water_mark = MIN(pv_hashed_low_water_mark, 16000);
 223         /* Alterable via sysctl */
 224         pv_hashed_kern_low_water_mark = MAX(PV_HASHED_KERN_LOW_WATER_MARK_DEFAULT, ((uint32_t)(sane_size >> 30)) * 1000);
 225         pv_hashed_kern_low_water_mark = MIN(pv_hashed_kern_low_water_mark, 16000);
 226         pv_hashed_kern_alloc_chunk = PV_HASHED_KERN_ALLOC_CHUNK_INITIAL;
 227         pv_hashed_alloc_chunk = PV_HASHED_ALLOC_CHUNK_INITIAL;
 228
 229         pv_cnt = 0;
 230         pvh_eh = pvh_et = PV_HASHED_ENTRY_NULL;
 231
 232         for (i = 0; i < (5 * PV_HASHED_ALLOC_CHUNK_INITIAL); i++) {
 233                 pvh_e = (pv_hashed_entry_t) zalloc(pv_hashed_list_zone);
 234
 235                 pvh_e->qlink.next = (queue_entry_t)pvh_eh;
 236                 pvh_eh = pvh_e;
 237
 238                 if (pvh_et == PV_HASHED_ENTRY_NULL)
 239                         pvh_et = pvh_e;
 240                 pv_cnt++;
 241         }
 242         PV_HASHED_FREE_LIST(pvh_eh, pvh_et, pv_cnt);
 243
 244         pv_cnt = 0;
 245         pvh_eh = pvh_et = PV_HASHED_ENTRY_NULL;
 246         for (i = 0; i < PV_HASHED_KERN_ALLOC_CHUNK_INITIAL; i++) {
 247                 pvh_e = (pv_hashed_entry_t) zalloc(pv_hashed_list_zone);
 248
 249                 pvh_e->qlink.next = (queue_entry_t)pvh_eh;
 250                 pvh_eh = pvh_e;
 251
 252                 if (pvh_et == PV_HASHED_ENTRY_NULL)
 253                         pvh_et = pvh_e;
 254                 pv_cnt++;
 255         }
 256         PV_HASHED_KERN_FREE_LIST(pvh_eh, pvh_et, pv_cnt);
 257 }
 258
 259 void mapping_replenish(void);
 260
 261 void mapping_adjust(void) {
 262         kern_return_t mres;
 263
 264         pmap_pagetable_corruption_log_setup();
 265
 266         mres = kernel_thread_start_priority((thread_continue_t)mapping_replenish, NULL, MAXPRI_KERNEL, &mapping_replenish_thread);
 267         if (mres != KERN_SUCCESS) {
 268                 panic("pmap: mapping_replenish_thread creation failed");
 269         }
 270         thread_deallocate(mapping_replenish_thread);
 271 }
 272
 273 unsigned pmap_mapping_thread_wakeups;
 274 unsigned pmap_kernel_reserve_replenish_stat;
 275 unsigned pmap_user_reserve_replenish_stat;
 276 unsigned pmap_kern_reserve_alloc_stat;
 277
 278 void mapping_replenish(void)
 279 {
 280         pv_hashed_entry_t       pvh_e;
 281         pv_hashed_entry_t       pvh_eh;
 282         pv_hashed_entry_t       pvh_et;
 283         int                     pv_cnt;
 284         unsigned                i;
 285
 286         /* We qualify for VM privileges...*/
 287         current_thread()->options |= TH_OPT_VMPRIV;
 288
 289         for (;;) {
 290
 291                 while (pv_hashed_kern_free_count < pv_hashed_kern_low_water_mark) {
 292                         pv_cnt = 0;
 293                         pvh_eh = pvh_et = PV_HASHED_ENTRY_NULL;
 294
 295                         for (i = 0; i < pv_hashed_kern_alloc_chunk; i++) {
 296                                 pvh_e = (pv_hashed_entry_t) zalloc(pv_hashed_list_zone);
 297                                 pvh_e->qlink.next = (queue_entry_t)pvh_eh;
 298                                 pvh_eh = pvh_e;
 299
 300                                 if (pvh_et == PV_HASHED_ENTRY_NULL)
 301                                         pvh_et = pvh_e;
 302                                 pv_cnt++;
 303                         }
 304                         pmap_kernel_reserve_replenish_stat += pv_cnt;
 305                         PV_HASHED_KERN_FREE_LIST(pvh_eh, pvh_et, pv_cnt);
 306                 }
 307
 308                 pv_cnt = 0;
 309                 pvh_eh = pvh_et = PV_HASHED_ENTRY_NULL;
 310
 311                 if (pv_hashed_free_count < pv_hashed_low_water_mark) {
 312                         for (i = 0; i < pv_hashed_alloc_chunk; i++) {
 313                                 pvh_e = (pv_hashed_entry_t) zalloc(pv_hashed_list_zone);
 314
 315                                 pvh_e->qlink.next = (queue_entry_t)pvh_eh;
 316                                 pvh_eh = pvh_e;
 317
 318                                 if (pvh_et == PV_HASHED_ENTRY_NULL)
 319                                         pvh_et = pvh_e;
 320                                 pv_cnt++;
 321                         }
 322                         pmap_user_reserve_replenish_stat += pv_cnt;
 323                         PV_HASHED_FREE_LIST(pvh_eh, pvh_et, pv_cnt);
 324                 }
 325 /* Wake threads throttled while the kernel reserve was being replenished.
 326  */
 327                 if (pmap_pv_throttled_waiters) {
 328                         pmap_pv_throttled_waiters = 0;
 329                         thread_wakeup(&pmap_user_pv_throttle_event);
 330                 }
 331                 /* Check if the kernel pool has been depleted since the
 332                  * first pass, to reduce refill latency.
 333                  */
 334                 if (pv_hashed_kern_free_count < pv_hashed_kern_low_water_mark)
 335                         continue;
 336                 /* Block sans continuation to avoid yielding kernel stack */
 337                 assert_wait(&mapping_replenish_event, THREAD_UNINT);
 338                 mappingrecurse = 0;
 339                 thread_block(THREAD_CONTINUE_NULL);
 340                 pmap_mapping_thread_wakeups++;
 341         }
 342 }
 343
 344 /*
 345  *      Set specified attribute bits.
 346  */
 347
 348 void
 349 phys_attribute_set(
 350         ppnum_t         pn,
 351         int             bits)
 352 {
 353         int             pai;
 354
 355         pmap_intr_assert();
 356         assert(pn != vm_page_fictitious_addr);
 357         if (pn == vm_page_guard_addr)
 358                 return;
 359
 360         pai = ppn_to_pai(pn);
 361
 362         if (!IS_MANAGED_PAGE(pai)) {
 363                 /* Not a managed page.  */
 364                 return;
 365         }
 366
 367         LOCK_PVH(pai);
 368         pmap_phys_attributes[pai] |= bits;
 369         UNLOCK_PVH(pai);
 370 }
 371
 372 /*
 373  *      Set the modify bit on the specified physical page.
 374  */
 375
 376 void
 377 pmap_set_modify(ppnum_t pn)
 378 {
 379         phys_attribute_set(pn, PHYS_MODIFIED);
 380 }
 381
 382 /*
 383  *      Clear the modify bits on the specified physical page.
 384  */
 385
 386 void
 387 pmap_clear_modify(ppnum_t pn)
 388 {
 389         phys_attribute_clear(pn, PHYS_MODIFIED);
 390 }
 391
 392 /*
 393  *      pmap_is_modified:
 394  *
 395  *      Return whether or not the specified physical page is modified
 396  *      by any physical maps.
 397  */
 398
 399 boolean_t
 400 pmap_is_modified(ppnum_t pn)
 401 {
 402         if (phys_attribute_test(pn, PHYS_MODIFIED))
 403                 return TRUE;
 404         return FALSE;
 405 }
 406
 407
 408 /*
 409  *      pmap_clear_reference:
 410  *
 411  *      Clear the reference bit on the specified physical page.
 412  */
 413
 414 void
 415 pmap_clear_reference(ppnum_t pn)
 416 {
 417         phys_attribute_clear(pn, PHYS_REFERENCED);
 418 }
 419
 420 void
 421 pmap_set_reference(ppnum_t pn)
 422 {
 423         phys_attribute_set(pn, PHYS_REFERENCED);
 424 }
 425
 426 /*
 427  *      pmap_is_referenced:
 428  *
 429  *      Return whether or not the specified physical page is referenced
 430  *      by any physical maps.
 431  */
 432
 433 boolean_t
 434 pmap_is_referenced(ppnum_t pn)
 435 {
 436         if (phys_attribute_test(pn, PHYS_REFERENCED))
 437                 return TRUE;
 438         return FALSE;
 439 }
 440
 441
 442 /*
 443  * pmap_get_refmod(phys)
 444  *  returns the referenced and modified bits of the specified
 445  *  physical page.
 446  */
 447 unsigned int
 448 pmap_get_refmod(ppnum_t pn)
 449 {
 450         int             refmod;
 451         unsigned int    retval = 0;
 452
 453         refmod = phys_attribute_test(pn, PHYS_MODIFIED | PHYS_REFERENCED);
 454
 455         if (refmod & PHYS_MODIFIED)
 456                 retval |= VM_MEM_MODIFIED;
 457         if (refmod & PHYS_REFERENCED)
 458                 retval |= VM_MEM_REFERENCED;
 459
 460         return (retval);
 461 }
 462
 463 /*
 464  * pmap_clear_refmod(phys, mask)
 465  *  clears the referenced and modified bits as specified by the mask
 466  *  of the specified physical page.
 467  */
 468 void
 469 pmap_clear_refmod(ppnum_t pn, unsigned int mask)
 470 {
 471         unsigned int  x86Mask;
 472
 473         x86Mask = (   ((mask &   VM_MEM_MODIFIED)?   PHYS_MODIFIED : 0)
 474                     | ((mask & VM_MEM_REFERENCED)? PHYS_REFERENCED : 0));
 475         phys_attribute_clear(pn, x86Mask);
 476 }
 477
 478 /*
 479  *      Routine:
 480  *              pmap_disconnect
 481  *
 482  *      Function:
 483  *              Disconnect all mappings for this page and return reference and change status
 484  *              in generic format.
 485  *
 486  */
 487 unsigned int
 488 pmap_disconnect(ppnum_t pa)
 489 {
 490         unsigned refmod, vmrefmod = 0;
 491
 492         pmap_page_protect(pa, 0);               /* disconnect the page */
 493
 494         pmap_assert(pa != vm_page_fictitious_addr);
 495         if ((pa == vm_page_guard_addr) || !IS_MANAGED_PAGE(pa))
 496                 return 0;
 497         refmod = pmap_phys_attributes[pa] & (PHYS_MODIFIED | PHYS_REFERENCED);
 498
 499         if (refmod & PHYS_MODIFIED)
 500                 vmrefmod |= VM_MEM_MODIFIED;
 501         if (refmod & PHYS_REFERENCED)
 502                 vmrefmod |= VM_MEM_REFERENCED;
 503
 504         return vmrefmod;
 505 }