osfmk/i386/pmap_common.c

   1 /*
   2  * Copyright (c) 2000-2010 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 #include <vm/pmap.h>
  29 #include <kern/ledger.h>
  30 #include <i386/pmap_internal.h>
  31
  32
  33 /*
  34  *      Each entry in the pv_head_table is locked by a bit in the
  35  *      pv_lock_table.  The lock bits are accessed by the physical
  36  *      address of the page they lock.
  37  */
  38
  39 char    *pv_lock_table;         /* pointer to array of bits */
  40 char    *pv_hash_lock_table;
  41
  42 pv_rooted_entry_t       pv_head_table;          /* array of entries, one per
  43                                                  * page */
  44 uint32_t                        pv_hashed_free_count = 0;
  45 uint32_t                        pv_hashed_kern_free_count = 0;
  46
  47 pmap_pagetable_corruption_record_t pmap_pagetable_corruption_records[PMAP_PAGETABLE_CORRUPTION_MAX_LOG];
  48 uint32_t pmap_pagetable_corruption_incidents;
  49 uint64_t pmap_pagetable_corruption_last_abstime = (~(0ULL) >> 1);
  50 uint64_t pmap_pagetable_corruption_interval_abstime;
  51 thread_call_t   pmap_pagetable_corruption_log_call;
  52 static thread_call_data_t       pmap_pagetable_corruption_log_call_data;
  53 boolean_t pmap_pagetable_corruption_timeout = FALSE;
  54
  55 volatile uint32_t       mappingrecurse = 0;
  56
  57 uint32_t  pv_hashed_low_water_mark, pv_hashed_kern_low_water_mark, pv_hashed_alloc_chunk, pv_hashed_kern_alloc_chunk;
  58
  59 thread_t mapping_replenish_thread;
  60 event_t mapping_replenish_event, pmap_user_pv_throttle_event;
  61
  62 uint64_t pmap_pv_throttle_stat, pmap_pv_throttled_waiters;
  63
  64 unsigned int pmap_cache_attributes(ppnum_t pn) {
  65         if (pmap_get_cache_attributes(pn, FALSE) & INTEL_PTE_NCACHE)
  66                 return (VM_WIMG_IO);
  67         else
  68                 return (VM_WIMG_COPYBACK);
  69 }
  70
  71 void    pmap_set_cache_attributes(ppnum_t pn, unsigned int cacheattr) {
  72         unsigned int current, template = 0;
  73         int pai;
  74
  75         if (cacheattr & VM_MEM_NOT_CACHEABLE) {
  76                 if(!(cacheattr & VM_MEM_GUARDED))
  77                         template |= PHYS_PTA;
  78                 template |= PHYS_NCACHE;
  79         }
  80
  81         pmap_intr_assert();
  82
  83         assert((pn != vm_page_fictitious_addr) && (pn != vm_page_guard_addr));
  84
  85         pai = ppn_to_pai(pn);
  86
  87         if (!IS_MANAGED_PAGE(pai)) {
  88                 return;
  89         }
  90
  91         /* override cache attributes for this phys page
  92          * Does not walk through existing mappings to adjust,
  93          * assumes page is disconnected
  94          */
  95
  96         LOCK_PVH(pai);
  97
  98         pmap_update_cache_attributes_locked(pn, template);
  99
 100         current = pmap_phys_attributes[pai] & PHYS_CACHEABILITY_MASK;
 101         pmap_phys_attributes[pai] &= ~PHYS_CACHEABILITY_MASK;
 102         pmap_phys_attributes[pai] |= template;
 103
 104         UNLOCK_PVH(pai);
 105
 106         if ((template & PHYS_NCACHE) && !(current & PHYS_NCACHE)) {
 107                 pmap_sync_page_attributes_phys(pn);
 108         }
 109 }
 110
 111 unsigned        pmap_get_cache_attributes(ppnum_t pn, boolean_t is_ept) {
 112         if (last_managed_page == 0)
 113                 return 0;
 114
 115         if (!IS_MANAGED_PAGE(ppn_to_pai(pn)))
 116             return PTE_NCACHE(is_ept);
 117
 118         /*
 119          * The cache attributes are read locklessly for efficiency.
 120          */
 121         unsigned int attr = pmap_phys_attributes[ppn_to_pai(pn)];
 122         unsigned int template = 0;
 123
 124         /*
 125          * The PTA bit is currently unsupported for EPT PTEs.
 126          */
 127         if ((attr & PHYS_PTA) && !is_ept)
 128                 template |= INTEL_PTE_PTA;
 129
 130         /*
 131          * If the page isn't marked as NCACHE, the default for EPT entries
 132          * is WB.
 133          */
 134         if (attr & PHYS_NCACHE)
 135                 template |= PTE_NCACHE(is_ept);
 136         else if (is_ept)
 137                 template |= INTEL_EPT_WB;
 138
 139         return template;
 140 }
 141
 142 boolean_t
 143 pmap_has_managed_page(ppnum_t first, ppnum_t last)
 144 {
 145         ppnum_t   pn;
 146     boolean_t result;
 147
 148     assert(last_managed_page);
 149     assert(first <= last);
 150
 151     for (result = FALSE, pn = first;
 152         !result
 153           && (pn <= last)
 154           && (pn <= last_managed_page);
 155          pn++)
 156     {
 157         result = (0 != (pmap_phys_attributes[pn] & PHYS_MANAGED));
 158     }
 159
 160         return (result);
 161 }
 162
 163 boolean_t
 164 pmap_is_noencrypt(ppnum_t pn)
 165 {
 166         int             pai;
 167
 168         pai = ppn_to_pai(pn);
 169
 170         if (!IS_MANAGED_PAGE(pai))
 171                 return (FALSE);
 172
 173         if (pmap_phys_attributes[pai] & PHYS_NOENCRYPT)
 174                 return (TRUE);
 175
 176         return (FALSE);
 177 }
 178
 179
 180 void
 181 pmap_set_noencrypt(ppnum_t pn)
 182 {
 183         int             pai;
 184
 185         pai = ppn_to_pai(pn);
 186
 187         if (IS_MANAGED_PAGE(pai)) {
 188                 LOCK_PVH(pai);
 189
 190                 pmap_phys_attributes[pai] |= PHYS_NOENCRYPT;
 191
 192                 UNLOCK_PVH(pai);
 193         }
 194 }
 195
 196
 197 void
 198 pmap_clear_noencrypt(ppnum_t pn)
 199 {
 200         int             pai;
 201
 202         pai = ppn_to_pai(pn);
 203
 204         if (IS_MANAGED_PAGE(pai)) {
 205                 /*
 206                  * synchronization at VM layer prevents PHYS_NOENCRYPT
 207                  * from changing state, so we don't need the lock to inspect
 208                  */
 209                 if (pmap_phys_attributes[pai] & PHYS_NOENCRYPT) {
 210                         LOCK_PVH(pai);
 211
 212                         pmap_phys_attributes[pai] &= ~PHYS_NOENCRYPT;
 213
 214                         UNLOCK_PVH(pai);
 215                 }
 216         }
 217 }
 218
 219 void
 220 compute_pmap_gc_throttle(void *arg __unused)
 221 {
 222
 223 }
 224
 225
 226 void
 227 pmap_lock_phys_page(ppnum_t pn)
 228 {
 229         int             pai;
 230
 231         pai = ppn_to_pai(pn);
 232
 233         if (IS_MANAGED_PAGE(pai)) {
 234                 LOCK_PVH(pai);
 235         } else
 236                 simple_lock(&phys_backup_lock);
 237 }
 238
 239
 240 void
 241 pmap_unlock_phys_page(ppnum_t pn)
 242 {
 243         int             pai;
 244
 245         pai = ppn_to_pai(pn);
 246
 247         if (IS_MANAGED_PAGE(pai)) {
 248                 UNLOCK_PVH(pai);
 249         } else
 250                 simple_unlock(&phys_backup_lock);
 251 }
 252
 253
 254
 255 __private_extern__ void
 256 pmap_pagetable_corruption_msg_log(int (*log_func)(const char * fmt, ...)__printflike(1,2)) {
 257         if (pmap_pagetable_corruption_incidents > 0) {
 258                 int i, e = MIN(pmap_pagetable_corruption_incidents, PMAP_PAGETABLE_CORRUPTION_MAX_LOG);
 259                 (*log_func)("%u pagetable corruption incident(s) detected, timeout: %u\n", pmap_pagetable_corruption_incidents, pmap_pagetable_corruption_timeout);
 260                 for (i = 0; i < e; i++) {
 261                         (*log_func)("Incident 0x%x, reason: 0x%x, action: 0x%x, time: 0x%llx\n", pmap_pagetable_corruption_records[i].incident,  pmap_pagetable_corruption_records[i].reason, pmap_pagetable_corruption_records[i].action, pmap_pagetable_corruption_records[i].abstime);
 262                 }
 263         }
 264 }
 265
 266 static inline void
 267 pmap_pagetable_corruption_log_setup(void) {
 268         if (pmap_pagetable_corruption_log_call == NULL) {
 269                 nanotime_to_absolutetime(PMAP_PAGETABLE_CORRUPTION_INTERVAL, 0, &pmap_pagetable_corruption_interval_abstime);
 270                 thread_call_setup(&pmap_pagetable_corruption_log_call_data,
 271                     (thread_call_func_t) pmap_pagetable_corruption_msg_log,
 272                     (thread_call_param_t) &printf);
 273                 pmap_pagetable_corruption_log_call = &pmap_pagetable_corruption_log_call_data;
 274         }
 275 }
 276
 277 void
 278 mapping_free_prime(void)
 279 {
 280         unsigned                i;
 281         pv_hashed_entry_t       pvh_e;
 282         pv_hashed_entry_t       pvh_eh;
 283         pv_hashed_entry_t       pvh_et;
 284         int                     pv_cnt;
 285
 286         /* Scale based on DRAM size */
 287         pv_hashed_low_water_mark = MAX(PV_HASHED_LOW_WATER_MARK_DEFAULT, ((uint32_t)(sane_size >> 30)) * 2000);
 288         pv_hashed_low_water_mark = MIN(pv_hashed_low_water_mark, 16000);
 289         /* Alterable via sysctl */
 290         pv_hashed_kern_low_water_mark = MAX(PV_HASHED_KERN_LOW_WATER_MARK_DEFAULT, ((uint32_t)(sane_size >> 30)) * 1000);
 291         pv_hashed_kern_low_water_mark = MIN(pv_hashed_kern_low_water_mark, 16000);
 292         pv_hashed_kern_alloc_chunk = PV_HASHED_KERN_ALLOC_CHUNK_INITIAL;
 293         pv_hashed_alloc_chunk = PV_HASHED_ALLOC_CHUNK_INITIAL;
 294
 295         pv_cnt = 0;
 296         pvh_eh = pvh_et = PV_HASHED_ENTRY_NULL;
 297
 298         for (i = 0; i < (5 * PV_HASHED_ALLOC_CHUNK_INITIAL); i++) {
 299                 pvh_e = (pv_hashed_entry_t) zalloc(pv_hashed_list_zone);
 300
 301                 pvh_e->qlink.next = (queue_entry_t)pvh_eh;
 302                 pvh_eh = pvh_e;
 303
 304                 if (pvh_et == PV_HASHED_ENTRY_NULL)
 305                         pvh_et = pvh_e;
 306                 pv_cnt++;
 307         }
 308         PV_HASHED_FREE_LIST(pvh_eh, pvh_et, pv_cnt);
 309
 310         pv_cnt = 0;
 311         pvh_eh = pvh_et = PV_HASHED_ENTRY_NULL;
 312         for (i = 0; i < PV_HASHED_KERN_ALLOC_CHUNK_INITIAL; i++) {
 313                 pvh_e = (pv_hashed_entry_t) zalloc(pv_hashed_list_zone);
 314
 315                 pvh_e->qlink.next = (queue_entry_t)pvh_eh;
 316                 pvh_eh = pvh_e;
 317
 318                 if (pvh_et == PV_HASHED_ENTRY_NULL)
 319                         pvh_et = pvh_e;
 320                 pv_cnt++;
 321         }
 322         PV_HASHED_KERN_FREE_LIST(pvh_eh, pvh_et, pv_cnt);
 323 }
 324
 325 void mapping_replenish(void);
 326
 327 void mapping_adjust(void) {
 328         kern_return_t mres;
 329
 330         pmap_pagetable_corruption_log_setup();
 331
 332         mres = kernel_thread_start_priority((thread_continue_t)mapping_replenish, NULL, MAXPRI_KERNEL, &mapping_replenish_thread);
 333         if (mres != KERN_SUCCESS) {
 334                 panic("pmap: mapping_replenish_thread creation failed");
 335         }
 336         thread_deallocate(mapping_replenish_thread);
 337 }
 338
 339 unsigned pmap_mapping_thread_wakeups;
 340 unsigned pmap_kernel_reserve_replenish_stat;
 341 unsigned pmap_user_reserve_replenish_stat;
 342 unsigned pmap_kern_reserve_alloc_stat;
 343
 344 void mapping_replenish(void)
 345 {
 346         pv_hashed_entry_t       pvh_e;
 347         pv_hashed_entry_t       pvh_eh;
 348         pv_hashed_entry_t       pvh_et;
 349         int                     pv_cnt;
 350         unsigned                i;
 351
 352         /* We qualify for VM privileges...*/
 353         current_thread()->options |= TH_OPT_VMPRIV;
 354
 355         for (;;) {
 356
 357                 while (pv_hashed_kern_free_count < pv_hashed_kern_low_water_mark) {
 358                         pv_cnt = 0;
 359                         pvh_eh = pvh_et = PV_HASHED_ENTRY_NULL;
 360
 361                         for (i = 0; i < pv_hashed_kern_alloc_chunk; i++) {
 362                                 pvh_e = (pv_hashed_entry_t) zalloc(pv_hashed_list_zone);
 363                                 pvh_e->qlink.next = (queue_entry_t)pvh_eh;
 364                                 pvh_eh = pvh_e;
 365
 366                                 if (pvh_et == PV_HASHED_ENTRY_NULL)
 367                                         pvh_et = pvh_e;
 368                                 pv_cnt++;
 369                         }
 370                         pmap_kernel_reserve_replenish_stat += pv_cnt;
 371                         PV_HASHED_KERN_FREE_LIST(pvh_eh, pvh_et, pv_cnt);
 372                 }
 373
 374                 pv_cnt = 0;
 375                 pvh_eh = pvh_et = PV_HASHED_ENTRY_NULL;
 376
 377                 if (pv_hashed_free_count < pv_hashed_low_water_mark) {
 378                         for (i = 0; i < pv_hashed_alloc_chunk; i++) {
 379                                 pvh_e = (pv_hashed_entry_t) zalloc(pv_hashed_list_zone);
 380
 381                                 pvh_e->qlink.next = (queue_entry_t)pvh_eh;
 382                                 pvh_eh = pvh_e;
 383
 384                                 if (pvh_et == PV_HASHED_ENTRY_NULL)
 385                                         pvh_et = pvh_e;
 386                                 pv_cnt++;
 387                         }
 388                         pmap_user_reserve_replenish_stat += pv_cnt;
 389                         PV_HASHED_FREE_LIST(pvh_eh, pvh_et, pv_cnt);
 390                 }
 391 /* Wake threads throttled while the kernel reserve was being replenished.
 392  */
 393                 if (pmap_pv_throttled_waiters) {
 394                         pmap_pv_throttled_waiters = 0;
 395                         thread_wakeup(&pmap_user_pv_throttle_event);
 396                 }
 397                 /* Check if the kernel pool has been depleted since the
 398                  * first pass, to reduce refill latency.
 399                  */
 400                 if (pv_hashed_kern_free_count < pv_hashed_kern_low_water_mark)
 401                         continue;
 402                 /* Block sans continuation to avoid yielding kernel stack */
 403                 assert_wait(&mapping_replenish_event, THREAD_UNINT);
 404                 mappingrecurse = 0;
 405                 thread_block(THREAD_CONTINUE_NULL);
 406                 pmap_mapping_thread_wakeups++;
 407         }
 408 }
 409
 410 /*
 411  *      Set specified attribute bits.
 412  */
 413
 414 void
 415 phys_attribute_set(
 416         ppnum_t         pn,
 417         int             bits)
 418 {
 419         int             pai;
 420
 421         pmap_intr_assert();
 422         assert(pn != vm_page_fictitious_addr);
 423         if (pn == vm_page_guard_addr)
 424                 return;
 425
 426         pai = ppn_to_pai(pn);
 427
 428         if (!IS_MANAGED_PAGE(pai)) {
 429                 /* Not a managed page.  */
 430                 return;
 431         }
 432
 433         LOCK_PVH(pai);
 434         pmap_phys_attributes[pai] |= bits;
 435         UNLOCK_PVH(pai);
 436 }
 437
 438 /*
 439  *      Set the modify bit on the specified physical page.
 440  */
 441
 442 void
 443 pmap_set_modify(ppnum_t pn)
 444 {
 445         phys_attribute_set(pn, PHYS_MODIFIED);
 446 }
 447
 448 /*
 449  *      Clear the modify bits on the specified physical page.
 450  */
 451
 452 void
 453 pmap_clear_modify(ppnum_t pn)
 454 {
 455         phys_attribute_clear(pn, PHYS_MODIFIED, 0, NULL);
 456 }
 457
 458 /*
 459  *      pmap_is_modified:
 460  *
 461  *      Return whether or not the specified physical page is modified
 462  *      by any physical maps.
 463  */
 464
 465 boolean_t
 466 pmap_is_modified(ppnum_t pn)
 467 {
 468         if (phys_attribute_test(pn, PHYS_MODIFIED))
 469                 return TRUE;
 470         return FALSE;
 471 }
 472
 473
 474 /*
 475  *      pmap_clear_reference:
 476  *
 477  *      Clear the reference bit on the specified physical page.
 478  */
 479
 480 void
 481 pmap_clear_reference(ppnum_t pn)
 482 {
 483         phys_attribute_clear(pn, PHYS_REFERENCED, 0, NULL);
 484 }
 485
 486 void
 487 pmap_set_reference(ppnum_t pn)
 488 {
 489         phys_attribute_set(pn, PHYS_REFERENCED);
 490 }
 491
 492 /*
 493  *      pmap_is_referenced:
 494  *
 495  *      Return whether or not the specified physical page is referenced
 496  *      by any physical maps.
 497  */
 498
 499 boolean_t
 500 pmap_is_referenced(ppnum_t pn)
 501 {
 502         if (phys_attribute_test(pn, PHYS_REFERENCED))
 503                 return TRUE;
 504         return FALSE;
 505 }
 506
 507
 508 /*
 509  * pmap_get_refmod(phys)
 510  *  returns the referenced and modified bits of the specified
 511  *  physical page.
 512  */
 513 unsigned int
 514 pmap_get_refmod(ppnum_t pn)
 515 {
 516         int             refmod;
 517         unsigned int    retval = 0;
 518
 519         refmod = phys_attribute_test(pn, PHYS_MODIFIED | PHYS_REFERENCED);
 520
 521         if (refmod & PHYS_MODIFIED)
 522                 retval |= VM_MEM_MODIFIED;
 523         if (refmod & PHYS_REFERENCED)
 524                 retval |= VM_MEM_REFERENCED;
 525
 526         return (retval);
 527 }
 528
 529
 530 void
 531 pmap_clear_refmod_options(ppnum_t pn, unsigned int mask, unsigned int options, void *arg)
 532 {
 533         unsigned int  x86Mask;
 534
 535         x86Mask = (   ((mask &   VM_MEM_MODIFIED)?   PHYS_MODIFIED : 0)
 536                       | ((mask & VM_MEM_REFERENCED)? PHYS_REFERENCED : 0));
 537
 538         phys_attribute_clear(pn, x86Mask, options, arg);
 539 }
 540
 541 /*
 542  * pmap_clear_refmod(phys, mask)
 543  *  clears the referenced and modified bits as specified by the mask
 544  *  of the specified physical page.
 545  */
 546 void
 547 pmap_clear_refmod(ppnum_t pn, unsigned int mask)
 548 {
 549         unsigned int  x86Mask;
 550
 551         x86Mask = (   ((mask &   VM_MEM_MODIFIED)?   PHYS_MODIFIED : 0)
 552                     | ((mask & VM_MEM_REFERENCED)? PHYS_REFERENCED : 0));
 553
 554         phys_attribute_clear(pn, x86Mask, 0, NULL);
 555 }
 556
 557 unsigned int
 558 pmap_disconnect(ppnum_t pa)
 559 {
 560         return (pmap_disconnect_options(pa, 0, NULL));
 561 }
 562
 563 /*
 564  *      Routine:
 565  *              pmap_disconnect_options
 566  *
 567  *      Function:
 568  *              Disconnect all mappings for this page and return reference and change status
 569  *              in generic format.
 570  *
 571  */
 572 unsigned int
 573 pmap_disconnect_options(ppnum_t pa, unsigned int options, void *arg)
 574 {
 575         unsigned refmod, vmrefmod = 0;
 576
 577         pmap_page_protect_options(pa, 0, options, arg);         /* disconnect the page */
 578
 579         pmap_assert(pa != vm_page_fictitious_addr);
 580         if ((pa == vm_page_guard_addr) || !IS_MANAGED_PAGE(pa) || (options & PMAP_OPTIONS_NOREFMOD))
 581                 return 0;
 582         refmod = pmap_phys_attributes[pa] & (PHYS_MODIFIED | PHYS_REFERENCED);
 583
 584         if (refmod & PHYS_MODIFIED)
 585                 vmrefmod |= VM_MEM_MODIFIED;
 586         if (refmod & PHYS_REFERENCED)
 587                 vmrefmod |= VM_MEM_REFERENCED;
 588
 589         return vmrefmod;
 590 }