osfmk/i386/pmap_common.c

   1 /*
   2  * Copyright (c) 2000-2010 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 #include <vm/pmap.h>
  29 #include <i386/pmap_internal.h>
  30
  31 /*
  32  *      Each entry in the pv_head_table is locked by a bit in the
  33  *      pv_lock_table.  The lock bits are accessed by the physical
  34  *      address of the page they lock.
  35  */
  36
  37 char    *pv_lock_table;         /* pointer to array of bits */
  38 char    *pv_hash_lock_table;
  39
  40 pv_rooted_entry_t       pv_head_table;          /* array of entries, one per
  41                                                  * page */
  42 uint32_t                        pv_hashed_free_count = 0;
  43 uint32_t                        pv_hashed_kern_free_count = 0;
  44
  45 pmap_pagetable_corruption_record_t pmap_pagetable_corruption_records[PMAP_PAGETABLE_CORRUPTION_MAX_LOG];
  46 uint32_t pmap_pagetable_corruption_incidents;
  47 uint64_t pmap_pagetable_corruption_last_abstime = (~(0ULL) >> 1);
  48 uint64_t pmap_pagetable_corruption_interval_abstime;
  49 thread_call_t   pmap_pagetable_corruption_log_call;
  50 static thread_call_data_t       pmap_pagetable_corruption_log_call_data;
  51 boolean_t pmap_pagetable_corruption_timeout = FALSE;
  52
  53 volatile uint32_t       mappingrecurse = 0;
  54
  55 uint32_t  pv_hashed_low_water_mark, pv_hashed_kern_low_water_mark, pv_hashed_alloc_chunk, pv_hashed_kern_alloc_chunk;
  56
  57 thread_t mapping_replenish_thread;
  58 event_t mapping_replenish_event, pmap_user_pv_throttle_event;
  59
  60 uint64_t pmap_pv_throttle_stat, pmap_pv_throttled_waiters;
  61
  62 unsigned int pmap_cache_attributes(ppnum_t pn) {
  63         if (pmap_get_cache_attributes(pn) & INTEL_PTE_NCACHE)
  64                 return (VM_WIMG_IO);
  65         else
  66                 return (VM_WIMG_COPYBACK);
  67 }
  68
  69 void    pmap_set_cache_attributes(ppnum_t pn, unsigned int cacheattr) {
  70         unsigned int current, template = 0;
  71         int pai;
  72
  73         if (cacheattr & VM_MEM_NOT_CACHEABLE) {
  74                 if(!(cacheattr & VM_MEM_GUARDED))
  75                         template |= PHYS_PTA;
  76                 template |= PHYS_NCACHE;
  77         }
  78
  79         pmap_intr_assert();
  80
  81         assert((pn != vm_page_fictitious_addr) && (pn != vm_page_guard_addr));
  82
  83         pai = ppn_to_pai(pn);
  84
  85         if (!IS_MANAGED_PAGE(pai)) {
  86                 return;
  87         }
  88
  89         /* override cache attributes for this phys page
  90          * Does not walk through existing mappings to adjust,
  91          * assumes page is disconnected
  92          */
  93
  94         LOCK_PVH(pai);
  95
  96         pmap_update_cache_attributes_locked(pn, template);
  97
  98         current = pmap_phys_attributes[pai] & PHYS_CACHEABILITY_MASK;
  99         pmap_phys_attributes[pai] &= ~PHYS_CACHEABILITY_MASK;
 100         pmap_phys_attributes[pai] |= template;
 101
 102         UNLOCK_PVH(pai);
 103
 104         if ((template & PHYS_NCACHE) && !(current & PHYS_NCACHE)) {
 105                 pmap_sync_page_attributes_phys(pn);
 106         }
 107 }
 108
 109 unsigned        pmap_get_cache_attributes(ppnum_t pn) {
 110         if (last_managed_page == 0)
 111                 return 0;
 112
 113         if (!IS_MANAGED_PAGE(ppn_to_pai(pn))) {
 114             return INTEL_PTE_NCACHE;
 115         }
 116
 117         /*
 118          * The cache attributes are read locklessly for efficiency.
 119          */
 120         unsigned int attr = pmap_phys_attributes[ppn_to_pai(pn)];
 121         unsigned int template = 0;
 122
 123         if (attr & PHYS_PTA)
 124                 template |= INTEL_PTE_PTA;
 125         if (attr & PHYS_NCACHE)
 126                 template |= INTEL_PTE_NCACHE;
 127         return template;
 128 }
 129
 130
 131
 132 boolean_t
 133 pmap_is_noencrypt(ppnum_t pn)
 134 {
 135         int             pai;
 136
 137         pai = ppn_to_pai(pn);
 138
 139         if (!IS_MANAGED_PAGE(pai))
 140                 return (FALSE);
 141
 142         if (pmap_phys_attributes[pai] & PHYS_NOENCRYPT)
 143                 return (TRUE);
 144
 145         return (FALSE);
 146 }
 147
 148
 149 void
 150 pmap_set_noencrypt(ppnum_t pn)
 151 {
 152         int             pai;
 153
 154         pai = ppn_to_pai(pn);
 155
 156         if (IS_MANAGED_PAGE(pai)) {
 157                 LOCK_PVH(pai);
 158
 159                 pmap_phys_attributes[pai] |= PHYS_NOENCRYPT;
 160
 161                 UNLOCK_PVH(pai);
 162         }
 163 }
 164
 165
 166 void
 167 pmap_clear_noencrypt(ppnum_t pn)
 168 {
 169         int             pai;
 170
 171         pai = ppn_to_pai(pn);
 172
 173         if (IS_MANAGED_PAGE(pai)) {
 174                 /*
 175                  * synchronization at VM layer prevents PHYS_NOENCRYPT
 176                  * from changing state, so we don't need the lock to inspect
 177                  */
 178                 if (pmap_phys_attributes[pai] & PHYS_NOENCRYPT) {
 179                         LOCK_PVH(pai);
 180
 181                         pmap_phys_attributes[pai] &= ~PHYS_NOENCRYPT;
 182
 183                         UNLOCK_PVH(pai);
 184                 }
 185         }
 186 }
 187
 188 void
 189 compute_pmap_gc_throttle(void *arg __unused)
 190 {
 191
 192 }
 193
 194
 195 __private_extern__ void
 196 pmap_pagetable_corruption_msg_log(int (*log_func)(const char * fmt, ...)__printflike(1,2)) {
 197         if (pmap_pagetable_corruption_incidents > 0) {
 198                 int i, e = MIN(pmap_pagetable_corruption_incidents, PMAP_PAGETABLE_CORRUPTION_MAX_LOG);
 199                 (*log_func)("%u pagetable corruption incident(s) detected, timeout: %u\n", pmap_pagetable_corruption_incidents, pmap_pagetable_corruption_timeout);
 200                 for (i = 0; i < e; i++) {
 201                         (*log_func)("Incident 0x%x, reason: 0x%x, action: 0x%x, time: 0x%llx\n", pmap_pagetable_corruption_records[i].incident,  pmap_pagetable_corruption_records[i].reason, pmap_pagetable_corruption_records[i].action, pmap_pagetable_corruption_records[i].abstime);
 202                 }
 203         }
 204 }
 205
 206 static inline void
 207 pmap_pagetable_corruption_log_setup(void) {
 208         if (pmap_pagetable_corruption_log_call == NULL) {
 209                 nanotime_to_absolutetime(PMAP_PAGETABLE_CORRUPTION_INTERVAL, 0, &pmap_pagetable_corruption_interval_abstime);
 210                 thread_call_setup(&pmap_pagetable_corruption_log_call_data,
 211                     (thread_call_func_t) pmap_pagetable_corruption_msg_log,
 212                     (thread_call_param_t) &printf);
 213                 pmap_pagetable_corruption_log_call = &pmap_pagetable_corruption_log_call_data;
 214         }
 215 }
 216
 217 void
 218 mapping_free_prime(void)
 219 {
 220         unsigned                i;
 221         pv_hashed_entry_t       pvh_e;
 222         pv_hashed_entry_t       pvh_eh;
 223         pv_hashed_entry_t       pvh_et;
 224         int                     pv_cnt;
 225
 226         /* Scale based on DRAM size */
 227         pv_hashed_low_water_mark = MAX(PV_HASHED_LOW_WATER_MARK_DEFAULT, ((uint32_t)(sane_size >> 30)) * 2000);
 228         pv_hashed_low_water_mark = MIN(pv_hashed_low_water_mark, 16000);
 229         /* Alterable via sysctl */
 230         pv_hashed_kern_low_water_mark = MAX(PV_HASHED_KERN_LOW_WATER_MARK_DEFAULT, ((uint32_t)(sane_size >> 30)) * 1000);
 231         pv_hashed_kern_low_water_mark = MIN(pv_hashed_kern_low_water_mark, 16000);
 232         pv_hashed_kern_alloc_chunk = PV_HASHED_KERN_ALLOC_CHUNK_INITIAL;
 233         pv_hashed_alloc_chunk = PV_HASHED_ALLOC_CHUNK_INITIAL;
 234
 235         pv_cnt = 0;
 236         pvh_eh = pvh_et = PV_HASHED_ENTRY_NULL;
 237
 238         for (i = 0; i < (5 * PV_HASHED_ALLOC_CHUNK_INITIAL); i++) {
 239                 pvh_e = (pv_hashed_entry_t) zalloc(pv_hashed_list_zone);
 240
 241                 pvh_e->qlink.next = (queue_entry_t)pvh_eh;
 242                 pvh_eh = pvh_e;
 243
 244                 if (pvh_et == PV_HASHED_ENTRY_NULL)
 245                         pvh_et = pvh_e;
 246                 pv_cnt++;
 247         }
 248         PV_HASHED_FREE_LIST(pvh_eh, pvh_et, pv_cnt);
 249
 250         pv_cnt = 0;
 251         pvh_eh = pvh_et = PV_HASHED_ENTRY_NULL;
 252         for (i = 0; i < PV_HASHED_KERN_ALLOC_CHUNK_INITIAL; i++) {
 253                 pvh_e = (pv_hashed_entry_t) zalloc(pv_hashed_list_zone);
 254
 255                 pvh_e->qlink.next = (queue_entry_t)pvh_eh;
 256                 pvh_eh = pvh_e;
 257
 258                 if (pvh_et == PV_HASHED_ENTRY_NULL)
 259                         pvh_et = pvh_e;
 260                 pv_cnt++;
 261         }
 262         PV_HASHED_KERN_FREE_LIST(pvh_eh, pvh_et, pv_cnt);
 263 }
 264
 265 void mapping_replenish(void);
 266
 267 void mapping_adjust(void) {
 268         kern_return_t mres;
 269
 270         pmap_pagetable_corruption_log_setup();
 271
 272         mres = kernel_thread_start_priority((thread_continue_t)mapping_replenish, NULL, MAXPRI_KERNEL, &mapping_replenish_thread);
 273         if (mres != KERN_SUCCESS) {
 274                 panic("pmap: mapping_replenish_thread creation failed");
 275         }
 276         thread_deallocate(mapping_replenish_thread);
 277 }
 278
 279 unsigned pmap_mapping_thread_wakeups;
 280 unsigned pmap_kernel_reserve_replenish_stat;
 281 unsigned pmap_user_reserve_replenish_stat;
 282 unsigned pmap_kern_reserve_alloc_stat;
 283
 284 void mapping_replenish(void)
 285 {
 286         pv_hashed_entry_t       pvh_e;
 287         pv_hashed_entry_t       pvh_eh;
 288         pv_hashed_entry_t       pvh_et;
 289         int                     pv_cnt;
 290         unsigned                i;
 291
 292         /* We qualify for VM privileges...*/
 293         current_thread()->options |= TH_OPT_VMPRIV;
 294
 295         for (;;) {
 296
 297                 while (pv_hashed_kern_free_count < pv_hashed_kern_low_water_mark) {
 298                         pv_cnt = 0;
 299                         pvh_eh = pvh_et = PV_HASHED_ENTRY_NULL;
 300
 301                         for (i = 0; i < pv_hashed_kern_alloc_chunk; i++) {
 302                                 pvh_e = (pv_hashed_entry_t) zalloc(pv_hashed_list_zone);
 303                                 pvh_e->qlink.next = (queue_entry_t)pvh_eh;
 304                                 pvh_eh = pvh_e;
 305
 306                                 if (pvh_et == PV_HASHED_ENTRY_NULL)
 307                                         pvh_et = pvh_e;
 308                                 pv_cnt++;
 309                         }
 310                         pmap_kernel_reserve_replenish_stat += pv_cnt;
 311                         PV_HASHED_KERN_FREE_LIST(pvh_eh, pvh_et, pv_cnt);
 312                 }
 313
 314                 pv_cnt = 0;
 315                 pvh_eh = pvh_et = PV_HASHED_ENTRY_NULL;
 316
 317                 if (pv_hashed_free_count < pv_hashed_low_water_mark) {
 318                         for (i = 0; i < pv_hashed_alloc_chunk; i++) {
 319                                 pvh_e = (pv_hashed_entry_t) zalloc(pv_hashed_list_zone);
 320
 321                                 pvh_e->qlink.next = (queue_entry_t)pvh_eh;
 322                                 pvh_eh = pvh_e;
 323
 324                                 if (pvh_et == PV_HASHED_ENTRY_NULL)
 325                                         pvh_et = pvh_e;
 326                                 pv_cnt++;
 327                         }
 328                         pmap_user_reserve_replenish_stat += pv_cnt;
 329                         PV_HASHED_FREE_LIST(pvh_eh, pvh_et, pv_cnt);
 330                 }
 331 /* Wake threads throttled while the kernel reserve was being replenished.
 332  */
 333                 if (pmap_pv_throttled_waiters) {
 334                         pmap_pv_throttled_waiters = 0;
 335                         thread_wakeup(&pmap_user_pv_throttle_event);
 336                 }
 337                 /* Check if the kernel pool has been depleted since the
 338                  * first pass, to reduce refill latency.
 339                  */
 340                 if (pv_hashed_kern_free_count < pv_hashed_kern_low_water_mark)
 341                         continue;
 342                 /* Block sans continuation to avoid yielding kernel stack */
 343                 assert_wait(&mapping_replenish_event, THREAD_UNINT);
 344                 mappingrecurse = 0;
 345                 thread_block(THREAD_CONTINUE_NULL);
 346                 pmap_mapping_thread_wakeups++;
 347         }
 348 }
 349
 350 /*
 351  *      Set specified attribute bits.
 352  */
 353
 354 void
 355 phys_attribute_set(
 356         ppnum_t         pn,
 357         int             bits)
 358 {
 359         int             pai;
 360
 361         pmap_intr_assert();
 362         assert(pn != vm_page_fictitious_addr);
 363         if (pn == vm_page_guard_addr)
 364                 return;
 365
 366         pai = ppn_to_pai(pn);
 367
 368         if (!IS_MANAGED_PAGE(pai)) {
 369                 /* Not a managed page.  */
 370                 return;
 371         }
 372
 373         LOCK_PVH(pai);
 374         pmap_phys_attributes[pai] |= bits;
 375         UNLOCK_PVH(pai);
 376 }
 377
 378 /*
 379  *      Set the modify bit on the specified physical page.
 380  */
 381
 382 void
 383 pmap_set_modify(ppnum_t pn)
 384 {
 385         phys_attribute_set(pn, PHYS_MODIFIED);
 386 }
 387
 388 /*
 389  *      Clear the modify bits on the specified physical page.
 390  */
 391
 392 void
 393 pmap_clear_modify(ppnum_t pn)
 394 {
 395         phys_attribute_clear(pn, PHYS_MODIFIED);
 396 }
 397
 398 /*
 399  *      pmap_is_modified:
 400  *
 401  *      Return whether or not the specified physical page is modified
 402  *      by any physical maps.
 403  */
 404
 405 boolean_t
 406 pmap_is_modified(ppnum_t pn)
 407 {
 408         if (phys_attribute_test(pn, PHYS_MODIFIED))
 409                 return TRUE;
 410         return FALSE;
 411 }
 412
 413
 414 /*
 415  *      pmap_clear_reference:
 416  *
 417  *      Clear the reference bit on the specified physical page.
 418  */
 419
 420 void
 421 pmap_clear_reference(ppnum_t pn)
 422 {
 423         phys_attribute_clear(pn, PHYS_REFERENCED);
 424 }
 425
 426 void
 427 pmap_set_reference(ppnum_t pn)
 428 {
 429         phys_attribute_set(pn, PHYS_REFERENCED);
 430 }
 431
 432 /*
 433  *      pmap_is_referenced:
 434  *
 435  *      Return whether or not the specified physical page is referenced
 436  *      by any physical maps.
 437  */
 438
 439 boolean_t
 440 pmap_is_referenced(ppnum_t pn)
 441 {
 442         if (phys_attribute_test(pn, PHYS_REFERENCED))
 443                 return TRUE;
 444         return FALSE;
 445 }
 446
 447
 448 /*
 449  * pmap_get_refmod(phys)
 450  *  returns the referenced and modified bits of the specified
 451  *  physical page.
 452  */
 453 unsigned int
 454 pmap_get_refmod(ppnum_t pn)
 455 {
 456         int             refmod;
 457         unsigned int    retval = 0;
 458
 459         refmod = phys_attribute_test(pn, PHYS_MODIFIED | PHYS_REFERENCED);
 460
 461         if (refmod & PHYS_MODIFIED)
 462                 retval |= VM_MEM_MODIFIED;
 463         if (refmod & PHYS_REFERENCED)
 464                 retval |= VM_MEM_REFERENCED;
 465
 466         return (retval);
 467 }
 468
 469 /*
 470  * pmap_clear_refmod(phys, mask)
 471  *  clears the referenced and modified bits as specified by the mask
 472  *  of the specified physical page.
 473  */
 474 void
 475 pmap_clear_refmod(ppnum_t pn, unsigned int mask)
 476 {
 477         unsigned int  x86Mask;
 478
 479         x86Mask = (   ((mask &   VM_MEM_MODIFIED)?   PHYS_MODIFIED : 0)
 480                     | ((mask & VM_MEM_REFERENCED)? PHYS_REFERENCED : 0));
 481         phys_attribute_clear(pn, x86Mask);
 482 }
 483
 484 /*
 485  *      Routine:
 486  *              pmap_disconnect
 487  *
 488  *      Function:
 489  *              Disconnect all mappings for this page and return reference and change status
 490  *              in generic format.
 491  *
 492  */
 493 unsigned int
 494 pmap_disconnect(ppnum_t pa)
 495 {
 496         unsigned refmod, vmrefmod = 0;
 497
 498         pmap_page_protect(pa, 0);               /* disconnect the page */
 499
 500         pmap_assert(pa != vm_page_fictitious_addr);
 501         if ((pa == vm_page_guard_addr) || !IS_MANAGED_PAGE(pa))
 502                 return 0;
 503         refmod = pmap_phys_attributes[pa] & (PHYS_MODIFIED | PHYS_REFERENCED);
 504
 505         if (refmod & PHYS_MODIFIED)
 506                 vmrefmod |= VM_MEM_MODIFIED;
 507         if (refmod & PHYS_REFERENCED)
 508                 vmrefmod |= VM_MEM_REFERENCED;
 509
 510         return vmrefmod;
 511 }