osfmk/i386/mtrr.c

   1 /*
   2  * Copyright (c) 2000-2011 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <mach/kern_return.h>
  30 #include <kern/zalloc.h>
  31 #include <kern/cpu_number.h>
  32 #include <kern/cpu_data.h>
  33 #include <i386/cpuid.h>
  34 #include <i386/mp.h>
  35 #include <i386/proc_reg.h>
  36 #include <i386/mtrr.h>
  37 #include <i386/machine_check.h>
  38
  39 struct mtrr_var_range {
  40         uint64_t  base;         /* in IA32_MTRR_PHYSBASE format */
  41         uint64_t  mask;         /* in IA32_MTRR_PHYSMASK format */
  42         uint32_t  refcnt;       /* var ranges reference count */
  43 };
  44
  45 struct mtrr_fix_range {
  46         uint64_t  types;        /* fixed-range type octet */
  47 };
  48
  49 typedef struct mtrr_var_range mtrr_var_range_t;
  50 typedef struct mtrr_fix_range mtrr_fix_range_t;
  51
  52 static struct {
  53         uint64_t            MTRRcap;
  54         uint64_t            MTRRdefType;
  55         mtrr_var_range_t *  var_range;
  56         unsigned int        var_count;
  57         mtrr_fix_range_t    fix_range[11];
  58 } mtrr_state;
  59
  60 static boolean_t mtrr_initialized = FALSE;
  61
  62 decl_simple_lock_data(static, mtrr_lock);
  63 #define MTRR_LOCK()     simple_lock(&mtrr_lock, LCK_GRP_NULL);
  64 #define MTRR_UNLOCK()   simple_unlock(&mtrr_lock);
  65
  66 //#define MTRR_DEBUG 1
  67 #if     MTRR_DEBUG
  68 #define DBG(x...)       kprintf(x)
  69 #else
  70 #define DBG(x...)
  71 #endif
  72
  73 /* Private functions */
  74 static void mtrr_get_var_ranges(mtrr_var_range_t * range, int count);
  75 static void mtrr_set_var_ranges(const mtrr_var_range_t * range, int count);
  76 static void mtrr_get_fix_ranges(mtrr_fix_range_t * range);
  77 static void mtrr_set_fix_ranges(const mtrr_fix_range_t * range);
  78 static void mtrr_update_setup(void * param);
  79 static void mtrr_update_teardown(void * param);
  80 static void mtrr_update_action(void * param);
  81 static void var_range_encode(mtrr_var_range_t * range, addr64_t address,
  82     uint64_t length, uint32_t type, int valid);
  83 static int  var_range_overlap(mtrr_var_range_t * range, addr64_t address,
  84     uint64_t length, uint32_t type);
  85
  86 #define CACHE_CONTROL_MTRR              (NULL)
  87 #define CACHE_CONTROL_PAT               ((void *)1)
  88
  89 /*
  90  * MTRR MSR bit fields.
  91  */
  92 #define IA32_MTRR_DEF_TYPE_MT           0x000000ff
  93 #define IA32_MTRR_DEF_TYPE_FE           0x00000400
  94 #define IA32_MTRR_DEF_TYPE_E            0x00000800
  95
  96 #define IA32_MTRRCAP_VCNT               0x000000ff
  97 #define IA32_MTRRCAP_FIX                0x00000100
  98 #define IA32_MTRRCAP_WC                 0x00000400
  99
 100 /* 0 < bits <= 64 */
 101 #define PHYS_BITS_TO_MASK(bits) \
 102         ((((1ULL << (bits-1)) - 1) << 1) | 1)
 103
 104 /*
 105  * Default mask for 36 physical address bits, this can
 106  * change depending on the cpu model.
 107  */
 108 static uint64_t mtrr_phys_mask = PHYS_BITS_TO_MASK(36);
 109
 110 #define IA32_MTRR_PHYMASK_VALID         0x0000000000000800ULL
 111 #define IA32_MTRR_PHYSBASE_MASK         (mtrr_phys_mask & ~0x0000000000000FFFULL)
 112 #define IA32_MTRR_PHYSBASE_TYPE         0x00000000000000FFULL
 113
 114 /*
 115  * Variable-range mask to/from length conversions.
 116  */
 117 #define MASK_TO_LEN(mask) \
 118         ((~((mask) & IA32_MTRR_PHYSBASE_MASK) & mtrr_phys_mask) + 1)
 119
 120 #define LEN_TO_MASK(len)  \
 121         (~((len) - 1) & IA32_MTRR_PHYSBASE_MASK)
 122
 123 #define LSB(x)          ((x) & (~((x) - 1)))
 124
 125 /*
 126  * Fetch variable-range MTRR register pairs.
 127  */
 128 static void
 129 mtrr_get_var_ranges(mtrr_var_range_t * range, int count)
 130 {
 131         int i;
 132
 133         for (i = 0; i < count; i++) {
 134                 range[i].base = rdmsr64(MSR_IA32_MTRR_PHYSBASE(i));
 135                 range[i].mask = rdmsr64(MSR_IA32_MTRR_PHYSMASK(i));
 136
 137                 /* bump ref count for firmware configured ranges */
 138                 if (range[i].mask & IA32_MTRR_PHYMASK_VALID) {
 139                         range[i].refcnt = 1;
 140                 } else {
 141                         range[i].refcnt = 0;
 142                 }
 143         }
 144 }
 145
 146 /*
 147  * Update variable-range MTRR register pairs.
 148  */
 149 static void
 150 mtrr_set_var_ranges(const mtrr_var_range_t * range, int count)
 151 {
 152         int i;
 153
 154         for (i = 0; i < count; i++) {
 155                 wrmsr64(MSR_IA32_MTRR_PHYSBASE(i), range[i].base);
 156                 wrmsr64(MSR_IA32_MTRR_PHYSMASK(i), range[i].mask);
 157         }
 158 }
 159
 160 /*
 161  * Fetch all fixed-range MTRR's. Note MSR offsets are not consecutive.
 162  */
 163 static void
 164 mtrr_get_fix_ranges(mtrr_fix_range_t * range)
 165 {
 166         int i;
 167
 168         /* assume 11 fix range registers */
 169         range[0].types = rdmsr64(MSR_IA32_MTRR_FIX64K_00000);
 170         range[1].types = rdmsr64(MSR_IA32_MTRR_FIX16K_80000);
 171         range[2].types = rdmsr64(MSR_IA32_MTRR_FIX16K_A0000);
 172         for (i = 0; i < 8; i++) {
 173                 range[3 + i].types = rdmsr64(MSR_IA32_MTRR_FIX4K_C0000 + i);
 174         }
 175 }
 176
 177 /*
 178  * Update all fixed-range MTRR's.
 179  */
 180 static void
 181 mtrr_set_fix_ranges(const struct mtrr_fix_range * range)
 182 {
 183         int i;
 184
 185         /* assume 11 fix range registers */
 186         wrmsr64(MSR_IA32_MTRR_FIX64K_00000, range[0].types);
 187         wrmsr64(MSR_IA32_MTRR_FIX16K_80000, range[1].types);
 188         wrmsr64(MSR_IA32_MTRR_FIX16K_A0000, range[2].types);
 189         for (i = 0; i < 8; i++) {
 190                 wrmsr64(MSR_IA32_MTRR_FIX4K_C0000 + i, range[3 + i].types);
 191         }
 192 }
 193
 194 static boolean_t
 195 mtrr_check_fix_ranges(const struct mtrr_fix_range * range)
 196 {
 197         int             i;
 198         boolean_t       match = TRUE;
 199
 200         DBG("CPU%d: %s\n", get_cpu_number(), __FUNCTION__);
 201
 202         /* assume 11 fix range registers */
 203         match = range[0].types == rdmsr64(MSR_IA32_MTRR_FIX64K_00000) &&
 204             range[1].types == rdmsr64(MSR_IA32_MTRR_FIX16K_80000) &&
 205             range[2].types == rdmsr64(MSR_IA32_MTRR_FIX16K_A0000);
 206         for (i = 0; match && i < 8; i++) {
 207                 match = range[3 + i].types ==
 208                     rdmsr64(MSR_IA32_MTRR_FIX4K_C0000 + i);
 209         }
 210
 211         return match;
 212 }
 213
 214 static boolean_t
 215 mtrr_check_var_ranges(mtrr_var_range_t * range, int count)
 216 {
 217         int             i;
 218         boolean_t       match = TRUE;
 219
 220         DBG("CPU%d: %s\n", get_cpu_number(), __FUNCTION__);
 221
 222         for (i = 0; match && i < count; i++) {
 223                 match = range[i].base == rdmsr64(MSR_IA32_MTRR_PHYSBASE(i)) &&
 224                     range[i].mask == rdmsr64(MSR_IA32_MTRR_PHYSMASK(i));
 225         }
 226
 227         return match;
 228 }
 229
 230 #if MTRR_DEBUG
 231 static void
 232 mtrr_msr_dump(void)
 233 {
 234         int i;
 235         int count = rdmsr64(MSR_IA32_MTRRCAP) & IA32_MTRRCAP_VCNT;
 236
 237         DBG("VAR -- BASE -------------- MASK -------------- SIZE\n");
 238         for (i = 0; i < count; i++) {
 239                 DBG(" %02x    0x%016llx  0x%016llx  0x%llx\n", i,
 240                     rdmsr64(MSR_IA32_MTRR_PHYSBASE(i)),
 241                     rdmsr64(MSR_IA32_MTRR_PHYSMASK(i)),
 242                     MASK_TO_LEN(rdmsr64(MSR_IA32_MTRR_PHYSMASK(i))));
 243         }
 244         DBG("\n");
 245
 246         DBG("FIX64K_00000: 0x%016llx\n", rdmsr64(MSR_IA32_MTRR_FIX64K_00000));
 247         DBG("FIX16K_80000: 0x%016llx\n", rdmsr64(MSR_IA32_MTRR_FIX16K_80000));
 248         DBG("FIX16K_A0000: 0x%016llx\n", rdmsr64(MSR_IA32_MTRR_FIX16K_A0000));
 249         DBG(" FIX4K_C0000: 0x%016llx\n", rdmsr64(MSR_IA32_MTRR_FIX4K_C0000));
 250         DBG(" FIX4K_C8000: 0x%016llx\n", rdmsr64(MSR_IA32_MTRR_FIX4K_C8000));
 251         DBG(" FIX4K_D0000: 0x%016llx\n", rdmsr64(MSR_IA32_MTRR_FIX4K_D0000));
 252         DBG(" FIX4K_D8000: 0x%016llx\n", rdmsr64(MSR_IA32_MTRR_FIX4K_D8000));
 253         DBG(" FIX4K_E0000: 0x%016llx\n", rdmsr64(MSR_IA32_MTRR_FIX4K_E0000));
 254         DBG(" FIX4K_E8000: 0x%016llx\n", rdmsr64(MSR_IA32_MTRR_FIX4K_E8000));
 255         DBG(" FIX4K_F0000: 0x%016llx\n", rdmsr64(MSR_IA32_MTRR_FIX4K_F0000));
 256         DBG(" FIX4K_F8000: 0x%016llx\n", rdmsr64(MSR_IA32_MTRR_FIX4K_F8000));
 257
 258         DBG("\nMTRRcap = 0x%llx MTRRdefType = 0x%llx\n",
 259             rdmsr64(MSR_IA32_MTRRCAP), rdmsr64(MSR_IA32_MTRR_DEF_TYPE));
 260 }
 261 #endif /* MTRR_DEBUG */
 262
 263 /*
 264  * Called by the boot processor (BP) early during boot to initialize MTRR
 265  * support.  The MTRR state on the BP is saved, any additional processors
 266  * will have the same settings applied to ensure MTRR consistency.
 267  */
 268 void
 269 mtrr_init(void)
 270 {
 271         /* no reason to init more than once */
 272         if (mtrr_initialized == TRUE) {
 273                 return;
 274         }
 275
 276         /* check for presence of MTRR feature on the processor */
 277         if ((cpuid_features() & CPUID_FEATURE_MTRR) == 0) {
 278                 return;  /* no MTRR feature */
 279         }
 280         /* use a lock to serialize MTRR changes */
 281         bzero((void *)&mtrr_state, sizeof(mtrr_state));
 282         simple_lock_init(&mtrr_lock, 0);
 283
 284         mtrr_state.MTRRcap     = rdmsr64(MSR_IA32_MTRRCAP);
 285         mtrr_state.MTRRdefType = rdmsr64(MSR_IA32_MTRR_DEF_TYPE);
 286         mtrr_state.var_count   = (unsigned int)(mtrr_state.MTRRcap & IA32_MTRRCAP_VCNT);
 287
 288         /* allocate storage for variable ranges (can block?) */
 289         if (mtrr_state.var_count) {
 290                 mtrr_state.var_range = (mtrr_var_range_t *)
 291                     zalloc_permanent(sizeof(mtrr_var_range_t) *
 292                     mtrr_state.var_count, ZALIGN(mtrr_var_range_t));
 293                 if (mtrr_state.var_range == NULL) {
 294                         mtrr_state.var_count = 0;
 295                 }
 296         }
 297
 298         /* fetch the initial firmware configured variable ranges */
 299         if (mtrr_state.var_count) {
 300                 mtrr_get_var_ranges(mtrr_state.var_range,
 301                     mtrr_state.var_count);
 302         }
 303
 304         /* fetch the initial firmware configured fixed ranges */
 305         if (mtrr_state.MTRRcap & IA32_MTRRCAP_FIX) {
 306                 mtrr_get_fix_ranges(mtrr_state.fix_range);
 307         }
 308
 309         mtrr_initialized = TRUE;
 310
 311 #if MTRR_DEBUG
 312         mtrr_msr_dump();        /* dump firmware settings */
 313 #endif
 314 }
 315
 316 /*
 317  * Performs the Intel recommended procedure for changing the MTRR
 318  * in a MP system. Leverage rendezvous mechanism for the required
 319  * barrier synchronization among all processors. This function is
 320  * called from the rendezvous IPI handler, and mtrr_update_cpu().
 321  */
 322 static void
 323 mtrr_update_action(void * cache_control_type)
 324 {
 325         uintptr_t cr0, cr4;
 326         uintptr_t tmp;
 327
 328         cr0 = get_cr0();
 329         cr4 = get_cr4();
 330
 331         /* enter no-fill cache mode */
 332         tmp = cr0 | CR0_CD;
 333         tmp &= ~CR0_NW;
 334         set_cr0(tmp);
 335
 336         /* flush caches */
 337         wbinvd();
 338
 339         /* clear the PGE flag in CR4 */
 340         if (cr4 & CR4_PGE) {
 341                 set_cr4(cr4 & ~CR4_PGE);
 342         } else {
 343                 set_cr3_raw(get_cr3_raw());
 344         }
 345
 346         if (CACHE_CONTROL_PAT == cache_control_type) {
 347                 /* Change PA6 attribute field to WC */
 348                 uint64_t pat = rdmsr64(MSR_IA32_CR_PAT);
 349                 DBG("CPU%d PAT: was 0x%016llx\n", get_cpu_number(), pat);
 350                 /*
 351                  * Intel doc states:
 352                  * "The IA32_PAT MSR contains eight page attribute fields: PA0 through PA7.
 353                  * The three low-order bits of each field are used to specify a memory type.
 354                  * The five high-order bits of each field are reserved, and must be set to all 0s."
 355                  * So, we zero-out the high 5 bits of the PA6 entry here:
 356                  */
 357                 pat &= ~(0xFFULL << 48);
 358                 pat |=  (0x01ULL << 48);
 359                 wrmsr64(MSR_IA32_CR_PAT, pat);
 360                 DBG("CPU%d PAT: is  0x%016llx\n",
 361                     get_cpu_number(), rdmsr64(MSR_IA32_CR_PAT));
 362         } else {
 363                 /* disable all MTRR ranges */
 364                 wrmsr64(MSR_IA32_MTRR_DEF_TYPE,
 365                     mtrr_state.MTRRdefType & ~IA32_MTRR_DEF_TYPE_E);
 366
 367                 /* apply MTRR settings */
 368                 if (mtrr_state.var_count) {
 369                         mtrr_set_var_ranges(mtrr_state.var_range,
 370                             mtrr_state.var_count);
 371                 }
 372
 373                 if (mtrr_state.MTRRcap & IA32_MTRRCAP_FIX) {
 374                         mtrr_set_fix_ranges(mtrr_state.fix_range);
 375                 }
 376
 377                 /* enable all MTRR range registers (what if E was not set?) */
 378                 wrmsr64(MSR_IA32_MTRR_DEF_TYPE,
 379                     mtrr_state.MTRRdefType | IA32_MTRR_DEF_TYPE_E);
 380         }
 381
 382         /* flush all caches and TLBs a second time */
 383         wbinvd();
 384         set_cr3_raw(get_cr3_raw());
 385         /* restore normal cache mode */
 386         set_cr0(cr0);
 387
 388         /* restore PGE flag */
 389         if (cr4 & CR4_PGE) {
 390                 set_cr4(cr4);
 391         }
 392
 393         DBG("CPU%d: %s\n", get_cpu_number(), __FUNCTION__);
 394 }
 395
 396 static void
 397 mtrr_update_setup(__unused void * param_not_used)
 398 {
 399         /* disable interrupts before the first barrier */
 400         current_cpu_datap()->cpu_iflag = ml_set_interrupts_enabled(FALSE);
 401         DBG("CPU%d: %s\n", get_cpu_number(), __FUNCTION__);
 402 }
 403
 404 static void
 405 mtrr_update_teardown(__unused void * param_not_used)
 406 {
 407         /* restore interrupt flag following MTRR changes */
 408         ml_set_interrupts_enabled(current_cpu_datap()->cpu_iflag);
 409         DBG("CPU%d: %s\n", get_cpu_number(), __FUNCTION__);
 410 }
 411
 412 /*
 413  * Update MTRR settings on all processors.
 414  */
 415 kern_return_t
 416 mtrr_update_all_cpus(void)
 417 {
 418         if (mtrr_initialized == FALSE) {
 419                 return KERN_NOT_SUPPORTED;
 420         }
 421
 422         MTRR_LOCK();
 423         mp_rendezvous(mtrr_update_setup,
 424             mtrr_update_action,
 425             mtrr_update_teardown, NULL);
 426         MTRR_UNLOCK();
 427
 428         return KERN_SUCCESS;
 429 }
 430
 431 /*
 432  * Verify that a processor has been set with the BSP's MTRR settings. Called
 433  * during slave processor initialization to check and set MTRR settings
 434  * discovered on the boot processor by mtrr_init().
 435  */
 436 kern_return_t
 437 mtrr_update_cpu(void)
 438 {
 439         boolean_t       match = TRUE;
 440
 441         if (mtrr_initialized == FALSE) {
 442                 return KERN_NOT_SUPPORTED;
 443         }
 444
 445         DBG("CPU%d: %s\n", get_cpu_number(), __FUNCTION__);
 446
 447         MTRR_LOCK();
 448
 449         /* Check MSR_IA32_MTRR_DEF_TYPE MSR */
 450         match = mtrr_state.MTRRdefType == rdmsr64(MSR_IA32_MTRR_DEF_TYPE);
 451
 452         /* Check MSR_IA32_MTRRCAP MSR */
 453         if (match) {
 454                 match = mtrr_state.MTRRcap == rdmsr64(MSR_IA32_MTRRCAP);
 455         }
 456
 457         /* Check variable ranges */
 458         if (match && mtrr_state.var_count) {
 459                 match = mtrr_check_var_ranges(mtrr_state.var_range,
 460                     mtrr_state.var_count);
 461         }
 462
 463         /* Check fixed ranges */
 464         if (match && (mtrr_state.MTRRcap & IA32_MTRRCAP_FIX)) {
 465                 match = mtrr_check_fix_ranges(mtrr_state.fix_range);
 466         }
 467
 468 #if MTRR_DEBUG
 469         if (!match) {
 470                 mtrr_msr_dump();
 471         }
 472 #endif
 473         if (!match) {
 474                 DBG("mtrr_update_cpu() setting MTRR for cpu %d\n",
 475                     get_cpu_number());
 476                 mtrr_update_action(NULL);
 477         }
 478 #if MTRR_DEBUG
 479         if (!match) {
 480                 mtrr_msr_dump();
 481         }
 482 #endif
 483
 484         MTRR_UNLOCK();
 485
 486         return KERN_SUCCESS;
 487 }
 488
 489 /*
 490  * Add a MTRR range to associate the physical memory range specified
 491  * with a given memory caching type.
 492  */
 493 kern_return_t
 494 mtrr_range_add(addr64_t address, uint64_t length, uint32_t type)
 495 {
 496         mtrr_var_range_t * vr;
 497         mtrr_var_range_t * free_range;
 498         kern_return_t      ret = KERN_NO_SPACE;
 499         int                overlap;
 500         unsigned int       i;
 501
 502         DBG("mtrr_range_add base = 0x%llx, size = 0x%llx, type = %d\n",
 503             address, length, type);
 504
 505         if (mtrr_initialized == FALSE) {
 506                 return KERN_NOT_SUPPORTED;
 507         }
 508
 509         /* check memory type (GPF exception for undefined types) */
 510         if ((type != MTRR_TYPE_UNCACHEABLE) &&
 511             (type != MTRR_TYPE_WRITECOMBINE) &&
 512             (type != MTRR_TYPE_WRITETHROUGH) &&
 513             (type != MTRR_TYPE_WRITEPROTECT) &&
 514             (type != MTRR_TYPE_WRITEBACK)) {
 515                 return KERN_INVALID_ARGUMENT;
 516         }
 517
 518         /* check WC support if requested */
 519         if ((type == MTRR_TYPE_WRITECOMBINE) &&
 520             (mtrr_state.MTRRcap & IA32_MTRRCAP_WC) == 0) {
 521                 return KERN_NOT_SUPPORTED;
 522         }
 523
 524         /* leave the fix range area below 1MB alone */
 525         if (address < 0x100000 || mtrr_state.var_count == 0) {
 526                 return KERN_NOT_SUPPORTED;
 527         }
 528
 529         /*
 530          * Length must be a power of 2 given by 2^n, where n >= 12.
 531          * Base address alignment must be larger than or equal to length.
 532          */
 533         if ((length < 0x1000) ||
 534             (LSB(length) != length) ||
 535             (address && (length > LSB(address)))) {
 536                 return KERN_INVALID_ARGUMENT;
 537         }
 538
 539         MTRR_LOCK();
 540
 541         /*
 542          * Check for overlap and locate a free range.
 543          */
 544         for (i = 0, free_range = NULL; i < mtrr_state.var_count; i++) {
 545                 vr = &mtrr_state.var_range[i];
 546
 547                 if (vr->refcnt == 0) {
 548                         /* free range candidate if no overlaps are found */
 549                         free_range = vr;
 550                         continue;
 551                 }
 552
 553                 overlap = var_range_overlap(vr, address, length, type);
 554                 if (overlap > 0) {
 555                         /*
 556                          * identical overlap permitted, increment ref count.
 557                          * no hardware update required.
 558                          */
 559                         free_range = vr;
 560                         break;
 561                 }
 562                 if (overlap < 0) {
 563                         /* unsupported overlapping of memory types */
 564                         free_range = NULL;
 565                         break;
 566                 }
 567         }
 568
 569         if (free_range) {
 570                 if (free_range->refcnt++ == 0) {
 571                         var_range_encode(free_range, address, length, type, 1);
 572                         mp_rendezvous(mtrr_update_setup,
 573                             mtrr_update_action,
 574                             mtrr_update_teardown, NULL);
 575                 }
 576                 ret = KERN_SUCCESS;
 577         }
 578
 579 #if MTRR_DEBUG
 580         mtrr_msr_dump();
 581 #endif
 582
 583         MTRR_UNLOCK();
 584
 585         return ret;
 586 }
 587
 588 /*
 589  * Remove a previously added MTRR range. The same arguments used for adding
 590  * the memory range must be supplied again.
 591  */
 592 kern_return_t
 593 mtrr_range_remove(addr64_t address, uint64_t length, uint32_t type)
 594 {
 595         mtrr_var_range_t * vr;
 596         int                result = KERN_FAILURE;
 597         int                cpu_update = 0;
 598         unsigned int       i;
 599
 600         DBG("mtrr_range_remove base = 0x%llx, size = 0x%llx, type = %d\n",
 601             address, length, type);
 602
 603         if (mtrr_initialized == FALSE) {
 604                 return KERN_NOT_SUPPORTED;
 605         }
 606
 607         MTRR_LOCK();
 608
 609         for (i = 0; i < mtrr_state.var_count; i++) {
 610                 vr = &mtrr_state.var_range[i];
 611
 612                 if (vr->refcnt &&
 613                     var_range_overlap(vr, address, length, type) > 0) {
 614                         /* found specified variable range */
 615                         if (--mtrr_state.var_range[i].refcnt == 0) {
 616                                 var_range_encode(vr, address, length, type, 0);
 617                                 cpu_update = 1;
 618                         }
 619                         result = KERN_SUCCESS;
 620                         break;
 621                 }
 622         }
 623
 624         if (cpu_update) {
 625                 mp_rendezvous(mtrr_update_setup,
 626                     mtrr_update_action,
 627                     mtrr_update_teardown, NULL);
 628                 result = KERN_SUCCESS;
 629         }
 630
 631 #if MTRR_DEBUG
 632         mtrr_msr_dump();
 633 #endif
 634
 635         MTRR_UNLOCK();
 636
 637         return result;
 638 }
 639
 640 /*
 641  * Variable range helper routines
 642  */
 643 static void
 644 var_range_encode(mtrr_var_range_t * range, addr64_t address,
 645     uint64_t length, uint32_t type, int valid)
 646 {
 647         range->base = (address & IA32_MTRR_PHYSBASE_MASK) |
 648             (type    & (uint32_t)IA32_MTRR_PHYSBASE_TYPE);
 649
 650         range->mask = LEN_TO_MASK(length) |
 651             (valid ? IA32_MTRR_PHYMASK_VALID : 0);
 652 }
 653
 654 static int
 655 var_range_overlap(mtrr_var_range_t * range, addr64_t address,
 656     uint64_t length, uint32_t type)
 657 {
 658         uint64_t  v_address, v_length;
 659         uint32_t  v_type;
 660         int       result = 0;  /* no overlap, or overlap ok */
 661
 662         v_address = range->base & IA32_MTRR_PHYSBASE_MASK;
 663         v_type    = (uint32_t)(range->base & IA32_MTRR_PHYSBASE_TYPE);
 664         v_length  = MASK_TO_LEN(range->mask);
 665
 666         /* detect range overlap */
 667         if ((v_address >= address && v_address < (address + length)) ||
 668             (address >= v_address && address < (v_address + v_length))) {
 669                 if (v_address == address && v_length == length && v_type == type) {
 670                         result = 1; /* identical overlap ok */
 671                 } else if (v_type == MTRR_TYPE_UNCACHEABLE &&
 672                     type == MTRR_TYPE_UNCACHEABLE) {
 673                         /* UC ranges can overlap */
 674                 } else if ((v_type == MTRR_TYPE_UNCACHEABLE &&
 675                     type == MTRR_TYPE_WRITEBACK) ||
 676                     (v_type == MTRR_TYPE_WRITEBACK &&
 677                     type == MTRR_TYPE_UNCACHEABLE)) {
 678                         /* UC/WB can overlap - effective type becomes UC */
 679                 } else {
 680                         /* anything else may cause undefined behavior */
 681                         result = -1;
 682                 }
 683         }
 684
 685         return result;
 686 }
 687
 688 /*
 689  * Initialize PAT (Page Attribute Table)
 690  */
 691 void
 692 pat_init(void)
 693 {
 694         boolean_t       istate;
 695         uint64_t        pat;
 696
 697         if (!(cpuid_features() & CPUID_FEATURE_PAT)) {
 698                 return;
 699         }
 700
 701         istate = ml_set_interrupts_enabled(FALSE);
 702
 703         pat = rdmsr64(MSR_IA32_CR_PAT);
 704         DBG("CPU%d PAT: was 0x%016llx\n", get_cpu_number(), pat);
 705
 706         /* Change PA6 attribute field to WC if required */
 707         if ((pat & (0x07ULL << 48)) != (0x01ULL << 48)) {
 708                 mtrr_update_action(CACHE_CONTROL_PAT);
 709         }
 710         ml_set_interrupts_enabled(istate);
 711 }