osfmk/i386/i386_vm_init.c

   1 /*
   2  * Copyright (c) 2003-2012 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 /*
  29  * @OSF_COPYRIGHT@
  30  */
  31 /*
  32  * Mach Operating System
  33  * Copyright (c) 1991,1990,1989, 1988 Carnegie Mellon University
  34  * All Rights Reserved.
  35  *
  36  * Permission to use, copy, modify and distribute this software and its
  37  * documentation is hereby granted, provided that both the copyright
  38  * notice and this permission notice appear in all copies of the
  39  * software, derivative works or modified versions, and any portions
  40  * thereof, and that both notices appear in supporting documentation.
  41  *
  42  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  43  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
  44  * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
  45  *
  46  * Carnegie Mellon requests users of this software to return to
  47  *
  48  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  49  *  School of Computer Science
  50  *  Carnegie Mellon University
  51  *  Pittsburgh PA 15213-3890
  52  *
  53  * any improvements or extensions that they make and grant Carnegie Mellon
  54  * the rights to redistribute these changes.
  55  */
  56
  57 #include <platforms.h>
  58
  59 #include <mach/i386/vm_param.h>
  60
  61 #include <string.h>
  62 #include <mach/vm_param.h>
  63 #include <mach/vm_prot.h>
  64 #include <mach/machine.h>
  65 #include <mach/time_value.h>
  66 #include <kern/spl.h>
  67 #include <kern/assert.h>
  68 #include <kern/debug.h>
  69 #include <kern/misc_protos.h>
  70 #include <kern/cpu_data.h>
  71 #include <kern/processor.h>
  72 #include <vm/vm_page.h>
  73 #include <vm/pmap.h>
  74 #include <vm/vm_kern.h>
  75 #include <i386/pmap.h>
  76 #include <i386/misc_protos.h>
  77 #include <i386/cpuid.h>
  78 #include <mach/thread_status.h>
  79 #include <pexpert/i386/efi.h>
  80 #include <i386/i386_lowmem.h>
  81 #include <x86_64/lowglobals.h>
  82 #include <i386/pal_routines.h>
  83
  84 #include <mach-o/loader.h>
  85 #include <libkern/kernel_mach_header.h>
  86
  87
  88 vm_size_t       mem_size = 0;
  89 pmap_paddr_t    first_avail = 0;/* first after page tables */
  90
  91 uint64_t        max_mem;        /* Size of physical memory (bytes), adjusted by maxmem */
  92 uint64_t        mem_actual;
  93 uint64_t        sane_size = 0;  /* Memory size for defaults calculations */
  94
  95 /*
  96  * KASLR parameters
  97  */
  98 ppnum_t         vm_kernel_base_page;
  99 vm_offset_t     vm_kernel_base;
 100 vm_offset_t     vm_kernel_top;
 101 vm_offset_t     vm_kernel_stext;
 102 vm_offset_t     vm_kernel_etext;
 103 vm_offset_t     vm_kernel_slide;
 104 vm_offset_t     vm_kext_base = VM_MIN_KERNEL_AND_KEXT_ADDRESS;
 105 vm_offset_t     vm_kext_top = VM_MIN_KERNEL_ADDRESS;
 106
 107 #define MAXLORESERVE    (32 * 1024 * 1024)
 108
 109 ppnum_t         max_ppnum = 0;
 110 ppnum_t         lowest_lo = 0;
 111 ppnum_t         lowest_hi = 0;
 112 ppnum_t         highest_hi = 0;
 113
 114 enum {PMAP_MAX_RESERVED_RANGES = 32};
 115 uint32_t pmap_reserved_pages_allocated = 0;
 116 uint32_t pmap_reserved_range_indices[PMAP_MAX_RESERVED_RANGES];
 117 uint32_t pmap_last_reserved_range_index = 0;
 118 uint32_t pmap_reserved_ranges = 0;
 119
 120 extern unsigned int bsd_mbuf_cluster_reserve(boolean_t *);
 121
 122 pmap_paddr_t     avail_start, avail_end;
 123 vm_offset_t     virtual_avail, virtual_end;
 124 static pmap_paddr_t     avail_remaining;
 125 vm_offset_t     static_memory_end = 0;
 126
 127 vm_offset_t     sHIB, eHIB, stext, etext, sdata, edata, sconstdata, econstdata, end;
 128
 129 /*
 130  * _mh_execute_header is the mach_header for the currently executing kernel
 131  */
 132 vm_offset_t segTEXTB; unsigned long segSizeTEXT;
 133 vm_offset_t segDATAB; unsigned long segSizeDATA;
 134 vm_offset_t segLINKB; unsigned long segSizeLINK;
 135 vm_offset_t segPRELINKB; unsigned long segSizePRELINK;
 136 vm_offset_t segHIBB; unsigned long segSizeHIB;
 137 vm_offset_t sectCONSTB; unsigned long sectSizeConst;
 138
 139 boolean_t doconstro_override = FALSE;
 140
 141 static kernel_segment_command_t *segTEXT, *segDATA;
 142 static kernel_section_t *cursectTEXT, *lastsectTEXT;
 143 static kernel_section_t *sectDCONST;
 144
 145 extern uint64_t firmware_Conventional_bytes;
 146 extern uint64_t firmware_RuntimeServices_bytes;
 147 extern uint64_t firmware_ACPIReclaim_bytes;
 148 extern uint64_t firmware_ACPINVS_bytes;
 149 extern uint64_t firmware_PalCode_bytes;
 150 extern uint64_t firmware_Reserved_bytes;
 151 extern uint64_t firmware_Unusable_bytes;
 152 extern uint64_t firmware_other_bytes;
 153 uint64_t firmware_MMIO_bytes;
 154
 155 /*
 156  * Linker magic to establish the highest address in the kernel.
 157  */
 158 extern void     *last_kernel_symbol;
 159
 160 #if     DEBUG
 161 #define PRINT_PMAP_MEMORY_TABLE
 162 #define DBG(x...)       kprintf(x)
 163 #else
 164 #define DBG(x...)
 165 #endif /* DEBUG */
 166 /*
 167  * Basic VM initialization.
 168  */
 169 void
 170 i386_vm_init(uint64_t   maxmem,
 171              boolean_t  IA32e,
 172              boot_args  *args)
 173 {
 174         pmap_memory_region_t *pmptr;
 175         pmap_memory_region_t *prev_pmptr;
 176         EfiMemoryRange *mptr;
 177         unsigned int mcount;
 178         unsigned int msize;
 179         ppnum_t fap;
 180         unsigned int i;
 181         ppnum_t maxpg = 0;
 182         uint32_t pmap_type;
 183         uint32_t maxloreserve;
 184         uint32_t maxdmaaddr;
 185         uint32_t  mbuf_reserve = 0;
 186         boolean_t mbuf_override = FALSE;
 187         boolean_t coalescing_permitted;
 188         vm_kernel_base_page = i386_btop(args->kaddr);
 189         vm_offset_t base_address;
 190         vm_offset_t static_base_address;
 191
 192         /*
 193          * Establish the KASLR parameters.
 194          */
 195         static_base_address = ml_static_ptovirt(KERNEL_BASE_OFFSET);
 196         base_address        = ml_static_ptovirt(args->kaddr);
 197         vm_kernel_slide     = base_address - static_base_address;
 198         if (args->kslide) {
 199                 kprintf("KASLR slide: 0x%016lx dynamic\n", vm_kernel_slide);
 200                 if (vm_kernel_slide != ((vm_offset_t)args->kslide))
 201                         panic("Kernel base inconsistent with slide - rebased?");
 202         } else {
 203                 /* No slide relative to on-disk symbols */
 204                 kprintf("KASLR slide: 0x%016lx static and ignored\n",
 205                         vm_kernel_slide);
 206                 vm_kernel_slide = 0;
 207         }
 208
 209         /*
 210          * Zero out local relocations to avoid confusing kxld.
 211          * TODO: might be better to move this code to OSKext::initialize
 212          */
 213         if (_mh_execute_header.flags & MH_PIE) {
 214                 struct load_command *loadcmd;
 215                 uint32_t cmd;
 216
 217                 loadcmd = (struct load_command *)((uintptr_t)&_mh_execute_header +
 218                                                   sizeof (_mh_execute_header));
 219
 220                 for (cmd = 0; cmd < _mh_execute_header.ncmds; cmd++) {
 221                         if (loadcmd->cmd == LC_DYSYMTAB) {
 222                                 struct dysymtab_command *dysymtab;
 223
 224                                 dysymtab = (struct dysymtab_command *)loadcmd;
 225                                 dysymtab->nlocrel = 0;
 226                                 dysymtab->locreloff = 0;
 227                                 kprintf("Hiding local relocations\n");
 228                                 break;
 229                         }
 230                         loadcmd = (struct load_command *)((uintptr_t)loadcmd + loadcmd->cmdsize);
 231                 }
 232         }
 233
 234         /*
 235          * Now retrieve addresses for end, edata, and etext
 236          * from MACH-O headers.
 237          */
 238         segTEXTB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header,
 239                                         "__TEXT", &segSizeTEXT);
 240         segDATAB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header,
 241                                         "__DATA", &segSizeDATA);
 242         segLINKB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header,
 243                                         "__LINKEDIT", &segSizeLINK);
 244         segHIBB  = (vm_offset_t) getsegdatafromheader(&_mh_execute_header,
 245                                         "__HIB", &segSizeHIB);
 246         segPRELINKB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header,
 247                                         "__PRELINK_TEXT", &segSizePRELINK);
 248         segTEXT = getsegbynamefromheader(&_mh_execute_header,
 249                                         "__TEXT");
 250         segDATA = getsegbynamefromheader(&_mh_execute_header,
 251                                         "__DATA");
 252         sectDCONST = getsectbynamefromheader(&_mh_execute_header,
 253                                         "__DATA", "__const");
 254         cursectTEXT = lastsectTEXT = firstsect(segTEXT);
 255         /* Discover the last TEXT section within the TEXT segment */
 256         while ((cursectTEXT = nextsect(segTEXT, cursectTEXT)) != NULL) {
 257                 lastsectTEXT = cursectTEXT;
 258         }
 259
 260         sHIB  = segHIBB;
 261         eHIB  = segHIBB + segSizeHIB;
 262         /* Zero-padded from ehib to stext if text is 2M-aligned */
 263         stext = segTEXTB;
 264         lowGlo.lgStext = stext;
 265         etext = (vm_offset_t) round_page_64(lastsectTEXT->addr + lastsectTEXT->size);
 266         /* Zero-padded from etext to sdata if text is 2M-aligned */
 267         sdata = segDATAB;
 268         edata = segDATAB + segSizeDATA;
 269
 270         sectCONSTB = (vm_offset_t) sectDCONST->addr;
 271         sectSizeConst = sectDCONST->size;
 272         sconstdata = sectCONSTB;
 273         econstdata = sectCONSTB + sectSizeConst;
 274
 275         if (sectSizeConst & PAGE_MASK) {
 276                 kernel_section_t *ns = nextsect(segDATA, sectDCONST);
 277                 if (ns && !(ns->addr & PAGE_MASK))
 278                         doconstro_override = TRUE;
 279         } else
 280                 doconstro_override = TRUE;
 281
 282         DBG("segTEXTB    = %p\n", (void *) segTEXTB);
 283         DBG("segDATAB    = %p\n", (void *) segDATAB);
 284         DBG("segLINKB    = %p\n", (void *) segLINKB);
 285         DBG("segHIBB     = %p\n", (void *) segHIBB);
 286         DBG("segPRELINKB = %p\n", (void *) segPRELINKB);
 287         DBG("sHIB        = %p\n", (void *) sHIB);
 288         DBG("eHIB        = %p\n", (void *) eHIB);
 289         DBG("stext       = %p\n", (void *) stext);
 290         DBG("etext       = %p\n", (void *) etext);
 291         DBG("sdata       = %p\n", (void *) sdata);
 292         DBG("edata       = %p\n", (void *) edata);
 293         DBG("sconstdata  = %p\n", (void *) sconstdata);
 294         DBG("econstdata  = %p\n", (void *) econstdata);
 295         DBG("kernel_top  = %p\n", (void *) &last_kernel_symbol);
 296
 297         vm_kernel_base  = sHIB;
 298         vm_kernel_top   = (vm_offset_t) &last_kernel_symbol;
 299         vm_kernel_stext = stext;
 300         vm_kernel_etext = etext;
 301
 302         vm_set_page_size();
 303
 304         /*
 305          * Compute the memory size.
 306          */
 307
 308         avail_remaining = 0;
 309         avail_end = 0;
 310         pmptr = pmap_memory_regions;
 311         prev_pmptr = 0;
 312         pmap_memory_region_count = pmap_memory_region_current = 0;
 313         fap = (ppnum_t) i386_btop(first_avail);
 314
 315         mptr = (EfiMemoryRange *)ml_static_ptovirt((vm_offset_t)args->MemoryMap);
 316         if (args->MemoryMapDescriptorSize == 0)
 317                 panic("Invalid memory map descriptor size");
 318         msize = args->MemoryMapDescriptorSize;
 319         mcount = args->MemoryMapSize / msize;
 320
 321 #define FOURGIG 0x0000000100000000ULL
 322 #define ONEGIG  0x0000000040000000ULL
 323
 324         for (i = 0; i < mcount; i++, mptr = (EfiMemoryRange *)(((vm_offset_t)mptr) + msize)) {
 325                 ppnum_t base, top;
 326                 uint64_t region_bytes = 0;
 327
 328                 if (pmap_memory_region_count >= PMAP_MEMORY_REGIONS_SIZE) {
 329                         kprintf("WARNING: truncating memory region count at %d\n", pmap_memory_region_count);
 330                         break;
 331                 }
 332                 base = (ppnum_t) (mptr->PhysicalStart >> I386_PGSHIFT);
 333                 top = (ppnum_t) (((mptr->PhysicalStart) >> I386_PGSHIFT) + mptr->NumberOfPages - 1);
 334
 335                 if (base == 0) {
 336                         /*
 337                          * Avoid having to deal with the edge case of the
 338                          * very first possible physical page and the roll-over
 339                          * to -1; just ignore that page.
 340                          */
 341                         kprintf("WARNING: ignoring first page in [0x%llx:0x%llx]\n", (uint64_t) base, (uint64_t) top);
 342                         base++;
 343                 }
 344                 if (top + 1 == 0) {
 345                         /*
 346                          * Avoid having to deal with the edge case of the
 347                          * very last possible physical page and the roll-over
 348                          * to 0; just ignore that page.
 349                          */
 350                         kprintf("WARNING: ignoring last page in [0x%llx:0x%llx]\n", (uint64_t) base, (uint64_t) top);
 351                         top--;
 352                 }
 353                 if (top < base) {
 354                         /*
 355                          * That was the only page in that region, so
 356                          * ignore the whole region.
 357                          */
 358                         continue;
 359                 }
 360
 361 #if     MR_RSV_TEST
 362                 static uint32_t nmr = 0;
 363                 if ((base > 0x20000) && (nmr++ < 4))
 364                         mptr->Attribute |= EFI_MEMORY_KERN_RESERVED;
 365 #endif
 366                 region_bytes = (uint64_t)(mptr->NumberOfPages << I386_PGSHIFT);
 367                 pmap_type = mptr->Type;
 368
 369                 switch (mptr->Type) {
 370                 case kEfiLoaderCode:
 371                 case kEfiLoaderData:
 372                 case kEfiBootServicesCode:
 373                 case kEfiBootServicesData:
 374                 case kEfiConventionalMemory:
 375                         /*
 376                          * Consolidate usable memory types into one.
 377                          */
 378                         pmap_type = kEfiConventionalMemory;
 379                         sane_size += region_bytes;
 380                         firmware_Conventional_bytes += region_bytes;
 381                         break;
 382                         /*
 383                          * sane_size should reflect the total amount of physical
 384                          * RAM in the system, not just the amount that is
 385                          * available for the OS to use.
 386                          * FIXME:Consider deriving this value from SMBIOS tables
 387                          * rather than reverse engineering the memory map.
 388                          * Alternatively, see
 389                          * <rdar://problem/4642773> Memory map should
 390                          * describe all memory
 391                          * Firmware on some systems guarantees that the memory
 392                          * map is complete via the "RomReservedMemoryTracked"
 393                          * feature field--consult that where possible to
 394                          * avoid the "round up to 128M" workaround below.
 395                          */
 396
 397                 case kEfiRuntimeServicesCode:
 398                 case kEfiRuntimeServicesData:
 399                         firmware_RuntimeServices_bytes += region_bytes;
 400                         sane_size += region_bytes;
 401                         break;
 402                 case kEfiACPIReclaimMemory:
 403                         firmware_ACPIReclaim_bytes += region_bytes;
 404                         sane_size += region_bytes;
 405                         break;
 406                 case kEfiACPIMemoryNVS:
 407                         firmware_ACPINVS_bytes += region_bytes;
 408                         sane_size += region_bytes;
 409                         break;
 410                 case kEfiPalCode:
 411                         firmware_PalCode_bytes += region_bytes;
 412                         sane_size += region_bytes;
 413                         break;
 414
 415                 case kEfiReservedMemoryType:
 416                         firmware_Reserved_bytes += region_bytes;
 417                         break;
 418                 case kEfiUnusableMemory:
 419                         firmware_Unusable_bytes += region_bytes;
 420                         break;
 421                 case kEfiMemoryMappedIO:
 422                 case kEfiMemoryMappedIOPortSpace:
 423                         firmware_MMIO_bytes += region_bytes;
 424                         break;
 425                 default:
 426                         firmware_other_bytes += region_bytes;
 427                         break;
 428                 }
 429
 430                 DBG("EFI region %d: type %u/%d, base 0x%x, top 0x%x %s\n",
 431                     i, mptr->Type, pmap_type, base, top,
 432                     (mptr->Attribute&EFI_MEMORY_KERN_RESERVED)? "RESERVED" :
 433                     (mptr->Attribute&EFI_MEMORY_RUNTIME)? "RUNTIME" : "");
 434
 435                 if (maxpg) {
 436                         if (base >= maxpg)
 437                                 break;
 438                         top = (top > maxpg) ? maxpg : top;
 439                 }
 440
 441                 /*
 442                  * handle each region
 443                  */
 444                 if ((mptr->Attribute & EFI_MEMORY_RUNTIME) == EFI_MEMORY_RUNTIME ||
 445                     pmap_type != kEfiConventionalMemory) {
 446                         prev_pmptr = 0;
 447                         continue;
 448                 } else {
 449                         /*
 450                          * Usable memory region
 451                          */
 452                         if (top < I386_LOWMEM_RESERVED ||
 453                             !pal_is_usable_memory(base, top)) {
 454                                 prev_pmptr = 0;
 455                                 continue;
 456                         }
 457                         /*
 458                          * A range may be marked with with the
 459                          * EFI_MEMORY_KERN_RESERVED attribute
 460                          * on some systems, to indicate that the range
 461                          * must not be made available to devices.
 462                          */
 463
 464                         if (mptr->Attribute & EFI_MEMORY_KERN_RESERVED) {
 465                                 if (++pmap_reserved_ranges > PMAP_MAX_RESERVED_RANGES) {
 466                                         panic("Too many reserved ranges %u\n", pmap_reserved_ranges);
 467                                 }
 468                         }
 469
 470                         if (top < fap) {
 471                                 /*
 472                                  * entire range below first_avail
 473                                  * salvage some low memory pages
 474                                  * we use some very low memory at startup
 475                                  * mark as already allocated here
 476                                  */
 477                                 if (base >= I386_LOWMEM_RESERVED)
 478                                         pmptr->base = base;
 479                                 else
 480                                         pmptr->base = I386_LOWMEM_RESERVED;
 481
 482                                 pmptr->end = top;
 483
 484
 485                                 if ((mptr->Attribute & EFI_MEMORY_KERN_RESERVED) &&
 486                                     (top < vm_kernel_base_page)) {
 487                                         pmptr->alloc_up = pmptr->base;
 488                                         pmptr->alloc_down = pmptr->end;
 489                                         pmap_reserved_range_indices[pmap_last_reserved_range_index++] = pmap_memory_region_count;
 490                                 }
 491                                 else {
 492                                         /*
 493                                          * mark as already mapped
 494                                          */
 495                                         pmptr->alloc_up = top + 1;
 496                                         pmptr->alloc_down = top;
 497                                 }
 498                                 pmptr->type = pmap_type;
 499                                 pmptr->attribute = mptr->Attribute;
 500                         }
 501                         else if ( (base < fap) && (top > fap) ) {
 502                                 /*
 503                                  * spans first_avail
 504                                  * put mem below first avail in table but
 505                                  * mark already allocated
 506                                  */
 507                                 pmptr->base = base;
 508                                 pmptr->end = (fap - 1);
 509                                 pmptr->alloc_up = pmptr->end + 1;
 510                                 pmptr->alloc_down = pmptr->end;
 511                                 pmptr->type = pmap_type;
 512                                 pmptr->attribute = mptr->Attribute;
 513                                 /*
 514                                  * we bump these here inline so the accounting
 515                                  * below works correctly
 516                                  */
 517                                 pmptr++;
 518                                 pmap_memory_region_count++;
 519
 520                                 pmptr->alloc_up = pmptr->base = fap;
 521                                 pmptr->type = pmap_type;
 522                                 pmptr->attribute = mptr->Attribute;
 523                                 pmptr->alloc_down = pmptr->end = top;
 524
 525                                 if (mptr->Attribute & EFI_MEMORY_KERN_RESERVED)
 526                                         pmap_reserved_range_indices[pmap_last_reserved_range_index++] = pmap_memory_region_count;
 527                         } else {
 528                                 /*
 529                                  * entire range useable
 530                                  */
 531                                 pmptr->alloc_up = pmptr->base = base;
 532                                 pmptr->type = pmap_type;
 533                                 pmptr->attribute = mptr->Attribute;
 534                                 pmptr->alloc_down = pmptr->end = top;
 535                                 if (mptr->Attribute & EFI_MEMORY_KERN_RESERVED)
 536                                         pmap_reserved_range_indices[pmap_last_reserved_range_index++] = pmap_memory_region_count;
 537                         }
 538
 539                         if (i386_ptob(pmptr->end) > avail_end )
 540                                 avail_end = i386_ptob(pmptr->end);
 541
 542                         avail_remaining += (pmptr->end - pmptr->base);
 543                         coalescing_permitted = (prev_pmptr && (pmptr->attribute == prev_pmptr->attribute) && ((pmptr->attribute & EFI_MEMORY_KERN_RESERVED) == 0));
 544                         /*
 545                          * Consolidate contiguous memory regions, if possible
 546                          */
 547                         if (prev_pmptr &&
 548                             (pmptr->type == prev_pmptr->type) &&
 549                             (coalescing_permitted) &&
 550                             (pmptr->base == pmptr->alloc_up) &&
 551                             (prev_pmptr->end == prev_pmptr->alloc_down) &&
 552                             (pmptr->base == (prev_pmptr->end + 1)))
 553                         {
 554                                 prev_pmptr->end = pmptr->end;
 555                                 prev_pmptr->alloc_down = pmptr->alloc_down;
 556                         } else {
 557                                 pmap_memory_region_count++;
 558                                 prev_pmptr = pmptr;
 559                                 pmptr++;
 560                         }
 561                 }
 562         }
 563
 564 #ifdef PRINT_PMAP_MEMORY_TABLE
 565         {
 566         unsigned int j;
 567         pmap_memory_region_t *p = pmap_memory_regions;
 568         addr64_t region_start, region_end;
 569         addr64_t efi_start, efi_end;
 570         for (j=0;j<pmap_memory_region_count;j++, p++) {
 571             kprintf("pmap region %d type %d base 0x%llx alloc_up 0x%llx alloc_down 0x%llx top 0x%llx\n",
 572                     j, p->type,
 573                     (addr64_t) p->base  << I386_PGSHIFT,
 574                     (addr64_t) p->alloc_up << I386_PGSHIFT,
 575                     (addr64_t) p->alloc_down << I386_PGSHIFT,
 576                     (addr64_t) p->end   << I386_PGSHIFT);
 577             region_start = (addr64_t) p->base << I386_PGSHIFT;
 578             region_end = ((addr64_t) p->end << I386_PGSHIFT) - 1;
 579             mptr = (EfiMemoryRange *) ml_static_ptovirt((vm_offset_t)args->MemoryMap);
 580             for (i=0; i<mcount; i++, mptr = (EfiMemoryRange *)(((vm_offset_t)mptr) + msize)) {
 581                 if (mptr->Type != kEfiLoaderCode &&
 582                     mptr->Type != kEfiLoaderData &&
 583                     mptr->Type != kEfiBootServicesCode &&
 584                     mptr->Type != kEfiBootServicesData &&
 585                     mptr->Type != kEfiConventionalMemory) {
 586                 efi_start = (addr64_t)mptr->PhysicalStart;
 587                 efi_end = efi_start + ((vm_offset_t)mptr->NumberOfPages << I386_PGSHIFT) - 1;
 588                 if ((efi_start >= region_start && efi_start <= region_end) ||
 589                     (efi_end >= region_start && efi_end <= region_end)) {
 590                     kprintf(" *** Overlapping region with EFI runtime region %d\n", i);
 591                 }
 592               }
 593             }
 594           }
 595         }
 596 #endif
 597
 598         avail_start = first_avail;
 599         mem_actual = sane_size;
 600
 601         /*
 602          * For user visible memory size, round up to 128 Mb - accounting for the various stolen memory
 603          * not reported by EFI.
 604          */
 605
 606         sane_size = (sane_size + 128 * MB - 1) & ~((uint64_t)(128 * MB - 1));
 607
 608         /*
 609          * We cap at KERNEL_MAXMEM bytes (currently 32GB for K32, 96GB for K64).
 610          * Unless overriden by the maxmem= boot-arg
 611          * -- which is a non-zero maxmem argument to this function.
 612          */
 613         if (maxmem == 0 && sane_size > KERNEL_MAXMEM) {
 614                 maxmem = KERNEL_MAXMEM;
 615                 printf("Physical memory %lld bytes capped at %dGB\n",
 616                         sane_size, (uint32_t) (KERNEL_MAXMEM/GB));
 617         }
 618
 619         /*
 620          * if user set maxmem, reduce memory sizes
 621          */
 622         if ( (maxmem > (uint64_t)first_avail) && (maxmem < sane_size)) {
 623                 ppnum_t discarded_pages  = (ppnum_t)((sane_size - maxmem) >> I386_PGSHIFT);
 624                 ppnum_t highest_pn = 0;
 625                 ppnum_t cur_end  = 0;
 626                 uint64_t        pages_to_use;
 627                 unsigned        cur_region = 0;
 628
 629                 sane_size = maxmem;
 630
 631                 if (avail_remaining > discarded_pages)
 632                         avail_remaining -= discarded_pages;
 633                 else
 634                         avail_remaining = 0;
 635
 636                 pages_to_use = avail_remaining;
 637
 638                 while (cur_region < pmap_memory_region_count && pages_to_use) {
 639                         for (cur_end = pmap_memory_regions[cur_region].base;
 640                              cur_end < pmap_memory_regions[cur_region].end && pages_to_use;
 641                              cur_end++) {
 642                                 if (cur_end > highest_pn)
 643                                         highest_pn = cur_end;
 644                                 pages_to_use--;
 645                         }
 646                         if (pages_to_use == 0) {
 647                                 pmap_memory_regions[cur_region].end = cur_end;
 648                                 pmap_memory_regions[cur_region].alloc_down = cur_end;
 649                         }
 650
 651                         cur_region++;
 652                 }
 653                 pmap_memory_region_count = cur_region;
 654
 655                 avail_end = i386_ptob(highest_pn + 1);
 656         }
 657
 658         /*
 659          * mem_size is only a 32 bit container... follow the PPC route
 660          * and pin it to a 2 Gbyte maximum
 661          */
 662         if (sane_size > (FOURGIG >> 1))
 663                 mem_size = (vm_size_t)(FOURGIG >> 1);
 664         else
 665                 mem_size = (vm_size_t)sane_size;
 666         max_mem = sane_size;
 667
 668         kprintf("Physical memory %llu MB\n", sane_size/MB);
 669
 670         max_valid_low_ppnum = (2 * GB) / PAGE_SIZE;
 671
 672         if (!PE_parse_boot_argn("max_valid_dma_addr", &maxdmaaddr, sizeof (maxdmaaddr))) {
 673                 max_valid_dma_address = (uint64_t)4 * (uint64_t)GB;
 674         } else {
 675                 max_valid_dma_address = ((uint64_t) maxdmaaddr) * MB;
 676
 677                 if ((max_valid_dma_address / PAGE_SIZE) < max_valid_low_ppnum)
 678                         max_valid_low_ppnum = (ppnum_t)(max_valid_dma_address / PAGE_SIZE);
 679         }
 680         if (avail_end >= max_valid_dma_address) {
 681
 682                 if (!PE_parse_boot_argn("maxloreserve", &maxloreserve, sizeof (maxloreserve))) {
 683
 684                         if (sane_size >= (ONEGIG * 15))
 685                                 maxloreserve = (MAXLORESERVE / PAGE_SIZE) * 4;
 686                         else if (sane_size >= (ONEGIG * 7))
 687                                 maxloreserve = (MAXLORESERVE / PAGE_SIZE) * 2;
 688                         else
 689                                 maxloreserve = MAXLORESERVE / PAGE_SIZE;
 690
 691 #if SOCKETS
 692                         mbuf_reserve = bsd_mbuf_cluster_reserve(&mbuf_override) / PAGE_SIZE;
 693 #endif
 694                 } else
 695                         maxloreserve = (maxloreserve * (1024 * 1024)) / PAGE_SIZE;
 696
 697                 if (maxloreserve) {
 698                         vm_lopage_free_limit = maxloreserve;
 699
 700                         if (mbuf_override == TRUE) {
 701                                 vm_lopage_free_limit += mbuf_reserve;
 702                                 vm_lopage_lowater = 0;
 703                         } else
 704                                 vm_lopage_lowater = vm_lopage_free_limit / 16;
 705
 706                         vm_lopage_refill = TRUE;
 707                         vm_lopage_needed = TRUE;
 708                 }
 709         }
 710
 711         /*
 712          *      Initialize kernel physical map.
 713          *      Kernel virtual address starts at VM_KERNEL_MIN_ADDRESS.
 714          */
 715         kprintf("avail_remaining = 0x%lx\n", (unsigned long)avail_remaining);
 716         pmap_bootstrap(0, IA32e);
 717 }
 718
 719
 720 unsigned int
 721 pmap_free_pages(void)
 722 {
 723         return (unsigned int)avail_remaining;
 724 }
 725
 726
 727 boolean_t pmap_next_page_reserved(ppnum_t *);
 728
 729 /*
 730  * Pick a page from a "kernel private" reserved range; works around
 731  * errata on some hardware.
 732  */
 733 boolean_t
 734 pmap_next_page_reserved(ppnum_t *pn) {
 735         if (pmap_reserved_ranges) {
 736                 uint32_t n;
 737                 pmap_memory_region_t *region;
 738                 for (n = 0; n < pmap_last_reserved_range_index; n++) {
 739                         uint32_t reserved_index = pmap_reserved_range_indices[n];
 740                         region = &pmap_memory_regions[reserved_index];
 741                         if (region->alloc_up <= region->alloc_down) {
 742                                 *pn = region->alloc_up++;
 743                                 avail_remaining--;
 744
 745                                 if (*pn > max_ppnum)
 746                                         max_ppnum = *pn;
 747
 748                                 if (lowest_lo == 0 || *pn < lowest_lo)
 749                                         lowest_lo = *pn;
 750
 751                                 pmap_reserved_pages_allocated++;
 752 #if DEBUG
 753                                 if (region->alloc_up > region->alloc_down) {
 754                                         kprintf("Exhausted reserved range index: %u, base: 0x%x end: 0x%x, type: 0x%x, attribute: 0x%llx\n", reserved_index, region->base, region->end, region->type, region->attribute);
 755                                 }
 756 #endif
 757                                 return TRUE;
 758                         }
 759                 }
 760         }
 761         return FALSE;
 762 }
 763
 764
 765 boolean_t
 766 pmap_next_page_hi(
 767                   ppnum_t *pn)
 768 {
 769         pmap_memory_region_t *region;
 770         int     n;
 771
 772         if (pmap_next_page_reserved(pn))
 773                 return TRUE;
 774
 775         if (avail_remaining) {
 776                 for (n = pmap_memory_region_count - 1; n >= 0; n--) {
 777                         region = &pmap_memory_regions[n];
 778
 779                         if (region->alloc_down >= region->alloc_up) {
 780                                 *pn = region->alloc_down--;
 781                                 avail_remaining--;
 782
 783                                 if (*pn > max_ppnum)
 784                                         max_ppnum = *pn;
 785
 786                                 if (lowest_lo == 0 || *pn < lowest_lo)
 787                                         lowest_lo = *pn;
 788
 789                                 if (lowest_hi == 0 || *pn < lowest_hi)
 790                                         lowest_hi = *pn;
 791
 792                                 if (*pn > highest_hi)
 793                                         highest_hi = *pn;
 794
 795                                 return TRUE;
 796                         }
 797                 }
 798         }
 799         return FALSE;
 800 }
 801
 802
 803 boolean_t
 804 pmap_next_page(
 805                ppnum_t *pn)
 806 {
 807         if (avail_remaining) while (pmap_memory_region_current < pmap_memory_region_count) {
 808                 if (pmap_memory_regions[pmap_memory_region_current].alloc_up >
 809                     pmap_memory_regions[pmap_memory_region_current].alloc_down) {
 810                         pmap_memory_region_current++;
 811                         continue;
 812                 }
 813                 *pn = pmap_memory_regions[pmap_memory_region_current].alloc_up++;
 814                 avail_remaining--;
 815
 816                 if (*pn > max_ppnum)
 817                         max_ppnum = *pn;
 818
 819                 if (lowest_lo == 0 || *pn < lowest_lo)
 820                         lowest_lo = *pn;
 821
 822                 return TRUE;
 823         }
 824         return FALSE;
 825 }
 826
 827
 828 boolean_t
 829 pmap_valid_page(
 830         ppnum_t pn)
 831 {
 832         unsigned int i;
 833         pmap_memory_region_t *pmptr = pmap_memory_regions;
 834
 835         for (i = 0; i < pmap_memory_region_count; i++, pmptr++) {
 836                 if ( (pn >= pmptr->base) && (pn <= pmptr->end) )
 837                         return TRUE;
 838         }
 839         return FALSE;
 840 }
 841