X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/6d2010ae8f7a6078e10b361c6962983bab233e0f..d9a64523371fa019c4575bb400cbbc3a50ac9903:/osfmk/i386/i386_vm_init.c diff --git a/osfmk/i386/i386_vm_init.c b/osfmk/i386/i386_vm_init.c index 989895eb0..9b74e8d44 100644 --- a/osfmk/i386/i386_vm_init.c +++ b/osfmk/i386/i386_vm_init.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2003-2008 Apple Computer, Inc. All rights reserved. + * Copyright (c) 2003-2012 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -54,8 +54,6 @@ * the rights to redistribute these changes. */ -#include -#include #include @@ -79,18 +77,44 @@ #include #include #include -#include +#include #include #include #include + vm_size_t mem_size = 0; pmap_paddr_t first_avail = 0;/* first after page tables */ uint64_t max_mem; /* Size of physical memory (bytes), adjusted by maxmem */ uint64_t mem_actual; -uint64_t sane_size = 0; /* Memory size to use for defaults calculations */ +uint64_t sane_size = 0; /* Memory size for defaults calculations */ + +/* + * KASLR parameters + */ +ppnum_t vm_kernel_base_page; +vm_offset_t vm_kernel_base; +vm_offset_t vm_kernel_top; +vm_offset_t vm_kernel_stext; +vm_offset_t vm_kernel_etext; +vm_offset_t vm_kernel_slide; +vm_offset_t vm_kernel_slid_base; +vm_offset_t vm_kernel_slid_top; +vm_offset_t vm_hib_base; +vm_offset_t vm_kext_base = VM_MIN_KERNEL_AND_KEXT_ADDRESS; +vm_offset_t vm_kext_top = VM_MIN_KERNEL_ADDRESS; + +vm_offset_t vm_prelink_stext; +vm_offset_t vm_prelink_etext; +vm_offset_t vm_prelink_sinfo; +vm_offset_t vm_prelink_einfo; +vm_offset_t vm_slinkedit; +vm_offset_t vm_elinkedit; + +vm_offset_t vm_kernel_builtinkmod_text; +vm_offset_t vm_kernel_builtinkmod_text_end; #define MAXLORESERVE (32 * 1024 * 1024) @@ -99,8 +123,10 @@ ppnum_t lowest_lo = 0; ppnum_t lowest_hi = 0; ppnum_t highest_hi = 0; +enum {PMAP_MAX_RESERVED_RANGES = 32}; uint32_t pmap_reserved_pages_allocated = 0; -uint32_t pmap_last_reserved_range = 0xFFFFFFFF; +uint32_t pmap_reserved_range_indices[PMAP_MAX_RESERVED_RANGES]; +uint32_t pmap_last_reserved_range_index = 0; uint32_t pmap_reserved_ranges = 0; extern unsigned int bsd_mbuf_cluster_reserve(boolean_t *); @@ -110,21 +136,22 @@ vm_offset_t virtual_avail, virtual_end; static pmap_paddr_t avail_remaining; vm_offset_t static_memory_end = 0; -vm_offset_t sHIB, eHIB, stext, etext, sdata, edata, end; +vm_offset_t sHIB, eHIB, stext, etext, sdata, edata, end, sconst, econst; /* * _mh_execute_header is the mach_header for the currently executing kernel */ -void *sectTEXTB; unsigned long sectSizeTEXT; -void *sectDATAB; unsigned long sectSizeDATA; -void *sectOBJCB; unsigned long sectSizeOBJC; -void *sectLINKB; unsigned long sectSizeLINK; -void *sectPRELINKB; unsigned long sectSizePRELINK; -void *sectHIBB; unsigned long sectSizeHIB; -void *sectINITPTB; unsigned long sectSizeINITPT; - -kernel_segment_command_t *segTEXT; -kernel_section_t *cursectTEXT, *lastsectTEXT; +vm_offset_t segTEXTB; unsigned long segSizeTEXT; +vm_offset_t segDATAB; unsigned long segSizeDATA; +vm_offset_t segLINKB; unsigned long segSizeLINK; +vm_offset_t segPRELINKTEXTB; unsigned long segSizePRELINKTEXT; +vm_offset_t segPRELINKINFOB; unsigned long segSizePRELINKINFO; +vm_offset_t segHIBB; unsigned long segSizeHIB; +unsigned long segSizeConst; + +static kernel_segment_command_t *segTEXT, *segDATA; +static kernel_section_t *cursectTEXT, *lastsectTEXT; +static kernel_segment_command_t *segCONST; extern uint64_t firmware_Conventional_bytes; extern uint64_t firmware_RuntimeServices_bytes; @@ -136,9 +163,62 @@ extern uint64_t firmware_Unusable_bytes; extern uint64_t firmware_other_bytes; uint64_t firmware_MMIO_bytes; -#if DEBUG -#define PRINT_PMAP_MEMORY_TABLE +/* + * Linker magic to establish the highest address in the kernel. + */ +extern void *last_kernel_symbol; + +boolean_t memmap = FALSE; +#if DEBUG || DEVELOPMENT +static void +kprint_memmap(vm_offset_t maddr, unsigned int msize, unsigned int mcount) { + unsigned int i; + unsigned int j; + pmap_memory_region_t *p = pmap_memory_regions; + EfiMemoryRange *mptr; + addr64_t region_start, region_end; + addr64_t efi_start, efi_end; + + for (j = 0; j < pmap_memory_region_count; j++, p++) { + kprintf("pmap region %d type %d base 0x%llx alloc_up 0x%llx alloc_down 0x%llx top 0x%llx\n", + j, p->type, + (addr64_t) p->base << I386_PGSHIFT, + (addr64_t) p->alloc_up << I386_PGSHIFT, + (addr64_t) p->alloc_down << I386_PGSHIFT, + (addr64_t) p->end << I386_PGSHIFT); + region_start = (addr64_t) p->base << I386_PGSHIFT; + region_end = ((addr64_t) p->end << I386_PGSHIFT) - 1; + mptr = (EfiMemoryRange *) maddr; + for (i = 0; + i < mcount; + i++, mptr = (EfiMemoryRange *)(((vm_offset_t)mptr) + msize)) { + if (mptr->Type != kEfiLoaderCode && + mptr->Type != kEfiLoaderData && + mptr->Type != kEfiBootServicesCode && + mptr->Type != kEfiBootServicesData && + mptr->Type != kEfiConventionalMemory) { + efi_start = (addr64_t)mptr->PhysicalStart; + efi_end = efi_start + ((vm_offset_t)mptr->NumberOfPages << I386_PGSHIFT) - 1; + if ((efi_start >= region_start && efi_start <= region_end) || + (efi_end >= region_start && efi_end <= region_end)) { + kprintf(" *** Overlapping region with EFI runtime region %d\n", i); + } + } + } + } +} +#define DPRINTF(x...) do { if (memmap) kprintf(x); } while (0) + +#else + +static void +kprint_memmap(vm_offset_t maddr, unsigned int msize, unsigned int mcount) { +#pragma unused(maddr, msize, mcount) +} + +#define DPRINTF(x...) #endif /* DEBUG */ + /* * Basic VM initialization. */ @@ -152,74 +232,137 @@ i386_vm_init(uint64_t maxmem, EfiMemoryRange *mptr; unsigned int mcount; unsigned int msize; + vm_offset_t maddr; ppnum_t fap; unsigned int i; - unsigned int safeboot; ppnum_t maxpg = 0; uint32_t pmap_type; uint32_t maxloreserve; uint32_t maxdmaaddr; uint32_t mbuf_reserve = 0; boolean_t mbuf_override = FALSE; + boolean_t coalescing_permitted; + vm_kernel_base_page = i386_btop(args->kaddr); + vm_offset_t base_address; + vm_offset_t static_base_address; + + PE_parse_boot_argn("memmap", &memmap, sizeof(memmap)); -#if DEBUG - kprintf("Boot args revision: %d version: %d", - args->Revision, args->Version); - kprintf(" commandline: \""); - for(i=0; iCommandLine[i]); - kprintf("\"\n"); -#endif + /* + * Establish the KASLR parameters. + */ + static_base_address = ml_static_ptovirt(KERNEL_BASE_OFFSET); + base_address = ml_static_ptovirt(args->kaddr); + vm_kernel_slide = base_address - static_base_address; + if (args->kslide) { + kprintf("KASLR slide: 0x%016lx dynamic\n", vm_kernel_slide); + if (vm_kernel_slide != ((vm_offset_t)args->kslide)) + panic("Kernel base inconsistent with slide - rebased?"); + } else { + /* No slide relative to on-disk symbols */ + kprintf("KASLR slide: 0x%016lx static and ignored\n", + vm_kernel_slide); + vm_kernel_slide = 0; + } + + /* + * Zero out local relocations to avoid confusing kxld. + * TODO: might be better to move this code to OSKext::initialize + */ + if (_mh_execute_header.flags & MH_PIE) { + struct load_command *loadcmd; + uint32_t cmd; + + loadcmd = (struct load_command *)((uintptr_t)&_mh_execute_header + + sizeof (_mh_execute_header)); + + for (cmd = 0; cmd < _mh_execute_header.ncmds; cmd++) { + if (loadcmd->cmd == LC_DYSYMTAB) { + struct dysymtab_command *dysymtab; + + dysymtab = (struct dysymtab_command *)loadcmd; + dysymtab->nlocrel = 0; + dysymtab->locreloff = 0; + kprintf("Hiding local relocations\n"); + break; + } + loadcmd = (struct load_command *)((uintptr_t)loadcmd + loadcmd->cmdsize); + } + } /* * Now retrieve addresses for end, edata, and etext * from MACH-O headers. */ - - sectTEXTB = (void *) getsegdatafromheader( - &_mh_execute_header, "__TEXT", §SizeTEXT); - sectDATAB = (void *) getsegdatafromheader( - &_mh_execute_header, "__DATA", §SizeDATA); - sectOBJCB = (void *) getsegdatafromheader( - &_mh_execute_header, "__OBJC", §SizeOBJC); - sectLINKB = (void *) getsegdatafromheader( - &_mh_execute_header, "__LINKEDIT", §SizeLINK); - sectHIBB = (void *)getsegdatafromheader( - &_mh_execute_header, "__HIB", §SizeHIB); - sectINITPTB = (void *)getsegdatafromheader( - &_mh_execute_header, "__INITPT", §SizeINITPT); - sectPRELINKB = (void *) getsegdatafromheader( - &_mh_execute_header, "__PRELINK_TEXT", §SizePRELINK); - - segTEXT = getsegbynamefromheader(&_mh_execute_header, "__TEXT"); + segTEXTB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, + "__TEXT", &segSizeTEXT); + segDATAB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, + "__DATA", &segSizeDATA); + segLINKB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, + "__LINKEDIT", &segSizeLINK); + segHIBB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, + "__HIB", &segSizeHIB); + segPRELINKTEXTB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, + "__PRELINK_TEXT", &segSizePRELINKTEXT); + segPRELINKINFOB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, + "__PRELINK_INFO", &segSizePRELINKINFO); + segTEXT = getsegbynamefromheader(&_mh_execute_header, + "__TEXT"); + segDATA = getsegbynamefromheader(&_mh_execute_header, + "__DATA"); + segCONST = getsegbynamefromheader(&_mh_execute_header, + "__CONST"); cursectTEXT = lastsectTEXT = firstsect(segTEXT); /* Discover the last TEXT section within the TEXT segment */ while ((cursectTEXT = nextsect(segTEXT, cursectTEXT)) != NULL) { lastsectTEXT = cursectTEXT; } - sHIB = (vm_offset_t) sectHIBB; - eHIB = (vm_offset_t) sectHIBB + sectSizeHIB; + sHIB = segHIBB; + eHIB = segHIBB + segSizeHIB; + vm_hib_base = sHIB; /* Zero-padded from ehib to stext if text is 2M-aligned */ - stext = (vm_offset_t) sectTEXTB; + stext = segTEXTB; + lowGlo.lgStext = stext; etext = (vm_offset_t) round_page_64(lastsectTEXT->addr + lastsectTEXT->size); /* Zero-padded from etext to sdata if text is 2M-aligned */ - sdata = (vm_offset_t) sectDATAB; - edata = (vm_offset_t) sectDATAB + sectSizeDATA; + sdata = segDATAB; + edata = segDATAB + segSizeDATA; -#if DEBUG - kprintf("sectTEXTB = %p\n", sectTEXTB); - kprintf("sectDATAB = %p\n", sectDATAB); - kprintf("sectOBJCB = %p\n", sectOBJCB); - kprintf("sectLINKB = %p\n", sectLINKB); - kprintf("sectHIBB = %p\n", sectHIBB); - kprintf("sectPRELINKB = %p\n", sectPRELINKB); - kprintf("eHIB = %p\n", (void *) eHIB); - kprintf("stext = %p\n", (void *) stext); - kprintf("etext = %p\n", (void *) etext); - kprintf("sdata = %p\n", (void *) sdata); - kprintf("edata = %p\n", (void *) edata); -#endif + sconst = segCONST->vmaddr; + segSizeConst = segCONST->vmsize; + econst = sconst + segSizeConst; + + assert(((sconst|econst) & PAGE_MASK) == 0); + + DPRINTF("segTEXTB = %p\n", (void *) segTEXTB); + DPRINTF("segDATAB = %p\n", (void *) segDATAB); + DPRINTF("segLINKB = %p\n", (void *) segLINKB); + DPRINTF("segHIBB = %p\n", (void *) segHIBB); + DPRINTF("segPRELINKTEXTB = %p\n", (void *) segPRELINKTEXTB); + DPRINTF("segPRELINKINFOB = %p\n", (void *) segPRELINKINFOB); + DPRINTF("sHIB = %p\n", (void *) sHIB); + DPRINTF("eHIB = %p\n", (void *) eHIB); + DPRINTF("stext = %p\n", (void *) stext); + DPRINTF("etext = %p\n", (void *) etext); + DPRINTF("sdata = %p\n", (void *) sdata); + DPRINTF("edata = %p\n", (void *) edata); + DPRINTF("sconst = %p\n", (void *) sconst); + DPRINTF("econst = %p\n", (void *) econst); + DPRINTF("kernel_top = %p\n", (void *) &last_kernel_symbol); + + vm_kernel_base = sHIB; + vm_kernel_top = (vm_offset_t) &last_kernel_symbol; + vm_kernel_stext = stext; + vm_kernel_etext = etext; + vm_prelink_stext = segPRELINKTEXTB; + vm_prelink_etext = segPRELINKTEXTB + segSizePRELINKTEXT; + vm_prelink_sinfo = segPRELINKINFOB; + vm_prelink_einfo = segPRELINKINFOB + segSizePRELINKINFO; + vm_slinkedit = segLINKB; + vm_elinkedit = segLINKB + segSizeLINK; + vm_kernel_slid_base = vm_kext_base + vm_kernel_slide; + vm_kernel_slid_top = vm_prelink_einfo; vm_set_page_size(); @@ -227,9 +370,6 @@ i386_vm_init(uint64_t maxmem, * Compute the memory size. */ - if ((1 == vm_himemory_mode) || PE_parse_boot_argn("-x", &safeboot, sizeof (safeboot))) { - maxpg = 1 << (32 - I386_PGSHIFT); - } avail_remaining = 0; avail_end = 0; pmptr = pmap_memory_regions; @@ -237,7 +377,8 @@ i386_vm_init(uint64_t maxmem, pmap_memory_region_count = pmap_memory_region_current = 0; fap = (ppnum_t) i386_btop(first_avail); - mptr = (EfiMemoryRange *)ml_static_ptovirt((vm_offset_t)args->MemoryMap); + maddr = ml_static_ptovirt((vm_offset_t)args->MemoryMap); + mptr = (EfiMemoryRange *)maddr; if (args->MemoryMapDescriptorSize == 0) panic("Invalid memory map descriptor size"); msize = args->MemoryMapDescriptorSize; @@ -256,6 +397,38 @@ i386_vm_init(uint64_t maxmem, } base = (ppnum_t) (mptr->PhysicalStart >> I386_PGSHIFT); top = (ppnum_t) (((mptr->PhysicalStart) >> I386_PGSHIFT) + mptr->NumberOfPages - 1); + + if (base == 0) { + /* + * Avoid having to deal with the edge case of the + * very first possible physical page and the roll-over + * to -1; just ignore that page. + */ + kprintf("WARNING: ignoring first page in [0x%llx:0x%llx]\n", (uint64_t) base, (uint64_t) top); + base++; + } + if (top + 1 == 0) { + /* + * Avoid having to deal with the edge case of the + * very last possible physical page and the roll-over + * to 0; just ignore that page. + */ + kprintf("WARNING: ignoring last page in [0x%llx:0x%llx]\n", (uint64_t) base, (uint64_t) top); + top--; + } + if (top < base) { + /* + * That was the only page in that region, so + * ignore the whole region. + */ + continue; + } + +#if MR_RSV_TEST + static uint32_t nmr = 0; + if ((base > 0x20000) && (nmr++ < 4)) + mptr->Attribute |= EFI_MEMORY_KERN_RESERVED; +#endif region_bytes = (uint64_t)(mptr->NumberOfPages << I386_PGSHIFT); pmap_type = mptr->Type; @@ -276,15 +449,10 @@ i386_vm_init(uint64_t maxmem, * sane_size should reflect the total amount of physical * RAM in the system, not just the amount that is * available for the OS to use. - * FIXME:Consider deriving this value from SMBIOS tables + * We now get this value from SMBIOS tables * rather than reverse engineering the memory map. - * Alternatively, see - * Memory map should - * describe all memory - * Firmware on some systems guarantees that the memory - * map is complete via the "RomReservedMemoryTracked" - * feature field--consult that where possible to - * avoid the "round up to 128M" workaround below. + * But the legacy computation of "sane_size" is kept + * for diagnostic information. */ case kEfiRuntimeServicesCode: @@ -320,10 +488,10 @@ i386_vm_init(uint64_t maxmem, break; } -#if DEBUG - kprintf("EFI region %d: type %u/%d, base 0x%x, top 0x%x\n", - i, mptr->Type, pmap_type, base, top); -#endif + DPRINTF("EFI region %d: type %u/%d, base 0x%x, top 0x%x %s\n", + i, mptr->Type, pmap_type, base, top, + (mptr->Attribute&EFI_MEMORY_KERN_RESERVED)? "RESERVED" : + (mptr->Attribute&EFI_MEMORY_RUNTIME)? "RUNTIME" : ""); if (maxpg) { if (base >= maxpg) @@ -347,6 +515,19 @@ i386_vm_init(uint64_t maxmem, prev_pmptr = 0; continue; } + /* + * A range may be marked with with the + * EFI_MEMORY_KERN_RESERVED attribute + * on some systems, to indicate that the range + * must not be made available to devices. + */ + + if (mptr->Attribute & EFI_MEMORY_KERN_RESERVED) { + if (++pmap_reserved_ranges > PMAP_MAX_RESERVED_RANGES) { + panic("Too many reserved ranges %u\n", pmap_reserved_ranges); + } + } + if (top < fap) { /* * entire range below first_avail @@ -361,29 +542,22 @@ i386_vm_init(uint64_t maxmem, pmptr->end = top; - /* - * A range may be marked with with the - * EFI_MEMORY_KERN_RESERVED attribute - * on some systems, to indicate that the range - * must not be made available to devices. - * Simplifying assumptions are made regarding - * the placement of the range. - */ - if (mptr->Attribute & EFI_MEMORY_KERN_RESERVED) - pmap_reserved_ranges++; if ((mptr->Attribute & EFI_MEMORY_KERN_RESERVED) && - (top < I386_KERNEL_IMAGE_BASE_PAGE)) { - pmptr->alloc = pmptr->base; - pmap_last_reserved_range = pmap_memory_region_count; + (top < vm_kernel_base_page)) { + pmptr->alloc_up = pmptr->base; + pmptr->alloc_down = pmptr->end; + pmap_reserved_range_indices[pmap_last_reserved_range_index++] = pmap_memory_region_count; } else { /* * mark as already mapped */ - pmptr->alloc = top; + pmptr->alloc_up = top + 1; + pmptr->alloc_down = top; } pmptr->type = pmap_type; + pmptr->attribute = mptr->Attribute; } else if ( (base < fap) && (top > fap) ) { /* @@ -392,43 +566,54 @@ i386_vm_init(uint64_t maxmem, * mark already allocated */ pmptr->base = base; - pmptr->alloc = pmptr->end = (fap - 1); + pmptr->end = (fap - 1); + pmptr->alloc_up = pmptr->end + 1; + pmptr->alloc_down = pmptr->end; pmptr->type = pmap_type; + pmptr->attribute = mptr->Attribute; /* * we bump these here inline so the accounting * below works correctly */ pmptr++; pmap_memory_region_count++; - pmptr->alloc = pmptr->base = fap; + + pmptr->alloc_up = pmptr->base = fap; pmptr->type = pmap_type; - pmptr->end = top; - } - else { + pmptr->attribute = mptr->Attribute; + pmptr->alloc_down = pmptr->end = top; + + if (mptr->Attribute & EFI_MEMORY_KERN_RESERVED) + pmap_reserved_range_indices[pmap_last_reserved_range_index++] = pmap_memory_region_count; + } else { /* * entire range useable */ - pmptr->alloc = pmptr->base = base; + pmptr->alloc_up = pmptr->base = base; pmptr->type = pmap_type; - pmptr->end = top; + pmptr->attribute = mptr->Attribute; + pmptr->alloc_down = pmptr->end = top; + if (mptr->Attribute & EFI_MEMORY_KERN_RESERVED) + pmap_reserved_range_indices[pmap_last_reserved_range_index++] = pmap_memory_region_count; } if (i386_ptob(pmptr->end) > avail_end ) avail_end = i386_ptob(pmptr->end); avail_remaining += (pmptr->end - pmptr->base); - + coalescing_permitted = (prev_pmptr && (pmptr->attribute == prev_pmptr->attribute) && ((pmptr->attribute & EFI_MEMORY_KERN_RESERVED) == 0)); /* * Consolidate contiguous memory regions, if possible */ if (prev_pmptr && - pmptr->type == prev_pmptr->type && - pmptr->base == pmptr->alloc && - pmptr->base == (prev_pmptr->end + 1)) + (pmptr->type == prev_pmptr->type) && + (coalescing_permitted) && + (pmptr->base == pmptr->alloc_up) && + (prev_pmptr->end == prev_pmptr->alloc_down) && + (pmptr->base == (prev_pmptr->end + 1))) { - if(prev_pmptr->end == prev_pmptr->alloc) - prev_pmptr->alloc = pmptr->base; prev_pmptr->end = pmptr->end; + prev_pmptr->alloc_down = pmptr->alloc_down; } else { pmap_memory_region_count++; prev_pmptr = pmptr; @@ -437,48 +622,24 @@ i386_vm_init(uint64_t maxmem, } } -#ifdef PRINT_PMAP_MEMORY_TABLE - { - unsigned int j; - pmap_memory_region_t *p = pmap_memory_regions; - addr64_t region_start, region_end; - addr64_t efi_start, efi_end; - for (j=0;jtype, - (addr64_t) p->base << I386_PGSHIFT, - (addr64_t) p->alloc << I386_PGSHIFT, - (addr64_t) p->end << I386_PGSHIFT); - region_start = (addr64_t) p->base << I386_PGSHIFT; - region_end = ((addr64_t) p->end << I386_PGSHIFT) - 1; - mptr = (EfiMemoryRange *) ml_static_ptovirt((vm_offset_t)args->MemoryMap); - for (i=0; iType != kEfiLoaderCode && - mptr->Type != kEfiLoaderData && - mptr->Type != kEfiBootServicesCode && - mptr->Type != kEfiBootServicesData && - mptr->Type != kEfiConventionalMemory) { - efi_start = (addr64_t)mptr->PhysicalStart; - efi_end = efi_start + ((vm_offset_t)mptr->NumberOfPages << I386_PGSHIFT) - 1; - if ((efi_start >= region_start && efi_start <= region_end) || - (efi_end >= region_start && efi_end <= region_end)) { - kprintf(" *** Overlapping region with EFI runtime region %d\n", i); - } - } - } - } + if (memmap) { + kprint_memmap(maddr, msize, mcount); } -#endif avail_start = first_avail; - mem_actual = sane_size; + mem_actual = args->PhysicalMemorySize; /* - * For user visible memory size, round up to 128 Mb - accounting for the various stolen memory - * not reported by EFI. + * For user visible memory size, round up to 128 Mb + * - accounting for the various stolen memory not reported by EFI. + * This is maintained for historical, comparison purposes but + * we now use the memory size reported by EFI/Booter. */ - sane_size = (sane_size + 128 * MB - 1) & ~((uint64_t)(128 * MB - 1)); + if (sane_size != mem_actual) + printf("mem_actual: 0x%llx\n legacy sane_size: 0x%llx\n", + mem_actual, sane_size); + sane_size = mem_actual; /* * We cap at KERNEL_MAXMEM bytes (currently 32GB for K32, 96GB for K64). @@ -497,7 +658,7 @@ i386_vm_init(uint64_t maxmem, if ( (maxmem > (uint64_t)first_avail) && (maxmem < sane_size)) { ppnum_t discarded_pages = (ppnum_t)((sane_size - maxmem) >> I386_PGSHIFT); ppnum_t highest_pn = 0; - ppnum_t cur_alloc = 0; + ppnum_t cur_end = 0; uint64_t pages_to_use; unsigned cur_region = 0; @@ -511,15 +672,17 @@ i386_vm_init(uint64_t maxmem, pages_to_use = avail_remaining; while (cur_region < pmap_memory_region_count && pages_to_use) { - for (cur_alloc = pmap_memory_regions[cur_region].alloc; - cur_alloc < pmap_memory_regions[cur_region].end && pages_to_use; - cur_alloc++) { - if (cur_alloc > highest_pn) - highest_pn = cur_alloc; + for (cur_end = pmap_memory_regions[cur_region].base; + cur_end < pmap_memory_regions[cur_region].end && pages_to_use; + cur_end++) { + if (cur_end > highest_pn) + highest_pn = cur_end; pages_to_use--; } - if (pages_to_use == 0) - pmap_memory_regions[cur_region].end = cur_alloc; + if (pages_to_use == 0) { + pmap_memory_regions[cur_region].end = cur_end; + pmap_memory_regions[cur_region].alloc_down = cur_end; + } cur_region++; } @@ -561,7 +724,9 @@ i386_vm_init(uint64_t maxmem, else maxloreserve = MAXLORESERVE / PAGE_SIZE; +#if SOCKETS mbuf_reserve = bsd_mbuf_cluster_reserve(&mbuf_override) / PAGE_SIZE; +#endif } else maxloreserve = (maxloreserve * (1024 * 1024)) / PAGE_SIZE; @@ -603,13 +768,14 @@ boolean_t pmap_next_page_reserved(ppnum_t *); */ boolean_t pmap_next_page_reserved(ppnum_t *pn) { - if (pmap_reserved_ranges && pmap_last_reserved_range != 0xFFFFFFFF) { + if (pmap_reserved_ranges) { uint32_t n; pmap_memory_region_t *region; - for (n = 0; n <= pmap_last_reserved_range; n++) { - region = &pmap_memory_regions[n]; - if (region->alloc < region->end) { - *pn = region->alloc++; + for (n = 0; n < pmap_last_reserved_range_index; n++) { + uint32_t reserved_index = pmap_reserved_range_indices[n]; + region = &pmap_memory_regions[reserved_index]; + if (region->alloc_up <= region->alloc_down) { + *pn = region->alloc_up++; avail_remaining--; if (*pn > max_ppnum) @@ -619,6 +785,11 @@ pmap_next_page_reserved(ppnum_t *pn) { lowest_lo = *pn; pmap_reserved_pages_allocated++; +#if DEBUG + if (region->alloc_up > region->alloc_down) { + kprintf("Exhausted reserved range index: %u, base: 0x%x end: 0x%x, type: 0x%x, attribute: 0x%llx\n", reserved_index, region->base, region->end, region->type, region->attribute); + } +#endif return TRUE; } } @@ -641,8 +812,8 @@ pmap_next_page_hi( for (n = pmap_memory_region_count - 1; n >= 0; n--) { region = &pmap_memory_regions[n]; - if (region->alloc != region->end) { - *pn = region->alloc++; + if (region->alloc_down >= region->alloc_up) { + *pn = region->alloc_down--; avail_remaining--; if (*pn > max_ppnum) @@ -670,12 +841,12 @@ pmap_next_page( ppnum_t *pn) { if (avail_remaining) while (pmap_memory_region_current < pmap_memory_region_count) { - if (pmap_memory_regions[pmap_memory_region_current].alloc == - pmap_memory_regions[pmap_memory_region_current].end) { + if (pmap_memory_regions[pmap_memory_region_current].alloc_up > + pmap_memory_regions[pmap_memory_region_current].alloc_down) { pmap_memory_region_current++; continue; } - *pn = pmap_memory_regions[pmap_memory_region_current].alloc++; + *pn = pmap_memory_regions[pmap_memory_region_current].alloc_up++; avail_remaining--; if (*pn > max_ppnum)