/*
- * Copyright (c) 2003-2008 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2012 Apple Inc. All rights reserved.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
*
* the rights to redistribute these changes.
*/
-#include <platforms.h>
-#include <mach_kdb.h>
#include <mach/i386/vm_param.h>
#include <vm/vm_kern.h>
#include <i386/pmap.h>
#include <i386/misc_protos.h>
-#include <i386/ipl.h>
#include <i386/cpuid.h>
#include <mach/thread_status.h>
#include <pexpert/i386/efi.h>
#include <i386/i386_lowmem.h>
-#include <i386/lowglobals.h>
+#include <x86_64/lowglobals.h>
+#include <i386/pal_routines.h>
#include <mach-o/loader.h>
#include <libkern/kernel_mach_header.h>
-#if DEBUG
-#define DBG(x...) kprintf("DBG: " x)
-#define PRINT_PMAP_MEMORY_TABLE
-#else
-#define DBG(x...)
-#endif
vm_size_t mem_size = 0;
-vm_offset_t first_avail = 0;/* first after page tables */
+pmap_paddr_t first_avail = 0;/* first after page tables */
uint64_t max_mem; /* Size of physical memory (bytes), adjusted by maxmem */
uint64_t mem_actual;
-uint64_t sane_size = 0; /* Memory size to use for defaults calculations */
+uint64_t sane_size = 0; /* Memory size for defaults calculations */
-#define MAXLORESERVE ( 32 * 1024 * 1024)
+/*
+ * KASLR parameters
+ */
+ppnum_t vm_kernel_base_page;
+vm_offset_t vm_kernel_base;
+vm_offset_t vm_kernel_top;
+vm_offset_t vm_kernel_stext;
+vm_offset_t vm_kernel_etext;
+vm_offset_t vm_kernel_slide;
+vm_offset_t vm_hib_base;
+vm_offset_t vm_kext_base = VM_MIN_KERNEL_AND_KEXT_ADDRESS;
+vm_offset_t vm_kext_top = VM_MIN_KERNEL_ADDRESS;
+
+vm_offset_t vm_prelink_stext;
+vm_offset_t vm_prelink_etext;
+vm_offset_t vm_prelink_sinfo;
+vm_offset_t vm_prelink_einfo;
+vm_offset_t vm_slinkedit;
+vm_offset_t vm_elinkedit;
+
+#define MAXLORESERVE (32 * 1024 * 1024)
ppnum_t max_ppnum = 0;
ppnum_t lowest_lo = 0;
ppnum_t lowest_hi = 0;
ppnum_t highest_hi = 0;
+enum {PMAP_MAX_RESERVED_RANGES = 32};
uint32_t pmap_reserved_pages_allocated = 0;
-uint32_t pmap_last_reserved_range = 0xFFFFFFFF;
+uint32_t pmap_reserved_range_indices[PMAP_MAX_RESERVED_RANGES];
+uint32_t pmap_last_reserved_range_index = 0;
uint32_t pmap_reserved_ranges = 0;
extern unsigned int bsd_mbuf_cluster_reserve(boolean_t *);
static pmap_paddr_t avail_remaining;
vm_offset_t static_memory_end = 0;
-vm_offset_t sHIB, eHIB, stext, etext, sdata, edata, end;
-
-boolean_t kernel_text_ps_4K = TRUE;
-boolean_t wpkernel = TRUE;
-
-extern void *KPTphys;
+vm_offset_t sHIB, eHIB, stext, etext, sdata, edata, sconstdata, econstdata, end;
/*
* _mh_execute_header is the mach_header for the currently executing kernel
*/
-void *sectTEXTB; unsigned long sectSizeTEXT;
-void *sectDATAB; unsigned long sectSizeDATA;
-void *sectOBJCB; unsigned long sectSizeOBJC;
-void *sectLINKB; unsigned long sectSizeLINK;
-void *sectPRELINKB; unsigned long sectSizePRELINK;
-void *sectHIBB; unsigned long sectSizeHIB;
-void *sectINITPTB; unsigned long sectSizeINITPT;
+vm_offset_t segTEXTB; unsigned long segSizeTEXT;
+vm_offset_t segDATAB; unsigned long segSizeDATA;
+vm_offset_t segLINKB; unsigned long segSizeLINK;
+vm_offset_t segPRELINKB; unsigned long segSizePRELINK;
+vm_offset_t segPRELINKINFOB; unsigned long segSizePRELINKINFO;
+vm_offset_t segHIBB; unsigned long segSizeHIB;
+vm_offset_t sectCONSTB; unsigned long sectSizeConst;
+
+boolean_t doconstro_override = FALSE;
+
+static kernel_segment_command_t *segTEXT, *segDATA;
+static kernel_section_t *cursectTEXT, *lastsectTEXT;
+static kernel_section_t *sectDCONST;
extern uint64_t firmware_Conventional_bytes;
extern uint64_t firmware_RuntimeServices_bytes;
extern uint64_t firmware_other_bytes;
uint64_t firmware_MMIO_bytes;
+/*
+ * Linker magic to establish the highest address in the kernel.
+ */
+extern void *last_kernel_symbol;
+
+#if DEBUG
+#define PRINT_PMAP_MEMORY_TABLE
+#define DBG(x...) kprintf(x)
+#else
+#define DBG(x...)
+#endif /* DEBUG */
/*
* Basic VM initialization.
*/
unsigned int msize;
ppnum_t fap;
unsigned int i;
- unsigned int safeboot;
ppnum_t maxpg = 0;
uint32_t pmap_type;
+ uint32_t maxloreserve;
uint32_t maxdmaaddr;
+ uint32_t mbuf_reserve = 0;
+ boolean_t mbuf_override = FALSE;
+ boolean_t coalescing_permitted;
+ vm_kernel_base_page = i386_btop(args->kaddr);
+ vm_offset_t base_address;
+ vm_offset_t static_base_address;
+
+ /*
+ * Establish the KASLR parameters.
+ */
+ static_base_address = ml_static_ptovirt(KERNEL_BASE_OFFSET);
+ base_address = ml_static_ptovirt(args->kaddr);
+ vm_kernel_slide = base_address - static_base_address;
+ if (args->kslide) {
+ kprintf("KASLR slide: 0x%016lx dynamic\n", vm_kernel_slide);
+ if (vm_kernel_slide != ((vm_offset_t)args->kslide))
+ panic("Kernel base inconsistent with slide - rebased?");
+ } else {
+ /* No slide relative to on-disk symbols */
+ kprintf("KASLR slide: 0x%016lx static and ignored\n",
+ vm_kernel_slide);
+ vm_kernel_slide = 0;
+ }
+
+ /*
+ * Zero out local relocations to avoid confusing kxld.
+ * TODO: might be better to move this code to OSKext::initialize
+ */
+ if (_mh_execute_header.flags & MH_PIE) {
+ struct load_command *loadcmd;
+ uint32_t cmd;
+
+ loadcmd = (struct load_command *)((uintptr_t)&_mh_execute_header +
+ sizeof (_mh_execute_header));
+
+ for (cmd = 0; cmd < _mh_execute_header.ncmds; cmd++) {
+ if (loadcmd->cmd == LC_DYSYMTAB) {
+ struct dysymtab_command *dysymtab;
+
+ dysymtab = (struct dysymtab_command *)loadcmd;
+ dysymtab->nlocrel = 0;
+ dysymtab->locreloff = 0;
+ kprintf("Hiding local relocations\n");
+ break;
+ }
+ loadcmd = (struct load_command *)((uintptr_t)loadcmd + loadcmd->cmdsize);
+ }
+ }
/*
* Now retrieve addresses for end, edata, and etext
* from MACH-O headers.
*/
+ segTEXTB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header,
+ "__TEXT", &segSizeTEXT);
+ segDATAB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header,
+ "__DATA", &segSizeDATA);
+ segLINKB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header,
+ "__LINKEDIT", &segSizeLINK);
+ segHIBB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header,
+ "__HIB", &segSizeHIB);
+ segPRELINKB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header,
+ "__PRELINK_TEXT", &segSizePRELINK);
+ segPRELINKINFOB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header,
+ "__PRELINK_INFO", &segSizePRELINKINFO);
+ segTEXT = getsegbynamefromheader(&_mh_execute_header,
+ "__TEXT");
+ segDATA = getsegbynamefromheader(&_mh_execute_header,
+ "__DATA");
+ sectDCONST = getsectbynamefromheader(&_mh_execute_header,
+ "__DATA", "__const");
+ cursectTEXT = lastsectTEXT = firstsect(segTEXT);
+ /* Discover the last TEXT section within the TEXT segment */
+ while ((cursectTEXT = nextsect(segTEXT, cursectTEXT)) != NULL) {
+ lastsectTEXT = cursectTEXT;
+ }
- sectTEXTB = (void *) getsegdatafromheader(
- &_mh_execute_header, "__TEXT", §SizeTEXT);
- sectDATAB = (void *) getsegdatafromheader(
- &_mh_execute_header, "__DATA", §SizeDATA);
- sectOBJCB = (void *) getsegdatafromheader(
- &_mh_execute_header, "__OBJC", §SizeOBJC);
- sectLINKB = (void *) getsegdatafromheader(
- &_mh_execute_header, "__LINKEDIT", §SizeLINK);
- sectHIBB = (void *)getsegdatafromheader(
- &_mh_execute_header, "__HIB", §SizeHIB);
- sectINITPTB = (void *)getsegdatafromheader(
- &_mh_execute_header, "__INITPT", §SizeINITPT);
- sectPRELINKB = (void *) getsegdatafromheader(
- &_mh_execute_header, "__PRELINK_TEXT", §SizePRELINK);
-
- sHIB = (vm_offset_t) sectHIBB;
- eHIB = (vm_offset_t) sectHIBB + sectSizeHIB;
+ sHIB = segHIBB;
+ eHIB = segHIBB + segSizeHIB;
+ vm_hib_base = sHIB;
/* Zero-padded from ehib to stext if text is 2M-aligned */
- stext = (vm_offset_t) sectTEXTB;
- etext = (vm_offset_t) sectTEXTB + sectSizeTEXT;
+ stext = segTEXTB;
+ lowGlo.lgStext = stext;
+ etext = (vm_offset_t) round_page_64(lastsectTEXT->addr + lastsectTEXT->size);
/* Zero-padded from etext to sdata if text is 2M-aligned */
- sdata = (vm_offset_t) sectDATAB;
- edata = (vm_offset_t) sectDATAB + sectSizeDATA;
-
-#if DEBUG
- kprintf("sectTEXTB = %p\n", sectTEXTB);
- kprintf("sectDATAB = %p\n", sectDATAB);
- kprintf("sectOBJCB = %p\n", sectOBJCB);
- kprintf("sectLINKB = %p\n", sectLINKB);
- kprintf("sectHIBB = %p\n", sectHIBB);
- kprintf("sectPRELINKB = %p\n", sectPRELINKB);
- kprintf("eHIB = %p\n", (void *) eHIB);
- kprintf("stext = %p\n", (void *) stext);
- kprintf("etext = %p\n", (void *) etext);
- kprintf("sdata = %p\n", (void *) sdata);
- kprintf("edata = %p\n", (void *) edata);
-#endif
+ sdata = segDATAB;
+ edata = segDATAB + segSizeDATA;
+
+ sectCONSTB = (vm_offset_t) sectDCONST->addr;
+ sectSizeConst = sectDCONST->size;
+ sconstdata = sectCONSTB;
+ econstdata = sectCONSTB + sectSizeConst;
+
+ if (sectSizeConst & PAGE_MASK) {
+ kernel_section_t *ns = nextsect(segDATA, sectDCONST);
+ if (ns && !(ns->addr & PAGE_MASK))
+ doconstro_override = TRUE;
+ } else
+ doconstro_override = TRUE;
+
+ DBG("segTEXTB = %p\n", (void *) segTEXTB);
+ DBG("segDATAB = %p\n", (void *) segDATAB);
+ DBG("segLINKB = %p\n", (void *) segLINKB);
+ DBG("segHIBB = %p\n", (void *) segHIBB);
+ DBG("segPRELINKB = %p\n", (void *) segPRELINKB);
+ DBG("segPRELINKINFOB = %p\n", (void *) segPRELINKINFOB);
+ DBG("sHIB = %p\n", (void *) sHIB);
+ DBG("eHIB = %p\n", (void *) eHIB);
+ DBG("stext = %p\n", (void *) stext);
+ DBG("etext = %p\n", (void *) etext);
+ DBG("sdata = %p\n", (void *) sdata);
+ DBG("edata = %p\n", (void *) edata);
+ DBG("sconstdata = %p\n", (void *) sconstdata);
+ DBG("econstdata = %p\n", (void *) econstdata);
+ DBG("kernel_top = %p\n", (void *) &last_kernel_symbol);
+
+ vm_kernel_base = sHIB;
+ vm_kernel_top = (vm_offset_t) &last_kernel_symbol;
+ vm_kernel_stext = stext;
+ vm_kernel_etext = etext;
+ vm_prelink_stext = segPRELINKB;
+ vm_prelink_etext = segPRELINKB + segSizePRELINK;
+ vm_prelink_sinfo = segPRELINKINFOB;
+ vm_prelink_einfo = segPRELINKINFOB + segSizePRELINKINFO;
+ vm_slinkedit = segLINKB;
+ vm_elinkedit = segLINKB + segSizePRELINK;
vm_set_page_size();
* Compute the memory size.
*/
- if ((1 == vm_himemory_mode) || PE_parse_boot_argn("-x", &safeboot, sizeof (safeboot))) {
- maxpg = 1 << (32 - I386_PGSHIFT);
- }
avail_remaining = 0;
avail_end = 0;
pmptr = pmap_memory_regions;
}
base = (ppnum_t) (mptr->PhysicalStart >> I386_PGSHIFT);
top = (ppnum_t) (((mptr->PhysicalStart) >> I386_PGSHIFT) + mptr->NumberOfPages - 1);
+
+ if (base == 0) {
+ /*
+ * Avoid having to deal with the edge case of the
+ * very first possible physical page and the roll-over
+ * to -1; just ignore that page.
+ */
+ kprintf("WARNING: ignoring first page in [0x%llx:0x%llx]\n", (uint64_t) base, (uint64_t) top);
+ base++;
+ }
+ if (top + 1 == 0) {
+ /*
+ * Avoid having to deal with the edge case of the
+ * very last possible physical page and the roll-over
+ * to 0; just ignore that page.
+ */
+ kprintf("WARNING: ignoring last page in [0x%llx:0x%llx]\n", (uint64_t) base, (uint64_t) top);
+ top--;
+ }
+ if (top < base) {
+ /*
+ * That was the only page in that region, so
+ * ignore the whole region.
+ */
+ continue;
+ }
+
+#if MR_RSV_TEST
+ static uint32_t nmr = 0;
+ if ((base > 0x20000) && (nmr++ < 4))
+ mptr->Attribute |= EFI_MEMORY_KERN_RESERVED;
+#endif
region_bytes = (uint64_t)(mptr->NumberOfPages << I386_PGSHIFT);
pmap_type = mptr->Type;
break;
}
- kprintf("EFI region %d: type %u/%d, base 0x%x, top 0x%x\n",
- i, mptr->Type, pmap_type, base, top);
+ DBG("EFI region %d: type %u/%d, base 0x%x, top 0x%x %s\n",
+ i, mptr->Type, pmap_type, base, top,
+ (mptr->Attribute&EFI_MEMORY_KERN_RESERVED)? "RESERVED" :
+ (mptr->Attribute&EFI_MEMORY_RUNTIME)? "RUNTIME" : "");
if (maxpg) {
if (base >= maxpg)
/*
* Usable memory region
*/
- if (top < I386_LOWMEM_RESERVED) {
+ if (top < I386_LOWMEM_RESERVED ||
+ !pal_is_usable_memory(base, top)) {
prev_pmptr = 0;
continue;
}
+ /*
+ * A range may be marked with with the
+ * EFI_MEMORY_KERN_RESERVED attribute
+ * on some systems, to indicate that the range
+ * must not be made available to devices.
+ */
+
+ if (mptr->Attribute & EFI_MEMORY_KERN_RESERVED) {
+ if (++pmap_reserved_ranges > PMAP_MAX_RESERVED_RANGES) {
+ panic("Too many reserved ranges %u\n", pmap_reserved_ranges);
+ }
+ }
+
if (top < fap) {
/*
* entire range below first_avail
pmptr->end = top;
- /*
- * A range may be marked with with the
- * EFI_MEMORY_KERN_RESERVED attribute
- * on some systems, to indicate that the range
- * must not be made available to devices.
- * Simplifying assumptions are made regarding
- * the placement of the range.
- */
- if (mptr->Attribute & EFI_MEMORY_KERN_RESERVED)
- pmap_reserved_ranges++;
if ((mptr->Attribute & EFI_MEMORY_KERN_RESERVED) &&
- (top < I386_KERNEL_IMAGE_BASE_PAGE)) {
- pmptr->alloc = pmptr->base;
- pmap_last_reserved_range = pmap_memory_region_count;
+ (top < vm_kernel_base_page)) {
+ pmptr->alloc_up = pmptr->base;
+ pmptr->alloc_down = pmptr->end;
+ pmap_reserved_range_indices[pmap_last_reserved_range_index++] = pmap_memory_region_count;
}
else {
/*
* mark as already mapped
*/
- pmptr->alloc = top;
+ pmptr->alloc_up = top + 1;
+ pmptr->alloc_down = top;
}
pmptr->type = pmap_type;
+ pmptr->attribute = mptr->Attribute;
}
else if ( (base < fap) && (top > fap) ) {
/*
* mark already allocated
*/
pmptr->base = base;
- pmptr->alloc = pmptr->end = (fap - 1);
+ pmptr->end = (fap - 1);
+ pmptr->alloc_up = pmptr->end + 1;
+ pmptr->alloc_down = pmptr->end;
pmptr->type = pmap_type;
+ pmptr->attribute = mptr->Attribute;
/*
* we bump these here inline so the accounting
* below works correctly
*/
pmptr++;
pmap_memory_region_count++;
- pmptr->alloc = pmptr->base = fap;
+
+ pmptr->alloc_up = pmptr->base = fap;
pmptr->type = pmap_type;
- pmptr->end = top;
- }
- else {
+ pmptr->attribute = mptr->Attribute;
+ pmptr->alloc_down = pmptr->end = top;
+
+ if (mptr->Attribute & EFI_MEMORY_KERN_RESERVED)
+ pmap_reserved_range_indices[pmap_last_reserved_range_index++] = pmap_memory_region_count;
+ } else {
/*
* entire range useable
*/
- pmptr->alloc = pmptr->base = base;
+ pmptr->alloc_up = pmptr->base = base;
pmptr->type = pmap_type;
- pmptr->end = top;
+ pmptr->attribute = mptr->Attribute;
+ pmptr->alloc_down = pmptr->end = top;
+ if (mptr->Attribute & EFI_MEMORY_KERN_RESERVED)
+ pmap_reserved_range_indices[pmap_last_reserved_range_index++] = pmap_memory_region_count;
}
if (i386_ptob(pmptr->end) > avail_end )
avail_end = i386_ptob(pmptr->end);
avail_remaining += (pmptr->end - pmptr->base);
-
+ coalescing_permitted = (prev_pmptr && (pmptr->attribute == prev_pmptr->attribute) && ((pmptr->attribute & EFI_MEMORY_KERN_RESERVED) == 0));
/*
* Consolidate contiguous memory regions, if possible
*/
if (prev_pmptr &&
- pmptr->type == prev_pmptr->type &&
- pmptr->base == pmptr->alloc &&
- pmptr->base == (prev_pmptr->end + 1)) {
- prev_pmptr->end = pmptr->end;
+ (pmptr->type == prev_pmptr->type) &&
+ (coalescing_permitted) &&
+ (pmptr->base == pmptr->alloc_up) &&
+ (prev_pmptr->end == prev_pmptr->alloc_down) &&
+ (pmptr->base == (prev_pmptr->end + 1)))
+ {
+ prev_pmptr->end = pmptr->end;
+ prev_pmptr->alloc_down = pmptr->alloc_down;
} else {
pmap_memory_region_count++;
prev_pmptr = pmptr;
addr64_t region_start, region_end;
addr64_t efi_start, efi_end;
for (j=0;j<pmap_memory_region_count;j++, p++) {
- kprintf("pmap region %d type %d base 0x%llx alloc 0x%llx top 0x%llx\n",
+ kprintf("pmap region %d type %d base 0x%llx alloc_up 0x%llx alloc_down 0x%llx top 0x%llx\n",
j, p->type,
(addr64_t) p->base << I386_PGSHIFT,
- (addr64_t) p->alloc << I386_PGSHIFT,
+ (addr64_t) p->alloc_up << I386_PGSHIFT,
+ (addr64_t) p->alloc_down << I386_PGSHIFT,
(addr64_t) p->end << I386_PGSHIFT);
region_start = (addr64_t) p->base << I386_PGSHIFT;
region_end = ((addr64_t) p->end << I386_PGSHIFT) - 1;
sane_size = (sane_size + 128 * MB - 1) & ~((uint64_t)(128 * MB - 1));
/*
- * We cap at KERNEL_MAXMEM bytes (currently 32GB for K32, 64GB for K64).
+ * We cap at KERNEL_MAXMEM bytes (currently 32GB for K32, 96GB for K64).
* Unless overriden by the maxmem= boot-arg
* -- which is a non-zero maxmem argument to this function.
*/
if ( (maxmem > (uint64_t)first_avail) && (maxmem < sane_size)) {
ppnum_t discarded_pages = (ppnum_t)((sane_size - maxmem) >> I386_PGSHIFT);
ppnum_t highest_pn = 0;
- ppnum_t cur_alloc = 0;
+ ppnum_t cur_end = 0;
uint64_t pages_to_use;
unsigned cur_region = 0;
pages_to_use = avail_remaining;
while (cur_region < pmap_memory_region_count && pages_to_use) {
- for (cur_alloc = pmap_memory_regions[cur_region].alloc;
- cur_alloc < pmap_memory_regions[cur_region].end && pages_to_use;
- cur_alloc++) {
- if (cur_alloc > highest_pn)
- highest_pn = cur_alloc;
+ for (cur_end = pmap_memory_regions[cur_region].base;
+ cur_end < pmap_memory_regions[cur_region].end && pages_to_use;
+ cur_end++) {
+ if (cur_end > highest_pn)
+ highest_pn = cur_end;
pages_to_use--;
}
- if (pages_to_use == 0)
- pmap_memory_regions[cur_region].end = cur_alloc;
+ if (pages_to_use == 0) {
+ pmap_memory_regions[cur_region].end = cur_end;
+ pmap_memory_regions[cur_region].alloc_down = cur_end;
+ }
cur_region++;
}
max_valid_low_ppnum = (ppnum_t)(max_valid_dma_address / PAGE_SIZE);
}
if (avail_end >= max_valid_dma_address) {
- uint32_t maxloreserve;
- uint32_t mbuf_reserve = 0;
- boolean_t mbuf_override = FALSE;
if (!PE_parse_boot_argn("maxloreserve", &maxloreserve, sizeof (maxloreserve))) {
else
maxloreserve = MAXLORESERVE / PAGE_SIZE;
+#if SOCKETS
mbuf_reserve = bsd_mbuf_cluster_reserve(&mbuf_override) / PAGE_SIZE;
+#endif
} else
maxloreserve = (maxloreserve * (1024 * 1024)) / PAGE_SIZE;
vm_lopage_needed = TRUE;
}
}
+
/*
* Initialize kernel physical map.
* Kernel virtual address starts at VM_KERNEL_MIN_ADDRESS.
*/
+ kprintf("avail_remaining = 0x%lx\n", (unsigned long)avail_remaining);
pmap_bootstrap(0, IA32e);
}
return (unsigned int)avail_remaining;
}
+
boolean_t pmap_next_page_reserved(ppnum_t *);
/*
*/
boolean_t
pmap_next_page_reserved(ppnum_t *pn) {
- if (pmap_reserved_ranges && pmap_last_reserved_range != 0xFFFFFFFF) {
+ if (pmap_reserved_ranges) {
uint32_t n;
pmap_memory_region_t *region;
- for (n = 0; n <= pmap_last_reserved_range; n++) {
- region = &pmap_memory_regions[n];
- if (region->alloc < region->end) {
- *pn = region->alloc++;
+ for (n = 0; n < pmap_last_reserved_range_index; n++) {
+ uint32_t reserved_index = pmap_reserved_range_indices[n];
+ region = &pmap_memory_regions[reserved_index];
+ if (region->alloc_up <= region->alloc_down) {
+ *pn = region->alloc_up++;
avail_remaining--;
if (*pn > max_ppnum)
lowest_lo = *pn;
pmap_reserved_pages_allocated++;
+#if DEBUG
+ if (region->alloc_up > region->alloc_down) {
+ kprintf("Exhausted reserved range index: %u, base: 0x%x end: 0x%x, type: 0x%x, attribute: 0x%llx\n", reserved_index, region->base, region->end, region->type, region->attribute);
+ }
+#endif
return TRUE;
}
}
for (n = pmap_memory_region_count - 1; n >= 0; n--) {
region = &pmap_memory_regions[n];
- if (region->alloc != region->end) {
- *pn = region->alloc++;
+ if (region->alloc_down >= region->alloc_up) {
+ *pn = region->alloc_down--;
avail_remaining--;
if (*pn > max_ppnum)
ppnum_t *pn)
{
if (avail_remaining) while (pmap_memory_region_current < pmap_memory_region_count) {
- if (pmap_memory_regions[pmap_memory_region_current].alloc ==
- pmap_memory_regions[pmap_memory_region_current].end) {
+ if (pmap_memory_regions[pmap_memory_region_current].alloc_up >
+ pmap_memory_regions[pmap_memory_region_current].alloc_down) {
pmap_memory_region_current++;
continue;
}
- *pn = pmap_memory_regions[pmap_memory_region_current].alloc++;
+ *pn = pmap_memory_regions[pmap_memory_region_current].alloc_up++;
avail_remaining--;
if (*pn > max_ppnum)
return FALSE;
}
-/*
- * Called once VM is fully initialized so that we can release unused
- * sections of low memory to the general pool.
- * Also complete the set-up of identity-mapped sections of the kernel:
- * 1) write-protect kernel text
- * 2) map kernel text using large pages if possible
- * 3) read and write-protect page zero (for K32)
- * 4) map the global page at the appropriate virtual address.
- *
- * Use of large pages
- * ------------------
- * To effectively map and write-protect all kernel text pages, the text
- * must be 2M-aligned at the base, and the data section above must also be
- * 2M-aligned. That is, there's padding below and above. This is achieved
- * through linker directives. Large pages are used only if this alignment
- * exists (and not overriden by the -kernel_text_page_4K boot-arg). The
- * memory layout is:
- *
- * : :
- * | __DATA |
- * sdata: ================== 2Meg
- * | |
- * | zero-padding |
- * | |
- * etext: ------------------
- * | |
- * : :
- * | |
- * | __TEXT |
- * | |
- * : :
- * | |
- * stext: ================== 2Meg
- * | |
- * | zero-padding |
- * | |
- * eHIB: ------------------
- * | __HIB |
- * : :
- *
- * Prior to changing the mapping from 4K to 2M, the zero-padding pages
- * [eHIB,stext] and [etext,sdata] are ml_static_mfree()'d. Then all the
- * 4K pages covering [stext,etext] are coalesced as 2M large pages.
- * The now unused level-1 PTE pages are also freed.
- */
-void
-pmap_lowmem_finalize(void)
-{
- spl_t spl;
- int i;
-
- /* Check the kernel is linked at the expected base address */
- if (i386_btop(kvtophys((vm_offset_t) &IdlePML4)) !=
- I386_KERNEL_IMAGE_BASE_PAGE)
- panic("pmap_lowmem_finalize() unexpected kernel base address");
-
- /*
- * Free all pages in pmap regions below the base:
- * rdar://6332712
- * We can't free all the pages to VM that EFI reports available.
- * Pages in the range 0xc0000-0xff000 aren't safe over sleep/wake.
- * There's also a size miscalculation here: pend is one page less
- * than it should be but this is not fixed to be backwards
- * compatible.
- * Due to this current EFI limitation, we take only the first
- * entry in the memory region table. However, the loop is retained
- * (with the intended termination criteria commented out) in the
- * hope that some day we can free all low-memory ranges.
- * This loop assumes the first range does not span the kernel
- * image base & avail_start. We skip this process on systems
- * with "kernel reserved" ranges, as the low memory reclamation
- * is handled in the initial memory map processing loop on
- * such systems.
- */
- for (i = 0;
-// pmap_memory_regions[i].end <= I386_KERNEL_IMAGE_BASE_PAGE;
- i < 1 && (pmap_reserved_ranges == 0);
- i++) {
- vm_offset_t pbase = (vm_offset_t)i386_ptob(pmap_memory_regions[i].base);
- vm_offset_t pend = (vm_offset_t)i386_ptob(pmap_memory_regions[i].end);
-// vm_offset_t pend = i386_ptob(pmap_memory_regions[i].end+1);
-
- DBG("ml_static_mfree(%p,%p) for pmap region %d\n",
- (void *) ml_static_ptovirt(pbase),
- (void *) (pend - pbase), i);
- ml_static_mfree(ml_static_ptovirt(pbase), pend - pbase);
- }
-
- /*
- * If text and data are both 2MB-aligned,
- * we can map text with large-pages,
- * unless the -kernel_text_ps_4K boot-arg overrides.
- */
- if ((stext & I386_LPGMASK) == 0 && (sdata & I386_LPGMASK) == 0) {
- kprintf("Kernel text is 2MB aligned");
- kernel_text_ps_4K = FALSE;
- if (PE_parse_boot_argn("-kernel_text_ps_4K",
- &kernel_text_ps_4K,
- sizeof (kernel_text_ps_4K)))
- kprintf(" but will be mapped with 4K pages\n");
- else
- kprintf(" and will be mapped with 2M pages\n");
- }
-
- (void) PE_parse_boot_argn("wpkernel", &wpkernel, sizeof (wpkernel));
- if (wpkernel)
- kprintf("Kernel text %p-%p to be write-protected\n",
- (void *) stext, (void *) etext);
-
- spl = splhigh();
-
- /*
- * Scan over text if mappings are to be changed:
- * - Remap kernel text readonly unless the "wpkernel" boot-arg is 0
- * - Change to large-pages if possible and not overriden.
- */
- if (kernel_text_ps_4K && wpkernel) {
- vm_offset_t myva;
- for (myva = stext; myva < etext; myva += PAGE_SIZE) {
- pt_entry_t *ptep;
-
- ptep = pmap_pte(kernel_pmap, (vm_map_offset_t)myva);
- if (ptep)
- pmap_store_pte(ptep, *ptep & ~INTEL_PTE_RW);
- }
- }
-
- if (!kernel_text_ps_4K) {
- vm_offset_t myva;
-
- /*
- * Release zero-filled page padding used for 2M-alignment.
- */
- DBG("ml_static_mfree(%p,%p) for padding below text\n",
- (void *) eHIB, (void *) (stext - eHIB));
- ml_static_mfree(eHIB, stext - eHIB);
- DBG("ml_static_mfree(%p,%p) for padding above text\n",
- (void *) etext, (void *) (sdata - etext));
- ml_static_mfree(etext, sdata - etext);
-
- /*
- * Coalesce text pages into large pages.
- */
- for (myva = stext; myva < sdata; myva += I386_LPGBYTES) {
- pt_entry_t *ptep;
- vm_offset_t pte_phys;
- pt_entry_t *pdep;
- pt_entry_t pde;
-
- pdep = pmap_pde(kernel_pmap, (vm_map_offset_t)myva);
- ptep = pmap_pte(kernel_pmap, (vm_map_offset_t)myva);
- DBG("myva: %p pdep: %p ptep: %p\n",
- (void *) myva, (void *) pdep, (void *) ptep);
- if ((*ptep & INTEL_PTE_VALID) == 0)
- continue;
- pte_phys = (vm_offset_t)(*ptep & PG_FRAME);
- pde = *pdep & PTMASK; /* page attributes from pde */
- pde |= INTEL_PTE_PS; /* make it a 2M entry */
- pde |= pte_phys; /* take page frame from pte */
-
- if (wpkernel)
- pde &= ~INTEL_PTE_RW;
- DBG("pmap_store_pte(%p,0x%llx)\n",
- (void *)pdep, pde);
- pmap_store_pte(pdep, pde);
-
- /*
- * Free the now-unused level-1 pte.
- * Note: ptep is a virtual address to the pte in the
- * recursive map. We can't use this address to free
- * the page. Instead we need to compute its address
- * in the Idle PTEs in "low memory".
- */
- vm_offset_t vm_ptep = (vm_offset_t) KPTphys
- + (pte_phys >> PTPGSHIFT);
- DBG("ml_static_mfree(%p,0x%x) for pte\n",
- (void *) vm_ptep, PAGE_SIZE);
- ml_static_mfree(vm_ptep, PAGE_SIZE);
- }
-
- /* Change variable read by sysctl machdep.pmap */
- pmap_kernel_text_ps = I386_LPGBYTES;
- }
-
-#if defined(__i386__)
- /* no matter what, kernel page zero is not accessible */
- pmap_store_pte(pmap_pte(kernel_pmap, 0), INTEL_PTE_INVALID);
-#endif
-
- /* map lowmem global page into fixed addr */
- pt_entry_t *pte = NULL;
- if (0 == (pte = pmap_pte(kernel_pmap,
- VM_MIN_KERNEL_LOADED_ADDRESS + 0x2000)))
- panic("lowmem pte");
- /* make sure it is defined on page boundary */
- assert(0 == ((vm_offset_t) &lowGlo & PAGE_MASK));
- pmap_store_pte(pte, kvtophys((vm_offset_t)&lowGlo)
- | INTEL_PTE_REF
- | INTEL_PTE_MOD
- | INTEL_PTE_WIRED
- | INTEL_PTE_VALID
- | INTEL_PTE_RW);
- splx(spl);
- flush_tlb();
-}
-
projects / apple / xnu.git / blobdiff