]> git.saurik.com Git - apple/xnu.git/blobdiff - osfmk/i386/i386_vm_init.c
projects / apple / xnu.git / blobdiff
? search:
summary | shortlog | log | commit | commitdiff | tree
raw | inline | side by side
xnu-7195.50.7.100.1.tar.gz
[apple/xnu.git] / osfmk / i386 / i386_vm_init.c
diff --git a/osfmk/i386/i386_vm_init.c b/osfmk/i386/i386_vm_init.c
index 81b9d6f51c8f239566b6d58254c2f3868e36e872..beba092e170aceb796d844858d11a681faf1230d 100644 (file)
--- a/osfmk/i386/i386_vm_init.c
+++ b/osfmk/i386/i386_vm_init.c
@@ -1,8 +1,8 @@
 /*
 /*
- * Copyright (c) 2003-2012 Apple Inc. All rights reserved.
+ * Copyright (c) 2003-2019 Apple Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
- * 
+ *
  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in
  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in
@@ -11,10 +11,10 @@
  * unlawful or unlicensed copies of an Apple operating system, or to
  * circumvent, violate, or enable the circumvention or violation of, any
  * terms of an Apple operating system software license agreement.
  * unlawful or unlicensed copies of an Apple operating system, or to
  * circumvent, violate, or enable the circumvention or violation of, any
  * terms of an Apple operating system software license agreement.
- * 
+ *
  * Please obtain a copy of the License at
  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  * Please obtain a copy of the License at
  * http://www.opensource.apple.com/apsl/ and read it before using this file.
- * 
+ *
  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
@@ -22,34 +22,34 @@
  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  * Please see the License for the specific language governing rights and
  * limitations under the License.
  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  * Please see the License for the specific language governing rights and
  * limitations under the License.
- * 
+ *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
  */
  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
  */
-/* 
+/*
  * Mach Operating System
  * Copyright (c) 1991,1990,1989, 1988 Carnegie Mellon University
  * All Rights Reserved.
  * Mach Operating System
  * Copyright (c) 1991,1990,1989, 1988 Carnegie Mellon University
  * All Rights Reserved.
- * 
+ *
  * Permission to use, copy, modify and distribute this software and its
  * documentation is hereby granted, provided that both the copyright
  * notice and this permission notice appear in all copies of the
  * software, derivative works or modified versions, and any portions
  * thereof, and that both notices appear in supporting documentation.
  * Permission to use, copy, modify and distribute this software and its
  * documentation is hereby granted, provided that both the copyright
  * notice and this permission notice appear in all copies of the
  * software, derivative works or modified versions, and any portions
  * thereof, and that both notices appear in supporting documentation.
- * 
+ *
  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
  * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
  * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
- * 
+ *
  * Carnegie Mellon requests users of this software to return to
  * Carnegie Mellon requests users of this software to return to
- * 
+ *
  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  *  School of Computer Science
  *  Carnegie Mellon University
  *  Pittsburgh PA 15213-3890
  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  *  School of Computer Science
  *  Carnegie Mellon University
  *  Pittsburgh PA 15213-3890
- * 
+ *
  * any improvements or extensions that they make and grant Carnegie Mellon
  * the rights to redistribute these changes.
  */
  * any improvements or extensions that they make and grant Carnegie Mellon
  * the rights to redistribute these changes.
  */
@@ -76,33 +76,40 @@
 #include <i386/cpuid.h>
 #include <mach/thread_status.h>
 #include <pexpert/i386/efi.h>
 #include <i386/cpuid.h>
 #include <mach/thread_status.h>
 #include <pexpert/i386/efi.h>
+#include <pexpert/pexpert.h>
 #include <i386/i386_lowmem.h>
 #include <i386/i386_lowmem.h>
+#include <i386/misc_protos.h>
 #include <x86_64/lowglobals.h>
 #include <i386/pal_routines.h>
 
 #include <mach-o/loader.h>
 #include <libkern/kernel_mach_header.h>
 
 #include <x86_64/lowglobals.h>
 #include <i386/pal_routines.h>
 
 #include <mach-o/loader.h>
 #include <libkern/kernel_mach_header.h>
 
+#define P2ROUNDUP(x, align)             (-(-(x) & -(align)))
 
 
-vm_size_t      mem_size = 0; 
-pmap_paddr_t   first_avail = 0;/* first after page tables */
+vm_size_t       mem_size = 0;
+pmap_paddr_t    first_avail = 0;/* first after page tables */
 
 
-uint64_t       max_mem;        /* Size of physical memory (bytes), adjusted by maxmem */
+uint64_t        max_mem;        /* Size of physical memory minus carveouts (bytes), adjusted by maxmem */
+uint64_t        max_mem_actual; /* Actual size of physical memory (bytes) adjusted by
+                                 * the maxmem boot-arg */
 uint64_t        mem_actual;
 uint64_t        mem_actual;
-uint64_t       sane_size = 0;  /* Memory size for defaults calculations */
+uint64_t        sane_size = 0;  /* Memory size for defaults calculations */
 
 /*
  * KASLR parameters
  */
 
 /*
  * KASLR parameters
  */
-ppnum_t                vm_kernel_base_page;
-vm_offset_t    vm_kernel_base;
-vm_offset_t    vm_kernel_top;
-vm_offset_t    vm_kernel_stext;
-vm_offset_t    vm_kernel_etext;
-vm_offset_t    vm_kernel_slide;
+ppnum_t         vm_kernel_base_page;
+vm_offset_t     vm_kernel_base;
+vm_offset_t     vm_kernel_top;
+vm_offset_t     vm_kernel_stext;
+vm_offset_t     vm_kernel_etext;
+vm_offset_t     vm_kernel_slide;
+vm_offset_t     vm_kernel_slid_base;
+vm_offset_t     vm_kernel_slid_top;
 vm_offset_t vm_hib_base;
 vm_offset_t vm_hib_base;
-vm_offset_t    vm_kext_base = VM_MIN_KERNEL_AND_KEXT_ADDRESS;
-vm_offset_t    vm_kext_top = VM_MIN_KERNEL_ADDRESS;
+vm_offset_t     vm_kext_base = VM_MIN_KERNEL_AND_KEXT_ADDRESS;
+vm_offset_t     vm_kext_top = VM_MIN_KERNEL_ADDRESS;
 
 vm_offset_t vm_prelink_stext;
 vm_offset_t vm_prelink_etext;
 
 vm_offset_t vm_prelink_stext;
 vm_offset_t vm_prelink_etext;
@@ -111,12 +118,25 @@ vm_offset_t vm_prelink_einfo;
 vm_offset_t vm_slinkedit;
 vm_offset_t vm_elinkedit;
 
 vm_offset_t vm_slinkedit;
 vm_offset_t vm_elinkedit;
 
-#define MAXLORESERVE   (32 * 1024 * 1024)
+vm_offset_t vm_kernel_builtinkmod_text;
+vm_offset_t vm_kernel_builtinkmod_text_end;
 
 
-ppnum_t                max_ppnum = 0;
-ppnum_t                lowest_lo = 0;
-ppnum_t                lowest_hi = 0;
-ppnum_t                highest_hi = 0;
+#define MAXLORESERVE    (32 * 1024 * 1024)
+
+ppnum_t         max_ppnum = 0;
+
+/*
+ * pmap_high_used* are the highest range of physical memory used for kernel
+ * internals (page tables, vm_pages) via pmap_steal_memory() that don't
+ * need to be encrypted in hibernation images. There can be one gap in
+ * the middle of this due to fragmentation when using a mix of small
+ * and large pages.  In that case, the fragment lives between the high
+ * and middle ranges.
+ */
+ppnum_t pmap_high_used_top = 0;
+ppnum_t pmap_high_used_bottom = 0;
+ppnum_t pmap_middle_used_top = 0;
+ppnum_t pmap_middle_used_bottom = 0;
 
 enum {PMAP_MAX_RESERVED_RANGES = 32};
 uint32_t pmap_reserved_pages_allocated = 0;
 
 enum {PMAP_MAX_RESERVED_RANGES = 32};
 uint32_t pmap_reserved_pages_allocated = 0;
@@ -127,11 +147,11 @@ uint32_t pmap_reserved_ranges = 0;
 extern unsigned int bsd_mbuf_cluster_reserve(boolean_t *);
 
 pmap_paddr_t     avail_start, avail_end;
 extern unsigned int bsd_mbuf_cluster_reserve(boolean_t *);
 
 pmap_paddr_t     avail_start, avail_end;
-vm_offset_t    virtual_avail, virtual_end;
-static pmap_paddr_t    avail_remaining;
+vm_offset_t     virtual_avail, virtual_end;
+static pmap_paddr_t     avail_remaining;
 vm_offset_t     static_memory_end = 0;
 
 vm_offset_t     static_memory_end = 0;
 
-vm_offset_t    sHIB, eHIB, stext, etext, sdata, edata, sconstdata, econstdata, end;
+vm_offset_t     sHIB, eHIB, stext, etext, sdata, edata, end, sconst, econst;
 
 /*
  * _mh_execute_header is the mach_header for the currently executing kernel
 
 /*
  * _mh_execute_header is the mach_header for the currently executing kernel
@@ -139,16 +159,14 @@ vm_offset_t       sHIB, eHIB, stext, etext, sdata, edata, sconstdata, econstdata, end;
 vm_offset_t segTEXTB; unsigned long segSizeTEXT;
 vm_offset_t segDATAB; unsigned long segSizeDATA;
 vm_offset_t segLINKB; unsigned long segSizeLINK;
 vm_offset_t segTEXTB; unsigned long segSizeTEXT;
 vm_offset_t segDATAB; unsigned long segSizeDATA;
 vm_offset_t segLINKB; unsigned long segSizeLINK;
-vm_offset_t segPRELINKB; unsigned long segSizePRELINK;
+vm_offset_t segPRELINKTEXTB; unsigned long segSizePRELINKTEXT;
 vm_offset_t segPRELINKINFOB; unsigned long segSizePRELINKINFO;
 vm_offset_t segHIBB; unsigned long segSizeHIB;
 vm_offset_t segPRELINKINFOB; unsigned long segSizePRELINKINFO;
 vm_offset_t segHIBB; unsigned long segSizeHIB;
-vm_offset_t sectCONSTB; unsigned long sectSizeConst;
-
-boolean_t doconstro_override = FALSE;
+unsigned long segSizeConst;
 
 static kernel_segment_command_t *segTEXT, *segDATA;
 static kernel_section_t *cursectTEXT, *lastsectTEXT;
 
 static kernel_segment_command_t *segTEXT, *segDATA;
 static kernel_section_t *cursectTEXT, *lastsectTEXT;
-static kernel_section_t *sectDCONST;
+static kernel_segment_command_t *segCONST;
 
 extern uint64_t firmware_Conventional_bytes;
 extern uint64_t firmware_RuntimeServices_bytes;
 
 extern uint64_t firmware_Conventional_bytes;
 extern uint64_t firmware_RuntimeServices_bytes;
@@ -163,31 +181,88 @@ uint64_t firmware_MMIO_bytes;
 /*
  * Linker magic to establish the highest address in the kernel.
  */
 /*
  * Linker magic to establish the highest address in the kernel.
  */
-extern void    *last_kernel_symbol;
+extern void     *last_kernel_symbol;
+
+#define LG_PPNUM_PAGES (I386_LPGBYTES >> PAGE_SHIFT)
+#define LG_PPNUM_MASK (I386_LPGMASK >> PAGE_SHIFT)
+
+/* set so no region large page fragment pages exist */
+#define RESET_FRAG(r) (((r)->alloc_frag_up = 1), ((r)->alloc_frag_down = 0))
+
+boolean_t       memmap = FALSE;
+#if     DEBUG || DEVELOPMENT
+static void
+kprint_memmap(vm_offset_t maddr, unsigned int msize, unsigned int mcount)
+{
+       unsigned int         i;
+       unsigned int         j;
+       pmap_memory_region_t *p = pmap_memory_regions;
+       EfiMemoryRange       *mptr;
+       addr64_t             region_start, region_end;
+       addr64_t             efi_start, efi_end;
+
+       for (j = 0; j < pmap_memory_region_count; j++, p++) {
+               kprintf("pmap region %d type %d base 0x%llx alloc_up 0x%llx alloc_down 0x%llx"
+                   " alloc_frag_up 0x%llx alloc_frag_down 0x%llx top 0x%llx\n",
+                   j, p->type,
+                   (addr64_t) p->base << I386_PGSHIFT,
+                   (addr64_t) p->alloc_up << I386_PGSHIFT,
+                   (addr64_t) p->alloc_down << I386_PGSHIFT,
+                   (addr64_t) p->alloc_frag_up << I386_PGSHIFT,
+                   (addr64_t) p->alloc_frag_down << I386_PGSHIFT,
+                   (addr64_t) p->end   << I386_PGSHIFT);
+               region_start = (addr64_t) p->base << I386_PGSHIFT;
+               region_end = ((addr64_t) p->end << I386_PGSHIFT) - 1;
+               mptr = (EfiMemoryRange *) maddr;
+               for (i = 0;
+                   i < mcount;
+                   i++, mptr = (EfiMemoryRange *)(((vm_offset_t)mptr) + msize)) {
+                       if (mptr->Type != kEfiLoaderCode &&
+                           mptr->Type != kEfiLoaderData &&
+                           mptr->Type != kEfiBootServicesCode &&
+                           mptr->Type != kEfiBootServicesData &&
+                           mptr->Type != kEfiConventionalMemory) {
+                               efi_start = (addr64_t)mptr->PhysicalStart;
+                               efi_end = efi_start + ((vm_offset_t)mptr->NumberOfPages << I386_PGSHIFT) - 1;
+                               if ((efi_start >= region_start && efi_start <= region_end) ||
+                                   (efi_end >= region_start && efi_end <= region_end)) {
+                                       kprintf(" *** Overlapping region with EFI runtime region %d\n", i);
+                               }
+                       }
+               }
+       }
+}
+#define DPRINTF(x...)   do { if (memmap) kprintf(x); } while (0)
 
 
-#if    DEBUG
-#define        PRINT_PMAP_MEMORY_TABLE
-#define DBG(x...)       kprintf(x)
 #else
 #else
-#define DBG(x...)
+
+static void
+kprint_memmap(vm_offset_t maddr, unsigned int msize, unsigned int mcount)
+{
+#pragma unused(maddr, msize, mcount)
+}
+
+#define DPRINTF(x...)
 #endif /* DEBUG */
 #endif /* DEBUG */
+
 /*
  * Basic VM initialization.
  */
 void
 /*
  * Basic VM initialization.
  */
 void
-i386_vm_init(uint64_t  maxmem,
-            boolean_t  IA32e,
-            boot_args  *args)
+i386_vm_init(uint64_t   maxmem,
+    boolean_t  IA32e,
+    boot_args  *args)
 {
        pmap_memory_region_t *pmptr;
 {
        pmap_memory_region_t *pmptr;
-        pmap_memory_region_t *prev_pmptr;
+       pmap_memory_region_t *prev_pmptr;
        EfiMemoryRange *mptr;
        EfiMemoryRange *mptr;
-        unsigned int mcount;
-        unsigned int msize;
+       unsigned int mcount;
+       unsigned int msize;
+       vm_offset_t maddr;
        ppnum_t fap;
        unsigned int i;
        ppnum_t maxpg = 0;
        ppnum_t fap;
        unsigned int i;
        ppnum_t maxpg = 0;
-        uint32_t pmap_type;
+       uint32_t pmap_type;
        uint32_t maxloreserve;
        uint32_t maxdmaaddr;
        uint32_t  mbuf_reserve = 0;
        uint32_t maxloreserve;
        uint32_t maxdmaaddr;
        uint32_t  mbuf_reserve = 0;
@@ -196,7 +271,9 @@ i386_vm_init(uint64_t       maxmem,
        vm_kernel_base_page = i386_btop(args->kaddr);
        vm_offset_t base_address;
        vm_offset_t static_base_address;
        vm_kernel_base_page = i386_btop(args->kaddr);
        vm_offset_t base_address;
        vm_offset_t static_base_address;
-    
+
+       PE_parse_boot_argn("memmap", &memmap, sizeof(memmap));
+
        /*
         * Establish the KASLR parameters.
         */
        /*
         * Establish the KASLR parameters.
         */
@@ -205,12 +282,13 @@ i386_vm_init(uint64_t     maxmem,
        vm_kernel_slide     = base_address - static_base_address;
        if (args->kslide) {
                kprintf("KASLR slide: 0x%016lx dynamic\n", vm_kernel_slide);
        vm_kernel_slide     = base_address - static_base_address;
        if (args->kslide) {
                kprintf("KASLR slide: 0x%016lx dynamic\n", vm_kernel_slide);
-               if (vm_kernel_slide != ((vm_offset_t)args->kslide))
+               if (vm_kernel_slide != ((vm_offset_t)args->kslide)) {
                        panic("Kernel base inconsistent with slide - rebased?");
                        panic("Kernel base inconsistent with slide - rebased?");
+               }
        } else {
                /* No slide relative to on-disk symbols */
                kprintf("KASLR slide: 0x%016lx static and ignored\n",
        } else {
                /* No slide relative to on-disk symbols */
                kprintf("KASLR slide: 0x%016lx static and ignored\n",
-                       vm_kernel_slide);
+                   vm_kernel_slide);
                vm_kernel_slide = 0;
        }
 
                vm_kernel_slide = 0;
        }
 
@@ -223,7 +301,7 @@ i386_vm_init(uint64_t       maxmem,
                uint32_t cmd;
 
                loadcmd = (struct load_command *)((uintptr_t)&_mh_execute_header +
                uint32_t cmd;
 
                loadcmd = (struct load_command *)((uintptr_t)&_mh_execute_header +
-                                                 sizeof (_mh_execute_header));
+                   sizeof(_mh_execute_header));
 
                for (cmd = 0; cmd < _mh_execute_header.ncmds; cmd++) {
                        if (loadcmd->cmd == LC_DYSYMTAB) {
 
                for (cmd = 0; cmd < _mh_execute_header.ncmds; cmd++) {
                        if (loadcmd->cmd == LC_DYSYMTAB) {
@@ -240,27 +318,27 @@ i386_vm_init(uint64_t     maxmem,
        }
 
        /*
        }
 
        /*
-        * Now retrieve addresses for end, edata, and etext 
+        * Now retrieve addresses for end, edata, and etext
         * from MACH-O headers.
         */
        segTEXTB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header,
         * from MACH-O headers.
         */
        segTEXTB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header,
-                                       "__TEXT", &segSizeTEXT);
+           "__TEXT", &segSizeTEXT);
        segDATAB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header,
        segDATAB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header,
-                                       "__DATA", &segSizeDATA);
+           "__DATA", &segSizeDATA);
        segLINKB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header,
        segLINKB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header,
-                                       "__LINKEDIT", &segSizeLINK);
+           "__LINKEDIT", &segSizeLINK);
        segHIBB  = (vm_offset_t) getsegdatafromheader(&_mh_execute_header,
        segHIBB  = (vm_offset_t) getsegdatafromheader(&_mh_execute_header,
-                                       "__HIB", &segSizeHIB);
-    segPRELINKB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header,
-                                                     "__PRELINK_TEXT", &segSizePRELINK);
-    segPRELINKINFOB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header,
-                                                     "__PRELINK_INFO", &segSizePRELINKINFO);
+           "__HIB", &segSizeHIB);
+       segPRELINKTEXTB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header,
+           "__PRELINK_TEXT", &segSizePRELINKTEXT);
+       segPRELINKINFOB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header,
+           "__PRELINK_INFO", &segSizePRELINKINFO);
        segTEXT = getsegbynamefromheader(&_mh_execute_header,
        segTEXT = getsegbynamefromheader(&_mh_execute_header,
-                                       "__TEXT");
+           "__TEXT");
        segDATA = getsegbynamefromheader(&_mh_execute_header,
        segDATA = getsegbynamefromheader(&_mh_execute_header,
-                                       "__DATA");
-       sectDCONST = getsectbynamefromheader(&_mh_execute_header,
-                                       "__DATA", "__const");
+           "__DATA");
+       segCONST = getsegbynamefromheader(&_mh_execute_header,
+           "__DATA_CONST");
        cursectTEXT = lastsectTEXT = firstsect(segTEXT);
        /* Discover the last TEXT section within the TEXT segment */
        while ((cursectTEXT = nextsect(segTEXT, cursectTEXT)) != NULL) {
        cursectTEXT = lastsectTEXT = firstsect(segTEXT);
        /* Discover the last TEXT section within the TEXT segment */
        while ((cursectTEXT = nextsect(segTEXT, cursectTEXT)) != NULL) {
@@ -278,45 +356,80 @@ i386_vm_init(uint64_t     maxmem,
        sdata = segDATAB;
        edata = segDATAB + segSizeDATA;
 
        sdata = segDATAB;
        edata = segDATAB + segSizeDATA;
 
-       sectCONSTB = (vm_offset_t) sectDCONST->addr;
-       sectSizeConst = sectDCONST->size;
-       sconstdata = sectCONSTB;
-       econstdata = sectCONSTB + sectSizeConst;
-
-       if (sectSizeConst & PAGE_MASK) {
-               kernel_section_t *ns = nextsect(segDATA, sectDCONST);
-               if (ns && !(ns->addr & PAGE_MASK))
-                       doconstro_override = TRUE;
-       } else
-               doconstro_override = TRUE;
-
-       DBG("segTEXTB    = %p\n", (void *) segTEXTB);
-       DBG("segDATAB    = %p\n", (void *) segDATAB);
-       DBG("segLINKB    = %p\n", (void *) segLINKB);
-       DBG("segHIBB     = %p\n", (void *) segHIBB);
-       DBG("segPRELINKB = %p\n", (void *) segPRELINKB);
-       DBG("segPRELINKINFOB = %p\n", (void *) segPRELINKINFOB);
-       DBG("sHIB        = %p\n", (void *) sHIB);
-       DBG("eHIB        = %p\n", (void *) eHIB);
-       DBG("stext       = %p\n", (void *) stext);
-       DBG("etext       = %p\n", (void *) etext);
-       DBG("sdata       = %p\n", (void *) sdata);
-       DBG("edata       = %p\n", (void *) edata);
-       DBG("sconstdata  = %p\n", (void *) sconstdata);
-       DBG("econstdata  = %p\n", (void *) econstdata);
-       DBG("kernel_top  = %p\n", (void *) &last_kernel_symbol);
+       sconst = segCONST->vmaddr;
+       segSizeConst = segCONST->vmsize;
+       econst = sconst + segSizeConst;
+
+       kc_format_t kc_format = KCFormatUnknown;
+
+       /* XXX: FIXME_IN_dyld: For new-style kernel caches, the ending address of __DATA_CONST may not be page-aligned */
+       if (PE_get_primary_kc_format(&kc_format) && kc_format == KCFormatFileset) {
+               /* Round up the end */
+               econst = P2ROUNDUP(econst, PAGE_SIZE);
+               edata = P2ROUNDUP(edata, PAGE_SIZE);
+       } else {
+               assert(((sconst | econst) & PAGE_MASK) == 0);
+               assert(((sdata | edata) & PAGE_MASK) == 0);
+       }
+
+       DPRINTF("segTEXTB    = %p\n", (void *) segTEXTB);
+       DPRINTF("segDATAB    = %p\n", (void *) segDATAB);
+       DPRINTF("segLINKB    = %p\n", (void *) segLINKB);
+       DPRINTF("segHIBB     = %p\n", (void *) segHIBB);
+       DPRINTF("segPRELINKTEXTB = %p\n", (void *) segPRELINKTEXTB);
+       DPRINTF("segPRELINKINFOB = %p\n", (void *) segPRELINKINFOB);
+       DPRINTF("sHIB        = %p\n", (void *) sHIB);
+       DPRINTF("eHIB        = %p\n", (void *) eHIB);
+       DPRINTF("stext       = %p\n", (void *) stext);
+       DPRINTF("etext       = %p\n", (void *) etext);
+       DPRINTF("sdata       = %p\n", (void *) sdata);
+       DPRINTF("edata       = %p\n", (void *) edata);
+       DPRINTF("sconst      = %p\n", (void *) sconst);
+       DPRINTF("econst      = %p\n", (void *) econst);
+       DPRINTF("kernel_top  = %p\n", (void *) &last_kernel_symbol);
 
        vm_kernel_base  = sHIB;
        vm_kernel_top   = (vm_offset_t) &last_kernel_symbol;
        vm_kernel_stext = stext;
        vm_kernel_etext = etext;
 
        vm_kernel_base  = sHIB;
        vm_kernel_top   = (vm_offset_t) &last_kernel_symbol;
        vm_kernel_stext = stext;
        vm_kernel_etext = etext;
+       vm_prelink_stext = segPRELINKTEXTB;
+       vm_prelink_etext = segPRELINKTEXTB + segSizePRELINKTEXT;
+       vm_prelink_sinfo = segPRELINKINFOB;
+       vm_prelink_einfo = segPRELINKINFOB + segSizePRELINKINFO;
+       vm_slinkedit = segLINKB;
+       vm_elinkedit = segLINKB + segSizeLINK;
+
+       /*
+        * In the fileset world, we want to be able to (un)slide addresses from
+        * the kernel or any of the kexts (e.g., for kernel logging metadata
+        * passed between the kernel and logd in userspace). VM_KERNEL_UNSLIDE
+        * (via VM_KERNEL_IS_SLID) should apply to the addresses in the range
+        * from the first basement address to the last boot kc address.
+        *
+        *                     ^
+        *                     :
+        *                     |
+        *  vm_kernel_slid_top - ---------------------------------------------
+        *                     |
+        *                     :
+        *                     : Boot kc (kexts in the boot kc here)
+        *                     : - - - - - - - - - - - - - - - - - - - - - - -
+        *                     :
+        *                     :
+        *                     | Boot kc (kernel here)
+        *                     - ---------------------------------------------
+        *                     |
+        *                     :
+        *                     | Basement (kexts in pageable and aux kcs here)
+        * vm_kernel_slid_base - ---------------------------------------------
+        *                     0
+        */
+
+       vm_kernel_slid_base = vm_kext_base + vm_kernel_slide;
+       vm_kernel_slid_top = (kc_format == KCFormatFileset) ?
+           vm_slinkedit : vm_prelink_einfo;
 
 
-    vm_prelink_stext = segPRELINKB;
-    vm_prelink_etext = segPRELINKB + segSizePRELINK;
-    vm_prelink_sinfo = segPRELINKINFOB;
-    vm_prelink_einfo = segPRELINKINFOB + segSizePRELINKINFO;
-    vm_slinkedit = segLINKB;
-    vm_elinkedit = segLINKB + segSizePRELINK;
+       vm_page_kernelcache_count = (unsigned int) (atop_64(vm_kernel_top - vm_kernel_base));
 
        vm_set_page_size();
 
 
        vm_set_page_size();
 
@@ -327,25 +440,27 @@ i386_vm_init(uint64_t     maxmem,
        avail_remaining = 0;
        avail_end = 0;
        pmptr = pmap_memory_regions;
        avail_remaining = 0;
        avail_end = 0;
        pmptr = pmap_memory_regions;
-        prev_pmptr = 0;
+       prev_pmptr = 0;
        pmap_memory_region_count = pmap_memory_region_current = 0;
        fap = (ppnum_t) i386_btop(first_avail);
 
        pmap_memory_region_count = pmap_memory_region_current = 0;
        fap = (ppnum_t) i386_btop(first_avail);
 
-       mptr = (EfiMemoryRange *)ml_static_ptovirt((vm_offset_t)args->MemoryMap);
-        if (args->MemoryMapDescriptorSize == 0)
-               panic("Invalid memory map descriptor size");
-        msize = args->MemoryMapDescriptorSize;
-        mcount = args->MemoryMapSize / msize;
+       maddr = ml_static_ptovirt((vm_offset_t)args->MemoryMap);
+       mptr = (EfiMemoryRange *)maddr;
+       if (args->MemoryMapDescriptorSize == 0) {
+               panic("Invalid memory map descriptor size");
+       }
+       msize = args->MemoryMapDescriptorSize;
+       mcount = args->MemoryMapSize / msize;
 
 #define FOURGIG 0x0000000100000000ULL
 #define ONEGIG  0x0000000040000000ULL
 
        for (i = 0; i < mcount; i++, mptr = (EfiMemoryRange *)(((vm_offset_t)mptr) + msize)) {
 
 #define FOURGIG 0x0000000100000000ULL
 #define ONEGIG  0x0000000040000000ULL
 
        for (i = 0; i < mcount; i++, mptr = (EfiMemoryRange *)(((vm_offset_t)mptr) + msize)) {
-               ppnum_t base, top;
+               ppnum_t base, top;
                uint64_t region_bytes = 0;
 
                if (pmap_memory_region_count >= PMAP_MEMORY_REGIONS_SIZE) {
                uint64_t region_bytes = 0;
 
                if (pmap_memory_region_count >= PMAP_MEMORY_REGIONS_SIZE) {
-                       kprintf("WARNING: truncating memory region count at %d\n", pmap_memory_region_count);
+                       kprintf("WARNING: truncating memory region count at %d\n", pmap_memory_region_count);
                        break;
                }
                base = (ppnum_t) (mptr->PhysicalStart >> I386_PGSHIFT);
                        break;
                }
                base = (ppnum_t) (mptr->PhysicalStart >> I386_PGSHIFT);
@@ -353,7 +468,7 @@ i386_vm_init(uint64_t       maxmem,
 
                if (base == 0) {
                        /*
 
                if (base == 0) {
                        /*
-                        * Avoid having to deal with the edge case of the 
+                        * Avoid having to deal with the edge case of the
                         * very first possible physical page and the roll-over
                         * to -1; just ignore that page.
                         */
                         * very first possible physical page and the roll-over
                         * to -1; just ignore that page.
                         */
@@ -362,7 +477,7 @@ i386_vm_init(uint64_t       maxmem,
                }
                if (top + 1 == 0) {
                        /*
                }
                if (top + 1 == 0) {
                        /*
-                        * Avoid having to deal with the edge case of the 
+                        * Avoid having to deal with the edge case of the
                         * very last possible physical page and the roll-over
                         * to 0; just ignore that page.
                         */
                         * very last possible physical page and the roll-over
                         * to 0; just ignore that page.
                         */
@@ -377,10 +492,11 @@ i386_vm_init(uint64_t     maxmem,
                        continue;
                }
 
                        continue;
                }
 
-#if    MR_RSV_TEST
+#if     MR_RSV_TEST
                static uint32_t nmr = 0;
                static uint32_t nmr = 0;
-               if ((base > 0x20000) && (nmr++ < 4))
+               if ((base > 0x20000) && (nmr++ < 4)) {
                        mptr->Attribute |= EFI_MEMORY_KERN_RESERVED;
                        mptr->Attribute |= EFI_MEMORY_KERN_RESERVED;
+               }
 #endif
                region_bytes = (uint64_t)(mptr->NumberOfPages << I386_PGSHIFT);
                pmap_type = mptr->Type;
 #endif
                region_bytes = (uint64_t)(mptr->NumberOfPages << I386_PGSHIFT);
                pmap_type = mptr->Type;
@@ -391,22 +507,22 @@ i386_vm_init(uint64_t     maxmem,
                case kEfiBootServicesCode:
                case kEfiBootServicesData:
                case kEfiConventionalMemory:
                case kEfiBootServicesCode:
                case kEfiBootServicesData:
                case kEfiConventionalMemory:
-                       /*
+                       /*
                         * Consolidate usable memory types into one.
                         */
                         * Consolidate usable memory types into one.
                         */
-                       pmap_type = kEfiConventionalMemory;
-                       sane_size += region_bytes;
+                       pmap_type = kEfiConventionalMemory;
+                       sane_size += region_bytes;
                        firmware_Conventional_bytes += region_bytes;
                        break;
                        firmware_Conventional_bytes += region_bytes;
                        break;
-                       /*
-                        * sane_size should reflect the total amount of physical
-                        * RAM in the system, not just the amount that is
-                        * available for the OS to use.
-                        * We now get this value from SMBIOS tables
-                        * rather than reverse engineering the memory map.
-                        * But the legacy computation of "sane_size" is kept
-                        * for diagnostic information.
-                        */
+               /*
+                * sane_size should reflect the total amount of physical
+                * RAM in the system, not just the amount that is
+                * available for the OS to use.
+                * We now get this value from SMBIOS tables
+                * rather than reverse engineering the memory map.
+                * But the legacy computation of "sane_size" is kept
+                * for diagnostic information.
+                */
 
                case kEfiRuntimeServicesCode:
                case kEfiRuntimeServicesData:
 
                case kEfiRuntimeServicesCode:
                case kEfiRuntimeServicesData:
@@ -423,7 +539,7 @@ i386_vm_init(uint64_t       maxmem,
                        break;
                case kEfiPalCode:
                        firmware_PalCode_bytes += region_bytes;
                        break;
                case kEfiPalCode:
                        firmware_PalCode_bytes += region_bytes;
-                       sane_size += region_bytes;
+                       sane_size += region_bytes;
                        break;
 
                case kEfiReservedMemoryType:
                        break;
 
                case kEfiReservedMemoryType:
@@ -441,15 +557,16 @@ i386_vm_init(uint64_t     maxmem,
                        break;
                }
 
                        break;
                }
 
-               DBG("EFI region %d: type %u/%d, base 0x%x, top 0x%x %s\n",
+               DPRINTF("EFI region %d: type %u/%d, base 0x%x, top 0x%x %s\n",
                    i, mptr->Type, pmap_type, base, top,
                    i, mptr->Type, pmap_type, base, top,
-                   (mptr->Attribute&EFI_MEMORY_KERN_RESERVED)? "RESERVED" :
-                   (mptr->Attribute&EFI_MEMORY_RUNTIME)? "RUNTIME" : "");
+                   (mptr->Attribute & EFI_MEMORY_KERN_RESERVED)? "RESERVED" :
+                   (mptr->Attribute & EFI_MEMORY_RUNTIME)? "RUNTIME" : "");
 
                if (maxpg) {
 
                if (maxpg) {
-                       if (base >= maxpg)
+                       if (base >= maxpg) {
                                break;
                                break;
-                       top = (top > maxpg) ? maxpg : top;
+                       }
+                       top = (top > maxpg) ? maxpg : top;
                }
 
                /*
                }
 
                /*
@@ -457,15 +574,15 @@ i386_vm_init(uint64_t     maxmem,
                 */
                if ((mptr->Attribute & EFI_MEMORY_RUNTIME) == EFI_MEMORY_RUNTIME ||
                    pmap_type != kEfiConventionalMemory) {
                 */
                if ((mptr->Attribute & EFI_MEMORY_RUNTIME) == EFI_MEMORY_RUNTIME ||
                    pmap_type != kEfiConventionalMemory) {
-                       prev_pmptr = 0;
+                       prev_pmptr = 0;
                        continue;
                } else {
                        continue;
                } else {
-                       /*
+                       /*
                         * Usable memory region
                         */
                         * Usable memory region
                         */
-                       if (top < I386_LOWMEM_RESERVED ||
+                       if (top < I386_LOWMEM_RESERVED ||
                            !pal_is_usable_memory(base, top)) {
                            !pal_is_usable_memory(base, top)) {
-                               prev_pmptr = 0;
+                               prev_pmptr = 0;
                                continue;
                        }
                        /*
                                continue;
                        }
                        /*
@@ -482,16 +599,17 @@ i386_vm_init(uint64_t     maxmem,
                        }
 
                        if (top < fap) {
                        }
 
                        if (top < fap) {
-                               /*
+                               /*
                                 * entire range below first_avail
                                 * entire range below first_avail
-                                * salvage some low memory pages
+                                * salvage some low memory pages
                                 * we use some very low memory at startup
                                 * mark as already allocated here
                                 */
                                 * we use some very low memory at startup
                                 * mark as already allocated here
                                 */
-                               if (base >= I386_LOWMEM_RESERVED)
-                                       pmptr->base = base;
-                               else
-                                       pmptr->base = I386_LOWMEM_RESERVED;
+                               if (base >= I386_LOWMEM_RESERVED) {
+                                       pmptr->base = base;
+                               } else {
+                                       pmptr->base = I386_LOWMEM_RESERVED;
+                               }
 
                                pmptr->end = top;
 
 
                                pmptr->end = top;
 
@@ -500,28 +618,29 @@ i386_vm_init(uint64_t     maxmem,
                                    (top < vm_kernel_base_page)) {
                                        pmptr->alloc_up = pmptr->base;
                                        pmptr->alloc_down = pmptr->end;
                                    (top < vm_kernel_base_page)) {
                                        pmptr->alloc_up = pmptr->base;
                                        pmptr->alloc_down = pmptr->end;
+                                       RESET_FRAG(pmptr);
                                        pmap_reserved_range_indices[pmap_last_reserved_range_index++] = pmap_memory_region_count;
                                        pmap_reserved_range_indices[pmap_last_reserved_range_index++] = pmap_memory_region_count;
-                               }
-                               else {
+                               } else {
                                        /*
                                         * mark as already mapped
                                         */
                                        pmptr->alloc_up = top + 1;
                                        pmptr->alloc_down = top;
                                        /*
                                         * mark as already mapped
                                         */
                                        pmptr->alloc_up = top + 1;
                                        pmptr->alloc_down = top;
+                                       RESET_FRAG(pmptr);
                                }
                                pmptr->type = pmap_type;
                                pmptr->attribute = mptr->Attribute;
                                }
                                pmptr->type = pmap_type;
                                pmptr->attribute = mptr->Attribute;
-                       }
-                       else if ( (base < fap) && (top > fap) ) {
-                               /*
+                       } else if ((base < fap) && (top > fap)) {
+                               /*
                                 * spans first_avail
                                 * put mem below first avail in table but
                                 * mark already allocated
                                 */
                                 * spans first_avail
                                 * put mem below first avail in table but
                                 * mark already allocated
                                 */
-                               pmptr->base = base;
+                               pmptr->base = base;
                                pmptr->end = (fap - 1);
                                pmptr->alloc_up = pmptr->end + 1;
                                pmptr->alloc_down = pmptr->end;
                                pmptr->end = (fap - 1);
                                pmptr->alloc_up = pmptr->end + 1;
                                pmptr->alloc_down = pmptr->end;
+                               RESET_FRAG(pmptr);
                                pmptr->type = pmap_type;
                                pmptr->attribute = mptr->Attribute;
                                /*
                                pmptr->type = pmap_type;
                                pmptr->attribute = mptr->Attribute;
                                /*
@@ -535,23 +654,28 @@ i386_vm_init(uint64_t     maxmem,
                                pmptr->type = pmap_type;
                                pmptr->attribute = mptr->Attribute;
                                pmptr->alloc_down = pmptr->end = top;
                                pmptr->type = pmap_type;
                                pmptr->attribute = mptr->Attribute;
                                pmptr->alloc_down = pmptr->end = top;
+                               RESET_FRAG(pmptr);
 
 
-                               if (mptr->Attribute & EFI_MEMORY_KERN_RESERVED)
+                               if (mptr->Attribute & EFI_MEMORY_KERN_RESERVED) {
                                        pmap_reserved_range_indices[pmap_last_reserved_range_index++] = pmap_memory_region_count;
                                        pmap_reserved_range_indices[pmap_last_reserved_range_index++] = pmap_memory_region_count;
+                               }
                        } else {
                        } else {
-                               /*
+                               /*
                                 * entire range useable
                                 */
                                 * entire range useable
                                 */
-                               pmptr->alloc_up = pmptr->base = base;
+                               pmptr->alloc_up = pmptr->base = base;
                                pmptr->type = pmap_type;
                                pmptr->attribute = mptr->Attribute;
                                pmptr->alloc_down = pmptr->end = top;
                                pmptr->type = pmap_type;
                                pmptr->attribute = mptr->Attribute;
                                pmptr->alloc_down = pmptr->end = top;
-                               if (mptr->Attribute & EFI_MEMORY_KERN_RESERVED)
+                               RESET_FRAG(pmptr);
+                               if (mptr->Attribute & EFI_MEMORY_KERN_RESERVED) {
                                        pmap_reserved_range_indices[pmap_last_reserved_range_index++] = pmap_memory_region_count;
                                        pmap_reserved_range_indices[pmap_last_reserved_range_index++] = pmap_memory_region_count;
+                               }
                        }
 
                        }
 
-                       if (i386_ptob(pmptr->end) > avail_end )
-                               avail_end = i386_ptob(pmptr->end);
+                       if (i386_ptob(pmptr->end) > avail_end) {
+                               avail_end = i386_ptob(pmptr->end);
+                       }
 
                        avail_remaining += (pmptr->end - pmptr->base);
                        coalescing_permitted = (prev_pmptr && (pmptr->attribute == prev_pmptr->attribute) && ((pmptr->attribute & EFI_MEMORY_KERN_RESERVED) == 0));
 
                        avail_remaining += (pmptr->end - pmptr->base);
                        coalescing_permitted = (prev_pmptr && (pmptr->attribute == prev_pmptr->attribute) && ((pmptr->attribute & EFI_MEMORY_KERN_RESERVED) == 0));
@@ -563,51 +687,21 @@ i386_vm_init(uint64_t     maxmem,
                            (coalescing_permitted) &&
                            (pmptr->base == pmptr->alloc_up) &&
                            (prev_pmptr->end == prev_pmptr->alloc_down) &&
                            (coalescing_permitted) &&
                            (pmptr->base == pmptr->alloc_up) &&
                            (prev_pmptr->end == prev_pmptr->alloc_down) &&
-                           (pmptr->base == (prev_pmptr->end + 1)))
-                       {
+                           (pmptr->base == (prev_pmptr->end + 1))) {
                                prev_pmptr->end = pmptr->end;
                                prev_pmptr->alloc_down = pmptr->alloc_down;
                                prev_pmptr->end = pmptr->end;
                                prev_pmptr->alloc_down = pmptr->alloc_down;
+                               RESET_FRAG(pmptr);
                        } else {
                        } else {
-                               pmap_memory_region_count++;
+                               pmap_memory_region_count++;
                                prev_pmptr = pmptr;
                                pmptr++;
                        }
                }
        }
 
                                prev_pmptr = pmptr;
                                pmptr++;
                        }
                }
        }
 
-#ifdef PRINT_PMAP_MEMORY_TABLE
-       {
-        unsigned int j;
-        pmap_memory_region_t *p = pmap_memory_regions;
-        addr64_t region_start, region_end;
-        addr64_t efi_start, efi_end;
-        for (j=0;j<pmap_memory_region_count;j++, p++) {
-            kprintf("pmap region %d type %d base 0x%llx alloc_up 0x%llx alloc_down 0x%llx top 0x%llx\n",
-                   j, p->type,
-                    (addr64_t) p->base  << I386_PGSHIFT,
-                   (addr64_t) p->alloc_up << I386_PGSHIFT,
-                   (addr64_t) p->alloc_down << I386_PGSHIFT,
-                   (addr64_t) p->end   << I386_PGSHIFT);
-            region_start = (addr64_t) p->base << I386_PGSHIFT;
-            region_end = ((addr64_t) p->end << I386_PGSHIFT) - 1;
-           mptr = (EfiMemoryRange *) ml_static_ptovirt((vm_offset_t)args->MemoryMap);
-            for (i=0; i<mcount; i++, mptr = (EfiMemoryRange *)(((vm_offset_t)mptr) + msize)) {
-                if (mptr->Type != kEfiLoaderCode &&
-                    mptr->Type != kEfiLoaderData &&
-                    mptr->Type != kEfiBootServicesCode &&
-                    mptr->Type != kEfiBootServicesData &&
-                    mptr->Type != kEfiConventionalMemory) {
-                efi_start = (addr64_t)mptr->PhysicalStart;
-                efi_end = efi_start + ((vm_offset_t)mptr->NumberOfPages << I386_PGSHIFT) - 1;
-                if ((efi_start >= region_start && efi_start <= region_end) ||
-                    (efi_end >= region_start && efi_end <= region_end)) {
-                    kprintf(" *** Overlapping region with EFI runtime region %d\n", i);
-                }
-              }
-            }
-          }
+       if (memmap) {
+               kprint_memmap(maddr, msize, mcount);
        }
        }
-#endif
 
        avail_start = first_avail;
        mem_actual = args->PhysicalMemorySize;
 
        avail_start = first_avail;
        mem_actual = args->PhysicalMemorySize;
@@ -619,52 +713,56 @@ i386_vm_init(uint64_t     maxmem,
         * we now use the memory size reported by EFI/Booter.
         */
        sane_size = (sane_size + 128 * MB - 1) & ~((uint64_t)(128 * MB - 1));
         * we now use the memory size reported by EFI/Booter.
         */
        sane_size = (sane_size + 128 * MB - 1) & ~((uint64_t)(128 * MB - 1));
-       if (sane_size != mem_actual)
+       if (sane_size != mem_actual) {
                printf("mem_actual: 0x%llx\n legacy sane_size: 0x%llx\n",
                printf("mem_actual: 0x%llx\n legacy sane_size: 0x%llx\n",
-                       mem_actual, sane_size);
+                   mem_actual, sane_size);
+       }
        sane_size = mem_actual;
 
        /*
        sane_size = mem_actual;
 
        /*
-        * We cap at KERNEL_MAXMEM bytes (currently 32GB for K32, 96GB for K64).
+        * We cap at KERNEL_MAXMEM bytes (currently 1536GB).
         * Unless overriden by the maxmem= boot-arg
         * -- which is a non-zero maxmem argument to this function.
         */
        if (maxmem == 0 && sane_size > KERNEL_MAXMEM) {
                maxmem = KERNEL_MAXMEM;
                printf("Physical memory %lld bytes capped at %dGB\n",
         * Unless overriden by the maxmem= boot-arg
         * -- which is a non-zero maxmem argument to this function.
         */
        if (maxmem == 0 && sane_size > KERNEL_MAXMEM) {
                maxmem = KERNEL_MAXMEM;
                printf("Physical memory %lld bytes capped at %dGB\n",
-                       sane_size, (uint32_t) (KERNEL_MAXMEM/GB));
+                   sane_size, (uint32_t) (KERNEL_MAXMEM / GB));
        }
 
        /*
         * if user set maxmem, reduce memory sizes
         */
        }
 
        /*
         * if user set maxmem, reduce memory sizes
         */
-       if ( (maxmem > (uint64_t)first_avail) && (maxmem < sane_size)) {
+       if ((maxmem > (uint64_t)first_avail) && (maxmem < sane_size)) {
                ppnum_t discarded_pages  = (ppnum_t)((sane_size - maxmem) >> I386_PGSHIFT);
                ppnum_t discarded_pages  = (ppnum_t)((sane_size - maxmem) >> I386_PGSHIFT);
-               ppnum_t highest_pn = 0;
-               ppnum_t cur_end  = 0;
-               uint64_t        pages_to_use;
-               unsigned        cur_region = 0;
+               ppnum_t highest_pn = 0;
+               ppnum_t cur_end  = 0;
+               uint64_t        pages_to_use;
+               unsigned        cur_region = 0;
 
                sane_size = maxmem;
 
 
                sane_size = maxmem;
 
-               if (avail_remaining > discarded_pages)
+               if (avail_remaining > discarded_pages) {
                        avail_remaining -= discarded_pages;
                        avail_remaining -= discarded_pages;
-               else
+               } else {
                        avail_remaining = 0;
                        avail_remaining = 0;
-               
+               }
+
                pages_to_use = avail_remaining;
 
                while (cur_region < pmap_memory_region_count && pages_to_use) {
                pages_to_use = avail_remaining;
 
                while (cur_region < pmap_memory_region_count && pages_to_use) {
-                       for (cur_end = pmap_memory_regions[cur_region].base;
-                            cur_end < pmap_memory_regions[cur_region].end && pages_to_use;
-                            cur_end++) {
-                               if (cur_end > highest_pn)
-                                       highest_pn = cur_end;
+                       for (cur_end = pmap_memory_regions[cur_region].base;
+                           cur_end < pmap_memory_regions[cur_region].end && pages_to_use;
+                           cur_end++) {
+                               if (cur_end > highest_pn) {
+                                       highest_pn = cur_end;
+                               }
                                pages_to_use--;
                        }
                        if (pages_to_use == 0) {
                                pages_to_use--;
                        }
                        if (pages_to_use == 0) {
-                               pmap_memory_regions[cur_region].end = cur_end;
-                               pmap_memory_regions[cur_region].alloc_down = cur_end;
+                               pmap_memory_regions[cur_region].end = cur_end;
+                               pmap_memory_regions[cur_region].alloc_down = cur_end;
+                               RESET_FRAG(&pmap_memory_regions[cur_region]);
                        }
 
                        cur_region++;
                        }
 
                        cur_region++;
@@ -678,55 +776,59 @@ i386_vm_init(uint64_t     maxmem,
         * mem_size is only a 32 bit container... follow the PPC route
         * and pin it to a 2 Gbyte maximum
         */
         * mem_size is only a 32 bit container... follow the PPC route
         * and pin it to a 2 Gbyte maximum
         */
-       if (sane_size > (FOURGIG >> 1))
-               mem_size = (vm_size_t)(FOURGIG >> 1);
-       else
-               mem_size = (vm_size_t)sane_size;
+       if (sane_size > (FOURGIG >> 1)) {
+               mem_size = (vm_size_t)(FOURGIG >> 1);
+       } else {
+               mem_size = (vm_size_t)sane_size;
+       }
        max_mem = sane_size;
        max_mem = sane_size;
+       max_mem_actual = sane_size;
 
 
-       kprintf("Physical memory %llu MB\n", sane_size/MB);
+       kprintf("Physical memory %llu MB\n", sane_size / MB);
 
        max_valid_low_ppnum = (2 * GB) / PAGE_SIZE;
 
 
        max_valid_low_ppnum = (2 * GB) / PAGE_SIZE;
 
-       if (!PE_parse_boot_argn("max_valid_dma_addr", &maxdmaaddr, sizeof (maxdmaaddr))) {
-               max_valid_dma_address = (uint64_t)4 * (uint64_t)GB;
+       if (!PE_parse_boot_argn("max_valid_dma_addr", &maxdmaaddr, sizeof(maxdmaaddr))) {
+               max_valid_dma_address = (uint64_t)4 * (uint64_t)GB;
        } else {
        } else {
-               max_valid_dma_address = ((uint64_t) maxdmaaddr) * MB;
+               max_valid_dma_address = ((uint64_t) maxdmaaddr) * MB;
 
 
-               if ((max_valid_dma_address / PAGE_SIZE) < max_valid_low_ppnum)
+               if ((max_valid_dma_address / PAGE_SIZE) < max_valid_low_ppnum) {
                        max_valid_low_ppnum = (ppnum_t)(max_valid_dma_address / PAGE_SIZE);
                        max_valid_low_ppnum = (ppnum_t)(max_valid_dma_address / PAGE_SIZE);
+               }
        }
        if (avail_end >= max_valid_dma_address) {
        }
        if (avail_end >= max_valid_dma_address) {
-
-               if (!PE_parse_boot_argn("maxloreserve", &maxloreserve, sizeof (maxloreserve))) {
-
-                       if (sane_size >= (ONEGIG * 15))
+               if (!PE_parse_boot_argn("maxloreserve", &maxloreserve, sizeof(maxloreserve))) {
+                       if (sane_size >= (ONEGIG * 15)) {
                                maxloreserve = (MAXLORESERVE / PAGE_SIZE) * 4;
                                maxloreserve = (MAXLORESERVE / PAGE_SIZE) * 4;
-                       else if (sane_size >= (ONEGIG * 7))
+                       } else if (sane_size >= (ONEGIG * 7)) {
                                maxloreserve = (MAXLORESERVE / PAGE_SIZE) * 2;
                                maxloreserve = (MAXLORESERVE / PAGE_SIZE) * 2;
-                       else
+                       } else {
                                maxloreserve = MAXLORESERVE / PAGE_SIZE;
                                maxloreserve = MAXLORESERVE / PAGE_SIZE;
+                       }
 
 #if SOCKETS
                        mbuf_reserve = bsd_mbuf_cluster_reserve(&mbuf_override) / PAGE_SIZE;
 #endif
 
 #if SOCKETS
                        mbuf_reserve = bsd_mbuf_cluster_reserve(&mbuf_override) / PAGE_SIZE;
 #endif
-               } else
+               } else {
                        maxloreserve = (maxloreserve * (1024 * 1024)) / PAGE_SIZE;
                        maxloreserve = (maxloreserve * (1024 * 1024)) / PAGE_SIZE;
+               }
 
                if (maxloreserve) {
 
                if (maxloreserve) {
-                       vm_lopage_free_limit = maxloreserve;
-                       
+                       vm_lopage_free_limit = maxloreserve;
+
                        if (mbuf_override == TRUE) {
                                vm_lopage_free_limit += mbuf_reserve;
                                vm_lopage_lowater = 0;
                        if (mbuf_override == TRUE) {
                                vm_lopage_free_limit += mbuf_reserve;
                                vm_lopage_lowater = 0;
-                       } else
+                       } else {
                                vm_lopage_lowater = vm_lopage_free_limit / 16;
                                vm_lopage_lowater = vm_lopage_free_limit / 16;
+                       }
 
                        vm_lopage_refill = TRUE;
                        vm_lopage_needed = TRUE;
                }
        }
 
                        vm_lopage_refill = TRUE;
                        vm_lopage_needed = TRUE;
                }
        }
-       
+
        /*
         *      Initialize kernel physical map.
         *      Kernel virtual address starts at VM_KERNEL_MIN_ADDRESS.
        /*
         *      Initialize kernel physical map.
         *      Kernel virtual address starts at VM_KERNEL_MIN_ADDRESS.
@@ -742,103 +844,222 @@ pmap_free_pages(void)
        return (unsigned int)avail_remaining;
 }
 
        return (unsigned int)avail_remaining;
 }
 
-
 boolean_t pmap_next_page_reserved(ppnum_t *);
 
 /*
  * Pick a page from a "kernel private" reserved range; works around
 boolean_t pmap_next_page_reserved(ppnum_t *);
 
 /*
  * Pick a page from a "kernel private" reserved range; works around
- * errata on some hardware.
+ * errata on some hardware. EFI marks pages which can't be used for
+ * certain kinds of I/O-ish activities as reserved. We reserve them for
+ * kernel internal usage and prevent them from ever going on regular
+ * free list.
  */
 boolean_t
  */
 boolean_t
-pmap_next_page_reserved(ppnum_t *pn) {
+pmap_next_page_reserved(
+       ppnum_t              *pn)
+{
+       uint32_t             n;
+       pmap_memory_region_t *region;
+       uint32_t             reserved_index;
+
        if (pmap_reserved_ranges) {
        if (pmap_reserved_ranges) {
-               uint32_t n;
-               pmap_memory_region_t *region;
                for (n = 0; n < pmap_last_reserved_range_index; n++) {
                for (n = 0; n < pmap_last_reserved_range_index; n++) {
-                       uint32_t reserved_index = pmap_reserved_range_indices[n];
+                       reserved_index = pmap_reserved_range_indices[n];
                        region = &pmap_memory_regions[reserved_index];
                        if (region->alloc_up <= region->alloc_down) {
                                *pn = region->alloc_up++;
                        region = &pmap_memory_regions[reserved_index];
                        if (region->alloc_up <= region->alloc_down) {
                                *pn = region->alloc_up++;
-                               avail_remaining--;
-
-                               if (*pn > max_ppnum)
-                                       max_ppnum = *pn;
+                       } else if (region->alloc_frag_up <= region->alloc_frag_down) {
+                               *pn = region->alloc_frag_up++;
+                       } else {
+                               continue;
+                       }
+                       avail_remaining--;
 
 
-                               if (lowest_lo == 0 || *pn < lowest_lo)
-                                       lowest_lo = *pn;
+                       if (*pn > max_ppnum) {
+                               max_ppnum = *pn;
+                       }
 
 
-                               pmap_reserved_pages_allocated++;
+                       pmap_reserved_pages_allocated++;
 #if DEBUG
 #if DEBUG
-                               if (region->alloc_up > region->alloc_down) {
-                                       kprintf("Exhausted reserved range index: %u, base: 0x%x end: 0x%x, type: 0x%x, attribute: 0x%llx\n", reserved_index, region->base, region->end, region->type, region->attribute);
-                               }
-#endif
-                               return TRUE;
+                       if (region->alloc_up > region->alloc_down) {
+                               kprintf("Exhausted reserved range index: %u, base: 0x%x end: 0x%x, type: 0x%x, attribute: 0x%llx\n", reserved_index, region->base, region->end, region->type, region->attribute);
                        }
                        }
+#endif
+                       return TRUE;
                }
        }
        return FALSE;
 }
 
                }
        }
        return FALSE;
 }
 
+/*
+ * Return the highest large page available. Fails once there are no more large pages.
+ */
+kern_return_t
+pmap_next_page_large(
+       ppnum_t              *pn)
+{
+       int                  r;
+       pmap_memory_region_t *region;
+       ppnum_t              frag_start;
+       ppnum_t              lgpg;
+
+       if (avail_remaining < LG_PPNUM_PAGES) {
+               return KERN_FAILURE;
+       }
+
+       for (r = pmap_memory_region_count - 1; r >= 0; r--) {
+               region = &pmap_memory_regions[r];
+
+               /*
+                * First check if there is enough memory.
+                */
+               if (region->alloc_down < region->alloc_up ||
+                   (region->alloc_down - region->alloc_up + 1) < LG_PPNUM_PAGES) {
+                       continue;
+               }
+
+               /*
+                * Find the starting large page, creating a fragment if needed.
+                */
+               if ((region->alloc_down & LG_PPNUM_MASK) == LG_PPNUM_MASK) {
+                       lgpg = (region->alloc_down & ~LG_PPNUM_MASK);
+               } else {
+                       /* Can only have 1 fragment per region at a time */
+                       if (region->alloc_frag_up <= region->alloc_frag_down) {
+                               continue;
+                       }
+
+                       /* Check for enough room below any fragment. */
+                       frag_start = (region->alloc_down & ~LG_PPNUM_MASK);
+                       if (frag_start < region->alloc_up ||
+                           frag_start - region->alloc_up < LG_PPNUM_PAGES) {
+                               continue;
+                       }
+
+                       lgpg = frag_start - LG_PPNUM_PAGES;
+                       region->alloc_frag_up = frag_start;
+                       region->alloc_frag_down = region->alloc_down;
+               }
+
+               *pn = lgpg;
+               region->alloc_down = lgpg - 1;
+
+
+               avail_remaining -= LG_PPNUM_PAGES;
+               if (*pn + LG_PPNUM_MASK > max_ppnum) {
+                       max_ppnum = *pn + LG_PPNUM_MASK;
+               }
+
+               return KERN_SUCCESS;
+       }
+       return KERN_FAILURE;
+}
 
 boolean_t
 pmap_next_page_hi(
 
 boolean_t
 pmap_next_page_hi(
-                 ppnum_t *pn)
+       ppnum_t              *pn,
+       boolean_t            might_free)
 {
        pmap_memory_region_t *region;
 {
        pmap_memory_region_t *region;
-       int     n;
+       int                  n;
 
 
-       if (pmap_next_page_reserved(pn))
+       if (!might_free && pmap_next_page_reserved(pn)) {
                return TRUE;
                return TRUE;
+       }
 
        if (avail_remaining) {
                for (n = pmap_memory_region_count - 1; n >= 0; n--) {
                        region = &pmap_memory_regions[n];
 
        if (avail_remaining) {
                for (n = pmap_memory_region_count - 1; n >= 0; n--) {
                        region = &pmap_memory_regions[n];
-
-                       if (region->alloc_down >= region->alloc_up) {
+                       if (region->alloc_frag_up <= region->alloc_frag_down) {
+                               *pn = region->alloc_frag_down--;
+                       } else if (region->alloc_down >= region->alloc_up) {
                                *pn = region->alloc_down--;
                                *pn = region->alloc_down--;
-                               avail_remaining--;
-
-                               if (*pn > max_ppnum)
-                                       max_ppnum = *pn;
-
-                                if (lowest_lo == 0 || *pn < lowest_lo)
-                                        lowest_lo = *pn;
-
-                                if (lowest_hi == 0 || *pn < lowest_hi)
-                                        lowest_hi = *pn;
+                       } else {
+                               continue;
+                       }
 
 
-                                if (*pn > highest_hi)
-                                        highest_hi = *pn;
+                       avail_remaining--;
 
 
-                               return TRUE;
+                       if (*pn > max_ppnum) {
+                               max_ppnum = *pn;
                        }
                        }
+
+                       return TRUE;
                }
        }
        return FALSE;
 }
 
                }
        }
        return FALSE;
 }
 
+/*
+ * Record which high pages have been allocated so far,
+ * so that pmap_init() can mark them PMAP_NOENCRYPT, which
+ * makes hibernation faster.
+ *
+ * Because of the code in pmap_next_page_large(), we could
+ * theoretically have fragments in several regions.
+ * In practice that just doesn't happen. The last pmap region
+ * is normally the largest and will satisfy all pmap_next_hi/large()
+ * allocations. Since this information is used as an optimization
+ * and it's ok to be conservative, we'll just record the information
+ * for the final region.
+ */
+void
+pmap_hi_pages_done(void)
+{
+       pmap_memory_region_t *r;
+
+       r = &pmap_memory_regions[pmap_memory_region_count - 1];
+       pmap_high_used_top = r->end;
+       if (r->alloc_frag_up <= r->alloc_frag_down) {
+               pmap_high_used_bottom = r->alloc_frag_down + 1;
+               pmap_middle_used_top = r->alloc_frag_up - 1;
+               if (r->alloc_up <= r->alloc_down) {
+                       pmap_middle_used_bottom = r->alloc_down + 1;
+               } else {
+                       pmap_high_used_bottom = r->base;
+               }
+       } else {
+               if (r->alloc_up <= r->alloc_down) {
+                       pmap_high_used_bottom = r->alloc_down + 1;
+               } else {
+                       pmap_high_used_bottom = r->base;
+               }
+       }
+#if     DEBUG || DEVELOPMENT
+       kprintf("pmap_high_used_top      0x%x\n", pmap_high_used_top);
+       kprintf("pmap_high_used_bottom   0x%x\n", pmap_high_used_bottom);
+       kprintf("pmap_middle_used_top    0x%x\n", pmap_middle_used_top);
+       kprintf("pmap_middle_used_bottom 0x%x\n", pmap_middle_used_bottom);
+#endif
+}
 
 
+/*
+ * Return the next available page from lowest memory for general use.
+ */
 boolean_t
 pmap_next_page(
 boolean_t
 pmap_next_page(
-              ppnum_t *pn)
+       ppnum_t              *pn)
 {
 {
-       if (avail_remaining) while (pmap_memory_region_current < pmap_memory_region_count) {
-               if (pmap_memory_regions[pmap_memory_region_current].alloc_up >
-                   pmap_memory_regions[pmap_memory_region_current].alloc_down) {
-                       pmap_memory_region_current++;
-                       continue;
-               }
-               *pn = pmap_memory_regions[pmap_memory_region_current].alloc_up++;
-               avail_remaining--;
+       pmap_memory_region_t *region;
 
 
-               if (*pn > max_ppnum)
-                       max_ppnum = *pn;
+       if (avail_remaining) {
+               while (pmap_memory_region_current < pmap_memory_region_count) {
+                       region = &pmap_memory_regions[pmap_memory_region_current];
+                       if (region->alloc_up <= region->alloc_down) {
+                               *pn = region->alloc_up++;
+                       } else if (region->alloc_frag_up <= region->alloc_frag_down) {
+                               *pn = region->alloc_frag_up++;
+                       } else {
+                               pmap_memory_region_current++;
+                               continue;
+                       }
+                       avail_remaining--;
 
 
-               if (lowest_lo == 0 || *pn < lowest_lo)
-                       lowest_lo = *pn;
+                       if (*pn > max_ppnum) {
+                               max_ppnum = *pn;
+                       }
 
 
-               return TRUE;
+                       return TRUE;
+               }
        }
        return FALSE;
 }
        }
        return FALSE;
 }
@@ -848,13 +1069,13 @@ boolean_t
 pmap_valid_page(
        ppnum_t pn)
 {
 pmap_valid_page(
        ppnum_t pn)
 {
-        unsigned int i;
+       unsigned int i;
        pmap_memory_region_t *pmptr = pmap_memory_regions;
 
        for (i = 0; i < pmap_memory_region_count; i++, pmptr++) {
        pmap_memory_region_t *pmptr = pmap_memory_regions;
 
        for (i = 0; i < pmap_memory_region_count; i++, pmptr++) {
-               if ( (pn >= pmptr->base) && (pn <= pmptr->end) )
-                       return TRUE;
+               if ((pn >= pmptr->base) && (pn <= pmptr->end)) {
+                       return TRUE;
+               }
        }
        return FALSE;
 }
        }
        return FALSE;
 }
-
mirror of XNU