xnu-6153.81.5.tar.gz

[apple/xnu.git] / osfmk / arm / arm_vm_init.c

diff --git a/osfmk/arm/arm_vm_init.c b/osfmk/arm/arm_vm_init.c
index 07bcfb9b263cf1e7cad4b6795e62eaf45b2433a4..08788e136d91c5a51a5db958092c9c1d4daae23d 100644 (file)
--- a/osfmk/arm/arm_vm_init.c
+++ b/osfmk/arm/arm_vm_init.c
@@ -43,11 +43,13 @@
 
 #include <arm/proc_reg.h>
 #include <arm/caches_internal.h>
 
 #include <arm/proc_reg.h>
 #include <arm/caches_internal.h>

+#include <arm/cpu_data_internal.h>

 #include <arm/pmap.h>
 #include <arm/misc_protos.h>
 #include <arm/lowglobals.h>
 
 #include <pexpert/arm/boot.h>
 #include <arm/pmap.h>
 #include <arm/misc_protos.h>
 #include <arm/lowglobals.h>
 
 #include <pexpert/arm/boot.h>

+#include <pexpert/device_tree.h>

 
 #include <libkern/kernel_mach_header.h>
 
 
 #include <libkern/kernel_mach_header.h>
 


@@ -77,7 +79,10 @@ vm_offset_t vm_elinkedit;
 vm_offset_t vm_prelink_sdata;
 vm_offset_t vm_prelink_edata;
 
 vm_offset_t vm_prelink_sdata;
 vm_offset_t vm_prelink_edata;
 

-unsigned long gVirtBase, gPhysBase, gPhysSize;     /* Used by <mach/arm/vm_param.h> */
+vm_offset_t vm_kernel_builtinkmod_text;
+vm_offset_t vm_kernel_builtinkmod_text_end;
+
+unsigned long gVirtBase, gPhysBase, gPhysSize;      /* Used by <mach/arm/vm_param.h> */

 
 vm_offset_t   mem_size;                             /* Size of actual physical memory present
                                                      * minus any performance buffer and possibly
 
 vm_offset_t   mem_size;                             /* Size of actual physical memory present
                                                      * minus any performance buffer and possibly


@@ -93,6 +98,9 @@ addr64_t      vm_last_addr = VM_MAX_KERNEL_ADDRESS; /* Highest kernel
                                                      * virtual address known
                                                      * to the VM system */
 
                                                      * virtual address known
                                                      * to the VM system */
 

+vm_offset_t            segEXTRADATA;
+unsigned long          segSizeEXTRADATA;
+vm_offset_t            segLOWESTTEXT;

 static vm_offset_t     segTEXTB;
 static unsigned long   segSizeTEXT;
 static vm_offset_t     segDATAB;
 static vm_offset_t     segTEXTB;
 static unsigned long   segSizeTEXT;
 static vm_offset_t     segDATAB;


@@ -105,6 +113,11 @@ static vm_offset_t     segLASTB;
 static unsigned long   segSizeLAST;
 static vm_offset_t     sectCONSTB;
 static unsigned long   sectSizeCONST;
 static unsigned long   segSizeLAST;
 static vm_offset_t     sectCONSTB;
 static unsigned long   sectSizeCONST;

+vm_offset_t            segBOOTDATAB;
+unsigned long          segSizeBOOTDATA;
+extern vm_offset_t     intstack_low_guard;
+extern vm_offset_t     intstack_high_guard;
+extern vm_offset_t     fiqstack_high_guard;

 
 vm_offset_t     segPRELINKTEXTB;
 unsigned long   segSizePRELINKTEXT;
 
 vm_offset_t     segPRELINKTEXTB;
 unsigned long   segSizePRELINKTEXT;


@@ -139,10 +152,15 @@ extern vm_offset_t ExceptionVectorsBase; /* the code we want to load there */
 #define round_x_table(x) \
        (((pmap_paddr_t)(x) + (ARM_PGBYTES<<2) - 1) & ~((ARM_PGBYTES<<2) - 1))
 
 #define round_x_table(x) \
        (((pmap_paddr_t)(x) + (ARM_PGBYTES<<2) - 1) & ~((ARM_PGBYTES<<2) - 1))
 

+vm_map_address_t
+phystokv(pmap_paddr_t pa)
+{
+       return pa - gPhysBase + gVirtBase;
+}

 
 static void
 
 static void

-arm_vm_page_granular_helper(vm_offset_t start, vm_offset_t _end, vm_offset_t va, 
-                            int pte_prot_APX, int pte_prot_XN)
+arm_vm_page_granular_helper(vm_offset_t start, vm_offset_t _end, vm_offset_t va,
+    int pte_prot_APX, int pte_prot_XN)

 {
        if (va & ARM_TT_L1_PT_OFFMASK) { /* ragged edge hanging over a ARM_TT_L1_PT_SIZE  boundary */
                va &= (~ARM_TT_L1_PT_OFFMASK);
 {
        if (va & ARM_TT_L1_PT_OFFMASK) { /* ragged edge hanging over a ARM_TT_L1_PT_SIZE  boundary */
                va &= (~ARM_TT_L1_PT_OFFMASK);


@@ -154,29 +172,42 @@ arm_vm_page_granular_helper(vm_offset_t start, vm_offset_t _end, vm_offset_t va,
 
                pa = va - gVirtBase + gPhysBase;
 
 
                pa = va - gVirtBase + gPhysBase;
 

+               if (pa >= avail_end) {
+                       return;
+               }
+
+               assert(_end >= va);
+

                if (ARM_TTE_TYPE_TABLE == (tmplate & ARM_TTE_TYPE_MASK)) {
                        /* pick up the existing page table. */
                        ppte = (pt_entry_t *)phystokv((tmplate & ARM_TTE_TABLE_MASK));
                } else {
                        /* TTE must be reincarnated COARSE. */
                        ppte = (pt_entry_t *)phystokv(avail_start);
                if (ARM_TTE_TYPE_TABLE == (tmplate & ARM_TTE_TYPE_MASK)) {
                        /* pick up the existing page table. */
                        ppte = (pt_entry_t *)phystokv((tmplate & ARM_TTE_TABLE_MASK));
                } else {
                        /* TTE must be reincarnated COARSE. */
                        ppte = (pt_entry_t *)phystokv(avail_start);

+                       pmap_paddr_t l2table = avail_start;

                        avail_start += ARM_PGBYTES;
                        avail_start += ARM_PGBYTES;

+                       bzero(ppte, ARM_PGBYTES);

 
 

-                       pmap_init_pte_static_page(kernel_pmap, ppte, pa);
+                       for (i = 0; i < 4; ++i) {
+                               tte[i] = pa_to_tte(l2table + (i * 0x400)) | ARM_TTE_TYPE_TABLE;
+                       }
+               }

 
 

-                       for (i = 0; i < 4; ++i)
-                               tte[i] = pa_to_tte(kvtophys((vm_offset_t)ppte) + (i * 0x400)) | ARM_TTE_TYPE_TABLE;
+               vm_offset_t len = _end - va;
+               if ((pa + len) > avail_end) {
+                       _end -= (pa + len - avail_end);

                }
                }

+               assert((start - gVirtBase + gPhysBase) >= gPhysBase);

 
                /* Apply the desired protections to the specified page range */
                for (i = 0; i < (ARM_PGBYTES / sizeof(*ppte)); i++) {
                        if (start <= va && va < _end) {
 
                /* Apply the desired protections to the specified page range */
                for (i = 0; i < (ARM_PGBYTES / sizeof(*ppte)); i++) {
                        if (start <= va && va < _end) {

-

                                ptmp = pa | ARM_PTE_AF | ARM_PTE_SH | ARM_PTE_TYPE;
                                ptmp = ptmp | ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DEFAULT);
                                ptmp = ptmp | ARM_PTE_AP(pte_prot_APX);
                                ptmp = pa | ARM_PTE_AF | ARM_PTE_SH | ARM_PTE_TYPE;
                                ptmp = ptmp | ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DEFAULT);
                                ptmp = ptmp | ARM_PTE_AP(pte_prot_APX);

-                               if (pte_prot_XN)
+                               if (pte_prot_XN) {

                                        ptmp = ptmp | ARM_PTE_NX;
                                        ptmp = ptmp | ARM_PTE_NX;

+                               }

 
                                ppte[i] = ptmp;
                        }
 
                                ppte[i] = ptmp;
                        }


@@ -188,8 +219,8 @@ arm_vm_page_granular_helper(vm_offset_t start, vm_offset_t _end, vm_offset_t va,
 }
 
 static void
 }
 
 static void

-arm_vm_page_granular_prot(vm_offset_t start, unsigned long size, 
-                          int tte_prot_XN, int pte_prot_APX, int pte_prot_XN, int forceCoarse)
+arm_vm_page_granular_prot(vm_offset_t start, unsigned long size,
+    int tte_prot_XN, int pte_prot_APX, int pte_prot_XN, int force_page_granule)

 {
        vm_offset_t _end = start + size;
        vm_offset_t align_start = (start + ARM_TT_L1_PT_OFFMASK) & ~ARM_TT_L1_PT_OFFMASK;
 {
        vm_offset_t _end = start + size;
        vm_offset_t align_start = (start + ARM_TT_L1_PT_OFFMASK) & ~ARM_TT_L1_PT_OFFMASK;


@@ -198,9 +229,9 @@ arm_vm_page_granular_prot(vm_offset_t start, unsigned long size,
        arm_vm_page_granular_helper(start, _end, start, pte_prot_APX, pte_prot_XN);
 
        while (align_start < align_end) {
        arm_vm_page_granular_helper(start, _end, start, pte_prot_APX, pte_prot_XN);
 
        while (align_start < align_end) {

-               if (forceCoarse) {
-                       arm_vm_page_granular_helper(align_start, align_end, align_start + 1, 
-                                                   pte_prot_APX, pte_prot_XN);
+               if (force_page_granule) {
+                       arm_vm_page_granular_helper(align_start, align_end, align_start + 1,
+                           pte_prot_APX, pte_prot_XN);

                } else {
                        tt_entry_t *tte = &cpu_tte[ttenum(align_start)];
                        for (int i = 0; i < 4; ++i) {
                } else {
                        tt_entry_t *tte = &cpu_tte[ttenum(align_start)];
                        for (int i = 0; i < 4; ++i) {


@@ -208,8 +239,9 @@ arm_vm_page_granular_prot(vm_offset_t start, unsigned long size,
 
                                tmplate = (tmplate & ~ARM_TTE_BLOCK_APMASK) | ARM_TTE_BLOCK_AP(pte_prot_APX);
                                tmplate = (tmplate & ~ARM_TTE_BLOCK_NX_MASK);
 
                                tmplate = (tmplate & ~ARM_TTE_BLOCK_APMASK) | ARM_TTE_BLOCK_AP(pte_prot_APX);
                                tmplate = (tmplate & ~ARM_TTE_BLOCK_NX_MASK);

-                               if (tte_prot_XN)
+                               if (tte_prot_XN) {

                                        tmplate = tmplate | ARM_TTE_BLOCK_NX;
                                        tmplate = tmplate | ARM_TTE_BLOCK_NX;

+                               }

 
                                tte[i] = tmplate;
                        }
 
                                tte[i] = tmplate;
                        }


@@ -221,27 +253,27 @@ arm_vm_page_granular_prot(vm_offset_t start, unsigned long size,
 }
 
 static inline void
 }
 
 static inline void

-arm_vm_page_granular_RNX(vm_offset_t start, unsigned long size, int forceCoarse)
+arm_vm_page_granular_RNX(vm_offset_t start, unsigned long size, int force_page_granule)

 {
 {

-       arm_vm_page_granular_prot(start, size, 1, AP_RONA, 1, forceCoarse);
+       arm_vm_page_granular_prot(start, size, 1, AP_RONA, 1, force_page_granule);

 }
 
 static inline void
 }
 
 static inline void

-arm_vm_page_granular_ROX(vm_offset_t start, unsigned long size, int forceCoarse)
+arm_vm_page_granular_ROX(vm_offset_t start, unsigned long size, int force_page_granule)

 {
 {

-       arm_vm_page_granular_prot(start, size, 0, AP_RONA, 0, forceCoarse);
+       arm_vm_page_granular_prot(start, size, 0, AP_RONA, 0, force_page_granule);

 }
 
 static inline void
 }
 
 static inline void

-arm_vm_page_granular_RWNX(vm_offset_t start, unsigned long size, int forceCoarse)
+arm_vm_page_granular_RWNX(vm_offset_t start, unsigned long size, int force_page_granule)

 {
 {

-       arm_vm_page_granular_prot(start, size, 1, AP_RWNA, 1, forceCoarse);
+       arm_vm_page_granular_prot(start, size, 1, AP_RWNA, 1, force_page_granule);

 }
 
 static inline void
 }
 
 static inline void

-arm_vm_page_granular_RWX(vm_offset_t start, unsigned long size, int forceCoarse)
+arm_vm_page_granular_RWX(vm_offset_t start, unsigned long size, int force_page_granule)

 {
 {

-       arm_vm_page_granular_prot(start, size, 0, AP_RWNA, 0, forceCoarse);
+       arm_vm_page_granular_prot(start, size, 0, AP_RWNA, 0, force_page_granule);

 }
 
 void
 }
 
 void


@@ -254,9 +286,9 @@ arm_vm_prot_init(boot_args * args)
 #endif
        /*
         * Enforce W^X protections on segments that have been identified so far. This will be
 #endif
        /*
         * Enforce W^X protections on segments that have been identified so far. This will be

-        * further refined for each KEXT's TEXT and DATA segments in readPrelinkedExtensions() 
+        * further refined for each KEXT's TEXT and DATA segments in readPrelinkedExtensions()

         */
         */

-       
+

        /*
         * Protection on kernel text is loose here to allow shenanigans early on (e.g. copying exception vectors)
         * and storing an address into "error_buffer" (see arm_init.c) !?!
        /*
         * Protection on kernel text is loose here to allow shenanigans early on (e.g. copying exception vectors)
         * and storing an address into "error_buffer" (see arm_init.c) !?!


@@ -276,14 +308,19 @@ arm_vm_prot_init(boot_args * args)
                /* If we aren't protecting const, just map DATA as a single blob. */
                arm_vm_page_granular_RWNX(segDATAB, segSizeDATA, FALSE);
        }
                /* If we aren't protecting const, just map DATA as a single blob. */
                arm_vm_page_granular_RWNX(segDATAB, segSizeDATA, FALSE);
        }

+       arm_vm_page_granular_RWNX(segBOOTDATAB, segSizeBOOTDATA, TRUE);
+       arm_vm_page_granular_RNX((vm_offset_t)&intstack_low_guard, PAGE_MAX_SIZE, TRUE);
+       arm_vm_page_granular_RNX((vm_offset_t)&intstack_high_guard, PAGE_MAX_SIZE, TRUE);
+       arm_vm_page_granular_RNX((vm_offset_t)&fiqstack_high_guard, PAGE_MAX_SIZE, TRUE);

 
        arm_vm_page_granular_ROX(segKLDB, segSizeKLD, force_coarse_physmap);
        arm_vm_page_granular_RWNX(segLINKB, segSizeLINK, force_coarse_physmap);
        arm_vm_page_granular_RWNX(segLASTB, segSizeLAST, FALSE); // __LAST may be empty, but we cannot assume this
        arm_vm_page_granular_RWNX(segPRELINKTEXTB, segSizePRELINKTEXT, TRUE); // Refined in OSKext::readPrelinkedExtensions
        arm_vm_page_granular_RWNX(segPRELINKTEXTB + segSizePRELINKTEXT,
 
        arm_vm_page_granular_ROX(segKLDB, segSizeKLD, force_coarse_physmap);
        arm_vm_page_granular_RWNX(segLINKB, segSizeLINK, force_coarse_physmap);
        arm_vm_page_granular_RWNX(segLASTB, segSizeLAST, FALSE); // __LAST may be empty, but we cannot assume this
        arm_vm_page_granular_RWNX(segPRELINKTEXTB, segSizePRELINKTEXT, TRUE); // Refined in OSKext::readPrelinkedExtensions
        arm_vm_page_granular_RWNX(segPRELINKTEXTB + segSizePRELINKTEXT,

-                                    end_kern - (segPRELINKTEXTB + segSizePRELINKTEXT), force_coarse_physmap); // PreLinkInfoDictionary
-       arm_vm_page_granular_RWNX(end_kern, phystokv(args->topOfKernelData) - end_kern, force_coarse_physmap); // Device Tree, RAM Disk (if present), bootArgs
+           end_kern - (segPRELINKTEXTB + segSizePRELINKTEXT), force_coarse_physmap);                          // PreLinkInfoDictionary
+       arm_vm_page_granular_RWNX(end_kern, phystokv(args->topOfKernelData) - end_kern, force_coarse_physmap); // Device Tree, RAM Disk (if present), bootArgs, trust caches
+       arm_vm_page_granular_RNX(segEXTRADATA, segSizeEXTRADATA, FALSE); // tighter trust cache protection

        arm_vm_page_granular_RWNX(phystokv(args->topOfKernelData), ARM_PGBYTES * 8, FALSE); // boot_tte, cpu_tte
 
        /*
        arm_vm_page_granular_RWNX(phystokv(args->topOfKernelData), ARM_PGBYTES * 8, FALSE); // boot_tte, cpu_tte
 
        /*


@@ -299,7 +336,7 @@ arm_vm_prot_init(boot_args * args)
 
        /* Map the remainder of xnu owned memory. */
        arm_vm_page_granular_RWNX(phystokv(args->topOfKernelData) + ARM_PGBYTES * 10,
 
        /* Map the remainder of xnu owned memory. */
        arm_vm_page_granular_RWNX(phystokv(args->topOfKernelData) + ARM_PGBYTES * 10,

-                                 static_memory_end - (phystokv(args->topOfKernelData) + ARM_PGBYTES * 10), force_coarse_physmap); /* rest of physmem */
+           static_memory_end - (phystokv(args->topOfKernelData) + ARM_PGBYTES * 10), force_coarse_physmap);                       /* rest of physmem */

 
        /*
         * Special case write protection for the mapping of ExceptionVectorsBase (EVB) at 0xFFFF0000.
 
        /*
         * Special case write protection for the mapping of ExceptionVectorsBase (EVB) at 0xFFFF0000.


@@ -307,8 +344,9 @@ arm_vm_prot_init(boot_args * args)
         */
        pmap_paddr_t p = (pmap_paddr_t)(args->topOfKernelData) + (ARM_PGBYTES * 9);
        pt_entry_t *ppte = (pt_entry_t *)phystokv(p);
         */
        pmap_paddr_t p = (pmap_paddr_t)(args->topOfKernelData) + (ARM_PGBYTES * 9);
        pt_entry_t *ppte = (pt_entry_t *)phystokv(p);

+       pmap_init_pte_page(kernel_pmap, ppte, HIGH_EXC_VECTORS & ~ARM_TT_L1_PT_OFFMASK, 2, TRUE, FALSE);

 
 

-       int idx = (HIGH_EXC_VECTORS & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT;
+       int idx = (HIGH_EXC_VECTORS & ARM_TT_L1_PT_OFFMASK) >> ARM_TT_L2_SHIFT;

        pt_entry_t ptmp = ppte[idx];
 
        ptmp = (ptmp & ~ARM_PTE_APMASK) | ARM_PTE_AP(AP_RONA);
        pt_entry_t ptmp = ppte[idx];
 
        ptmp = (ptmp & ~ARM_PTE_APMASK) | ARM_PTE_AP(AP_RONA);


@@ -319,6 +357,8 @@ arm_vm_prot_init(boot_args * args)
 void
 arm_vm_prot_finalize(boot_args * args)
 {
 void
 arm_vm_prot_finalize(boot_args * args)
 {

+       cpu_stack_alloc(&BootCpuData);
+       ml_static_mfree(segBOOTDATAB, segSizeBOOTDATA);

        /*
         * Naively we could have:
         * arm_vm_page_granular_ROX(segTEXTB, segSizeTEXT, FALSE);
        /*
         * Naively we could have:
         * arm_vm_page_granular_ROX(segTEXTB, segSizeTEXT, FALSE);


@@ -329,12 +369,16 @@ arm_vm_prot_finalize(boot_args * args)
 
        arm_vm_page_granular_RWNX(phystokv(args->topOfKernelData) + ARM_PGBYTES * 9, ARM_PGBYTES, FALSE); /* commpage, EVB */
 
 
        arm_vm_page_granular_RWNX(phystokv(args->topOfKernelData) + ARM_PGBYTES * 9, ARM_PGBYTES, FALSE); /* commpage, EVB */
 

-#ifndef  __ARM_L1_PTW__
-       FlushPoC_Dcache();
-#endif

        flush_mmu_tlb();
 }
 
        flush_mmu_tlb();
 }
 

+/* used in the chosen/memory-map node, populated by iBoot. */
+typedef struct MemoryMapFileInfo {
+       vm_offset_t paddr;
+       size_t length;
+} MemoryMapFileInfo;
+
+

 void
 arm_vm_init(uint64_t memory_size, boot_args * args)
 {
 void
 arm_vm_init(uint64_t memory_size, boot_args * args)
 {


@@ -352,10 +396,12 @@ arm_vm_init(uint64_t memory_size, boot_args * args)
        gPhysBase = args->physBase;
        gPhysSize = args->memSize;
        mem_size = args->memSize;
        gPhysBase = args->physBase;
        gPhysSize = args->memSize;
        mem_size = args->memSize;

-       if ((memory_size != 0) && (mem_size > memory_size))
+       if ((memory_size != 0) && (mem_size > memory_size)) {

                mem_size = memory_size;
                mem_size = memory_size;

-       if (mem_size > MEM_SIZE_MAX )
+       }
+       if (mem_size > MEM_SIZE_MAX) {

                mem_size = MEM_SIZE_MAX;
                mem_size = MEM_SIZE_MAX;

+       }

        static_memory_end = gVirtBase + mem_size;
 
        /* Calculate the nubmer of ~256MB segments of memory */
        static_memory_end = gVirtBase + mem_size;
 
        /* Calculate the nubmer of ~256MB segments of memory */


@@ -383,18 +429,20 @@ arm_vm_init(uint64_t memory_size, boot_args * args)
 
        /* Hands off [gVirtBase, gVirtBase + gPhysSize) please. */
        if (gPhysBase < gVirtBase) {
 
        /* Hands off [gVirtBase, gVirtBase + gPhysSize) please. */
        if (gPhysBase < gVirtBase) {

-               if (gPhysBase + gPhysSize > gVirtBase)
+               if (gPhysBase + gPhysSize > gVirtBase) {

                        tte_limit = &cpu_tte[ttenum(gVirtBase)];
                        tte_limit = &cpu_tte[ttenum(gVirtBase)];

+               }

        } else {
        } else {

-               if (gPhysBase < gVirtBase + gPhysSize)
+               if (gPhysBase < gVirtBase + gPhysSize) {

                        tte = &cpu_tte[ttenum(gVirtBase + gPhysSize)];
                        tte = &cpu_tte[ttenum(gVirtBase + gPhysSize)];

+               }

        }
 
        while (tte < tte_limit) {
        }
 
        while (tte < tte_limit) {

-                       *tte = ARM_TTE_TYPE_FAULT; 
-                       tte++;
-               }
-               
+               *tte = ARM_TTE_TYPE_FAULT;
+               tte++;
+       }
+

        /* Skip 6 pages (four L1 + two L2 entries) */
        avail_start = cpu_ttep + ARM_PGBYTES * 6;
        avail_end = gPhysBase + mem_size;
        /* Skip 6 pages (four L1 + two L2 entries) */
        avail_start = cpu_ttep + ARM_PGBYTES * 6;
        avail_end = gPhysBase + mem_size;


@@ -404,12 +452,33 @@ arm_vm_init(uint64_t memory_size, boot_args * args)
         * from MACH-O headers for the currently running 32 bit kernel.
         */
        segTEXTB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__TEXT", &segSizeTEXT);
         * from MACH-O headers for the currently running 32 bit kernel.
         */
        segTEXTB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__TEXT", &segSizeTEXT);

+       segLOWESTTEXT = segTEXTB;

        segDATAB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__DATA", &segSizeDATA);
        segLINKB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__LINKEDIT", &segSizeLINK);
        segKLDB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__KLD", &segSizeKLD);
        segLASTB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__LAST", &segSizeLAST);
        segPRELINKTEXTB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__PRELINK_TEXT", &segSizePRELINKTEXT);
        segPRELINKINFOB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__PRELINK_INFO", &segSizePRELINKINFO);
        segDATAB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__DATA", &segSizeDATA);
        segLINKB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__LINKEDIT", &segSizeLINK);
        segKLDB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__KLD", &segSizeKLD);
        segLASTB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__LAST", &segSizeLAST);
        segPRELINKTEXTB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__PRELINK_TEXT", &segSizePRELINKTEXT);
        segPRELINKINFOB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__PRELINK_INFO", &segSizePRELINKINFO);

+       segBOOTDATAB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__BOOTDATA", &segSizeBOOTDATA);
+
+       segEXTRADATA = 0;
+       segSizeEXTRADATA = 0;
+
+       DTEntry memory_map;
+       MemoryMapFileInfo *trustCacheRange;
+       unsigned int trustCacheRangeSize;
+       int err;
+
+       err = DTLookupEntry(NULL, "chosen/memory-map", &memory_map);
+       assert(err == kSuccess);
+
+       err = DTGetProperty(memory_map, "TrustCache", (void**)&trustCacheRange, &trustCacheRangeSize);
+       if (err == kSuccess) {
+               assert(trustCacheRangeSize == sizeof(MemoryMapFileInfo));
+
+               segEXTRADATA = phystokv(trustCacheRange->paddr);
+               segSizeEXTRADATA = trustCacheRange->length;
+       }

 
        etext = (vm_offset_t) segTEXTB + segSizeTEXT;
        sdata = (vm_offset_t) segDATAB;
 
        etext = (vm_offset_t) segTEXTB + segSizeTEXT;
        sdata = (vm_offset_t) segDATAB;


@@ -420,18 +489,13 @@ arm_vm_init(uint64_t memory_size, boot_args * args)
         * Special handling for the __DATA,__const *section*.
         * A page of padding named lastkerneldataconst is at the end of the __DATA,__const
         * so we can safely truncate the size. __DATA,__const is also aligned, but
         * Special handling for the __DATA,__const *section*.
         * A page of padding named lastkerneldataconst is at the end of the __DATA,__const
         * so we can safely truncate the size. __DATA,__const is also aligned, but

-        * just in case we will round that to a page, too. 
+        * just in case we will round that to a page, too.

         */
        segDATA = getsegbynamefromheader(&_mh_execute_header, "__DATA");
        sectDCONST = getsectbynamefromheader(&_mh_execute_header, "__DATA", "__const");
        sectCONSTB = sectDCONST->addr;
        sectSizeCONST = sectDCONST->size;
 
         */
        segDATA = getsegbynamefromheader(&_mh_execute_header, "__DATA");
        sectDCONST = getsectbynamefromheader(&_mh_execute_header, "__DATA", "__const");
        sectCONSTB = sectDCONST->addr;
        sectSizeCONST = sectDCONST->size;
 

-#if !SECURE_KERNEL
-       /* doconstro is true by default, but we allow a boot-arg to disable it */
-       (void) PE_parse_boot_argn("dataconstro", &doconstro, sizeof(doconstro));
-#endif
-

        if (doconstro) {
                extern vm_offset_t _lastkerneldataconst;
                extern vm_size_t _lastkerneldataconst_padsize;
        if (doconstro) {
                extern vm_offset_t _lastkerneldataconst;
                extern vm_size_t _lastkerneldataconst_padsize;


@@ -464,25 +528,6 @@ arm_vm_init(uint64_t memory_size, boot_args * args)
 
        vm_set_page_size();
 
 
        vm_set_page_size();
 

-#ifndef __ARM_L1_PTW__
-       FlushPoC_Dcache();
-#endif
-       set_mmu_ttb(cpu_ttep);
-       set_mmu_ttb_alternate(cpu_ttep);
-       flush_mmu_tlb();
-#if __arm__ && __ARM_USER_PROTECT__
-       {
-               unsigned int ttbr0_val, ttbr1_val, ttbcr_val;
-               thread_t thread = current_thread();
-
-               __asm__ volatile("mrc p15,0,%0,c2,c0,0\n" : "=r"(ttbr0_val));
-               __asm__ volatile("mrc p15,0,%0,c2,c0,1\n" : "=r"(ttbr1_val));
-               __asm__ volatile("mrc p15,0,%0,c2,c0,2\n" : "=r"(ttbcr_val));
-               thread->machine.uptw_ttb = ttbr0_val;
-               thread->machine.kptw_ttb = ttbr1_val;
-               thread->machine.uptw_ttc = ttbcr_val;
-       }
-#endif

        vm_prelink_stext = segPRELINKTEXTB;
        vm_prelink_etext = segPRELINKTEXTB + segSizePRELINKTEXT;
        vm_prelink_sinfo = segPRELINKINFOB;
        vm_prelink_stext = segPRELINKTEXTB;
        vm_prelink_etext = segPRELINKTEXTB + segSizePRELINKTEXT;
        vm_prelink_sinfo = segPRELINKINFOB;


@@ -492,7 +537,7 @@ arm_vm_init(uint64_t memory_size, boot_args * args)
 
        sane_size = mem_size - (avail_start - gPhysBase);
        max_mem = mem_size;
 
        sane_size = mem_size - (avail_start - gPhysBase);
        max_mem = mem_size;

-       vm_kernel_slide = gVirtBase-0x80000000;
+       vm_kernel_slide = gVirtBase - VM_KERNEL_LINK_ADDRESS;

        vm_kernel_stext = segTEXTB;
        vm_kernel_etext = segTEXTB + segSizeTEXT;
        vm_kernel_base = gVirtBase;
        vm_kernel_stext = segTEXTB;
        vm_kernel_etext = segTEXTB + segSizeTEXT;
        vm_kernel_base = gVirtBase;


@@ -502,7 +547,7 @@ arm_vm_init(uint64_t memory_size, boot_args * args)
        vm_kernel_slid_base = segTEXTB;
        vm_kernel_slid_top = vm_kext_top;
 
        vm_kernel_slid_base = segTEXTB;
        vm_kernel_slid_top = vm_kext_top;
 

-       pmap_bootstrap((gVirtBase+MEM_SIZE_MAX+0x3FFFFF) & 0xFFC00000);
+       pmap_bootstrap((gVirtBase + MEM_SIZE_MAX + 0x3FFFFF) & 0xFFC00000);

 
        arm_vm_prot_init(args);
 
 
        arm_vm_prot_init(args);
 


@@ -514,24 +559,39 @@ arm_vm_init(uint64_t memory_size, boot_args * args)
        off_end = (2 + (mem_segments * 3)) << 20;
        off_end += (unsigned int) round_page(args->Video.v_height * args->Video.v_rowBytes);
 
        off_end = (2 + (mem_segments * 3)) << 20;
        off_end += (unsigned int) round_page(args->Video.v_height * args->Video.v_rowBytes);
 

-       for (off = 0, va = (gVirtBase+MEM_SIZE_MAX+0x3FFFFF) & 0xFFC00000; off < off_end; off += ARM_TT_L1_PT_SIZE) {
+       for (off = 0, va = (gVirtBase + MEM_SIZE_MAX + 0x3FFFFF) & 0xFFC00000; off < off_end; off += ARM_TT_L1_PT_SIZE) {

                pt_entry_t   *ptp;
                pmap_paddr_t ptp_phys;
 
                ptp = (pt_entry_t *) phystokv(avail_start);
                ptp_phys = (pmap_paddr_t)avail_start;
                avail_start += ARM_PGBYTES;
                pt_entry_t   *ptp;
                pmap_paddr_t ptp_phys;
 
                ptp = (pt_entry_t *) phystokv(avail_start);
                ptp_phys = (pmap_paddr_t)avail_start;
                avail_start += ARM_PGBYTES;

-               pmap_init_pte_page(kernel_pmap, ptp, va + off, 2, TRUE);
+               pmap_init_pte_page(kernel_pmap, ptp, va + off, 2, TRUE, TRUE);

                tte = &cpu_tte[ttenum(va + off)];
                tte = &cpu_tte[ttenum(va + off)];

-               *tte     = pa_to_tte((ptp_phys        )) | ARM_TTE_TYPE_TABLE;;
-               *(tte+1) = pa_to_tte((ptp_phys + 0x400)) | ARM_TTE_TYPE_TABLE;;
-               *(tte+2) = pa_to_tte((ptp_phys + 0x800)) | ARM_TTE_TYPE_TABLE;;
-               *(tte+3) = pa_to_tte((ptp_phys + 0xC00)) | ARM_TTE_TYPE_TABLE;;
+               *tte     = pa_to_tte((ptp_phys)) | ARM_TTE_TYPE_TABLE;
+               *(tte + 1) = pa_to_tte((ptp_phys + 0x400)) | ARM_TTE_TYPE_TABLE;
+               *(tte + 2) = pa_to_tte((ptp_phys + 0x800)) | ARM_TTE_TYPE_TABLE;
+               *(tte + 3) = pa_to_tte((ptp_phys + 0xC00)) | ARM_TTE_TYPE_TABLE;

        }
 
        }
 

+       set_mmu_ttb(cpu_ttep);
+       set_mmu_ttb_alternate(cpu_ttep);
+       flush_mmu_tlb();
+#if __arm__ && __ARM_USER_PROTECT__
+       {
+               unsigned int ttbr0_val, ttbr1_val, ttbcr_val;
+               thread_t thread = current_thread();
+
+               __asm__ volatile ("mrc p15,0,%0,c2,c0,0\n" : "=r"(ttbr0_val));
+               __asm__ volatile ("mrc p15,0,%0,c2,c0,1\n" : "=r"(ttbr1_val));
+               __asm__ volatile ("mrc p15,0,%0,c2,c0,2\n" : "=r"(ttbcr_val));
+               thread->machine.uptw_ttb = ttbr0_val;
+               thread->machine.kptw_ttb = ttbr1_val;
+               thread->machine.uptw_ttc = ttbcr_val;
+       }
+#endif

        avail_start = (avail_start + PAGE_MASK) & ~PAGE_MASK;
 
        first_avail = avail_start;
        patch_low_glo_static_region(args->topOfKernelData, avail_start - args->topOfKernelData);
 }
        avail_start = (avail_start + PAGE_MASK) & ~PAGE_MASK;
 
        first_avail = avail_start;
        patch_low_glo_static_region(args->topOfKernelData, avail_start - args->topOfKernelData);
 }

-