[apple/xnu.git] / osfmk / arm64 / amcc_rorgn.c

/*
 * Copyright (c) 2019-2020 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 *
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */

#include <pexpert/arm64/board_config.h>

#if defined(KERNEL_INTEGRITY_KTRR) || defined(KERNEL_INTEGRITY_CTRR)

#include <vm/pmap.h>
#include <libkern/section_keywords.h>
#include <libkern/kernel_mach_header.h>
#include <pexpert/pexpert.h>
#include <pexpert/device_tree.h>
#include <machine/atomic.h>
#include <arm/cpu_internal.h>
#include <arm/caches_internal.h>
#include <arm/machine_routines.h>
#include <arm/pmap.h>
#include <arm64/tlb.h>
#include <arm64/amcc_rorgn.h>
#include <memmap_types.h>

#if HIBERNATION
#include <arm64/pal_hibernate.h>
#endif /* HIBERNATION */

#if HAS_IOA
#define MAX_LOCK_GROUPS 2                                       // 2 lock groups (AMCC, IOA)
#define IOA_LOCK_GROUP  1                                       // IOA lock group index
#else
#define MAX_LOCK_GROUPS 1                                       // 1 lock group (AMCC)
#endif
#define AMCC_LOCK_GROUP 0                                       // AMCC lock group index
#define MAX_APERTURES   16                                      // Maximum number of register apertures
#define MAX_PLANES      16                                      // Maximum number of planes within each aperture

#define LOCK_GROUP_HAS_CACHE_STATUS_REG (1 << 0)                // Look for cache status register in the lock group
#define LOCK_GROUP_HAS_MASTER_LOCK_REG  (1 << 1)                // Look for master lock register in the lock group

#define LOCK_TYPE_HAS_LOCK_REG          (1 << 0)                // Look for lock register in the lock type

extern vm_offset_t   segLOWESTRO;
extern vm_offset_t   segHIGHESTRO;

extern vm_offset_t   segLASTB;
extern vm_offset_t   segTEXTEXECB;
extern unsigned long segSizeLAST;
extern unsigned long segSizeLASTDATACONST;
extern unsigned long segSizeTEXTEXEC;

typedef struct lock_reg {
        uint32_t        reg_offset;                             // Register offset
        uint32_t        reg_mask;                               // Register mask
        uint32_t        reg_value;                              // Regsiter value
} lock_reg_t;

typedef struct lock_type {
        uint32_t        page_size_shift;                        // page shift used in lower/upper limit registers
        lock_reg_t      lower_limit_reg;                        // Lower limit register description
        lock_reg_t      upper_limit_reg;                        // Upper limit register description
        lock_reg_t      enable_reg;                             // Enable register description
        lock_reg_t      write_disable_reg;                      // Write disable register description
        lock_reg_t      lock_reg;                               // Lock register description
} lock_type_t;

typedef struct lock_group {
        uint32_t        aperture_count;                         // Aperture count
        uint32_t        aperture_size;                          // Aperture size
        uint32_t        plane_count;                            // Number of planes in the aperture
        uint32_t        plane_stride;                           // Stride between planes in the aperture
        uint64_t        aperture_phys_addr[MAX_APERTURES];      // Apreture physical addresses
        lock_reg_t      cache_status_reg;                       // Cache status register description
#if HAS_IOA
        lock_reg_t      master_lock_reg;                        // Master lock register description
#endif
        lock_type_t     ctrr_a;                                 // CTRR-A (KTRR) lock
} lock_group_t;

SECURITY_READ_ONLY_LATE(lock_group_t) _lock_group[MAX_LOCK_GROUPS] = { {0} };
SECURITY_READ_ONLY_LATE(bool) lock_regs_set = false;

static vm_offset_t rorgn_begin = 0;
static vm_offset_t rorgn_end = 0;
SECURITY_READ_ONLY_LATE(vm_offset_t) ctrr_begin = 0;
SECURITY_READ_ONLY_LATE(vm_offset_t) ctrr_end = 0;

static uint64_t lock_group_va[MAX_LOCK_GROUPS][MAX_APERTURES];

#if CONFIG_CSR_FROM_DT
SECURITY_READ_ONLY_LATE(bool) csr_unsafe_kernel_text = false;
#endif

#if defined(KERNEL_INTEGRITY_KTRR)
#define CTRR_LOCK_MSR ARM64_REG_KTRR_LOCK_EL1
#elif defined(KERNEL_INTEGRITY_CTRR)
#define CTRR_LOCK_MSR ARM64_REG_CTRR_LOCK_EL1
#endif

/*
 * lock_group_t - describes all the parameters xnu needs to know to
 * lock down the AMCC/IOA (Lock Group) Read Only Region(s) on cold start.
 * This description assumes that each AMCC/IOA in a given system will
 * be identical, respectively. The only variable are the number of
 * apertures present and the physical base address of each aperture.
 *
 * General xnu lock group lockdown flow:
 * - for each lock group:
 *   - ml_io_map all present lock group physical base addresses
 *   - assert all lock group begin/end page numbers set by iboot are identical
 *   - convert lock group begin/end page number to physical address
 *   - assert lock group begin/end page numbers match xnu view of read only region
 *   - assert lock group is not currently locked
 *   - ensure lock group master cache is disabled
 *   - write enable/lock registers to enable/lock the lock group read only region
 */

static bool
_dt_get_uint32(DTEntry node, char const *name, uint32_t *dest)
{
        uint32_t const *value;
        unsigned int size;

        if (SecureDTGetProperty(node, name, (void const **)&value, &size) != kSuccess) {
                return false;
        }

        if (size != sizeof(uint32_t)) {
                panic("lock-regs: unexpected size %u", size);
        }

        *dest = *value;

        return true;
}

static uint32_t
_dt_get_uint32_required(DTEntry node, char const *name)
{
        uint32_t value;

        if (!_dt_get_uint32(node, name, &value)) {
                panic("lock-regs: cannot find required property '%s'", name);
        }

        return value;
}

static bool
_dt_get_lock_reg(DTEntry node, lock_reg_t *reg, const char *parent_name, const char *reg_name, bool required, bool with_value)
{
        char prop_name[32];
        bool found;

        snprintf(prop_name, sizeof(prop_name), "%s-reg-offset", reg_name);
        found = _dt_get_uint32(node, prop_name, &reg->reg_offset);
        if (!found) {
                if (required) {
                        panic("%s: missing property '%s'", parent_name, prop_name);
                } else {
                        return false;
                }
        }

        snprintf(prop_name, sizeof(prop_name), "%s-reg-mask", reg_name);
        found = _dt_get_uint32(node, prop_name, &reg->reg_mask);
        if (!found) {
                panic("%s: missing property '%s'", parent_name, prop_name);
        }

        if (with_value) {
                snprintf(prop_name, sizeof(prop_name), "%s-reg-value", reg_name);
                found = _dt_get_uint32(node, prop_name, &reg->reg_value);
                if (!found) {
                        panic("%s: missing property '%s'", parent_name, prop_name);
                }
        }

        return true;
}

static DTEntry
_dt_get_lock_group(DTEntry lock_regs_node, lock_group_t* lock_group, const char *group_name, uint32_t options)
{
        DTEntry group_node;

        // Find the lock group node.
        if (SecureDTLookupEntry(lock_regs_node, group_name, &group_node) != kSuccess) {
                panic("lock-regs: /chosen/lock-regs/%s not found", group_name);
        }

        lock_group->aperture_count = _dt_get_uint32_required(group_node, "aperture-count");

        if (lock_group->aperture_count > MAX_APERTURES) {
                panic("%s: %s %u exceeds maximum %u", group_name, "aperture-count", lock_group->aperture_count, MAX_APERTURES);
        }

        lock_group->aperture_size = _dt_get_uint32_required(group_node, "aperture-size");

        if ((lock_group->aperture_count > 0) && (lock_group->aperture_size == 0)) {
                panic("%s: have %u apertures, but 0 size", group_name, lock_group->aperture_count);
        }

        lock_group->plane_count = _dt_get_uint32_required(group_node, "plane-count");

        if (lock_group->plane_count > MAX_PLANES) {
                panic("%s: %s %u exceeds maximum %u", group_name, "plane-count", lock_group->plane_count, MAX_PLANES);
        }

        if (!_dt_get_uint32(group_node, "plane-stride", &lock_group->plane_stride)) {
                lock_group->plane_stride = 0;
        }

        if (lock_group->plane_count > 1) {
                uint32_t aperture_size;

                if (lock_group->plane_stride == 0) {
                        panic("%s: plane-count (%u) > 1, but stride is 0/missing", group_name, lock_group->plane_count);
                }

                if (os_mul_overflow(lock_group->plane_count, lock_group->plane_stride, &aperture_size)
                    || (aperture_size > lock_group->aperture_size)) {
                        panic("%s: aperture-size (%#x) is insufficent to cover plane-count (%#x) of plane-stride (%#x) bytes", group_name, lock_group->aperture_size, lock_group->plane_count, lock_group->plane_stride);
                }
        }

        uint64_t const *phys_bases = NULL;
        unsigned int prop_size;
        if (SecureDTGetProperty(group_node, "aperture-phys-addr", (const void**)&phys_bases, &prop_size) != kSuccess) {
                panic("%s: missing required %s", group_name, "aperture-phys-addr");
        }

        if (prop_size != lock_group->aperture_count * sizeof(lock_group->aperture_phys_addr[0])) {
                panic("%s: aperture-phys-addr size (%#x) != (aperture-count (%#x) * PA size (%#zx) = %#lx)",
                    group_name, prop_size, lock_group->aperture_count, sizeof(lock_group->aperture_phys_addr[0]),
                    lock_group->aperture_count * sizeof(lock_group->aperture_phys_addr[0]));
        }

        memcpy(lock_group->aperture_phys_addr, phys_bases, prop_size);

        if (options & LOCK_GROUP_HAS_CACHE_STATUS_REG) {
                _dt_get_lock_reg(group_node, &lock_group->cache_status_reg, group_name, "cache-status", true, true);
        }

#if HAS_IOA
        if (options & LOCK_GROUP_HAS_MASTER_LOCK_REG) {
                _dt_get_lock_reg(group_node, &lock_group->master_lock_reg, group_name, "master-lock", true, true);
        }
#endif

        return group_node;
}

static void
_dt_get_lock_type(DTEntry group_node, lock_type_t *lock_type, const char *group_name, const char *type_name, uint32_t options)
{
        DTEntry type_node;
        bool has_lock = options & LOCK_TYPE_HAS_LOCK_REG;

        // Find the lock type type_node.
        if (SecureDTLookupEntry(group_node, type_name, &type_node) != kSuccess) {
                panic("lock-regs: /chosen/lock-regs/%s/%s not found", group_name, type_name);
        }

        lock_type->page_size_shift = _dt_get_uint32_required(type_node, "page-size-shift");

        // Find all of the regsiters for this lock type.
        //               Parent     Register Descriptor            Parent Name Reg Name        Required Value
        _dt_get_lock_reg(type_node, &lock_type->lower_limit_reg, type_name, "lower-limit", true, false);
        _dt_get_lock_reg(type_node, &lock_type->upper_limit_reg, type_name, "upper-limit", true, false);
        _dt_get_lock_reg(type_node, &lock_type->lock_reg, type_name, "lock", has_lock, true);
        _dt_get_lock_reg(type_node, &lock_type->enable_reg, type_name, "enable", false, true);
        _dt_get_lock_reg(type_node, &lock_type->write_disable_reg, type_name, "write-disable", false, true);
}

/*
 * find_lock_group_data:
 *
 * finds and gathers lock group (AMCC/IOA) data from device tree, returns it as lock_group_t
 *
 * called first time before IOKit start while still uniprocessor
 *
 */
static lock_group_t const * _Nonnull
find_lock_group_data(void)
{
        DTEntry lock_regs_node = NULL;
        DTEntry amcc_node = NULL;

        // Return the lock group data pointer if we already found and populated one.
        if (lock_regs_set) {
                return _lock_group;
        }

        if (SecureDTLookupEntry(NULL, "/chosen/lock-regs", &lock_regs_node) != kSuccess) {
                panic("lock-regs: /chosen/lock-regs not found (your iBoot or EDT may be too old)");
        }

        amcc_node = _dt_get_lock_group(lock_regs_node, &_lock_group[AMCC_LOCK_GROUP], "amcc", LOCK_GROUP_HAS_CACHE_STATUS_REG);
        _dt_get_lock_type(amcc_node, &_lock_group[AMCC_LOCK_GROUP].ctrr_a, "amcc", "amcc-ctrr-a", LOCK_TYPE_HAS_LOCK_REG);

#if HAS_IOA
        DTEntry ioa_node = _dt_get_lock_group(lock_regs_node, &_lock_group[IOA_LOCK_GROUP], "ioa", LOCK_GROUP_HAS_MASTER_LOCK_REG);
        _dt_get_lock_type(ioa_node, &_lock_group[IOA_LOCK_GROUP].ctrr_a, "ioa", "ioa-ctrr-a", 0);
#endif

        lock_regs_set = true;

        return _lock_group;
}

void
rorgn_stash_range(void)
{
#if DEVELOPMENT || DEBUG || CONFIG_DTRACE || CONFIG_CSR_FROM_DT
        boolean_t rorgn_disable = FALSE;

#if DEVELOPMENT || DEBUG
        PE_parse_boot_argn("-unsafe_kernel_text", &rorgn_disable, sizeof(rorgn_disable));
#endif

#if CONFIG_CSR_FROM_DT
        if (csr_unsafe_kernel_text) {
                rorgn_disable = true;
        }
#endif

        if (rorgn_disable) {
                /* take early out if boot arg present, don't query any machine registers to avoid
                 * dependency on amcc DT entry
                 */
                return;
        }
#endif
        lock_group_t const * const lock_group = find_lock_group_data();

        /* Get the lock group read-only region range values, and stash them into rorgn_begin, rorgn_end. */
        uint64_t rorgn_begin_page[MAX_LOCK_GROUPS][MAX_APERTURES][MAX_PLANES];
        uint64_t rorgn_end_page[MAX_LOCK_GROUPS][MAX_APERTURES][MAX_PLANES];

        for (unsigned int lg = 0; lg < MAX_LOCK_GROUPS; lg++) {
                for (unsigned int aperture = 0; aperture < lock_group[lg].aperture_count; aperture++) {
                        const uint64_t amcc_pa = lock_group[lg].aperture_phys_addr[aperture];

                        // VA space will be unmapped and freed after lockdown complete in rorgn_lockdown()
                        lock_group_va[lg][aperture] = ml_io_map(amcc_pa, lock_group[lg].aperture_size);

                        if (lock_group_va[lg][aperture] == 0) {
                                panic("map aperture_phys_addr[%u]/%#x failed", aperture, lock_group[lg].aperture_size);
                        }

                        for (unsigned int plane = 0; plane < lock_group[lg].plane_count; plane++) {
                                uint64_t reg_addr;

                                reg_addr = lock_group_va[lg][aperture] + (plane * lock_group[lg].plane_stride) + lock_group[lg].ctrr_a.lower_limit_reg.reg_offset;
                                rorgn_begin_page[lg][aperture][plane] = *(volatile uint32_t *)reg_addr;
                                reg_addr = lock_group_va[lg][aperture] + (plane * lock_group[lg].plane_stride) + lock_group[lg].ctrr_a.upper_limit_reg.reg_offset;
                                rorgn_end_page[lg][aperture][plane] = *(volatile uint32_t *)reg_addr;
                        }
                }

                assert(rorgn_end_page[lg][0][0] > rorgn_begin_page[lg][0][0]);

                for (unsigned int aperture = 0; aperture < lock_group[lg].aperture_count; aperture++) {
                        for (unsigned int plane = 0; plane < lock_group[lg].plane_count; plane++) {
                                if ((rorgn_begin_page[lg][aperture][plane] != rorgn_begin_page[0][0][0])
                                    || (rorgn_end_page[lg][aperture][plane] != rorgn_end_page[0][0][0])) {
                                        panic("Inconsistent memory config");
                                }
                        }
                }

                uint64_t page_bytes = 1ULL << lock_group[lg].ctrr_a.page_size_shift;

                /* rorgn_begin and rorgn_end are first and last byte inclusive of lock group read only region as determined by iBoot. */
                rorgn_begin = (rorgn_begin_page[0][0][0] << lock_group[lg].ctrr_a.page_size_shift) + gDramBase;
                rorgn_end = (rorgn_end_page[0][0][0] << lock_group[lg].ctrr_a.page_size_shift) + gDramBase + page_bytes - 1;
        }

        assert(segLOWESTRO && gVirtBase && gPhysBase);

        /* ctrr_begin and end are first and last bytes inclusive of MMU KTRR/CTRR region */
        ctrr_begin = kvtophys(segLOWESTRO);

#if defined(KERNEL_INTEGRITY_KTRR)

        /* __LAST is not part of the MMU KTRR region (it is however part of the AMCC read only region)
         *
         * +------------------+-----------+-----------------------------------+
         * | Largest Address  |    LAST   | <- AMCC RO Region End (rorgn_end) |
         * +------------------+-----------+-----------------------------------+
         * |                  | TEXT_EXEC | <- KTRR RO Region End (ctrr_end)  |
         * +------------------+-----------+-----------------------------------+
         * |                  |    ...    |                                   |
         * +------------------+-----------+-----------------------------------+
         * | Smallest Address |   LOWEST  | <- KTRR/AMCC RO Region Begin      |
         * |                  |           |     (ctrr_begin/rorgn_begin)      |
         * +------------------+-----------+-----------------------------------+
         *
         */

        ctrr_end = kvtophys(segLASTB) - segSizeLASTDATACONST - 1;

        /* assert not booted from kernel collection */
        assert(!segHIGHESTRO);

        /* assert that __LAST segment containing privileged insns is only a single page */
        assert(segSizeLAST == PAGE_SIZE);

        /* assert that segLAST is contiguous and just after/above/numerically higher than KTRR end */
        assert((ctrr_end + 1) == kvtophys(segTEXTEXECB) + segSizeTEXTEXEC);

        /* ensure that iboot and xnu agree on the amcc rorgn range */
        assert((rorgn_begin == ctrr_begin) && (rorgn_end == (ctrr_end + segSizeLASTDATACONST + segSizeLAST)));
#elif defined(KERNEL_INTEGRITY_CTRR)

        /* __LAST is part of MMU CTRR region. Can't use the KTRR style method of making
         * __pinst no execute because PXN applies with MMU off in CTRR.
         *
         * +------------------+-----------+------------------------------+
         * | Largest Address  |    LAST   |  <- CTRR/AMCC RO Region End  |
         * |                  |           |     (ctrr_end/rorgn_end)     |
         * +------------------+-----------+------------------------------+
         * |                  | TEXT_EXEC |                              |
         * +------------------+-----------+------------------------------+
         * |                  |    ...    |                              |
         * +------------------+-----------+------------------------------+
         * | Smallest Address |   LOWEST  | <- CTRR/AMCC RO Region Begin |
         * |                  |           |    (ctrr_begin/rorgn_begin)  |
         * +------------------+-----------+------------------------------+
         *
         */

        if (segHIGHESTRO) {
                /*
                 * kernel collections may have additional kext RO data after kernel LAST
                 */
                assert(segLASTB + segSizeLAST <= segHIGHESTRO);
                ctrr_end = kvtophys(segHIGHESTRO) - 1;
        } else {
                ctrr_end = kvtophys(segLASTB) + segSizeLAST - 1;
        }

        /* ensure that iboot and xnu agree on the amcc rorgn range */
        assert((rorgn_begin == ctrr_begin) && (rorgn_end == ctrr_end));
#endif
}

#if DEVELOPMENT || DEBUG
static void
assert_all_lock_groups_unlocked(lock_group_t const *lock_groups)
{
        uint64_t reg_addr;
        uint64_t ctrr_lock = 0;
        bool locked = false;
        bool write_disabled = false;;

        assert(lock_groups);

        for (unsigned int lg = 0; lg < MAX_LOCK_GROUPS; lg++) {
                for (unsigned int aperture = 0; aperture < lock_groups[lg].aperture_count; aperture++) {
#if HAS_IOA
                        // Does the lock group define a master lock register?
                        if (lock_groups[lg].master_lock_reg.reg_mask != 0) {
                                reg_addr = lock_group_va[lg][aperture] + lock_groups[lg].master_lock_reg.reg_offset;
                                locked |= ((*(volatile uint32_t *)reg_addr & lock_groups[lg].master_lock_reg.reg_mask) == lock_groups[lg].master_lock_reg.reg_value);
                        }
#endif
                        for (unsigned int plane = 0; plane < lock_groups[lg].plane_count; plane++) {
                                // Does the lock group define a write disable register?
                                if (lock_groups[lg].ctrr_a.write_disable_reg.reg_mask != 0) {
                                        reg_addr = lock_group_va[lg][aperture] + (plane * lock_groups[lg].plane_stride) + lock_groups[lg].ctrr_a.write_disable_reg.reg_offset;
                                        write_disabled |= ((*(volatile uint32_t *)reg_addr & lock_groups[lg].ctrr_a.write_disable_reg.reg_mask) == lock_groups[lg].ctrr_a.write_disable_reg.reg_value);
                                }

                                // Does the lock group define a lock register?
                                if (lock_groups[lg].ctrr_a.lock_reg.reg_mask != 0) {
                                        reg_addr = lock_group_va[lg][aperture] + (plane * lock_groups[lg].plane_stride) + lock_groups[lg].ctrr_a.lock_reg.reg_offset;
                                        locked |= ((*(volatile uint32_t *)reg_addr & lock_groups[lg].ctrr_a.lock_reg.reg_mask) == lock_groups[lg].ctrr_a.lock_reg.reg_value);
                                }
                        }
                }
        }

        ctrr_lock = __builtin_arm_rsr64(CTRR_LOCK_MSR);

        assert(!ctrr_lock);
        assert(!write_disabled && !locked);
}
#endif

static void
lock_all_lock_groups(lock_group_t const *lock_group, vm_offset_t begin, vm_offset_t end)
{
        uint64_t reg_addr;
        assert(lock_group);

        /*
         * [x] - ensure all in flight writes are flushed to the lock group before enabling RO Region Lock
         *
         * begin and end are first and last byte inclusive of lock group read only region
         */

        CleanPoC_DcacheRegion_Force(begin, end - begin + 1);

        for (unsigned int lg = 0; lg < MAX_LOCK_GROUPS; lg++) {
                for (unsigned int aperture = 0; aperture < lock_group[lg].aperture_count; aperture++) {
                        /* lock planes in reverse order: plane 0 should be locked last */
                        unsigned int plane = lock_group[lg].plane_count - 1;
                        do {
                                // Enable the protection region if the lock group defines an enable register.
                                if (lock_group[lg].ctrr_a.enable_reg.reg_mask != 0) {
                                        reg_addr = lock_group_va[lg][aperture] + (plane * lock_group[lg].plane_stride) + lock_group[lg].ctrr_a.enable_reg.reg_offset;
                                        *(volatile uint32_t *)reg_addr = lock_group[lg].ctrr_a.enable_reg.reg_value;
                                }

                                // Disable writes if the lock group defines a write disable register.
                                if (lock_group[lg].ctrr_a.write_disable_reg.reg_mask != 0) {
                                        reg_addr = lock_group_va[lg][aperture] + (plane * lock_group[lg].plane_stride) + lock_group[lg].ctrr_a.write_disable_reg.reg_offset;
                                        *(volatile uint32_t *)reg_addr = lock_group[lg].ctrr_a.write_disable_reg.reg_value;
                                }

                                // Lock the lock if the lock group defines an enable register.
                                if (lock_group[lg].ctrr_a.lock_reg.reg_mask != 0) {
                                        reg_addr = lock_group_va[lg][aperture] + (plane * lock_group[lg].plane_stride) + lock_group[lg].ctrr_a.lock_reg.reg_offset;
                                        *(volatile uint32_t *)reg_addr = lock_group[lg].ctrr_a.lock_reg.reg_value;
                                }

                                __builtin_arm_isb(ISB_SY);
                        } while (plane-- > 0);
#if HAS_IOA
                        // Lock the master lock if the lock group define a master lock register.
                        if (lock_group[lg].master_lock_reg.reg_mask != 0) {
                                reg_addr = lock_group_va[lg][aperture] + lock_group[lg].master_lock_reg.reg_offset;
                                *(volatile uint32_t *)reg_addr = lock_group[lg].master_lock_reg.reg_value;
                        }
                        __builtin_arm_isb(ISB_SY);
#endif
                }
        }
}

static void
lock_mmu(uint64_t begin, uint64_t end)
{
#if defined(KERNEL_INTEGRITY_KTRR)

        __builtin_arm_wsr64(ARM64_REG_KTRR_LOWER_EL1, begin);
        __builtin_arm_wsr64(ARM64_REG_KTRR_UPPER_EL1, end);
        __builtin_arm_wsr64(ARM64_REG_KTRR_LOCK_EL1, 1ULL);

        /* flush TLB */

        __builtin_arm_isb(ISB_SY);
        flush_mmu_tlb();

#elif defined (KERNEL_INTEGRITY_CTRR)
        /* this will lock the entire bootstrap cluster. non bootstrap clusters
         * will be locked by respective cluster master in start.s */

        __builtin_arm_wsr64(ARM64_REG_CTRR_A_LWR_EL1, begin);
        __builtin_arm_wsr64(ARM64_REG_CTRR_A_UPR_EL1, end);

#if !defined(APPLEVORTEX)
        /* H12+ changed sequence, must invalidate TLB immediately after setting CTRR bounds */
        __builtin_arm_isb(ISB_SY); /* ensure all prior MSRs are complete */
        flush_mmu_tlb();
#endif /* !defined(APPLEVORTEX) */

        __builtin_arm_wsr64(ARM64_REG_CTRR_CTL_EL1, CTRR_CTL_EL1_A_PXN | CTRR_CTL_EL1_A_MMUON_WRPROTECT);
        __builtin_arm_wsr64(ARM64_REG_CTRR_LOCK_EL1, 1ULL);

        uint64_t current_el = __builtin_arm_rsr64("CurrentEL");
        if (current_el == PSR64_MODE_EL2) {
                // CTRR v2 has explicit registers for cluster config. they can only be written in EL2

                __builtin_arm_wsr64(ACC_CTRR_A_LWR_EL2, begin);
                __builtin_arm_wsr64(ACC_CTRR_A_UPR_EL2, end);
                __builtin_arm_wsr64(ACC_CTRR_CTL_EL2, CTRR_CTL_EL1_A_PXN | CTRR_CTL_EL1_A_MMUON_WRPROTECT);
                __builtin_arm_wsr64(ACC_CTRR_LOCK_EL2, 1ULL);
        }

        __builtin_arm_isb(ISB_SY); /* ensure all prior MSRs are complete */
#if defined(APPLEVORTEX)
        flush_mmu_tlb();
#endif /* defined(APPLEVORTEX) */

#else /* defined(KERNEL_INTEGRITY_KTRR) */
#error KERNEL_INTEGRITY config error
#endif /* defined(KERNEL_INTEGRITY_KTRR) */
}

#if DEVELOPMENT || DEBUG
static void
assert_amcc_cache_disabled(lock_group_t const *lock_group)
{
        assert(lock_group);

        const lock_reg_t *cache_status_reg = &lock_group[AMCC_LOCK_GROUP].cache_status_reg;

        // If the platform does not define a cache status register, then we're done here.
        if (cache_status_reg->reg_mask != 0) {
                return;
        }

        for (unsigned int aperture = 0; aperture < lock_group[AMCC_LOCK_GROUP].aperture_count; aperture++) {
                for (unsigned int plane = 0; plane < lock_group[AMCC_LOCK_GROUP].plane_count; plane++) {
                        uint64_t reg_addr = lock_group_va[AMCC_LOCK_GROUP][aperture] + (plane * lock_group[AMCC_LOCK_GROUP].plane_stride) + cache_status_reg->reg_offset;
                        uint32_t reg_value = *(volatile uint32_t *)reg_addr;
                        assert((reg_value & cache_status_reg->reg_mask) == cache_status_reg->reg_value);
                }
        }
}
#endif /* DEVELOPMENT || DEBUG */

/*
 * void rorgn_lockdown(void)
 *
 * Lock the MMU and AMCC RORegion within lower and upper boundaries if not already locked
 *
 * [ ] - ensure this is being called ASAP on secondary CPUs: KTRR programming and lockdown handled in
 *       start.s:start_cpu() for subsequent wake/resume of all cores
 */
void
rorgn_lockdown(void)
{
        boolean_t ctrr_disable = FALSE;

#if DEVELOPMENT || DEBUG
        PE_parse_boot_argn("-unsafe_kernel_text", &ctrr_disable, sizeof(ctrr_disable));
#endif /* DEVELOPMENT || DEBUG */

#if CONFIG_CSR_FROM_DT
        if (csr_unsafe_kernel_text) {
                ctrr_disable = true;
        }
#endif /* CONFIG_CSR_FROM_DT */

        if (!ctrr_disable) {
                lock_group_t const * const lock_group = find_lock_group_data();

#if DEVELOPMENT || DEBUG
                assert_all_lock_groups_unlocked(lock_group);

                printf("RO Region Begin: %p End: %p\n", (void *)rorgn_begin, (void *)rorgn_end);
                printf("CTRR (MMU) Begin: %p End: %p, setting lockdown\n", (void *)ctrr_begin, (void *)ctrr_end);

                assert_amcc_cache_disabled(lock_group);
#endif /* DEVELOPMENT || DEBUG */

                // Lock the AMCC/IOA PIO lock registers.
                lock_all_lock_groups(lock_group, phystokv(rorgn_begin), phystokv(rorgn_end));

                /*
                 * KTRR/CTRR registers are inclusive of the smallest page size granule supported by processor MMU
                 * rather than the actual page size in use. Load the last byte of the end page, and let the HW
                 * truncate per the smallest page granule supported. Must use same treament in start.s for warm
                 * start of APs.
                 */
                lock_mmu(ctrr_begin, ctrr_end);

                // Unmap and free PIO VA space needed to lockdown the lock groups.
                for (unsigned int lg = 0; lg < MAX_LOCK_GROUPS; lg++) {
                        for (unsigned int aperture = 0; aperture < lock_group[lg].aperture_count; aperture++) {
                                ml_io_unmap(lock_group_va[lg][aperture], lock_group[lg].aperture_size);
                        }
                }
        }

#if defined(KERNEL_INTEGRITY_CTRR)
        /* wake any threads blocked on cluster master lockdown */
        cpu_data_t *cdp;

        cdp = getCpuDatap();

        cdp->cpu_cluster_id = ml_get_cluster_number_local();
        assert(cdp->cpu_cluster_id <= (uint32_t)ml_get_max_cluster_number());
        ctrr_cluster_locked[cdp->cpu_cluster_id] = CTRR_LOCKED;
        thread_wakeup(&ctrr_cluster_locked[cdp->cpu_cluster_id]);
#endif
}

#endif /* defined(KERNEL_INTEGRITY_KTRR) || defined(KERNEL_INTEGRITY_CTRR) */