xnu-4570.20.62.tar.gz

[apple/xnu.git] / osfmk / arm64 / start.s

/*
 * Copyright (c) 2007-2013 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 *
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */
#include <arm/proc_reg.h>
#include <arm64/asm.h>
#include <arm64/proc_reg.h>
#include <pexpert/arm64/board_config.h>
#include <pexpert/arm64/cyclone.h>
#include <pexpert/arm64/hurricane.h>
#include <mach_assert.h>
#include <machine/asm.h>
#include "assym.s"


.macro MSR_VBAR_EL1_X0
#if defined(KERNEL_INTEGRITY_KTRR)
	mov	x1, lr
	bl		EXT(pinst_set_vbar)
	mov	lr, x1
#else
	msr		VBAR_EL1, x0
#endif
.endmacro

.macro MSR_TCR_EL1_X1
#if defined(KERNEL_INTEGRITY_KTRR)
	mov		x0, x1
	mov		x1, lr
	bl		_pinst_set_tcr
	mov		lr, x1
#else
	msr		TCR_EL1, x1
#endif
.endmacro

.macro MSR_TTBR1_EL1_X0
#if defined(KERNEL_INTEGRITY_KTRR)
	mov		x1, lr
	bl		_pinst_set_ttbr1
	mov		lr, x1
#else
	msr		TTBR1_EL1, x0
#endif
.endmacro

.macro MSR_SCTLR_EL1_X0
#if defined(KERNEL_INTEGRITY_KTRR) 
	mov		x1, lr

	// This may abort, do so on SP1
	bl		_pinst_spsel_1

	bl		_pinst_set_sctlr
	msr		SPSel, #0									// Back to SP0
	mov		lr, x1
#else
	msr		SCTLR_EL1, x0
#endif /* defined(KERNEL_INTEGRITY_KTRR) */
.endmacro

/*
 * Checks the reset handler for global and CPU-specific reset-assist functions,
 * then jumps to the reset handler with boot args and cpu data. This is copied
 * to the first physical page during CPU bootstrap (see cpu.c).
 *
 * Variables:
 *	x19 - Reset handler data pointer
 *	x20 - Boot args pointer
 *	x21 - CPU data pointer
 */
	.text
	.align 12
	.globl EXT(LowResetVectorBase)
LEXT(LowResetVectorBase)
	// Preserve x0 for start_first_cpu, if called

	// Unlock the core for debugging
	msr		OSLAR_EL1, xzr

#if !(defined(KERNEL_INTEGRITY_KTRR))
	// Set low reset vector before attempting any loads
	adrp    x0, EXT(LowExceptionVectorBase)@page
	add     x0, x0, EXT(LowExceptionVectorBase)@pageoff
	msr     VBAR_EL1, x0
#endif


#if defined(KERNEL_INTEGRITY_KTRR)
	/*
	 * Set KTRR registers immediately after wake/resume
	 *
	 * During power on reset, XNU stashed the kernel text region range values
	 * into __DATA,__const which should be protected by AMCC RoRgn at this point.
	 * Read this data and program/lock KTRR registers accordingly.
	 * If either values are zero, we're debugging kernel so skip programming KTRR.
	 */

	// load stashed rorgn_begin
	adrp	x17, EXT(rorgn_begin)@page
	add		x17, x17, EXT(rorgn_begin)@pageoff
	ldr		x17, [x17]
	// if rorgn_begin is zero, we're debugging. skip enabling ktrr
	cbz		x17, 1f

	// load stashed rorgn_end
	adrp	x19, EXT(rorgn_end)@page
	add		x19, x19, EXT(rorgn_end)@pageoff
	ldr		x19, [x19]
	cbz		x19, 1f

	// program and lock down KTRR
	// subtract one page from rorgn_end to make pinst insns NX
	msr		ARM64_REG_KTRR_LOWER_EL1, x17
	sub		x19, x19, #(1 << (ARM_PTE_SHIFT-12)), lsl #12 
	msr		ARM64_REG_KTRR_UPPER_EL1, x19
	mov		x17, #1
	msr		ARM64_REG_KTRR_LOCK_EL1, x17

1:
#endif /* defined(KERNEL_INTEGRITY_KTRR) */

	// Process reset handlers
	adrp	x19, EXT(ResetHandlerData)@page			// Get address of the reset handler data
	add		x19, x19, EXT(ResetHandlerData)@pageoff
	mrs		x15, MPIDR_EL1						// Load MPIDR to get CPU number
	and		x0, x15, #0xFF						// CPU number is in MPIDR Affinity Level 0
	ldr		x1, [x19, CPU_DATA_ENTRIES]			// Load start of data entries
	add		x3, x1, MAX_CPUS * 16				// end addr of data entries = start + (16 * MAX_CPUS)  
Lcheck_cpu_data_entry:
	ldr		x21, [x1, CPU_DATA_PADDR]			// Load physical CPU data address
	cbz		x21, Lnext_cpu_data_entry
	ldr		w2, [x21, CPU_PHYS_ID]				// Load ccc cpu phys id
	cmp		x0, x2						// Compare cpu data phys cpu and MPIDR_EL1 phys cpu
	b.eq		Lfound_cpu_data_entry				// Branch if match
Lnext_cpu_data_entry:
	add		x1, x1, #16					// Increment to the next cpu data entry
	cmp		x1, x3
	b.eq		Lskip_cpu_reset_handler				// Not found
	b		Lcheck_cpu_data_entry	// loop
Lfound_cpu_data_entry:
	adrp		x20, EXT(const_boot_args)@page
	add		x20, x20, EXT(const_boot_args)@pageoff
	ldr		x0, [x21, CPU_RESET_HANDLER]		// Call CPU reset handler
	cbz		x0, Lskip_cpu_reset_handler

	// Validate that our handler is one of the two expected handlers
	adrp	x2, EXT(resume_idle_cpu)@page
	add		x2, x2, EXT(resume_idle_cpu)@pageoff
	cmp		x0, x2
	beq		1f
	adrp	x2, EXT(start_cpu)@page
	add		x2, x2, EXT(start_cpu)@pageoff
	cmp		x0, x2
	bne	Lskip_cpu_reset_handler
1:



	blr		x0
Lskip_cpu_reset_handler:
	b		.									// Hang if the handler is NULL or returns

	.align	3
	.globl  EXT(ResetHandlerData)
LEXT(ResetHandlerData)
	.space  (rhdSize_NUM),0		// (filled with 0s)

	.align	3
	.global EXT(LowResetVectorEnd)
LEXT(LowResetVectorEnd)
	.global	EXT(SleepToken)
#if WITH_CLASSIC_S2R
LEXT(SleepToken)
	.space	(stSize_NUM),0
#endif


/*
 * __start trampoline is located at a position relative to LowResetVectorBase
 * so that iBoot can compute the reset vector position to set IORVBAR using
 * only the kernel entry point.  Reset vector = (__start & ~0xfff)
 */
	.align	3
	.globl EXT(_start)
LEXT(_start)
	b	EXT(start_first_cpu)


/*
 * Provides an early-boot exception vector so that the processor will spin
 * and preserve exception information (e.g., ELR_EL1) when early CPU bootstrap
 * code triggers an exception. This is copied to the second physical page
 * during CPU bootstrap (see cpu.c).
 */
	.align 12, 0
	.global	EXT(LowExceptionVectorBase)
LEXT(LowExceptionVectorBase)
	/* EL1 SP 0 */
	b		.
	.align	7
	b		.
	.align	7
	b		.
	.align	7
	b		.
	/* EL1 SP1 */
	.align	7
	b		.
	.align	7
	b		.
	.align	7
	b		.
	.align	7
	b		.
	/* EL0 64 */
	.align	7
	b		.
	.align	7
	b		.
	.align	7
	b		.
	.align	7
	b		.
	/* EL0 32 */
	.align	7
	b		.
	.align	7
	b		.
	.align	7
	b		.
	.align	7
	b		.
	.align 12, 0

#if defined(KERNEL_INTEGRITY_KTRR)
/*
 * Provide a global symbol so that we can narrow the V=P mapping to cover
 * this page during arm_vm_init.
 */
.align ARM_PGSHIFT
.globl EXT(bootstrap_instructions)
LEXT(bootstrap_instructions)
#endif /* defined(KERNEL_INTEGRITY_KTRR)*/
	.align 2
	.globl EXT(resume_idle_cpu)
LEXT(resume_idle_cpu)
	adrp	lr, EXT(arm_init_idle_cpu)@page
	add		lr, lr, EXT(arm_init_idle_cpu)@pageoff
	b		start_cpu

	.align 2
	.globl EXT(start_cpu)
LEXT(start_cpu)
	adrp	lr, EXT(arm_init_cpu)@page
	add		lr, lr, EXT(arm_init_cpu)@pageoff
	b		start_cpu

	.align 2
start_cpu:
#if defined(KERNEL_INTEGRITY_KTRR)
	// This is done right away in reset vector for pre-KTRR devices
	// Set low reset vector now that we are in the KTRR-free zone
	adrp	x0, EXT(LowExceptionVectorBase)@page
	add		x0, x0, EXT(LowExceptionVectorBase)@pageoff
	MSR_VBAR_EL1_X0
#endif /* defined(KERNEL_INTEGRITY_KTRR)*/

	// x20 set to BootArgs phys address
	// x21 set to cpu data phys address
	msr		DAIFSet, #(DAIFSC_ALL)				// Disable all interrupts

	// Get the kernel memory parameters from the boot args
	ldr		x22, [x20, BA_VIRT_BASE]			// Get the kernel virt base
	ldr		x23, [x20, BA_PHYS_BASE]			// Get the kernel phys base
	ldr		x24, [x20, BA_MEM_SIZE]				// Get the physical memory size
	ldr		x25, [x20, BA_TOP_OF_KERNEL_DATA]	// Get the top of the kernel data

	// Set TPIDRRO_EL0 with the CPU number
	ldr		x0, [x21, CPU_NUMBER_GS]
	msr		TPIDRRO_EL0, x0

	// Set the exception stack pointer
	ldr		x0, [x21, CPU_EXCEPSTACK_TOP]


	// Set SP_EL1 to exception stack
#if defined(KERNEL_INTEGRITY_KTRR)
	mov		x1, lr
	bl		_pinst_spsel_1
	mov		lr, x1
#else
	msr		SPSel, #1
#endif
	mov		sp, x0

	// Set the interrupt stack pointer
	ldr		x0, [x21, CPU_INTSTACK_TOP]
	msr		SPSel, #0
	mov		sp, x0

	// Convert lr to KVA
	add		lr, lr, x22
	sub		lr, lr, x23

	b		common_start

/*
 * create_l1_table_entry
 *
 * Given a virtual address, creates a table entry in an L1 translation table
 * to point to an L2 translation table.
 *   arg0 - Virtual address
 *   arg1 - L1 table address
 *   arg2 - L2 table address
 *   arg3 - Scratch register
 *   arg4 - Scratch register
 *   arg5 - Scratch register
 */
.macro create_l1_table_entry
	and		$3,	$0, #(ARM_TT_L1_INDEX_MASK)
	lsr		$3, $3, #(ARM_TT_L1_SHIFT)			// Get index in L1 table for L2 table
	lsl		$3, $3, #(TTE_SHIFT)				// Convert index into pointer offset
	add		$3, $1, $3							// Get L1 entry pointer
	mov		$4, #(ARM_TTE_BOOT_TABLE)			// Get L1 table entry template
	and		$5, $2, #(ARM_TTE_TABLE_MASK)		// Get address bits of L2 table
	orr		$5, $4, $5 							// Create table entry for L2 table
	str		$5, [$3]							// Write entry to L1 table
.endmacro

/*
 * create_l2_block_entries
 *
 * Given base virtual and physical addresses, creates consecutive block entries
 * in an L2 translation table.
 *   arg0 - Virtual address
 *   arg1 - Physical address
 *   arg2 - L2 table address
 *   arg3 - Number of entries
 *   arg4 - Scratch register
 *   arg5 - Scratch register
 *   arg6 - Scratch register
 *   arg7 - Scratch register
 */
.macro create_l2_block_entries
	and		$4,	$0, #(ARM_TT_L2_INDEX_MASK)
	lsr		$4, $4, #(ARM_TTE_BLOCK_L2_SHIFT)	// Get index in L2 table for block entry
	lsl		$4, $4, #(TTE_SHIFT)				// Convert index into pointer offset
	add		$4, $2, $4							// Get L2 entry pointer
	mov		$5, #(ARM_TTE_BOOT_BLOCK)			// Get L2 block entry template
	and		$6, $1, #(ARM_TTE_BLOCK_L2_MASK)	// Get address bits of block mapping
	orr		$6, $5, $6
	mov		$5, $3
	mov		$7, #(ARM_TT_L2_SIZE)
1:
	str		$6, [$4], #(1 << TTE_SHIFT)			// Write entry to L2 table and advance
	add		$6, $6, $7							// Increment the output address
	subs	$5, $5, #1							// Decrement the number of entries
	b.ne	1b
.endmacro

/*
 * _start_first_cpu
 * Cold boot init routine.  Called from __start
 *   x0 - Boot args
 */
	.align 2
	.globl EXT(start_first_cpu)
LEXT(start_first_cpu)

	// Unlock the core for debugging
	msr		OSLAR_EL1, xzr
	mov		x20, x0
	mov		x21, xzr

	// Set low reset vector before attempting any loads
	adrp	x0, EXT(LowExceptionVectorBase)@page
	add		x0, x0, EXT(LowExceptionVectorBase)@pageoff
	MSR_VBAR_EL1_X0



	// Get the kernel memory parameters from the boot args
	ldr		x22, [x20, BA_VIRT_BASE]			// Get the kernel virt base
	ldr		x23, [x20, BA_PHYS_BASE]			// Get the kernel phys base
	ldr		x24, [x20, BA_MEM_SIZE]				// Get the physical memory size
	ldr		x25, [x20, BA_TOP_OF_KERNEL_DATA]	// Get the top of the kernel data

	// Set CPU number to 0
	msr		TPIDRRO_EL0, x21

	// Set up exception stack pointer
	adrp	x0, EXT(excepstack_top)@page		// Load top of exception stack
	add		x0, x0, EXT(excepstack_top)@pageoff
	add		x0, x0, x22							// Convert to KVA
	sub		x0, x0, x23

	// Set SP_EL1 to exception stack
#if defined(KERNEL_INTEGRITY_KTRR)
	bl		_pinst_spsel_1
#else
	msr		SPSel, #1
#endif

	mov		sp, x0

	// Set up interrupt stack pointer
	adrp	x0, EXT(intstack_top)@page			// Load top of irq stack
	add		x0, x0, EXT(intstack_top)@pageoff
	add		x0, x0, x22							// Convert to KVA
	sub		x0, x0, x23
	msr		SPSel, #0							// Set SP_EL0 to interrupt stack
	mov		sp, x0

	// Load address to the C init routine into link register
	adrp	lr, EXT(arm_init)@page
	add		lr, lr, EXT(arm_init)@pageoff
	add		lr, lr, x22							// Convert to KVA
	sub		lr, lr, x23

	/*
	 * Set up the bootstrap page tables with a single block entry for the V=P
	 * mapping, a single block entry for the trampolined kernel address (KVA),
	 * and all else invalid. This requires four pages:
	 *	Page 1 - V=P L1 table
	 *	Page 2 - V=P L2 table
	 *	Page 3 - KVA L1 table
	 *	Page 4 - KVA L2 table
	 */
#if __ARM64_TWO_LEVEL_PMAP__
	/*
	 * If we are using a two level scheme, we don't need the L1 entries, so:
	 *      Page 1 - V=P L2 table
	 *      Page 2 - KVA L2 table
	 */
#endif

	// Invalidate all entries in the bootstrap page tables
	mov		x0, #(ARM_TTE_EMPTY)				// Load invalid entry template
	mov		x1, x25								// Start at top of kernel
	mov		x2, #(TTE_PGENTRIES)				// Load number of entries per page
#if __ARM64_TWO_LEVEL_PMAP__
	lsl		x2, x2, #1							// Shift by 1 for num entries on 2 pages
#else
	lsl		x2, x2, #2							// Shift by 2 for num entries on 4 pages
#endif
	sub		x2, x2, #1							// Subtract one to terminate on last entry
Linvalidate_bootstrap:							// do {
	str		x0, [x1], #(1 << TTE_SHIFT)			//   Invalidate and advance
	subs	x2, x2, #1							//   entries--
	b.ne	Linvalidate_bootstrap				// } while (entries != 0)

	/* Load addresses for page table construction macros
	 *  x0 - Physical base (used to identify V=P section to set up)
	 *	x1 - V=P L1 table base
	 *	x2 - V=P L2 table base
	 *	x3 - KVA L1 table base
	 *	x4 - KVA L2 table base
	 *	x5 - Mem size in entries (up to 1GB)
	 */

	/*
	 * In order to reclaim memory on targets where TZ0 (or some other entity)
	 * must be located at the base of memory, iBoot may set the virtual and
	 * physical base addresses to immediately follow whatever lies at the
	 * base of physical memory.
	 *
	 * If the base address belongs to TZ0, it may be dangerous for xnu to map
	 * it (as it may be prefetched, despite being technically inaccessible).
	 * In order to avoid this issue while keeping the mapping code simple, we
	 * may continue to use block mappings, but we will only map xnu's mach
	 * header to the end of memory.
	 *
	 * Given that iBoot guarantees that the unslid kernelcache base address
	 * will begin on an L2 boundary, this should prevent us from accidentally
	 * mapping TZ0.
	 */
	adrp	x0, EXT(_mh_execute_header)@page	// Use xnu's mach header as the start address
	add		x0, x0, EXT(_mh_execute_header)@pageoff
#if __ARM64_TWO_LEVEL_PMAP__
	/*
	 * We don't need the L1 entries in this case, so skip them.
	 */
	mov		x2, x25								// Load V=P L2 table address
	add		x4, x2, PGBYTES						// Load KVA L2 table address
#else
	mov		x1, x25								// Load V=P L1 table address
	add		x2, x1, PGBYTES						// Load V=P L2 table address
	add		x3, x2, PGBYTES						// Load KVA L1 table address
	add		x4, x3, PGBYTES						// Load KVA L2 table address
#endif
	/*
	 * We must adjust the amount we wish to map in order to account for the
	 * memory preceeding xnu's mach header.
	 */
	sub		x5, x0, x23							// Map from the mach header up to the end of our memory
	sub		x5, x24, x5
	lsr		x5, x5, #(ARM_TT_L2_SHIFT)
	mov		x6, #(TTE_PGENTRIES)				// Load number of L2 entries per page
	cmp		x5, x6								// If memsize requires more than 1 page of entries
	csel	x5, x5, x6, lt						// ... round down to a single page (first 1GB)

#if !__ARM64_TWO_LEVEL_PMAP__
	/* Create entry for L2 table in V=P L1 table
	 * create_l1_table_entry(V=P, L1 table, L2 table, scratch1, scratch2, scratch3)
	 */
	create_l1_table_entry	x0, x1, x2, x10, x11, x12
#endif

	/* Create block entry in V=P L2 table
	 * create_l2_block_entries(V=P virt, V=P phys, L2 table, num_ents, scratch1, scratch2, scratch3)
	 */
	create_l2_block_entries x0, x0, x2, x5, x10, x11, x12, x13

#if !__ARM64_TWO_LEVEL_PMAP__
	/* Create entry for L2 table in KVA L1 table
	 * create_l1_table_entry(virt_base, L1 table, L2 table, scratch1, scratch2, scratch3)
	 */
	create_l1_table_entry	x22, x3, x4, x10, x11, x12
#endif

	/* Create block entries in KVA L2 table
	 * create_l2_block_entries(virt_base, phys_base, L2 table, num_ents, scratch1, scratch2, scratch3)
	 */
	create_l2_block_entries	x22, x23, x4, x5, x10, x11, x12, x13

	/* Ensure TTEs are visible */
	dsb		ish

	b		common_start

/*
 * Begin common CPU initialization
 *
 * Regster state:
 *	x20 - PA of boot args
 *	x21 - zero on cold boot, PA of cpu data on warm reset
 *	x22 - Kernel virtual base
 *	x23 - Kernel physical base
 *	x24	- Physical memory size
 *	x25 - PA of the end of the kernl
 *	 lr - KVA of C init routine
 *	 sp - SP_EL0 selected
 *
 *	SP_EL0 - KVA of CPU's interrupt stack
 *	SP_EL1 - KVA of CPU's exception stack
 *	TPIDRRO_EL0 - CPU number
 */
common_start:
	// Set the translation control register.
	adrp	x0,     EXT(sysreg_restore)@page		// Load TCR value from the system register restore structure
	add		x0, x0, EXT(sysreg_restore)@pageoff
	ldr		x1, [x0, SR_RESTORE_TCR_EL1]
	MSR_TCR_EL1_X1

	/* Set up translation table base registers.
	 *	TTBR0 - V=P table @ top of kernel
	 *	TTBR1 - KVA table @ top of kernel + 2 pages
	 */
#if defined(KERNEL_INTEGRITY_KTRR)
	/* Note that for KTRR configurations, the V=P map will be modified by
	 * arm_vm_init.c.
	 */
#endif
	and		x0, x25, #(TTBR_BADDR_MASK)
	msr		TTBR0_EL1, x0
#if __ARM64_TWO_LEVEL_PMAP__
	/*
	 * If we're using a two level pmap, we'll only need a
	 * single page per bootstrap pmap.
	 */
	mov		x12, #1
#else
	/*
	 * If we're using a three level pmap, we'll need two
	 * pages per bootstrap pmap.
	 */
	mov		x12, #2
#endif
	add		x0, x25, x12, lsl PGSHIFT
	and		x0, x0, #(TTBR_BADDR_MASK)
	MSR_TTBR1_EL1_X0

	// Set up MAIR attr0 for normal memory, attr1 for device memory
	mov		x0, xzr
	mov		x1, #(MAIR_WRITEBACK << MAIR_ATTR_SHIFT(CACHE_ATTRINDX_WRITEBACK))
	orr		x0, x0, x1
	mov		x1, #(MAIR_INNERWRITEBACK << MAIR_ATTR_SHIFT(CACHE_ATTRINDX_INNERWRITEBACK))
	orr		x0, x0, x1
	mov		x1, #(MAIR_DISABLE << MAIR_ATTR_SHIFT(CACHE_ATTRINDX_DISABLE))
	orr		x0, x0, x1
	mov		x1, #(MAIR_WRITETHRU << MAIR_ATTR_SHIFT(CACHE_ATTRINDX_WRITETHRU))
	orr		x0, x0, x1
	mov		x1, #(MAIR_WRITECOMB << MAIR_ATTR_SHIFT(CACHE_ATTRINDX_WRITECOMB))
	orr		x0, x0, x1
	mov		x1, #(MAIR_POSTED << MAIR_ATTR_SHIFT(CACHE_ATTRINDX_POSTED))
	orr		x0, x0, x1
	msr		MAIR_EL1, x0

	// Disable interrupts
	msr     DAIFSet, #(DAIFSC_IRQF | DAIFSC_FIQF)

#if defined(APPLEHURRICANE)

	// <rdar://problem/26726624> Increase Snoop reservation in EDB to reduce starvation risk
	// Needs to be done before MMU is enabled
	mrs	x12, ARM64_REG_HID5
	and	x12, x12, (~ARM64_REG_HID5_CrdEdbSnpRsvd_mask)
	orr x12, x12, ARM64_REG_HID5_CrdEdbSnpRsvd_VALUE
	msr	ARM64_REG_HID5, x12

#endif


#ifndef __ARM_IC_NOALIAS_ICACHE__
	/* Invalidate the TLB and icache on systems that do not guarantee that the
	 * caches are invalidated on reset.
	 */
	tlbi	vmalle1
	ic		iallu
#endif

	/* If x21 is not 0, then this is either the start_cpu path or
	 * the resume_idle_cpu path.  cpu_ttep should already be
	 * populated, so just switch to the kernel_pmap now.
	 */

	cbz		x21, 1f
	adrp	x0, EXT(cpu_ttep)@page
	add		x0, x0, EXT(cpu_ttep)@pageoff
	ldr		x0, [x0]
	MSR_TTBR1_EL1_X0
1:

	// Set up the exception vectors
	adrp	x0, EXT(ExceptionVectorsBase)@page			// Load exception vectors base address
	add		x0, x0, EXT(ExceptionVectorsBase)@pageoff
	add		x0, x0, x22									// Convert exception vector address to KVA
	sub		x0, x0, x23
	MSR_VBAR_EL1_X0


	// Enable caches and MMU
	mov		x0, #(SCTLR_EL1_DEFAULT & 0xFFFF)
	mov		x1, #(SCTLR_EL1_DEFAULT & 0xFFFF0000)
	orr		x0, x0, x1
	MSR_SCTLR_EL1_X0
	isb		sy

#if (!CONFIG_KERNEL_INTEGRITY || (CONFIG_KERNEL_INTEGRITY && !defined(KERNEL_INTEGRITY_WT)))
	/* Watchtower
	 *
	 * If we have a Watchtower monitor it will setup CPACR_EL1 for us, touching
	 * it here would trap to EL3.
	 */

	// Enable NEON
	mov		x0, #(CPACR_FPEN_ENABLE)
	msr		CPACR_EL1, x0
#endif

	// Clear thread pointer
	mov		x0, #0
	msr		TPIDR_EL1, x0						// Set thread register

#if defined(APPLECYCLONE) || defined(APPLETYPHOON)
	//
	// Cyclone/Typhoon-Specific initialization
	// For tunable summary, see <rdar://problem/13503621> Alcatraz/H6: Confirm Cyclone CPU tunables have been set
	//

	//
	// Disable LSP flush with context switch to work around bug in LSP
	// that can cause Cyclone to wedge when CONTEXTIDR is written.
	// <rdar://problem/12387704> Innsbruck11A175: panic(cpu 0 caller 0xffffff800024e30c): "wait queue deadlock - wq=0xffffff805a7a63c0, cpu=0\n"
	//

	mrs		x12, ARM64_REG_HID0
	orr		x12, x12, ARM64_REG_HID0_LoopBuffDisb
	msr		ARM64_REG_HID0, x12
	
	mrs		x12, ARM64_REG_HID1
	orr		x12, x12, ARM64_REG_HID1_rccDisStallInactiveIexCtl
#if defined(APPLECYCLONE)
	orr		x12, x12, ARM64_REG_HID1_disLspFlushWithContextSwitch
#endif
	msr		ARM64_REG_HID1, x12

	mrs		x12, ARM64_REG_HID3
	orr		x12, x12, ARM64_REG_HID3_DisXmonSnpEvictTriggerL2StarvationMode
	msr		ARM64_REG_HID3, x12

	// Do not disable cache ops -- XNU's cache operations already are no-op'ed for Cyclone, but explicit _Force variants are provided
	// for when we really do need the L2 cache to be cleaned: <rdar://problem/14350417> Innsbruck11A416: Panic logs not preserved on h6
/*
	mrs		x12, ARM64_REG_HID4
	orr		x12, x12, ARM64_REG_HID4_DisDcMVAOps
	orr		x12, x12, ARM64_REG_HID4_DisDcSWL2Ops
	msr		ARM64_REG_HID4, x12
*/

	mrs		x12, ARM64_REG_HID5
	and		x12, x12, (~ARM64_REG_HID5_DisHwpLd)
	and		x12, x12, (~ARM64_REG_HID5_DisHwpSt)
	msr		ARM64_REG_HID5, x12

	// Change the default memcache data set ID from 0 to 15 for all agents
	mrs		x12, ARM64_REG_HID8
	orr		x12, x12, (ARM64_REG_HID8_DataSetID0_VALUE | ARM64_REG_HID8_DataSetID1_VALUE)
#if ARM64_BOARD_CONFIG_T7001
	orr		x12, x12, ARM64_REG_HID8_DataSetID2_VALUE
#endif	// ARM64_BOARD_CONFIG_T7001
	msr		ARM64_REG_HID8, x12
	isb		sy
#endif	// APPLECYCLONE || APPLETYPHOON

#if defined(APPLETWISTER)
	mrs 	x12, ARM64_REG_HID11
	and 	x12, x12, (~ARM64_REG_HID11_DisFillC1BubOpt)
	msr 	ARM64_REG_HID11, x12

	// Change the default memcache data set ID from 0 to 15 for all agents
	mrs		x12, ARM64_REG_HID8
	orr		x12, x12, (ARM64_REG_HID8_DataSetID0_VALUE | ARM64_REG_HID8_DataSetID1_VALUE)
	orr 	x12, x12, (ARM64_REG_HID8_DataSetID2_VALUE | ARM64_REG_HID8_DataSetID3_VALUE)
	msr		ARM64_REG_HID8, x12

	// Use 4-cycle MUL latency to avoid denormal stalls
	mrs 	x12, ARM64_REG_HID7
	orr 	x12, x12, #ARM64_REG_HID7_disNexFastFmul
	msr 	ARM64_REG_HID7, x12

	// disable reporting of TLB-multi-hit-error
	// <rdar://problem/22163216> 
	mrs		x12, ARM64_REG_LSU_ERR_STS
	and		x12, x12, (~ARM64_REG_LSU_ERR_STS_L1DTlbMultiHitEN)
	msr		ARM64_REG_LSU_ERR_STS, x12

	isb		sy
#endif	// APPLETWISTER

#if defined(APPLEHURRICANE)

	// IC prefetch configuration
	// <rdar://problem/23019425>
	mrs		x12, ARM64_REG_HID0
	and		x12, x12, (~ARM64_REG_HID0_ICPrefDepth_bmsk)
	orr		x12, x12, (1 << ARM64_REG_HID0_ICPrefDepth_bshift)
	orr		x12, x12, ARM64_REG_HID0_ICPrefLimitOneBrn
	msr		ARM64_REG_HID0, x12

	// disable reporting of TLB-multi-hit-error
	// <rdar://problem/22163216> 
	mrs		x12, ARM64_REG_LSU_ERR_CTL
	and		x12, x12, (~ARM64_REG_LSU_ERR_CTL_L1DTlbMultiHitEN)
	msr		ARM64_REG_LSU_ERR_CTL, x12

	// disable crypto fusion across decode groups
	// <rdar://problem/27306424>
	mrs		x12, ARM64_REG_HID1
	orr		x12, x12, ARM64_REG_HID1_disAESFuseAcrossGrp
	msr		ARM64_REG_HID1, x12

#if defined(ARM64_BOARD_CONFIG_T8011)
	// Clear DisDcZvaCmdOnly 
	// Per Myst A0/B0 tunables document
	// https://seg-docs.csg.apple.com/projects/myst//release/UserManual/tunables_a0/ACC.html
	// <rdar://problem/27627428> Myst: Confirm ACC Per-CPU Tunables
	mrs		x12, ARM64_REG_HID3
	and             x12, x12, ~ARM64_REG_HID3_DisDcZvaCmdOnly
	msr             ARM64_REG_HID3, x12

	mrs		x12, ARM64_REG_EHID3
	and             x12, x12, ~ARM64_REG_EHID3_DisDcZvaCmdOnly
	msr             ARM64_REG_EHID3, x12
#endif /* defined(ARM64_BOARD_CONFIG_T8011) */

#endif // APPLEHURRICANE


	// If x21 != 0, we're doing a warm reset, so we need to trampoline to the kernel pmap.
	cbnz	x21, Ltrampoline

	// Set KVA of boot args as first arg
	add		x0, x20, x22
	sub		x0, x0, x23

#if KASAN
	mov	x20, x0
	mov	x21, lr

	// x0: boot args
	// x1: KVA page table phys base
	mrs	x1, TTBR1_EL1
	bl	_kasan_bootstrap

	mov	x0, x20
	mov	lr, x21
#endif

	// Return to arm_init()
	ret

Ltrampoline:
	// Load VA of the trampoline
	adrp	x0, arm_init_tramp@page
	add		x0, x0, arm_init_tramp@pageoff
	add		x0, x0, x22
	sub		x0, x0, x23

	// Branch to the trampoline
	br		x0

/*
 * V=P to KVA trampoline.
 *	x0 - KVA of cpu data pointer
 */
	.text
	.align 2
arm_init_tramp:
	/* On a warm boot, the full kernel translation table is initialized in
	 * addition to the bootstrap tables. The layout is as follows:
	 *
	 *  +--Top of Memory--+
	 *         ...
	 *  |                 |
	 *  |  Primary Kernel |
	 *  |   Trans. Table  |
	 *  |                 |
	 *  +--Top + 5 pages--+
	 *  |                 |
	 *  |  Invalid Table  |
	 *  |                 |
	 *  +--Top + 4 pages--+
	 *  |                 |
	 *  |    KVA Table    |
	 *  |                 |
	 *  +--Top + 2 pages--+
	 *  |                 |
	 *  |    V=P Table    |
	 *  |                 |
	 *  +--Top of Kernel--+
	 *  |                 |
	 *  |  Kernel Mach-O  |
	 *  |                 |
	 *         ...
	 *  +---Kernel Base---+
	 */


	adrp	x0, EXT(invalid_ttep)@page
	add		x0, x0, EXT(invalid_ttep)@pageoff
	ldr		x0, [x0]

	msr		TTBR0_EL1, x0

	// Convert CPU data PA to VA and set as first argument
	add		x0, x21, x22
	sub		x0, x0, x23
	mov		x1, #0

	// Make sure that the TLB flush happens after the registers are set!
	isb		sy

	// Synchronize system for TTBR updates
	tlbi	vmalle1
	dsb		sy
	isb		sy

	/* Return to arm_init() */
	ret

//#include	"globals_asm.h"

/* vim: set ts=4: */