[apple/xnu.git] / osfmk / arm / start.s

/*
 * Copyright (c) 2007-2014 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 *
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */

#include <arm/asm.h>
#include <arm/proc_reg.h>
#include <mach_kdp.h>
#include "assym.s"

	.text
	.align 12

	.align 2
	.globl EXT(resume_idle_cpu)
LEXT(resume_idle_cpu)
	// r0 set to BootArgs phys address
	// r1 set to cpu data phys address
	LOAD_ADDR(lr, arm_init_idle_cpu)
	b		L_start_cpu_0

	.globl EXT(start_cpu)
LEXT(start_cpu)
	// r0 set to BootArgs phys address
	// r1 set to cpu data phys address
	LOAD_ADDR(lr, arm_init_cpu)
	b		L_start_cpu_0

L_start_cpu_0:
	cpsid	if									// Disable IRQ FIQ

	// Turn on L1 I-Cache, Branch prediction early
	mcr		p15, 0, r11, c7, c5, 0				// invalidate the icache
	isb											// before moving on
	mrc		p15, 0, r11, c1, c0, 0				// read mmu control into r11
	orr		r11, r11, #(SCTLR_ICACHE | SCTLR_PREDIC)	// enable i-cache, b-prediction
	mcr		p15, 0, r11, c1, c0, 0				// set mmu control
	dsb											// ensure mmu settings are inplace
	isb											// before moving on

	// Get the kernel's phys & virt addr, and size from BootArgs
	ldr		r8, [r0, BA_PHYS_BASE]				// Get the phys base in r8
	ldr		r9, [r0, BA_VIRT_BASE]				// Get the virt base in r9
	ldr		r10, [r0, BA_MEM_SIZE]				// Get the mem size in r10

	// Set the base of the translation table into the MMU
	ldr		r4, [r0, BA_TOP_OF_KERNEL_DATA]		// Get the top of kernel data
	orr		r5, r4, #(TTBR_SETUP & 0x00FF)		// Setup PTWs memory attribute
	orr		r5, r5, #(TTBR_SETUP & 0xFF00)		// Setup PTWs memory attribute
	mcr		p15, 0, r5, c2, c0, 0				// write kernel to translation table base 0
	mcr		p15, 0, r5, c2, c0, 1				// also to translation table base 1
	mov		r5, #TTBCR_N_SETUP					// identify the split between 0 and 1
	mcr		p15, 0, r5, c2, c0, 2				// and set up the translation control reg
	ldr		r2, [r1, CPU_NUMBER_GS]				// Get cpu number
	mcr		p15, 0, r2, c13, c0, 3				// Write TPIDRURO
	ldr		sp, [r1, CPU_INTSTACK_TOP]			// Get interrupt stack top
	sub		sp, sp, SS_SIZE						// Set stack pointer
	sub		r0, r1, r8							// Convert to virtual address
	add		r0, r0, r9
	b		join_start

	.align 2
	.globl EXT(_start)
LEXT(_start)
	// r0 has the boot-args pointer 
	// r1 set to zero
	mov		r1, #0
	LOAD_ADDR(lr, arm_init)
	cpsid	if									// Disable IRQ FIQ

	// Turn on L1 I-Cache, Branch prediction early
	mcr		p15, 0, r11, c7, c5, 0				// invalidate the icache
	isb											// before moving on
	mrc		p15, 0, r11, c1, c0, 0				// read mmu control into r11
	orr		r11, r11, #(SCTLR_ICACHE | SCTLR_PREDIC)	// enable i-cache, b-prediction
	mcr		p15, 0, r11, c1, c0, 0				// set mmu control
	dsb											// ensure mmu settings are inplace
	isb											// before moving on

	// Get the kernel's phys & virt addr, and size from boot_args.
	ldr		r8, [r0, BA_PHYS_BASE]				// Get the phys base in r8
	ldr		r9, [r0, BA_VIRT_BASE]				// Get the virt base in r9
	ldr		r10, [r0, BA_MEM_SIZE]				// Get the mem size in r10

#define LOAD_PHYS_ADDR(reg, label) \
	LOAD_ADDR(reg, label); \
	sub		reg, reg, r9; \
	add		reg, reg, r8

	// Take this opportunity to patch the targets for the exception vectors
	LOAD_ADDR(r4, fleh_reset)
	LOAD_PHYS_ADDR(r5, ExceptionVectorsTable)
	str		r4, [r5]
	LOAD_ADDR(r4, fleh_undef)
	add		r5, #4
	str		r4, [r5]
	LOAD_ADDR(r4, fleh_swi)
	add		r5, #4
	str		r4, [r5]
	LOAD_ADDR(r4, fleh_prefabt)
	add		r5, #4
	str		r4, [r5]
	LOAD_ADDR(r4, fleh_dataabt)
	add		r5, #4
	str		r4, [r5]
	LOAD_ADDR(r4, fleh_addrexc)
	add		r5, #4
	str		r4, [r5]
	LOAD_ADDR(r4, fleh_irq)
	add		r5, #4
	str		r4, [r5]
	LOAD_ADDR(r4, fleh_decirq)
	add		r5, #4
	str		r4, [r5]

	// arm_init_tramp is sensitive, so for the moment, take the opportunity to store the
	// virtual address locally, so that we don't run into issues retrieving it later.
	// This is a pretty miserable solution, but it should be enough for the moment
	LOAD_ADDR(r4, arm_init_tramp)
	adr		r5, arm_init_tramp_addr
	str		r4, [r5]

#undef LOAD_PHYS_ADDR

	// Set the base of the translation table into the MMU
	ldr		r4, [r0, BA_TOP_OF_KERNEL_DATA]		// Get the top of kernel data
	orr		r5, r4, #(TTBR_SETUP & 0x00FF)		// Setup PTWs memory attribute
	orr		r5, r5, #(TTBR_SETUP & 0xFF00)		// Setup PTWs memory attribute
	mcr		p15, 0, r5, c2, c0, 0				// write kernel to translation table base 0
	mcr		p15, 0, r5, c2, c0, 1				// also to translation table base 1
	mov		r5, #TTBCR_N_SETUP					// identify the split between 0 and 1
	mcr		p15, 0, r5, c2, c0, 2				// and set up the translation control reg
		
	// Mark the entries invalid in the 4 page trampoline translation table
	// Mark the entries invalid in the 4 page CPU translation table
	// Mark the entries invalid in the one page table for the final 1MB (if used)
	// Mark the entries invalid in the one page table for HIGH_EXC_VECTORS
	mov		r5, r4								// local copy of base
	mov		r11, #ARM_TTE_TYPE_FAULT			// invalid entry template
	mov		r2, PGBYTES >> 2					// number of ttes/page
	add		r2, r2, r2, LSL #2					// 8 ttes + 2 ptes to clear. Multiply by 5...
	mov		r2, r2, LSL #1						// ...then multiply by 2
invalidate_tte:
	str		r11, [r5]							// store the invalid tte
	add		r5, r5, #4							// increment tte pointer
	subs	r2, r2, #1							// decrement count
	bne		invalidate_tte

	// create default section tte template
	mov		r6, #ARM_TTE_TYPE_BLOCK				// use block mapping entries
	mov		r7, #(ARM_TTE_BLOCK_ATTRINDX(CACHE_ATTRINDX_DEFAULT) & 0xFF)
	orr		r7, r7, #(ARM_TTE_BLOCK_ATTRINDX(CACHE_ATTRINDX_DEFAULT) & 0xFF00)
	orr		r7, r7, #(ARM_TTE_BLOCK_ATTRINDX(CACHE_ATTRINDX_DEFAULT) & 0xF0000)
	orr		r6, r6, r7							// with default cache attrs
	mov		r7, #ARM_TTE_BLOCK_AP(AP_RWNA)		// Set kernel rw, user no access
	orr		r7, r7, #(ARM_TTE_BLOCK_AP(AP_RWNA) & 0xFF00)
	orr		r7, r7, #(ARM_TTE_BLOCK_AP(AP_RWNA) & 0xF0000)
	orr		r6, r6, r7							// Set RWNA protection 

	orr		r6, r6, #ARM_TTE_BLOCK_AF			// Set access protection 
	orr		r6, r6, #ARM_TTE_BLOCK_SH			// Set shareability

	// Set up the V=P mapping for the 1 MB section around the current pc
	lsr		r7, pc, #ARM_TT_L1_SHIFT			// Extract tte index for pc addr
	add		r5, r4, r7, LSL #2					// convert tte index to tte pointer
	lsl		r7, r7, #ARM_TT_L1_SHIFT			// Truncate pc to 1MB aligned addr
	orr		r11, r7, r6							// make tte entry value
	str		r11, [r5]							// store tte

	// Set up the virtual mapping for the kernel using 1Mb direct section TTE entries
	mov		r7, r8								// Save original phys base
	add		r5, r4, r9, LSR #ARM_TT_L1_SHIFT-2	// convert vaddr to tte pointer
	mov		r3, #ARM_TT_L1_SIZE					// set 1MB boundary
	
mapveqp:
	cmp		r3, r10								// Check if we're beyond the last 1MB section
	bgt		mapveqpL2							// If so, a coarse entry is required

	orr		r11, r7, r6							// make tte entry value
	str		r11, [r5], #4						// store tte and move to next
	add		r7, r7, #ARM_TT_L1_SIZE				// move to next phys addr
	subs	r10, r10, #ARM_TT_L1_SIZE			// subtract tte size
	bne		mapveqp
	b		doneveqp							// end is 1MB aligned, and we're done
	
mapveqpL2:
	// The end is not 1MB aligned, so steal a page and set up L2 entries within
	
	// Coarse entry first
	add		r6, r4, PGBYTES * 8					// add L2 offset
	mov 		r11, r6
	
	orr		r6, #ARM_TTE_TYPE_TABLE				// coarse entry
	
	str		r6, [r5]							// store coarse tte entry	
	
	// Fill in the L2 entries
	mov 		r5, r11
	
	// create pte template
	mov		r2, #ARM_PTE_TYPE					// default pte type
	orr		r2, r2, #(ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DEFAULT) & 0xff)	// with default cache attrs
	orr		r2, r2, #(ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DEFAULT) & 0xff00)
	orr		r2, r2, #(ARM_PTE_AP(AP_RWNA) & 0xff)	// with default cache attrs
	orr		r2, r2, #(ARM_PTE_AP(AP_RWNA) & 0xff00)
	orr		r2, r2, #ARM_PTE_AF					// Set access 
	orr		r2, r2, #ARM_PTE_SH					// Set shareability 
	
storepte:
	orr		r11, r7, r2							// make pte entry value
	str		r11, [r5], #4						// store pte and move to next
	add		r7, r7,  PGBYTES					// move to next phys addr
	subs	r10, r10, PGBYTES					// subtract pte size
	bne		storepte

doneveqp:
	// Insert page table page for high address exception vectors into translation table
	mov		r5, #0xff000000						// part of virt HIGH_EXC_VECTORS (HACK!)
	orr		r5, r5, #0x00ff0000					// rest of virt HIGH_EXC_VECTORS (HACK!)
	mov		r5, r5, LSR #ARM_TT_L1_SHIFT		// convert virt addr to index
	add		r5, r4, r5, LSL #2					// convert to tte pointer

	add		r6, r4, PGBYTES * 9					// get page table base (past 4 + 4 + 1 tte/pte pages)
	add		r6, r6, #0xc00						// adjust to last 1MB section
	mov		r7, #(ARM_TTE_TABLE_MASK & 0xFFFF) 	// ARM_TTE_TABLE_MASK low halfword
	movt		r7, #(ARM_TTE_TABLE_MASK >> 16)		// ARM_TTE_TABLE_MASK top halfword 
	and		r11, r6, r7							// apply mask
	orr		r11, r11, #ARM_TTE_TYPE_TABLE		// mark it as a coarse page table
	str		r11, [r5]							// store tte entry for page table

	// create pte template
	mov		r2, #ARM_PTE_TYPE					// pte type
	orr		r2, r2, #(ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DEFAULT) & 0x00ff)	// default cache attrs
	orr		r2, r2, #(ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DEFAULT) & 0xff00)
	orr		r2, r2, #(ARM_PTE_AP(AP_RWNA) & 0x00ff)	// set  RWNA protection
	orr		r2, r2, #(ARM_PTE_AP(AP_RWNA) & 0xff00)
	orr		r2, r2, #ARM_PTE_AF					// Set access 
	orr		r2, r2, #ARM_PTE_SH					// Set shareability 

	// Now initialize the page table entry for the exception vectors
	mov		r5, #0xff000000						// part of HIGH_EXC_VECTORS
	orr		r5, r5, #0x00ff0000					// rest of HIGH_EXC_VECTORS
	mov		r7, #(ARM_TT_L2_INDEX_MASK & 0xFFFF) // ARM_TT_L2_INDEX_MASK low halfword
	movt	r7, #(ARM_TT_L2_INDEX_MASK >> 16)	// ARM_TT_L2_INDEX_MASK top halfword 
	and		r5, r5, r7 							// mask for getting index 
	mov		r5, r5, LSR #ARM_TT_L2_SHIFT		// get page table index
	add		r5, r6, r5, LSL #2					// convert to pte pointer

	LOAD_ADDR(r11, ExceptionVectorsBase)		// get address of vectors addr
	sub		r11, r11, r9						// convert to physical address
	add		r11, r11, r8

	mov		r7, #(ARM_PTE_PAGE_MASK & 0xFFFF) 	// ARM_PTE_PAGE_MASK low halfword
	movt	r7, #(ARM_PTE_PAGE_MASK >> 16)		// ARM_PTE_PAGE_MASK top halfword 
	and		r11, r11, r7						// insert masked address into pte
	orr		r11, r11, r2						// add template bits
	str		r11, [r5]							// store pte by base and index

	// clean the dcache
	mov		r11, #0
cleanflushway:
cleanflushline:		
	mcr		p15, 0, r11, c7, c14, 2				 // cleanflush dcache line by way/set
	add		r11, r11, #1 << MMU_I7SET			 // increment set index
	tst		r11, #1 << (MMU_NSET + MMU_I7SET)	 // look for overflow
	beq		cleanflushline
	bic		r11, r11, #1 << (MMU_NSET + MMU_I7SET) // clear set overflow
	adds	r11, r11, #1 << MMU_I7WAY			 // increment way
	bcc		cleanflushway				 		 // loop

#if	__ARM_L2CACHE__
	// Invalidate L2 cache
	mov		r11, #2
invall2flushway:
invall2flushline:		
	mcr		p15, 0, r11, c7, c14, 2				 // Invalidate dcache line by way/set
	add		r11, r11, #1 << L2_I7SET			 // increment set index
	tst		r11, #1 << (L2_NSET + L2_I7SET)		 // look for overflow
	beq		invall2flushline
	bic		r11, r11, #1 << (L2_NSET + L2_I7SET) // clear set overflow
	adds	r11, r11, #1 << L2_I7WAY			 // increment way
	bcc		invall2flushway				 		 // loop

#endif

	mov		r11, #0
	mcr		p15, 0, r11, c13, c0, 3				// Write TPIDRURO
	LOAD_ADDR(sp, intstack_top)					// Get interrupt stack top
	sub		sp, sp, SS_SIZE						// Set stack pointer
	sub		r0, r0, r8							// Convert to virtual address
	add		r0, r0, r9

join_start:
	// kernel page table is setup
	// lr set to return handler function virtual address
	// r0 set to return handler argument virtual address
	// sp set to interrupt context stack virtual address

	// Cpu specific configuration

#ifdef  ARMA7
#if	 __ARMA7_SMP__
	mrc		p15, 0, r11, c1, c0, 1
	orr		r11, r11, #(1<<6)						// SMP
	mcr		p15, 0, r11, c1, c0, 1
	isb
#endif
#endif

	mrs		r11, cpsr							// Get cpsr
	bic		r11, #0x100							// Allow async aborts
	msr		cpsr_x, r11							// Update cpsr

	mov		r11, #0
	mcr		p15, 0, r11, c8, c7, 0				// invalidate all TLB entries
	mcr		p15, 0, r11, c7, c5, 0				// invalidate the icache

	// set DACR
	mov		r11, #(ARM_DAC_SETUP & 0xFFFF) 		// ARM_DAC_SETUP low halfword
	movt	r11, #(ARM_DAC_SETUP >> 16)			// ARM_DAC_SETUP top halfword 
	mcr		p15, 0, r11, c3, c0, 0				// write to dac register

	// Set PRRR
	mov		r11, #(PRRR_SETUP & 0xFFFF) 		// PRRR_SETUP low halfword
	movt	r11, #(PRRR_SETUP >> 16)			// PRRR_SETUP top halfword 
	mcr		p15, 0, r11, c10,c2,0				// write to PRRR register

	// Set NMRR
	mov		r11, #(NMRR_SETUP & 0xFFFF)			// NMRR_SETUP low halfword
	movt	r11, #(NMRR_SETUP >> 16)			// NMRR_SETUP top halfword 
	mcr		p15, 0, r11, c10,c2,1				// write to NMRR register

	// set SCTLR
	mrc		p15, 0, r11, c1, c0, 0				// read  system control

	bic		r11, r11, #SCTLR_ALIGN				// force off alignment exceptions
	mov		r7, #(SCTLR_AFE|SCTLR_TRE)			// Access flag, TEX remap
	orr		r7, r7, #(SCTLR_HIGHVEC | SCTLR_ICACHE | SCTLR_PREDIC)
	orr		r7, r7, #(SCTLR_DCACHE | SCTLR_ENABLE)
#if  (__ARM_ENABLE_SWAP__ == 1)
	orr		r7, r7, #SCTLR_SW					// SWP/SWPB Enable
#endif
	orr		r11, r11, r7						// or in the default settings
	mcr		p15, 0, r11, c1, c0, 0				// set mmu control

	dsb											// ensure mmu settings are inplace
	isb											// before moving on

#if __ARM_VFP__
	// Initialize the VFP coprocessors.
	mrc		p15, 0, r2, c1, c0, 2				// read coprocessor control register
	mov		r3, #15								// 0xF
	orr		r2, r2, r3, LSL #20					// enable 10 and 11
	mcr		p15, 0, r2, c1, c0, 2				// write coprocessor control register
	isb
#endif	/* __ARM_VFP__ */
		
	// Running virtual.  Prepare to call init code
	cmp		r1, #0								// Test if invoked from start
	beq		join_start_1						// Branch if yes
	ldr		r7, arm_init_tramp_addr				// Load trampoline address
	bx		r7									// Branch to virtual trampoline address

	// Loading the virtual address for arm_init_tramp is a rather ugly
	// problem.  There is probably a better solution, but for the moment,
	// patch the address in locally so that loading it is trivial
arm_init_tramp_addr:
	.long	0
	.globl EXT(arm_init_tramp)
LEXT(arm_init_tramp)
	mrc		p15, 0, r5, c2, c0, 0				// Read to translation table base 0
	add		r5, r5, PGBYTES * 4 				// get kernel page table base (past 4 boot tte pages)
	mcr		p15, 0, r5, c2, c0, 0				// write kernel to translation table base 0
	mcr		p15, 0, r5, c2, c0, 1				// also to translation table base 1
	isb
	mov		r5, #0
	mcr		p15, 0, r5, c8, c7, 0				// Flush all TLB entries
	dsb											// ensure mmu settings are inplace
	isb											// before moving on

join_start_1:
#if __ARM_VFP__
	// Enable VFP for the bootstrap thread context.
	// VFP is enabled for the arm_init path as we may
	// execute VFP code before we can handle an undef.
	fmrx	r2, fpexc							// get fpexc
	orr		r2, #FPEXC_EN						// set the enable bit
	fmxr	fpexc, r2							// set fpexc
	mov		r2, #FPSCR_DEFAULT					// set default fpscr
	fmxr	fpscr, r2							// set fpscr
#endif	/* __ARM_VFP__ */

	mov		r7, #0								// Set stack frame 0
	bx		lr

LOAD_ADDR_GEN_DEF(arm_init)
LOAD_ADDR_GEN_DEF(arm_init_cpu)
LOAD_ADDR_GEN_DEF(arm_init_idle_cpu)
LOAD_ADDR_GEN_DEF(arm_init_tramp)
LOAD_ADDR_GEN_DEF(fleh_reset)
LOAD_ADDR_GEN_DEF(ExceptionVectorsTable)
LOAD_ADDR_GEN_DEF(fleh_undef)
LOAD_ADDR_GEN_DEF(fleh_swi)
LOAD_ADDR_GEN_DEF(fleh_prefabt)
LOAD_ADDR_GEN_DEF(fleh_dataabt)
LOAD_ADDR_GEN_DEF(fleh_addrexc)
LOAD_ADDR_GEN_DEF(fleh_irq)
LOAD_ADDR_GEN_DEF(fleh_decirq)

#include "globals_asm.h"

/* vim: set ts=4: */
Commit	Line	Data
5ba3f43e A	1	/*
	2	* Copyright (c) 2007-2014 Apple Inc. All rights reserved.
	3	*
	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
	5	*
	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
	14	*
	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
	25	*
	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
	27	*/
	28
	29	#include <arm/asm.h>
	30	#include <arm/proc_reg.h>
	31	#include <mach_kdp.h>
	32	#include "assym.s"
	33
	34	.text
	35	.align 12
	36
	37	.align 2
	38	.globl EXT(resume_idle_cpu)
	39	LEXT(resume_idle_cpu)
	40	// r0 set to BootArgs phys address
	41	// r1 set to cpu data phys address
	42	LOAD_ADDR(lr, arm_init_idle_cpu)
	43	b L_start_cpu_0
	44
	45	.globl EXT(start_cpu)
	46	LEXT(start_cpu)
	47	// r0 set to BootArgs phys address
	48	// r1 set to cpu data phys address
	49	LOAD_ADDR(lr, arm_init_cpu)
	50	b L_start_cpu_0
	51
	52	L_start_cpu_0:
	53	cpsid if // Disable IRQ FIQ
	54
	55	// Turn on L1 I-Cache, Branch prediction early
	56	mcr p15, 0, r11, c7, c5, 0 // invalidate the icache
	57	isb // before moving on
	58	mrc p15, 0, r11, c1, c0, 0 // read mmu control into r11
	59	orr r11, r11, #(SCTLR_ICACHE \| SCTLR_PREDIC) // enable i-cache, b-prediction
	60	mcr p15, 0, r11, c1, c0, 0 // set mmu control
	61	dsb // ensure mmu settings are inplace
	62	isb // before moving on
	63
	64	// Get the kernel's phys & virt addr, and size from BootArgs
65	ldr r8, [r0, BA_PHYS_BASE] // Get the phys base in r8
66	ldr r9, [r0, BA_VIRT_BASE] // Get the virt base in r9
67	ldr r10, [r0, BA_MEM_SIZE] // Get the mem size in r10
68
69	// Set the base of the translation table into the MMU
70	ldr r4, [r0, BA_TOP_OF_KERNEL_DATA] // Get the top of kernel data
71	orr r5, r4, #(TTBR_SETUP & 0x00FF) // Setup PTWs memory attribute
72	orr r5, r5, #(TTBR_SETUP & 0xFF00) // Setup PTWs memory attribute
73	mcr p15, 0, r5, c2, c0, 0 // write kernel to translation table base 0
74	mcr p15, 0, r5, c2, c0, 1 // also to translation table base 1
f427ee49	75	mov r5, #TTBCR_N_SETUP // identify the split between 0 and 1
5ba3f43e A	76	mcr p15, 0, r5, c2, c0, 2 // and set up the translation control reg
	77	ldr r2, [r1, CPU_NUMBER_GS] // Get cpu number
	78	mcr p15, 0, r2, c13, c0, 3 // Write TPIDRURO
	79	ldr sp, [r1, CPU_INTSTACK_TOP] // Get interrupt stack top
	80	sub sp, sp, SS_SIZE // Set stack pointer
	81	sub r0, r1, r8 // Convert to virtual address
	82	add r0, r0, r9
	83	b join_start
	84
	85	.align 2
	86	.globl EXT(_start)
	87	LEXT(_start)
	88	// r0 has the boot-args pointer
	89	// r1 set to zero
	90	mov r1, #0
	91	LOAD_ADDR(lr, arm_init)
	92	cpsid if // Disable IRQ FIQ
	93
	94	// Turn on L1 I-Cache, Branch prediction early
	95	mcr p15, 0, r11, c7, c5, 0 // invalidate the icache
	96	isb // before moving on
	97	mrc p15, 0, r11, c1, c0, 0 // read mmu control into r11
	98	orr r11, r11, #(SCTLR_ICACHE \| SCTLR_PREDIC) // enable i-cache, b-prediction
	99	mcr p15, 0, r11, c1, c0, 0 // set mmu control
	100	dsb // ensure mmu settings are inplace
	101	isb // before moving on
	102
	103	// Get the kernel's phys & virt addr, and size from boot_args.
	104	ldr r8, [r0, BA_PHYS_BASE] // Get the phys base in r8
	105	ldr r9, [r0, BA_VIRT_BASE] // Get the virt base in r9
	106	ldr r10, [r0, BA_MEM_SIZE] // Get the mem size in r10
	107
	108	#define LOAD_PHYS_ADDR(reg, label) \
	109	LOAD_ADDR(reg, label); \
	110	sub reg, reg, r9; \
	111	add reg, reg, r8
	112
	113	// Take this opportunity to patch the targets for the exception vectors
	114	LOAD_ADDR(r4, fleh_reset)
	115	LOAD_PHYS_ADDR(r5, ExceptionVectorsTable)
	116	str r4, [r5]
	117	LOAD_ADDR(r4, fleh_undef)
	118	add r5, #4
	119	str r4, [r5]
	120	LOAD_ADDR(r4, fleh_swi)
	121	add r5, #4
	122	str r4, [r5]
	123	LOAD_ADDR(r4, fleh_prefabt)
	124	add r5, #4
	125	str r4, [r5]
	126	LOAD_ADDR(r4, fleh_dataabt)
	127	add r5, #4
	128	str r4, [r5]
	129	LOAD_ADDR(r4, fleh_addrexc)
	130	add r5, #4
	131	str r4, [r5]
	132	LOAD_ADDR(r4, fleh_irq)
	133	add r5, #4
	134	str r4, [r5]
	135	LOAD_ADDR(r4, fleh_decirq)
	136	add r5, #4
	137	str r4, [r5]
	138
	139	// arm_init_tramp is sensitive, so for the moment, take the opportunity to store the
140	// virtual address locally, so that we don't run into issues retrieving it later.
141	// This is a pretty miserable solution, but it should be enough for the moment
142	LOAD_ADDR(r4, arm_init_tramp)
143	adr r5, arm_init_tramp_addr
144	str r4, [r5]
145
146	#undef LOAD_PHYS_ADDR
147
148	// Set the base of the translation table into the MMU
149	ldr r4, [r0, BA_TOP_OF_KERNEL_DATA] // Get the top of kernel data
150	orr r5, r4, #(TTBR_SETUP & 0x00FF) // Setup PTWs memory attribute
151	orr r5, r5, #(TTBR_SETUP & 0xFF00) // Setup PTWs memory attribute
152	mcr p15, 0, r5, c2, c0, 0 // write kernel to translation table base 0
153	mcr p15, 0, r5, c2, c0, 1 // also to translation table base 1
f427ee49	154	mov r5, #TTBCR_N_SETUP // identify the split between 0 and 1
5ba3f43e A	155	mcr p15, 0, r5, c2, c0, 2 // and set up the translation control reg
	156
	157	// Mark the entries invalid in the 4 page trampoline translation table
	158	// Mark the entries invalid in the 4 page CPU translation table
	159	// Mark the entries invalid in the one page table for the final 1MB (if used)
	160	// Mark the entries invalid in the one page table for HIGH_EXC_VECTORS
	161	mov r5, r4 // local copy of base
	162	mov r11, #ARM_TTE_TYPE_FAULT // invalid entry template
	163	mov r2, PGBYTES >> 2 // number of ttes/page
	164	add r2, r2, r2, LSL #2 // 8 ttes + 2 ptes to clear. Multiply by 5...
	165	mov r2, r2, LSL #1 // ...then multiply by 2
	166	invalidate_tte:
	167	str r11, [r5] // store the invalid tte
	168	add r5, r5, #4 // increment tte pointer
	169	subs r2, r2, #1 // decrement count
	170	bne invalidate_tte
	171
	172	// create default section tte template
	173	mov r6, #ARM_TTE_TYPE_BLOCK // use block mapping entries
	174	mov r7, #(ARM_TTE_BLOCK_ATTRINDX(CACHE_ATTRINDX_DEFAULT) & 0xFF)
	175	orr r7, r7, #(ARM_TTE_BLOCK_ATTRINDX(CACHE_ATTRINDX_DEFAULT) & 0xFF00)
	176	orr r7, r7, #(ARM_TTE_BLOCK_ATTRINDX(CACHE_ATTRINDX_DEFAULT) & 0xF0000)
	177	orr r6, r6, r7 // with default cache attrs
	178	mov r7, #ARM_TTE_BLOCK_AP(AP_RWNA) // Set kernel rw, user no access
	179	orr r7, r7, #(ARM_TTE_BLOCK_AP(AP_RWNA) & 0xFF00)
	180	orr r7, r7, #(ARM_TTE_BLOCK_AP(AP_RWNA) & 0xF0000)
	181	orr r6, r6, r7 // Set RWNA protection
	182
	183	orr r6, r6, #ARM_TTE_BLOCK_AF // Set access protection
	184	orr r6, r6, #ARM_TTE_BLOCK_SH // Set shareability
	185
	186	// Set up the V=P mapping for the 1 MB section around the current pc
	187	lsr r7, pc, #ARM_TT_L1_SHIFT // Extract tte index for pc addr
	188	add r5, r4, r7, LSL #2 // convert tte index to tte pointer
	189	lsl r7, r7, #ARM_TT_L1_SHIFT // Truncate pc to 1MB aligned addr
	190	orr r11, r7, r6 // make tte entry value
	191	str r11, [r5] // store tte
	192
	193	// Set up the virtual mapping for the kernel using 1Mb direct section TTE entries
	194	mov r7, r8 // Save original phys base
	195	add r5, r4, r9, LSR #ARM_TT_L1_SHIFT-2 // convert vaddr to tte pointer
	196	mov r3, #ARM_TT_L1_SIZE // set 1MB boundary
	197
	198	mapveqp:
	199	cmp r3, r10 // Check if we're beyond the last 1MB section
	200	bgt mapveqpL2 // If so, a coarse entry is required
	201
	202	orr r11, r7, r6 // make tte entry value
	203	str r11, [r5], #4 // store tte and move to next
	204	add r7, r7, #ARM_TT_L1_SIZE // move to next phys addr
	205	subs r10, r10, #ARM_TT_L1_SIZE // subtract tte size
	206	bne mapveqp
	207	b doneveqp // end is 1MB aligned, and we're done
	208
	209	mapveqpL2:
	210	// The end is not 1MB aligned, so steal a page and set up L2 entries within
	211
	212	// Coarse entry first
	213	add r6, r4, PGBYTES * 8 // add L2 offset
	214	mov r11, r6
	215
	216	orr r6, #ARM_TTE_TYPE_TABLE // coarse entry
	217
	218	str r6, [r5] // store coarse tte entry
219
220	// Fill in the L2 entries
221	mov r5, r11
222
223	// create pte template
224	mov r2, #ARM_PTE_TYPE // default pte type
225	orr r2, r2, #(ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DEFAULT) & 0xff) // with default cache attrs
226	orr r2, r2, #(ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DEFAULT) & 0xff00)
227	orr r2, r2, #(ARM_PTE_AP(AP_RWNA) & 0xff) // with default cache attrs
228	orr r2, r2, #(ARM_PTE_AP(AP_RWNA) & 0xff00)
229	orr r2, r2, #ARM_PTE_AF // Set access
230	orr r2, r2, #ARM_PTE_SH // Set shareability
231
232	storepte:
233	orr r11, r7, r2 // make pte entry value
234	str r11, [r5], #4 // store pte and move to next
235	add r7, r7, PGBYTES // move to next phys addr
236	subs r10, r10, PGBYTES // subtract pte size
237	bne storepte
238
239	doneveqp:
240	// Insert page table page for high address exception vectors into translation table
241	mov r5, #0xff000000 // part of virt HIGH_EXC_VECTORS (HACK!)
242	orr r5, r5, #0x00ff0000 // rest of virt HIGH_EXC_VECTORS (HACK!)
243	mov r5, r5, LSR #ARM_TT_L1_SHIFT // convert virt addr to index
244	add r5, r4, r5, LSL #2 // convert to tte pointer
245
246	add r6, r4, PGBYTES * 9 // get page table base (past 4 + 4 + 1 tte/pte pages)
cb323159	247	add r6, r6, #0xc00 // adjust to last 1MB section
5ba3f43e	248	mov r7, #(ARM_TTE_TABLE_MASK & 0xFFFF) // ARM_TTE_TABLE_MASK low halfword
cb323159	249	movt r7, #(ARM_TTE_TABLE_MASK >> 16) // ARM_TTE_TABLE_MASK top halfword
5ba3f43e A	250	and r11, r6, r7 // apply mask
	251	orr r11, r11, #ARM_TTE_TYPE_TABLE // mark it as a coarse page table
	252	str r11, [r5] // store tte entry for page table
	253
	254	// create pte template
	255	mov r2, #ARM_PTE_TYPE // pte type
	256	orr r2, r2, #(ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DEFAULT) & 0x00ff) // default cache attrs
	257	orr r2, r2, #(ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DEFAULT) & 0xff00)
	258	orr r2, r2, #(ARM_PTE_AP(AP_RWNA) & 0x00ff) // set RWNA protection
	259	orr r2, r2, #(ARM_PTE_AP(AP_RWNA) & 0xff00)
	260	orr r2, r2, #ARM_PTE_AF // Set access
	261	orr r2, r2, #ARM_PTE_SH // Set shareability
	262
	263	// Now initialize the page table entry for the exception vectors
	264	mov r5, #0xff000000 // part of HIGH_EXC_VECTORS
	265	orr r5, r5, #0x00ff0000 // rest of HIGH_EXC_VECTORS
	266	mov r7, #(ARM_TT_L2_INDEX_MASK & 0xFFFF) // ARM_TT_L2_INDEX_MASK low halfword
	267	movt r7, #(ARM_TT_L2_INDEX_MASK >> 16) // ARM_TT_L2_INDEX_MASK top halfword
	268	and r5, r5, r7 // mask for getting index
	269	mov r5, r5, LSR #ARM_TT_L2_SHIFT // get page table index
	270	add r5, r6, r5, LSL #2 // convert to pte pointer
	271
	272	LOAD_ADDR(r11, ExceptionVectorsBase) // get address of vectors addr
	273	sub r11, r11, r9 // convert to physical address
	274	add r11, r11, r8
	275
	276	mov r7, #(ARM_PTE_PAGE_MASK & 0xFFFF) // ARM_PTE_PAGE_MASK low halfword
	277	movt r7, #(ARM_PTE_PAGE_MASK >> 16) // ARM_PTE_PAGE_MASK top halfword
	278	and r11, r11, r7 // insert masked address into pte
	279	orr r11, r11, r2 // add template bits
	280	str r11, [r5] // store pte by base and index
	281
	282	// clean the dcache
	283	mov r11, #0
	284	cleanflushway:
	285	cleanflushline:
	286	mcr p15, 0, r11, c7, c14, 2 // cleanflush dcache line by way/set
2a1bd2d3 A	287	add r11, r11, #1 << MMU_I7SET // increment set index
2a1bd2d3 A	288	tst r11, #1 << (MMU_NSET + MMU_I7SET) // look for overflow
5ba3f43e	289	beq cleanflushline
2a1bd2d3 A	290	bic r11, r11, #1 << (MMU_NSET + MMU_I7SET) // clear set overflow
2a1bd2d3 A	291	adds r11, r11, #1 << MMU_I7WAY // increment way
5ba3f43e	292	bcc cleanflushway // loop
2a1bd2d3 A	293
2a1bd2d3 A	294	#if __ARM_L2CACHE__
5ba3f43e A	295	// Invalidate L2 cache
	296	mov r11, #2
	297	invall2flushway:
	298	invall2flushline:
	299	mcr p15, 0, r11, c7, c14, 2 // Invalidate dcache line by way/set
2a1bd2d3 A	300	add r11, r11, #1 << L2_I7SET // increment set index
2a1bd2d3 A	301	tst r11, #1 << (L2_NSET + L2_I7SET) // look for overflow
5ba3f43e	302	beq invall2flushline
2a1bd2d3 A	303	bic r11, r11, #1 << (L2_NSET + L2_I7SET) // clear set overflow
2a1bd2d3 A	304	adds r11, r11, #1 << L2_I7WAY // increment way
5ba3f43e	305	bcc invall2flushway // loop
2a1bd2d3 A	306
	307	#endif
	308
5ba3f43e A	309	mov r11, #0
	310	mcr p15, 0, r11, c13, c0, 3 // Write TPIDRURO
	311	LOAD_ADDR(sp, intstack_top) // Get interrupt stack top
	312	sub sp, sp, SS_SIZE // Set stack pointer
	313	sub r0, r0, r8 // Convert to virtual address
	314	add r0, r0, r9
	315
	316	join_start:
	317	// kernel page table is setup
	318	// lr set to return handler function virtual address
	319	// r0 set to return handler argument virtual address
	320	// sp set to interrupt context stack virtual address
	321
	322	// Cpu specific configuration
	323
	324	#ifdef ARMA7
	325	#if __ARMA7_SMP__
	326	mrc p15, 0, r11, c1, c0, 1
	327	orr r11, r11, #(1<<6) // SMP
	328	mcr p15, 0, r11, c1, c0, 1
	329	isb
	330	#endif
	331	#endif
	332
	333	mrs r11, cpsr // Get cpsr
	334	bic r11, #0x100 // Allow async aborts
	335	msr cpsr_x, r11 // Update cpsr
	336
	337	mov r11, #0
	338	mcr p15, 0, r11, c8, c7, 0 // invalidate all TLB entries
	339	mcr p15, 0, r11, c7, c5, 0 // invalidate the icache
	340
	341	// set DACR
	342	mov r11, #(ARM_DAC_SETUP & 0xFFFF) // ARM_DAC_SETUP low halfword
	343	movt r11, #(ARM_DAC_SETUP >> 16) // ARM_DAC_SETUP top halfword
	344	mcr p15, 0, r11, c3, c0, 0 // write to dac register
	345
	346	// Set PRRR
	347	mov r11, #(PRRR_SETUP & 0xFFFF) // PRRR_SETUP low halfword
	348	movt r11, #(PRRR_SETUP >> 16) // PRRR_SETUP top halfword
	349	mcr p15, 0, r11, c10,c2,0 // write to PRRR register
	350
	351	// Set NMRR
	352	mov r11, #(NMRR_SETUP & 0xFFFF) // NMRR_SETUP low halfword
	353	movt r11, #(NMRR_SETUP >> 16) // NMRR_SETUP top halfword
	354	mcr p15, 0, r11, c10,c2,1 // write to NMRR register
	355
	356	// set SCTLR
	357	mrc p15, 0, r11, c1, c0, 0 // read system control
	358
	359	bic r11, r11, #SCTLR_ALIGN // force off alignment exceptions
	360	mov r7, #(SCTLR_AFE\|SCTLR_TRE) // Access flag, TEX remap
	361	orr r7, r7, #(SCTLR_HIGHVEC \| SCTLR_ICACHE \| SCTLR_PREDIC)
	362	orr r7, r7, #(SCTLR_DCACHE \| SCTLR_ENABLE)
	363	#if (__ARM_ENABLE_SWAP__ == 1)
	364	orr r7, r7, #SCTLR_SW // SWP/SWPB Enable
	365	#endif
	366	orr r11, r11, r7 // or in the default settings
	367	mcr p15, 0, r11, c1, c0, 0 // set mmu control
	368
	369	dsb // ensure mmu settings are inplace
	370	isb // before moving on
	371
	372	#if __ARM_VFP__
373	// Initialize the VFP coprocessors.
374	mrc p15, 0, r2, c1, c0, 2 // read coprocessor control register
375	mov r3, #15 // 0xF
376	orr r2, r2, r3, LSL #20 // enable 10 and 11
377	mcr p15, 0, r2, c1, c0, 2 // write coprocessor control register
378	isb
379	#endif /* __ARM_VFP__ */
380
381	// Running virtual. Prepare to call init code
382	cmp r1, #0 // Test if invoked from start
383	beq join_start_1 // Branch if yes
384	ldr r7, arm_init_tramp_addr // Load trampoline address
385	bx r7 // Branch to virtual trampoline address
386
387	// Loading the virtual address for arm_init_tramp is a rather ugly
388	// problem. There is probably a better solution, but for the moment,
389	// patch the address in locally so that loading it is trivial
390	arm_init_tramp_addr:
391	.long 0
392	.globl EXT(arm_init_tramp)
393	LEXT(arm_init_tramp)
394	mrc p15, 0, r5, c2, c0, 0 // Read to translation table base 0
395	add r5, r5, PGBYTES * 4 // get kernel page table base (past 4 boot tte pages)
396	mcr p15, 0, r5, c2, c0, 0 // write kernel to translation table base 0
397	mcr p15, 0, r5, c2, c0, 1 // also to translation table base 1
398	isb
399	mov r5, #0
400	mcr p15, 0, r5, c8, c7, 0 // Flush all TLB entries
401	dsb // ensure mmu settings are inplace
402	isb // before moving on
403
404	join_start_1:
405	#if __ARM_VFP__
406	// Enable VFP for the bootstrap thread context.
407	// VFP is enabled for the arm_init path as we may
408	// execute VFP code before we can handle an undef.
409	fmrx r2, fpexc // get fpexc
410	orr r2, #FPEXC_EN // set the enable bit
411	fmxr fpexc, r2 // set fpexc
412	mov r2, #FPSCR_DEFAULT // set default fpscr
413	fmxr fpscr, r2 // set fpscr
414	#endif /* __ARM_VFP__ */
415
416	mov r7, #0 // Set stack frame 0
417	bx lr
418
419	LOAD_ADDR_GEN_DEF(arm_init)
420	LOAD_ADDR_GEN_DEF(arm_init_cpu)
421	LOAD_ADDR_GEN_DEF(arm_init_idle_cpu)
422	LOAD_ADDR_GEN_DEF(arm_init_tramp)
423	LOAD_ADDR_GEN_DEF(fleh_reset)
424	LOAD_ADDR_GEN_DEF(ExceptionVectorsTable)
425	LOAD_ADDR_GEN_DEF(fleh_undef)
426	LOAD_ADDR_GEN_DEF(fleh_swi)
427	LOAD_ADDR_GEN_DEF(fleh_prefabt)
428	LOAD_ADDR_GEN_DEF(fleh_dataabt)
429	LOAD_ADDR_GEN_DEF(fleh_addrexc)
430	LOAD_ADDR_GEN_DEF(fleh_irq)
431	LOAD_ADDR_GEN_DEF(fleh_decirq)
432
433	#include "globals_asm.h"
434
435	/* vim: set ts=4: */