#include <machine/asm.h>
#include <arm/proc_reg.h>
#include <pexpert/arm/board_config.h>
+#include <mach/arm/traps.h>
#include <mach/exception_types.h>
#include <mach_kdp.h>
#include <mach_assert.h>
#include <config_dtrace.h>
#include "assym.s"
+#include "dwarf_unwind.h"
#define TRACE_SYSCALL 0
adr r4, EXT(ResetHandlerData)
ldr r0, [r4, BOOT_ARGS]
ldr r1, [r4, CPU_DATA_ENTRIES]
-#if __ARM_SMP__
#if defined(ARMA7)
// physical cpu number is stored in MPIDR Affinity level 0
mrc p15, 0, r6, c0, c0, 5 // Read MPIDR
#else
#error missing Who Am I implementation
#endif
-#else
- mov r6, #0
-#endif /* __ARM_SMP__ */
// physical cpu number matches cpu number
//#if cdeSize != 16
//#error cpu_data_entry is not 16bytes in size
.align 2
.globl EXT(fleh_undef)
-LEXT(fleh_undef)
- mrs sp, spsr // Check the previous mode
+/*
+ * Ensures the stack is safely aligned, usually in preparation for an external branch
+ * arg0: temp register for storing the stack offset
+ * arg1: temp register for storing the previous stack pointer
+ */
+.macro ALIGN_STACK
+/*
+ * For armv7k ABI, the stack needs to be 16-byte aligned
+ */
+#if __BIGGEST_ALIGNMENT__ > 4
+ and $0, sp, #0x0F // sp mod 16-bytes
+ cmp $0, #4 // need space for the sp on the stack
+ addlt $0, $0, #0x10 // make room if needed, but keep stack aligned
+ mov $1, sp // get current sp
+ sub sp, sp, $0 // align stack
+ str $1, [sp] // store previous sp on stack
+#endif
+.endmacro
+
+/*
+ * Restores the stack pointer to its previous value following an ALIGN_STACK call
+ */
+.macro UNALIGN_STACK
+#if __BIGGEST_ALIGNMENT__ > 4
+ ldr sp, [sp]
+#endif
+.endmacro
+
+/*
+ * Checks that cpu is currently in the expected mode, panics if not.
+ * arg0: the expected mode, should be one of the PSR_*_MODE defines
+ */
+.macro VERIFY_EXCEPTION_MODE
+ mrs sp, cpsr // Read cpsr
+ and sp, sp, #PSR_MODE_MASK // Extract current mode
+ cmp sp, $0 // Check specified mode
+ movne r0, sp
+ bne EXT(ExceptionVectorPanic)
+.endmacro
+
+/*
+ * Checks previous processor mode. If usermode, will execute the code
+ * following the macro to handle the userspace exception. Otherwise,
+ * will branch to a ELSE_IF_KERNELMODE_EXCEPTION call with the same
+ * argument.
+ * arg0: arbitrary string indicating the exception class, e.g. 'dataabt'
+ */
+.macro IF_USERMODE_EXCEPTION
+ mrs sp, spsr
+ and sp, sp, #PSR_MODE_MASK // Is it from user?
+ cmp sp, #PSR_USER_MODE
+ beq $0_from_user
+ cmp sp, #PSR_IRQ_MODE
+ beq $0_from_irq
+ cmp sp, #PSR_FIQ_MODE
+ beq $0_from_fiq
+ bne $0_from_svc
+$0_from_user:
+.endmacro
+
+/*
+ * Handles an exception taken from kernelmode (IRQ/FIQ/SVC/etc).
+ * Places the processor into the correct mode and executes the
+ * code following the macro to handle the kernel exception.
+ * Intended to be paired with a prior call to IF_USERMODE_EXCEPTION.
+ * arg0: arbitrary string indicating the exception class, e.g. 'dataabt'
+ */
+.macro ELSE_IF_KERNELMODE_EXCEPTION
+$0_from_irq:
+ cpsid i, #PSR_IRQ_MODE
+ b $0_from_kernel
+$0_from_fiq:
+ cpsid i, #PSR_FIQ_MODE
+ b $0_from_kernel
+$0_from_svc:
+ cpsid i, #PSR_SVC_MODE
+$0_from_kernel:
+.endmacro
+
+LEXT(fleh_undef)
+VERIFY_EXCEPTION_MODE PSR_UND_MODE
+ mrs sp, spsr // For check the previous mode
tst sp, #PSR_TF // Is it Thumb?
subeq lr, lr, #4
subne lr, lr, #2
- tst sp, #0x0f // Is it from user?
- bne undef_from_kernel
-
-undef_from_user:
+IF_USERMODE_EXCEPTION undef
mrc p15, 0, sp, c13, c0, 4 // Read TPIDRPRW
add sp, sp, ACT_PCBDATA // Get current thread PCB pointer
stmia sp, {r0-r12, sp, lr}^ // Save user context on PCB
mov r7, #0 // Zero the frame pointer
nop
-
+
mov r0, sp // Store arm_saved_state pointer
// for argument
mrs r4, spsr
str r4, [sp, SS_CPSR] // Save user mode cpsr
- mrs r4, cpsr // Read cpsr
cpsid i, #PSR_SVC_MODE
mrs r3, cpsr // Read cpsr
msr spsr_cxsf, r3 // Set spsr(svc mode cpsr)
mcr p15, 0, r3, c13, c0, 1 // Set CONTEXTIDR
isb
#endif
- and r0, r4, #PSR_MODE_MASK // Extract current mode
- cmp r0, #PSR_UND_MODE // Check undef mode
- bne EXT(ExceptionVectorPanic)
mvn r0, #0
str r0, [r9, TH_IOTIER_OVERRIDE] // Reset IO tier override to -1 before handling abort from userspace
// sleh will enable interrupt
b load_and_go_user
-undef_from_kernel:
- mrs sp, cpsr // Read cpsr
- and sp, sp, #PSR_MODE_MASK // Extract current mode
- cmp sp, #PSR_UND_MODE // Check undef mode
- movne r0, sp
- bne EXT(ExceptionVectorPanic)
- mrs sp, spsr // Check the previous mode
-
+ELSE_IF_KERNELMODE_EXCEPTION undef
/*
* We have a kernel stack already, and I will use it to save contexts
* IRQ is disabled
*/
-
-#if CONFIG_DTRACE
- /*
- * See if we came here from IRQ or SVC mode, and go back to that mode
- */
-
- and sp, sp, #PSR_MODE_MASK
- cmp sp, #PSR_IRQ_MODE
- bne undef_from_kernel_svc
-
- cpsid i, #PSR_IRQ_MODE
- b handle_undef
-#endif
-
-undef_from_kernel_svc:
- cpsid i, #PSR_SVC_MODE
-
-handle_undef:
#if CONFIG_DTRACE
// We need a frame for backtracing. The LR here is the LR of supervisor mode, not the location where the exception
// took place. We'll store that later after we switch to undef mode and pull out the LR from there.
#if CONFIG_DTRACE
add r7, sp, EXC_CTX_SIZE // Save frame pointer
#endif
-
- mov ip, sp // Stack transfer
- cpsid i, #PSR_UND_MODE
+ mrs r4, lr_und
+ str r4, [sp, SS_PC] // Save complete
+ mrs r4, spsr_und
+ str r4, [sp, SS_CPSR]
- str lr, [ip, SS_PC] // Save complete
- mrs r4, spsr
- str r4, [ip, SS_CPSR]
-
-#if CONFIG_DTRACE
- /*
- * Go back to previous mode for mode specific regs
- */
- and r4, r4, #PSR_MODE_MASK
- cmp r4, #PSR_IRQ_MODE
- bne handle_undef_from_svc
-
- cpsid i, #PSR_IRQ_MODE
- b handle_undef2
-#endif
-
-handle_undef_from_svc:
- cpsid i, #PSR_SVC_MODE
-
-handle_undef2:
+ mov ip, sp
/*
sp - stack pointer
#endif
mov r0, sp // Argument
-/*
- * For armv7k ABI, the stack needs to be 16-byte aligned
- */
-#if __BIGGEST_ALIGNMENT__ > 4
- and r1, sp, #0x0F // sp mod 16-bytes
- cmp r1, #4 // need space for the sp on the stack
- addlt r1, r1, #0x10 // make room if needed, but keep stack aligned
- mov r2, sp // get current sp
- sub sp, sp, r1 // align stack
- str r2, [sp] // store previous sp on stack
-#endif
-
+ ALIGN_STACK r2, r3
bl EXT(sleh_undef) // Call second level handler
-
-#if __BIGGEST_ALIGNMENT__ > 4
- ldr sp, [sp] // restore stack
-#endif
+ UNALIGN_STACK
#if __ARM_USER_PROTECT__
mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW
str r0, [sp, SS_SP] // Save supervisor mode sp
str lr, [sp, SS_LR] // Save supervisor mode lr
+ ALIGN_STACK r0, r1
adr r0, L_kernel_swi_panic_str // Load panic messages and panic()
blx EXT(panic)
b .
/* Check for special mach_absolute_time trap value.
* This is intended to be a super-lightweight call to ml_get_timebase(), which
* is handrolled assembly and does not use the stack, thus not requiring us to setup a kernel stack. */
- cmp r12, #-3
+ cmp r12, #MACH_ARM_TRAP_ABSTIME
beq fleh_swi_trap_tb
stmia sp, {r0-r12, sp, lr}^ // Save user context on PCB
mov r7, #0 // Zero the frame pointer
#if !CONFIG_SKIP_PRECISE_USER_KERNEL_TIME
bl EXT(timer_state_event_user_to_kernel)
mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW
- add r8, r9, ACT_PCBDATA // Reload arm_saved_state pointer
+ add r8, r9, ACT_PCBDATA // Reload arm_saved_state pointer
#endif
- ldr r10, [r9, ACT_TASK] // Load the current task
+ ldr r10, [r9, ACT_TASK] // Load the current task
/* enable interrupts */
- cpsie i // Enable IRQ
+ cpsie i // Enable IRQ
- cmp r11, #-4 // Special value for mach_continuous_time
+ cmp r11, #MACH_ARM_TRAP_CONTTIME // Special value for mach_continuous_time
beq fleh_swi_trap_mct
cmp r11, #0x80000000
dsb ish
isb
#endif
- mov r4, r0
- mov r5, r1
- bl EXT(CleanPoU_DcacheRegion)
- mov r0, r4
- mov r1, r5
bl EXT(InvalidatePoU_IcacheRegion)
mrc p15, 0, r9, c13, c0, 4 // Reload r9 from TPIDRPRW
#if __ARM_USER_PROTECT__
mov r1, sp
str r2, [sp]
str r3, [sp, #4]
+ ALIGN_STACK r2, r3
mov r2, #2
bl EXT(exception_triage)
b .
.globl EXT(fleh_prefabt)
LEXT(fleh_prefabt)
+VERIFY_EXCEPTION_MODE PSR_ABT_MODE
sub lr, lr, #4
-
- mrs sp, spsr // For check the previous mode
- tst sp, #0x0f // Is it from user?
- bne prefabt_from_kernel
-prefabt_from_user:
+IF_USERMODE_EXCEPTION prefabt
mrc p15, 0, sp, c13, c0, 4 // Read TPIDRPRW
add sp, sp, ACT_PCBDATA // Get User PCB
mrs r4, spsr
str r4, [sp, SS_CPSR] // Save user mode cpsr
- mrs r4, cpsr // Read cpsr
cpsid i, #PSR_SVC_MODE
mrs r3, cpsr // Read cpsr
msr spsr_cxsf, r3 // Set spsr(svc mode cpsr)
mcr p15, 0, r3, c13, c0, 1 // Set CONTEXTIDR
isb
#endif
- and r0, r4, #PSR_MODE_MASK // Extract current mode
- cmp r0, #PSR_ABT_MODE // Check abort mode
- bne EXT(ExceptionVectorPanic)
mvn r0, #0
str r0, [r9, TH_IOTIER_OVERRIDE] // Reset IO tier override to -1 before handling abort from userspace
// Sleh will enable interrupt
b load_and_go_user
-prefabt_from_kernel:
- mrs sp, cpsr // Read cpsr
- and sp, sp, #PSR_MODE_MASK // Extract current mode
- cmp sp, #PSR_ABT_MODE // Check abort mode
- movne r0, sp
- bne EXT(ExceptionVectorPanic)
- mrs sp, spsr // Check the previous mode
-
+ELSE_IF_KERNELMODE_EXCEPTION prefabt
+
+UNWIND_PROLOGUE
+
/*
* We have a kernel stack already, and I will use it to save contexts:
* ------------------
*
* IRQ is disabled
*/
- cpsid i, #PSR_SVC_MODE
-
sub sp, sp, EXC_CTX_SIZE
stmia sp, {r0-r12}
add r0, sp, EXC_CTX_SIZE
mcr p15, 0, r3, c13, c0, 1 // Set CONTEXTIDR
isb
#endif
- mov ip, sp
- cpsid i, #PSR_ABT_MODE
-
- str lr, [ip, SS_PC] // Save pc to pc and
+ mrs r4, lr_abt
+ str r4, [sp, SS_PC] // Save pc
mrc p15, 0, r5, c6, c0, 2 // Read IFAR
- str r5, [ip, SS_VADDR] // and fault address of pcb
+ str r5, [sp, SS_VADDR] // and fault address of pcb
mrc p15, 0, r5, c5, c0, 1 // Read (instruction) Fault Status
- str r5, [ip, SS_STATUS] // Save fault status register to pcb
-
- mrs r4, spsr
- str r4, [ip, SS_CPSR]
+ str r5, [sp, SS_STATUS] // Save fault status register to pcb
- cpsid i, #PSR_SVC_MODE
+ mrs r4, spsr_abt
+ str r4, [sp, SS_CPSR]
- mov r0, sp
+ mov r0, sp
+ ALIGN_STACK r1, r2
+ mov r1, T_PREFETCH_ABT // Pass abort type
-/*
- * For armv7k ABI, the stack needs to be 16-byte aligned
- */
-#if __BIGGEST_ALIGNMENT__ > 4
- and r1, sp, #0x0F // sp mod 16-bytes
- cmp r1, #4 // need space for the sp on the stack
- addlt r1, r1, #0x10 // make room if needed, but keep stack aligned
- mov r2, sp // get current sp
- sub sp, sp, r1 // align stack
- str r2, [sp] // store previous sp on stack
-#endif
- mov r1, T_PREFETCH_ABT // Pass abort type
+UNWIND_DIRECTIVES
+
bl EXT(sleh_abort) // Call second level handler
-
-#if __BIGGEST_ALIGNMENT__ > 4
- ldr sp, [sp] // restore stack
-#endif
+ UNALIGN_STACK
mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW
#if __ARM_USER_PROTECT__
b load_and_go_sys
+UNWIND_EPILOGUE
/*
* First Level Exception Handler for Data Abort
.globl EXT(fleh_dataabt)
LEXT(fleh_dataabt)
+VERIFY_EXCEPTION_MODE PSR_ABT_MODE
sub lr, lr, #8
-
- mrs sp, spsr // For check the previous mode
- tst sp, #0x0f // Is it from kernel?
- bne dataabt_from_kernel
-
-dataabt_from_user:
+IF_USERMODE_EXCEPTION dataabt
mrc p15, 0, sp, c13, c0, 4 // Read TPIDRPRW
add sp, sp, ACT_PCBDATA // Get User PCB
str r5, [sp, SS_STATUS] // Save fault status register to pcb
str r6, [sp, SS_VADDR] // Save fault address to pcb
- mrs r4, cpsr // Read cpsr
cpsid i, #PSR_SVC_MODE
mrs r3, cpsr // Read cpsr
msr spsr_cxsf, r3 // Set spsr(svc mode cpsr)
mcr p15, 0, r3, c13, c0, 1 // Set CONTEXTIDR
isb
#endif
- and r0, r4, #PSR_MODE_MASK // Extract current mode
- cmp r0, #PSR_ABT_MODE // Check abort mode
- bne EXT(ExceptionVectorPanic)
mvn r0, #0
str r0, [r9, TH_IOTIER_OVERRIDE] // Reset IO tier override to -1 before handling abort from userspace
// Sleh will enable irq
b load_and_go_user
-dataabt_from_kernel:
- mrs sp, cpsr // Read cpsr
- and sp, sp, #PSR_MODE_MASK // Extract current mode
- cmp sp, #PSR_ABT_MODE // Check abort mode
- movne r0, sp
- bne EXT(ExceptionVectorPanic)
- mrs sp, spsr // Check the previous mode
+ELSE_IF_KERNELMODE_EXCEPTION dataabt
+UNWIND_PROLOGUE
+
/*
* We have a kernel stack already, and I will use it to save contexts:
* ------------------
*
* IRQ is disabled
*/
- cpsid i, #PSR_SVC_MODE
-
sub sp, sp, EXC_CTX_SIZE
stmia sp, {r0-r12}
add r0, sp, EXC_CTX_SIZE
fmxr fpscr, r4 // And shove it into FPSCR
#endif
- mov ip, sp
-
- cpsid i, #PSR_ABT_MODE
-
- str lr, [ip, SS_PC]
- mrs r4, spsr
- str r4, [ip, SS_CPSR]
-
- cpsid i, #PSR_SVC_MODE
+ mrs r4, lr_abt
+ str r4, [sp, SS_PC]
+ mrs r4, spsr_abt
+ str r4, [sp, SS_CPSR]
#if __ARM_USER_PROTECT__
mrc p15, 0, r10, c2, c0, 0 // Get TTBR0
str r6, [sp, SS_VADDR] // Save fault address to pcb
mov r0, sp // Argument
-
-/*
- * For armv7k ABI, the stack needs to be 16-byte aligned
- */
-#if __BIGGEST_ALIGNMENT__ > 4
- and r1, sp, #0x0F // sp mod 16-bytes
- cmp r1, #4 // need space for the sp on the stack
- addlt r1, r1, #0x10 // make room if needed, but keep stack aligned
- mov r2, sp // get current sp
- sub sp, sp, r1 // align stack
- str r2, [sp] // store previous sp on stack
-#endif
-
+ ALIGN_STACK r1, r2
mov r1, T_DATA_ABT // Pass abort type
- bl EXT(sleh_abort) // Call second level handler
-#if __BIGGEST_ALIGNMENT__ > 4
- ldr sp, [sp] // restore stack (removed align padding)
-#endif
+UNWIND_DIRECTIVES
+
+ bl EXT(sleh_abort) // Call second level handler
+ UNALIGN_STACK
mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW
#if __ARM_USER_PROTECT__
ldr lr, [sp, SS_LR] // Restore the link register
stmfd sp!, {r7, lr} // Push a fake frame
- /* TODO: Should this be setting r7? I think so. */
- mov r7, sp // Set the frame pointer
-
-#if __BIGGEST_ALIGNMENT__ > 4
- and r2, sp, #0x0F // sp mod 16-bytes
- cmp r2, #4 // need space for the sp on the stack
- addlt r2, r2, #0x10 // make room if needed, but keep stack aligned
- mov r3, sp // get current sp
- sub sp, sp, r2 // align stack
- str r3, [sp] // store previous sp on stack
-#endif
-
+ ALIGN_STACK r2, r3
bl EXT(ast_taken_kernel) // Handle AST_URGENT
-
-#if __BIGGEST_ALIGNMENT__ > 4
- ldr sp, [sp]
-#endif
-
+ UNALIGN_STACK
ldmfd sp!, {r7, lr} // Pop the fake frame
mrc p15, 0, r9, c13, c0, 4 // Reload r9 from TPIDRPRW
ldmia sp, {r0-r12} // Restore other registers
movs pc, lr // Return to sys (svc, irq, fiq)
-
+
+UNWIND_EPILOGUE
+
/*
* First Level Exception Handler for address exception
* Not supported
ldr r2, [r2]
movs r2, r2
beq 1f
+ mov r1, #0 // (not a PMI record)
bl EXT(telemetry_mark_curthread) // ...if so, mark the current thread...
mrc p15, 0, r9, c13, c0, 4 // ...and restore the thread pointer from TPIDRPRW
1:
ldr r2, [r2]
movs r2, r2
beq 1f
+ mov r1, #0 // (not a PMI record)
bl EXT(telemetry_mark_curthread) // ...if so, mark the current thread...
mrc p15, 0, r9, c13, c0, 4 // ...and restore the thread pointer from TPIDRPRW
1:
mrc p15, 0, r9, c13, c0, 4 // Reload r9 from TPIDRPRW
bl EXT(ml_get_timebase) // get current timebase
LOAD_ADDR(r3, EntropyData)
- ldr r2, [r3, ENTROPY_INDEX_PTR]
- add r1, r3, ENTROPY_DATA_SIZE
- add r2, r2, #4
- cmp r2, r1
- addge r2, r3, ENTROPY_BUFFER
- ldr r4, [r2]
- eor r0, r0, r4, ROR #9
- str r0, [r2] // Update gEntropie
- str r2, [r3, ENTROPY_INDEX_PTR]
-
+ ldr r1, [r3, ENTROPY_SAMPLE_COUNT]
+ ldr r2, [r3, ENTROPY_BUFFER_INDEX_MASK]
+ add r4, r1, 1
+ and r5, r1, r2
+ str r4, [r3, ENTROPY_SAMPLE_COUNT]
+ ldr r1, [r3, ENTROPY_BUFFER]
+ ldr r2, [r3, ENTROPY_BUFFER_ROR_MASK]
+ ldr r4, [r1, r5, lsl #2]
+ and r4, r4, r2
+ eor r0, r0, r4, ror #9
+ str r0, [r1, r5, lsl #2]
return_from_irq:
mov r5, #0
ldr r4, [r9, ACT_CPUDATAP] // Get current cpu
ldr r2, [r2]
movs r2, r2
beq 1f
+ mov r1, #0 // (not a PMI record)
bl EXT(telemetry_mark_curthread) // ...if so, mark the current thread...
mrc p15, 0, r9, c13, c0, 4 // ...and restore the thread pointer from TPIDRPRW
1:
ldr r2, [r2]
movs r2, r2
beq 1f
+ mov r1, #0 // (not a pmi record)
bl EXT(telemetry_mark_curthread) // ...if so, mark the current thread...
mrc p15, 0, r9, c13, c0, 4 // ...and restore the thread pointer from TPIDRPRW
1:
mcr p15, 0, r3, c13, c0, 1 // Set CONTEXTIDR
isb
#endif
+
+ ALIGN_STACK r0, r1
mov r0, r8 // Get current cpu in arg 0
mov r1, SIGPdec // Decrementer signal in arg1
mov r2, #0
mov r3, #0
bl EXT(cpu_signal) // Call cpu_signal
+ UNALIGN_STACK
mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW
cpsid i, #PSR_IRQ_MODE
cpsie f
mov sp, r6 // Restore the stack pointer
+ ALIGN_STACK r2, r3
msr spsr_cxsf, r4 // Restore the spsr
ldr r2, [r9, ACT_PREEMPT_CNT] // Load preemption count
add r2, r2, #1 // Increment count
#if CONFIG_TELEMETRY
LOAD_ADDR(r4, telemetry_needs_record) // Check if a telemetry record was requested...
ldr r4, [r4]
- movs r4, r4
+ movs r4, r4
beq 6f
+ mov r1, #0 // (not a PMI record)
bl EXT(telemetry_mark_curthread) // ...if so, mark the current thread...
mrc p15, 0, r9, c13, c0, 4 // ...and restore the thread pointer from TPIDRPRW
6:
movs r4, r4
COND_EXTERN_BLNE(interrupt_trace_exit)
#endif
+ UNALIGN_STACK
mrc p15, 0, r9, c13, c0, 4 // Reload r9 from TPIDRPRW
cmp r5, #0 // Test if ASTs pending
beq return_to_user_now // Branch if no ASTs
-#if __BIGGEST_ALIGNMENT__ > 4
- and r2, sp, #0x0F // sp mod 16-bytes
- cmp r2, #4 // need space for the sp on the stack
- addlt r2, r2, #0x10 // make room if needed, but keep stack aligned
- mov r3, sp // get current sp
- sub sp, sp, r2 // align stack
- str r3, [sp] // store previous sp on stack
-#endif
-
bl EXT(ast_taken_user) // Handle all ASTs (may continue via thread_exception_return)
-#if __BIGGEST_ALIGNMENT__ > 4
- ldr sp, [sp] // Restore the stack pointer
-#endif
-
mrc p15, 0, r9, c13, c0, 4 // Reload r9 from TPIDRPRW
b load_and_go_user // Loop back
/*
* Assert that the preemption level is zero prior to the return to user space
*/
- ldr r1, [r9, ACT_PREEMPT_CNT] // Load preemption count
- movs r1, r1 // Test
- beq 0f // Continue if zero, or...
- adr r0, L_lagu_panic_str // Load the panic string...
- blx EXT(panic) // Finally, panic
-0:
- ldr r2, [r9, TH_RWLOCK_CNT] // Load RW lock count
- movs r2, r2 // Test
- beq 0f // Continue if zero, or...
- adr r0, L_lagu_rwlock_cnt_panic_str // Load the panic string...
- mov r1, r9 // Thread argument for panic string
- blx EXT(panic) // Finally, panic
+ ldr r1, [r9, ACT_PREEMPT_CNT] // Load preemption count
+ cmp r1, #0 // Test
+ bne L_lagu_preempt_panic // Panic if not zero
+
+/*
+ * Assert that the preemption level is zero prior to the return to user space
+ */
+ ldr r2, [r9, TH_RWLOCK_CNT] // Load RW lock count
+ cmp r2, #0 // Test
+ bne L_lagu_rwlock_cnt_panic // Panic if not zero
#endif
-0:
+/*
+ * Assert that we aren't leaking KHEAP_TEMP allocations prior to the return to user space
+ */
+ ldr r1, [r9, TH_TMP_ALLOC_CNT] // Load temp alloc count
+ cmp r1, #0 // Test
+ bne L_lagu_temp_alloc_cnt_panic // Panic if not zero
+
#if !CONFIG_SKIP_PRECISE_USER_KERNEL_TIME
bl EXT(timer_state_event_kernel_to_user)
mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW
nop // Hardware problem
movs pc, lr // Return to user
+/*
+ * r1: tmp alloc count
+ * r9: current_thread()
+ */
+L_lagu_temp_alloc_cnt_panic:
+ mov r0, r9 // Thread argument
+ blx EXT(kheap_temp_leak_panic) // Finally, panic
+
+#if MACH_ASSERT
+/*
+ * r1: current preemption count
+ * r9: current_thread()
+ */
+L_lagu_preempt_panic:
+ adr r0, L_lagu_preempt_panic_str // Load the panic string...
+ blx EXT(panic) // Finally, panic
+
+/*
+ * r2: rwlock count
+ * r9: current_thread()
+ */
+L_lagu_rwlock_cnt_panic:
+ adr r0, L_lagu_rwlock_cnt_panic_str // Load the panic string...
+ mov r1, r9 // Thread argument for panic string
+ blx EXT(panic) // Finally, panic
+
.align 2
-L_lagu_panic_str:
+L_lagu_preempt_panic_str:
.asciz "load_and_go_user: preemption_level %d"
.align 2
L_lagu_rwlock_cnt_panic_str:
.asciz "load_and_go_user: RW lock count not 0 on thread %p (%u)"
.align 2
+#endif /* MACH_ASSERT */
- .align 2
+ .align 2
L_evimpanic_str:
- .ascii "Exception Vector: Illegal Mode: 0x%08X\n\000"
- .align 2
+ .ascii "Exception Vector: Illegal Mode: 0x%08X\n\000"
+ .align 2
.text
.align 2
LEXT(ExceptionVectorPanic)
cpsid i, #PSR_SVC_MODE
+ ALIGN_STACK r1, r2
mov r1, r0
adr r0, L_evimpanic_str
blx EXT(panic)
projects / apple / xnu.git / blobdiff