]> git.saurik.com Git - apple/xnu.git/blob - osfmk/arm/locore.s
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-7195.60.75.tar.gz
[apple/xnu.git] / osfmk / arm / locore.s
1 /*
2 * Copyright (c) 2007-2011 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56
57 #include <machine/asm.h>
58 #include <arm/proc_reg.h>
59 #include <pexpert/arm/board_config.h>
60 #include <mach/arm/traps.h>
61 #include <mach/exception_types.h>
62 #include <mach_kdp.h>
63 #include <mach_assert.h>
64 #include <config_dtrace.h>
65 #include "assym.s"
66 #include "dwarf_unwind.h"
67
68 #define TRACE_SYSCALL 0
69
70 /*
71 * Copied to low physical memory in arm_init,
72 * so the kernel must be linked virtually at
73 * 0xc0001000 or higher to leave space for it.
74 */
75 .syntax unified
76 .text
77 .align 12
78 .globl EXT(ExceptionLowVectorsBase)
79
80 LEXT(ExceptionLowVectorsBase)
81 adr pc, Lreset_low_vector
82 b . // Undef
83 b . // SWI
84 b . // Prefetch Abort
85 b . // Data Abort
86 b . // Address Exception
87 b . // IRQ
88 b . // FIQ/DEC
89 LEXT(ResetPrivateData)
90 .space (480),0 // (filled with 0s)
91 // ExceptionLowVectorsBase + 0x200
92 Lreset_low_vector:
93 adr r4, EXT(ResetHandlerData)
94 ldr r0, [r4, ASSIST_RESET_HANDLER]
95 movs r0, r0
96 blxne r0
97 adr r4, EXT(ResetHandlerData)
98 ldr r1, [r4, CPU_DATA_ENTRIES]
99 ldr r1, [r1, CPU_DATA_PADDR]
100 ldr r5, [r1, CPU_RESET_ASSIST]
101 movs r5, r5
102 blxne r5
103 adr r4, EXT(ResetHandlerData)
104 ldr r0, [r4, BOOT_ARGS]
105 ldr r1, [r4, CPU_DATA_ENTRIES]
106 #if defined(ARMA7)
107 // physical cpu number is stored in MPIDR Affinity level 0
108 mrc p15, 0, r6, c0, c0, 5 // Read MPIDR
109 and r6, r6, #0xFF // Extract Affinity level 0
110 #else
111 #error missing Who Am I implementation
112 #endif
113 // physical cpu number matches cpu number
114 //#if cdeSize != 16
115 //#error cpu_data_entry is not 16bytes in size
116 //#endif
117 lsl r6, r6, #4 // Get CpuDataEntry offset
118 add r1, r1, r6 // Get cpu_data_entry pointer
119 ldr r1, [r1, CPU_DATA_PADDR]
120 ldr r5, [r1, CPU_RESET_HANDLER]
121 movs r5, r5
122 blxne r5 // Branch to cpu reset handler
123 b . // Unexpected reset
124 .globl EXT(ResetHandlerData)
125 LEXT(ResetHandlerData)
126 .space (rhdSize_NUM),0 // (filled with 0s)
127
128
129 .globl EXT(ExceptionLowVectorsEnd)
130 LEXT(ExceptionLowVectorsEnd)
131
132 .text
133 .align 12
134 .globl EXT(ExceptionVectorsBase)
135
136 LEXT(ExceptionVectorsBase)
137
138 adr pc, Lexc_reset_vector
139 adr pc, Lexc_undefined_inst_vector
140 adr pc, Lexc_swi_vector
141 adr pc, Lexc_prefetch_abort_vector
142 adr pc, Lexc_data_abort_vector
143 adr pc, Lexc_address_exception_vector
144 adr pc, Lexc_irq_vector
145 #if __ARM_TIME__
146 adr pc, Lexc_decirq_vector
147 #else /* ! __ARM_TIME__ */
148 mov pc, r9
149 #endif /* __ARM_TIME__ */
150
151 Lexc_reset_vector:
152 b .
153 .long 0x0
154 .long 0x0
155 .long 0x0
156 Lexc_undefined_inst_vector:
157 mrc p15, 0, sp, c13, c0, 4 // Read TPIDRPRW
158 ldr sp, [sp, ACT_CPUDATAP] // Get current cpu data
159 ldr sp, [sp, CPU_EXC_VECTORS] // Get exception vector table
160 ldr pc, [sp, #4] // Branch to exception handler
161 Lexc_swi_vector:
162 mrc p15, 0, sp, c13, c0, 4 // Read TPIDRPRW
163 ldr sp, [sp, ACT_CPUDATAP] // Get current cpu data
164 ldr sp, [sp, CPU_EXC_VECTORS] // Get exception vector table
165 ldr pc, [sp, #8] // Branch to exception handler
166 Lexc_prefetch_abort_vector:
167 mrc p15, 0, sp, c13, c0, 4 // Read TPIDRPRW
168 ldr sp, [sp, ACT_CPUDATAP] // Get current cpu data
169 ldr sp, [sp, CPU_EXC_VECTORS] // Get exception vector table
170 ldr pc, [sp, #0xC] // Branch to exception handler
171 Lexc_data_abort_vector:
172 mrc p15, 0, sp, c13, c0, 4 // Read TPIDRPRW
173 ldr sp, [sp, ACT_CPUDATAP] // Get current cpu data
174 ldr sp, [sp, CPU_EXC_VECTORS] // Get exception vector table
175 ldr pc, [sp, #0x10] // Branch to exception handler
176 Lexc_address_exception_vector:
177 mrc p15, 0, sp, c13, c0, 4 // Read TPIDRPRW
178 ldr sp, [sp, ACT_CPUDATAP] // Get current cpu data
179 ldr sp, [sp, CPU_EXC_VECTORS] // Get exception vector table
180 ldr pc, [sp, #0x14] // Branch to exception handler
181 Lexc_irq_vector:
182 mrc p15, 0, sp, c13, c0, 4 // Read TPIDRPRW
183 ldr sp, [sp, ACT_CPUDATAP] // Get current cpu data
184 ldr sp, [sp, CPU_EXC_VECTORS] // Get exception vector table
185 ldr pc, [sp, #0x18] // Branch to exception handler
186 #if __ARM_TIME__
187 Lexc_decirq_vector:
188 mrc p15, 0, sp, c13, c0, 4 // Read TPIDRPRW
189 ldr sp, [sp, ACT_CPUDATAP] // Get current cpu data
190 ldr sp, [sp, CPU_EXC_VECTORS] // Get exception vector table
191 ldr pc, [sp, #0x1C] // Branch to exception handler
192 #else /* ! __ARM_TIME__ */
193 .long 0x0
194 .long 0x0
195 .long 0x0
196 .long 0x0
197 #endif /* __ARM_TIME__ */
198
199 .fill 984, 4, 0 // Push to the 4KB page boundary
200
201 .globl EXT(ExceptionVectorsEnd)
202 LEXT(ExceptionVectorsEnd)
203
204
205 /*
206 * Targets for the exception vectors; we patch these during boot (to allow
207 * for position independent code without complicating the vectors; see start.s).
208 */
209 .globl EXT(ExceptionVectorsTable)
210 LEXT(ExceptionVectorsTable)
211 Lreset_vector:
212 .long 0x0
213 Lundefined_inst_vector:
214 .long 0x0
215 Lswi_vector:
216 .long 0x0
217 Lprefetch_abort_vector:
218 .long 0x0
219 Ldata_abort_vector:
220 .long 0x0
221 Laddress_exception_vector:
222 .long 0x0
223 Lirq_vector:
224 .long 0x0
225 Ldecirq_vector:
226 .long 0x0
227
228
229 /*
230 * First Level Exception Handlers
231 */
232 .text
233 .align 2
234 .globl EXT(fleh_reset)
235 LEXT(fleh_reset)
236 b . // Never return
237
238 /*
239 * First Level Exception Handler for Undefined Instruction.
240 */
241 .text
242 .align 2
243 .globl EXT(fleh_undef)
244
245 /*
246 * Ensures the stack is safely aligned, usually in preparation for an external branch
247 * arg0: temp register for storing the stack offset
248 * arg1: temp register for storing the previous stack pointer
249 */
250 .macro ALIGN_STACK
251 /*
252 * For armv7k ABI, the stack needs to be 16-byte aligned
253 */
254 #if __BIGGEST_ALIGNMENT__ > 4
255 and $0, sp, #0x0F // sp mod 16-bytes
256 cmp $0, #4 // need space for the sp on the stack
257 addlt $0, $0, #0x10 // make room if needed, but keep stack aligned
258 mov $1, sp // get current sp
259 sub sp, sp, $0 // align stack
260 str $1, [sp] // store previous sp on stack
261 #endif
262 .endmacro
263
264 /*
265 * Restores the stack pointer to its previous value following an ALIGN_STACK call
266 */
267 .macro UNALIGN_STACK
268 #if __BIGGEST_ALIGNMENT__ > 4
269 ldr sp, [sp]
270 #endif
271 .endmacro
272
273 /*
274 * Checks that cpu is currently in the expected mode, panics if not.
275 * arg0: the expected mode, should be one of the PSR_*_MODE defines
276 */
277 .macro VERIFY_EXCEPTION_MODE
278 mrs sp, cpsr // Read cpsr
279 and sp, sp, #PSR_MODE_MASK // Extract current mode
280 cmp sp, $0 // Check specified mode
281 movne r0, sp
282 bne EXT(ExceptionVectorPanic)
283 .endmacro
284
285 /*
286 * Checks previous processor mode. If usermode, will execute the code
287 * following the macro to handle the userspace exception. Otherwise,
288 * will branch to a ELSE_IF_KERNELMODE_EXCEPTION call with the same
289 * argument.
290 * arg0: arbitrary string indicating the exception class, e.g. 'dataabt'
291 */
292 .macro IF_USERMODE_EXCEPTION
293 mrs sp, spsr
294 and sp, sp, #PSR_MODE_MASK // Is it from user?
295 cmp sp, #PSR_USER_MODE
296 beq $0_from_user
297 cmp sp, #PSR_IRQ_MODE
298 beq $0_from_irq
299 cmp sp, #PSR_FIQ_MODE
300 beq $0_from_fiq
301 bne $0_from_svc
302 $0_from_user:
303 .endmacro
304
305 /*
306 * Handles an exception taken from kernelmode (IRQ/FIQ/SVC/etc).
307 * Places the processor into the correct mode and executes the
308 * code following the macro to handle the kernel exception.
309 * Intended to be paired with a prior call to IF_USERMODE_EXCEPTION.
310 * arg0: arbitrary string indicating the exception class, e.g. 'dataabt'
311 */
312 .macro ELSE_IF_KERNELMODE_EXCEPTION
313 $0_from_irq:
314 cpsid i, #PSR_IRQ_MODE
315 b $0_from_kernel
316 $0_from_fiq:
317 cpsid i, #PSR_FIQ_MODE
318 b $0_from_kernel
319 $0_from_svc:
320 cpsid i, #PSR_SVC_MODE
321 $0_from_kernel:
322 .endmacro
323
324 LEXT(fleh_undef)
325 VERIFY_EXCEPTION_MODE PSR_UND_MODE
326 mrs sp, spsr // For check the previous mode
327 tst sp, #PSR_TF // Is it Thumb?
328 subeq lr, lr, #4
329 subne lr, lr, #2
330 IF_USERMODE_EXCEPTION undef
331 mrc p15, 0, sp, c13, c0, 4 // Read TPIDRPRW
332 add sp, sp, ACT_PCBDATA // Get current thread PCB pointer
333
334 stmia sp, {r0-r12, sp, lr}^ // Save user context on PCB
335 mov r7, #0 // Zero the frame pointer
336 nop
337
338 mov r0, sp // Store arm_saved_state pointer
339 // for argument
340
341 str lr, [sp, SS_PC] // Save user mode pc register
342
343 mrs r4, spsr
344 str r4, [sp, SS_CPSR] // Save user mode cpsr
345
346 cpsid i, #PSR_SVC_MODE
347 mrs r3, cpsr // Read cpsr
348 msr spsr_cxsf, r3 // Set spsr(svc mode cpsr)
349 mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW
350 ldr sp, [r9, TH_KSTACKPTR] // Load kernel stack
351 #if __ARM_USER_PROTECT__
352 ldr r3, [r9, ACT_KPTW_TTB] // Load kernel ttb
353 mcr p15, 0, r3, c2, c0, 0 // Set TTBR0
354 mov r3, #0 // Load kernel asid
355 mcr p15, 0, r3, c13, c0, 1 // Set CONTEXTIDR
356 isb
357 #endif
358
359 mvn r0, #0
360 str r0, [r9, TH_IOTIER_OVERRIDE] // Reset IO tier override to -1 before handling abort from userspace
361
362 #if !CONFIG_SKIP_PRECISE_USER_KERNEL_TIME
363 bl EXT(timer_state_event_user_to_kernel)
364 mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW
365 #endif
366
367 #if __ARM_VFP__
368 add r0, r9, ACT_UVFP // Get the address of the user VFP save area
369 bl EXT(vfp_save) // Save the current VFP state to ACT_UVFP
370 mov r3, #FPSCR_DEFAULT // Load up the default FPSCR value...
371 fmxr fpscr, r3 // And shove it into FPSCR
372 add r1, r9, ACT_UVFP // Reload the pointer to the save state
373 add r0, r9, ACT_PCBDATA // Reload the VFP save state argument
374 #else
375 mov r1, #0 // Clear the VFP save state argument
376 add r0, r9, ACT_PCBDATA // Reload arm_saved_state pointer
377 #endif
378
379 bl EXT(sleh_undef) // Call second level handler
380 // sleh will enable interrupt
381 b load_and_go_user
382
383 ELSE_IF_KERNELMODE_EXCEPTION undef
384 /*
385 * We have a kernel stack already, and I will use it to save contexts
386 * IRQ is disabled
387 */
388 #if CONFIG_DTRACE
389 // We need a frame for backtracing. The LR here is the LR of supervisor mode, not the location where the exception
390 // took place. We'll store that later after we switch to undef mode and pull out the LR from there.
391
392 // This frame is consumed by fbt_invop. Any changes with the size or location of this frame will probably require
393 // changes in fbt_invop also.
394 stmfd sp!, { r7, lr }
395 #endif
396
397 sub sp, sp, EXC_CTX_SIZE // Reserve for arm_saved_state
398
399 stmia sp, {r0-r12} // Save on supervisor mode stack
400 str lr, [sp, SS_LR]
401
402 #if CONFIG_DTRACE
403 add r7, sp, EXC_CTX_SIZE // Save frame pointer
404 #endif
405
406 mrs r4, lr_und
407 str r4, [sp, SS_PC] // Save complete
408 mrs r4, spsr_und
409 str r4, [sp, SS_CPSR]
410
411 mov ip, sp
412
413 /*
414 sp - stack pointer
415 ip - stack pointer
416 r7 - frame pointer state
417 */
418
419
420 #if CONFIG_DTRACE
421 ldr r0, [ip, SS_PC] // Get the exception pc to store later
422 #endif
423
424 add ip, ip, EXC_CTX_SIZE // Send stack pointer to debugger
425 #if CONFIG_DTRACE
426 str r0, [ip, #4]
427 add ip, ip, #8
428 #endif
429 str ip, [sp, SS_SP] // for accessing local variable
430 #if CONFIG_DTRACE
431 sub ip, ip, #8
432 #endif
433 sub ip, ip, EXC_CTX_SIZE
434
435 #if __ARM_VFP__
436 mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW
437 add r0, sp, SS_SIZE // Get vfp state pointer
438 bic r0, #(VSS_ALIGN_NUM - 1) // Align to arm_vfpsaved_state alignment
439 add r0, VSS_ALIGN // Get the actual vfp save area
440 mov r5, r0 // Stash the save area in another register
441 bl EXT(vfp_save) // Save the current VFP state to the stack
442 mov r1, r5 // Load the VFP save area argument
443 mov r4, #FPSCR_DEFAULT // Load up the default FPSCR value...
444 fmxr fpscr, r4 // And shove it into FPSCR
445 #else
446 mov r1, #0 // Clear the facility context argument
447 #endif
448 #if __ARM_USER_PROTECT__
449 mrc p15, 0, r10, c2, c0, 0 // Get TTBR0
450 ldr r3, [r9, ACT_KPTW_TTB] // Load kernel ttb
451 cmp r3, r10
452 beq 1f
453 mcr p15, 0, r3, c2, c0, 0 // Set TTBR0
454 1:
455 mrc p15, 0, r11, c13, c0, 1 // Save CONTEXTIDR
456 mov r3, #0 // Load kernel asid
457 mcr p15, 0, r3, c13, c0, 1 // Set CONTEXTIDR
458 isb
459 #endif
460 mov r0, sp // Argument
461
462 ALIGN_STACK r2, r3
463 bl EXT(sleh_undef) // Call second level handler
464 UNALIGN_STACK
465
466 #if __ARM_USER_PROTECT__
467 mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW
468 ldr r0, [r9, ACT_KPTW_TTB] // Load kernel ttb
469 cmp r10, r0
470 beq 1f
471 ldr r10, [r9, ACT_UPTW_TTB] // Load thread ttb
472 cmp r10, r0
473 beq 1f
474 mcr p15, 0, r10, c2, c0, 0 // Set TTBR0
475 ldr r11, [r9, ACT_ASID] // Load thread asid
476 1:
477 mcr p15, 0, r11, c13, c0, 1 // set CONTEXTIDR
478 isb
479 #endif
480 b load_and_go_sys
481
482
483 /*
484 * First Level Exception Handler for Software Interrupt
485 *
486 * We assert that only user level can use the "SWI" instruction for a system
487 * call on development kernels, and assume it's true on release.
488 *
489 * System call number is stored in r12.
490 * System call arguments are stored in r0 to r6 and r8 (we skip r7)
491 *
492 */
493 .text
494 .align 5
495 .globl EXT(fleh_swi)
496
497 LEXT(fleh_swi)
498 cpsid i, #PSR_ABT_MODE
499 mov sp, ip // Save ip
500 cpsid i, #PSR_SVC_MODE
501 mrs ip, spsr // Check the previous mode
502 tst ip, #0x0f
503 cpsid i, #PSR_ABT_MODE
504 mov ip, sp // Restore ip
505 cpsid i, #PSR_SVC_MODE
506 beq swi_from_user
507
508 /* Only user mode can use SWI. Panic if the kernel tries. */
509 swi_from_kernel:
510 sub sp, sp, EXC_CTX_SIZE
511 stmia sp, {r0-r12}
512 add r0, sp, EXC_CTX_SIZE
513
514 str r0, [sp, SS_SP] // Save supervisor mode sp
515 str lr, [sp, SS_LR] // Save supervisor mode lr
516
517 ALIGN_STACK r0, r1
518 adr r0, L_kernel_swi_panic_str // Load panic messages and panic()
519 blx EXT(panic)
520 b .
521
522 swi_from_user:
523 mrc p15, 0, sp, c13, c0, 4 // Read TPIDRPRW
524 add sp, sp, ACT_PCBDATA // Get User PCB
525
526
527 /* Check for special mach_absolute_time trap value.
528 * This is intended to be a super-lightweight call to ml_get_timebase(), which
529 * is handrolled assembly and does not use the stack, thus not requiring us to setup a kernel stack. */
530 cmp r12, #MACH_ARM_TRAP_ABSTIME
531 beq fleh_swi_trap_tb
532 stmia sp, {r0-r12, sp, lr}^ // Save user context on PCB
533 mov r7, #0 // Zero the frame pointer
534 nop
535 mov r8, sp // Store arm_saved_state pointer
536 add sp, sp, SS_PC
537 srsia sp, #PSR_SVC_MODE
538 mrs r3, cpsr // Read cpsr
539 msr spsr_cxsf, r3 // Set spsr(svc mode cpsr)
540 sub r9, sp, ACT_PCBDATA_PC
541
542 ldr sp, [r9, TH_KSTACKPTR] // Load kernel stack
543 mov r11, r12 // save the syscall vector in a nontrashed register
544
545 #if __ARM_VFP__
546 add r0, r9, ACT_UVFP // Get the address of the user VFP save area
547 bl EXT(vfp_save) // Save the current VFP state to ACT_UVFP
548 mov r4, #FPSCR_DEFAULT // Load up the default FPSCR value...
549 fmxr fpscr, r4 // And shove it into FPSCR
550 #endif
551 #if __ARM_USER_PROTECT__
552 ldr r3, [r9, ACT_KPTW_TTB] // Load kernel ttb
553 mcr p15, 0, r3, c2, c0, 0 // Set TTBR0
554 mov r3, #0 // Load kernel asid
555 mcr p15, 0, r3, c13, c0, 1 // Set CONTEXTIDR
556 isb
557 #endif
558
559 mvn r0, #0
560 str r0, [r9, TH_IOTIER_OVERRIDE] // Reset IO tier override to -1 before handling SWI from userspace
561
562 #if !CONFIG_SKIP_PRECISE_USER_KERNEL_TIME
563 bl EXT(timer_state_event_user_to_kernel)
564 mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW
565 add r8, r9, ACT_PCBDATA // Reload arm_saved_state pointer
566 #endif
567 ldr r10, [r9, ACT_TASK] // Load the current task
568
569 /* enable interrupts */
570 cpsie i // Enable IRQ
571
572 cmp r11, #MACH_ARM_TRAP_CONTTIME // Special value for mach_continuous_time
573 beq fleh_swi_trap_mct
574
575 cmp r11, #0x80000000
576 beq fleh_swi_trap
577 fleh_swi_trap_ret:
578
579 #if TRACE_SYSCALL
580 /* trace the syscall */
581 mov r0, r8
582 bl EXT(syscall_trace)
583 #endif
584
585 bl EXT(mach_kauth_cred_uthread_update)
586 mrc p15, 0, r9, c13, c0, 4 // Reload r9 from TPIDRPRW
587 /* unix syscall? */
588 rsbs r5, r11, #0 // make the syscall positive (if negative)
589 ble fleh_swi_unix // positive syscalls are unix (note reverse logic here)
590
591 fleh_swi_mach:
592 /* note that mach_syscall_trace can modify r9, so increment the thread
593 * syscall count before the call : */
594 ldr r2, [r9, TH_MACH_SYSCALLS]
595 add r2, r2, #1
596 str r2, [r9, TH_MACH_SYSCALLS]
597
598 LOAD_ADDR(r1, mach_trap_table) // load mach_trap_table
599 #if MACH_TRAP_TABLE_ENTRY_SIZE_NUM == 12
600 add r11, r5, r5, lsl #1 // syscall * 3
601 add r6, r1, r11, lsl #2 // trap_table + syscall * 12
602 #elif MACH_TRAP_TABLE_ENTRY_SIZE_NUM == 16
603 add r6, r1, r5, lsl #4 // trap_table + syscall * 16
604 #elif MACH_TRAP_TABLE_ENTRY_SIZE_NUM == 20
605 add r11, r5, r5, lsl #2 // syscall * 5
606 add r6, r1, r11, lsl #2 // trap_table + syscall * 20
607 #else
608 #error mach_trap_t size unhandled (see MACH_TRAP_TABLE_ENTRY_SIZE)!
609 #endif
610
611 #ifndef NO_KDEBUG
612 LOAD_ADDR(r4, kdebug_enable)
613 ldr r4, [r4]
614 movs r4, r4
615 movne r0, r8 // ready the reg state pointer as an arg to the call
616 movne r1, r5 // syscall number as 2nd arg
617 COND_EXTERN_BLNE(mach_syscall_trace)
618 #endif
619 adr lr, fleh_swi_exit // any calls from here on out will return to our exit path
620 cmp r5, MACH_TRAP_TABLE_COUNT // check syscall number range
621 bge fleh_swi_mach_error
622
623 /*
624 * For arm32 ABI where 64-bit types are aligned to even registers and
625 * 64-bits on stack, we need to unpack registers differently. So
626 * we use the mungers for marshalling in arguments from user space.
627 * Currently this is just ARMv7k.
628 */
629 #if __BIGGEST_ALIGNMENT__ > 4
630 sub sp, #0x40 // allocate buffer and keep stack 128-bit aligned
631 // it should be big enough for all syscall arguments
632 ldr r11, [r6, #8] // get mach_trap_table[call_number].mach_trap_arg_munge32
633 teq r11, #0 // check if we have a munger
634 moveq r0, #0
635 movne r0, r8 // ready the reg state pointer as an arg to the call
636 movne r1, sp // stack will hold arguments buffer
637 blxne r11 // call munger to get arguments from userspace
638 adr lr, fleh_swi_exit // any calls from here on out will return to our exit path
639 teq r0, #0
640 bne fleh_swi_mach_error // exit if the munger returned non-zero status
641 #endif
642
643 ldr r1, [r6, #4] // load the syscall vector
644
645 LOAD_ADDR(r2, kern_invalid) // test to make sure the trap is not kern_invalid
646 teq r1, r2
647 beq fleh_swi_mach_error
648
649 #if __BIGGEST_ALIGNMENT__ > 4
650 mov r0, sp // argument buffer on stack
651 bx r1 // call the syscall handler
652 #else
653 mov r0, r8 // ready the reg state pointer as an arg to the call
654 bx r1 // call the syscall handler
655 #endif
656
657 fleh_swi_exit64:
658 str r1, [r8, #4] // top of 64-bit return
659 fleh_swi_exit:
660 str r0, [r8] // save the return value
661 #ifndef NO_KDEBUG
662 movs r4, r4
663 movne r1, r5
664 COND_EXTERN_BLNE(mach_syscall_trace_exit)
665 #endif
666 #if TRACE_SYSCALL
667 bl EXT(syscall_trace_exit)
668 #endif
669
670 mov r0, #1
671 bl EXT(throttle_lowpri_io) // throttle_lowpri_io(1);
672
673 bl EXT(thread_exception_return)
674 b .
675
676 fleh_swi_mach_error:
677 mov r0, #EXC_SYSCALL
678 sub r1, sp, #4
679 mov r2, #1
680 bl EXT(exception_triage)
681 b .
682
683 .align 5
684 fleh_swi_unix:
685 ldr r1, [r9, TH_UNIX_SYSCALLS]
686 mov r0, r8 // reg state structure is arg
687 add r1, r1, #1
688 str r1, [r9, TH_UNIX_SYSCALLS]
689 mov r1, r9 // current thread in arg1
690 ldr r2, [r9, TH_UTHREAD] // current uthread in arg2
691 ldr r3, [r10, TASK_BSD_INFO] // current proc in arg3
692 bl EXT(unix_syscall)
693 b .
694
695 fleh_swi_trap:
696 ldmia r8, {r0-r3}
697 cmp r3, #3
698 addls pc, pc, r3, LSL#2
699 b fleh_swi_trap_ret
700 b icache_invalidate_trap
701 b dcache_flush_trap
702 b thread_set_cthread_trap
703 b thread_get_cthread_trap
704
705 icache_invalidate_trap:
706 add r3, r0, r1
707 cmp r3, VM_MAX_ADDRESS
708 subhi r3, r3, #1<<MMU_CLINE
709 bhi cache_trap_error
710 adr r11, cache_trap_jmp
711 ldr r6, [r9, TH_RECOVER] // Save existing recovery routine
712 str r11, [r9, TH_RECOVER]
713 #if __ARM_USER_PROTECT__
714 ldr r5, [r9, ACT_UPTW_TTB] // Load thread ttb
715 mcr p15, 0, r5, c2, c0, 0 // Set TTBR0
716 ldr r5, [r9, ACT_ASID] // Load thread asid
717 mcr p15, 0, r5, c13, c0, 1 // Set CONTEXTIDR
718 dsb ish
719 isb
720 #endif
721 bl EXT(InvalidatePoU_IcacheRegion)
722 mrc p15, 0, r9, c13, c0, 4 // Reload r9 from TPIDRPRW
723 #if __ARM_USER_PROTECT__
724 ldr r4, [r9, ACT_KPTW_TTB] // Load kernel ttb
725 mcr p15, 0, r4, c2, c0, 0 // Set TTBR0
726 mov r4, #0 // Load kernel asid
727 mcr p15, 0, r4, c13, c0, 1 // Set CONTEXTIDR
728 isb
729 #endif
730 str r6, [r9, TH_RECOVER]
731 bl EXT(thread_exception_return)
732 b .
733
734 dcache_flush_trap:
735 add r3, r0, r1
736 cmp r3, VM_MAX_ADDRESS
737 subhi r3, r3, #1<<MMU_CLINE
738 bhi cache_trap_error
739 adr r11, cache_trap_jmp
740 ldr r4, [r9, TH_RECOVER] // Save existing recovery routine
741 str r11, [r9, TH_RECOVER]
742 #if __ARM_USER_PROTECT__
743 ldr r6, [r9, ACT_UPTW_TTB] // Load thread ttb
744 mcr p15, 0, r6, c2, c0, 0 // Set TTBR0
745 ldr r5, [r9, ACT_ASID] // Load thread asid
746 mcr p15, 0, r5, c13, c0, 1 // Set CONTEXTIDR
747 isb
748 #endif
749 bl EXT(flush_dcache_syscall)
750 mrc p15, 0, r9, c13, c0, 4 // Reload r9 from TPIDRPRW
751 #if __ARM_USER_PROTECT__
752 ldr r5, [r9, ACT_KPTW_TTB] // Load kernel ttb
753 mcr p15, 0, r5, c2, c0, 0 // Set TTBR0
754 mov r5, #0 // Load kernel asid
755 mcr p15, 0, r5, c13, c0, 1 // Set CONTEXTIDR
756 isb
757 #endif
758 str r4, [r9, TH_RECOVER]
759 bl EXT(thread_exception_return)
760 b .
761
762 thread_set_cthread_trap:
763 bl EXT(thread_set_cthread_self)
764 bl EXT(thread_exception_return)
765 b .
766
767 thread_get_cthread_trap:
768 bl EXT(thread_get_cthread_self)
769 mrc p15, 0, r9, c13, c0, 4 // Reload r9 from TPIDRPRW
770 add r1, r9, ACT_PCBDATA // Get User PCB
771 str r0, [r1, SS_R0] // set return value
772 bl EXT(thread_exception_return)
773 b .
774
775 cache_trap_jmp:
776 #if __ARM_USER_PROTECT__
777 mrc p15, 0, r9, c13, c0, 4 // Reload r9 from TPIDRPRW
778 ldr r5, [r9, ACT_KPTW_TTB] // Load kernel ttb
779 mcr p15, 0, r5, c2, c0, 0 // Set TTBR0
780 mov r5, #0 // Load kernel asid
781 mcr p15, 0, r5, c13, c0, 1 // Set CONTEXTIDR
782 isb
783 #endif
784 mrc p15, 0, r3, c6, c0 // Read Fault Address
785 cache_trap_error:
786 mrc p15, 0, r9, c13, c0, 4 // Reload r9 from TPIDRPRW
787 add r0, r9, ACT_PCBDATA // Get User PCB
788 ldr r1, [r0, SS_PC] // Save user mode pc register as pc
789 sub r1, r1, #4 // Backtrack current pc
790 str r1, [r0, SS_PC] // pc at cache assist swi
791 str r3, [r0, SS_VADDR] // Fault Address
792 mov r0, #EXC_BAD_ACCESS
793 mov r2, KERN_INVALID_ADDRESS
794 sub sp, sp, #8
795 mov r1, sp
796 str r2, [sp]
797 str r3, [sp, #4]
798 ALIGN_STACK r2, r3
799 mov r2, #2
800 bl EXT(exception_triage)
801 b .
802
803 fleh_swi_trap_mct:
804 bl EXT(mach_continuous_time)
805 mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW
806 add r9, r9, ACT_PCBDATA_R0 // Get User register state
807 stmia r9, {r0, r1} // set 64-bit return value
808 bl EXT(thread_exception_return)
809 b .
810
811 fleh_swi_trap_tb:
812 str lr, [sp, SS_PC]
813 bl EXT(ml_get_timebase) // ml_get_timebase() (64-bit return)
814 ldr lr, [sp, SS_PC]
815 nop
816 movs pc, lr // Return to user
817
818 .align 2
819 L_kernel_swi_panic_str:
820 .asciz "fleh_swi: took SWI from kernel mode\n"
821 .align 2
822
823 /*
824 * First Level Exception Handler for Prefetching Abort.
825 */
826 .text
827 .align 2
828 .globl EXT(fleh_prefabt)
829
830 LEXT(fleh_prefabt)
831 VERIFY_EXCEPTION_MODE PSR_ABT_MODE
832 sub lr, lr, #4
833
834 IF_USERMODE_EXCEPTION prefabt
835 mrc p15, 0, sp, c13, c0, 4 // Read TPIDRPRW
836 add sp, sp, ACT_PCBDATA // Get User PCB
837
838 stmia sp, {r0-r12, sp, lr}^ // Save user context on PCB
839 mov r7, #0 // Zero the frame pointer
840 nop
841 mov r0, sp // Store arm_saved_state pointer
842 // For argument
843 str lr, [sp, SS_PC] // Save user mode pc register as pc
844 mrc p15, 0, r1, c6, c0, 2 // Read IFAR
845 str r1, [sp, SS_VADDR] // and fault address of pcb
846
847 mrc p15, 0, r5, c5, c0, 1 // Read Fault Status
848 str r5, [sp, SS_STATUS] // Save fault status register to pcb
849
850 mrs r4, spsr
851 str r4, [sp, SS_CPSR] // Save user mode cpsr
852
853 cpsid i, #PSR_SVC_MODE
854 mrs r3, cpsr // Read cpsr
855 msr spsr_cxsf, r3 // Set spsr(svc mode cpsr)
856 mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW
857 ldr sp, [r9, TH_KSTACKPTR] // Load kernel stack
858
859 #if __ARM_VFP__
860 add r0, r9, ACT_UVFP // Get the address of the user VFP save area
861 bl EXT(vfp_save) // Save the current VFP state to ACT_UVFP
862 mov r3, #FPSCR_DEFAULT // Load up the default FPSCR value...
863 fmxr fpscr, r3 // And shove it into FPSCR
864 #endif
865 #if __ARM_USER_PROTECT__
866 ldr r3, [r9, ACT_KPTW_TTB] // Load kernel ttb
867 mcr p15, 0, r3, c2, c0, 0 // Set TTBR0
868 mov r3, #0 // Load kernel asid
869 mcr p15, 0, r3, c13, c0, 1 // Set CONTEXTIDR
870 isb
871 #endif
872
873 mvn r0, #0
874 str r0, [r9, TH_IOTIER_OVERRIDE] // Reset IO tier override to -1 before handling abort from userspace
875
876 #if !CONFIG_SKIP_PRECISE_USER_KERNEL_TIME
877 bl EXT(timer_state_event_user_to_kernel)
878 mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW
879 #endif
880
881 add r0, r9, ACT_PCBDATA // Reload arm_saved_state pointer
882 mov r1, T_PREFETCH_ABT // Pass abort type
883 bl EXT(sleh_abort) // Call second level handler
884 // Sleh will enable interrupt
885 b load_and_go_user
886
887 ELSE_IF_KERNELMODE_EXCEPTION prefabt
888
889 UNWIND_PROLOGUE
890
891 /*
892 * We have a kernel stack already, and I will use it to save contexts:
893 * ------------------
894 * | VFP saved state |
895 * |------------------|
896 * | ARM saved state |
897 * SP ------------------
898 *
899 * IRQ is disabled
900 */
901 sub sp, sp, EXC_CTX_SIZE
902 stmia sp, {r0-r12}
903 add r0, sp, EXC_CTX_SIZE
904
905 str r0, [sp, SS_SP] // Save supervisor mode sp
906 str lr, [sp, SS_LR] // Save supervisor mode lr
907
908 mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW
909
910 #if __ARM_VFP__
911 add r0, sp, SS_SIZE // Get vfp state pointer
912 bic r0, #(VSS_ALIGN_NUM - 1) // Align to arm_vfpsaved_state alignment
913 add r0, VSS_ALIGN // Get the actual vfp save area
914 bl EXT(vfp_save) // Save the current VFP state to the stack
915 mov r4, #FPSCR_DEFAULT // Load up the default FPSCR value...
916 fmxr fpscr, r4 // And shove it into FPSCR
917 #endif
918 #if __ARM_USER_PROTECT__
919 mrc p15, 0, r10, c2, c0, 0 // Get TTBR0
920 ldr r3, [r9, ACT_KPTW_TTB] // Load kernel ttb
921 cmp r3, r10
922 beq 1f
923 mcr p15, 0, r3, c2, c0, 0 // Set TTBR0
924 1:
925 mrc p15, 0, r11, c13, c0, 1 // Save CONTEXTIDR
926 mov r3, #0 // Load kernel asid
927 mcr p15, 0, r3, c13, c0, 1 // Set CONTEXTIDR
928 isb
929 #endif
930
931 mrs r4, lr_abt
932 str r4, [sp, SS_PC] // Save pc
933
934 mrc p15, 0, r5, c6, c0, 2 // Read IFAR
935 str r5, [sp, SS_VADDR] // and fault address of pcb
936 mrc p15, 0, r5, c5, c0, 1 // Read (instruction) Fault Status
937 str r5, [sp, SS_STATUS] // Save fault status register to pcb
938
939 mrs r4, spsr_abt
940 str r4, [sp, SS_CPSR]
941
942 mov r0, sp
943 ALIGN_STACK r1, r2
944 mov r1, T_PREFETCH_ABT // Pass abort type
945
946
947 UNWIND_DIRECTIVES
948
949 bl EXT(sleh_abort) // Call second level handler
950 UNALIGN_STACK
951
952 mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW
953 #if __ARM_USER_PROTECT__
954 ldr r0, [r9, ACT_KPTW_TTB] // Load kernel ttb
955 cmp r10, r0
956 beq 1f
957 ldr r10, [r9, ACT_UPTW_TTB] // Load thread ttb
958 cmp r10, r0
959 beq 1f
960 mcr p15, 0, r10, c2, c0, 0 // Set TTBR0
961 ldr r11, [r9, ACT_ASID] // Load thread asid
962 1:
963 mcr p15, 0, r11, c13, c0, 1 // set CONTEXTIDR
964 isb
965 #endif
966
967 b load_and_go_sys
968
969 UNWIND_EPILOGUE
970
971 /*
972 * First Level Exception Handler for Data Abort
973 */
974 .text
975 .align 2
976 .globl EXT(fleh_dataabt)
977
978 LEXT(fleh_dataabt)
979 VERIFY_EXCEPTION_MODE PSR_ABT_MODE
980 sub lr, lr, #8
981 IF_USERMODE_EXCEPTION dataabt
982 mrc p15, 0, sp, c13, c0, 4 // Read TPIDRPRW
983 add sp, sp, ACT_PCBDATA // Get User PCB
984
985 stmia sp, {r0-r12, sp, lr}^ // Save user context on PCB
986 mov r7, #0 // Zero the frame pointer
987 nop
988
989 mov r0, sp // Store arm_saved_state pointer
990 // For argument
991
992 str lr, [sp, SS_PC] // Save user mode pc register
993
994 mrs r4, spsr
995 str r4, [sp, SS_CPSR] // Save user mode cpsr
996
997 mrc p15, 0, r5, c5, c0 // Read Fault Status
998 mrc p15, 0, r6, c6, c0 // Read Fault Address
999 str r5, [sp, SS_STATUS] // Save fault status register to pcb
1000 str r6, [sp, SS_VADDR] // Save fault address to pcb
1001
1002 cpsid i, #PSR_SVC_MODE
1003 mrs r3, cpsr // Read cpsr
1004 msr spsr_cxsf, r3 // Set spsr(svc mode cpsr)
1005 mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW
1006 ldr sp, [r9, TH_KSTACKPTR] // Load kernel stack
1007
1008 #if __ARM_VFP__
1009 add r0, r9, ACT_UVFP // Get the address of the user VFP save area
1010 bl EXT(vfp_save) // Save the current VFP state to ACT_UVFP
1011 mov r3, #FPSCR_DEFAULT // Load up the default FPSCR value...
1012 fmxr fpscr, r3 // And shove it into FPSCR
1013 #endif
1014 #if __ARM_USER_PROTECT__
1015 ldr r3, [r9, ACT_KPTW_TTB] // Load kernel ttb
1016 mcr p15, 0, r3, c2, c0, 0 // Set TTBR0
1017 mov r3, #0 // Load kernel asid
1018 mcr p15, 0, r3, c13, c0, 1 // Set CONTEXTIDR
1019 isb
1020 #endif
1021
1022 mvn r0, #0
1023 str r0, [r9, TH_IOTIER_OVERRIDE] // Reset IO tier override to -1 before handling abort from userspace
1024
1025 #if !CONFIG_SKIP_PRECISE_USER_KERNEL_TIME
1026 bl EXT(timer_state_event_user_to_kernel)
1027 mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW
1028 #endif
1029
1030 add r0, r9, ACT_PCBDATA // Reload arm_saved_state pointer
1031 mov r1, T_DATA_ABT // Pass abort type
1032 bl EXT(sleh_abort) // Call second level handler
1033 // Sleh will enable irq
1034 b load_and_go_user
1035
1036 ELSE_IF_KERNELMODE_EXCEPTION dataabt
1037
1038 UNWIND_PROLOGUE
1039
1040 /*
1041 * We have a kernel stack already, and I will use it to save contexts:
1042 * ------------------
1043 * | VFP saved state |
1044 * |------------------|
1045 * | ARM saved state |
1046 * SP ------------------
1047 *
1048 * IRQ is disabled
1049 */
1050 sub sp, sp, EXC_CTX_SIZE
1051 stmia sp, {r0-r12}
1052 add r0, sp, EXC_CTX_SIZE
1053
1054 str r0, [sp, SS_SP] // Save supervisor mode sp
1055 str lr, [sp, SS_LR] // Save supervisor mode lr
1056
1057 mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW
1058
1059 #if __ARM_VFP__
1060 add r0, sp, SS_SIZE // Get vfp state pointer
1061 bic r0, #(VSS_ALIGN_NUM - 1) // Align to arm_vfpsaved_state alignment
1062 add r0, VSS_ALIGN // Get the actual vfp save area
1063 bl EXT(vfp_save) // Save the current VFP state to the stack
1064 mov r4, #FPSCR_DEFAULT // Load up the default FPSCR value...
1065 fmxr fpscr, r4 // And shove it into FPSCR
1066 #endif
1067
1068 mrs r4, lr_abt
1069 str r4, [sp, SS_PC]
1070 mrs r4, spsr_abt
1071 str r4, [sp, SS_CPSR]
1072
1073 #if __ARM_USER_PROTECT__
1074 mrc p15, 0, r10, c2, c0, 0 // Get TTBR0
1075 ldr r3, [r9, ACT_KPTW_TTB] // Load kernel ttb
1076 cmp r3, r10
1077 beq 1f
1078 mcr p15, 0, r3, c2, c0, 0 // Set TTBR0
1079 1:
1080 mrc p15, 0, r11, c13, c0, 1 // Save CONTEXTIDR
1081 mov r3, #0 // Load kernel asid
1082 mcr p15, 0, r3, c13, c0, 1 // Set CONTEXTIDR
1083 isb
1084 #endif
1085 mrc p15, 0, r5, c5, c0 // Read Fault Status
1086 mrc p15, 0, r6, c6, c0 // Read Fault Address
1087 str r5, [sp, SS_STATUS] // Save fault status register to pcb
1088 str r6, [sp, SS_VADDR] // Save fault address to pcb
1089
1090 mov r0, sp // Argument
1091 ALIGN_STACK r1, r2
1092 mov r1, T_DATA_ABT // Pass abort type
1093
1094 UNWIND_DIRECTIVES
1095
1096 bl EXT(sleh_abort) // Call second level handler
1097 UNALIGN_STACK
1098
1099 mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW
1100 #if __ARM_USER_PROTECT__
1101 ldr r0, [r9, ACT_KPTW_TTB] // Load kernel ttb
1102 cmp r10, r0
1103 beq 1f
1104 ldr r10, [r9, ACT_UPTW_TTB] // Load thread ttb
1105 cmp r10, r0
1106 beq 1f
1107 mcr p15, 0, r10, c2, c0, 0 // Set TTBR0
1108 ldr r11, [r9, ACT_ASID] // Load thread asid
1109 1:
1110 mcr p15, 0, r11, c13, c0, 1 // set CONTEXTIDR
1111 isb
1112 #endif
1113
1114 load_and_go_sys:
1115 mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW
1116
1117 ldr r4, [sp, SS_CPSR] // Load saved cpsr
1118 tst r4, #PSR_IRQF // Test IRQ set
1119 bne lags1 // Branch if IRQ disabled
1120
1121 cpsid i // Disable IRQ
1122 ldr r2, [r9, ACT_PREEMPT_CNT] // Load preemption count
1123 movs r2, r2 // Test if null
1124 ldr r8, [r9, ACT_CPUDATAP] // Get current cpu
1125 bne lags1 // Branch if count not null
1126 ldr r5, [r8, CPU_PENDING_AST] // Get ASTs
1127 ands r5, r5, AST_URGENT // Get the requests we do honor
1128 beq lags1 // Branch if no ASTs
1129 #if __ARM_USER_PROTECT__
1130 mrc p15, 0, r10, c2, c0, 0 // Get TTBR0
1131 ldr r3, [r9, ACT_KPTW_TTB] // Load kernel ttb
1132 cmp r3, r10
1133 beq 1f
1134 mcr p15, 0, r3, c2, c0, 0 // Set TTBR0
1135 1:
1136 mrc p15, 0, r11, c13, c0, 1 // Save CONTEXTIDR
1137 mov r3, #0 // Load kernel asid
1138 mcr p15, 0, r3, c13, c0, 1 // Set CONTEXTIDR
1139 isb
1140 #endif
1141 ldr lr, [sp, SS_LR] // Restore the link register
1142 stmfd sp!, {r7, lr} // Push a fake frame
1143
1144 ALIGN_STACK r2, r3
1145 bl EXT(ast_taken_kernel) // Handle AST_URGENT
1146 UNALIGN_STACK
1147
1148 ldmfd sp!, {r7, lr} // Pop the fake frame
1149 mrc p15, 0, r9, c13, c0, 4 // Reload r9 from TPIDRPRW
1150 ldr r8, [r9, ACT_CPUDATAP] // Get current cpu
1151 #if __ARM_USER_PROTECT__
1152 ldr r0, [r9, ACT_KPTW_TTB] // Load kernel ttb
1153 cmp r10, r0
1154 beq 1f
1155 ldr r10, [r9, ACT_UPTW_TTB] // Load thread ttb
1156 cmp r10, r0
1157 beq 1f
1158 mcr p15, 0, r10, c2, c0, 0 // Set TTBR0
1159 ldr r11, [r9, ACT_ASID] // Load thread asid
1160 1:
1161 mcr p15, 0, r11, c13, c0, 1 // set CONTEXTIDR
1162 isb
1163 #endif
1164 lags1:
1165 ldr lr, [sp, SS_LR]
1166
1167 mov ip, sp // Save pointer to contexts for abort mode
1168 ldr sp, [ip, SS_SP] // Restore stack pointer
1169
1170 cpsid if, #PSR_ABT_MODE
1171
1172 mov sp, ip
1173
1174 ldr r4, [sp, SS_CPSR]
1175 msr spsr_cxsf, r4 // Restore spsr
1176
1177 clrex // clear exclusive memory tag
1178 #if __ARM_ENABLE_WFE_
1179 sev
1180 #endif
1181
1182 #if __ARM_VFP__
1183 add r0, sp, SS_SIZE // Get vfp state pointer
1184 bic r0, #(VSS_ALIGN_NUM - 1) // Align to arm_vfpsaved_state alignment
1185 add r0, VSS_ALIGN // Get the actual vfp save area
1186 bl EXT(vfp_load) // Load the desired VFP state from the stack
1187 #endif
1188
1189 ldr lr, [sp, SS_PC] // Restore lr
1190
1191 ldmia sp, {r0-r12} // Restore other registers
1192
1193 movs pc, lr // Return to sys (svc, irq, fiq)
1194
1195 UNWIND_EPILOGUE
1196
1197 /*
1198 * First Level Exception Handler for address exception
1199 * Not supported
1200 */
1201 .text
1202 .align 2
1203 .globl EXT(fleh_addrexc)
1204
1205 LEXT(fleh_addrexc)
1206 b .
1207
1208
1209 /*
1210 * First Level Exception Handler for IRQ
1211 * Current mode : IRQ
1212 * IRQ and FIQ are always disabled while running in FIQ handler
1213 * We do not permit nested interrupt.
1214 *
1215 * Saving area: from user : PCB.
1216 * from kernel : interrupt stack.
1217 */
1218
1219 .text
1220 .align 2
1221 .globl EXT(fleh_irq)
1222
1223 LEXT(fleh_irq)
1224 sub lr, lr, #4
1225
1226 cpsie a // Re-enable async aborts
1227
1228 mrs sp, spsr
1229 tst sp, #0x0f // From user? or kernel?
1230 bne fleh_irq_kernel
1231
1232 fleh_irq_user:
1233 mrc p15, 0, sp, c13, c0, 4 // Read TPIDRPRW
1234 add sp, sp, ACT_PCBDATA // Get User PCB
1235 stmia sp, {r0-r12, sp, lr}^
1236 mov r7, #0 // Zero the frame pointer
1237 nop
1238 str lr, [sp, SS_PC]
1239 mrs r4, spsr
1240 str r4, [sp, SS_CPSR]
1241 mov r5, sp // Saved context in r5
1242 mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW
1243 ldr r6, [r9, ACT_CPUDATAP] // Get current cpu
1244 ldr sp, [r6, CPU_ISTACKPTR] // Set interrupt stack
1245 cpsid i, #PSR_SVC_MODE
1246 ldr sp, [r9, TH_KSTACKPTR] // Set kernel stack
1247 cpsid i, #PSR_IRQ_MODE
1248
1249 #if __ARM_VFP__
1250 add r0, r9, ACT_UVFP // Get the address of the user VFP save area
1251 bl EXT(vfp_save) // Save the current VFP state to ACT_UVFP
1252 mov r4, #FPSCR_DEFAULT // Load up the default FPSCR value...
1253 fmxr fpscr, r4 // And shove it into FPSCR
1254 #endif
1255 #if __ARM_USER_PROTECT__
1256 ldr r3, [r9, ACT_KPTW_TTB] // Load kernel ttb
1257 mcr p15, 0, r3, c2, c0, 0 // Set TTBR0
1258 mov r3, #0 // Load kernel asid
1259 mcr p15, 0, r3, c13, c0, 1 // Set CONTEXTIDR
1260 isb
1261 #endif
1262 #if !CONFIG_SKIP_PRECISE_USER_KERNEL_TIME
1263 bl EXT(timer_state_event_user_to_kernel)
1264 mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW
1265 #endif
1266 #if CONFIG_TELEMETRY
1267 LOAD_ADDR(r2, telemetry_needs_record) // Check if a telemetry record was requested...
1268 mov r0, #1
1269 ldr r2, [r2]
1270 movs r2, r2
1271 beq 1f
1272 mov r1, #0 // (not a PMI record)
1273 bl EXT(telemetry_mark_curthread) // ...if so, mark the current thread...
1274 mrc p15, 0, r9, c13, c0, 4 // ...and restore the thread pointer from TPIDRPRW
1275 1:
1276 #endif
1277
1278 b fleh_irq_handler
1279
1280 fleh_irq_kernel:
1281 cpsid i, #PSR_SVC_MODE
1282
1283 sub sp, sp, EXC_CTX_SIZE
1284 stmia sp, {r0-r12}
1285 add r0, sp, EXC_CTX_SIZE
1286
1287 str r0, [sp, SS_SP] // Save supervisor mode sp
1288 str lr, [sp, SS_LR] // Save supervisor mode lr
1289
1290 mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW
1291
1292 #if __ARM_VFP__
1293 add r0, sp, SS_SIZE // Get vfp state pointer
1294 bic r0, #(VSS_ALIGN_NUM - 1) // Align to arm_vfpsaved_state alignment
1295 add r0, VSS_ALIGN // Get the actual vfp save area
1296 bl EXT(vfp_save) // Save the current VFP state to the stack
1297 mov r4, #FPSCR_DEFAULT // Load up the default FPSCR value...
1298 fmxr fpscr, r4 // And shove it into FPSCR
1299 #endif
1300 #if __ARM_USER_PROTECT__
1301 mrc p15, 0, r10, c2, c0, 0 // Get TTBR0
1302 ldr r3, [r9, ACT_KPTW_TTB] // Load kernel ttb
1303 mcr p15, 0, r3, c2, c0, 0 // Set TTBR0
1304 mrc p15, 0, r11, c13, c0, 1 // Get CONTEXTIDR
1305 mov r3, #0 // Load kernel asid
1306 mcr p15, 0, r3, c13, c0, 1 // Set CONTEXTIDR
1307 isb
1308 #endif
1309 mov r5, sp // Saved context in r5
1310
1311 cpsid i, #PSR_IRQ_MODE
1312
1313 str lr, [r5, SS_PC] // Save LR as the return PC
1314 mrs r4, spsr
1315 str r4, [r5, SS_CPSR] // Save the cpsr of the interrupted mode
1316
1317 ldr sp, [r9, ACT_CPUDATAP] // Get current cpu
1318 ldr sp, [sp, CPU_ISTACKPTR] // Set interrupt stack
1319
1320 #if CONFIG_TELEMETRY
1321 LOAD_ADDR(r2, telemetry_needs_record) // Check if a telemetry record was requested...
1322 mov r0, #0
1323 ldr r2, [r2]
1324 movs r2, r2
1325 beq 1f
1326 mov r1, #0 // (not a PMI record)
1327 bl EXT(telemetry_mark_curthread) // ...if so, mark the current thread...
1328 mrc p15, 0, r9, c13, c0, 4 // ...and restore the thread pointer from TPIDRPRW
1329 1:
1330 #endif
1331
1332 fleh_irq_handler:
1333 ldr r2, [r9, ACT_PREEMPT_CNT] // Load preemption count
1334 add r2, r2, #1 // Increment count
1335 str r2, [r9, ACT_PREEMPT_CNT] // Update preemption count
1336 #ifndef NO_KDEBUG
1337 LOAD_ADDR(r8, kdebug_enable)
1338 ldr r8, [r8]
1339 movs r8, r8
1340 movne r0, r5
1341 COND_EXTERN_BLNE(interrupt_trace)
1342 #endif
1343 bl EXT(interrupt_stats) // Record interrupt statistics
1344 mrc p15, 0, r9, c13, c0, 4 // Reload r9 from TPIDRPRW
1345 ldr r4, [r9, ACT_CPUDATAP] // Get current cpu
1346 str r5, [r4, CPU_INT_STATE] // Saved context in cpu_int_state
1347 ldr r3, [r4, CPU_STAT_IRQ] // Get IRQ count
1348 add r3, r3, #1 // Increment count
1349 str r3, [r4, CPU_STAT_IRQ] // Update IRQ count
1350 ldr r3, [r4, CPU_STAT_IRQ_WAKE] // Get post-wake IRQ count
1351 add r3, r3, #1 // Increment count
1352 str r3, [r4, CPU_STAT_IRQ_WAKE] // Update post-wake IRQ count
1353 ldr r0, [r4, INTERRUPT_TARGET]
1354 ldr r1, [r4, INTERRUPT_REFCON]
1355 ldr r2, [r4, INTERRUPT_NUB]
1356 ldr r3, [r4, INTERRUPT_SOURCE]
1357 ldr r5, [r4, INTERRUPT_HANDLER] // Call second level exception handler
1358 blx r5
1359 #ifndef NO_KDEBUG
1360 movs r8, r8
1361 COND_EXTERN_BLNE(interrupt_trace_exit)
1362 #endif
1363 bl EXT(entropy_collect)
1364 return_from_irq:
1365 mrc p15, 0, r9, c13, c0, 4 // Reload r9 from TPIDRPRW
1366 mov r5, #0
1367 ldr r4, [r9, ACT_CPUDATAP] // Get current cpu
1368 str r5, [r4, CPU_INT_STATE] // Clear cpu_int_state
1369 ldr r2, [r9, ACT_PREEMPT_CNT] // Load preemption count
1370 #if MACH_ASSERT
1371 cmp r2, #0 // verify positive count
1372 bgt 1f
1373 push {r7, lr}
1374 mov r7, sp
1375 adr r0, L_preemption_count_zero_str
1376 blx EXT(panic)
1377 b .
1378 1:
1379 #endif
1380 sub r2, r2, #1 // Decrement count
1381 str r2, [r9, ACT_PREEMPT_CNT] // Update preemption count
1382
1383 mrs r0, spsr // For check the previous mode
1384
1385 cpsid i, #PSR_SVC_MODE
1386
1387 tst r0, #0x0f // Check if the previous is from user
1388 ldreq sp, [r9, TH_KSTACKPTR] // ...If so, reload the kernel stack pointer
1389 beq load_and_go_user // ...and return
1390
1391 #if __ARM_USER_PROTECT__
1392 ldr r0, [r9, ACT_KPTW_TTB] // Load kernel ttb
1393 cmp r10, r0
1394 beq 1f
1395 ldr r10, [r9, ACT_UPTW_TTB] // Load thread ttb
1396 cmp r10, r0
1397 beq 1f
1398 mcr p15, 0, r10, c2, c0, 0 // Set TTBR0
1399 ldr r11, [r9, ACT_ASID] // Load thread asid
1400 1:
1401 mcr p15, 0, r11, c13, c0, 1 // set CONTEXTIDR
1402 isb
1403 #endif
1404 b load_and_go_sys
1405
1406 .align 2
1407 L_preemption_count_zero_str:
1408 .ascii "locore.s: preemption count is zero \000"
1409 .align 2
1410 /*
1411 * First Level Exception Handler for DEC
1412 * Current mode : IRQ
1413 * IRQ and FIQ are always disabled while running in FIQ handler
1414 * We do not permit nested interrupt.
1415 *
1416 * Saving area: from user : PCB.
1417 * from kernel : interrupt stack.
1418 */
1419
1420 .text
1421 .align 2
1422 .globl EXT(fleh_decirq)
1423
1424 LEXT(fleh_decirq)
1425 sub lr, lr, #4
1426
1427 cpsie af // Re-enable async aborts/FIQ
1428
1429 mrs sp, spsr
1430 tst sp, #0x0f // From user? or kernel?
1431 bne fleh_decirq_kernel
1432
1433 fleh_decirq_user:
1434 mrc p15, 0, sp, c13, c0, 4 // Read TPIDRPRW
1435 add sp, sp, ACT_PCBDATA // Get User PCB
1436 stmia sp, {r0-r12, sp, lr}^
1437 mov r7, #0 // Zero the frame pointer
1438 nop
1439 str lr, [sp, SS_PC]
1440 mrs r4, spsr
1441 str r4, [sp, SS_CPSR]
1442 mov r5, sp // Saved context in r5
1443 mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW
1444 ldr r6, [r9, ACT_CPUDATAP] // Get current cpu
1445 ldr sp, [r6, CPU_ISTACKPTR] // Set interrupt stack
1446 cpsid i, #PSR_SVC_MODE
1447 ldr sp, [r9, TH_KSTACKPTR] // Set kernel stack
1448 cpsid i, #PSR_IRQ_MODE
1449
1450 #if __ARM_VFP__
1451 add r0, r9, ACT_UVFP // Get the address of the user VFP save area
1452 bl EXT(vfp_save) // Save the current VFP state to ACT_UVFP
1453 mov r4, #FPSCR_DEFAULT // Load up the default FPSCR value...
1454 fmxr fpscr, r4 // And shove it into FPSCR
1455 #endif
1456 #if __ARM_USER_PROTECT__
1457 ldr r3, [r9, ACT_KPTW_TTB] // Load kernel ttb
1458 mcr p15, 0, r3, c2, c0, 0 // Set TTBR0
1459 mov r3, #0 // Load kernel asid
1460 mcr p15, 0, r3, c13, c0, 1 // Set CONTEXTIDR
1461 isb
1462 #endif
1463 #if !CONFIG_SKIP_PRECISE_USER_KERNEL_TIME
1464 bl EXT(timer_state_event_user_to_kernel)
1465 mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW
1466 #endif
1467 #if CONFIG_TELEMETRY
1468 LOAD_ADDR(r2, telemetry_needs_record) // Check if a telemetry record was requested...
1469 mov r0, #1
1470 ldr r2, [r2]
1471 movs r2, r2
1472 beq 1f
1473 mov r1, #0 // (not a PMI record)
1474 bl EXT(telemetry_mark_curthread) // ...if so, mark the current thread...
1475 mrc p15, 0, r9, c13, c0, 4 // ...and restore the thread pointer from TPIDRPRW
1476 1:
1477 #endif
1478
1479 b fleh_decirq_handler
1480
1481 fleh_decirq_kernel:
1482 cpsid i, #PSR_SVC_MODE
1483
1484 sub sp, sp, EXC_CTX_SIZE
1485 stmia sp, {r0-r12}
1486 add r0, sp, EXC_CTX_SIZE
1487
1488 str r0, [sp, SS_SP] // Save supervisor mode sp
1489 str lr, [sp, SS_LR] // Save supervisor mode lr
1490
1491 mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW
1492
1493 #if __ARM_VFP__
1494 add r0, sp, SS_SIZE // Get vfp state pointer
1495 bic r0, #(VSS_ALIGN_NUM - 1) // Align to arm_vfpsaved_state alignment
1496 add r0, VSS_ALIGN // Get the actual vfp save area
1497 bl EXT(vfp_save) // Save the current VFP state to the stack
1498 mov r4, #FPSCR_DEFAULT // Load up the default FPSCR value...
1499 fmxr fpscr, r4 // And shove it into FPSCR
1500 #endif
1501 #if __ARM_USER_PROTECT__
1502 mrc p15, 0, r10, c2, c0, 0 // Get TTBR0
1503 ldr r3, [r9, ACT_KPTW_TTB] // Load kernel ttb
1504 mcr p15, 0, r3, c2, c0, 0 // Set TTBR0
1505 mrc p15, 0, r11, c13, c0, 1 // Get CONTEXTIDR
1506 mov r3, #0 // Load kernel asid
1507 mcr p15, 0, r3, c13, c0, 1 // Set CONTEXTIDR
1508 isb
1509 #endif
1510 mov r5, sp // Saved context in r5
1511
1512 cpsid i, #PSR_IRQ_MODE
1513
1514 str lr, [r5, SS_PC] // Save LR as the return PC
1515 mrs r4, spsr
1516 str r4, [r5, SS_CPSR] // Save the cpsr of the interrupted mode
1517
1518 ldr sp, [r9, ACT_CPUDATAP] // Get current cpu
1519 ldr sp, [sp, CPU_ISTACKPTR] // Set interrupt stack
1520
1521 #if CONFIG_TELEMETRY
1522 LOAD_ADDR(r2, telemetry_needs_record) // Check if a telemetry record was requested...
1523 mov r0, #0
1524 ldr r2, [r2]
1525 movs r2, r2
1526 beq 1f
1527 mov r1, #0 // (not a pmi record)
1528 bl EXT(telemetry_mark_curthread) // ...if so, mark the current thread...
1529 mrc p15, 0, r9, c13, c0, 4 // ...and restore the thread pointer from TPIDRPRW
1530 1:
1531 #endif
1532
1533 fleh_decirq_handler:
1534 ldr r2, [r9, ACT_PREEMPT_CNT] // Load preemption count
1535 add r2, r2, #1 // Increment count
1536 str r2, [r9, ACT_PREEMPT_CNT] // Update preemption count
1537 ldr r2, [r9, ACT_CPUDATAP] // Get current cpu
1538 str r5, [r2, CPU_INT_STATE] // Saved context in cpu_int_state
1539 ldr r3, [r2, CPU_STAT_IRQ] // Get IRQ count
1540 add r3, r3, #1 // Increment count
1541 str r3, [r2, CPU_STAT_IRQ] // Update IRQ count
1542 ldr r3, [r2, CPU_STAT_IRQ_WAKE] // Get post-wake IRQ count
1543 add r3, r3, #1 // Increment count
1544 str r3, [r2, CPU_STAT_IRQ_WAKE] // Update post-wake IRQ count
1545 #ifndef NO_KDEBUG
1546 LOAD_ADDR(r4, kdebug_enable)
1547 ldr r4, [r4]
1548 movs r4, r4
1549 movne r0, r5 // Pass saved context
1550 COND_EXTERN_BLNE(interrupt_trace)
1551 #endif
1552 bl EXT(interrupt_stats) // Record interrupt statistics
1553 mov r0, #0
1554 bl EXT(rtclock_intr) // Call second level exception handler
1555 #ifndef NO_KDEBUG
1556 movs r4, r4
1557 COND_EXTERN_BLNE(interrupt_trace_exit)
1558 #endif
1559
1560 b return_from_irq
1561
1562
1563 /*
1564 * First Level Exception Handler for FIQ
1565 * Current mode : FIQ
1566 * IRQ and FIQ are always disabled while running in FIQ handler
1567 * We do not permit nested interrupt.
1568 *
1569 * Saving area: from user : PCB.
1570 * from kernel : interrupt stack.
1571 *
1572 * We have 7 added shadow registers in FIQ mode for fast services.
1573 * So only we have to save is just 8 general registers and LR.
1574 * But if the current thread was running on user mode before the FIQ interrupt,
1575 * All user registers be saved for ast handler routine.
1576 */
1577 .text
1578 .align 2
1579 .globl EXT(fleh_fiq_generic)
1580
1581 LEXT(fleh_fiq_generic)
1582 str r11, [r10] // Clear the FIQ source
1583
1584 ldr r13, [r8, CPU_TIMEBASE_LOW] // Load TBL
1585 adds r13, r13, #1 // Increment TBL
1586 str r13, [r8, CPU_TIMEBASE_LOW] // Store TBL
1587 ldreq r13, [r8, CPU_TIMEBASE_HIGH] // Load TBU
1588 addeq r13, r13, #1 // Increment TBU
1589 streq r13, [r8, CPU_TIMEBASE_HIGH] // Store TBU
1590 subs r12, r12, #1 // Decrement, DEC
1591 str r12, [r8, CPU_DECREMENTER] // Store DEC
1592 subspl pc, lr, #4 // Return unless DEC < 0
1593 b EXT(fleh_dec)
1594
1595 .text
1596 .align 2
1597 .globl EXT(fleh_dec)
1598 LEXT(fleh_dec)
1599 mrs sp, spsr // Get the spsr
1600 sub lr, lr, #4
1601 tst sp, #0x0f // From user? or kernel?
1602 bne 2f
1603
1604 /* From user */
1605 mrc p15, 0, sp, c13, c0, 4 // Read TPIDRPRW
1606 add sp, sp, ACT_PCBDATA // Get User PCB
1607
1608 stmia sp, {r0-r12, sp, lr}^
1609 mov r7, #0 // Zero the frame pointer
1610 nop
1611 str lr, [sp, SS_PC]
1612
1613 mrs r4, spsr
1614 str r4, [sp, SS_CPSR]
1615 mov r5, sp
1616 sub sp, sp, ACT_PCBDATA // Get User PCB
1617 ldr sp, [sp, ACT_CPUDATAP] // Get current cpu
1618 ldr sp, [sp, CPU_ISTACKPTR] // Set interrupt stack
1619 mov r6, sp
1620 cpsid i, #PSR_SVC_MODE
1621 mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW
1622 ldr sp, [r9, TH_KSTACKPTR] // Set kernel stack
1623
1624 #if __ARM_VFP__
1625 add r0, r9, ACT_UVFP // Get the address of the user VFP save area
1626 bl EXT(vfp_save) // Save the current VFP state to ACT_UVFP
1627 mov r4, #FPSCR_DEFAULT // Load up the default FPSCR value...
1628 fmxr fpscr, r4 // And shove it into FPSCR
1629 #endif
1630 #if __ARM_USER_PROTECT__
1631 mrc p15, 0, r10, c2, c0, 0 // Get TTBR0
1632 ldr r3, [r9, ACT_KPTW_TTB] // Load kernel ttb
1633 mcr p15, 0, r3, c2, c0, 0 // Set TTBR0
1634 mrc p15, 0, r11, c13, c0, 1 // Get CONTEXTIDR
1635 mov r3, #0 // Load kernel asid
1636 mcr p15, 0, r3, c13, c0, 1 // Set CONTEXTIDR
1637 isb
1638 #endif
1639 mov r0, #1 // Mark this as coming from user context
1640 b 4f
1641
1642 2:
1643 /* From kernel */
1644 tst sp, #PSR_IRQF // Test for IRQ masked
1645 bne 3f // We're on the cpu_signal path
1646
1647 cpsid if, #PSR_SVC_MODE
1648
1649 sub sp, sp, EXC_CTX_SIZE
1650 stmia sp, {r0-r12}
1651 add r0, sp, EXC_CTX_SIZE
1652
1653 str r0, [sp, SS_SP] // Save supervisor mode sp
1654 str lr, [sp, SS_LR] // Save supervisor mode lr
1655
1656 mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW
1657
1658 #if __ARM_VFP__
1659 add r0, sp, SS_SIZE // Get vfp state pointer
1660 bic r0, #(VSS_ALIGN_NUM - 1) // Align to arm_vfpsaved_state alignment
1661 add r0, VSS_ALIGN // Get the actual vfp save area
1662 bl EXT(vfp_save) // Save the current VFP state to the stack
1663 mov r4, #FPSCR_DEFAULT // Load up the default FPSCR value...
1664 fmxr fpscr, r4 // And shove it into FPSCR
1665 #endif
1666 #if __ARM_USER_PROTECT__
1667 mrc p15, 0, r10, c2, c0, 0 // Get TTBR0
1668 ldr r3, [r9, ACT_KPTW_TTB] // Load kernel ttb
1669 mcr p15, 0, r3, c2, c0, 0 // Set TTBR0
1670 mrc p15, 0, r11, c13, c0, 1 // Get CONTEXTIDR
1671 mov r3, #0 // Load kernel asid
1672 mcr p15, 0, r3, c13, c0, 1 // Set CONTEXTIDR
1673 isb
1674 #endif
1675 mov r5, sp // Saved context in r5
1676
1677 cpsid if, #PSR_FIQ_MODE
1678
1679 mrc p15, 0, r1, c13, c0, 4 // Read TPIDRPRW
1680
1681 str lr, [r5, SS_PC] // Save LR as the return PC
1682 mrs r4, spsr
1683 str r4, [r5, SS_CPSR] // Save the cpsr of the interrupted mode
1684
1685 ldr r6, [r1, ACT_CPUDATAP] // Get current cpu
1686 ldr r6, [r6, CPU_ISTACKPTR] // Set interrupt stack
1687
1688 mov r0, #0 // Mark this as coming from kernel context
1689 b 4f
1690
1691 3:
1692 /* cpu_signal path */
1693 mrc p15, 0, sp, c13, c0, 4 // Read TPIDRPRW
1694 ldr sp, [sp, ACT_CPUDATAP] // Get current cpu
1695 ldr sp, [sp, CPU_FIQSTACKPTR] // Set fiq stack
1696 sub sp, sp, EXC_CTX_SIZE
1697 stmia sp, {r0-r12}
1698 str lr, [sp, SS_PC]
1699 mrs r4, spsr
1700 str r4, [sp, SS_CPSR]
1701 mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW
1702
1703 #if __ARM_VFP__
1704 add r0, sp, SS_SIZE // Get vfp state pointer
1705 bic r0, #(VSS_ALIGN_NUM - 1) // Align to arm_vfpsaved_state alignment
1706 add r0, VSS_ALIGN // Get the actual vfp save area
1707 bl EXT(vfp_save) // Save the current VFP state to the stack
1708 mov r4, #FPSCR_DEFAULT // Load up the default FPSCR value...
1709 fmxr fpscr, r4 // And shove it into FPSCR
1710 #endif
1711 #if __ARM_USER_PROTECT__
1712 mrc p15, 0, r10, c2, c0, 0 // Get TTBR0
1713 ldr r3, [r9, ACT_KPTW_TTB] // Load kernel ttb
1714 mcr p15, 0, r3, c2, c0, 0 // Set TTBR0
1715 mrc p15, 0, r11, c13, c0, 1 // Get CONTEXTIDR
1716 mov r3, #0 // Load kernel asid
1717 mcr p15, 0, r3, c13, c0, 1 // Set CONTEXTIDR
1718 isb
1719 #endif
1720
1721 ALIGN_STACK r0, r1
1722 mov r0, r8 // Get current cpu in arg 0
1723 mov r1, SIGPdec // Decrementer signal in arg1
1724 mov r2, #0
1725 mov r3, #0
1726 bl EXT(cpu_signal) // Call cpu_signal
1727 UNALIGN_STACK
1728
1729 mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW
1730
1731 #if __ARM_VFP__
1732 add r0, sp, SS_SIZE // Get vfp state pointer
1733 bic r0, #(VSS_ALIGN_NUM - 1) // Align to arm_vfpsaved_state alignment
1734 add r0, VSS_ALIGN // Get the actual vfp save area
1735 bl EXT(vfp_load) // Load the desired VFP state from the stack
1736 #endif
1737
1738 clrex // clear exclusive memory tag
1739 #if __ARM_ENABLE_WFE_
1740 sev
1741 #endif
1742 #if __ARM_USER_PROTECT__
1743 mcr p15, 0, r10, c2, c0, 0 // Set TTBR0
1744 mcr p15, 0, r11, c13, c0, 1 // Set CONTEXTIDR
1745 isb
1746 #endif
1747 ldr lr, [sp, SS_PC]
1748 ldmia sp, {r0-r12} // Restore saved registers
1749 movs pc, lr // Return from fiq
1750
1751 4:
1752 cpsid i, #PSR_IRQ_MODE
1753 cpsie f
1754 mov sp, r6 // Restore the stack pointer
1755 ALIGN_STACK r2, r3
1756 msr spsr_cxsf, r4 // Restore the spsr
1757 ldr r2, [r9, ACT_PREEMPT_CNT] // Load preemption count
1758 add r2, r2, #1 // Increment count
1759 str r2, [r9, ACT_PREEMPT_CNT] // Update preemption count
1760 ldr r4, [r9, ACT_CPUDATAP] // Get current cpu
1761 str r5, [r4, CPU_INT_STATE]
1762 ldr r3, [r4, CPU_STAT_IRQ] // Get IRQ count
1763 add r3, r3, #1 // Increment count
1764 str r3, [r4, CPU_STAT_IRQ] // Update IRQ count
1765 ldr r3, [r4, CPU_STAT_IRQ_WAKE] // Get post-wake IRQ count
1766 add r3, r3, #1 // Increment count
1767 str r3, [r4, CPU_STAT_IRQ_WAKE] // Update post-wake IRQ count
1768 #if !CONFIG_SKIP_PRECISE_USER_KERNEL_TIME
1769 movs r0, r0
1770 beq 5f
1771 mov r8, r0 // Stash our "from_user" boolean value
1772 bl EXT(timer_state_event_user_to_kernel)
1773 mov r0, r8 // Restore our "from_user" value
1774 mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW
1775 5:
1776 #endif
1777 #if CONFIG_TELEMETRY
1778 LOAD_ADDR(r4, telemetry_needs_record) // Check if a telemetry record was requested...
1779 ldr r4, [r4]
1780 movs r4, r4
1781 beq 6f
1782 mov r1, #0 // (not a PMI record)
1783 bl EXT(telemetry_mark_curthread) // ...if so, mark the current thread...
1784 mrc p15, 0, r9, c13, c0, 4 // ...and restore the thread pointer from TPIDRPRW
1785 6:
1786 #endif
1787
1788 #ifndef NO_KDEBUG
1789 LOAD_ADDR(r4, kdebug_enable)
1790 ldr r4, [r4]
1791 movs r4, r4
1792 ldrne r1, [r9, ACT_CPUDATAP] // Get current cpu
1793 ldrne r0, [r1, CPU_INT_STATE]
1794 COND_EXTERN_BLNE(interrupt_trace)
1795 #endif
1796 bl EXT(interrupt_stats) // Record interrupt statistics
1797 mov r0, #0
1798 bl EXT(rtclock_intr) // Call second level exception handler
1799 #ifndef NO_KDEBUG
1800 movs r4, r4
1801 COND_EXTERN_BLNE(interrupt_trace_exit)
1802 #endif
1803 UNALIGN_STACK
1804
1805 b return_from_irq
1806
1807 /*
1808 * void thread_syscall_return(kern_return_t r0)
1809 *
1810 */
1811 .text
1812 .align 2
1813 .globl EXT(thread_syscall_return)
1814
1815 LEXT(thread_syscall_return)
1816 mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW
1817 add r1, r9, ACT_PCBDATA // Get User PCB
1818 str r0, [r1, SS_R0] // set return value
1819 #ifndef NO_KDEBUG
1820 LOAD_ADDR(r4, kdebug_enable)
1821 ldr r4, [r4]
1822 movs r4, r4
1823 beq load_and_go_user
1824 ldr r12, [r1, SS_R12] // Load syscall number
1825 rsbs r1, r12, #0 // make the syscall positive (if negative)
1826 COND_EXTERN_BLGT(mach_syscall_trace_exit)
1827 #endif
1828 b load_and_go_user
1829
1830 /*
1831 * void thread_exception_return(void)
1832 * void thread_bootstrap_return(void)
1833 *
1834 */
1835 .text
1836 .globl EXT(thread_exception_return)
1837 .globl EXT(thread_bootstrap_return)
1838
1839 LEXT(thread_bootstrap_return)
1840 #if CONFIG_DTRACE
1841 bl EXT(dtrace_thread_bootstrap)
1842 #endif
1843 // Fall through
1844
1845 LEXT(thread_exception_return)
1846
1847 load_and_go_user:
1848 /*
1849 * Restore user mode states and go back to user mode
1850 */
1851 cpsid i // Disable irq
1852 mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW
1853
1854 mvn r0, #0
1855 str r0, [r9, TH_IOTIER_OVERRIDE] // Reset IO tier override to -1 before returning to user
1856
1857 ldr r8, [r9, ACT_CPUDATAP] // Get current cpu
1858 ldr r5, [r8, CPU_PENDING_AST] // Get ASTs
1859 cmp r5, #0 // Test if ASTs pending
1860 beq return_to_user_now // Branch if no ASTs
1861
1862 bl EXT(ast_taken_user) // Handle all ASTs (may continue via thread_exception_return)
1863
1864 mrc p15, 0, r9, c13, c0, 4 // Reload r9 from TPIDRPRW
1865 b load_and_go_user // Loop back
1866
1867 return_to_user_now:
1868
1869 #if MACH_ASSERT
1870 /*
1871 * Assert that the preemption level is zero prior to the return to user space
1872 */
1873 ldr r1, [r9, ACT_PREEMPT_CNT] // Load preemption count
1874 cmp r1, #0 // Test
1875 bne L_lagu_preempt_panic // Panic if not zero
1876
1877 /*
1878 * Assert that the preemption level is zero prior to the return to user space
1879 */
1880 ldr r2, [r9, TH_RWLOCK_CNT] // Load RW lock count
1881 cmp r2, #0 // Test
1882 bne L_lagu_rwlock_cnt_panic // Panic if not zero
1883 #endif
1884
1885 /*
1886 * Assert that we aren't leaking KHEAP_TEMP allocations prior to the return to user space
1887 */
1888 ldr r1, [r9, TH_TMP_ALLOC_CNT] // Load temp alloc count
1889 cmp r1, #0 // Test
1890 bne L_lagu_temp_alloc_cnt_panic // Panic if not zero
1891
1892 #if !CONFIG_SKIP_PRECISE_USER_KERNEL_TIME
1893 bl EXT(timer_state_event_kernel_to_user)
1894 mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW
1895 ldr r8, [r9, ACT_CPUDATAP] // Get current cpu data
1896 #endif /* !CONFIG_SKIP_PRECISE_USER_KERNEL_TIME */
1897 #if __ARM_DEBUG__ >= 6
1898 ldr r0, [r9, ACT_DEBUGDATA]
1899 ldr r6, [r8, CPU_USER_DEBUG]
1900 cmp r0, r6 // test if debug registers need to be changed
1901 beq 1f
1902 bl EXT(arm_debug_set) // argument is already in r0
1903 mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW
1904 1:
1905 #endif
1906 #if __ARM_VFP__
1907 add r0, r9, ACT_UVFP // Get the address of the user VFP save area
1908 bl EXT(vfp_load) // Load the desired VFP state from ACT_UVFP
1909 #endif
1910 add r0, r9, ACT_PCBDATA // Get User PCB
1911 ldr r4, [r0, SS_CPSR] // Get saved cpsr
1912 and r3, r4, #PSR_MODE_MASK // Extract current mode
1913 cmp r3, #PSR_USER_MODE // Check user mode
1914 movne r0, r3
1915 bne EXT(ExceptionVectorPanic)
1916
1917 msr spsr_cxsf, r4 // Restore spsr(user mode cpsr)
1918 mov sp, r0 // Get User PCB
1919
1920 clrex // clear exclusive memory tag
1921 #if __ARM_ENABLE_WFE_
1922 sev
1923 #endif
1924 #if __ARM_USER_PROTECT__
1925 ldr r3, [r9, ACT_UPTW_TTB] // Load thread ttb
1926 mcr p15, 0, r3, c2, c0, 0 // Set TTBR0
1927 ldr r2, [r9, ACT_ASID] // Load thread asid
1928 mcr p15, 0, r2, c13, c0, 1
1929 isb
1930 #endif
1931 ldr lr, [sp, SS_PC] // Restore user mode pc
1932 ldmia sp, {r0-r12, sp, lr}^ // Restore the other user mode registers
1933 nop // Hardware problem
1934 movs pc, lr // Return to user
1935
1936 /*
1937 * r1: tmp alloc count
1938 * r9: current_thread()
1939 */
1940 L_lagu_temp_alloc_cnt_panic:
1941 mov r0, r9 // Thread argument
1942 blx EXT(kheap_temp_leak_panic) // Finally, panic
1943
1944 #if MACH_ASSERT
1945 /*
1946 * r1: current preemption count
1947 * r9: current_thread()
1948 */
1949 L_lagu_preempt_panic:
1950 adr r0, L_lagu_preempt_panic_str // Load the panic string...
1951 blx EXT(panic) // Finally, panic
1952
1953 /*
1954 * r2: rwlock count
1955 * r9: current_thread()
1956 */
1957 L_lagu_rwlock_cnt_panic:
1958 adr r0, L_lagu_rwlock_cnt_panic_str // Load the panic string...
1959 mov r1, r9 // Thread argument for panic string
1960 blx EXT(panic) // Finally, panic
1961
1962 .align 2
1963 L_lagu_preempt_panic_str:
1964 .asciz "load_and_go_user: preemption_level %d"
1965 .align 2
1966
1967 .align 2
1968 L_lagu_rwlock_cnt_panic_str:
1969 .asciz "load_and_go_user: RW lock count not 0 on thread %p (%u)"
1970 .align 2
1971 #endif /* MACH_ASSERT */
1972
1973 .align 2
1974 L_evimpanic_str:
1975 .ascii "Exception Vector: Illegal Mode: 0x%08X\n\000"
1976 .align 2
1977
1978 .text
1979 .align 2
1980 .globl EXT(ExceptionVectorPanic)
1981
1982 LEXT(ExceptionVectorPanic)
1983 cpsid i, #PSR_SVC_MODE
1984 ALIGN_STACK r1, r2
1985 mov r1, r0
1986 adr r0, L_evimpanic_str
1987 blx EXT(panic)
1988 b .
1989
1990 #include "globals_asm.h"
1991
1992 LOAD_ADDR_GEN_DEF(mach_trap_table)
1993 LOAD_ADDR_GEN_DEF(kern_invalid)
1994
1995 /* vim: set ts=4: */
mirror of XNU