]> git.saurik.com Git - apple/xnu.git/blob - osfmk/arm64/machine_routines_asm.s
projects / apple / xnu.git / blob
? search:


64fd61152e622e2d9c860b6991de65c9741e780c
[apple/xnu.git] / osfmk / arm64 / machine_routines_asm.s
1 /*
2 * Copyright (c) 2007-2015 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <machine/asm.h>
30 #include <arm64/machine_machdep.h>
31 #include <arm64/proc_reg.h>
32 #include <arm/pmap.h>
33 #include <pexpert/arm64/board_config.h>
34 #include <sys/errno.h>
35 #include "assym.s"
36
37
38 #if defined(HAS_APPLE_PAC)
39 /*
40 * void
41 * ml_set_kernelkey_enabled(boolean_t enable)
42 *
43 * Toggle pointer auth kernel domain key diversification. Assembly to prevent compiler reordering.
44 *
45 */
46
47 .align 2
48 .globl EXT(ml_set_kernelkey_enabled)
49 LEXT(ml_set_kernelkey_enabled)
50 mrs x1, ARM64_REG_APCTL_EL1
51 orr x2, x1, #APCTL_EL1_KernKeyEn
52 and x1, x1, #~APCTL_EL1_KernKeyEn
53 cmp w0, #0
54 csel x1, x1, x2, eq
55 msr ARM64_REG_APCTL_EL1, x1
56 isb
57 ret
58
59 #endif /* defined(HAS_APPLE_PAC) */
60
61
62
63 /* uint32_t get_fpscr(void):
64 * Returns (FPSR | FPCR).
65 */
66 .align 2
67 .globl EXT(get_fpscr)
68 LEXT(get_fpscr)
69 #if __ARM_VFP__
70 mrs x1, FPSR // Grab FPSR
71 mov x4, #(FPSR_MASK & 0xFFFF)
72 mov x5, #(FPSR_MASK & 0xFFFF0000)
73 orr x0, x4, x5
74 and x1, x1, x0 // Be paranoid, and clear bits we expect to
75 // be clear
76 mrs x2, FPCR // Grab FPCR
77 mov x4, #(FPCR_MASK & 0xFFFF)
78 mov x5, #(FPCR_MASK & 0xFFFF0000)
79 orr x0, x4, x5
80 and x2, x2, x0 // Be paranoid, and clear bits we expect to
81 // be clear
82 orr x0, x1, x2 // OR them to get FPSCR equivalent state
83 #else
84 mov x0, #0
85 #endif
86 ret
87 .align 2
88 .globl EXT(set_fpscr)
89 /* void set_fpscr(uint32_t value):
90 * Set the FPCR and FPSR registers, based on the given value; a
91 * noteworthy point is that unlike 32-bit mode, 64-bit mode FPSR
92 * and FPCR are not responsible for condition codes.
93 */
94 LEXT(set_fpscr)
95 #if __ARM_VFP__
96 mov x4, #(FPSR_MASK & 0xFFFF)
97 mov x5, #(FPSR_MASK & 0xFFFF0000)
98 orr x1, x4, x5
99 and x1, x1, x0 // Clear the bits that don't apply to FPSR
100 mov x4, #(FPCR_MASK & 0xFFFF)
101 mov x5, #(FPCR_MASK & 0xFFFF0000)
102 orr x2, x4, x5
103 and x2, x2, x0 // Clear the bits that don't apply to FPCR
104 msr FPSR, x1 // Write FPCR
105 msr FPCR, x2 // Write FPSR
106 dsb ish // FPCR requires synchronization
107 #endif
108 ret
109
110 /*
111 * void update_mdscr(unsigned long clear, unsigned long set)
112 * Clears and sets the specified bits in MDSCR_EL1.
113 *
114 * Setting breakpoints in EL1 is effectively a KTRR bypass. The ability to do so is
115 * controlled by MDSCR.KDE. The MSR to set MDSCR must be present to allow
116 * self-hosted user mode debug. Any checks before the MRS can be skipped with ROP,
117 * so we need to put the checks after the MRS where they can't be skipped. That
118 * still leaves a small window if a breakpoint is set on the instruction
119 * immediately after the MRS. To handle that, we also do a check and then set of
120 * the breakpoint control registers. This allows us to guarantee that a given
121 * core will never have both KDE set and a breakpoint targeting EL1.
122 *
123 * If KDE gets set, unset it and then panic
124 */
125 .align 2
126 .globl EXT(update_mdscr)
127 LEXT(update_mdscr)
128 mov x4, #0
129 mrs x2, MDSCR_EL1
130 bic x2, x2, x0
131 orr x2, x2, x1
132 1:
133 bic x2, x2, #0x2000
134 msr MDSCR_EL1, x2
135 #if defined(CONFIG_KERNEL_INTEGRITY)
136 /*
137 * verify KDE didn't get set (including via ROP)
138 * If set, clear it and then panic
139 */
140 ands x3, x2, #0x2000
141 orr x4, x4, x3
142 bne 1b
143 cmp x4, xzr
144 b.ne Lupdate_mdscr_panic
145 #endif
146 ret
147
148 Lupdate_mdscr_panic:
149 adrp x0, Lupdate_mdscr_panic_str@page
150 add x0, x0, Lupdate_mdscr_panic_str@pageoff
151 b EXT(panic)
152 b .
153
154 Lupdate_mdscr_panic_str:
155 .asciz "MDSCR.KDE was set"
156
157
158 /*
159 * Set MMU Translation Table Base Alternate
160 */
161 .text
162 .align 2
163 .globl EXT(set_mmu_ttb_alternate)
164 LEXT(set_mmu_ttb_alternate)
165 dsb sy
166 #if defined(KERNEL_INTEGRITY_KTRR)
167 mov x1, lr
168 bl EXT(pinst_set_ttbr1)
169 mov lr, x1
170 #else
171 msr TTBR1_EL1, x0
172 #endif /* defined(KERNEL_INTEGRITY_KTRR) */
173 isb sy
174 ret
175
176 .text
177 .align 2
178 .globl EXT(set_mmu_ttb)
179 LEXT(set_mmu_ttb)
180 #if __ARM_KERNEL_PROTECT__
181 /* All EL1-mode ASIDs are odd. */
182 orr x0, x0, #(1 << TTBR_ASID_SHIFT)
183 #endif /* __ARM_KERNEL_PROTECT__ */
184 dsb ish
185 msr TTBR0_EL1, x0
186 isb sy
187 ret
188
189 /*
190 * set AUX control register
191 */
192 .text
193 .align 2
194 .globl EXT(set_aux_control)
195 LEXT(set_aux_control)
196 msr ACTLR_EL1, x0
197 // Synchronize system
198 dsb sy
199 isb sy
200 ret
201
202 #if __ARM_KERNEL_PROTECT__
203 .text
204 .align 2
205 .globl EXT(set_vbar_el1)
206 LEXT(set_vbar_el1)
207 #if defined(KERNEL_INTEGRITY_KTRR)
208 b EXT(pinst_set_vbar)
209 #else
210 msr VBAR_EL1, x0
211 ret
212 #endif
213 #endif /* __ARM_KERNEL_PROTECT__ */
214
215
216 /*
217 * set translation control register
218 */
219 .text
220 .align 2
221 .globl EXT(set_tcr)
222 LEXT(set_tcr)
223 #if defined(APPLE_ARM64_ARCH_FAMILY)
224 // Assert that T0Z is always equal to T1Z
225 eor x1, x0, x0, lsr #(TCR_T1SZ_SHIFT - TCR_T0SZ_SHIFT)
226 and x1, x1, #(TCR_TSZ_MASK << TCR_T0SZ_SHIFT)
227 cbnz x1, L_set_tcr_panic
228 #if defined(KERNEL_INTEGRITY_KTRR)
229 mov x1, lr
230 bl EXT(pinst_set_tcr)
231 mov lr, x1
232 #else
233 msr TCR_EL1, x0
234 #endif /* defined(KERNEL_INTRITY_KTRR) */
235 isb sy
236 ret
237
238 L_set_tcr_panic:
239 PUSH_FRAME
240 sub sp, sp, #16
241 str x0, [sp]
242 adr x0, L_set_tcr_panic_str
243 BRANCH_EXTERN panic
244
245 L_set_locked_reg_panic:
246 PUSH_FRAME
247 sub sp, sp, #16
248 str x0, [sp]
249 adr x0, L_set_locked_reg_panic_str
250 BRANCH_EXTERN panic
251 b .
252
253 L_set_tcr_panic_str:
254 .asciz "set_tcr: t0sz, t1sz not equal (%llx)\n"
255
256
257 L_set_locked_reg_panic_str:
258 .asciz "attempt to set locked register: (%llx)\n"
259 #else
260 #if defined(KERNEL_INTEGRITY_KTRR)
261 mov x1, lr
262 bl EXT(pinst_set_tcr)
263 mov lr, x1
264 #else
265 msr TCR_EL1, x0
266 #endif
267 isb sy
268 ret
269 #endif // defined(APPLE_ARM64_ARCH_FAMILY)
270
271 /*
272 * MMU kernel virtual to physical address translation
273 */
274 .text
275 .align 2
276 .globl EXT(mmu_kvtop)
277 LEXT(mmu_kvtop)
278 mrs x2, DAIF // Load current DAIF
279 msr DAIFSet, #(DAIFSC_IRQF | DAIFSC_FIQF) // Disable IRQ
280 at s1e1r, x0 // Translation Stage 1 EL1
281 mrs x1, PAR_EL1 // Read result
282 msr DAIF, x2 // Restore interrupt state
283 tbnz x1, #0, L_mmu_kvtop_invalid // Test Translation not valid
284 bfm x1, x0, #0, #11 // Add page offset
285 and x0, x1, #0x0000ffffffffffff // Clear non-address bits
286 ret
287 L_mmu_kvtop_invalid:
288 mov x0, #0 // Return invalid
289 ret
290
291 /*
292 * MMU user virtual to physical address translation
293 */
294 .text
295 .align 2
296 .globl EXT(mmu_uvtop)
297 LEXT(mmu_uvtop)
298 lsr x8, x0, #56 // Extract top byte
299 cbnz x8, L_mmu_uvtop_invalid // Tagged pointers are invalid
300 mrs x2, DAIF // Load current DAIF
301 msr DAIFSet, #(DAIFSC_IRQF | DAIFSC_FIQF) // Disable IRQ
302 at s1e0r, x0 // Translation Stage 1 EL0
303 mrs x1, PAR_EL1 // Read result
304 msr DAIF, x2 // Restore interrupt state
305 tbnz x1, #0, L_mmu_uvtop_invalid // Test Translation not valid
306 bfm x1, x0, #0, #11 // Add page offset
307 and x0, x1, #0x0000ffffffffffff // Clear non-address bits
308 ret
309 L_mmu_uvtop_invalid:
310 mov x0, #0 // Return invalid
311 ret
312
313 /*
314 * MMU kernel virtual to physical address preflight write access
315 */
316 .text
317 .align 2
318 .globl EXT(mmu_kvtop_wpreflight)
319 LEXT(mmu_kvtop_wpreflight)
320 mrs x2, DAIF // Load current DAIF
321 msr DAIFSet, #(DAIFSC_IRQF | DAIFSC_FIQF) // Disable IRQ
322 at s1e1w, x0 // Translation Stage 1 EL1
323 mrs x1, PAR_EL1 // Read result
324 msr DAIF, x2 // Restore interrupt state
325 tbnz x1, #0, L_mmu_kvtop_wpreflight_invalid // Test Translation not valid
326 bfm x1, x0, #0, #11 // Add page offset
327 and x0, x1, #0x0000ffffffffffff // Clear non-address bits
328 ret
329 L_mmu_kvtop_wpreflight_invalid:
330 mov x0, #0 // Return invalid
331 ret
332
333 /*
334 * SET_RECOVERY_HANDLER
335 *
336 * Sets up a page fault recovery handler
337 *
338 * arg0 - persisted thread pointer
339 * arg1 - persisted recovery handler
340 * arg2 - scratch reg
341 * arg3 - recovery label
342 */
343 .macro SET_RECOVERY_HANDLER
344 mrs $0, TPIDR_EL1 // Load thread pointer
345 adrp $2, $3@page // Load the recovery handler address
346 add $2, $2, $3@pageoff
347 #if defined(HAS_APPLE_PAC)
348 add $1, $0, TH_RECOVER
349 movk $1, #PAC_DISCRIMINATOR_RECOVER, lsl 48
350 pacia $2, $1 // Sign with IAKey + blended discriminator
351 #endif
352
353 ldr $1, [$0, TH_RECOVER] // Save previous recovery handler
354 str $2, [$0, TH_RECOVER] // Set new signed recovery handler
355 .endmacro
356
357 /*
358 * CLEAR_RECOVERY_HANDLER
359 *
360 * Clears page fault handler set by SET_RECOVERY_HANDLER
361 *
362 * arg0 - thread pointer saved by SET_RECOVERY_HANDLER
363 * arg1 - old recovery handler saved by SET_RECOVERY_HANDLER
364 */
365 .macro CLEAR_RECOVERY_HANDLER
366 str $1, [$0, TH_RECOVER] // Restore the previous recovery handler
367 .endmacro
368
369
370 .text
371 .align 2
372 copyio_error:
373 CLEAR_RECOVERY_HANDLER x10, x11
374 mov x0, #EFAULT // Return an EFAULT error
375 POP_FRAME
376 ARM64_STACK_EPILOG
377
378 /*
379 * int _bcopyin(const char *src, char *dst, vm_size_t len)
380 */
381 .text
382 .align 2
383 .globl EXT(_bcopyin)
384 LEXT(_bcopyin)
385 ARM64_STACK_PROLOG
386 PUSH_FRAME
387 SET_RECOVERY_HANDLER x10, x11, x3, copyio_error
388 /* If len is less than 16 bytes, just do a bytewise copy */
389 cmp x2, #16
390 b.lt 2f
391 sub x2, x2, #16
392 1:
393 /* 16 bytes at a time */
394 ldp x3, x4, [x0], #16
395 stp x3, x4, [x1], #16
396 subs x2, x2, #16
397 b.ge 1b
398 /* Fixup the len and test for completion */
399 adds x2, x2, #16
400 b.eq 3f
401 2: /* Bytewise */
402 subs x2, x2, #1
403 ldrb w3, [x0], #1
404 strb w3, [x1], #1
405 b.hi 2b
406 3:
407 CLEAR_RECOVERY_HANDLER x10, x11
408 mov x0, #0
409 POP_FRAME
410 ARM64_STACK_EPILOG
411
412 /*
413 * int _copyin_atomic32(const char *src, uint32_t *dst)
414 */
415 .text
416 .align 2
417 .globl EXT(_copyin_atomic32)
418 LEXT(_copyin_atomic32)
419 ARM64_STACK_PROLOG
420 PUSH_FRAME
421 SET_RECOVERY_HANDLER x10, x11, x3, copyio_error
422 ldr w8, [x0]
423 str w8, [x1]
424 mov x0, #0
425 CLEAR_RECOVERY_HANDLER x10, x11
426 POP_FRAME
427 ARM64_STACK_EPILOG
428
429 /*
430 * int _copyin_atomic32_wait_if_equals(const char *src, uint32_t value)
431 */
432 .text
433 .align 2
434 .globl EXT(_copyin_atomic32_wait_if_equals)
435 LEXT(_copyin_atomic32_wait_if_equals)
436 ARM64_STACK_PROLOG
437 PUSH_FRAME
438 SET_RECOVERY_HANDLER x10, x11, x3, copyio_error
439 ldxr w8, [x0]
440 cmp w8, w1
441 mov x0, ESTALE
442 b.ne 1f
443 mov x0, #0
444 wfe
445 1:
446 clrex
447 CLEAR_RECOVERY_HANDLER x10, x11
448 POP_FRAME
449 ARM64_STACK_EPILOG
450
451 /*
452 * int _copyin_atomic64(const char *src, uint32_t *dst)
453 */
454 .text
455 .align 2
456 .globl EXT(_copyin_atomic64)
457 LEXT(_copyin_atomic64)
458 ARM64_STACK_PROLOG
459 PUSH_FRAME
460 SET_RECOVERY_HANDLER x10, x11, x3, copyio_error
461 ldr x8, [x0]
462 str x8, [x1]
463 mov x0, #0
464 CLEAR_RECOVERY_HANDLER x10, x11
465 POP_FRAME
466 ARM64_STACK_EPILOG
467
468
469 /*
470 * int _copyout_atomic32(uint32_t value, char *dst)
471 */
472 .text
473 .align 2
474 .globl EXT(_copyout_atomic32)
475 LEXT(_copyout_atomic32)
476 ARM64_STACK_PROLOG
477 PUSH_FRAME
478 SET_RECOVERY_HANDLER x10, x11, x3, copyio_error
479 str w0, [x1]
480 mov x0, #0
481 CLEAR_RECOVERY_HANDLER x10, x11
482 POP_FRAME
483 ARM64_STACK_EPILOG
484
485 /*
486 * int _copyout_atomic64(uint64_t value, char *dst)
487 */
488 .text
489 .align 2
490 .globl EXT(_copyout_atomic64)
491 LEXT(_copyout_atomic64)
492 ARM64_STACK_PROLOG
493 PUSH_FRAME
494 SET_RECOVERY_HANDLER x10, x11, x3, copyio_error
495 str x0, [x1]
496 mov x0, #0
497 CLEAR_RECOVERY_HANDLER x10, x11
498 POP_FRAME
499 ARM64_STACK_EPILOG
500
501
502 /*
503 * int _bcopyout(const char *src, char *dst, vm_size_t len)
504 */
505 .text
506 .align 2
507 .globl EXT(_bcopyout)
508 LEXT(_bcopyout)
509 ARM64_STACK_PROLOG
510 PUSH_FRAME
511 SET_RECOVERY_HANDLER x10, x11, x3, copyio_error
512 /* If len is less than 16 bytes, just do a bytewise copy */
513 cmp x2, #16
514 b.lt 2f
515 sub x2, x2, #16
516 1:
517 /* 16 bytes at a time */
518 ldp x3, x4, [x0], #16
519 stp x3, x4, [x1], #16
520 subs x2, x2, #16
521 b.ge 1b
522 /* Fixup the len and test for completion */
523 adds x2, x2, #16
524 b.eq 3f
525 2: /* Bytewise */
526 subs x2, x2, #1
527 ldrb w3, [x0], #1
528 strb w3, [x1], #1
529 b.hi 2b
530 3:
531 CLEAR_RECOVERY_HANDLER x10, x11
532 mov x0, #0
533 POP_FRAME
534 ARM64_STACK_EPILOG
535
536 /*
537 * int _bcopyinstr(
538 * const user_addr_t user_addr,
539 * char *kernel_addr,
540 * vm_size_t max,
541 * vm_size_t *actual)
542 */
543 .text
544 .align 2
545 .globl EXT(_bcopyinstr)
546 LEXT(_bcopyinstr)
547 ARM64_STACK_PROLOG
548 PUSH_FRAME
549 adr x4, Lcopyinstr_error // Get address for recover
550 mrs x10, TPIDR_EL1 // Get thread pointer
551 ldr x11, [x10, TH_RECOVER] // Save previous recover
552
553 #if defined(HAS_APPLE_PAC)
554 add x5, x10, TH_RECOVER // Sign new pointer with IAKey + blended discriminator
555 movk x5, #PAC_DISCRIMINATOR_RECOVER, lsl 48
556 pacia x4, x5
557 #endif
558 str x4, [x10, TH_RECOVER] // Store new recover
559
560 mov x4, #0 // x4 - total bytes copied
561 Lcopyinstr_loop:
562 ldrb w5, [x0], #1 // Load a byte from the user source
563 strb w5, [x1], #1 // Store a byte to the kernel dest
564 add x4, x4, #1 // Increment bytes copied
565 cbz x5, Lcopyinstr_done // If this byte is null, we're done
566 cmp x4, x2 // If we're out of space, return an error
567 b.ne Lcopyinstr_loop
568 Lcopyinstr_too_long:
569 mov x5, #ENAMETOOLONG // Set current byte to error code for later return
570 Lcopyinstr_done:
571 str x4, [x3] // Return number of bytes copied
572 mov x0, x5 // Set error code (0 on success, ENAMETOOLONG on failure)
573 b Lcopyinstr_exit
574 Lcopyinstr_error:
575 mov x0, #EFAULT // Return EFAULT on error
576 Lcopyinstr_exit:
577 str x11, [x10, TH_RECOVER] // Restore old recover
578 POP_FRAME
579 ARM64_STACK_EPILOG
580
581 /*
582 * int copyinframe(const vm_address_t frame_addr, char *kernel_addr, bool is64bit)
583 *
584 * Safely copy sixteen bytes (the fixed top of an ARM64 frame) from
585 * either user or kernel memory, or 8 bytes (AArch32) from user only.
586 *
587 * x0 : address of frame to copy.
588 * x1 : kernel address at which to store data.
589 * w2 : whether to copy an AArch32 or AArch64 frame.
590 * x3 : temp
591 * x5 : temp (kernel virtual base)
592 * x9 : temp
593 * x10 : thread pointer (set by SET_RECOVERY_HANDLER)
594 * x11 : old recovery function (set by SET_RECOVERY_HANDLER)
595 * x12, x13 : backtrace data
596 *
597 */
598 .text
599 .align 2
600 .globl EXT(copyinframe)
601 LEXT(copyinframe)
602 ARM64_STACK_PROLOG
603 PUSH_FRAME
604 SET_RECOVERY_HANDLER x10, x11, x3, copyio_error
605 cbnz w2, Lcopyinframe64 // Check frame size
606 adrp x5, EXT(gVirtBase)@page // For 32-bit frame, make sure we're not trying to copy from kernel
607 add x5, x5, EXT(gVirtBase)@pageoff
608 ldr x5, [x5]
609 cmp x5, x0 // See if address is in kernel virtual range
610 b.hi Lcopyinframe32 // If below kernel virtual range, proceed.
611 mov w0, #EFAULT // Should never have a 32-bit frame in kernel virtual range
612 b Lcopyinframe_done
613
614 Lcopyinframe32:
615 ldr x12, [x0] // Copy 8 bytes
616 str x12, [x1]
617 mov w0, #0 // Success
618 b Lcopyinframe_done
619
620 Lcopyinframe64:
621 mov x3, VM_MIN_KERNEL_ADDRESS // Check if kernel address
622 orr x9, x0, TBI_MASK // Hide tags in address comparison
623 cmp x9, x3 // If in kernel address range, skip tag test
624 b.hs Lcopyinframe_valid
625 tst x0, TBI_MASK // Detect tagged pointers
626 b.eq Lcopyinframe_valid
627 mov w0, #EFAULT // Tagged address, fail
628 b Lcopyinframe_done
629 Lcopyinframe_valid:
630 ldp x12, x13, [x0] // Copy 16 bytes
631 stp x12, x13, [x1]
632 mov w0, #0 // Success
633
634 Lcopyinframe_done:
635 CLEAR_RECOVERY_HANDLER x10, x11
636 POP_FRAME
637 ARM64_STACK_EPILOG
638
639
640 /*
641 * uint32_t arm_debug_read_dscr(void)
642 */
643 .text
644 .align 2
645 .globl EXT(arm_debug_read_dscr)
646 LEXT(arm_debug_read_dscr)
647 PANIC_UNIMPLEMENTED
648
649 /*
650 * void arm_debug_set_cp14(arm_debug_state_t *debug_state)
651 *
652 * Set debug registers to match the current thread state
653 * (NULL to disable). Assume 6 breakpoints and 2
654 * watchpoints, since that has been the case in all cores
655 * thus far.
656 */
657 .text
658 .align 2
659 .globl EXT(arm_debug_set_cp14)
660 LEXT(arm_debug_set_cp14)
661 PANIC_UNIMPLEMENTED
662
663 #if defined(APPLE_ARM64_ARCH_FAMILY)
664 /*
665 * Note: still have to ISB before executing wfi!
666 */
667 .text
668 .align 2
669 .globl EXT(arm64_prepare_for_sleep)
670 LEXT(arm64_prepare_for_sleep)
671 PUSH_FRAME
672
673 #if defined(APPLETYPHOON)
674 // <rdar://problem/15827409>
675 mrs x0, ARM64_REG_HID2 // Read HID2
676 orr x0, x0, #(ARM64_REG_HID2_disMMUmtlbPrefetch) // Set HID.DisableMTLBPrefetch
677 msr ARM64_REG_HID2, x0 // Write HID2
678 dsb sy
679 isb sy
680 #endif
681
682 #if __ARM_GLOBAL_SLEEP_BIT__
683 // Enable deep sleep
684 mrs x1, ARM64_REG_ACC_OVRD
685 orr x1, x1, #(ARM64_REG_ACC_OVRD_enDeepSleep)
686 and x1, x1, #(~(ARM64_REG_ACC_OVRD_disL2Flush4AccSlp_mask))
687 orr x1, x1, #( ARM64_REG_ACC_OVRD_disL2Flush4AccSlp_deepsleep)
688 and x1, x1, #(~(ARM64_REG_ACC_OVRD_ok2PwrDnSRM_mask))
689 orr x1, x1, #( ARM64_REG_ACC_OVRD_ok2PwrDnSRM_deepsleep)
690 and x1, x1, #(~(ARM64_REG_ACC_OVRD_ok2TrDnLnk_mask))
691 orr x1, x1, #( ARM64_REG_ACC_OVRD_ok2TrDnLnk_deepsleep)
692 and x1, x1, #(~(ARM64_REG_ACC_OVRD_ok2PwrDnCPM_mask))
693 orr x1, x1, #( ARM64_REG_ACC_OVRD_ok2PwrDnCPM_deepsleep)
694 msr ARM64_REG_ACC_OVRD, x1
695
696
697 #else
698 // Enable deep sleep
699 mov x1, ARM64_REG_CYC_CFG_deepSleep
700 msr ARM64_REG_CYC_CFG, x1
701 #endif
702 // Set "OK to power down" (<rdar://problem/12390433>)
703 mrs x0, ARM64_REG_CYC_OVRD
704 orr x0, x0, #(ARM64_REG_CYC_OVRD_ok2pwrdn_force_down)
705 msr ARM64_REG_CYC_OVRD, x0
706
707 #if defined(APPLEMONSOON)
708 ARM64_IS_PCORE x0
709 cbz x0, Lwfi_inst // skip if not p-core
710
711 /* <rdar://problem/32512947>: Flush the GUPS prefetcher prior to
712 * wfi. A Skye HW bug can cause the GUPS prefetcher on p-cores
713 * to be left with valid entries that fail to drain if a
714 * subsequent wfi is issued. This can prevent the core from
715 * power-gating. For the idle case that is recoverable, but
716 * for the deep-sleep (S2R) case in which cores MUST power-gate,
717 * it can lead to a hang. This can be prevented by disabling
718 * and re-enabling GUPS, which forces the prefetch queue to
719 * drain. This should be done as close to wfi as possible, i.e.
720 * at the very end of arm64_prepare_for_sleep(). */
721 mrs x0, ARM64_REG_HID10
722 orr x0, x0, #(ARM64_REG_HID10_DisHwpGups)
723 msr ARM64_REG_HID10, x0
724 isb sy
725 and x0, x0, #(~(ARM64_REG_HID10_DisHwpGups))
726 msr ARM64_REG_HID10, x0
727 isb sy
728 #endif
729 Lwfi_inst:
730 dsb sy
731 isb sy
732 wfi
733 b Lwfi_inst
734
735 /*
736 * Force WFI to use clock gating only
737 *
738 */
739 .text
740 .align 2
741 .globl EXT(arm64_force_wfi_clock_gate)
742 LEXT(arm64_force_wfi_clock_gate)
743 ARM64_STACK_PROLOG
744 PUSH_FRAME
745
746 mrs x0, ARM64_REG_CYC_OVRD
747 orr x0, x0, #(ARM64_REG_CYC_OVRD_ok2pwrdn_force_up)
748 msr ARM64_REG_CYC_OVRD, x0
749
750 POP_FRAME
751 ARM64_STACK_EPILOG
752
753
754
755 #if defined(APPLETYPHOON)
756
757 .text
758 .align 2
759 .globl EXT(typhoon_prepare_for_wfi)
760
761 LEXT(typhoon_prepare_for_wfi)
762 PUSH_FRAME
763
764 // <rdar://problem/15827409>
765 mrs x0, ARM64_REG_HID2 // Read HID2
766 orr x0, x0, #(ARM64_REG_HID2_disMMUmtlbPrefetch) // Set HID.DisableMTLBPrefetch
767 msr ARM64_REG_HID2, x0 // Write HID2
768 dsb sy
769 isb sy
770
771 POP_FRAME
772 ret
773
774
775 .text
776 .align 2
777 .globl EXT(typhoon_return_from_wfi)
778 LEXT(typhoon_return_from_wfi)
779 PUSH_FRAME
780
781 // <rdar://problem/15827409>
782 mrs x0, ARM64_REG_HID2 // Read HID2
783 mov x1, #(ARM64_REG_HID2_disMMUmtlbPrefetch) //
784 bic x0, x0, x1 // Clear HID.DisableMTLBPrefetchMTLBPrefetch
785 msr ARM64_REG_HID2, x0 // Write HID2
786 dsb sy
787 isb sy
788
789 POP_FRAME
790 ret
791 #endif
792
793 #ifdef APPLETYPHOON
794
795 #define HID0_DEFEATURES_1 0x0000a0c000064010ULL
796 #define HID1_DEFEATURES_1 0x000000004005bf20ULL
797 #define HID2_DEFEATURES_1 0x0000000000102074ULL
798 #define HID3_DEFEATURES_1 0x0000000000400003ULL
799 #define HID4_DEFEATURES_1 0x83ff00e100000268ULL
800 #define HID7_DEFEATURES_1 0x000000000000000eULL
801
802 #define HID0_DEFEATURES_2 0x0000a1c000020010ULL
803 #define HID1_DEFEATURES_2 0x000000000005d720ULL
804 #define HID2_DEFEATURES_2 0x0000000000002074ULL
805 #define HID3_DEFEATURES_2 0x0000000000400001ULL
806 #define HID4_DEFEATURES_2 0x8390000200000208ULL
807 #define HID7_DEFEATURES_2 0x0000000000000000ULL
808
809 /*
810 arg0 = target register
811 arg1 = 64-bit constant
812 */
813 .macro LOAD_UINT64
814 movz $0, #(($1 >> 48) & 0xffff), lsl #48
815 movk $0, #(($1 >> 32) & 0xffff), lsl #32
816 movk $0, #(($1 >> 16) & 0xffff), lsl #16
817 movk $0, #(($1) & 0xffff)
818 .endmacro
819
820 .text
821 .align 2
822 .globl EXT(cpu_defeatures_set)
823 LEXT(cpu_defeatures_set)
824 PUSH_FRAME
825 cmp x0, #2
826 b.eq cpu_defeatures_set_2
827 cmp x0, #1
828 b.ne cpu_defeatures_set_ret
829 LOAD_UINT64 x1, HID0_DEFEATURES_1
830 mrs x0, ARM64_REG_HID0
831 orr x0, x0, x1
832 msr ARM64_REG_HID0, x0
833 LOAD_UINT64 x1, HID1_DEFEATURES_1
834 mrs x0, ARM64_REG_HID1
835 orr x0, x0, x1
836 msr ARM64_REG_HID1, x0
837 LOAD_UINT64 x1, HID2_DEFEATURES_1
838 mrs x0, ARM64_REG_HID2
839 orr x0, x0, x1
840 msr ARM64_REG_HID2, x0
841 LOAD_UINT64 x1, HID3_DEFEATURES_1
842 mrs x0, ARM64_REG_HID3
843 orr x0, x0, x1
844 msr ARM64_REG_HID3, x0
845 LOAD_UINT64 x1, HID4_DEFEATURES_1
846 mrs x0, ARM64_REG_HID4
847 orr x0, x0, x1
848 msr ARM64_REG_HID4, x0
849 LOAD_UINT64 x1, HID7_DEFEATURES_1
850 mrs x0, ARM64_REG_HID7
851 orr x0, x0, x1
852 msr ARM64_REG_HID7, x0
853 dsb sy
854 isb sy
855 b cpu_defeatures_set_ret
856 cpu_defeatures_set_2:
857 LOAD_UINT64 x1, HID0_DEFEATURES_2
858 mrs x0, ARM64_REG_HID0
859 orr x0, x0, x1
860 msr ARM64_REG_HID0, x0
861 LOAD_UINT64 x1, HID1_DEFEATURES_2
862 mrs x0, ARM64_REG_HID1
863 orr x0, x0, x1
864 msr ARM64_REG_HID1, x0
865 LOAD_UINT64 x1, HID2_DEFEATURES_2
866 mrs x0, ARM64_REG_HID2
867 orr x0, x0, x1
868 msr ARM64_REG_HID2, x0
869 LOAD_UINT64 x1, HID3_DEFEATURES_2
870 mrs x0, ARM64_REG_HID3
871 orr x0, x0, x1
872 msr ARM64_REG_HID3, x0
873 LOAD_UINT64 x1, HID4_DEFEATURES_2
874 mrs x0, ARM64_REG_HID4
875 orr x0, x0, x1
876 msr ARM64_REG_HID4, x0
877 LOAD_UINT64 x1, HID7_DEFEATURES_2
878 mrs x0, ARM64_REG_HID7
879 orr x0, x0, x1
880 msr ARM64_REG_HID7, x0
881 dsb sy
882 isb sy
883 b cpu_defeatures_set_ret
884 cpu_defeatures_set_ret:
885 POP_FRAME
886 ret
887 #endif
888
889 #else /* !defined(APPLE_ARM64_ARCH_FAMILY) */
890 .text
891 .align 2
892 .globl EXT(arm64_prepare_for_sleep)
893 LEXT(arm64_prepare_for_sleep)
894 PUSH_FRAME
895 Lwfi_inst:
896 dsb sy
897 isb sy
898 wfi
899 b Lwfi_inst
900
901 /*
902 * Force WFI to use clock gating only
903 * Note: for non-Apple device, do nothing.
904 */
905 .text
906 .align 2
907 .globl EXT(arm64_force_wfi_clock_gate)
908 LEXT(arm64_force_wfi_clock_gate)
909 PUSH_FRAME
910 nop
911 POP_FRAME
912
913 #endif /* defined(APPLE_ARM64_ARCH_FAMILY) */
914
915 /*
916 * void arm64_replace_bootstack(cpu_data_t *cpu_data)
917 *
918 * This must be called from a kernel thread context running on the boot CPU,
919 * after setting up new exception stacks in per-CPU data. That will guarantee
920 * that the stack(s) we're trying to replace aren't currently in use. For
921 * KTRR-protected devices, this must also be called prior to VM prot finalization
922 * and lockdown, as updating SP1 requires a sensitive instruction.
923 */
924 .text
925 .align 2
926 .globl EXT(arm64_replace_bootstack)
927 LEXT(arm64_replace_bootstack)
928 ARM64_STACK_PROLOG
929 PUSH_FRAME
930 // Set the exception stack pointer
931 ldr x0, [x0, CPU_EXCEPSTACK_TOP]
932 mrs x4, DAIF // Load current DAIF; use x4 as pinst may trash x1-x3
933 msr DAIFSet, #(DAIFSC_IRQF | DAIFSC_FIQF | DAIFSC_ASYNCF) // Disable IRQ/FIQ/serror
934 // Set SP_EL1 to exception stack
935 #if defined(KERNEL_INTEGRITY_KTRR)
936 mov x1, lr
937 bl EXT(pinst_spsel_1)
938 mov lr, x1
939 #else
940 msr SPSel, #1
941 #endif
942 mov sp, x0
943 msr SPSel, #0
944 msr DAIF, x4 // Restore interrupt state
945 POP_FRAME
946 ARM64_STACK_EPILOG
947
948 #ifdef MONITOR
949 /*
950 * unsigned long monitor_call(uintptr_t callnum, uintptr_t arg1,
951 uintptr_t arg2, uintptr_t arg3)
952 *
953 * Call the EL3 monitor with 4 arguments in registers
954 * The monitor interface maintains the same ABI as the C function call standard. Callee-saved
955 * registers are preserved, temporary registers are not. Parameters and results are passed in
956 * the usual manner.
957 */
958 .text
959 .align 2
960 .globl EXT(monitor_call)
961 LEXT(monitor_call)
962 smc 0x11
963 ret
964 #endif
965
966 #ifdef HAS_APPLE_PAC
967 /**
968 * void ml_sign_thread_state(arm_saved_state_t *ss, uint64_t pc,
969 * uint32_t cpsr, uint64_t lr, uint64_t x16,
970 * uint64_t x17)
971 */
972 .text
973 .align 2
974 .globl EXT(ml_sign_thread_state)
975 LEXT(ml_sign_thread_state)
976 pacga x1, x1, x0 /* PC hash (gkey + &arm_saved_state) */
977 /*
978 * Mask off the carry flag so we don't need to re-sign when that flag is
979 * touched by the system call return path.
980 */
981 bic x2, x2, PSR_CF
982 pacga x1, x2, x1 /* SPSR hash (gkey + pc hash) */
983 pacga x1, x3, x1 /* LR Hash (gkey + spsr hash) */
984 pacga x1, x4, x1 /* X16 hash (gkey + lr hash) */
985 pacga x1, x5, x1 /* X17 hash (gkey + x16 hash) */
986 str x1, [x0, SS64_JOPHASH]
987 ret
988
989 /**
990 * void ml_check_signed_state(arm_saved_state_t *ss, uint64_t pc,
991 * uint32_t cpsr, uint64_t lr, uint64_t x16,
992 * uint64_t x17)
993 */
994 .text
995 .align 2
996 .globl EXT(ml_check_signed_state)
997 LEXT(ml_check_signed_state)
998 pacga x1, x1, x0 /* PC hash (gkey + &arm_saved_state) */
999 /*
1000 * Mask off the carry flag so we don't need to re-sign when that flag is
1001 * touched by the system call return path.
1002 */
1003 bic x2, x2, PSR_CF
1004 pacga x1, x2, x1 /* SPSR hash (gkey + pc hash) */
1005 pacga x1, x3, x1 /* LR Hash (gkey + spsr hash) */
1006 pacga x1, x4, x1 /* X16 hash (gkey + lr hash) */
1007 pacga x1, x5, x1 /* X17 hash (gkey + x16 hash) */
1008 ldr x2, [x0, SS64_JOPHASH]
1009 cmp x1, x2
1010 b.ne Lcheck_hash_panic
1011 ret
1012 Lcheck_hash_panic:
1013 mov x1, x0
1014 adr x0, Lcheck_hash_str
1015 CALL_EXTERN panic_with_thread_kernel_state
1016 Lcheck_hash_str:
1017 .asciz "JOP Hash Mismatch Detected (PC, CPSR, or LR corruption)"
1018 #endif /* HAS_APPLE_PAC */
1019
1020 .text
1021 .align 2
1022 .globl EXT(fill32_dczva)
1023 LEXT(fill32_dczva)
1024 0:
1025 dc zva, x0
1026 add x0, x0, #64
1027 subs x1, x1, #64
1028 b.hi 0b
1029 ret
1030
1031 .text
1032 .align 2
1033 .globl EXT(fill32_nt)
1034 LEXT(fill32_nt)
1035 dup.4s v0, w2
1036 0:
1037 stnp q0, q0, [x0]
1038 stnp q0, q0, [x0, #0x20]
1039 stnp q0, q0, [x0, #0x40]
1040 stnp q0, q0, [x0, #0x60]
1041 add x0, x0, #128
1042 subs x1, x1, #128
1043 b.hi 0b
1044 ret
1045
1046 /* vim: set sw=4 ts=4: */
mirror of XNU