]> git.saurik.com Git - apple/xnu.git/blob - osfmk/arm64/machine_routines_asm.s
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-7195.81.3.tar.gz
[apple/xnu.git] / osfmk / arm64 / machine_routines_asm.s
1 /*
2 * Copyright (c) 2007-2015 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <machine/asm.h>
30 #include <arm64/exception_asm.h>
31 #include <arm64/machine_machdep.h>
32 #include <arm64/proc_reg.h>
33 #include <arm/pmap.h>
34 #include <pexpert/arm64/board_config.h>
35 #include <sys/errno.h>
36 #include "assym.s"
37
38
39
40 #if HAS_BP_RET
41
42 /*
43 * void set_bp_ret(void)
44 * Helper function to enable branch predictor state retention
45 * across ACC sleep
46 */
47
48 .align 2
49 .globl EXT(set_bp_ret)
50 LEXT(set_bp_ret)
51 // Load bpret boot-arg
52 adrp x14, EXT(bp_ret)@page
53 add x14, x14, EXT(bp_ret)@pageoff
54 ldr w14, [x14]
55
56 mrs x13, ARM64_REG_ACC_CFG
57 and x13, x13, (~(ARM64_REG_ACC_CFG_bpSlp_mask << ARM64_REG_ACC_CFG_bpSlp_shift))
58 and x14, x14, #(ARM64_REG_ACC_CFG_bpSlp_mask)
59 orr x13, x13, x14, lsl #(ARM64_REG_ACC_CFG_bpSlp_shift)
60 msr ARM64_REG_ACC_CFG, x13
61
62 ret
63 #endif // HAS_BP_RET
64
65 #if HAS_NEX_PG
66 .align 2
67 .globl EXT(set_nex_pg)
68 LEXT(set_nex_pg)
69 mrs x14, MPIDR_EL1
70 // Skip if this isn't a p-core; NEX powergating isn't available for e-cores
71 and x14, x14, #(MPIDR_PNE)
72 cbz x14, Lnex_pg_done
73
74 // Set the SEG-recommended value of 12 additional reset cycles
75 HID_INSERT_BITS ARM64_REG_HID13, ARM64_REG_HID13_RstCyc_mask, ARM64_REG_HID13_RstCyc_val, x13
76 HID_SET_BITS ARM64_REG_HID14, ARM64_REG_HID14_NexPwgEn, x13
77
78 Lnex_pg_done:
79 ret
80
81 #endif // HAS_NEX_PG
82
83 /* uint32_t get_fpscr(void):
84 * Returns (FPSR | FPCR).
85 */
86 .align 2
87 .globl EXT(get_fpscr)
88 LEXT(get_fpscr)
89 #if __ARM_VFP__
90 mrs x1, FPSR // Grab FPSR
91 mov x4, #(FPSR_MASK & 0xFFFF)
92 mov x5, #(FPSR_MASK & 0xFFFF0000)
93 orr x0, x4, x5
94 and x1, x1, x0 // Be paranoid, and clear bits we expect to
95 // be clear
96 mrs x2, FPCR // Grab FPCR
97 mov x4, #(FPCR_MASK & 0xFFFF)
98 mov x5, #(FPCR_MASK & 0xFFFF0000)
99 orr x0, x4, x5
100 and x2, x2, x0 // Be paranoid, and clear bits we expect to
101 // be clear
102 orr x0, x1, x2 // OR them to get FPSCR equivalent state
103 #else
104 mov x0, #0
105 #endif
106 ret
107 .align 2
108 .globl EXT(set_fpscr)
109 /* void set_fpscr(uint32_t value):
110 * Set the FPCR and FPSR registers, based on the given value; a
111 * noteworthy point is that unlike 32-bit mode, 64-bit mode FPSR
112 * and FPCR are not responsible for condition codes.
113 */
114 LEXT(set_fpscr)
115 #if __ARM_VFP__
116 mov x4, #(FPSR_MASK & 0xFFFF)
117 mov x5, #(FPSR_MASK & 0xFFFF0000)
118 orr x1, x4, x5
119 and x1, x1, x0 // Clear the bits that don't apply to FPSR
120 mov x4, #(FPCR_MASK & 0xFFFF)
121 mov x5, #(FPCR_MASK & 0xFFFF0000)
122 orr x2, x4, x5
123 and x2, x2, x0 // Clear the bits that don't apply to FPCR
124 msr FPSR, x1 // Write FPCR
125 msr FPCR, x2 // Write FPSR
126 dsb ish // FPCR requires synchronization
127 #endif
128 ret
129
130 /*
131 * void update_mdscr(unsigned long clear, unsigned long set)
132 * Clears and sets the specified bits in MDSCR_EL1.
133 *
134 * Setting breakpoints in EL1 is effectively a KTRR bypass. The ability to do so is
135 * controlled by MDSCR.KDE. The MSR to set MDSCR must be present to allow
136 * self-hosted user mode debug. Any checks before the MRS can be skipped with ROP,
137 * so we need to put the checks after the MRS where they can't be skipped. That
138 * still leaves a small window if a breakpoint is set on the instruction
139 * immediately after the MRS. To handle that, we also do a check and then set of
140 * the breakpoint control registers. This allows us to guarantee that a given
141 * core will never have both KDE set and a breakpoint targeting EL1.
142 *
143 * If KDE gets set, unset it and then panic
144 */
145 .align 2
146 .globl EXT(update_mdscr)
147 LEXT(update_mdscr)
148 mov x4, #0
149 mrs x2, MDSCR_EL1
150 bic x2, x2, x0
151 orr x2, x2, x1
152 1:
153 bic x2, x2, #0x2000
154 msr MDSCR_EL1, x2
155 #if defined(CONFIG_KERNEL_INTEGRITY)
156 /*
157 * verify KDE didn't get set (including via ROP)
158 * If set, clear it and then panic
159 */
160 ands x3, x2, #0x2000
161 orr x4, x4, x3
162 bne 1b
163 cmp x4, xzr
164 b.ne Lupdate_mdscr_panic
165 #endif
166 ret
167
168 Lupdate_mdscr_panic:
169 adrp x0, Lupdate_mdscr_panic_str@page
170 add x0, x0, Lupdate_mdscr_panic_str@pageoff
171 b EXT(panic)
172 b .
173
174 Lupdate_mdscr_panic_str:
175 .asciz "MDSCR.KDE was set"
176
177
178 /*
179 * Set MMU Translation Table Base Alternate
180 */
181 .text
182 .align 2
183 .globl EXT(set_mmu_ttb_alternate)
184 LEXT(set_mmu_ttb_alternate)
185 dsb sy
186 #if defined(KERNEL_INTEGRITY_KTRR)
187 mov x1, lr
188 bl EXT(pinst_set_ttbr1)
189 mov lr, x1
190 #else
191 #if defined(HAS_VMSA_LOCK)
192 #if DEBUG || DEVELOPMENT
193 mrs x1, ARM64_REG_VMSA_LOCK_EL1
194 and x1, x1, #(VMSA_LOCK_TTBR1_EL1)
195 cbnz x1, L_set_locked_reg_panic
196 #endif /* DEBUG || DEVELOPMENT */
197 #endif /* defined(HAS_VMSA_LOCK) */
198 msr TTBR1_EL1, x0
199 #endif /* defined(KERNEL_INTEGRITY_KTRR) */
200 isb sy
201 ret
202
203 #if XNU_MONITOR
204 .section __PPLTEXT,__text,regular,pure_instructions
205 #else
206 .text
207 #endif
208 .align 2
209 .globl EXT(set_mmu_ttb)
210 LEXT(set_mmu_ttb)
211 #if __ARM_KERNEL_PROTECT__
212 /* All EL1-mode ASIDs are odd. */
213 orr x0, x0, #(1 << TTBR_ASID_SHIFT)
214 #endif /* __ARM_KERNEL_PROTECT__ */
215 dsb ish
216 msr TTBR0_EL1, x0
217 isb sy
218 ret
219
220
221 #if XNU_MONITOR
222 .text
223 .align 2
224 .globl EXT(ml_get_ppl_cpu_data)
225 LEXT(ml_get_ppl_cpu_data)
226 LOAD_PMAP_CPU_DATA x0, x1, x2
227 ret
228 #endif
229
230 /*
231 * set AUX control register
232 */
233 .text
234 .align 2
235 .globl EXT(set_aux_control)
236 LEXT(set_aux_control)
237 msr ACTLR_EL1, x0
238 // Synchronize system
239 isb sy
240 ret
241
242 #if __ARM_KERNEL_PROTECT__
243 .text
244 .align 2
245 .globl EXT(set_vbar_el1)
246 LEXT(set_vbar_el1)
247 #if defined(KERNEL_INTEGRITY_KTRR)
248 b EXT(pinst_set_vbar)
249 #else
250 msr VBAR_EL1, x0
251 ret
252 #endif
253 #endif /* __ARM_KERNEL_PROTECT__ */
254
255 #if defined(HAS_VMSA_LOCK)
256 .text
257 .align 2
258 .globl EXT(vmsa_lock)
259 LEXT(vmsa_lock)
260 isb sy
261 mov x1, #(VMSA_LOCK_SCTLR_M_BIT)
262 #if __ARM_MIXED_PAGE_SIZE__
263 mov x0, #(VMSA_LOCK_TTBR1_EL1 | VMSA_LOCK_VBAR_EL1)
264 #else
265 mov x0, #(VMSA_LOCK_TTBR1_EL1 | VMSA_LOCK_TCR_EL1 | VMSA_LOCK_VBAR_EL1)
266 #endif
267 orr x0, x0, x1
268 msr ARM64_REG_VMSA_LOCK_EL1, x0
269 isb sy
270 ret
271 #endif /* defined(HAS_VMSA_LOCK) */
272
273 /*
274 * set translation control register
275 */
276 .text
277 .align 2
278 .globl EXT(set_tcr)
279 LEXT(set_tcr)
280 #if defined(APPLE_ARM64_ARCH_FAMILY)
281 #if DEBUG || DEVELOPMENT
282 // Assert that T0Z is always equal to T1Z
283 eor x1, x0, x0, lsr #(TCR_T1SZ_SHIFT - TCR_T0SZ_SHIFT)
284 and x1, x1, #(TCR_TSZ_MASK << TCR_T0SZ_SHIFT)
285 cbnz x1, L_set_tcr_panic
286 #endif /* DEBUG || DEVELOPMENT */
287 #endif /* defined(APPLE_ARM64_ARCH_FAMILY) */
288 #if defined(KERNEL_INTEGRITY_KTRR)
289 mov x1, lr
290 bl EXT(pinst_set_tcr)
291 mov lr, x1
292 #else
293 #if defined(HAS_VMSA_LOCK)
294 #if DEBUG || DEVELOPMENT
295 // assert TCR unlocked
296 mrs x1, ARM64_REG_VMSA_LOCK_EL1
297 and x1, x1, #(VMSA_LOCK_TCR_EL1)
298 cbnz x1, L_set_locked_reg_panic
299 #endif /* DEBUG || DEVELOPMENT */
300 #endif /* defined(HAS_VMSA_LOCK) */
301 msr TCR_EL1, x0
302 #endif /* defined(KERNEL_INTRITY_KTRR) */
303 isb sy
304 ret
305
306 #if DEBUG || DEVELOPMENT
307 L_set_tcr_panic:
308 PUSH_FRAME
309 sub sp, sp, #16
310 str x0, [sp]
311 adr x0, L_set_tcr_panic_str
312 BRANCH_EXTERN panic
313
314 L_set_locked_reg_panic:
315 PUSH_FRAME
316 sub sp, sp, #16
317 str x0, [sp]
318 adr x0, L_set_locked_reg_panic_str
319 BRANCH_EXTERN panic
320 b .
321
322 L_set_tcr_panic_str:
323 .asciz "set_tcr: t0sz, t1sz not equal (%llx)\n"
324
325
326 L_set_locked_reg_panic_str:
327 .asciz "attempt to set locked register: (%llx)\n"
328 #endif /* DEBUG || DEVELOPMENT */
329
330 /*
331 * MMU kernel virtual to physical address translation
332 */
333 .text
334 .align 2
335 .globl EXT(mmu_kvtop)
336 LEXT(mmu_kvtop)
337 mrs x2, DAIF // Load current DAIF
338 msr DAIFSet, #(DAIFSC_IRQF | DAIFSC_FIQF) // Disable IRQ
339 at s1e1r, x0 // Translation Stage 1 EL1
340 isb sy
341 mrs x1, PAR_EL1 // Read result
342 msr DAIF, x2 // Restore interrupt state
343 tbnz x1, #0, L_mmu_kvtop_invalid // Test Translation not valid
344 bfm x1, x0, #0, #11 // Add page offset
345 and x0, x1, #0x0000ffffffffffff // Clear non-address bits
346 ret
347 L_mmu_kvtop_invalid:
348 mov x0, #0 // Return invalid
349 ret
350
351 /*
352 * MMU user virtual to physical address translation
353 */
354 .text
355 .align 2
356 .globl EXT(mmu_uvtop)
357 LEXT(mmu_uvtop)
358 lsr x8, x0, #56 // Extract top byte
359 cbnz x8, L_mmu_uvtop_invalid // Tagged pointers are invalid
360 mrs x2, DAIF // Load current DAIF
361 msr DAIFSet, #(DAIFSC_IRQF | DAIFSC_FIQF) // Disable IRQ
362 at s1e0r, x0 // Translation Stage 1 EL0
363 isb sy
364 mrs x1, PAR_EL1 // Read result
365 msr DAIF, x2 // Restore interrupt state
366 tbnz x1, #0, L_mmu_uvtop_invalid // Test Translation not valid
367 bfm x1, x0, #0, #11 // Add page offset
368 and x0, x1, #0x0000ffffffffffff // Clear non-address bits
369 ret
370 L_mmu_uvtop_invalid:
371 mov x0, #0 // Return invalid
372 ret
373
374 /*
375 * MMU kernel virtual to physical address preflight write access
376 */
377 .text
378 .align 2
379 .globl EXT(mmu_kvtop_wpreflight)
380 LEXT(mmu_kvtop_wpreflight)
381 mrs x2, DAIF // Load current DAIF
382 msr DAIFSet, #(DAIFSC_IRQF | DAIFSC_FIQF) // Disable IRQ
383 at s1e1w, x0 // Translation Stage 1 EL1
384 mrs x1, PAR_EL1 // Read result
385 msr DAIF, x2 // Restore interrupt state
386 tbnz x1, #0, L_mmu_kvtop_wpreflight_invalid // Test Translation not valid
387 bfm x1, x0, #0, #11 // Add page offset
388 and x0, x1, #0x0000ffffffffffff // Clear non-address bits
389 ret
390 L_mmu_kvtop_wpreflight_invalid:
391 mov x0, #0 // Return invalid
392 ret
393
394 /*
395 * SET_RECOVERY_HANDLER
396 *
397 * Sets up a page fault recovery handler. This macro clobbers x16 and x17.
398 *
399 * label - recovery label
400 * tpidr - persisted thread pointer
401 * old_handler - persisted recovery handler
402 * label_in_adr_range - whether \label is within 1 MB of PC
403 */
404 .macro SET_RECOVERY_HANDLER label, tpidr=x16, old_handler=x10, label_in_adr_range=0
405 // Note: x16 and x17 are designated for use as temporaries in
406 // interruptible PAC routines. DO NOT CHANGE THESE REGISTER ASSIGNMENTS.
407 .if \label_in_adr_range==1 // Load the recovery handler address
408 adr x17, \label
409 .else
410 adrp x17, \label@page
411 add x17, x17, \label@pageoff
412 .endif
413 #if defined(HAS_APPLE_PAC)
414 mrs x16, TPIDR_EL1
415 add x16, x16, TH_RECOVER
416 movk x16, #PAC_DISCRIMINATOR_RECOVER, lsl 48
417 pacia x17, x16 // Sign with IAKey + blended discriminator
418 #endif
419
420 mrs \tpidr, TPIDR_EL1 // Load thread pointer
421 ldr \old_handler, [\tpidr, TH_RECOVER] // Save previous recovery handler
422 str x17, [\tpidr, TH_RECOVER] // Set new signed recovery handler
423 .endmacro
424
425 /*
426 * CLEAR_RECOVERY_HANDLER
427 *
428 * Clears page fault handler set by SET_RECOVERY_HANDLER
429 *
430 * tpidr - thread pointer saved by SET_RECOVERY_HANDLER
431 * old_handler - old recovery handler saved by SET_RECOVERY_HANDLER
432 */
433 .macro CLEAR_RECOVERY_HANDLER tpidr=x16, old_handler=x10
434 str \old_handler, [\tpidr, TH_RECOVER] // Restore the previous recovery handler
435 .endmacro
436
437
438 .text
439 .align 2
440 copyio_error:
441 CLEAR_RECOVERY_HANDLER
442 mov x0, #EFAULT // Return an EFAULT error
443 POP_FRAME
444 ARM64_STACK_EPILOG
445
446 /*
447 * int _bcopyin(const char *src, char *dst, vm_size_t len)
448 */
449 .text
450 .align 2
451 .globl EXT(_bcopyin)
452 LEXT(_bcopyin)
453 ARM64_STACK_PROLOG
454 PUSH_FRAME
455 SET_RECOVERY_HANDLER copyio_error
456 /* If len is less than 16 bytes, just do a bytewise copy */
457 cmp x2, #16
458 b.lt 2f
459 sub x2, x2, #16
460 1:
461 /* 16 bytes at a time */
462 ldp x3, x4, [x0], #16
463 stp x3, x4, [x1], #16
464 subs x2, x2, #16
465 b.ge 1b
466 /* Fixup the len and test for completion */
467 adds x2, x2, #16
468 b.eq 3f
469 2: /* Bytewise */
470 subs x2, x2, #1
471 ldrb w3, [x0], #1
472 strb w3, [x1], #1
473 b.hi 2b
474 3:
475 CLEAR_RECOVERY_HANDLER
476 mov x0, #0
477 POP_FRAME
478 ARM64_STACK_EPILOG
479
480 /*
481 * int _copyin_atomic32(const char *src, uint32_t *dst)
482 */
483 .text
484 .align 2
485 .globl EXT(_copyin_atomic32)
486 LEXT(_copyin_atomic32)
487 ARM64_STACK_PROLOG
488 PUSH_FRAME
489 SET_RECOVERY_HANDLER copyio_error
490 ldr w8, [x0]
491 str w8, [x1]
492 mov x0, #0
493 CLEAR_RECOVERY_HANDLER
494 POP_FRAME
495 ARM64_STACK_EPILOG
496
497 /*
498 * int _copyin_atomic32_wait_if_equals(const char *src, uint32_t value)
499 */
500 .text
501 .align 2
502 .globl EXT(_copyin_atomic32_wait_if_equals)
503 LEXT(_copyin_atomic32_wait_if_equals)
504 ARM64_STACK_PROLOG
505 PUSH_FRAME
506 SET_RECOVERY_HANDLER copyio_error
507 ldxr w8, [x0]
508 cmp w8, w1
509 mov x0, ESTALE
510 b.ne 1f
511 mov x0, #0
512 wfe
513 1:
514 clrex
515 CLEAR_RECOVERY_HANDLER
516 POP_FRAME
517 ARM64_STACK_EPILOG
518
519 /*
520 * int _copyin_atomic64(const char *src, uint32_t *dst)
521 */
522 .text
523 .align 2
524 .globl EXT(_copyin_atomic64)
525 LEXT(_copyin_atomic64)
526 ARM64_STACK_PROLOG
527 PUSH_FRAME
528 SET_RECOVERY_HANDLER copyio_error
529 ldr x8, [x0]
530 str x8, [x1]
531 mov x0, #0
532 CLEAR_RECOVERY_HANDLER
533 POP_FRAME
534 ARM64_STACK_EPILOG
535
536
537 /*
538 * int _copyout_atomic32(uint32_t value, char *dst)
539 */
540 .text
541 .align 2
542 .globl EXT(_copyout_atomic32)
543 LEXT(_copyout_atomic32)
544 ARM64_STACK_PROLOG
545 PUSH_FRAME
546 SET_RECOVERY_HANDLER copyio_error
547 str w0, [x1]
548 mov x0, #0
549 CLEAR_RECOVERY_HANDLER
550 POP_FRAME
551 ARM64_STACK_EPILOG
552
553 /*
554 * int _copyout_atomic64(uint64_t value, char *dst)
555 */
556 .text
557 .align 2
558 .globl EXT(_copyout_atomic64)
559 LEXT(_copyout_atomic64)
560 ARM64_STACK_PROLOG
561 PUSH_FRAME
562 SET_RECOVERY_HANDLER copyio_error
563 str x0, [x1]
564 mov x0, #0
565 CLEAR_RECOVERY_HANDLER
566 POP_FRAME
567 ARM64_STACK_EPILOG
568
569
570 /*
571 * int _bcopyout(const char *src, char *dst, vm_size_t len)
572 */
573 .text
574 .align 2
575 .globl EXT(_bcopyout)
576 LEXT(_bcopyout)
577 ARM64_STACK_PROLOG
578 PUSH_FRAME
579 SET_RECOVERY_HANDLER copyio_error
580 /* If len is less than 16 bytes, just do a bytewise copy */
581 cmp x2, #16
582 b.lt 2f
583 sub x2, x2, #16
584 1:
585 /* 16 bytes at a time */
586 ldp x3, x4, [x0], #16
587 stp x3, x4, [x1], #16
588 subs x2, x2, #16
589 b.ge 1b
590 /* Fixup the len and test for completion */
591 adds x2, x2, #16
592 b.eq 3f
593 2: /* Bytewise */
594 subs x2, x2, #1
595 ldrb w3, [x0], #1
596 strb w3, [x1], #1
597 b.hi 2b
598 3:
599 CLEAR_RECOVERY_HANDLER
600 mov x0, #0
601 POP_FRAME
602 ARM64_STACK_EPILOG
603
604 /*
605 * int _bcopyinstr(
606 * const user_addr_t user_addr,
607 * char *kernel_addr,
608 * vm_size_t max,
609 * vm_size_t *actual)
610 */
611 .text
612 .align 2
613 .globl EXT(_bcopyinstr)
614 LEXT(_bcopyinstr)
615 ARM64_STACK_PROLOG
616 PUSH_FRAME
617 SET_RECOVERY_HANDLER Lcopyinstr_error, label_in_adr_range=1
618 mov x4, #0 // x4 - total bytes copied
619 Lcopyinstr_loop:
620 ldrb w5, [x0], #1 // Load a byte from the user source
621 strb w5, [x1], #1 // Store a byte to the kernel dest
622 add x4, x4, #1 // Increment bytes copied
623 cbz x5, Lcopyinstr_done // If this byte is null, we're done
624 cmp x4, x2 // If we're out of space, return an error
625 b.ne Lcopyinstr_loop
626 Lcopyinstr_too_long:
627 mov x5, #ENAMETOOLONG // Set current byte to error code for later return
628 Lcopyinstr_done:
629 str x4, [x3] // Return number of bytes copied
630 mov x0, x5 // Set error code (0 on success, ENAMETOOLONG on failure)
631 b Lcopyinstr_exit
632 Lcopyinstr_error:
633 mov x0, #EFAULT // Return EFAULT on error
634 Lcopyinstr_exit:
635 CLEAR_RECOVERY_HANDLER
636 POP_FRAME
637 ARM64_STACK_EPILOG
638
639 /*
640 * int copyinframe(const vm_address_t frame_addr, char *kernel_addr, bool is64bit)
641 *
642 * Safely copy sixteen bytes (the fixed top of an ARM64 frame) from
643 * either user or kernel memory, or 8 bytes (AArch32) from user only.
644 *
645 * x0 : address of frame to copy.
646 * x1 : kernel address at which to store data.
647 * w2 : whether to copy an AArch32 or AArch64 frame.
648 * x3 : temp
649 * x5 : temp (kernel virtual base)
650 * x9 : temp
651 * x10 : old recovery function (set by SET_RECOVERY_HANDLER)
652 * x12, x13 : backtrace data
653 * x16 : thread pointer (set by SET_RECOVERY_HANDLER)
654 *
655 */
656 .text
657 .align 2
658 .globl EXT(copyinframe)
659 LEXT(copyinframe)
660 ARM64_STACK_PROLOG
661 PUSH_FRAME
662 SET_RECOVERY_HANDLER copyio_error
663 cbnz w2, Lcopyinframe64 // Check frame size
664 adrp x5, EXT(gVirtBase)@page // For 32-bit frame, make sure we're not trying to copy from kernel
665 add x5, x5, EXT(gVirtBase)@pageoff
666 ldr x5, [x5]
667 cmp x5, x0 // See if address is in kernel virtual range
668 b.hi Lcopyinframe32 // If below kernel virtual range, proceed.
669 mov w0, #EFAULT // Should never have a 32-bit frame in kernel virtual range
670 b Lcopyinframe_done
671
672 Lcopyinframe32:
673 ldr x12, [x0] // Copy 8 bytes
674 str x12, [x1]
675 mov w0, #0 // Success
676 b Lcopyinframe_done
677
678 Lcopyinframe64:
679 mov x3, VM_MIN_KERNEL_ADDRESS // Check if kernel address
680 orr x9, x0, TBI_MASK // Hide tags in address comparison
681 cmp x9, x3 // If in kernel address range, skip tag test
682 b.hs Lcopyinframe_valid
683 tst x0, TBI_MASK // Detect tagged pointers
684 b.eq Lcopyinframe_valid
685 mov w0, #EFAULT // Tagged address, fail
686 b Lcopyinframe_done
687 Lcopyinframe_valid:
688 ldp x12, x13, [x0] // Copy 16 bytes
689 stp x12, x13, [x1]
690 mov w0, #0 // Success
691
692 Lcopyinframe_done:
693 CLEAR_RECOVERY_HANDLER
694 POP_FRAME
695 ARM64_STACK_EPILOG
696
697
698 /*
699 * uint32_t arm_debug_read_dscr(void)
700 */
701 .text
702 .align 2
703 .globl EXT(arm_debug_read_dscr)
704 LEXT(arm_debug_read_dscr)
705 PANIC_UNIMPLEMENTED
706
707 /*
708 * void arm_debug_set_cp14(arm_debug_state_t *debug_state)
709 *
710 * Set debug registers to match the current thread state
711 * (NULL to disable). Assume 6 breakpoints and 2
712 * watchpoints, since that has been the case in all cores
713 * thus far.
714 */
715 .text
716 .align 2
717 .globl EXT(arm_debug_set_cp14)
718 LEXT(arm_debug_set_cp14)
719 PANIC_UNIMPLEMENTED
720
721 #if defined(APPLE_ARM64_ARCH_FAMILY)
722 /*
723 * Note: still have to ISB before executing wfi!
724 */
725 .text
726 .align 2
727 .globl EXT(arm64_prepare_for_sleep)
728 LEXT(arm64_prepare_for_sleep)
729 PUSH_FRAME
730
731 #if defined(APPLETYPHOON)
732 // <rdar://problem/15827409>
733 HID_SET_BITS ARM64_REG_HID2, ARM64_REG_HID2_disMMUmtlbPrefetch, x9
734 dsb sy
735 isb sy
736 #endif
737
738 #if HAS_CLUSTER
739 cbnz x0, 1f // Skip if deep_sleep == true
740 // Mask FIQ and IRQ to avoid spurious wakeups
741 mrs x9, ARM64_REG_CYC_OVRD
742 and x9, x9, #(~(ARM64_REG_CYC_OVRD_irq_mask | ARM64_REG_CYC_OVRD_fiq_mask))
743 mov x10, #(ARM64_REG_CYC_OVRD_irq_disable | ARM64_REG_CYC_OVRD_fiq_disable)
744 orr x9, x9, x10
745 msr ARM64_REG_CYC_OVRD, x9
746 isb
747 1:
748 #endif
749
750 cbz x0, 1f // Skip if deep_sleep == false
751 #if __ARM_GLOBAL_SLEEP_BIT__
752 // Enable deep sleep
753 mrs x1, ARM64_REG_ACC_OVRD
754 orr x1, x1, #(ARM64_REG_ACC_OVRD_enDeepSleep)
755 and x1, x1, #(~(ARM64_REG_ACC_OVRD_disL2Flush4AccSlp_mask))
756 orr x1, x1, #( ARM64_REG_ACC_OVRD_disL2Flush4AccSlp_deepsleep)
757 and x1, x1, #(~(ARM64_REG_ACC_OVRD_ok2PwrDnSRM_mask))
758 orr x1, x1, #( ARM64_REG_ACC_OVRD_ok2PwrDnSRM_deepsleep)
759 and x1, x1, #(~(ARM64_REG_ACC_OVRD_ok2TrDnLnk_mask))
760 orr x1, x1, #( ARM64_REG_ACC_OVRD_ok2TrDnLnk_deepsleep)
761 and x1, x1, #(~(ARM64_REG_ACC_OVRD_ok2PwrDnCPM_mask))
762 orr x1, x1, #( ARM64_REG_ACC_OVRD_ok2PwrDnCPM_deepsleep)
763 #if HAS_RETENTION_STATE
764 orr x1, x1, #(ARM64_REG_ACC_OVRD_disPioOnWfiCpu)
765 #endif
766 msr ARM64_REG_ACC_OVRD, x1
767
768
769 #else
770 // Enable deep sleep
771 mov x1, ARM64_REG_CYC_CFG_deepSleep
772 msr ARM64_REG_CYC_CFG, x1
773 #endif
774
775 1:
776 // Set "OK to power down" (<rdar://problem/12390433>)
777 mrs x9, ARM64_REG_CYC_OVRD
778 orr x9, x9, #(ARM64_REG_CYC_OVRD_ok2pwrdn_force_down)
779 #if HAS_RETENTION_STATE
780 orr x9, x9, #(ARM64_REG_CYC_OVRD_disWfiRetn)
781 #endif
782 msr ARM64_REG_CYC_OVRD, x9
783
784 #if defined(APPLEMONSOON) || defined(APPLEVORTEX)
785 ARM64_IS_PCORE x9
786 cbz x9, Lwfi_inst // skip if not p-core
787
788 /* <rdar://problem/32512947>: Flush the GUPS prefetcher prior to
789 * wfi. A Skye HW bug can cause the GUPS prefetcher on p-cores
790 * to be left with valid entries that fail to drain if a
791 * subsequent wfi is issued. This can prevent the core from
792 * power-gating. For the idle case that is recoverable, but
793 * for the deep-sleep (S2R) case in which cores MUST power-gate,
794 * it can lead to a hang. This can be prevented by disabling
795 * and re-enabling GUPS, which forces the prefetch queue to
796 * drain. This should be done as close to wfi as possible, i.e.
797 * at the very end of arm64_prepare_for_sleep(). */
798 #if defined(APPLEVORTEX)
799 /* <rdar://problem/32821461>: Cyprus A0/A1 parts have a similar
800 * bug in the HSP prefetcher that can be worked around through
801 * the same method mentioned above for Skye. */
802 mrs x9, MIDR_EL1
803 EXEC_COREALL_REVLO CPU_VERSION_B0, x9, x10
804 #endif
805 mrs x9, ARM64_REG_HID10
806 orr x9, x9, #(ARM64_REG_HID10_DisHwpGups)
807 msr ARM64_REG_HID10, x9
808 isb sy
809 and x9, x9, #(~(ARM64_REG_HID10_DisHwpGups))
810 msr ARM64_REG_HID10, x9
811 isb sy
812 #endif
813 EXEC_END
814
815 Lwfi_inst:
816 dsb sy
817 isb sy
818 wfi
819 b Lwfi_inst
820
821 /*
822 * Force WFI to use clock gating only
823 *
824 */
825 .text
826 .align 2
827 .globl EXT(arm64_force_wfi_clock_gate)
828 LEXT(arm64_force_wfi_clock_gate)
829 ARM64_STACK_PROLOG
830 PUSH_FRAME
831
832 mrs x0, ARM64_REG_CYC_OVRD
833 orr x0, x0, #(ARM64_REG_CYC_OVRD_ok2pwrdn_force_up)
834 msr ARM64_REG_CYC_OVRD, x0
835
836 POP_FRAME
837 ARM64_STACK_EPILOG
838
839
840 #if HAS_RETENTION_STATE
841 .text
842 .align 2
843 .globl EXT(arm64_retention_wfi)
844 LEXT(arm64_retention_wfi)
845 wfi
846 cbz lr, Lwfi_retention // If lr is 0, we entered retention state and lost all GPRs except sp and pc
847 ret // Otherwise just return to cpu_idle()
848 Lwfi_retention:
849 mov x0, #1
850 bl EXT(ClearIdlePop)
851 mov x0, #0
852 bl EXT(cpu_idle_exit) // cpu_idle_exit(from_reset = FALSE)
853 b . // cpu_idle_exit() should never return
854 #endif
855
856 #if defined(APPLETYPHOON)
857
858 .text
859 .align 2
860 .globl EXT(typhoon_prepare_for_wfi)
861
862 LEXT(typhoon_prepare_for_wfi)
863 PUSH_FRAME
864
865 // <rdar://problem/15827409>
866 HID_SET_BITS ARM64_REG_HID2, ARM64_REG_HID2_disMMUmtlbPrefetch, x0
867 dsb sy
868 isb sy
869
870 POP_FRAME
871 ret
872
873
874 .text
875 .align 2
876 .globl EXT(typhoon_return_from_wfi)
877 LEXT(typhoon_return_from_wfi)
878 PUSH_FRAME
879
880 // <rdar://problem/15827409>
881 HID_CLEAR_BITS ARM64_REG_HID2, ARM64_REG_HID2_disMMUmtlbPrefetch, x0
882 dsb sy
883 isb sy
884
885 POP_FRAME
886 ret
887 #endif
888
889 #ifdef APPLETYPHOON
890
891 #define HID0_DEFEATURES_1 0x0000a0c000064010ULL
892 #define HID1_DEFEATURES_1 0x000000004005bf20ULL
893 #define HID2_DEFEATURES_1 0x0000000000102074ULL
894 #define HID3_DEFEATURES_1 0x0000000000400003ULL
895 #define HID4_DEFEATURES_1 0x83ff00e100000268ULL
896 #define HID7_DEFEATURES_1 0x000000000000000eULL
897
898 #define HID0_DEFEATURES_2 0x0000a1c000020010ULL
899 #define HID1_DEFEATURES_2 0x000000000005d720ULL
900 #define HID2_DEFEATURES_2 0x0000000000002074ULL
901 #define HID3_DEFEATURES_2 0x0000000000400001ULL
902 #define HID4_DEFEATURES_2 0x8390000200000208ULL
903 #define HID7_DEFEATURES_2 0x0000000000000000ULL
904
905 /*
906 arg0 = target register
907 arg1 = 64-bit constant
908 */
909 .macro LOAD_UINT64
910 movz $0, #(($1 >> 48) & 0xffff), lsl #48
911 movk $0, #(($1 >> 32) & 0xffff), lsl #32
912 movk $0, #(($1 >> 16) & 0xffff), lsl #16
913 movk $0, #(($1) & 0xffff)
914 .endmacro
915
916 .text
917 .align 2
918 .globl EXT(cpu_defeatures_set)
919 LEXT(cpu_defeatures_set)
920 PUSH_FRAME
921 cmp x0, #2
922 b.eq cpu_defeatures_set_2
923 cmp x0, #1
924 b.ne cpu_defeatures_set_ret
925 LOAD_UINT64 x1, HID0_DEFEATURES_1
926 mrs x0, ARM64_REG_HID0
927 orr x0, x0, x1
928 msr ARM64_REG_HID0, x0
929 LOAD_UINT64 x1, HID1_DEFEATURES_1
930 mrs x0, ARM64_REG_HID1
931 orr x0, x0, x1
932 msr ARM64_REG_HID1, x0
933 LOAD_UINT64 x1, HID2_DEFEATURES_1
934 mrs x0, ARM64_REG_HID2
935 orr x0, x0, x1
936 msr ARM64_REG_HID2, x0
937 LOAD_UINT64 x1, HID3_DEFEATURES_1
938 mrs x0, ARM64_REG_HID3
939 orr x0, x0, x1
940 msr ARM64_REG_HID3, x0
941 LOAD_UINT64 x1, HID4_DEFEATURES_1
942 mrs x0, ARM64_REG_HID4
943 orr x0, x0, x1
944 msr ARM64_REG_HID4, x0
945 LOAD_UINT64 x1, HID7_DEFEATURES_1
946 mrs x0, ARM64_REG_HID7
947 orr x0, x0, x1
948 msr ARM64_REG_HID7, x0
949 dsb sy
950 isb sy
951 b cpu_defeatures_set_ret
952 cpu_defeatures_set_2:
953 LOAD_UINT64 x1, HID0_DEFEATURES_2
954 mrs x0, ARM64_REG_HID0
955 orr x0, x0, x1
956 msr ARM64_REG_HID0, x0
957 LOAD_UINT64 x1, HID1_DEFEATURES_2
958 mrs x0, ARM64_REG_HID1
959 orr x0, x0, x1
960 msr ARM64_REG_HID1, x0
961 LOAD_UINT64 x1, HID2_DEFEATURES_2
962 mrs x0, ARM64_REG_HID2
963 orr x0, x0, x1
964 msr ARM64_REG_HID2, x0
965 LOAD_UINT64 x1, HID3_DEFEATURES_2
966 mrs x0, ARM64_REG_HID3
967 orr x0, x0, x1
968 msr ARM64_REG_HID3, x0
969 LOAD_UINT64 x1, HID4_DEFEATURES_2
970 mrs x0, ARM64_REG_HID4
971 orr x0, x0, x1
972 msr ARM64_REG_HID4, x0
973 LOAD_UINT64 x1, HID7_DEFEATURES_2
974 mrs x0, ARM64_REG_HID7
975 orr x0, x0, x1
976 msr ARM64_REG_HID7, x0
977 dsb sy
978 isb sy
979 b cpu_defeatures_set_ret
980 cpu_defeatures_set_ret:
981 POP_FRAME
982 ret
983 #endif
984
985 #else /* !defined(APPLE_ARM64_ARCH_FAMILY) */
986 .text
987 .align 2
988 .globl EXT(arm64_prepare_for_sleep)
989 LEXT(arm64_prepare_for_sleep)
990 PUSH_FRAME
991 Lwfi_inst:
992 dsb sy
993 isb sy
994 wfi
995 b Lwfi_inst
996
997 /*
998 * Force WFI to use clock gating only
999 * Note: for non-Apple device, do nothing.
1000 */
1001 .text
1002 .align 2
1003 .globl EXT(arm64_force_wfi_clock_gate)
1004 LEXT(arm64_force_wfi_clock_gate)
1005 PUSH_FRAME
1006 nop
1007 POP_FRAME
1008
1009 #endif /* defined(APPLE_ARM64_ARCH_FAMILY) */
1010
1011 /*
1012 * void arm64_replace_bootstack(cpu_data_t *cpu_data)
1013 *
1014 * This must be called from a kernel thread context running on the boot CPU,
1015 * after setting up new exception stacks in per-CPU data. That will guarantee
1016 * that the stack(s) we're trying to replace aren't currently in use. For
1017 * KTRR-protected devices, this must also be called prior to VM prot finalization
1018 * and lockdown, as updating SP1 requires a sensitive instruction.
1019 */
1020 .text
1021 .align 2
1022 .globl EXT(arm64_replace_bootstack)
1023 LEXT(arm64_replace_bootstack)
1024 ARM64_STACK_PROLOG
1025 PUSH_FRAME
1026 // Set the exception stack pointer
1027 ldr x0, [x0, CPU_EXCEPSTACK_TOP]
1028 mrs x4, DAIF // Load current DAIF; use x4 as pinst may trash x1-x3
1029 msr DAIFSet, #(DAIFSC_IRQF | DAIFSC_FIQF | DAIFSC_ASYNCF) // Disable IRQ/FIQ/serror
1030 // Set SP_EL1 to exception stack
1031 #if defined(KERNEL_INTEGRITY_KTRR) || defined(KERNEL_INTEGRITY_CTRR)
1032 mov x1, lr
1033 bl EXT(pinst_spsel_1)
1034 mov lr, x1
1035 #else
1036 msr SPSel, #1
1037 #endif
1038 mov sp, x0
1039 msr SPSel, #0
1040 msr DAIF, x4 // Restore interrupt state
1041 POP_FRAME
1042 ARM64_STACK_EPILOG
1043
1044 #ifdef MONITOR
1045 /*
1046 * unsigned long monitor_call(uintptr_t callnum, uintptr_t arg1,
1047 uintptr_t arg2, uintptr_t arg3)
1048 *
1049 * Call the EL3 monitor with 4 arguments in registers
1050 * The monitor interface maintains the same ABI as the C function call standard. Callee-saved
1051 * registers are preserved, temporary registers are not. Parameters and results are passed in
1052 * the usual manner.
1053 */
1054 .text
1055 .align 2
1056 .globl EXT(monitor_call)
1057 LEXT(monitor_call)
1058 smc 0x11
1059 ret
1060 #endif
1061
1062 #ifdef HAS_APPLE_PAC
1063 /*
1064 * SIGN_THREAD_STATE
1065 *
1066 * Macro that signs thread state.
1067 * $0 - Offset in arm_saved_state to store JOPHASH value.
1068 */
1069 .macro SIGN_THREAD_STATE
1070 pacga x1, x1, x0 /* PC hash (gkey + &arm_saved_state) */
1071 /*
1072 * Mask off the carry flag so we don't need to re-sign when that flag is
1073 * touched by the system call return path.
1074 */
1075 bic x2, x2, PSR_CF
1076 pacga x1, x2, x1 /* SPSR hash (gkey + pc hash) */
1077 pacga x1, x3, x1 /* LR Hash (gkey + spsr hash) */
1078 pacga x1, x4, x1 /* X16 hash (gkey + lr hash) */
1079 pacga x1, x5, x1 /* X17 hash (gkey + x16 hash) */
1080 str x1, [x0, $0]
1081 #if DEBUG || DEVELOPMENT
1082 mrs x1, DAIF
1083 tbz x1, #DAIF_IRQF_SHIFT, Lintr_enabled_panic
1084 #endif /* DEBUG || DEVELOPMENT */
1085 .endmacro
1086
1087 /*
1088 * CHECK_SIGNED_STATE
1089 *
1090 * Macro that checks signed thread state.
1091 * $0 - Offset in arm_saved_state to to read the JOPHASH value from.
1092 * $1 - Label to jump to when check is unsuccessful.
1093 */
1094 .macro CHECK_SIGNED_STATE
1095 pacga x1, x1, x0 /* PC hash (gkey + &arm_saved_state) */
1096 /*
1097 * Mask off the carry flag so we don't need to re-sign when that flag is
1098 * touched by the system call return path.
1099 */
1100 bic x2, x2, PSR_CF
1101 pacga x1, x2, x1 /* SPSR hash (gkey + pc hash) */
1102 pacga x1, x3, x1 /* LR Hash (gkey + spsr hash) */
1103 pacga x1, x4, x1 /* X16 hash (gkey + lr hash) */
1104 pacga x1, x5, x1 /* X17 hash (gkey + x16 hash) */
1105 ldr x2, [x0, $0]
1106 cmp x1, x2
1107 b.ne $1
1108 #if DEBUG || DEVELOPMENT
1109 mrs x1, DAIF
1110 tbz x1, #DAIF_IRQF_SHIFT, Lintr_enabled_panic
1111 #endif /* DEBUG || DEVELOPMENT */
1112 .endmacro
1113
1114 /**
1115 * void ml_sign_thread_state(arm_saved_state_t *ss, uint64_t pc,
1116 * uint32_t cpsr, uint64_t lr, uint64_t x16,
1117 * uint64_t x17)
1118 */
1119 .text
1120 .align 2
1121 .globl EXT(ml_sign_thread_state)
1122 LEXT(ml_sign_thread_state)
1123 SIGN_THREAD_STATE SS64_JOPHASH
1124 ret
1125
1126 /**
1127 * void ml_sign_kernel_thread_state(arm_kernel_saved_state *ss, uint64_t pc,
1128 * uint32_t cpsr, uint64_t lr, uint64_t x16,
1129 * uint64_t x17)
1130 */
1131 .text
1132 .align 2
1133 .globl EXT(ml_sign_kernel_thread_state)
1134 LEXT(ml_sign_kernel_thread_state)
1135 SIGN_THREAD_STATE SS64_KERNEL_JOPHASH
1136 ret
1137
1138 /**
1139 * void ml_check_signed_state(arm_saved_state_t *ss, uint64_t pc,
1140 * uint32_t cpsr, uint64_t lr, uint64_t x16,
1141 * uint64_t x17)
1142 */
1143 .text
1144 .align 2
1145 .globl EXT(ml_check_signed_state)
1146 LEXT(ml_check_signed_state)
1147 CHECK_SIGNED_STATE SS64_JOPHASH, Lcheck_hash_panic
1148 ret
1149 Lcheck_hash_panic:
1150 /*
1151 * ml_check_signed_state normally doesn't set up a stack frame, since it
1152 * needs to work in the face of attackers that can modify the stack.
1153 * However we lazily create one in the panic path: at this point we're
1154 * *only* using the stack frame for unwinding purposes, and without one
1155 * we'd be missing information about the caller.
1156 */
1157 ARM64_STACK_PROLOG
1158 PUSH_FRAME
1159 mov x1, x0
1160 adr x0, Lcheck_hash_str
1161 CALL_EXTERN panic_with_thread_kernel_state
1162
1163 /**
1164 * void ml_check_kernel_signed_state(arm_kernel_saved_state *ss, uint64_t pc,
1165 * uint32_t cpsr, uint64_t lr, uint64_t x16,
1166 * uint64_t x17)
1167 */
1168 .text
1169 .align 2
1170 .globl EXT(ml_check_kernel_signed_state)
1171 LEXT(ml_check_kernel_signed_state)
1172 CHECK_SIGNED_STATE SS64_KERNEL_JOPHASH, Lcheck_kernel_hash_panic
1173 ret
1174 Lcheck_kernel_hash_panic:
1175 ARM64_STACK_PROLOG
1176 PUSH_FRAME
1177 adr x0, Lcheck_hash_str
1178 CALL_EXTERN panic
1179
1180 Lcheck_hash_str:
1181 .asciz "JOP Hash Mismatch Detected (PC, CPSR, or LR corruption)"
1182
1183 #if DEBUG || DEVELOPMENT
1184 Lintr_enabled_panic:
1185 ARM64_STACK_PROLOG
1186 PUSH_FRAME
1187 adr x0, Lintr_enabled_str
1188 CALL_EXTERN panic
1189 Lintr_enabled_str:
1190 /*
1191 * Please see the "Signing spilled register state" section of doc/pac.md
1192 * for an explanation of why this is bad and how it should be fixed.
1193 */
1194 .asciz "Signed thread state manipulated with interrupts enabled"
1195 #endif /* DEBUG || DEVELOPMENT */
1196
1197 /**
1198 * void ml_auth_thread_state_invalid_cpsr(arm_saved_state_t *ss)
1199 *
1200 * Panics due to an invalid CPSR value in ss.
1201 */
1202 .text
1203 .align 2
1204 .globl EXT(ml_auth_thread_state_invalid_cpsr)
1205 LEXT(ml_auth_thread_state_invalid_cpsr)
1206 ARM64_STACK_PROLOG
1207 PUSH_FRAME
1208 mov x1, x0
1209 adr x0, Linvalid_cpsr_str
1210 CALL_EXTERN panic_with_thread_kernel_state
1211
1212 Linvalid_cpsr_str:
1213 .asciz "Thread state corruption detected (PE mode == 0)"
1214 #endif /* HAS_APPLE_PAC */
1215
1216 .text
1217 .align 2
1218 .globl EXT(fill32_dczva)
1219 LEXT(fill32_dczva)
1220 0:
1221 dc zva, x0
1222 add x0, x0, #64
1223 subs x1, x1, #64
1224 b.hi 0b
1225 ret
1226
1227 .text
1228 .align 2
1229 .globl EXT(fill32_nt)
1230 LEXT(fill32_nt)
1231 dup.4s v0, w2
1232 0:
1233 stnp q0, q0, [x0]
1234 stnp q0, q0, [x0, #0x20]
1235 stnp q0, q0, [x0, #0x40]
1236 stnp q0, q0, [x0, #0x60]
1237 add x0, x0, #128
1238 subs x1, x1, #128
1239 b.hi 0b
1240 ret
1241
1242 /* vim: set sw=4 ts=4: */
mirror of XNU