]> git.saurik.com Git - apple/xnu.git/blame - osfmk/arm64/machine_routines_asm.s
projects / apple / xnu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
xnu-4570.1.46.tar.gz
[apple/xnu.git] / osfmk / arm64 / machine_routines_asm.s
CommitLineData
5ba3f43e
A		 1/*
2 * Copyright (c) 2007-2015 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <machine/asm.h>
30#include <arm64/machine_machdep.h>
31#include <arm64/proc_reg.h>
32#include <arm/pmap.h>
33#include <pexpert/arm64/board_config.h>
34#include <sys/errno.h>
35#include "assym.s"
36
37
38/* uint32_t get_fpscr(void):
39 * Returns (FPSR | FPCR).
40 */
41 .align 2
42 .globl EXT(get_fpscr)
43LEXT(get_fpscr)
44#if __ARM_VFP__
45 mrs x1, FPSR // Grab FPSR
46 mov x4, #(FPSR_MASK & 0xFFFF)
47 mov x5, #(FPSR_MASK & 0xFFFF0000)
48 orr x0, x4, x5
49 and x1, x1, x0 // Be paranoid, and clear bits we expect to
50 // be clear
51 mrs x2, FPCR // Grab FPCR
52 mov x4, #(FPCR_MASK & 0xFFFF)
53 mov x5, #(FPCR_MASK & 0xFFFF0000)
54 orr x0, x4, x5
55 and x2, x2, x0 // Be paranoid, and clear bits we expect to
56 // be clear
57 orr x0, x1, x2 // OR them to get FPSCR equivalent state
58#else
59 mov x0, #0
60#endif
61 ret
62 .align 2
63 .globl EXT(set_fpscr)
64/* void set_fpscr(uint32_t value):
65 * Set the FPCR and FPSR registers, based on the given value; a
66 * noteworthy point is that unlike 32-bit mode, 64-bit mode FPSR
67 * and FPCR are not responsible for condition codes.
68 */
69LEXT(set_fpscr)
70#if __ARM_VFP__
71 mov x4, #(FPSR_MASK & 0xFFFF)
72 mov x5, #(FPSR_MASK & 0xFFFF0000)
73 orr x1, x4, x5
74 and x1, x1, x0 // Clear the bits that don't apply to FPSR
75 mov x4, #(FPCR_MASK & 0xFFFF)
76 mov x5, #(FPCR_MASK & 0xFFFF0000)
77 orr x2, x4, x5
78 and x2, x2, x0 // Clear the bits that don't apply to FPCR
79 msr FPSR, x1 // Write FPCR
80 msr FPCR, x2 // Write FPSR
81 dsb ish // FPCR requires synchronization
82#endif
83 ret
84
85#if (__ARM_VFP__ >= 3)
86 .align 2
87 .globl EXT(get_mvfr0)
88LEXT(get_mvfr0)
89 mrs x0, MVFR0_EL1
90 ret
91
92 .globl EXT(get_mvfr1)
93LEXT(get_mvfr1)
94 mrs x0, MVFR1_EL1
95 ret
96
97#endif
98
99/*
100 * void flush_mmu_tlb(void)
101 *
102 * Flush all TLBs
103 */
104 .text
105 .align 2
106 .globl EXT(flush_mmu_tlb)
107LEXT(flush_mmu_tlb)
108 tlbi vmalle1is
109 dsb ish
110 isb sy
111 ret
112
113/*
114 * void flush_core_tlb(void)
115 *
116 * Flush core TLB
117 */
118 .text
119 .align 2
120 .globl EXT(flush_core_tlb)
121LEXT(flush_core_tlb)
122 tlbi vmalle1
123 dsb ish
124 isb sy
125 ret
126
127/*
128 * void flush_mmu_tlb_allentries(uint64_t, uint64_t)
129 *
130 * Flush TLB entries
131 */
132 .text
133 .align 2
134 .globl EXT(flush_mmu_tlb_allentries)
135LEXT(flush_mmu_tlb_allentries)
136#if __ARM_16K_PG__
137 and x0, x0, #~0x3
138
139 /*
140 * The code below is not necessarily correct. From an overview of
141 * the client code, the expected contract for TLB flushes is that
142 * we will expand from an "address, length" pair to "start address,
143 * end address" in the course of a TLB flush. This suggests that
144 * a flush for "X, X+4" is actually only asking for a flush of a
145 * single 16KB page. At the same time, we'd like to be prepared
146 * for bad inputs (X, X+3), so add 3 and then truncate the 4KB page
147 * number to a 16KB page boundary. This should deal correctly with
148 * unaligned inputs.
149 *
150 * If our expecations about client behavior are wrong however, this
151 * will lead to occasional TLB corruption on platforms with 16KB
152 * pages.
153 */
154 add x1, x1, #0x3
155 and x1, x1, #~0x3
156#endif
157
1581:
159 tlbi vaae1is, x0
160 add x0, x0, #(ARM_PGBYTES / 4096) // Units are 4KB pages, as defined by the ISA
161 cmp x0, x1
162 b.lt 1b
163 dsb ish
164 isb sy
165 ret
166
167/*
168 * void flush_mmu_tlb_entry(uint64_t)
169 *
170 * Flush TLB entry
171 */
172 .text
173 .align 2
174 .globl EXT(flush_mmu_tlb_entry)
175LEXT(flush_mmu_tlb_entry)
176 tlbi vae1is, x0
177 dsb ish
178 isb sy
179 ret
180
181/*
182 * void flush_mmu_tlb_entries(uint64_t, uint64_t)
183 *
184 * Flush TLB entries
185 */
186 .text
187 .align 2
188 .globl EXT(flush_mmu_tlb_entries)
189LEXT(flush_mmu_tlb_entries)
190#if __ARM_16K_PG__
191 and x0, x0, #~0x3
192
193 /*
194 * The code below is not necessarily correct. From an overview of
195 * the client code, the expected contract for TLB flushes is that
196 * we will expand from an "address, length" pair to "start address,
197 * end address" in the course of a TLB flush. This suggests that
198 * a flush for "X, X+4" is actually only asking for a flush of a
199 * single 16KB page. At the same time, we'd like to be prepared
200 * for bad inputs (X, X+3), so add 3 and then truncate the 4KB page
201 * number to a 16KB page boundary. This should deal correctly with
202 * unaligned inputs.
203 *
204 * If our expecations about client behavior are wrong however, this
205 * will lead to occasional TLB corruption on platforms with 16KB
206 * pages.
207 */
208 add x1, x1, #0x3
209 and x1, x1, #~0x3
210#endif
211
2121:
213 tlbi vae1is, x0
214 add x0, x0, #(ARM_PGBYTES / 4096) // Units are pages
215 cmp x0, x1
216 b.lt 1b
217 dsb ish
218 isb sy
219 ret
220
221/*
222 * void flush_mmu_tlb_asid(uint64_t)
223 *
224 * Flush TLB entriesfor requested asid
225 */
226 .text
227 .align 2
228 .globl EXT(flush_mmu_tlb_asid)
229LEXT(flush_mmu_tlb_asid)
230 tlbi aside1is, x0
231 dsb ish
232 isb sy
233 ret
234
235/*
236 * void flush_core_tlb_asid(uint64_t)
237 *
238 * Flush TLB entries for core for requested asid
239 */
240 .text
241 .align 2
242 .globl EXT(flush_core_tlb_asid)
243LEXT(flush_core_tlb_asid)
244 tlbi aside1, x0
245 dsb ish
246 isb sy
247 ret
248
249/*
250 * Set MMU Translation Table Base Alternate
251 */
252 .text
253 .align 2
254 .globl EXT(set_mmu_ttb_alternate)
255LEXT(set_mmu_ttb_alternate)
256 dsb sy
257#if defined(KERNEL_INTEGRITY_KTRR)
258 mov x1, lr
259 bl EXT(pinst_set_ttbr1)
260 mov lr, x1
261#else
262 msr TTBR1_EL1, x0
263#endif /* defined(KERNEL_INTEGRITY_KTRR) */
264 isb sy
265 ret
266
267/*
268 * set AUX control register
269 */
270 .text
271 .align 2
272 .globl EXT(set_aux_control)
273LEXT(set_aux_control)
274 msr ACTLR_EL1, x0
275 // Synchronize system
276 dsb sy
277 isb sy
278 ret
279
280#if (DEVELOPMENT || DEBUG)
281/*
282 * set MMU control register
283 */
284 .text
285 .align 2
286 .globl EXT(set_mmu_control)
287LEXT(set_mmu_control)
288 msr SCTLR_EL1, x0
289 dsb sy
290 isb sy
291 ret
292#endif
293
294
295/*
296 * set translation control register
297 */
298 .text
299 .align 2
300 .globl EXT(set_tcr)
301LEXT(set_tcr)
302#if defined(APPLE_ARM64_ARCH_FAMILY)
303 // Assert that T0Z is always equal to T1Z
304 eor x1, x0, x0, lsr #(TCR_T1SZ_SHIFT - TCR_T0SZ_SHIFT)
305 and x1, x1, #(TCR_TSZ_MASK << TCR_T0SZ_SHIFT)
306 cbnz x1, L_set_tcr_panic
307#if defined(KERNEL_INTEGRITY_KTRR)
308 mov x1, lr
309 bl _pinst_set_tcr
310 mov lr, x1
311#else
312 msr TCR_EL1, x0
313#endif /* defined(KERNEL_INTRITY_KTRR) */
314 isb sy
315 ret
316
317L_set_tcr_panic:
318 PUSH_FRAME
319 sub sp, sp, #16
320 str x0, [sp]
321 adr x0, L_set_tcr_panic_str
322 BRANCH_EXTERN panic
323
324L_set_locked_reg_panic:
325 PUSH_FRAME
326 sub sp, sp, #16
327 str x0, [sp]
328 adr x0, L_set_locked_reg_panic_str
329 BRANCH_EXTERN panic
330 b .
331
332L_set_tcr_panic_str:
333 .asciz "set_tcr: t0sz, t1sz not equal (%llx)\n"
334
335
336L_set_locked_reg_panic_str:
337 .asciz "attempt to set locked register: (%llx)\n"
338#else
339#if defined(KERNEL_INTEGRITY_KTRR)
340 mov x1, lr
341 bl _pinst_set_tcr
342 mov lr, x1
343#else
344 msr TCR_EL1, x0
345#endif
346 isb sy
347 ret
348#endif // defined(APPLE_ARM64_ARCH_FAMILY)
349
350/*
351 * MMU kernel virtual to physical address translation
352 */
353 .text
354 .align 2
355 .globl EXT(mmu_kvtop)
356LEXT(mmu_kvtop)
357 mrs x2, DAIF // Load current DAIF
358 msr DAIFSet, #(DAIFSC_IRQF | DAIFSC_FIQF) // Disable IRQ
359 at s1e1r, x0 // Translation Stage 1 EL1
360 mrs x1, PAR_EL1 // Read result
361 msr DAIF, x2 // Restore interrupt state
362 tbnz x1, #0, L_mmu_kvtop_invalid // Test Translation not valid
363 bfm x1, x0, #0, #11 // Add page offset
364 and x0, x1, #0x0000ffffffffffff // Clear non-address bits
365 ret
366L_mmu_kvtop_invalid:
367 mov x0, xzr // Return invalid
368 ret
369
370/*
371 * MMU user virtual to physical address translation
372 */
373 .text
374 .align 2
375 .globl EXT(mmu_uvtop)
376LEXT(mmu_uvtop)
377 lsr x8, x0, #56 // Extract top byte
378 cbnz x8, L_mmu_uvtop_invalid // Tagged pointers are invalid
379 mrs x2, DAIF // Load current DAIF
380 msr DAIFSet, #(DAIFSC_IRQF | DAIFSC_FIQF) // Disable IRQ
381 at s1e0r, x0 // Translation Stage 1 EL0
382 mrs x1, PAR_EL1 // Read result
383 msr DAIF, x2 // Restore interrupt state
384 tbnz x1, #0, L_mmu_uvtop_invalid // Test Translation not valid
385 bfm x1, x0, #0, #11 // Add page offset
386 and x0, x1, #0x0000ffffffffffff // Clear non-address bits
387 ret
388L_mmu_uvtop_invalid:
389 mov x0, xzr // Return invalid
390 ret
391
392/*
393 * MMU kernel virtual to physical address preflight write access
394 */
395 .text
396 .align 2
397 .globl EXT(mmu_kvtop_wpreflight)
398LEXT(mmu_kvtop_wpreflight)
399 mrs x2, DAIF // Load current DAIF
400 msr DAIFSet, #(DAIFSC_IRQF | DAIFSC_FIQF) // Disable IRQ
401 at s1e1w, x0 // Translation Stage 1 EL1
402 mrs x1, PAR_EL1 // Read result
403 msr DAIF, x2 // Restore interrupt state
404 tbnz x1, #0, L_mmu_kvtop_wpreflight_invalid // Test Translation not valid
405 bfm x1, x0, #0, #11 // Add page offset
406 and x0, x1, #0x0000ffffffffffff // Clear non-address bits
407 ret
408L_mmu_kvtop_wpreflight_invalid:
409 mov x0, xzr // Return invalid
410 ret
411
412/*
413 * SET_RECOVERY_HANDLER
414 *
415 * Sets up a page fault recovery handler
416 *
417 * arg0 - persisted thread pointer
418 * arg1 - persisted recovery handler
419 * arg2 - scratch reg
420 * arg3 - recovery label
421 */
422.macro SET_RECOVERY_HANDLER
423 mrs $0, TPIDR_EL1 // Load thread pointer
424 ldr $1, [$0, TH_RECOVER] // Save previous recovery handler
425 adrp $2, $3@page // Load the recovery handler address
426 add $2, $2, $3@pageoff
427 str $2, [$0, TH_RECOVER] // Set new recovery handler
428.endmacro
429
430/*
431 * CLEAR_RECOVERY_HANDLER
432 *
433 * Clears page fault handler set by SET_RECOVERY_HANDLER
434 *
435 * arg0 - thread pointer saved by SET_RECOVERY_HANDLER
436 * arg1 - old recovery handler saved by SET_RECOVERY_HANDLER
437 */
438.macro CLEAR_RECOVERY_HANDLER
439 str $1, [$0, TH_RECOVER] // Restore the previous recovery handler
440.endmacro
441
442
443 .text
444 .align 2
445copyio_error:
446 CLEAR_RECOVERY_HANDLER x10, x11
447 mov x0, #EFAULT // Return an EFAULT error
448 POP_FRAME
449 ret
450
451/*
452 * int _bcopyin(const char *src, char *dst, vm_size_t len)
453 */
454 .text
455 .align 2
456 .globl EXT(_bcopyin)
457LEXT(_bcopyin)
458 PUSH_FRAME
459 SET_RECOVERY_HANDLER x10, x11, x3, copyio_error
460 /* If len is less than 16 bytes, just do a bytewise copy */
461 cmp x2, #16
462 b.lt 2f
463 sub x2, x2, #16
4641:
465 /* 16 bytes at a time */
466 ldp x3, x4, [x0], #16
467 stp x3, x4, [x1], #16
468 subs x2, x2, #16
469 b.ge 1b
470 /* Fixup the len and test for completion */
471 adds x2, x2, #16
472 b.eq 3f
4732: /* Bytewise */
474 subs x2, x2, #1
475 ldrb w3, [x0], #1
476 strb w3, [x1], #1
477 b.hi 2b
4783:
479 CLEAR_RECOVERY_HANDLER x10, x11
480 mov x0, xzr
481 POP_FRAME
482 ret
483
484/*
485 * int _copyin_word(const char *src, uint64_t *dst, vm_size_t len)
486 */
487 .text
488 .align 2
489 .globl EXT(_copyin_word)
490LEXT(_copyin_word)
491 PUSH_FRAME
492 SET_RECOVERY_HANDLER x10, x11, x3, copyio_error
493 cmp x2, #4
494 b.eq L_copyin_word_4
495 cmp x2, #8
496 b.eq L_copyin_word_8
497 mov x0, EINVAL
498 b L_copying_exit
499L_copyin_word_4:
500 ldr w8, [x0]
501 b L_copyin_word_store
502L_copyin_word_8:
503 ldr x8, [x0]
504L_copyin_word_store:
505 str x8, [x1]
506 mov x0, xzr
507 CLEAR_RECOVERY_HANDLER x10, x11
508L_copying_exit:
509 POP_FRAME
510 ret
511
512
513/*
514 * int _bcopyout(const char *src, char *dst, vm_size_t len)
515 */
516 .text
517 .align 2
518 .globl EXT(_bcopyout)
519LEXT(_bcopyout)
520 PUSH_FRAME
521 SET_RECOVERY_HANDLER x10, x11, x3, copyio_error
522 /* If len is less than 16 bytes, just do a bytewise copy */
523 cmp x2, #16
524 b.lt 2f
525 sub x2, x2, #16
5261:
527 /* 16 bytes at a time */
528 ldp x3, x4, [x0], #16
529 stp x3, x4, [x1], #16
530 subs x2, x2, #16
531 b.ge 1b
532 /* Fixup the len and test for completion */
533 adds x2, x2, #16
534 b.eq 3f
5352: /* Bytewise */
536 subs x2, x2, #1
537 ldrb w3, [x0], #1
538 strb w3, [x1], #1
539 b.hi 2b
5403:
541 CLEAR_RECOVERY_HANDLER x10, x11
542 mov x0, xzr
543 POP_FRAME
544 ret
545
546/*
547 * int _bcopyinstr(
548 * const user_addr_t user_addr,
549 * char *kernel_addr,
550 * vm_size_t max,
551 * vm_size_t *actual)
552 */
553 .text
554 .align 2
555 .globl EXT(_bcopyinstr)
556LEXT(_bcopyinstr)
557 PUSH_FRAME
558 adr x4, Lcopyinstr_error // Get address for recover
559 mrs x10, TPIDR_EL1 // Get thread pointer
560 ldr x11, [x10, TH_RECOVER] // Save previous recover
561 str x4, [x10, TH_RECOVER] // Store new recover
562 mov x4, xzr // x4 - total bytes copied
563Lcopyinstr_loop:
564 ldrb w5, [x0], #1 // Load a byte from the user source
565 strb w5, [x1], #1 // Store a byte to the kernel dest
566 add x4, x4, #1 // Increment bytes copied
567 cbz x5, Lcopyinstr_done // If this byte is null, we're done
568 cmp x4, x2 // If we're out of space, return an error
569 b.ne Lcopyinstr_loop
570Lcopyinstr_too_long:
571 mov x5, #ENAMETOOLONG // Set current byte to error code for later return
572Lcopyinstr_done:
573 str x4, [x3] // Return number of bytes copied
574 mov x0, x5 // Set error code (0 on success, ENAMETOOLONG on failure)
575 b Lcopyinstr_exit
576Lcopyinstr_error:
577 mov x0, #EFAULT // Return EFAULT on error
578Lcopyinstr_exit:
579 str x11, [x10, TH_RECOVER] // Restore old recover
580 POP_FRAME
581 ret
582
583/*
584 * int copyinframe(const vm_address_t frame_addr, char *kernel_addr, bool is64bit)
585 *
586 * Safely copy sixteen bytes (the fixed top of an ARM64 frame) from
587 * either user or kernel memory, or 8 bytes (AArch32) from user only.
588 *
589 * x0 : address of frame to copy.
590 * x1 : kernel address at which to store data.
591 * w2 : whether to copy an AArch32 or AArch64 frame.
592 * x3 : temp
593 * x5 : temp (kernel virtual base)
594 * x9 : temp
595 * x10 : thread pointer (set by SET_RECOVERY_HANDLER)
596 * x11 : old recovery function (set by SET_RECOVERY_HANDLER)
597 * x12, x13 : backtrace data
598 *
599 */
600 .text
601 .align 2
602 .globl EXT(copyinframe)
603LEXT(copyinframe)
604 PUSH_FRAME
605 SET_RECOVERY_HANDLER x10, x11, x3, copyio_error
606 cbnz w2, Lcopyinframe64 // Check frame size
607 adrp x5, EXT(gVirtBase)@page // For 32-bit frame, make sure we're not trying to copy from kernel
608 add x5, x5, EXT(gVirtBase)@pageoff
609 ldr x5, [x5]
610 cmp x5, x0 // See if address is in kernel virtual range
611 b.hi Lcopyinframe32 // If below kernel virtual range, proceed.
612 mov w0, #EFAULT // Should never have a 32-bit frame in kernel virtual range
613 b Lcopyinframe_done
614
615Lcopyinframe32:
616 ldr x12, [x0] // Copy 8 bytes
617 str x12, [x1]
618 mov w0, #0 // Success
619 b Lcopyinframe_done
620
621Lcopyinframe64:
622 mov x3, VM_MIN_KERNEL_ADDRESS // Check if kernel address
623 orr x9, x0, TBI_MASK // Hide tags in address comparison
624 cmp x9, x3 // If in kernel address range, skip tag test
625 b.hs Lcopyinframe_valid
626 tst x0, TBI_MASK // Detect tagged pointers
627 b.eq Lcopyinframe_valid
628 mov w0, #EFAULT // Tagged address, fail
629 b Lcopyinframe_done
630Lcopyinframe_valid:
631 ldp x12, x13, [x0] // Copy 16 bytes
632 stp x12, x13, [x1]
633 mov w0, #0 // Success
634
635Lcopyinframe_done:
636 CLEAR_RECOVERY_HANDLER x10, x11
637 POP_FRAME
638 ret
639
640
641/*
642 * int _emulate_swp(user_addr_t addr, uint32_t newval, uint32_t *oldval)
643 *
644 * Securely emulates the swp instruction removed from armv8.
645 * Returns true on success.
646 * Returns false if the user address is not user accessible.
647 *
648 * x0 : address to swap
649 * x1 : new value to store
650 * x2 : address to save old value
651 * x3 : scratch reg
652 * x10 : thread pointer (set by SET_RECOVERY_HANDLER)
653 * x11 : old recovery handler (set by SET_RECOVERY_HANDLER)
654 * x12 : interrupt state
655 * x13 : return value
656 */
657 .text
658 .align 2
659 .globl EXT(_emulate_swp)
660LEXT(_emulate_swp)
661 PUSH_FRAME
662 SET_RECOVERY_HANDLER x10, x11, x3, swp_error
663
664 // Perform swap
665Lswp_try:
666 ldxr w3, [x0] // Load data at target address
667 stxr w4, w1, [x0] // Store new value to target address
668 cbnz w4, Lswp_try // Retry if store failed
669 str w3, [x2] // Save old value
670 mov x13, #1 // Set successful return value
671
672Lswp_exit:
673 mov x0, x13 // Set return value
674 CLEAR_RECOVERY_HANDLER x10, x11
675 POP_FRAME
676 ret
677
678/*
679 * int _emulate_swpb(user_addr_t addr, uint32_t newval, uint32_t *oldval)
680 *
681 * Securely emulates the swpb instruction removed from armv8.
682 * Returns true on success.
683 * Returns false if the user address is not user accessible.
684 *
685 * x0 : address to swap
686 * x1 : new value to store
687 * x2 : address to save old value
688 * x3 : scratch reg
689 * x10 : thread pointer (set by SET_RECOVERY_HANDLER)
690 * x11 : old recovery handler (set by SET_RECOVERY_HANDLER)
691 * x12 : interrupt state
692 * x13 : return value
693 */
694 .text
695 .align 2
696 .globl EXT(_emulate_swpb)
697LEXT(_emulate_swpb)
698 PUSH_FRAME
699 SET_RECOVERY_HANDLER x10, x11, x3, swp_error
700
701 // Perform swap
702Lswpb_try:
703 ldxrb w3, [x0] // Load data at target address
704 stxrb w4, w1, [x0] // Store new value to target address
705 cbnz w4, Lswp_try // Retry if store failed
706 str w3, [x2] // Save old value
707 mov x13, #1 // Set successful return value
708
709Lswpb_exit:
710 mov x0, x13 // Set return value
711 CLEAR_RECOVERY_HANDLER x10, x11
712 POP_FRAME
713 ret
714
715 .text
716 .align 2
717swp_error:
718 mov x0, xzr // Return false
719 CLEAR_RECOVERY_HANDLER x10, x11
720 POP_FRAME
721 ret
722
723/*
724 * uint32_t arm_debug_read_dscr(void)
725 */
726 .text
727 .align 2
728 .globl EXT(arm_debug_read_dscr)
729LEXT(arm_debug_read_dscr)
730 PANIC_UNIMPLEMENTED
731
732/*
733 * void arm_debug_set_cp14(arm_debug_state_t *debug_state)
734 *
735 * Set debug registers to match the current thread state
736 * (NULL to disable). Assume 6 breakpoints and 2
737 * watchpoints, since that has been the case in all cores
738 * thus far.
739 */
740 .text
741 .align 2
742 .globl EXT(arm_debug_set_cp14)
743LEXT(arm_debug_set_cp14)
744 PANIC_UNIMPLEMENTED
745
746
747#if defined(APPLE_ARM64_ARCH_FAMILY)
748/*
749 * Note: still have to ISB before executing wfi!
750 */
751 .text
752 .align 2
753 .globl EXT(arm64_prepare_for_sleep)
754LEXT(arm64_prepare_for_sleep)
755 PUSH_FRAME
756
757#if defined(APPLECYCLONE) || defined(APPLETYPHOON)
758 // <rdar://problem/15827409> CPU1 Stuck in WFIWT Because of MMU Prefetch
759 mrs x0, ARM64_REG_HID2 // Read HID2
760 orr x0, x0, #(ARM64_REG_HID2_disMMUmtlbPrefetch) // Set HID.DisableMTLBPrefetch
761 msr ARM64_REG_HID2, x0 // Write HID2
762 dsb sy
763 isb sy
764#endif
765
766#if __ARM_GLOBAL_SLEEP_BIT__
767 // Enable deep sleep
768 mrs x1, ARM64_REG_ACC_OVRD
769 orr x1, x1, #(ARM64_REG_ACC_OVRD_enDeepSleep)
770 and x1, x1, #(~(ARM64_REG_ACC_OVRD_disL2Flush4AccSlp_mask))
771 orr x1, x1, #( ARM64_REG_ACC_OVRD_disL2Flush4AccSlp_deepsleep)
772 and x1, x1, #(~(ARM64_REG_ACC_OVRD_ok2PwrDnSRM_mask))
773 orr x1, x1, #( ARM64_REG_ACC_OVRD_ok2PwrDnSRM_deepsleep)
774 and x1, x1, #(~(ARM64_REG_ACC_OVRD_ok2TrDnLnk_mask))
775 orr x1, x1, #( ARM64_REG_ACC_OVRD_ok2TrDnLnk_deepsleep)
776 and x1, x1, #(~(ARM64_REG_ACC_OVRD_ok2PwrDnCPM_mask))
777 orr x1, x1, #( ARM64_REG_ACC_OVRD_ok2PwrDnCPM_deepsleep)
778 msr ARM64_REG_ACC_OVRD, x1
779
780
781#else
782 // Enable deep sleep
783 mov x1, ARM64_REG_CYC_CFG_deepSleep
784 msr ARM64_REG_CYC_CFG, x1
785#endif
786 // Set "OK to power down" (<rdar://problem/12390433>)
787 mrs x0, ARM64_REG_CYC_OVRD
788 orr x0, x0, #(ARM64_REG_CYC_OVRD_ok2pwrdn_force_down)
789 msr ARM64_REG_CYC_OVRD, x0
790
791Lwfi_inst:
792 dsb sy
793 isb sy
794 wfi
795 b Lwfi_inst
796
797/*
798 * Force WFI to use clock gating only
799 *
800 */
801 .text
802 .align 2
803 .globl EXT(arm64_force_wfi_clock_gate)
804LEXT(arm64_force_wfi_clock_gate)
805 PUSH_FRAME
806
807 mrs x0, ARM64_REG_CYC_OVRD
808 orr x0, x0, #(ARM64_REG_CYC_OVRD_ok2pwrdn_force_up)
809 msr ARM64_REG_CYC_OVRD, x0
810
811 POP_FRAME
812 ret
813
814
815
816#if defined(APPLECYCLONE) || defined(APPLETYPHOON)
817
818 .text
819 .align 2
820 .globl EXT(cyclone_typhoon_prepare_for_wfi)
821
822LEXT(cyclone_typhoon_prepare_for_wfi)
823 PUSH_FRAME
824
825 // <rdar://problem/15827409> CPU1 Stuck in WFIWT Because of MMU Prefetch
826 mrs x0, ARM64_REG_HID2 // Read HID2
827 orr x0, x0, #(ARM64_REG_HID2_disMMUmtlbPrefetch) // Set HID.DisableMTLBPrefetch
828 msr ARM64_REG_HID2, x0 // Write HID2
829 dsb sy
830 isb sy
831
832 POP_FRAME
833 ret
834
835
836 .text
837 .align 2
838 .globl EXT(cyclone_typhoon_return_from_wfi)
839LEXT(cyclone_typhoon_return_from_wfi)
840 PUSH_FRAME
841
842 // <rdar://problem/15827409> CPU1 Stuck in WFIWT Because of MMU Prefetch
843 mrs x0, ARM64_REG_HID2 // Read HID2
844 mov x1, #(ARM64_REG_HID2_disMMUmtlbPrefetch) //
845 bic x0, x0, x1 // Clear HID.DisableMTLBPrefetchMTLBPrefetch
846 msr ARM64_REG_HID2, x0 // Write HID2
847 dsb sy
848 isb sy
849
850 POP_FRAME
851 ret
852#endif
853
854#ifdef APPLETYPHOON
855
856#define HID0_DEFEATURES_1 0x0000a0c000064010ULL
857#define HID1_DEFEATURES_1 0x000000004005bf20ULL
858#define HID2_DEFEATURES_1 0x0000000000102074ULL
859#define HID3_DEFEATURES_1 0x0000000000400003ULL
860#define HID4_DEFEATURES_1 0x83ff00e100000268ULL
861#define HID7_DEFEATURES_1 0x000000000000000eULL
862
863#define HID0_DEFEATURES_2 0x0000a1c000020010ULL
864#define HID1_DEFEATURES_2 0x000000000005d720ULL
865#define HID2_DEFEATURES_2 0x0000000000002074ULL
866#define HID3_DEFEATURES_2 0x0000000000400001ULL
867#define HID4_DEFEATURES_2 0x8390000200000208ULL
868#define HID7_DEFEATURES_2 0x0000000000000000ULL
869
870/*
871 arg0 = target register
872 arg1 = 64-bit constant
873*/
874.macro LOAD_UINT64
875 movz $0, #(($1 >> 48) & 0xffff), lsl #48
876 movk $0, #(($1 >> 32) & 0xffff), lsl #32
877 movk $0, #(($1 >> 16) & 0xffff), lsl #16
878 movk $0, #(($1) & 0xffff)
879.endmacro
880
881 .text
882 .align 2
883 .globl EXT(cpu_defeatures_set)
884LEXT(cpu_defeatures_set)
885 PUSH_FRAME
886 cmp x0, #2
887 b.eq cpu_defeatures_set_2
888 cmp x0, #1
889 b.ne cpu_defeatures_set_ret
890 LOAD_UINT64 x1, HID0_DEFEATURES_1
891 mrs x0, ARM64_REG_HID0
892 orr x0, x0, x1
893 msr ARM64_REG_HID0, x0
894 LOAD_UINT64 x1, HID1_DEFEATURES_1
895 mrs x0, ARM64_REG_HID1
896 orr x0, x0, x1
897 msr ARM64_REG_HID1, x0
898 LOAD_UINT64 x1, HID2_DEFEATURES_1
899 mrs x0, ARM64_REG_HID2
900 orr x0, x0, x1
901 msr ARM64_REG_HID2, x0
902 LOAD_UINT64 x1, HID3_DEFEATURES_1
903 mrs x0, ARM64_REG_HID3
904 orr x0, x0, x1
905 msr ARM64_REG_HID3, x0
906 LOAD_UINT64 x1, HID4_DEFEATURES_1
907 mrs x0, ARM64_REG_HID4
908 orr x0, x0, x1
909 msr ARM64_REG_HID4, x0
910 LOAD_UINT64 x1, HID7_DEFEATURES_1
911 mrs x0, ARM64_REG_HID7
912 orr x0, x0, x1
913 msr ARM64_REG_HID7, x0
914 dsb sy
915 isb sy
916 b cpu_defeatures_set_ret
917cpu_defeatures_set_2:
918 LOAD_UINT64 x1, HID0_DEFEATURES_2
919 mrs x0, ARM64_REG_HID0
920 orr x0, x0, x1
921 msr ARM64_REG_HID0, x0
922 LOAD_UINT64 x1, HID1_DEFEATURES_2
923 mrs x0, ARM64_REG_HID1
924 orr x0, x0, x1
925 msr ARM64_REG_HID1, x0
926 LOAD_UINT64 x1, HID2_DEFEATURES_2
927 mrs x0, ARM64_REG_HID2
928 orr x0, x0, x1
929 msr ARM64_REG_HID2, x0
930 LOAD_UINT64 x1, HID3_DEFEATURES_2
931 mrs x0, ARM64_REG_HID3
932 orr x0, x0, x1
933 msr ARM64_REG_HID3, x0
934 LOAD_UINT64 x1, HID4_DEFEATURES_2
935 mrs x0, ARM64_REG_HID4
936 orr x0, x0, x1
937 msr ARM64_REG_HID4, x0
938 LOAD_UINT64 x1, HID7_DEFEATURES_2
939 mrs x0, ARM64_REG_HID7
940 orr x0, x0, x1
941 msr ARM64_REG_HID7, x0
942 dsb sy
943 isb sy
944 b cpu_defeatures_set_ret
945cpu_defeatures_set_ret:
946 POP_FRAME
947 ret
948#endif
949
950#endif
951
952#ifdef MONITOR
953/*
954 * unsigned long monitor_call(uintptr_t callnum, uintptr_t arg1,
955 uintptr_t arg2, uintptr_t arg3)
956 *
957 * Call the EL3 monitor with 4 arguments in registers
958 * The monitor interface maintains the same ABI as the C function call standard. Callee-saved
959 * registers are preserved, temporary registers are not. Parameters and results are passed in
960 * the usual manner.
961 */
962 .text
963 .align 2
964 .globl EXT(monitor_call)
965LEXT(monitor_call)
966 smc 0x11
967 ret
968#endif
969
970/* vim: set sw=4 ts=4: */
mirror of XNU