osfmk/arm64/machine_routines_asm.s

   1 /*
   2  * Copyright (c) 2007-2015 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <machine/asm.h>
  30 #include <arm64/machine_machdep.h>
  31 #include <arm64/proc_reg.h>
  32 #include <arm/pmap.h>
  33 #include <pexpert/arm64/board_config.h>
  34 #include <sys/errno.h>
  35 #include "assym.s"
  36
  37
  38
  39 /*      uint32_t get_fpscr(void):
  40  *              Returns (FPSR | FPCR).
  41  */
  42         .align  2
  43         .globl  EXT(get_fpscr)
  44 LEXT(get_fpscr)
  45 #if     __ARM_VFP__
  46         mrs     x1, FPSR                        // Grab FPSR
  47         mov     x4, #(FPSR_MASK & 0xFFFF)
  48         mov     x5, #(FPSR_MASK & 0xFFFF0000)
  49         orr     x0, x4, x5
  50         and     x1, x1, x0                      // Be paranoid, and clear bits we expect to
  51                                                 // be clear
  52         mrs     x2, FPCR                        // Grab FPCR
  53         mov     x4, #(FPCR_MASK & 0xFFFF)
  54         mov     x5, #(FPCR_MASK & 0xFFFF0000)
  55         orr     x0, x4, x5
  56         and     x2, x2, x0                      // Be paranoid, and clear bits we expect to
  57                                                 // be clear
  58         orr     x0, x1, x2                      // OR them to get FPSCR equivalent state
  59 #else
  60         mov     x0, #0
  61 #endif
  62         ret
  63         .align  2
  64         .globl  EXT(set_fpscr)
  65 /*      void set_fpscr(uint32_t value):
  66  *              Set the FPCR and FPSR registers, based on the given value; a
  67  *              noteworthy point is that unlike 32-bit mode, 64-bit mode FPSR
  68  *              and FPCR are not responsible for condition codes.
  69  */
  70 LEXT(set_fpscr)
  71 #if     __ARM_VFP__
  72         mov     x4, #(FPSR_MASK & 0xFFFF)
  73         mov     x5, #(FPSR_MASK & 0xFFFF0000)
  74         orr     x1, x4, x5
  75         and     x1, x1, x0                      // Clear the bits that don't apply to FPSR
  76         mov     x4, #(FPCR_MASK & 0xFFFF)
  77         mov     x5, #(FPCR_MASK & 0xFFFF0000)
  78         orr     x2, x4, x5
  79         and     x2, x2, x0                      // Clear the bits that don't apply to FPCR
  80         msr     FPSR, x1                        // Write FPCR
  81         msr     FPCR, x2                        // Write FPSR
  82         dsb     ish                             // FPCR requires synchronization
  83 #endif
  84         ret
  85
  86 /*
  87  * void update_mdscr(unsigned long clear, unsigned long set)
  88  *   Clears and sets the specified bits in MDSCR_EL1.
  89  *
  90  * Setting breakpoints in EL1 is effectively a KTRR bypass. The ability to do so is
  91  * controlled by MDSCR.KDE. The MSR to set MDSCR must be present to allow
  92  * self-hosted user mode debug. Any checks before the MRS can be skipped with ROP,
  93  * so we need to put the checks after the MRS where they can't be skipped. That
  94  * still leaves a small window if a breakpoint is set on the instruction
  95  * immediately after the MRS. To handle that, we also do a check and then set of
  96  * the breakpoint control registers. This allows us to guarantee that a given
  97  * core will never have both KDE set and a breakpoint targeting EL1.
  98  *
  99  * If KDE gets set, unset it and then panic
 100  */
 101         .align 2
 102         .globl EXT(update_mdscr)
 103 LEXT(update_mdscr)
 104         mov     x4, #0
 105         mrs     x2, MDSCR_EL1
 106         bic     x2, x2, x0
 107         orr     x2, x2, x1
 108 1:
 109         bic     x2, x2, #0x2000
 110         msr     MDSCR_EL1, x2
 111 #if defined(CONFIG_KERNEL_INTEGRITY)
 112         /*
 113          * verify KDE didn't get set (including via ROP)
 114          * If set, clear it and then panic
 115          */
 116         ands    x3, x2, #0x2000
 117         orr     x4, x4, x3
 118         bne     1b
 119         cmp     x4, xzr
 120         b.ne    Lupdate_mdscr_panic
 121 #endif
 122         ret
 123
 124 Lupdate_mdscr_panic:
 125         adrp    x0, Lupdate_mdscr_panic_str@page
 126         add     x0, x0, Lupdate_mdscr_panic_str@pageoff
 127         b       EXT(panic)
 128         b       .
 129
 130 Lupdate_mdscr_panic_str:
 131         .asciz "MDSCR.KDE was set"
 132
 133
 134 #if __ARM_KERNEL_PROTECT__
 135 /*
 136  * __ARM_KERNEL_PROTECT__ adds two complications to TLB management:
 137  *
 138  * 1. As each pmap has two ASIDs, every TLB operation that targets an ASID must
 139  *   target both ASIDs for the pmap that owns the target ASID.
 140  *
 141  * 2. Any TLB operation targeting the kernel_pmap ASID (ASID 0) must target all
 142  *   ASIDs (as kernel_pmap mappings may be referenced while using an ASID that
 143  *   belongs to another pmap).  We expect these routines to be called with the
 144  *   EL0 ASID for the target; not the EL1 ASID.
 145  */
 146 #endif /* __ARM_KERNEL_PROTECT__ */
 147
 148 .macro SYNC_TLB_FLUSH
 149         dsb     ish
 150         isb     sy
 151 .endmacro
 152
 153
 154 /*
 155  *      void sync_tlb_flush(void)
 156  *
 157  *              Synchronize one or more prior TLB flush operations
 158  */
 159         .text
 160         .align 2
 161         .globl EXT(sync_tlb_flush)
 162 LEXT(sync_tlb_flush)
 163         SYNC_TLB_FLUSH
 164         ret
 165
 166
 167 .macro FLUSH_MMU_TLB
 168         tlbi    vmalle1is
 169 .endmacro
 170 /*
 171  *      void flush_mmu_tlb_async(void)
 172  *
 173  *              Flush all TLBs, don't wait for completion
 174  */
 175         .text
 176         .align 2
 177         .globl EXT(flush_mmu_tlb_async)
 178 LEXT(flush_mmu_tlb_async)
 179         FLUSH_MMU_TLB
 180         ret
 181
 182 /*
 183  *      void flush_mmu_tlb(void)
 184  *
 185  *              Flush all TLBs
 186  */
 187         .text
 188         .align 2
 189         .globl EXT(flush_mmu_tlb)
 190 LEXT(flush_mmu_tlb)
 191         FLUSH_MMU_TLB
 192         SYNC_TLB_FLUSH
 193         ret
 194
 195 .macro FLUSH_CORE_TLB
 196         tlbi    vmalle1
 197 .endmacro
 198
 199 /*
 200  *      void flush_core_tlb_async(void)
 201  *
 202  *              Flush local core TLB, don't wait for completion
 203  */
 204         .text
 205         .align 2
 206         .globl EXT(flush_core_tlb_async)
 207 LEXT(flush_core_tlb_async)
 208         FLUSH_CORE_TLB
 209         ret
 210
 211 /*
 212  *      void flush_core_tlb(void)
 213  *
 214  *              Flush local core TLB
 215  */
 216         .text
 217         .align 2
 218         .globl EXT(flush_core_tlb)
 219 LEXT(flush_core_tlb)
 220         FLUSH_CORE_TLB
 221         SYNC_TLB_FLUSH
 222         ret
 223
 224 .macro FLUSH_MMU_TLB_ALLENTRIES
 225 #if __ARM_16K_PG__
 226         and             x0, x0, #~0x3
 227
 228         /*
 229          * The code below is not necessarily correct.  From an overview of
 230          * the client code, the expected contract for TLB flushes is that
 231          * we will expand from an "address, length" pair to "start address,
 232          * end address" in the course of a TLB flush.  This suggests that
 233          * a flush for "X, X+4" is actually only asking for a flush of a
 234          * single 16KB page.  At the same time, we'd like to be prepared
 235          * for bad inputs (X, X+3), so add 3 and then truncate the 4KB page
 236          * number to a 16KB page boundary.  This should deal correctly with
 237          * unaligned inputs.
 238          *
 239          * If our expecations about client behavior are wrong however, this
 240          * will lead to occasional TLB corruption on platforms with 16KB
 241          * pages.
 242          */
 243         add             x1, x1, #0x3
 244         and             x1, x1, #~0x3
 245 #endif
 246 1: // Lflush_mmu_tlb_allentries_loop:
 247         tlbi    vaae1is, x0
 248         add             x0, x0, #(ARM_PGBYTES / 4096)   // Units are 4KB pages, as defined by the ISA
 249         cmp             x0, x1
 250         b.lt    1b // Lflush_mmu_tlb_allentries_loop
 251 .endmacro
 252
 253 /*
 254  *      void flush_mmu_tlb_allentries_async(uint64_t, uint64_t)
 255  *
 256  *              Flush TLB entries, don't wait for completion
 257  */
 258         .text
 259         .align 2
 260         .globl EXT(flush_mmu_tlb_allentries_async)
 261 LEXT(flush_mmu_tlb_allentries_async)
 262         FLUSH_MMU_TLB_ALLENTRIES
 263         ret
 264
 265 /*
 266  *      void flush_mmu_tlb_allentries(uint64_t, uint64_t)
 267  *
 268  *              Flush TLB entries
 269  */
 270         .globl EXT(flush_mmu_tlb_allentries)
 271 LEXT(flush_mmu_tlb_allentries)
 272         FLUSH_MMU_TLB_ALLENTRIES
 273         SYNC_TLB_FLUSH
 274         ret
 275
 276 .macro FLUSH_MMU_TLB_ENTRY
 277 #if __ARM_KERNEL_PROTECT__
 278         /*
 279          * If we are flushing ASID 0, this is a kernel operation.  With this
 280          * ASID scheme, this means we should flush all ASIDs.
 281          */
 282         lsr             x2, x0, #TLBI_ASID_SHIFT
 283         cmp             x2, #0
 284         b.eq            1f // Lflush_mmu_tlb_entry_globally
 285
 286         bic             x0, x0, #(1 << TLBI_ASID_SHIFT)
 287         tlbi    vae1is, x0
 288         orr             x0, x0, #(1 << TLBI_ASID_SHIFT)
 289 #endif /* __ARM_KERNEL_PROTECT__ */
 290         tlbi    vae1is, x0
 291 #if __ARM_KERNEL_PROTECT__
 292         b               2f // Lflush_mmu_tlb_entry_done
 293 1: // Lflush_mmu_tlb_entry_globally:
 294         tlbi    vaae1is, x0
 295 2: // Lflush_mmu_tlb_entry_done
 296 #endif /* __ARM_KERNEL_PROTECT__ */
 297 .endmacro
 298 /*
 299  *      void flush_mmu_tlb_entry_async(uint64_t)
 300  *
 301  *              Flush TLB entry, don't wait for completion
 302  */
 303         .text
 304         .align 2
 305         .globl EXT(flush_mmu_tlb_entry_async)
 306 LEXT(flush_mmu_tlb_entry_async)
 307         FLUSH_MMU_TLB_ENTRY
 308         ret
 309
 310 /*
 311  *      void flush_mmu_tlb_entry(uint64_t)
 312  *
 313  *              Flush TLB entry
 314  */
 315         .text
 316         .align 2
 317         .globl EXT(flush_mmu_tlb_entry)
 318 LEXT(flush_mmu_tlb_entry)
 319         FLUSH_MMU_TLB_ENTRY
 320         SYNC_TLB_FLUSH
 321         ret
 322
 323 .macro FLUSH_MMU_TLB_ENTRIES
 324 #if __ARM_16K_PG__
 325         and             x0, x0, #~0x3
 326
 327         /*
 328          * The code below is not necessarily correct.  From an overview of
 329          * the client code, the expected contract for TLB flushes is that
 330          * we will expand from an "address, length" pair to "start address,
 331          * end address" in the course of a TLB flush.  This suggests that
 332          * a flush for "X, X+4" is actually only asking for a flush of a
 333          * single 16KB page.  At the same time, we'd like to be prepared
 334          * for bad inputs (X, X+3), so add 3 and then truncate the 4KB page
 335          * number to a 16KB page boundary.  This should deal correctly with
 336          * unaligned inputs.
 337          *
 338          * If our expecations about client behavior are wrong however, this
 339          * will lead to occasional TLB corruption on platforms with 16KB
 340          * pages.
 341          */
 342         add             x1, x1, #0x3
 343         and             x1, x1, #~0x3
 344 #endif /* __ARM_16K_PG__ */
 345 #if __ARM_KERNEL_PROTECT__
 346         /*
 347          * If we are flushing ASID 0, this is a kernel operation.  With this
 348          * ASID scheme, this means we should flush all ASIDs.
 349          */
 350         lsr             x2, x0, #TLBI_ASID_SHIFT
 351         cmp             x2, #0
 352         b.eq            2f // Lflush_mmu_tlb_entries_globally_loop
 353
 354         bic             x0, x0, #(1 << TLBI_ASID_SHIFT)
 355 #endif /* __ARM_KERNEL_PROTECT__ */
 356 1: // Lflush_mmu_tlb_entries_loop
 357         tlbi    vae1is, x0
 358 #if __ARM_KERNEL_PROTECT__
 359         orr             x0, x0, #(1 << TLBI_ASID_SHIFT)
 360         tlbi    vae1is, x0
 361         bic             x0, x0, #(1 << TLBI_ASID_SHIFT)
 362 #endif /* __ARM_KERNEL_PROTECT__ */
 363         add             x0, x0, #(ARM_PGBYTES / 4096)   // Units are pages
 364         cmp             x0, x1
 365         b.lt            1b // Lflush_mmu_tlb_entries_loop
 366 #if __ARM_KERNEL_PROTECT__
 367         b               3f // Lflush_mmu_tlb_entries_done
 368 2: // Lflush_mmu_tlb_entries_globally_loop:
 369         tlbi    vaae1is, x0
 370         add             x0, x0, #(ARM_PGBYTES / 4096)   // Units are pages
 371         cmp             x0, x1
 372         b.lt            2b // Lflush_mmu_tlb_entries_globally_loop
 373 3: // Lflush_mmu_tlb_entries_done
 374 #endif /* __ARM_KERNEL_PROTECT__ */
 375 .endmacro
 376
 377 /*
 378  *      void flush_mmu_tlb_entries_async(uint64_t, uint64_t)
 379  *
 380  *              Flush TLB entries, don't wait for completion
 381  */
 382         .text
 383         .align 2
 384         .globl EXT(flush_mmu_tlb_entries_async)
 385 LEXT(flush_mmu_tlb_entries_async)
 386         FLUSH_MMU_TLB_ENTRIES
 387         ret
 388
 389 /*
 390  *      void flush_mmu_tlb_entries(uint64_t, uint64_t)
 391  *
 392  *              Flush TLB entries
 393  */
 394         .text
 395         .align 2
 396         .globl EXT(flush_mmu_tlb_entries)
 397 LEXT(flush_mmu_tlb_entries)
 398         FLUSH_MMU_TLB_ENTRIES
 399         SYNC_TLB_FLUSH
 400         ret
 401
 402 .macro FLUSH_MMU_TLB_ASID
 403 #if __ARM_KERNEL_PROTECT__
 404         /*
 405          * If we are flushing ASID 0, this is a kernel operation.  With this
 406          * ASID scheme, this means we should flush all ASIDs.
 407          */
 408         lsr             x1, x0, #TLBI_ASID_SHIFT
 409         cmp             x1, #0
 410         b.eq            1f // Lflush_mmu_tlb_globally
 411
 412         bic             x0, x0, #(1 << TLBI_ASID_SHIFT)
 413         tlbi    aside1is, x0
 414         orr             x0, x0, #(1 << TLBI_ASID_SHIFT)
 415 #endif /* __ARM_KERNEL_PROTECT__ */
 416         tlbi    aside1is, x0
 417 #if __ARM_KERNEL_PROTECT__
 418         b               2f // Lflush_mmu_tlb_asid_done
 419 1: // Lflush_mmu_tlb_globally:
 420         tlbi    vmalle1is
 421 2: // Lflush_mmu_tlb_asid_done:
 422 #endif /* __ARM_KERNEL_PROTECT__ */
 423 .endmacro
 424
 425 /*
 426  *      void flush_mmu_tlb_asid_async(uint64_t)
 427  *
 428  *              Flush TLB entriesfor requested asid, don't wait for completion
 429  */
 430         .text
 431         .align 2
 432         .globl EXT(flush_mmu_tlb_asid_async)
 433 LEXT(flush_mmu_tlb_asid_async)
 434         FLUSH_MMU_TLB_ASID
 435         ret
 436
 437 /*
 438  *      void flush_mmu_tlb_asid(uint64_t)
 439  *
 440  *              Flush TLB entriesfor requested asid
 441  */
 442         .text
 443         .align 2
 444         .globl EXT(flush_mmu_tlb_asid)
 445 LEXT(flush_mmu_tlb_asid)
 446         FLUSH_MMU_TLB_ASID
 447         SYNC_TLB_FLUSH
 448         ret
 449
 450 .macro FLUSH_CORE_TLB_ASID
 451 #if __ARM_KERNEL_PROTECT__
 452         /*
 453          * If we are flushing ASID 0, this is a kernel operation.  With this
 454          * ASID scheme, this means we should flush all ASIDs.
 455          */
 456         lsr             x1, x0, #TLBI_ASID_SHIFT
 457         cmp             x1, #0
 458         b.eq            1f // Lflush_core_tlb_asid_globally
 459
 460         bic             x0, x0, #(1 << TLBI_ASID_SHIFT)
 461         tlbi    aside1, x0
 462         orr             x0, x0, #(1 << TLBI_ASID_SHIFT)
 463 #endif /* __ARM_KERNEL_PROTECT__ */
 464         tlbi    aside1, x0
 465 #if __ARM_KERNEL_PROTECT__
 466         b               2f // Lflush_core_tlb_asid_done
 467 1: // Lflush_core_tlb_asid_globally:
 468         tlbi    vmalle1
 469 2: // Lflush_core_tlb_asid_done:
 470 #endif /* __ARM_KERNEL_PROTECT__ */
 471 .endmacro
 472
 473 /*
 474  *      void flush_core_tlb_asid_async(uint64_t)
 475  *
 476  *              Flush TLB entries for core for requested asid, don't wait for completion
 477  */
 478         .text
 479         .align 2
 480         .globl EXT(flush_core_tlb_asid_async)
 481 LEXT(flush_core_tlb_asid_async)
 482         FLUSH_CORE_TLB_ASID
 483         ret
 484 /*
 485  *      void flush_core_tlb_asid(uint64_t)
 486  *
 487  *              Flush TLB entries for core for requested asid
 488  */
 489         .text
 490         .align 2
 491         .globl EXT(flush_core_tlb_asid)
 492 LEXT(flush_core_tlb_asid)
 493         FLUSH_CORE_TLB_ASID
 494         SYNC_TLB_FLUSH
 495         ret
 496
 497 /*
 498  *      Set MMU Translation Table Base Alternate
 499  */
 500         .text
 501         .align 2
 502         .globl EXT(set_mmu_ttb_alternate)
 503 LEXT(set_mmu_ttb_alternate)
 504         dsb             sy
 505 #if defined(KERNEL_INTEGRITY_KTRR)
 506         mov             x1, lr
 507         bl              EXT(pinst_set_ttbr1)
 508         mov             lr, x1
 509 #else
 510         msr             TTBR1_EL1, x0
 511 #endif /* defined(KERNEL_INTEGRITY_KTRR) */
 512         isb             sy
 513         ret
 514
 515         .text
 516         .align 2
 517         .globl EXT(set_mmu_ttb)
 518 LEXT(set_mmu_ttb)
 519 #if __ARM_KERNEL_PROTECT__
 520         /* All EL1-mode ASIDs are odd. */
 521         orr             x0, x0, #(1 << TTBR_ASID_SHIFT)
 522 #endif /* __ARM_KERNEL_PROTECT__ */
 523         dsb             ish
 524         msr             TTBR0_EL1, x0
 525         isb             sy
 526         ret
 527
 528 /*
 529  *      set AUX control register
 530  */
 531         .text
 532         .align 2
 533         .globl EXT(set_aux_control)
 534 LEXT(set_aux_control)
 535         msr             ACTLR_EL1, x0
 536         // Synchronize system
 537         dsb             sy
 538         isb             sy
 539         ret
 540
 541 #if __ARM_KERNEL_PROTECT__
 542         .text
 543         .align 2
 544         .globl EXT(set_vbar_el1)
 545 LEXT(set_vbar_el1)
 546 #if defined(KERNEL_INTEGRITY_KTRR)
 547         b               EXT(pinst_set_vbar)
 548 #else
 549         msr             VBAR_EL1, x0
 550         ret
 551 #endif
 552 #endif /* __ARM_KERNEL_PROTECT__ */
 553
 554
 555 /*
 556  *      set translation control register
 557  */
 558         .text
 559         .align 2
 560         .globl EXT(set_tcr)
 561 LEXT(set_tcr)
 562 #if defined(APPLE_ARM64_ARCH_FAMILY)
 563         // Assert that T0Z is always equal to T1Z
 564         eor             x1, x0, x0, lsr #(TCR_T1SZ_SHIFT - TCR_T0SZ_SHIFT)
 565         and             x1, x1, #(TCR_TSZ_MASK << TCR_T0SZ_SHIFT)
 566         cbnz    x1, L_set_tcr_panic
 567 #if defined(KERNEL_INTEGRITY_KTRR)
 568         mov             x1, lr
 569         bl              _pinst_set_tcr
 570         mov             lr, x1
 571 #else
 572         msr             TCR_EL1, x0
 573 #endif /* defined(KERNEL_INTRITY_KTRR) */
 574         isb             sy
 575         ret
 576
 577 L_set_tcr_panic:
 578         PUSH_FRAME
 579         sub             sp, sp, #16
 580         str             x0, [sp]
 581         adr             x0, L_set_tcr_panic_str
 582         BRANCH_EXTERN panic
 583
 584 L_set_locked_reg_panic:
 585         PUSH_FRAME
 586         sub             sp, sp, #16
 587         str             x0, [sp]
 588         adr             x0, L_set_locked_reg_panic_str
 589         BRANCH_EXTERN panic
 590         b .
 591
 592 L_set_tcr_panic_str:
 593         .asciz  "set_tcr: t0sz, t1sz not equal (%llx)\n"
 594
 595
 596 L_set_locked_reg_panic_str:
 597         .asciz  "attempt to set locked register: (%llx)\n"
 598 #else
 599 #if defined(KERNEL_INTEGRITY_KTRR)
 600         mov             x1, lr
 601         bl              _pinst_set_tcr
 602         mov             lr, x1
 603 #else
 604         msr             TCR_EL1, x0
 605 #endif
 606         isb             sy
 607         ret
 608 #endif // defined(APPLE_ARM64_ARCH_FAMILY)
 609
 610 /*
 611  *      MMU kernel virtual to physical address translation
 612  */
 613         .text
 614         .align 2
 615         .globl EXT(mmu_kvtop)
 616 LEXT(mmu_kvtop)
 617         mrs             x2, DAIF                                                                        // Load current DAIF
 618         msr             DAIFSet, #(DAIFSC_IRQF | DAIFSC_FIQF)           // Disable IRQ
 619         at              s1e1r, x0                                                                       // Translation Stage 1 EL1
 620         mrs             x1, PAR_EL1                                                                     // Read result
 621         msr             DAIF, x2                                                                        // Restore interrupt state
 622         tbnz    x1, #0, L_mmu_kvtop_invalid                                     // Test Translation not valid
 623         bfm             x1, x0, #0, #11                                                         // Add page offset
 624         and             x0, x1, #0x0000ffffffffffff                                     // Clear non-address bits
 625         ret
 626 L_mmu_kvtop_invalid:
 627         mov             x0, #0                                                                          // Return invalid
 628         ret
 629
 630 /*
 631  *      MMU user virtual to physical address translation
 632  */
 633         .text
 634         .align 2
 635         .globl EXT(mmu_uvtop)
 636 LEXT(mmu_uvtop)
 637         lsr             x8, x0, #56                                                                     // Extract top byte
 638         cbnz    x8, L_mmu_uvtop_invalid                                         // Tagged pointers are invalid
 639         mrs             x2, DAIF                                                                        // Load current DAIF
 640         msr             DAIFSet, #(DAIFSC_IRQF | DAIFSC_FIQF)           // Disable IRQ
 641         at              s1e0r, x0                                                                       // Translation Stage 1 EL0
 642         mrs             x1, PAR_EL1                                                                     // Read result
 643         msr             DAIF, x2                                                                        // Restore interrupt state
 644         tbnz    x1, #0, L_mmu_uvtop_invalid                                     // Test Translation not valid
 645         bfm             x1, x0, #0, #11                                                         // Add page offset
 646         and             x0, x1, #0x0000ffffffffffff                                     // Clear non-address bits
 647         ret
 648 L_mmu_uvtop_invalid:
 649         mov             x0, #0                                                                          // Return invalid
 650         ret
 651
 652 /*
 653  *      MMU kernel virtual to physical address preflight write access
 654  */
 655         .text
 656         .align 2
 657         .globl EXT(mmu_kvtop_wpreflight)
 658 LEXT(mmu_kvtop_wpreflight)
 659         mrs             x2, DAIF                                                                        // Load current DAIF
 660         msr             DAIFSet, #(DAIFSC_IRQF | DAIFSC_FIQF)           // Disable IRQ
 661         at              s1e1w, x0                                                                       // Translation Stage 1 EL1
 662         mrs             x1, PAR_EL1                                                                     // Read result
 663         msr             DAIF, x2                                                                        // Restore interrupt state
 664         tbnz    x1, #0, L_mmu_kvtop_wpreflight_invalid          // Test Translation not valid
 665         bfm             x1, x0, #0, #11                                                         // Add page offset
 666         and             x0, x1, #0x0000ffffffffffff                                     // Clear non-address bits
 667         ret
 668 L_mmu_kvtop_wpreflight_invalid:
 669         mov             x0, #0                                                                          // Return invalid
 670         ret
 671
 672 /*
 673  * SET_RECOVERY_HANDLER
 674  *
 675  *      Sets up a page fault recovery handler
 676  *
 677  *      arg0 - persisted thread pointer
 678  *      arg1 - persisted recovery handler
 679  *      arg2 - scratch reg
 680  *      arg3 - recovery label
 681  */
 682 .macro SET_RECOVERY_HANDLER
 683         mrs             $0, TPIDR_EL1                                   // Load thread pointer
 684         adrp    $2, $3@page                                             // Load the recovery handler address
 685         add             $2, $2, $3@pageoff
 686
 687         ldr             $1, [$0, TH_RECOVER]                    // Save previous recovery handler
 688         str             $2, [$0, TH_RECOVER]                    // Set new signed recovery handler
 689 .endmacro
 690
 691 /*
 692  * CLEAR_RECOVERY_HANDLER
 693  *
 694  *      Clears page fault handler set by SET_RECOVERY_HANDLER
 695  *
 696  *      arg0 - thread pointer saved by SET_RECOVERY_HANDLER
 697  *      arg1 - old recovery handler saved by SET_RECOVERY_HANDLER
 698  */
 699 .macro CLEAR_RECOVERY_HANDLER
 700         str             $1, [$0, TH_RECOVER]            // Restore the previous recovery handler
 701 .endmacro
 702
 703
 704         .text
 705         .align 2
 706 copyio_error:
 707         CLEAR_RECOVERY_HANDLER x10, x11
 708         mov             x0, #EFAULT                                     // Return an EFAULT error
 709         POP_FRAME
 710         ARM64_STACK_EPILOG
 711
 712 /*
 713  * int _bcopyin(const char *src, char *dst, vm_size_t len)
 714  */
 715         .text
 716         .align 2
 717         .globl EXT(_bcopyin)
 718 LEXT(_bcopyin)
 719         ARM64_STACK_PROLOG
 720         PUSH_FRAME
 721         SET_RECOVERY_HANDLER x10, x11, x3, copyio_error
 722         /* If len is less than 16 bytes, just do a bytewise copy */
 723         cmp             x2, #16
 724         b.lt    2f
 725         sub             x2, x2, #16
 726 1:
 727         /* 16 bytes at a time */
 728         ldp             x3, x4, [x0], #16
 729         stp             x3, x4, [x1], #16
 730         subs    x2, x2, #16
 731         b.ge    1b
 732         /* Fixup the len and test for completion */
 733         adds    x2, x2, #16
 734         b.eq    3f
 735 2:      /* Bytewise */
 736         subs    x2, x2, #1
 737         ldrb    w3, [x0], #1
 738         strb    w3, [x1], #1
 739         b.hi    2b
 740 3:
 741         CLEAR_RECOVERY_HANDLER x10, x11
 742         mov             x0, #0
 743         POP_FRAME
 744         ARM64_STACK_EPILOG
 745
 746 /*
 747  * int _copyin_word(const char *src, uint64_t *dst, vm_size_t len)
 748  */
 749         .text
 750         .align 2
 751         .globl EXT(_copyin_word)
 752 LEXT(_copyin_word)
 753         ARM64_STACK_PROLOG
 754         PUSH_FRAME
 755         SET_RECOVERY_HANDLER x10, x11, x3, copyio_error
 756         cmp             x2, #4
 757         b.eq    L_copyin_word_4
 758         cmp             x2, #8
 759         b.eq    L_copyin_word_8
 760         mov             x0, EINVAL
 761         b               L_copying_exit
 762 L_copyin_word_4:
 763         ldr             w8, [x0]
 764         b               L_copyin_word_store
 765 L_copyin_word_8:
 766         ldr             x8, [x0]
 767 L_copyin_word_store:
 768         str             x8, [x1]
 769         mov             x0, #0
 770         CLEAR_RECOVERY_HANDLER x10, x11
 771 L_copying_exit:
 772         POP_FRAME
 773         ARM64_STACK_EPILOG
 774
 775
 776
 777 /*
 778  * int _bcopyout(const char *src, char *dst, vm_size_t len)
 779  */
 780         .text
 781         .align 2
 782         .globl EXT(_bcopyout)
 783 LEXT(_bcopyout)
 784         ARM64_STACK_PROLOG
 785         PUSH_FRAME
 786         SET_RECOVERY_HANDLER x10, x11, x3, copyio_error
 787         /* If len is less than 16 bytes, just do a bytewise copy */
 788         cmp             x2, #16
 789         b.lt    2f
 790         sub             x2, x2, #16
 791 1:
 792         /* 16 bytes at a time */
 793         ldp             x3, x4, [x0], #16
 794         stp             x3, x4, [x1], #16
 795         subs    x2, x2, #16
 796         b.ge    1b
 797         /* Fixup the len and test for completion */
 798         adds    x2, x2, #16
 799         b.eq    3f
 800 2:  /* Bytewise */
 801         subs    x2, x2, #1
 802         ldrb    w3, [x0], #1
 803         strb    w3, [x1], #1
 804         b.hi    2b
 805 3:
 806         CLEAR_RECOVERY_HANDLER x10, x11
 807         mov             x0, #0
 808         POP_FRAME
 809         ARM64_STACK_EPILOG
 810
 811 /*
 812  * int _bcopyinstr(
 813  *        const user_addr_t user_addr,
 814  *        char *kernel_addr,
 815  *        vm_size_t max,
 816  *        vm_size_t *actual)
 817  */
 818         .text
 819         .align 2
 820         .globl EXT(_bcopyinstr)
 821 LEXT(_bcopyinstr)
 822         ARM64_STACK_PROLOG
 823         PUSH_FRAME
 824         adr             x4, Lcopyinstr_error            // Get address for recover
 825         mrs             x10, TPIDR_EL1                          // Get thread pointer
 826         ldr             x11, [x10, TH_RECOVER]          // Save previous recover
 827
 828         str             x4, [x10, TH_RECOVER]           // Store new recover
 829
 830         mov             x4, #0                                          // x4 - total bytes copied
 831 Lcopyinstr_loop:
 832         ldrb    w5, [x0], #1                                    // Load a byte from the user source
 833         strb    w5, [x1], #1                            // Store a byte to the kernel dest
 834         add             x4, x4, #1                                      // Increment bytes copied
 835         cbz     x5, Lcopyinstr_done                     // If this byte is null, we're done
 836         cmp             x4, x2                                          // If we're out of space, return an error
 837         b.ne    Lcopyinstr_loop
 838 Lcopyinstr_too_long:
 839         mov             x5, #ENAMETOOLONG                       // Set current byte to error code for later return
 840 Lcopyinstr_done:
 841         str             x4, [x3]                                        // Return number of bytes copied
 842         mov             x0, x5                                          // Set error code (0 on success, ENAMETOOLONG on failure)
 843         b               Lcopyinstr_exit
 844 Lcopyinstr_error:
 845         mov             x0, #EFAULT                                     // Return EFAULT on error
 846 Lcopyinstr_exit:
 847         str             x11, [x10, TH_RECOVER]          // Restore old recover
 848         POP_FRAME
 849         ARM64_STACK_EPILOG
 850
 851 /*
 852  * int copyinframe(const vm_address_t frame_addr, char *kernel_addr, bool is64bit)
 853  *
 854  *      Safely copy sixteen bytes (the fixed top of an ARM64 frame) from
 855  *      either user or kernel memory, or 8 bytes (AArch32) from user only.
 856  *
 857  *      x0 : address of frame to copy.
 858  *      x1 : kernel address at which to store data.
 859  *      w2 : whether to copy an AArch32 or AArch64 frame.
 860  *      x3 : temp
 861  *      x5 : temp (kernel virtual base)
 862  *      x9 : temp
 863  *      x10 : thread pointer (set by SET_RECOVERY_HANDLER)
 864  *      x11 : old recovery function (set by SET_RECOVERY_HANDLER)
 865  *      x12, x13 : backtrace data
 866  *
 867  */
 868         .text
 869         .align 2
 870         .globl EXT(copyinframe)
 871 LEXT(copyinframe)
 872         ARM64_STACK_PROLOG
 873         PUSH_FRAME
 874         SET_RECOVERY_HANDLER x10, x11, x3, copyio_error
 875         cbnz    w2, Lcopyinframe64              // Check frame size
 876         adrp    x5, EXT(gVirtBase)@page // For 32-bit frame, make sure we're not trying to copy from kernel
 877         add             x5, x5, EXT(gVirtBase)@pageoff
 878         ldr             x5, [x5]
 879         cmp     x5, x0                                  // See if address is in kernel virtual range
 880         b.hi    Lcopyinframe32                  // If below kernel virtual range, proceed.
 881         mov             w0, #EFAULT                             // Should never have a 32-bit frame in kernel virtual range
 882         b               Lcopyinframe_done
 883
 884 Lcopyinframe32:
 885         ldr             x12, [x0]                               // Copy 8 bytes
 886         str             x12, [x1]
 887         mov     w0, #0                                  // Success
 888         b               Lcopyinframe_done
 889
 890 Lcopyinframe64:
 891         mov             x3, VM_MIN_KERNEL_ADDRESS               // Check if kernel address
 892         orr             x9, x0, TBI_MASK                                // Hide tags in address comparison
 893         cmp             x9, x3                                                  // If in kernel address range, skip tag test
 894         b.hs    Lcopyinframe_valid
 895         tst             x0, TBI_MASK                                    // Detect tagged pointers
 896         b.eq    Lcopyinframe_valid
 897         mov             w0, #EFAULT                                             // Tagged address, fail
 898         b               Lcopyinframe_done
 899 Lcopyinframe_valid:
 900         ldp             x12, x13, [x0]                  // Copy 16 bytes
 901         stp             x12, x13, [x1]
 902         mov     w0, #0                                  // Success
 903
 904 Lcopyinframe_done:
 905         CLEAR_RECOVERY_HANDLER x10, x11
 906         POP_FRAME
 907         ARM64_STACK_EPILOG
 908
 909
 910 /*
 911  * uint32_t arm_debug_read_dscr(void)
 912  */
 913         .text
 914         .align 2
 915         .globl EXT(arm_debug_read_dscr)
 916 LEXT(arm_debug_read_dscr)
 917         PANIC_UNIMPLEMENTED
 918
 919 /*
 920  * void arm_debug_set_cp14(arm_debug_state_t *debug_state)
 921  *
 922  *     Set debug registers to match the current thread state
 923  *      (NULL to disable).  Assume 6 breakpoints and 2
 924  *      watchpoints, since that has been the case in all cores
 925  *      thus far.
 926  */
 927        .text
 928        .align 2
 929        .globl EXT(arm_debug_set_cp14)
 930 LEXT(arm_debug_set_cp14)
 931         PANIC_UNIMPLEMENTED
 932
 933 #if defined(APPLE_ARM64_ARCH_FAMILY)
 934 /*
 935  * Note: still have to ISB before executing wfi!
 936  */
 937         .text
 938         .align 2
 939         .globl EXT(arm64_prepare_for_sleep)
 940 LEXT(arm64_prepare_for_sleep)
 941         PUSH_FRAME
 942
 943 #if defined(APPLECYCLONE) || defined(APPLETYPHOON)
 944         // <rdar://problem/15827409> CPU1 Stuck in WFIWT Because of MMU Prefetch
 945         mrs             x0, ARM64_REG_HID2                              // Read HID2
 946         orr             x0, x0, #(ARM64_REG_HID2_disMMUmtlbPrefetch)    // Set HID.DisableMTLBPrefetch
 947         msr             ARM64_REG_HID2, x0                              // Write HID2
 948         dsb             sy
 949         isb             sy
 950 #endif
 951
 952 #if __ARM_GLOBAL_SLEEP_BIT__
 953         // Enable deep sleep
 954         mrs             x1, ARM64_REG_ACC_OVRD
 955         orr             x1, x1, #(ARM64_REG_ACC_OVRD_enDeepSleep)
 956         and             x1, x1, #(~(ARM64_REG_ACC_OVRD_disL2Flush4AccSlp_mask))
 957         orr             x1, x1, #(  ARM64_REG_ACC_OVRD_disL2Flush4AccSlp_deepsleep)
 958         and             x1, x1, #(~(ARM64_REG_ACC_OVRD_ok2PwrDnSRM_mask))
 959         orr             x1, x1, #(  ARM64_REG_ACC_OVRD_ok2PwrDnSRM_deepsleep)
 960         and             x1, x1, #(~(ARM64_REG_ACC_OVRD_ok2TrDnLnk_mask))
 961         orr             x1, x1, #(  ARM64_REG_ACC_OVRD_ok2TrDnLnk_deepsleep)
 962         and             x1, x1, #(~(ARM64_REG_ACC_OVRD_ok2PwrDnCPM_mask))
 963         orr             x1, x1, #(  ARM64_REG_ACC_OVRD_ok2PwrDnCPM_deepsleep)
 964         msr             ARM64_REG_ACC_OVRD, x1
 965
 966
 967 #else
 968         // Enable deep sleep
 969         mov             x1, ARM64_REG_CYC_CFG_deepSleep
 970         msr             ARM64_REG_CYC_CFG, x1
 971 #endif
 972         // Set "OK to power down" (<rdar://problem/12390433>)
 973         mrs             x0, ARM64_REG_CYC_OVRD
 974         orr             x0, x0, #(ARM64_REG_CYC_OVRD_ok2pwrdn_force_down)
 975         msr             ARM64_REG_CYC_OVRD, x0
 976
 977 #if defined(APPLEMONSOON)
 978         ARM64_IS_PCORE x0
 979         cbz             x0, Lwfi_inst // skip if not p-core
 980
 981         /* <rdar://problem/32512947>: Flush the GUPS prefetcher prior to
 982          * wfi.  A Skye HW bug can cause the GUPS prefetcher on p-cores
 983          * to be left with valid entries that fail to drain if a
 984          * subsequent wfi is issued.  This can prevent the core from
 985          * power-gating.  For the idle case that is recoverable, but
 986          * for the deep-sleep (S2R) case in which cores MUST power-gate,
 987          * it can lead to a hang.  This can be prevented by disabling
 988          * and re-enabling GUPS, which forces the prefetch queue to
 989          * drain.  This should be done as close to wfi as possible, i.e.
 990          * at the very end of arm64_prepare_for_sleep(). */
 991         mrs             x0, ARM64_REG_HID10
 992         orr             x0, x0, #(ARM64_REG_HID10_DisHwpGups)
 993         msr             ARM64_REG_HID10, x0
 994         isb             sy
 995         and             x0, x0, #(~(ARM64_REG_HID10_DisHwpGups))
 996         msr             ARM64_REG_HID10, x0
 997         isb             sy
 998 #endif
 999 Lwfi_inst:
1000         dsb             sy
1001         isb             sy
1002         wfi
1003         b               Lwfi_inst
1004
1005 /*
1006  * Force WFI to use clock gating only
1007  *
1008  */
1009         .text
1010         .align 2
1011         .globl EXT(arm64_force_wfi_clock_gate)
1012 LEXT(arm64_force_wfi_clock_gate)
1013         ARM64_STACK_PROLOG
1014         PUSH_FRAME
1015
1016         mrs             x0, ARM64_REG_CYC_OVRD
1017         orr             x0, x0, #(ARM64_REG_CYC_OVRD_ok2pwrdn_force_up)
1018         msr             ARM64_REG_CYC_OVRD, x0
1019
1020         POP_FRAME
1021         ARM64_STACK_EPILOG
1022
1023
1024
1025 #if defined(APPLECYCLONE) || defined(APPLETYPHOON)
1026
1027         .text
1028         .align 2
1029         .globl EXT(cyclone_typhoon_prepare_for_wfi)
1030
1031 LEXT(cyclone_typhoon_prepare_for_wfi)
1032         PUSH_FRAME
1033
1034         // <rdar://problem/15827409> CPU1 Stuck in WFIWT Because of MMU Prefetch
1035         mrs             x0, ARM64_REG_HID2                              // Read HID2
1036         orr             x0, x0, #(ARM64_REG_HID2_disMMUmtlbPrefetch)    // Set HID.DisableMTLBPrefetch
1037         msr             ARM64_REG_HID2, x0                              // Write HID2
1038         dsb             sy
1039         isb             sy
1040
1041         POP_FRAME
1042         ret
1043
1044
1045         .text
1046         .align 2
1047         .globl EXT(cyclone_typhoon_return_from_wfi)
1048 LEXT(cyclone_typhoon_return_from_wfi)
1049         PUSH_FRAME
1050
1051         // <rdar://problem/15827409> CPU1 Stuck in WFIWT Because of MMU Prefetch
1052         mrs             x0, ARM64_REG_HID2                              // Read HID2
1053         mov             x1, #(ARM64_REG_HID2_disMMUmtlbPrefetch)        //
1054         bic             x0, x0, x1                                      // Clear HID.DisableMTLBPrefetchMTLBPrefetch
1055         msr             ARM64_REG_HID2, x0                              // Write HID2
1056         dsb             sy
1057         isb             sy
1058
1059         POP_FRAME
1060         ret
1061 #endif
1062
1063 #ifdef  APPLETYPHOON
1064
1065 #define HID0_DEFEATURES_1 0x0000a0c000064010ULL
1066 #define HID1_DEFEATURES_1 0x000000004005bf20ULL
1067 #define HID2_DEFEATURES_1 0x0000000000102074ULL
1068 #define HID3_DEFEATURES_1 0x0000000000400003ULL
1069 #define HID4_DEFEATURES_1 0x83ff00e100000268ULL
1070 #define HID7_DEFEATURES_1 0x000000000000000eULL
1071
1072 #define HID0_DEFEATURES_2 0x0000a1c000020010ULL
1073 #define HID1_DEFEATURES_2 0x000000000005d720ULL
1074 #define HID2_DEFEATURES_2 0x0000000000002074ULL
1075 #define HID3_DEFEATURES_2 0x0000000000400001ULL
1076 #define HID4_DEFEATURES_2 0x8390000200000208ULL
1077 #define HID7_DEFEATURES_2 0x0000000000000000ULL
1078
1079 /*
1080         arg0 = target register
1081         arg1 = 64-bit constant
1082 */
1083 .macro LOAD_UINT64
1084         movz    $0, #(($1 >> 48) & 0xffff), lsl #48
1085         movk    $0, #(($1 >> 32) & 0xffff), lsl #32
1086         movk    $0, #(($1 >> 16) & 0xffff), lsl #16
1087         movk    $0, #(($1)       & 0xffff)
1088 .endmacro
1089
1090         .text
1091         .align 2
1092         .globl EXT(cpu_defeatures_set)
1093 LEXT(cpu_defeatures_set)
1094         PUSH_FRAME
1095         cmp             x0, #2
1096         b.eq            cpu_defeatures_set_2
1097         cmp             x0, #1
1098         b.ne            cpu_defeatures_set_ret
1099         LOAD_UINT64     x1, HID0_DEFEATURES_1
1100         mrs             x0, ARM64_REG_HID0
1101         orr             x0, x0, x1
1102         msr             ARM64_REG_HID0, x0
1103         LOAD_UINT64     x1, HID1_DEFEATURES_1
1104         mrs             x0, ARM64_REG_HID1
1105         orr             x0, x0, x1
1106         msr             ARM64_REG_HID1, x0
1107         LOAD_UINT64     x1, HID2_DEFEATURES_1
1108         mrs             x0, ARM64_REG_HID2
1109         orr             x0, x0, x1
1110         msr             ARM64_REG_HID2, x0
1111         LOAD_UINT64     x1, HID3_DEFEATURES_1
1112         mrs             x0, ARM64_REG_HID3
1113         orr             x0, x0, x1
1114         msr             ARM64_REG_HID3, x0
1115         LOAD_UINT64     x1, HID4_DEFEATURES_1
1116         mrs             x0, ARM64_REG_HID4
1117         orr             x0, x0, x1
1118         msr             ARM64_REG_HID4, x0
1119         LOAD_UINT64     x1, HID7_DEFEATURES_1
1120         mrs             x0, ARM64_REG_HID7
1121         orr             x0, x0, x1
1122         msr             ARM64_REG_HID7, x0
1123         dsb             sy
1124         isb             sy
1125         b               cpu_defeatures_set_ret
1126 cpu_defeatures_set_2:
1127         LOAD_UINT64     x1, HID0_DEFEATURES_2
1128         mrs             x0, ARM64_REG_HID0
1129         orr             x0, x0, x1
1130         msr             ARM64_REG_HID0, x0
1131         LOAD_UINT64     x1, HID1_DEFEATURES_2
1132         mrs             x0, ARM64_REG_HID1
1133         orr             x0, x0, x1
1134         msr             ARM64_REG_HID1, x0
1135         LOAD_UINT64     x1, HID2_DEFEATURES_2
1136         mrs             x0, ARM64_REG_HID2
1137         orr             x0, x0, x1
1138         msr             ARM64_REG_HID2, x0
1139         LOAD_UINT64     x1, HID3_DEFEATURES_2
1140         mrs             x0, ARM64_REG_HID3
1141         orr             x0, x0, x1
1142         msr             ARM64_REG_HID3, x0
1143         LOAD_UINT64     x1, HID4_DEFEATURES_2
1144         mrs             x0, ARM64_REG_HID4
1145         orr             x0, x0, x1
1146         msr             ARM64_REG_HID4, x0
1147         LOAD_UINT64     x1, HID7_DEFEATURES_2
1148         mrs             x0, ARM64_REG_HID7
1149         orr             x0, x0, x1
1150         msr             ARM64_REG_HID7, x0
1151         dsb             sy
1152         isb             sy
1153         b               cpu_defeatures_set_ret
1154 cpu_defeatures_set_ret:
1155         POP_FRAME
1156         ret
1157 #endif
1158
1159 #else /* !defined(APPLE_ARM64_ARCH_FAMILY) */
1160         .text
1161         .align 2
1162         .globl EXT(arm64_prepare_for_sleep)
1163 LEXT(arm64_prepare_for_sleep)
1164         PUSH_FRAME
1165 Lwfi_inst:
1166         dsb             sy
1167         isb             sy
1168         wfi
1169         b               Lwfi_inst
1170
1171 /*
1172  * Force WFI to use clock gating only
1173  * Note: for non-Apple device, do nothing.
1174  */
1175         .text
1176         .align 2
1177         .globl EXT(arm64_force_wfi_clock_gate)
1178 LEXT(arm64_force_wfi_clock_gate)
1179         PUSH_FRAME
1180         nop
1181         POP_FRAME
1182
1183 #endif /* defined(APPLE_ARM64_ARCH_FAMILY) */
1184
1185 /*
1186  * void arm64_replace_bootstack(cpu_data_t *cpu_data)
1187  *
1188  * This must be called from a kernel thread context running on the boot CPU,
1189  * after setting up new exception stacks in per-CPU data. That will guarantee
1190  * that the stack(s) we're trying to replace aren't currently in use.  For
1191  * KTRR-protected devices, this must also be called prior to VM prot finalization
1192  * and lockdown, as updating SP1 requires a sensitive instruction.
1193  */
1194         .text
1195         .align 2
1196         .globl EXT(arm64_replace_bootstack)
1197 LEXT(arm64_replace_bootstack)
1198         ARM64_STACK_PROLOG
1199         PUSH_FRAME
1200         // Set the exception stack pointer
1201         ldr             x0, [x0, CPU_EXCEPSTACK_TOP]
1202         mrs             x4, DAIF                                        // Load current DAIF; use x4 as pinst may trash x1-x3
1203         msr             DAIFSet, #(DAIFSC_IRQF | DAIFSC_FIQF | DAIFSC_ASYNCF)           // Disable IRQ/FIQ/serror
1204         // Set SP_EL1 to exception stack
1205 #if defined(KERNEL_INTEGRITY_KTRR)
1206         mov             x1, lr
1207         bl              _pinst_spsel_1
1208         mov             lr, x1
1209 #else
1210         msr             SPSel, #1
1211 #endif
1212         mov             sp, x0
1213         msr             SPSel, #0
1214         msr             DAIF, x4                                        // Restore interrupt state
1215         POP_FRAME
1216         ARM64_STACK_EPILOG
1217
1218 #ifdef MONITOR
1219 /*
1220  * unsigned long monitor_call(uintptr_t callnum, uintptr_t arg1,
1221                                                           uintptr_t arg2, uintptr_t arg3)
1222  *
1223  * Call the EL3 monitor with 4 arguments in registers
1224  * The monitor interface maintains the same ABI as the C function call standard.  Callee-saved
1225  * registers are preserved, temporary registers are not.  Parameters and results are passed in
1226  * the usual manner.
1227  */
1228         .text
1229         .align 2
1230         .globl EXT(monitor_call)
1231 LEXT(monitor_call)
1232         smc     0x11
1233         ret
1234 #endif
1235
1236
1237 /* vim: set sw=4 ts=4: */