osfmk/mach/arm/thread_status.h

   1 /*
   2  * Copyright (c) 2007 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 /*
  29  * FILE_ID: thread_status.h
  30  */
  31
  32
  33 #ifndef _ARM_THREAD_STATUS_H_
  34 #define _ARM_THREAD_STATUS_H_
  35
  36 #include <mach/machine/_structs.h>
  37 #include <mach/message.h>
  38 #include <mach/vm_types.h>
  39 #include <mach/arm/thread_state.h>
  40
  41 /*
  42  *    Support for determining the state of a thread
  43  */
  44
  45
  46 /*
  47  *  Flavors
  48  */
  49
  50 #define ARM_THREAD_STATE         1
  51 #define ARM_UNIFIED_THREAD_STATE ARM_THREAD_STATE
  52 #define ARM_VFP_STATE            2
  53 #define ARM_EXCEPTION_STATE      3
  54 #define ARM_DEBUG_STATE          4 /* pre-armv8 */
  55 #define THREAD_STATE_NONE        5
  56 #define ARM_THREAD_STATE64       6
  57 #define ARM_EXCEPTION_STATE64    7
  58 //      ARM_THREAD_STATE_LAST    8 /* legacy */
  59 #define ARM_THREAD_STATE32       9
  60
  61 /* API */
  62 #define ARM_DEBUG_STATE32        14
  63 #define ARM_DEBUG_STATE64        15
  64 #define ARM_NEON_STATE           16
  65 #define ARM_NEON_STATE64         17
  66 #define ARM_CPMU_STATE64         18
  67
  68 #ifdef XNU_KERNEL_PRIVATE
  69 /* For kernel use */
  70 #define ARM_SAVED_STATE32        20
  71 #define ARM_SAVED_STATE64        21
  72 #define ARM_NEON_SAVED_STATE32   22
  73 #define ARM_NEON_SAVED_STATE64   23
  74 #endif /* XNU_KERNEL_PRIVATE */
  75
  76
  77 #define ARM_STATE_FLAVOR_IS_OTHER_VALID(_flavor_) 0
  78
  79 #define ARM_PAGEIN_STATE         27
  80
  81 #define VALID_THREAD_STATE_FLAVOR(x) \
  82         ((x == ARM_THREAD_STATE) ||           \
  83          (x == ARM_VFP_STATE) ||              \
  84          (x == ARM_EXCEPTION_STATE) ||        \
  85          (x == ARM_DEBUG_STATE) ||            \
  86          (x == THREAD_STATE_NONE) ||          \
  87          (x == ARM_THREAD_STATE32) ||         \
  88          (x == ARM_THREAD_STATE64) ||         \
  89          (x == ARM_EXCEPTION_STATE64) ||      \
  90          (x == ARM_NEON_STATE) ||             \
  91          (x == ARM_NEON_STATE64) ||           \
  92          (x == ARM_DEBUG_STATE32) ||          \
  93          (x == ARM_DEBUG_STATE64) ||          \
  94          (x == ARM_PAGEIN_STATE) ||           \
  95          (ARM_STATE_FLAVOR_IS_OTHER_VALID(x)))
  96
  97 struct arm_state_hdr {
  98         uint32_t flavor;
  99         uint32_t count;
 100 };
 101 typedef struct arm_state_hdr arm_state_hdr_t;
 102
 103 typedef _STRUCT_ARM_THREAD_STATE   arm_thread_state_t;
 104 typedef _STRUCT_ARM_THREAD_STATE   arm_thread_state32_t;
 105 typedef _STRUCT_ARM_THREAD_STATE64 arm_thread_state64_t;
 106
 107 #if !defined(KERNEL)
 108 #if __DARWIN_C_LEVEL >= __DARWIN_C_FULL && defined(__arm64__)
 109
 110 /* Accessor macros for arm_thread_state64_t pointer fields */
 111
 112 /* Return pc field of arm_thread_state64_t as a data pointer value */
 113 #define arm_thread_state64_get_pc(ts) \
 114                 __darwin_arm_thread_state64_get_pc(ts)
 115 /* Return pc field of arm_thread_state64_t as a function pointer. May return
 116  * NULL if a valid function pointer cannot be constructed, the caller should
 117  * fall back to the arm_thread_state64_get_pc() macro in that case. */
 118 #define arm_thread_state64_get_pc_fptr(ts) \
 119                 __darwin_arm_thread_state64_get_pc_fptr(ts)
 120 /* Set pc field of arm_thread_state64_t to a function pointer */
 121 #define arm_thread_state64_set_pc_fptr(ts, fptr) \
 122                 __darwin_arm_thread_state64_set_pc_fptr(ts, fptr)
 123 /* Return lr field of arm_thread_state64_t as a data pointer value */
 124 #define arm_thread_state64_get_lr(ts) \
 125                 __darwin_arm_thread_state64_get_lr(ts)
 126 /* Return lr field of arm_thread_state64_t as a function pointer. May return
 127  * NULL if a valid function pointer cannot be constructed, the caller should
 128  * fall back to the arm_thread_state64_get_lr() macro in that case. */
 129 #define arm_thread_state64_get_lr_fptr(ts) \
 130                 __darwin_arm_thread_state64_get_lr_fptr(ts)
 131 /* Set lr field of arm_thread_state64_t to a function pointer */
 132 #define arm_thread_state64_set_lr_fptr(ts, fptr) \
 133                 __darwin_arm_thread_state64_set_lr_fptr(ts, fptr)
 134 /* Return sp field of arm_thread_state64_t as a data pointer value */
 135 #define arm_thread_state64_get_sp(ts) \
 136                 __darwin_arm_thread_state64_get_sp(ts)
 137 /* Set sp field of arm_thread_state64_t to a data pointer value */
 138 #define arm_thread_state64_set_sp(ts, ptr) \
 139                 __darwin_arm_thread_state64_set_sp(ts, ptr)
 140 /* Return fp field of arm_thread_state64_t as a data pointer value */
 141 #define arm_thread_state64_get_fp(ts) \
 142                 __darwin_arm_thread_state64_get_fp(ts)
 143 /* Set fp field of arm_thread_state64_t to a data pointer value */
 144 #define arm_thread_state64_set_fp(ts, ptr) \
 145                 __darwin_arm_thread_state64_set_fp(ts, ptr)
 146
 147 #endif /* __DARWIN_C_LEVEL >= __DARWIN_C_FULL && defined(__arm64__) */
 148 #endif /* !defined(KERNEL) */
 149
 150 struct arm_unified_thread_state {
 151         arm_state_hdr_t ash;
 152         union {
 153                 arm_thread_state32_t ts_32;
 154                 arm_thread_state64_t ts_64;
 155         } uts;
 156 };
 157 #define ts_32 uts.ts_32
 158 #define ts_64 uts.ts_64
 159 typedef struct arm_unified_thread_state arm_unified_thread_state_t;
 160
 161 #define ARM_THREAD_STATE_COUNT ((mach_msg_type_number_t) \
 162         (sizeof (arm_thread_state_t)/sizeof(uint32_t)))
 163 #define ARM_THREAD_STATE32_COUNT ((mach_msg_type_number_t) \
 164         (sizeof (arm_thread_state32_t)/sizeof(uint32_t)))
 165 #define ARM_THREAD_STATE64_COUNT ((mach_msg_type_number_t) \
 166         (sizeof (arm_thread_state64_t)/sizeof(uint32_t)))
 167 #define ARM_UNIFIED_THREAD_STATE_COUNT ((mach_msg_type_number_t) \
 168         (sizeof (arm_unified_thread_state_t)/sizeof(uint32_t)))
 169
 170
 171 typedef _STRUCT_ARM_VFP_STATE         arm_vfp_state_t;
 172 typedef _STRUCT_ARM_NEON_STATE        arm_neon_state_t;
 173 typedef _STRUCT_ARM_NEON_STATE        arm_neon_state32_t;
 174 typedef _STRUCT_ARM_NEON_STATE64      arm_neon_state64_t;
 175
 176
 177 typedef _STRUCT_ARM_EXCEPTION_STATE   arm_exception_state_t;
 178 typedef _STRUCT_ARM_EXCEPTION_STATE   arm_exception_state32_t;
 179 typedef _STRUCT_ARM_EXCEPTION_STATE64 arm_exception_state64_t;
 180
 181 typedef _STRUCT_ARM_DEBUG_STATE32     arm_debug_state32_t;
 182 typedef _STRUCT_ARM_DEBUG_STATE64     arm_debug_state64_t;
 183
 184 typedef _STRUCT_ARM_PAGEIN_STATE      arm_pagein_state_t;
 185
 186 #if defined(XNU_KERNEL_PRIVATE) && defined(__arm64__)
 187 /* See below for ARM64 kernel structure definition for arm_debug_state. */
 188 #else /* defined(XNU_KERNEL_PRIVATE) && defined(__arm64__) */
 189 /*
 190  * Otherwise not ARM64 kernel and we must preserve legacy ARM definitions of
 191  * arm_debug_state for binary compatability of userland consumers of this file.
 192  */
 193 #if defined(__arm__)
 194 typedef _STRUCT_ARM_DEBUG_STATE        arm_debug_state_t;
 195 #elif defined(__arm64__)
 196 typedef _STRUCT_ARM_LEGACY_DEBUG_STATE arm_debug_state_t;
 197 #else /* defined(__arm__) */
 198 #error Undefined architecture
 199 #endif /* defined(__arm__) */
 200 #endif /* defined(XNU_KERNEL_PRIVATE) && defined(__arm64__) */
 201
 202 #define ARM_VFP_STATE_COUNT ((mach_msg_type_number_t) \
 203         (sizeof (arm_vfp_state_t)/sizeof(uint32_t)))
 204
 205 #define ARM_EXCEPTION_STATE_COUNT ((mach_msg_type_number_t) \
 206         (sizeof (arm_exception_state_t)/sizeof(uint32_t)))
 207
 208 #define ARM_EXCEPTION_STATE64_COUNT ((mach_msg_type_number_t) \
 209         (sizeof (arm_exception_state64_t)/sizeof(uint32_t)))
 210
 211 #define ARM_DEBUG_STATE_COUNT ((mach_msg_type_number_t) \
 212         (sizeof (arm_debug_state_t)/sizeof(uint32_t)))
 213
 214 #define ARM_DEBUG_STATE32_COUNT ((mach_msg_type_number_t) \
 215         (sizeof (arm_debug_state32_t)/sizeof(uint32_t)))
 216
 217 #define ARM_PAGEIN_STATE_COUNT ((mach_msg_type_number_t) \
 218         (sizeof (arm_pagein_state_t)/sizeof(uint32_t)))
 219
 220 #define ARM_DEBUG_STATE64_COUNT ((mach_msg_type_number_t) \
 221         (sizeof (arm_debug_state64_t)/sizeof(uint32_t)))
 222
 223 #define ARM_NEON_STATE_COUNT ((mach_msg_type_number_t) \
 224         (sizeof (arm_neon_state_t)/sizeof(uint32_t)))
 225
 226 #define ARM_NEON_STATE64_COUNT ((mach_msg_type_number_t) \
 227         (sizeof (arm_neon_state64_t)/sizeof(uint32_t)))
 228
 229 #define MACHINE_THREAD_STATE       ARM_THREAD_STATE
 230 #define MACHINE_THREAD_STATE_COUNT ARM_UNIFIED_THREAD_STATE_COUNT
 231
 232
 233 /*
 234  * Largest state on this machine:
 235  */
 236 #define THREAD_MACHINE_STATE_MAX THREAD_STATE_MAX
 237
 238 #ifdef XNU_KERNEL_PRIVATE
 239
 240 static inline boolean_t
 241 is_thread_state32(const arm_unified_thread_state_t *its)
 242 {
 243         return its->ash.flavor == ARM_THREAD_STATE32;
 244 }
 245
 246 static inline boolean_t
 247 is_thread_state64(const arm_unified_thread_state_t *its)
 248 {
 249         return its->ash.flavor == ARM_THREAD_STATE64;
 250 }
 251
 252 static inline arm_thread_state32_t*
 253 thread_state32(arm_unified_thread_state_t *its)
 254 {
 255         return &its->ts_32;
 256 }
 257
 258 static inline arm_thread_state64_t*
 259 thread_state64(arm_unified_thread_state_t *its)
 260 {
 261         return &its->ts_64;
 262 }
 263
 264 static inline const arm_thread_state32_t*
 265 const_thread_state32(const arm_unified_thread_state_t *its)
 266 {
 267         return &its->ts_32;
 268 }
 269
 270 static inline const arm_thread_state64_t*
 271 const_thread_state64(const arm_unified_thread_state_t *its)
 272 {
 273         return &its->ts_64;
 274 }
 275
 276 #if defined(__arm__)
 277 #include <arm/proc_reg.h>
 278
 279 #define ARM_SAVED_STATE (THREAD_STATE_NONE + 1)
 280
 281 #if __ARM_VFP__
 282 #define VFPSAVE_ALIGN  16
 283 #define VFPSAVE_ATTRIB __attribute__((aligned (VFPSAVE_ALIGN)))
 284 #define THREAD_ALIGN   VFPSAVE_ALIGN
 285
 286 /*
 287  * vector floating point saved state
 288  */
 289 struct arm_vfpsaved_state {
 290         uint32_t r[64];
 291         uint32_t fpscr;
 292         uint32_t fpexc;
 293 };
 294 #endif
 295
 296 struct arm_saved_state {
 297         uint32_t r[13];     /* General purpose register r0-r12 */
 298         uint32_t sp;        /* Stack pointer r13 */
 299         uint32_t lr;        /* Link register r14 */
 300         uint32_t pc;        /* Program counter r15 */
 301         uint32_t cpsr;      /* Current program status register */
 302         uint32_t fsr;       /* Fault status */
 303         uint32_t far;       /* Virtual Fault Address */
 304         uint32_t exception; /* exception number */
 305
 306 #if __ARM_VFP__
 307         /* VFP state */
 308         struct arm_vfpsaved_state VFPdata VFPSAVE_ATTRIB;
 309         // for packing reasons chtread_self and DebugData
 310         // are inside the the PcbData when __ARM_VFP__ is set
 311         arm_debug_state_t        *VFPpadding_DebugData;
 312         vm_address_t              VFPpadding_cthread_self;
 313 #endif
 314 };
 315 typedef struct arm_saved_state arm_saved_state_t;
 316
 317 /*
 318  * Just for coexistence with AArch64 code.
 319  */
 320 typedef struct arm_saved_state arm_saved_state32_t;
 321
 322 static inline void
 323 copy_signed_thread_state(arm_saved_state_t *dst, const arm_saved_state_t *src)
 324 {
 325         *dst = *src;
 326 }
 327
 328 static inline arm_saved_state32_t*
 329 saved_state32(arm_saved_state_t *iss)
 330 {
 331         return iss;
 332 }
 333
 334 static inline boolean_t
 335 is_saved_state32(const arm_saved_state_t *iss __unused)
 336 {
 337         return TRUE;
 338 }
 339
 340
 341 struct arm_saved_state_tagged {
 342         uint32_t               tag;
 343         struct arm_saved_state state;
 344 };
 345 typedef struct arm_saved_state_tagged arm_saved_state_tagged_t;
 346
 347 #define ARM_SAVED_STATE32_COUNT ((mach_msg_type_number_t) \
 348         (sizeof (arm_saved_state_t)/sizeof(unsigned int)))
 349
 350
 351 static inline register_t
 352 get_saved_state_pc(const arm_saved_state_t *iss)
 353 {
 354         return iss->pc;
 355 }
 356
 357 static inline void
 358 add_saved_state_pc(arm_saved_state_t *iss, int diff)
 359 {
 360         iss->pc += diff;
 361 }
 362
 363 static inline void
 364 set_saved_state_pc(arm_saved_state_t *iss, register_t pc)
 365 {
 366         iss->pc = pc;
 367 }
 368
 369 static inline register_t
 370 get_saved_state_sp(const arm_saved_state_t *iss)
 371 {
 372         return iss->sp;
 373 }
 374
 375 static inline void
 376 set_saved_state_sp(arm_saved_state_t *iss, register_t sp)
 377 {
 378         iss->sp = sp;
 379 }
 380
 381 static inline register_t
 382 get_saved_state_fp(const arm_saved_state_t *iss)
 383 {
 384         return iss->r[7];
 385 }
 386
 387 static inline void
 388 set_saved_state_fp(arm_saved_state_t *iss, register_t fp)
 389 {
 390         iss->r[7] = fp;
 391 }
 392
 393 static inline register_t
 394 get_saved_state_lr(const arm_saved_state_t *iss)
 395 {
 396         return iss->lr;
 397 }
 398
 399 static inline void
 400 set_saved_state_lr(arm_saved_state_t *iss, register_t lr)
 401 {
 402         iss->lr = lr;
 403 }
 404
 405 static inline register_t
 406 get_saved_state_cpsr(const arm_saved_state_t *iss)
 407 {
 408         return iss->cpsr;
 409 }
 410
 411 static inline void
 412 mask_saved_state_cpsr(arm_saved_state_t *iss, uint32_t set_bits, uint32_t clear_bits)
 413 {
 414         iss->cpsr |= set_bits;
 415         iss->cpsr &= ~clear_bits;
 416 }
 417
 418 static inline void
 419 set_saved_state_cpsr(arm_saved_state_t *iss, register_t cpsr)
 420 {
 421         iss->cpsr = cpsr;
 422 }
 423
 424 static inline register_t
 425 get_saved_state_reg(const arm_saved_state_t *iss, unsigned regno)
 426 {
 427         return iss->r[regno];
 428 }
 429
 430 static inline void
 431 set_saved_state_reg(arm_saved_state_t *iss, unsigned regno, register_t val)
 432 {
 433         iss->r[regno] = val;
 434 }
 435
 436 #elif defined(__arm64__)
 437
 438 #include <kern/assert.h>
 439 #include <arm64/proc_reg.h>
 440 #define CAST_ASSERT_SAFE(type, val) (assert((val) == ((type)(val))), (type)(val))
 441
 442 /*
 443  * GPR context
 444  */
 445
 446 struct arm_saved_state32 {
 447         uint32_t r[13];     /* General purpose register r0-r12 */
 448         uint32_t sp;        /* Stack pointer r13 */
 449         uint32_t lr;        /* Link register r14 */
 450         uint32_t pc;        /* Program counter r15 */
 451         uint32_t cpsr;      /* Current program status register */
 452         uint32_t far;       /* Virtual fault address */
 453         uint32_t esr;       /* Exception syndrome register */
 454         uint32_t exception; /* Exception number */
 455 };
 456 typedef struct arm_saved_state32 arm_saved_state32_t;
 457
 458 struct arm_saved_state32_tagged {
 459         uint32_t                 tag;
 460         struct arm_saved_state32 state;
 461 };
 462 typedef struct arm_saved_state32_tagged arm_saved_state32_tagged_t;
 463
 464 #define ARM_SAVED_STATE32_COUNT ((mach_msg_type_number_t) \
 465                 (sizeof(arm_saved_state32_t)/sizeof(unsigned int)))
 466
 467 struct arm_saved_state64 {
 468         uint64_t x[29];     /* General purpose registers x0-x28 */
 469         uint64_t fp;        /* Frame pointer x29 */
 470         uint64_t lr;        /* Link register x30 */
 471         uint64_t sp;        /* Stack pointer x31 */
 472         uint64_t pc;        /* Program counter */
 473         uint32_t cpsr;      /* Current program status register */
 474         uint32_t reserved;  /* Reserved padding */
 475         uint64_t far;       /* Virtual fault address */
 476         uint32_t esr;       /* Exception syndrome register */
 477         uint32_t exception; /* Exception number */
 478 #if defined(HAS_APPLE_PAC)
 479         uint64_t jophash;
 480 #endif /* defined(HAS_APPLE_PAC) */
 481 };
 482 typedef struct arm_saved_state64 arm_saved_state64_t;
 483
 484 #define ARM_SAVED_STATE64_COUNT ((mach_msg_type_number_t) \
 485         (sizeof(arm_saved_state64_t)/sizeof(unsigned int)))
 486
 487 struct arm_saved_state64_tagged {
 488         uint32_t                 tag;
 489         struct arm_saved_state64 state;
 490 };
 491 typedef struct arm_saved_state64_tagged arm_saved_state64_tagged_t;
 492
 493 struct arm_saved_state {
 494         arm_state_hdr_t ash;
 495         union {
 496                 struct arm_saved_state32 ss_32;
 497                 struct arm_saved_state64 ss_64;
 498         } uss;
 499 } __attribute__((aligned(16)));
 500 #define ss_32 uss.ss_32
 501 #define ss_64 uss.ss_64
 502
 503 typedef struct arm_saved_state arm_saved_state_t;
 504
 505 #if defined(XNU_KERNEL_PRIVATE)
 506 #if defined(HAS_APPLE_PAC)
 507
 508 #include <sys/cdefs.h>
 509
 510 /*
 511  * Used by MANIPULATE_SIGNED_THREAD_STATE(), potentially from C++ (IOKit) code.
 512  * Open-coded to prevent a circular dependency between mach/arm/thread_status.h
 513  * and osfmk/arm/machine_routines.h.
 514  */
 515 __BEGIN_DECLS
 516 extern boolean_t ml_set_interrupts_enabled(boolean_t);
 517 __END_DECLS
 518
 519 /*
 520  * Methods used to sign and check thread state to detect corruptions of saved
 521  * thread state across exceptions and context switches.
 522  */
 523 extern void ml_sign_thread_state(arm_saved_state_t *, uint64_t, uint32_t, uint64_t, uint64_t, uint64_t);
 524
 525 extern void ml_check_signed_state(const arm_saved_state_t *, uint64_t, uint32_t, uint64_t, uint64_t, uint64_t);
 526
 527 /* XXX: including stddef.f here breaks ctfmerge on some builds, so use __builtin_offsetof() instead of offsetof() */
 528 #define ss64_offsetof(x) __builtin_offsetof(struct arm_saved_state, ss_64.x)
 529
 530 /**
 531  * Verify the signed thread state in _iss, execute the assembly instructions
 532  * _instr, and re-sign the modified thread state.  Varargs specify additional
 533  * inputs.
 534  *
 535  * _instr may read or modify the thread state in the following registers:
 536  *
 537  * x0: _iss
 538  * x1: authed _iss->ss_64.pc
 539  * w2: authed _iss->ss_64.cpsr
 540  * x3: authed _iss->ss_64.lr
 541  * x4: authed _iss->ss_64.x16
 542  * x5: authed _iss->ss_64.x17
 543  * x6: scratch register
 544  * x7: scratch register
 545  */
 546 #define MANIPULATE_SIGNED_THREAD_STATE(_iss, _instr, ...)                       \
 547         do {                                                                    \
 548                 boolean_t _intr = ml_set_interrupts_enabled(FALSE);             \
 549                 asm volatile (                                                  \
 550                         "mov    x8, lr"                         "\n"            \
 551                         "mov    x0, %[iss]"                     "\n"            \
 552                         "ldp    x4, x5, [x0, %[SS64_X16]]"      "\n"            \
 553                         "ldr    x6, [x0, %[SS64_PC]]"           "\n"            \
 554                         "ldr    w7, [x0, %[SS64_CPSR]]"         "\n"            \
 555                         "ldr    x3, [x0, %[SS64_LR]]"           "\n"            \
 556                         "mov    x1, x6"                         "\n"            \
 557                         "mov    w2, w7"                         "\n"            \
 558                         "bl     _ml_check_signed_state"         "\n"            \
 559                         "mov    x1, x6"                         "\n"            \
 560                         "mov    w2, w7"                         "\n"            \
 561                         _instr                                  "\n"            \
 562                         "bl     _ml_sign_thread_state"          "\n"            \
 563                         "mov    lr, x8"                         "\n"            \
 564                         :                                                       \
 565                         : [iss]         "r"(_iss),                              \
 566                           [SS64_X16]    "i"(ss64_offsetof(x[16])),              \
 567                           [SS64_PC]     "i"(ss64_offsetof(pc)),                 \
 568                           [SS64_CPSR]   "i"(ss64_offsetof(cpsr)),               \
 569                           [SS64_LR]     "i"(ss64_offsetof(lr)),##__VA_ARGS__    \
 570                         : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8"  \
 571                 );                                                              \
 572                 ml_set_interrupts_enabled(_intr);                               \
 573         } while (0)
 574
 575 static inline void
 576 check_and_sign_copied_thread_state(arm_saved_state_t *dst, const arm_saved_state_t *src)
 577 {
 578         MANIPULATE_SIGNED_THREAD_STATE(src,
 579             "mov        x0, %[dst]",
 580             [dst] "r"(dst)
 581             );
 582 }
 583 #endif /* defined(HAS_APPLE_PAC) */
 584
 585 static inline void
 586 copy_signed_thread_state(arm_saved_state_t *dst, const arm_saved_state_t *src)
 587 {
 588         *dst = *src;
 589 #if defined(HAS_APPLE_PAC)
 590         check_and_sign_copied_thread_state(dst, src);
 591 #endif
 592 }
 593
 594 #endif /* defined(XNU_KERNEL_PRIVATE) */
 595
 596 static inline boolean_t
 597 is_saved_state32(const arm_saved_state_t *iss)
 598 {
 599         return iss->ash.flavor == ARM_SAVED_STATE32;
 600 }
 601
 602 static inline boolean_t
 603 is_saved_state64(const arm_saved_state_t *iss)
 604 {
 605         return iss->ash.flavor == ARM_SAVED_STATE64;
 606 }
 607
 608 static inline arm_saved_state32_t*
 609 saved_state32(arm_saved_state_t *iss)
 610 {
 611         return &iss->ss_32;
 612 }
 613
 614 static inline const arm_saved_state32_t*
 615 const_saved_state32(const arm_saved_state_t *iss)
 616 {
 617         return &iss->ss_32;
 618 }
 619
 620 static inline arm_saved_state64_t*
 621 saved_state64(arm_saved_state_t *iss)
 622 {
 623         return &iss->ss_64;
 624 }
 625
 626 static inline const arm_saved_state64_t*
 627 const_saved_state64(const arm_saved_state_t *iss)
 628 {
 629         return &iss->ss_64;
 630 }
 631
 632 static inline register_t
 633 get_saved_state_pc(const arm_saved_state_t *iss)
 634 {
 635         return is_saved_state32(iss) ? const_saved_state32(iss)->pc : const_saved_state64(iss)->pc;
 636 }
 637
 638 static inline void
 639 add_saved_state_pc(arm_saved_state_t *iss, int diff)
 640 {
 641         if (is_saved_state32(iss)) {
 642                 uint64_t pc = saved_state32(iss)->pc + diff;
 643                 saved_state32(iss)->pc = CAST_ASSERT_SAFE(uint32_t, pc);
 644         } else {
 645 #if defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC)
 646                 MANIPULATE_SIGNED_THREAD_STATE(iss,
 647                     "mov        w6, %w[diff]            \n"
 648                     "add        x1, x1, w6, sxtw        \n"
 649                     "str        x1, [x0, %[SS64_PC]]    \n",
 650                     [diff] "r"(diff)
 651                     );
 652 #else
 653                 saved_state64(iss)->pc += diff;
 654 #endif /* defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC) */
 655         }
 656 }
 657
 658 static inline void
 659 set_saved_state_pc(arm_saved_state_t *iss, register_t pc)
 660 {
 661         if (is_saved_state32(iss)) {
 662                 saved_state32(iss)->pc = CAST_ASSERT_SAFE(uint32_t, pc);
 663         } else {
 664 #if defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC)
 665                 MANIPULATE_SIGNED_THREAD_STATE(iss,
 666                     "mov        x1, %[pc]               \n"
 667                     "str        x1, [x0, %[SS64_PC]]    \n",
 668                     [pc] "r"(pc)
 669                     );
 670 #else
 671                 saved_state64(iss)->pc = pc;
 672 #endif /* defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC) */
 673         }
 674 }
 675
 676 static inline register_t
 677 get_saved_state_sp(const arm_saved_state_t *iss)
 678 {
 679         return is_saved_state32(iss) ? const_saved_state32(iss)->sp : const_saved_state64(iss)->sp;
 680 }
 681
 682 static inline void
 683 set_saved_state_sp(arm_saved_state_t *iss, register_t sp)
 684 {
 685         if (is_saved_state32(iss)) {
 686                 saved_state32(iss)->sp = CAST_ASSERT_SAFE(uint32_t, sp);
 687         } else {
 688                 saved_state64(iss)->sp = sp;
 689         }
 690 }
 691
 692 static inline register_t
 693 get_saved_state_lr(const arm_saved_state_t *iss)
 694 {
 695         return is_saved_state32(iss) ? const_saved_state32(iss)->lr : const_saved_state64(iss)->lr;
 696 }
 697
 698 static inline void
 699 set_saved_state_lr(arm_saved_state_t *iss, register_t lr)
 700 {
 701         if (is_saved_state32(iss)) {
 702                 saved_state32(iss)->lr = CAST_ASSERT_SAFE(uint32_t, lr);
 703         } else {
 704 #if defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC)
 705                 MANIPULATE_SIGNED_THREAD_STATE(iss,
 706                     "mov        x3, %[lr]               \n"
 707                     "str        x3, [x0, %[SS64_LR]]    \n",
 708                     [lr] "r"(lr)
 709                     );
 710 #else
 711                 saved_state64(iss)->lr = lr;
 712 #endif /* defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC) */
 713         }
 714 }
 715
 716 static inline register_t
 717 get_saved_state_fp(const arm_saved_state_t *iss)
 718 {
 719         return is_saved_state32(iss) ? const_saved_state32(iss)->r[7] : const_saved_state64(iss)->fp;
 720 }
 721
 722 static inline void
 723 set_saved_state_fp(arm_saved_state_t *iss, register_t fp)
 724 {
 725         if (is_saved_state32(iss)) {
 726                 saved_state32(iss)->r[7] = CAST_ASSERT_SAFE(uint32_t, fp);
 727         } else {
 728                 saved_state64(iss)->fp = fp;
 729         }
 730 }
 731
 732 static inline int
 733 check_saved_state_reglimit(const arm_saved_state_t *iss, unsigned reg)
 734 {
 735         return is_saved_state32(iss) ? (reg < ARM_SAVED_STATE32_COUNT) : (reg < ARM_SAVED_STATE64_COUNT);
 736 }
 737
 738 static inline register_t
 739 get_saved_state_reg(const arm_saved_state_t *iss, unsigned reg)
 740 {
 741         if (!check_saved_state_reglimit(iss, reg)) {
 742                 return 0;
 743         }
 744
 745         return is_saved_state32(iss) ? (const_saved_state32(iss)->r[reg]) : (const_saved_state64(iss)->x[reg]);
 746 }
 747
 748 static inline void
 749 set_saved_state_reg(arm_saved_state_t *iss, unsigned reg, register_t value)
 750 {
 751         if (!check_saved_state_reglimit(iss, reg)) {
 752                 return;
 753         }
 754
 755         if (is_saved_state32(iss)) {
 756                 saved_state32(iss)->r[reg] = CAST_ASSERT_SAFE(uint32_t, value);
 757         } else {
 758 #if defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC)
 759                 /* x16 and x17 are part of the jophash */
 760                 if (reg == 16) {
 761                         MANIPULATE_SIGNED_THREAD_STATE(iss,
 762                             "mov        x4, %[value]            \n"
 763                             "str        x4, [x0, %[SS64_X16]]   \n",
 764                             [value] "r"(value)
 765                             );
 766                         return;
 767                 } else if (reg == 17) {
 768                         MANIPULATE_SIGNED_THREAD_STATE(iss,
 769                             "mov        x5, %[value]            \n"
 770                             "str        x5, [x0, %[SS64_X17]]   \n",
 771                             [value] "r"(value),
 772                             [SS64_X17] "i"(ss64_offsetof(x[17]))
 773                             );
 774                         return;
 775                 }
 776 #endif
 777                 saved_state64(iss)->x[reg] = value;
 778         }
 779 }
 780
 781 static inline uint32_t
 782 get_saved_state_cpsr(const arm_saved_state_t *iss)
 783 {
 784         return is_saved_state32(iss) ? const_saved_state32(iss)->cpsr : const_saved_state64(iss)->cpsr;
 785 }
 786
 787 static inline void
 788 mask_saved_state_cpsr(arm_saved_state_t *iss, uint32_t set_bits, uint32_t clear_bits)
 789 {
 790         if (is_saved_state32(iss)) {
 791                 saved_state32(iss)->cpsr |= set_bits;
 792                 saved_state32(iss)->cpsr &= ~clear_bits;
 793         } else {
 794 #if defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC)
 795                 MANIPULATE_SIGNED_THREAD_STATE(iss,
 796                     "mov        w6, %w[set_bits]        \n"
 797                     "orr        w2, w2, w6, lsl #0      \n"
 798                     "mov        w6, %w[clear_bits]      \n"
 799                     "bic        w2, w2, w6, lsl #0      \n"
 800                     "str        w2, [x0, %[SS64_CPSR]]  \n",
 801                     [set_bits] "r"(set_bits),
 802                     [clear_bits] "r"(clear_bits)
 803                     );
 804 #else
 805                 saved_state64(iss)->cpsr |= set_bits;
 806                 saved_state64(iss)->cpsr &= ~clear_bits;
 807 #endif /* defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC) */
 808         }
 809 }
 810
 811 static inline void
 812 set_saved_state_cpsr(arm_saved_state_t *iss, uint32_t cpsr)
 813 {
 814         if (is_saved_state32(iss)) {
 815                 saved_state32(iss)->cpsr = cpsr;
 816         } else {
 817 #if defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC)
 818                 MANIPULATE_SIGNED_THREAD_STATE(iss,
 819                     "mov        w2, %w[cpsr]            \n"
 820                     "str        w2, [x0, %[SS64_CPSR]]  \n",
 821                     [cpsr] "r"(cpsr)
 822                     );
 823 #else
 824                 saved_state64(iss)->cpsr = cpsr;
 825 #endif /* defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC) */
 826         }
 827 }
 828
 829 static inline register_t
 830 get_saved_state_far(const arm_saved_state_t *iss)
 831 {
 832         return is_saved_state32(iss) ? const_saved_state32(iss)->far : const_saved_state64(iss)->far;
 833 }
 834
 835 static inline void
 836 set_saved_state_far(arm_saved_state_t *iss, register_t far)
 837 {
 838         if (is_saved_state32(iss)) {
 839                 saved_state32(iss)->far = CAST_ASSERT_SAFE(uint32_t, far);
 840         } else {
 841                 saved_state64(iss)->far = far;
 842         }
 843 }
 844
 845 static inline uint32_t
 846 get_saved_state_esr(const arm_saved_state_t *iss)
 847 {
 848         return is_saved_state32(iss) ? const_saved_state32(iss)->esr : const_saved_state64(iss)->esr;
 849 }
 850
 851 static inline void
 852 set_saved_state_esr(arm_saved_state_t *iss, uint32_t esr)
 853 {
 854         if (is_saved_state32(iss)) {
 855                 saved_state32(iss)->esr = esr;
 856         } else {
 857                 saved_state64(iss)->esr = esr;
 858         }
 859 }
 860
 861 static inline uint32_t
 862 get_saved_state_exc(const arm_saved_state_t *iss)
 863 {
 864         return is_saved_state32(iss) ? const_saved_state32(iss)->exception : const_saved_state64(iss)->exception;
 865 }
 866
 867 static inline void
 868 set_saved_state_exc(arm_saved_state_t *iss, uint32_t exc)
 869 {
 870         if (is_saved_state32(iss)) {
 871                 saved_state32(iss)->exception = exc;
 872         } else {
 873                 saved_state64(iss)->exception = exc;
 874         }
 875 }
 876
 877 extern void panic_unimplemented(void);
 878
 879 static inline int
 880 get_saved_state_svc_number(const arm_saved_state_t *iss)
 881 {
 882         return is_saved_state32(iss) ? (int)const_saved_state32(iss)->r[12] : (int)const_saved_state64(iss)->x[ARM64_SYSCALL_CODE_REG_NUM]; /* Only first word counts here */
 883 }
 884
 885 typedef _STRUCT_ARM_LEGACY_DEBUG_STATE arm_legacy_debug_state_t;
 886
 887 struct arm_debug_aggregate_state {
 888         arm_state_hdr_t dsh;
 889         union {
 890                 arm_debug_state32_t ds32;
 891                 arm_debug_state64_t ds64;
 892         } uds;
 893 } __attribute__((aligned(16)));
 894
 895 typedef struct arm_debug_aggregate_state arm_debug_state_t;
 896
 897 #define ARM_LEGACY_DEBUG_STATE_COUNT ((mach_msg_type_number_t) \
 898         (sizeof (arm_legacy_debug_state_t)/sizeof(uint32_t)))
 899
 900 /*
 901  * NEON context
 902  */
 903 typedef __uint128_t uint128_t;
 904 typedef uint64_t uint64x2_t __attribute__((ext_vector_type(2)));
 905 typedef uint32_t uint32x4_t __attribute__((ext_vector_type(4)));
 906
 907 struct arm_neon_saved_state32 {
 908         union {
 909                 uint128_t q[16];
 910                 uint64_t  d[32];
 911                 uint32_t  s[32];
 912         } v;
 913         uint32_t fpsr;
 914         uint32_t fpcr;
 915 };
 916 typedef struct arm_neon_saved_state32 arm_neon_saved_state32_t;
 917
 918 #define ARM_NEON_SAVED_STATE32_COUNT ((mach_msg_type_number_t) \
 919         (sizeof (arm_neon_saved_state32_t)/sizeof(unsigned int)))
 920
 921 struct arm_neon_saved_state64 {
 922         union {
 923                 uint128_t  q[32];
 924                 uint64x2_t d[32];
 925                 uint32x4_t s[32];
 926         } v;
 927         uint32_t fpsr;
 928         uint32_t fpcr;
 929 };
 930 typedef struct arm_neon_saved_state64 arm_neon_saved_state64_t;
 931
 932 #define ARM_NEON_SAVED_STATE64_COUNT ((mach_msg_type_number_t) \
 933         (sizeof (arm_neon_saved_state64_t)/sizeof(unsigned int)))
 934
 935 struct arm_neon_saved_state {
 936         arm_state_hdr_t nsh;
 937         union {
 938                 struct arm_neon_saved_state32 ns_32;
 939                 struct arm_neon_saved_state64 ns_64;
 940         } uns;
 941 };
 942 typedef struct arm_neon_saved_state arm_neon_saved_state_t;
 943 #define ns_32 uns.ns_32
 944 #define ns_64 uns.ns_64
 945
 946 static inline boolean_t
 947 is_neon_saved_state32(const arm_neon_saved_state_t *state)
 948 {
 949         return state->nsh.flavor == ARM_NEON_SAVED_STATE32;
 950 }
 951
 952 static inline boolean_t
 953 is_neon_saved_state64(const arm_neon_saved_state_t *state)
 954 {
 955         return state->nsh.flavor == ARM_NEON_SAVED_STATE64;
 956 }
 957
 958 static inline arm_neon_saved_state32_t *
 959 neon_state32(arm_neon_saved_state_t *state)
 960 {
 961         return &state->ns_32;
 962 }
 963
 964 static inline arm_neon_saved_state64_t *
 965 neon_state64(arm_neon_saved_state_t *state)
 966 {
 967         return &state->ns_64;
 968 }
 969
 970
 971 /*
 972  * Aggregated context
 973  */
 974
 975 struct arm_context {
 976         struct arm_saved_state ss;
 977         struct arm_neon_saved_state ns;
 978 };
 979 typedef struct arm_context arm_context_t;
 980
 981 extern void saved_state_to_thread_state64(const arm_saved_state_t*, arm_thread_state64_t*);
 982 extern void thread_state64_to_saved_state(const arm_thread_state64_t*, arm_saved_state_t*);
 983
 984 #else /* defined(__arm__) */
 985 #error Unknown arch
 986 #endif /* defined(__arm__) */
 987
 988 extern void saved_state_to_thread_state32(const arm_saved_state_t*, arm_thread_state32_t*);
 989 extern void thread_state32_to_saved_state(const arm_thread_state32_t*, arm_saved_state_t*);
 990
 991 #endif /* XNU_KERNEL_PRIVATE */
 992
 993 #endif /* _ARM_THREAD_STATUS_H_ */