osfmk/mach/arm/thread_status.h

   1 /*
   2  * Copyright (c) 2007 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 /*
  29  * FILE_ID: thread_status.h
  30  */
  31
  32
  33 #ifndef _ARM_THREAD_STATUS_H_
  34 #define _ARM_THREAD_STATUS_H_
  35
  36 #include <mach/machine/_structs.h>
  37 #include <mach/message.h>
  38 #include <mach/vm_types.h>
  39 #include <mach/arm/thread_state.h>
  40
  41 /*
  42  *    Support for determining the state of a thread
  43  */
  44
  45
  46 /*
  47  *  Flavors
  48  */
  49
  50 #define ARM_THREAD_STATE         1
  51 #define ARM_UNIFIED_THREAD_STATE ARM_THREAD_STATE
  52 #define ARM_VFP_STATE            2
  53 #define ARM_EXCEPTION_STATE      3
  54 #define ARM_DEBUG_STATE          4 /* pre-armv8 */
  55 #define THREAD_STATE_NONE        5
  56 #define ARM_THREAD_STATE64       6
  57 #define ARM_EXCEPTION_STATE64    7
  58 //      ARM_THREAD_STATE_LAST    8 /* legacy */
  59 #define ARM_THREAD_STATE32       9
  60
  61 /* API */
  62 #define ARM_DEBUG_STATE32        14
  63 #define ARM_DEBUG_STATE64        15
  64 #define ARM_NEON_STATE           16
  65 #define ARM_NEON_STATE64         17
  66 #define ARM_CPMU_STATE64         18
  67
  68 #ifdef XNU_KERNEL_PRIVATE
  69 /* For kernel use */
  70 #define ARM_SAVED_STATE32        20
  71 #define ARM_SAVED_STATE64        21
  72 #define ARM_NEON_SAVED_STATE32   22
  73 #define ARM_NEON_SAVED_STATE64   23
  74 #endif /* XNU_KERNEL_PRIVATE */
  75
  76
  77 #define ARM_STATE_FLAVOR_IS_OTHER_VALID(_flavor_) 0
  78
  79 #define ARM_PAGEIN_STATE         27
  80
  81 #define VALID_THREAD_STATE_FLAVOR(x) \
  82         ((x == ARM_THREAD_STATE) ||           \
  83          (x == ARM_VFP_STATE) ||              \
  84          (x == ARM_EXCEPTION_STATE) ||        \
  85          (x == ARM_DEBUG_STATE) ||            \
  86          (x == THREAD_STATE_NONE) ||          \
  87          (x == ARM_THREAD_STATE32) ||         \
  88          (x == ARM_THREAD_STATE64) ||         \
  89          (x == ARM_EXCEPTION_STATE64) ||      \
  90          (x == ARM_NEON_STATE) ||             \
  91          (x == ARM_NEON_STATE64) ||           \
  92          (x == ARM_DEBUG_STATE32) ||          \
  93          (x == ARM_DEBUG_STATE64) ||          \
  94          (x == ARM_PAGEIN_STATE) ||           \
  95          (ARM_STATE_FLAVOR_IS_OTHER_VALID(x)))
  96
  97 struct arm_state_hdr {
  98         uint32_t flavor;
  99         uint32_t count;
 100 };
 101 typedef struct arm_state_hdr arm_state_hdr_t;
 102
 103 typedef _STRUCT_ARM_THREAD_STATE   arm_thread_state_t;
 104 typedef _STRUCT_ARM_THREAD_STATE   arm_thread_state32_t;
 105 typedef _STRUCT_ARM_THREAD_STATE64 arm_thread_state64_t;
 106
 107 #if !defined(KERNEL)
 108 #if __DARWIN_C_LEVEL >= __DARWIN_C_FULL && defined(__arm64__)
 109
 110 /* Accessor macros for arm_thread_state64_t pointer fields */
 111
 112 /* Return pc field of arm_thread_state64_t as a data pointer value */
 113 #define arm_thread_state64_get_pc(ts) \
 114                 __darwin_arm_thread_state64_get_pc(ts)
 115 /* Return pc field of arm_thread_state64_t as a function pointer. May return
 116  * NULL if a valid function pointer cannot be constructed, the caller should
 117  * fall back to the arm_thread_state64_get_pc() macro in that case. */
 118 #define arm_thread_state64_get_pc_fptr(ts) \
 119                 __darwin_arm_thread_state64_get_pc_fptr(ts)
 120 /* Set pc field of arm_thread_state64_t to a function pointer */
 121 #define arm_thread_state64_set_pc_fptr(ts, fptr) \
 122                 __darwin_arm_thread_state64_set_pc_fptr(ts, fptr)
 123 /* Return lr field of arm_thread_state64_t as a data pointer value */
 124 #define arm_thread_state64_get_lr(ts) \
 125                 __darwin_arm_thread_state64_get_lr(ts)
 126 /* Return lr field of arm_thread_state64_t as a function pointer. May return
 127  * NULL if a valid function pointer cannot be constructed, the caller should
 128  * fall back to the arm_thread_state64_get_lr() macro in that case. */
 129 #define arm_thread_state64_get_lr_fptr(ts) \
 130                 __darwin_arm_thread_state64_get_lr_fptr(ts)
 131 /* Set lr field of arm_thread_state64_t to a function pointer */
 132 #define arm_thread_state64_set_lr_fptr(ts, fptr) \
 133                 __darwin_arm_thread_state64_set_lr_fptr(ts, fptr)
 134 /* Return sp field of arm_thread_state64_t as a data pointer value */
 135 #define arm_thread_state64_get_sp(ts) \
 136                 __darwin_arm_thread_state64_get_sp(ts)
 137 /* Set sp field of arm_thread_state64_t to a data pointer value */
 138 #define arm_thread_state64_set_sp(ts, ptr) \
 139                 __darwin_arm_thread_state64_set_sp(ts, ptr)
 140 /* Return fp field of arm_thread_state64_t as a data pointer value */
 141 #define arm_thread_state64_get_fp(ts) \
 142                 __darwin_arm_thread_state64_get_fp(ts)
 143 /* Set fp field of arm_thread_state64_t to a data pointer value */
 144 #define arm_thread_state64_set_fp(ts, ptr) \
 145                 __darwin_arm_thread_state64_set_fp(ts, ptr)
 146
 147 #endif /* __DARWIN_C_LEVEL >= __DARWIN_C_FULL && defined(__arm64__) */
 148 #endif /* !defined(KERNEL) */
 149
 150 struct arm_unified_thread_state {
 151         arm_state_hdr_t ash;
 152         union {
 153                 arm_thread_state32_t ts_32;
 154                 arm_thread_state64_t ts_64;
 155         } uts;
 156 };
 157 #define ts_32 uts.ts_32
 158 #define ts_64 uts.ts_64
 159 typedef struct arm_unified_thread_state arm_unified_thread_state_t;
 160
 161 #define ARM_THREAD_STATE_COUNT ((mach_msg_type_number_t) \
 162         (sizeof (arm_thread_state_t)/sizeof(uint32_t)))
 163 #define ARM_THREAD_STATE32_COUNT ((mach_msg_type_number_t) \
 164         (sizeof (arm_thread_state32_t)/sizeof(uint32_t)))
 165 #define ARM_THREAD_STATE64_COUNT ((mach_msg_type_number_t) \
 166         (sizeof (arm_thread_state64_t)/sizeof(uint32_t)))
 167 #define ARM_UNIFIED_THREAD_STATE_COUNT ((mach_msg_type_number_t) \
 168         (sizeof (arm_unified_thread_state_t)/sizeof(uint32_t)))
 169
 170
 171 typedef _STRUCT_ARM_VFP_STATE         arm_vfp_state_t;
 172 typedef _STRUCT_ARM_NEON_STATE        arm_neon_state_t;
 173 typedef _STRUCT_ARM_NEON_STATE        arm_neon_state32_t;
 174 typedef _STRUCT_ARM_NEON_STATE64      arm_neon_state64_t;
 175
 176
 177 typedef _STRUCT_ARM_EXCEPTION_STATE   arm_exception_state_t;
 178 typedef _STRUCT_ARM_EXCEPTION_STATE   arm_exception_state32_t;
 179 typedef _STRUCT_ARM_EXCEPTION_STATE64 arm_exception_state64_t;
 180
 181 typedef _STRUCT_ARM_DEBUG_STATE32     arm_debug_state32_t;
 182 typedef _STRUCT_ARM_DEBUG_STATE64     arm_debug_state64_t;
 183
 184 typedef _STRUCT_ARM_PAGEIN_STATE      arm_pagein_state_t;
 185
 186 #if defined(XNU_KERNEL_PRIVATE) && defined(__arm64__)
 187 /* See below for ARM64 kernel structure definition for arm_debug_state. */
 188 #else /* defined(XNU_KERNEL_PRIVATE) && defined(__arm64__) */
 189 /*
 190  * Otherwise not ARM64 kernel and we must preserve legacy ARM definitions of
 191  * arm_debug_state for binary compatability of userland consumers of this file.
 192  */
 193 #if defined(__arm__)
 194 typedef _STRUCT_ARM_DEBUG_STATE        arm_debug_state_t;
 195 #elif defined(__arm64__)
 196 typedef _STRUCT_ARM_LEGACY_DEBUG_STATE arm_debug_state_t;
 197 #else /* defined(__arm__) */
 198 #error Undefined architecture
 199 #endif /* defined(__arm__) */
 200 #endif /* defined(XNU_KERNEL_PRIVATE) && defined(__arm64__) */
 201
 202 #define ARM_VFP_STATE_COUNT ((mach_msg_type_number_t) \
 203         (sizeof (arm_vfp_state_t)/sizeof(uint32_t)))
 204
 205 #define ARM_EXCEPTION_STATE_COUNT ((mach_msg_type_number_t) \
 206         (sizeof (arm_exception_state_t)/sizeof(uint32_t)))
 207
 208 #define ARM_EXCEPTION_STATE64_COUNT ((mach_msg_type_number_t) \
 209         (sizeof (arm_exception_state64_t)/sizeof(uint32_t)))
 210
 211 #define ARM_DEBUG_STATE_COUNT ((mach_msg_type_number_t) \
 212         (sizeof (arm_debug_state_t)/sizeof(uint32_t)))
 213
 214 #define ARM_DEBUG_STATE32_COUNT ((mach_msg_type_number_t) \
 215         (sizeof (arm_debug_state32_t)/sizeof(uint32_t)))
 216
 217 #define ARM_PAGEIN_STATE_COUNT ((mach_msg_type_number_t) \
 218         (sizeof (arm_pagein_state_t)/sizeof(uint32_t)))
 219
 220 #define ARM_DEBUG_STATE64_COUNT ((mach_msg_type_number_t) \
 221         (sizeof (arm_debug_state64_t)/sizeof(uint32_t)))
 222
 223 #define ARM_NEON_STATE_COUNT ((mach_msg_type_number_t) \
 224         (sizeof (arm_neon_state_t)/sizeof(uint32_t)))
 225
 226 #define ARM_NEON_STATE64_COUNT ((mach_msg_type_number_t) \
 227         (sizeof (arm_neon_state64_t)/sizeof(uint32_t)))
 228
 229 #define MACHINE_THREAD_STATE       ARM_THREAD_STATE
 230 #define MACHINE_THREAD_STATE_COUNT ARM_UNIFIED_THREAD_STATE_COUNT
 231
 232
 233 /*
 234  * Largest state on this machine:
 235  */
 236 #define THREAD_MACHINE_STATE_MAX THREAD_STATE_MAX
 237
 238 #ifdef XNU_KERNEL_PRIVATE
 239
 240 static inline boolean_t
 241 is_thread_state32(const arm_unified_thread_state_t *its)
 242 {
 243         return its->ash.flavor == ARM_THREAD_STATE32;
 244 }
 245
 246 static inline boolean_t
 247 is_thread_state64(const arm_unified_thread_state_t *its)
 248 {
 249         return its->ash.flavor == ARM_THREAD_STATE64;
 250 }
 251
 252 static inline arm_thread_state32_t*
 253 thread_state32(arm_unified_thread_state_t *its)
 254 {
 255         return &its->ts_32;
 256 }
 257
 258 static inline arm_thread_state64_t*
 259 thread_state64(arm_unified_thread_state_t *its)
 260 {
 261         return &its->ts_64;
 262 }
 263
 264 static inline const arm_thread_state32_t*
 265 const_thread_state32(const arm_unified_thread_state_t *its)
 266 {
 267         return &its->ts_32;
 268 }
 269
 270 static inline const arm_thread_state64_t*
 271 const_thread_state64(const arm_unified_thread_state_t *its)
 272 {
 273         return &its->ts_64;
 274 }
 275
 276 #if defined(__arm__)
 277 #include <arm/proc_reg.h>
 278
 279 #define ARM_SAVED_STATE (THREAD_STATE_NONE + 1)
 280
 281 #if __ARM_VFP__
 282 #define VFPSAVE_ALIGN  16
 283 #define VFPSAVE_ATTRIB __attribute__((aligned (VFPSAVE_ALIGN)))
 284 #define THREAD_ALIGN   VFPSAVE_ALIGN
 285
 286 /*
 287  * vector floating point saved state
 288  */
 289 struct arm_vfpsaved_state {
 290         uint32_t r[64];
 291         uint32_t fpscr;
 292         uint32_t fpexc;
 293 };
 294 #endif
 295
 296 struct arm_saved_state {
 297         uint32_t r[13];     /* General purpose register r0-r12 */
 298         uint32_t sp;        /* Stack pointer r13 */
 299         uint32_t lr;        /* Link register r14 */
 300         uint32_t pc;        /* Program counter r15 */
 301         uint32_t cpsr;      /* Current program status register */
 302         uint32_t fsr;       /* Fault status */
 303         uint32_t far;       /* Virtual Fault Address */
 304         uint32_t exception; /* exception number */
 305
 306 #if __ARM_VFP__
 307         /* VFP state */
 308         struct arm_vfpsaved_state VFPdata VFPSAVE_ATTRIB;
 309         // for packing reasons chtread_self and DebugData
 310         // are inside the the PcbData when __ARM_VFP__ is set
 311         arm_debug_state_t        *VFPpadding_DebugData;
 312         vm_address_t              VFPpadding_cthread_self;
 313 #endif
 314 };
 315 typedef struct arm_saved_state arm_saved_state_t;
 316
 317 /*
 318  * Just for coexistence with AArch64 code.
 319  */
 320 typedef struct arm_saved_state arm_saved_state32_t;
 321
 322 static inline void
 323 copy_signed_thread_state(arm_saved_state_t *dst, const arm_saved_state_t *src)
 324 {
 325         *dst = *src;
 326 }
 327
 328 static inline arm_saved_state32_t*
 329 saved_state32(arm_saved_state_t *iss)
 330 {
 331         return iss;
 332 }
 333
 334 static inline boolean_t
 335 is_saved_state32(const arm_saved_state_t *iss __unused)
 336 {
 337         return TRUE;
 338 }
 339
 340
 341 struct arm_saved_state_tagged {
 342         uint32_t               tag;
 343         struct arm_saved_state state;
 344 };
 345 typedef struct arm_saved_state_tagged arm_saved_state_tagged_t;
 346
 347 #define ARM_SAVED_STATE32_COUNT ((mach_msg_type_number_t) \
 348         (sizeof (arm_saved_state_t)/sizeof(unsigned int)))
 349
 350
 351 static inline register_t
 352 get_saved_state_pc(const arm_saved_state_t *iss)
 353 {
 354         return iss->pc;
 355 }
 356
 357 static inline void
 358 add_saved_state_pc(arm_saved_state_t *iss, int diff)
 359 {
 360         iss->pc += diff;
 361 }
 362
 363 static inline void
 364 set_saved_state_pc(arm_saved_state_t *iss, register_t pc)
 365 {
 366         iss->pc = pc;
 367 }
 368
 369 static inline register_t
 370 get_saved_state_sp(const arm_saved_state_t *iss)
 371 {
 372         return iss->sp;
 373 }
 374
 375 static inline void
 376 set_saved_state_sp(arm_saved_state_t *iss, register_t sp)
 377 {
 378         iss->sp = sp;
 379 }
 380
 381 static inline register_t
 382 get_saved_state_fp(const arm_saved_state_t *iss)
 383 {
 384         return iss->r[7];
 385 }
 386
 387 static inline void
 388 set_saved_state_fp(arm_saved_state_t *iss, register_t fp)
 389 {
 390         iss->r[7] = fp;
 391 }
 392
 393 static inline register_t
 394 get_saved_state_lr(const arm_saved_state_t *iss)
 395 {
 396         return iss->lr;
 397 }
 398
 399 static inline void
 400 set_saved_state_lr(arm_saved_state_t *iss, register_t lr)
 401 {
 402         iss->lr = lr;
 403 }
 404
 405 static inline register_t
 406 get_saved_state_cpsr(const arm_saved_state_t *iss)
 407 {
 408         return iss->cpsr;
 409 }
 410
 411 static inline void
 412 mask_saved_state_cpsr(arm_saved_state_t *iss, uint32_t set_bits, uint32_t clear_bits)
 413 {
 414         iss->cpsr |= set_bits;
 415         iss->cpsr &= ~clear_bits;
 416 }
 417
 418 static inline void
 419 set_saved_state_cpsr(arm_saved_state_t *iss, register_t cpsr)
 420 {
 421         iss->cpsr = cpsr;
 422 }
 423
 424 static inline register_t
 425 get_saved_state_reg(const arm_saved_state_t *iss, unsigned regno)
 426 {
 427         return iss->r[regno];
 428 }
 429
 430 static inline void
 431 set_saved_state_reg(arm_saved_state_t *iss, unsigned regno, register_t val)
 432 {
 433         iss->r[regno] = val;
 434 }
 435
 436 #elif defined(__arm64__)
 437
 438 #include <kern/assert.h>
 439 #include <arm64/proc_reg.h>
 440 #define CAST_ASSERT_SAFE(type, val) (assert((val) == ((type)(val))), (type)(val))
 441
 442 /*
 443  * GPR context
 444  */
 445
 446 struct arm_saved_state32 {
 447         uint32_t r[13];     /* General purpose register r0-r12 */
 448         uint32_t sp;        /* Stack pointer r13 */
 449         uint32_t lr;        /* Link register r14 */
 450         uint32_t pc;        /* Program counter r15 */
 451         uint32_t cpsr;      /* Current program status register */
 452         uint32_t far;       /* Virtual fault address */
 453         uint32_t esr;       /* Exception syndrome register */
 454         uint32_t exception; /* Exception number */
 455 };
 456 typedef struct arm_saved_state32 arm_saved_state32_t;
 457
 458 struct arm_saved_state32_tagged {
 459         uint32_t                 tag;
 460         struct arm_saved_state32 state;
 461 };
 462 typedef struct arm_saved_state32_tagged arm_saved_state32_tagged_t;
 463
 464 #define ARM_SAVED_STATE32_COUNT ((mach_msg_type_number_t) \
 465                 (sizeof(arm_saved_state32_t)/sizeof(unsigned int)))
 466
 467 struct arm_saved_state64 {
 468         uint64_t x[29];     /* General purpose registers x0-x28 */
 469         uint64_t fp;        /* Frame pointer x29 */
 470         uint64_t lr;        /* Link register x30 */
 471         uint64_t sp;        /* Stack pointer x31 */
 472         uint64_t pc;        /* Program counter */
 473         uint32_t cpsr;      /* Current program status register */
 474         uint32_t reserved;  /* Reserved padding */
 475         uint64_t far;       /* Virtual fault address */
 476         uint32_t esr;       /* Exception syndrome register */
 477         uint32_t exception; /* Exception number */
 478 #if defined(HAS_APPLE_PAC)
 479         uint64_t jophash;
 480 #endif /* defined(HAS_APPLE_PAC) */
 481 };
 482 typedef struct arm_saved_state64 arm_saved_state64_t;
 483
 484 #define ARM_SAVED_STATE64_COUNT ((mach_msg_type_number_t) \
 485         (sizeof(arm_saved_state64_t)/sizeof(unsigned int)))
 486
 487 struct arm_saved_state64_tagged {
 488         uint32_t                 tag;
 489         struct arm_saved_state64 state;
 490 };
 491 typedef struct arm_saved_state64_tagged arm_saved_state64_tagged_t;
 492
 493 struct arm_saved_state {
 494         arm_state_hdr_t ash;
 495         union {
 496                 struct arm_saved_state32 ss_32;
 497                 struct arm_saved_state64 ss_64;
 498         } uss;
 499 } __attribute__((aligned(16)));
 500 #define ss_32 uss.ss_32
 501 #define ss_64 uss.ss_64
 502
 503 typedef struct arm_saved_state arm_saved_state_t;
 504
 505 #if defined(XNU_KERNEL_PRIVATE)
 506 #if defined(HAS_APPLE_PAC)
 507 /*
 508  * Methods used to sign and check thread state to detect corruptions of saved
 509  * thread state across exceptions and context switches.
 510  */
 511 extern void ml_sign_thread_state(arm_saved_state_t *, uint64_t, uint32_t, uint64_t, uint64_t, uint64_t);
 512
 513 extern void ml_check_signed_state(const arm_saved_state_t *, uint64_t, uint32_t, uint64_t, uint64_t, uint64_t);
 514
 515 /* XXX: including stddef.f here breaks ctfmerge on some builds, so use __builtin_offsetof() instead of offsetof() */
 516 #define ss64_offsetof(x) __builtin_offsetof(struct arm_saved_state, ss_64.x)
 517
 518 /**
 519  * Verify the signed thread state in _iss, execute the assembly instructions
 520  * _instr, and re-sign the modified thread state.  Varargs specify additional
 521  * inputs.
 522  *
 523  * _instr may read or modify the thread state in the following registers:
 524  *
 525  * x0: _iss
 526  * x1: authed _iss->ss_64.pc
 527  * w2: authed _iss->ss_64.cpsr
 528  * x3: authed _iss->ss_64.lr
 529  * x4: authed _iss->ss_64.x16
 530  * x5: authed _iss->ss_64.x17
 531  * x6: scratch register
 532  * x7: scratch register
 533  */
 534 #define MANIPULATE_SIGNED_THREAD_STATE(_iss, _instr, ...)               \
 535         asm volatile (                                                  \
 536                 "mov    x8, lr"                         "\n"            \
 537                 "mov    x0, %[iss]"                     "\n"            \
 538                 "ldp    x4, x5, [x0, %[SS64_X16]]"      "\n"            \
 539                 "ldr    x6, [x0, %[SS64_PC]]"           "\n"            \
 540                 "ldr    w7, [x0, %[SS64_CPSR]]"         "\n"            \
 541                 "ldr    x3, [x0, %[SS64_LR]]"           "\n"            \
 542                 "mov    x1, x6"                         "\n"            \
 543                 "mov    w2, w7"                         "\n"            \
 544                 "bl     _ml_check_signed_state"         "\n"            \
 545                 "mov    x1, x6"                         "\n"            \
 546                 "mov    w2, w7"                         "\n"            \
 547                 _instr                                  "\n"            \
 548                 "bl     _ml_sign_thread_state"          "\n"            \
 549                 "mov    lr, x8"                         "\n"            \
 550                 :                                                       \
 551                 : [iss]         "r"(_iss),                              \
 552                   [SS64_X16]    "i"(ss64_offsetof(x[16])),              \
 553                   [SS64_PC]     "i"(ss64_offsetof(pc)),                 \
 554                   [SS64_CPSR]   "i"(ss64_offsetof(cpsr)),               \
 555                   [SS64_LR]     "i"(ss64_offsetof(lr)),##__VA_ARGS__    \
 556                 : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8"  \
 557         )
 558
 559 static inline void
 560 check_and_sign_copied_thread_state(arm_saved_state_t *dst, const arm_saved_state_t *src)
 561 {
 562         MANIPULATE_SIGNED_THREAD_STATE(src,
 563             "mov        x0, %[dst]",
 564             [dst] "r"(dst)
 565             );
 566 }
 567 #endif /* defined(HAS_APPLE_PAC) */
 568
 569 static inline void
 570 copy_signed_thread_state(arm_saved_state_t *dst, const arm_saved_state_t *src)
 571 {
 572         *dst = *src;
 573 #if defined(HAS_APPLE_PAC)
 574         check_and_sign_copied_thread_state(dst, src);
 575 #endif
 576 }
 577
 578 #endif /* defined(XNU_KERNEL_PRIVATE) */
 579
 580 static inline boolean_t
 581 is_saved_state32(const arm_saved_state_t *iss)
 582 {
 583         return iss->ash.flavor == ARM_SAVED_STATE32;
 584 }
 585
 586 static inline boolean_t
 587 is_saved_state64(const arm_saved_state_t *iss)
 588 {
 589         return iss->ash.flavor == ARM_SAVED_STATE64;
 590 }
 591
 592 static inline arm_saved_state32_t*
 593 saved_state32(arm_saved_state_t *iss)
 594 {
 595         return &iss->ss_32;
 596 }
 597
 598 static inline const arm_saved_state32_t*
 599 const_saved_state32(const arm_saved_state_t *iss)
 600 {
 601         return &iss->ss_32;
 602 }
 603
 604 static inline arm_saved_state64_t*
 605 saved_state64(arm_saved_state_t *iss)
 606 {
 607         return &iss->ss_64;
 608 }
 609
 610 static inline const arm_saved_state64_t*
 611 const_saved_state64(const arm_saved_state_t *iss)
 612 {
 613         return &iss->ss_64;
 614 }
 615
 616 static inline register_t
 617 get_saved_state_pc(const arm_saved_state_t *iss)
 618 {
 619         return is_saved_state32(iss) ? const_saved_state32(iss)->pc : const_saved_state64(iss)->pc;
 620 }
 621
 622 static inline void
 623 add_saved_state_pc(arm_saved_state_t *iss, int diff)
 624 {
 625         if (is_saved_state32(iss)) {
 626                 uint64_t pc = saved_state32(iss)->pc + diff;
 627                 saved_state32(iss)->pc = CAST_ASSERT_SAFE(uint32_t, pc);
 628         } else {
 629 #if defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC)
 630                 MANIPULATE_SIGNED_THREAD_STATE(iss,
 631                     "mov        w6, %w[diff]            \n"
 632                     "add        x1, x1, w6, sxtw        \n"
 633                     "str        x1, [x0, %[SS64_PC]]    \n",
 634                     [diff] "r"(diff)
 635                     );
 636 #else
 637                 saved_state64(iss)->pc += diff;
 638 #endif /* defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC) */
 639         }
 640 }
 641
 642 static inline void
 643 set_saved_state_pc(arm_saved_state_t *iss, register_t pc)
 644 {
 645         if (is_saved_state32(iss)) {
 646                 saved_state32(iss)->pc = CAST_ASSERT_SAFE(uint32_t, pc);
 647         } else {
 648 #if defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC)
 649                 MANIPULATE_SIGNED_THREAD_STATE(iss,
 650                     "mov        x1, %[pc]               \n"
 651                     "str        x1, [x0, %[SS64_PC]]    \n",
 652                     [pc] "r"(pc)
 653                     );
 654 #else
 655                 saved_state64(iss)->pc = pc;
 656 #endif /* defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC) */
 657         }
 658 }
 659
 660 static inline register_t
 661 get_saved_state_sp(const arm_saved_state_t *iss)
 662 {
 663         return is_saved_state32(iss) ? const_saved_state32(iss)->sp : const_saved_state64(iss)->sp;
 664 }
 665
 666 static inline void
 667 set_saved_state_sp(arm_saved_state_t *iss, register_t sp)
 668 {
 669         if (is_saved_state32(iss)) {
 670                 saved_state32(iss)->sp = CAST_ASSERT_SAFE(uint32_t, sp);
 671         } else {
 672                 saved_state64(iss)->sp = sp;
 673         }
 674 }
 675
 676 static inline register_t
 677 get_saved_state_lr(const arm_saved_state_t *iss)
 678 {
 679         return is_saved_state32(iss) ? const_saved_state32(iss)->lr : const_saved_state64(iss)->lr;
 680 }
 681
 682 static inline void
 683 set_saved_state_lr(arm_saved_state_t *iss, register_t lr)
 684 {
 685         if (is_saved_state32(iss)) {
 686                 saved_state32(iss)->lr = CAST_ASSERT_SAFE(uint32_t, lr);
 687         } else {
 688 #if defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC)
 689                 MANIPULATE_SIGNED_THREAD_STATE(iss,
 690                     "mov        x3, %[lr]               \n"
 691                     "str        x3, [x0, %[SS64_LR]]    \n",
 692                     [lr] "r"(lr)
 693                     );
 694 #else
 695                 saved_state64(iss)->lr = lr;
 696 #endif /* defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC) */
 697         }
 698 }
 699
 700 static inline register_t
 701 get_saved_state_fp(const arm_saved_state_t *iss)
 702 {
 703         return is_saved_state32(iss) ? const_saved_state32(iss)->r[7] : const_saved_state64(iss)->fp;
 704 }
 705
 706 static inline void
 707 set_saved_state_fp(arm_saved_state_t *iss, register_t fp)
 708 {
 709         if (is_saved_state32(iss)) {
 710                 saved_state32(iss)->r[7] = CAST_ASSERT_SAFE(uint32_t, fp);
 711         } else {
 712                 saved_state64(iss)->fp = fp;
 713         }
 714 }
 715
 716 static inline int
 717 check_saved_state_reglimit(const arm_saved_state_t *iss, unsigned reg)
 718 {
 719         return is_saved_state32(iss) ? (reg < ARM_SAVED_STATE32_COUNT) : (reg < ARM_SAVED_STATE64_COUNT);
 720 }
 721
 722 static inline register_t
 723 get_saved_state_reg(const arm_saved_state_t *iss, unsigned reg)
 724 {
 725         if (!check_saved_state_reglimit(iss, reg)) {
 726                 return 0;
 727         }
 728
 729         return is_saved_state32(iss) ? (const_saved_state32(iss)->r[reg]) : (const_saved_state64(iss)->x[reg]);
 730 }
 731
 732 static inline void
 733 set_saved_state_reg(arm_saved_state_t *iss, unsigned reg, register_t value)
 734 {
 735         if (!check_saved_state_reglimit(iss, reg)) {
 736                 return;
 737         }
 738
 739         if (is_saved_state32(iss)) {
 740                 saved_state32(iss)->r[reg] = CAST_ASSERT_SAFE(uint32_t, value);
 741         } else {
 742 #if defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC)
 743                 /* x16 and x17 are part of the jophash */
 744                 if (reg == 16) {
 745                         MANIPULATE_SIGNED_THREAD_STATE(iss,
 746                             "mov        x4, %[value]            \n"
 747                             "str        x4, [x0, %[SS64_X16]]   \n",
 748                             [value] "r"(value)
 749                             );
 750                         return;
 751                 } else if (reg == 17) {
 752                         MANIPULATE_SIGNED_THREAD_STATE(iss,
 753                             "mov        x5, %[value]            \n"
 754                             "str        x5, [x0, %[SS64_X17]]   \n",
 755                             [value] "r"(value),
 756                             [SS64_X17] "i"(ss64_offsetof(x[17]))
 757                             );
 758                         return;
 759                 }
 760 #endif
 761                 saved_state64(iss)->x[reg] = value;
 762         }
 763 }
 764
 765 static inline uint32_t
 766 get_saved_state_cpsr(const arm_saved_state_t *iss)
 767 {
 768         return is_saved_state32(iss) ? const_saved_state32(iss)->cpsr : const_saved_state64(iss)->cpsr;
 769 }
 770
 771 static inline void
 772 mask_saved_state_cpsr(arm_saved_state_t *iss, uint32_t set_bits, uint32_t clear_bits)
 773 {
 774         if (is_saved_state32(iss)) {
 775                 saved_state32(iss)->cpsr |= set_bits;
 776                 saved_state32(iss)->cpsr &= ~clear_bits;
 777         } else {
 778 #if defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC)
 779                 MANIPULATE_SIGNED_THREAD_STATE(iss,
 780                     "mov        w6, %w[set_bits]        \n"
 781                     "orr        w2, w2, w6, lsl #0      \n"
 782                     "mov        w6, %w[clear_bits]      \n"
 783                     "bic        w2, w2, w6, lsl #0      \n"
 784                     "str        w2, [x0, %[SS64_CPSR]]  \n",
 785                     [set_bits] "r"(set_bits),
 786                     [clear_bits] "r"(clear_bits)
 787                     );
 788 #else
 789                 saved_state64(iss)->cpsr |= set_bits;
 790                 saved_state64(iss)->cpsr &= ~clear_bits;
 791 #endif /* defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC) */
 792         }
 793 }
 794
 795 static inline void
 796 set_saved_state_cpsr(arm_saved_state_t *iss, uint32_t cpsr)
 797 {
 798         if (is_saved_state32(iss)) {
 799                 saved_state32(iss)->cpsr = cpsr;
 800         } else {
 801 #if defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC)
 802                 MANIPULATE_SIGNED_THREAD_STATE(iss,
 803                     "mov        w2, %w[cpsr]            \n"
 804                     "str        w2, [x0, %[SS64_CPSR]]  \n",
 805                     [cpsr] "r"(cpsr)
 806                     );
 807 #else
 808                 saved_state64(iss)->cpsr = cpsr;
 809 #endif /* defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC) */
 810         }
 811 }
 812
 813 static inline register_t
 814 get_saved_state_far(const arm_saved_state_t *iss)
 815 {
 816         return is_saved_state32(iss) ? const_saved_state32(iss)->far : const_saved_state64(iss)->far;
 817 }
 818
 819 static inline void
 820 set_saved_state_far(arm_saved_state_t *iss, register_t far)
 821 {
 822         if (is_saved_state32(iss)) {
 823                 saved_state32(iss)->far = CAST_ASSERT_SAFE(uint32_t, far);
 824         } else {
 825                 saved_state64(iss)->far = far;
 826         }
 827 }
 828
 829 static inline uint32_t
 830 get_saved_state_esr(const arm_saved_state_t *iss)
 831 {
 832         return is_saved_state32(iss) ? const_saved_state32(iss)->esr : const_saved_state64(iss)->esr;
 833 }
 834
 835 static inline void
 836 set_saved_state_esr(arm_saved_state_t *iss, uint32_t esr)
 837 {
 838         if (is_saved_state32(iss)) {
 839                 saved_state32(iss)->esr = esr;
 840         } else {
 841                 saved_state64(iss)->esr = esr;
 842         }
 843 }
 844
 845 static inline uint32_t
 846 get_saved_state_exc(const arm_saved_state_t *iss)
 847 {
 848         return is_saved_state32(iss) ? const_saved_state32(iss)->exception : const_saved_state64(iss)->exception;
 849 }
 850
 851 static inline void
 852 set_saved_state_exc(arm_saved_state_t *iss, uint32_t exc)
 853 {
 854         if (is_saved_state32(iss)) {
 855                 saved_state32(iss)->exception = exc;
 856         } else {
 857                 saved_state64(iss)->exception = exc;
 858         }
 859 }
 860
 861 extern void panic_unimplemented(void);
 862
 863 static inline int
 864 get_saved_state_svc_number(const arm_saved_state_t *iss)
 865 {
 866         return is_saved_state32(iss) ? (int)const_saved_state32(iss)->r[12] : (int)const_saved_state64(iss)->x[ARM64_SYSCALL_CODE_REG_NUM]; /* Only first word counts here */
 867 }
 868
 869 typedef _STRUCT_ARM_LEGACY_DEBUG_STATE arm_legacy_debug_state_t;
 870
 871 struct arm_debug_aggregate_state {
 872         arm_state_hdr_t dsh;
 873         union {
 874                 arm_debug_state32_t ds32;
 875                 arm_debug_state64_t ds64;
 876         } uds;
 877 } __attribute__((aligned(16)));
 878
 879 typedef struct arm_debug_aggregate_state arm_debug_state_t;
 880
 881 #define ARM_LEGACY_DEBUG_STATE_COUNT ((mach_msg_type_number_t) \
 882         (sizeof (arm_legacy_debug_state_t)/sizeof(uint32_t)))
 883
 884 /*
 885  * NEON context
 886  */
 887 typedef __uint128_t uint128_t;
 888 typedef uint64_t uint64x2_t __attribute__((ext_vector_type(2)));
 889 typedef uint32_t uint32x4_t __attribute__((ext_vector_type(4)));
 890
 891 struct arm_neon_saved_state32 {
 892         union {
 893                 uint128_t q[16];
 894                 uint64_t  d[32];
 895                 uint32_t  s[32];
 896         } v;
 897         uint32_t fpsr;
 898         uint32_t fpcr;
 899 };
 900 typedef struct arm_neon_saved_state32 arm_neon_saved_state32_t;
 901
 902 #define ARM_NEON_SAVED_STATE32_COUNT ((mach_msg_type_number_t) \
 903         (sizeof (arm_neon_saved_state32_t)/sizeof(unsigned int)))
 904
 905 struct arm_neon_saved_state64 {
 906         union {
 907                 uint128_t  q[32];
 908                 uint64x2_t d[32];
 909                 uint32x4_t s[32];
 910         } v;
 911         uint32_t fpsr;
 912         uint32_t fpcr;
 913 };
 914 typedef struct arm_neon_saved_state64 arm_neon_saved_state64_t;
 915
 916 #define ARM_NEON_SAVED_STATE64_COUNT ((mach_msg_type_number_t) \
 917         (sizeof (arm_neon_saved_state64_t)/sizeof(unsigned int)))
 918
 919 struct arm_neon_saved_state {
 920         arm_state_hdr_t nsh;
 921         union {
 922                 struct arm_neon_saved_state32 ns_32;
 923                 struct arm_neon_saved_state64 ns_64;
 924         } uns;
 925 };
 926 typedef struct arm_neon_saved_state arm_neon_saved_state_t;
 927 #define ns_32 uns.ns_32
 928 #define ns_64 uns.ns_64
 929
 930 static inline boolean_t
 931 is_neon_saved_state32(const arm_neon_saved_state_t *state)
 932 {
 933         return state->nsh.flavor == ARM_NEON_SAVED_STATE32;
 934 }
 935
 936 static inline boolean_t
 937 is_neon_saved_state64(const arm_neon_saved_state_t *state)
 938 {
 939         return state->nsh.flavor == ARM_NEON_SAVED_STATE64;
 940 }
 941
 942 static inline arm_neon_saved_state32_t *
 943 neon_state32(arm_neon_saved_state_t *state)
 944 {
 945         return &state->ns_32;
 946 }
 947
 948 static inline arm_neon_saved_state64_t *
 949 neon_state64(arm_neon_saved_state_t *state)
 950 {
 951         return &state->ns_64;
 952 }
 953
 954
 955 /*
 956  * Aggregated context
 957  */
 958
 959 struct arm_context {
 960         struct arm_saved_state ss;
 961         struct arm_neon_saved_state ns;
 962 };
 963 typedef struct arm_context arm_context_t;
 964
 965 extern void saved_state_to_thread_state64(const arm_saved_state_t*, arm_thread_state64_t*);
 966 extern void thread_state64_to_saved_state(const arm_thread_state64_t*, arm_saved_state_t*);
 967
 968 #else /* defined(__arm__) */
 969 #error Unknown arch
 970 #endif /* defined(__arm__) */
 971
 972 extern void saved_state_to_thread_state32(const arm_saved_state_t*, arm_thread_state32_t*);
 973 extern void thread_state32_to_saved_state(const arm_thread_state32_t*, arm_saved_state_t*);
 974
 975 #endif /* XNU_KERNEL_PRIVATE */
 976
 977 #endif /* _ARM_THREAD_STATUS_H_ */