osfmk/mach/arm/thread_status.h

   1 /*
   2  * Copyright (c) 2007 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 /*
  29  * FILE_ID: thread_status.h
  30  */
  31
  32
  33 #ifndef _ARM_THREAD_STATUS_H_
  34 #define _ARM_THREAD_STATUS_H_
  35
  36 #include <mach/machine/_structs.h>
  37 #include <mach/message.h>
  38 #include <mach/arm/thread_state.h>
  39
  40 /*
  41  *    Support for determining the state of a thread
  42  */
  43
  44
  45 /*
  46  *  Flavors
  47  */
  48
  49 #define ARM_THREAD_STATE         1
  50 #define ARM_UNIFIED_THREAD_STATE ARM_THREAD_STATE
  51 #define ARM_VFP_STATE            2
  52 #define ARM_EXCEPTION_STATE      3
  53 #define ARM_DEBUG_STATE          4 /* pre-armv8 */
  54 #define THREAD_STATE_NONE        5
  55 #define ARM_THREAD_STATE64       6
  56 #define ARM_EXCEPTION_STATE64    7
  57 //      ARM_THREAD_STATE_LAST    8 /* legacy */
  58 #define ARM_THREAD_STATE32       9
  59
  60 /* API */
  61 #define ARM_DEBUG_STATE32        14
  62 #define ARM_DEBUG_STATE64        15
  63 #define ARM_NEON_STATE           16
  64 #define ARM_NEON_STATE64         17
  65 #define ARM_CPMU_STATE64         18
  66
  67 #ifdef XNU_KERNEL_PRIVATE
  68 /* For kernel use */
  69 #define ARM_SAVED_STATE32        20
  70 #define ARM_SAVED_STATE64        21
  71 #define ARM_NEON_SAVED_STATE32   22
  72 #define ARM_NEON_SAVED_STATE64   23
  73 #endif /* XNU_KERNEL_PRIVATE */
  74
  75
  76 #define ARM_STATE_FLAVOR_IS_OTHER_VALID(_flavor_) 0
  77
  78 #define ARM_PAGEIN_STATE         27
  79
  80 #define VALID_THREAD_STATE_FLAVOR(x) \
  81         ((x == ARM_THREAD_STATE) ||           \
  82          (x == ARM_VFP_STATE) ||              \
  83          (x == ARM_EXCEPTION_STATE) ||        \
  84          (x == ARM_DEBUG_STATE) ||            \
  85          (x == THREAD_STATE_NONE) ||          \
  86          (x == ARM_THREAD_STATE32) ||         \
  87          (x == ARM_THREAD_STATE64) ||         \
  88          (x == ARM_EXCEPTION_STATE64) ||      \
  89          (x == ARM_NEON_STATE) ||             \
  90          (x == ARM_NEON_STATE64) ||           \
  91          (x == ARM_DEBUG_STATE32) ||          \
  92          (x == ARM_DEBUG_STATE64) ||          \
  93          (x == ARM_PAGEIN_STATE) ||           \
  94          (ARM_STATE_FLAVOR_IS_OTHER_VALID(x)))
  95
  96 struct arm_state_hdr {
  97         uint32_t flavor;
  98         uint32_t count;
  99 };
 100 typedef struct arm_state_hdr arm_state_hdr_t;
 101
 102 typedef _STRUCT_ARM_THREAD_STATE   arm_thread_state_t;
 103 typedef _STRUCT_ARM_THREAD_STATE   arm_thread_state32_t;
 104 typedef _STRUCT_ARM_THREAD_STATE64 arm_thread_state64_t;
 105
 106 #if !defined(KERNEL)
 107 #if __DARWIN_C_LEVEL >= __DARWIN_C_FULL && defined(__arm64__)
 108
 109 /* Accessor macros for arm_thread_state64_t pointer fields */
 110
 111 /* Return pc field of arm_thread_state64_t as a data pointer value */
 112 #define arm_thread_state64_get_pc(ts) \
 113                 __darwin_arm_thread_state64_get_pc(ts)
 114 /* Return pc field of arm_thread_state64_t as a function pointer. May return
 115  * NULL if a valid function pointer cannot be constructed, the caller should
 116  * fall back to the arm_thread_state64_get_pc() macro in that case. */
 117 #define arm_thread_state64_get_pc_fptr(ts) \
 118                 __darwin_arm_thread_state64_get_pc_fptr(ts)
 119 /* Set pc field of arm_thread_state64_t to a function pointer */
 120 #define arm_thread_state64_set_pc_fptr(ts, fptr) \
 121                 __darwin_arm_thread_state64_set_pc_fptr(ts, fptr)
 122 /* Return lr field of arm_thread_state64_t as a data pointer value */
 123 #define arm_thread_state64_get_lr(ts) \
 124                 __darwin_arm_thread_state64_get_lr(ts)
 125 /* Return lr field of arm_thread_state64_t as a function pointer. May return
 126  * NULL if a valid function pointer cannot be constructed, the caller should
 127  * fall back to the arm_thread_state64_get_lr() macro in that case. */
 128 #define arm_thread_state64_get_lr_fptr(ts) \
 129                 __darwin_arm_thread_state64_get_lr_fptr(ts)
 130 /* Set lr field of arm_thread_state64_t to a function pointer */
 131 #define arm_thread_state64_set_lr_fptr(ts, fptr) \
 132                 __darwin_arm_thread_state64_set_lr_fptr(ts, fptr)
 133 /* Return sp field of arm_thread_state64_t as a data pointer value */
 134 #define arm_thread_state64_get_sp(ts) \
 135                 __darwin_arm_thread_state64_get_sp(ts)
 136 /* Set sp field of arm_thread_state64_t to a data pointer value */
 137 #define arm_thread_state64_set_sp(ts, ptr) \
 138                 __darwin_arm_thread_state64_set_sp(ts, ptr)
 139 /* Return fp field of arm_thread_state64_t as a data pointer value */
 140 #define arm_thread_state64_get_fp(ts) \
 141                 __darwin_arm_thread_state64_get_fp(ts)
 142 /* Set fp field of arm_thread_state64_t to a data pointer value */
 143 #define arm_thread_state64_set_fp(ts, ptr) \
 144                 __darwin_arm_thread_state64_set_fp(ts, ptr)
 145
 146 #endif /* __DARWIN_C_LEVEL >= __DARWIN_C_FULL && defined(__arm64__) */
 147 #endif /* !defined(KERNEL) */
 148
 149 struct arm_unified_thread_state {
 150         arm_state_hdr_t ash;
 151         union {
 152                 arm_thread_state32_t ts_32;
 153                 arm_thread_state64_t ts_64;
 154         } uts;
 155 };
 156 #define ts_32 uts.ts_32
 157 #define ts_64 uts.ts_64
 158 typedef struct arm_unified_thread_state arm_unified_thread_state_t;
 159
 160 #define ARM_THREAD_STATE_COUNT ((mach_msg_type_number_t) \
 161         (sizeof (arm_thread_state_t)/sizeof(uint32_t)))
 162 #define ARM_THREAD_STATE32_COUNT ((mach_msg_type_number_t) \
 163         (sizeof (arm_thread_state32_t)/sizeof(uint32_t)))
 164 #define ARM_THREAD_STATE64_COUNT ((mach_msg_type_number_t) \
 165         (sizeof (arm_thread_state64_t)/sizeof(uint32_t)))
 166 #define ARM_UNIFIED_THREAD_STATE_COUNT ((mach_msg_type_number_t) \
 167         (sizeof (arm_unified_thread_state_t)/sizeof(uint32_t)))
 168
 169
 170 typedef _STRUCT_ARM_VFP_STATE         arm_vfp_state_t;
 171 typedef _STRUCT_ARM_NEON_STATE        arm_neon_state_t;
 172 typedef _STRUCT_ARM_NEON_STATE        arm_neon_state32_t;
 173 typedef _STRUCT_ARM_NEON_STATE64      arm_neon_state64_t;
 174
 175
 176 typedef _STRUCT_ARM_EXCEPTION_STATE   arm_exception_state_t;
 177 typedef _STRUCT_ARM_EXCEPTION_STATE   arm_exception_state32_t;
 178 typedef _STRUCT_ARM_EXCEPTION_STATE64 arm_exception_state64_t;
 179
 180 typedef _STRUCT_ARM_DEBUG_STATE32     arm_debug_state32_t;
 181 typedef _STRUCT_ARM_DEBUG_STATE64     arm_debug_state64_t;
 182
 183 typedef _STRUCT_ARM_PAGEIN_STATE      arm_pagein_state_t;
 184
 185 #if defined(XNU_KERNEL_PRIVATE) && defined(__arm64__)
 186 /* See below for ARM64 kernel structure definition for arm_debug_state. */
 187 #else /* defined(XNU_KERNEL_PRIVATE) && defined(__arm64__) */
 188 /*
 189  * Otherwise not ARM64 kernel and we must preserve legacy ARM definitions of
 190  * arm_debug_state for binary compatability of userland consumers of this file.
 191  */
 192 #if defined(__arm__)
 193 typedef _STRUCT_ARM_DEBUG_STATE        arm_debug_state_t;
 194 #elif defined(__arm64__)
 195 typedef _STRUCT_ARM_LEGACY_DEBUG_STATE arm_debug_state_t;
 196 #else /* defined(__arm__) */
 197 #error Undefined architecture
 198 #endif /* defined(__arm__) */
 199 #endif /* defined(XNU_KERNEL_PRIVATE) && defined(__arm64__) */
 200
 201 #define ARM_VFP_STATE_COUNT ((mach_msg_type_number_t) \
 202         (sizeof (arm_vfp_state_t)/sizeof(uint32_t)))
 203
 204 #define ARM_EXCEPTION_STATE_COUNT ((mach_msg_type_number_t) \
 205         (sizeof (arm_exception_state_t)/sizeof(uint32_t)))
 206
 207 #define ARM_EXCEPTION_STATE64_COUNT ((mach_msg_type_number_t) \
 208         (sizeof (arm_exception_state64_t)/sizeof(uint32_t)))
 209
 210 #define ARM_DEBUG_STATE_COUNT ((mach_msg_type_number_t) \
 211         (sizeof (arm_debug_state_t)/sizeof(uint32_t)))
 212
 213 #define ARM_DEBUG_STATE32_COUNT ((mach_msg_type_number_t) \
 214         (sizeof (arm_debug_state32_t)/sizeof(uint32_t)))
 215
 216 #define ARM_PAGEIN_STATE_COUNT ((mach_msg_type_number_t) \
 217         (sizeof (arm_pagein_state_t)/sizeof(uint32_t)))
 218
 219 #define ARM_DEBUG_STATE64_COUNT ((mach_msg_type_number_t) \
 220         (sizeof (arm_debug_state64_t)/sizeof(uint32_t)))
 221
 222 #define ARM_NEON_STATE_COUNT ((mach_msg_type_number_t) \
 223         (sizeof (arm_neon_state_t)/sizeof(uint32_t)))
 224
 225 #define ARM_NEON_STATE64_COUNT ((mach_msg_type_number_t) \
 226         (sizeof (arm_neon_state64_t)/sizeof(uint32_t)))
 227
 228 #define MACHINE_THREAD_STATE       ARM_THREAD_STATE
 229 #define MACHINE_THREAD_STATE_COUNT ARM_UNIFIED_THREAD_STATE_COUNT
 230
 231
 232 /*
 233  * Largest state on this machine:
 234  */
 235 #define THREAD_MACHINE_STATE_MAX THREAD_STATE_MAX
 236
 237 #ifdef XNU_KERNEL_PRIVATE
 238
 239 static inline boolean_t
 240 is_thread_state32(const arm_unified_thread_state_t *its)
 241 {
 242         return its->ash.flavor == ARM_THREAD_STATE32;
 243 }
 244
 245 static inline boolean_t
 246 is_thread_state64(const arm_unified_thread_state_t *its)
 247 {
 248         return its->ash.flavor == ARM_THREAD_STATE64;
 249 }
 250
 251 static inline arm_thread_state32_t*
 252 thread_state32(arm_unified_thread_state_t *its)
 253 {
 254         return &its->ts_32;
 255 }
 256
 257 static inline arm_thread_state64_t*
 258 thread_state64(arm_unified_thread_state_t *its)
 259 {
 260         return &its->ts_64;
 261 }
 262
 263 static inline const arm_thread_state32_t*
 264 const_thread_state32(const arm_unified_thread_state_t *its)
 265 {
 266         return &its->ts_32;
 267 }
 268
 269 static inline const arm_thread_state64_t*
 270 const_thread_state64(const arm_unified_thread_state_t *its)
 271 {
 272         return &its->ts_64;
 273 }
 274
 275 #if defined(__arm__)
 276 #include <arm/proc_reg.h>
 277
 278 #define ARM_SAVED_STATE (THREAD_STATE_NONE + 1)
 279
 280 struct arm_saved_state {
 281         uint32_t r[13];     /* General purpose register r0-r12 */
 282         uint32_t sp;        /* Stack pointer r13 */
 283         uint32_t lr;        /* Link register r14 */
 284         uint32_t pc;        /* Program counter r15 */
 285         uint32_t cpsr;      /* Current program status register */
 286         uint32_t fsr;       /* Fault status */
 287         uint32_t far;       /* Virtual Fault Address */
 288         uint32_t exception; /* exception number */
 289 };
 290 typedef struct arm_saved_state arm_saved_state_t;
 291
 292 /*
 293  * Just for coexistence with AArch64 code.
 294  */
 295 typedef struct arm_saved_state arm_saved_state32_t;
 296
 297 static inline void
 298 copy_signed_thread_state(arm_saved_state_t *dst, const arm_saved_state_t *src)
 299 {
 300         *dst = *src;
 301 }
 302
 303 static inline arm_saved_state32_t*
 304 saved_state32(arm_saved_state_t *iss)
 305 {
 306         return iss;
 307 }
 308
 309 static inline boolean_t
 310 is_saved_state32(const arm_saved_state_t *iss __unused)
 311 {
 312         return TRUE;
 313 }
 314
 315
 316 struct arm_saved_state_tagged {
 317         uint32_t               tag;
 318         struct arm_saved_state state;
 319 };
 320 typedef struct arm_saved_state_tagged arm_saved_state_tagged_t;
 321
 322 #define ARM_SAVED_STATE32_COUNT ((mach_msg_type_number_t) \
 323         (sizeof (arm_saved_state_t)/sizeof(unsigned int)))
 324
 325
 326 static inline register_t
 327 get_saved_state_pc(const arm_saved_state_t *iss)
 328 {
 329         return iss->pc;
 330 }
 331
 332 static inline void
 333 add_saved_state_pc(arm_saved_state_t *iss, int diff)
 334 {
 335         iss->pc += diff;
 336 }
 337
 338 static inline void
 339 set_saved_state_pc(arm_saved_state_t *iss, register_t pc)
 340 {
 341         iss->pc = pc;
 342 }
 343
 344 static inline register_t
 345 get_saved_state_sp(const arm_saved_state_t *iss)
 346 {
 347         return iss->sp;
 348 }
 349
 350 static inline void
 351 set_saved_state_sp(arm_saved_state_t *iss, register_t sp)
 352 {
 353         iss->sp = sp;
 354 }
 355
 356 static inline register_t
 357 get_saved_state_fp(const arm_saved_state_t *iss)
 358 {
 359         return iss->r[7];
 360 }
 361
 362 static inline void
 363 set_saved_state_fp(arm_saved_state_t *iss, register_t fp)
 364 {
 365         iss->r[7] = fp;
 366 }
 367
 368 static inline register_t
 369 get_saved_state_lr(const arm_saved_state_t *iss)
 370 {
 371         return iss->lr;
 372 }
 373
 374 static inline void
 375 set_saved_state_lr(arm_saved_state_t *iss, register_t lr)
 376 {
 377         iss->lr = lr;
 378 }
 379
 380 static inline register_t
 381 get_saved_state_cpsr(const arm_saved_state_t *iss)
 382 {
 383         return iss->cpsr;
 384 }
 385
 386 static inline void
 387 mask_saved_state_cpsr(arm_saved_state_t *iss, uint32_t set_bits, uint32_t clear_bits)
 388 {
 389         iss->cpsr |= set_bits;
 390         iss->cpsr &= clear_bits;
 391 }
 392
 393 static inline void
 394 set_saved_state_cpsr(arm_saved_state_t *iss, register_t cpsr)
 395 {
 396         iss->cpsr = cpsr;
 397 }
 398
 399 static inline register_t
 400 get_saved_state_reg(const arm_saved_state_t *iss, unsigned regno)
 401 {
 402         return iss->r[regno];
 403 }
 404
 405 static inline void
 406 set_saved_state_reg(arm_saved_state_t *iss, unsigned regno, register_t val)
 407 {
 408         iss->r[regno] = val;
 409 }
 410
 411 #elif defined(__arm64__)
 412
 413 #include <kern/assert.h>
 414 #include <arm64/proc_reg.h>
 415 #define CAST_ASSERT_SAFE(type, val) (assert((val) == ((type)(val))), (type)(val))
 416
 417 /*
 418  * GPR context
 419  */
 420
 421 struct arm_saved_state32 {
 422         uint32_t r[13];     /* General purpose register r0-r12 */
 423         uint32_t sp;        /* Stack pointer r13 */
 424         uint32_t lr;        /* Link register r14 */
 425         uint32_t pc;        /* Program counter r15 */
 426         uint32_t cpsr;      /* Current program status register */
 427         uint32_t far;       /* Virtual fault address */
 428         uint32_t esr;       /* Exception syndrome register */
 429         uint32_t exception; /* Exception number */
 430 };
 431 typedef struct arm_saved_state32 arm_saved_state32_t;
 432
 433 struct arm_saved_state32_tagged {
 434         uint32_t                 tag;
 435         struct arm_saved_state32 state;
 436 };
 437 typedef struct arm_saved_state32_tagged arm_saved_state32_tagged_t;
 438
 439 #define ARM_SAVED_STATE32_COUNT ((mach_msg_type_number_t) \
 440                 (sizeof(arm_saved_state32_t)/sizeof(unsigned int)))
 441
 442 struct arm_saved_state64 {
 443         uint64_t x[29];     /* General purpose registers x0-x28 */
 444         uint64_t fp;        /* Frame pointer x29 */
 445         uint64_t lr;        /* Link register x30 */
 446         uint64_t sp;        /* Stack pointer x31 */
 447         uint64_t pc;        /* Program counter */
 448         uint32_t cpsr;      /* Current program status register */
 449         uint32_t reserved;  /* Reserved padding */
 450         uint64_t far;       /* Virtual fault address */
 451         uint32_t esr;       /* Exception syndrome register */
 452         uint32_t exception; /* Exception number */
 453 #if defined(HAS_APPLE_PAC)
 454         uint64_t jophash;
 455 #endif /* defined(HAS_APPLE_PAC) */
 456 };
 457 typedef struct arm_saved_state64 arm_saved_state64_t;
 458
 459 #define ARM_SAVED_STATE64_COUNT ((mach_msg_type_number_t) \
 460         (sizeof(arm_saved_state64_t)/sizeof(unsigned int)))
 461
 462 struct arm_saved_state64_tagged {
 463         uint32_t                 tag;
 464         struct arm_saved_state64 state;
 465 };
 466 typedef struct arm_saved_state64_tagged arm_saved_state64_tagged_t;
 467
 468 struct arm_saved_state {
 469         arm_state_hdr_t ash;
 470         union {
 471                 struct arm_saved_state32 ss_32;
 472                 struct arm_saved_state64 ss_64;
 473         } uss;
 474 } __attribute__((aligned(16)));
 475 #define ss_32 uss.ss_32
 476 #define ss_64 uss.ss_64
 477
 478 typedef struct arm_saved_state arm_saved_state_t;
 479
 480 #if defined(XNU_KERNEL_PRIVATE)
 481 #if defined(HAS_APPLE_PAC)
 482 /*
 483  * Methods used to sign and check thread state to detect corruptions of saved
 484  * thread state across exceptions and context switches.
 485  */
 486 extern void ml_sign_thread_state(arm_saved_state_t *, uint64_t, uint32_t, uint64_t, uint64_t, uint64_t);
 487
 488 extern void ml_check_signed_state(const arm_saved_state_t *, uint64_t, uint32_t, uint64_t, uint64_t, uint64_t);
 489
 490 /* XXX: including stddef.f here breaks ctfmerge on some builds, so use __builtin_offsetof() instead of offsetof() */
 491 #define ss64_offsetof(x) __builtin_offsetof(struct arm_saved_state, ss_64.x)
 492
 493 /**
 494  * Verify the signed thread state in _iss, execute the assembly instructions
 495  * _instr, and re-sign the modified thread state.  Varargs specify additional
 496  * inputs.
 497  *
 498  * _instr may read or modify the thread state in the following registers:
 499  *
 500  * x0: _iss
 501  * x1: authed _iss->ss_64.pc
 502  * w2: authed _iss->ss_64.cpsr
 503  * x3: authed _iss->ss_64.lr
 504  * x4: authed _iss->ss_64.x16
 505  * x5: authed _iss->ss_64.x17
 506  * x6: scratch register
 507  * x7: scratch register
 508  */
 509 #define MANIPULATE_SIGNED_THREAD_STATE(_iss, _instr, ...)               \
 510         asm volatile (                                                  \
 511                 "mov    x8, lr"                         "\n"            \
 512                 "mov    x0, %[iss]"                     "\n"            \
 513                 "ldp    x4, x5, [x0, %[SS64_X16]]"      "\n"            \
 514                 "ldr    x6, [x0, %[SS64_PC]]"           "\n"            \
 515                 "ldr    w7, [x0, %[SS64_CPSR]]"         "\n"            \
 516                 "ldr    x3, [x0, %[SS64_LR]]"           "\n"            \
 517                 "mov    x1, x6"                         "\n"            \
 518                 "mov    w2, w7"                         "\n"            \
 519                 "bl     _ml_check_signed_state"         "\n"            \
 520                 "mov    x1, x6"                         "\n"            \
 521                 "mov    w2, w7"                         "\n"            \
 522                 _instr                                  "\n"            \
 523                 "bl     _ml_sign_thread_state"          "\n"            \
 524                 "mov    lr, x8"                         "\n"            \
 525                 :                                                       \
 526                 : [iss]         "r"(_iss),                              \
 527                   [SS64_X16]    "i"(ss64_offsetof(x[16])),              \
 528                   [SS64_PC]     "i"(ss64_offsetof(pc)),                 \
 529                   [SS64_CPSR]   "i"(ss64_offsetof(cpsr)),               \
 530                   [SS64_LR]     "i"(ss64_offsetof(lr)),##__VA_ARGS__    \
 531                 : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8"  \
 532         )
 533
 534 static inline void
 535 check_and_sign_copied_thread_state(arm_saved_state_t *dst, const arm_saved_state_t *src)
 536 {
 537         MANIPULATE_SIGNED_THREAD_STATE(src,
 538             "mov        x0, %[dst]",
 539             [dst] "r"(dst)
 540             );
 541 }
 542 #endif /* defined(HAS_APPLE_PAC) */
 543
 544 static inline void
 545 copy_signed_thread_state(arm_saved_state_t *dst, const arm_saved_state_t *src)
 546 {
 547         *dst = *src;
 548 #if defined(HAS_APPLE_PAC)
 549         check_and_sign_copied_thread_state(dst, src);
 550 #endif
 551 }
 552
 553 #endif /* defined(XNU_KERNEL_PRIVATE) */
 554
 555 static inline boolean_t
 556 is_saved_state32(const arm_saved_state_t *iss)
 557 {
 558         return iss->ash.flavor == ARM_SAVED_STATE32;
 559 }
 560
 561 static inline boolean_t
 562 is_saved_state64(const arm_saved_state_t *iss)
 563 {
 564         return iss->ash.flavor == ARM_SAVED_STATE64;
 565 }
 566
 567 static inline arm_saved_state32_t*
 568 saved_state32(arm_saved_state_t *iss)
 569 {
 570         return &iss->ss_32;
 571 }
 572
 573 static inline const arm_saved_state32_t*
 574 const_saved_state32(const arm_saved_state_t *iss)
 575 {
 576         return &iss->ss_32;
 577 }
 578
 579 static inline arm_saved_state64_t*
 580 saved_state64(arm_saved_state_t *iss)
 581 {
 582         return &iss->ss_64;
 583 }
 584
 585 static inline const arm_saved_state64_t*
 586 const_saved_state64(const arm_saved_state_t *iss)
 587 {
 588         return &iss->ss_64;
 589 }
 590
 591 static inline register_t
 592 get_saved_state_pc(const arm_saved_state_t *iss)
 593 {
 594         return is_saved_state32(iss) ? const_saved_state32(iss)->pc : const_saved_state64(iss)->pc;
 595 }
 596
 597 static inline void
 598 add_saved_state_pc(arm_saved_state_t *iss, int diff)
 599 {
 600         if (is_saved_state32(iss)) {
 601                 uint64_t pc = saved_state32(iss)->pc + diff;
 602                 saved_state32(iss)->pc = CAST_ASSERT_SAFE(uint32_t, pc);
 603         } else {
 604 #if defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC)
 605                 MANIPULATE_SIGNED_THREAD_STATE(iss,
 606                     "mov        w6, %w[diff]            \n"
 607                     "add        x1, x1, w6, sxtw        \n"
 608                     "str        x1, [x0, %[SS64_PC]]    \n",
 609                     [diff] "r"(diff)
 610                     );
 611 #else
 612                 saved_state64(iss)->pc += diff;
 613 #endif /* defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC) */
 614         }
 615 }
 616
 617 static inline void
 618 set_saved_state_pc(arm_saved_state_t *iss, register_t pc)
 619 {
 620         if (is_saved_state32(iss)) {
 621                 saved_state32(iss)->pc = CAST_ASSERT_SAFE(uint32_t, pc);
 622         } else {
 623 #if defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC)
 624                 MANIPULATE_SIGNED_THREAD_STATE(iss,
 625                     "mov        x1, %[pc]               \n"
 626                     "str        x1, [x0, %[SS64_PC]]    \n",
 627                     [pc] "r"(pc)
 628                     );
 629 #else
 630                 saved_state64(iss)->pc = pc;
 631 #endif /* defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC) */
 632         }
 633 }
 634
 635 static inline register_t
 636 get_saved_state_sp(const arm_saved_state_t *iss)
 637 {
 638         return is_saved_state32(iss) ? const_saved_state32(iss)->sp : const_saved_state64(iss)->sp;
 639 }
 640
 641 static inline void
 642 set_saved_state_sp(arm_saved_state_t *iss, register_t sp)
 643 {
 644         if (is_saved_state32(iss)) {
 645                 saved_state32(iss)->sp = CAST_ASSERT_SAFE(uint32_t, sp);
 646         } else {
 647                 saved_state64(iss)->sp = sp;
 648         }
 649 }
 650
 651 static inline register_t
 652 get_saved_state_lr(const arm_saved_state_t *iss)
 653 {
 654         return is_saved_state32(iss) ? const_saved_state32(iss)->lr : const_saved_state64(iss)->lr;
 655 }
 656
 657 static inline void
 658 set_saved_state_lr(arm_saved_state_t *iss, register_t lr)
 659 {
 660         if (is_saved_state32(iss)) {
 661                 saved_state32(iss)->lr = CAST_ASSERT_SAFE(uint32_t, lr);
 662         } else {
 663 #if defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC)
 664                 MANIPULATE_SIGNED_THREAD_STATE(iss,
 665                     "mov        x3, %[lr]               \n"
 666                     "str        x3, [x0, %[SS64_LR]]    \n",
 667                     [lr] "r"(lr)
 668                     );
 669 #else
 670                 saved_state64(iss)->lr = lr;
 671 #endif /* defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC) */
 672         }
 673 }
 674
 675 static inline register_t
 676 get_saved_state_fp(const arm_saved_state_t *iss)
 677 {
 678         return is_saved_state32(iss) ? const_saved_state32(iss)->r[7] : const_saved_state64(iss)->fp;
 679 }
 680
 681 static inline void
 682 set_saved_state_fp(arm_saved_state_t *iss, register_t fp)
 683 {
 684         if (is_saved_state32(iss)) {
 685                 saved_state32(iss)->r[7] = CAST_ASSERT_SAFE(uint32_t, fp);
 686         } else {
 687                 saved_state64(iss)->fp = fp;
 688         }
 689 }
 690
 691 static inline int
 692 check_saved_state_reglimit(const arm_saved_state_t *iss, unsigned reg)
 693 {
 694         return is_saved_state32(iss) ? (reg < ARM_SAVED_STATE32_COUNT) : (reg < ARM_SAVED_STATE64_COUNT);
 695 }
 696
 697 static inline register_t
 698 get_saved_state_reg(const arm_saved_state_t *iss, unsigned reg)
 699 {
 700         if (!check_saved_state_reglimit(iss, reg)) {
 701                 return 0;
 702         }
 703
 704         return is_saved_state32(iss) ? (const_saved_state32(iss)->r[reg]) : (const_saved_state64(iss)->x[reg]);
 705 }
 706
 707 static inline void
 708 set_saved_state_reg(arm_saved_state_t *iss, unsigned reg, register_t value)
 709 {
 710         if (!check_saved_state_reglimit(iss, reg)) {
 711                 return;
 712         }
 713
 714         if (is_saved_state32(iss)) {
 715                 saved_state32(iss)->r[reg] = CAST_ASSERT_SAFE(uint32_t, value);
 716         } else {
 717 #if defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC)
 718                 /* x16 and x17 are part of the jophash */
 719                 if (reg == 16) {
 720                         MANIPULATE_SIGNED_THREAD_STATE(iss,
 721                             "mov        x4, %[value]            \n"
 722                             "str        x4, [x0, %[SS64_X16]]   \n",
 723                             [value] "r"(value)
 724                             );
 725                         return;
 726                 } else if (reg == 17) {
 727                         MANIPULATE_SIGNED_THREAD_STATE(iss,
 728                             "mov        x5, %[value]            \n"
 729                             "str        x5, [x0, %[SS64_X17]]   \n",
 730                             [value] "r"(value),
 731                             [SS64_X17] "i"(ss64_offsetof(x[17]))
 732                             );
 733                         return;
 734                 }
 735 #endif
 736                 saved_state64(iss)->x[reg] = value;
 737         }
 738 }
 739
 740 static inline uint32_t
 741 get_saved_state_cpsr(const arm_saved_state_t *iss)
 742 {
 743         return is_saved_state32(iss) ? const_saved_state32(iss)->cpsr : const_saved_state64(iss)->cpsr;
 744 }
 745
 746 static inline void
 747 mask_saved_state_cpsr(arm_saved_state_t *iss, uint32_t set_bits, uint32_t clear_bits)
 748 {
 749         if (is_saved_state32(iss)) {
 750                 saved_state32(iss)->cpsr |= set_bits;
 751                 saved_state32(iss)->cpsr &= ~clear_bits;
 752         } else {
 753 #if defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC)
 754                 MANIPULATE_SIGNED_THREAD_STATE(iss,
 755                     "mov        w6, %w[set_bits]        \n"
 756                     "orr        w2, w2, w6, lsl #0      \n"
 757                     "mov        w6, %w[clear_bits]      \n"
 758                     "bic        w2, w2, w6, lsl #0      \n"
 759                     "str        w2, [x0, %[SS64_CPSR]]  \n",
 760                     [set_bits] "r"(set_bits),
 761                     [clear_bits] "r"(clear_bits)
 762                     );
 763 #else
 764                 saved_state64(iss)->cpsr |= set_bits;
 765                 saved_state64(iss)->cpsr &= ~clear_bits;
 766 #endif /* defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC) */
 767         }
 768 }
 769
 770 static inline void
 771 set_saved_state_cpsr(arm_saved_state_t *iss, uint32_t cpsr)
 772 {
 773         if (is_saved_state32(iss)) {
 774                 saved_state32(iss)->cpsr = cpsr;
 775         } else {
 776 #if defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC)
 777                 MANIPULATE_SIGNED_THREAD_STATE(iss,
 778                     "mov        w2, %w[cpsr]            \n"
 779                     "str        w2, [x0, %[SS64_CPSR]]  \n",
 780                     [cpsr] "r"(cpsr)
 781                     );
 782 #else
 783                 saved_state64(iss)->cpsr = cpsr;
 784 #endif /* defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC) */
 785         }
 786 }
 787
 788 static inline register_t
 789 get_saved_state_far(const arm_saved_state_t *iss)
 790 {
 791         return is_saved_state32(iss) ? const_saved_state32(iss)->far : const_saved_state64(iss)->far;
 792 }
 793
 794 static inline void
 795 set_saved_state_far(arm_saved_state_t *iss, register_t far)
 796 {
 797         if (is_saved_state32(iss)) {
 798                 saved_state32(iss)->far = CAST_ASSERT_SAFE(uint32_t, far);
 799         } else {
 800                 saved_state64(iss)->far = far;
 801         }
 802 }
 803
 804 static inline uint32_t
 805 get_saved_state_esr(const arm_saved_state_t *iss)
 806 {
 807         return is_saved_state32(iss) ? const_saved_state32(iss)->esr : const_saved_state64(iss)->esr;
 808 }
 809
 810 static inline void
 811 set_saved_state_esr(arm_saved_state_t *iss, uint32_t esr)
 812 {
 813         if (is_saved_state32(iss)) {
 814                 saved_state32(iss)->esr = esr;
 815         } else {
 816                 saved_state64(iss)->esr = esr;
 817         }
 818 }
 819
 820 static inline uint32_t
 821 get_saved_state_exc(const arm_saved_state_t *iss)
 822 {
 823         return is_saved_state32(iss) ? const_saved_state32(iss)->exception : const_saved_state64(iss)->exception;
 824 }
 825
 826 static inline void
 827 set_saved_state_exc(arm_saved_state_t *iss, uint32_t exc)
 828 {
 829         if (is_saved_state32(iss)) {
 830                 saved_state32(iss)->exception = exc;
 831         } else {
 832                 saved_state64(iss)->exception = exc;
 833         }
 834 }
 835
 836 extern void panic_unimplemented(void);
 837
 838 static inline int
 839 get_saved_state_svc_number(const arm_saved_state_t *iss)
 840 {
 841         return is_saved_state32(iss) ? (int)const_saved_state32(iss)->r[12] : (int)const_saved_state64(iss)->x[ARM64_SYSCALL_CODE_REG_NUM]; /* Only first word counts here */
 842 }
 843
 844 typedef _STRUCT_ARM_LEGACY_DEBUG_STATE arm_legacy_debug_state_t;
 845
 846 struct arm_debug_aggregate_state {
 847         arm_state_hdr_t dsh;
 848         union {
 849                 arm_debug_state32_t ds32;
 850                 arm_debug_state64_t ds64;
 851         } uds;
 852 } __attribute__((aligned(16)));
 853
 854 typedef struct arm_debug_aggregate_state arm_debug_state_t;
 855
 856 #define ARM_LEGACY_DEBUG_STATE_COUNT ((mach_msg_type_number_t) \
 857         (sizeof (arm_legacy_debug_state_t)/sizeof(uint32_t)))
 858
 859 /*
 860  * NEON context
 861  */
 862 typedef __uint128_t uint128_t;
 863 typedef uint64_t uint64x2_t __attribute__((ext_vector_type(2)));
 864 typedef uint32_t uint32x4_t __attribute__((ext_vector_type(4)));
 865
 866 struct arm_neon_saved_state32 {
 867         union {
 868                 uint128_t q[16];
 869                 uint64_t  d[32];
 870                 uint32_t  s[32];
 871         } v;
 872         uint32_t fpsr;
 873         uint32_t fpcr;
 874 };
 875 typedef struct arm_neon_saved_state32 arm_neon_saved_state32_t;
 876
 877 #define ARM_NEON_SAVED_STATE32_COUNT ((mach_msg_type_number_t) \
 878         (sizeof (arm_neon_saved_state32_t)/sizeof(unsigned int)))
 879
 880 struct arm_neon_saved_state64 {
 881         union {
 882                 uint128_t  q[32];
 883                 uint64x2_t d[32];
 884                 uint32x4_t s[32];
 885         } v;
 886         uint32_t fpsr;
 887         uint32_t fpcr;
 888 };
 889 typedef struct arm_neon_saved_state64 arm_neon_saved_state64_t;
 890
 891 #define ARM_NEON_SAVED_STATE64_COUNT ((mach_msg_type_number_t) \
 892         (sizeof (arm_neon_saved_state64_t)/sizeof(unsigned int)))
 893
 894 struct arm_neon_saved_state {
 895         arm_state_hdr_t nsh;
 896         union {
 897                 struct arm_neon_saved_state32 ns_32;
 898                 struct arm_neon_saved_state64 ns_64;
 899         } uns;
 900 };
 901 typedef struct arm_neon_saved_state arm_neon_saved_state_t;
 902 #define ns_32 uns.ns_32
 903 #define ns_64 uns.ns_64
 904
 905 static inline boolean_t
 906 is_neon_saved_state32(const arm_neon_saved_state_t *state)
 907 {
 908         return state->nsh.flavor == ARM_NEON_SAVED_STATE32;
 909 }
 910
 911 static inline boolean_t
 912 is_neon_saved_state64(const arm_neon_saved_state_t *state)
 913 {
 914         return state->nsh.flavor == ARM_NEON_SAVED_STATE64;
 915 }
 916
 917 static inline arm_neon_saved_state32_t *
 918 neon_state32(arm_neon_saved_state_t *state)
 919 {
 920         return &state->ns_32;
 921 }
 922
 923 static inline arm_neon_saved_state64_t *
 924 neon_state64(arm_neon_saved_state_t *state)
 925 {
 926         return &state->ns_64;
 927 }
 928
 929
 930 /*
 931  * Aggregated context
 932  */
 933
 934 struct arm_context {
 935         struct arm_saved_state ss;
 936         struct arm_neon_saved_state ns;
 937 };
 938 typedef struct arm_context arm_context_t;
 939
 940 extern void saved_state_to_thread_state64(const arm_saved_state_t*, arm_thread_state64_t*);
 941 extern void thread_state64_to_saved_state(const arm_thread_state64_t*, arm_saved_state_t*);
 942
 943 #else /* defined(__arm__) */
 944 #error Unknown arch
 945 #endif /* defined(__arm__) */
 946
 947 extern void saved_state_to_thread_state32(const arm_saved_state_t*, arm_thread_state32_t*);
 948 extern void thread_state32_to_saved_state(const arm_thread_state32_t*, arm_saved_state_t*);
 949
 950 #endif /* XNU_KERNEL_PRIVATE */
 951
 952 #endif /* _ARM_THREAD_STATUS_H_ */