osfmk/arm/cpu_data_internal.h

   1 /*
   2  * Copyright (c) 2007 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 /*
  29  * @OSF_COPYRIGHT@
  30  *
  31  */
  32
  33 #ifndef ARM_CPU_DATA_INTERNAL
  34 #define ARM_CPU_DATA_INTERNAL
  35
  36 #include <mach_assert.h>
  37 #include <kern/assert.h>
  38 #include <kern/kern_types.h>
  39 #include <kern/percpu.h>
  40 #include <kern/processor.h>
  41 #include <pexpert/pexpert.h>
  42 #include <arm/dbgwrap.h>
  43 #include <arm/machine_routines.h>
  44 #include <arm/proc_reg.h>
  45 #include <arm/thread.h>
  46 #include <arm/pmap.h>
  47
  48 #if MONOTONIC
  49 #include <machine/monotonic.h>
  50 #endif /* MONOTONIC */
  51
  52 #define NSEC_PER_HZ     (NSEC_PER_SEC / 100)
  53
  54 typedef struct reset_handler_data {
  55         vm_offset_t     assist_reset_handler;           /* Assist handler phys address */
  56         vm_offset_t     cpu_data_entries;                       /* CpuDataEntries phys address */
  57 #if !__arm64__
  58         vm_offset_t     boot_args;                                      /* BootArgs phys address */
  59 #endif
  60 } reset_handler_data_t;
  61
  62 extern  reset_handler_data_t    ResetHandlerData;
  63
  64 /* Put the static check for cpumap_t here as it's defined in <kern/processor.h> */
  65 static_assert(sizeof(cpumap_t) * CHAR_BIT >= MAX_CPUS, "cpumap_t bitvector is too small for current MAX_CPUS value");
  66
  67 #ifdef  __arm__
  68 #define CPUWINDOWS_BASE_MASK            0xFFF00000UL
  69 #else
  70 #define CPUWINDOWS_BASE_MASK            0xFFFFFFFFFFE00000UL
  71 #endif
  72 #define CPUWINDOWS_BASE                 (VM_MAX_KERNEL_ADDRESS & CPUWINDOWS_BASE_MASK)
  73 #define CPUWINDOWS_TOP                  (CPUWINDOWS_BASE + (MAX_CPUS * CPUWINDOWS_MAX * ARM_PGBYTES))
  74
  75 static_assert((CPUWINDOWS_BASE >= VM_MIN_KERNEL_ADDRESS) && (CPUWINDOWS_TOP <= VM_MAX_KERNEL_ADDRESS),
  76     "CPU copy windows too large for CPUWINDOWS_BASE_MASK value");
  77
  78 typedef struct cpu_data_entry {
  79         void                           *cpu_data_paddr;         /* Cpu data physical address */
  80         struct  cpu_data               *cpu_data_vaddr;         /* Cpu data virtual address */
  81 #if __arm__
  82         uint32_t                        cpu_data_offset_8;
  83         uint32_t                        cpu_data_offset_12;
  84 #elif __arm64__
  85 #else
  86 #error Check cpu_data_entry padding for this architecture
  87 #endif
  88 } cpu_data_entry_t;
  89
  90
  91 typedef struct rtclock_timer {
  92         mpqueue_head_t                  queue;
  93         uint64_t                        deadline;
  94         uint32_t                        is_set:1,
  95             has_expired:1,
  96         :0;
  97 } rtclock_timer_t;
  98
  99 typedef struct {
 100         /*
 101          * The wake variants of these counters are reset to 0 when the CPU wakes.
 102          */
 103         uint64_t irq_ex_cnt;
 104         uint64_t irq_ex_cnt_wake;
 105         uint64_t ipi_cnt;
 106         uint64_t ipi_cnt_wake;
 107         uint64_t timer_cnt;
 108 #if MONOTONIC
 109         uint64_t pmi_cnt_wake;
 110 #endif /* MONOTONIC */
 111         uint64_t undef_ex_cnt;
 112         uint64_t unaligned_cnt;
 113         uint64_t vfp_cnt;
 114         uint64_t data_ex_cnt;
 115         uint64_t instr_ex_cnt;
 116 } cpu_stat_t;
 117
 118 typedef struct cpu_data {
 119         unsigned short                  cpu_number;
 120         unsigned short                  cpu_flags;
 121         int                             cpu_type;
 122         int                             cpu_subtype;
 123         int                             cpu_threadtype;
 124
 125         vm_offset_t                     istackptr;
 126         vm_offset_t                     intstack_top;
 127 #if __arm64__
 128         vm_offset_t                     excepstackptr;
 129         vm_offset_t                     excepstack_top;
 130 #else
 131         vm_offset_t                     fiqstackptr;
 132         vm_offset_t                     fiqstack_top;
 133 #endif
 134         thread_t                        cpu_active_thread;
 135         vm_offset_t                     cpu_active_stack;
 136         cpu_id_t                        cpu_id;
 137         unsigned volatile int           cpu_signal;
 138         ast_t                           cpu_pending_ast;
 139         cache_dispatch_t                cpu_cache_dispatch;
 140
 141 #if __arm64__
 142         uint64_t                        cpu_base_timebase;
 143         uint64_t                        cpu_timebase;
 144 #else
 145         union {
 146                 struct {
 147                         uint32_t        low;
 148                         uint32_t        high;
 149                 } split;
 150                 struct {
 151                         uint64_t        val;
 152                 } raw;
 153         } cbtb;
 154 #define cpu_base_timebase_low cbtb.split.low
 155 #define cpu_base_timebase_high cbtb.split.high
 156
 157         union {
 158                 struct {
 159                         uint32_t        low;
 160                         uint32_t        high;
 161                 } split;
 162                 struct {
 163                         uint64_t        val;
 164                 } raw;
 165         } ctb;
 166 #define cpu_timebase_low ctb.split.low
 167 #define cpu_timebase_high ctb.split.high
 168 #endif
 169         bool                            cpu_hibernate; /* This cpu is currently hibernating the system */
 170         bool                            cpu_running;
 171         bool                            cluster_master;
 172         /* true if processor_start() or processor_exit() is operating on this CPU */
 173         bool                            in_state_transition;
 174
 175         uint32_t                        cpu_decrementer;
 176         get_decrementer_t               cpu_get_decrementer_func;
 177         set_decrementer_t               cpu_set_decrementer_func;
 178         fiq_handler_t                   cpu_get_fiq_handler;
 179
 180         void                            *cpu_tbd_hardware_addr;
 181         void                            *cpu_tbd_hardware_val;
 182
 183         void                            *cpu_console_buf;
 184
 185         processor_idle_t                cpu_idle_notify;
 186         uint64_t                        cpu_idle_latency;
 187         uint64_t                        cpu_idle_pop;
 188
 189 #if     __arm__ || __ARM_KERNEL_PROTECT__
 190         vm_offset_t                     cpu_exc_vectors;
 191 #endif /* __ARM_KERNEL_PROTECT__ */
 192         vm_offset_t                     cpu_reset_handler;
 193         uintptr_t                       cpu_reset_assist;
 194         uint32_t                        cpu_reset_type;
 195
 196         unsigned int                    interrupt_source;
 197         void                            *cpu_int_state;
 198         IOInterruptHandler              interrupt_handler;
 199         void                            *interrupt_nub;
 200         void                            *interrupt_target;
 201         void                            *interrupt_refCon;
 202
 203         idle_timer_t                    idle_timer_notify;
 204         void                            *idle_timer_refcon;
 205         uint64_t                        idle_timer_deadline;
 206
 207         uint64_t                        rtcPop;
 208         rtclock_timer_t                 rtclock_timer;
 209         struct _rtclock_data_           *rtclock_datap;
 210
 211         arm_debug_state_t               *cpu_user_debug; /* Current debug state */
 212         vm_offset_t                     cpu_debug_interface_map;
 213
 214         volatile int                    debugger_active;
 215         volatile int                    PAB_active; /* Tells the console if we are dumping backtraces */
 216
 217         void                            *cpu_xcall_p0;
 218         void                            *cpu_xcall_p1;
 219         void                            *cpu_imm_xcall_p0;
 220         void                            *cpu_imm_xcall_p1;
 221
 222 #if     defined(ARMA7)
 223         volatile uint32_t               cpu_CLW_active;
 224         volatile uint64_t               cpu_CLWFlush_req;
 225         volatile uint64_t               cpu_CLWFlush_last;
 226         volatile uint64_t               cpu_CLWClean_req;
 227         volatile uint64_t               cpu_CLWClean_last;
 228 #endif
 229
 230 #if     __arm64__
 231         vm_offset_t                     coresight_base[CORESIGHT_REGIONS];
 232 #endif
 233
 234         /* CCC ARMv8 registers */
 235         uint64_t                        cpu_regmap_paddr;
 236
 237         uint32_t                        cpu_phys_id;
 238         uint32_t                        cpu_l2_access_penalty;
 239         platform_error_handler_t        platform_error_handler;
 240
 241         int                             cpu_mcount_off;
 242
 243         #define ARM_CPU_ON_SLEEP_PATH   0x50535553UL
 244         volatile unsigned int           cpu_sleep_token;
 245         unsigned int                    cpu_sleep_token_last;
 246
 247         cluster_type_t                  cpu_cluster_type;
 248         uint32_t                        cpu_cluster_id;
 249         uint32_t                        cpu_l2_id;
 250         uint32_t                        cpu_l2_size;
 251         uint32_t                        cpu_l3_id;
 252         uint32_t                        cpu_l3_size;
 253
 254         enum {
 255                 CPU_NOT_HALTED = 0,
 256                 CPU_HALTED,
 257                 CPU_HALTED_WITH_STATE
 258         }                               halt_status;
 259 #if defined(HAS_APPLE_PAC)
 260         uint64_t                        rop_key;
 261         uint64_t                        jop_key;
 262 #endif /* defined(HAS_APPLE_PAC) */
 263
 264         /* large structs with large alignment requirements */
 265 #if KPC
 266         /* double-buffered performance counter data */
 267         uint64_t                        *cpu_kpc_buf[2];
 268         /* PMC shadow and reload value buffers */
 269         uint64_t                        *cpu_kpc_shadow;
 270         uint64_t                        *cpu_kpc_reload;
 271 #endif
 272 #if MONOTONIC
 273         struct mt_cpu                   cpu_monotonic;
 274 #endif /* MONOTONIC */
 275         cpu_stat_t                      cpu_stat;
 276 #if !XNU_MONITOR
 277         struct pmap_cpu_data            cpu_pmap_cpu_data;
 278 #endif
 279         dbgwrap_thread_state_t          halt_state;
 280 #if DEVELOPMENT || DEBUG
 281         uint64_t                        wfe_count;
 282         uint64_t                        wfe_deadline_checks;
 283         uint64_t                        wfe_terminations;
 284 #endif
 285 } cpu_data_t;
 286
 287 /*
 288  * cpu_flags
 289  */
 290 #define SleepState                      0x0800
 291 #define StartedState                    0x1000
 292
 293 extern  cpu_data_entry_t                CpuDataEntries[MAX_CPUS];
 294 PERCPU_DECL(cpu_data_t, cpu_data);
 295 #define BootCpuData                     __PERCPU_NAME(cpu_data)
 296 extern  boot_args                      *BootArgs;
 297
 298 #if __arm__
 299 extern  unsigned int                   *ExceptionLowVectorsBase;
 300 extern  unsigned int                   *ExceptionVectorsTable;
 301 #elif __arm64__
 302 extern  unsigned int                    LowResetVectorBase;
 303 extern  unsigned int                    LowResetVectorEnd;
 304 #if WITH_CLASSIC_S2R
 305 extern  uint8_t                         SleepToken[8];
 306 #endif
 307 extern  unsigned int                    LowExceptionVectorBase;
 308 #else
 309 #error Unknown arch
 310 #endif
 311
 312 extern cpu_data_t      *cpu_datap(int cpu);
 313 extern cpu_data_t      *cpu_data_alloc(boolean_t is_boot);
 314 extern void             cpu_stack_alloc(cpu_data_t*);
 315 extern void             cpu_data_init(cpu_data_t *cpu_data_ptr);
 316 extern void             cpu_data_free(cpu_data_t *cpu_data_ptr);
 317 extern kern_return_t    cpu_data_register(cpu_data_t *cpu_data_ptr);
 318 extern cpu_data_t      *processor_to_cpu_datap( processor_t processor);
 319
 320 #if __arm64__
 321 typedef struct sysreg_restore {
 322         uint64_t                tcr_el1;
 323 } sysreg_restore_t;
 324
 325 extern sysreg_restore_t sysreg_restore;
 326 #endif  /* __arm64__ */
 327
 328 #endif  /* ARM_CPU_DATA_INTERNAL */