osfmk/x86_64/monotonic_x86_64.c

   1 /*
   2  * Copyright (c) 2017 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <i386/cpu_data.h>
  30 #include <i386/cpuid.h>
  31 #include <i386/lapic.h>
  32 #include <i386/mp.h>
  33 #include <i386/proc_reg.h>
  34 #include <kern/assert.h> /* static_assert, assert */
  35 #include <kern/monotonic.h>
  36 #include <os/overflow.h>
  37 #include <sys/errno.h>
  38 #include <sys/monotonic.h>
  39 #include <x86_64/monotonic.h>
  40 #include <kern/kpc.h>
  41
  42 /*
  43  * Sanity check the compiler.
  44  */
  45
  46 #ifndef __has_builtin
  47 #define __has_builtin(x) 0
  48 #endif /* !defined(__has_builtin) */
  49 #if !__has_builtin(__builtin_ia32_rdpmc)
  50 #error requires __builtin_ia32_rdpmc builtin
  51 #endif /* !__has_builtin(__builtin_ia32_rdpmc) */
  52
  53 #pragma mark core counters
  54
  55 bool mt_core_supported = false;
  56
  57 /*
  58  * PMC[0-2]_{RD,WR} allow reading and writing the fixed PMCs.
  59  *
  60  * There are separate defines for access type because the read side goes through
  61  * the rdpmc instruction, which has a different counter encoding than the msr
  62  * path.
  63  */
  64 #define PMC_FIXED_RD(CTR) ((UINT64_C(1) << 30) | (CTR))
  65 #define PMC_FIXED_WR(CTR) (MSR_IA32_PERF_FIXED_CTR0 + (CTR))
  66 #define PMC0_RD PMC_FIXED_RD(0)
  67 #define PMC0_WR PMC_FIXED_WR(0)
  68 #define PMC1_RD PMC_FIXED_RD(1)
  69 #define PMC1_WR PMC_FIXED_WR(1)
  70 #define PMC2_RD PMC_FIXED_RD(2)
  71 #define PMC2_WR PMC_FIXED_WR(2)
  72
  73 struct mt_cpu *
  74 mt_cur_cpu(void)
  75 {
  76         return &current_cpu_datap()->cpu_monotonic;
  77 }
  78
  79 uint64_t
  80 mt_core_snap(unsigned int ctr)
  81 {
  82         if (!mt_core_supported) {
  83                 return 0;
  84         }
  85
  86         switch (ctr) {
  87         case 0:
  88                 return __builtin_ia32_rdpmc(PMC0_RD);
  89         case 1:
  90                 return __builtin_ia32_rdpmc(PMC1_RD);
  91         case 2:
  92                 return __builtin_ia32_rdpmc(PMC2_RD);
  93         default:
  94                 panic("monotonic: invalid core counter read: %u", ctr);
  95                 __builtin_unreachable();
  96         }
  97 }
  98
  99 void
 100 mt_core_set_snap(unsigned int ctr, uint64_t count)
 101 {
 102         if (!mt_core_supported) {
 103                 return;
 104         }
 105
 106         switch (ctr) {
 107         case 0:
 108                 wrmsr64(PMC0_WR, count);
 109                 break;
 110         case 1:
 111                 wrmsr64(PMC1_WR, count);
 112                 break;
 113         case 2:
 114                 wrmsr64(PMC2_WR, count);
 115                 break;
 116         default:
 117                 panic("monotonic: invalid core counter write: %u", ctr);
 118                 __builtin_unreachable();
 119         }
 120 }
 121
 122 /*
 123  * FIXED_CTR_CTRL controls which rings fixed counters are enabled in and if they
 124  * deliver PMIs.
 125  *
 126  * Each fixed counters has 4 bits: [0:1] controls which ring it's enabled in,
 127  * [2] counts all hardware threads in each logical core (we don't want this),
 128  * and [3] enables PMIs on overflow.
 129  */
 130
 131 #define FIXED_CTR_CTRL 0x38d
 132
 133 /*
 134  * Fixed counters are enabled in all rings, so hard-code this register state to
 135  * enable in all rings and deliver PMIs.
 136  */
 137 #define FIXED_CTR_CTRL_INIT (0x888)
 138 #define FIXED_CTR_CTRL_ENABLE (0x333)
 139
 140 /*
 141  * GLOBAL_CTRL controls which counters are enabled -- the high 32-bits control
 142  * the fixed counters and the lower half is for the configurable counters.
 143  */
 144
 145 #define GLOBAL_CTRL 0x38f
 146
 147 /*
 148  * Fixed counters are always enabled -- and there are three of them.
 149  */
 150 #define GLOBAL_CTRL_FIXED_EN (((UINT64_C(1) << 3) - 1) << 32)
 151
 152 /*
 153  * GLOBAL_STATUS reports the state of counters, like those that have overflowed.
 154  */
 155 #define GLOBAL_STATUS 0x38e
 156
 157 #define CTR_MAX ((UINT64_C(1) << 48) - 1)
 158 #define CTR_FIX_POS(CTR) ((UINT64_C(1) << (CTR)) << 32)
 159
 160 #define GLOBAL_OVF 0x390
 161
 162 static void mt_check_for_pmi(struct mt_cpu *mtc, x86_saved_state_t *state);
 163
 164 static void
 165 enable_counters(void)
 166 {
 167         wrmsr64(FIXED_CTR_CTRL, FIXED_CTR_CTRL_INIT | FIXED_CTR_CTRL_ENABLE);
 168
 169         uint64_t global_en = GLOBAL_CTRL_FIXED_EN;
 170         if (kpc_get_running() & KPC_CLASS_CONFIGURABLE_MASK) {
 171                 global_en |= kpc_get_configurable_pmc_mask(KPC_CLASS_CONFIGURABLE_MASK);
 172         }
 173
 174         wrmsr64(GLOBAL_CTRL, global_en);
 175 }
 176
 177 static void
 178 disable_counters(void)
 179 {
 180         wrmsr64(GLOBAL_CTRL, 0);
 181 }
 182
 183 static void
 184 core_down(cpu_data_t *cpu)
 185 {
 186         if (!mt_core_supported) {
 187                 return;
 188         }
 189         assert(ml_get_interrupts_enabled() == FALSE);
 190         struct mt_cpu *mtc = &cpu->cpu_monotonic;
 191
 192         disable_counters();
 193         mt_mtc_update_fixed_counts(mtc, NULL, NULL);
 194         mtc->mtc_active = false;
 195 }
 196
 197 static void
 198 core_up(cpu_data_t *cpu)
 199 {
 200         struct mt_cpu *mtc;
 201
 202         if (!mt_core_supported) {
 203                 return;
 204         }
 205
 206         assert(ml_get_interrupts_enabled() == FALSE);
 207
 208         mtc = &cpu->cpu_monotonic;
 209
 210         for (int i = 0; i < MT_CORE_NFIXED; i++) {
 211                 mt_core_set_snap(i, mtc->mtc_snaps[i]);
 212         }
 213         enable_counters();
 214         mtc->mtc_active = true;
 215 }
 216
 217 void
 218 mt_cpu_down(cpu_data_t *cpu)
 219 {
 220         core_down(cpu);
 221 }
 222
 223 void
 224 mt_cpu_up(cpu_data_t *cpu)
 225 {
 226         boolean_t intrs_en;
 227         intrs_en = ml_set_interrupts_enabled(FALSE);
 228         core_up(cpu);
 229         ml_set_interrupts_enabled(intrs_en);
 230 }
 231
 232 uint64_t
 233 mt_count_pmis(void)
 234 {
 235         uint64_t npmis = 0;
 236         for (unsigned int i = 0; i < real_ncpus; i++) {
 237                 cpu_data_t *cpu = cpu_data_ptr[i];
 238                 npmis += cpu->cpu_monotonic.mtc_npmis;
 239         }
 240         return npmis;
 241 }
 242
 243 static void
 244 mt_check_for_pmi(struct mt_cpu *mtc, x86_saved_state_t *state)
 245 {
 246         uint64_t status = rdmsr64(GLOBAL_STATUS);
 247
 248         mtc->mtc_npmis += 1;
 249
 250         if (mtc->mtc_active) {
 251                 disable_counters();
 252         }
 253
 254         for (unsigned int i = 0; i < MT_CORE_NFIXED; i++) {
 255                 if (status & CTR_FIX_POS(i)) {
 256                         uint64_t prior = CTR_MAX - mtc->mtc_snaps[i];
 257                         assert(prior <= CTR_MAX);
 258                         prior += 1; /* wrapped */
 259
 260                         uint64_t delta = mt_mtc_update_count(mtc, i);
 261                         mtc->mtc_counts[i] += delta;
 262
 263                         if (mt_microstackshots && mt_microstackshot_ctr == i) {
 264                                 bool user_mode = false;
 265                                 if (state) {
 266                                         x86_saved_state64_t *state64 = saved_state64(state);
 267                                         user_mode = (state64->isf.cs & 0x3) != 0;
 268                                 }
 269                                 KDBG_RELEASE(KDBG_EVENTID(DBG_MONOTONIC, DBG_MT_DEBUG, 1),
 270                                     mt_microstackshot_ctr, user_mode);
 271                                 mt_microstackshot_pmi_handler(user_mode, mt_microstackshot_ctx);
 272                         } else if (mt_debug) {
 273                                 KDBG(KDBG_EVENTID(DBG_MONOTONIC, DBG_MT_DEBUG, 2),
 274                                     mt_microstackshot_ctr, i);
 275                         }
 276
 277                         mtc->mtc_snaps[i] = mt_core_reset_values[i];
 278                         mt_core_set_snap(i, mt_core_reset_values[i]);
 279                 }
 280         }
 281
 282         /* if any of the configurable counters overflowed, tell kpc */
 283         if (status & ((UINT64_C(1) << 4) - 1)) {
 284                 extern void kpc_pmi_handler(void);
 285                 kpc_pmi_handler();
 286         }
 287
 288         if (mtc->mtc_active) {
 289                 enable_counters();
 290         }
 291 }
 292
 293 static int
 294 mt_pmi_x86_64(x86_saved_state_t *state)
 295 {
 296         assert(ml_get_interrupts_enabled() == FALSE);
 297         mt_check_for_pmi(mt_cur_cpu(), state);
 298         return 0;
 299 }
 300
 301 static void
 302 mt_microstackshot_start_remote(__unused void *arg)
 303 {
 304         struct mt_cpu *mtc = mt_cur_cpu();
 305
 306         wrmsr64(FIXED_CTR_CTRL, FIXED_CTR_CTRL_INIT);
 307
 308         for (int i = 0; i < MT_CORE_NFIXED; i++) {
 309                 uint64_t delta = mt_mtc_update_count(mtc, i);
 310                 mtc->mtc_counts[i] += delta;
 311                 mt_core_set_snap(i, mt_core_reset_values[i]);
 312                 mtc->mtc_snaps[i] = mt_core_reset_values[i];
 313         }
 314
 315         wrmsr64(FIXED_CTR_CTRL, FIXED_CTR_CTRL_INIT | FIXED_CTR_CTRL_ENABLE);
 316 }
 317
 318 int
 319 mt_microstackshot_start_arch(uint64_t period)
 320 {
 321         if (!mt_core_supported) {
 322                 return ENOTSUP;
 323         }
 324
 325         uint64_t reset_value = 0;
 326         int ovf = os_sub_overflow(CTR_MAX, period, &reset_value);
 327         if (ovf) {
 328                 return ERANGE;
 329         }
 330
 331         mt_core_reset_values[mt_microstackshot_ctr] = CTR_MAX - period;
 332         mp_cpus_call(CPUMASK_ALL, ASYNC, mt_microstackshot_start_remote,
 333             NULL);
 334         return 0;
 335 }
 336
 337 void
 338 mt_early_init(void)
 339 {
 340         if (PE_parse_boot_argn("-nomt_core", NULL, 0)) {
 341                 return;
 342         }
 343         i386_cpu_info_t *info = cpuid_info();
 344         if (info->cpuid_arch_perf_leaf.version >= 2) {
 345                 lapic_set_pmi_func((i386_intr_func_t)mt_pmi_x86_64);
 346                 mt_core_supported = true;
 347         }
 348 }
 349
 350 static int
 351 core_init(__unused mt_device_t dev)
 352 {
 353         return ENOTSUP;
 354 }
 355
 356 #pragma mark common hooks
 357
 358 struct mt_device mt_devices[] = {
 359         [0] = {
 360                 .mtd_name = "core",
 361                 .mtd_init = core_init
 362         }
 363 };
 364
 365 static_assert(
 366         (sizeof(mt_devices) / sizeof(mt_devices[0])) == MT_NDEVS,
 367         "MT_NDEVS macro should be same as the length of mt_devices");