osfmk/x86_64/monotonic_x86_64.c

   1 /*
   2  * Copyright (c) 2017 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <i386/cpu_data.h>
  30 #include <i386/cpuid.h>
  31 #include <i386/lapic.h>
  32 #include <i386/mp.h>
  33 #include <i386/proc_reg.h>
  34 #include <kern/assert.h> /* static_assert, assert */
  35 #include <kern/monotonic.h>
  36 #include <os/overflow.h>
  37 #include <sys/errno.h>
  38 #include <sys/monotonic.h>
  39 #include <x86_64/monotonic.h>
  40
  41 /*
  42  * Sanity check the compiler.
  43  */
  44
  45 #ifndef __has_builtin
  46 #define __has_builtin(x) 0
  47 #endif /* !defined(__has_builtin) */
  48 #if !__has_builtin(__builtin_ia32_rdpmc)
  49 #error requires __builtin_ia32_rdpmc builtin
  50 #endif /* !__has_builtin(__builtin_ia32_rdpmc) */
  51
  52 #pragma mark core counters
  53
  54 bool mt_core_supported = false;
  55
  56 /*
  57  * PMC[0-2]_{RD,WR} allow reading and writing the fixed PMCs.
  58  *
  59  * There are separate defines for access type because the read side goes through
  60  * the rdpmc instruction, which has a different counter encoding than the msr
  61  * path.
  62  */
  63 #define PMC_FIXED_RD(CTR) ((UINT64_C(1) << 30) | (CTR))
  64 #define PMC_FIXED_WR(CTR) (MSR_IA32_PERF_FIXED_CTR0 + (CTR))
  65 #define PMC0_RD PMC_FIXED_RD(0)
  66 #define PMC0_WR PMC_FIXED_WR(0)
  67 #define PMC1_RD PMC_FIXED_RD(1)
  68 #define PMC1_WR PMC_FIXED_WR(1)
  69 #define PMC2_RD PMC_FIXED_RD(2)
  70 #define PMC2_WR PMC_FIXED_WR(2)
  71
  72 struct mt_cpu *
  73 mt_cur_cpu(void)
  74 {
  75         return &current_cpu_datap()->cpu_monotonic;
  76 }
  77
  78 uint64_t
  79 mt_core_snap(unsigned int ctr)
  80 {
  81         if (!mt_core_supported) {
  82                 return 0;
  83         }
  84
  85         switch (ctr) {
  86         case 0:
  87                 return __builtin_ia32_rdpmc(PMC0_RD);
  88         case 1:
  89                 return __builtin_ia32_rdpmc(PMC1_RD);
  90         case 2:
  91                 return __builtin_ia32_rdpmc(PMC2_RD);
  92         default:
  93                 panic("monotonic: invalid core counter read: %u", ctr);
  94                 __builtin_unreachable();
  95         }
  96 }
  97
  98 void
  99 mt_core_set_snap(unsigned int ctr, uint64_t count)
 100 {
 101         if (!mt_core_supported) {
 102                 return;
 103         }
 104
 105         switch (ctr) {
 106         case 0:
 107                 wrmsr64(PMC0_WR, count);
 108                 break;
 109         case 1:
 110                 wrmsr64(PMC1_WR, count);
 111                 break;
 112         case 2:
 113                 wrmsr64(PMC2_WR, count);
 114                 break;
 115         default:
 116                 panic("monotonic: invalid core counter write: %u", ctr);
 117                 __builtin_unreachable();
 118         }
 119 }
 120
 121 /*
 122  * FIXED_CTR_CTRL controls which rings fixed counters are enabled in and if they
 123  * deliver PMIs.
 124  *
 125  * Each fixed counters has 4 bits: [0:1] controls which ring it's enabled in,
 126  * [2] counts all hardware threads in each logical core (we don't want this),
 127  * and [3] enables PMIs on overflow.
 128  */
 129
 130 #define FIXED_CTR_CTRL 0x38d
 131
 132 /*
 133  * Fixed counters are enabled in all rings, so hard-code this register state to
 134  * enable in all rings and deliver PMIs.
 135  */
 136 #define FIXED_CTR_CTRL_INIT (0x888)
 137 #define FIXED_CTR_CTRL_ENABLE (0x333)
 138
 139 /*
 140  * GLOBAL_CTRL controls which counters are enabled -- the high 32-bits control
 141  * the fixed counters and the lower half is for the configurable counters.
 142  */
 143
 144 #define GLOBAL_CTRL 0x38f
 145
 146 /*
 147  * Fixed counters are always enabled -- and there are three of them.
 148  */
 149 #define GLOBAL_CTRL_FIXED_EN (((UINT64_C(1) << 3) - 1) << 32)
 150
 151 /*
 152  * GLOBAL_STATUS reports the state of counters, like those that have overflowed.
 153  */
 154 #define GLOBAL_STATUS 0x38e
 155
 156 #define CTR_MAX ((UINT64_C(1) << 48) - 1)
 157 #define CTR_FIX_POS(CTR) ((UINT64_C(1) << (CTR)) << 32)
 158
 159 #define GLOBAL_OVF 0x390
 160
 161 static void mt_check_for_pmi(struct mt_cpu *mtc, x86_saved_state_t *state);
 162
 163 static void
 164 enable_counters(void)
 165 {
 166         wrmsr64(FIXED_CTR_CTRL, FIXED_CTR_CTRL_INIT | FIXED_CTR_CTRL_ENABLE);
 167         wrmsr64(GLOBAL_CTRL, GLOBAL_CTRL_FIXED_EN);
 168 }
 169
 170 static void
 171 disable_counters(void)
 172 {
 173         wrmsr64(GLOBAL_CTRL, 0);
 174 }
 175
 176 static void
 177 core_down(cpu_data_t *cpu)
 178 {
 179         if (!mt_core_supported) {
 180                 return;
 181         }
 182         assert(ml_get_interrupts_enabled() == FALSE);
 183         struct mt_cpu *mtc = &cpu->cpu_monotonic;
 184
 185         disable_counters();
 186         mt_mtc_update_fixed_counts(mtc, NULL, NULL);
 187         mtc->mtc_active = false;
 188 }
 189
 190 static void
 191 core_up(cpu_data_t *cpu)
 192 {
 193         struct mt_cpu *mtc;
 194
 195         if (!mt_core_supported) {
 196                 return;
 197         }
 198
 199         assert(ml_get_interrupts_enabled() == FALSE);
 200
 201         mtc = &cpu->cpu_monotonic;
 202
 203         for (int i = 0; i < MT_CORE_NFIXED; i++) {
 204                 mt_core_set_snap(i, mtc->mtc_snaps[i]);
 205         }
 206         enable_counters();
 207         mtc->mtc_active = true;
 208 }
 209
 210 void
 211 mt_cpu_down(cpu_data_t *cpu)
 212 {
 213         core_down(cpu);
 214 }
 215
 216 void
 217 mt_cpu_up(cpu_data_t *cpu)
 218 {
 219         boolean_t intrs_en;
 220         intrs_en = ml_set_interrupts_enabled(FALSE);
 221         core_up(cpu);
 222         ml_set_interrupts_enabled(intrs_en);
 223 }
 224
 225 uint64_t
 226 mt_count_pmis(void)
 227 {
 228         uint64_t npmis = 0;
 229         for (unsigned int i = 0; i < real_ncpus; i++) {
 230                 cpu_data_t *cpu = cpu_data_ptr[i];
 231                 npmis += cpu->cpu_monotonic.mtc_npmis;
 232         }
 233         return npmis;
 234 }
 235
 236 static void
 237 mt_check_for_pmi(struct mt_cpu *mtc, x86_saved_state_t *state)
 238 {
 239         uint64_t status = rdmsr64(GLOBAL_STATUS);
 240
 241         mtc->mtc_npmis += 1;
 242
 243         if (mtc->mtc_active) {
 244                 disable_counters();
 245         }
 246
 247         for (unsigned int i = 0; i < MT_CORE_NFIXED; i++) {
 248                 if (status & CTR_FIX_POS(i)) {
 249                         uint64_t prior = CTR_MAX - mtc->mtc_snaps[i];
 250                         assert(prior <= CTR_MAX);
 251                         prior += 1; /* wrapped */
 252
 253                         uint64_t delta = mt_mtc_update_count(mtc, i);
 254                         mtc->mtc_counts[i] += delta;
 255
 256                         if (mt_microstackshots && mt_microstackshot_ctr == i) {
 257                                 bool user_mode = false;
 258                                 if (state) {
 259                                         x86_saved_state64_t *state64 = saved_state64(state);
 260                                         user_mode = (state64->isf.cs & 0x3) != 0;
 261                                 }
 262                                 KDBG_RELEASE(KDBG_EVENTID(DBG_MONOTONIC, DBG_MT_DEBUG, 1),
 263                                     mt_microstackshot_ctr, user_mode);
 264                                 mt_microstackshot_pmi_handler(user_mode, mt_microstackshot_ctx);
 265                         } else if (mt_debug) {
 266                                 KDBG(KDBG_EVENTID(DBG_MONOTONIC, DBG_MT_DEBUG, 2),
 267                                     mt_microstackshot_ctr, i);
 268                         }
 269
 270                         mtc->mtc_snaps[i] = mt_core_reset_values[i];
 271                         mt_core_set_snap(i, mt_core_reset_values[i]);
 272                 }
 273         }
 274
 275         /* if any of the configurable counters overflowed, tell kpc */
 276         if (status & ((UINT64_C(1) << 4) - 1)) {
 277                 extern void kpc_pmi_handler(void);
 278                 kpc_pmi_handler();
 279         }
 280
 281         if (mtc->mtc_active) {
 282                 enable_counters();
 283         }
 284 }
 285
 286 static int
 287 mt_pmi_x86_64(x86_saved_state_t *state)
 288 {
 289         assert(ml_get_interrupts_enabled() == FALSE);
 290         mt_check_for_pmi(mt_cur_cpu(), state);
 291         return 0;
 292 }
 293
 294 static void
 295 mt_microstackshot_start_remote(__unused void *arg)
 296 {
 297         struct mt_cpu *mtc = mt_cur_cpu();
 298
 299         wrmsr64(FIXED_CTR_CTRL, FIXED_CTR_CTRL_INIT);
 300
 301         for (int i = 0; i < MT_CORE_NFIXED; i++) {
 302                 uint64_t delta = mt_mtc_update_count(mtc, i);
 303                 mtc->mtc_counts[i] += delta;
 304                 mt_core_set_snap(i, mt_core_reset_values[i]);
 305                 mtc->mtc_snaps[i] = mt_core_reset_values[i];
 306         }
 307
 308         wrmsr64(FIXED_CTR_CTRL, FIXED_CTR_CTRL_INIT | FIXED_CTR_CTRL_ENABLE);
 309 }
 310
 311 int
 312 mt_microstackshot_start_arch(uint64_t period)
 313 {
 314         if (!mt_core_supported) {
 315                 return ENOTSUP;
 316         }
 317
 318         uint64_t reset_value = 0;
 319         int ovf = os_sub_overflow(CTR_MAX, period, &reset_value);
 320         if (ovf) {
 321                 return ERANGE;
 322         }
 323
 324         mt_core_reset_values[mt_microstackshot_ctr] = CTR_MAX - period;
 325         mp_cpus_call(CPUMASK_ALL, ASYNC, mt_microstackshot_start_remote,
 326             NULL);
 327         return 0;
 328 }
 329
 330 void
 331 mt_early_init(void)
 332 {
 333         if (PE_parse_boot_argn("-nomt_core", NULL, 0)) {
 334                 return;
 335         }
 336         i386_cpu_info_t *info = cpuid_info();
 337         if (info->cpuid_arch_perf_leaf.version >= 2) {
 338                 lapic_set_pmi_func((i386_intr_func_t)mt_pmi_x86_64);
 339                 mt_core_supported = true;
 340         }
 341 }
 342
 343 static int
 344 core_init(__unused mt_device_t dev)
 345 {
 346         return ENOTSUP;
 347 }
 348
 349 #pragma mark common hooks
 350
 351 struct mt_device mt_devices[] = {
 352         [0] = {
 353                 .mtd_name = "core",
 354                 .mtd_init = core_init
 355         }
 356 };
 357
 358 static_assert(
 359         (sizeof(mt_devices) / sizeof(mt_devices[0])) == MT_NDEVS,
 360         "MT_NDEVS macro should be same as the length of mt_devices");