osfmk/i386/tsc.c

   1 /*
   2  * Copyright (c) 2005-2007 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 /*
  29  * @OSF_COPYRIGHT@
  30  */
  31
  32 /*
  33  *      File:           i386/tsc.c
  34  *      Purpose:        Initializes the TSC and the various conversion
  35  *                      factors needed by other parts of the system.
  36  */
  37
  38
  39 #include <mach/mach_types.h>
  40
  41 #include <kern/cpu_data.h>
  42 #include <kern/cpu_number.h>
  43 #include <kern/clock.h>
  44 #include <kern/host_notify.h>
  45 #include <kern/macro_help.h>
  46 #include <kern/misc_protos.h>
  47 #include <kern/spl.h>
  48 #include <kern/assert.h>
  49 #include <mach/vm_prot.h>
  50 #include <vm/pmap.h>
  51 #include <vm/vm_kern.h>         /* for kernel_map */
  52 #include <architecture/i386/pio.h>
  53 #include <i386/machine_cpu.h>
  54 #include <i386/cpuid.h>
  55 #include <i386/mp.h>
  56 #include <i386/machine_routines.h>
  57 #include <i386/proc_reg.h>
  58 #include <i386/tsc.h>
  59 #include <i386/misc_protos.h>
  60 #include <pexpert/pexpert.h>
  61 #include <machine/limits.h>
  62 #include <machine/commpage.h>
  63 #include <sys/kdebug.h>
  64 #include <pexpert/device_tree.h>
  65
  66 uint64_t        busFCvtt2n = 0;
  67 uint64_t        busFCvtn2t = 0;
  68 uint64_t        tscFreq = 0;
  69 uint64_t        tscFCvtt2n = 0;
  70 uint64_t        tscFCvtn2t = 0;
  71 uint64_t        tscGranularity = 0;
  72 uint64_t        bus2tsc = 0;
  73 uint64_t        busFreq = 0;
  74 uint32_t        flex_ratio = 0;
  75 uint32_t        flex_ratio_min = 0;
  76 uint32_t        flex_ratio_max = 0;
  77
  78 uint64_t        tsc_at_boot = 0;
  79
  80 #define bit(n)          (1ULL << (n))
  81 #define bitmask(h, l)    ((bit(h)|(bit(h)-1)) & ~(bit(l)-1))
  82 #define bitfield(x, h, l) (((x) & bitmask(h,l)) >> l)
  83
  84 /* Decimal powers: */
  85 #define kilo (1000ULL)
  86 #define Mega (kilo * kilo)
  87 #define Giga (kilo * Mega)
  88 #define Tera (kilo * Giga)
  89 #define Peta (kilo * Tera)
  90
  91 #define CPU_FAMILY_PENTIUM_M    (0x6)
  92
  93 /*
  94  * This routine extracts a frequency property in Hz from the device tree.
  95  * Also reads any initial TSC value at boot from the device tree.
  96  */
  97 static uint64_t
  98 EFI_get_frequency(const char *prop)
  99 {
 100         uint64_t        frequency = 0;
 101         DTEntry         entry;
 102         void const      *value;
 103         unsigned int    size;
 104
 105         if (SecureDTLookupEntry(0, "/efi/platform", &entry) != kSuccess) {
 106                 kprintf("EFI_get_frequency: didn't find /efi/platform\n");
 107                 return 0;
 108         }
 109
 110         /*
 111          * While we're here, see if EFI published an initial TSC value.
 112          */
 113         if (SecureDTGetProperty(entry, "InitialTSC", &value, &size) == kSuccess) {
 114                 if (size == sizeof(uint64_t)) {
 115                         tsc_at_boot = *(uint64_t const *) value;
 116                         kprintf("EFI_get_frequency: read InitialTSC: %llu\n",
 117                             tsc_at_boot);
 118                 }
 119         }
 120
 121         if (SecureDTGetProperty(entry, prop, &value, &size) != kSuccess) {
 122                 kprintf("EFI_get_frequency: property %s not found\n", prop);
 123                 return 0;
 124         }
 125         if (size == sizeof(uint64_t)) {
 126                 frequency = *(uint64_t const *) value;
 127                 kprintf("EFI_get_frequency: read %s value: %llu\n",
 128                     prop, frequency);
 129         }
 130
 131         return frequency;
 132 }
 133
 134 /*
 135  * Initialize the various conversion factors needed by code referencing
 136  * the TSC.
 137  */
 138 void
 139 tsc_init(void)
 140 {
 141         boolean_t       N_by_2_bus_ratio = FALSE;
 142
 143         if (cpuid_vmm_present()) {
 144                 kprintf("VMM vendor %s TSC frequency %u KHz bus frequency %u KHz\n",
 145                     cpuid_vmm_family_string(),
 146                     cpuid_vmm_info()->cpuid_vmm_tsc_frequency,
 147                     cpuid_vmm_info()->cpuid_vmm_bus_frequency);
 148
 149                 if (cpuid_vmm_info()->cpuid_vmm_tsc_frequency &&
 150                     cpuid_vmm_info()->cpuid_vmm_bus_frequency) {
 151                         busFreq = (uint64_t)cpuid_vmm_info()->cpuid_vmm_bus_frequency * kilo;
 152                         busFCvtt2n = ((1 * Giga) << 32) / busFreq;
 153                         busFCvtn2t = 0xFFFFFFFFFFFFFFFFULL / busFCvtt2n;
 154
 155                         tscFreq = (uint64_t)cpuid_vmm_info()->cpuid_vmm_tsc_frequency * kilo;
 156                         tscFCvtt2n = ((1 * Giga) << 32) / tscFreq;
 157                         tscFCvtn2t = 0xFFFFFFFFFFFFFFFFULL / tscFCvtt2n;
 158
 159                         tscGranularity = tscFreq / busFreq;
 160
 161                         bus2tsc = tmrCvt(busFCvtt2n, tscFCvtn2t);
 162
 163                         return;
 164                 }
 165         }
 166
 167         switch (cpuid_cpufamily()) {
 168         case CPUFAMILY_INTEL_KABYLAKE:
 169         case CPUFAMILY_INTEL_ICELAKE:
 170         case CPUFAMILY_INTEL_SKYLAKE: {
 171                 /*
 172                  * SkyLake and later has an Always Running Timer (ART) providing
 173                  * the reference frequency. CPUID leaf 0x15 determines the
 174                  * rationship between this and the TSC frequency expressed as
 175                  *   -  multiplier (numerator, N), and
 176                  *   -  divisor (denominator, M).
 177                  * So that TSC = ART * N / M.
 178                  */
 179                 i386_cpu_info_t *infop = cpuid_info();
 180                 cpuid_tsc_leaf_t *tsc_leafp = &infop->cpuid_tsc_leaf;
 181                 uint64_t         N = (uint64_t) tsc_leafp->numerator;
 182                 uint64_t         M = (uint64_t) tsc_leafp->denominator;
 183                 uint64_t         refFreq;
 184
 185                 refFreq = EFI_get_frequency("ARTFrequency");
 186                 if (refFreq == 0) {
 187                         /*
 188                          * Intel Scalable Processor (Xeon-SP) CPUs use a different
 189                          * ART frequency.  Use that default here if EFI didn't
 190                          * specify the frequency.  Since Xeon-SP uses the same
 191                          * DisplayModel / DisplayFamily as Xeon-W, we need to
 192                          * use the platform ID (or, as XNU calls it, the "processor
 193                          * flag") to differentiate the two.
 194                          */
 195                         if (cpuid_family() == 0x06 &&
 196                             infop->cpuid_model == CPUID_MODEL_SKYLAKE_W &&
 197                             is_xeon_sp(infop->cpuid_processor_flag)) {
 198                                 refFreq = BASE_ART_CLOCK_SOURCE_SP;
 199                         } else {
 200                                 refFreq = BASE_ART_CLOCK_SOURCE;
 201                         }
 202                 }
 203
 204                 assert(N != 0);
 205                 assert(M != 1);
 206                 tscFreq = refFreq * N / M;
 207                 busFreq = tscFreq;              /* bus is APIC frequency */
 208
 209                 kprintf(" ART: Frequency = %6d.%06dMHz, N/M = %lld/%llu\n",
 210                     (uint32_t)(refFreq / Mega),
 211                     (uint32_t)(refFreq % Mega),
 212                     N, M);
 213
 214                 break;
 215         }
 216         default: {
 217                 uint64_t msr_flex_ratio;
 218                 uint64_t msr_platform_info;
 219
 220                 /* See if FLEX_RATIO is being used */
 221                 msr_flex_ratio = rdmsr64(MSR_FLEX_RATIO);
 222                 msr_platform_info = rdmsr64(MSR_PLATFORM_INFO);
 223                 flex_ratio_min = (uint32_t)bitfield(msr_platform_info, 47, 40);
 224                 flex_ratio_max = (uint32_t)bitfield(msr_platform_info, 15, 8);
 225                 /* No BIOS-programed flex ratio. Use hardware max as default */
 226                 tscGranularity = flex_ratio_max;
 227                 if (msr_flex_ratio & bit(16)) {
 228                         /* Flex Enabled: Use this MSR if less than max */
 229                         flex_ratio = (uint32_t)bitfield(msr_flex_ratio, 15, 8);
 230                         if (flex_ratio < flex_ratio_max) {
 231                                 tscGranularity = flex_ratio;
 232                         }
 233                 }
 234
 235                 busFreq = EFI_get_frequency("FSBFrequency");
 236                 /* If EFI isn't configured correctly, use a constant
 237                  * value. See 6036811.
 238                  */
 239                 if (busFreq == 0) {
 240                         busFreq = BASE_NHM_CLOCK_SOURCE;
 241                 }
 242
 243                 break;
 244         }
 245         case CPUFAMILY_INTEL_PENRYN: {
 246                 uint64_t        prfsts;
 247
 248                 prfsts = rdmsr64(IA32_PERF_STS);
 249                 tscGranularity = (uint32_t)bitfield(prfsts, 44, 40);
 250                 N_by_2_bus_ratio = (prfsts & bit(46)) != 0;
 251
 252                 busFreq = EFI_get_frequency("FSBFrequency");
 253         }
 254         }
 255
 256         if (busFreq != 0) {
 257                 busFCvtt2n = ((1 * Giga) << 32) / busFreq;
 258                 busFCvtn2t = 0xFFFFFFFFFFFFFFFFULL / busFCvtt2n;
 259         } else {
 260                 panic("tsc_init: EFI not supported!\n");
 261         }
 262
 263         kprintf(" BUS: Frequency = %6d.%06dMHz, "
 264             "cvtt2n = %08X.%08X, cvtn2t = %08X.%08X\n",
 265             (uint32_t)(busFreq / Mega),
 266             (uint32_t)(busFreq % Mega),
 267             (uint32_t)(busFCvtt2n >> 32), (uint32_t)busFCvtt2n,
 268             (uint32_t)(busFCvtn2t >> 32), (uint32_t)busFCvtn2t);
 269
 270         if (tscFreq == busFreq) {
 271                 bus2tsc = 1;
 272                 tscGranularity = 1;
 273                 tscFCvtn2t = busFCvtn2t;
 274                 tscFCvtt2n = busFCvtt2n;
 275         } else {
 276                 /*
 277                  * Get the TSC increment.  The TSC is incremented by this
 278                  * on every bus tick.  Calculate the TSC conversion factors
 279                  * to and from nano-seconds.
 280                  * The tsc granularity is also called the "bus ratio".
 281                  * If the N/2 bit is set this indicates the bus ration is
 282                  * 0.5 more than this - i.e.  that the true bus ratio
 283                  * is (2*tscGranularity + 1)/2.
 284                  */
 285                 if (N_by_2_bus_ratio) {
 286                         tscFCvtt2n = busFCvtt2n * 2 / (1 + 2 * tscGranularity);
 287                 } else {
 288                         tscFCvtt2n = busFCvtt2n / tscGranularity;
 289                 }
 290
 291                 tscFreq = ((1 * Giga) << 32) / tscFCvtt2n;
 292                 tscFCvtn2t = 0xFFFFFFFFFFFFFFFFULL / tscFCvtt2n;
 293
 294                 /*
 295                  * Calculate conversion from BUS to TSC
 296                  */
 297                 bus2tsc = tmrCvt(busFCvtt2n, tscFCvtn2t);
 298         }
 299
 300         kprintf(" TSC: Frequency = %6d.%06dMHz, "
 301             "cvtt2n = %08X.%08X, cvtn2t = %08X.%08X, gran = %lld%s\n",
 302             (uint32_t)(tscFreq / Mega),
 303             (uint32_t)(tscFreq % Mega),
 304             (uint32_t)(tscFCvtt2n >> 32), (uint32_t)tscFCvtt2n,
 305             (uint32_t)(tscFCvtn2t >> 32), (uint32_t)tscFCvtn2t,
 306             tscGranularity, N_by_2_bus_ratio ? " (N/2)" : "");
 307 }
 308
 309 void
 310 tsc_get_info(tscInfo_t *info)
 311 {
 312         info->busFCvtt2n     = busFCvtt2n;
 313         info->busFCvtn2t     = busFCvtn2t;
 314         info->tscFreq        = tscFreq;
 315         info->tscFCvtt2n     = tscFCvtt2n;
 316         info->tscFCvtn2t     = tscFCvtn2t;
 317         info->tscGranularity = tscGranularity;
 318         info->bus2tsc        = bus2tsc;
 319         info->busFreq        = busFreq;
 320         info->flex_ratio     = flex_ratio;
 321         info->flex_ratio_min = flex_ratio_min;
 322         info->flex_ratio_max = flex_ratio_max;
 323 }
 324
 325 #if DEVELOPMENT || DEBUG
 326 void
 327 cpu_data_tsc_sync_deltas_string(char *buf, uint32_t buflen,
 328     uint32_t start_cpu, uint32_t end_cpu)
 329 {
 330         int cnt;
 331         uint32_t offset = 0;
 332
 333         if (start_cpu >= real_ncpus || end_cpu >= real_ncpus) {
 334                 if (buflen >= 1) {
 335                         buf[0] = 0;
 336                 }
 337                 return;
 338         }
 339
 340         for (uint32_t curcpu = start_cpu; curcpu <= end_cpu; curcpu++) {
 341                 cnt = snprintf(buf + offset, buflen - offset, "0x%llx ", cpu_datap(curcpu)->tsc_sync_delta);
 342                 if (cnt < 0 || (offset + (unsigned) cnt >= buflen)) {
 343                         break;
 344                 }
 345                 offset += cnt;
 346         }
 347         if (offset >= 1) {
 348                 buf[offset - 1] = 0;    /* Clip the final, trailing space */
 349         }
 350 }
 351 #endif /* DEVELOPMENT || DEBUG */