xnu-7195.101.1.tar.gz

[apple/xnu.git] / osfmk / arm64 / monotonic_arm64.c

/*
 * Copyright (c) 2017-2020 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 *
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */

#include <arm/cpu_data_internal.h>
#include <arm/machine_routines.h>
#include <arm64/monotonic.h>
#include <kern/assert.h>
#include <kern/debug.h> /* panic */
#include <kern/kpc.h>
#include <kern/monotonic.h>
#include <machine/atomic.h>
#include <machine/limits.h> /* CHAR_BIT */
#include <os/overflow.h>
#include <pexpert/arm64/board_config.h>
#include <pexpert/device_tree.h> /* SecureDTFindEntry */
#include <pexpert/pexpert.h>
#include <stdatomic.h>
#include <stdint.h>
#include <string.h>
#include <sys/errno.h>
#include <sys/monotonic.h>

/*
 * Ensure that control registers read back what was written under MACH_ASSERT
 * kernels.
 *
 * A static inline function cannot be used due to passing the register through
 * the builtin -- it requires a constant string as its first argument, since
 * MSRs registers are encoded as an immediate in the instruction.
 */
#if MACH_ASSERT
#define CTRL_REG_SET(reg, val) do { \
        __builtin_arm_wsr64((reg), (val)); \
        uint64_t __check_reg = __builtin_arm_rsr64((reg)); \
        if (__check_reg != (val)) { \
                panic("value written to %s was not read back (wrote %llx, read %llx)", \
                    #reg, (val), __check_reg); \
        } \
} while (0)
#else /* MACH_ASSERT */
#define CTRL_REG_SET(reg, val) __builtin_arm_wsr64((reg), (val))
#endif /* MACH_ASSERT */

#pragma mark core counters

bool mt_core_supported = true;

static const ml_topology_info_t *topology_info;

/*
 * PMC[0-1] are the 48-bit fixed counters -- PMC0 is cycles and PMC1 is
 * instructions (see arm64/monotonic.h).
 *
 * PMC2+ are currently handled by kpc.
 */
#define PMC_0_7(X, A) X(0, A); X(1, A); X(2, A); X(3, A); X(4, A); X(5, A); \
    X(6, A); X(7, A)

#if CORE_NCTRS > 8
#define PMC_8_9(X, A) X(8, A); X(9, A)
#else // CORE_NCTRS > 8
#define PMC_8_9(X, A)
#endif // CORE_NCTRS > 8

#define PMC_ALL(X, A) PMC_0_7(X, A); PMC_8_9(X, A)

#define CTR_MAX ((UINT64_C(1) << 47) - 1)

#define CYCLES 0
#define INSTRS 1

/*
 * PMC0's offset into a core's PIO range.
 *
 * This allows cores to remotely query another core's counters.
 */

#define PIO_PMC0_OFFSET (0x200)

/*
 * The offset of the counter in the configuration registers.  Post-Hurricane
 * devices have additional counters that need a larger shift than the original
 * counters.
 *
 * XXX For now, just support the lower-numbered counters.
 */
#define CTR_POS(CTR) (CTR)

/*
 * PMCR0 is the main control register for the performance monitor.  It
 * controls whether the counters are enabled, how they deliver interrupts, and
 * other features.
 */

#define PMCR0_CTR_EN(CTR) (UINT64_C(1) << CTR_POS(CTR))
#define PMCR0_FIXED_EN (PMCR0_CTR_EN(CYCLES) | PMCR0_CTR_EN(INSTRS))
/* how interrupts are delivered on a PMI */
enum {
        PMCR0_INTGEN_OFF = 0,
        PMCR0_INTGEN_PMI = 1,
        PMCR0_INTGEN_AIC = 2,
        PMCR0_INTGEN_HALT = 3,
        PMCR0_INTGEN_FIQ = 4,
};
#define PMCR0_INTGEN_SET(X) ((uint64_t)(X) << 8)

#if CPMU_AIC_PMI
#define PMCR0_INTGEN_INIT PMCR0_INTGEN_SET(PMCR0_INTGEN_AIC)
#else /* CPMU_AIC_PMI */
#define PMCR0_INTGEN_INIT PMCR0_INTGEN_SET(PMCR0_INTGEN_FIQ)
#endif /* !CPMU_AIC_PMI */

#define PMCR0_PMI_SHIFT (12)
#define PMCR0_CTR_GE8_PMI_SHIFT (44)
#define PMCR0_PMI_EN(CTR) (UINT64_C(1) << (PMCR0_PMI_SHIFT + CTR_POS(CTR)))
/* fixed counters are always counting */
#define PMCR0_PMI_INIT (PMCR0_PMI_EN(CYCLES) | PMCR0_PMI_EN(INSTRS))
/* disable counting on a PMI */
#define PMCR0_DISCNT_EN (UINT64_C(1) << 20)
/* block PMIs until ERET retires */
#define PMCR0_WFRFE_EN (UINT64_C(1) << 22)
/* count global (not just core-local) L2C events */
#define PMCR0_L2CGLOBAL_EN (UINT64_C(1) << 23)
/* user mode access to configuration registers */
#define PMCR0_USEREN_EN (UINT64_C(1) << 30)
#define PMCR0_CTR_GE8_EN_SHIFT (32)

#define PMCR0_INIT (PMCR0_INTGEN_INIT | PMCR0_PMI_INIT)

/*
 * PMCR1 controls which execution modes count events.
 */
#define PMCR1_EL0A32_EN(CTR) (UINT64_C(1) << (0 + CTR_POS(CTR)))
#define PMCR1_EL0A64_EN(CTR) (UINT64_C(1) << (8 + CTR_POS(CTR)))
#define PMCR1_EL1A64_EN(CTR) (UINT64_C(1) << (16 + CTR_POS(CTR)))
/* PMCR1_EL3A64 is not supported on systems with no monitor */
#if defined(APPLEHURRICANE)
#define PMCR1_EL3A64_EN(CTR) UINT64_C(0)
#else
#define PMCR1_EL3A64_EN(CTR) (UINT64_C(1) << (24 + CTR_POS(CTR)))
#endif
#define PMCR1_ALL_EN(CTR) (PMCR1_EL0A32_EN(CTR) | PMCR1_EL0A64_EN(CTR) | \
                           PMCR1_EL1A64_EN(CTR) | PMCR1_EL3A64_EN(CTR))

/* fixed counters always count in all modes */
#define PMCR1_INIT (PMCR1_ALL_EN(CYCLES) | PMCR1_ALL_EN(INSTRS))

static inline void
core_init_execution_modes(void)
{
        uint64_t pmcr1;

        pmcr1 = __builtin_arm_rsr64("PMCR1_EL1");
        pmcr1 |= PMCR1_INIT;
        __builtin_arm_wsr64("PMCR1_EL1", pmcr1);
}

#define PMSR_OVF(CTR) (1ULL << (CTR))

static int
core_init(__unused mt_device_t dev)
{
        /* the dev node interface to the core counters is still unsupported */
        return ENOTSUP;
}

struct mt_cpu *
mt_cur_cpu(void)
{
        return &getCpuDatap()->cpu_monotonic;
}

uint64_t
mt_core_snap(unsigned int ctr)
{
        switch (ctr) {
#define PMC_RD(CTR, UNUSED) case (CTR): return __builtin_arm_rsr64(__MSR_STR(PMC ## CTR))
                PMC_ALL(PMC_RD, 0);
#undef PMC_RD
        default:
                panic("monotonic: invalid core counter read: %u", ctr);
                __builtin_unreachable();
        }
}

void
mt_core_set_snap(unsigned int ctr, uint64_t count)
{
        switch (ctr) {
        case 0:
                __builtin_arm_wsr64("PMC0", count);
                break;
        case 1:
                __builtin_arm_wsr64("PMC1", count);
                break;
        default:
                panic("monotonic: invalid core counter %u write %llu", ctr, count);
                __builtin_unreachable();
        }
}

static void
core_set_enabled(void)
{
        uint64_t pmcr0 = __builtin_arm_rsr64("PMCR0_EL1");
        pmcr0 |= PMCR0_INIT | PMCR0_FIXED_EN;

        if (kpc_get_running() & KPC_CLASS_CONFIGURABLE_MASK) {
                uint64_t kpc_ctrs = kpc_get_configurable_pmc_mask(
                        KPC_CLASS_CONFIGURABLE_MASK) << MT_CORE_NFIXED;
#if KPC_ARM64_CONFIGURABLE_COUNT > 6
                uint64_t ctrs_ge8 = kpc_ctrs >> 8;
                pmcr0 |= ctrs_ge8 << PMCR0_CTR_GE8_EN_SHIFT;
                pmcr0 |= ctrs_ge8 << PMCR0_CTR_GE8_PMI_SHIFT;
                kpc_ctrs &= (1ULL << 8) - 1;
#endif /* KPC_ARM64_CONFIGURABLE_COUNT > 6 */
                kpc_ctrs |= kpc_ctrs << PMCR0_PMI_SHIFT;
                pmcr0 |= kpc_ctrs;
        }

        __builtin_arm_wsr64("PMCR0_EL1", pmcr0);
#if MACH_ASSERT
        /*
         * Only check for the values that were ORed in.
         */
        uint64_t pmcr0_check = __builtin_arm_rsr64("PMCR0_EL1");
        if ((pmcr0_check & (PMCR0_INIT | PMCR0_FIXED_EN)) != (PMCR0_INIT | PMCR0_FIXED_EN)) {
                panic("monotonic: hardware ignored enable (read %llx, wrote %llx)",
                    pmcr0_check, pmcr0);
        }
#endif /* MACH_ASSERT */
}

static void
core_idle(__unused cpu_data_t *cpu)
{
        assert(cpu != NULL);
        assert(ml_get_interrupts_enabled() == FALSE);

#if DEBUG
        uint64_t pmcr0 = __builtin_arm_rsr64("PMCR0_EL1");
        if ((pmcr0 & PMCR0_FIXED_EN) == 0) {
                panic("monotonic: counters disabled before idling, pmcr0 = 0x%llx\n", pmcr0);
        }
        uint64_t pmcr1 = __builtin_arm_rsr64("PMCR1_EL1");
        if ((pmcr1 & PMCR1_INIT) == 0) {
                panic("monotonic: counter modes disabled before idling, pmcr1 = 0x%llx\n", pmcr1);
        }
#endif /* DEBUG */

        /* disable counters before updating */
        __builtin_arm_wsr64("PMCR0_EL1", PMCR0_INIT);

        mt_update_fixed_counts();
}

#pragma mark uncore performance monitor

#if HAS_UNCORE_CTRS

static bool mt_uncore_initted = false;

/*
 * Uncore Performance Monitor
 *
 * Uncore performance monitors provide event-counting for the last-level caches
 * (LLCs).  Each LLC has its own uncore performance monitor, which can only be
 * accessed by cores that use that LLC.  Like the core performance monitoring
 * unit, uncore counters are configured globally.  If there is more than one
 * LLC on the system, PIO reads must be used to satisfy uncore requests (using
 * the `_r` remote variants of the access functions).  Otherwise, local MSRs
 * suffice (using the `_l` local variants of the access functions).
 */

#if UNCORE_PER_CLUSTER
#define MAX_NMONITORS MAX_CPU_CLUSTERS
static uintptr_t cpm_impl[MAX_NMONITORS] = {};
#else
#define MAX_NMONITORS (1)
#endif /* UNCORE_PER_CLUSTER */

#if UNCORE_VERSION >= 2
/*
 * V2 uncore monitors feature a CTI mechanism -- the second bit of UPMSR is
 * used to track if a CTI has been triggered due to an overflow.
 */
#define UPMSR_OVF_POS 2
#else /* UNCORE_VERSION >= 2 */
#define UPMSR_OVF_POS 1
#endif /* UNCORE_VERSION < 2 */
#define UPMSR_OVF(R, CTR) ((R) >> ((CTR) + UPMSR_OVF_POS) & 0x1)
#define UPMSR_OVF_MASK    (((UINT64_C(1) << UNCORE_NCTRS) - 1) << UPMSR_OVF_POS)

#define UPMPCM_CORE(ID) (UINT64_C(1) << (ID))

/*
 * The uncore_pmi_mask is a bitmask of CPUs that receive uncore PMIs.  It's
 * initialized by uncore_init and controllable by the uncore_pmi_mask boot-arg.
 */
static int32_t uncore_pmi_mask = 0;

/*
 * The uncore_active_ctrs is a bitmask of uncore counters that are currently
 * requested.
 */
static uint16_t uncore_active_ctrs = 0;
static_assert(sizeof(uncore_active_ctrs) * CHAR_BIT >= UNCORE_NCTRS,
    "counter mask should fit the full range of counters");

/*
 * mt_uncore_enabled is true when any uncore counters are active.
 */
bool mt_uncore_enabled = false;

/*
 * The uncore_events are the event configurations for each uncore counter -- as
 * a union to make it easy to program the hardware registers.
 */
static struct uncore_config {
        union {
                uint8_t uce_ctrs[UNCORE_NCTRS];
                uint64_t uce_regs[UNCORE_NCTRS / 8];
        } uc_events;
        union {
                uint16_t uccm_masks[UNCORE_NCTRS];
                uint64_t uccm_regs[UNCORE_NCTRS / 4];
        } uc_cpu_masks[MAX_NMONITORS];
} uncore_config;

static struct uncore_monitor {
        /*
         * The last snapshot of each of the hardware counter values.
         */
        uint64_t um_snaps[UNCORE_NCTRS];

        /*
         * The accumulated counts for each counter.
         */
        uint64_t um_counts[UNCORE_NCTRS];

        /*
         * Protects accessing the hardware registers and fields in this structure.
         */
        lck_spin_t um_lock;

        /*
         * Whether this monitor needs its registers restored after wake.
         */
        bool um_sleeping;
} uncore_monitors[MAX_NMONITORS];

/*
 * Each uncore unit has its own monitor, corresponding to the memory hierarchy
 * of the LLCs.
 */
static unsigned int
uncore_nmonitors(void)
{
#if UNCORE_PER_CLUSTER
        return topology_info->num_clusters;
#else /* UNCORE_PER_CLUSTER */
        return 1;
#endif /* !UNCORE_PER_CLUSTER */
}

static unsigned int
uncmon_get_curid(void)
{
#if UNCORE_PER_CLUSTER
        // Pointer arithmetic to translate cluster_id into a clusters[] index.
        return cpu_cluster_id();
#else /* UNCORE_PER_CLUSTER */
        return 0;
#endif /* !UNCORE_PER_CLUSTER */
}

/*
 * Per-monitor locks are required to prevent races with the PMI handlers, not
 * from other CPUs that are configuring (those are serialized with monotonic's
 * per-device lock).
 */

static int
uncmon_lock(struct uncore_monitor *mon)
{
        int intrs_en = ml_set_interrupts_enabled(FALSE);
        lck_spin_lock(&mon->um_lock);
        return intrs_en;
}

static void
uncmon_unlock(struct uncore_monitor *mon, int intrs_en)
{
        lck_spin_unlock(&mon->um_lock);
        (void)ml_set_interrupts_enabled(intrs_en);
}

/*
 * Helper functions for accessing the hardware -- these require the monitor be
 * locked to prevent other CPUs' PMI handlers from making local modifications
 * or updating the counts.
 */

#if UNCORE_VERSION >= 2
#define UPMCR0_INTEN_POS 20
#define UPMCR0_INTGEN_POS 16
#else /* UNCORE_VERSION >= 2 */
#define UPMCR0_INTEN_POS 12
#define UPMCR0_INTGEN_POS 8
#endif /* UNCORE_VERSION < 2 */
enum {
        UPMCR0_INTGEN_OFF = 0,
        /* fast PMIs are only supported on core CPMU */
        UPMCR0_INTGEN_AIC = 2,
        UPMCR0_INTGEN_HALT = 3,
        UPMCR0_INTGEN_FIQ = 4,
};
/* always enable interrupts for all counters */
#define UPMCR0_INTEN (((1ULL << UNCORE_NCTRS) - 1) << UPMCR0_INTEN_POS)
/* route uncore PMIs through the FIQ path */
#define UPMCR0_INIT (UPMCR0_INTEN | (UPMCR0_INTGEN_FIQ << UPMCR0_INTGEN_POS))

/*
 * Turn counting on for counters set in the `enctrmask` and off, otherwise.
 */
static inline void
uncmon_set_counting_locked_l(__unused unsigned int monid, uint64_t enctrmask)
{
        /*
         * UPMCR0 controls which counters are enabled and how interrupts are generated
         * for overflows.
         */
        __builtin_arm_wsr64("UPMCR0_EL1", UPMCR0_INIT | enctrmask);
}

#if UNCORE_PER_CLUSTER

/*
 * Turn counting on for counters set in the `enctrmask` and off, otherwise.
 */
static inline void
uncmon_set_counting_locked_r(unsigned int monid, uint64_t enctrmask)
{
        const uintptr_t upmcr0_offset = 0x4180;
        *(uint64_t *)(cpm_impl[monid] + upmcr0_offset) = UPMCR0_INIT | enctrmask;
}

#endif /* UNCORE_PER_CLUSTER */

/*
 * The uncore performance monitoring counters (UPMCs) are 48-bits wide.  The
 * high bit is an overflow bit, triggering a PMI, providing 47 usable bits.
 */

#define UPMC_MAX ((UINT64_C(1) << 48) - 1)

/*
 * The `__builtin_arm_{r,w}sr` functions require constant strings, since the
 * MSR/MRS instructions encode the registers as immediates.  Otherwise, this
 * would be indexing into an array of strings.
 */

#define UPMC_0_7(X, A) X(0, A); X(1, A); X(2, A); X(3, A); X(4, A); X(5, A); \
                X(6, A); X(7, A)
#if UNCORE_NCTRS <= 8
#define UPMC_ALL(X, A) UPMC_0_7(X, A)
#else /* UNCORE_NCTRS <= 8 */
#define UPMC_8_15(X, A) X(8, A); X(9, A); X(10, A); X(11, A); X(12, A); \
                X(13, A); X(14, A); X(15, A)
#define UPMC_ALL(X, A) UPMC_0_7(X, A); UPMC_8_15(X, A)
#endif /* UNCORE_NCTRS > 8 */

static inline uint64_t
uncmon_read_counter_locked_l(__unused unsigned int monid, unsigned int ctr)
{
        assert(ctr < UNCORE_NCTRS);
        switch (ctr) {
#define UPMC_RD(CTR, UNUSED) case (CTR): return __builtin_arm_rsr64(__MSR_STR(UPMC ## CTR))
                UPMC_ALL(UPMC_RD, 0);
#undef UPMC_RD
        default:
                panic("monotonic: invalid counter read %u", ctr);
                __builtin_unreachable();
        }
}

static inline void
uncmon_write_counter_locked_l(__unused unsigned int monid, unsigned int ctr,
    uint64_t count)
{
        assert(count < UPMC_MAX);
        assert(ctr < UNCORE_NCTRS);
        switch (ctr) {
#define UPMC_WR(CTR, COUNT) case (CTR): \
                return __builtin_arm_wsr64(__MSR_STR(UPMC ## CTR), (COUNT))
                UPMC_ALL(UPMC_WR, count);
#undef UPMC_WR
        default:
                panic("monotonic: invalid counter write %u", ctr);
        }
}

#if UNCORE_PER_CLUSTER

uintptr_t upmc_offs[UNCORE_NCTRS] = {
        [0] = 0x4100, [1] = 0x4248, [2] = 0x4110, [3] = 0x4250, [4] = 0x4120,
        [5] = 0x4258, [6] = 0x4130, [7] = 0x4260, [8] = 0x4140, [9] = 0x4268,
        [10] = 0x4150, [11] = 0x4270, [12] = 0x4160, [13] = 0x4278,
        [14] = 0x4170, [15] = 0x4280,
};

static inline uint64_t
uncmon_read_counter_locked_r(unsigned int mon_id, unsigned int ctr)
{
        assert(mon_id < uncore_nmonitors());
        assert(ctr < UNCORE_NCTRS);
        return *(uint64_t *)(cpm_impl[mon_id] + upmc_offs[ctr]);
}

static inline void
uncmon_write_counter_locked_r(unsigned int mon_id, unsigned int ctr,
    uint64_t count)
{
        assert(count < UPMC_MAX);
        assert(ctr < UNCORE_NCTRS);
        assert(mon_id < uncore_nmonitors());
        *(uint64_t *)(cpm_impl[mon_id] + upmc_offs[ctr]) = count;
}

#endif /* UNCORE_PER_CLUSTER */

static inline void
uncmon_update_locked(unsigned int monid, unsigned int curid, unsigned int ctr)
{
        struct uncore_monitor *mon = &uncore_monitors[monid];
        uint64_t snap = 0;
        if (curid == monid) {
                snap = uncmon_read_counter_locked_l(monid, ctr);
        } else {
#if UNCORE_PER_CLUSTER
                snap = uncmon_read_counter_locked_r(monid, ctr);
#endif /* UNCORE_PER_CLUSTER */
        }
        /* counters should increase monotonically */
        assert(snap >= mon->um_snaps[ctr]);
        mon->um_counts[ctr] += snap - mon->um_snaps[ctr];
        mon->um_snaps[ctr] = snap;
}

static inline void
uncmon_program_events_locked_l(unsigned int monid)
{
        /*
         * UPMESR[01] is the event selection register that determines which event a
         * counter will count.
         */
        CTRL_REG_SET("UPMESR0_EL1", uncore_config.uc_events.uce_regs[0]);

#if UNCORE_NCTRS > 8
        CTRL_REG_SET("UPMESR1_EL1", uncore_config.uc_events.uce_regs[1]);
#endif /* UNCORE_NCTRS > 8 */

        /*
         * UPMECM[0123] are the event core masks for each counter -- whether or not
         * that counter counts events generated by an agent.  These are set to all
         * ones so the uncore counters count events from all cores.
         *
         * The bits are based off the start of the cluster -- e.g. even if a core
         * has a CPU ID of 4, it might be the first CPU in a cluster.  Shift the
         * registers right by the ID of the first CPU in the cluster.
         */
        CTRL_REG_SET("UPMECM0_EL1",
            uncore_config.uc_cpu_masks[monid].uccm_regs[0]);
        CTRL_REG_SET("UPMECM1_EL1",
            uncore_config.uc_cpu_masks[monid].uccm_regs[1]);

#if UNCORE_NCTRS > 8
        CTRL_REG_SET("UPMECM2_EL1",
            uncore_config.uc_cpu_masks[monid].uccm_regs[2]);
        CTRL_REG_SET("UPMECM3_EL1",
            uncore_config.uc_cpu_masks[monid].uccm_regs[3]);
#endif /* UNCORE_NCTRS > 8 */
}

#if UNCORE_PER_CLUSTER

static inline void
uncmon_program_events_locked_r(unsigned int monid)
{
        const uintptr_t upmesr_offs[2] = {[0] = 0x41b0, [1] = 0x41b8, };

        for (unsigned int i = 0; i < sizeof(upmesr_offs) / sizeof(upmesr_offs[0]);
            i++) {
                *(uint64_t *)(cpm_impl[monid] + upmesr_offs[i]) =
                    uncore_config.uc_events.uce_regs[i];
        }

        const uintptr_t upmecm_offs[4] = {
                [0] = 0x4190, [1] = 0x4198, [2] = 0x41a0, [3] = 0x41a8,
        };

        for (unsigned int i = 0; i < sizeof(upmecm_offs) / sizeof(upmecm_offs[0]);
            i++) {
                *(uint64_t *)(cpm_impl[monid] + upmecm_offs[i]) =
                    uncore_config.uc_cpu_masks[monid].uccm_regs[i];
        }
}

#endif /* UNCORE_PER_CLUSTER */

static void
uncmon_clear_int_locked_l(__unused unsigned int monid)
{
        __builtin_arm_wsr64("UPMSR_EL1", 0);
}

#if UNCORE_PER_CLUSTER

static void
uncmon_clear_int_locked_r(unsigned int monid)
{
        const uintptr_t upmsr_off = 0x41c0;
        *(uint64_t *)(cpm_impl[monid] + upmsr_off) = 0;
}

#endif /* UNCORE_PER_CLUSTER */

/*
 * Get the PMI mask for the provided `monid` -- that is, the bitmap of CPUs
 * that should be sent PMIs for a particular monitor.
 */
static uint64_t
uncmon_get_pmi_mask(unsigned int monid)
{
        uint64_t pmi_mask = uncore_pmi_mask;

#if UNCORE_PER_CLUSTER
        pmi_mask &= topology_info->clusters[monid].cpu_mask;
#else /* UNCORE_PER_CLUSTER */
#pragma unused(monid)
#endif /* !UNCORE_PER_CLUSTER */

        return pmi_mask;
}

/*
 * Initialization routines for the uncore counters.
 */

static void
uncmon_init_locked_l(unsigned int monid)
{
        /*
         * UPMPCM defines the PMI core mask for the UPMCs -- which cores should
         * receive interrupts on overflow.
         */
        CTRL_REG_SET("UPMPCM_EL1", uncmon_get_pmi_mask(monid));
        uncmon_set_counting_locked_l(monid,
            mt_uncore_enabled ? uncore_active_ctrs : 0);
}

#if UNCORE_PER_CLUSTER

static uintptr_t acc_impl[MAX_NMONITORS] = {};

static void
uncmon_init_locked_r(unsigned int monid)
{
        const uintptr_t upmpcm_off = 0x1010;

        *(uint64_t *)(acc_impl[monid] + upmpcm_off) = uncmon_get_pmi_mask(monid);
        uncmon_set_counting_locked_r(monid,
            mt_uncore_enabled ? uncore_active_ctrs : 0);
}

#endif /* UNCORE_PER_CLUSTER */

/*
 * Initialize the uncore device for monotonic.
 */
static int
uncore_init(__unused mt_device_t dev)
{
#if HAS_UNCORE_CTRS
        assert(MT_NDEVS > 0);
        mt_devices[MT_NDEVS - 1].mtd_nmonitors = (uint8_t)uncore_nmonitors();
#endif

#if DEVELOPMENT || DEBUG
        /*
         * Development and debug kernels observe the `uncore_pmi_mask` boot-arg,
         * allowing PMIs to be routed to the CPUs present in the supplied bitmap.
         * Do some sanity checks on the value provided.
         */
        bool parsed_arg = PE_parse_boot_argn("uncore_pmi_mask", &uncore_pmi_mask,
            sizeof(uncore_pmi_mask));
        if (parsed_arg) {
#if UNCORE_PER_CLUSTER
                if (__builtin_popcount(uncore_pmi_mask) != (int)uncore_nmonitors()) {
                        panic("monotonic: invalid uncore PMI mask 0x%x", uncore_pmi_mask);
                }
                for (unsigned int i = 0; i < uncore_nmonitors(); i++) {
                        if (__builtin_popcountll(uncmon_get_pmi_mask(i)) != 1) {
                                panic("monotonic: invalid uncore PMI CPU for cluster %d in mask 0x%x",
                                    i, uncore_pmi_mask);
                        }
                }
#else /* UNCORE_PER_CLUSTER */
                if (__builtin_popcount(uncore_pmi_mask) != 1) {
                        panic("monotonic: invalid uncore PMI mask 0x%x", uncore_pmi_mask);
                }
#endif /* !UNCORE_PER_CLUSTER */
        } else
#endif /* DEVELOPMENT || DEBUG */
        {
#if UNCORE_PER_CLUSTER
                for (unsigned int i = 0; i < topology_info->num_clusters; i++) {
                        uncore_pmi_mask |= 1ULL << topology_info->clusters[i].first_cpu_id;
                }
#else /* UNCORE_PER_CLUSTER */
                /* arbitrarily route to core 0 */
                uncore_pmi_mask |= 1;
#endif /* !UNCORE_PER_CLUSTER */
        }
        assert(uncore_pmi_mask != 0);

        unsigned int curmonid = uncmon_get_curid();

        for (unsigned int monid = 0; monid < uncore_nmonitors(); monid++) {
#if UNCORE_PER_CLUSTER
                ml_topology_cluster_t *cluster = &topology_info->clusters[monid];
                cpm_impl[monid] = (uintptr_t)cluster->cpm_IMPL_regs;
                acc_impl[monid] = (uintptr_t)cluster->acc_IMPL_regs;
                assert(cpm_impl[monid] != 0 && acc_impl[monid] != 0);
#endif /* UNCORE_PER_CLUSTER */

                struct uncore_monitor *mon = &uncore_monitors[monid];
                lck_spin_init(&mon->um_lock, &mt_lock_grp, LCK_ATTR_NULL);

                int intrs_en = uncmon_lock(mon);
                if (monid != curmonid) {
#if UNCORE_PER_CLUSTER
                        uncmon_init_locked_r(monid);
#endif /* UNCORE_PER_CLUSTER */
                } else {
                        uncmon_init_locked_l(monid);
                }
                uncmon_unlock(mon, intrs_en);
        }

        mt_uncore_initted = true;

        return 0;
}

/*
 * Support for monotonic's mtd_read function.
 */

static void
uncmon_read_all_counters(unsigned int monid, unsigned int curmonid,
    uint64_t ctr_mask, uint64_t *counts)
{
        struct uncore_monitor *mon = &uncore_monitors[monid];

        int intrs_en = uncmon_lock(mon);

        for (unsigned int ctr = 0; ctr < UNCORE_NCTRS; ctr++) {
                if (ctr_mask & (1ULL << ctr)) {
                        uncmon_update_locked(monid, curmonid, ctr);
                        counts[ctr] = mon->um_counts[ctr];
                }
        }

        uncmon_unlock(mon, intrs_en);
}

/*
 * Read all monitor's counters.
 */
static int
uncore_read(uint64_t ctr_mask, uint64_t *counts_out)
{
        assert(ctr_mask != 0);
        assert(counts_out != NULL);

        if (!uncore_active_ctrs) {
                return EPWROFF;
        }
        if (ctr_mask & ~uncore_active_ctrs) {
                return EINVAL;
        }

        unsigned int curmonid = uncmon_get_curid();
        for (unsigned int monid = 0; monid < uncore_nmonitors(); monid++) {
                /*
                 * Find this monitor's starting offset into the `counts_out` array.
                 */
                uint64_t *counts = counts_out + (UNCORE_NCTRS * monid);

                uncmon_read_all_counters(monid, curmonid, ctr_mask, counts);
        }

        return 0;
}

/*
 * Support for monotonic's mtd_add function.
 */

/*
 * Add an event to the current uncore configuration.  This doesn't take effect
 * until the counters are enabled again, so there's no need to involve the
 * monitors.
 */
static int
uncore_add(struct monotonic_config *config, uint32_t *ctr_out)
{
        if (mt_uncore_enabled) {
                return EBUSY;
        }

        uint32_t available = ~uncore_active_ctrs & config->allowed_ctr_mask;

        if (available == 0) {
                return ENOSPC;
        }

        uint32_t valid_ctrs = (UINT32_C(1) << UNCORE_NCTRS) - 1;
        if ((available & valid_ctrs) == 0) {
                return E2BIG;
        }

        uint32_t ctr = __builtin_ffsll(available) - 1;

        uncore_active_ctrs |= UINT64_C(1) << ctr;
        uncore_config.uc_events.uce_ctrs[ctr] = (uint8_t)config->event;
        uint64_t cpu_mask = UINT64_MAX;
        if (config->cpu_mask != 0) {
                cpu_mask = config->cpu_mask;
        }
        for (unsigned int i = 0; i < uncore_nmonitors(); i++) {
#if UNCORE_PER_CLUSTER
                const unsigned int shift = topology_info->clusters[i].first_cpu_id;
#else /* UNCORE_PER_CLUSTER */
                const unsigned int shift = 0;
#endif /* !UNCORE_PER_CLUSTER */
                uncore_config.uc_cpu_masks[i].uccm_masks[ctr] = (uint16_t)(cpu_mask >> shift);
        }

        *ctr_out = ctr;
        return 0;
}

/*
 * Support for monotonic's mtd_reset function.
 */

/*
 * Reset all configuration and disable the counters if they're currently
 * counting.
 */
static void
uncore_reset(void)
{
        mt_uncore_enabled = false;

        unsigned int curmonid = uncmon_get_curid();

        for (unsigned int monid = 0; monid < uncore_nmonitors(); monid++) {
                struct uncore_monitor *mon = &uncore_monitors[monid];
                bool remote = monid != curmonid;

                int intrs_en = uncmon_lock(mon);
                if (remote) {
#if UNCORE_PER_CLUSTER
                        uncmon_set_counting_locked_r(monid, 0);
#endif /* UNCORE_PER_CLUSTER */
                } else {
                        uncmon_set_counting_locked_l(monid, 0);
                }

                for (int ctr = 0; ctr < UNCORE_NCTRS; ctr++) {
                        if (uncore_active_ctrs & (1U << ctr)) {
                                if (remote) {
#if UNCORE_PER_CLUSTER
                                        uncmon_write_counter_locked_r(monid, ctr, 0);
#endif /* UNCORE_PER_CLUSTER */
                                } else {
                                        uncmon_write_counter_locked_l(monid, ctr, 0);
                                }
                        }
                }

                memset(&mon->um_snaps, 0, sizeof(mon->um_snaps));
                memset(&mon->um_counts, 0, sizeof(mon->um_counts));
                if (remote) {
#if UNCORE_PER_CLUSTER
                        uncmon_clear_int_locked_r(monid);
#endif /* UNCORE_PER_CLUSTER */
                } else {
                        uncmon_clear_int_locked_l(monid);
                }

                uncmon_unlock(mon, intrs_en);
        }

        uncore_active_ctrs = 0;
        memset(&uncore_config, 0, sizeof(uncore_config));

        for (unsigned int monid = 0; monid < uncore_nmonitors(); monid++) {
                struct uncore_monitor *mon = &uncore_monitors[monid];
                bool remote = monid != curmonid;

                int intrs_en = uncmon_lock(mon);
                if (remote) {
#if UNCORE_PER_CLUSTER
                        uncmon_program_events_locked_r(monid);
#endif /* UNCORE_PER_CLUSTER */
                } else {
                        uncmon_program_events_locked_l(monid);
                }
                uncmon_unlock(mon, intrs_en);
        }
}

/*
 * Support for monotonic's mtd_enable function.
 */

static void
uncmon_set_enabled_l(unsigned int monid, bool enable)
{
        struct uncore_monitor *mon = &uncore_monitors[monid];
        int intrs_en = uncmon_lock(mon);

        if (enable) {
                uncmon_program_events_locked_l(monid);
                uncmon_set_counting_locked_l(monid, uncore_active_ctrs);
        } else {
                uncmon_set_counting_locked_l(monid, 0);
        }

        uncmon_unlock(mon, intrs_en);
}

#if UNCORE_PER_CLUSTER

static void
uncmon_set_enabled_r(unsigned int monid, bool enable)
{
        struct uncore_monitor *mon = &uncore_monitors[monid];
        int intrs_en = uncmon_lock(mon);

        if (enable) {
                uncmon_program_events_locked_r(monid);
                uncmon_set_counting_locked_r(monid, uncore_active_ctrs);
        } else {
                uncmon_set_counting_locked_r(monid, 0);
        }

        uncmon_unlock(mon, intrs_en);
}

#endif /* UNCORE_PER_CLUSTER */

static void
uncore_set_enabled(bool enable)
{
        mt_uncore_enabled = enable;

        unsigned int curmonid = uncmon_get_curid();
        for (unsigned int monid = 0; monid < uncore_nmonitors(); monid++) {
                if (monid != curmonid) {
#if UNCORE_PER_CLUSTER
                        uncmon_set_enabled_r(monid, enable);
#endif /* UNCORE_PER_CLUSTER */
                } else {
                        uncmon_set_enabled_l(monid, enable);
                }
        }
}

/*
 * Hooks in the machine layer.
 */

static void
uncore_fiq(uint64_t upmsr)
{
        /*
         * Determine which counters overflowed.
         */
        uint64_t disable_ctr_mask = (upmsr & UPMSR_OVF_MASK) >> UPMSR_OVF_POS;
        /* should not receive interrupts from inactive counters */
        assert(!(disable_ctr_mask & ~uncore_active_ctrs));

        unsigned int monid = uncmon_get_curid();
        struct uncore_monitor *mon = &uncore_monitors[monid];

        int intrs_en = uncmon_lock(mon);

        /*
         * Disable any counters that overflowed.
         */
        uncmon_set_counting_locked_l(monid,
            uncore_active_ctrs & ~disable_ctr_mask);

        /*
         * With the overflowing counters disabled, capture their counts and reset
         * the UPMCs and their snapshots to 0.
         */
        for (unsigned int ctr = 0; ctr < UNCORE_NCTRS; ctr++) {
                if (UPMSR_OVF(upmsr, ctr)) {
                        uncmon_update_locked(monid, monid, ctr);
                        mon->um_snaps[ctr] = 0;
                        uncmon_write_counter_locked_l(monid, ctr, 0);
                }
        }

        /*
         * Acknowledge the interrupt, now that any overflowed PMCs have been reset.
         */
        uncmon_clear_int_locked_l(monid);

        /*
         * Re-enable all active counters.
         */
        uncmon_set_counting_locked_l(monid, uncore_active_ctrs);

        uncmon_unlock(mon, intrs_en);
}

static void
uncore_save(void)
{
        if (!uncore_active_ctrs) {
                return;
        }

        unsigned int curmonid = uncmon_get_curid();

        for (unsigned int monid = 0; monid < uncore_nmonitors(); monid++) {
                struct uncore_monitor *mon = &uncore_monitors[monid];
                int intrs_en = uncmon_lock(mon);

                if (mt_uncore_enabled) {
                        if (monid != curmonid) {
#if UNCORE_PER_CLUSTER
                                uncmon_set_counting_locked_r(monid, 0);
#endif /* UNCORE_PER_CLUSTER */
                        } else {
                                uncmon_set_counting_locked_l(monid, 0);
                        }
                }

                for (unsigned int ctr = 0; ctr < UNCORE_NCTRS; ctr++) {
                        if (uncore_active_ctrs & (1U << ctr)) {
                                uncmon_update_locked(monid, curmonid, ctr);
                        }
                }

                mon->um_sleeping = true;
                uncmon_unlock(mon, intrs_en);
        }
}

static void
uncore_restore(void)
{
        if (!uncore_active_ctrs) {
                return;
        }
        unsigned int curmonid = uncmon_get_curid();

        struct uncore_monitor *mon = &uncore_monitors[curmonid];
        int intrs_en = uncmon_lock(mon);
        if (!mon->um_sleeping) {
                goto out;
        }

        for (unsigned int ctr = 0; ctr < UNCORE_NCTRS; ctr++) {
                if (uncore_active_ctrs & (1U << ctr)) {
                        uncmon_write_counter_locked_l(curmonid, ctr, mon->um_snaps[ctr]);
                }
        }
        uncmon_program_events_locked_l(curmonid);
        uncmon_init_locked_l(curmonid);
        mon->um_sleeping = false;

out:
        uncmon_unlock(mon, intrs_en);
}

#endif /* HAS_UNCORE_CTRS */

#pragma mark common hooks

void
mt_early_init(void)
{
        topology_info = ml_get_topology_info();
}

void
mt_cpu_idle(cpu_data_t *cpu)
{
        core_idle(cpu);
}

void
mt_cpu_run(cpu_data_t *cpu)
{
        struct mt_cpu *mtc;

        assert(cpu != NULL);
        assert(ml_get_interrupts_enabled() == FALSE);

        mtc = &cpu->cpu_monotonic;

        for (int i = 0; i < MT_CORE_NFIXED; i++) {
                mt_core_set_snap(i, mtc->mtc_snaps[i]);
        }

        /* re-enable the counters */
        core_init_execution_modes();

        core_set_enabled();
}

void
mt_cpu_down(cpu_data_t *cpu)
{
        mt_cpu_idle(cpu);
}

void
mt_cpu_up(cpu_data_t *cpu)
{
        mt_cpu_run(cpu);
}

void
mt_sleep(void)
{
#if HAS_UNCORE_CTRS
        uncore_save();
#endif /* HAS_UNCORE_CTRS */
}

void
mt_wake_per_core(void)
{
#if HAS_UNCORE_CTRS
        if (mt_uncore_initted) {
                uncore_restore();
        }
#endif /* HAS_UNCORE_CTRS */
}

uint64_t
mt_count_pmis(void)
{
        uint64_t npmis = 0;
        for (unsigned int i = 0; i < topology_info->num_cpus; i++) {
                cpu_data_t *cpu = (cpu_data_t *)CpuDataEntries[topology_info->cpus[i].cpu_id].cpu_data_vaddr;
                npmis += cpu->cpu_monotonic.mtc_npmis;
        }
        return npmis;
}

static void
mt_cpu_pmi(cpu_data_t *cpu, uint64_t pmcr0)
{
        assert(cpu != NULL);
        assert(ml_get_interrupts_enabled() == FALSE);

        __builtin_arm_wsr64("PMCR0_EL1", PMCR0_INIT);
        /*
         * Ensure the CPMU has flushed any increments at this point, so PMSR is up
         * to date.
         */
        __builtin_arm_isb(ISB_SY);

        cpu->cpu_monotonic.mtc_npmis += 1;
        cpu->cpu_stat.pmi_cnt_wake += 1;

#if MONOTONIC_DEBUG
        if (!PMCR0_PMI(pmcr0)) {
                kprintf("monotonic: mt_cpu_pmi but no PMI (PMCR0 = %#llx)\n",
                    pmcr0);
        }
#else /* MONOTONIC_DEBUG */
#pragma unused(pmcr0)
#endif /* !MONOTONIC_DEBUG */

        uint64_t pmsr = __builtin_arm_rsr64("PMSR_EL1");

#if MONOTONIC_DEBUG
        printf("monotonic: cpu = %d, PMSR = 0x%llx, PMCR0 = 0x%llx\n",
            cpu_number(), pmsr, pmcr0);
#endif /* MONOTONIC_DEBUG */

#if MACH_ASSERT
        uint64_t handled = 0;
#endif /* MACH_ASSERT */

        /*
         * monotonic handles any fixed counter PMIs.
         */
        for (unsigned int i = 0; i < MT_CORE_NFIXED; i++) {
                if ((pmsr & PMSR_OVF(i)) == 0) {
                        continue;
                }

#if MACH_ASSERT
                handled |= 1ULL << i;
#endif /* MACH_ASSERT */
                uint64_t count = mt_cpu_update_count(cpu, i);
                cpu->cpu_monotonic.mtc_counts[i] += count;
                mt_core_set_snap(i, mt_core_reset_values[i]);
                cpu->cpu_monotonic.mtc_snaps[i] = mt_core_reset_values[i];

                if (mt_microstackshots && mt_microstackshot_ctr == i) {
                        bool user_mode = false;
                        arm_saved_state_t *state = get_user_regs(current_thread());
                        if (state) {
                                user_mode = PSR64_IS_USER(get_saved_state_cpsr(state));
                        }
                        KDBG_RELEASE(KDBG_EVENTID(DBG_MONOTONIC, DBG_MT_DEBUG, 1),
                            mt_microstackshot_ctr, user_mode);
                        mt_microstackshot_pmi_handler(user_mode, mt_microstackshot_ctx);
                } else if (mt_debug) {
                        KDBG_RELEASE(KDBG_EVENTID(DBG_MONOTONIC, DBG_MT_DEBUG, 2),
                            i, count);
                }
        }

        /*
         * KPC handles the configurable counter PMIs.
         */
        for (unsigned int i = MT_CORE_NFIXED; i < CORE_NCTRS; i++) {
                if (pmsr & PMSR_OVF(i)) {
#if MACH_ASSERT
                        handled |= 1ULL << i;
#endif /* MACH_ASSERT */
                        extern void kpc_pmi_handler(unsigned int ctr);
                        kpc_pmi_handler(i);
                }
        }

#if MACH_ASSERT
        uint64_t pmsr_after_handling = __builtin_arm_rsr64("PMSR_EL1");
        if (pmsr_after_handling != 0) {
                unsigned int first_ctr_ovf = __builtin_ffsll(pmsr_after_handling) - 1;
                uint64_t count = 0;
                const char *extra = "";
                if (first_ctr_ovf >= CORE_NCTRS) {
                        extra = " (invalid counter)";
                } else {
                        count = mt_core_snap(first_ctr_ovf);
                }

                panic("monotonic: PMI status not cleared on exit from handler, "
                    "PMSR = 0x%llx HANDLE -> -> 0x%llx, handled 0x%llx, "
                    "PMCR0 = 0x%llx, PMC%d = 0x%llx%s", pmsr, pmsr_after_handling,
                    handled, __builtin_arm_rsr64("PMCR0_EL1"), first_ctr_ovf, count, extra);
        }
#endif /* MACH_ASSERT */

        core_set_enabled();
}

#if CPMU_AIC_PMI
void
mt_cpmu_aic_pmi(cpu_id_t source)
{
        struct cpu_data *curcpu = getCpuDatap();
        if (source != curcpu->interrupt_nub) {
                panic("monotonic: PMI from IOCPU %p delivered to %p", source,
                    curcpu->interrupt_nub);
        }
        mt_cpu_pmi(curcpu, __builtin_arm_rsr64("PMCR0_EL1"));
}
#endif /* CPMU_AIC_PMI */

void
mt_fiq(void *cpu, uint64_t pmcr0, uint64_t upmsr)
{
#if CPMU_AIC_PMI
#pragma unused(cpu, pmcr0)
#else /* CPMU_AIC_PMI */
        mt_cpu_pmi(cpu, pmcr0);
#endif /* !CPMU_AIC_PMI */

#if HAS_UNCORE_CTRS
        uncore_fiq(upmsr);
#else /* HAS_UNCORE_CTRS */
#pragma unused(upmsr)
#endif /* !HAS_UNCORE_CTRS */
}

static uint32_t mt_xc_sync;

static void
mt_microstackshot_start_remote(__unused void *arg)
{
        cpu_data_t *cpu = getCpuDatap();

        __builtin_arm_wsr64("PMCR0_EL1", PMCR0_INIT);

        for (int i = 0; i < MT_CORE_NFIXED; i++) {
                uint64_t count = mt_cpu_update_count(cpu, i);
                cpu->cpu_monotonic.mtc_counts[i] += count;
                mt_core_set_snap(i, mt_core_reset_values[i]);
                cpu->cpu_monotonic.mtc_snaps[i] = mt_core_reset_values[i];
        }

        core_set_enabled();

        if (os_atomic_dec(&mt_xc_sync, relaxed) == 0) {
                thread_wakeup((event_t)&mt_xc_sync);
        }
}

int
mt_microstackshot_start_arch(uint64_t period)
{
        uint64_t reset_value = 0;
        int ovf = os_sub_overflow(CTR_MAX, period, &reset_value);
        if (ovf) {
                return ERANGE;
        }

        mt_core_reset_values[mt_microstackshot_ctr] = reset_value;
        cpu_broadcast_xcall(&mt_xc_sync, TRUE, mt_microstackshot_start_remote,
            mt_microstackshot_start_remote /* cannot pass NULL */);
        return 0;
}

#pragma mark dev nodes

struct mt_device mt_devices[] = {
        [0] = {
                .mtd_name = "core",
                .mtd_init = core_init,
        },
#if HAS_UNCORE_CTRS
        [1] = {
                .mtd_name = "uncore",
                .mtd_init = uncore_init,
                .mtd_add = uncore_add,
                .mtd_reset = uncore_reset,
                .mtd_enable = uncore_set_enabled,
                .mtd_read = uncore_read,

                .mtd_ncounters = UNCORE_NCTRS,
        }
#endif /* HAS_UNCORE_CTRS */
};

static_assert(
        (sizeof(mt_devices) / sizeof(mt_devices[0])) == MT_NDEVS,
        "MT_NDEVS macro should be same as the length of mt_devices");