[apple/xnu.git] / osfmk / pmc / pmc.c

/*
 * Copyright (c) 2009 Apple Inc. All rights reserved.
 *
 * @APPLE_LICENSE_HEADER_START@
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this
 * file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_LICENSE_HEADER_END@
 */

#include <kern/kalloc.h>
#include <kern/kern_types.h>
#include <kern/locks.h>
#include <kern/misc_protos.h>
#include <kern/task.h>
#include <kern/thread.h>
#include <kern/zalloc.h>
#include <machine/machine_cpu.h>

#include <pmc/pmc.h>

#include <libkern/OSAtomic.h>

#if defined(__i386__) || defined(__x86_64__)
#include <i386/mp.h>
#endif

#if CONFIG_COUNTERS

/* various debug logging enable */
#undef DEBUG_COUNTERS

typedef uint8_t pmc_state_event_t;

#define PMC_STATE_EVENT_START                           0
#define PMC_STATE_EVENT_STOP                            1
#define PMC_STATE_EVENT_FREE                            2
#define PMC_STATE_EVENT_INTERRUPT                       3
#define PMC_STATE_EVENT_END_OF_INTERRUPT        4
#define PMC_STATE_EVENT_CONTEXT_IN                      5
#define PMC_STATE_EVENT_CONTEXT_OUT                     6
#define PMC_STATE_EVENT_LOAD_FINISHED           7
#define PMC_STATE_EVENT_STORE_FINISHED          8

/* PMC spin timeouts */
#define PMC_SPIN_THRESHOLD      10      /* Number of spins to allow before checking mach_absolute_time() */
#define PMC_SPIN_TIMEOUT_US     10      /* Time in microseconds before the spin causes an assert */

uint64_t pmc_spin_timeout_count = 0;    /* Number of times where a PMC spin loop causes a timeout */

#ifdef DEBUG_COUNTERS
#       include <pexpert/pexpert.h>
#       define COUNTER_DEBUG(...) \
        do { \
                kprintf("[%s:%s][%u] ", __FILE__, __PRETTY_FUNCTION__, cpu_number()); \
                kprintf(__VA_ARGS__); \
        } while(0)

#       define PRINT_PERF_MON(x)        \
        do { \
                kprintf("perfmon: %p (obj: %p refCt: %u switchable: %u)\n", \
                        x, x->object, x->useCount, \
                        x->methods.supports_context_switching ? \
                        x->methods.supports_context_switching(x->object) : 0); \
        } while(0)

static const char const * pmc_state_state_name(pmc_state_t state) {
        switch (PMC_STATE_STATE(state)) {
                case PMC_STATE_STATE_INVALID:
                        return "INVALID";
                case PMC_STATE_STATE_STOP:
                        return "STOP";
                case PMC_STATE_STATE_CAN_RUN:
                        return "CAN_RUN";
                case PMC_STATE_STATE_LOAD:
                        return "LOAD";
                case PMC_STATE_STATE_RUN:
                        return "RUN";
                case PMC_STATE_STATE_STORE:
                        return "STORE";
                case PMC_STATE_STATE_INTERRUPT:
                        return "INTERRUPT";
                case PMC_STATE_STATE_DEALLOC:
                        return "DEALLOC";
                default:
                        return "UNKNOWN";
        }
}

static const char const * pmc_state_event_name(pmc_state_event_t event) {
        switch (event) {
                case PMC_STATE_EVENT_START:
                        return "START";
                case PMC_STATE_EVENT_STOP:
                        return "STOP";
                case PMC_STATE_EVENT_FREE:
                        return "FREE";
                case PMC_STATE_EVENT_INTERRUPT:
                        return "INTERRUPT";
                case PMC_STATE_EVENT_END_OF_INTERRUPT:
                        return "END OF INTERRUPT";
                case PMC_STATE_EVENT_CONTEXT_IN:
                        return "CONTEXT IN";
                case PMC_STATE_EVENT_CONTEXT_OUT:
                        return "CONTEXT OUT";
                case PMC_STATE_EVENT_LOAD_FINISHED:
                        return "LOAD_FINISHED";
                case PMC_STATE_EVENT_STORE_FINISHED:
                        return "STORE_FINISHED";
                default:
                        return "UNKNOWN";
        }
}

#       define PMC_STATE_FORMAT "<%s, %u, %s%s%s>"
#       define PMC_STATE_ARGS(x)        pmc_state_state_name(x), PMC_STATE_CONTEXT_COUNT(x), ((PMC_STATE_FLAGS(x) & PMC_STATE_FLAGS_INTERRUPTING) ? "I" : ""), \
                                        ((PMC_STATE_FLAGS(x) & PMC_STATE_FLAGS_STOPPING) ? "S" : ""), ((PMC_STATE_FLAGS(x) & PMC_STATE_FLAGS_DEALLOCING) ? "D" : "")
#else
#       define COUNTER_DEBUG(...)
#       define PRINT_PERF_MON(x)
#       define PMC_STATE_FORMAT
#       define PMC_STATE_ARGS(x)
#endif

/*!struct
 * pmc_config is the data behind a pmc_config_t.
 * @member object A pointer to an instance of IOPerformanceCounterConfiguration
 * @member method A pointer to a method to call to handle PMI.
 * @member interrupt_after_value Cause a PMI after the counter counts this many
 * events.
 * @member refCon Passed to the @method method as the refCon argument.
 */
struct pmc_config {
        pmc_config_object_t object;
        volatile pmc_interrupt_method_t method;
        uint64_t interrupt_after_value;
        void *refCon;
};

/*
 * Allocation Zones
 * 
 * Two allocation zones - Perf zone small and Perf zone big.
 * Each zone has associated maximums, defined below.
 * The small zone is the max of the smallest allocation objects (all sizes on
 * K64):
 *      perf_monitor_t - 48 bytes
 *              perf_monitor_methods_t - 28 bytes
 *      pmc_reservation_t - 48 bytes
 *  pmc_config_t - 32 bytes
 * perf_small_zone unit size is (on K64) 48 bytes
 * perf_small_zone max count must be max number of perf monitors, plus (max
 * number of reservations * 2). The "*2" is because each reservation has a
 * pmc_config_t within.
 *
 * Big zone is max of the larger allocation units
 *      pmc_t - 144 bytes
 *              pmc_methods_t - 116 bytes
 * perf_big_zone unit size is (on K64) 144 bytes
 * perf_big_zone max count is the max number of PMCs we support.
 */

static zone_t perf_small_zone = NULL;
#define MAX_PERF_SMALLS         (256 + 8196 + 8196)
#define PERF_SMALL_UNIT_SZ      (MAX(MAX(sizeof(struct perf_monitor), \
        sizeof(struct pmc_reservation)), sizeof(struct pmc_config))) 

static zone_t perf_big_zone = NULL;
#define MAX_PERF_BIGS           (1024)
#define PERF_BIG_UNIT_SZ        (sizeof(struct pmc))

/*
 * Locks and Lock groups
 */
static lck_grp_t *pmc_lock_grp = LCK_GRP_NULL;
static lck_grp_attr_t *pmc_lock_grp_attr;
static lck_attr_t *pmc_lock_attr;

/* PMC tracking queue locks */
static lck_spin_t perf_monitor_queue_spin;              /* protects adding and removing from queue */
static lck_spin_t perf_counters_queue_spin;             /* protects adding and removing from queue */

/* Reservation tracking queues lock */
static lck_spin_t reservations_spin;

/*
 * Tracking queues
 *
 * Keeps track of registered perf monitors and perf counters
 */
static queue_t perf_monitors_queue = NULL;
static volatile uint32_t perf_monitors_count = 0U;

static queue_t perf_counters_queue = NULL;
static volatile uint32_t perf_counters_count = 0U;

/* 
 * Reservation queues
 *
 * Keeps track of all system, task, and thread-level reservations (both active and
 * inactive).
 *
 * We track them all here (rather than in their respective task or thread only)
 * so that we can inspect our tracking data directly (rather than peeking at
 * every task and thread) to determine if/when a new reservation would
 * constitute a conflict.
 */
static queue_t system_reservations = NULL;
static volatile uint32_t system_reservation_count = 0U;

static queue_t task_reservations = NULL;
static volatile uint32_t task_reservation_count = 0U;

static queue_t thread_reservations = NULL;
static volatile uint32_t thread_reservation_count = 0U;


#if XNU_KERNEL_PRIVATE

/*
 * init_pmc_locks creates and initializes all the locks and lock groups and lock
 * attributes required for the pmc sub-system.
 */
static void init_pmc_locks(void) {
        pmc_lock_attr = lck_attr_alloc_init();
        assert(pmc_lock_attr);

        pmc_lock_grp_attr = lck_grp_attr_alloc_init();
        assert(pmc_lock_grp_attr);

        pmc_lock_grp = lck_grp_alloc_init("pmc", pmc_lock_grp_attr);
        assert(pmc_lock_grp);

        lck_spin_init(&perf_monitor_queue_spin, pmc_lock_grp, pmc_lock_attr);
        lck_spin_init(&perf_counters_queue_spin, pmc_lock_grp, pmc_lock_attr);

        lck_spin_init(&reservations_spin, pmc_lock_grp, pmc_lock_attr);
}

/*
 * init_pmc_zones initializes the allocation zones used by the pmc subsystem
 */
static void init_pmc_zones(void) {
        perf_small_zone = zinit(PERF_SMALL_UNIT_SZ, 
                MAX_PERF_SMALLS * PERF_SMALL_UNIT_SZ, MAX_PERF_SMALLS, 
                "pmc.small zone");

        assert(perf_small_zone);

        perf_big_zone = zinit(PERF_BIG_UNIT_SZ,
                MAX_PERF_BIGS * PERF_BIG_UNIT_SZ, MAX_PERF_BIGS, 
                "pmc.big zone");

        assert(perf_big_zone);
}

/*
 * init_pmc_queues allocates and initializes the tracking queues for
 * registering and reserving individual pmcs and perf monitors.
 */
static void init_pmc_queues(void) {
        perf_monitors_queue = (queue_t)kalloc(sizeof(queue_t));
        assert(perf_monitors_queue);

        queue_init(perf_monitors_queue);

        perf_counters_queue = (queue_t)kalloc(sizeof(queue_t));
        assert(perf_counters_queue);

        queue_init(perf_counters_queue);

        system_reservations = (queue_t)kalloc(sizeof(queue_t));
        assert(system_reservations);

        queue_init(system_reservations);

        task_reservations = (queue_t)kalloc(sizeof(queue_t));
        assert(task_reservations);

        queue_init(task_reservations);

        thread_reservations = (queue_t)kalloc(sizeof(queue_t));
        assert(thread_reservations);

        queue_init(thread_reservations);
}

/*
 * pmc_bootstrap brings up all the necessary infrastructure required to use the
 * pmc sub-system.
 */
__private_extern__
void pmc_bootstrap(void) {
        /* build our alloc zones */
        init_pmc_zones();

        /* build the locks */
        init_pmc_locks();

        /* build our tracking queues */
        init_pmc_queues();
}

#endif /* XNU_KERNEL_PRIVATE */

/*
 * Perf Monitor Internals
 */

static perf_monitor_t perf_monitor_alloc(void) {
        /* perf monitors come from the perf small zone */
        return (perf_monitor_t)zalloc(perf_small_zone);
}

static void perf_monitor_free(void *pm) {
        zfree(perf_small_zone, pm);
}

static void perf_monitor_init(perf_monitor_t pm) {
        assert(pm);

        pm->object = NULL;

        bzero(&(pm->methods), sizeof(perf_monitor_methods_t));

        pm->useCount = 1;       /* initial retain count of 1, for caller */

        pm->link.next = pm->link.prev = (queue_entry_t)NULL;
}

/*
 * perf_monitor_dequeue removes the given perf_monitor_t from the
 * perf_monitor_queue, thereby unregistering it with the system.
 */
static void perf_monitor_dequeue(perf_monitor_t pm) {
        lck_spin_lock(&perf_monitor_queue_spin);
        
        /* 
         * remove the @pm object from the @perf_monitor_queue queue (it is of type
         * <perf_monitor_t> and has a field called @link that is the queue_link_t
         */
        queue_remove(perf_monitors_queue, pm, perf_monitor_t, link);

        perf_monitors_count--;

        lck_spin_unlock(&perf_monitor_queue_spin);
}

/*
 * perf_monitor_enqueue adds the given perf_monitor_t to the perf_monitor_queue,
 * thereby registering it for use with the system.
 */
static void perf_monitor_enqueue(perf_monitor_t pm) {
        lck_spin_lock(&perf_monitor_queue_spin);

        queue_enter(perf_monitors_queue, pm, perf_monitor_t, link);

        perf_monitors_count++;

        lck_spin_unlock(&perf_monitor_queue_spin);
}

/*
 * perf_monitor_reference increments the reference count for the given
 * perf_monitor_t.
 */
static void perf_monitor_reference(perf_monitor_t pm) {
        assert(pm);

        OSIncrementAtomic(&(pm->useCount));
}

/*
 * perf_monitor_deallocate decrements the reference count for the given
 * perf_monitor_t.  If the reference count hits 0, the object is released back
 * to the perf_small_zone via a call to perf_monitor_free().
 */
static void perf_monitor_deallocate(perf_monitor_t pm) {
        assert(pm);

        /* If we just removed the last reference count */
        if(1 == OSDecrementAtomic(&(pm->useCount))) {
                /* Free the object */
                perf_monitor_free(pm);
        }
}

/*
 * perf_monitor_find attempts to find a perf_monitor_t that corresponds to the
 * given C++ object pointer that was used when registering with the subsystem.
 *
 * If found, the method returns the perf_monitor_t with an extra reference 
 * placed on the object (or NULL if not
 * found).
 *
 * NOTE: Caller must use perf_monitor_deallocate to remove the extra reference after
 * calling perf_monitor_find.
 */
static perf_monitor_t perf_monitor_find(perf_monitor_object_t monitor) {
        assert(monitor);
        perf_monitor_t element = NULL;
        perf_monitor_t found = NULL;

        lck_spin_lock(&perf_monitor_queue_spin);
        
        queue_iterate(perf_monitors_queue, element, perf_monitor_t, link) {
                if(element && element->object == monitor) {
                        /* We found it - reference the object. */
                        perf_monitor_reference(element);
                        found = element;
                        break;
                }
        }

        lck_spin_unlock(&perf_monitor_queue_spin);

        return found;
}

/*
 * perf_monitor_add_pmc adds a newly registered PMC to the perf monitor it is
 * aassociated with.
 */
static void perf_monitor_add_pmc(perf_monitor_t pm, pmc_t pmc __unused) {
        assert(pm);
        assert(pmc);

        /* Today, we merely add a reference count now that a new pmc is attached */
        perf_monitor_reference(pm);
}

/*
 * perf_monitor_remove_pmc removes a newly *un*registered PMC from the perf
 * monitor it is associated with.
 */
static void perf_monitor_remove_pmc(perf_monitor_t pm, pmc_t pmc __unused) {
        assert(pm);
        assert(pmc);

        /* Today, we merely remove a reference count now that the pmc is detached */
        perf_monitor_deallocate(pm);
}

/*
 * Perf Counter internals
 */

static pmc_t pmc_alloc(void) {
        return (pmc_t)zalloc(perf_big_zone);
}

static void pmc_free(void *pmc) {
        zfree(perf_big_zone, pmc);
}

/*
 * pmc_init initializes a newly allocated pmc_t
 */
static void pmc_init(pmc_t pmc) {
        assert(pmc);

        pmc->object = NULL;
        pmc->monitor = NULL;

        bzero(&pmc->methods, sizeof(pmc_methods_t));

        /* One reference for the caller */
        pmc->useCount = 1;
}

/*
 * pmc_reference increments the reference count of the given pmc_t
 */
static void pmc_reference(pmc_t pmc) {
        assert(pmc);

        OSIncrementAtomic(&(pmc->useCount));
}

/*
 * pmc_deallocate decrements the reference count of the given pmc_t. If the
 * reference count hits zero, the given pmc_t is deallocated and released back
 * to the allocation zone.
 */
static void pmc_deallocate(pmc_t pmc) {
        assert(pmc);

        /* If we just removed the last reference count */
        if(1 == OSDecrementAtomic(&(pmc->useCount))) {
                /* Free the pmc */
                pmc_free(pmc);
        }
}

/*
 * pmc_dequeue removes the given, newly *un*registered pmc from the
 * perf_counters_queue.
 */
static void pmc_dequeue(pmc_t pmc) {
        lck_spin_lock(&perf_counters_queue_spin);

        queue_remove(perf_counters_queue, pmc, pmc_t, link);

        perf_counters_count--;

        lck_spin_unlock(&perf_counters_queue_spin);
}

/*
 * pmc_enqueue adds the given, newly registered pmc to the perf_counters_queue
 */
static void pmc_enqueue(pmc_t pmc) {
        lck_spin_lock(&perf_counters_queue_spin);

        queue_enter(perf_counters_queue, pmc, pmc_t, link);

        perf_counters_count++;

        lck_spin_unlock(&perf_counters_queue_spin);
}

/*
 * pmc_find attempts to locate a pmc_t that was registered with the given
 * pmc_object_t pointer.  If found, it returns the pmc_t with an extra reference
 * which must be dropped by the caller by calling pmc_deallocate().
 */
static pmc_t pmc_find(pmc_object_t object) {
        assert(object);

        lck_spin_lock(&perf_counters_queue_spin);
        
        pmc_t element = NULL;
        pmc_t found = NULL;

        queue_iterate(perf_counters_queue, element, pmc_t, link) {
                if(element && element->object == object) {
                        pmc_reference(element);

                        found = element;
                        break;
                }
        }

        lck_spin_unlock(&perf_counters_queue_spin);

        return found;
}

/*
 * Config internals
 */

/* Allocate a pmc_config_t */
static pmc_config_t pmc_config_alloc(pmc_t pmc __unused) {
        return (pmc_config_t)zalloc(perf_small_zone);
}

/* Free a pmc_config_t, and underlying pmc_config_object_t (if needed) */
static void pmc_config_free(pmc_t pmc, pmc_config_t config) {
        assert(pmc);
        assert(config);

        if(config->object) {
                pmc->methods.free_config(pmc->object, config->object);
                config->object = NULL;
        }

        zfree(perf_small_zone, config);
}

static kern_return_t pmc_open(pmc_t pmc) {
        assert(pmc);
        assert(pmc->object);
        assert(pmc->open_object);

        return pmc->methods.open(pmc->object, pmc->open_object);
}

static kern_return_t pmc_close(pmc_t pmc) {
        assert(pmc);
        assert(pmc->object);
        assert(pmc->open_object);

        return pmc->methods.close(pmc->object, pmc->open_object);
}

/*
 * Reservation Internals
 */

static kern_return_t pmc_internal_reservation_set_pmc(pmc_reservation_t resv, pmc_t pmc);
static void pmc_internal_reservation_store(pmc_reservation_t reservation);
static void pmc_internal_reservation_load(pmc_reservation_t reservation);

static pmc_reservation_t reservation_alloc(void) {
        /* pmc reservations come from the perf small zone */
        return (pmc_reservation_t)zalloc(perf_small_zone);
}

/*
 * reservation_free deallocates and releases all resources associated with the
 * given pmc_reservation_t.  This includes freeing the config used to create the
 * reservation, decrementing the reference count for the pmc used to create the
 * reservation, and deallocating the reservation's memory.
 */
static void reservation_free(pmc_reservation_t resv) {
        /* Free config */
        if(resv->config) {
                assert(resv->pmc);

                pmc_free_config(resv->pmc, resv->config);

                resv->config = NULL;
        }

        /* release PMC */
        (void)pmc_internal_reservation_set_pmc(resv, NULL);

        /* Free reservation */
        zfree(perf_small_zone, resv);
}

/*
 * reservation_init initializes a newly created reservation.
 */
static void reservation_init(pmc_reservation_t resv) {
        assert(resv);

        resv->pmc = NULL;
        resv->config = NULL;
        resv->value = 0ULL;

        resv->flags = 0U;
        resv->state = PMC_STATE(PMC_STATE_STATE_STOP, 0, 0);
        resv->active_last_context_in = 0U;

        /*
         * Since this member is a union, we only need to set either the task 
         * or thread to NULL.
         */
        resv->task = TASK_NULL;
}

/*
 * pmc_internal_reservation_set_pmc sets the pmc associated with the reservation object. If
 * there was one set already, it is deallocated (reference is dropped) before
 * the new one is set.  This methods increases the reference count of the given
 * pmc_t.
 *
 * NOTE: It is okay to pass NULL as the pmc_t - this will have the effect of
 * dropping the reference on any previously set pmc, and setting the reservation
 * to having no pmc set.
 */
static kern_return_t pmc_internal_reservation_set_pmc(pmc_reservation_t resv, pmc_t pmc) {
        assert(resv);

        if(resv->pmc) {
                (void)pmc_close(resv->pmc);
                pmc_deallocate(resv->pmc);
                resv->pmc = NULL;
        }

        resv->pmc = pmc;

        if(resv->pmc) {
                pmc_reference(resv->pmc);
                if(KERN_SUCCESS != pmc_open(resv->pmc)) {
                        pmc_deallocate(resv->pmc);
                        resv->pmc = NULL;

                        return KERN_FAILURE;
                }
        }

        return KERN_SUCCESS;
}

/* 
 * Used to place reservation into one of the system, task, and thread queues
 * Assumes the queue's spin lock is already held.
 */
static void pmc_internal_reservation_enqueue(queue_t queue, pmc_reservation_t resv) {
        assert(queue);
        assert(resv);

        queue_enter(queue, resv, pmc_reservation_t, link);
}

static void pmc_internal_reservation_dequeue(queue_t queue, pmc_reservation_t resv) {
        assert(queue);
        assert(resv);

        queue_remove(queue, resv, pmc_reservation_t, link);
}

/* Returns TRUE if the reservation applies to the current execution context */
static boolean_t pmc_internal_reservation_matches_context(pmc_reservation_t resv) {
        boolean_t ret = FALSE;
        assert(resv);

        if(PMC_FLAG_IS_SYSTEM_SCOPE(resv->flags)) {
                ret = TRUE;
        } else if(PMC_FLAG_IS_TASK_SCOPE(resv->flags)) {
                if(current_task() == resv->task) {
                        ret = TRUE;
                }
        } else if(PMC_FLAG_IS_THREAD_SCOPE(resv->flags)) {
                if(current_thread() == resv->thread) {
                        ret = TRUE;
                }
        }

        return ret;
}

/*
 * pmc_accessible_core_count returns the number of logical cores that can access
 * a given @pmc.  0 means every core in the system.
 */
static uint32_t pmc_accessible_core_count(pmc_t pmc) {
        assert(pmc);

        uint32_t *cores = NULL;
        size_t coreCt = 0UL;

        if(KERN_SUCCESS != pmc->methods.accessible_cores(pmc->object,
                &cores, &coreCt)) {
                coreCt = 0U;
        }

        return (uint32_t)coreCt;
}

/* spin lock for the queue must already be held */
/*
 * This method will inspect the task/thread of the reservation to see if it
 * matches the new incoming one (for thread/task reservations only).  Will only
 * return TRUE if the task/thread matches.
 */
static boolean_t pmc_internal_reservation_queue_contains_pmc(queue_t queue, pmc_reservation_t
resv) {
        assert(queue);
        assert(resv);

        boolean_t ret = FALSE;
        pmc_reservation_t tmp = NULL;

        queue_iterate(queue, tmp, pmc_reservation_t, link) {
                if(tmp) {
                        if(tmp->pmc == resv->pmc) {
                                /* PMC matches - make sure scope matches first */
                                switch(PMC_FLAG_SCOPE(tmp->flags)) {
                                        case PMC_FLAG_SCOPE_SYSTEM:
                                                /*
                                                 * Found a reservation in system queue with same pmc - always a
                                                 * conflict.
                                                 */
                                                ret = TRUE;
                                                break;
                                        case PMC_FLAG_SCOPE_THREAD:
                                                /*
                                                 * Found one in thread queue with the same PMC as the
                                                 * argument. Only a conflict if argument scope isn't
                                                 * thread or system, or the threads match.
                                                 */
                                                ret = (PMC_FLAG_SCOPE(resv->flags) != PMC_FLAG_SCOPE_THREAD) || 
                                                        (tmp->thread == resv->thread);

                                                if(!ret) {
                                                        /*
                                                         * so far, no conflict - check that the pmc that is
                                                         * being reserved isn't accessible from more than
                                                         * one core, if it is, we need to say it's already
                                                         * taken.
                                                         */
                                                        if(1 != pmc_accessible_core_count(tmp->pmc)) {
                                                                ret = TRUE;
                                                        }
                                                }
                                                break;
                                        case PMC_FLAG_SCOPE_TASK:
                                                /* 
                                                 * Follow similar semantics for task scope.
                                                 */

                                                ret = (PMC_FLAG_SCOPE(resv->flags) != PMC_FLAG_SCOPE_TASK) ||
                                                        (tmp->task == resv->task);
                                                if(!ret) {
                                                        /*
                                                         * so far, no conflict - check that the pmc that is
                                                         * being reserved isn't accessible from more than
                                                         * one core, if it is, we need to say it's already
                                                         * taken.
                                                         */
                                                        if(1 != pmc_accessible_core_count(tmp->pmc)) {
                                                                ret = TRUE;
                                                        }
                                                }

                                                break;
                                }

                                if(ret) break;
                        }
                }
        }

        return ret;
}

/*
 * pmc_internal_reservation_validate_for_pmc returns TRUE if the given reservation can be 
 * added to its target queue without createing conflicts (target queue is 
 * determined by the reservation's scope flags). Further, this method returns
 * FALSE if any level contains a reservation for a PMC that can be accessed from
 * more than just 1 core, and the given reservation also wants the same PMC.
 */
static boolean_t pmc_internal_reservation_validate_for_pmc(pmc_reservation_t resv) {
        assert(resv);
        boolean_t ret = TRUE;

        if(pmc_internal_reservation_queue_contains_pmc(system_reservations, resv) ||
                pmc_internal_reservation_queue_contains_pmc(task_reservations, resv) ||
                pmc_internal_reservation_queue_contains_pmc(thread_reservations, resv)) {
                ret = FALSE;
        }

        return ret;
}

static void pmc_internal_update_thread_flag(thread_t thread, boolean_t newFlag) {
        assert(thread);

        /* See if this thread needs it's PMC flag set */
        pmc_reservation_t tmp = NULL;

        if(!newFlag) {
                /*
                 * If the parent task just dropped its reservation, iterate the thread
                 * reservations to see if we need to keep the pmc flag set for the given
                 * thread or not.
                 */
                lck_spin_lock(&reservations_spin);
        
                queue_iterate(thread_reservations, tmp, pmc_reservation_t, link) {
                        if(tmp->thread == thread) {
                                newFlag = TRUE;
                                break;
                        }
                }

                lck_spin_unlock(&reservations_spin);
        }

        if(newFlag) {
                OSBitOrAtomic(THREAD_PMC_FLAG, &thread->t_chud);
        } else {
                OSBitAndAtomic(~(THREAD_PMC_FLAG), &thread->t_chud);
        }
}

/* 
 * This operation is (worst case) O(N*M) where N is number of threads in the
 * given task, and M is the number of thread reservations in our system.
 */
static void pmc_internal_update_task_flag(task_t task, boolean_t newFlag) {
        assert(task);
        thread_t thread = NULL;

        if(newFlag) {
                OSBitOrAtomic(TASK_PMC_FLAG, &task->t_chud);
        } else {
                OSBitAndAtomic(~(TASK_PMC_FLAG), &task->t_chud);
        }

        task_lock(task);

        queue_iterate(&task->threads, thread, thread_t, task_threads) {
                /* propagate the task's mask down to each thread  */
                pmc_internal_update_thread_flag(thread, newFlag);
        }

        task_unlock(task);
}

/*
 * pmc_internal_reservation_add adds a reservation to the global tracking queues after
 * ensuring there are no reservation conflicts.  To do this, it takes all the
 * spin locks for all the queue (to ensure no other core goes and adds a
 * reservation for the same pmc to a queue that has already been checked).
 */
static boolean_t pmc_internal_reservation_add(pmc_reservation_t resv) {
        assert(resv);

        boolean_t ret = FALSE;

        /* always lock all three in the same order */
        lck_spin_lock(&reservations_spin);

        /* Check if the reservation can be added without conflicts */
        if(pmc_internal_reservation_validate_for_pmc(resv)) {
                ret = TRUE;
        }

        if(ret) {
                /* add reservation to appropriate scope */
                switch(PMC_FLAG_SCOPE(resv->flags)) {

                        /* System-wide counter */
                        case PMC_FLAG_SCOPE_SYSTEM:
                                /* Simply add it to the system queue */
                                pmc_internal_reservation_enqueue(system_reservations, resv);
                                system_reservation_count++;
                                
                                lck_spin_unlock(&reservations_spin);

                                break;

                        /* Task-switched counter */
                        case PMC_FLAG_SCOPE_TASK:
                                assert(resv->task);

                                /* Not only do we enqueue it in our local queue for tracking */
                                pmc_internal_reservation_enqueue(task_reservations, resv);
                                task_reservation_count++;

                                lck_spin_unlock(&reservations_spin);

                                /* update the task mask, and propagate it to existing threads */
                                pmc_internal_update_task_flag(resv->task, TRUE);
                                break;

                        /* Thread-switched counter */
                        case PMC_FLAG_SCOPE_THREAD:
                                assert(resv->thread);

                                /*
                                 * Works the same as a task-switched counter, only at
                                 * thread-scope
                                 */

                                pmc_internal_reservation_enqueue(thread_reservations, resv);
                                thread_reservation_count++;

                                lck_spin_unlock(&reservations_spin);
                                
                                pmc_internal_update_thread_flag(resv->thread, TRUE);
                                break;
                        }
        } else {
                lck_spin_unlock(&reservations_spin);
        }                       
        
        return ret;
}

static void pmc_internal_reservation_broadcast(pmc_reservation_t reservation, void (*action_func)(void *)) {
        uint32_t * cores;
        size_t core_cnt;
        
        /* Get the list of accessible cores */
        if (KERN_SUCCESS == pmc_get_accessible_core_list(reservation->pmc, &cores, &core_cnt)) {
                boolean_t intrs_enabled = ml_set_interrupts_enabled(FALSE);

                /* Fast case: the PMC is only accessible from one core and we happen to be on it */
                if (core_cnt == 1 && cores[0] == (uint32_t)cpu_number()) {
                        action_func(reservation);
                } else {
                        /* Call action_func on every accessible core */
#if defined(__i386__) || defined(__x86_64__)
                        size_t ii;
                        cpumask_t mask = 0;
                        
                        /* Build a mask for the accessible cores */
                        if (core_cnt > 0) {
                                for (ii = 0; ii < core_cnt; ii++) {
                                        mask |= cpu_to_cpumask(cores[ii]);
                                }
                        } else {
                                /* core_cnt = 0 really means all cpus */
                                mask = CPUMASK_ALL;
                        }
                        
                        /* Have each core run pmc_internal_reservation_stop_cpu asynchronously. */
                        mp_cpus_call(mask, ASYNC, action_func, reservation);
#else
#error pmc_reservation_interrupt needs an inter-processor method invocation mechanism for this architecture
#endif
                }

                ml_set_interrupts_enabled(intrs_enabled);
        }
        
}

/*
 * pmc_internal_reservation_remove removes the given reservation from the appropriate
 * reservation queue according to its scope. 
 *
 * NOTE: The scope flag must have been set for this method to function.
 */
static void pmc_internal_reservation_remove(pmc_reservation_t resv) {
        assert(resv);

        /*
         * Due to the way the macros are written, we can't just blindly queue-remove
         * the reservation without knowing which queue it's in. We figure this out
         * using the reservation's scope flags.
         */

        switch(PMC_FLAG_SCOPE(resv->flags)) {

                case PMC_FLAG_SCOPE_SYSTEM:
                        lck_spin_lock(&reservations_spin);
                        pmc_internal_reservation_dequeue(system_reservations, resv);
                        system_reservation_count--;
                        lck_spin_unlock(&reservations_spin);
                        break;

                case PMC_FLAG_SCOPE_TASK:
                        
                        /* Lock the global spin lock */
                        lck_spin_lock(&reservations_spin);

                        /* remove from the global queue */
                        pmc_internal_reservation_dequeue(task_reservations, resv);
                        task_reservation_count--;

                        /* unlock the global */
                        lck_spin_unlock(&reservations_spin);

                        /* Recalculate task's counter mask */
                        pmc_internal_update_task_flag(resv->task, FALSE);
                        break;

                case PMC_FLAG_SCOPE_THREAD:
                        lck_spin_lock(&reservations_spin);

                        pmc_internal_reservation_dequeue(thread_reservations, resv);
                        thread_reservation_count--;

                        lck_spin_unlock(&reservations_spin);

                        /* recalculate the thread's counter mask */
                        pmc_internal_update_thread_flag(resv->thread, FALSE);

                        break;
        }
}

/* Reservation State Machine
 *
 * The PMC subsystem uses a 3-tuple of state information packed into a 32-bit quantity and a 
 * set of 9 events to provide MP-safe bookkeeping and control flow.  The 3-tuple is comprised 
 * of a state, a count of active contexts, and a set of modifier flags.  A state machine defines
 * the possible transitions at each event point given the current 3-tuple.  Atomicity is handled
 * by reading the current 3-tuple, applying the transformations indicated by the state machine
 * and then attempting to OSCompareAndSwap the transformed value.  If the OSCompareAndSwap fails,
 * the process is repeated until either the OSCompareAndSwap succeeds or not valid transitions are
 * available.
 *
 * The state machine is described using tuple notation for the current state and a related notation
 * for describing the transformations.  For concisness, the flag and state names are abbreviated as
 * follows:
 * 
 * states:
 * S = STOP
 * CR = CAN_RUN
 * L = LOAD
 * R = RUN
 * ST = STORE
 * I = INTERRUPT
 * D = DEALLOC
 *
 * flags:
 *
 * S = STOPPING
 * D = DEALLOCING
 * I = INTERRUPTING
 *
 * The tuple notation is formed from the following pattern:
 *
 * tuple = < state, active-context-count, flags >
 * state = S | CR | L | R | ST | I | D
 * active-context-count = 0 | >0 | 1 | >1
 * flags = flags flag | blank
 * flag = S | D | I
 *
 * The transform notation is similar, but only describes the modifications made to the current state.
 * The notation is formed from the following pattern:
 * 
 * transform = < state, active-context-count, flags >
 * state = S | CR | L | R | ST | I | D
 * active-context-count = + | - | blank
 * flags = flags flag | flags !flag | blank
 * flag = S | D | I
 *
 * And now for the state machine:
 * State                Start           Stop            Free            Interrupt               End Interrupt           Context In              Context Out     Load Finished           Store Finished
 * <CR, 0, >                            <S, , >         <D, , >                 <L, +, >
 * <D, 0, >
 * <D, 1, D>                                                                    < , -, !D>
 * <D, >1, D>                                                                   < , -, >
 * <I, 0, D>                                                                    <D, , !D>
 * <I, 0, S>    < , , !S>                               < , , !SD>              <S, , !S>
 * <I, 0, >                                     < , , S>        < , , D>        <CR, , >
 * <L, 1, D>                                                                    <ST, -, >
 * <L, 1, ID>                                                                   <ST, -, >
 * <L, 1, IS>                                                   < , , !SD>      <ST, -, >
 * <L, 1, S>    < , , !S>                               < , , !SD>              <ST, -, >
 * <L, 1, >                                     < , , S>        < , , D>        < , , IS>                                                       < , +, >        <R, , >
 * <L, >1, D>                                                                   < , -, >                <R, -, >
 * <L, >1, ID>                                                                  < , -, >                <R, -, >
 * <L, >1, IS>                                                  < , , !SD>      < , -, >                <R, -, >
 * <L, >1, S>   < , , !S>                               < , , !SD>              < , -, >                <R, -, >
 * <L, >1, >                            < , , S>        < , , D>        < , , IS>                                                       < , +, >                < , -, >                <R, , >
 * <R, 1, D>                                                                    <ST, -, >
 * <R, 1, ID>                                                                   <ST, -, >
 * <R, 1, IS>                                                   < , , !SD>      <ST, -, >
 * <R, 1, S>    < , , !S>                               < , , !SD>              <ST, -, >
 * <R, 1, >                                     < , , S>        < , , D>        < , , IS>                                                       < , +, >        <ST, -, >
 * <R, >1, D>                                                                   < , -, >
 * <R, >1, ID>                                                                  < , -, >
 * <R, >1, IS>                                                  < , , !SD>      < , -, >
 * <R, >1, S>   < , , !S>                               < , , !SD>              < , -, >
 * <R, >1, >                            < , , S>        < , , D>        < , , IS>                                                       < , +, >                < , -, >
 * <S, 0, >             <CR, , >                                <D, , >
 * <S, 1, ID>                                                                   <I, -, !I>
 * <S, 1, IS>                                                   < , , !SD>      <I, -, !I>
 * <S, 1, S>    < , , !S>                               <D, , !SD>              < , -, !S>
 * <S, 1, >                                     < , , S>        <D, , D>        <L, +, >                <CR, -, >
 * <S, >1, ID>                                                                  < , -, >
 * <S, >1, IS>                                                  < , , !SD>      < , -, >
 * <S, >1, S>   < , , !S>                               <D, , !SD>              < , -, >
 * <S, >1, >                            < , , S>        <D, , D>                <L, +, >                < , -, >
 * <ST, 0, D>                                                                   <D, , !D>
 * <ST, 0, ID>                                                                  <I, , !I>
 * <ST, 0, IS>                                                  < , , !SD>      <I, , !I>
 * <ST, 0, S>   < , , !S>                               < , , !SD>              <S, , !S>
 * <ST, 0, >                            < , , S>        < , , D>        < , , IS>                                                       < , +, >                <CR, , >
 * <ST, >0, D>                                                                  < , -, >                                                        <D, , >
 * <ST, >0, ID>                                                         < , -, >                                                        <S, , >
 * <ST, >0, IS>                                                 < , , !SD>                                                                              < , -, >                        <S, , >
 * <ST, >0, S>  < , , !S>                               < , , !SD>              < , -, >                                                        <S, , >
 * <ST, >0, >                           < , , S>        < , , D>        < , , IS>                                                       < , +, >                < , -, >                        <L, , >
 */

static uint32_t pmc_internal_reservation_next_state(uint32_t current_state, pmc_state_event_t event) {
        uint32_t new_state = PMC_STATE(PMC_STATE_STATE_INVALID, 0, 0);
        
        switch (event) {
                case PMC_STATE_EVENT_START:
                        switch (current_state & ~(PMC_STATE_CONTEXT_COUNT_MASK)) {
                                case PMC_STATE(PMC_STATE_STATE_INTERRUPT, 0, PMC_STATE_FLAGS_STOPPING):
                                case PMC_STATE(PMC_STATE_STATE_LOAD, 0, PMC_STATE_FLAGS_STOPPING):
                                case PMC_STATE(PMC_STATE_STATE_RUN, 0, PMC_STATE_FLAGS_STOPPING):
                                case PMC_STATE(PMC_STATE_STATE_STOP, 0, PMC_STATE_FLAGS_STOPPING):
                                case PMC_STATE(PMC_STATE_STATE_STORE, 0, PMC_STATE_FLAGS_STOPPING):
                                        new_state = PMC_STATE_MODIFY(current_state, 0, 0, PMC_STATE_FLAGS_STOPPING);
                                        break;
                                case PMC_STATE(PMC_STATE_STATE_STOP, 0, 0):
                                        if (PMC_STATE_CONTEXT_COUNT(current_state) == 0) {
                                                new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_CAN_RUN, 0, 0, 0);
                                        }
                                        break;
                        }
                        break;
                case PMC_STATE_EVENT_STOP:
                        switch (current_state & ~(PMC_STATE_CONTEXT_COUNT_MASK)) {
                                case PMC_STATE(PMC_STATE_STATE_CAN_RUN, 0, 0):
                                        new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_STOP, 0, 0, 0);
                                        break;
                                case PMC_STATE(PMC_STATE_STATE_INTERRUPT, 0, 0):
                                case PMC_STATE(PMC_STATE_STATE_LOAD, 0, 0):
                                case PMC_STATE(PMC_STATE_STATE_RUN, 0, 0):
                                case PMC_STATE(PMC_STATE_STATE_STORE, 0, 0):
                                        new_state = PMC_STATE_MODIFY(current_state, 0, PMC_STATE_FLAGS_STOPPING, 0);
                                        break;
                                case PMC_STATE(PMC_STATE_STATE_STOP, 0, 0):
                                        if (PMC_STATE_CONTEXT_COUNT(current_state) > 0) {
                                                new_state = PMC_STATE_MODIFY(current_state, 0, PMC_STATE_FLAGS_STOPPING, 0);
                                        }
                                        break;
                        }
                        break;
                case PMC_STATE_EVENT_FREE:
                        switch (current_state & ~(PMC_STATE_CONTEXT_COUNT_MASK)) {
                                case PMC_STATE(PMC_STATE_STATE_CAN_RUN, 0, 0):
                                        new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_DEALLOC, 0, 0, 0);
                                        break;
                                case PMC_STATE(PMC_STATE_STATE_INTERRUPT, 0, PMC_STATE_FLAGS_STOPPING):
                                case PMC_STATE(PMC_STATE_STATE_LOAD, 0, PMC_STATE_FLAGS_INTERRUPTING | PMC_STATE_FLAGS_STOPPING):
                                case PMC_STATE(PMC_STATE_STATE_LOAD, 0, PMC_STATE_FLAGS_STOPPING):
                                case PMC_STATE(PMC_STATE_STATE_RUN, 0, PMC_STATE_FLAGS_INTERRUPTING | PMC_STATE_FLAGS_STOPPING):
                                case PMC_STATE(PMC_STATE_STATE_RUN, 0, PMC_STATE_FLAGS_STOPPING):
                                case PMC_STATE(PMC_STATE_STATE_STOP, 0, PMC_STATE_FLAGS_INTERRUPTING | PMC_STATE_FLAGS_STOPPING):
                                case PMC_STATE(PMC_STATE_STATE_STORE, 0, PMC_STATE_FLAGS_INTERRUPTING | PMC_STATE_FLAGS_STOPPING):
                                case PMC_STATE(PMC_STATE_STATE_STORE, 0, PMC_STATE_FLAGS_STOPPING):
                                        new_state = PMC_STATE_MODIFY(current_state, 0, PMC_STATE_FLAGS_DEALLOCING, PMC_STATE_FLAGS_STOPPING);
                                        break;
                                case PMC_STATE(PMC_STATE_STATE_INTERRUPT, 0, 0):
                                case PMC_STATE(PMC_STATE_STATE_LOAD, 0, 0):
                                case PMC_STATE(PMC_STATE_STATE_RUN, 0, 0):
                                case PMC_STATE(PMC_STATE_STATE_STORE, 0, 0):
                                        new_state = PMC_STATE_MODIFY(current_state, 0, PMC_STATE_FLAGS_DEALLOCING, 0);
                                        break;
                                case PMC_STATE(PMC_STATE_STATE_STOP, 0, PMC_STATE_FLAGS_STOPPING):
                                        new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_DEALLOC, 0, PMC_STATE_FLAGS_DEALLOCING, PMC_STATE_FLAGS_STOPPING);
                                        break;
                                case PMC_STATE(PMC_STATE_STATE_STOP, 0, 0):
                                        if (PMC_STATE_CONTEXT_COUNT(current_state) > 0) {
                                                new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_DEALLOC, 0, PMC_STATE_FLAGS_DEALLOCING, 0);
                                        } else {
                                                new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_DEALLOC, 0, 0, 0);
                                        }
                                        break;
                        }
                        break;
                case PMC_STATE_EVENT_INTERRUPT:
                        switch (current_state & ~(PMC_STATE_CONTEXT_COUNT_MASK)) {
                                case PMC_STATE(PMC_STATE_STATE_LOAD, 0, 0):
                                case PMC_STATE(PMC_STATE_STATE_RUN, 0, 0):
                                case PMC_STATE(PMC_STATE_STATE_STORE, 0, 0):
                                        new_state = PMC_STATE_MODIFY(current_state, 0, PMC_STATE_FLAGS_INTERRUPTING | PMC_STATE_FLAGS_STOPPING, 0);
                                        break;
                        }
                        break;
                case PMC_STATE_EVENT_END_OF_INTERRUPT:
                        switch (current_state & ~(PMC_STATE_CONTEXT_COUNT_MASK)) {
                                case PMC_STATE(PMC_STATE_STATE_INTERRUPT, 0, PMC_STATE_FLAGS_DEALLOCING):
                                        new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_DEALLOC, 0, 0, PMC_STATE_FLAGS_DEALLOCING);
                                        break;
                                case PMC_STATE(PMC_STATE_STATE_INTERRUPT, 0, PMC_STATE_FLAGS_STOPPING):
                                        new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_STOP, 0, 0, PMC_STATE_FLAGS_STOPPING);
                                        break;
                                case PMC_STATE(PMC_STATE_STATE_INTERRUPT, 0, 0):
                                        new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_CAN_RUN, 0, 0, 0);
                                        break;
                        }
                        break;
                case PMC_STATE_EVENT_CONTEXT_IN:
                        switch (current_state & ~(PMC_STATE_CONTEXT_COUNT_MASK)) {
                                case PMC_STATE(PMC_STATE_STATE_CAN_RUN, 0, 0):
                                        new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_LOAD, 1, 0, 0);
                                        break;
                                case PMC_STATE(PMC_STATE_STATE_LOAD, 0, 0):
                                case PMC_STATE(PMC_STATE_STATE_RUN, 0, 0):
                                case PMC_STATE(PMC_STATE_STATE_STORE, 0, 0):
                                        new_state = PMC_STATE_MODIFY(current_state, 1, 0, 0);
                                        break;
                                case PMC_STATE(PMC_STATE_STATE_STOP, 0, 0):
                                        if (PMC_STATE_CONTEXT_COUNT(current_state) > 0) {
                                                new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_LOAD, 1, 0, 0);
                                        }
                                        break;
                        }
                        break;
                case PMC_STATE_EVENT_CONTEXT_OUT:
                        switch (current_state & ~(PMC_STATE_CONTEXT_COUNT_MASK)) {
                                case PMC_STATE(PMC_STATE_STATE_DEALLOC, 0, PMC_STATE_FLAGS_DEALLOCING):
                                        if (PMC_STATE_CONTEXT_COUNT(current_state) > 1) {
                                                new_state = PMC_STATE_MODIFY(current_state, -1, 0, PMC_STATE_FLAGS_DEALLOCING);
                                        } else {
                                                new_state = PMC_STATE_MODIFY(current_state, -1, 0, 0);
                                        }                                       
                                        break;
                                case PMC_STATE(PMC_STATE_STATE_LOAD, 0, PMC_STATE_FLAGS_DEALLOCING):
                                case PMC_STATE(PMC_STATE_STATE_LOAD, 0, PMC_STATE_FLAGS_INTERRUPTING | PMC_STATE_FLAGS_DEALLOCING):
                                case PMC_STATE(PMC_STATE_STATE_LOAD, 0, PMC_STATE_FLAGS_INTERRUPTING | PMC_STATE_FLAGS_STOPPING):
                                case PMC_STATE(PMC_STATE_STATE_LOAD, 0, PMC_STATE_FLAGS_STOPPING):
                                case PMC_STATE(PMC_STATE_STATE_LOAD, 0, 0):
                                        if (PMC_STATE_CONTEXT_COUNT(current_state) > 1) {
                                                new_state = PMC_STATE_MODIFY(current_state, -1, 0, 0);
                                        }
                                        break;
                                case PMC_STATE(PMC_STATE_STATE_RUN, 0, PMC_STATE_FLAGS_DEALLOCING):
                                case PMC_STATE(PMC_STATE_STATE_RUN, 0, PMC_STATE_FLAGS_INTERRUPTING | PMC_STATE_FLAGS_DEALLOCING):
                                case PMC_STATE(PMC_STATE_STATE_RUN, 0, PMC_STATE_FLAGS_INTERRUPTING | PMC_STATE_FLAGS_STOPPING):
                                case PMC_STATE(PMC_STATE_STATE_RUN, 0, PMC_STATE_FLAGS_STOPPING):
                                case PMC_STATE(PMC_STATE_STATE_RUN, 0, 0):
                                        if (PMC_STATE_CONTEXT_COUNT(current_state) == 1) {
                                                new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_STORE, -1, 0, 0);
                                        } else {
                                                new_state = PMC_STATE_MODIFY(current_state, -1, 0, 0);
                                        }
                                        break;
                                case PMC_STATE(PMC_STATE_STATE_STOP, 0, PMC_STATE_FLAGS_INTERRUPTING | PMC_STATE_FLAGS_DEALLOCING):
                                case PMC_STATE(PMC_STATE_STATE_STOP, 0, PMC_STATE_FLAGS_INTERRUPTING | PMC_STATE_FLAGS_STOPPING):
                                        if (PMC_STATE_CONTEXT_COUNT(current_state) == 1) {
                                                new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_INTERRUPT, -1, 0, PMC_STATE_FLAGS_INTERRUPTING);
                                        } else {
                                                new_state = PMC_STATE_MODIFY(current_state, -1, 0, 0);
                                        }
                                        break;
                                case PMC_STATE(PMC_STATE_STATE_STOP, 0, PMC_STATE_FLAGS_STOPPING):
                                        if (PMC_STATE_CONTEXT_COUNT(current_state) == 1) {
                                                new_state = PMC_STATE_MODIFY(current_state, -1, 0, PMC_STATE_FLAGS_STOPPING);
                                        } else {
                                                new_state = PMC_STATE_MODIFY(current_state, -1, 0, 0);
                                        }
                                        break;
                                case PMC_STATE(PMC_STATE_STATE_STOP, 0, 0):
                                        if (PMC_STATE_CONTEXT_COUNT(current_state) > 0) {
                                                if (PMC_STATE_CONTEXT_COUNT(current_state) == 1) {
                                                        new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_CAN_RUN, -1, 0, 0);
                                                } else {
                                                        new_state = PMC_STATE_MODIFY(current_state, -1, 0, 0);
                                                }
                                        }
                                        break;
                                case PMC_STATE(PMC_STATE_STATE_STORE, 0, PMC_STATE_FLAGS_DEALLOCING):
                                case PMC_STATE(PMC_STATE_STATE_STORE, 0, PMC_STATE_FLAGS_INTERRUPTING | PMC_STATE_FLAGS_DEALLOCING):
                                case PMC_STATE(PMC_STATE_STATE_STORE, 0, PMC_STATE_FLAGS_INTERRUPTING | PMC_STATE_FLAGS_STOPPING):
                                case PMC_STATE(PMC_STATE_STATE_STORE, 0, PMC_STATE_FLAGS_STOPPING):
                                case PMC_STATE(PMC_STATE_STATE_STORE, 0, 0):
                                        if (PMC_STATE_CONTEXT_COUNT(current_state) > 0) {
                                                new_state = PMC_STATE_MODIFY(current_state, -1, 0, 0);
                                        }
                                        break;
                        }
                        break;
                case PMC_STATE_EVENT_LOAD_FINISHED:
                        switch (current_state & ~(PMC_STATE_CONTEXT_COUNT_MASK)) {
                                case PMC_STATE(PMC_STATE_STATE_LOAD, 0, PMC_STATE_FLAGS_DEALLOCING):
                                case PMC_STATE(PMC_STATE_STATE_LOAD, 0, PMC_STATE_FLAGS_INTERRUPTING | PMC_STATE_FLAGS_DEALLOCING):
                                case PMC_STATE(PMC_STATE_STATE_LOAD, 0, PMC_STATE_FLAGS_INTERRUPTING | PMC_STATE_FLAGS_STOPPING):
                                case PMC_STATE(PMC_STATE_STATE_LOAD, 0, PMC_STATE_FLAGS_STOPPING):
                                        if (PMC_STATE_CONTEXT_COUNT(current_state) > 1) {
                                                new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_RUN, -1, 0, 0);
                                        } else {
                                                new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_STORE, -1, 0, 0);
                                        }
                                        break;
                                case PMC_STATE(PMC_STATE_STATE_LOAD, 0, 0):
                                        new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_RUN, 0, 0, 0);
                                        break;
                        }
                        break;
                case PMC_STATE_EVENT_STORE_FINISHED:
                        switch (current_state & ~(PMC_STATE_CONTEXT_COUNT_MASK)) {
                                case PMC_STATE(PMC_STATE_STATE_STORE, 0, PMC_STATE_FLAGS_DEALLOCING):
                                        if (PMC_STATE_CONTEXT_COUNT(current_state) == 0) {
                                                new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_DEALLOC, 0, 0, PMC_STATE_FLAGS_DEALLOCING);
                                        } else {
                                                new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_DEALLOC, 0, 0, 0);
                                        }
                                        break;
                                case PMC_STATE(PMC_STATE_STATE_STORE, 0, PMC_STATE_FLAGS_INTERRUPTING | PMC_STATE_FLAGS_DEALLOCING):
                                case PMC_STATE(PMC_STATE_STATE_STORE, 0, PMC_STATE_FLAGS_INTERRUPTING | PMC_STATE_FLAGS_STOPPING):
                                        if (PMC_STATE_CONTEXT_COUNT(current_state) == 0) {
                                                new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_INTERRUPT, 0, 0, PMC_STATE_FLAGS_INTERRUPTING);
                                        } else {
                                                new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_STOP, 0, 0, 0);
                                        }
                                        break;
                                case PMC_STATE(PMC_STATE_STATE_STORE, 0, PMC_STATE_FLAGS_STOPPING):
                                        if (PMC_STATE_CONTEXT_COUNT(current_state) == 0) {
                                                new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_STOP, 0, 0, PMC_STATE_FLAGS_STOPPING);
                                        } else {
                                                new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_STOP, 0, 0, 0);
                                        }
                                        break;
                                case PMC_STATE(PMC_STATE_STATE_STORE, 0, 0):
                                        if (PMC_STATE_CONTEXT_COUNT(current_state) == 0) {
                                                new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_CAN_RUN, 0, 0, 0);
                                        } else {
                                                new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_LOAD, 0, 0, 0);
                                        }
                                        break;
                        }
                        break;
        }
        
        return new_state;
}

static uint32_t pmc_internal_reservation_move_for_event(pmc_reservation_t reservation, pmc_state_event_t event, pmc_state_t *old_state_out) {
        pmc_state_t oldState;
        pmc_state_t newState;

        assert(reservation);
        
        /* Determine what state change, if any, we need to do.  Keep trying until either we succeed doing a transition
         * or the there is no valid move.
         */     
        do {
                oldState = reservation->state;
                newState = pmc_internal_reservation_next_state(oldState, event);
        } while (newState != PMC_STATE_INVALID && !OSCompareAndSwap(oldState, newState, &(reservation->state)));
        
        if (newState != PMC_STATE_INVALID) {
                COUNTER_DEBUG("Moved reservation %p from state "PMC_STATE_FORMAT" to state "PMC_STATE_FORMAT" for event %s\n", reservation, PMC_STATE_ARGS(oldState), PMC_STATE_ARGS(newState), pmc_state_event_name(event));
        } else {
                COUNTER_DEBUG("No valid moves for reservation %p in state "PMC_STATE_FORMAT" for event %s\n", reservation, PMC_STATE_ARGS(oldState), pmc_state_event_name(event));
        }
        
        if (old_state_out != NULL) {
                *old_state_out = oldState;
        }
        
        return newState;
}
                                        
static void pmc_internal_reservation_context_out(pmc_reservation_t reservation) {
        assert(reservation);
        pmc_state_t newState;
        pmc_state_t oldState;

        /* Clear that the this reservation was active when this cpu did its last context in */
        OSBitAndAtomic(~(1U << cpu_number()), &(reservation->active_last_context_in));
        
        /* Move the state machine */
        if (PMC_STATE_INVALID == (newState = pmc_internal_reservation_move_for_event(reservation, PMC_STATE_EVENT_CONTEXT_OUT, &oldState))) {
                return;
        }
        
        /* Do any actions required based on the state change */
        if (PMC_STATE_STATE(newState) == PMC_STATE_STATE_STORE && PMC_STATE_STATE(oldState) != PMC_STATE_STATE_STORE) {
                /* Just moved into STORE, so store the reservation. */
                pmc_internal_reservation_store(reservation);
        } else if (PMC_STATE_STATE(newState) == PMC_STATE_STATE_DEALLOC && PMC_STATE_CONTEXT_COUNT(newState) == 0 && PMC_STATE_FLAGS(newState) == 0) {
                /* Wakeup any thread blocking for this reservation to hit <DEALLOC, 0, > */
                thread_wakeup((event_t)reservation);
        }
        
}

static void pmc_internal_reservation_context_in(pmc_reservation_t reservation) {
        assert(reservation);
        pmc_state_t oldState;
        pmc_state_t newState;
        
        /* Move the state machine */
        if (PMC_STATE_INVALID == (newState = pmc_internal_reservation_move_for_event(reservation, PMC_STATE_EVENT_CONTEXT_IN, &oldState))) {
                return;
        }

        /* Mark that the reservation was active when this cpu did its last context in */
        OSBitOrAtomic(1U << cpu_number(), &(reservation->active_last_context_in));
                
        /* Do any actions required based on the state change */
        if (PMC_STATE_STATE(newState) == PMC_STATE_STATE_LOAD && PMC_STATE_STATE(oldState) != PMC_STATE_STATE_LOAD) {
                /* Just moved into LOAD, so load the reservation. */
                pmc_internal_reservation_load(reservation);
        }
        
}

static void pmc_internal_reservation_store(pmc_reservation_t reservation) {
        assert(reservation);
        assert(PMC_STATE_STATE(reservation->state) == PMC_STATE_STATE_STORE);
        
        assert(reservation->pmc);
        assert(reservation->config);

        pmc_state_t newState;
        kern_return_t ret = KERN_SUCCESS;
        
        pmc_t store_pmc = reservation->pmc;
        pmc_object_t store_pmc_obj = store_pmc->object;
        perf_monitor_t store_pm = store_pmc->monitor;

        /* 
         * Instruct the Perf Monitor that contains this counter to turn 
         * off the global disable for this counter.
         */
        ret = store_pm->methods.disable_counters(store_pm->object, &store_pmc_obj, 1);
        if(KERN_SUCCESS != ret) {
                COUNTER_DEBUG(" [error] disable_counters: 0x%x\n", ret);
                return;
        }

        /* Instruct the counter to disable itself */
        ret = store_pmc->methods.disable(store_pmc_obj);
        if(KERN_SUCCESS != ret) {
                COUNTER_DEBUG("  [error] disable: 0x%x\n", ret);
        }

        /*
         * At this point, we're off the hardware, so we don't have to
         * set_on_hardare(TRUE) if anything fails from here on.
         */

        /* store the counter value into the reservation's stored count */
        ret = store_pmc->methods.get_count(store_pmc_obj, &reservation->value);
        if(KERN_SUCCESS != ret) {
                COUNTER_DEBUG("  [error] get_count: 0x%x\n", ret);
                return;
        }
                
        /* Advance the state machine now that the STORE is finished */
        if (PMC_STATE_INVALID == (newState = pmc_internal_reservation_move_for_event(reservation, PMC_STATE_EVENT_STORE_FINISHED, NULL))) {
                return;
        }

        /* Do any actions required based on the state change */
        if (PMC_STATE_STATE(newState) == PMC_STATE_STATE_LOAD) {
                /* Just moved into LOAD, so load the reservation. */
                pmc_internal_reservation_load(reservation);
        } else if (PMC_STATE_STATE(newState) == PMC_STATE_STATE_DEALLOC && PMC_STATE_CONTEXT_COUNT(newState) == 0 && PMC_STATE_FLAGS(newState) == 0) {
                /* Wakeup any thread blocking for this reservation to hit <DEALLOC, 0, > */
                thread_wakeup((event_t)reservation);
        }
        
}

static void pmc_internal_reservation_load(pmc_reservation_t reservation) {
        assert(reservation);
        assert(PMC_STATE_STATE(reservation->state) == PMC_STATE_STATE_LOAD);

        pmc_state_t newState;
        kern_return_t ret = KERN_SUCCESS;

        assert(reservation->pmc);
        assert(reservation->config);
        
        pmc_t load_pmc = reservation->pmc;
        pmc_object_t load_pmc_obj = load_pmc->object;
        perf_monitor_t load_pm = load_pmc->monitor;

        /* Set the control register up with the stored configuration */
        ret = load_pmc->methods.set_config(load_pmc_obj, reservation->config->object);
        if(KERN_SUCCESS != ret) {
                COUNTER_DEBUG("  [error] set_config: 0x%x\n", ret);
                return;
        }

        /* load the counter value */
        ret = load_pmc->methods.set_count(load_pmc_obj, reservation->value);
        if(KERN_SUCCESS != ret) {
                COUNTER_DEBUG("  [error] set_count: 0x%x\n", ret);
                return;
        }

        /* Locally enable the counter */
        ret = load_pmc->methods.enable(load_pmc_obj);
        if(KERN_SUCCESS != ret) {
                COUNTER_DEBUG("  [error] enable: 0x%x\n", ret);
                return;
        }

        /*
         * Instruct the Perf Monitor containing the pmc to enable the
         * counter.
         */
        ret = load_pm->methods.enable_counters(load_pm->object, &load_pmc_obj, 1);
        if(KERN_SUCCESS != ret) {
                COUNTER_DEBUG("  [error] enable_counters: 0x%x\n", ret);
                /* not on the hardware. */
                return;
        }
        
        /* Advance the state machine now that the STORE is finished */
        if (PMC_STATE_INVALID == (newState = pmc_internal_reservation_move_for_event(reservation, PMC_STATE_EVENT_LOAD_FINISHED, NULL))) {
                return;
        }

        /* Do any actions required based on the state change */
        if (PMC_STATE_STATE(newState) == PMC_STATE_STATE_STORE) {
                /* Just moved into STORE, so store the reservation. */
                pmc_internal_reservation_store(reservation);
        }
        
}

static void pmc_internal_reservation_start_cpu(void * arg) {
        pmc_reservation_t reservation = (pmc_reservation_t)arg;
        
        assert(reservation);
        
        if (pmc_internal_reservation_matches_context(reservation)) {
                /* We are in context, but the reservation may have already had the context_in method run.  Attempt
                 * to set this cpu's bit in the active_last_context_in mask.  If we set it, call context_in.
                 */
                uint32_t oldMask = OSBitOrAtomic(1U << cpu_number(), &(reservation->active_last_context_in));
                
                if ((oldMask & (1U << cpu_number())) == 0) {
                        COUNTER_DEBUG("Starting already in-context reservation %p for cpu %d\n", reservation, cpu_number());
                        
                        pmc_internal_reservation_context_in(reservation);
                }
        }
}

static void pmc_internal_reservation_stop_cpu(void * arg) {
        pmc_reservation_t reservation = (pmc_reservation_t)arg;
        
        assert(reservation);
        
        if (pmc_internal_reservation_matches_context(reservation)) {
                COUNTER_DEBUG("Stopping in-context reservation %p for cpu %d\n", reservation, cpu_number());

                pmc_internal_reservation_context_out(reservation);
        }
}       

/*!fn
 * pmc_reservation_interrupt is called when a PMC reservation which was setup
 * with an interrupt threshold counts the requested number of events. When the
 * underlying counter hits the threshold, an interrupt is generated, and this
 * method is called. This method marks the reservation as stopped, and passes
 * control off to the user-registered callback method, along with the
 * reservation (so that the user can, for example, write a 0 to the counter, and
 * restart the reservation).
 * This method assumes the reservation has a valid pmc_config_t within.
 *
 * @param target The pmc_reservation_t that caused the interrupt.
 * @param refCon User specified reference constant.
 */
static void pmc_reservation_interrupt(void *target, void *refCon) {
        pmc_reservation_t reservation = (pmc_reservation_t)target;
        pmc_state_t newState;
        uint64_t timeout;
        uint32_t spins;

        assert(reservation);

        /* Move the state machine */
        if (PMC_STATE_INVALID == pmc_internal_reservation_move_for_event(reservation, PMC_STATE_EVENT_INTERRUPT, NULL)) {
                return;
        }

        /* A valid state move has been made, but won't be picked up until a context switch occurs.  To cause matching
         * contexts that are currently running to update, we do an inter-processor message to run pmc_internal_reservation_stop_cpu
         * on every cpu that can access the PMC.
         */
        pmc_internal_reservation_broadcast(reservation, pmc_internal_reservation_stop_cpu);
                        
        /* Spin waiting for the state to turn to INTERRUPT */
        nanoseconds_to_absolutetime(PMC_SPIN_TIMEOUT_US * 1000, &timeout);
        timeout += mach_absolute_time();
        spins = 0;
        while (PMC_STATE_STATE(reservation->state) != PMC_STATE_STATE_INTERRUPT) {
                /* Assert if this takes longer than PMC_SPIN_TIMEOUT_US */
                if (++spins > PMC_SPIN_THRESHOLD) {
                        if (mach_absolute_time() > timeout) {
                                pmc_spin_timeout_count++;
                                assert(0);
                        }
                }

                cpu_pause();
        }
                        
        assert(reservation->config);
        assert(reservation->config->method);                    
                
        /* Call the registered callback handler */
#if DEBUG_COUNTERS
        uint64_t start = mach_absolute_time();
#endif /* DEBUG */
        
        (void)reservation->config->method(reservation, refCon);
        
#if DEBUG_COUNTERS
        uint64_t end = mach_absolute_time();
        if((end - start) > 5000ULL) {
                kprintf("%s - user method %p took %llu ns\n", __FUNCTION__, 
                                reservation->config->method, (end - start));
        }
#endif
        
        /* Move the state machine */
        if (PMC_STATE_INVALID == (newState = pmc_internal_reservation_move_for_event(reservation, PMC_STATE_EVENT_END_OF_INTERRUPT, NULL))) {
                return;
        }
        
        /* Do any post-move actions necessary */
        if (PMC_STATE_STATE(newState) == PMC_STATE_STATE_CAN_RUN) {
                pmc_internal_reservation_broadcast(reservation, pmc_internal_reservation_start_cpu);
        } else if (PMC_STATE_STATE(newState) == PMC_STATE_STATE_DEALLOC && PMC_STATE_CONTEXT_COUNT(newState) == 0 && PMC_STATE_FLAGS(newState) == 0) {
                /* Wakeup any thread blocking for this reservation to hit <DEALLOC, 0, > */
                thread_wakeup((event_t)reservation);
        }
}       

/*
 * Apple-private KPI for Apple kext's (IOProfileFamily) only
 */

#if 0
#pragma mark -
#pragma mark IOProfileFamily private KPI
#endif

/*
 * perf_monitor_register registers a new Performance Monitor, and its associated
 * callback methods.  The given perf_monitor_object_t is the first argument to
 * each callback when they are called.
 */
kern_return_t perf_monitor_register(perf_monitor_object_t monitor,
        perf_monitor_methods_t *methods) {

        COUNTER_DEBUG("registering perf monitor %p\n", monitor);

        if(!monitor || !methods) {
                return KERN_INVALID_ARGUMENT;
        }

        /* Protect against out-of-date driver kexts */
        if(MACH_PERFMON_METHODS_VERSION != methods->perf_monitor_methods_version) {
                return KERN_INVALID_ARGUMENT;
        }

        /* All methods are required */
        if(!methods->supports_context_switching || !methods->enable_counters ||
                !methods->disable_counters) {
                return KERN_INVALID_ARGUMENT;
        }

        /* prevent dupes. */
        perf_monitor_t dupe = perf_monitor_find(monitor);
        if(dupe) {
                COUNTER_DEBUG("Duplicate registration for %p\n", monitor);
                perf_monitor_deallocate(dupe);
                return KERN_FAILURE;
        }

        perf_monitor_t pm = perf_monitor_alloc();
        if(!pm) {
                return KERN_RESOURCE_SHORTAGE;
        }

        /* initialize the object */
        perf_monitor_init(pm);

        /* copy in the registration info */
        pm->object = monitor;
        memcpy(&(pm->methods), methods, sizeof(perf_monitor_methods_t));

        /* place it in the tracking queue */
        perf_monitor_enqueue(pm);

        /* debug it */
        PRINT_PERF_MON(pm);

        return KERN_SUCCESS;
}

/*
 * perf_monitor_unregister unregisters a previously registered Perf Monitor,
 * looking it up by reference pointer (the same that was used in
 * perf_monitor_register()).
 */
kern_return_t perf_monitor_unregister(perf_monitor_object_t monitor) {
        kern_return_t ret = KERN_FAILURE;

        COUNTER_DEBUG("unregistering perf monitor %p\n", monitor);

        if(!monitor) {
                return KERN_INVALID_ARGUMENT;
        }

        perf_monitor_t pm = perf_monitor_find(monitor);
        if(pm) {
                /* Remove it from the queue. */
                perf_monitor_dequeue(pm);

                /* drop extra retain from find */
                perf_monitor_deallocate(pm);

                /* and release the object */
                perf_monitor_deallocate(pm);

                ret = KERN_SUCCESS;
        } else {
                COUNTER_DEBUG("could not find a registered pm that matches!\n");
        }

        return ret;
}

/*
 * pmc_register registers a new PMC for use with the pmc subsystem. Each PMC is
 * associated with a Perf Monitor.  Perf Monitors are looked up by the reference
 * pointer that was used to previously register them. 
 *
 * PMCs are registered with a reference pointer (@pmc_object), and a set of
 * callback methods.  When the given callback methods are called from xnu, the
 * first argument will always be the reference pointer used to register the PMC.
 *
 * NOTE: @monitor must have been successfully registered via
 * perf_monitor_register before this method will succeed.
 */
kern_return_t pmc_register(perf_monitor_object_t monitor, pmc_object_t pmc_object,
        pmc_methods_t *methods, void *object) {

        COUNTER_DEBUG("%p %p\n", monitor, pmc_object);

        if(!monitor || !pmc_object || !methods || !object) {
                return KERN_INVALID_ARGUMENT;
        }

        /* Prevent version mismatches */
        if(MACH_PMC_METHODS_VERSION != methods->pmc_methods_version) {
                COUNTER_DEBUG("version mismatch\n");
                return KERN_INVALID_ARGUMENT;
        }

        /* All methods are required. */
        if(!methods->create_config || 
                !methods->free_config ||
                !methods->config_set_value || 
                !methods->config_set_threshold || 
                !methods->config_set_handler ||
                !methods->set_config || 
                !methods->get_monitor || 
                !methods->get_name ||
                !methods->accessible_from_core || 
                !methods->accessible_cores ||
                !methods->get_count || 
                !methods->set_count ||
                !methods->disable ||
                !methods->enable ||
                !methods->open || 
                !methods->close) {
                return KERN_INVALID_ARGUMENT;
        }

        /* make sure this perf monitor object is already registered */
        /*
         * NOTE: this adds a reference to the parent, so we'll have to drop it in
         * any failure code paths from here on out.
         */
        perf_monitor_t pm = perf_monitor_find(monitor);
        if(!pm) {
                COUNTER_DEBUG("Could not find perf monitor for %p\n", monitor);
                return KERN_INVALID_ARGUMENT;
        }

        /* make a new pmc */
        pmc_t pmc = pmc_alloc();
        if(!pmc) {
                /* drop the extra reference from perf_monitor_find() */
                perf_monitor_deallocate(pm);
                return KERN_RESOURCE_SHORTAGE;
        }

        /* init it */
        pmc_init(pmc);

        pmc->object = pmc_object;
        pmc->open_object = object;

        /* copy the callbacks in */
        memcpy(&(pmc->methods), methods, sizeof(pmc_methods_t));

        pmc->monitor = pm;

        perf_monitor_add_pmc(pmc->monitor, pmc);

        /* enqueue it in our tracking queue */
        pmc_enqueue(pmc);

        /* drop extra reference from perf_monitor_find() */
        perf_monitor_deallocate(pm);

        return KERN_SUCCESS;
}

/*
 * pmc_unregister unregisters a previously registered PMC, looking it up by
 * reference point to *both* the Perf Monitor it was created with, and the PMC's
 * reference pointer itself.
 */
kern_return_t pmc_unregister(perf_monitor_object_t monitor, pmc_object_t pmc_object) {
        COUNTER_DEBUG("%p %p\n", monitor, pmc_object);

        if(!monitor || !pmc_object) {
                return KERN_INVALID_ARGUMENT;
        }

        pmc_t pmc = pmc_find(pmc_object);
        if(!pmc) {
                COUNTER_DEBUG("Could not find a matching pmc.\n");
                return KERN_FAILURE;
        }

        /* remove it from the global queue */
        pmc_dequeue(pmc);

        perf_monitor_remove_pmc(pmc->monitor, pmc);

        /* remove extra reference count from pmc_find() */
        pmc_deallocate(pmc);

        /* dealloc the pmc */
        pmc_deallocate(pmc);

        return KERN_SUCCESS;
}

#if 0
#pragma mark -
#pragma mark KPI
#endif

/*
 * Begin in-kernel and in-kext KPI methods
 */

/*
 * pmc_create_config creates a new configuration area from a given @pmc.
 *
 * NOTE: This method is not interrupt safe.
 */
kern_return_t pmc_create_config(pmc_t pmc, pmc_config_t *config) {
        pmc_config_t tmp = NULL;

        if(!pmc || !config) {
                return KERN_INVALID_ARGUMENT;
        }

        pmc_reference(pmc);

        tmp = pmc_config_alloc(pmc);
        if(tmp) {
                tmp->object = pmc->methods.create_config(pmc->object);

                if(!tmp->object) {
                        pmc_config_free(pmc, tmp);
                        tmp = NULL;
                } else {
                        tmp->interrupt_after_value = 0ULL;
                        tmp->method = NULL;
                        tmp->refCon = NULL;
                }
        }

        pmc_deallocate(pmc);

        if(!tmp) {
                return KERN_RESOURCE_SHORTAGE;
        }

        *config = tmp;

        return KERN_SUCCESS;
}

/*
 * pmc_free_config frees a configuration area created from a given @pmc
 *
 * NOTE: This method is not interrupt safe.
 */
void pmc_free_config(pmc_t pmc, pmc_config_t config) {
        assert(pmc);
        assert(config);

        pmc_reference(pmc);

        pmc_config_free(pmc, config);

        pmc_deallocate(pmc);
}

/*
 * pmc_config_set_value sets up configuration area key-value pairs.  These pairs
 * are to be either pre-known, or looked up via CoreProfile.framework.
 *
 * NOTE: This method is not interrupt safe.
 */
kern_return_t pmc_config_set_value(pmc_t pmc, pmc_config_t config,
        uint8_t id, uint64_t value) {

        kern_return_t ret = KERN_INVALID_ARGUMENT;
        
        if(!pmc || !config) {
                return ret;
        }

        pmc_reference(pmc);

        ret = pmc->methods.config_set_value(config->object, id, value);

        pmc_deallocate(pmc);

        return ret;
}

/*
 * pmc_config_set_interrupt_threshold modifies a config object, instructing
 * the pmc that it should generate a call to the given pmc_interrupt_method_t
 * after the counter counts @threshold events.
 *
 * PMC Threshold handler methods will have the pmc_reservation_t that generated the interrupt
 * as the first argument when the interrupt handler is invoked, and the given
 * @refCon (which may be NULL) as the second.
 *
 * See pmc_interrupt_method_t.
 *
 * NOTE: This method is not interrupt safe.
 */
kern_return_t pmc_config_set_interrupt_threshold(pmc_t pmc, pmc_config_t config, 
        uint64_t threshold, pmc_interrupt_method_t method, void *refCon) {
        kern_return_t ret = KERN_INVALID_ARGUMENT;

        if(!config || !pmc) {
                return ret;
        }
        
        assert(config);
        assert(pmc);

        pmc_reference(pmc);

        do {
                /*
                 * We have a minor annoyance to side-step here. The driver layer expects
                 * the config to never change once a reservation has been taken out with
                 * it.  However, in order to have the PMI method have the reservation as
                 * the first argument (in order to allow the user-method to, for
                 * example, write a 0 to it, and restart it), we need to create the
                 * pmc_reservation_t before setting it up in the config object.
                 * We overcome this by caching the method in the pmc_config_t stand-in,
                 * and mutating the pmc_config_object_t just before returning a
                 * reservation (in pmc_reserve() and friends, below).
                 */

                /* might as well stash this away too. */
                config->interrupt_after_value = threshold;
                config->method = method;
                config->refCon = refCon;

                ret = KERN_SUCCESS;

        }while(0);

        pmc_deallocate(pmc);

        return ret;
}

/*
 * pmc_get_pmc_list returns an allocated list of pmc_t's, as well as the number
 * of pmc_t's returned. Callers should free this list with a call to
 * pmc_free_pmc_list().
 *
 * NOTE: This method is not interrupt safe.
 */
kern_return_t pmc_get_pmc_list(pmc_t **pmcs, size_t *pmcCount) {
        pmc_t *array = NULL;
        pmc_t pmc = NULL;
        size_t count = 0UL;
        
        do {
                /* Copy down (to the stack) the count of perf counters */
                vm_size_t size = perf_counters_count;

                /* Allocate that sized chunk */
                array = (pmc_t *)kalloc(sizeof(pmc_t) * size);
                if(!array) {
                        return KERN_RESOURCE_SHORTAGE;
                }

                /* Take the spin lock */
                lck_spin_lock(&perf_counters_queue_spin);

                /* verify the size didn't change while we were allocating */
                if(size != perf_counters_count) {
                        /*
                         * queue size has changed between alloc and now - go back and
                         * make another pass.
                         */

                        /* drop the lock */
                        lck_spin_unlock(&perf_counters_queue_spin);

                        /* free the block */
                        kfree(array, sizeof(pmc_t) * size);
                        array = NULL;
                }

                /* if we get here, and array is NULL, we try again. */
        }while(!array);

        /* copy the bits out */
        queue_iterate(perf_counters_queue, pmc, pmc_t, link) {
                if(pmc) {
                        /* copy out the pointer */
                        array[count++] = pmc;
                }
        }

        lck_spin_unlock(&perf_counters_queue_spin);

        /* return the list and the size */
        *pmcs = array;
        *pmcCount = count;

        return KERN_SUCCESS;
}

/*
 * pmc_free_pmc_list frees an array of pmc_t that has been returned from
 * pmc_get_pmc_list.
 * 
 * NOTE: This method is not interrupt safe.
 */
void pmc_free_pmc_list(pmc_t *pmcs, size_t pmcCount) {
        if(pmcs && pmcCount) {
                COUNTER_DEBUG("pmcs: %p pmcCount: %lu\n", pmcs, pmcCount);

                kfree(pmcs, pmcCount * sizeof(pmc_t));
        }
}

kern_return_t pmc_find_by_name(const char *name, pmc_t **pmcs, size_t *pmcCount) {
        kern_return_t ret = KERN_INVALID_ARGUMENT;

        if(!name || !pmcs || !pmcCount) {
                return ret;
        }

        pmc_t *list = NULL;
        size_t count = 0UL;

        if(KERN_SUCCESS == (ret = pmc_get_pmc_list(&list, &count))) {
                size_t matchCount = 0UL, ii = 0UL, swapPtr = 0UL;
                size_t len = strlen(name);

                for(ii = 0UL; ii < count; ii++) {
                        const char *pmcName = pmc_get_name(list[ii]);

                        if(strlen(pmcName) < len) {
                                /*
                                 * If the pmc name is shorter than the requested match, it's no 
                                 * match, as we're looking for the most specific match(es).
                                 */
                                continue;
                        }

                        if(0 == strncmp(name, pmcName, len)) {
                                pmc_t temp = list[ii];
                                
                                // move matches to the head of the array.
                                list[ii] = list[swapPtr];
                                list[swapPtr] = temp;
                                swapPtr++;

                                // keep a count of the matches
                                matchCount++;
                        }
                }

                if(matchCount) {
                        /*
                         * If we have matches, they are all at the head of the array, so
                         * just allocate enough space for @matchCount pmc_t's, and copy the
                         * head of the array to the new allocation.  Then free the old
                         * allocation.
                         */

                        pmc_t *result = (pmc_t *)kalloc(sizeof(pmc_t) * matchCount);
                        if(result) {
                                // copy the matches
                                memcpy(result, list, sizeof(pmc_t) * matchCount);

                                ret = KERN_SUCCESS;
                        }

                        pmc_free_pmc_list(list, count);

                        if(!result) {
                                *pmcs = NULL;
                                *pmcCount = 0UL;
                                return KERN_RESOURCE_SHORTAGE;
                        }

                        *pmcs = result;
                        *pmcCount = matchCount;
                } else {
                        *pmcs = NULL;
                        *pmcCount = 0UL;
                }
        }

        return ret;
}

/*
 * pmc_get_name returns a pointer (not copied) to the human-readable name of the
 * given pmc.
 *
 * NOTE: Driver authors must take care to not allocate during this method, as
 * this method *IS* interrupt safe.
 */
const char *pmc_get_name(pmc_t pmc) {
        assert(pmc);

        const char *name = pmc->methods.get_name(pmc->object);

        return name;
}

/*
 * pmc_get_accessible_core_list returns a pointer to an array of logical core
 * numbers (as well as the size of that array) that represent the local cores
 * (hardware threads) from which the given @pmc can be accessed directly.
 *
 * NOTE: This method is interrupt safe.
 */
kern_return_t pmc_get_accessible_core_list(pmc_t pmc, uint32_t **logicalCores,
        size_t *logicalCoreCt) {

        kern_return_t ret = KERN_INVALID_ARGUMENT;

        if(!pmc || !logicalCores || !logicalCoreCt) {
                return ret;
        }

        ret = pmc->methods.accessible_cores(pmc->object, logicalCores, logicalCoreCt);

        return ret;
}

/*
 * pmc_accessible_from_core will return TRUE if the given @pmc is directly
 * (e.g., hardware) readable from the given logical core.
 *
 * NOTE: This method is interrupt safe.
 */
boolean_t pmc_accessible_from_core(pmc_t pmc, uint32_t logicalCore) {
        boolean_t ret = FALSE;

        assert(pmc);

        ret = pmc->methods.accessible_from_core(pmc->object, logicalCore);

        return ret;
}

static boolean_t pmc_reservation_setup_pmi(pmc_reservation_t resv, pmc_config_t config) {
        assert(resv);
        assert(resv->pmc);
        assert(config);
        assert(config->object);

        /* If there's no PMI to setup, return success */
        if(config->interrupt_after_value && config->method) {

                /* set the threshold */
                kern_return_t ret = resv->pmc->methods.config_set_threshold(config->object,
                        config->interrupt_after_value);

                if(KERN_SUCCESS != ret) {
                        /*
                         * This is the most useful error message here, as this only happens
                         * as a result of pmc_reserve*()
                         */
                        COUNTER_DEBUG("Failed to set threshold for pmc %p\n", resv->pmc);
                        return FALSE;
                }

                if(KERN_SUCCESS != resv->pmc->methods.config_set_handler(config->object, 
                        (void *)resv, &pmc_reservation_interrupt, config->refCon)) {

                        COUNTER_DEBUG("Failed to set handler for pmc %p\n", resv->pmc);
                        return FALSE;
                }
        }

        return TRUE;
}

/*
 * pmc_reserve will attempt to reserve the given @pmc, with a given
 * configuration object, for counting system-wide. This method will fail with
 * KERN_FAILURE if the given pmc is already reserved at any scope.
 *
 * This method consumes the given configuration object if it returns
 * KERN_SUCCESS. Any other return value indicates the caller
 * must free the config object via pmc_free_config().
 *
 * NOTE: This method is NOT interrupt safe.
 */
kern_return_t pmc_reserve(pmc_t pmc, pmc_config_t config,
        pmc_reservation_t *reservation) {

        if(!pmc || !config || !reservation) {
                return KERN_INVALID_ARGUMENT;
        }

        pmc_reservation_t resv = reservation_alloc();
        if(!resv) {
                return KERN_RESOURCE_SHORTAGE;
        }

        reservation_init(resv);

        resv->flags |= PMC_FLAG_SCOPE_SYSTEM;
        resv->config = config;

        if(KERN_SUCCESS != pmc_internal_reservation_set_pmc(resv, pmc)) {
                resv->config = NULL;
                return KERN_FAILURE;
        }
        
        /* enqueue reservation in proper place */
        if(!pmc_internal_reservation_add(resv) || !pmc_reservation_setup_pmi(resv, config)) {
                /* Prevent free of config object */
                resv->config = NULL;
                
                reservation_free(resv);
                return KERN_FAILURE;
        }

        /* Here's where we setup the PMI method (if needed) */
        
        *reservation = resv;

        return KERN_SUCCESS;
}

/*
 * pmc_reserve_task will attempt to reserve the given @pmc with a given
 * configuration object, for counting when the given @task is running on any
 * logical core that can directly access the given @pmc.  This method will fail
 * with KERN_FAILURE if the given pmc is already reserved at either system or
 * thread scope.  
 *
 * This method consumes the given configuration object if it returns
 * KERN_SUCCESS. Any other return value indicates the caller
 * must free the config object via pmc_free_config().
 *
 * NOTE: You can reserve the same pmc for N different tasks concurrently.
 * NOTE: This method is NOT interrupt safe.
 */
kern_return_t pmc_reserve_task(pmc_t pmc, pmc_config_t config, 
        task_t task, pmc_reservation_t *reservation) {

        if(!pmc || !config || !reservation || !task) {
                return KERN_INVALID_ARGUMENT;
        }

        if(!pmc->monitor->methods.supports_context_switching(pmc->monitor->object)) {
                COUNTER_DEBUG("pmc %p cannot be context switched!\n", pmc);
                return KERN_INVALID_ARGUMENT;
        }

        pmc_reservation_t resv = reservation_alloc();
        if(!resv) {
                return KERN_RESOURCE_SHORTAGE;
        }

        reservation_init(resv);

        resv->flags |= PMC_FLAG_SCOPE_TASK;
        resv->task = task;

        resv->config = config;

        if(KERN_SUCCESS != pmc_internal_reservation_set_pmc(resv, pmc)) {
                resv->config = NULL;
                return KERN_FAILURE;
        }
        
        /* enqueue reservation in proper place */
        if(!pmc_internal_reservation_add(resv) || !pmc_reservation_setup_pmi(resv, config)) {
                /* Prevent free of config object */
                resv->config = NULL;

                reservation_free(resv);
                return KERN_FAILURE;
        }

        *reservation = resv;

        return KERN_SUCCESS;
}

/*
 * pmc_reserve_thread will attempt to reserve the given @pmc with a given
 * configuration object, for counting when the given @thread is running on any
 * logical core that can directly access the given @pmc.  This method will fail
 * with KERN_FAILURE if the given pmc is already reserved at either system or
 * task scope.  
 *
 * This method consumes the given configuration object if it returns
 * KERN_SUCCESS. Any other return value indicates the caller
 * must free the config object via pmc_free_config().
 *
 * NOTE: You can reserve the same pmc for N different threads concurrently.
 * NOTE: This method is NOT interrupt safe.
 */
kern_return_t pmc_reserve_thread(pmc_t pmc, pmc_config_t config, 
        thread_t thread, pmc_reservation_t *reservation) {
        if(!pmc || !config || !reservation || !thread) {
                return KERN_INVALID_ARGUMENT;
        }

        if(!pmc->monitor->methods.supports_context_switching(pmc->monitor->object)) {
                COUNTER_DEBUG("pmc %p cannot be context switched!\n", pmc);
                return KERN_INVALID_ARGUMENT;
        }

        pmc_reservation_t resv = reservation_alloc();
        if(!resv) {
                return KERN_RESOURCE_SHORTAGE;
        }

        reservation_init(resv);

        resv->flags |= PMC_FLAG_SCOPE_THREAD;
        resv->thread = thread;

        resv->config = config;

        if(KERN_SUCCESS != pmc_internal_reservation_set_pmc(resv, pmc)) {
                resv->config = NULL;
                return KERN_FAILURE;
        }
        
        /* enqueue reservation in proper place */
        if(!pmc_internal_reservation_add(resv) || !pmc_reservation_setup_pmi(resv, config)) {
                /* Prevent free of config object */
                resv->config = NULL;

                reservation_free(resv);
                return KERN_FAILURE;
        }

        *reservation = resv;

        return KERN_SUCCESS;
}

/*
 * pmc_reservation_start instructs the given reservation to start counting as
 * soon as possible. 
 *
 * NOTE: This method is interrupt safe.
 */
kern_return_t pmc_reservation_start(pmc_reservation_t reservation) {
        pmc_state_t newState;

        if(!reservation) {
                return KERN_INVALID_ARGUMENT;
        }

        /* Move the state machine */
        if (PMC_STATE_INVALID == (newState = pmc_internal_reservation_move_for_event(reservation, PMC_STATE_EVENT_START, NULL))) {
                return KERN_FAILURE;
        }
        
        /* If we are currently in an interrupt, don't bother to broadcast since it won't do anything now and the interrupt will
         * broadcast right before it leaves
         */
        if (PMC_STATE_STATE(newState) != PMC_STATE_STATE_INTERRUPT) {   
                /* A valid state move has been made, but won't be picked up until a context switch occurs.  To cause matching
                 * contexts that are currently running to update, we do an inter-processor message to run pmc_internal_reservation_start_cpu
                 * on every cpu that can access the PMC.
                 */
                pmc_internal_reservation_broadcast(reservation, pmc_internal_reservation_start_cpu);
        }
        
        return KERN_SUCCESS;                     
}

/*
 * pmc_reservation_stop instructs the given reservation to stop counting as
 * soon as possible.  When this method returns, the pmc will be marked as stopping
 * and subsequent calls to pmc_reservation_start will succeed.  This does not mean
 * that the pmc hardware has _actually_ stopped running.  Assuming no other changes
 * to the reservation state, the pmc hardware _will_ stop shortly.
 *
 */
kern_return_t pmc_reservation_stop(pmc_reservation_t reservation) {
        pmc_state_t newState;

        if(!reservation) {
                return KERN_INVALID_ARGUMENT;
        }
        
        /* Move the state machine */
        if (PMC_STATE_INVALID == (newState = pmc_internal_reservation_move_for_event(reservation, PMC_STATE_EVENT_STOP, NULL))) {
                return KERN_FAILURE;
        }
        
        /* If we are currently in an interrupt, don't bother to broadcast since it won't do anything now and the interrupt will
         * broadcast right before it leaves.  Similarly, if we just moved directly to STOP, don't bother broadcasting.
         */
        if (PMC_STATE_STATE(newState) != PMC_STATE_STATE_INTERRUPT && PMC_STATE_STATE(newState) != PMC_STATE_STATE_STOP) {      
                /* A valid state move has been made, but won't be picked up until a context switch occurs.  To cause matching
                         * contexts that are currently running to update, we do an inter-processor message to run pmc_internal_reservation_stop_cpu
                 * on every cpu that can access the PMC.
                 */
                
                pmc_internal_reservation_broadcast(reservation, pmc_internal_reservation_stop_cpu);
        }
        
        return KERN_SUCCESS;
}

/*
 * pmc_reservation_read will read the event count associated with a reservation.
 * If the caller is current executing in a context that both a) matches the
 * reservation's context, and b) can access the reservation's pmc directly, the
 * value will be read from hardware.  Otherwise, this returns the reservation's
 * stored value.
 *
 * NOTE: This method is interrupt safe.
 * NOTE: When not on the interrupt stack, this method may block.
 */
kern_return_t pmc_reservation_read(pmc_reservation_t reservation, uint64_t *value) {
        kern_return_t ret = KERN_FAILURE;
        uint64_t timeout;
        uint32_t spins;

        if(!reservation || !value) {
                return KERN_INVALID_ARGUMENT;
        }

        nanoseconds_to_absolutetime(PMC_SPIN_TIMEOUT_US * 1000, &timeout);
        timeout += mach_absolute_time();
        spins = 0;
        do {
                uint32_t state = reservation->state;
                
                if((PMC_STATE_STATE(state) == PMC_STATE_STATE_RUN)) {
                        /* Attempt read from hardware via drivers. */

                        assert(reservation->pmc);

                        ret = reservation->pmc->methods.get_count(reservation->pmc->object, value);
                        
                        break;
                } else if ((PMC_STATE_STATE(state) == PMC_STATE_STATE_STORE) ||
                                   (PMC_STATE_STATE(state) == PMC_STATE_STATE_LOAD)) {
                        /* Spin */
                        /* Assert if this takes longer than PMC_SPIN_TIMEOUT_US */
                        if (++spins > PMC_SPIN_THRESHOLD) {
                                if (mach_absolute_time() > timeout) {
                                        pmc_spin_timeout_count++;
                                        assert(0);
                                }
                        }

                        cpu_pause();
                } else {
                        break;
                }
        } while (1);

        /* If the direct hardware read failed (for whatever reason) */
        if(KERN_SUCCESS != ret) {
                /* Read stored value */
                *value = reservation->value;
        }

        return KERN_SUCCESS;
}

/*
 * pmc_reservation_write will write the event count associated with a reservation.
 * If the caller is current executing in a context that both a) matches the
 * reservation's context, and b) can access the reservation's pmc directly, the
 * value will be written to hardware.  Otherwise, this writes the reservation's
 * stored value.
 *
 * NOTE: This method is interrupt safe.
 * NOTE: When not on the interrupt stack, this method may block.
 */
kern_return_t pmc_reservation_write(pmc_reservation_t reservation, uint64_t value) {
        kern_return_t ret = KERN_FAILURE;
        uint64_t timeout;
        uint32_t spins;

        if(!reservation) {
                return KERN_INVALID_ARGUMENT;
        }

        nanoseconds_to_absolutetime(PMC_SPIN_TIMEOUT_US * 1000, &timeout);
        timeout += mach_absolute_time();
        spins = 0;
        do {
                uint32_t state = reservation->state;
                
                if((PMC_STATE_STATE(state) == PMC_STATE_STATE_RUN)) {
                                /* Write to hardware via drivers. */
                        assert(reservation->pmc);

                        ret = reservation->pmc->methods.set_count(reservation->pmc->object, value);
                        break;
                } else if ((PMC_STATE_STATE(state) == PMC_STATE_STATE_STORE) ||
                                   (PMC_STATE_STATE(state) == PMC_STATE_STATE_LOAD)) {
                        /* Spin */
                        /* Assert if this takes longer than PMC_SPIN_TIMEOUT_US */
                        if (++spins > PMC_SPIN_THRESHOLD) {
                                if (mach_absolute_time() > timeout) {
                                        pmc_spin_timeout_count++;
                                        assert(0);
                                }
                        }

                        cpu_pause();
                } else {
                        break;
                }
        } while (1);
        
        if(KERN_SUCCESS != ret) {
                /* Write stored value */
                reservation->value = value;
        }

        return KERN_SUCCESS;
}

/* 
 * pmc_reservation_free releases a reservation and all associated resources.
 *
 * NOTE: This method is NOT interrupt safe.
 */
kern_return_t pmc_reservation_free(pmc_reservation_t reservation) {
        pmc_state_t newState;
        
        if(!reservation) {
                return KERN_INVALID_ARGUMENT;
        }
        
        /* Move the state machine */
        if (PMC_STATE_INVALID == (newState = pmc_internal_reservation_move_for_event(reservation, PMC_STATE_EVENT_FREE, NULL))) {
                return KERN_FAILURE;
        }

        /* If we didn't move directly to DEALLOC, help things along */  
        if (PMC_STATE_STATE(newState) != PMC_STATE_STATE_DEALLOC) {     
                /* A valid state move has been made, but won't be picked up until a context switch occurs.  To cause matching
                 * contexts that are currently running to update, we do an inter-processor message to run pmc_internal_reservation_stop_cpu
                 * on every cpu that can access the PMC.
                 */
                pmc_internal_reservation_broadcast(reservation, pmc_internal_reservation_stop_cpu);
        }

        /* Block until the reservation hits the <DEALLOC, 0, > state */
        while (!(PMC_STATE_STATE(reservation->state) == PMC_STATE_STATE_DEALLOC && PMC_STATE_CONTEXT_COUNT(reservation->state) == 0 && PMC_STATE_FLAGS(reservation->state) == 0)) {
                assert_wait((event_t)reservation, THREAD_UNINT);
                thread_block(THREAD_CONTINUE_NULL);
        }

        /* remove from queues */
        pmc_internal_reservation_remove(reservation);
                
        /* free reservation */
        reservation_free(reservation);

        return KERN_SUCCESS;
}

/*
 * pmc_context_switch performs all context switching necessary to save all pmc
 * state associated with @oldThread (and the task to which @oldThread belongs),
 * as well as to restore all pmc state associated with @newThread (and the task
 * to which @newThread belongs).
 *
 * NOTE: This method IS interrupt safe.
 */
boolean_t pmc_context_switch(thread_t oldThread, thread_t newThread) {
        pmc_reservation_t resv = NULL;
        uint32_t cpuNum = cpu_number();

        /* Out going thread: save pmc state */
        lck_spin_lock(&reservations_spin);

        /* interate over any reservations */
        queue_iterate(thread_reservations, resv, pmc_reservation_t, link) {
                if(resv && oldThread == resv->thread) {

                        /* check if we can read the associated pmc from this core. */
                        if(pmc_accessible_from_core(resv->pmc, cpuNum)) {
                                /* save the state At this point, if it fails, it fails. */
                                (void)pmc_internal_reservation_context_out(resv);
                        }
                }
        }
        
        queue_iterate(task_reservations, resv, pmc_reservation_t, link) {
                if(resv && resv->task == oldThread->task) {
                        if(pmc_accessible_from_core(resv->pmc, cpuNum)) {
                                (void)pmc_internal_reservation_context_out(resv);
                        }
                }
        }
        
        /* Incoming task: restore */

        queue_iterate(thread_reservations, resv, pmc_reservation_t, link) {
                if(resv && resv->thread == newThread) {
                        if(pmc_accessible_from_core(resv->pmc, cpuNum)) {
                                (void)pmc_internal_reservation_context_in(resv);
                        }
                }
        }
        

        queue_iterate(task_reservations, resv, pmc_reservation_t, link) {
                if(resv && resv->task == newThread->task) {
                        if(pmc_accessible_from_core(resv->pmc, cpuNum)) {
                                (void)pmc_internal_reservation_context_in(resv);
                        }
                }
        }
        
        lck_spin_unlock(&reservations_spin);

        return TRUE;
}

#else /* !CONFIG_COUNTERS */

#if 0
#pragma mark -
#pragma mark Dummy functions
#endif

/*
 * In the case that someone has chosen not to include the PMC KPI in some
 * configuration, we still have exports for kexts, so we'll need to define stub
 * methods that return failures.
 */
kern_return_t perf_monitor_register(perf_monitor_object_t monitor __unused,
        perf_monitor_methods_t *methods __unused) {
        return KERN_FAILURE;
}

kern_return_t perf_monitor_unregister(perf_monitor_object_t monitor __unused) {
        return KERN_FAILURE;
}

kern_return_t pmc_register(perf_monitor_object_t monitor __unused, 
        pmc_object_t pmc __unused, pmc_methods_t *methods __unused, void *object __unused) {
        return KERN_FAILURE;
}

kern_return_t pmc_unregister(perf_monitor_object_t monitor __unused,
        pmc_object_t pmc __unused) {
        return KERN_FAILURE;
}

kern_return_t pmc_create_config(pmc_t pmc __unused, 
        pmc_config_t *config __unused) {
        return KERN_FAILURE;
}

void pmc_free_config(pmc_t pmc __unused, pmc_config_t config __unused) {
}

kern_return_t pmc_config_set_value(pmc_t pmc __unused, 
        pmc_config_t config __unused, uint8_t id __unused, 
        uint64_t value __unused) {
        return KERN_FAILURE;
}

kern_return_t pmc_config_set_interrupt_threshold(pmc_t pmc __unused, 
        pmc_config_t config __unused, uint64_t threshold __unused, 
        pmc_interrupt_method_t method __unused, void *refCon __unused) {
        return KERN_FAILURE;
}

kern_return_t pmc_get_pmc_list(pmc_t **pmcs __unused, size_t *pmcCount __unused) {
        return KERN_FAILURE;
}

void pmc_free_pmc_list(pmc_t *pmcs __unused, size_t pmcCount __unused) {
}

kern_return_t pmc_find_by_name(const char *name __unused, pmc_t **pmcs __unused, 
        size_t *pmcCount __unused) {
        return KERN_FAILURE;
}

const char *pmc_get_name(pmc_t pmc __unused) {
        return "";
}

kern_return_t pmc_get_accessible_core_list(pmc_t pmc __unused, 
        uint32_t **logicalCores __unused, size_t *logicalCoreCt __unused) {
        return KERN_FAILURE;
}

boolean_t pmc_accessible_from_core(pmc_t pmc __unused, 
        uint32_t logicalCore __unused) {
        return FALSE;
}

kern_return_t pmc_reserve(pmc_t pmc __unused, 
        pmc_config_t config __unused, pmc_reservation_t *reservation __unused) {
        return KERN_FAILURE;
}

kern_return_t pmc_reserve_task(pmc_t pmc __unused, 
        pmc_config_t config __unused, task_t task __unused, 
        pmc_reservation_t *reservation __unused) {
        return KERN_FAILURE;
}

kern_return_t pmc_reserve_thread(pmc_t pmc __unused, 
        pmc_config_t config __unused, thread_t thread __unused, 
        pmc_reservation_t *reservation __unused) {
        return KERN_FAILURE;
}

kern_return_t pmc_reservation_start(pmc_reservation_t reservation __unused) {
        return KERN_FAILURE;
}

kern_return_t pmc_reservation_stop(pmc_reservation_t reservation __unused) {
        return KERN_FAILURE;
}

kern_return_t pmc_reservation_read(pmc_reservation_t reservation __unused, 
        uint64_t *value __unused) {
        return KERN_FAILURE;
}

kern_return_t pmc_reservation_write(pmc_reservation_t reservation __unused, 
        uint64_t value __unused) {
        return KERN_FAILURE;
}

kern_return_t pmc_reservation_free(pmc_reservation_t reservation __unused) {
        return KERN_FAILURE;
}


#endif /* !CONFIG_COUNTERS */