xnu-7195.101.1.tar.gz

[apple/xnu.git] / osfmk / i386 / cpu_topology.c

diff --git a/osfmk/i386/cpu_topology.c b/osfmk/i386/cpu_topology.c
index 6be77e6ffb3b04f2a97cb6ce149ca3ca72fa54a6..1ed64e3df88e22743458f004527819e494b9aff4 100644 (file)
--- a/osfmk/i386/cpu_topology.c
+++ b/osfmk/i386/cpu_topology.c
@@ -2,7 +2,7 @@
  * Copyright (c) 2007-2010 Apple Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * Copyright (c) 2007-2010 Apple Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@

- * 
+ *

  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in
  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in


@@ -11,10 +11,10 @@
  * unlawful or unlicensed copies of an Apple operating system, or to
  * circumvent, violate, or enable the circumvention or violation of, any
  * terms of an Apple operating system software license agreement.
  * unlawful or unlicensed copies of an Apple operating system, or to
  * circumvent, violate, or enable the circumvention or violation of, any
  * terms of an Apple operating system software license agreement.

- * 
+ *

  * Please obtain a copy of the License at
  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  * Please obtain a copy of the License at
  * http://www.opensource.apple.com/apsl/ and read it before using this file.

- * 
+ *

  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,


@@ -22,7 +22,7 @@
  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  * Please see the License for the specific language governing rights and
  * limitations under the License.
  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  * Please see the License for the specific language governing rights and
  * limitations under the License.

- * 
+ *

  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 


@@ -33,39 +33,61 @@
 #include <i386/cpu_topology.h>
 #include <i386/cpu_threads.h>
 #include <i386/machine_cpu.h>
 #include <i386/cpu_topology.h>
 #include <i386/cpu_threads.h>
 #include <i386/machine_cpu.h>

-#include <i386/lock.h>
+#include <i386/bit_routines.h>

 #include <i386/cpu_data.h>
 #include <i386/lapic.h>
 #include <i386/machine_routines.h>
 #include <i386/cpu_data.h>
 #include <i386/lapic.h>
 #include <i386/machine_routines.h>

+#include <stddef.h>

 
 __private_extern__ void qsort(
 
 __private_extern__ void qsort(

-    void * array,
-    size_t nmembers,
-    size_t member_size,
-    int (*)(const void *, const void *));
+       void * array,
+       size_t nmembers,
+       size_t member_size,
+       int (*)(const void *, const void *));

 
 static int lapicid_cmp(const void *x, const void *y);
 static x86_affinity_set_t *find_cache_affinity(x86_cpu_cache_t *L2_cachep);
 
 
 static int lapicid_cmp(const void *x, const void *y);
 static x86_affinity_set_t *find_cache_affinity(x86_cpu_cache_t *L2_cachep);
 

-x86_affinity_set_t     *x86_affinities = NULL;
-static int             x86_affinity_count = 0;
+x86_affinity_set_t      *x86_affinities = NULL;
+static int              x86_affinity_count = 0;
+
+extern cpu_data_t cpshadows[];
+
+#if DEVELOPMENT || DEBUG
+void iotrace_init(int ncpus);
+void traptrace_init(int ncpus);
+#endif /* DEVELOPMENT || DEBUG */
+
+
+/* Re-sort double-mapped CPU data shadows after topology discovery sorts the
+ * primary CPU data structures by physical/APIC CPU ID.
+ */
+static void
+cpu_shadow_sort(int ncpus)
+{
+       for (int i = 0; i < ncpus; i++) {
+               cpu_data_t      *cpup = cpu_datap(i);
+               ptrdiff_t       coff = cpup - cpu_datap(0);
+
+               cpup->cd_shadow = &cpshadows[coff];
+       }
+}

 
 /*
 
 /*

- * cpu_topology_sort() is called after all processors have been registered
- * but before any non-boot processor id started.
- * We establish canonical logical processor numbering - logical cpus must be
- * contiguous, zero-based and assigned in physical (local apic id) order.
- * This step is required because the discovery/registration order is
- * non-deterministic - cores are registered in differing orders over boots.
- * Enforcing canonical numbering simplifies identification
- * of processors - in particular, for stopping/starting from CHUD.
- */ 
+ * cpu_topology_sort() is called after all processors have been registered but
+ * before any non-boot processor is started.  We establish canonical logical
+ * processor numbering - logical cpus must be contiguous, zero-based and
+ * assigned in physical (local apic id) order.  This step is required because
+ * the discovery/registration order is non-deterministic - cores are registered
+ * in differing orders over boots.  Enforcing canonical numbering simplifies
+ * identification of processors.
+ */

 void
 cpu_topology_sort(int ncpus)
 {
 void
 cpu_topology_sort(int ncpus)
 {

-       int             i;
-       boolean_t       istate;
-       processor_t             lprim = NULL;
+       int             i;
+       boolean_t       istate;
+       processor_t             lprim = NULL;

 
        assert(machine_info.physical_cpu == 1);
        assert(machine_info.logical_cpu == 1);
 
        assert(machine_info.physical_cpu == 1);
        assert(machine_info.logical_cpu == 1);


@@ -73,16 +95,22 @@ cpu_topology_sort(int ncpus)
        assert(cpu_number() == 0);
        assert(cpu_datap(0)->cpu_number == 0);
 
        assert(cpu_number() == 0);
        assert(cpu_datap(0)->cpu_number == 0);
 

+       uint32_t cpus_per_pset = 0;
+
+#if DEVELOPMENT || DEBUG
+       PE_parse_boot_argn("cpus_per_pset", &cpus_per_pset, sizeof(cpus_per_pset));
+#endif
+

        /* Lights out for this */
        istate = ml_set_interrupts_enabled(FALSE);
 
        if (topo_dbg) {
                TOPO_DBG("cpu_topology_start() %d cpu%s registered\n",
        /* Lights out for this */
        istate = ml_set_interrupts_enabled(FALSE);
 
        if (topo_dbg) {
                TOPO_DBG("cpu_topology_start() %d cpu%s registered\n",

-                       ncpus, (ncpus > 1) ? "s" : "");
+                   ncpus, (ncpus > 1) ? "s" : "");

                for (i = 0; i < ncpus; i++) {
                for (i = 0; i < ncpus; i++) {

-                       cpu_data_t      *cpup = cpu_datap(i);
+                       cpu_data_t      *cpup = cpu_datap(i);

                        TOPO_DBG("\tcpu_data[%d]:%p local apic 0x%x\n",
                        TOPO_DBG("\tcpu_data[%d]:%p local apic 0x%x\n",

-                               i, (void *) cpup, cpup->cpu_phys_number);
+                           i, (void *) cpup, cpup->cpu_phys_number);

                }
        }
 
                }
        }
 


@@ -92,53 +120,47 @@ cpu_topology_sort(int ncpus)
         */
        if (ncpus > 1) {
                qsort((void *) &cpu_data_ptr[1],
         */
        if (ncpus > 1) {
                qsort((void *) &cpu_data_ptr[1],

-                       ncpus - 1,
-                       sizeof(cpu_data_t *),
-                       lapicid_cmp);
+                   ncpus - 1,
+                   sizeof(cpu_data_t *),
+                   lapicid_cmp);

        }
        if (topo_dbg) {
                TOPO_DBG("cpu_topology_start() after sorting:\n");
                for (i = 0; i < ncpus; i++) {
        }
        if (topo_dbg) {
                TOPO_DBG("cpu_topology_start() after sorting:\n");
                for (i = 0; i < ncpus; i++) {

-                       cpu_data_t      *cpup = cpu_datap(i);
+                       cpu_data_t      *cpup = cpu_datap(i);

                        TOPO_DBG("\tcpu_data[%d]:%p local apic 0x%x\n",
                        TOPO_DBG("\tcpu_data[%d]:%p local apic 0x%x\n",

-                               i, (void *) cpup, cpup->cpu_phys_number);
+                           i, (void *) cpup, cpup->cpu_phys_number);

                }
        }
 
        /*
                }
        }
 
        /*

-        * Fix up logical numbers and reset the map kept by the lapic code.
+        * Finalize logical numbers and map kept by the lapic code.

         */
         */

-       for (i = 1; i < ncpus; i++) {
-               cpu_data_t      *cpup = cpu_datap(i);
-               x86_core_t      *core = cpup->lcpu.core;
-               x86_die_t       *die  = cpup->lcpu.die;
-               x86_pkg_t       *pkg  = cpup->lcpu.package;
-
-               assert(core != NULL);
-               assert(die != NULL);
-               assert(pkg != NULL);
+       for (i = 0; i < ncpus; i++) {
+               cpu_data_t      *cpup = cpu_datap(i);

 
                if (cpup->cpu_number != i) {
                        kprintf("cpu_datap(%d):%p local apic id 0x%x "
 
                if (cpup->cpu_number != i) {
                        kprintf("cpu_datap(%d):%p local apic id 0x%x "

-                               "remapped from %d\n",
-                               i, cpup, cpup->cpu_phys_number,
-                               cpup->cpu_number);
+                           "remapped from %d\n",
+                           i, cpup, cpup->cpu_phys_number,
+                           cpup->cpu_number);

                }
                cpup->cpu_number = i;
                }
                cpup->cpu_number = i;

-               cpup->lcpu.cpu_num = i;
-               cpup->lcpu.pnum = cpup->cpu_phys_number;

                lapic_cpu_map(cpup->cpu_phys_number, i);
                lapic_cpu_map(cpup->cpu_phys_number, i);

-               x86_set_lcpu_numbers(&cpup->lcpu);
-               x86_set_core_numbers(core, &cpup->lcpu);
-               x86_set_die_numbers(die, &cpup->lcpu);
-               x86_set_pkg_numbers(pkg, &cpup->lcpu);
+               x86_set_logical_topology(&cpup->lcpu, cpup->cpu_phys_number, i);

        }
 
        }
 

-       validate_topology();
+       cpu_shadow_sort(ncpus);
+       x86_validate_topology();

 
        ml_set_interrupts_enabled(istate);
        TOPO_DBG("cpu_topology_start() LLC is L%d\n", topoParms.LLCDepth + 1);
 
 
        ml_set_interrupts_enabled(istate);
        TOPO_DBG("cpu_topology_start() LLC is L%d\n", topoParms.LLCDepth + 1);
 

+#if DEVELOPMENT || DEBUG
+       iotrace_init(ncpus);
+       traptrace_init(ncpus);
+#endif /* DEVELOPMENT || DEBUG */
+

        /*
         * Let the CPU Power Management know that the topology is stable.
         */
        /*
         * Let the CPU Power Management know that the topology is stable.
         */


@@ -150,46 +172,61 @@ cpu_topology_sort(int ncpus)
         * for their LLC cache. Each affinity set possesses a processor set
         * into which each logical processor is added.
         */
         * for their LLC cache. Each affinity set possesses a processor set
         * into which each logical processor is added.
         */

-       TOPO_DBG("cpu_topology_start() creating affinity sets:\n");
-       for (i = 0; i < ncpus; i++) {
-               cpu_data_t              *cpup = cpu_datap(i);
-               x86_lcpu_t              *lcpup = cpu_to_lcpu(i);
-               x86_cpu_cache_t         *LLC_cachep;
-               x86_affinity_set_t      *aset;
+       TOPO_DBG("cpu_topology_start() creating affinity sets:ncpus=%d max_cpus=%d\n", ncpus, machine_info.max_cpus);
+       for (i = 0; i < machine_info.max_cpus; i++) {
+               cpu_data_t              *cpup = cpu_datap(i);
+               x86_lcpu_t              *lcpup = cpu_to_lcpu(i);
+               x86_cpu_cache_t         *LLC_cachep;
+               x86_affinity_set_t      *aset;

 
                LLC_cachep = lcpup->caches[topoParms.LLCDepth];
                assert(LLC_cachep->type == CPU_CACHE_TYPE_UNIF);
 
                LLC_cachep = lcpup->caches[topoParms.LLCDepth];
                assert(LLC_cachep->type == CPU_CACHE_TYPE_UNIF);

-               aset = find_cache_affinity(LLC_cachep); 
-               if (aset == NULL) {
+               aset = find_cache_affinity(LLC_cachep);
+               if ((aset == NULL) || ((cpus_per_pset != 0) && (i % cpus_per_pset) == 0)) {

                        aset = (x86_affinity_set_t *) kalloc(sizeof(*aset));
                        aset = (x86_affinity_set_t *) kalloc(sizeof(*aset));

-                       if (aset == NULL)
+                       if (aset == NULL) {

                                panic("cpu_topology_start() failed aset alloc");
                                panic("cpu_topology_start() failed aset alloc");

+                       }

                        aset->next = x86_affinities;
                        x86_affinities = aset;
                        aset->num = x86_affinity_count++;
                        aset->cache = LLC_cachep;
                        aset->pset = (i == master_cpu) ?
                        aset->next = x86_affinities;
                        x86_affinities = aset;
                        aset->num = x86_affinity_count++;
                        aset->cache = LLC_cachep;
                        aset->pset = (i == master_cpu) ?

-                                       processor_pset(master_processor) :
-                                       pset_create(pset_node_root());
-                       if (aset->pset == PROCESSOR_SET_NULL)
+                           processor_pset(master_processor) :
+                           pset_create(pset_node_root());
+                       if (aset->pset == PROCESSOR_SET_NULL) {

                                panic("cpu_topology_start: pset_create");
                                panic("cpu_topology_start: pset_create");

+                       }

                        TOPO_DBG("\tnew set %p(%d) pset %p for cache %p\n",
                        TOPO_DBG("\tnew set %p(%d) pset %p for cache %p\n",

-                               aset, aset->num, aset->pset, aset->cache);
+                           aset, aset->num, aset->pset, aset->cache);

                }
 
                TOPO_DBG("\tprocessor_init set %p(%d) lcpup %p(%d) cpu %p processor %p\n",
                }
 
                TOPO_DBG("\tprocessor_init set %p(%d) lcpup %p(%d) cpu %p processor %p\n",

-                       aset, aset->num, lcpup, lcpup->cpu_num, cpup, cpup->cpu_processor);
+                   aset, aset->num, lcpup, lcpup->cpu_num, cpup, cpup->cpu_processor);

 
 

-               if (i != master_cpu)
+               if (i != master_cpu) {

                        processor_init(cpup->cpu_processor, i, aset->pset);
                        processor_init(cpup->cpu_processor, i, aset->pset);

+               }

 
                if (lcpup->core->num_lcpus > 1) {
 
                if (lcpup->core->num_lcpus > 1) {

-                       if (lcpup->lnum == 0)
+                       if (lcpup->lnum == 0) {

                                lprim = cpup->cpu_processor;
                                lprim = cpup->cpu_processor;

+                       }

 
 

-                       processor_meta_init(cpup->cpu_processor, lprim);
+                       processor_set_primary(cpup->cpu_processor, lprim);

                }
        }
                }
        }

+
+       if (machine_info.max_cpus < machine_info.logical_cpu_max) {
+               /* boot-args cpus=n is set, so adjust max numbers to match */
+               int logical_max = machine_info.max_cpus;
+               int physical_max = logical_max;
+               if (machine_info.logical_cpu_max != machine_info.physical_cpu_max) {
+                       physical_max = (logical_max + 1) / 2;
+               }
+               machine_info.logical_cpu_max = logical_max;
+               machine_info.physical_cpu_max = physical_max;
+       }

 }
 
 /* We got a request to start a CPU. Check that this CPU is within the
 }
 
 /* We got a request to start a CPU. Check that this CPU is within the


@@ -198,46 +235,48 @@ cpu_topology_sort(int ncpus)
 kern_return_t
 cpu_topology_start_cpu( int cpunum )
 {
 kern_return_t
 cpu_topology_start_cpu( int cpunum )
 {

-       int             ncpus = machine_info.max_cpus;
-       int             i = cpunum;
+       int             ncpus = machine_info.max_cpus;
+       int             i = cpunum;

 
        /* Decide whether to start a CPU, and actually start it */
        TOPO_DBG("cpu_topology_start() processor_start():\n");
 
        /* Decide whether to start a CPU, and actually start it */
        TOPO_DBG("cpu_topology_start() processor_start():\n");

-       if( i < ncpus)
-       {
+       if (i < ncpus) {

                TOPO_DBG("\tlcpu %d\n", cpu_datap(i)->cpu_number);
                TOPO_DBG("\tlcpu %d\n", cpu_datap(i)->cpu_number);

-               processor_start(cpu_datap(i)->cpu_processor); 
+               processor_start(cpu_datap(i)->cpu_processor);

                return KERN_SUCCESS;
                return KERN_SUCCESS;

+       } else {
+               return KERN_FAILURE;

        }
        }

-       else
-           return KERN_FAILURE;

 }
 
 static int
 lapicid_cmp(const void *x, const void *y)
 {
 }
 
 static int
 lapicid_cmp(const void *x, const void *y)
 {

-       cpu_data_t      *cpu_x = *((cpu_data_t **)(uintptr_t)x);
-       cpu_data_t      *cpu_y = *((cpu_data_t **)(uintptr_t)y);
+       cpu_data_t      *cpu_x = *((cpu_data_t **)(uintptr_t)x);
+       cpu_data_t      *cpu_y = *((cpu_data_t **)(uintptr_t)y);

 
        TOPO_DBG("lapicid_cmp(%p,%p) (%d,%d)\n",
 
        TOPO_DBG("lapicid_cmp(%p,%p) (%d,%d)\n",

-               x, y, cpu_x->cpu_phys_number, cpu_y->cpu_phys_number);
-       if (cpu_x->cpu_phys_number < cpu_y->cpu_phys_number)
+           x, y, cpu_x->cpu_phys_number, cpu_y->cpu_phys_number);
+       if (cpu_x->cpu_phys_number < cpu_y->cpu_phys_number) {

                return -1;
                return -1;

-       if (cpu_x->cpu_phys_number == cpu_y->cpu_phys_number)
+       }
+       if (cpu_x->cpu_phys_number == cpu_y->cpu_phys_number) {

                return 0;
                return 0;

+       }

        return 1;
 }
 
 static x86_affinity_set_t *
 find_cache_affinity(x86_cpu_cache_t *l2_cachep)
 {
        return 1;
 }
 
 static x86_affinity_set_t *
 find_cache_affinity(x86_cpu_cache_t *l2_cachep)
 {

-       x86_affinity_set_t      *aset;
+       x86_affinity_set_t      *aset;

 
        for (aset = x86_affinities; aset != NULL; aset = aset->next) {
 
        for (aset = x86_affinities; aset != NULL; aset = aset->next) {

-               if (l2_cachep == aset->cache)
+               if (l2_cachep == aset->cache) {

                        break;
                        break;

+               }

        }
        }

-       return aset;                    
+       return aset;

 }
 
 int
 }
 
 int


@@ -247,13 +286,14 @@ ml_get_max_affinity_sets(void)
 }
 
 processor_set_t
 }
 
 processor_set_t

-ml_affinity_to_pset(uint32_t affinity_num) 
+ml_affinity_to_pset(uint32_t affinity_num)

 {
 {

-       x86_affinity_set_t      *aset;
+       x86_affinity_set_t      *aset;

 
        for (aset = x86_affinities; aset != NULL; aset = aset->next) {
 
        for (aset = x86_affinities; aset != NULL; aset = aset->next) {

-               if (affinity_num == aset->num)
+               if (affinity_num == aset->num) {

                        break;
                        break;

+               }

        }
        return (aset == NULL) ? PROCESSOR_SET_NULL : aset->pset;
 }
        }
        return (aset == NULL) ? PROCESSOR_SET_NULL : aset->pset;
 }


@@ -261,12 +301,12 @@ ml_affinity_to_pset(uint32_t affinity_num)
 uint64_t
 ml_cpu_cache_size(unsigned int level)
 {
 uint64_t
 ml_cpu_cache_size(unsigned int level)
 {

-       x86_cpu_cache_t *cachep;
+       x86_cpu_cache_t *cachep;

 
        if (level == 0) {
                return machine_info.max_mem;
 
        if (level == 0) {
                return machine_info.max_mem;

-       } else if ( 1 <= level && level <= MAX_CACHE_DEPTH) {
-               cachep = current_cpu_datap()->lcpu.caches[level-1];
+       } else if (1 <= level && level <= MAX_CACHE_DEPTH) {
+               cachep = current_cpu_datap()->lcpu.caches[level - 1];

                return cachep ? cachep->cache_size : 0;
        } else {
                return 0;
                return cachep ? cachep->cache_size : 0;
        } else {
                return 0;


@@ -276,15 +316,136 @@ ml_cpu_cache_size(unsigned int level)
 uint64_t
 ml_cpu_cache_sharing(unsigned int level)
 {
 uint64_t
 ml_cpu_cache_sharing(unsigned int level)
 {

-       x86_cpu_cache_t *cachep;
+       x86_cpu_cache_t *cachep;

 
        if (level == 0) {
                return machine_info.max_cpus;
 
        if (level == 0) {
                return machine_info.max_cpus;

-       } else if ( 1 <= level && level <= MAX_CACHE_DEPTH) {
-               cachep = current_cpu_datap()->lcpu.caches[level-1];
+       } else if (1 <= level && level <= MAX_CACHE_DEPTH) {
+               cachep = current_cpu_datap()->lcpu.caches[level - 1];

                return cachep ? cachep->nlcpus : 0;
        } else {
                return 0;
        }
 }
 
                return cachep ? cachep->nlcpus : 0;
        } else {
                return 0;
        }
 }
 

+#if     DEVELOPMENT || DEBUG
+volatile int mmiotrace_enabled = 1;
+int iotrace_generators = 0;
+int iotrace_entries_per_cpu = 0;
+int *iotrace_next;
+iotrace_entry_t **iotrace_ring;
+
+volatile int traptrace_enabled = 1;
+int traptrace_generators = 0;
+int traptrace_entries_per_cpu = 0;
+int *traptrace_next;
+traptrace_entry_t **traptrace_ring;
+
+static void
+init_trace_bufs(int cpucnt, int entries_per_cpu, void ***ring, int entry_size,
+    int **next_array, int *allocated_entries_per_cpu, int *allocated_generator_count)
+{
+       int i;
+
+       *next_array = kalloc_tag(cpucnt * sizeof(int), VM_KERN_MEMORY_DIAG);
+       if (__improbable(*next_array == NULL)) {
+               *allocated_generator_count = 0;
+               return;
+       } else {
+               bzero(*next_array, cpucnt * sizeof(int));
+       }
+
+       *ring = kalloc_tag(cpucnt * sizeof(void *), VM_KERN_MEMORY_DIAG);
+       if (__improbable(*ring == NULL)) {
+               kfree(*next_array, cpucnt * sizeof(int));
+               *next_array = NULL;
+               *allocated_generator_count = 0;
+               return;
+       }
+       for (i = 0; i < cpucnt; i++) {
+               (*ring)[i] = kalloc_tag(entries_per_cpu * entry_size, VM_KERN_MEMORY_DIAG);
+               if (__improbable((*ring)[i] == NULL)) {
+                       kfree(*next_array, cpucnt * sizeof(int));
+                       *next_array = NULL;
+                       for (int j = 0; j < i; j++) {
+                               kfree((*ring)[j], entries_per_cpu * entry_size);
+                       }
+                       kfree(*ring, cpucnt * sizeof(void *));
+                       *ring = NULL;
+                       return;
+               }
+               bzero((*ring)[i], entries_per_cpu * entry_size);
+       }
+
+       *allocated_entries_per_cpu = entries_per_cpu;
+       *allocated_generator_count = cpucnt;
+}
+
+
+static void
+init_iotrace_bufs(int cpucnt, int entries_per_cpu)
+{
+       init_trace_bufs(cpucnt, entries_per_cpu, (void ***)&iotrace_ring, sizeof(iotrace_entry_t),
+           &iotrace_next, &iotrace_entries_per_cpu, &iotrace_generators);
+}
+
+static void
+init_traptrace_bufs(int cpucnt, int entries_per_cpu)
+{
+       init_trace_bufs(cpucnt, entries_per_cpu, (void ***)&traptrace_ring, sizeof(traptrace_entry_t),
+           &traptrace_next, &traptrace_entries_per_cpu, &traptrace_generators);
+}
+
+static void
+gentrace_configure_from_bootargs(const char *ena_prop, int *ena_valp, const char *epc_prop,
+    int *epcp, int max_epc, int def_epc, int override)
+{
+       if (kern_feature_override(override)) {
+               *ena_valp = 0;
+       }
+
+       (void) PE_parse_boot_argn(ena_prop, ena_valp, sizeof(*ena_valp));
+
+       if (*ena_valp == 0) {
+               return;
+       }
+
+       if (PE_parse_boot_argn(epc_prop, epcp, sizeof(*epcp)) &&
+           (*epcp < 1 || *epcp > max_epc)) {
+               *epcp = def_epc;
+       }
+}
+
+void
+iotrace_init(int ncpus)
+{
+       int entries_per_cpu = DEFAULT_IOTRACE_ENTRIES_PER_CPU;
+       int enable = mmiotrace_enabled;
+
+       gentrace_configure_from_bootargs("iotrace", &enable, "iotrace_epc", &entries_per_cpu,
+           IOTRACE_MAX_ENTRIES_PER_CPU, DEFAULT_IOTRACE_ENTRIES_PER_CPU, KF_IOTRACE_OVRD);
+
+       mmiotrace_enabled = enable;
+
+       if (mmiotrace_enabled) {
+               init_iotrace_bufs(ncpus, entries_per_cpu);
+       }
+}
+
+void
+traptrace_init(int ncpus)
+{
+       int entries_per_cpu = DEFAULT_TRAPTRACE_ENTRIES_PER_CPU;
+       int enable = traptrace_enabled;
+
+       gentrace_configure_from_bootargs("traptrace", &enable, "traptrace_epc", &entries_per_cpu,
+           TRAPTRACE_MAX_ENTRIES_PER_CPU, DEFAULT_TRAPTRACE_ENTRIES_PER_CPU, KF_TRAPTRACE_OVRD);
+
+       traptrace_enabled = enable;
+
+       if (traptrace_enabled) {
+               init_traptrace_bufs(ncpus, entries_per_cpu);
+       }
+}
+
+#endif /* DEVELOPMENT || DEBUG */