xnu-1699.22.81.tar.gz

[apple/xnu.git] / osfmk / kern / processor.c

diff --git a/osfmk/kern/processor.c b/osfmk/kern/processor.c
index 0a413bcf1f51b92ee08f72b2f0bdbb6c9be70d7b..b0771351f96e6b18374d70f3bba7e44aa4b49ac6 100644 (file)
--- a/osfmk/kern/processor.c
+++ b/osfmk/kern/processor.c
@@ -1,5 +1,5 @@
 /*
 /*

- * Copyright (c) 2000-2008 Apple Inc. All rights reserved.
+ * Copyright (c) 2000-2009 Apple Inc. All rights reserved.

  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 


@@ -89,10 +89,11 @@ struct pset_node            pset_node0;
 decl_simple_lock_data(static,pset_node_lock)
 
 queue_head_t                   tasks;
 decl_simple_lock_data(static,pset_node_lock)
 
 queue_head_t                   tasks;

+queue_head_t                   terminated_tasks;       /* To be used ONLY for stackshot. */

 int                                            tasks_count;
 queue_head_t                   threads;
 int                                            threads_count;
 int                                            tasks_count;
 queue_head_t                   threads;
 int                                            threads_count;

-decl_mutex_data(,tasks_threads_lock)
+decl_lck_mtx_data(,tasks_threads_lock)

 
 processor_t                            processor_list;
 unsigned int                   processor_count;
 
 processor_t                            processor_list;
 unsigned int                   processor_count;


@@ -101,8 +102,9 @@ decl_simple_lock_data(,processor_list_lock)
 
 uint32_t                               processor_avail_count;
 
 
 uint32_t                               processor_avail_count;
 

-processor_t    master_processor;
-int            master_cpu = 0;
+processor_t            master_processor;
+int                    master_cpu = 0;
+boolean_t              sched_stats_active = FALSE;

 
 /* Forwards */
 kern_return_t  processor_set_things(
 
 /* Forwards */
 kern_return_t  processor_set_things(


@@ -119,8 +121,8 @@ processor_bootstrap(void)
 
        simple_lock_init(&pset_node_lock, 0);
 
 
        simple_lock_init(&pset_node_lock, 0);
 

-       mutex_init(&tasks_threads_lock, 0);

        queue_init(&tasks);
        queue_init(&tasks);

+       queue_init(&terminated_tasks);

        queue_init(&threads);
 
        simple_lock_init(&processor_list_lock, 0);
        queue_init(&threads);
 
        simple_lock_init(&processor_list_lock, 0);


@@ -132,40 +134,71 @@ processor_bootstrap(void)
 
 /*
  *     Initialize the given processor for the cpu
 
 /*
  *     Initialize the given processor for the cpu

- *     indicated by slot_num, and assign to the
+ *     indicated by cpu_id, and assign to the

  *     specified processor set.
  */
 void
 processor_init(
  *     specified processor set.
  */
 void
 processor_init(

-       processor_t             p,
-       int                             slot_num,
-       processor_set_t pset)
+       processor_t                     processor,
+       int                                     cpu_id,
+       processor_set_t         pset)

 {
 {

-       run_queue_init(&p->runq);
-
-       p->state = PROCESSOR_OFF_LINE;
-       p->active_thread = p->next_thread = p->idle_thread = THREAD_NULL;
-       p->processor_set = pset;
-       p->current_pri = MINPRI;
-       timer_call_setup(&p->quantum_timer, thread_quantum_expire, p);
-       p->deadline = UINT64_MAX;
-       p->timeslice = 0;
-       p->processor_self = IP_NULL;
-       simple_lock_init(&p->lock, 0);
-       processor_data_init(p);
-       PROCESSOR_DATA(p, slot_num) = slot_num;
-       p->processor_list = NULL;
+       if (processor != master_processor) {
+               /* Scheduler state deferred until sched_init() */
+               SCHED(processor_init)(processor);
+       }
+
+       processor->state = PROCESSOR_OFF_LINE;
+       processor->active_thread = processor->next_thread = processor->idle_thread = THREAD_NULL;
+       processor->processor_set = pset;
+       processor->current_pri = MINPRI;
+       processor->current_thmode = TH_MODE_NONE;
+       processor->cpu_id = cpu_id;
+       timer_call_setup(&processor->quantum_timer, thread_quantum_expire, processor);
+       processor->deadline = UINT64_MAX;
+       processor->timeslice = 0;
+       processor->processor_meta = PROCESSOR_META_NULL;
+       processor->processor_self = IP_NULL;
+       processor_data_init(processor);
+       processor->processor_list = NULL;
+
+       pset_lock(pset);
+       if (pset->cpu_set_count++ == 0)
+               pset->cpu_set_low = pset->cpu_set_hi = cpu_id;
+       else {
+               pset->cpu_set_low = (cpu_id < pset->cpu_set_low)? cpu_id: pset->cpu_set_low;
+               pset->cpu_set_hi = (cpu_id > pset->cpu_set_hi)? cpu_id: pset->cpu_set_hi;
+       }
+       pset_unlock(pset);

 
        simple_lock(&processor_list_lock);
        if (processor_list == NULL)
 
        simple_lock(&processor_list_lock);
        if (processor_list == NULL)

-               processor_list = p;
+               processor_list = processor;

        else
        else

-               processor_list_tail->processor_list = p;
-       processor_list_tail = p;
+               processor_list_tail->processor_list = processor;
+       processor_list_tail = processor;

        processor_count++;
        simple_unlock(&processor_list_lock);
 }
 
        processor_count++;
        simple_unlock(&processor_list_lock);
 }
 

+void
+processor_meta_init(
+       processor_t             processor,
+       processor_t             primary)
+{
+       processor_meta_t        pmeta = primary->processor_meta;
+
+       if (pmeta == PROCESSOR_META_NULL) {
+               pmeta = kalloc(sizeof (*pmeta));
+
+               queue_init(&pmeta->idle_queue);
+
+               pmeta->primary = primary;
+       }
+
+       processor->processor_meta = pmeta;
+}
+

 processor_set_t
 processor_pset(
        processor_t     processor)
 processor_set_t
 processor_pset(
        processor_t     processor)


@@ -210,11 +243,18 @@ pset_init(
        processor_set_t         pset,
        pset_node_t                     node)
 {
        processor_set_t         pset,
        pset_node_t                     node)
 {

+       if (pset != &pset0) {
+               /* Scheduler state deferred until sched_init() */
+               SCHED(pset_init)(pset);
+       }
+

        queue_init(&pset->active_queue);
        queue_init(&pset->idle_queue);
        queue_init(&pset->active_queue);
        queue_init(&pset->idle_queue);

-       pset->idle_count = 0;
-       pset->processor_count = 0;
-       pset->low_pri = PROCESSOR_NULL;
+       pset->online_processor_count = 0;
+       pset_pri_init_hint(pset, PROCESSOR_NULL);
+       pset_count_init_hint(pset, PROCESSOR_NULL);
+       pset->cpu_set_low = pset->cpu_set_hi = 0;
+       pset->cpu_set_count = 0;

        pset_lock_init(pset);
        pset->pset_self = IP_NULL;
        pset->pset_name_self = IP_NULL;
        pset_lock_init(pset);
        pset->pset_self = IP_NULL;
        pset->pset_name_self = IP_NULL;


@@ -253,13 +293,13 @@ processor_info(
        processor_info_t                info,
        mach_msg_type_number_t  *count)
 {
        processor_info_t                info,
        mach_msg_type_number_t  *count)
 {

-       register int    slot_num, state;
+       register int    cpu_id, state;

        kern_return_t   result;
 
        if (processor == PROCESSOR_NULL)
                return (KERN_INVALID_ARGUMENT);
 
        kern_return_t   result;
 
        if (processor == PROCESSOR_NULL)
                return (KERN_INVALID_ARGUMENT);
 

-       slot_num = PROCESSOR_DATA(processor, slot_num);
+       cpu_id = processor->cpu_id;

 
        switch (flavor) {
 
 
        switch (flavor) {
 


@@ -271,14 +311,14 @@ processor_info(
                        return (KERN_FAILURE);
 
                basic_info = (processor_basic_info_t) info;
                        return (KERN_FAILURE);
 
                basic_info = (processor_basic_info_t) info;

-               basic_info->cpu_type = slot_type(slot_num);
-               basic_info->cpu_subtype = slot_subtype(slot_num);
+               basic_info->cpu_type = slot_type(cpu_id);
+               basic_info->cpu_subtype = slot_subtype(cpu_id);

                state = processor->state;
                if (state == PROCESSOR_OFF_LINE)
                        basic_info->running = FALSE;
                else
                        basic_info->running = TRUE;
                state = processor->state;
                if (state == PROCESSOR_OFF_LINE)
                        basic_info->running = FALSE;
                else
                        basic_info->running = TRUE;

-               basic_info->slot_num = slot_num;
+               basic_info->slot_num = cpu_id;

                if (processor == master_processor) 
                        basic_info->is_master = TRUE;
                else
                if (processor == master_processor) 
                        basic_info->is_master = TRUE;
                else


@@ -294,16 +334,32 @@ processor_info(
        {
                register processor_cpu_load_info_t      cpu_load_info;
 
        {
                register processor_cpu_load_info_t      cpu_load_info;
 

-           if (*count < PROCESSOR_CPU_LOAD_INFO_COUNT)
+               if (*count < PROCESSOR_CPU_LOAD_INFO_COUNT)

                        return (KERN_FAILURE);
 
                        return (KERN_FAILURE);
 

-           cpu_load_info = (processor_cpu_load_info_t) info;
+               cpu_load_info = (processor_cpu_load_info_t) info;

                cpu_load_info->cpu_ticks[CPU_STATE_USER] =
                cpu_load_info->cpu_ticks[CPU_STATE_USER] =

-                                                       timer_grab(&PROCESSOR_DATA(processor, user_state)) / hz_tick_interval;
+                                                       (uint32_t)(timer_grab(&PROCESSOR_DATA(processor, user_state)) / hz_tick_interval);

                cpu_load_info->cpu_ticks[CPU_STATE_SYSTEM] =
                cpu_load_info->cpu_ticks[CPU_STATE_SYSTEM] =

-                                                       timer_grab(&PROCESSOR_DATA(processor, system_state)) / hz_tick_interval;
-               cpu_load_info->cpu_ticks[CPU_STATE_IDLE] =
-                                                       timer_grab(&PROCESSOR_DATA(processor, idle_state)) / hz_tick_interval;
+                                                       (uint32_t)(timer_grab(&PROCESSOR_DATA(processor, system_state)) / hz_tick_interval);
+               {
+               timer_data_t    idle_temp;
+               timer_t         idle_state;
+
+               idle_state = &PROCESSOR_DATA(processor, idle_state);
+               idle_temp = *idle_state;
+
+               if (PROCESSOR_DATA(processor, current_state) != idle_state ||
+                   timer_grab(&idle_temp) != timer_grab(idle_state))
+                       cpu_load_info->cpu_ticks[CPU_STATE_IDLE] =
+                                                       (uint32_t)(timer_grab(&PROCESSOR_DATA(processor, idle_state)) / hz_tick_interval);
+               else {
+                       timer_advance(&idle_temp, mach_absolute_time() - idle_temp.tstamp);
+                               
+                       cpu_load_info->cpu_ticks[CPU_STATE_IDLE] =
+                               (uint32_t)(timer_grab(&idle_temp) / hz_tick_interval);
+               }
+               }

                cpu_load_info->cpu_ticks[CPU_STATE_NICE] = 0;
 
            *count = PROCESSOR_CPU_LOAD_INFO_COUNT;
                cpu_load_info->cpu_ticks[CPU_STATE_NICE] = 0;
 
            *count = PROCESSOR_CPU_LOAD_INFO_COUNT;


@@ -313,7 +369,7 @@ processor_info(
        }
 
        default:
        }
 
        default:

-           result = cpu_info(flavor, slot_num, info, count);
+           result = cpu_info(flavor, cpu_id, info, count);

            if (result == KERN_SUCCESS)
                        *host = &realhost;                 
 
            if (result == KERN_SUCCESS)
                        *host = &realhost;                 
 


@@ -339,7 +395,7 @@ processor_start(
                prev = thread_bind(processor);
                thread_block(THREAD_CONTINUE_NULL);
 
                prev = thread_bind(processor);
                thread_block(THREAD_CONTINUE_NULL);
 

-               result = cpu_start(PROCESSOR_DATA(processor, slot_num));
+               result = cpu_start(processor->cpu_id);

 
                thread_bind(prev);
 
 
                thread_bind(prev);
 


@@ -408,12 +464,11 @@ processor_start(
        if (processor->processor_self == IP_NULL)
                ipc_processor_init(processor);
 
        if (processor->processor_self == IP_NULL)
                ipc_processor_init(processor);
 

-       result = cpu_start(PROCESSOR_DATA(processor, slot_num));
+       result = cpu_start(processor->cpu_id);

        if (result != KERN_SUCCESS) {
                s = splsched();
                pset_lock(pset);
                processor->state = PROCESSOR_OFF_LINE;
        if (result != KERN_SUCCESS) {
                s = splsched();
                pset_lock(pset);
                processor->state = PROCESSOR_OFF_LINE;

-               timer_call_shutdown(processor);

                pset_unlock(pset);
                splx(s);
 
                pset_unlock(pset);
                splx(s);
 


@@ -444,7 +499,7 @@ processor_control(
        if (processor == PROCESSOR_NULL)
                return(KERN_INVALID_ARGUMENT);
 
        if (processor == PROCESSOR_NULL)
                return(KERN_INVALID_ARGUMENT);
 

-       return(cpu_control(PROCESSOR_DATA(processor, slot_num), info, count));
+       return(cpu_control(processor->cpu_id, info, count));

 }
            
 kern_return_t
 }
            
 kern_return_t


@@ -712,7 +767,7 @@ processor_set_things(
        addr = NULL;
 
        for (;;) {
        addr = NULL;
 
        for (;;) {

-               mutex_lock(&tasks_threads_lock);
+               lck_mtx_lock(&tasks_threads_lock);

 
                if (type == THING_TASK)
                        maxthings = tasks_count;
 
                if (type == THING_TASK)
                        maxthings = tasks_count;


@@ -726,7 +781,7 @@ processor_set_things(
                        break;
 
                /* unlock and allocate more memory */
                        break;
 
                /* unlock and allocate more memory */

-               mutex_unlock(&tasks_threads_lock);
+               lck_mtx_unlock(&tasks_threads_lock);

 
                if (size != 0)
                        kfree(addr, size);
 
                if (size != 0)
                        kfree(addr, size);


@@ -778,7 +833,7 @@ processor_set_things(
 
        }
                
 
        }
                

-       mutex_unlock(&tasks_threads_lock);
+       lck_mtx_unlock(&tasks_threads_lock);

 
        if (actual < maxthings)
                size_needed = actual * sizeof (mach_port_t);
 
        if (actual < maxthings)
                size_needed = actual * sizeof (mach_port_t);