X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/8f6c56a50524aa785f7e596d52dddfb331e18961..d9a64523371fa019c4575bb400cbbc3a50ac9903:/osfmk/kern/processor.c?ds=sidebyside diff --git a/osfmk/kern/processor.c b/osfmk/kern/processor.c index e3853ac7e..479094c30 100644 --- a/osfmk/kern/processor.c +++ b/osfmk/kern/processor.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved. + * Copyright (c) 2000-2009 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -78,293 +78,322 @@ #include #include +#include + +#if defined(CONFIG_XNUPOST) + +#include + +#endif /* CONFIG_XNUPOST */ + /* * Exported interface */ #include #include -/* - * Exported variables. - */ -struct processor_set default_pset; +struct processor_set pset0; +struct pset_node pset_node0; +decl_simple_lock_data(static,pset_node_lock) + +queue_head_t tasks; +queue_head_t terminated_tasks; /* To be used ONLY for stackshot. */ +queue_head_t corpse_tasks; +int tasks_count; +int terminated_tasks_count; +queue_head_t threads; +int threads_count; +decl_lck_mtx_data(,tasks_threads_lock) +decl_lck_mtx_data(,tasks_corpse_lock) processor_t processor_list; unsigned int processor_count; static processor_t processor_list_tail; decl_simple_lock_data(,processor_list_lock) -processor_t master_processor; -int master_cpu = 0; +uint32_t processor_avail_count; -/* Forwards */ -kern_return_t processor_set_base( - processor_set_t pset, - policy_t policy, - policy_base_t base, - boolean_t change); +processor_t master_processor; +int master_cpu = 0; +boolean_t sched_stats_active = FALSE; -kern_return_t processor_set_limit( - processor_set_t pset, - policy_t policy, - policy_limit_t limit, - boolean_t change); +processor_t processor_array[MAX_SCHED_CPUS] = { 0 }; -kern_return_t processor_set_things( - processor_set_t pset, - mach_port_t **thing_list, - mach_msg_type_number_t *count, - int type); +#if defined(CONFIG_XNUPOST) +kern_return_t ipi_test(void); +extern void arm64_ipi_test(void); -void -processor_bootstrap(void) +kern_return_t +ipi_test() { - simple_lock_init(&processor_list_lock, 0); +#if __arm64__ + processor_t p; - master_processor = cpu_to_processor(master_cpu); + for (p = processor_list; p != NULL; p = p->processor_list) { + thread_bind(p); + thread_block(THREAD_CONTINUE_NULL); + kprintf("Running IPI test on cpu %d\n", p->cpu_id); + arm64_ipi_test(); + } + + /* unbind thread from specific cpu */ + thread_bind(PROCESSOR_NULL); + thread_block(THREAD_CONTINUE_NULL); - processor_init(master_processor, master_cpu); + T_PASS("Done running IPI tests"); +#else + T_PASS("Unsupported platform. Not running IPI tests"); + +#endif /* __arm64__ */ + + return KERN_SUCCESS; } +#endif /* defined(CONFIG_XNUPOST) */ -/* - * Initialize the given processor_set structure. - */ void -pset_init( - register processor_set_t pset) +processor_bootstrap(void) { - register int i; - - /* setup run queue */ - pset->runq.highq = IDLEPRI; - for (i = 0; i < NRQBM; i++) - pset->runq.bitmap[i] = 0; - setbit(MAXPRI - IDLEPRI, pset->runq.bitmap); - pset->runq.urgency = pset->runq.count = 0; - for (i = 0; i < NRQS; i++) - queue_init(&pset->runq.queues[i]); - - queue_init(&pset->idle_queue); - pset->idle_count = 0; - queue_init(&pset->active_queue); - simple_lock_init(&pset->sched_lock, 0); - pset->run_count = pset->share_count = 0; - pset->mach_factor = pset->load_average = 0; - pset->pri_shift = INT8_MAX; - queue_init(&pset->processors); - pset->processor_count = 0; - queue_init(&pset->tasks); - pset->task_count = 0; - queue_init(&pset->threads); - pset->thread_count = 0; - pset->ref_count = 1; - pset->active = TRUE; - mutex_init(&pset->lock, 0); - pset->pset_self = IP_NULL; - pset->pset_name_self = IP_NULL; - pset->timeshare_quanta = 1; + pset_init(&pset0, &pset_node0); + pset_node0.psets = &pset0; + + simple_lock_init(&pset_node_lock, 0); + + queue_init(&tasks); + queue_init(&terminated_tasks); + queue_init(&threads); + queue_init(&corpse_tasks); + + simple_lock_init(&processor_list_lock, 0); + + master_processor = cpu_to_processor(master_cpu); + + processor_init(master_processor, master_cpu, &pset0); } /* - * Initialize the given processor structure for the processor in - * the slot specified by slot_num. + * Initialize the given processor for the cpu + * indicated by cpu_id, and assign to the + * specified processor set. */ void processor_init( - register processor_t p, - int slot_num) + processor_t processor, + int cpu_id, + processor_set_t pset) { - register int i; - - /* setup run queue */ - p->runq.highq = IDLEPRI; - for (i = 0; i < NRQBM; i++) - p->runq.bitmap[i] = 0; - setbit(MAXPRI - IDLEPRI, p->runq.bitmap); - p->runq.urgency = p->runq.count = 0; - for (i = 0; i < NRQS; i++) - queue_init(&p->runq.queues[i]); - - p->state = PROCESSOR_OFF_LINE; - p->active_thread = p->next_thread = p->idle_thread = THREAD_NULL; - p->processor_set = PROCESSOR_SET_NULL; - p->current_pri = MINPRI; - p->deadline = UINT64_MAX; - timer_call_setup(&p->quantum_timer, thread_quantum_expire, p); - p->timeslice = 0; - simple_lock_init(&p->lock, 0); - p->processor_self = IP_NULL; - processor_data_init(p); - PROCESSOR_DATA(p, slot_num) = slot_num; + spl_t s; + + if (processor != master_processor) { + /* Scheduler state for master_processor initialized in sched_init() */ + SCHED(processor_init)(processor); + } + + assert(cpu_id < MAX_SCHED_CPUS); + + processor->state = PROCESSOR_OFF_LINE; + processor->active_thread = processor->next_thread = processor->idle_thread = THREAD_NULL; + processor->processor_set = pset; + processor_state_update_idle(processor); + processor->starting_pri = MINPRI; + processor->cpu_id = cpu_id; + timer_call_setup(&processor->quantum_timer, thread_quantum_expire, processor); + processor->quantum_end = UINT64_MAX; + processor->deadline = UINT64_MAX; + processor->first_timeslice = FALSE; + processor->processor_primary = processor; /* no SMT relationship known at this point */ + processor->processor_secondary = NULL; + processor->is_SMT = FALSE; + processor->is_recommended = (pset->recommended_bitmask & (1ULL << cpu_id)) ? TRUE : FALSE; + processor->processor_self = IP_NULL; + processor_data_init(processor); + processor->processor_list = NULL; + processor->cpu_quiesce_state = CPU_QUIESCE_COUNTER_NONE; + processor->cpu_quiesce_last_checkin = 0; + + s = splsched(); + pset_lock(pset); + bit_set(pset->cpu_bitmask, cpu_id); + if (pset->cpu_set_count++ == 0) + pset->cpu_set_low = pset->cpu_set_hi = cpu_id; + else { + pset->cpu_set_low = (cpu_id < pset->cpu_set_low)? cpu_id: pset->cpu_set_low; + pset->cpu_set_hi = (cpu_id > pset->cpu_set_hi)? cpu_id: pset->cpu_set_hi; + } + pset_unlock(pset); + splx(s); simple_lock(&processor_list_lock); if (processor_list == NULL) - processor_list = p; + processor_list = processor; else - processor_list_tail->processor_list = p; - processor_list_tail = p; + processor_list_tail->processor_list = processor; + processor_list_tail = processor; processor_count++; - p->processor_list = NULL; + processor_array[cpu_id] = processor; simple_unlock(&processor_list_lock); } -/* - * pset_deallocate: - * - * Remove one reference to the processor set. Destroy processor_set - * if this was the last reference. - */ void -pset_deallocate( - processor_set_t pset) +processor_set_primary( + processor_t processor, + processor_t primary) { - if (pset == PROCESSOR_SET_NULL) - return; - - assert(pset == &default_pset); - return; + assert(processor->processor_primary == primary || processor->processor_primary == processor); + /* Re-adjust primary point for this (possibly) secondary processor */ + processor->processor_primary = primary; + + assert(primary->processor_secondary == NULL || primary->processor_secondary == processor); + if (primary != processor) { + /* Link primary to secondary, assumes a 2-way SMT model + * We'll need to move to a queue if any future architecture + * requires otherwise. + */ + assert(processor->processor_secondary == NULL); + primary->processor_secondary = processor; + /* Mark both processors as SMT siblings */ + primary->is_SMT = TRUE; + processor->is_SMT = TRUE; + + processor_set_t pset = processor->processor_set; + atomic_bit_clear(&pset->primary_map, processor->cpu_id, memory_order_relaxed); + } } -/* - * pset_reference: - * - * Add one reference to the processor set. - */ -void -pset_reference( - processor_set_t pset) +processor_set_t +processor_pset( + processor_t processor) { - if (pset == PROCESSOR_SET_NULL) - return; - - assert(pset == &default_pset); + return (processor->processor_set); } -#define pset_reference_locked(pset) assert(pset == &default_pset) - -/* - * pset_remove_processor() removes a processor from a processor_set. - * It can only be called on the current processor. Caller must - * hold lock on current processor and processor set. - */ void -pset_remove_processor( - processor_set_t pset, - processor_t processor) +processor_state_update_idle(processor_t processor) { - if (pset != processor->processor_set) - panic("pset_remove_processor: wrong pset"); - - queue_remove(&pset->processors, processor, processor_t, processors); - processor->processor_set = PROCESSOR_SET_NULL; - pset->processor_count--; - timeshare_quanta_update(pset); + processor->current_pri = IDLEPRI; + processor->current_sfi_class = SFI_CLASS_KERNEL; + processor->current_recommended_pset_type = PSET_SMP; + processor->current_perfctl_class = PERFCONTROL_CLASS_IDLE; } -/* - * pset_add_processor() adds a processor to a processor_set. - * It can only be called on the current processor. Caller must - * hold lock on curent processor and on pset. No reference counting on - * processors. Processor reference to pset is implicit. - */ void -pset_add_processor( - processor_set_t pset, - processor_t processor) +processor_state_update_from_thread(processor_t processor, thread_t thread) { - queue_enter(&pset->processors, processor, processor_t, processors); - processor->processor_set = pset; - pset->processor_count++; - timeshare_quanta_update(pset); + processor->current_pri = thread->sched_pri; + processor->current_sfi_class = thread->sfi_class; + processor->current_recommended_pset_type = recommended_pset_type(thread); + processor->current_perfctl_class = thread_get_perfcontrol_class(thread); } -/* - * pset_remove_task() removes a task from a processor_set. - * Caller must hold locks on pset and task (unless task has - * no references left, in which case just the pset lock is - * needed). Pset reference count is not decremented; - * caller must explicitly pset_deallocate. - */ void -pset_remove_task( - processor_set_t pset, - task_t task) +processor_state_update_explicit(processor_t processor, int pri, sfi_class_id_t sfi_class, + pset_cluster_type_t pset_type, perfcontrol_class_t perfctl_class) { - if (pset != task->processor_set) - return; - - queue_remove(&pset->tasks, task, task_t, pset_tasks); - pset->task_count--; + processor->current_pri = pri; + processor->current_sfi_class = sfi_class; + processor->current_recommended_pset_type = pset_type; + processor->current_perfctl_class = perfctl_class; } -/* - * pset_add_task() adds a task to a processor_set. - * Caller must hold locks on pset and task. Pset references to - * tasks are implicit. - */ -void -pset_add_task( - processor_set_t pset, - task_t task) +pset_node_t +pset_node_root(void) { - queue_enter(&pset->tasks, task, task_t, pset_tasks); - task->processor_set = pset; - pset->task_count++; - pset_reference_locked(pset); + return &pset_node0; } -/* - * pset_remove_thread() removes a thread from a processor_set. - * Caller must hold locks on pset and thread (but only if thread - * has outstanding references that could be used to lookup the pset). - * The pset reference count is not decremented; caller must explicitly - * pset_deallocate. - */ -void -pset_remove_thread( - processor_set_t pset, - thread_t thread) +processor_set_t +pset_create( + pset_node_t node) { - queue_remove(&pset->threads, thread, thread_t, pset_threads); - pset->thread_count--; + /* some schedulers do not support multiple psets */ + if (SCHED(multiple_psets_enabled) == FALSE) + return processor_pset(master_processor); + + processor_set_t *prev, pset = kalloc(sizeof (*pset)); + + if (pset != PROCESSOR_SET_NULL) { + pset_init(pset, node); + + simple_lock(&pset_node_lock); + + prev = &node->psets; + while (*prev != PROCESSOR_SET_NULL) + prev = &(*prev)->pset_list; + + *prev = pset; + + simple_unlock(&pset_node_lock); + } + + return (pset); } /* - * pset_add_thread() adds a thread to a processor_set. - * Caller must hold locks on pset and thread. Pset references to - * threads are implicit. + * Find processor set in specified node with specified cluster_id. + * Returns default_pset if not found. */ -void -pset_add_thread( - processor_set_t pset, - thread_t thread) +processor_set_t +pset_find( + uint32_t cluster_id, + processor_set_t default_pset) { - queue_enter(&pset->threads, thread, thread_t, pset_threads); - thread->processor_set = pset; - pset->thread_count++; - pset_reference_locked(pset); + simple_lock(&pset_node_lock); + pset_node_t node = &pset_node0; + processor_set_t pset = NULL; + + do { + pset = node->psets; + while (pset != NULL) { + if (pset->pset_cluster_id == cluster_id) + break; + pset = pset->pset_list; + } + } while ((node = node->node_list) != NULL); + simple_unlock(&pset_node_lock); + if (pset == NULL) + return default_pset; + return (pset); } /* - * thread_change_psets() changes the pset of a thread. Caller must - * hold locks on both psets and thread. The old pset must be - * explicitly pset_deallocat()'ed by caller. + * Initialize the given processor_set structure. */ void -thread_change_psets( - thread_t thread, - processor_set_t old_pset, - processor_set_t new_pset) +pset_init( + processor_set_t pset, + pset_node_t node) { - queue_remove(&old_pset->threads, thread, thread_t, pset_threads); - old_pset->thread_count--; - queue_enter(&new_pset->threads, thread, thread_t, pset_threads); - thread->processor_set = new_pset; - new_pset->thread_count++; - pset_reference_locked(new_pset); -} + if (pset != &pset0) { + /* Scheduler state for pset0 initialized in sched_init() */ + SCHED(pset_init)(pset); + SCHED(rt_init)(pset); + } + pset->online_processor_count = 0; + pset->load_average = 0; + pset->cpu_set_low = pset->cpu_set_hi = 0; + pset->cpu_set_count = 0; + pset->last_chosen = -1; + pset->cpu_bitmask = 0; + pset->recommended_bitmask = ~0ULL; + pset->primary_map = ~0ULL; + pset->cpu_state_map[PROCESSOR_OFF_LINE] = ~0ULL; + for (uint i = PROCESSOR_SHUTDOWN; i < PROCESSOR_STATE_LEN; i++) { + pset->cpu_state_map[i] = 0; + } + pset->pending_AST_cpu_mask = 0; +#if defined(CONFIG_SCHED_DEFERRED_AST) + pset->pending_deferred_AST_cpu_mask = 0; +#endif + pset->pending_spill_cpu_mask = 0; + pset_lock_init(pset); + pset->pset_self = IP_NULL; + pset->pset_name_self = IP_NULL; + pset->pset_list = PROCESSOR_SET_NULL; + pset->node = node; + pset->pset_cluster_type = PSET_SMP; + pset->pset_cluster_id = 0; +} kern_return_t processor_info_count( @@ -391,38 +420,38 @@ processor_info_count( kern_return_t processor_info( - register processor_t processor, + processor_t processor, processor_flavor_t flavor, host_t *host, processor_info_t info, mach_msg_type_number_t *count) { - register int i, slot_num, state; + int cpu_id, state; kern_return_t result; if (processor == PROCESSOR_NULL) return (KERN_INVALID_ARGUMENT); - slot_num = PROCESSOR_DATA(processor, slot_num); + cpu_id = processor->cpu_id; switch (flavor) { case PROCESSOR_BASIC_INFO: { - register processor_basic_info_t basic_info; + processor_basic_info_t basic_info; if (*count < PROCESSOR_BASIC_INFO_COUNT) return (KERN_FAILURE); basic_info = (processor_basic_info_t) info; - basic_info->cpu_type = slot_type(slot_num); - basic_info->cpu_subtype = slot_subtype(slot_num); + basic_info->cpu_type = slot_type(cpu_id); + basic_info->cpu_subtype = slot_subtype(cpu_id); state = processor->state; if (state == PROCESSOR_OFF_LINE) basic_info->running = FALSE; else basic_info->running = TRUE; - basic_info->slot_num = slot_num; + basic_info->slot_num = cpu_id; if (processor == master_processor) basic_info->is_master = TRUE; else @@ -436,16 +465,72 @@ processor_info( case PROCESSOR_CPU_LOAD_INFO: { - register processor_cpu_load_info_t cpu_load_info; - register integer_t *cpu_ticks; - - if (*count < PROCESSOR_CPU_LOAD_INFO_COUNT) + processor_cpu_load_info_t cpu_load_info; + timer_t idle_state; + uint64_t idle_time_snapshot1, idle_time_snapshot2; + uint64_t idle_time_tstamp1, idle_time_tstamp2; + + /* + * We capture the accumulated idle time twice over + * the course of this function, as well as the timestamps + * when each were last updated. Since these are + * all done using non-atomic racy mechanisms, the + * most we can infer is whether values are stable. + * timer_grab() is the only function that can be + * used reliably on another processor's per-processor + * data. + */ + + if (*count < PROCESSOR_CPU_LOAD_INFO_COUNT) return (KERN_FAILURE); - cpu_load_info = (processor_cpu_load_info_t) info; - cpu_ticks = PROCESSOR_DATA(processor, cpu_ticks); - for (i=0; i < CPU_STATE_MAX; i++) - cpu_load_info->cpu_ticks[i] = cpu_ticks[i]; + cpu_load_info = (processor_cpu_load_info_t) info; + if (precise_user_kernel_time) { + cpu_load_info->cpu_ticks[CPU_STATE_USER] = + (uint32_t)(timer_grab(&PROCESSOR_DATA(processor, user_state)) / hz_tick_interval); + cpu_load_info->cpu_ticks[CPU_STATE_SYSTEM] = + (uint32_t)(timer_grab(&PROCESSOR_DATA(processor, system_state)) / hz_tick_interval); + } else { + uint64_t tval = timer_grab(&PROCESSOR_DATA(processor, user_state)) + + timer_grab(&PROCESSOR_DATA(processor, system_state)); + + cpu_load_info->cpu_ticks[CPU_STATE_USER] = (uint32_t)(tval / hz_tick_interval); + cpu_load_info->cpu_ticks[CPU_STATE_SYSTEM] = 0; + } + + idle_state = &PROCESSOR_DATA(processor, idle_state); + idle_time_snapshot1 = timer_grab(idle_state); + idle_time_tstamp1 = idle_state->tstamp; + + /* + * Idle processors are not continually updating their + * per-processor idle timer, so it may be extremely + * out of date, resulting in an over-representation + * of non-idle time between two measurement + * intervals by e.g. top(1). If we are non-idle, or + * have evidence that the timer is being updated + * concurrently, we consider its value up-to-date. + */ + if (PROCESSOR_DATA(processor, current_state) != idle_state) { + cpu_load_info->cpu_ticks[CPU_STATE_IDLE] = + (uint32_t)(idle_time_snapshot1 / hz_tick_interval); + } else if ((idle_time_snapshot1 != (idle_time_snapshot2 = timer_grab(idle_state))) || + (idle_time_tstamp1 != (idle_time_tstamp2 = idle_state->tstamp))){ + /* Idle timer is being updated concurrently, second stamp is good enough */ + cpu_load_info->cpu_ticks[CPU_STATE_IDLE] = + (uint32_t)(idle_time_snapshot2 / hz_tick_interval); + } else { + /* + * Idle timer may be very stale. Fortunately we have established + * that idle_time_snapshot1 and idle_time_tstamp1 are unchanging + */ + idle_time_snapshot1 += mach_absolute_time() - idle_time_tstamp1; + + cpu_load_info->cpu_ticks[CPU_STATE_IDLE] = + (uint32_t)(idle_time_snapshot1 / hz_tick_interval); + } + + cpu_load_info->cpu_ticks[CPU_STATE_NICE] = 0; *count = PROCESSOR_CPU_LOAD_INFO_COUNT; *host = &realhost; @@ -454,7 +539,7 @@ processor_info( } default: - result = cpu_info(flavor, slot_num, info, count); + result = cpu_info(flavor, cpu_id, info, count); if (result == KERN_SUCCESS) *host = &realhost; @@ -464,40 +549,41 @@ processor_info( kern_return_t processor_start( - processor_t processor) + processor_t processor) { - kern_return_t result; - thread_t thread; - spl_t s; + processor_set_t pset; + thread_t thread; + kern_return_t result; + spl_t s; - if (processor == PROCESSOR_NULL) + if (processor == PROCESSOR_NULL || processor->processor_set == PROCESSOR_SET_NULL) return (KERN_INVALID_ARGUMENT); if (processor == master_processor) { - thread_t self = current_thread(); processor_t prev; - prev = thread_bind(self, processor); + prev = thread_bind(processor); thread_block(THREAD_CONTINUE_NULL); - result = cpu_start(PROCESSOR_DATA(processor, slot_num)); + result = cpu_start(processor->cpu_id); - thread_bind(self, prev); + thread_bind(prev); return (result); } s = splsched(); - processor_lock(processor); + pset = processor->processor_set; + pset_lock(pset); if (processor->state != PROCESSOR_OFF_LINE) { - processor_unlock(processor); + pset_unlock(pset); splx(s); return (KERN_FAILURE); } - processor->state = PROCESSOR_START; - processor_unlock(processor); + pset_update_processor_state(pset, processor, PROCESSOR_START); + pset_unlock(pset); splx(s); /* @@ -507,9 +593,9 @@ processor_start( result = idle_thread_create(processor); if (result != KERN_SUCCESS) { s = splsched(); - processor_lock(processor); - processor->state = PROCESSOR_OFF_LINE; - processor_unlock(processor); + pset_lock(pset); + pset_update_processor_state(pset, processor, PROCESSOR_OFF_LINE); + pset_unlock(pset); splx(s); return (result); @@ -526,9 +612,9 @@ processor_start( result = kernel_thread_create((thread_continue_t)processor_start_thread, NULL, MAXPRI_KERNEL, &thread); if (result != KERN_SUCCESS) { s = splsched(); - processor_lock(processor); - processor->state = PROCESSOR_OFF_LINE; - processor_unlock(processor); + pset_lock(pset); + pset_update_processor_state(pset, processor, PROCESSOR_OFF_LINE); + pset_unlock(pset); splx(s); return (result); @@ -539,6 +625,7 @@ processor_start( thread->bound_processor = processor; processor->next_thread = thread; thread->state = TH_RUN; + thread->last_made_runnable_time = mach_absolute_time(); thread_unlock(thread); splx(s); @@ -548,13 +635,12 @@ processor_start( if (processor->processor_self == IP_NULL) ipc_processor_init(processor); - result = cpu_start(PROCESSOR_DATA(processor, slot_num)); + result = cpu_start(processor->cpu_id); if (result != KERN_SUCCESS) { s = splsched(); - processor_lock(processor); - processor->state = PROCESSOR_OFF_LINE; - timer_call_shutdown(processor); - processor_unlock(processor); + pset_lock(pset); + pset_update_processor_state(pset, processor, PROCESSOR_OFF_LINE); + pset_unlock(pset); splx(s); return (result); @@ -584,29 +670,7 @@ processor_control( if (processor == PROCESSOR_NULL) return(KERN_INVALID_ARGUMENT); - return(cpu_control(PROCESSOR_DATA(processor, slot_num), info, count)); -} - -/* - * Calculate the appropriate timesharing quanta based on set load. - */ - -void -timeshare_quanta_update( - processor_set_t pset) -{ - int pcount = pset->processor_count; - int i = pset->runq.count; - - if (i >= pcount) - i = 1; - else - if (i <= 1) - i = pcount; - else - i = (pcount + (i / 2)) / i; - - pset->timeshare_quanta = i; + return(cpu_control(processor->cpu_id, info, count)); } kern_return_t @@ -630,14 +694,17 @@ processor_get_assignment( processor_t processor, processor_set_t *pset) { - int state; + int state; + + if (processor == PROCESSOR_NULL) + return(KERN_INVALID_ARGUMENT); state = processor->state; if (state == PROCESSOR_SHUTDOWN || state == PROCESSOR_OFF_LINE) return(KERN_FAILURE); - *pset = processor->processor_set; - pset_reference(*pset); + *pset = &pset0; + return(KERN_SUCCESS); } @@ -653,13 +720,13 @@ processor_set_info( return(KERN_INVALID_ARGUMENT); if (flavor == PROCESSOR_SET_BASIC_INFO) { - register processor_set_basic_info_t basic_info; + processor_set_basic_info_t basic_info; if (*count < PROCESSOR_SET_BASIC_INFO_COUNT) return(KERN_FAILURE); basic_info = (processor_set_basic_info_t) info; - basic_info->processor_count = pset->processor_count; + basic_info->processor_count = processor_avail_count; basic_info->default_policy = POLICY_TIMESHARE; *count = PROCESSOR_SET_BASIC_INFO_COUNT; @@ -667,7 +734,7 @@ processor_set_info( return(KERN_SUCCESS); } else if (flavor == PROCESSOR_SET_TIMESHARE_DEFAULT) { - register policy_timeshare_base_t ts_base; + policy_timeshare_base_t ts_base; if (*count < POLICY_TIMESHARE_BASE_COUNT) return(KERN_FAILURE); @@ -680,7 +747,7 @@ processor_set_info( return(KERN_SUCCESS); } else if (flavor == PROCESSOR_SET_FIFO_DEFAULT) { - register policy_fifo_base_t fifo_base; + policy_fifo_base_t fifo_base; if (*count < POLICY_FIFO_BASE_COUNT) return(KERN_FAILURE); @@ -693,7 +760,7 @@ processor_set_info( return(KERN_SUCCESS); } else if (flavor == PROCESSOR_SET_RR_DEFAULT) { - register policy_rr_base_t rr_base; + policy_rr_base_t rr_base; if (*count < POLICY_RR_BASE_COUNT) return(KERN_FAILURE); @@ -707,7 +774,7 @@ processor_set_info( return(KERN_SUCCESS); } else if (flavor == PROCESSOR_SET_TIMESHARE_LIMITS) { - register policy_timeshare_limit_t ts_limit; + policy_timeshare_limit_t ts_limit; if (*count < POLICY_TIMESHARE_LIMIT_COUNT) return(KERN_FAILURE); @@ -720,7 +787,7 @@ processor_set_info( return(KERN_SUCCESS); } else if (flavor == PROCESSOR_SET_FIFO_LIMITS) { - register policy_fifo_limit_t fifo_limit; + policy_fifo_limit_t fifo_limit; if (*count < POLICY_FIFO_LIMIT_COUNT) return(KERN_FAILURE); @@ -733,7 +800,7 @@ processor_set_info( return(KERN_SUCCESS); } else if (flavor == PROCESSOR_SET_RR_LIMITS) { - register policy_rr_limit_t rr_limit; + policy_rr_limit_t rr_limit; if (*count < POLICY_RR_LIMIT_COUNT) return(KERN_FAILURE); @@ -746,7 +813,7 @@ processor_set_info( return(KERN_SUCCESS); } else if (flavor == PROCESSOR_SET_ENABLED_POLICIES) { - register int *enabled; + int *enabled; if (*count < (sizeof(*enabled)/sizeof(int))) return(KERN_FAILURE); @@ -776,29 +843,28 @@ processor_set_statistics( processor_set_info_t info, mach_msg_type_number_t *count) { - if (pset == PROCESSOR_SET_NULL) - return (KERN_INVALID_PROCESSOR_SET); + if (pset == PROCESSOR_SET_NULL || pset != &pset0) + return (KERN_INVALID_PROCESSOR_SET); + + if (flavor == PROCESSOR_SET_LOAD_INFO) { + processor_set_load_info_t load_info; - if (flavor == PROCESSOR_SET_LOAD_INFO) { - register processor_set_load_info_t load_info; + if (*count < PROCESSOR_SET_LOAD_INFO_COUNT) + return(KERN_FAILURE); - if (*count < PROCESSOR_SET_LOAD_INFO_COUNT) - return(KERN_FAILURE); + load_info = (processor_set_load_info_t) info; - load_info = (processor_set_load_info_t) info; + load_info->mach_factor = sched_mach_factor; + load_info->load_average = sched_load_average; - pset_lock(pset); - load_info->task_count = pset->task_count; - load_info->thread_count = pset->thread_count; - load_info->mach_factor = pset->mach_factor; - load_info->load_average = pset->load_average; - pset_unlock(pset); + load_info->task_count = tasks_count; + load_info->thread_count = threads_count; - *count = PROCESSOR_SET_LOAD_INFO_COUNT; - return(KERN_SUCCESS); - } + *count = PROCESSOR_SET_LOAD_INFO_COUNT; + return(KERN_SUCCESS); + } - return(KERN_INVALID_ARGUMENT); + return(KERN_INVALID_ARGUMENT); } /* @@ -846,9 +912,6 @@ processor_set_policy_disable( return (KERN_INVALID_ARGUMENT); } -#define THING_TASK 0 -#define THING_THREAD 1 - /* * processor_set_things: * @@ -856,168 +919,227 @@ processor_set_policy_disable( */ kern_return_t processor_set_things( - processor_set_t pset, - mach_port_t **thing_list, - mach_msg_type_number_t *count, - int type) + processor_set_t pset, + void **thing_list, + mach_msg_type_number_t *count, + int type) { - unsigned int actual; /* this many things */ - unsigned int maxthings; unsigned int i; + task_t task; + thread_t thread; + + task_t *task_list; + unsigned int actual_tasks; + vm_size_t task_size, task_size_needed; + thread_t *thread_list; + unsigned int actual_threads; + vm_size_t thread_size, thread_size_needed; + + void *addr, *newaddr; vm_size_t size, size_needed; - void *addr; - if (pset == PROCESSOR_SET_NULL) + if (pset == PROCESSOR_SET_NULL || pset != &pset0) return (KERN_INVALID_ARGUMENT); - size = 0; addr = 0; + task_size = 0; + task_size_needed = 0; + task_list = NULL; + actual_tasks = 0; - for (;;) { - pset_lock(pset); - if (!pset->active) { - pset_unlock(pset); + thread_size = 0; + thread_size_needed = 0; + thread_list = NULL; + actual_threads = 0; - return (KERN_FAILURE); - } - - if (type == THING_TASK) - maxthings = pset->task_count; - else - maxthings = pset->thread_count; + for (;;) { + lck_mtx_lock(&tasks_threads_lock); /* do we have the memory we need? */ + if (type == PSET_THING_THREAD) + thread_size_needed = threads_count * sizeof(void *); +#if !CONFIG_MACF + else +#endif + task_size_needed = tasks_count * sizeof(void *); - size_needed = maxthings * sizeof (mach_port_t); - if (size_needed <= size) + if (task_size_needed <= task_size && + thread_size_needed <= thread_size) break; - /* unlock the pset and allocate more memory */ - pset_unlock(pset); - - if (size != 0) - kfree(addr, size); + /* unlock and allocate more memory */ + lck_mtx_unlock(&tasks_threads_lock); - assert(size_needed > 0); - size = size_needed; + /* grow task array */ + if (task_size_needed > task_size) { + if (task_size != 0) + kfree(task_list, task_size); - addr = kalloc(size); - if (addr == 0) - return (KERN_RESOURCE_SHORTAGE); - } + assert(task_size_needed > 0); + task_size = task_size_needed; - /* OK, have memory and the processor_set is locked & active */ + task_list = (task_t *)kalloc(task_size); + if (task_list == NULL) { + if (thread_size != 0) + kfree(thread_list, thread_size); + return (KERN_RESOURCE_SHORTAGE); + } + } - actual = 0; - switch (type) { + /* grow thread array */ + if (thread_size_needed > thread_size) { + if (thread_size != 0) + kfree(thread_list, thread_size); - case THING_TASK: - { - task_t task, *tasks = (task_t *)addr; + assert(thread_size_needed > 0); + thread_size = thread_size_needed; - for (task = (task_t)queue_first(&pset->tasks); - !queue_end(&pset->tasks, (queue_entry_t)task); - task = (task_t)queue_next(&task->pset_tasks)) { - task_reference_internal(task); - tasks[actual++] = task; + thread_list = (thread_t *)kalloc(thread_size); + if (thread_list == 0) { + if (task_size != 0) + kfree(task_list, task_size); + return (KERN_RESOURCE_SHORTAGE); + } } - - break; } - case THING_THREAD: - { - thread_t thread, *threads = (thread_t *)addr; - - for (i = 0, thread = (thread_t)queue_first(&pset->threads); - !queue_end(&pset->threads, (queue_entry_t)thread); - thread = (thread_t)queue_next(&thread->pset_threads)) { - thread_reference_internal(thread); - threads[actual++] = thread; + /* OK, have memory and the list locked */ + + /* If we need it, get the thread list */ + if (type == PSET_THING_THREAD) { + for (thread = (thread_t)queue_first(&threads); + !queue_end(&threads, (queue_entry_t)thread); + thread = (thread_t)queue_next(&thread->threads)) { +#if defined(SECURE_KERNEL) + if (thread->task != kernel_task) { +#endif + thread_reference_internal(thread); + thread_list[actual_threads++] = thread; +#if defined(SECURE_KERNEL) + } +#endif } - - break; } +#if !CONFIG_MACF + else { +#endif + /* get a list of the tasks */ + for (task = (task_t)queue_first(&tasks); + !queue_end(&tasks, (queue_entry_t)task); + task = (task_t)queue_next(&task->tasks)) { +#if defined(SECURE_KERNEL) + if (task != kernel_task) { +#endif + task_reference_internal(task); + task_list[actual_tasks++] = task; +#if defined(SECURE_KERNEL) + } +#endif + } +#if !CONFIG_MACF } - - pset_unlock(pset); +#endif - if (actual < maxthings) - size_needed = actual * sizeof (mach_port_t); + lck_mtx_unlock(&tasks_threads_lock); - if (actual == 0) { - /* no things, so return null pointer and deallocate memory */ - *thing_list = 0; - *count = 0; +#if CONFIG_MACF + unsigned int j, used; - if (size != 0) - kfree(addr, size); + /* for each task, make sure we are allowed to examine it */ + for (i = used = 0; i < actual_tasks; i++) { + if (mac_task_check_expose_task(task_list[i])) { + task_deallocate(task_list[i]); + continue; + } + task_list[used++] = task_list[i]; } - else { - /* if we allocated too much, must copy */ - - if (size_needed < size) { - void *newaddr; - - newaddr = kalloc(size_needed); - if (newaddr == 0) { - switch (type) { + actual_tasks = used; + task_size_needed = actual_tasks * sizeof(void *); - case THING_TASK: - { - task_t *tasks = (task_t *)addr; - - for (i = 0; i < actual; i++) - task_deallocate(tasks[i]); - break; - } + if (type == PSET_THING_THREAD) { - case THING_THREAD: - { - thread_t *threads = (thread_t *)addr; + /* for each thread (if any), make sure it's task is in the allowed list */ + for (i = used = 0; i < actual_threads; i++) { + boolean_t found_task = FALSE; - for (i = 0; i < actual; i++) - thread_deallocate(threads[i]); + task = thread_list[i]->task; + for (j = 0; j < actual_tasks; j++) { + if (task_list[j] == task) { + found_task = TRUE; break; } - } - - kfree(addr, size); - return (KERN_RESOURCE_SHORTAGE); } - - bcopy((void *) addr, (void *) newaddr, size_needed); - kfree(addr, size); - addr = newaddr; + if (found_task) + thread_list[used++] = thread_list[i]; + else + thread_deallocate(thread_list[i]); } + actual_threads = used; + thread_size_needed = actual_threads * sizeof(void *); + + /* done with the task list */ + for (i = 0; i < actual_tasks; i++) + task_deallocate(task_list[i]); + kfree(task_list, task_size); + task_size = 0; + actual_tasks = 0; + task_list = NULL; + } +#endif + + if (type == PSET_THING_THREAD) { + if (actual_threads == 0) { + /* no threads available to return */ + assert(task_size == 0); + if (thread_size != 0) + kfree(thread_list, thread_size); + *thing_list = NULL; + *count = 0; + return KERN_SUCCESS; + } + size_needed = actual_threads * sizeof(void *); + size = thread_size; + addr = thread_list; + } else { + if (actual_tasks == 0) { + /* no tasks available to return */ + assert(thread_size == 0); + if (task_size != 0) + kfree(task_list, task_size); + *thing_list = NULL; + *count = 0; + return KERN_SUCCESS; + } + size_needed = actual_tasks * sizeof(void *); + size = task_size; + addr = task_list; + } - *thing_list = (mach_port_t *)addr; - *count = actual; - - /* do the conversion that Mig should handle */ - - switch (type) { - - case THING_TASK: - { - task_t *tasks = (task_t *)addr; - - for (i = 0; i < actual; i++) - (*thing_list)[i] = convert_task_to_port(tasks[i]); - break; + /* if we allocated too much, must copy */ + if (size_needed < size) { + newaddr = kalloc(size_needed); + if (newaddr == 0) { + for (i = 0; i < actual_tasks; i++) { + if (type == PSET_THING_THREAD) + thread_deallocate(thread_list[i]); + else + task_deallocate(task_list[i]); + } + if (size) + kfree(addr, size); + return (KERN_RESOURCE_SHORTAGE); } - case THING_THREAD: - { - thread_t *threads = (thread_t *)addr; + bcopy((void *) addr, (void *) newaddr, size_needed); + kfree(addr, size); - for (i = 0; i < actual; i++) - (*thing_list)[i] = convert_thread_to_port(threads[i]); - break; - } - } + addr = newaddr; + size = size_needed; } + *thing_list = (void **)addr; + *count = (unsigned int)size / sizeof(void *); + return (KERN_SUCCESS); } @@ -1033,7 +1155,17 @@ processor_set_tasks( task_array_t *task_list, mach_msg_type_number_t *count) { - return(processor_set_things(pset, (mach_port_t **)task_list, count, THING_TASK)); + kern_return_t ret; + mach_msg_type_number_t i; + + ret = processor_set_things(pset, (void **)task_list, count, PSET_THING_TASK); + if (ret != KERN_SUCCESS) + return ret; + + /* do the conversion that Mig should handle */ + for (i = 0; i < *count; i++) + (*task_list)[i] = (task_t)convert_task_to_port((*task_list)[i]); + return KERN_SUCCESS; } /* @@ -1041,48 +1173,44 @@ processor_set_tasks( * * List all threads in the processor set. */ +#if defined(SECURE_KERNEL) kern_return_t processor_set_threads( - processor_set_t pset, - thread_array_t *thread_list, - mach_msg_type_number_t *count) + __unused processor_set_t pset, + __unused thread_array_t *thread_list, + __unused mach_msg_type_number_t *count) { - return(processor_set_things(pset, (mach_port_t **)thread_list, count, THING_THREAD)); + return KERN_FAILURE; } - -/* - * processor_set_base: - * - * Specify per-policy base priority for a processor set. Set processor - * set default policy to the given policy. This affects newly created - * and assigned threads. Optionally change existing ones. - */ +#elif defined(CONFIG_EMBEDDED) kern_return_t -processor_set_base( - __unused processor_set_t pset, - __unused policy_t policy, - __unused policy_base_t base, - __unused boolean_t change) +processor_set_threads( + __unused processor_set_t pset, + __unused thread_array_t *thread_list, + __unused mach_msg_type_number_t *count) { - return (KERN_INVALID_ARGUMENT); + return KERN_NOT_SUPPORTED; } - -/* - * processor_set_limit: - * - * Specify per-policy limits for a processor set. This affects - * newly created and assigned threads. Optionally change existing - * ones. - */ +#else kern_return_t -processor_set_limit( - __unused processor_set_t pset, - __unused policy_t policy, - __unused policy_limit_t limit, - __unused boolean_t change) +processor_set_threads( + processor_set_t pset, + thread_array_t *thread_list, + mach_msg_type_number_t *count) { - return (KERN_POLICY_LIMIT); + kern_return_t ret; + mach_msg_type_number_t i; + + ret = processor_set_things(pset, (void **)thread_list, count, PSET_THING_THREAD); + if (ret != KERN_SUCCESS) + return ret; + + /* do the conversion that Mig should handle */ + for (i = 0; i < *count; i++) + (*thread_list)[i] = (thread_t)convert_thread_to_port((*thread_list)[i]); + return KERN_SUCCESS; } +#endif /* * processor_set_policy_control @@ -1101,3 +1229,28 @@ processor_set_policy_control( { return (KERN_INVALID_ARGUMENT); } + +#undef pset_deallocate +void pset_deallocate(processor_set_t pset); +void +pset_deallocate( +__unused processor_set_t pset) +{ + return; +} + +#undef pset_reference +void pset_reference(processor_set_t pset); +void +pset_reference( +__unused processor_set_t pset) +{ + return; +} + +pset_cluster_type_t +recommended_pset_type(thread_t thread) +{ + (void)thread; + return PSET_SMP; +}