X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/6d2010ae8f7a6078e10b361c6962983bab233e0f..f427ee49d309d8fc33ebf3042c3a775f2f530ded:/osfmk/kern/sched_proto.c diff --git a/osfmk/kern/sched_proto.c b/osfmk/kern/sched_proto.c index e31cb0590..9693756fd 100644 --- a/osfmk/kern/sched_proto.c +++ b/osfmk/kern/sched_proto.c @@ -2,7 +2,7 @@ * Copyright (c) 2009 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * + * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in @@ -11,10 +11,10 @@ * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. - * + * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. - * + * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, @@ -22,7 +22,7 @@ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. - * + * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ @@ -42,7 +42,6 @@ #include #include #include -#include #include #include #include @@ -53,7 +52,6 @@ #include #include #include -#include #include #include @@ -79,44 +77,44 @@ static void sched_proto_maintenance_continuation(void); static thread_t -sched_proto_choose_thread(processor_t processor, - int priority); +sched_proto_choose_thread(processor_t processor, + int priority, + ast_t reason); static thread_t -sched_proto_steal_thread(processor_set_t pset); +sched_proto_steal_thread(processor_set_t pset); -static void -sched_proto_compute_priority(thread_t thread, - boolean_t override_depress); +static int +sched_proto_compute_priority(thread_t thread); static processor_t -sched_proto_choose_processor( processor_set_t pset, - processor_t processor, - thread_t thread); +sched_proto_choose_processor( processor_set_t pset, + processor_t processor, + thread_t thread); static boolean_t sched_proto_processor_enqueue( - processor_t processor, - thread_t thread, - integer_t options); + processor_t processor, + thread_t thread, + sched_options_t options); static void sched_proto_processor_queue_shutdown( - processor_t processor); + processor_t processor); static boolean_t sched_proto_processor_queue_remove( - processor_t processor, - thread_t thread); + processor_t processor, + thread_t thread); static boolean_t -sched_proto_processor_queue_empty(processor_t processor); +sched_proto_processor_queue_empty(processor_t processor); static boolean_t -sched_proto_processor_queue_has_priority(processor_t processor, - int priority, - boolean_t gte); +sched_proto_processor_queue_has_priority(processor_t processor, + int priority, + boolean_t gte); static boolean_t sched_proto_priority_is_urgent(int priority); @@ -131,22 +129,16 @@ static sched_mode_t sched_proto_initial_thread_sched_mode(task_t parent_task); static boolean_t -sched_proto_supports_timeshare_mode(void); - -static boolean_t -sched_proto_can_update_priority(thread_t thread); +sched_proto_can_update_priority(thread_t thread); static void -sched_proto_update_priority(thread_t thread); +sched_proto_update_priority(thread_t thread); static void -sched_proto_lightweight_update_priority(thread_t thread); +sched_proto_lightweight_update_priority(thread_t thread); static void -sched_proto_quantum_expire(thread_t thread); - -static boolean_t -sched_proto_should_current_thread_rechoose_processor(processor_t processor); +sched_proto_quantum_expire(thread_t thread); static int sched_proto_processor_runq_count(processor_t processor); @@ -154,93 +146,115 @@ sched_proto_processor_runq_count(processor_t processor); static uint64_t sched_proto_processor_runq_stats_count_sum(processor_t processor); +static int +sched_proto_processor_bound_count(processor_t processor); + +static void +sched_proto_thread_update_scan(sched_update_scan_context_t scan_context); + + const struct sched_dispatch_table sched_proto_dispatch = { - sched_proto_init, - sched_proto_timebase_init, - sched_proto_processor_init, - sched_proto_pset_init, - sched_proto_maintenance_continuation, - sched_proto_choose_thread, - sched_proto_steal_thread, - sched_proto_compute_priority, - sched_proto_choose_processor, - sched_proto_processor_enqueue, - sched_proto_processor_queue_shutdown, - sched_proto_processor_queue_remove, - sched_proto_processor_queue_empty, - sched_proto_priority_is_urgent, - sched_proto_processor_csw_check, - sched_proto_processor_queue_has_priority, - sched_proto_initial_quantum_size, - sched_proto_initial_thread_sched_mode, - sched_proto_supports_timeshare_mode, - sched_proto_can_update_priority, - sched_proto_update_priority, - sched_proto_lightweight_update_priority, - sched_proto_quantum_expire, - sched_proto_should_current_thread_rechoose_processor, - sched_proto_processor_runq_count, - sched_proto_processor_runq_stats_count_sum, - sched_traditional_fairshare_init, - sched_traditional_fairshare_runq_count, - sched_traditional_fairshare_runq_stats_count_sum, - sched_traditional_fairshare_enqueue, - sched_traditional_fairshare_dequeue, - sched_traditional_fairshare_queue_remove, - TRUE /* direct_dispatch_to_idle_processors */ + .sched_name = "proto", + .init = sched_proto_init, + .timebase_init = sched_proto_timebase_init, + .processor_init = sched_proto_processor_init, + .pset_init = sched_proto_pset_init, + .maintenance_continuation = sched_proto_maintenance_continuation, + .choose_thread = sched_proto_choose_thread, + .steal_thread_enabled = sched_steal_thread_DISABLED, + .steal_thread = sched_proto_steal_thread, + .compute_timeshare_priority = sched_proto_compute_priority, + .choose_node = sched_choose_node, + .choose_processor = sched_proto_choose_processor, + .processor_enqueue = sched_proto_processor_enqueue, + .processor_queue_shutdown = sched_proto_processor_queue_shutdown, + .processor_queue_remove = sched_proto_processor_queue_remove, + .processor_queue_empty = sched_proto_processor_queue_empty, + .priority_is_urgent = sched_proto_priority_is_urgent, + .processor_csw_check = sched_proto_processor_csw_check, + .processor_queue_has_priority = sched_proto_processor_queue_has_priority, + .initial_quantum_size = sched_proto_initial_quantum_size, + .initial_thread_sched_mode = sched_proto_initial_thread_sched_mode, + .can_update_priority = sched_proto_can_update_priority, + .update_priority = sched_proto_update_priority, + .lightweight_update_priority = sched_proto_lightweight_update_priority, + .quantum_expire = sched_proto_quantum_expire, + .processor_runq_count = sched_proto_processor_runq_count, + .processor_runq_stats_count_sum = sched_proto_processor_runq_stats_count_sum, + .processor_bound_count = sched_proto_processor_bound_count, + .thread_update_scan = sched_proto_thread_update_scan, + .multiple_psets_enabled = TRUE, + .sched_groups_enabled = FALSE, + .avoid_processor_enabled = FALSE, + .thread_avoid_processor = NULL, + .processor_balance = sched_SMT_balance, + + .rt_runq = sched_rtlocal_runq, + .rt_init = sched_rtlocal_init, + .rt_queue_shutdown = sched_rtlocal_queue_shutdown, + .rt_runq_scan = sched_rtlocal_runq_scan, + .rt_runq_count_sum = sched_rtlocal_runq_count_sum, + + .qos_max_parallelism = sched_qos_max_parallelism, + .check_spill = sched_check_spill, + .ipi_policy = sched_ipi_policy, + .thread_should_yield = sched_thread_should_yield, + .run_count_incr = sched_run_incr, + .run_count_decr = sched_run_decr, + .update_thread_bucket = sched_update_thread_bucket, + .pset_made_schedulable = sched_pset_made_schedulable, }; -static struct run_queue *global_runq; -static struct run_queue global_runq_storage; +static struct run_queue *global_runq; +static struct run_queue global_runq_storage; -#define GLOBAL_RUNQ ((processor_t)-2) -decl_simple_lock_data(static,global_runq_lock); +#define GLOBAL_RUNQ ((processor_t)-2) +decl_simple_lock_data(static, global_runq_lock); -extern int max_unsafe_quanta; +extern int max_unsafe_quanta; static uint32_t proto_quantum_us; static uint32_t proto_quantum; -static uint32_t runqueue_generation; +static uint32_t runqueue_generation; static processor_t proto_processor; -static uint64_t sched_proto_tick_deadline; -static uint32_t sched_proto_tick; +static uint64_t sched_proto_tick_deadline; +static uint32_t sched_proto_tick; static void sched_proto_init(void) { - proto_quantum_us = 10*1000; - + proto_quantum_us = 10 * 1000; + printf("standard proto timeslicing quantum is %d us\n", proto_quantum_us); simple_lock_init(&global_runq_lock, 0); global_runq = &global_runq_storage; run_queue_init(global_runq); runqueue_generation = 0; - + proto_processor = master_processor; } static void sched_proto_timebase_init(void) { - uint64_t abstime; + uint64_t abstime; /* standard timeslicing quantum */ clock_interval_to_absolutetime_interval( - proto_quantum_us, NSEC_PER_USEC, &abstime); + proto_quantum_us, NSEC_PER_USEC, &abstime); assert((abstime >> 32) == 0 && (uint32_t)abstime != 0); proto_quantum = (uint32_t)abstime; - + thread_depress_time = 1 * proto_quantum; default_timeshare_computation = proto_quantum / 2; default_timeshare_constraint = proto_quantum; - + max_unsafe_computation = max_unsafe_quanta * proto_quantum; sched_safe_duration = 2 * max_unsafe_quanta * proto_quantum; - } static void @@ -257,54 +271,57 @@ sched_proto_pset_init(processor_set_t pset __unused) static void sched_proto_maintenance_continuation(void) { - uint64_t abstime = mach_absolute_time(); - + uint64_t abstime = mach_absolute_time(); + sched_proto_tick++; - + /* Every 8 seconds, switch to another processor */ if ((sched_proto_tick & 0x7) == 0) { processor_t new_processor; - + new_processor = proto_processor->processor_list; - if (new_processor == PROCESSOR_NULL) + if (new_processor == PROCESSOR_NULL) { proto_processor = master_processor; - else + } else { proto_processor = new_processor; + } } - - + + /* * Compute various averages. */ - compute_averages(); - - if (sched_proto_tick_deadline == 0) + compute_averages(1); + + if (sched_proto_tick_deadline == 0) { sched_proto_tick_deadline = abstime; - + } + clock_deadline_for_periodic_event(sched_one_second_interval, abstime, - &sched_proto_tick_deadline); - + &sched_proto_tick_deadline); + assert_wait_deadline((event_t)sched_proto_maintenance_continuation, THREAD_UNINT, sched_proto_tick_deadline); thread_block((thread_continue_t)sched_proto_maintenance_continuation); /*NOTREACHED*/ } static thread_t -sched_proto_choose_thread(processor_t processor, - int priority) +sched_proto_choose_thread(processor_t processor, + int priority, + ast_t reason __unused) { - run_queue_t rq = global_runq; - queue_t queue; - int pri, count; - thread_t thread; - - - simple_lock(&global_runq_lock); - + run_queue_t rq = global_runq; + circle_queue_t queue; + int pri, count; + thread_t thread; + + + simple_lock(&global_runq_lock, LCK_GRP_NULL); + queue = rq->queues + rq->highq; pri = rq->highq; count = rq->count; - + /* * Since we don't depress priorities, a high priority thread * may get selected over and over again. Put a runqueue @@ -314,99 +331,95 @@ sched_proto_choose_thread(processor_t processor, * perfect, especially if the number of runnable threads always * stays high, but is a workable approximation */ - + while (count > 0 && pri >= priority) { - thread = (thread_t)queue_first(queue); - while (!queue_end(queue, (queue_entry_t)thread)) { + cqe_foreach_element_safe(thread, queue, runq_links) { if ((thread->bound_processor == PROCESSOR_NULL || - thread->bound_processor == processor) && - runqueue_generation != thread->runqueue_generation) { - remqueue((queue_entry_t)thread); - + thread->bound_processor == processor) && + runqueue_generation != thread->runqueue_generation) { + circle_dequeue(queue, &thread->runq_links); + thread->runq = PROCESSOR_NULL; thread->runqueue_generation = runqueue_generation; SCHED_STATS_RUNQ_CHANGE(&rq->runq_stats, rq->count); rq->count--; - if (queue_empty(queue)) { - if (pri != IDLEPRI) - clrbit(MAXPRI - pri, rq->bitmap); - rq->highq = MAXPRI - ffsbit(rq->bitmap); + if (circle_queue_empty(queue)) { + bitmap_clear(rq->bitmap, pri); + rq->highq = bitmap_first(rq->bitmap, NRQS); } - + simple_unlock(&global_runq_lock); - return (thread); + return thread; } count--; - + thread = (thread_t)queue_next((queue_entry_t)thread); } - + queue--; pri--; } - + runqueue_generation++; - + simple_unlock(&global_runq_lock); - return (THREAD_NULL); + return THREAD_NULL; } static thread_t -sched_proto_steal_thread(processor_set_t pset) +sched_proto_steal_thread(processor_set_t pset) { pset_unlock(pset); - - return (THREAD_NULL); - + + return THREAD_NULL; } -static void -sched_proto_compute_priority(thread_t thread, - boolean_t override_depress __unused) +static int +sched_proto_compute_priority(thread_t thread) { - set_sched_pri(thread, thread->priority); + return thread->base_pri; } static processor_t -sched_proto_choose_processor( processor_set_t pset, - processor_t processor, - thread_t thread __unused) +sched_proto_choose_processor( processor_set_t pset, + processor_t processor, + thread_t thread __unused) { processor = proto_processor; - + /* * Check that the correct processor set is * returned locked. */ if (pset != processor->processor_set) { pset_unlock(pset); - + pset = processor->processor_set; pset_lock(pset); } - - return (processor); + + return processor; } static boolean_t sched_proto_processor_enqueue( - processor_t processor __unused, - thread_t thread, - integer_t options) + processor_t processor __unused, + thread_t thread, + sched_options_t options) { - run_queue_t rq = global_runq; - boolean_t result; - - simple_lock(&global_runq_lock); + run_queue_t rq = global_runq; + boolean_t result; + + simple_lock(&global_runq_lock, LCK_GRP_NULL); result = run_queue_enqueue(rq, thread, options); thread->runq = GLOBAL_RUNQ; simple_unlock(&global_runq_lock); - - return (result); + + return result; } static void sched_proto_processor_queue_shutdown( - processor_t processor) + processor_t processor) { /* With a global runqueue, just stop choosing this processor */ (void)processor; @@ -414,77 +427,63 @@ sched_proto_processor_queue_shutdown( static boolean_t sched_proto_processor_queue_remove( - processor_t processor, - thread_t thread) + processor_t processor, + thread_t thread) { - void * rqlock; - run_queue_t rq; - + void * rqlock; + run_queue_t rq; + rqlock = &global_runq_lock; rq = global_runq; - - simple_lock(rqlock); + + simple_lock(rqlock, LCK_GRP_NULL); if (processor == thread->runq) { /* * Thread is on a run queue and we have a lock on * that run queue. */ - remqueue((queue_entry_t)thread); - SCHED_STATS_RUNQ_CHANGE(&rq->runq_stats, rq->count); - rq->count--; - if (SCHED(priority_is_urgent)(thread->sched_pri)) { - rq->urgency--; assert(rq->urgency >= 0); - } - - if (queue_empty(rq->queues + thread->sched_pri)) { - /* update run queue status */ - if (thread->sched_pri != IDLEPRI) - clrbit(MAXPRI - thread->sched_pri, rq->bitmap); - rq->highq = MAXPRI - ffsbit(rq->bitmap); - } - - thread->runq = PROCESSOR_NULL; - } - else { + run_queue_remove(rq, thread); + } else { /* * The thread left the run queue before we could - * lock the run queue. + * lock the run queue. */ assert(thread->runq == PROCESSOR_NULL); processor = PROCESSOR_NULL; } - + simple_unlock(rqlock); - - return (processor != PROCESSOR_NULL); + + return processor != PROCESSOR_NULL; } static boolean_t -sched_proto_processor_queue_empty(processor_t processor __unused) +sched_proto_processor_queue_empty(processor_t processor __unused) { boolean_t result; - + result = (global_runq->count == 0); - + return result; } static boolean_t -sched_proto_processor_queue_has_priority(processor_t processor __unused, - int priority, - boolean_t gte) +sched_proto_processor_queue_has_priority(processor_t processor __unused, + int priority, + boolean_t gte) { boolean_t result; - - simple_lock(&global_runq_lock); - if (gte) - result = global_runq->highq >= priority; - else + simple_lock(&global_runq_lock, LCK_GRP_NULL); + + if (gte) { result = global_runq->highq >= priority; + } else { + result = global_runq->highq > priority; + } simple_unlock(&global_runq_lock); - + return result; } @@ -492,35 +491,43 @@ sched_proto_processor_queue_has_priority(processor_t processor __unused, static boolean_t sched_proto_priority_is_urgent(int priority) { - if (priority <= BASEPRI_FOREGROUND) + if (priority <= BASEPRI_FOREGROUND) { return FALSE; - - if (priority < MINPRI_KERNEL) + } + + if (priority < MINPRI_KERNEL) { return TRUE; + } - if (priority >= BASEPRI_PREEMPT) + if (priority >= BASEPRI_PREEMPT) { return TRUE; - + } + return FALSE; } static ast_t -sched_proto_processor_csw_check(processor_t processor __unused) +sched_proto_processor_csw_check(processor_t processor) { - run_queue_t runq; - int count, urgency; - + run_queue_t runq; + int count, urgency; + runq = global_runq; count = runq->count; urgency = runq->urgency; - + if (count > 0) { - if (urgency > 0) - return (AST_PREEMPT | AST_URGENT); - + if (urgency > 0) { + return AST_PREEMPT | AST_URGENT; + } + + return AST_PREEMPT; + } + + if (proto_processor != processor) { return AST_PREEMPT; } - + return AST_NONE; } @@ -533,46 +540,32 @@ sched_proto_initial_quantum_size(thread_t thread __unused) static sched_mode_t sched_proto_initial_thread_sched_mode(task_t parent_task) { - if (parent_task == kernel_task) + if (parent_task == kernel_task) { return TH_MODE_FIXED; - else - return TH_MODE_TIMESHARE; -} - -static boolean_t -sched_proto_supports_timeshare_mode(void) -{ - return TRUE; + } else { + return TH_MODE_TIMESHARE; + } } static boolean_t -sched_proto_can_update_priority(thread_t thread __unused) +sched_proto_can_update_priority(thread_t thread __unused) { return FALSE; } static void -sched_proto_update_priority(thread_t thread __unused) +sched_proto_update_priority(thread_t thread __unused) { - } static void -sched_proto_lightweight_update_priority(thread_t thread __unused) +sched_proto_lightweight_update_priority(thread_t thread __unused) { - } static void -sched_proto_quantum_expire(thread_t thread __unused) +sched_proto_quantum_expire(thread_t thread __unused) { - -} - -static boolean_t -sched_proto_should_current_thread_rechoose_processor(processor_t processor) -{ - return (proto_processor != processor); } static int @@ -582,7 +575,7 @@ sched_proto_processor_runq_count(processor_t processor) return global_runq->count; } else { return 0; - } + } } static uint64_t @@ -595,3 +588,13 @@ sched_proto_processor_runq_stats_count_sum(processor_t processor) } } +static int +sched_proto_processor_bound_count(__unused processor_t processor) +{ + return 0; +} + +static void +sched_proto_thread_update_scan(__unused sched_update_scan_context_t scan_context) +{ +}