2 * Copyright (c) 2013 Apple Inc. All rights reserved.
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
29 #include <mach/mach_types.h>
30 #include <mach/machine.h>
32 #include <machine/machine_routines.h>
33 #include <machine/sched_param.h>
34 #include <machine/machine_cpu.h>
36 #include <kern/kern_types.h>
37 #include <kern/debug.h>
38 #include <kern/machine.h>
39 #include <kern/misc_protos.h>
40 #include <kern/processor.h>
41 #include <kern/queue.h>
42 #include <kern/sched.h>
43 #include <kern/sched_prim.h>
44 #include <kern/task.h>
45 #include <kern/thread.h>
47 #include <sys/kdebug.h>
50 sched_dualq_init(void);
53 sched_dualq_steal_thread(processor_set_t pset
);
56 sched_dualq_thread_update_scan(sched_update_scan_context_t scan_context
);
59 sched_dualq_processor_enqueue(processor_t processor
, thread_t thread
, integer_t options
);
62 sched_dualq_processor_queue_remove(processor_t processor
, thread_t thread
);
65 sched_dualq_processor_csw_check(processor_t processor
);
68 sched_dualq_processor_queue_has_priority(processor_t processor
, int priority
, boolean_t gte
);
71 sched_dualq_runq_count(processor_t processor
);
74 sched_dualq_processor_queue_empty(processor_t processor
);
77 sched_dualq_runq_stats_count_sum(processor_t processor
);
80 sched_dualq_processor_bound_count(processor_t processor
);
83 sched_dualq_pset_init(processor_set_t pset
);
86 sched_dualq_processor_init(processor_t processor
);
89 sched_dualq_choose_thread(processor_t processor
, int priority
, ast_t reason
);
92 sched_dualq_processor_queue_shutdown(processor_t processor
);
95 sched_dualq_initial_thread_sched_mode(task_t parent_task
);
97 const struct sched_dispatch_table sched_dualq_dispatch
= {
98 .sched_name
= "dualq",
99 .init
= sched_dualq_init
,
100 .timebase_init
= sched_timeshare_timebase_init
,
101 .processor_init
= sched_dualq_processor_init
,
102 .pset_init
= sched_dualq_pset_init
,
103 .maintenance_continuation
= sched_timeshare_maintenance_continue
,
104 .choose_thread
= sched_dualq_choose_thread
,
105 .steal_thread_enabled
= TRUE
,
106 .steal_thread
= sched_dualq_steal_thread
,
107 .compute_timeshare_priority
= sched_compute_timeshare_priority
,
108 .choose_processor
= choose_processor
,
109 .processor_enqueue
= sched_dualq_processor_enqueue
,
110 .processor_queue_shutdown
= sched_dualq_processor_queue_shutdown
,
111 .processor_queue_remove
= sched_dualq_processor_queue_remove
,
112 .processor_queue_empty
= sched_dualq_processor_queue_empty
,
113 .priority_is_urgent
= priority_is_urgent
,
114 .processor_csw_check
= sched_dualq_processor_csw_check
,
115 .processor_queue_has_priority
= sched_dualq_processor_queue_has_priority
,
116 .initial_quantum_size
= sched_timeshare_initial_quantum_size
,
117 .initial_thread_sched_mode
= sched_dualq_initial_thread_sched_mode
,
118 .can_update_priority
= can_update_priority
,
119 .update_priority
= update_priority
,
120 .lightweight_update_priority
= lightweight_update_priority
,
121 .quantum_expire
= sched_default_quantum_expire
,
122 .processor_runq_count
= sched_dualq_runq_count
,
123 .processor_runq_stats_count_sum
= sched_dualq_runq_stats_count_sum
,
124 .processor_bound_count
= sched_dualq_processor_bound_count
,
125 .thread_update_scan
= sched_dualq_thread_update_scan
,
126 .direct_dispatch_to_idle_processors
= FALSE
,
127 .multiple_psets_enabled
= TRUE
,
128 .sched_groups_enabled
= FALSE
,
129 .avoid_processor_enabled
= FALSE
,
130 .thread_avoid_processor
= NULL
,
131 .processor_balance
= sched_SMT_balance
,
133 .rt_runq
= sched_rtglobal_runq
,
134 .rt_init
= sched_rtglobal_init
,
135 .rt_queue_shutdown
= sched_rtglobal_queue_shutdown
,
136 .rt_runq_scan
= sched_rtglobal_runq_scan
,
137 .rt_runq_count_sum
= sched_rtglobal_runq_count_sum
,
139 .qos_max_parallelism
= sched_qos_max_parallelism
,
140 .check_spill
= sched_check_spill
,
141 .ipi_policy
= sched_ipi_policy
,
142 .thread_should_yield
= sched_thread_should_yield
,
145 __attribute__((always_inline
))
146 static inline run_queue_t
dualq_main_runq(processor_t processor
)
148 return &processor
->processor_set
->pset_runq
;
151 __attribute__((always_inline
))
152 static inline run_queue_t
dualq_bound_runq(processor_t processor
)
154 return &processor
->runq
;
157 __attribute__((always_inline
))
158 static inline run_queue_t
dualq_runq_for_thread(processor_t processor
, thread_t thread
)
160 if (thread
->bound_processor
== PROCESSOR_NULL
) {
161 return dualq_main_runq(processor
);
163 assert(thread
->bound_processor
== processor
);
164 return dualq_bound_runq(processor
);
169 sched_dualq_initial_thread_sched_mode(task_t parent_task
)
171 if (parent_task
== kernel_task
)
172 return TH_MODE_FIXED
;
174 return TH_MODE_TIMESHARE
;
178 sched_dualq_processor_init(processor_t processor
)
180 run_queue_init(&processor
->runq
);
184 sched_dualq_pset_init(processor_set_t pset
)
186 run_queue_init(&pset
->pset_runq
);
190 sched_dualq_init(void)
192 sched_timeshare_init();
196 sched_dualq_choose_thread(
197 processor_t processor
,
199 __unused ast_t reason
)
201 run_queue_t main_runq
= dualq_main_runq(processor
);
202 run_queue_t bound_runq
= dualq_bound_runq(processor
);
203 run_queue_t chosen_runq
;
205 if (bound_runq
->highq
< priority
&&
206 main_runq
->highq
< priority
)
209 if (bound_runq
->count
&& main_runq
->count
) {
210 if (bound_runq
->highq
>= main_runq
->highq
) {
211 chosen_runq
= bound_runq
;
213 chosen_runq
= main_runq
;
215 } else if (bound_runq
->count
) {
216 chosen_runq
= bound_runq
;
217 } else if (main_runq
->count
) {
218 chosen_runq
= main_runq
;
220 return (THREAD_NULL
);
223 return run_queue_dequeue(chosen_runq
, SCHED_HEADQ
);
227 sched_dualq_processor_enqueue(
228 processor_t processor
,
232 run_queue_t rq
= dualq_runq_for_thread(processor
, thread
);
235 result
= run_queue_enqueue(rq
, thread
, options
);
236 thread
->runq
= processor
;
242 sched_dualq_processor_queue_empty(processor_t processor
)
244 return dualq_main_runq(processor
)->count
== 0 &&
245 dualq_bound_runq(processor
)->count
== 0;
249 sched_dualq_processor_csw_check(processor_t processor
)
251 boolean_t has_higher
;
254 run_queue_t main_runq
= dualq_main_runq(processor
);
255 run_queue_t bound_runq
= dualq_bound_runq(processor
);
257 assert(processor
->active_thread
!= NULL
);
259 pri
= MAX(main_runq
->highq
, bound_runq
->highq
);
261 if (processor
->first_timeslice
) {
262 has_higher
= (pri
> processor
->current_pri
);
264 has_higher
= (pri
>= processor
->current_pri
);
268 if (main_runq
->urgency
> 0)
269 return (AST_PREEMPT
| AST_URGENT
);
271 if (bound_runq
->urgency
> 0)
272 return (AST_PREEMPT
| AST_URGENT
);
281 sched_dualq_processor_queue_has_priority(processor_t processor
,
285 run_queue_t main_runq
= dualq_main_runq(processor
);
286 run_queue_t bound_runq
= dualq_bound_runq(processor
);
288 int qpri
= MAX(main_runq
->highq
, bound_runq
->highq
);
291 return qpri
>= priority
;
293 return qpri
> priority
;
297 sched_dualq_runq_count(processor_t processor
)
299 return dualq_main_runq(processor
)->count
+ dualq_bound_runq(processor
)->count
;
303 sched_dualq_runq_stats_count_sum(processor_t processor
)
305 uint64_t bound_sum
= dualq_bound_runq(processor
)->runq_stats
.count_sum
;
307 if (processor
->cpu_id
== processor
->processor_set
->cpu_set_low
)
308 return bound_sum
+ dualq_main_runq(processor
)->runq_stats
.count_sum
;
313 sched_dualq_processor_bound_count(processor_t processor
)
315 return dualq_bound_runq(processor
)->count
;
319 sched_dualq_processor_queue_shutdown(processor_t processor
)
321 processor_set_t pset
= processor
->processor_set
;
322 run_queue_t rq
= dualq_main_runq(processor
);
326 /* We only need to migrate threads if this is the last active processor in the pset */
327 if (pset
->online_processor_count
> 0) {
334 while (rq
->count
> 0) {
335 thread
= run_queue_dequeue(rq
, SCHED_HEADQ
);
336 enqueue_tail(&tqueue
, &thread
->runq_links
);
341 qe_foreach_element_safe(thread
, &tqueue
, runq_links
) {
343 remqueue(&thread
->runq_links
);
347 thread_setrun(thread
, SCHED_TAILQ
);
349 thread_unlock(thread
);
354 sched_dualq_processor_queue_remove(
355 processor_t processor
,
359 processor_set_t pset
= processor
->processor_set
;
363 rq
= dualq_runq_for_thread(processor
, thread
);
365 if (processor
== thread
->runq
) {
367 * Thread is on a run queue and we have a lock on
370 run_queue_remove(rq
, thread
);
374 * The thread left the run queue before we could
375 * lock the run queue.
377 assert(thread
->runq
== PROCESSOR_NULL
);
378 processor
= PROCESSOR_NULL
;
383 return (processor
!= PROCESSOR_NULL
);
387 sched_dualq_steal_thread(processor_set_t pset
)
389 processor_set_t nset
, cset
= pset
;
393 if (cset
->pset_runq
.count
> 0) {
394 thread
= run_queue_dequeue(&cset
->pset_runq
, SCHED_HEADQ
);
399 nset
= next_pset(cset
);
407 } while (nset
!= pset
);
411 return (THREAD_NULL
);
415 sched_dualq_thread_update_scan(sched_update_scan_context_t scan_context
)
417 boolean_t restart_needed
= FALSE
;
418 processor_t processor
= processor_list
;
419 processor_set_t pset
;
424 * We update the threads associated with each processor (bound and idle threads)
425 * and then update the threads in each pset runqueue.
430 pset
= processor
->processor_set
;
435 restart_needed
= runq_scan(dualq_bound_runq(processor
), scan_context
);
443 thread
= processor
->idle_thread
;
444 if (thread
!= THREAD_NULL
&& thread
->sched_stamp
!= sched_tick
) {
445 if (thread_update_add_thread(thread
) == FALSE
) {
446 restart_needed
= TRUE
;
450 } while ((processor
= processor
->processor_list
) != NULL
);
452 /* Ok, we now have a collection of candidates -- fix them. */
453 thread_update_process_threads();
455 } while (restart_needed
);
464 restart_needed
= runq_scan(&pset
->pset_runq
, scan_context
);
471 } while ((pset
= pset
->pset_list
) != NULL
);
473 /* Ok, we now have a collection of candidates -- fix them. */
474 thread_update_process_threads();
476 } while (restart_needed
);