]>
Commit | Line | Data |
---|---|---|
1 | /* | |
2 | * Copyright (c) 2000-2012 Apple Inc. All rights reserved. | |
3 | * | |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ | |
5 | * | |
6 | * This file contains Original Code and/or Modifications of Original Code | |
7 | * as defined in and that are subject to the Apple Public Source License | |
8 | * Version 2.0 (the 'License'). You may not use this file except in | |
9 | * compliance with the License. The rights granted to you under the License | |
10 | * may not be used to create, or enable the creation or redistribution of, | |
11 | * unlawful or unlicensed copies of an Apple operating system, or to | |
12 | * circumvent, violate, or enable the circumvention or violation of, any | |
13 | * terms of an Apple operating system software license agreement. | |
14 | * | |
15 | * Please obtain a copy of the License at | |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. | |
17 | * | |
18 | * The Original Code and all software distributed under the License are | |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER | |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, | |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, | |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. | |
23 | * Please see the License for the specific language governing rights and | |
24 | * limitations under the License. | |
25 | * | |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ | |
27 | */ | |
28 | /* | |
29 | * @OSF_COPYRIGHT@ | |
30 | */ | |
31 | /* | |
32 | * Mach Operating System | |
33 | * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University | |
34 | * All Rights Reserved. | |
35 | * | |
36 | * Permission to use, copy, modify and distribute this software and its | |
37 | * documentation is hereby granted, provided that both the copyright | |
38 | * notice and this permission notice appear in all copies of the | |
39 | * software, derivative works or modified versions, and any portions | |
40 | * thereof, and that both notices appear in supporting documentation. | |
41 | * | |
42 | * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" | |
43 | * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR | |
44 | * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. | |
45 | * | |
46 | * Carnegie Mellon requests users of this software to return to | |
47 | * | |
48 | * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU | |
49 | * School of Computer Science | |
50 | * Carnegie Mellon University | |
51 | * Pittsburgh PA 15213-3890 | |
52 | * | |
53 | * any improvements or extensions that they make and grant Carnegie Mellon | |
54 | * the rights to redistribute these changes. | |
55 | */ | |
56 | /* | |
57 | */ | |
58 | /* | |
59 | * File: sched_prim.h | |
60 | * Author: David Golub | |
61 | * | |
62 | * Scheduling primitive definitions file | |
63 | * | |
64 | */ | |
65 | ||
66 | #ifndef _KERN_SCHED_PRIM_H_ | |
67 | #define _KERN_SCHED_PRIM_H_ | |
68 | ||
69 | #include <mach/boolean.h> | |
70 | #include <mach/machine/vm_types.h> | |
71 | #include <mach/kern_return.h> | |
72 | #include <kern/clock.h> | |
73 | #include <kern/kern_types.h> | |
74 | #include <kern/thread.h> | |
75 | #include <sys/cdefs.h> | |
76 | #include <kern/block_hint.h> | |
77 | ||
78 | #ifdef MACH_KERNEL_PRIVATE | |
79 | ||
80 | /* Initialization */ | |
81 | extern void sched_init(void); | |
82 | ||
83 | extern void sched_startup(void); | |
84 | ||
85 | extern void sched_timebase_init(void); | |
86 | ||
87 | extern void pset_rt_init(processor_set_t pset); | |
88 | ||
89 | extern void sched_rtglobal_init(processor_set_t pset); | |
90 | ||
91 | extern rt_queue_t sched_rtglobal_runq(processor_set_t pset); | |
92 | ||
93 | extern void sched_rtglobal_queue_shutdown(processor_t processor); | |
94 | ||
95 | extern int64_t sched_rtglobal_runq_count_sum(void); | |
96 | ||
97 | extern void sched_check_spill(processor_set_t pset, thread_t thread); | |
98 | ||
99 | extern bool sched_thread_should_yield(processor_t processor, thread_t thread); | |
100 | ||
101 | /* Force a preemption point for a thread and wait for it to stop running */ | |
102 | extern boolean_t thread_stop( | |
103 | thread_t thread, | |
104 | boolean_t until_not_runnable); | |
105 | ||
106 | /* Release a previous stop request */ | |
107 | extern void thread_unstop( | |
108 | thread_t thread); | |
109 | ||
110 | /* Wait for a thread to stop running */ | |
111 | extern void thread_wait( | |
112 | thread_t thread, | |
113 | boolean_t until_not_runnable); | |
114 | ||
115 | /* Unblock thread on wake up */ | |
116 | extern boolean_t thread_unblock( | |
117 | thread_t thread, | |
118 | wait_result_t wresult); | |
119 | ||
120 | /* Unblock and dispatch thread */ | |
121 | extern kern_return_t thread_go( | |
122 | thread_t thread, | |
123 | wait_result_t wresult); | |
124 | ||
125 | /* Handle threads at context switch */ | |
126 | extern void thread_dispatch( | |
127 | thread_t old_thread, | |
128 | thread_t new_thread); | |
129 | ||
130 | /* Switch directly to a particular thread */ | |
131 | extern int thread_run( | |
132 | thread_t self, | |
133 | thread_continue_t continuation, | |
134 | void *parameter, | |
135 | thread_t new_thread); | |
136 | ||
137 | /* Resume thread with new stack */ | |
138 | extern void thread_continue( | |
139 | thread_t old_thread); | |
140 | ||
141 | /* Invoke continuation */ | |
142 | extern void call_continuation( | |
143 | thread_continue_t continuation, | |
144 | void *parameter, | |
145 | wait_result_t wresult, | |
146 | boolean_t enable_interrupts); | |
147 | ||
148 | /* | |
149 | * Flags that can be passed to set_sched_pri | |
150 | * to skip side effects | |
151 | */ | |
152 | typedef enum { | |
153 | SETPRI_DEFAULT = 0x0, | |
154 | SETPRI_LAZY = 0x1, /* Avoid setting AST flags or sending IPIs */ | |
155 | } set_sched_pri_options_t; | |
156 | ||
157 | /* Set the current scheduled priority */ | |
158 | extern void set_sched_pri( | |
159 | thread_t thread, | |
160 | int priority, | |
161 | set_sched_pri_options_t options); | |
162 | ||
163 | /* Set base priority of the specified thread */ | |
164 | extern void sched_set_thread_base_priority( | |
165 | thread_t thread, | |
166 | int priority); | |
167 | ||
168 | /* Set the thread's true scheduling mode */ | |
169 | extern void sched_set_thread_mode(thread_t thread, | |
170 | sched_mode_t mode); | |
171 | /* Demote the true scheduler mode */ | |
172 | extern void sched_thread_mode_demote(thread_t thread, | |
173 | uint32_t reason); | |
174 | /* Un-demote the true scheduler mode */ | |
175 | extern void sched_thread_mode_undemote(thread_t thread, | |
176 | uint32_t reason); | |
177 | ||
178 | extern void sched_thread_promote_to_pri(thread_t thread, int priority, uintptr_t trace_obj); | |
179 | extern void sched_thread_update_promotion_to_pri(thread_t thread, int priority, uintptr_t trace_obj); | |
180 | extern void sched_thread_unpromote(thread_t thread, uintptr_t trace_obj); | |
181 | ||
182 | extern void assert_promotions_invariant(thread_t thread); | |
183 | ||
184 | extern void sched_thread_promote_reason(thread_t thread, uint32_t reason, uintptr_t trace_obj); | |
185 | extern void sched_thread_unpromote_reason(thread_t thread, uint32_t reason, uintptr_t trace_obj); | |
186 | ||
187 | /* Re-evaluate base priority of thread (thread locked) */ | |
188 | void thread_recompute_priority(thread_t thread); | |
189 | ||
190 | /* Re-evaluate scheduled priority of thread (thread locked) */ | |
191 | extern void thread_recompute_sched_pri( | |
192 | thread_t thread, | |
193 | set_sched_pri_options_t options); | |
194 | ||
195 | /* Periodic scheduler activity */ | |
196 | extern void sched_init_thread(void (*)(void)); | |
197 | ||
198 | /* Perform sched_tick housekeeping activities */ | |
199 | extern boolean_t can_update_priority( | |
200 | thread_t thread); | |
201 | ||
202 | extern void update_priority( | |
203 | thread_t thread); | |
204 | ||
205 | extern void lightweight_update_priority( | |
206 | thread_t thread); | |
207 | ||
208 | extern void sched_default_quantum_expire(thread_t thread); | |
209 | ||
210 | /* Idle processor thread */ | |
211 | extern void idle_thread(void); | |
212 | ||
213 | extern kern_return_t idle_thread_create( | |
214 | processor_t processor); | |
215 | ||
216 | /* Continuation return from syscall */ | |
217 | extern void thread_syscall_return( | |
218 | kern_return_t ret); | |
219 | ||
220 | /* Context switch */ | |
221 | extern wait_result_t thread_block_reason( | |
222 | thread_continue_t continuation, | |
223 | void *parameter, | |
224 | ast_t reason); | |
225 | ||
226 | /* Reschedule thread for execution */ | |
227 | extern void thread_setrun( | |
228 | thread_t thread, | |
229 | integer_t options); | |
230 | ||
231 | typedef enum { | |
232 | SCHED_NONE = 0x0, | |
233 | SCHED_TAILQ = 0x1, | |
234 | SCHED_HEADQ = 0x2, | |
235 | SCHED_PREEMPT = 0x4, | |
236 | SCHED_REBALANCE = 0x8, | |
237 | } sched_options_t; | |
238 | ||
239 | extern processor_set_t task_choose_pset( | |
240 | task_t task); | |
241 | ||
242 | /* Bind the current thread to a particular processor */ | |
243 | extern processor_t thread_bind( | |
244 | processor_t processor); | |
245 | ||
246 | /* Choose the best processor to run a thread */ | |
247 | extern processor_t choose_processor( | |
248 | processor_set_t pset, | |
249 | processor_t processor, | |
250 | thread_t thread); | |
251 | ||
252 | extern void sched_SMT_balance( | |
253 | processor_t processor, | |
254 | processor_set_t pset); | |
255 | ||
256 | extern void thread_quantum_init( | |
257 | thread_t thread); | |
258 | ||
259 | extern void run_queue_init( | |
260 | run_queue_t runq); | |
261 | ||
262 | extern thread_t run_queue_dequeue( | |
263 | run_queue_t runq, | |
264 | integer_t options); | |
265 | ||
266 | extern boolean_t run_queue_enqueue( | |
267 | run_queue_t runq, | |
268 | thread_t thread, | |
269 | integer_t options); | |
270 | ||
271 | extern void run_queue_remove( | |
272 | run_queue_t runq, | |
273 | thread_t thread); | |
274 | ||
275 | struct sched_update_scan_context | |
276 | { | |
277 | uint64_t earliest_bg_make_runnable_time; | |
278 | uint64_t earliest_normal_make_runnable_time; | |
279 | uint64_t earliest_rt_make_runnable_time; | |
280 | }; | |
281 | typedef struct sched_update_scan_context *sched_update_scan_context_t; | |
282 | ||
283 | extern void sched_rtglobal_runq_scan(sched_update_scan_context_t scan_context); | |
284 | ||
285 | /* | |
286 | * Enum to define various events which need IPIs. The IPI policy | |
287 | * engine decides what kind of IPI to use based on destination | |
288 | * processor state, thread and one of the following scheduling events. | |
289 | */ | |
290 | typedef enum { | |
291 | SCHED_IPI_EVENT_BOUND_THR = 0x1, | |
292 | SCHED_IPI_EVENT_PREEMPT = 0x2, | |
293 | SCHED_IPI_EVENT_SMT_REBAL = 0x3, | |
294 | SCHED_IPI_EVENT_SPILL = 0x4, | |
295 | SCHED_IPI_EVENT_REBALANCE = 0x5, | |
296 | } sched_ipi_event_t; | |
297 | ||
298 | ||
299 | /* Enum to define various IPI types used by the scheduler */ | |
300 | typedef enum { | |
301 | SCHED_IPI_NONE = 0x0, | |
302 | SCHED_IPI_IMMEDIATE = 0x1, | |
303 | SCHED_IPI_IDLE = 0x2, | |
304 | SCHED_IPI_DEFERRED = 0x3, | |
305 | } sched_ipi_type_t; | |
306 | ||
307 | /* The IPI policy engine behaves in the following manner: | |
308 | * - All scheduler events which need an IPI invoke sched_ipi_action() with | |
309 | * the appropriate destination processor, thread and event. | |
310 | * - sched_ipi_action() performs basic checks, invokes the scheduler specific | |
311 | * ipi_policy routine and sets pending_AST bits based on the result. | |
312 | * - Once the pset lock is dropped, the scheduler invokes sched_ipi_perform() | |
313 | * routine which actually sends the appropriate IPI to the destination core. | |
314 | */ | |
315 | extern sched_ipi_type_t sched_ipi_action(processor_t dst, thread_t thread, | |
316 | boolean_t dst_idle, sched_ipi_event_t event); | |
317 | extern void sched_ipi_perform(processor_t dst, sched_ipi_type_t ipi); | |
318 | ||
319 | /* sched_ipi_policy() is the global default IPI policy for all schedulers */ | |
320 | extern sched_ipi_type_t sched_ipi_policy(processor_t dst, thread_t thread, | |
321 | boolean_t dst_idle, sched_ipi_event_t event); | |
322 | ||
323 | /* sched_ipi_deferred_policy() is the global default deferred IPI policy for all schedulers */ | |
324 | extern sched_ipi_type_t sched_ipi_deferred_policy(processor_set_t pset, | |
325 | processor_t dst, sched_ipi_event_t event); | |
326 | ||
327 | #if defined(CONFIG_SCHED_TIMESHARE_CORE) | |
328 | ||
329 | extern boolean_t thread_update_add_thread(thread_t thread); | |
330 | extern void thread_update_process_threads(void); | |
331 | extern boolean_t runq_scan(run_queue_t runq, sched_update_scan_context_t scan_context); | |
332 | ||
333 | extern void sched_timeshare_init(void); | |
334 | extern void sched_timeshare_timebase_init(void); | |
335 | extern void sched_timeshare_maintenance_continue(void); | |
336 | ||
337 | extern boolean_t priority_is_urgent(int priority); | |
338 | extern uint32_t sched_timeshare_initial_quantum_size(thread_t thread); | |
339 | ||
340 | extern int sched_compute_timeshare_priority(thread_t thread); | |
341 | ||
342 | #endif /* CONFIG_SCHED_TIMESHARE_CORE */ | |
343 | ||
344 | /* Remove thread from its run queue */ | |
345 | extern boolean_t thread_run_queue_remove(thread_t thread); | |
346 | thread_t thread_run_queue_remove_for_handoff(thread_t thread); | |
347 | ||
348 | /* Put a thread back in the run queue after being yanked */ | |
349 | extern void thread_run_queue_reinsert(thread_t thread, integer_t options); | |
350 | ||
351 | extern void thread_timer_expire( | |
352 | void *thread, | |
353 | void *p1); | |
354 | ||
355 | extern boolean_t thread_eager_preemption( | |
356 | thread_t thread); | |
357 | ||
358 | extern boolean_t sched_generic_direct_dispatch_to_idle_processors; | |
359 | ||
360 | /* Set the maximum interrupt level for the thread */ | |
361 | __private_extern__ wait_interrupt_t thread_interrupt_level( | |
362 | wait_interrupt_t interruptible); | |
363 | ||
364 | __private_extern__ wait_result_t thread_mark_wait_locked( | |
365 | thread_t thread, | |
366 | wait_interrupt_t interruptible); | |
367 | ||
368 | /* Wake up locked thread directly, passing result */ | |
369 | __private_extern__ kern_return_t clear_wait_internal( | |
370 | thread_t thread, | |
371 | wait_result_t result); | |
372 | ||
373 | extern void sched_stats_handle_csw( | |
374 | processor_t processor, | |
375 | int reasons, | |
376 | int selfpri, | |
377 | int otherpri); | |
378 | ||
379 | extern void sched_stats_handle_runq_change( | |
380 | struct runq_stats *stats, | |
381 | int old_count); | |
382 | ||
383 | ||
384 | #if DEBUG | |
385 | ||
386 | #define SCHED_STATS_CSW(processor, reasons, selfpri, otherpri) \ | |
387 | do { \ | |
388 | if (__builtin_expect(sched_stats_active, 0)) { \ | |
389 | sched_stats_handle_csw((processor), \ | |
390 | (reasons), (selfpri), (otherpri)); \ | |
391 | } \ | |
392 | } while (0) | |
393 | ||
394 | ||
395 | #define SCHED_STATS_RUNQ_CHANGE(stats, old_count) \ | |
396 | do { \ | |
397 | if (__builtin_expect(sched_stats_active, 0)) { \ | |
398 | sched_stats_handle_runq_change((stats), \ | |
399 | (old_count)); \ | |
400 | } \ | |
401 | } while (0) | |
402 | ||
403 | #else /* DEBUG */ | |
404 | ||
405 | #define SCHED_STATS_CSW(processor, reasons, selfpri, otherpri) do { }while(0) | |
406 | #define SCHED_STATS_RUNQ_CHANGE(stats, old_count) do { }while(0) | |
407 | ||
408 | #endif /* DEBUG */ | |
409 | ||
410 | extern uint32_t sched_debug_flags; | |
411 | #define SCHED_DEBUG_FLAG_PLATFORM_TRACEPOINTS 0x00000001 | |
412 | #define SCHED_DEBUG_FLAG_CHOOSE_PROCESSOR_TRACEPOINTS 0x00000002 | |
413 | ||
414 | #define SCHED_DEBUG_PLATFORM_KERNEL_DEBUG_CONSTANT(...) do { \ | |
415 | if (__improbable(sched_debug_flags & SCHED_DEBUG_FLAG_PLATFORM_TRACEPOINTS)) { \ | |
416 | KERNEL_DEBUG_CONSTANT(__VA_ARGS__); \ | |
417 | } \ | |
418 | } while(0) | |
419 | ||
420 | #define SCHED_DEBUG_CHOOSE_PROCESSOR_KERNEL_DEBUG_CONSTANT(...) do { \ | |
421 | if (__improbable(sched_debug_flags & SCHED_DEBUG_FLAG_CHOOSE_PROCESSOR_TRACEPOINTS)) { \ | |
422 | KERNEL_DEBUG_CONSTANT(__VA_ARGS__); \ | |
423 | } \ | |
424 | } while(0) | |
425 | ||
426 | #define THREAD_URGENCY_NONE 0 /* indicates that there is no currently runnable */ | |
427 | #define THREAD_URGENCY_BACKGROUND 1 /* indicates that the thread is marked as a "background" thread */ | |
428 | #define THREAD_URGENCY_NORMAL 2 /* indicates that the thread is marked as a "normal" thread */ | |
429 | #define THREAD_URGENCY_REAL_TIME 3 /* indicates that the thread is marked as a "real-time" or urgent thread */ | |
430 | #define THREAD_URGENCY_MAX 4 /* Marker */ | |
431 | /* Returns the "urgency" of a thread (provided by scheduler) */ | |
432 | extern int thread_get_urgency( | |
433 | thread_t thread, | |
434 | uint64_t *rt_period, | |
435 | uint64_t *rt_deadline); | |
436 | ||
437 | /* Tells the "urgency" of the just scheduled thread (provided by CPU PM) */ | |
438 | extern void thread_tell_urgency( | |
439 | int urgency, | |
440 | uint64_t rt_period, | |
441 | uint64_t rt_deadline, | |
442 | uint64_t sched_latency, | |
443 | thread_t nthread); | |
444 | ||
445 | /* Tells if there are "active" RT threads in the system (provided by CPU PM) */ | |
446 | extern void active_rt_threads( | |
447 | boolean_t active); | |
448 | ||
449 | /* Returns the perfcontrol attribute for the thread */ | |
450 | extern perfcontrol_class_t thread_get_perfcontrol_class( | |
451 | thread_t thread); | |
452 | ||
453 | /* Generic routine for Non-AMP schedulers to calculate parallelism */ | |
454 | extern uint32_t sched_qos_max_parallelism(int qos, uint64_t options); | |
455 | ||
456 | #endif /* MACH_KERNEL_PRIVATE */ | |
457 | ||
458 | __BEGIN_DECLS | |
459 | ||
460 | #ifdef XNU_KERNEL_PRIVATE | |
461 | ||
462 | /* Toggles a global override to turn off CPU Throttling */ | |
463 | extern void sys_override_cpu_throttle(boolean_t enable_override); | |
464 | ||
465 | /* | |
466 | ****************** Only exported until BSD stops using ******************** | |
467 | */ | |
468 | ||
469 | extern void thread_vm_bind_group_add(void); | |
470 | ||
471 | /* Wake up thread directly, passing result */ | |
472 | extern kern_return_t clear_wait( | |
473 | thread_t thread, | |
474 | wait_result_t result); | |
475 | ||
476 | /* Start thread running */ | |
477 | extern void thread_bootstrap_return(void) __attribute__((noreturn)); | |
478 | ||
479 | /* Return from exception (BSD-visible interface) */ | |
480 | extern void thread_exception_return(void) __dead2; | |
481 | ||
482 | #define SCHED_STRING_MAX_LENGTH (48) | |
483 | /* String declaring the name of the current scheduler */ | |
484 | extern char sched_string[SCHED_STRING_MAX_LENGTH]; | |
485 | ||
486 | extern thread_t port_name_to_thread_for_ulock(mach_port_name_t thread_name); | |
487 | ||
488 | /* Attempt to context switch to a specific runnable thread */ | |
489 | extern wait_result_t thread_handoff_deallocate(thread_t thread); | |
490 | ||
491 | __attribute__((nonnull(1, 2))) | |
492 | extern void thread_handoff_parameter(thread_t thread, | |
493 | thread_continue_t continuation, void *parameter) __dead2; | |
494 | ||
495 | extern struct waitq *assert_wait_queue(event_t event); | |
496 | ||
497 | extern kern_return_t thread_wakeup_one_with_pri(event_t event, int priority); | |
498 | ||
499 | extern thread_t thread_wakeup_identify(event_t event, int priority); | |
500 | ||
501 | #endif /* XNU_KERNEL_PRIVATE */ | |
502 | ||
503 | #ifdef KERNEL_PRIVATE | |
504 | /* Set pending block hint for a particular object before we go into a wait state */ | |
505 | extern void thread_set_pending_block_hint( | |
506 | thread_t thread, | |
507 | block_hint_t block_hint); | |
508 | ||
509 | #define QOS_PARALLELISM_COUNT_LOGICAL 0x1 | |
510 | #define QOS_PARALLELISM_REALTIME 0x2 | |
511 | extern uint32_t qos_max_parallelism(int qos, uint64_t options); | |
512 | #endif /* KERNEL_PRIVATE */ | |
513 | ||
514 | #if XNU_KERNEL_PRIVATE | |
515 | extern void thread_yield_with_continuation( | |
516 | thread_continue_t continuation, | |
517 | void *parameter) __dead2; | |
518 | #endif | |
519 | ||
520 | /* Context switch */ | |
521 | extern wait_result_t thread_block( | |
522 | thread_continue_t continuation); | |
523 | ||
524 | extern wait_result_t thread_block_parameter( | |
525 | thread_continue_t continuation, | |
526 | void *parameter); | |
527 | ||
528 | /* Declare thread will wait on a particular event */ | |
529 | extern wait_result_t assert_wait( | |
530 | event_t event, | |
531 | wait_interrupt_t interruptible); | |
532 | ||
533 | /* Assert that the thread intends to wait with a timeout */ | |
534 | extern wait_result_t assert_wait_timeout( | |
535 | event_t event, | |
536 | wait_interrupt_t interruptible, | |
537 | uint32_t interval, | |
538 | uint32_t scale_factor); | |
539 | ||
540 | /* Assert that the thread intends to wait with an urgency, timeout and leeway */ | |
541 | extern wait_result_t assert_wait_timeout_with_leeway( | |
542 | event_t event, | |
543 | wait_interrupt_t interruptible, | |
544 | wait_timeout_urgency_t urgency, | |
545 | uint32_t interval, | |
546 | uint32_t leeway, | |
547 | uint32_t scale_factor); | |
548 | ||
549 | extern wait_result_t assert_wait_deadline( | |
550 | event_t event, | |
551 | wait_interrupt_t interruptible, | |
552 | uint64_t deadline); | |
553 | ||
554 | /* Assert that the thread intends to wait with an urgency, deadline, and leeway */ | |
555 | extern wait_result_t assert_wait_deadline_with_leeway( | |
556 | event_t event, | |
557 | wait_interrupt_t interruptible, | |
558 | wait_timeout_urgency_t urgency, | |
559 | uint64_t deadline, | |
560 | uint64_t leeway); | |
561 | ||
562 | /* Wake up thread (or threads) waiting on a particular event */ | |
563 | extern kern_return_t thread_wakeup_prim( | |
564 | event_t event, | |
565 | boolean_t one_thread, | |
566 | wait_result_t result); | |
567 | ||
568 | #define thread_wakeup(x) \ | |
569 | thread_wakeup_prim((x), FALSE, THREAD_AWAKENED) | |
570 | #define thread_wakeup_with_result(x, z) \ | |
571 | thread_wakeup_prim((x), FALSE, (z)) | |
572 | #define thread_wakeup_one(x) \ | |
573 | thread_wakeup_prim((x), TRUE, THREAD_AWAKENED) | |
574 | ||
575 | /* Wakeup the specified thread if it is waiting on this event */ | |
576 | extern kern_return_t thread_wakeup_thread(event_t event, thread_t thread); | |
577 | ||
578 | extern boolean_t preemption_enabled(void); | |
579 | ||
580 | #ifdef MACH_KERNEL_PRIVATE | |
581 | ||
582 | /* | |
583 | * Scheduler algorithm indirection. If only one algorithm is | |
584 | * enabled at compile-time, a direction function call is used. | |
585 | * If more than one is enabled, calls are dispatched through | |
586 | * a function pointer table. | |
587 | */ | |
588 | ||
589 | #if !defined(CONFIG_SCHED_TRADITIONAL) && !defined(CONFIG_SCHED_PROTO) && !defined(CONFIG_SCHED_GRRR) && !defined(CONFIG_SCHED_MULTIQ) | |
590 | #error Enable at least one scheduler algorithm in osfmk/conf/MASTER.XXX | |
591 | #endif | |
592 | ||
593 | #if DEBUG | |
594 | #define SCHED(f) (sched_current_dispatch->f) | |
595 | #else /* DEBUG */ | |
596 | ||
597 | /* | |
598 | * For DEV & REL kernels, use a static dispatch table instead of | |
599 | * using the indirect function table. | |
600 | */ | |
601 | extern const struct sched_dispatch_table sched_dualq_dispatch; | |
602 | #define SCHED(f) (sched_dualq_dispatch.f) | |
603 | ||
604 | #endif /* DEBUG */ | |
605 | ||
606 | struct sched_dispatch_table { | |
607 | const char *sched_name; | |
608 | void (*init)(void); /* Init global state */ | |
609 | void (*timebase_init)(void); /* Timebase-dependent initialization */ | |
610 | void (*processor_init)(processor_t processor); /* Per-processor scheduler init */ | |
611 | void (*pset_init)(processor_set_t pset); /* Per-processor set scheduler init */ | |
612 | ||
613 | void (*maintenance_continuation)(void); /* Function called regularly */ | |
614 | ||
615 | /* | |
616 | * Choose a thread of greater or equal priority from the per-processor | |
617 | * runqueue for timeshare/fixed threads | |
618 | */ | |
619 | thread_t (*choose_thread)( | |
620 | processor_t processor, | |
621 | int priority, | |
622 | ast_t reason); | |
623 | ||
624 | /* True if scheduler supports stealing threads */ | |
625 | boolean_t steal_thread_enabled; | |
626 | ||
627 | /* | |
628 | * Steal a thread from another processor in the pset so that it can run | |
629 | * immediately | |
630 | */ | |
631 | thread_t (*steal_thread)( | |
632 | processor_set_t pset); | |
633 | ||
634 | /* | |
635 | * Compute priority for a timeshare thread based on base priority. | |
636 | */ | |
637 | int (*compute_timeshare_priority)(thread_t thread); | |
638 | ||
639 | /* | |
640 | * Pick the best processor for a thread (any kind of thread) to run on. | |
641 | */ | |
642 | processor_t (*choose_processor)( | |
643 | processor_set_t pset, | |
644 | processor_t processor, | |
645 | thread_t thread); | |
646 | /* | |
647 | * Enqueue a timeshare or fixed priority thread onto the per-processor | |
648 | * runqueue | |
649 | */ | |
650 | boolean_t (*processor_enqueue)( | |
651 | processor_t processor, | |
652 | thread_t thread, | |
653 | integer_t options); | |
654 | ||
655 | /* Migrate threads away in preparation for processor shutdown */ | |
656 | void (*processor_queue_shutdown)( | |
657 | processor_t processor); | |
658 | ||
659 | /* Remove the specific thread from the per-processor runqueue */ | |
660 | boolean_t (*processor_queue_remove)( | |
661 | processor_t processor, | |
662 | thread_t thread); | |
663 | ||
664 | /* | |
665 | * Does the per-processor runqueue have any timeshare or fixed priority | |
666 | * threads on it? Called without pset lock held, so should | |
667 | * not assume immutability while executing. | |
668 | */ | |
669 | boolean_t (*processor_queue_empty)(processor_t processor); | |
670 | ||
671 | /* | |
672 | * Would this priority trigger an urgent preemption if it's sitting | |
673 | * on the per-processor runqueue? | |
674 | */ | |
675 | boolean_t (*priority_is_urgent)(int priority); | |
676 | ||
677 | /* | |
678 | * Does the per-processor runqueue contain runnable threads that | |
679 | * should cause the currently-running thread to be preempted? | |
680 | */ | |
681 | ast_t (*processor_csw_check)(processor_t processor); | |
682 | ||
683 | /* | |
684 | * Does the per-processor runqueue contain a runnable thread | |
685 | * of > or >= priority, as a preflight for choose_thread() or other | |
686 | * thread selection | |
687 | */ | |
688 | boolean_t (*processor_queue_has_priority)(processor_t processor, | |
689 | int priority, | |
690 | boolean_t gte); | |
691 | ||
692 | /* Quantum size for the specified non-realtime thread. */ | |
693 | uint32_t (*initial_quantum_size)(thread_t thread); | |
694 | ||
695 | /* Scheduler mode for a new thread */ | |
696 | sched_mode_t (*initial_thread_sched_mode)(task_t parent_task); | |
697 | ||
698 | /* | |
699 | * Is it safe to call update_priority, which may change a thread's | |
700 | * runqueue or other state. This can be used to throttle changes | |
701 | * to dynamic priority. | |
702 | */ | |
703 | boolean_t (*can_update_priority)(thread_t thread); | |
704 | ||
705 | /* | |
706 | * Update both scheduled priority and other persistent state. | |
707 | * Side effects may including migration to another processor's runqueue. | |
708 | */ | |
709 | void (*update_priority)(thread_t thread); | |
710 | ||
711 | /* Lower overhead update to scheduled priority and state. */ | |
712 | void (*lightweight_update_priority)(thread_t thread); | |
713 | ||
714 | /* Callback for non-realtime threads when the quantum timer fires */ | |
715 | void (*quantum_expire)(thread_t thread); | |
716 | ||
717 | /* | |
718 | * Runnable threads on per-processor runqueue. Should only | |
719 | * be used for relative comparisons of load between processors. | |
720 | */ | |
721 | int (*processor_runq_count)(processor_t processor); | |
722 | ||
723 | /* Aggregate runcount statistics for per-processor runqueue */ | |
724 | uint64_t (*processor_runq_stats_count_sum)(processor_t processor); | |
725 | ||
726 | boolean_t (*processor_bound_count)(processor_t processor); | |
727 | ||
728 | void (*thread_update_scan)(sched_update_scan_context_t scan_context); | |
729 | ||
730 | /* | |
731 | * Use processor->next_thread to pin a thread to an idle | |
732 | * processor. If FALSE, threads are enqueued and can | |
733 | * be stolen by other processors. | |
734 | */ | |
735 | boolean_t direct_dispatch_to_idle_processors; | |
736 | ||
737 | /* Supports more than one pset */ | |
738 | boolean_t multiple_psets_enabled; | |
739 | /* Supports scheduler groups */ | |
740 | boolean_t sched_groups_enabled; | |
741 | ||
742 | /* Supports avoid-processor */ | |
743 | boolean_t avoid_processor_enabled; | |
744 | ||
745 | /* Returns true if this processor should avoid running this thread. */ | |
746 | bool (*thread_avoid_processor)(processor_t processor, thread_t thread); | |
747 | ||
748 | /* | |
749 | * Invoked when a processor is about to choose the idle thread | |
750 | * Used to send IPIs to a processor which would be preferred to be idle instead. | |
751 | * Called with pset lock held, returns pset lock unlocked. | |
752 | */ | |
753 | void (*processor_balance)(processor_t processor, processor_set_t pset); | |
754 | rt_queue_t (*rt_runq)(processor_set_t pset); | |
755 | void (*rt_init)(processor_set_t pset); | |
756 | void (*rt_queue_shutdown)(processor_t processor); | |
757 | void (*rt_runq_scan)(sched_update_scan_context_t scan_context); | |
758 | int64_t (*rt_runq_count_sum)(void); | |
759 | ||
760 | uint32_t (*qos_max_parallelism)(int qos, uint64_t options); | |
761 | void (*check_spill)(processor_set_t pset, thread_t thread); | |
762 | sched_ipi_type_t (*ipi_policy)(processor_t dst, thread_t thread, boolean_t dst_idle, sched_ipi_event_t event); | |
763 | bool (*thread_should_yield)(processor_t processor, thread_t thread); | |
764 | }; | |
765 | ||
766 | #if defined(CONFIG_SCHED_TRADITIONAL) | |
767 | extern const struct sched_dispatch_table sched_traditional_dispatch; | |
768 | extern const struct sched_dispatch_table sched_traditional_with_pset_runqueue_dispatch; | |
769 | #endif | |
770 | ||
771 | #if defined(CONFIG_SCHED_MULTIQ) | |
772 | extern const struct sched_dispatch_table sched_multiq_dispatch; | |
773 | extern const struct sched_dispatch_table sched_dualq_dispatch; | |
774 | #endif | |
775 | ||
776 | #if defined(CONFIG_SCHED_PROTO) | |
777 | extern const struct sched_dispatch_table sched_proto_dispatch; | |
778 | #endif | |
779 | ||
780 | #if defined(CONFIG_SCHED_GRRR) | |
781 | extern const struct sched_dispatch_table sched_grrr_dispatch; | |
782 | #endif | |
783 | ||
784 | /* | |
785 | * It is an error to invoke any scheduler-related code | |
786 | * before this is set up | |
787 | */ | |
788 | extern const struct sched_dispatch_table *sched_current_dispatch; | |
789 | ||
790 | #endif /* MACH_KERNEL_PRIVATE */ | |
791 | ||
792 | __END_DECLS | |
793 | ||
794 | #endif /* _KERN_SCHED_PRIM_H_ */ |