]> git.saurik.com Git - apple/xnu.git/blob - osfmk/kern/processor.c
projects / apple / xnu.git / blob
? search:


486efc100f6589b8ec95415243c3611879b34361
[apple/xnu.git] / osfmk / kern / processor.c
1 /*
2 * Copyright (c) 2000-2009 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56 /*
57 */
58
59 /*
60 * processor.c: processor and processor_set manipulation routines.
61 */
62
63 #include <mach/boolean.h>
64 #include <mach/policy.h>
65 #include <mach/processor.h>
66 #include <mach/processor_info.h>
67 #include <mach/vm_param.h>
68 #include <kern/cpu_number.h>
69 #include <kern/host.h>
70 #include <kern/machine.h>
71 #include <kern/misc_protos.h>
72 #include <kern/processor.h>
73 #include <kern/sched.h>
74 #include <kern/task.h>
75 #include <kern/thread.h>
76 #include <kern/ipc_host.h>
77 #include <kern/ipc_tt.h>
78 #include <ipc/ipc_port.h>
79 #include <kern/kalloc.h>
80
81 #include <security/mac_mach_internal.h>
82
83 #if defined(CONFIG_XNUPOST)
84
85 #include <tests/xnupost.h>
86
87 #endif /* CONFIG_XNUPOST */
88
89 /*
90 * Exported interface
91 */
92 #include <mach/mach_host_server.h>
93 #include <mach/processor_set_server.h>
94
95 struct processor_set pset0;
96 struct pset_node pset_node0;
97 decl_simple_lock_data(static, pset_node_lock)
98
99 lck_grp_t pset_lck_grp;
100
101 queue_head_t tasks;
102 queue_head_t terminated_tasks; /* To be used ONLY for stackshot. */
103 queue_head_t corpse_tasks;
104 int tasks_count;
105 int terminated_tasks_count;
106 queue_head_t threads;
107 int threads_count;
108 decl_lck_mtx_data(, tasks_threads_lock)
109 decl_lck_mtx_data(, tasks_corpse_lock)
110
111 processor_t processor_list;
112 unsigned int processor_count;
113 static processor_t processor_list_tail;
114 decl_simple_lock_data(, processor_list_lock)
115
116 uint32_t processor_avail_count;
117 uint32_t processor_avail_count_user;
118
119 processor_t master_processor;
120 int master_cpu = 0;
121 boolean_t sched_stats_active = FALSE;
122
123 processor_t processor_array[MAX_SCHED_CPUS] = { 0 };
124
125 #if defined(CONFIG_XNUPOST)
126 kern_return_t ipi_test(void);
127 extern void arm64_ipi_test(void);
128
129 kern_return_t
130 ipi_test()
131 {
132 #if __arm64__
133 processor_t p;
134
135 for (p = processor_list; p != NULL; p = p->processor_list) {
136 thread_bind(p);
137 thread_block(THREAD_CONTINUE_NULL);
138 kprintf("Running IPI test on cpu %d\n", p->cpu_id);
139 arm64_ipi_test();
140 }
141
142 /* unbind thread from specific cpu */
143 thread_bind(PROCESSOR_NULL);
144 thread_block(THREAD_CONTINUE_NULL);
145
146 T_PASS("Done running IPI tests");
147 #else
148 T_PASS("Unsupported platform. Not running IPI tests");
149
150 #endif /* __arm64__ */
151
152 return KERN_SUCCESS;
153 }
154 #endif /* defined(CONFIG_XNUPOST) */
155
156 int sched_enable_smt = 1;
157
158 void
159 processor_bootstrap(void)
160 {
161 lck_grp_init(&pset_lck_grp, "pset", LCK_GRP_ATTR_NULL);
162
163 simple_lock_init(&pset_node_lock, 0);
164
165 pset_node0.psets = &pset0;
166 pset_init(&pset0, &pset_node0);
167
168 queue_init(&tasks);
169 queue_init(&terminated_tasks);
170 queue_init(&threads);
171 queue_init(&corpse_tasks);
172
173 simple_lock_init(&processor_list_lock, 0);
174
175 master_processor = cpu_to_processor(master_cpu);
176
177 processor_init(master_processor, master_cpu, &pset0);
178 }
179
180 /*
181 * Initialize the given processor for the cpu
182 * indicated by cpu_id, and assign to the
183 * specified processor set.
184 */
185 void
186 processor_init(
187 processor_t processor,
188 int cpu_id,
189 processor_set_t pset)
190 {
191 spl_t s;
192
193 if (processor != master_processor) {
194 /* Scheduler state for master_processor initialized in sched_init() */
195 SCHED(processor_init)(processor);
196 }
197
198 assert(cpu_id < MAX_SCHED_CPUS);
199
200 processor->state = PROCESSOR_OFF_LINE;
201 processor->active_thread = processor->next_thread = processor->idle_thread = THREAD_NULL;
202 processor->processor_set = pset;
203 processor_state_update_idle(processor);
204 processor->starting_pri = MINPRI;
205 processor->cpu_id = cpu_id;
206 timer_call_setup(&processor->quantum_timer, thread_quantum_expire, processor);
207 processor->quantum_end = UINT64_MAX;
208 processor->deadline = UINT64_MAX;
209 processor->first_timeslice = FALSE;
210 processor->processor_primary = processor; /* no SMT relationship known at this point */
211 processor->processor_secondary = NULL;
212 processor->is_SMT = FALSE;
213 processor->is_recommended = (pset->recommended_bitmask & (1ULL << cpu_id)) ? TRUE : FALSE;
214 processor->processor_self = IP_NULL;
215 processor_data_init(processor);
216 processor->processor_list = NULL;
217 processor->cpu_quiesce_state = CPU_QUIESCE_COUNTER_NONE;
218 processor->cpu_quiesce_last_checkin = 0;
219 processor->must_idle = false;
220
221 s = splsched();
222 pset_lock(pset);
223 bit_set(pset->cpu_bitmask, cpu_id);
224 if (pset->cpu_set_count++ == 0) {
225 pset->cpu_set_low = pset->cpu_set_hi = cpu_id;
226 } else {
227 pset->cpu_set_low = (cpu_id < pset->cpu_set_low)? cpu_id: pset->cpu_set_low;
228 pset->cpu_set_hi = (cpu_id > pset->cpu_set_hi)? cpu_id: pset->cpu_set_hi;
229 }
230 pset_unlock(pset);
231 splx(s);
232
233 simple_lock(&processor_list_lock, LCK_GRP_NULL);
234 if (processor_list == NULL) {
235 processor_list = processor;
236 } else {
237 processor_list_tail->processor_list = processor;
238 }
239 processor_list_tail = processor;
240 processor_count++;
241 processor_array[cpu_id] = processor;
242 simple_unlock(&processor_list_lock);
243 }
244
245 void
246 processor_set_primary(
247 processor_t processor,
248 processor_t primary)
249 {
250 assert(processor->processor_primary == primary || processor->processor_primary == processor);
251 /* Re-adjust primary point for this (possibly) secondary processor */
252 processor->processor_primary = primary;
253
254 assert(primary->processor_secondary == NULL || primary->processor_secondary == processor);
255 if (primary != processor) {
256 /* Link primary to secondary, assumes a 2-way SMT model
257 * We'll need to move to a queue if any future architecture
258 * requires otherwise.
259 */
260 assert(processor->processor_secondary == NULL);
261 primary->processor_secondary = processor;
262 /* Mark both processors as SMT siblings */
263 primary->is_SMT = TRUE;
264 processor->is_SMT = TRUE;
265
266 processor_set_t pset = processor->processor_set;
267 spl_t s = splsched();
268 pset_lock(pset);
269 bit_clear(pset->primary_map, processor->cpu_id);
270 pset_unlock(pset);
271 splx(s);
272 }
273 }
274
275 processor_set_t
276 processor_pset(
277 processor_t processor)
278 {
279 return processor->processor_set;
280 }
281
282 void
283 processor_state_update_idle(processor_t processor)
284 {
285 processor->current_pri = IDLEPRI;
286 processor->current_sfi_class = SFI_CLASS_KERNEL;
287 processor->current_recommended_pset_type = PSET_SMP;
288 processor->current_perfctl_class = PERFCONTROL_CLASS_IDLE;
289 processor->current_urgency = THREAD_URGENCY_NONE;
290 processor->current_is_NO_SMT = false;
291 processor->current_is_bound = false;
292 }
293
294 void
295 processor_state_update_from_thread(processor_t processor, thread_t thread)
296 {
297 processor->current_pri = thread->sched_pri;
298 processor->current_sfi_class = thread->sfi_class;
299 processor->current_recommended_pset_type = recommended_pset_type(thread);
300 processor->current_perfctl_class = thread_get_perfcontrol_class(thread);
301 processor->current_urgency = thread_get_urgency(thread, NULL, NULL);
302 #if DEBUG || DEVELOPMENT
303 processor->current_is_NO_SMT = (thread->sched_flags & TH_SFLAG_NO_SMT) || (thread->task->t_flags & TF_NO_SMT);
304 #else
305 processor->current_is_NO_SMT = (thread->sched_flags & TH_SFLAG_NO_SMT);
306 #endif
307 processor->current_is_bound = thread->bound_processor != PROCESSOR_NULL;
308 }
309
310 void
311 processor_state_update_explicit(processor_t processor, int pri, sfi_class_id_t sfi_class,
312 pset_cluster_type_t pset_type, perfcontrol_class_t perfctl_class, thread_urgency_t urgency)
313 {
314 processor->current_pri = pri;
315 processor->current_sfi_class = sfi_class;
316 processor->current_recommended_pset_type = pset_type;
317 processor->current_perfctl_class = perfctl_class;
318 processor->current_urgency = urgency;
319 }
320
321 pset_node_t
322 pset_node_root(void)
323 {
324 return &pset_node0;
325 }
326
327 processor_set_t
328 pset_create(
329 pset_node_t node)
330 {
331 /* some schedulers do not support multiple psets */
332 if (SCHED(multiple_psets_enabled) == FALSE) {
333 return processor_pset(master_processor);
334 }
335
336 processor_set_t *prev, pset = kalloc(sizeof(*pset));
337
338 if (pset != PROCESSOR_SET_NULL) {
339 pset_init(pset, node);
340
341 simple_lock(&pset_node_lock, LCK_GRP_NULL);
342
343 prev = &node->psets;
344 while (*prev != PROCESSOR_SET_NULL) {
345 prev = &(*prev)->pset_list;
346 }
347
348 *prev = pset;
349
350 simple_unlock(&pset_node_lock);
351 }
352
353 return pset;
354 }
355
356 /*
357 * Find processor set in specified node with specified cluster_id.
358 * Returns default_pset if not found.
359 */
360 processor_set_t
361 pset_find(
362 uint32_t cluster_id,
363 processor_set_t default_pset)
364 {
365 simple_lock(&pset_node_lock, LCK_GRP_NULL);
366 pset_node_t node = &pset_node0;
367 processor_set_t pset = NULL;
368
369 do {
370 pset = node->psets;
371 while (pset != NULL) {
372 if (pset->pset_cluster_id == cluster_id) {
373 break;
374 }
375 pset = pset->pset_list;
376 }
377 } while ((node = node->node_list) != NULL);
378 simple_unlock(&pset_node_lock);
379 if (pset == NULL) {
380 return default_pset;
381 }
382 return pset;
383 }
384
385 /*
386 * Initialize the given processor_set structure.
387 */
388 void
389 pset_init(
390 processor_set_t pset,
391 pset_node_t node)
392 {
393 if (pset != &pset0) {
394 /* Scheduler state for pset0 initialized in sched_init() */
395 SCHED(pset_init)(pset);
396 SCHED(rt_init)(pset);
397 }
398
399 pset->online_processor_count = 0;
400 pset->load_average = 0;
401 pset->cpu_set_low = pset->cpu_set_hi = 0;
402 pset->cpu_set_count = 0;
403 pset->last_chosen = -1;
404 pset->cpu_bitmask = 0;
405 pset->recommended_bitmask = ~0ULL;
406 pset->primary_map = ~0ULL;
407 pset->cpu_state_map[PROCESSOR_OFF_LINE] = ~0ULL;
408 for (uint i = PROCESSOR_SHUTDOWN; i < PROCESSOR_STATE_LEN; i++) {
409 pset->cpu_state_map[i] = 0;
410 }
411 pset->pending_AST_URGENT_cpu_mask = 0;
412 pset->pending_AST_PREEMPT_cpu_mask = 0;
413 #if defined(CONFIG_SCHED_DEFERRED_AST)
414 pset->pending_deferred_AST_cpu_mask = 0;
415 #endif
416 pset->pending_spill_cpu_mask = 0;
417 pset_lock_init(pset);
418 pset->pset_self = IP_NULL;
419 pset->pset_name_self = IP_NULL;
420 pset->pset_list = PROCESSOR_SET_NULL;
421 pset->node = node;
422 pset->pset_cluster_type = PSET_SMP;
423 pset->pset_cluster_id = 0;
424
425 simple_lock(&pset_node_lock, LCK_GRP_NULL);
426 node->pset_count++;
427 simple_unlock(&pset_node_lock);
428 }
429
430 kern_return_t
431 processor_info_count(
432 processor_flavor_t flavor,
433 mach_msg_type_number_t *count)
434 {
435 switch (flavor) {
436 case PROCESSOR_BASIC_INFO:
437 *count = PROCESSOR_BASIC_INFO_COUNT;
438 break;
439
440 case PROCESSOR_CPU_LOAD_INFO:
441 *count = PROCESSOR_CPU_LOAD_INFO_COUNT;
442 break;
443
444 default:
445 return cpu_info_count(flavor, count);
446 }
447
448 return KERN_SUCCESS;
449 }
450
451
452 kern_return_t
453 processor_info(
454 processor_t processor,
455 processor_flavor_t flavor,
456 host_t *host,
457 processor_info_t info,
458 mach_msg_type_number_t *count)
459 {
460 int cpu_id, state;
461 kern_return_t result;
462
463 if (processor == PROCESSOR_NULL) {
464 return KERN_INVALID_ARGUMENT;
465 }
466
467 cpu_id = processor->cpu_id;
468
469 switch (flavor) {
470 case PROCESSOR_BASIC_INFO:
471 {
472 processor_basic_info_t basic_info;
473
474 if (*count < PROCESSOR_BASIC_INFO_COUNT) {
475 return KERN_FAILURE;
476 }
477
478 basic_info = (processor_basic_info_t) info;
479 basic_info->cpu_type = slot_type(cpu_id);
480 basic_info->cpu_subtype = slot_subtype(cpu_id);
481 state = processor->state;
482 if (state == PROCESSOR_OFF_LINE
483 #if defined(__x86_64__)
484 || !processor->is_recommended
485 #endif
486 ) {
487 basic_info->running = FALSE;
488 } else {
489 basic_info->running = TRUE;
490 }
491 basic_info->slot_num = cpu_id;
492 if (processor == master_processor) {
493 basic_info->is_master = TRUE;
494 } else {
495 basic_info->is_master = FALSE;
496 }
497
498 *count = PROCESSOR_BASIC_INFO_COUNT;
499 *host = &realhost;
500
501 return KERN_SUCCESS;
502 }
503
504 case PROCESSOR_CPU_LOAD_INFO:
505 {
506 processor_cpu_load_info_t cpu_load_info;
507 timer_t idle_state;
508 uint64_t idle_time_snapshot1, idle_time_snapshot2;
509 uint64_t idle_time_tstamp1, idle_time_tstamp2;
510
511 /*
512 * We capture the accumulated idle time twice over
513 * the course of this function, as well as the timestamps
514 * when each were last updated. Since these are
515 * all done using non-atomic racy mechanisms, the
516 * most we can infer is whether values are stable.
517 * timer_grab() is the only function that can be
518 * used reliably on another processor's per-processor
519 * data.
520 */
521
522 if (*count < PROCESSOR_CPU_LOAD_INFO_COUNT) {
523 return KERN_FAILURE;
524 }
525
526 cpu_load_info = (processor_cpu_load_info_t) info;
527 if (precise_user_kernel_time) {
528 cpu_load_info->cpu_ticks[CPU_STATE_USER] =
529 (uint32_t)(timer_grab(&PROCESSOR_DATA(processor, user_state)) / hz_tick_interval);
530 cpu_load_info->cpu_ticks[CPU_STATE_SYSTEM] =
531 (uint32_t)(timer_grab(&PROCESSOR_DATA(processor, system_state)) / hz_tick_interval);
532 } else {
533 uint64_t tval = timer_grab(&PROCESSOR_DATA(processor, user_state)) +
534 timer_grab(&PROCESSOR_DATA(processor, system_state));
535
536 cpu_load_info->cpu_ticks[CPU_STATE_USER] = (uint32_t)(tval / hz_tick_interval);
537 cpu_load_info->cpu_ticks[CPU_STATE_SYSTEM] = 0;
538 }
539
540 idle_state = &PROCESSOR_DATA(processor, idle_state);
541 idle_time_snapshot1 = timer_grab(idle_state);
542 idle_time_tstamp1 = idle_state->tstamp;
543
544 /*
545 * Idle processors are not continually updating their
546 * per-processor idle timer, so it may be extremely
547 * out of date, resulting in an over-representation
548 * of non-idle time between two measurement
549 * intervals by e.g. top(1). If we are non-idle, or
550 * have evidence that the timer is being updated
551 * concurrently, we consider its value up-to-date.
552 */
553 if (PROCESSOR_DATA(processor, current_state) != idle_state) {
554 cpu_load_info->cpu_ticks[CPU_STATE_IDLE] =
555 (uint32_t)(idle_time_snapshot1 / hz_tick_interval);
556 } else if ((idle_time_snapshot1 != (idle_time_snapshot2 = timer_grab(idle_state))) ||
557 (idle_time_tstamp1 != (idle_time_tstamp2 = idle_state->tstamp))) {
558 /* Idle timer is being updated concurrently, second stamp is good enough */
559 cpu_load_info->cpu_ticks[CPU_STATE_IDLE] =
560 (uint32_t)(idle_time_snapshot2 / hz_tick_interval);
561 } else {
562 /*
563 * Idle timer may be very stale. Fortunately we have established
564 * that idle_time_snapshot1 and idle_time_tstamp1 are unchanging
565 */
566 idle_time_snapshot1 += mach_absolute_time() - idle_time_tstamp1;
567
568 cpu_load_info->cpu_ticks[CPU_STATE_IDLE] =
569 (uint32_t)(idle_time_snapshot1 / hz_tick_interval);
570 }
571
572 cpu_load_info->cpu_ticks[CPU_STATE_NICE] = 0;
573
574 *count = PROCESSOR_CPU_LOAD_INFO_COUNT;
575 *host = &realhost;
576
577 return KERN_SUCCESS;
578 }
579
580 default:
581 result = cpu_info(flavor, cpu_id, info, count);
582 if (result == KERN_SUCCESS) {
583 *host = &realhost;
584 }
585
586 return result;
587 }
588 }
589
590 kern_return_t
591 processor_start(
592 processor_t processor)
593 {
594 processor_set_t pset;
595 thread_t thread;
596 kern_return_t result;
597 spl_t s;
598
599 if (processor == PROCESSOR_NULL || processor->processor_set == PROCESSOR_SET_NULL) {
600 return KERN_INVALID_ARGUMENT;
601 }
602
603 if (processor == master_processor) {
604 processor_t prev;
605
606 prev = thread_bind(processor);
607 thread_block(THREAD_CONTINUE_NULL);
608
609 result = cpu_start(processor->cpu_id);
610
611 thread_bind(prev);
612
613 return result;
614 }
615
616 bool scheduler_disable = false;
617
618 if ((processor->processor_primary != processor) && (sched_enable_smt == 0)) {
619 if (cpu_can_exit(processor->cpu_id)) {
620 return KERN_SUCCESS;
621 }
622 /*
623 * This secondary SMT processor must start in order to service interrupts,
624 * so instead it will be disabled at the scheduler level.
625 */
626 scheduler_disable = true;
627 }
628
629 s = splsched();
630 pset = processor->processor_set;
631 pset_lock(pset);
632 if (processor->state != PROCESSOR_OFF_LINE) {
633 pset_unlock(pset);
634 splx(s);
635
636 return KERN_FAILURE;
637 }
638
639 pset_update_processor_state(pset, processor, PROCESSOR_START);
640 pset_unlock(pset);
641 splx(s);
642
643 /*
644 * Create the idle processor thread.
645 */
646 if (processor->idle_thread == THREAD_NULL) {
647 result = idle_thread_create(processor);
648 if (result != KERN_SUCCESS) {
649 s = splsched();
650 pset_lock(pset);
651 pset_update_processor_state(pset, processor, PROCESSOR_OFF_LINE);
652 pset_unlock(pset);
653 splx(s);
654
655 return result;
656 }
657 }
658
659 /*
660 * If there is no active thread, the processor
661 * has never been started. Create a dedicated
662 * start up thread.
663 */
664 if (processor->active_thread == THREAD_NULL &&
665 processor->next_thread == THREAD_NULL) {
666 result = kernel_thread_create((thread_continue_t)processor_start_thread, NULL, MAXPRI_KERNEL, &thread);
667 if (result != KERN_SUCCESS) {
668 s = splsched();
669 pset_lock(pset);
670 pset_update_processor_state(pset, processor, PROCESSOR_OFF_LINE);
671 pset_unlock(pset);
672 splx(s);
673
674 return result;
675 }
676
677 s = splsched();
678 thread_lock(thread);
679 thread->bound_processor = processor;
680 processor->next_thread = thread;
681 thread->state = TH_RUN;
682 thread->last_made_runnable_time = mach_absolute_time();
683 thread_unlock(thread);
684 splx(s);
685
686 thread_deallocate(thread);
687 }
688
689 if (processor->processor_self == IP_NULL) {
690 ipc_processor_init(processor);
691 }
692
693 result = cpu_start(processor->cpu_id);
694 if (result != KERN_SUCCESS) {
695 s = splsched();
696 pset_lock(pset);
697 pset_update_processor_state(pset, processor, PROCESSOR_OFF_LINE);
698 pset_unlock(pset);
699 splx(s);
700
701 return result;
702 }
703 if (scheduler_disable) {
704 assert(processor->processor_primary != processor);
705 sched_processor_enable(processor, FALSE);
706 }
707
708 ipc_processor_enable(processor);
709
710 return KERN_SUCCESS;
711 }
712
713
714 kern_return_t
715 processor_exit(
716 processor_t processor)
717 {
718 if (processor == PROCESSOR_NULL) {
719 return KERN_INVALID_ARGUMENT;
720 }
721
722 return processor_shutdown(processor);
723 }
724
725
726 kern_return_t
727 processor_start_from_user(
728 processor_t processor)
729 {
730 kern_return_t ret;
731
732 if (processor == PROCESSOR_NULL) {
733 return KERN_INVALID_ARGUMENT;
734 }
735
736 if (!cpu_can_exit(processor->cpu_id)) {
737 ret = sched_processor_enable(processor, TRUE);
738 } else {
739 ret = processor_start(processor);
740 }
741
742 return ret;
743 }
744
745 kern_return_t
746 processor_exit_from_user(
747 processor_t processor)
748 {
749 kern_return_t ret;
750
751 if (processor == PROCESSOR_NULL) {
752 return KERN_INVALID_ARGUMENT;
753 }
754
755 if (!cpu_can_exit(processor->cpu_id)) {
756 ret = sched_processor_enable(processor, FALSE);
757 } else {
758 ret = processor_shutdown(processor);
759 }
760
761 return ret;
762 }
763
764 kern_return_t
765 enable_smt_processors(bool enable)
766 {
767 if (machine_info.logical_cpu_max == machine_info.physical_cpu_max) {
768 /* Not an SMT system */
769 return KERN_INVALID_ARGUMENT;
770 }
771
772 int ncpus = machine_info.logical_cpu_max;
773
774 for (int i = 1; i < ncpus; i++) {
775 processor_t processor = processor_array[i];
776
777 if (processor->processor_primary != processor) {
778 if (enable) {
779 processor_start_from_user(processor);
780 } else { /* Disable */
781 processor_exit_from_user(processor);
782 }
783 }
784 }
785
786 #define BSD_HOST 1
787 host_basic_info_data_t hinfo;
788 mach_msg_type_number_t count = HOST_BASIC_INFO_COUNT;
789 kern_return_t kret = host_info((host_t)BSD_HOST, HOST_BASIC_INFO, (host_info_t)&hinfo, &count);
790 if (kret != KERN_SUCCESS) {
791 return kret;
792 }
793
794 if (enable && (hinfo.logical_cpu != hinfo.logical_cpu_max)) {
795 return KERN_FAILURE;
796 }
797
798 if (!enable && (hinfo.logical_cpu != hinfo.physical_cpu)) {
799 return KERN_FAILURE;
800 }
801
802 return KERN_SUCCESS;
803 }
804
805 kern_return_t
806 processor_control(
807 processor_t processor,
808 processor_info_t info,
809 mach_msg_type_number_t count)
810 {
811 if (processor == PROCESSOR_NULL) {
812 return KERN_INVALID_ARGUMENT;
813 }
814
815 return cpu_control(processor->cpu_id, info, count);
816 }
817
818 kern_return_t
819 processor_set_create(
820 __unused host_t host,
821 __unused processor_set_t *new_set,
822 __unused processor_set_t *new_name)
823 {
824 return KERN_FAILURE;
825 }
826
827 kern_return_t
828 processor_set_destroy(
829 __unused processor_set_t pset)
830 {
831 return KERN_FAILURE;
832 }
833
834 kern_return_t
835 processor_get_assignment(
836 processor_t processor,
837 processor_set_t *pset)
838 {
839 int state;
840
841 if (processor == PROCESSOR_NULL) {
842 return KERN_INVALID_ARGUMENT;
843 }
844
845 state = processor->state;
846 if (state == PROCESSOR_SHUTDOWN || state == PROCESSOR_OFF_LINE) {
847 return KERN_FAILURE;
848 }
849
850 *pset = &pset0;
851
852 return KERN_SUCCESS;
853 }
854
855 kern_return_t
856 processor_set_info(
857 processor_set_t pset,
858 int flavor,
859 host_t *host,
860 processor_set_info_t info,
861 mach_msg_type_number_t *count)
862 {
863 if (pset == PROCESSOR_SET_NULL) {
864 return KERN_INVALID_ARGUMENT;
865 }
866
867 if (flavor == PROCESSOR_SET_BASIC_INFO) {
868 processor_set_basic_info_t basic_info;
869
870 if (*count < PROCESSOR_SET_BASIC_INFO_COUNT) {
871 return KERN_FAILURE;
872 }
873
874 basic_info = (processor_set_basic_info_t) info;
875 #if defined(__x86_64__)
876 basic_info->processor_count = processor_avail_count_user;
877 #else
878 basic_info->processor_count = processor_avail_count;
879 #endif
880 basic_info->default_policy = POLICY_TIMESHARE;
881
882 *count = PROCESSOR_SET_BASIC_INFO_COUNT;
883 *host = &realhost;
884 return KERN_SUCCESS;
885 } else if (flavor == PROCESSOR_SET_TIMESHARE_DEFAULT) {
886 policy_timeshare_base_t ts_base;
887
888 if (*count < POLICY_TIMESHARE_BASE_COUNT) {
889 return KERN_FAILURE;
890 }
891
892 ts_base = (policy_timeshare_base_t) info;
893 ts_base->base_priority = BASEPRI_DEFAULT;
894
895 *count = POLICY_TIMESHARE_BASE_COUNT;
896 *host = &realhost;
897 return KERN_SUCCESS;
898 } else if (flavor == PROCESSOR_SET_FIFO_DEFAULT) {
899 policy_fifo_base_t fifo_base;
900
901 if (*count < POLICY_FIFO_BASE_COUNT) {
902 return KERN_FAILURE;
903 }
904
905 fifo_base = (policy_fifo_base_t) info;
906 fifo_base->base_priority = BASEPRI_DEFAULT;
907
908 *count = POLICY_FIFO_BASE_COUNT;
909 *host = &realhost;
910 return KERN_SUCCESS;
911 } else if (flavor == PROCESSOR_SET_RR_DEFAULT) {
912 policy_rr_base_t rr_base;
913
914 if (*count < POLICY_RR_BASE_COUNT) {
915 return KERN_FAILURE;
916 }
917
918 rr_base = (policy_rr_base_t) info;
919 rr_base->base_priority = BASEPRI_DEFAULT;
920 rr_base->quantum = 1;
921
922 *count = POLICY_RR_BASE_COUNT;
923 *host = &realhost;
924 return KERN_SUCCESS;
925 } else if (flavor == PROCESSOR_SET_TIMESHARE_LIMITS) {
926 policy_timeshare_limit_t ts_limit;
927
928 if (*count < POLICY_TIMESHARE_LIMIT_COUNT) {
929 return KERN_FAILURE;
930 }
931
932 ts_limit = (policy_timeshare_limit_t) info;
933 ts_limit->max_priority = MAXPRI_KERNEL;
934
935 *count = POLICY_TIMESHARE_LIMIT_COUNT;
936 *host = &realhost;
937 return KERN_SUCCESS;
938 } else if (flavor == PROCESSOR_SET_FIFO_LIMITS) {
939 policy_fifo_limit_t fifo_limit;
940
941 if (*count < POLICY_FIFO_LIMIT_COUNT) {
942 return KERN_FAILURE;
943 }
944
945 fifo_limit = (policy_fifo_limit_t) info;
946 fifo_limit->max_priority = MAXPRI_KERNEL;
947
948 *count = POLICY_FIFO_LIMIT_COUNT;
949 *host = &realhost;
950 return KERN_SUCCESS;
951 } else if (flavor == PROCESSOR_SET_RR_LIMITS) {
952 policy_rr_limit_t rr_limit;
953
954 if (*count < POLICY_RR_LIMIT_COUNT) {
955 return KERN_FAILURE;
956 }
957
958 rr_limit = (policy_rr_limit_t) info;
959 rr_limit->max_priority = MAXPRI_KERNEL;
960
961 *count = POLICY_RR_LIMIT_COUNT;
962 *host = &realhost;
963 return KERN_SUCCESS;
964 } else if (flavor == PROCESSOR_SET_ENABLED_POLICIES) {
965 int *enabled;
966
967 if (*count < (sizeof(*enabled) / sizeof(int))) {
968 return KERN_FAILURE;
969 }
970
971 enabled = (int *) info;
972 *enabled = POLICY_TIMESHARE | POLICY_RR | POLICY_FIFO;
973
974 *count = sizeof(*enabled) / sizeof(int);
975 *host = &realhost;
976 return KERN_SUCCESS;
977 }
978
979
980 *host = HOST_NULL;
981 return KERN_INVALID_ARGUMENT;
982 }
983
984 /*
985 * processor_set_statistics
986 *
987 * Returns scheduling statistics for a processor set.
988 */
989 kern_return_t
990 processor_set_statistics(
991 processor_set_t pset,
992 int flavor,
993 processor_set_info_t info,
994 mach_msg_type_number_t *count)
995 {
996 if (pset == PROCESSOR_SET_NULL || pset != &pset0) {
997 return KERN_INVALID_PROCESSOR_SET;
998 }
999
1000 if (flavor == PROCESSOR_SET_LOAD_INFO) {
1001 processor_set_load_info_t load_info;
1002
1003 if (*count < PROCESSOR_SET_LOAD_INFO_COUNT) {
1004 return KERN_FAILURE;
1005 }
1006
1007 load_info = (processor_set_load_info_t) info;
1008
1009 load_info->mach_factor = sched_mach_factor;
1010 load_info->load_average = sched_load_average;
1011
1012 load_info->task_count = tasks_count;
1013 load_info->thread_count = threads_count;
1014
1015 *count = PROCESSOR_SET_LOAD_INFO_COUNT;
1016 return KERN_SUCCESS;
1017 }
1018
1019 return KERN_INVALID_ARGUMENT;
1020 }
1021
1022 /*
1023 * processor_set_max_priority:
1024 *
1025 * Specify max priority permitted on processor set. This affects
1026 * newly created and assigned threads. Optionally change existing
1027 * ones.
1028 */
1029 kern_return_t
1030 processor_set_max_priority(
1031 __unused processor_set_t pset,
1032 __unused int max_priority,
1033 __unused boolean_t change_threads)
1034 {
1035 return KERN_INVALID_ARGUMENT;
1036 }
1037
1038 /*
1039 * processor_set_policy_enable:
1040 *
1041 * Allow indicated policy on processor set.
1042 */
1043
1044 kern_return_t
1045 processor_set_policy_enable(
1046 __unused processor_set_t pset,
1047 __unused int policy)
1048 {
1049 return KERN_INVALID_ARGUMENT;
1050 }
1051
1052 /*
1053 * processor_set_policy_disable:
1054 *
1055 * Forbid indicated policy on processor set. Time sharing cannot
1056 * be forbidden.
1057 */
1058 kern_return_t
1059 processor_set_policy_disable(
1060 __unused processor_set_t pset,
1061 __unused int policy,
1062 __unused boolean_t change_threads)
1063 {
1064 return KERN_INVALID_ARGUMENT;
1065 }
1066
1067 /*
1068 * processor_set_things:
1069 *
1070 * Common internals for processor_set_{threads,tasks}
1071 */
1072 kern_return_t
1073 processor_set_things(
1074 processor_set_t pset,
1075 void **thing_list,
1076 mach_msg_type_number_t *count,
1077 int type)
1078 {
1079 unsigned int i;
1080 task_t task;
1081 thread_t thread;
1082
1083 task_t *task_list;
1084 unsigned int actual_tasks;
1085 vm_size_t task_size, task_size_needed;
1086
1087 thread_t *thread_list;
1088 unsigned int actual_threads;
1089 vm_size_t thread_size, thread_size_needed;
1090
1091 void *addr, *newaddr;
1092 vm_size_t size, size_needed;
1093
1094 if (pset == PROCESSOR_SET_NULL || pset != &pset0) {
1095 return KERN_INVALID_ARGUMENT;
1096 }
1097
1098 task_size = 0;
1099 task_size_needed = 0;
1100 task_list = NULL;
1101 actual_tasks = 0;
1102
1103 thread_size = 0;
1104 thread_size_needed = 0;
1105 thread_list = NULL;
1106 actual_threads = 0;
1107
1108 for (;;) {
1109 lck_mtx_lock(&tasks_threads_lock);
1110
1111 /* do we have the memory we need? */
1112 if (type == PSET_THING_THREAD) {
1113 thread_size_needed = threads_count * sizeof(void *);
1114 }
1115 #if !CONFIG_MACF
1116 else
1117 #endif
1118 task_size_needed = tasks_count * sizeof(void *);
1119
1120 if (task_size_needed <= task_size &&
1121 thread_size_needed <= thread_size) {
1122 break;
1123 }
1124
1125 /* unlock and allocate more memory */
1126 lck_mtx_unlock(&tasks_threads_lock);
1127
1128 /* grow task array */
1129 if (task_size_needed > task_size) {
1130 if (task_size != 0) {
1131 kfree(task_list, task_size);
1132 }
1133
1134 assert(task_size_needed > 0);
1135 task_size = task_size_needed;
1136
1137 task_list = (task_t *)kalloc(task_size);
1138 if (task_list == NULL) {
1139 if (thread_size != 0) {
1140 kfree(thread_list, thread_size);
1141 }
1142 return KERN_RESOURCE_SHORTAGE;
1143 }
1144 }
1145
1146 /* grow thread array */
1147 if (thread_size_needed > thread_size) {
1148 if (thread_size != 0) {
1149 kfree(thread_list, thread_size);
1150 }
1151
1152 assert(thread_size_needed > 0);
1153 thread_size = thread_size_needed;
1154
1155 thread_list = (thread_t *)kalloc(thread_size);
1156 if (thread_list == 0) {
1157 if (task_size != 0) {
1158 kfree(task_list, task_size);
1159 }
1160 return KERN_RESOURCE_SHORTAGE;
1161 }
1162 }
1163 }
1164
1165 /* OK, have memory and the list locked */
1166
1167 /* If we need it, get the thread list */
1168 if (type == PSET_THING_THREAD) {
1169 for (thread = (thread_t)queue_first(&threads);
1170 !queue_end(&threads, (queue_entry_t)thread);
1171 thread = (thread_t)queue_next(&thread->threads)) {
1172 #if defined(SECURE_KERNEL)
1173 if (thread->task != kernel_task) {
1174 #endif
1175 thread_reference_internal(thread);
1176 thread_list[actual_threads++] = thread;
1177 #if defined(SECURE_KERNEL)
1178 }
1179 #endif
1180 }
1181 }
1182 #if !CONFIG_MACF
1183 else {
1184 #endif
1185 /* get a list of the tasks */
1186 for (task = (task_t)queue_first(&tasks);
1187 !queue_end(&tasks, (queue_entry_t)task);
1188 task = (task_t)queue_next(&task->tasks)) {
1189 #if defined(SECURE_KERNEL)
1190 if (task != kernel_task) {
1191 #endif
1192 task_reference_internal(task);
1193 task_list[actual_tasks++] = task;
1194 #if defined(SECURE_KERNEL)
1195 }
1196 #endif
1197 }
1198 #if !CONFIG_MACF
1199 }
1200 #endif
1201
1202 lck_mtx_unlock(&tasks_threads_lock);
1203
1204 #if CONFIG_MACF
1205 unsigned int j, used;
1206
1207 /* for each task, make sure we are allowed to examine it */
1208 for (i = used = 0; i < actual_tasks; i++) {
1209 if (mac_task_check_expose_task(task_list[i])) {
1210 task_deallocate(task_list[i]);
1211 continue;
1212 }
1213 task_list[used++] = task_list[i];
1214 }
1215 actual_tasks = used;
1216 task_size_needed = actual_tasks * sizeof(void *);
1217
1218 if (type == PSET_THING_THREAD) {
1219 /* for each thread (if any), make sure it's task is in the allowed list */
1220 for (i = used = 0; i < actual_threads; i++) {
1221 boolean_t found_task = FALSE;
1222
1223 task = thread_list[i]->task;
1224 for (j = 0; j < actual_tasks; j++) {
1225 if (task_list[j] == task) {
1226 found_task = TRUE;
1227 break;
1228 }
1229 }
1230 if (found_task) {
1231 thread_list[used++] = thread_list[i];
1232 } else {
1233 thread_deallocate(thread_list[i]);
1234 }
1235 }
1236 actual_threads = used;
1237 thread_size_needed = actual_threads * sizeof(void *);
1238
1239 /* done with the task list */
1240 for (i = 0; i < actual_tasks; i++) {
1241 task_deallocate(task_list[i]);
1242 }
1243 kfree(task_list, task_size);
1244 task_size = 0;
1245 actual_tasks = 0;
1246 task_list = NULL;
1247 }
1248 #endif
1249
1250 if (type == PSET_THING_THREAD) {
1251 if (actual_threads == 0) {
1252 /* no threads available to return */
1253 assert(task_size == 0);
1254 if (thread_size != 0) {
1255 kfree(thread_list, thread_size);
1256 }
1257 *thing_list = NULL;
1258 *count = 0;
1259 return KERN_SUCCESS;
1260 }
1261 size_needed = actual_threads * sizeof(void *);
1262 size = thread_size;
1263 addr = thread_list;
1264 } else {
1265 if (actual_tasks == 0) {
1266 /* no tasks available to return */
1267 assert(thread_size == 0);
1268 if (task_size != 0) {
1269 kfree(task_list, task_size);
1270 }
1271 *thing_list = NULL;
1272 *count = 0;
1273 return KERN_SUCCESS;
1274 }
1275 size_needed = actual_tasks * sizeof(void *);
1276 size = task_size;
1277 addr = task_list;
1278 }
1279
1280 /* if we allocated too much, must copy */
1281 if (size_needed < size) {
1282 newaddr = kalloc(size_needed);
1283 if (newaddr == 0) {
1284 for (i = 0; i < actual_tasks; i++) {
1285 if (type == PSET_THING_THREAD) {
1286 thread_deallocate(thread_list[i]);
1287 } else {
1288 task_deallocate(task_list[i]);
1289 }
1290 }
1291 if (size) {
1292 kfree(addr, size);
1293 }
1294 return KERN_RESOURCE_SHORTAGE;
1295 }
1296
1297 bcopy((void *) addr, (void *) newaddr, size_needed);
1298 kfree(addr, size);
1299
1300 addr = newaddr;
1301 size = size_needed;
1302 }
1303
1304 *thing_list = (void **)addr;
1305 *count = (unsigned int)size / sizeof(void *);
1306
1307 return KERN_SUCCESS;
1308 }
1309
1310
1311 /*
1312 * processor_set_tasks:
1313 *
1314 * List all tasks in the processor set.
1315 */
1316 kern_return_t
1317 processor_set_tasks(
1318 processor_set_t pset,
1319 task_array_t *task_list,
1320 mach_msg_type_number_t *count)
1321 {
1322 kern_return_t ret;
1323 mach_msg_type_number_t i;
1324
1325 ret = processor_set_things(pset, (void **)task_list, count, PSET_THING_TASK);
1326 if (ret != KERN_SUCCESS) {
1327 return ret;
1328 }
1329
1330 /* do the conversion that Mig should handle */
1331 for (i = 0; i < *count; i++) {
1332 (*task_list)[i] = (task_t)convert_task_to_port((*task_list)[i]);
1333 }
1334 return KERN_SUCCESS;
1335 }
1336
1337 /*
1338 * processor_set_threads:
1339 *
1340 * List all threads in the processor set.
1341 */
1342 #if defined(SECURE_KERNEL)
1343 kern_return_t
1344 processor_set_threads(
1345 __unused processor_set_t pset,
1346 __unused thread_array_t *thread_list,
1347 __unused mach_msg_type_number_t *count)
1348 {
1349 return KERN_FAILURE;
1350 }
1351 #elif defined(CONFIG_EMBEDDED)
1352 kern_return_t
1353 processor_set_threads(
1354 __unused processor_set_t pset,
1355 __unused thread_array_t *thread_list,
1356 __unused mach_msg_type_number_t *count)
1357 {
1358 return KERN_NOT_SUPPORTED;
1359 }
1360 #else
1361 kern_return_t
1362 processor_set_threads(
1363 processor_set_t pset,
1364 thread_array_t *thread_list,
1365 mach_msg_type_number_t *count)
1366 {
1367 kern_return_t ret;
1368 mach_msg_type_number_t i;
1369
1370 ret = processor_set_things(pset, (void **)thread_list, count, PSET_THING_THREAD);
1371 if (ret != KERN_SUCCESS) {
1372 return ret;
1373 }
1374
1375 /* do the conversion that Mig should handle */
1376 for (i = 0; i < *count; i++) {
1377 (*thread_list)[i] = (thread_t)convert_thread_to_port((*thread_list)[i]);
1378 }
1379 return KERN_SUCCESS;
1380 }
1381 #endif
1382
1383 /*
1384 * processor_set_policy_control
1385 *
1386 * Controls the scheduling attributes governing the processor set.
1387 * Allows control of enabled policies, and per-policy base and limit
1388 * priorities.
1389 */
1390 kern_return_t
1391 processor_set_policy_control(
1392 __unused processor_set_t pset,
1393 __unused int flavor,
1394 __unused processor_set_info_t policy_info,
1395 __unused mach_msg_type_number_t count,
1396 __unused boolean_t change)
1397 {
1398 return KERN_INVALID_ARGUMENT;
1399 }
1400
1401 #undef pset_deallocate
1402 void pset_deallocate(processor_set_t pset);
1403 void
1404 pset_deallocate(
1405 __unused processor_set_t pset)
1406 {
1407 return;
1408 }
1409
1410 #undef pset_reference
1411 void pset_reference(processor_set_t pset);
1412 void
1413 pset_reference(
1414 __unused processor_set_t pset)
1415 {
1416 return;
1417 }
1418
1419 pset_cluster_type_t
1420 recommended_pset_type(thread_t thread)
1421 {
1422 (void)thread;
1423 return PSET_SMP;
1424 }
mirror of XNU