]> git.saurik.com Git - apple/xnu.git/blob - osfmk/kern/thread.c
projects / apple / xnu.git / blob
? search:


b9a7ae0ebe5a749c2d0e7902d507a0389069eadd
[apple/xnu.git] / osfmk / kern / thread.c
1 /*
2 * Copyright (c) 2000-2014 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_FREE_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56 /*
57 */
58 /*
59 * File: kern/thread.c
60 * Author: Avadis Tevanian, Jr., Michael Wayne Young, David Golub
61 * Date: 1986
62 *
63 * Thread management primitives implementation.
64 */
65 /*
66 * Copyright (c) 1993 The University of Utah and
67 * the Computer Systems Laboratory (CSL). All rights reserved.
68 *
69 * Permission to use, copy, modify and distribute this software and its
70 * documentation is hereby granted, provided that both the copyright
71 * notice and this permission notice appear in all copies of the
72 * software, derivative works or modified versions, and any portions
73 * thereof, and that both notices appear in supporting documentation.
74 *
75 * THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF THIS SOFTWARE IN ITS "AS
76 * IS" CONDITION. THE UNIVERSITY OF UTAH AND CSL DISCLAIM ANY LIABILITY OF
77 * ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
78 *
79 * CSL requests users of this software to return to csl-dist@cs.utah.edu any
80 * improvements that they make and grant CSL redistribution rights.
81 *
82 */
83
84 #include <mach/mach_types.h>
85 #include <mach/boolean.h>
86 #include <mach/policy.h>
87 #include <mach/thread_info.h>
88 #include <mach/thread_special_ports.h>
89 #include <mach/thread_status.h>
90 #include <mach/time_value.h>
91 #include <mach/vm_param.h>
92
93 #include <machine/thread.h>
94 #include <machine/pal_routines.h>
95 #include <machine/limits.h>
96
97 #include <kern/kern_types.h>
98 #include <kern/kalloc.h>
99 #include <kern/cpu_data.h>
100 #include <kern/counters.h>
101 #include <kern/extmod_statistics.h>
102 #include <kern/ipc_mig.h>
103 #include <kern/ipc_tt.h>
104 #include <kern/mach_param.h>
105 #include <kern/machine.h>
106 #include <kern/misc_protos.h>
107 #include <kern/processor.h>
108 #include <kern/queue.h>
109 #include <kern/sched.h>
110 #include <kern/sched_prim.h>
111 #include <kern/sync_lock.h>
112 #include <kern/syscall_subr.h>
113 #include <kern/task.h>
114 #include <kern/thread.h>
115 #include <kern/host.h>
116 #include <kern/zalloc.h>
117 #include <kern/assert.h>
118 #include <kern/exc_resource.h>
119 #include <kern/telemetry.h>
120 #if KPC
121 #include <kern/kpc.h>
122 #endif
123
124 #include <ipc/ipc_kmsg.h>
125 #include <ipc/ipc_port.h>
126 #include <bank/bank_types.h>
127
128 #include <vm/vm_kern.h>
129 #include <vm/vm_pageout.h>
130
131 #include <sys/kdebug.h>
132
133 #include <mach/sdt.h>
134
135 /*
136 * Exported interfaces
137 */
138 #include <mach/task_server.h>
139 #include <mach/thread_act_server.h>
140 #include <mach/mach_host_server.h>
141 #include <mach/host_priv_server.h>
142 #include <mach/mach_voucher_server.h>
143
144 static struct zone *thread_zone;
145 static lck_grp_attr_t thread_lck_grp_attr;
146 lck_attr_t thread_lck_attr;
147 lck_grp_t thread_lck_grp;
148
149 struct zone *thread_qos_override_zone;
150
151 decl_simple_lock_data(static,thread_stack_lock)
152 static queue_head_t thread_stack_queue;
153
154 decl_simple_lock_data(static,thread_terminate_lock)
155 static queue_head_t thread_terminate_queue;
156
157 static struct thread thread_template, init_thread;
158
159 static void sched_call_null(
160 int type,
161 thread_t thread);
162
163 #ifdef MACH_BSD
164 extern void proc_exit(void *);
165 extern uint64_t get_dispatchqueue_offset_from_proc(void *);
166 extern int proc_selfpid(void);
167 extern char * proc_name_address(void *p);
168 #endif /* MACH_BSD */
169
170 extern int disable_exc_resource;
171 extern int audio_active;
172 extern int debug_task;
173 int thread_max = CONFIG_THREAD_MAX; /* Max number of threads */
174 int task_threadmax = CONFIG_THREAD_MAX;
175
176 static uint64_t thread_unique_id = 100;
177
178 struct _thread_ledger_indices thread_ledgers = { -1 };
179 static ledger_template_t thread_ledger_template = NULL;
180 void init_thread_ledgers(void);
181 int task_disable_cpumon(task_t task);
182
183 #if CONFIG_JETSAM
184 void jetsam_on_ledger_cpulimit_exceeded(void);
185 #endif
186
187 /*
188 * Level (in terms of percentage of the limit) at which the CPU usage monitor triggers telemetry.
189 *
190 * (ie when any thread's CPU consumption exceeds 70% of the limit, start taking user
191 * stacktraces, aka micro-stackshots)
192 */
193 #define CPUMON_USTACKSHOTS_TRIGGER_DEFAULT_PCT 70
194
195 int cpumon_ustackshots_trigger_pct; /* Percentage. Level at which we start gathering telemetry. */
196 void __attribute__((noinline)) THIS_THREAD_IS_CONSUMING_TOO_MUCH_CPU__SENDING_EXC_RESOURCE(void);
197
198 /*
199 * The smallest interval over which we support limiting CPU consumption is 1ms
200 */
201 #define MINIMUM_CPULIMIT_INTERVAL_MS 1
202
203 void
204 thread_bootstrap(void)
205 {
206 /*
207 * Fill in a template thread for fast initialization.
208 */
209
210 thread_template.runq = PROCESSOR_NULL;
211
212 thread_template.ref_count = 2;
213
214 thread_template.reason = AST_NONE;
215 thread_template.at_safe_point = FALSE;
216 thread_template.wait_event = NO_EVENT64;
217 thread_template.wait_queue = WAIT_QUEUE_NULL;
218 thread_template.wait_result = THREAD_WAITING;
219 thread_template.options = THREAD_ABORTSAFE;
220 thread_template.state = TH_WAIT | TH_UNINT;
221 thread_template.wake_active = FALSE;
222 thread_template.continuation = THREAD_CONTINUE_NULL;
223 thread_template.parameter = NULL;
224
225 thread_template.importance = 0;
226 thread_template.sched_mode = TH_MODE_NONE;
227 thread_template.sched_flags = 0;
228 thread_template.saved_mode = TH_MODE_NONE;
229 thread_template.safe_release = 0;
230
231 thread_template.sfi_class = SFI_CLASS_UNSPECIFIED;
232 thread_template.sfi_wait_class = SFI_CLASS_UNSPECIFIED;
233
234 thread_template.active = 0;
235 thread_template.started = 0;
236 thread_template.static_param = 0;
237 thread_template.policy_reset = 0;
238
239 thread_template.priority = 0;
240 thread_template.sched_pri = 0;
241 thread_template.max_priority = 0;
242 thread_template.task_priority = 0;
243 thread_template.promotions = 0;
244 thread_template.pending_promoter_index = 0;
245 thread_template.pending_promoter[0] =
246 thread_template.pending_promoter[1] = NULL;
247 thread_template.rwlock_count = 0;
248
249 #if MACH_ASSERT
250 thread_template.SHARE_COUNT = 0;
251 thread_template.BG_COUNT = 0;
252 #endif /* MACH_ASSERT */
253
254 thread_template.realtime.deadline = UINT64_MAX;
255
256 thread_template.quantum_remaining = 0;
257 thread_template.last_run_time = 0;
258
259 thread_template.computation_metered = 0;
260 thread_template.computation_epoch = 0;
261
262 #if defined(CONFIG_SCHED_TIMESHARE_CORE)
263 thread_template.sched_stamp = 0;
264 thread_template.pri_shift = INT8_MAX;
265 thread_template.sched_usage = 0;
266 thread_template.cpu_usage = thread_template.cpu_delta = 0;
267 #endif
268 thread_template.c_switch = thread_template.p_switch = thread_template.ps_switch = 0;
269
270 thread_template.bound_processor = PROCESSOR_NULL;
271 thread_template.last_processor = PROCESSOR_NULL;
272
273 thread_template.sched_call = sched_call_null;
274
275 timer_init(&thread_template.user_timer);
276 timer_init(&thread_template.system_timer);
277 thread_template.user_timer_save = 0;
278 thread_template.system_timer_save = 0;
279 thread_template.vtimer_user_save = 0;
280 thread_template.vtimer_prof_save = 0;
281 thread_template.vtimer_rlim_save = 0;
282
283 thread_template.wait_sfi_begin_time = 0;
284
285 thread_template.wait_timer_is_set = FALSE;
286 thread_template.wait_timer_active = 0;
287
288 thread_template.depress_timer_active = 0;
289
290 thread_template.special_handler.handler = special_handler;
291 thread_template.special_handler.next = NULL;
292
293 thread_template.recover = (vm_offset_t)NULL;
294
295 thread_template.map = VM_MAP_NULL;
296
297 #if CONFIG_DTRACE
298 thread_template.t_dtrace_predcache = 0;
299 thread_template.t_dtrace_vtime = 0;
300 thread_template.t_dtrace_tracing = 0;
301 #endif /* CONFIG_DTRACE */
302
303 #if KPC
304 thread_template.kpc_buf = NULL;
305 #endif
306
307 #if HYPERVISOR
308 thread_template.hv_thread_target = NULL;
309 #endif /* HYPERVISOR */
310
311 thread_template.t_chud = 0;
312 thread_template.t_page_creation_count = 0;
313 thread_template.t_page_creation_time = 0;
314
315 thread_template.affinity_set = NULL;
316
317 thread_template.syscalls_unix = 0;
318 thread_template.syscalls_mach = 0;
319
320 thread_template.t_ledger = LEDGER_NULL;
321 thread_template.t_threadledger = LEDGER_NULL;
322 #ifdef CONFIG_BANK
323 thread_template.t_bankledger = LEDGER_NULL;
324 thread_template.t_deduct_bank_ledger_time = 0;
325 #endif
326
327 thread_template.requested_policy = default_task_requested_policy;
328 thread_template.effective_policy = default_task_effective_policy;
329 thread_template.pended_policy = default_task_pended_policy;
330
331 bzero(&thread_template.overrides, sizeof(thread_template.overrides));
332
333 thread_template.iotier_override = THROTTLE_LEVEL_NONE;
334 thread_template.thread_io_stats = NULL;
335 thread_template.thread_callout_interrupt_wakeups = thread_template.thread_callout_platform_idle_wakeups = 0;
336
337 thread_template.thread_timer_wakeups_bin_1 = thread_template.thread_timer_wakeups_bin_2 = 0;
338 thread_template.callout_woken_from_icontext = thread_template.callout_woken_from_platform_idle = 0;
339
340 thread_template.thread_tag = 0;
341
342 thread_template.ith_voucher_name = MACH_PORT_NULL;
343 thread_template.ith_voucher = IPC_VOUCHER_NULL;
344
345 init_thread = thread_template;
346 machine_set_current_thread(&init_thread);
347 }
348
349 extern boolean_t allow_qos_policy_set;
350
351 void
352 thread_init(void)
353 {
354 thread_zone = zinit(
355 sizeof(struct thread),
356 thread_max * sizeof(struct thread),
357 THREAD_CHUNK * sizeof(struct thread),
358 "threads");
359
360 thread_qos_override_zone = zinit(
361 sizeof(struct thread_qos_override),
362 4 * thread_max * sizeof(struct thread_qos_override),
363 PAGE_SIZE,
364 "thread qos override");
365 zone_change(thread_qos_override_zone, Z_EXPAND, TRUE);
366 zone_change(thread_qos_override_zone, Z_COLLECT, TRUE);
367 zone_change(thread_qos_override_zone, Z_CALLERACCT, FALSE);
368 zone_change(thread_qos_override_zone, Z_NOENCRYPT, TRUE);
369
370 lck_grp_attr_setdefault(&thread_lck_grp_attr);
371 lck_grp_init(&thread_lck_grp, "thread", &thread_lck_grp_attr);
372 lck_attr_setdefault(&thread_lck_attr);
373
374 stack_init();
375
376 /*
377 * Initialize any machine-dependent
378 * per-thread structures necessary.
379 */
380 machine_thread_init();
381
382 if (!PE_parse_boot_argn("cpumon_ustackshots_trigger_pct", &cpumon_ustackshots_trigger_pct,
383 sizeof (cpumon_ustackshots_trigger_pct))) {
384 cpumon_ustackshots_trigger_pct = CPUMON_USTACKSHOTS_TRIGGER_DEFAULT_PCT;
385 }
386
387 PE_parse_boot_argn("-qos-policy-allow", &allow_qos_policy_set, sizeof(allow_qos_policy_set));
388
389 init_thread_ledgers();
390 }
391
392 static void
393 thread_terminate_continue(void)
394 {
395 panic("thread_terminate_continue");
396 /*NOTREACHED*/
397 }
398
399 /*
400 * thread_terminate_self:
401 */
402 void
403 thread_terminate_self(void)
404 {
405 thread_t thread = current_thread();
406
407 task_t task;
408 spl_t s;
409 int threadcnt;
410
411 pal_thread_terminate_self(thread);
412
413 DTRACE_PROC(lwp__exit);
414
415 thread_mtx_lock(thread);
416
417 ipc_thread_disable(thread);
418
419 thread_mtx_unlock(thread);
420
421 s = splsched();
422 thread_lock(thread);
423
424 assert_thread_sched_count(thread);
425
426 /*
427 * Cancel priority depression, wait for concurrent expirations
428 * on other processors.
429 */
430 if (thread->sched_flags & TH_SFLAG_DEPRESSED_MASK) {
431 thread->sched_flags &= ~TH_SFLAG_DEPRESSED_MASK;
432
433 /* If our priority was low because of a depressed yield, restore it in case we block below */
434 set_sched_pri(thread, thread->priority);
435
436 if (timer_call_cancel(&thread->depress_timer))
437 thread->depress_timer_active--;
438 }
439
440 while (thread->depress_timer_active > 0) {
441 thread_unlock(thread);
442 splx(s);
443
444 delay(1);
445
446 s = splsched();
447 thread_lock(thread);
448 }
449
450 thread_sched_call(thread, NULL);
451
452 thread_unlock(thread);
453 splx(s);
454
455
456 thread_mtx_lock(thread);
457
458 thread_policy_reset(thread);
459
460 thread_mtx_unlock(thread);
461
462 task = thread->task;
463 uthread_cleanup(task, thread->uthread, task->bsd_info);
464 threadcnt = hw_atomic_sub(&task->active_thread_count, 1);
465
466 /*
467 * If we are the last thread to terminate and the task is
468 * associated with a BSD process, perform BSD process exit.
469 */
470 if (threadcnt == 0 && task->bsd_info != NULL)
471 proc_exit(task->bsd_info);
472
473 uthread_cred_free(thread->uthread);
474
475 s = splsched();
476 thread_lock(thread);
477
478 /*
479 * Cancel wait timer, and wait for
480 * concurrent expirations.
481 */
482 if (thread->wait_timer_is_set) {
483 thread->wait_timer_is_set = FALSE;
484
485 if (timer_call_cancel(&thread->wait_timer))
486 thread->wait_timer_active--;
487 }
488
489 while (thread->wait_timer_active > 0) {
490 thread_unlock(thread);
491 splx(s);
492
493 delay(1);
494
495 s = splsched();
496 thread_lock(thread);
497 }
498
499 /*
500 * If there is a reserved stack, release it.
501 */
502 if (thread->reserved_stack != 0) {
503 stack_free_reserved(thread);
504 thread->reserved_stack = 0;
505 }
506
507 /*
508 * Mark thread as terminating, and block.
509 */
510 thread->state |= TH_TERMINATE;
511 thread_mark_wait_locked(thread, THREAD_UNINT);
512 assert(thread->promotions == 0);
513 assert(thread->rwlock_count == 0);
514 thread_unlock(thread);
515 /* splsched */
516
517 thread_block((thread_continue_t)thread_terminate_continue);
518 /*NOTREACHED*/
519 }
520
521 void
522 thread_deallocate(
523 thread_t thread)
524 {
525 task_t task;
526
527 if (thread == THREAD_NULL)
528 return;
529
530 if (thread_deallocate_internal(thread) > 0)
531 return;
532
533 if(!(thread->state & TH_TERMINATE2))
534 panic("thread_deallocate: thread not properly terminated\n");
535
536 #if KPC
537 kpc_thread_destroy(thread);
538 #endif
539
540 ipc_thread_terminate(thread);
541
542 proc_thread_qos_deallocate(thread);
543
544 task = thread->task;
545
546 #ifdef MACH_BSD
547 {
548 void *ut = thread->uthread;
549
550 thread->uthread = NULL;
551 uthread_zone_free(ut);
552 }
553 #endif /* MACH_BSD */
554
555 if (thread->t_ledger)
556 ledger_dereference(thread->t_ledger);
557 if (thread->t_threadledger)
558 ledger_dereference(thread->t_threadledger);
559
560 if (IPC_VOUCHER_NULL != thread->ith_voucher)
561 ipc_voucher_release(thread->ith_voucher);
562
563 if (thread->thread_io_stats)
564 kfree(thread->thread_io_stats, sizeof(struct io_stat_info));
565
566 if (thread->kernel_stack != 0)
567 stack_free(thread);
568
569 lck_mtx_destroy(&thread->mutex, &thread_lck_grp);
570 machine_thread_destroy(thread);
571
572 task_deallocate(task);
573
574 zfree(thread_zone, thread);
575 }
576
577 /*
578 * thread_terminate_daemon:
579 *
580 * Perform final clean up for terminating threads.
581 */
582 static void
583 thread_terminate_daemon(void)
584 {
585 thread_t self, thread;
586 task_t task;
587
588 self = current_thread();
589 self->options |= TH_OPT_SYSTEM_CRITICAL;
590
591 (void)splsched();
592 simple_lock(&thread_terminate_lock);
593
594 while ((thread = (thread_t)dequeue_head(&thread_terminate_queue)) != THREAD_NULL) {
595 simple_unlock(&thread_terminate_lock);
596 (void)spllo();
597
598 assert(thread->SHARE_COUNT == 0);
599 assert(thread->BG_COUNT == 0);
600
601 task = thread->task;
602
603 task_lock(task);
604 task->total_user_time += timer_grab(&thread->user_timer);
605 if (thread->precise_user_kernel_time) {
606 task->total_system_time += timer_grab(&thread->system_timer);
607 } else {
608 task->total_user_time += timer_grab(&thread->system_timer);
609 }
610
611 task->c_switch += thread->c_switch;
612 task->p_switch += thread->p_switch;
613 task->ps_switch += thread->ps_switch;
614
615 task->syscalls_unix += thread->syscalls_unix;
616 task->syscalls_mach += thread->syscalls_mach;
617
618 task->task_timer_wakeups_bin_1 += thread->thread_timer_wakeups_bin_1;
619 task->task_timer_wakeups_bin_2 += thread->thread_timer_wakeups_bin_2;
620 task->task_gpu_ns += ml_gpu_stat(thread);
621
622 thread_update_qos_cpu_time(thread, FALSE);
623 queue_remove(&task->threads, thread, thread_t, task_threads);
624 task->thread_count--;
625
626 /*
627 * If the task is being halted, and there is only one thread
628 * left in the task after this one, then wakeup that thread.
629 */
630 if (task->thread_count == 1 && task->halting)
631 thread_wakeup((event_t)&task->halting);
632
633 task_unlock(task);
634
635 lck_mtx_lock(&tasks_threads_lock);
636 queue_remove(&threads, thread, thread_t, threads);
637 threads_count--;
638 lck_mtx_unlock(&tasks_threads_lock);
639
640 thread_deallocate(thread);
641
642 (void)splsched();
643 simple_lock(&thread_terminate_lock);
644 }
645
646 assert_wait((event_t)&thread_terminate_queue, THREAD_UNINT);
647 simple_unlock(&thread_terminate_lock);
648 /* splsched */
649
650 self->options &= ~TH_OPT_SYSTEM_CRITICAL;
651 thread_block((thread_continue_t)thread_terminate_daemon);
652 /*NOTREACHED*/
653 }
654
655 /*
656 * thread_terminate_enqueue:
657 *
658 * Enqueue a terminating thread for final disposition.
659 *
660 * Called at splsched.
661 */
662 void
663 thread_terminate_enqueue(
664 thread_t thread)
665 {
666 KERNEL_DEBUG_CONSTANT(TRACEDBG_CODE(DBG_TRACE_DATA, TRACE_DATA_THREAD_TERMINATE) | DBG_FUNC_NONE, thread->thread_id, 0, 0, 0, 0);
667
668 simple_lock(&thread_terminate_lock);
669 enqueue_tail(&thread_terminate_queue, (queue_entry_t)thread);
670 simple_unlock(&thread_terminate_lock);
671
672 thread_wakeup((event_t)&thread_terminate_queue);
673 }
674
675 /*
676 * thread_stack_daemon:
677 *
678 * Perform stack allocation as required due to
679 * invoke failures.
680 */
681 static void
682 thread_stack_daemon(void)
683 {
684 thread_t thread;
685 spl_t s;
686
687 s = splsched();
688 simple_lock(&thread_stack_lock);
689
690 while ((thread = (thread_t)dequeue_head(&thread_stack_queue)) != THREAD_NULL) {
691 simple_unlock(&thread_stack_lock);
692 splx(s);
693
694 /* allocate stack with interrupts enabled so that we can call into VM */
695 stack_alloc(thread);
696
697 s = splsched();
698 thread_lock(thread);
699 thread_setrun(thread, SCHED_PREEMPT | SCHED_TAILQ);
700 thread_unlock(thread);
701
702 simple_lock(&thread_stack_lock);
703 }
704
705 assert_wait((event_t)&thread_stack_queue, THREAD_UNINT);
706 simple_unlock(&thread_stack_lock);
707 splx(s);
708
709 thread_block((thread_continue_t)thread_stack_daemon);
710 /*NOTREACHED*/
711 }
712
713 /*
714 * thread_stack_enqueue:
715 *
716 * Enqueue a thread for stack allocation.
717 *
718 * Called at splsched.
719 */
720 void
721 thread_stack_enqueue(
722 thread_t thread)
723 {
724 simple_lock(&thread_stack_lock);
725 enqueue_tail(&thread_stack_queue, (queue_entry_t)thread);
726 simple_unlock(&thread_stack_lock);
727
728 thread_wakeup((event_t)&thread_stack_queue);
729 }
730
731 void
732 thread_daemon_init(void)
733 {
734 kern_return_t result;
735 thread_t thread = NULL;
736
737 simple_lock_init(&thread_terminate_lock, 0);
738 queue_init(&thread_terminate_queue);
739
740 result = kernel_thread_start_priority((thread_continue_t)thread_terminate_daemon, NULL, MINPRI_KERNEL, &thread);
741 if (result != KERN_SUCCESS)
742 panic("thread_daemon_init: thread_terminate_daemon");
743
744 thread_deallocate(thread);
745
746 simple_lock_init(&thread_stack_lock, 0);
747 queue_init(&thread_stack_queue);
748
749 result = kernel_thread_start_priority((thread_continue_t)thread_stack_daemon, NULL, BASEPRI_PREEMPT, &thread);
750 if (result != KERN_SUCCESS)
751 panic("thread_daemon_init: thread_stack_daemon");
752
753 thread_deallocate(thread);
754 }
755
756 /*
757 * Create a new thread.
758 * Doesn't start the thread running.
759 *
760 * Task and tasks_threads_lock are returned locked on success.
761 */
762 static kern_return_t
763 thread_create_internal(
764 task_t parent_task,
765 integer_t priority,
766 thread_continue_t continuation,
767 int options,
768 #define TH_OPTION_NONE 0x00
769 #define TH_OPTION_NOCRED 0x01
770 #define TH_OPTION_NOSUSP 0x02
771 thread_t *out_thread)
772 {
773 thread_t new_thread;
774 static thread_t first_thread;
775
776 /*
777 * Allocate a thread and initialize static fields
778 */
779 if (first_thread == THREAD_NULL)
780 new_thread = first_thread = current_thread();
781 else
782 new_thread = (thread_t)zalloc(thread_zone);
783 if (new_thread == THREAD_NULL)
784 return (KERN_RESOURCE_SHORTAGE);
785
786 if (new_thread != first_thread)
787 *new_thread = thread_template;
788
789 #ifdef MACH_BSD
790 new_thread->uthread = uthread_alloc(parent_task, new_thread, (options & TH_OPTION_NOCRED) != 0);
791 if (new_thread->uthread == NULL) {
792 zfree(thread_zone, new_thread);
793 return (KERN_RESOURCE_SHORTAGE);
794 }
795 #endif /* MACH_BSD */
796
797 if (machine_thread_create(new_thread, parent_task) != KERN_SUCCESS) {
798 #ifdef MACH_BSD
799 void *ut = new_thread->uthread;
800
801 new_thread->uthread = NULL;
802 /* cred free may not be necessary */
803 uthread_cleanup(parent_task, ut, parent_task->bsd_info);
804 uthread_cred_free(ut);
805 uthread_zone_free(ut);
806 #endif /* MACH_BSD */
807
808 zfree(thread_zone, new_thread);
809 return (KERN_FAILURE);
810 }
811
812 new_thread->task = parent_task;
813
814 thread_lock_init(new_thread);
815 wake_lock_init(new_thread);
816
817 lck_mtx_init(&new_thread->mutex, &thread_lck_grp, &thread_lck_attr);
818
819 ipc_thread_init(new_thread);
820
821 new_thread->continuation = continuation;
822
823 /* Allocate I/O Statistics structure */
824 new_thread->thread_io_stats = (io_stat_info_t)kalloc(sizeof(struct io_stat_info));
825 assert(new_thread->thread_io_stats != NULL);
826 bzero(new_thread->thread_io_stats, sizeof(struct io_stat_info));
827
828 #if CONFIG_IOSCHED
829 /* Clear out the I/O Scheduling info for AppleFSCompression */
830 new_thread->decmp_upl = NULL;
831 #endif /* CONFIG_IOSCHED */
832
833 lck_mtx_lock(&tasks_threads_lock);
834 task_lock(parent_task);
835
836 if ( !parent_task->active || parent_task->halting ||
837 ((options & TH_OPTION_NOSUSP) != 0 &&
838 parent_task->suspend_count > 0) ||
839 (parent_task->thread_count >= task_threadmax &&
840 parent_task != kernel_task) ) {
841 task_unlock(parent_task);
842 lck_mtx_unlock(&tasks_threads_lock);
843
844 #ifdef MACH_BSD
845 {
846 void *ut = new_thread->uthread;
847
848 new_thread->uthread = NULL;
849 uthread_cleanup(parent_task, ut, parent_task->bsd_info);
850 /* cred free may not be necessary */
851 uthread_cred_free(ut);
852 uthread_zone_free(ut);
853 }
854 #endif /* MACH_BSD */
855 ipc_thread_disable(new_thread);
856 ipc_thread_terminate(new_thread);
857 kfree(new_thread->thread_io_stats, sizeof(struct io_stat_info));
858 lck_mtx_destroy(&new_thread->mutex, &thread_lck_grp);
859 machine_thread_destroy(new_thread);
860 zfree(thread_zone, new_thread);
861 return (KERN_FAILURE);
862 }
863
864 /* New threads inherit any default state on the task */
865 machine_thread_inherit_taskwide(new_thread, parent_task);
866
867 task_reference_internal(parent_task);
868
869 if (new_thread->task->rusage_cpu_flags & TASK_RUSECPU_FLAGS_PERTHR_LIMIT) {
870 /*
871 * This task has a per-thread CPU limit; make sure this new thread
872 * gets its limit set too, before it gets out of the kernel.
873 */
874 set_astledger(new_thread);
875 }
876
877 /* Instantiate a thread ledger. Do not fail thread creation if ledger creation fails. */
878 if ((new_thread->t_threadledger = ledger_instantiate(thread_ledger_template,
879 LEDGER_CREATE_INACTIVE_ENTRIES)) != LEDGER_NULL) {
880
881 ledger_entry_setactive(new_thread->t_threadledger, thread_ledgers.cpu_time);
882 }
883
884 new_thread->cpu_time_last_qos = 0;
885 #ifdef CONFIG_BANK
886 new_thread->t_bankledger = LEDGER_NULL;
887 new_thread->t_deduct_bank_ledger_time = 0;
888 #endif
889
890 new_thread->t_ledger = new_thread->task->ledger;
891 if (new_thread->t_ledger)
892 ledger_reference(new_thread->t_ledger);
893
894 #if defined(CONFIG_SCHED_MULTIQ)
895 /* Cache the task's sched_group */
896 new_thread->sched_group = parent_task->sched_group;
897 #endif /* defined(CONFIG_SCHED_MULTIQ) */
898
899 /* Cache the task's map */
900 new_thread->map = parent_task->map;
901
902 timer_call_setup(&new_thread->wait_timer, thread_timer_expire, new_thread);
903 timer_call_setup(&new_thread->depress_timer, thread_depress_expire, new_thread);
904
905 #if CONFIG_COUNTERS
906 /*
907 * If parent task has any reservations, they need to be propagated to this
908 * thread.
909 */
910 new_thread->t_chud = (TASK_PMC_FLAG == (parent_task->t_chud & TASK_PMC_FLAG)) ?
911 THREAD_PMC_FLAG : 0U;
912 #endif
913 #if KPC
914 kpc_thread_create(new_thread);
915 #endif
916
917 /* Only need to update policies pushed from task to thread */
918 new_thread->requested_policy.bg_iotier = parent_task->effective_policy.bg_iotier;
919 new_thread->requested_policy.terminated = parent_task->effective_policy.terminated;
920
921 /* Set the thread's scheduling parameters */
922 new_thread->sched_mode = SCHED(initial_thread_sched_mode)(parent_task);
923 new_thread->sched_flags = 0;
924 new_thread->max_priority = parent_task->max_priority;
925 new_thread->task_priority = parent_task->priority;
926 new_thread->priority = (priority < 0)? parent_task->priority: priority;
927 if (new_thread->priority > new_thread->max_priority)
928 new_thread->priority = new_thread->max_priority;
929 new_thread->importance = new_thread->priority - new_thread->task_priority;
930 new_thread->saved_importance = new_thread->importance;
931
932 #if defined(CONFIG_SCHED_TIMESHARE_CORE)
933 new_thread->sched_stamp = sched_tick;
934 new_thread->pri_shift = sched_pri_shift;
935 #endif /* defined(CONFIG_SCHED_TIMESHARE_CORE) */
936
937 if (parent_task->max_priority <= MAXPRI_THROTTLE) {
938 sched_set_thread_throttled(new_thread, TRUE);
939 }
940
941 SCHED(compute_priority)(new_thread, FALSE);
942
943 thread_policy_create(new_thread);
944
945 /* Chain the thread onto the task's list */
946 queue_enter(&parent_task->threads, new_thread, thread_t, task_threads);
947 parent_task->thread_count++;
948
949 /* So terminating threads don't need to take the task lock to decrement */
950 hw_atomic_add(&parent_task->active_thread_count, 1);
951
952 /* Protected by the tasks_threads_lock */
953 new_thread->thread_id = ++thread_unique_id;
954
955 queue_enter(&threads, new_thread, thread_t, threads);
956 threads_count++;
957
958 new_thread->active = TRUE;
959
960 *out_thread = new_thread;
961
962 {
963 long dbg_arg1, dbg_arg2, dbg_arg3, dbg_arg4;
964
965 kdbg_trace_data(parent_task->bsd_info, &dbg_arg2);
966
967 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
968 TRACEDBG_CODE(DBG_TRACE_DATA, 1) | DBG_FUNC_NONE,
969 (vm_address_t)(uintptr_t)thread_tid(new_thread), dbg_arg2, 0, 0, 0);
970
971 kdbg_trace_string(parent_task->bsd_info,
972 &dbg_arg1, &dbg_arg2, &dbg_arg3, &dbg_arg4);
973
974 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
975 TRACEDBG_CODE(DBG_TRACE_STRING, 1) | DBG_FUNC_NONE,
976 dbg_arg1, dbg_arg2, dbg_arg3, dbg_arg4, 0);
977 }
978
979 DTRACE_PROC1(lwp__create, thread_t, *out_thread);
980
981 return (KERN_SUCCESS);
982 }
983
984 static kern_return_t
985 thread_create_internal2(
986 task_t task,
987 thread_t *new_thread,
988 boolean_t from_user)
989 {
990 kern_return_t result;
991 thread_t thread;
992
993 if (task == TASK_NULL || task == kernel_task)
994 return (KERN_INVALID_ARGUMENT);
995
996 result = thread_create_internal(task, -1, (thread_continue_t)thread_bootstrap_return, TH_OPTION_NONE, &thread);
997 if (result != KERN_SUCCESS)
998 return (result);
999
1000 thread->user_stop_count = 1;
1001 thread_hold(thread);
1002 if (task->suspend_count > 0)
1003 thread_hold(thread);
1004
1005 if (from_user)
1006 extmod_statistics_incr_thread_create(task);
1007
1008 task_unlock(task);
1009 lck_mtx_unlock(&tasks_threads_lock);
1010
1011 *new_thread = thread;
1012
1013 return (KERN_SUCCESS);
1014 }
1015
1016 /* No prototype, since task_server.h has the _from_user version if KERNEL_SERVER */
1017 kern_return_t
1018 thread_create(
1019 task_t task,
1020 thread_t *new_thread);
1021
1022 kern_return_t
1023 thread_create(
1024 task_t task,
1025 thread_t *new_thread)
1026 {
1027 return thread_create_internal2(task, new_thread, FALSE);
1028 }
1029
1030 kern_return_t
1031 thread_create_from_user(
1032 task_t task,
1033 thread_t *new_thread)
1034 {
1035 return thread_create_internal2(task, new_thread, TRUE);
1036 }
1037
1038 static kern_return_t
1039 thread_create_running_internal2(
1040 register task_t task,
1041 int flavor,
1042 thread_state_t new_state,
1043 mach_msg_type_number_t new_state_count,
1044 thread_t *new_thread,
1045 boolean_t from_user)
1046 {
1047 register kern_return_t result;
1048 thread_t thread;
1049
1050 if (task == TASK_NULL || task == kernel_task)
1051 return (KERN_INVALID_ARGUMENT);
1052
1053 result = thread_create_internal(task, -1, (thread_continue_t)thread_bootstrap_return, TH_OPTION_NONE, &thread);
1054 if (result != KERN_SUCCESS)
1055 return (result);
1056
1057 result = machine_thread_set_state(
1058 thread, flavor, new_state, new_state_count);
1059 if (result != KERN_SUCCESS) {
1060 task_unlock(task);
1061 lck_mtx_unlock(&tasks_threads_lock);
1062
1063 thread_terminate(thread);
1064 thread_deallocate(thread);
1065 return (result);
1066 }
1067
1068 thread_mtx_lock(thread);
1069 thread_start_internal(thread);
1070 thread_mtx_unlock(thread);
1071
1072 if (from_user)
1073 extmod_statistics_incr_thread_create(task);
1074
1075 task_unlock(task);
1076 lck_mtx_unlock(&tasks_threads_lock);
1077
1078 *new_thread = thread;
1079
1080 return (result);
1081 }
1082
1083 /* Prototype, see justification above */
1084 kern_return_t
1085 thread_create_running(
1086 register task_t task,
1087 int flavor,
1088 thread_state_t new_state,
1089 mach_msg_type_number_t new_state_count,
1090 thread_t *new_thread);
1091
1092 kern_return_t
1093 thread_create_running(
1094 register task_t task,
1095 int flavor,
1096 thread_state_t new_state,
1097 mach_msg_type_number_t new_state_count,
1098 thread_t *new_thread)
1099 {
1100 return thread_create_running_internal2(
1101 task, flavor, new_state, new_state_count,
1102 new_thread, FALSE);
1103 }
1104
1105 kern_return_t
1106 thread_create_running_from_user(
1107 register task_t task,
1108 int flavor,
1109 thread_state_t new_state,
1110 mach_msg_type_number_t new_state_count,
1111 thread_t *new_thread)
1112 {
1113 return thread_create_running_internal2(
1114 task, flavor, new_state, new_state_count,
1115 new_thread, TRUE);
1116 }
1117
1118 kern_return_t
1119 thread_create_workq(
1120 task_t task,
1121 thread_continue_t thread_return,
1122 thread_t *new_thread)
1123 {
1124 kern_return_t result;
1125 thread_t thread;
1126
1127 if (task == TASK_NULL || task == kernel_task)
1128 return (KERN_INVALID_ARGUMENT);
1129
1130 result = thread_create_internal(task, -1, thread_return, TH_OPTION_NOCRED | TH_OPTION_NOSUSP, &thread);
1131 if (result != KERN_SUCCESS)
1132 return (result);
1133
1134 thread->user_stop_count = 1;
1135 thread_hold(thread);
1136 if (task->suspend_count > 0)
1137 thread_hold(thread);
1138
1139 task_unlock(task);
1140 lck_mtx_unlock(&tasks_threads_lock);
1141
1142 *new_thread = thread;
1143
1144 return (KERN_SUCCESS);
1145 }
1146
1147 /*
1148 * kernel_thread_create:
1149 *
1150 * Create a thread in the kernel task
1151 * to execute in kernel context.
1152 */
1153 kern_return_t
1154 kernel_thread_create(
1155 thread_continue_t continuation,
1156 void *parameter,
1157 integer_t priority,
1158 thread_t *new_thread)
1159 {
1160 kern_return_t result;
1161 thread_t thread;
1162 task_t task = kernel_task;
1163
1164 result = thread_create_internal(task, priority, continuation, TH_OPTION_NONE, &thread);
1165 if (result != KERN_SUCCESS)
1166 return (result);
1167
1168 task_unlock(task);
1169 lck_mtx_unlock(&tasks_threads_lock);
1170
1171 stack_alloc(thread);
1172 assert(thread->kernel_stack != 0);
1173 thread->reserved_stack = thread->kernel_stack;
1174
1175 thread->parameter = parameter;
1176
1177 if(debug_task & 1)
1178 kprintf("kernel_thread_create: thread = %p continuation = %p\n", thread, continuation);
1179 *new_thread = thread;
1180
1181 return (result);
1182 }
1183
1184 kern_return_t
1185 kernel_thread_start_priority(
1186 thread_continue_t continuation,
1187 void *parameter,
1188 integer_t priority,
1189 thread_t *new_thread)
1190 {
1191 kern_return_t result;
1192 thread_t thread;
1193
1194 result = kernel_thread_create(continuation, parameter, priority, &thread);
1195 if (result != KERN_SUCCESS)
1196 return (result);
1197
1198 *new_thread = thread;
1199
1200 thread_mtx_lock(thread);
1201 thread_start_internal(thread);
1202 thread_mtx_unlock(thread);
1203
1204 return (result);
1205 }
1206
1207 kern_return_t
1208 kernel_thread_start(
1209 thread_continue_t continuation,
1210 void *parameter,
1211 thread_t *new_thread)
1212 {
1213 return kernel_thread_start_priority(continuation, parameter, -1, new_thread);
1214 }
1215
1216
1217 kern_return_t
1218 thread_info_internal(
1219 register thread_t thread,
1220 thread_flavor_t flavor,
1221 thread_info_t thread_info_out, /* ptr to OUT array */
1222 mach_msg_type_number_t *thread_info_count) /*IN/OUT*/
1223 {
1224 int state, flags;
1225 spl_t s;
1226
1227 if (thread == THREAD_NULL)
1228 return (KERN_INVALID_ARGUMENT);
1229
1230 if (flavor == THREAD_BASIC_INFO) {
1231 register thread_basic_info_t basic_info;
1232
1233 if (*thread_info_count < THREAD_BASIC_INFO_COUNT)
1234 return (KERN_INVALID_ARGUMENT);
1235
1236 basic_info = (thread_basic_info_t) thread_info_out;
1237
1238 s = splsched();
1239 thread_lock(thread);
1240
1241 /* fill in info */
1242
1243 thread_read_times(thread, &basic_info->user_time,
1244 &basic_info->system_time);
1245
1246 /*
1247 * Update lazy-evaluated scheduler info because someone wants it.
1248 */
1249 if (SCHED(can_update_priority)(thread))
1250 SCHED(update_priority)(thread);
1251
1252 basic_info->sleep_time = 0;
1253
1254 /*
1255 * To calculate cpu_usage, first correct for timer rate,
1256 * then for 5/8 ageing. The correction factor [3/5] is
1257 * (1/(5/8) - 1).
1258 */
1259 basic_info->cpu_usage = 0;
1260 #if defined(CONFIG_SCHED_TIMESHARE_CORE)
1261 if (sched_tick_interval) {
1262 basic_info->cpu_usage = (integer_t)(((uint64_t)thread->cpu_usage
1263 * TH_USAGE_SCALE) / sched_tick_interval);
1264 basic_info->cpu_usage = (basic_info->cpu_usage * 3) / 5;
1265 }
1266 #endif
1267
1268 if (basic_info->cpu_usage > TH_USAGE_SCALE)
1269 basic_info->cpu_usage = TH_USAGE_SCALE;
1270
1271 basic_info->policy = ((thread->sched_mode == TH_MODE_TIMESHARE)?
1272 POLICY_TIMESHARE: POLICY_RR);
1273
1274 flags = 0;
1275 if (thread->options & TH_OPT_IDLE_THREAD)
1276 flags |= TH_FLAGS_IDLE;
1277
1278 if (!thread->kernel_stack)
1279 flags |= TH_FLAGS_SWAPPED;
1280
1281 state = 0;
1282 if (thread->state & TH_TERMINATE)
1283 state = TH_STATE_HALTED;
1284 else
1285 if (thread->state & TH_RUN)
1286 state = TH_STATE_RUNNING;
1287 else
1288 if (thread->state & TH_UNINT)
1289 state = TH_STATE_UNINTERRUPTIBLE;
1290 else
1291 if (thread->state & TH_SUSP)
1292 state = TH_STATE_STOPPED;
1293 else
1294 if (thread->state & TH_WAIT)
1295 state = TH_STATE_WAITING;
1296
1297 basic_info->run_state = state;
1298 basic_info->flags = flags;
1299
1300 basic_info->suspend_count = thread->user_stop_count;
1301
1302 thread_unlock(thread);
1303 splx(s);
1304
1305 *thread_info_count = THREAD_BASIC_INFO_COUNT;
1306
1307 return (KERN_SUCCESS);
1308 }
1309 else
1310 if (flavor == THREAD_IDENTIFIER_INFO) {
1311 register thread_identifier_info_t identifier_info;
1312
1313 if (*thread_info_count < THREAD_IDENTIFIER_INFO_COUNT)
1314 return (KERN_INVALID_ARGUMENT);
1315
1316 identifier_info = (thread_identifier_info_t) thread_info_out;
1317
1318 s = splsched();
1319 thread_lock(thread);
1320
1321 identifier_info->thread_id = thread->thread_id;
1322 identifier_info->thread_handle = thread->machine.cthread_self;
1323 if(thread->task->bsd_info) {
1324 identifier_info->dispatch_qaddr = identifier_info->thread_handle + get_dispatchqueue_offset_from_proc(thread->task->bsd_info);
1325 } else {
1326 thread_unlock(thread);
1327 splx(s);
1328 return KERN_INVALID_ARGUMENT;
1329 }
1330
1331 thread_unlock(thread);
1332 splx(s);
1333 return KERN_SUCCESS;
1334 }
1335 else
1336 if (flavor == THREAD_SCHED_TIMESHARE_INFO) {
1337 policy_timeshare_info_t ts_info;
1338
1339 if (*thread_info_count < POLICY_TIMESHARE_INFO_COUNT)
1340 return (KERN_INVALID_ARGUMENT);
1341
1342 ts_info = (policy_timeshare_info_t)thread_info_out;
1343
1344 s = splsched();
1345 thread_lock(thread);
1346
1347 if (thread->sched_mode != TH_MODE_TIMESHARE) {
1348 thread_unlock(thread);
1349 splx(s);
1350
1351 return (KERN_INVALID_POLICY);
1352 }
1353
1354 ts_info->depressed = (thread->sched_flags & TH_SFLAG_DEPRESSED_MASK) != 0;
1355 if (ts_info->depressed) {
1356 ts_info->base_priority = DEPRESSPRI;
1357 ts_info->depress_priority = thread->priority;
1358 }
1359 else {
1360 ts_info->base_priority = thread->priority;
1361 ts_info->depress_priority = -1;
1362 }
1363
1364 ts_info->cur_priority = thread->sched_pri;
1365 ts_info->max_priority = thread->max_priority;
1366
1367 thread_unlock(thread);
1368 splx(s);
1369
1370 *thread_info_count = POLICY_TIMESHARE_INFO_COUNT;
1371
1372 return (KERN_SUCCESS);
1373 }
1374 else
1375 if (flavor == THREAD_SCHED_FIFO_INFO) {
1376 if (*thread_info_count < POLICY_FIFO_INFO_COUNT)
1377 return (KERN_INVALID_ARGUMENT);
1378
1379 return (KERN_INVALID_POLICY);
1380 }
1381 else
1382 if (flavor == THREAD_SCHED_RR_INFO) {
1383 policy_rr_info_t rr_info;
1384 uint32_t quantum_time;
1385 uint64_t quantum_ns;
1386
1387 if (*thread_info_count < POLICY_RR_INFO_COUNT)
1388 return (KERN_INVALID_ARGUMENT);
1389
1390 rr_info = (policy_rr_info_t) thread_info_out;
1391
1392 s = splsched();
1393 thread_lock(thread);
1394
1395 if (thread->sched_mode == TH_MODE_TIMESHARE) {
1396 thread_unlock(thread);
1397 splx(s);
1398
1399 return (KERN_INVALID_POLICY);
1400 }
1401
1402 rr_info->depressed = (thread->sched_flags & TH_SFLAG_DEPRESSED_MASK) != 0;
1403 if (rr_info->depressed) {
1404 rr_info->base_priority = DEPRESSPRI;
1405 rr_info->depress_priority = thread->priority;
1406 }
1407 else {
1408 rr_info->base_priority = thread->priority;
1409 rr_info->depress_priority = -1;
1410 }
1411
1412 quantum_time = SCHED(initial_quantum_size)(THREAD_NULL);
1413 absolutetime_to_nanoseconds(quantum_time, &quantum_ns);
1414
1415 rr_info->max_priority = thread->max_priority;
1416 rr_info->quantum = (uint32_t)(quantum_ns / 1000 / 1000);
1417
1418 thread_unlock(thread);
1419 splx(s);
1420
1421 *thread_info_count = POLICY_RR_INFO_COUNT;
1422
1423 return (KERN_SUCCESS);
1424 }
1425
1426 return (KERN_INVALID_ARGUMENT);
1427 }
1428
1429 void
1430 thread_read_times(
1431 thread_t thread,
1432 time_value_t *user_time,
1433 time_value_t *system_time)
1434 {
1435 clock_sec_t secs;
1436 clock_usec_t usecs;
1437 uint64_t tval_user, tval_system;
1438
1439 tval_user = timer_grab(&thread->user_timer);
1440 tval_system = timer_grab(&thread->system_timer);
1441
1442 if (thread->precise_user_kernel_time) {
1443 absolutetime_to_microtime(tval_user, &secs, &usecs);
1444 user_time->seconds = (typeof(user_time->seconds))secs;
1445 user_time->microseconds = usecs;
1446
1447 absolutetime_to_microtime(tval_system, &secs, &usecs);
1448 system_time->seconds = (typeof(system_time->seconds))secs;
1449 system_time->microseconds = usecs;
1450 } else {
1451 /* system_timer may represent either sys or user */
1452 tval_user += tval_system;
1453 absolutetime_to_microtime(tval_user, &secs, &usecs);
1454 user_time->seconds = (typeof(user_time->seconds))secs;
1455 user_time->microseconds = usecs;
1456
1457 system_time->seconds = 0;
1458 system_time->microseconds = 0;
1459 }
1460 }
1461
1462 uint64_t thread_get_runtime_self(void)
1463 {
1464 boolean_t interrupt_state;
1465 uint64_t runtime;
1466 thread_t thread = NULL;
1467 processor_t processor = NULL;
1468
1469 thread = current_thread();
1470
1471 /* Not interrupt safe, as the scheduler may otherwise update timer values underneath us */
1472 interrupt_state = ml_set_interrupts_enabled(FALSE);
1473 processor = current_processor();
1474 timer_switch(PROCESSOR_DATA(processor, thread_timer), mach_absolute_time(), PROCESSOR_DATA(processor, thread_timer));
1475 runtime = (timer_grab(&thread->user_timer) + timer_grab(&thread->system_timer));
1476 ml_set_interrupts_enabled(interrupt_state);
1477
1478 return runtime;
1479 }
1480
1481 kern_return_t
1482 thread_assign(
1483 __unused thread_t thread,
1484 __unused processor_set_t new_pset)
1485 {
1486 return (KERN_FAILURE);
1487 }
1488
1489 /*
1490 * thread_assign_default:
1491 *
1492 * Special version of thread_assign for assigning threads to default
1493 * processor set.
1494 */
1495 kern_return_t
1496 thread_assign_default(
1497 thread_t thread)
1498 {
1499 return (thread_assign(thread, &pset0));
1500 }
1501
1502 /*
1503 * thread_get_assignment
1504 *
1505 * Return current assignment for this thread.
1506 */
1507 kern_return_t
1508 thread_get_assignment(
1509 thread_t thread,
1510 processor_set_t *pset)
1511 {
1512 if (thread == NULL)
1513 return (KERN_INVALID_ARGUMENT);
1514
1515 *pset = &pset0;
1516
1517 return (KERN_SUCCESS);
1518 }
1519
1520 /*
1521 * thread_wire_internal:
1522 *
1523 * Specify that the target thread must always be able
1524 * to run and to allocate memory.
1525 */
1526 kern_return_t
1527 thread_wire_internal(
1528 host_priv_t host_priv,
1529 thread_t thread,
1530 boolean_t wired,
1531 boolean_t *prev_state)
1532 {
1533 if (host_priv == NULL || thread != current_thread())
1534 return (KERN_INVALID_ARGUMENT);
1535
1536 assert(host_priv == &realhost);
1537
1538 if (prev_state)
1539 *prev_state = (thread->options & TH_OPT_VMPRIV) != 0;
1540
1541 if (wired) {
1542 if (!(thread->options & TH_OPT_VMPRIV))
1543 vm_page_free_reserve(1); /* XXX */
1544 thread->options |= TH_OPT_VMPRIV;
1545 }
1546 else {
1547 if (thread->options & TH_OPT_VMPRIV)
1548 vm_page_free_reserve(-1); /* XXX */
1549 thread->options &= ~TH_OPT_VMPRIV;
1550 }
1551
1552 return (KERN_SUCCESS);
1553 }
1554
1555
1556 /*
1557 * thread_wire:
1558 *
1559 * User-api wrapper for thread_wire_internal()
1560 */
1561 kern_return_t
1562 thread_wire(
1563 host_priv_t host_priv,
1564 thread_t thread,
1565 boolean_t wired)
1566 {
1567 return (thread_wire_internal(host_priv, thread, wired, NULL));
1568 }
1569
1570
1571 boolean_t
1572 set_vm_privilege(boolean_t privileged)
1573 {
1574 boolean_t was_vmpriv;
1575
1576 if (current_thread()->options & TH_OPT_VMPRIV)
1577 was_vmpriv = TRUE;
1578 else
1579 was_vmpriv = FALSE;
1580
1581 if (privileged != FALSE)
1582 current_thread()->options |= TH_OPT_VMPRIV;
1583 else
1584 current_thread()->options &= ~TH_OPT_VMPRIV;
1585
1586 return (was_vmpriv);
1587 }
1588
1589
1590 /*
1591 * XXX assuming current thread only, for now...
1592 */
1593 void
1594 thread_guard_violation(thread_t thread, unsigned type)
1595 {
1596 assert(thread == current_thread());
1597
1598 spl_t s = splsched();
1599 /*
1600 * Use the saved state area of the thread structure
1601 * to store all info required to handle the AST when
1602 * returning to userspace
1603 */
1604 thread->guard_exc_info.type = type;
1605 thread_ast_set(thread, AST_GUARD);
1606 ast_propagate(thread->ast);
1607
1608 splx(s);
1609 }
1610
1611 /*
1612 * guard_ast:
1613 *
1614 * Handle AST_GUARD for a thread. This routine looks at the
1615 * state saved in the thread structure to determine the cause
1616 * of this exception. Based on this value, it invokes the
1617 * appropriate routine which determines other exception related
1618 * info and raises the exception.
1619 */
1620 void
1621 guard_ast(thread_t thread)
1622 {
1623 if (thread->guard_exc_info.type == GUARD_TYPE_MACH_PORT)
1624 mach_port_guard_ast(thread);
1625 else
1626 fd_guard_ast(thread);
1627 }
1628
1629 static void
1630 thread_cputime_callback(int warning, __unused const void *arg0, __unused const void *arg1)
1631 {
1632 if (warning == LEDGER_WARNING_ROSE_ABOVE) {
1633 #if CONFIG_TELEMETRY
1634 /*
1635 * This thread is in danger of violating the CPU usage monitor. Enable telemetry
1636 * on the entire task so there are micro-stackshots available if and when
1637 * EXC_RESOURCE is triggered. We could have chosen to enable micro-stackshots
1638 * for this thread only; but now that this task is suspect, knowing what all of
1639 * its threads are up to will be useful.
1640 */
1641 telemetry_task_ctl(current_task(), TF_CPUMON_WARNING, 1);
1642 #endif
1643 return;
1644 }
1645
1646 #if CONFIG_TELEMETRY
1647 /*
1648 * If the balance has dipped below the warning level (LEDGER_WARNING_DIPPED_BELOW) or
1649 * exceeded the limit, turn telemetry off for the task.
1650 */
1651 telemetry_task_ctl(current_task(), TF_CPUMON_WARNING, 0);
1652 #endif
1653
1654 if (warning == 0) {
1655 THIS_THREAD_IS_CONSUMING_TOO_MUCH_CPU__SENDING_EXC_RESOURCE();
1656 }
1657 }
1658
1659 void __attribute__((noinline))
1660 THIS_THREAD_IS_CONSUMING_TOO_MUCH_CPU__SENDING_EXC_RESOURCE(void)
1661 {
1662 int pid = 0;
1663 task_t task = current_task();
1664 thread_t thread = current_thread();
1665 uint64_t tid = thread->thread_id;
1666 char *procname = (char *) "unknown";
1667 time_value_t thread_total_time = {0, 0};
1668 time_value_t thread_system_time;
1669 time_value_t thread_user_time;
1670 int action;
1671 uint8_t percentage;
1672 uint32_t limit_percent;
1673 uint32_t usage_percent;
1674 uint32_t interval_sec;
1675 uint64_t interval_ns;
1676 uint64_t balance_ns;
1677 boolean_t fatal = FALSE;
1678
1679 mach_exception_data_type_t code[EXCEPTION_CODE_MAX];
1680 struct ledger_entry_info lei;
1681
1682 assert(thread->t_threadledger != LEDGER_NULL);
1683
1684 /*
1685 * Now that a thread has tripped the monitor, disable it for the entire task.
1686 */
1687 task_lock(task);
1688 if ((task->rusage_cpu_flags & TASK_RUSECPU_FLAGS_PERTHR_LIMIT) == 0) {
1689 /*
1690 * The CPU usage monitor has been disabled on our task, so some other
1691 * thread must have gotten here first. We only send one exception per
1692 * task lifetime, so there's nothing left for us to do here.
1693 */
1694 task_unlock(task);
1695 return;
1696 }
1697 if (task->rusage_cpu_flags & TASK_RUSECPU_FLAGS_FATAL_CPUMON) {
1698 fatal = TRUE;
1699 }
1700 task_disable_cpumon(task);
1701 task_unlock(task);
1702
1703 #ifdef MACH_BSD
1704 pid = proc_selfpid();
1705 if (task->bsd_info != NULL)
1706 procname = proc_name_address(task->bsd_info);
1707 #endif
1708
1709 thread_get_cpulimit(&action, &percentage, &interval_ns);
1710
1711 interval_sec = (uint32_t)(interval_ns / NSEC_PER_SEC);
1712
1713 thread_read_times(thread, &thread_user_time, &thread_system_time);
1714 time_value_add(&thread_total_time, &thread_user_time);
1715 time_value_add(&thread_total_time, &thread_system_time);
1716
1717 ledger_get_entry_info(thread->t_threadledger, thread_ledgers.cpu_time, &lei);
1718
1719 absolutetime_to_nanoseconds(lei.lei_balance, &balance_ns);
1720 usage_percent = (uint32_t) ((balance_ns * 100ULL) / lei.lei_last_refill);
1721
1722 /* Show refill period in the same units as balance, limit, etc */
1723 nanoseconds_to_absolutetime(lei.lei_refill_period, &lei.lei_refill_period);
1724
1725 limit_percent = (uint32_t) ((lei.lei_limit * 100ULL) / lei.lei_refill_period);
1726
1727 /* TODO: show task total runtime as well? see TASK_ABSOLUTETIME_INFO */
1728
1729 if (disable_exc_resource) {
1730 printf("process %s[%d] thread %llu caught burning CPU!; EXC_RESOURCE "
1731 "supressed by a boot-arg\n", procname, pid, tid);
1732 return;
1733 }
1734
1735 if (audio_active) {
1736 printf("process %s[%d] thread %llu caught burning CPU!; EXC_RESOURCE "
1737 "supressed due to audio playback\n", procname, pid, tid);
1738 return;
1739 }
1740 printf("process %s[%d] thread %llu caught burning CPU! "
1741 "It used more than %d%% CPU (Actual recent usage: %d%%) over %d seconds. "
1742 "thread lifetime cpu usage %d.%06d seconds, (%d.%06d user, %d.%06d system) "
1743 "ledger info: balance: %lld credit: %lld debit: %lld limit: %llu (%d%%) "
1744 "period: %llu time since last refill (ns): %llu %s\n",
1745 procname, pid, tid,
1746 percentage, usage_percent, interval_sec,
1747 thread_total_time.seconds, thread_total_time.microseconds,
1748 thread_user_time.seconds, thread_user_time.microseconds,
1749 thread_system_time.seconds, thread_system_time.microseconds,
1750 lei.lei_balance,
1751 lei.lei_credit, lei.lei_debit,
1752 lei.lei_limit, limit_percent,
1753 lei.lei_refill_period, lei.lei_last_refill,
1754 (fatal ? "[fatal violation]" : ""));
1755
1756
1757 code[0] = code[1] = 0;
1758 EXC_RESOURCE_ENCODE_TYPE(code[0], RESOURCE_TYPE_CPU);
1759 if (fatal) {
1760 EXC_RESOURCE_ENCODE_FLAVOR(code[0], FLAVOR_CPU_MONITOR_FATAL);
1761 }else {
1762 EXC_RESOURCE_ENCODE_FLAVOR(code[0], FLAVOR_CPU_MONITOR);
1763 }
1764 EXC_RESOURCE_CPUMONITOR_ENCODE_INTERVAL(code[0], interval_sec);
1765 EXC_RESOURCE_CPUMONITOR_ENCODE_PERCENTAGE(code[0], limit_percent);
1766 EXC_RESOURCE_CPUMONITOR_ENCODE_PERCENTAGE(code[1], usage_percent);
1767 exception_triage(EXC_RESOURCE, code, EXCEPTION_CODE_MAX);
1768
1769 if (fatal) {
1770 #if CONFIG_JETSAM
1771 jetsam_on_ledger_cpulimit_exceeded();
1772 #else
1773 task_terminate_internal(task);
1774 #endif
1775 }
1776 }
1777
1778 #define UPDATE_IO_STATS(info, size) \
1779 { \
1780 info.count++; \
1781 info.size += size; \
1782 }
1783
1784 #define UPDATE_IO_STATS_ATOMIC(info, size) \
1785 { \
1786 OSIncrementAtomic64((SInt64 *)&(info.count)); \
1787 OSAddAtomic64(size, (SInt64 *)&(info.size)); \
1788 }
1789
1790 void thread_update_io_stats(thread_t thread, int size, int io_flags)
1791 {
1792 int io_tier;
1793
1794 if (thread->thread_io_stats == NULL || thread->task->task_io_stats == NULL)
1795 return;
1796
1797 if (io_flags & DKIO_READ) {
1798 UPDATE_IO_STATS(thread->thread_io_stats->disk_reads, size);
1799 UPDATE_IO_STATS_ATOMIC(thread->task->task_io_stats->disk_reads, size);
1800 }
1801
1802 if (io_flags & DKIO_META) {
1803 UPDATE_IO_STATS(thread->thread_io_stats->metadata, size);
1804 UPDATE_IO_STATS_ATOMIC(thread->task->task_io_stats->metadata, size);
1805 }
1806
1807 if (io_flags & DKIO_PAGING) {
1808 UPDATE_IO_STATS(thread->thread_io_stats->paging, size);
1809 UPDATE_IO_STATS_ATOMIC(thread->task->task_io_stats->paging, size);
1810 }
1811
1812 io_tier = ((io_flags & DKIO_TIER_MASK) >> DKIO_TIER_SHIFT);
1813 assert (io_tier < IO_NUM_PRIORITIES);
1814
1815 UPDATE_IO_STATS(thread->thread_io_stats->io_priority[io_tier], size);
1816 UPDATE_IO_STATS_ATOMIC(thread->task->task_io_stats->io_priority[io_tier], size);
1817
1818 /* Update Total I/O Counts */
1819 UPDATE_IO_STATS(thread->thread_io_stats->total_io, size);
1820 UPDATE_IO_STATS_ATOMIC(thread->task->task_io_stats->total_io, size);
1821
1822 }
1823
1824 void
1825 init_thread_ledgers(void) {
1826 ledger_template_t t;
1827 int idx;
1828
1829 assert(thread_ledger_template == NULL);
1830
1831 if ((t = ledger_template_create("Per-thread ledger")) == NULL)
1832 panic("couldn't create thread ledger template");
1833
1834 if ((idx = ledger_entry_add(t, "cpu_time", "sched", "ns")) < 0) {
1835 panic("couldn't create cpu_time entry for thread ledger template");
1836 }
1837
1838 if (ledger_set_callback(t, idx, thread_cputime_callback, NULL, NULL) < 0) {
1839 panic("couldn't set thread ledger callback for cpu_time entry");
1840 }
1841
1842 thread_ledgers.cpu_time = idx;
1843
1844 thread_ledger_template = t;
1845 }
1846
1847 /*
1848 * Returns currently applied CPU usage limit, or 0/0 if none is applied.
1849 */
1850 int
1851 thread_get_cpulimit(int *action, uint8_t *percentage, uint64_t *interval_ns)
1852 {
1853 int64_t abstime = 0;
1854 uint64_t limittime = 0;
1855 thread_t thread = current_thread();
1856
1857 *percentage = 0;
1858 *interval_ns = 0;
1859 *action = 0;
1860
1861 if (thread->t_threadledger == LEDGER_NULL) {
1862 /*
1863 * This thread has no per-thread ledger, so it can't possibly
1864 * have a CPU limit applied.
1865 */
1866 return (KERN_SUCCESS);
1867 }
1868
1869 ledger_get_period(thread->t_threadledger, thread_ledgers.cpu_time, interval_ns);
1870 ledger_get_limit(thread->t_threadledger, thread_ledgers.cpu_time, &abstime);
1871
1872 if ((abstime == LEDGER_LIMIT_INFINITY) || (*interval_ns == 0)) {
1873 /*
1874 * This thread's CPU time ledger has no period or limit; so it
1875 * doesn't have a CPU limit applied.
1876 */
1877 return (KERN_SUCCESS);
1878 }
1879
1880 /*
1881 * This calculation is the converse to the one in thread_set_cpulimit().
1882 */
1883 absolutetime_to_nanoseconds(abstime, &limittime);
1884 *percentage = (limittime * 100ULL) / *interval_ns;
1885 assert(*percentage <= 100);
1886
1887 if (thread->options & TH_OPT_PROC_CPULIMIT) {
1888 assert((thread->options & TH_OPT_PRVT_CPULIMIT) == 0);
1889
1890 *action = THREAD_CPULIMIT_BLOCK;
1891 } else if (thread->options & TH_OPT_PRVT_CPULIMIT) {
1892 assert((thread->options & TH_OPT_PROC_CPULIMIT) == 0);
1893
1894 *action = THREAD_CPULIMIT_EXCEPTION;
1895 } else {
1896 *action = THREAD_CPULIMIT_DISABLE;
1897 }
1898
1899 return (KERN_SUCCESS);
1900 }
1901
1902 /*
1903 * Set CPU usage limit on a thread.
1904 *
1905 * Calling with percentage of 0 will unset the limit for this thread.
1906 */
1907 int
1908 thread_set_cpulimit(int action, uint8_t percentage, uint64_t interval_ns)
1909 {
1910 thread_t thread = current_thread();
1911 ledger_t l;
1912 uint64_t limittime = 0;
1913 uint64_t abstime = 0;
1914
1915 assert(percentage <= 100);
1916
1917 if (action == THREAD_CPULIMIT_DISABLE) {
1918 /*
1919 * Remove CPU limit, if any exists.
1920 */
1921 if (thread->t_threadledger != LEDGER_NULL) {
1922 l = thread->t_threadledger;
1923 ledger_set_limit(l, thread_ledgers.cpu_time, LEDGER_LIMIT_INFINITY, 0);
1924 ledger_set_action(l, thread_ledgers.cpu_time, LEDGER_ACTION_IGNORE);
1925 thread->options &= ~(TH_OPT_PROC_CPULIMIT | TH_OPT_PRVT_CPULIMIT);
1926 }
1927
1928 return (0);
1929 }
1930
1931 if (interval_ns < MINIMUM_CPULIMIT_INTERVAL_MS * NSEC_PER_MSEC) {
1932 return (KERN_INVALID_ARGUMENT);
1933 }
1934
1935 l = thread->t_threadledger;
1936 if (l == LEDGER_NULL) {
1937 /*
1938 * This thread doesn't yet have a per-thread ledger; so create one with the CPU time entry active.
1939 */
1940 if ((l = ledger_instantiate(thread_ledger_template, LEDGER_CREATE_INACTIVE_ENTRIES)) == LEDGER_NULL)
1941 return (KERN_RESOURCE_SHORTAGE);
1942
1943 /*
1944 * We are the first to create this thread's ledger, so only activate our entry.
1945 */
1946 ledger_entry_setactive(l, thread_ledgers.cpu_time);
1947 thread->t_threadledger = l;
1948 }
1949
1950 /*
1951 * The limit is specified as a percentage of CPU over an interval in nanoseconds.
1952 * Calculate the amount of CPU time that the thread needs to consume in order to hit the limit.
1953 */
1954 limittime = (interval_ns * percentage) / 100;
1955 nanoseconds_to_absolutetime(limittime, &abstime);
1956 ledger_set_limit(l, thread_ledgers.cpu_time, abstime, cpumon_ustackshots_trigger_pct);
1957 /*
1958 * Refill the thread's allotted CPU time every interval_ns nanoseconds.
1959 */
1960 ledger_set_period(l, thread_ledgers.cpu_time, interval_ns);
1961
1962 if (action == THREAD_CPULIMIT_EXCEPTION) {
1963 /*
1964 * We don't support programming the CPU usage monitor on a task if any of its
1965 * threads have a per-thread blocking CPU limit configured.
1966 */
1967 if (thread->options & TH_OPT_PRVT_CPULIMIT) {
1968 panic("CPU usage monitor activated, but blocking thread limit exists");
1969 }
1970
1971 /*
1972 * Make a note that this thread's CPU limit is being used for the task-wide CPU
1973 * usage monitor. We don't have to arm the callback which will trigger the
1974 * exception, because that was done for us in ledger_instantiate (because the
1975 * ledger template used has a default callback).
1976 */
1977 thread->options |= TH_OPT_PROC_CPULIMIT;
1978 } else {
1979 /*
1980 * We deliberately override any CPU limit imposed by a task-wide limit (eg
1981 * CPU usage monitor).
1982 */
1983 thread->options &= ~TH_OPT_PROC_CPULIMIT;
1984
1985 thread->options |= TH_OPT_PRVT_CPULIMIT;
1986 /* The per-thread ledger template by default has a callback for CPU time */
1987 ledger_disable_callback(l, thread_ledgers.cpu_time);
1988 ledger_set_action(l, thread_ledgers.cpu_time, LEDGER_ACTION_BLOCK);
1989 }
1990
1991 return (0);
1992 }
1993
1994 static void
1995 sched_call_null(
1996 __unused int type,
1997 __unused thread_t thread)
1998 {
1999 return;
2000 }
2001
2002 void
2003 thread_sched_call(
2004 thread_t thread,
2005 sched_call_t call)
2006 {
2007 thread->sched_call = (call != NULL)? call: sched_call_null;
2008 }
2009
2010 void
2011 thread_static_param(
2012 thread_t thread,
2013 boolean_t state)
2014 {
2015 thread_mtx_lock(thread);
2016 thread->static_param = state;
2017 thread_mtx_unlock(thread);
2018 }
2019
2020 uint64_t
2021 thread_tid(
2022 thread_t thread)
2023 {
2024 return (thread != THREAD_NULL? thread->thread_id: 0);
2025 }
2026
2027 uint16_t thread_set_tag(thread_t th, uint16_t tag) {
2028 return thread_set_tag_internal(th, tag);
2029 }
2030 uint16_t thread_get_tag(thread_t th) {
2031 return thread_get_tag_internal(th);
2032 }
2033
2034 uint64_t
2035 thread_dispatchqaddr(
2036 thread_t thread)
2037 {
2038 uint64_t dispatchqueue_addr = 0;
2039 uint64_t thread_handle = 0;
2040
2041 if (thread != THREAD_NULL) {
2042 thread_handle = thread->machine.cthread_self;
2043
2044 if (thread->task->bsd_info)
2045 dispatchqueue_addr = thread_handle + get_dispatchqueue_offset_from_proc(thread->task->bsd_info);
2046 }
2047
2048 return (dispatchqueue_addr);
2049 }
2050
2051 /*
2052 * Export routines to other components for things that are done as macros
2053 * within the osfmk component.
2054 */
2055
2056 #undef thread_reference
2057 void thread_reference(thread_t thread);
2058 void
2059 thread_reference(
2060 thread_t thread)
2061 {
2062 if (thread != THREAD_NULL)
2063 thread_reference_internal(thread);
2064 }
2065
2066 #undef thread_should_halt
2067
2068 boolean_t
2069 thread_should_halt(
2070 thread_t th)
2071 {
2072 return (thread_should_halt_fast(th));
2073 }
2074
2075 /*
2076 * thread_set_voucher_name - reset the voucher port name bound to this thread
2077 *
2078 * Conditions: nothing locked
2079 *
2080 * If we already converted the previous name to a cached voucher
2081 * reference, then we discard that reference here. The next lookup
2082 * will cache it again.
2083 */
2084
2085 kern_return_t
2086 thread_set_voucher_name(mach_port_name_t voucher_name)
2087 {
2088 thread_t thread = current_thread();
2089 ipc_voucher_t new_voucher = IPC_VOUCHER_NULL;
2090 ipc_voucher_t voucher;
2091 #ifdef CONFIG_BANK
2092 ledger_t bankledger = NULL;
2093 #endif
2094
2095 if (MACH_PORT_DEAD == voucher_name)
2096 return KERN_INVALID_RIGHT;
2097
2098 /*
2099 * agressively convert to voucher reference
2100 */
2101 if (MACH_PORT_VALID(voucher_name)) {
2102 new_voucher = convert_port_name_to_voucher(voucher_name);
2103 if (IPC_VOUCHER_NULL == new_voucher)
2104 return KERN_INVALID_ARGUMENT;
2105 }
2106 #ifdef CONFIG_BANK
2107 bankledger = bank_get_voucher_ledger(new_voucher);
2108 #endif
2109
2110 thread_mtx_lock(thread);
2111 voucher = thread->ith_voucher;
2112 thread->ith_voucher_name = voucher_name;
2113 thread->ith_voucher = new_voucher;
2114 #ifdef CONFIG_BANK
2115 bank_swap_thread_bank_ledger(thread, bankledger);
2116 #endif
2117 thread_mtx_unlock(thread);
2118
2119 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
2120 MACHDBG_CODE(DBG_MACH_IPC,MACH_THREAD_SET_VOUCHER) | DBG_FUNC_NONE,
2121 (uintptr_t)thread_tid(thread),
2122 (uintptr_t)voucher_name,
2123 VM_KERNEL_ADDRPERM((uintptr_t)new_voucher),
2124 1, 0);
2125
2126 if (IPC_VOUCHER_NULL != voucher)
2127 ipc_voucher_release(voucher);
2128
2129 return KERN_SUCCESS;
2130 }
2131
2132 /*
2133 * thread_get_mach_voucher - return a voucher reference for the specified thread voucher
2134 *
2135 * Conditions: nothing locked
2136 *
2137 * A reference to the voucher may be lazily pending, if someone set the voucher name
2138 * but nobody has done a lookup yet. In that case, we'll have to do the equivalent
2139 * lookup here.
2140 *
2141 * NOTE: At the moment, there is no distinction between the current and effective
2142 * vouchers because we only set them at the thread level currently.
2143 */
2144 kern_return_t
2145 thread_get_mach_voucher(
2146 thread_act_t thread,
2147 mach_voucher_selector_t __unused which,
2148 ipc_voucher_t *voucherp)
2149 {
2150 ipc_voucher_t voucher;
2151 mach_port_name_t voucher_name;
2152
2153 if (THREAD_NULL == thread)
2154 return KERN_INVALID_ARGUMENT;
2155
2156 thread_mtx_lock(thread);
2157 voucher = thread->ith_voucher;
2158
2159 /* if already cached, just return a ref */
2160 if (IPC_VOUCHER_NULL != voucher) {
2161 ipc_voucher_reference(voucher);
2162 thread_mtx_unlock(thread);
2163 *voucherp = voucher;
2164 return KERN_SUCCESS;
2165 }
2166
2167 voucher_name = thread->ith_voucher_name;
2168
2169 /* convert the name to a port, then voucher reference */
2170 if (MACH_PORT_VALID(voucher_name)) {
2171 ipc_port_t port;
2172
2173 if (KERN_SUCCESS !=
2174 ipc_object_copyin(thread->task->itk_space, voucher_name,
2175 MACH_MSG_TYPE_COPY_SEND, (ipc_object_t *)&port)) {
2176 thread->ith_voucher_name = MACH_PORT_NULL;
2177 thread_mtx_unlock(thread);
2178 *voucherp = IPC_VOUCHER_NULL;
2179 return KERN_SUCCESS;
2180 }
2181
2182 /* convert to a voucher ref to return, and cache a ref on thread */
2183 voucher = convert_port_to_voucher(port);
2184 ipc_voucher_reference(voucher);
2185 thread->ith_voucher = voucher;
2186 thread_mtx_unlock(thread);
2187
2188 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
2189 MACHDBG_CODE(DBG_MACH_IPC,MACH_THREAD_SET_VOUCHER) | DBG_FUNC_NONE,
2190 (uintptr_t)thread_tid(thread),
2191 (uintptr_t)port,
2192 VM_KERNEL_ADDRPERM((uintptr_t)voucher),
2193 2, 0);
2194
2195
2196 ipc_port_release_send(port);
2197 } else
2198 thread_mtx_unlock(thread);
2199
2200 *voucherp = voucher;
2201 return KERN_SUCCESS;
2202 }
2203
2204 /*
2205 * thread_set_mach_voucher - set a voucher reference for the specified thread voucher
2206 *
2207 * Conditions: callers holds a reference on the voucher.
2208 * nothing locked.
2209 *
2210 * We grab another reference to the voucher and bind it to the thread. Any lazy
2211 * binding is erased. The old voucher reference associated with the thread is
2212 * discarded.
2213 */
2214 kern_return_t
2215 thread_set_mach_voucher(
2216 thread_t thread,
2217 ipc_voucher_t voucher)
2218 {
2219 ipc_voucher_t old_voucher;
2220 #ifdef CONFIG_BANK
2221 ledger_t bankledger = NULL;
2222 #endif
2223
2224 if (THREAD_NULL == thread)
2225 return KERN_INVALID_ARGUMENT;
2226
2227 if (thread != current_thread() || thread->started)
2228 return KERN_INVALID_ARGUMENT;
2229
2230
2231 ipc_voucher_reference(voucher);
2232 #ifdef CONFIG_BANK
2233 bankledger = bank_get_voucher_ledger(voucher);
2234 #endif
2235 thread_mtx_lock(thread);
2236 old_voucher = thread->ith_voucher;
2237 thread->ith_voucher = voucher;
2238 thread->ith_voucher_name = MACH_PORT_NULL;
2239 #ifdef CONFIG_BANK
2240 bank_swap_thread_bank_ledger(thread, bankledger);
2241 #endif
2242 thread_mtx_unlock(thread);
2243
2244 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
2245 MACHDBG_CODE(DBG_MACH_IPC,MACH_THREAD_SET_VOUCHER) | DBG_FUNC_NONE,
2246 (uintptr_t)thread_tid(thread),
2247 (uintptr_t)MACH_PORT_NULL,
2248 VM_KERNEL_ADDRPERM((uintptr_t)voucher),
2249 3, 0);
2250
2251 ipc_voucher_release(old_voucher);
2252
2253 return KERN_SUCCESS;
2254 }
2255
2256 /*
2257 * thread_swap_mach_voucher - swap a voucher reference for the specified thread voucher
2258 *
2259 * Conditions: callers holds a reference on the new and presumed old voucher(s).
2260 * nothing locked.
2261 *
2262 * If the old voucher is still the same as passed in, replace it with new voucher
2263 * and discard the old (and the reference passed in). Otherwise, discard the new
2264 * and return an updated old voucher.
2265 */
2266 kern_return_t
2267 thread_swap_mach_voucher(
2268 thread_t thread,
2269 ipc_voucher_t new_voucher,
2270 ipc_voucher_t *in_out_old_voucher)
2271 {
2272 mach_port_name_t old_voucher_name;
2273 ipc_voucher_t old_voucher;
2274 #ifdef CONFIG_BANK
2275 ledger_t bankledger = NULL;
2276 #endif
2277
2278 if (THREAD_NULL == thread)
2279 return KERN_INVALID_TASK;
2280
2281 if (thread != current_thread() || thread->started)
2282 return KERN_INVALID_ARGUMENT;
2283
2284 #ifdef CONFIG_BANK
2285 bankledger = bank_get_voucher_ledger(new_voucher);
2286 #endif
2287
2288 thread_mtx_lock(thread);
2289
2290 old_voucher = thread->ith_voucher;
2291
2292 if (IPC_VOUCHER_NULL == old_voucher) {
2293 old_voucher_name = thread->ith_voucher_name;
2294
2295 /* perform lazy binding if needed */
2296 if (MACH_PORT_VALID(old_voucher_name)) {
2297 old_voucher = convert_port_name_to_voucher(old_voucher_name);
2298 thread->ith_voucher_name = MACH_PORT_NULL;
2299 thread->ith_voucher = old_voucher;
2300
2301 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
2302 MACHDBG_CODE(DBG_MACH_IPC,MACH_THREAD_SET_VOUCHER) | DBG_FUNC_NONE,
2303 (uintptr_t)thread_tid(thread),
2304 (uintptr_t)old_voucher_name,
2305 VM_KERNEL_ADDRPERM((uintptr_t)old_voucher),
2306 4, 0);
2307
2308 }
2309 }
2310
2311 /* swap in new voucher, if old voucher matches the one supplied */
2312 if (old_voucher == *in_out_old_voucher) {
2313 ipc_voucher_reference(new_voucher);
2314 thread->ith_voucher = new_voucher;
2315 thread->ith_voucher_name = MACH_PORT_NULL;
2316 #ifdef CONFIG_BANK
2317 bank_swap_thread_bank_ledger(thread, bankledger);
2318 #endif
2319 thread_mtx_unlock(thread);
2320
2321 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
2322 MACHDBG_CODE(DBG_MACH_IPC,MACH_THREAD_SET_VOUCHER) | DBG_FUNC_NONE,
2323 (uintptr_t)thread_tid(thread),
2324 (uintptr_t)MACH_PORT_NULL,
2325 VM_KERNEL_ADDRPERM((uintptr_t)new_voucher),
2326 5, 0);
2327
2328 ipc_voucher_release(old_voucher);
2329
2330 *in_out_old_voucher = IPC_VOUCHER_NULL;
2331 return KERN_SUCCESS;
2332 }
2333
2334 /* Otherwise, just return old voucher reference */
2335 ipc_voucher_reference(old_voucher);
2336 thread_mtx_unlock(thread);
2337 *in_out_old_voucher = old_voucher;
2338 return KERN_SUCCESS;
2339 }
2340
2341 /*
2342 * thread_get_current_voucher_origin_pid - get the pid of the originator of the current voucher.
2343 */
2344 kern_return_t
2345 thread_get_current_voucher_origin_pid(
2346 int32_t *pid)
2347 {
2348 uint32_t buf_size;
2349 kern_return_t kr;
2350 thread_t thread = current_thread();
2351
2352 buf_size = sizeof(*pid);
2353 kr = mach_voucher_attr_command(thread->ith_voucher,
2354 MACH_VOUCHER_ATTR_KEY_BANK,
2355 BANK_ORIGINATOR_PID,
2356 NULL,
2357 0,
2358 (mach_voucher_attr_content_t)pid,
2359 &buf_size);
2360
2361 return kr;
2362 }
2363
2364 #if CONFIG_DTRACE
2365 uint32_t dtrace_get_thread_predcache(thread_t thread)
2366 {
2367 if (thread != THREAD_NULL)
2368 return thread->t_dtrace_predcache;
2369 else
2370 return 0;
2371 }
2372
2373 int64_t dtrace_get_thread_vtime(thread_t thread)
2374 {
2375 if (thread != THREAD_NULL)
2376 return thread->t_dtrace_vtime;
2377 else
2378 return 0;
2379 }
2380
2381 int64_t dtrace_get_thread_tracing(thread_t thread)
2382 {
2383 if (thread != THREAD_NULL)
2384 return thread->t_dtrace_tracing;
2385 else
2386 return 0;
2387 }
2388
2389 boolean_t dtrace_get_thread_reentering(thread_t thread)
2390 {
2391 if (thread != THREAD_NULL)
2392 return (thread->options & TH_OPT_DTRACE) ? TRUE : FALSE;
2393 else
2394 return 0;
2395 }
2396
2397 vm_offset_t dtrace_get_kernel_stack(thread_t thread)
2398 {
2399 if (thread != THREAD_NULL)
2400 return thread->kernel_stack;
2401 else
2402 return 0;
2403 }
2404
2405 int64_t dtrace_calc_thread_recent_vtime(thread_t thread)
2406 {
2407 if (thread != THREAD_NULL) {
2408 processor_t processor = current_processor();
2409 uint64_t abstime = mach_absolute_time();
2410 timer_t timer;
2411
2412 timer = PROCESSOR_DATA(processor, thread_timer);
2413
2414 return timer_grab(&(thread->system_timer)) + timer_grab(&(thread->user_timer)) +
2415 (abstime - timer->tstamp); /* XXX need interrupts off to prevent missed time? */
2416 } else
2417 return 0;
2418 }
2419
2420 void dtrace_set_thread_predcache(thread_t thread, uint32_t predcache)
2421 {
2422 if (thread != THREAD_NULL)
2423 thread->t_dtrace_predcache = predcache;
2424 }
2425
2426 void dtrace_set_thread_vtime(thread_t thread, int64_t vtime)
2427 {
2428 if (thread != THREAD_NULL)
2429 thread->t_dtrace_vtime = vtime;
2430 }
2431
2432 void dtrace_set_thread_tracing(thread_t thread, int64_t accum)
2433 {
2434 if (thread != THREAD_NULL)
2435 thread->t_dtrace_tracing = accum;
2436 }
2437
2438 void dtrace_set_thread_reentering(thread_t thread, boolean_t vbool)
2439 {
2440 if (thread != THREAD_NULL) {
2441 if (vbool)
2442 thread->options |= TH_OPT_DTRACE;
2443 else
2444 thread->options &= (~TH_OPT_DTRACE);
2445 }
2446 }
2447
2448 vm_offset_t dtrace_set_thread_recover(thread_t thread, vm_offset_t recover)
2449 {
2450 vm_offset_t prev = 0;
2451
2452 if (thread != THREAD_NULL) {
2453 prev = thread->recover;
2454 thread->recover = recover;
2455 }
2456 return prev;
2457 }
2458
2459 void dtrace_thread_bootstrap(void)
2460 {
2461 task_t task = current_task();
2462
2463 if (task->thread_count == 1) {
2464 thread_t thread = current_thread();
2465 if (thread->t_dtrace_flags & TH_DTRACE_EXECSUCCESS) {
2466 thread->t_dtrace_flags &= ~TH_DTRACE_EXECSUCCESS;
2467 DTRACE_PROC(exec__success);
2468 }
2469 DTRACE_PROC(start);
2470 }
2471 DTRACE_PROC(lwp__start);
2472
2473 }
2474
2475 void
2476 dtrace_thread_didexec(thread_t thread)
2477 {
2478 thread->t_dtrace_flags |= TH_DTRACE_EXECSUCCESS;
2479 }
2480 #endif /* CONFIG_DTRACE */
mirror of XNU