]> git.saurik.com Git - apple/xnu.git/blob - osfmk/kern/thread.c
projects / apple / xnu.git / blob
? search:


7dfcb02b3e0864d0866e956aa67fdde63d4b4023
[apple/xnu.git] / osfmk / kern / thread.c
1 /*
2 * Copyright (c) 2000-2015 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_FREE_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56 /*
57 */
58 /*
59 * File: kern/thread.c
60 * Author: Avadis Tevanian, Jr., Michael Wayne Young, David Golub
61 * Date: 1986
62 *
63 * Thread management primitives implementation.
64 */
65 /*
66 * Copyright (c) 1993 The University of Utah and
67 * the Computer Systems Laboratory (CSL). All rights reserved.
68 *
69 * Permission to use, copy, modify and distribute this software and its
70 * documentation is hereby granted, provided that both the copyright
71 * notice and this permission notice appear in all copies of the
72 * software, derivative works or modified versions, and any portions
73 * thereof, and that both notices appear in supporting documentation.
74 *
75 * THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF THIS SOFTWARE IN ITS "AS
76 * IS" CONDITION. THE UNIVERSITY OF UTAH AND CSL DISCLAIM ANY LIABILITY OF
77 * ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
78 *
79 * CSL requests users of this software to return to csl-dist@cs.utah.edu any
80 * improvements that they make and grant CSL redistribution rights.
81 *
82 */
83
84 #include <mach/mach_types.h>
85 #include <mach/boolean.h>
86 #include <mach/policy.h>
87 #include <mach/thread_info.h>
88 #include <mach/thread_special_ports.h>
89 #include <mach/thread_status.h>
90 #include <mach/time_value.h>
91 #include <mach/vm_param.h>
92
93 #include <machine/thread.h>
94 #include <machine/pal_routines.h>
95 #include <machine/limits.h>
96
97 #include <kern/kern_types.h>
98 #include <kern/kalloc.h>
99 #include <kern/cpu_data.h>
100 #include <kern/counters.h>
101 #include <kern/extmod_statistics.h>
102 #include <kern/ipc_mig.h>
103 #include <kern/ipc_tt.h>
104 #include <kern/mach_param.h>
105 #include <kern/machine.h>
106 #include <kern/misc_protos.h>
107 #include <kern/processor.h>
108 #include <kern/queue.h>
109 #include <kern/sched.h>
110 #include <kern/sched_prim.h>
111 #include <kern/sync_lock.h>
112 #include <kern/syscall_subr.h>
113 #include <kern/task.h>
114 #include <kern/thread.h>
115 #include <kern/thread_group.h>
116 #include <kern/coalition.h>
117 #include <kern/host.h>
118 #include <kern/zalloc.h>
119 #include <kern/assert.h>
120 #include <kern/exc_resource.h>
121 #include <kern/exc_guard.h>
122 #include <kern/telemetry.h>
123 #include <kern/policy_internal.h>
124 #include <kern/turnstile.h>
125
126 #include <corpses/task_corpse.h>
127 #if KPC
128 #include <kern/kpc.h>
129 #endif
130
131 #if MONOTONIC
132 #include <kern/monotonic.h>
133 #include <machine/monotonic.h>
134 #endif /* MONOTONIC */
135
136 #include <ipc/ipc_kmsg.h>
137 #include <ipc/ipc_port.h>
138 #include <bank/bank_types.h>
139
140 #include <vm/vm_kern.h>
141 #include <vm/vm_pageout.h>
142
143 #include <sys/kdebug.h>
144 #include <sys/bsdtask_info.h>
145 #include <mach/sdt.h>
146 #include <san/kasan.h>
147
148 #include <stdatomic.h>
149
150
151 /*
152 * Exported interfaces
153 */
154 #include <mach/task_server.h>
155 #include <mach/thread_act_server.h>
156 #include <mach/mach_host_server.h>
157 #include <mach/host_priv_server.h>
158 #include <mach/mach_voucher_server.h>
159 #include <kern/policy_internal.h>
160
161 static struct zone *thread_zone;
162 static lck_grp_attr_t thread_lck_grp_attr;
163 lck_attr_t thread_lck_attr;
164 lck_grp_t thread_lck_grp;
165
166 struct zone *thread_qos_override_zone;
167
168 decl_simple_lock_data(static, thread_stack_lock)
169 static queue_head_t thread_stack_queue;
170
171 decl_simple_lock_data(static, thread_terminate_lock)
172 static queue_head_t thread_terminate_queue;
173
174 static queue_head_t thread_deallocate_queue;
175
176 static queue_head_t turnstile_deallocate_queue;
177
178 static queue_head_t crashed_threads_queue;
179
180 static queue_head_t workq_deallocate_queue;
181
182 decl_simple_lock_data(static, thread_exception_lock)
183 static queue_head_t thread_exception_queue;
184
185 struct thread_exception_elt {
186 queue_chain_t elt;
187 exception_type_t exception_type;
188 task_t exception_task;
189 thread_t exception_thread;
190 };
191
192 static struct thread thread_template, init_thread;
193 static void thread_deallocate_enqueue(thread_t thread);
194 static void thread_deallocate_complete(thread_t thread);
195
196 #ifdef MACH_BSD
197 extern void proc_exit(void *);
198 extern mach_exception_data_type_t proc_encode_exit_exception_code(void *);
199 extern uint64_t get_dispatchqueue_offset_from_proc(void *);
200 extern uint64_t get_return_to_kernel_offset_from_proc(void *p);
201 extern int proc_selfpid(void);
202 extern void proc_name(int, char*, int);
203 extern char * proc_name_address(void *p);
204 #endif /* MACH_BSD */
205
206 extern int disable_exc_resource;
207 extern int audio_active;
208 extern int debug_task;
209 int thread_max = CONFIG_THREAD_MAX; /* Max number of threads */
210 int task_threadmax = CONFIG_THREAD_MAX;
211
212 static uint64_t thread_unique_id = 100;
213
214 struct _thread_ledger_indices thread_ledgers = { -1 };
215 static ledger_template_t thread_ledger_template = NULL;
216 static void init_thread_ledgers(void);
217
218 #if CONFIG_JETSAM
219 void jetsam_on_ledger_cpulimit_exceeded(void);
220 #endif
221
222 extern int task_thread_soft_limit;
223 extern int exc_via_corpse_forking;
224
225 #if DEVELOPMENT || DEBUG
226 extern int exc_resource_threads_enabled;
227 #endif /* DEVELOPMENT || DEBUG */
228
229 /*
230 * Level (in terms of percentage of the limit) at which the CPU usage monitor triggers telemetry.
231 *
232 * (ie when any thread's CPU consumption exceeds 70% of the limit, start taking user
233 * stacktraces, aka micro-stackshots)
234 */
235 #define CPUMON_USTACKSHOTS_TRIGGER_DEFAULT_PCT 70
236
237 int cpumon_ustackshots_trigger_pct; /* Percentage. Level at which we start gathering telemetry. */
238 void __attribute__((noinline)) SENDING_NOTIFICATION__THIS_THREAD_IS_CONSUMING_TOO_MUCH_CPU(void);
239 #if DEVELOPMENT || DEBUG
240 void __attribute__((noinline)) SENDING_NOTIFICATION__TASK_HAS_TOO_MANY_THREADS(task_t, int);
241 #endif /* DEVELOPMENT || DEBUG */
242
243 /*
244 * The smallest interval over which we support limiting CPU consumption is 1ms
245 */
246 #define MINIMUM_CPULIMIT_INTERVAL_MS 1
247
248 os_refgrp_decl(static, thread_refgrp, "thread", NULL);
249
250 void
251 thread_bootstrap(void)
252 {
253 /*
254 * Fill in a template thread for fast initialization.
255 */
256
257 #if MACH_ASSERT
258 thread_template.thread_magic = THREAD_MAGIC;
259 #endif /* MACH_ASSERT */
260
261 thread_template.runq = PROCESSOR_NULL;
262
263 thread_template.reason = AST_NONE;
264 thread_template.at_safe_point = FALSE;
265 thread_template.wait_event = NO_EVENT64;
266 thread_template.waitq = NULL;
267 thread_template.wait_result = THREAD_WAITING;
268 thread_template.options = THREAD_ABORTSAFE;
269 thread_template.state = TH_WAIT | TH_UNINT;
270 thread_template.wake_active = FALSE;
271 thread_template.continuation = THREAD_CONTINUE_NULL;
272 thread_template.parameter = NULL;
273
274 thread_template.importance = 0;
275 thread_template.sched_mode = TH_MODE_NONE;
276 thread_template.sched_flags = 0;
277 thread_template.saved_mode = TH_MODE_NONE;
278 thread_template.safe_release = 0;
279 thread_template.th_sched_bucket = TH_BUCKET_RUN;
280
281 thread_template.sfi_class = SFI_CLASS_UNSPECIFIED;
282 thread_template.sfi_wait_class = SFI_CLASS_UNSPECIFIED;
283
284 thread_template.active = 0;
285 thread_template.started = 0;
286 thread_template.static_param = 0;
287 thread_template.policy_reset = 0;
288
289 thread_template.base_pri = BASEPRI_DEFAULT;
290 thread_template.sched_pri = 0;
291 thread_template.max_priority = 0;
292 thread_template.task_priority = 0;
293 thread_template.promotions = 0;
294 thread_template.rwlock_count = 0;
295 thread_template.waiting_for_mutex = NULL;
296
297
298 thread_template.realtime.deadline = UINT64_MAX;
299
300 thread_template.quantum_remaining = 0;
301 thread_template.last_run_time = 0;
302 thread_template.last_made_runnable_time = THREAD_NOT_RUNNABLE;
303 thread_template.last_basepri_change_time = THREAD_NOT_RUNNABLE;
304 thread_template.same_pri_latency = 0;
305
306 thread_template.computation_metered = 0;
307 thread_template.computation_epoch = 0;
308
309 #if defined(CONFIG_SCHED_TIMESHARE_CORE)
310 thread_template.sched_stamp = 0;
311 thread_template.pri_shift = INT8_MAX;
312 thread_template.sched_usage = 0;
313 thread_template.cpu_usage = thread_template.cpu_delta = 0;
314 #endif
315 thread_template.c_switch = thread_template.p_switch = thread_template.ps_switch = 0;
316
317 #if MONOTONIC
318 memset(&thread_template.t_monotonic, 0,
319 sizeof(thread_template.t_monotonic));
320 #endif /* MONOTONIC */
321
322 thread_template.bound_processor = PROCESSOR_NULL;
323 thread_template.last_processor = PROCESSOR_NULL;
324
325 thread_template.sched_call = NULL;
326
327 timer_init(&thread_template.user_timer);
328 timer_init(&thread_template.system_timer);
329 timer_init(&thread_template.ptime);
330 timer_init(&thread_template.runnable_timer);
331 thread_template.user_timer_save = 0;
332 thread_template.system_timer_save = 0;
333 thread_template.vtimer_user_save = 0;
334 thread_template.vtimer_prof_save = 0;
335 thread_template.vtimer_rlim_save = 0;
336 thread_template.vtimer_qos_save = 0;
337
338 #if CONFIG_SCHED_SFI
339 thread_template.wait_sfi_begin_time = 0;
340 #endif
341
342 thread_template.wait_timer_is_set = FALSE;
343 thread_template.wait_timer_active = 0;
344
345 thread_template.depress_timer_active = 0;
346
347 thread_template.recover = (vm_offset_t)NULL;
348
349 thread_template.map = VM_MAP_NULL;
350 #if DEVELOPMENT || DEBUG
351 thread_template.pmap_footprint_suspended = FALSE;
352 #endif /* DEVELOPMENT || DEBUG */
353
354 #if CONFIG_DTRACE
355 thread_template.t_dtrace_predcache = 0;
356 thread_template.t_dtrace_vtime = 0;
357 thread_template.t_dtrace_tracing = 0;
358 #endif /* CONFIG_DTRACE */
359
360 #if KPERF
361 thread_template.kperf_flags = 0;
362 thread_template.kperf_pet_gen = 0;
363 thread_template.kperf_c_switch = 0;
364 thread_template.kperf_pet_cnt = 0;
365 #endif
366
367 #if KPC
368 thread_template.kpc_buf = NULL;
369 #endif
370
371 #if HYPERVISOR
372 thread_template.hv_thread_target = NULL;
373 #endif /* HYPERVISOR */
374
375 #if (DEVELOPMENT || DEBUG)
376 thread_template.t_page_creation_throttled_hard = 0;
377 thread_template.t_page_creation_throttled_soft = 0;
378 #endif /* DEVELOPMENT || DEBUG */
379 thread_template.t_page_creation_throttled = 0;
380 thread_template.t_page_creation_count = 0;
381 thread_template.t_page_creation_time = 0;
382
383 thread_template.affinity_set = NULL;
384
385 thread_template.syscalls_unix = 0;
386 thread_template.syscalls_mach = 0;
387
388 thread_template.t_ledger = LEDGER_NULL;
389 thread_template.t_threadledger = LEDGER_NULL;
390 thread_template.t_bankledger = LEDGER_NULL;
391 thread_template.t_deduct_bank_ledger_time = 0;
392
393 thread_template.requested_policy = (struct thread_requested_policy) {};
394 thread_template.effective_policy = (struct thread_effective_policy) {};
395
396 bzero(&thread_template.overrides, sizeof(thread_template.overrides));
397 thread_template.sync_ipc_overrides = 0;
398
399 thread_template.iotier_override = THROTTLE_LEVEL_NONE;
400 thread_template.thread_io_stats = NULL;
401 #if CONFIG_EMBEDDED
402 thread_template.taskwatch = NULL;
403 #endif /* CONFIG_EMBEDDED */
404 thread_template.thread_callout_interrupt_wakeups = thread_template.thread_callout_platform_idle_wakeups = 0;
405
406 thread_template.thread_timer_wakeups_bin_1 = thread_template.thread_timer_wakeups_bin_2 = 0;
407 thread_template.callout_woken_from_icontext = thread_template.callout_woken_from_platform_idle = 0;
408
409 thread_template.thread_tag = 0;
410
411 thread_template.ith_voucher_name = MACH_PORT_NULL;
412 thread_template.ith_voucher = IPC_VOUCHER_NULL;
413
414 thread_template.th_work_interval = NULL;
415
416 init_thread = thread_template;
417
418 /* fiddle with init thread to skip asserts in set_sched_pri */
419 init_thread.sched_pri = MAXPRI_KERNEL;
420
421 machine_set_current_thread(&init_thread);
422 }
423
424 extern boolean_t allow_qos_policy_set;
425
426 void
427 thread_init(void)
428 {
429 thread_zone = zinit(
430 sizeof(struct thread),
431 thread_max * sizeof(struct thread),
432 THREAD_CHUNK * sizeof(struct thread),
433 "threads");
434
435 thread_qos_override_zone = zinit(
436 sizeof(struct thread_qos_override),
437 4 * thread_max * sizeof(struct thread_qos_override),
438 PAGE_SIZE,
439 "thread qos override");
440 zone_change(thread_qos_override_zone, Z_EXPAND, TRUE);
441 zone_change(thread_qos_override_zone, Z_COLLECT, TRUE);
442 zone_change(thread_qos_override_zone, Z_CALLERACCT, FALSE);
443 zone_change(thread_qos_override_zone, Z_NOENCRYPT, TRUE);
444
445 lck_grp_attr_setdefault(&thread_lck_grp_attr);
446 lck_grp_init(&thread_lck_grp, "thread", &thread_lck_grp_attr);
447 lck_attr_setdefault(&thread_lck_attr);
448
449 stack_init();
450
451 thread_policy_init();
452
453 /*
454 * Initialize any machine-dependent
455 * per-thread structures necessary.
456 */
457 machine_thread_init();
458
459 if (!PE_parse_boot_argn("cpumon_ustackshots_trigger_pct", &cpumon_ustackshots_trigger_pct,
460 sizeof(cpumon_ustackshots_trigger_pct))) {
461 cpumon_ustackshots_trigger_pct = CPUMON_USTACKSHOTS_TRIGGER_DEFAULT_PCT;
462 }
463
464 PE_parse_boot_argn("-qos-policy-allow", &allow_qos_policy_set, sizeof(allow_qos_policy_set));
465
466 init_thread_ledgers();
467 }
468
469 boolean_t
470 thread_is_active(thread_t thread)
471 {
472 return thread->active;
473 }
474
475 void
476 thread_corpse_continue(void)
477 {
478 thread_t thread = current_thread();
479
480 thread_terminate_internal(thread);
481
482 /*
483 * Handle the thread termination directly
484 * here instead of returning to userspace.
485 */
486 assert(thread->active == FALSE);
487 thread_ast_clear(thread, AST_APC);
488 thread_apc_ast(thread);
489
490 panic("thread_corpse_continue");
491 /*NOTREACHED*/
492 }
493
494 static void
495 thread_terminate_continue(void)
496 {
497 panic("thread_terminate_continue");
498 /*NOTREACHED*/
499 }
500
501 /*
502 * thread_terminate_self:
503 */
504 void
505 thread_terminate_self(void)
506 {
507 thread_t thread = current_thread();
508 task_t task;
509 int threadcnt;
510
511 pal_thread_terminate_self(thread);
512
513 DTRACE_PROC(lwp__exit);
514
515 thread_mtx_lock(thread);
516
517 ipc_thread_disable(thread);
518
519 thread_mtx_unlock(thread);
520
521 thread_sched_call(thread, NULL);
522
523 spl_t s = splsched();
524 thread_lock(thread);
525
526 thread_depress_abort_locked(thread);
527
528 thread_unlock(thread);
529 splx(s);
530
531 #if CONFIG_EMBEDDED
532 thead_remove_taskwatch(thread);
533 #endif /* CONFIG_EMBEDDED */
534
535 work_interval_thread_terminate(thread);
536
537 thread_mtx_lock(thread);
538
539 thread_policy_reset(thread);
540
541 thread_mtx_unlock(thread);
542
543 bank_swap_thread_bank_ledger(thread, NULL);
544
545 if (kdebug_enable && bsd_hasthreadname(thread->uthread)) {
546 char threadname[MAXTHREADNAMESIZE];
547 bsd_getthreadname(thread->uthread, threadname);
548 kernel_debug_string_simple(TRACE_STRING_THREADNAME_PREV, threadname);
549 }
550
551 task = thread->task;
552 uthread_cleanup(task, thread->uthread, task->bsd_info);
553
554 if (kdebug_enable && task->bsd_info && !task_is_exec_copy(task)) {
555 /* trace out pid before we sign off */
556 long dbg_arg1 = 0;
557 long dbg_arg2 = 0;
558
559 kdbg_trace_data(thread->task->bsd_info, &dbg_arg1, &dbg_arg2);
560 KDBG_RELEASE(TRACE_DATA_THREAD_TERMINATE_PID, dbg_arg1, dbg_arg2);
561 }
562
563 /*
564 * After this subtraction, this thread should never access
565 * task->bsd_info unless it got 0 back from the hw_atomic_sub. It
566 * could be racing with other threads to be the last thread in the
567 * process, and the last thread in the process will tear down the proc
568 * structure and zero-out task->bsd_info.
569 */
570 threadcnt = hw_atomic_sub(&task->active_thread_count, 1);
571
572 /*
573 * If we are the last thread to terminate and the task is
574 * associated with a BSD process, perform BSD process exit.
575 */
576 if (threadcnt == 0 && task->bsd_info != NULL && !task_is_exec_copy(task)) {
577 mach_exception_data_type_t subcode = 0;
578 if (kdebug_enable) {
579 /* since we're the last thread in this process, trace out the command name too */
580 long args[4] = {};
581 kdbg_trace_string(thread->task->bsd_info, &args[0], &args[1], &args[2], &args[3]);
582 KDBG_RELEASE(TRACE_STRING_PROC_EXIT, args[0], args[1], args[2], args[3]);
583 }
584
585 /* Get the exit reason before proc_exit */
586 subcode = proc_encode_exit_exception_code(task->bsd_info);
587 proc_exit(task->bsd_info);
588 /*
589 * if there is crash info in task
590 * then do the deliver action since this is
591 * last thread for this task.
592 */
593 if (task->corpse_info) {
594 task_deliver_crash_notification(task, current_thread(), EXC_RESOURCE, subcode);
595 }
596 }
597
598 if (threadcnt == 0) {
599 task_lock(task);
600 if (task_is_a_corpse_fork(task)) {
601 thread_wakeup((event_t)&task->active_thread_count);
602 }
603 task_unlock(task);
604 }
605
606 uthread_cred_free(thread->uthread);
607
608 s = splsched();
609 thread_lock(thread);
610
611 /*
612 * Ensure that the depress timer is no longer enqueued,
613 * so the timer (stored in the thread) can be safely deallocated
614 *
615 * TODO: build timer_call_cancel_wait
616 */
617
618 assert((thread->sched_flags & TH_SFLAG_DEPRESSED_MASK) == 0);
619
620 uint32_t delay_us = 1;
621
622 while (thread->depress_timer_active > 0) {
623 thread_unlock(thread);
624 splx(s);
625
626 delay(delay_us++);
627
628 if (delay_us > USEC_PER_SEC) {
629 panic("depress timer failed to inactivate!"
630 "thread: %p depress_timer_active: %d",
631 thread, thread->depress_timer_active);
632 }
633
634 s = splsched();
635 thread_lock(thread);
636 }
637
638 /*
639 * Cancel wait timer, and wait for
640 * concurrent expirations.
641 */
642 if (thread->wait_timer_is_set) {
643 thread->wait_timer_is_set = FALSE;
644
645 if (timer_call_cancel(&thread->wait_timer)) {
646 thread->wait_timer_active--;
647 }
648 }
649
650 delay_us = 1;
651
652 while (thread->wait_timer_active > 0) {
653 thread_unlock(thread);
654 splx(s);
655
656 delay(delay_us++);
657
658 if (delay_us > USEC_PER_SEC) {
659 panic("wait timer failed to inactivate!"
660 "thread: %p wait_timer_active: %d",
661 thread, thread->wait_timer_active);
662 }
663
664 s = splsched();
665 thread_lock(thread);
666 }
667
668 /*
669 * If there is a reserved stack, release it.
670 */
671 if (thread->reserved_stack != 0) {
672 stack_free_reserved(thread);
673 thread->reserved_stack = 0;
674 }
675
676 /*
677 * Mark thread as terminating, and block.
678 */
679 thread->state |= TH_TERMINATE;
680 thread_mark_wait_locked(thread, THREAD_UNINT);
681
682 assert((thread->sched_flags & TH_SFLAG_WAITQ_PROMOTED) == 0);
683 assert((thread->sched_flags & TH_SFLAG_RW_PROMOTED) == 0);
684 assert((thread->sched_flags & TH_SFLAG_EXEC_PROMOTED) == 0);
685 assert((thread->sched_flags & TH_SFLAG_PROMOTED) == 0);
686 assert(thread->promotions == 0);
687 assert(thread->was_promoted_on_wakeup == 0);
688 assert(thread->waiting_for_mutex == NULL);
689 assert(thread->rwlock_count == 0);
690
691 thread_unlock(thread);
692 /* splsched */
693
694 thread_block((thread_continue_t)thread_terminate_continue);
695 /*NOTREACHED*/
696 }
697
698 static bool
699 thread_ref_release(thread_t thread)
700 {
701 if (thread == THREAD_NULL) {
702 return false;
703 }
704
705 assert_thread_magic(thread);
706
707 return os_ref_release(&thread->ref_count) == 0;
708 }
709
710 /* Drop a thread refcount safely without triggering a zfree */
711 void
712 thread_deallocate_safe(thread_t thread)
713 {
714 if (__improbable(thread_ref_release(thread))) {
715 /* enqueue the thread for thread deallocate deamon to call thread_deallocate_complete */
716 thread_deallocate_enqueue(thread);
717 }
718 }
719
720 void
721 thread_deallocate(thread_t thread)
722 {
723 if (__improbable(thread_ref_release(thread))) {
724 thread_deallocate_complete(thread);
725 }
726 }
727
728 void
729 thread_deallocate_complete(
730 thread_t thread)
731 {
732 task_t task;
733
734 assert_thread_magic(thread);
735
736 assert(os_ref_get_count(&thread->ref_count) == 0);
737
738 assert(thread_owned_workloops_count(thread) == 0);
739
740 if (!(thread->state & TH_TERMINATE2)) {
741 panic("thread_deallocate: thread not properly terminated\n");
742 }
743
744 assert(thread->runq == PROCESSOR_NULL);
745
746 #if KPC
747 kpc_thread_destroy(thread);
748 #endif
749
750 ipc_thread_terminate(thread);
751
752 proc_thread_qos_deallocate(thread);
753
754 task = thread->task;
755
756 #ifdef MACH_BSD
757 {
758 void *ut = thread->uthread;
759
760 thread->uthread = NULL;
761 uthread_zone_free(ut);
762 }
763 #endif /* MACH_BSD */
764
765 if (thread->t_ledger) {
766 ledger_dereference(thread->t_ledger);
767 }
768 if (thread->t_threadledger) {
769 ledger_dereference(thread->t_threadledger);
770 }
771
772 assert(thread->turnstile != TURNSTILE_NULL);
773 if (thread->turnstile) {
774 turnstile_deallocate(thread->turnstile);
775 }
776
777 if (IPC_VOUCHER_NULL != thread->ith_voucher) {
778 ipc_voucher_release(thread->ith_voucher);
779 }
780
781 if (thread->thread_io_stats) {
782 kfree(thread->thread_io_stats, sizeof(struct io_stat_info));
783 }
784
785 if (thread->kernel_stack != 0) {
786 stack_free(thread);
787 }
788
789 lck_mtx_destroy(&thread->mutex, &thread_lck_grp);
790 machine_thread_destroy(thread);
791
792 task_deallocate(task);
793
794 #if MACH_ASSERT
795 assert_thread_magic(thread);
796 thread->thread_magic = 0;
797 #endif /* MACH_ASSERT */
798
799 zfree(thread_zone, thread);
800 }
801
802 void
803 thread_starts_owning_workloop(thread_t thread)
804 {
805 atomic_fetch_add_explicit(&thread->kqwl_owning_count, 1,
806 memory_order_relaxed);
807 }
808
809 void
810 thread_ends_owning_workloop(thread_t thread)
811 {
812 __assert_only uint32_t count;
813 count = atomic_fetch_sub_explicit(&thread->kqwl_owning_count, 1,
814 memory_order_relaxed);
815 assert(count > 0);
816 }
817
818 uint32_t
819 thread_owned_workloops_count(thread_t thread)
820 {
821 return atomic_load_explicit(&thread->kqwl_owning_count,
822 memory_order_relaxed);
823 }
824
825 /*
826 * thread_inspect_deallocate:
827 *
828 * Drop a thread inspection reference.
829 */
830 void
831 thread_inspect_deallocate(
832 thread_inspect_t thread_inspect)
833 {
834 return thread_deallocate((thread_t)thread_inspect);
835 }
836
837 /*
838 * thread_exception_daemon:
839 *
840 * Deliver EXC_{RESOURCE,GUARD} exception
841 */
842 static void
843 thread_exception_daemon(void)
844 {
845 struct thread_exception_elt *elt;
846 task_t task;
847 thread_t thread;
848 exception_type_t etype;
849
850 simple_lock(&thread_exception_lock, LCK_GRP_NULL);
851 while ((elt = (struct thread_exception_elt *)dequeue_head(&thread_exception_queue)) != NULL) {
852 simple_unlock(&thread_exception_lock);
853
854 etype = elt->exception_type;
855 task = elt->exception_task;
856 thread = elt->exception_thread;
857 assert_thread_magic(thread);
858
859 kfree(elt, sizeof(*elt));
860
861 /* wait for all the threads in the task to terminate */
862 task_lock(task);
863 task_wait_till_threads_terminate_locked(task);
864 task_unlock(task);
865
866 /* Consumes the task ref returned by task_generate_corpse_internal */
867 task_deallocate(task);
868 /* Consumes the thread ref returned by task_generate_corpse_internal */
869 thread_deallocate(thread);
870
871 /* Deliver the notification, also clears the corpse. */
872 task_deliver_crash_notification(task, thread, etype, 0);
873
874 simple_lock(&thread_exception_lock, LCK_GRP_NULL);
875 }
876
877 assert_wait((event_t)&thread_exception_queue, THREAD_UNINT);
878 simple_unlock(&thread_exception_lock);
879
880 thread_block((thread_continue_t)thread_exception_daemon);
881 }
882
883 /*
884 * thread_exception_enqueue:
885 *
886 * Enqueue a corpse port to be delivered an EXC_{RESOURCE,GUARD}.
887 */
888 void
889 thread_exception_enqueue(
890 task_t task,
891 thread_t thread,
892 exception_type_t etype)
893 {
894 assert(EXC_RESOURCE == etype || EXC_GUARD == etype);
895 struct thread_exception_elt *elt = kalloc(sizeof(*elt));
896 elt->exception_type = etype;
897 elt->exception_task = task;
898 elt->exception_thread = thread;
899
900 simple_lock(&thread_exception_lock, LCK_GRP_NULL);
901 enqueue_tail(&thread_exception_queue, (queue_entry_t)elt);
902 simple_unlock(&thread_exception_lock);
903
904 thread_wakeup((event_t)&thread_exception_queue);
905 }
906
907 /*
908 * thread_copy_resource_info
909 *
910 * Copy the resource info counters from source
911 * thread to destination thread.
912 */
913 void
914 thread_copy_resource_info(
915 thread_t dst_thread,
916 thread_t src_thread)
917 {
918 dst_thread->c_switch = src_thread->c_switch;
919 dst_thread->p_switch = src_thread->p_switch;
920 dst_thread->ps_switch = src_thread->ps_switch;
921 dst_thread->precise_user_kernel_time = src_thread->precise_user_kernel_time;
922 dst_thread->user_timer = src_thread->user_timer;
923 dst_thread->user_timer_save = src_thread->user_timer_save;
924 dst_thread->system_timer = src_thread->system_timer;
925 dst_thread->system_timer_save = src_thread->system_timer_save;
926 dst_thread->runnable_timer = src_thread->runnable_timer;
927 dst_thread->vtimer_user_save = src_thread->vtimer_user_save;
928 dst_thread->vtimer_prof_save = src_thread->vtimer_prof_save;
929 dst_thread->vtimer_rlim_save = src_thread->vtimer_rlim_save;
930 dst_thread->vtimer_qos_save = src_thread->vtimer_qos_save;
931 dst_thread->syscalls_unix = src_thread->syscalls_unix;
932 dst_thread->syscalls_mach = src_thread->syscalls_mach;
933 ledger_rollup(dst_thread->t_threadledger, src_thread->t_threadledger);
934 *dst_thread->thread_io_stats = *src_thread->thread_io_stats;
935 }
936
937 /*
938 * thread_terminate_daemon:
939 *
940 * Perform final clean up for terminating threads.
941 */
942 static void
943 thread_terminate_daemon(void)
944 {
945 thread_t self, thread;
946 task_t task;
947
948 self = current_thread();
949 self->options |= TH_OPT_SYSTEM_CRITICAL;
950
951 (void)splsched();
952 simple_lock(&thread_terminate_lock, LCK_GRP_NULL);
953
954 thread_terminate_start:
955 while ((thread = qe_dequeue_head(&thread_terminate_queue, struct thread, runq_links)) != THREAD_NULL) {
956 assert_thread_magic(thread);
957
958 /*
959 * if marked for crash reporting, skip reaping.
960 * The corpse delivery thread will clear bit and enqueue
961 * for reaping when done
962 */
963 if (thread->inspection) {
964 enqueue_tail(&crashed_threads_queue, &thread->runq_links);
965 continue;
966 }
967
968 simple_unlock(&thread_terminate_lock);
969 (void)spllo();
970
971 task = thread->task;
972
973 task_lock(task);
974 task->total_user_time += timer_grab(&thread->user_timer);
975 task->total_ptime += timer_grab(&thread->ptime);
976 task->total_runnable_time += timer_grab(&thread->runnable_timer);
977 if (thread->precise_user_kernel_time) {
978 task->total_system_time += timer_grab(&thread->system_timer);
979 } else {
980 task->total_user_time += timer_grab(&thread->system_timer);
981 }
982
983 task->c_switch += thread->c_switch;
984 task->p_switch += thread->p_switch;
985 task->ps_switch += thread->ps_switch;
986
987 task->syscalls_unix += thread->syscalls_unix;
988 task->syscalls_mach += thread->syscalls_mach;
989
990 task->task_timer_wakeups_bin_1 += thread->thread_timer_wakeups_bin_1;
991 task->task_timer_wakeups_bin_2 += thread->thread_timer_wakeups_bin_2;
992 task->task_gpu_ns += ml_gpu_stat(thread);
993 task->task_energy += ml_energy_stat(thread);
994
995 #if MONOTONIC
996 mt_terminate_update(task, thread);
997 #endif /* MONOTONIC */
998
999 thread_update_qos_cpu_time(thread);
1000
1001 queue_remove(&task->threads, thread, thread_t, task_threads);
1002 task->thread_count--;
1003
1004 /*
1005 * If the task is being halted, and there is only one thread
1006 * left in the task after this one, then wakeup that thread.
1007 */
1008 if (task->thread_count == 1 && task->halting) {
1009 thread_wakeup((event_t)&task->halting);
1010 }
1011
1012 task_unlock(task);
1013
1014 lck_mtx_lock(&tasks_threads_lock);
1015 queue_remove(&threads, thread, thread_t, threads);
1016 threads_count--;
1017 lck_mtx_unlock(&tasks_threads_lock);
1018
1019 thread_deallocate(thread);
1020
1021 (void)splsched();
1022 simple_lock(&thread_terminate_lock, LCK_GRP_NULL);
1023 }
1024
1025 while ((thread = qe_dequeue_head(&thread_deallocate_queue, struct thread, runq_links)) != THREAD_NULL) {
1026 assert_thread_magic(thread);
1027
1028 simple_unlock(&thread_terminate_lock);
1029 (void)spllo();
1030
1031 thread_deallocate_complete(thread);
1032
1033 (void)splsched();
1034 simple_lock(&thread_terminate_lock, LCK_GRP_NULL);
1035 }
1036
1037 struct turnstile *turnstile;
1038 while ((turnstile = qe_dequeue_head(&turnstile_deallocate_queue, struct turnstile, ts_deallocate_link)) != TURNSTILE_NULL) {
1039 simple_unlock(&thread_terminate_lock);
1040 (void)spllo();
1041
1042 turnstile_destroy(turnstile);
1043
1044 (void)splsched();
1045 simple_lock(&thread_terminate_lock, LCK_GRP_NULL);
1046 }
1047
1048 queue_entry_t qe;
1049
1050 /*
1051 * see workq_deallocate_enqueue: struct workqueue is opaque to thread.c and
1052 * we just link pieces of memory here
1053 */
1054 while ((qe = dequeue_head(&workq_deallocate_queue))) {
1055 simple_unlock(&thread_terminate_lock);
1056 (void)spllo();
1057
1058 workq_destroy((struct workqueue *)qe);
1059
1060 (void)splsched();
1061 simple_lock(&thread_terminate_lock, LCK_GRP_NULL);
1062 }
1063
1064 /*
1065 * Check if something enqueued in thread terminate/deallocate queue
1066 * while processing workq deallocate queue
1067 */
1068 if (!queue_empty(&thread_terminate_queue) ||
1069 !queue_empty(&thread_deallocate_queue) ||
1070 !queue_empty(&turnstile_deallocate_queue)) {
1071 goto thread_terminate_start;
1072 }
1073
1074 assert_wait((event_t)&thread_terminate_queue, THREAD_UNINT);
1075 simple_unlock(&thread_terminate_lock);
1076 /* splsched */
1077
1078 self->options &= ~TH_OPT_SYSTEM_CRITICAL;
1079 thread_block((thread_continue_t)thread_terminate_daemon);
1080 /*NOTREACHED*/
1081 }
1082
1083 /*
1084 * thread_terminate_enqueue:
1085 *
1086 * Enqueue a terminating thread for final disposition.
1087 *
1088 * Called at splsched.
1089 */
1090 void
1091 thread_terminate_enqueue(
1092 thread_t thread)
1093 {
1094 KDBG_RELEASE(TRACE_DATA_THREAD_TERMINATE, thread->thread_id);
1095
1096 simple_lock(&thread_terminate_lock, LCK_GRP_NULL);
1097 enqueue_tail(&thread_terminate_queue, &thread->runq_links);
1098 simple_unlock(&thread_terminate_lock);
1099
1100 thread_wakeup((event_t)&thread_terminate_queue);
1101 }
1102
1103 /*
1104 * thread_deallocate_enqueue:
1105 *
1106 * Enqueue a thread for final deallocation.
1107 */
1108 static void
1109 thread_deallocate_enqueue(
1110 thread_t thread)
1111 {
1112 spl_t s = splsched();
1113
1114 simple_lock(&thread_terminate_lock, LCK_GRP_NULL);
1115 enqueue_tail(&thread_deallocate_queue, &thread->runq_links);
1116 simple_unlock(&thread_terminate_lock);
1117
1118 thread_wakeup((event_t)&thread_terminate_queue);
1119 splx(s);
1120 }
1121
1122 /*
1123 * turnstile_deallocate_enqueue:
1124 *
1125 * Enqueue a turnstile for final deallocation.
1126 */
1127 void
1128 turnstile_deallocate_enqueue(
1129 struct turnstile *turnstile)
1130 {
1131 spl_t s = splsched();
1132
1133 simple_lock(&thread_terminate_lock, LCK_GRP_NULL);
1134 enqueue_tail(&turnstile_deallocate_queue, &turnstile->ts_deallocate_link);
1135 simple_unlock(&thread_terminate_lock);
1136
1137 thread_wakeup((event_t)&thread_terminate_queue);
1138 splx(s);
1139 }
1140
1141 /*
1142 * workq_deallocate_enqueue:
1143 *
1144 * Enqueue a workqueue for final deallocation.
1145 */
1146 void
1147 workq_deallocate_enqueue(
1148 struct workqueue *wq)
1149 {
1150 spl_t s = splsched();
1151
1152 simple_lock(&thread_terminate_lock, LCK_GRP_NULL);
1153 /*
1154 * this is just to delay a zfree(), so we link the memory with no regards
1155 * for how the struct looks like.
1156 */
1157 enqueue_tail(&workq_deallocate_queue, (queue_entry_t)wq);
1158 simple_unlock(&thread_terminate_lock);
1159
1160 thread_wakeup((event_t)&thread_terminate_queue);
1161 splx(s);
1162 }
1163
1164 /*
1165 * thread_terminate_crashed_threads:
1166 * walk the list of crashed threads and put back set of threads
1167 * who are no longer being inspected.
1168 */
1169 void
1170 thread_terminate_crashed_threads()
1171 {
1172 thread_t th_remove;
1173 boolean_t should_wake_terminate_queue = FALSE;
1174 spl_t s = splsched();
1175
1176 simple_lock(&thread_terminate_lock, LCK_GRP_NULL);
1177 /*
1178 * loop through the crashed threads queue
1179 * to put any threads that are not being inspected anymore
1180 */
1181
1182 qe_foreach_element_safe(th_remove, &crashed_threads_queue, runq_links) {
1183 /* make sure current_thread is never in crashed queue */
1184 assert(th_remove != current_thread());
1185
1186 if (th_remove->inspection == FALSE) {
1187 re_queue_tail(&thread_terminate_queue, &th_remove->runq_links);
1188 should_wake_terminate_queue = TRUE;
1189 }
1190 }
1191
1192 simple_unlock(&thread_terminate_lock);
1193 splx(s);
1194 if (should_wake_terminate_queue == TRUE) {
1195 thread_wakeup((event_t)&thread_terminate_queue);
1196 }
1197 }
1198
1199 /*
1200 * thread_stack_daemon:
1201 *
1202 * Perform stack allocation as required due to
1203 * invoke failures.
1204 */
1205 static void
1206 thread_stack_daemon(void)
1207 {
1208 thread_t thread;
1209 spl_t s;
1210
1211 s = splsched();
1212 simple_lock(&thread_stack_lock, LCK_GRP_NULL);
1213
1214 while ((thread = qe_dequeue_head(&thread_stack_queue, struct thread, runq_links)) != THREAD_NULL) {
1215 assert_thread_magic(thread);
1216
1217 simple_unlock(&thread_stack_lock);
1218 splx(s);
1219
1220 /* allocate stack with interrupts enabled so that we can call into VM */
1221 stack_alloc(thread);
1222
1223 KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_STACK_WAIT) | DBG_FUNC_END, thread_tid(thread), 0, 0, 0, 0);
1224
1225 s = splsched();
1226 thread_lock(thread);
1227 thread_setrun(thread, SCHED_PREEMPT | SCHED_TAILQ);
1228 thread_unlock(thread);
1229
1230 simple_lock(&thread_stack_lock, LCK_GRP_NULL);
1231 }
1232
1233 assert_wait((event_t)&thread_stack_queue, THREAD_UNINT);
1234 simple_unlock(&thread_stack_lock);
1235 splx(s);
1236
1237 thread_block((thread_continue_t)thread_stack_daemon);
1238 /*NOTREACHED*/
1239 }
1240
1241 /*
1242 * thread_stack_enqueue:
1243 *
1244 * Enqueue a thread for stack allocation.
1245 *
1246 * Called at splsched.
1247 */
1248 void
1249 thread_stack_enqueue(
1250 thread_t thread)
1251 {
1252 KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_STACK_WAIT) | DBG_FUNC_START, thread_tid(thread), 0, 0, 0, 0);
1253 assert_thread_magic(thread);
1254
1255 simple_lock(&thread_stack_lock, LCK_GRP_NULL);
1256 enqueue_tail(&thread_stack_queue, &thread->runq_links);
1257 simple_unlock(&thread_stack_lock);
1258
1259 thread_wakeup((event_t)&thread_stack_queue);
1260 }
1261
1262 void
1263 thread_daemon_init(void)
1264 {
1265 kern_return_t result;
1266 thread_t thread = NULL;
1267
1268 simple_lock_init(&thread_terminate_lock, 0);
1269 queue_init(&thread_terminate_queue);
1270 queue_init(&thread_deallocate_queue);
1271 queue_init(&workq_deallocate_queue);
1272 queue_init(&turnstile_deallocate_queue);
1273 queue_init(&crashed_threads_queue);
1274
1275 result = kernel_thread_start_priority((thread_continue_t)thread_terminate_daemon, NULL, MINPRI_KERNEL, &thread);
1276 if (result != KERN_SUCCESS) {
1277 panic("thread_daemon_init: thread_terminate_daemon");
1278 }
1279
1280 thread_deallocate(thread);
1281
1282 simple_lock_init(&thread_stack_lock, 0);
1283 queue_init(&thread_stack_queue);
1284
1285 result = kernel_thread_start_priority((thread_continue_t)thread_stack_daemon, NULL, BASEPRI_PREEMPT_HIGH, &thread);
1286 if (result != KERN_SUCCESS) {
1287 panic("thread_daemon_init: thread_stack_daemon");
1288 }
1289
1290 thread_deallocate(thread);
1291
1292 simple_lock_init(&thread_exception_lock, 0);
1293 queue_init(&thread_exception_queue);
1294
1295 result = kernel_thread_start_priority((thread_continue_t)thread_exception_daemon, NULL, MINPRI_KERNEL, &thread);
1296 if (result != KERN_SUCCESS) {
1297 panic("thread_daemon_init: thread_exception_daemon");
1298 }
1299
1300 thread_deallocate(thread);
1301 }
1302
1303 #define TH_OPTION_NONE 0x00
1304 #define TH_OPTION_NOCRED 0x01
1305 #define TH_OPTION_NOSUSP 0x02
1306 #define TH_OPTION_WORKQ 0x04
1307
1308 /*
1309 * Create a new thread.
1310 * Doesn't start the thread running.
1311 *
1312 * Task and tasks_threads_lock are returned locked on success.
1313 */
1314 static kern_return_t
1315 thread_create_internal(
1316 task_t parent_task,
1317 integer_t priority,
1318 thread_continue_t continuation,
1319 void *parameter,
1320 int options,
1321 thread_t *out_thread)
1322 {
1323 thread_t new_thread;
1324 static thread_t first_thread;
1325
1326 /*
1327 * Allocate a thread and initialize static fields
1328 */
1329 if (first_thread == THREAD_NULL) {
1330 new_thread = first_thread = current_thread();
1331 } else {
1332 new_thread = (thread_t)zalloc(thread_zone);
1333 }
1334 if (new_thread == THREAD_NULL) {
1335 return KERN_RESOURCE_SHORTAGE;
1336 }
1337
1338 if (new_thread != first_thread) {
1339 *new_thread = thread_template;
1340 }
1341
1342 os_ref_init_count(&new_thread->ref_count, &thread_refgrp, 2);
1343
1344 #ifdef MACH_BSD
1345 new_thread->uthread = uthread_alloc(parent_task, new_thread, (options & TH_OPTION_NOCRED) != 0);
1346 if (new_thread->uthread == NULL) {
1347 #if MACH_ASSERT
1348 new_thread->thread_magic = 0;
1349 #endif /* MACH_ASSERT */
1350
1351 zfree(thread_zone, new_thread);
1352 return KERN_RESOURCE_SHORTAGE;
1353 }
1354 #endif /* MACH_BSD */
1355
1356 if (machine_thread_create(new_thread, parent_task) != KERN_SUCCESS) {
1357 #ifdef MACH_BSD
1358 void *ut = new_thread->uthread;
1359
1360 new_thread->uthread = NULL;
1361 /* cred free may not be necessary */
1362 uthread_cleanup(parent_task, ut, parent_task->bsd_info);
1363 uthread_cred_free(ut);
1364 uthread_zone_free(ut);
1365 #endif /* MACH_BSD */
1366
1367 #if MACH_ASSERT
1368 new_thread->thread_magic = 0;
1369 #endif /* MACH_ASSERT */
1370
1371 zfree(thread_zone, new_thread);
1372 return KERN_FAILURE;
1373 }
1374
1375 new_thread->task = parent_task;
1376
1377 thread_lock_init(new_thread);
1378 wake_lock_init(new_thread);
1379
1380 lck_mtx_init(&new_thread->mutex, &thread_lck_grp, &thread_lck_attr);
1381
1382 ipc_thread_init(new_thread);
1383
1384 new_thread->continuation = continuation;
1385 new_thread->parameter = parameter;
1386 new_thread->inheritor_flags = TURNSTILE_UPDATE_FLAGS_NONE;
1387 priority_queue_init(&new_thread->inheritor_queue,
1388 PRIORITY_QUEUE_BUILTIN_MAX_HEAP);
1389
1390 /* Allocate I/O Statistics structure */
1391 new_thread->thread_io_stats = (io_stat_info_t)kalloc(sizeof(struct io_stat_info));
1392 assert(new_thread->thread_io_stats != NULL);
1393 bzero(new_thread->thread_io_stats, sizeof(struct io_stat_info));
1394 new_thread->sync_ipc_overrides = 0;
1395
1396 #if KASAN
1397 kasan_init_thread(&new_thread->kasan_data);
1398 #endif
1399
1400 #if CONFIG_IOSCHED
1401 /* Clear out the I/O Scheduling info for AppleFSCompression */
1402 new_thread->decmp_upl = NULL;
1403 #endif /* CONFIG_IOSCHED */
1404
1405 #if DEVELOPMENT || DEBUG
1406 task_lock(parent_task);
1407 uint16_t thread_limit = parent_task->task_thread_limit;
1408 if (exc_resource_threads_enabled &&
1409 thread_limit > 0 &&
1410 parent_task->thread_count >= thread_limit &&
1411 !parent_task->task_has_crossed_thread_limit &&
1412 !(parent_task->t_flags & TF_CORPSE)) {
1413 int thread_count = parent_task->thread_count;
1414 parent_task->task_has_crossed_thread_limit = TRUE;
1415 task_unlock(parent_task);
1416 SENDING_NOTIFICATION__TASK_HAS_TOO_MANY_THREADS(parent_task, thread_count);
1417 } else {
1418 task_unlock(parent_task);
1419 }
1420 #endif
1421
1422 lck_mtx_lock(&tasks_threads_lock);
1423 task_lock(parent_task);
1424
1425 /*
1426 * Fail thread creation if parent task is being torn down or has too many threads
1427 * If the caller asked for TH_OPTION_NOSUSP, also fail if the parent task is suspended
1428 */
1429 if (parent_task->active == 0 || parent_task->halting ||
1430 (parent_task->suspend_count > 0 && (options & TH_OPTION_NOSUSP) != 0) ||
1431 (parent_task->thread_count >= task_threadmax && parent_task != kernel_task)) {
1432 task_unlock(parent_task);
1433 lck_mtx_unlock(&tasks_threads_lock);
1434
1435 #ifdef MACH_BSD
1436 {
1437 void *ut = new_thread->uthread;
1438
1439 new_thread->uthread = NULL;
1440 uthread_cleanup(parent_task, ut, parent_task->bsd_info);
1441 /* cred free may not be necessary */
1442 uthread_cred_free(ut);
1443 uthread_zone_free(ut);
1444 }
1445 #endif /* MACH_BSD */
1446 ipc_thread_disable(new_thread);
1447 ipc_thread_terminate(new_thread);
1448 kfree(new_thread->thread_io_stats, sizeof(struct io_stat_info));
1449 lck_mtx_destroy(&new_thread->mutex, &thread_lck_grp);
1450 machine_thread_destroy(new_thread);
1451 zfree(thread_zone, new_thread);
1452 return KERN_FAILURE;
1453 }
1454
1455 /* Protected by the tasks_threads_lock */
1456 new_thread->thread_id = ++thread_unique_id;
1457
1458 /* New threads inherit any default state on the task */
1459 machine_thread_inherit_taskwide(new_thread, parent_task);
1460
1461 task_reference_internal(parent_task);
1462
1463 if (new_thread->task->rusage_cpu_flags & TASK_RUSECPU_FLAGS_PERTHR_LIMIT) {
1464 /*
1465 * This task has a per-thread CPU limit; make sure this new thread
1466 * gets its limit set too, before it gets out of the kernel.
1467 */
1468 act_set_astledger(new_thread);
1469 }
1470
1471 /* Instantiate a thread ledger. Do not fail thread creation if ledger creation fails. */
1472 if ((new_thread->t_threadledger = ledger_instantiate(thread_ledger_template,
1473 LEDGER_CREATE_INACTIVE_ENTRIES)) != LEDGER_NULL) {
1474 ledger_entry_setactive(new_thread->t_threadledger, thread_ledgers.cpu_time);
1475 }
1476
1477 new_thread->t_bankledger = LEDGER_NULL;
1478 new_thread->t_deduct_bank_ledger_time = 0;
1479 new_thread->t_deduct_bank_ledger_energy = 0;
1480
1481 new_thread->t_ledger = new_thread->task->ledger;
1482 if (new_thread->t_ledger) {
1483 ledger_reference(new_thread->t_ledger);
1484 }
1485
1486 #if defined(CONFIG_SCHED_MULTIQ)
1487 /* Cache the task's sched_group */
1488 new_thread->sched_group = parent_task->sched_group;
1489 #endif /* defined(CONFIG_SCHED_MULTIQ) */
1490
1491 /* Cache the task's map */
1492 new_thread->map = parent_task->map;
1493
1494 timer_call_setup(&new_thread->wait_timer, thread_timer_expire, new_thread);
1495 timer_call_setup(&new_thread->depress_timer, thread_depress_expire, new_thread);
1496
1497 #if KPC
1498 kpc_thread_create(new_thread);
1499 #endif
1500
1501 /* Set the thread's scheduling parameters */
1502 new_thread->sched_mode = SCHED(initial_thread_sched_mode)(parent_task);
1503 new_thread->max_priority = parent_task->max_priority;
1504 new_thread->task_priority = parent_task->priority;
1505
1506 int new_priority = (priority < 0) ? parent_task->priority: priority;
1507 new_priority = (priority < 0)? parent_task->priority: priority;
1508 if (new_priority > new_thread->max_priority) {
1509 new_priority = new_thread->max_priority;
1510 }
1511 #if CONFIG_EMBEDDED
1512 if (new_priority < MAXPRI_THROTTLE) {
1513 new_priority = MAXPRI_THROTTLE;
1514 }
1515 #endif /* CONFIG_EMBEDDED */
1516
1517 new_thread->importance = new_priority - new_thread->task_priority;
1518
1519 sched_set_thread_base_priority(new_thread, new_priority);
1520
1521 #if defined(CONFIG_SCHED_TIMESHARE_CORE)
1522 new_thread->sched_stamp = sched_tick;
1523 new_thread->pri_shift = sched_pri_shifts[new_thread->th_sched_bucket];
1524 #endif /* defined(CONFIG_SCHED_TIMESHARE_CORE) */
1525
1526 #if CONFIG_EMBEDDED
1527 if (parent_task->max_priority <= MAXPRI_THROTTLE) {
1528 sched_thread_mode_demote(new_thread, TH_SFLAG_THROTTLED);
1529 }
1530 #endif /* CONFIG_EMBEDDED */
1531
1532 thread_policy_create(new_thread);
1533
1534 /* Chain the thread onto the task's list */
1535 queue_enter(&parent_task->threads, new_thread, thread_t, task_threads);
1536 parent_task->thread_count++;
1537
1538 /* So terminating threads don't need to take the task lock to decrement */
1539 hw_atomic_add(&parent_task->active_thread_count, 1);
1540
1541
1542 queue_enter(&threads, new_thread, thread_t, threads);
1543 threads_count++;
1544
1545 new_thread->active = TRUE;
1546 if (task_is_a_corpse_fork(parent_task)) {
1547 /* Set the inspection bit if the task is a corpse fork */
1548 new_thread->inspection = TRUE;
1549 } else {
1550 new_thread->inspection = FALSE;
1551 }
1552 new_thread->corpse_dup = FALSE;
1553 new_thread->turnstile = turnstile_alloc();
1554 *out_thread = new_thread;
1555
1556 if (kdebug_enable) {
1557 long args[4] = {};
1558
1559 kdbg_trace_data(parent_task->bsd_info, &args[1], &args[3]);
1560
1561 /*
1562 * Starting with 26604425, exec'ing creates a new task/thread.
1563 *
1564 * NEWTHREAD in the current process has two possible meanings:
1565 *
1566 * 1) Create a new thread for this process.
1567 * 2) Create a new thread for the future process this will become in an
1568 * exec.
1569 *
1570 * To disambiguate these, arg3 will be set to TRUE for case #2.
1571 *
1572 * The value we need to find (TPF_EXEC_COPY) is stable in the case of a
1573 * task exec'ing. The read of t_procflags does not take the proc_lock.
1574 */
1575 args[2] = task_is_exec_copy(parent_task) ? 1 : 0;
1576
1577 KDBG_RELEASE(TRACE_DATA_NEWTHREAD, (uintptr_t)thread_tid(new_thread),
1578 args[1], args[2], args[3]);
1579
1580 kdbg_trace_string(parent_task->bsd_info, &args[0], &args[1],
1581 &args[2], &args[3]);
1582 KDBG_RELEASE(TRACE_STRING_NEWTHREAD, args[0], args[1], args[2],
1583 args[3]);
1584 }
1585
1586 DTRACE_PROC1(lwp__create, thread_t, *out_thread);
1587
1588 return KERN_SUCCESS;
1589 }
1590
1591 static kern_return_t
1592 thread_create_internal2(
1593 task_t task,
1594 thread_t *new_thread,
1595 boolean_t from_user,
1596 thread_continue_t continuation)
1597 {
1598 kern_return_t result;
1599 thread_t thread;
1600
1601 if (task == TASK_NULL || task == kernel_task) {
1602 return KERN_INVALID_ARGUMENT;
1603 }
1604
1605 result = thread_create_internal(task, -1, continuation, NULL, TH_OPTION_NONE, &thread);
1606 if (result != KERN_SUCCESS) {
1607 return result;
1608 }
1609
1610 thread->user_stop_count = 1;
1611 thread_hold(thread);
1612 if (task->suspend_count > 0) {
1613 thread_hold(thread);
1614 }
1615
1616 if (from_user) {
1617 extmod_statistics_incr_thread_create(task);
1618 }
1619
1620 task_unlock(task);
1621 lck_mtx_unlock(&tasks_threads_lock);
1622
1623 *new_thread = thread;
1624
1625 return KERN_SUCCESS;
1626 }
1627
1628 /* No prototype, since task_server.h has the _from_user version if KERNEL_SERVER */
1629 kern_return_t
1630 thread_create(
1631 task_t task,
1632 thread_t *new_thread);
1633
1634 kern_return_t
1635 thread_create(
1636 task_t task,
1637 thread_t *new_thread)
1638 {
1639 return thread_create_internal2(task, new_thread, FALSE, (thread_continue_t)thread_bootstrap_return);
1640 }
1641
1642 kern_return_t
1643 thread_create_from_user(
1644 task_t task,
1645 thread_t *new_thread)
1646 {
1647 return thread_create_internal2(task, new_thread, TRUE, (thread_continue_t)thread_bootstrap_return);
1648 }
1649
1650 kern_return_t
1651 thread_create_with_continuation(
1652 task_t task,
1653 thread_t *new_thread,
1654 thread_continue_t continuation)
1655 {
1656 return thread_create_internal2(task, new_thread, FALSE, continuation);
1657 }
1658
1659 /*
1660 * Create a thread that is already started, but is waiting on an event
1661 */
1662 static kern_return_t
1663 thread_create_waiting_internal(
1664 task_t task,
1665 thread_continue_t continuation,
1666 event_t event,
1667 block_hint_t block_hint,
1668 int options,
1669 thread_t *new_thread)
1670 {
1671 kern_return_t result;
1672 thread_t thread;
1673
1674 if (task == TASK_NULL || task == kernel_task) {
1675 return KERN_INVALID_ARGUMENT;
1676 }
1677
1678 result = thread_create_internal(task, -1, continuation, NULL,
1679 options, &thread);
1680 if (result != KERN_SUCCESS) {
1681 return result;
1682 }
1683
1684 /* note no user_stop_count or thread_hold here */
1685
1686 if (task->suspend_count > 0) {
1687 thread_hold(thread);
1688 }
1689
1690 thread_mtx_lock(thread);
1691 thread_set_pending_block_hint(thread, block_hint);
1692 if (options & TH_OPTION_WORKQ) {
1693 thread->static_param = true;
1694 event = workq_thread_init_and_wq_lock(task, thread);
1695 }
1696 thread_start_in_assert_wait(thread, event, THREAD_INTERRUPTIBLE);
1697 thread_mtx_unlock(thread);
1698
1699 task_unlock(task);
1700 lck_mtx_unlock(&tasks_threads_lock);
1701
1702 *new_thread = thread;
1703
1704 return KERN_SUCCESS;
1705 }
1706
1707 kern_return_t
1708 thread_create_waiting(
1709 task_t task,
1710 thread_continue_t continuation,
1711 event_t event,
1712 thread_t *new_thread)
1713 {
1714 return thread_create_waiting_internal(task, continuation, event,
1715 kThreadWaitNone, TH_OPTION_NONE, new_thread);
1716 }
1717
1718
1719 static kern_return_t
1720 thread_create_running_internal2(
1721 task_t task,
1722 int flavor,
1723 thread_state_t new_state,
1724 mach_msg_type_number_t new_state_count,
1725 thread_t *new_thread,
1726 boolean_t from_user)
1727 {
1728 kern_return_t result;
1729 thread_t thread;
1730
1731 if (task == TASK_NULL || task == kernel_task) {
1732 return KERN_INVALID_ARGUMENT;
1733 }
1734
1735 result = thread_create_internal(task, -1,
1736 (thread_continue_t)thread_bootstrap_return, NULL,
1737 TH_OPTION_NONE, &thread);
1738 if (result != KERN_SUCCESS) {
1739 return result;
1740 }
1741
1742 if (task->suspend_count > 0) {
1743 thread_hold(thread);
1744 }
1745
1746 if (from_user) {
1747 result = machine_thread_state_convert_from_user(thread, flavor,
1748 new_state, new_state_count);
1749 }
1750 if (result == KERN_SUCCESS) {
1751 result = machine_thread_set_state(thread, flavor, new_state,
1752 new_state_count);
1753 }
1754 if (result != KERN_SUCCESS) {
1755 task_unlock(task);
1756 lck_mtx_unlock(&tasks_threads_lock);
1757
1758 thread_terminate(thread);
1759 thread_deallocate(thread);
1760 return result;
1761 }
1762
1763 thread_mtx_lock(thread);
1764 thread_start(thread);
1765 thread_mtx_unlock(thread);
1766
1767 if (from_user) {
1768 extmod_statistics_incr_thread_create(task);
1769 }
1770
1771 task_unlock(task);
1772 lck_mtx_unlock(&tasks_threads_lock);
1773
1774 *new_thread = thread;
1775
1776 return result;
1777 }
1778
1779 /* Prototype, see justification above */
1780 kern_return_t
1781 thread_create_running(
1782 task_t task,
1783 int flavor,
1784 thread_state_t new_state,
1785 mach_msg_type_number_t new_state_count,
1786 thread_t *new_thread);
1787
1788 kern_return_t
1789 thread_create_running(
1790 task_t task,
1791 int flavor,
1792 thread_state_t new_state,
1793 mach_msg_type_number_t new_state_count,
1794 thread_t *new_thread)
1795 {
1796 return thread_create_running_internal2(
1797 task, flavor, new_state, new_state_count,
1798 new_thread, FALSE);
1799 }
1800
1801 kern_return_t
1802 thread_create_running_from_user(
1803 task_t task,
1804 int flavor,
1805 thread_state_t new_state,
1806 mach_msg_type_number_t new_state_count,
1807 thread_t *new_thread)
1808 {
1809 return thread_create_running_internal2(
1810 task, flavor, new_state, new_state_count,
1811 new_thread, TRUE);
1812 }
1813
1814 kern_return_t
1815 thread_create_workq_waiting(
1816 task_t task,
1817 thread_continue_t continuation,
1818 thread_t *new_thread)
1819 {
1820 int options = TH_OPTION_NOCRED | TH_OPTION_NOSUSP | TH_OPTION_WORKQ;
1821 return thread_create_waiting_internal(task, continuation, NULL,
1822 kThreadWaitParkedWorkQueue, options, new_thread);
1823 }
1824
1825 /*
1826 * kernel_thread_create:
1827 *
1828 * Create a thread in the kernel task
1829 * to execute in kernel context.
1830 */
1831 kern_return_t
1832 kernel_thread_create(
1833 thread_continue_t continuation,
1834 void *parameter,
1835 integer_t priority,
1836 thread_t *new_thread)
1837 {
1838 kern_return_t result;
1839 thread_t thread;
1840 task_t task = kernel_task;
1841
1842 result = thread_create_internal(task, priority, continuation, parameter,
1843 TH_OPTION_NOCRED | TH_OPTION_NONE, &thread);
1844 if (result != KERN_SUCCESS) {
1845 return result;
1846 }
1847
1848 task_unlock(task);
1849 lck_mtx_unlock(&tasks_threads_lock);
1850
1851 stack_alloc(thread);
1852 assert(thread->kernel_stack != 0);
1853 #if CONFIG_EMBEDDED
1854 if (priority > BASEPRI_KERNEL)
1855 #endif
1856 thread->reserved_stack = thread->kernel_stack;
1857
1858 if (debug_task & 1) {
1859 kprintf("kernel_thread_create: thread = %p continuation = %p\n", thread, continuation);
1860 }
1861 *new_thread = thread;
1862
1863 return result;
1864 }
1865
1866 kern_return_t
1867 kernel_thread_start_priority(
1868 thread_continue_t continuation,
1869 void *parameter,
1870 integer_t priority,
1871 thread_t *new_thread)
1872 {
1873 kern_return_t result;
1874 thread_t thread;
1875
1876 result = kernel_thread_create(continuation, parameter, priority, &thread);
1877 if (result != KERN_SUCCESS) {
1878 return result;
1879 }
1880
1881 *new_thread = thread;
1882
1883 thread_mtx_lock(thread);
1884 thread_start(thread);
1885 thread_mtx_unlock(thread);
1886
1887 return result;
1888 }
1889
1890 kern_return_t
1891 kernel_thread_start(
1892 thread_continue_t continuation,
1893 void *parameter,
1894 thread_t *new_thread)
1895 {
1896 return kernel_thread_start_priority(continuation, parameter, -1, new_thread);
1897 }
1898
1899 /* Separated into helper function so it can be used by THREAD_BASIC_INFO and THREAD_EXTENDED_INFO */
1900 /* it is assumed that the thread is locked by the caller */
1901 static void
1902 retrieve_thread_basic_info(thread_t thread, thread_basic_info_t basic_info)
1903 {
1904 int state, flags;
1905
1906 /* fill in info */
1907
1908 thread_read_times(thread, &basic_info->user_time,
1909 &basic_info->system_time, NULL);
1910
1911 /*
1912 * Update lazy-evaluated scheduler info because someone wants it.
1913 */
1914 if (SCHED(can_update_priority)(thread)) {
1915 SCHED(update_priority)(thread);
1916 }
1917
1918 basic_info->sleep_time = 0;
1919
1920 /*
1921 * To calculate cpu_usage, first correct for timer rate,
1922 * then for 5/8 ageing. The correction factor [3/5] is
1923 * (1/(5/8) - 1).
1924 */
1925 basic_info->cpu_usage = 0;
1926 #if defined(CONFIG_SCHED_TIMESHARE_CORE)
1927 if (sched_tick_interval) {
1928 basic_info->cpu_usage = (integer_t)(((uint64_t)thread->cpu_usage
1929 * TH_USAGE_SCALE) / sched_tick_interval);
1930 basic_info->cpu_usage = (basic_info->cpu_usage * 3) / 5;
1931 }
1932 #endif
1933
1934 if (basic_info->cpu_usage > TH_USAGE_SCALE) {
1935 basic_info->cpu_usage = TH_USAGE_SCALE;
1936 }
1937
1938 basic_info->policy = ((thread->sched_mode == TH_MODE_TIMESHARE)?
1939 POLICY_TIMESHARE: POLICY_RR);
1940
1941 flags = 0;
1942 if (thread->options & TH_OPT_IDLE_THREAD) {
1943 flags |= TH_FLAGS_IDLE;
1944 }
1945
1946 if (thread->options & TH_OPT_GLOBAL_FORCED_IDLE) {
1947 flags |= TH_FLAGS_GLOBAL_FORCED_IDLE;
1948 }
1949
1950 if (!thread->kernel_stack) {
1951 flags |= TH_FLAGS_SWAPPED;
1952 }
1953
1954 state = 0;
1955 if (thread->state & TH_TERMINATE) {
1956 state = TH_STATE_HALTED;
1957 } else if (thread->state & TH_RUN) {
1958 state = TH_STATE_RUNNING;
1959 } else if (thread->state & TH_UNINT) {
1960 state = TH_STATE_UNINTERRUPTIBLE;
1961 } else if (thread->state & TH_SUSP) {
1962 state = TH_STATE_STOPPED;
1963 } else if (thread->state & TH_WAIT) {
1964 state = TH_STATE_WAITING;
1965 }
1966
1967 basic_info->run_state = state;
1968 basic_info->flags = flags;
1969
1970 basic_info->suspend_count = thread->user_stop_count;
1971
1972 return;
1973 }
1974
1975 kern_return_t
1976 thread_info_internal(
1977 thread_t thread,
1978 thread_flavor_t flavor,
1979 thread_info_t thread_info_out, /* ptr to OUT array */
1980 mach_msg_type_number_t *thread_info_count) /*IN/OUT*/
1981 {
1982 spl_t s;
1983
1984 if (thread == THREAD_NULL) {
1985 return KERN_INVALID_ARGUMENT;
1986 }
1987
1988 if (flavor == THREAD_BASIC_INFO) {
1989 if (*thread_info_count < THREAD_BASIC_INFO_COUNT) {
1990 return KERN_INVALID_ARGUMENT;
1991 }
1992
1993 s = splsched();
1994 thread_lock(thread);
1995
1996 retrieve_thread_basic_info(thread, (thread_basic_info_t) thread_info_out);
1997
1998 thread_unlock(thread);
1999 splx(s);
2000
2001 *thread_info_count = THREAD_BASIC_INFO_COUNT;
2002
2003 return KERN_SUCCESS;
2004 } else if (flavor == THREAD_IDENTIFIER_INFO) {
2005 thread_identifier_info_t identifier_info;
2006
2007 if (*thread_info_count < THREAD_IDENTIFIER_INFO_COUNT) {
2008 return KERN_INVALID_ARGUMENT;
2009 }
2010
2011 identifier_info = (thread_identifier_info_t) thread_info_out;
2012
2013 s = splsched();
2014 thread_lock(thread);
2015
2016 identifier_info->thread_id = thread->thread_id;
2017 identifier_info->thread_handle = thread->machine.cthread_self;
2018 identifier_info->dispatch_qaddr = thread_dispatchqaddr(thread);
2019
2020 thread_unlock(thread);
2021 splx(s);
2022 return KERN_SUCCESS;
2023 } else if (flavor == THREAD_SCHED_TIMESHARE_INFO) {
2024 policy_timeshare_info_t ts_info;
2025
2026 if (*thread_info_count < POLICY_TIMESHARE_INFO_COUNT) {
2027 return KERN_INVALID_ARGUMENT;
2028 }
2029
2030 ts_info = (policy_timeshare_info_t)thread_info_out;
2031
2032 s = splsched();
2033 thread_lock(thread);
2034
2035 if (thread->sched_mode != TH_MODE_TIMESHARE) {
2036 thread_unlock(thread);
2037 splx(s);
2038 return KERN_INVALID_POLICY;
2039 }
2040
2041 ts_info->depressed = (thread->sched_flags & TH_SFLAG_DEPRESSED_MASK) != 0;
2042 if (ts_info->depressed) {
2043 ts_info->base_priority = DEPRESSPRI;
2044 ts_info->depress_priority = thread->base_pri;
2045 } else {
2046 ts_info->base_priority = thread->base_pri;
2047 ts_info->depress_priority = -1;
2048 }
2049
2050 ts_info->cur_priority = thread->sched_pri;
2051 ts_info->max_priority = thread->max_priority;
2052
2053 thread_unlock(thread);
2054 splx(s);
2055
2056 *thread_info_count = POLICY_TIMESHARE_INFO_COUNT;
2057
2058 return KERN_SUCCESS;
2059 } else if (flavor == THREAD_SCHED_FIFO_INFO) {
2060 if (*thread_info_count < POLICY_FIFO_INFO_COUNT) {
2061 return KERN_INVALID_ARGUMENT;
2062 }
2063
2064 return KERN_INVALID_POLICY;
2065 } else if (flavor == THREAD_SCHED_RR_INFO) {
2066 policy_rr_info_t rr_info;
2067 uint32_t quantum_time;
2068 uint64_t quantum_ns;
2069
2070 if (*thread_info_count < POLICY_RR_INFO_COUNT) {
2071 return KERN_INVALID_ARGUMENT;
2072 }
2073
2074 rr_info = (policy_rr_info_t) thread_info_out;
2075
2076 s = splsched();
2077 thread_lock(thread);
2078
2079 if (thread->sched_mode == TH_MODE_TIMESHARE) {
2080 thread_unlock(thread);
2081 splx(s);
2082
2083 return KERN_INVALID_POLICY;
2084 }
2085
2086 rr_info->depressed = (thread->sched_flags & TH_SFLAG_DEPRESSED_MASK) != 0;
2087 if (rr_info->depressed) {
2088 rr_info->base_priority = DEPRESSPRI;
2089 rr_info->depress_priority = thread->base_pri;
2090 } else {
2091 rr_info->base_priority = thread->base_pri;
2092 rr_info->depress_priority = -1;
2093 }
2094
2095 quantum_time = SCHED(initial_quantum_size)(THREAD_NULL);
2096 absolutetime_to_nanoseconds(quantum_time, &quantum_ns);
2097
2098 rr_info->max_priority = thread->max_priority;
2099 rr_info->quantum = (uint32_t)(quantum_ns / 1000 / 1000);
2100
2101 thread_unlock(thread);
2102 splx(s);
2103
2104 *thread_info_count = POLICY_RR_INFO_COUNT;
2105
2106 return KERN_SUCCESS;
2107 } else if (flavor == THREAD_EXTENDED_INFO) {
2108 thread_basic_info_data_t basic_info;
2109 thread_extended_info_t extended_info = (thread_extended_info_t) thread_info_out;
2110
2111 if (*thread_info_count < THREAD_EXTENDED_INFO_COUNT) {
2112 return KERN_INVALID_ARGUMENT;
2113 }
2114
2115 s = splsched();
2116 thread_lock(thread);
2117
2118 /* NOTE: This mimics fill_taskthreadinfo(), which is the function used by proc_pidinfo() for
2119 * the PROC_PIDTHREADINFO flavor (which can't be used on corpses)
2120 */
2121 retrieve_thread_basic_info(thread, &basic_info);
2122 extended_info->pth_user_time = ((basic_info.user_time.seconds * (integer_t)NSEC_PER_SEC) + (basic_info.user_time.microseconds * (integer_t)NSEC_PER_USEC));
2123 extended_info->pth_system_time = ((basic_info.system_time.seconds * (integer_t)NSEC_PER_SEC) + (basic_info.system_time.microseconds * (integer_t)NSEC_PER_USEC));
2124
2125 extended_info->pth_cpu_usage = basic_info.cpu_usage;
2126 extended_info->pth_policy = basic_info.policy;
2127 extended_info->pth_run_state = basic_info.run_state;
2128 extended_info->pth_flags = basic_info.flags;
2129 extended_info->pth_sleep_time = basic_info.sleep_time;
2130 extended_info->pth_curpri = thread->sched_pri;
2131 extended_info->pth_priority = thread->base_pri;
2132 extended_info->pth_maxpriority = thread->max_priority;
2133
2134 bsd_getthreadname(thread->uthread, extended_info->pth_name);
2135
2136 thread_unlock(thread);
2137 splx(s);
2138
2139 *thread_info_count = THREAD_EXTENDED_INFO_COUNT;
2140
2141 return KERN_SUCCESS;
2142 } else if (flavor == THREAD_DEBUG_INFO_INTERNAL) {
2143 #if DEVELOPMENT || DEBUG
2144 thread_debug_info_internal_t dbg_info;
2145 if (*thread_info_count < THREAD_DEBUG_INFO_INTERNAL_COUNT) {
2146 return KERN_NOT_SUPPORTED;
2147 }
2148
2149 if (thread_info_out == NULL) {
2150 return KERN_INVALID_ARGUMENT;
2151 }
2152
2153 dbg_info = (thread_debug_info_internal_t) thread_info_out;
2154 dbg_info->page_creation_count = thread->t_page_creation_count;
2155
2156 *thread_info_count = THREAD_DEBUG_INFO_INTERNAL_COUNT;
2157 return KERN_SUCCESS;
2158 #endif /* DEVELOPMENT || DEBUG */
2159 return KERN_NOT_SUPPORTED;
2160 }
2161
2162 return KERN_INVALID_ARGUMENT;
2163 }
2164
2165 void
2166 thread_read_times(
2167 thread_t thread,
2168 time_value_t *user_time,
2169 time_value_t *system_time,
2170 time_value_t *runnable_time)
2171 {
2172 clock_sec_t secs;
2173 clock_usec_t usecs;
2174 uint64_t tval_user, tval_system;
2175
2176 tval_user = timer_grab(&thread->user_timer);
2177 tval_system = timer_grab(&thread->system_timer);
2178
2179 if (thread->precise_user_kernel_time) {
2180 absolutetime_to_microtime(tval_user, &secs, &usecs);
2181 user_time->seconds = (typeof(user_time->seconds))secs;
2182 user_time->microseconds = usecs;
2183
2184 absolutetime_to_microtime(tval_system, &secs, &usecs);
2185 system_time->seconds = (typeof(system_time->seconds))secs;
2186 system_time->microseconds = usecs;
2187 } else {
2188 /* system_timer may represent either sys or user */
2189 tval_user += tval_system;
2190 absolutetime_to_microtime(tval_user, &secs, &usecs);
2191 user_time->seconds = (typeof(user_time->seconds))secs;
2192 user_time->microseconds = usecs;
2193
2194 system_time->seconds = 0;
2195 system_time->microseconds = 0;
2196 }
2197
2198 if (runnable_time) {
2199 uint64_t tval_runnable = timer_grab(&thread->runnable_timer);
2200 absolutetime_to_microtime(tval_runnable, &secs, &usecs);
2201 runnable_time->seconds = (typeof(runnable_time->seconds))secs;
2202 runnable_time->microseconds = usecs;
2203 }
2204 }
2205
2206 uint64_t
2207 thread_get_runtime_self(void)
2208 {
2209 boolean_t interrupt_state;
2210 uint64_t runtime;
2211 thread_t thread = NULL;
2212 processor_t processor = NULL;
2213
2214 thread = current_thread();
2215
2216 /* Not interrupt safe, as the scheduler may otherwise update timer values underneath us */
2217 interrupt_state = ml_set_interrupts_enabled(FALSE);
2218 processor = current_processor();
2219 timer_update(PROCESSOR_DATA(processor, thread_timer), mach_absolute_time());
2220 runtime = (timer_grab(&thread->user_timer) + timer_grab(&thread->system_timer));
2221 ml_set_interrupts_enabled(interrupt_state);
2222
2223 return runtime;
2224 }
2225
2226 kern_return_t
2227 thread_assign(
2228 __unused thread_t thread,
2229 __unused processor_set_t new_pset)
2230 {
2231 return KERN_FAILURE;
2232 }
2233
2234 /*
2235 * thread_assign_default:
2236 *
2237 * Special version of thread_assign for assigning threads to default
2238 * processor set.
2239 */
2240 kern_return_t
2241 thread_assign_default(
2242 thread_t thread)
2243 {
2244 return thread_assign(thread, &pset0);
2245 }
2246
2247 /*
2248 * thread_get_assignment
2249 *
2250 * Return current assignment for this thread.
2251 */
2252 kern_return_t
2253 thread_get_assignment(
2254 thread_t thread,
2255 processor_set_t *pset)
2256 {
2257 if (thread == NULL) {
2258 return KERN_INVALID_ARGUMENT;
2259 }
2260
2261 *pset = &pset0;
2262
2263 return KERN_SUCCESS;
2264 }
2265
2266 /*
2267 * thread_wire_internal:
2268 *
2269 * Specify that the target thread must always be able
2270 * to run and to allocate memory.
2271 */
2272 kern_return_t
2273 thread_wire_internal(
2274 host_priv_t host_priv,
2275 thread_t thread,
2276 boolean_t wired,
2277 boolean_t *prev_state)
2278 {
2279 if (host_priv == NULL || thread != current_thread()) {
2280 return KERN_INVALID_ARGUMENT;
2281 }
2282
2283 assert(host_priv == &realhost);
2284
2285 if (prev_state) {
2286 *prev_state = (thread->options & TH_OPT_VMPRIV) != 0;
2287 }
2288
2289 if (wired) {
2290 if (!(thread->options & TH_OPT_VMPRIV)) {
2291 vm_page_free_reserve(1); /* XXX */
2292 }
2293 thread->options |= TH_OPT_VMPRIV;
2294 } else {
2295 if (thread->options & TH_OPT_VMPRIV) {
2296 vm_page_free_reserve(-1); /* XXX */
2297 }
2298 thread->options &= ~TH_OPT_VMPRIV;
2299 }
2300
2301 return KERN_SUCCESS;
2302 }
2303
2304
2305 /*
2306 * thread_wire:
2307 *
2308 * User-api wrapper for thread_wire_internal()
2309 */
2310 kern_return_t
2311 thread_wire(
2312 host_priv_t host_priv,
2313 thread_t thread,
2314 boolean_t wired)
2315 {
2316 return thread_wire_internal(host_priv, thread, wired, NULL);
2317 }
2318
2319
2320 boolean_t
2321 is_vm_privileged(void)
2322 {
2323 return current_thread()->options & TH_OPT_VMPRIV ? TRUE : FALSE;
2324 }
2325
2326 boolean_t
2327 set_vm_privilege(boolean_t privileged)
2328 {
2329 boolean_t was_vmpriv;
2330
2331 if (current_thread()->options & TH_OPT_VMPRIV) {
2332 was_vmpriv = TRUE;
2333 } else {
2334 was_vmpriv = FALSE;
2335 }
2336
2337 if (privileged != FALSE) {
2338 current_thread()->options |= TH_OPT_VMPRIV;
2339 } else {
2340 current_thread()->options &= ~TH_OPT_VMPRIV;
2341 }
2342
2343 return was_vmpriv;
2344 }
2345
2346 void
2347 set_thread_rwlock_boost(void)
2348 {
2349 current_thread()->rwlock_count++;
2350 }
2351
2352 void
2353 clear_thread_rwlock_boost(void)
2354 {
2355 thread_t thread = current_thread();
2356
2357 if ((thread->rwlock_count-- == 1) && (thread->sched_flags & TH_SFLAG_RW_PROMOTED)) {
2358 lck_rw_clear_promotion(thread, 0);
2359 }
2360 }
2361
2362
2363 /*
2364 * XXX assuming current thread only, for now...
2365 */
2366 void
2367 thread_guard_violation(thread_t thread,
2368 mach_exception_data_type_t code, mach_exception_data_type_t subcode)
2369 {
2370 assert(thread == current_thread());
2371
2372 /* don't set up the AST for kernel threads */
2373 if (thread->task == kernel_task) {
2374 return;
2375 }
2376
2377 spl_t s = splsched();
2378 /*
2379 * Use the saved state area of the thread structure
2380 * to store all info required to handle the AST when
2381 * returning to userspace
2382 */
2383 assert(EXC_GUARD_DECODE_GUARD_TYPE(code));
2384 thread->guard_exc_info.code = code;
2385 thread->guard_exc_info.subcode = subcode;
2386 thread_ast_set(thread, AST_GUARD);
2387 ast_propagate(thread);
2388
2389 splx(s);
2390 }
2391
2392 /*
2393 * guard_ast:
2394 *
2395 * Handle AST_GUARD for a thread. This routine looks at the
2396 * state saved in the thread structure to determine the cause
2397 * of this exception. Based on this value, it invokes the
2398 * appropriate routine which determines other exception related
2399 * info and raises the exception.
2400 */
2401 void
2402 guard_ast(thread_t t)
2403 {
2404 const mach_exception_data_type_t
2405 code = t->guard_exc_info.code,
2406 subcode = t->guard_exc_info.subcode;
2407
2408 t->guard_exc_info.code = 0;
2409 t->guard_exc_info.subcode = 0;
2410
2411 switch (EXC_GUARD_DECODE_GUARD_TYPE(code)) {
2412 case GUARD_TYPE_NONE:
2413 /* lingering AST_GUARD on the processor? */
2414 break;
2415 case GUARD_TYPE_MACH_PORT:
2416 mach_port_guard_ast(t, code, subcode);
2417 break;
2418 case GUARD_TYPE_FD:
2419 fd_guard_ast(t, code, subcode);
2420 break;
2421 #if CONFIG_VNGUARD
2422 case GUARD_TYPE_VN:
2423 vn_guard_ast(t, code, subcode);
2424 break;
2425 #endif
2426 case GUARD_TYPE_VIRT_MEMORY:
2427 virt_memory_guard_ast(t, code, subcode);
2428 break;
2429 default:
2430 panic("guard_exc_info %llx %llx", code, subcode);
2431 }
2432 }
2433
2434 static void
2435 thread_cputime_callback(int warning, __unused const void *arg0, __unused const void *arg1)
2436 {
2437 if (warning == LEDGER_WARNING_ROSE_ABOVE) {
2438 #if CONFIG_TELEMETRY
2439 /*
2440 * This thread is in danger of violating the CPU usage monitor. Enable telemetry
2441 * on the entire task so there are micro-stackshots available if and when
2442 * EXC_RESOURCE is triggered. We could have chosen to enable micro-stackshots
2443 * for this thread only; but now that this task is suspect, knowing what all of
2444 * its threads are up to will be useful.
2445 */
2446 telemetry_task_ctl(current_task(), TF_CPUMON_WARNING, 1);
2447 #endif
2448 return;
2449 }
2450
2451 #if CONFIG_TELEMETRY
2452 /*
2453 * If the balance has dipped below the warning level (LEDGER_WARNING_DIPPED_BELOW) or
2454 * exceeded the limit, turn telemetry off for the task.
2455 */
2456 telemetry_task_ctl(current_task(), TF_CPUMON_WARNING, 0);
2457 #endif
2458
2459 if (warning == 0) {
2460 SENDING_NOTIFICATION__THIS_THREAD_IS_CONSUMING_TOO_MUCH_CPU();
2461 }
2462 }
2463
2464 void __attribute__((noinline))
2465 SENDING_NOTIFICATION__THIS_THREAD_IS_CONSUMING_TOO_MUCH_CPU(void)
2466 {
2467 int pid = 0;
2468 task_t task = current_task();
2469 thread_t thread = current_thread();
2470 uint64_t tid = thread->thread_id;
2471 const char *procname = "unknown";
2472 time_value_t thread_total_time = {0, 0};
2473 time_value_t thread_system_time;
2474 time_value_t thread_user_time;
2475 int action;
2476 uint8_t percentage;
2477 uint32_t usage_percent = 0;
2478 uint32_t interval_sec;
2479 uint64_t interval_ns;
2480 uint64_t balance_ns;
2481 boolean_t fatal = FALSE;
2482 boolean_t send_exc_resource = TRUE; /* in addition to RESOURCE_NOTIFY */
2483 kern_return_t kr;
2484
2485 #ifdef EXC_RESOURCE_MONITORS
2486 mach_exception_data_type_t code[EXCEPTION_CODE_MAX];
2487 #endif /* EXC_RESOURCE_MONITORS */
2488 struct ledger_entry_info lei;
2489
2490 assert(thread->t_threadledger != LEDGER_NULL);
2491
2492 /*
2493 * Extract the fatal bit and suspend the monitor (which clears the bit).
2494 */
2495 task_lock(task);
2496 if (task->rusage_cpu_flags & TASK_RUSECPU_FLAGS_FATAL_CPUMON) {
2497 fatal = TRUE;
2498 send_exc_resource = TRUE;
2499 }
2500 /* Only one thread can be here at a time. Whichever makes it through
2501 * first will successfully suspend the monitor and proceed to send the
2502 * notification. Other threads will get an error trying to suspend the
2503 * monitor and give up on sending the notification. In the first release,
2504 * the monitor won't be resumed for a number of seconds, but we may
2505 * eventually need to handle low-latency resume.
2506 */
2507 kr = task_suspend_cpumon(task);
2508 task_unlock(task);
2509 if (kr == KERN_INVALID_ARGUMENT) {
2510 return;
2511 }
2512
2513 #ifdef MACH_BSD
2514 pid = proc_selfpid();
2515 if (task->bsd_info != NULL) {
2516 procname = proc_name_address(task->bsd_info);
2517 }
2518 #endif
2519
2520 thread_get_cpulimit(&action, &percentage, &interval_ns);
2521
2522 interval_sec = (uint32_t)(interval_ns / NSEC_PER_SEC);
2523
2524 thread_read_times(thread, &thread_user_time, &thread_system_time, NULL);
2525 time_value_add(&thread_total_time, &thread_user_time);
2526 time_value_add(&thread_total_time, &thread_system_time);
2527 ledger_get_entry_info(thread->t_threadledger, thread_ledgers.cpu_time, &lei);
2528
2529 /* credit/debit/balance/limit are in absolute time units;
2530 * the refill info is in nanoseconds. */
2531 absolutetime_to_nanoseconds(lei.lei_balance, &balance_ns);
2532 if (lei.lei_last_refill > 0) {
2533 usage_percent = (uint32_t)((balance_ns * 100ULL) / lei.lei_last_refill);
2534 }
2535
2536 /* TODO: show task total runtime (via TASK_ABSOLUTETIME_INFO)? */
2537 printf("process %s[%d] thread %llu caught burning CPU! "
2538 "It used more than %d%% CPU over %u seconds "
2539 "(actual recent usage: %d%% over ~%llu seconds). "
2540 "Thread lifetime cpu usage %d.%06ds, (%d.%06d user, %d.%06d sys) "
2541 "ledger balance: %lld mabs credit: %lld mabs debit: %lld mabs "
2542 "limit: %llu mabs period: %llu ns last refill: %llu ns%s.\n",
2543 procname, pid, tid,
2544 percentage, interval_sec,
2545 usage_percent,
2546 (lei.lei_last_refill + NSEC_PER_SEC / 2) / NSEC_PER_SEC,
2547 thread_total_time.seconds, thread_total_time.microseconds,
2548 thread_user_time.seconds, thread_user_time.microseconds,
2549 thread_system_time.seconds, thread_system_time.microseconds,
2550 lei.lei_balance, lei.lei_credit, lei.lei_debit,
2551 lei.lei_limit, lei.lei_refill_period, lei.lei_last_refill,
2552 (fatal ? " [fatal violation]" : ""));
2553
2554 /*
2555 * For now, send RESOURCE_NOTIFY in parallel with EXC_RESOURCE. Once
2556 * we have logging parity, we will stop sending EXC_RESOURCE (24508922).
2557 */
2558
2559 /* RESOURCE_NOTIFY MIG specifies nanoseconds of CPU time */
2560 lei.lei_balance = balance_ns;
2561 absolutetime_to_nanoseconds(lei.lei_limit, &lei.lei_limit);
2562 trace_resource_violation(RMON_CPUUSAGE_VIOLATED, &lei);
2563 kr = send_resource_violation(send_cpu_usage_violation, task, &lei,
2564 fatal ? kRNFatalLimitFlag : 0);
2565 if (kr) {
2566 printf("send_resource_violation(CPU usage, ...): error %#x\n", kr);
2567 }
2568
2569 #ifdef EXC_RESOURCE_MONITORS
2570 if (send_exc_resource) {
2571 if (disable_exc_resource) {
2572 printf("process %s[%d] thread %llu caught burning CPU! "
2573 "EXC_RESOURCE%s supressed by a boot-arg\n",
2574 procname, pid, tid, fatal ? " (and termination)" : "");
2575 return;
2576 }
2577
2578 if (audio_active) {
2579 printf("process %s[%d] thread %llu caught burning CPU! "
2580 "EXC_RESOURCE & termination supressed due to audio playback\n",
2581 procname, pid, tid);
2582 return;
2583 }
2584 }
2585
2586
2587 if (send_exc_resource) {
2588 code[0] = code[1] = 0;
2589 EXC_RESOURCE_ENCODE_TYPE(code[0], RESOURCE_TYPE_CPU);
2590 if (fatal) {
2591 EXC_RESOURCE_ENCODE_FLAVOR(code[0], FLAVOR_CPU_MONITOR_FATAL);
2592 } else {
2593 EXC_RESOURCE_ENCODE_FLAVOR(code[0], FLAVOR_CPU_MONITOR);
2594 }
2595 EXC_RESOURCE_CPUMONITOR_ENCODE_INTERVAL(code[0], interval_sec);
2596 EXC_RESOURCE_CPUMONITOR_ENCODE_PERCENTAGE(code[0], percentage);
2597 EXC_RESOURCE_CPUMONITOR_ENCODE_PERCENTAGE(code[1], usage_percent);
2598 exception_triage(EXC_RESOURCE, code, EXCEPTION_CODE_MAX);
2599 }
2600 #endif /* EXC_RESOURCE_MONITORS */
2601
2602 if (fatal) {
2603 #if CONFIG_JETSAM
2604 jetsam_on_ledger_cpulimit_exceeded();
2605 #else
2606 task_terminate_internal(task);
2607 #endif
2608 }
2609 }
2610
2611 #if DEVELOPMENT || DEBUG
2612 void __attribute__((noinline))
2613 SENDING_NOTIFICATION__TASK_HAS_TOO_MANY_THREADS(task_t task, int thread_count)
2614 {
2615 mach_exception_data_type_t code[EXCEPTION_CODE_MAX] = {0};
2616 int pid = task_pid(task);
2617 char procname[MAXCOMLEN + 1] = "unknown";
2618
2619 if (pid == 1) {
2620 /*
2621 * Cannot suspend launchd
2622 */
2623 return;
2624 }
2625
2626 proc_name(pid, procname, sizeof(procname));
2627
2628 if (disable_exc_resource) {
2629 printf("process %s[%d] crossed thread count high watermark (%d), EXC_RESOURCE "
2630 "supressed by a boot-arg. \n", procname, pid, thread_count);
2631 return;
2632 }
2633
2634 if (audio_active) {
2635 printf("process %s[%d] crossed thread count high watermark (%d), EXC_RESOURCE "
2636 "supressed due to audio playback.\n", procname, pid, thread_count);
2637 return;
2638 }
2639
2640 if (exc_via_corpse_forking == 0) {
2641 printf("process %s[%d] crossed thread count high watermark (%d), EXC_RESOURCE "
2642 "supressed due to corpse forking being disabled.\n", procname, pid,
2643 thread_count);
2644 return;
2645 }
2646
2647 printf("process %s[%d] crossed thread count high watermark (%d), sending "
2648 "EXC_RESOURCE\n", procname, pid, thread_count);
2649
2650 EXC_RESOURCE_ENCODE_TYPE(code[0], RESOURCE_TYPE_THREADS);
2651 EXC_RESOURCE_ENCODE_FLAVOR(code[0], FLAVOR_THREADS_HIGH_WATERMARK);
2652 EXC_RESOURCE_THREADS_ENCODE_THREADS(code[0], thread_count);
2653
2654 task_enqueue_exception_with_corpse(task, EXC_RESOURCE, code, EXCEPTION_CODE_MAX, NULL);
2655 }
2656 #endif /* DEVELOPMENT || DEBUG */
2657
2658 void
2659 thread_update_io_stats(thread_t thread, int size, int io_flags)
2660 {
2661 int io_tier;
2662
2663 if (thread->thread_io_stats == NULL || thread->task->task_io_stats == NULL) {
2664 return;
2665 }
2666
2667 if (io_flags & DKIO_READ) {
2668 UPDATE_IO_STATS(thread->thread_io_stats->disk_reads, size);
2669 UPDATE_IO_STATS_ATOMIC(thread->task->task_io_stats->disk_reads, size);
2670 }
2671
2672 if (io_flags & DKIO_META) {
2673 UPDATE_IO_STATS(thread->thread_io_stats->metadata, size);
2674 UPDATE_IO_STATS_ATOMIC(thread->task->task_io_stats->metadata, size);
2675 }
2676
2677 if (io_flags & DKIO_PAGING) {
2678 UPDATE_IO_STATS(thread->thread_io_stats->paging, size);
2679 UPDATE_IO_STATS_ATOMIC(thread->task->task_io_stats->paging, size);
2680 }
2681
2682 io_tier = ((io_flags & DKIO_TIER_MASK) >> DKIO_TIER_SHIFT);
2683 assert(io_tier < IO_NUM_PRIORITIES);
2684
2685 UPDATE_IO_STATS(thread->thread_io_stats->io_priority[io_tier], size);
2686 UPDATE_IO_STATS_ATOMIC(thread->task->task_io_stats->io_priority[io_tier], size);
2687
2688 /* Update Total I/O Counts */
2689 UPDATE_IO_STATS(thread->thread_io_stats->total_io, size);
2690 UPDATE_IO_STATS_ATOMIC(thread->task->task_io_stats->total_io, size);
2691
2692 if (!(io_flags & DKIO_READ)) {
2693 DTRACE_IO3(physical_writes, struct task *, thread->task, uint32_t, size, int, io_flags);
2694 ledger_credit(thread->task->ledger, task_ledgers.physical_writes, size);
2695 }
2696 }
2697
2698 static void
2699 init_thread_ledgers(void)
2700 {
2701 ledger_template_t t;
2702 int idx;
2703
2704 assert(thread_ledger_template == NULL);
2705
2706 if ((t = ledger_template_create("Per-thread ledger")) == NULL) {
2707 panic("couldn't create thread ledger template");
2708 }
2709
2710 if ((idx = ledger_entry_add(t, "cpu_time", "sched", "ns")) < 0) {
2711 panic("couldn't create cpu_time entry for thread ledger template");
2712 }
2713
2714 if (ledger_set_callback(t, idx, thread_cputime_callback, NULL, NULL) < 0) {
2715 panic("couldn't set thread ledger callback for cpu_time entry");
2716 }
2717
2718 thread_ledgers.cpu_time = idx;
2719
2720 ledger_template_complete(t);
2721 thread_ledger_template = t;
2722 }
2723
2724 /*
2725 * Returns currently applied CPU usage limit, or 0/0 if none is applied.
2726 */
2727 int
2728 thread_get_cpulimit(int *action, uint8_t *percentage, uint64_t *interval_ns)
2729 {
2730 int64_t abstime = 0;
2731 uint64_t limittime = 0;
2732 thread_t thread = current_thread();
2733
2734 *percentage = 0;
2735 *interval_ns = 0;
2736 *action = 0;
2737
2738 if (thread->t_threadledger == LEDGER_NULL) {
2739 /*
2740 * This thread has no per-thread ledger, so it can't possibly
2741 * have a CPU limit applied.
2742 */
2743 return KERN_SUCCESS;
2744 }
2745
2746 ledger_get_period(thread->t_threadledger, thread_ledgers.cpu_time, interval_ns);
2747 ledger_get_limit(thread->t_threadledger, thread_ledgers.cpu_time, &abstime);
2748
2749 if ((abstime == LEDGER_LIMIT_INFINITY) || (*interval_ns == 0)) {
2750 /*
2751 * This thread's CPU time ledger has no period or limit; so it
2752 * doesn't have a CPU limit applied.
2753 */
2754 return KERN_SUCCESS;
2755 }
2756
2757 /*
2758 * This calculation is the converse to the one in thread_set_cpulimit().
2759 */
2760 absolutetime_to_nanoseconds(abstime, &limittime);
2761 *percentage = (limittime * 100ULL) / *interval_ns;
2762 assert(*percentage <= 100);
2763
2764 if (thread->options & TH_OPT_PROC_CPULIMIT) {
2765 assert((thread->options & TH_OPT_PRVT_CPULIMIT) == 0);
2766
2767 *action = THREAD_CPULIMIT_BLOCK;
2768 } else if (thread->options & TH_OPT_PRVT_CPULIMIT) {
2769 assert((thread->options & TH_OPT_PROC_CPULIMIT) == 0);
2770
2771 *action = THREAD_CPULIMIT_EXCEPTION;
2772 } else {
2773 *action = THREAD_CPULIMIT_DISABLE;
2774 }
2775
2776 return KERN_SUCCESS;
2777 }
2778
2779 /*
2780 * Set CPU usage limit on a thread.
2781 *
2782 * Calling with percentage of 0 will unset the limit for this thread.
2783 */
2784 int
2785 thread_set_cpulimit(int action, uint8_t percentage, uint64_t interval_ns)
2786 {
2787 thread_t thread = current_thread();
2788 ledger_t l;
2789 uint64_t limittime = 0;
2790 uint64_t abstime = 0;
2791
2792 assert(percentage <= 100);
2793
2794 if (action == THREAD_CPULIMIT_DISABLE) {
2795 /*
2796 * Remove CPU limit, if any exists.
2797 */
2798 if (thread->t_threadledger != LEDGER_NULL) {
2799 l = thread->t_threadledger;
2800 ledger_set_limit(l, thread_ledgers.cpu_time, LEDGER_LIMIT_INFINITY, 0);
2801 ledger_set_action(l, thread_ledgers.cpu_time, LEDGER_ACTION_IGNORE);
2802 thread->options &= ~(TH_OPT_PROC_CPULIMIT | TH_OPT_PRVT_CPULIMIT);
2803 }
2804
2805 return 0;
2806 }
2807
2808 if (interval_ns < MINIMUM_CPULIMIT_INTERVAL_MS * NSEC_PER_MSEC) {
2809 return KERN_INVALID_ARGUMENT;
2810 }
2811
2812 l = thread->t_threadledger;
2813 if (l == LEDGER_NULL) {
2814 /*
2815 * This thread doesn't yet have a per-thread ledger; so create one with the CPU time entry active.
2816 */
2817 if ((l = ledger_instantiate(thread_ledger_template, LEDGER_CREATE_INACTIVE_ENTRIES)) == LEDGER_NULL) {
2818 return KERN_RESOURCE_SHORTAGE;
2819 }
2820
2821 /*
2822 * We are the first to create this thread's ledger, so only activate our entry.
2823 */
2824 ledger_entry_setactive(l, thread_ledgers.cpu_time);
2825 thread->t_threadledger = l;
2826 }
2827
2828 /*
2829 * The limit is specified as a percentage of CPU over an interval in nanoseconds.
2830 * Calculate the amount of CPU time that the thread needs to consume in order to hit the limit.
2831 */
2832 limittime = (interval_ns * percentage) / 100;
2833 nanoseconds_to_absolutetime(limittime, &abstime);
2834 ledger_set_limit(l, thread_ledgers.cpu_time, abstime, cpumon_ustackshots_trigger_pct);
2835 /*
2836 * Refill the thread's allotted CPU time every interval_ns nanoseconds.
2837 */
2838 ledger_set_period(l, thread_ledgers.cpu_time, interval_ns);
2839
2840 if (action == THREAD_CPULIMIT_EXCEPTION) {
2841 /*
2842 * We don't support programming the CPU usage monitor on a task if any of its
2843 * threads have a per-thread blocking CPU limit configured.
2844 */
2845 if (thread->options & TH_OPT_PRVT_CPULIMIT) {
2846 panic("CPU usage monitor activated, but blocking thread limit exists");
2847 }
2848
2849 /*
2850 * Make a note that this thread's CPU limit is being used for the task-wide CPU
2851 * usage monitor. We don't have to arm the callback which will trigger the
2852 * exception, because that was done for us in ledger_instantiate (because the
2853 * ledger template used has a default callback).
2854 */
2855 thread->options |= TH_OPT_PROC_CPULIMIT;
2856 } else {
2857 /*
2858 * We deliberately override any CPU limit imposed by a task-wide limit (eg
2859 * CPU usage monitor).
2860 */
2861 thread->options &= ~TH_OPT_PROC_CPULIMIT;
2862
2863 thread->options |= TH_OPT_PRVT_CPULIMIT;
2864 /* The per-thread ledger template by default has a callback for CPU time */
2865 ledger_disable_callback(l, thread_ledgers.cpu_time);
2866 ledger_set_action(l, thread_ledgers.cpu_time, LEDGER_ACTION_BLOCK);
2867 }
2868
2869 return 0;
2870 }
2871
2872 void
2873 thread_sched_call(
2874 thread_t thread,
2875 sched_call_t call)
2876 {
2877 assert((thread->state & TH_WAIT_REPORT) == 0);
2878 thread->sched_call = call;
2879 }
2880
2881 uint64_t
2882 thread_tid(
2883 thread_t thread)
2884 {
2885 return thread != THREAD_NULL? thread->thread_id: 0;
2886 }
2887
2888 uint16_t
2889 thread_set_tag(thread_t th, uint16_t tag)
2890 {
2891 return thread_set_tag_internal(th, tag);
2892 }
2893
2894 uint16_t
2895 thread_get_tag(thread_t th)
2896 {
2897 return thread_get_tag_internal(th);
2898 }
2899
2900 uint64_t
2901 thread_last_run_time(thread_t th)
2902 {
2903 return th->last_run_time;
2904 }
2905
2906 uint64_t
2907 thread_dispatchqaddr(
2908 thread_t thread)
2909 {
2910 uint64_t dispatchqueue_addr;
2911 uint64_t thread_handle;
2912
2913 if (thread == THREAD_NULL) {
2914 return 0;
2915 }
2916
2917 thread_handle = thread->machine.cthread_self;
2918 if (thread_handle == 0) {
2919 return 0;
2920 }
2921
2922 if (thread->inspection == TRUE) {
2923 dispatchqueue_addr = thread_handle + get_task_dispatchqueue_offset(thread->task);
2924 } else if (thread->task->bsd_info) {
2925 dispatchqueue_addr = thread_handle + get_dispatchqueue_offset_from_proc(thread->task->bsd_info);
2926 } else {
2927 dispatchqueue_addr = 0;
2928 }
2929
2930 return dispatchqueue_addr;
2931 }
2932
2933 uint64_t
2934 thread_rettokern_addr(
2935 thread_t thread)
2936 {
2937 uint64_t rettokern_addr;
2938 uint64_t rettokern_offset;
2939 uint64_t thread_handle;
2940
2941 if (thread == THREAD_NULL) {
2942 return 0;
2943 }
2944
2945 thread_handle = thread->machine.cthread_self;
2946 if (thread_handle == 0) {
2947 return 0;
2948 }
2949
2950 if (thread->task->bsd_info) {
2951 rettokern_offset = get_return_to_kernel_offset_from_proc(thread->task->bsd_info);
2952
2953 /* Return 0 if return to kernel offset is not initialized. */
2954 if (rettokern_offset == 0) {
2955 rettokern_addr = 0;
2956 } else {
2957 rettokern_addr = thread_handle + rettokern_offset;
2958 }
2959 } else {
2960 rettokern_addr = 0;
2961 }
2962
2963 return rettokern_addr;
2964 }
2965
2966 /*
2967 * Export routines to other components for things that are done as macros
2968 * within the osfmk component.
2969 */
2970
2971 #undef thread_mtx_lock
2972 void thread_mtx_lock(thread_t thread);
2973 void
2974 thread_mtx_lock(thread_t thread)
2975 {
2976 lck_mtx_lock(&thread->mutex);
2977 }
2978
2979 #undef thread_mtx_unlock
2980 void thread_mtx_unlock(thread_t thread);
2981 void
2982 thread_mtx_unlock(thread_t thread)
2983 {
2984 lck_mtx_unlock(&thread->mutex);
2985 }
2986
2987 #undef thread_reference
2988 void thread_reference(thread_t thread);
2989 void
2990 thread_reference(
2991 thread_t thread)
2992 {
2993 if (thread != THREAD_NULL) {
2994 thread_reference_internal(thread);
2995 }
2996 }
2997
2998 #undef thread_should_halt
2999
3000 boolean_t
3001 thread_should_halt(
3002 thread_t th)
3003 {
3004 return thread_should_halt_fast(th);
3005 }
3006
3007 /*
3008 * thread_set_voucher_name - reset the voucher port name bound to this thread
3009 *
3010 * Conditions: nothing locked
3011 *
3012 * If we already converted the previous name to a cached voucher
3013 * reference, then we discard that reference here. The next lookup
3014 * will cache it again.
3015 */
3016
3017 kern_return_t
3018 thread_set_voucher_name(mach_port_name_t voucher_name)
3019 {
3020 thread_t thread = current_thread();
3021 ipc_voucher_t new_voucher = IPC_VOUCHER_NULL;
3022 ipc_voucher_t voucher;
3023 ledger_t bankledger = NULL;
3024 struct thread_group *banktg = NULL;
3025
3026 if (MACH_PORT_DEAD == voucher_name) {
3027 return KERN_INVALID_RIGHT;
3028 }
3029
3030 /*
3031 * agressively convert to voucher reference
3032 */
3033 if (MACH_PORT_VALID(voucher_name)) {
3034 new_voucher = convert_port_name_to_voucher(voucher_name);
3035 if (IPC_VOUCHER_NULL == new_voucher) {
3036 return KERN_INVALID_ARGUMENT;
3037 }
3038 }
3039 bank_get_bank_ledger_and_thread_group(new_voucher, &bankledger, &banktg);
3040
3041 thread_mtx_lock(thread);
3042 voucher = thread->ith_voucher;
3043 thread->ith_voucher_name = voucher_name;
3044 thread->ith_voucher = new_voucher;
3045 thread_mtx_unlock(thread);
3046
3047 bank_swap_thread_bank_ledger(thread, bankledger);
3048
3049 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
3050 MACHDBG_CODE(DBG_MACH_IPC, MACH_THREAD_SET_VOUCHER) | DBG_FUNC_NONE,
3051 (uintptr_t)thread_tid(thread),
3052 (uintptr_t)voucher_name,
3053 VM_KERNEL_ADDRPERM((uintptr_t)new_voucher),
3054 1, 0);
3055
3056 if (IPC_VOUCHER_NULL != voucher) {
3057 ipc_voucher_release(voucher);
3058 }
3059
3060 return KERN_SUCCESS;
3061 }
3062
3063 /*
3064 * thread_get_mach_voucher - return a voucher reference for the specified thread voucher
3065 *
3066 * Conditions: nothing locked
3067 *
3068 * A reference to the voucher may be lazily pending, if someone set the voucher name
3069 * but nobody has done a lookup yet. In that case, we'll have to do the equivalent
3070 * lookup here.
3071 *
3072 * NOTE: At the moment, there is no distinction between the current and effective
3073 * vouchers because we only set them at the thread level currently.
3074 */
3075 kern_return_t
3076 thread_get_mach_voucher(
3077 thread_act_t thread,
3078 mach_voucher_selector_t __unused which,
3079 ipc_voucher_t *voucherp)
3080 {
3081 ipc_voucher_t voucher;
3082 mach_port_name_t voucher_name;
3083
3084 if (THREAD_NULL == thread) {
3085 return KERN_INVALID_ARGUMENT;
3086 }
3087
3088 thread_mtx_lock(thread);
3089 voucher = thread->ith_voucher;
3090
3091 /* if already cached, just return a ref */
3092 if (IPC_VOUCHER_NULL != voucher) {
3093 ipc_voucher_reference(voucher);
3094 thread_mtx_unlock(thread);
3095 *voucherp = voucher;
3096 return KERN_SUCCESS;
3097 }
3098
3099 voucher_name = thread->ith_voucher_name;
3100
3101 /* convert the name to a port, then voucher reference */
3102 if (MACH_PORT_VALID(voucher_name)) {
3103 ipc_port_t port;
3104
3105 if (KERN_SUCCESS !=
3106 ipc_object_copyin(thread->task->itk_space, voucher_name,
3107 MACH_MSG_TYPE_COPY_SEND, (ipc_object_t *)&port)) {
3108 thread->ith_voucher_name = MACH_PORT_NULL;
3109 thread_mtx_unlock(thread);
3110 *voucherp = IPC_VOUCHER_NULL;
3111 return KERN_SUCCESS;
3112 }
3113
3114 /* convert to a voucher ref to return, and cache a ref on thread */
3115 voucher = convert_port_to_voucher(port);
3116 ipc_voucher_reference(voucher);
3117 thread->ith_voucher = voucher;
3118 thread_mtx_unlock(thread);
3119
3120 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
3121 MACHDBG_CODE(DBG_MACH_IPC, MACH_THREAD_SET_VOUCHER) | DBG_FUNC_NONE,
3122 (uintptr_t)thread_tid(thread),
3123 (uintptr_t)port,
3124 VM_KERNEL_ADDRPERM((uintptr_t)voucher),
3125 2, 0);
3126
3127
3128 ipc_port_release_send(port);
3129 } else {
3130 thread_mtx_unlock(thread);
3131 }
3132
3133 *voucherp = voucher;
3134 return KERN_SUCCESS;
3135 }
3136
3137 /*
3138 * thread_set_mach_voucher - set a voucher reference for the specified thread voucher
3139 *
3140 * Conditions: callers holds a reference on the voucher.
3141 * nothing locked.
3142 *
3143 * We grab another reference to the voucher and bind it to the thread. Any lazy
3144 * binding is erased. The old voucher reference associated with the thread is
3145 * discarded.
3146 */
3147 kern_return_t
3148 thread_set_mach_voucher(
3149 thread_t thread,
3150 ipc_voucher_t voucher)
3151 {
3152 ipc_voucher_t old_voucher;
3153 ledger_t bankledger = NULL;
3154 struct thread_group *banktg = NULL;
3155
3156 if (THREAD_NULL == thread) {
3157 return KERN_INVALID_ARGUMENT;
3158 }
3159
3160 if (thread != current_thread() && thread->started) {
3161 return KERN_INVALID_ARGUMENT;
3162 }
3163
3164 ipc_voucher_reference(voucher);
3165 bank_get_bank_ledger_and_thread_group(voucher, &bankledger, &banktg);
3166
3167 thread_mtx_lock(thread);
3168 old_voucher = thread->ith_voucher;
3169 thread->ith_voucher = voucher;
3170 thread->ith_voucher_name = MACH_PORT_NULL;
3171 thread_mtx_unlock(thread);
3172
3173 bank_swap_thread_bank_ledger(thread, bankledger);
3174
3175 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
3176 MACHDBG_CODE(DBG_MACH_IPC, MACH_THREAD_SET_VOUCHER) | DBG_FUNC_NONE,
3177 (uintptr_t)thread_tid(thread),
3178 (uintptr_t)MACH_PORT_NULL,
3179 VM_KERNEL_ADDRPERM((uintptr_t)voucher),
3180 3, 0);
3181
3182 ipc_voucher_release(old_voucher);
3183
3184 return KERN_SUCCESS;
3185 }
3186
3187 /*
3188 * thread_swap_mach_voucher - swap a voucher reference for the specified thread voucher
3189 *
3190 * Conditions: callers holds a reference on the new and presumed old voucher(s).
3191 * nothing locked.
3192 *
3193 * This function is no longer supported.
3194 */
3195 kern_return_t
3196 thread_swap_mach_voucher(
3197 __unused thread_t thread,
3198 __unused ipc_voucher_t new_voucher,
3199 ipc_voucher_t *in_out_old_voucher)
3200 {
3201 /*
3202 * Currently this function is only called from a MIG generated
3203 * routine which doesn't release the reference on the voucher
3204 * addressed by in_out_old_voucher. To avoid leaking this reference,
3205 * a call to release it has been added here.
3206 */
3207 ipc_voucher_release(*in_out_old_voucher);
3208 return KERN_NOT_SUPPORTED;
3209 }
3210
3211 /*
3212 * thread_get_current_voucher_origin_pid - get the pid of the originator of the current voucher.
3213 */
3214 kern_return_t
3215 thread_get_current_voucher_origin_pid(
3216 int32_t *pid)
3217 {
3218 uint32_t buf_size;
3219 kern_return_t kr;
3220 thread_t thread = current_thread();
3221
3222 buf_size = sizeof(*pid);
3223 kr = mach_voucher_attr_command(thread->ith_voucher,
3224 MACH_VOUCHER_ATTR_KEY_BANK,
3225 BANK_ORIGINATOR_PID,
3226 NULL,
3227 0,
3228 (mach_voucher_attr_content_t)pid,
3229 &buf_size);
3230
3231 return kr;
3232 }
3233
3234
3235 boolean_t
3236 thread_has_thread_name(thread_t th)
3237 {
3238 if ((th) && (th->uthread)) {
3239 return bsd_hasthreadname(th->uthread);
3240 }
3241
3242 /*
3243 * This is an odd case; clients may set the thread name based on the lack of
3244 * a name, but in this context there is no uthread to attach the name to.
3245 */
3246 return FALSE;
3247 }
3248
3249 void
3250 thread_set_thread_name(thread_t th, const char* name)
3251 {
3252 if ((th) && (th->uthread) && name) {
3253 bsd_setthreadname(th->uthread, name);
3254 }
3255 }
3256
3257 void
3258 thread_set_honor_qlimit(thread_t thread)
3259 {
3260 thread->options |= TH_OPT_HONOR_QLIMIT;
3261 }
3262
3263 void
3264 thread_clear_honor_qlimit(thread_t thread)
3265 {
3266 thread->options &= (~TH_OPT_HONOR_QLIMIT);
3267 }
3268
3269 /*
3270 * thread_enable_send_importance - set/clear the SEND_IMPORTANCE thread option bit.
3271 */
3272 void
3273 thread_enable_send_importance(thread_t thread, boolean_t enable)
3274 {
3275 if (enable == TRUE) {
3276 thread->options |= TH_OPT_SEND_IMPORTANCE;
3277 } else {
3278 thread->options &= ~TH_OPT_SEND_IMPORTANCE;
3279 }
3280 }
3281
3282 /*
3283 * thread_set_allocation_name - .
3284 */
3285
3286 kern_allocation_name_t
3287 thread_set_allocation_name(kern_allocation_name_t new_name)
3288 {
3289 kern_allocation_name_t ret;
3290 thread_kernel_state_t kstate = thread_get_kernel_state(current_thread());
3291 ret = kstate->allocation_name;
3292 // fifo
3293 if (!new_name || !kstate->allocation_name) {
3294 kstate->allocation_name = new_name;
3295 }
3296 return ret;
3297 }
3298
3299 uint64_t
3300 thread_get_last_wait_duration(thread_t thread)
3301 {
3302 return thread->last_made_runnable_time - thread->last_run_time;
3303 }
3304
3305 #if CONFIG_DTRACE
3306 uint32_t
3307 dtrace_get_thread_predcache(thread_t thread)
3308 {
3309 if (thread != THREAD_NULL) {
3310 return thread->t_dtrace_predcache;
3311 } else {
3312 return 0;
3313 }
3314 }
3315
3316 int64_t
3317 dtrace_get_thread_vtime(thread_t thread)
3318 {
3319 if (thread != THREAD_NULL) {
3320 return thread->t_dtrace_vtime;
3321 } else {
3322 return 0;
3323 }
3324 }
3325
3326 int
3327 dtrace_get_thread_last_cpu_id(thread_t thread)
3328 {
3329 if ((thread != THREAD_NULL) && (thread->last_processor != PROCESSOR_NULL)) {
3330 return thread->last_processor->cpu_id;
3331 } else {
3332 return -1;
3333 }
3334 }
3335
3336 int64_t
3337 dtrace_get_thread_tracing(thread_t thread)
3338 {
3339 if (thread != THREAD_NULL) {
3340 return thread->t_dtrace_tracing;
3341 } else {
3342 return 0;
3343 }
3344 }
3345
3346 boolean_t
3347 dtrace_get_thread_reentering(thread_t thread)
3348 {
3349 if (thread != THREAD_NULL) {
3350 return (thread->options & TH_OPT_DTRACE) ? TRUE : FALSE;
3351 } else {
3352 return 0;
3353 }
3354 }
3355
3356 vm_offset_t
3357 dtrace_get_kernel_stack(thread_t thread)
3358 {
3359 if (thread != THREAD_NULL) {
3360 return thread->kernel_stack;
3361 } else {
3362 return 0;
3363 }
3364 }
3365
3366 #if KASAN
3367 struct kasan_thread_data *
3368 kasan_get_thread_data(thread_t thread)
3369 {
3370 return &thread->kasan_data;
3371 }
3372 #endif
3373
3374 int64_t
3375 dtrace_calc_thread_recent_vtime(thread_t thread)
3376 {
3377 if (thread != THREAD_NULL) {
3378 processor_t processor = current_processor();
3379 uint64_t abstime = mach_absolute_time();
3380 timer_t timer;
3381
3382 timer = PROCESSOR_DATA(processor, thread_timer);
3383
3384 return timer_grab(&(thread->system_timer)) + timer_grab(&(thread->user_timer)) +
3385 (abstime - timer->tstamp); /* XXX need interrupts off to prevent missed time? */
3386 } else {
3387 return 0;
3388 }
3389 }
3390
3391 void
3392 dtrace_set_thread_predcache(thread_t thread, uint32_t predcache)
3393 {
3394 if (thread != THREAD_NULL) {
3395 thread->t_dtrace_predcache = predcache;
3396 }
3397 }
3398
3399 void
3400 dtrace_set_thread_vtime(thread_t thread, int64_t vtime)
3401 {
3402 if (thread != THREAD_NULL) {
3403 thread->t_dtrace_vtime = vtime;
3404 }
3405 }
3406
3407 void
3408 dtrace_set_thread_tracing(thread_t thread, int64_t accum)
3409 {
3410 if (thread != THREAD_NULL) {
3411 thread->t_dtrace_tracing = accum;
3412 }
3413 }
3414
3415 void
3416 dtrace_set_thread_reentering(thread_t thread, boolean_t vbool)
3417 {
3418 if (thread != THREAD_NULL) {
3419 if (vbool) {
3420 thread->options |= TH_OPT_DTRACE;
3421 } else {
3422 thread->options &= (~TH_OPT_DTRACE);
3423 }
3424 }
3425 }
3426
3427 vm_offset_t
3428 dtrace_set_thread_recover(thread_t thread, vm_offset_t recover)
3429 {
3430 vm_offset_t prev = 0;
3431
3432 if (thread != THREAD_NULL) {
3433 prev = thread->recover;
3434 thread->recover = recover;
3435 }
3436 return prev;
3437 }
3438
3439 vm_offset_t
3440 dtrace_sign_and_set_thread_recover(thread_t thread, vm_offset_t recover)
3441 {
3442 return dtrace_set_thread_recover(thread, recover);
3443 }
3444
3445 void
3446 dtrace_thread_bootstrap(void)
3447 {
3448 task_t task = current_task();
3449
3450 if (task->thread_count == 1) {
3451 thread_t thread = current_thread();
3452 if (thread->t_dtrace_flags & TH_DTRACE_EXECSUCCESS) {
3453 thread->t_dtrace_flags &= ~TH_DTRACE_EXECSUCCESS;
3454 DTRACE_PROC(exec__success);
3455 KDBG(BSDDBG_CODE(DBG_BSD_PROC, BSD_PROC_EXEC),
3456 task_pid(task));
3457 }
3458 DTRACE_PROC(start);
3459 }
3460 DTRACE_PROC(lwp__start);
3461 }
3462
3463 void
3464 dtrace_thread_didexec(thread_t thread)
3465 {
3466 thread->t_dtrace_flags |= TH_DTRACE_EXECSUCCESS;
3467 }
3468 #endif /* CONFIG_DTRACE */
mirror of XNU