]> git.saurik.com Git - apple/xnu.git/blob - osfmk/kern/thread.c
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-6153.141.1.tar.gz
[apple/xnu.git] / osfmk / kern / thread.c
1 /*
2 * Copyright (c) 2000-2019 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_FREE_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56 /*
57 */
58 /*
59 * File: kern/thread.c
60 * Author: Avadis Tevanian, Jr., Michael Wayne Young, David Golub
61 * Date: 1986
62 *
63 * Thread management primitives implementation.
64 */
65 /*
66 * Copyright (c) 1993 The University of Utah and
67 * the Computer Systems Laboratory (CSL). All rights reserved.
68 *
69 * Permission to use, copy, modify and distribute this software and its
70 * documentation is hereby granted, provided that both the copyright
71 * notice and this permission notice appear in all copies of the
72 * software, derivative works or modified versions, and any portions
73 * thereof, and that both notices appear in supporting documentation.
74 *
75 * THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF THIS SOFTWARE IN ITS "AS
76 * IS" CONDITION. THE UNIVERSITY OF UTAH AND CSL DISCLAIM ANY LIABILITY OF
77 * ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
78 *
79 * CSL requests users of this software to return to csl-dist@cs.utah.edu any
80 * improvements that they make and grant CSL redistribution rights.
81 *
82 */
83
84 #include <mach/mach_types.h>
85 #include <mach/boolean.h>
86 #include <mach/policy.h>
87 #include <mach/thread_info.h>
88 #include <mach/thread_special_ports.h>
89 #include <mach/thread_status.h>
90 #include <mach/time_value.h>
91 #include <mach/vm_param.h>
92
93 #include <machine/thread.h>
94 #include <machine/pal_routines.h>
95 #include <machine/limits.h>
96
97 #include <kern/kern_types.h>
98 #include <kern/kalloc.h>
99 #include <kern/cpu_data.h>
100 #include <kern/counters.h>
101 #include <kern/extmod_statistics.h>
102 #include <kern/ipc_mig.h>
103 #include <kern/ipc_tt.h>
104 #include <kern/mach_param.h>
105 #include <kern/machine.h>
106 #include <kern/misc_protos.h>
107 #include <kern/processor.h>
108 #include <kern/queue.h>
109 #include <kern/sched.h>
110 #include <kern/sched_prim.h>
111 #include <kern/sync_lock.h>
112 #include <kern/syscall_subr.h>
113 #include <kern/task.h>
114 #include <kern/thread.h>
115 #include <kern/thread_group.h>
116 #include <kern/coalition.h>
117 #include <kern/host.h>
118 #include <kern/zalloc.h>
119 #include <kern/assert.h>
120 #include <kern/exc_resource.h>
121 #include <kern/exc_guard.h>
122 #include <kern/telemetry.h>
123 #include <kern/policy_internal.h>
124 #include <kern/turnstile.h>
125 #include <kern/sched_clutch.h>
126
127 #include <corpses/task_corpse.h>
128 #if KPC
129 #include <kern/kpc.h>
130 #endif
131
132 #if MONOTONIC
133 #include <kern/monotonic.h>
134 #include <machine/monotonic.h>
135 #endif /* MONOTONIC */
136
137 #include <ipc/ipc_kmsg.h>
138 #include <ipc/ipc_port.h>
139 #include <bank/bank_types.h>
140
141 #include <vm/vm_kern.h>
142 #include <vm/vm_pageout.h>
143
144 #include <sys/kdebug.h>
145 #include <sys/bsdtask_info.h>
146 #include <mach/sdt.h>
147 #include <san/kasan.h>
148 #if CONFIG_KSANCOV
149 #include <san/ksancov.h>
150 #endif
151
152 #include <stdatomic.h>
153
154 #if defined(HAS_APPLE_PAC)
155 #include <ptrauth.h>
156 #include <arm64/proc_reg.h>
157 #endif /* defined(HAS_APPLE_PAC) */
158
159 /*
160 * Exported interfaces
161 */
162 #include <mach/task_server.h>
163 #include <mach/thread_act_server.h>
164 #include <mach/mach_host_server.h>
165 #include <mach/host_priv_server.h>
166 #include <mach/mach_voucher_server.h>
167 #include <kern/policy_internal.h>
168
169 static struct zone *thread_zone;
170 static lck_grp_attr_t thread_lck_grp_attr;
171 lck_attr_t thread_lck_attr;
172 lck_grp_t thread_lck_grp;
173
174 struct zone *thread_qos_override_zone;
175
176 static struct mpsc_daemon_queue thread_stack_queue;
177 static struct mpsc_daemon_queue thread_terminate_queue;
178 static struct mpsc_daemon_queue thread_deallocate_queue;
179 static struct mpsc_daemon_queue thread_exception_queue;
180
181 decl_simple_lock_data(static, crashed_threads_lock);
182 static queue_head_t crashed_threads_queue;
183
184 struct thread_exception_elt {
185 struct mpsc_queue_chain link;
186 exception_type_t exception_type;
187 task_t exception_task;
188 thread_t exception_thread;
189 };
190
191 static struct thread thread_template, init_thread;
192 static void thread_deallocate_enqueue(thread_t thread);
193 static void thread_deallocate_complete(thread_t thread);
194
195 #ifdef MACH_BSD
196 extern void proc_exit(void *);
197 extern mach_exception_data_type_t proc_encode_exit_exception_code(void *);
198 extern uint64_t get_dispatchqueue_offset_from_proc(void *);
199 extern uint64_t get_return_to_kernel_offset_from_proc(void *p);
200 extern int proc_selfpid(void);
201 extern void proc_name(int, char*, int);
202 extern char * proc_name_address(void *p);
203 #endif /* MACH_BSD */
204
205 extern int disable_exc_resource;
206 extern int audio_active;
207 extern int debug_task;
208 int thread_max = CONFIG_THREAD_MAX; /* Max number of threads */
209 int task_threadmax = CONFIG_THREAD_MAX;
210
211 static uint64_t thread_unique_id = 100;
212
213 struct _thread_ledger_indices thread_ledgers = { .cpu_time = -1 };
214 static ledger_template_t thread_ledger_template = NULL;
215 static void init_thread_ledgers(void);
216
217 #if CONFIG_JETSAM
218 void jetsam_on_ledger_cpulimit_exceeded(void);
219 #endif
220
221 extern int task_thread_soft_limit;
222 extern int exc_via_corpse_forking;
223
224 #if DEVELOPMENT || DEBUG
225 extern int exc_resource_threads_enabled;
226 #endif /* DEVELOPMENT || DEBUG */
227
228 /*
229 * Level (in terms of percentage of the limit) at which the CPU usage monitor triggers telemetry.
230 *
231 * (ie when any thread's CPU consumption exceeds 70% of the limit, start taking user
232 * stacktraces, aka micro-stackshots)
233 */
234 #define CPUMON_USTACKSHOTS_TRIGGER_DEFAULT_PCT 70
235
236 int cpumon_ustackshots_trigger_pct; /* Percentage. Level at which we start gathering telemetry. */
237 void __attribute__((noinline)) SENDING_NOTIFICATION__THIS_THREAD_IS_CONSUMING_TOO_MUCH_CPU(void);
238 #if DEVELOPMENT || DEBUG
239 void __attribute__((noinline)) SENDING_NOTIFICATION__TASK_HAS_TOO_MANY_THREADS(task_t, int);
240 #endif /* DEVELOPMENT || DEBUG */
241
242 /*
243 * The smallest interval over which we support limiting CPU consumption is 1ms
244 */
245 #define MINIMUM_CPULIMIT_INTERVAL_MS 1
246
247 os_refgrp_decl(static, thread_refgrp, "thread", NULL);
248
249 thread_t
250 thread_bootstrap(void)
251 {
252 /*
253 * Fill in a template thread for fast initialization.
254 */
255
256 #if MACH_ASSERT
257 thread_template.thread_magic = THREAD_MAGIC;
258 #endif /* MACH_ASSERT */
259
260 thread_template.runq = PROCESSOR_NULL;
261
262 thread_template.reason = AST_NONE;
263 thread_template.at_safe_point = FALSE;
264 thread_template.wait_event = NO_EVENT64;
265 thread_template.waitq = NULL;
266 thread_template.wait_result = THREAD_WAITING;
267 thread_template.options = THREAD_ABORTSAFE;
268 thread_template.state = TH_WAIT | TH_UNINT;
269 thread_template.wake_active = FALSE;
270 thread_template.continuation = THREAD_CONTINUE_NULL;
271 thread_template.parameter = NULL;
272
273 thread_template.importance = 0;
274 thread_template.sched_mode = TH_MODE_NONE;
275 thread_template.sched_flags = 0;
276 thread_template.saved_mode = TH_MODE_NONE;
277 thread_template.safe_release = 0;
278 thread_template.th_sched_bucket = TH_BUCKET_RUN;
279
280 thread_template.sfi_class = SFI_CLASS_UNSPECIFIED;
281 thread_template.sfi_wait_class = SFI_CLASS_UNSPECIFIED;
282
283 thread_template.active = 0;
284 thread_template.started = 0;
285 thread_template.static_param = 0;
286 thread_template.policy_reset = 0;
287
288 thread_template.base_pri = BASEPRI_DEFAULT;
289 thread_template.sched_pri = 0;
290 thread_template.max_priority = 0;
291 thread_template.task_priority = 0;
292 thread_template.rwlock_count = 0;
293 thread_template.waiting_for_mutex = NULL;
294
295
296 thread_template.realtime.deadline = UINT64_MAX;
297
298 thread_template.quantum_remaining = 0;
299 thread_template.last_run_time = 0;
300 thread_template.last_made_runnable_time = THREAD_NOT_RUNNABLE;
301 thread_template.last_basepri_change_time = THREAD_NOT_RUNNABLE;
302 thread_template.same_pri_latency = 0;
303
304 thread_template.computation_metered = 0;
305 thread_template.computation_epoch = 0;
306
307 #if defined(CONFIG_SCHED_TIMESHARE_CORE)
308 thread_template.sched_stamp = 0;
309 thread_template.pri_shift = INT8_MAX;
310 thread_template.sched_usage = 0;
311 thread_template.cpu_usage = thread_template.cpu_delta = 0;
312 #endif
313 thread_template.c_switch = thread_template.p_switch = thread_template.ps_switch = 0;
314
315 #if MONOTONIC
316 memset(&thread_template.t_monotonic, 0,
317 sizeof(thread_template.t_monotonic));
318 #endif /* MONOTONIC */
319
320 thread_template.bound_processor = PROCESSOR_NULL;
321 thread_template.last_processor = PROCESSOR_NULL;
322
323 thread_template.sched_call = NULL;
324
325 timer_init(&thread_template.user_timer);
326 timer_init(&thread_template.system_timer);
327 timer_init(&thread_template.ptime);
328 timer_init(&thread_template.runnable_timer);
329 thread_template.user_timer_save = 0;
330 thread_template.system_timer_save = 0;
331 thread_template.vtimer_user_save = 0;
332 thread_template.vtimer_prof_save = 0;
333 thread_template.vtimer_rlim_save = 0;
334 thread_template.vtimer_qos_save = 0;
335
336 #if CONFIG_SCHED_SFI
337 thread_template.wait_sfi_begin_time = 0;
338 #endif
339
340 thread_template.wait_timer_is_set = FALSE;
341 thread_template.wait_timer_active = 0;
342
343 thread_template.depress_timer_active = 0;
344
345 thread_template.recover = (vm_offset_t)NULL;
346
347 thread_template.map = VM_MAP_NULL;
348 #if DEVELOPMENT || DEBUG
349 thread_template.pmap_footprint_suspended = FALSE;
350 #endif /* DEVELOPMENT || DEBUG */
351
352 #if CONFIG_DTRACE
353 thread_template.t_dtrace_predcache = 0;
354 thread_template.t_dtrace_vtime = 0;
355 thread_template.t_dtrace_tracing = 0;
356 #endif /* CONFIG_DTRACE */
357
358 #if KPERF
359 thread_template.kperf_ast = 0;
360 thread_template.kperf_pet_gen = 0;
361 thread_template.kperf_c_switch = 0;
362 thread_template.kperf_pet_cnt = 0;
363 #endif
364
365 #if KPC
366 thread_template.kpc_buf = NULL;
367 #endif
368
369 #if HYPERVISOR
370 thread_template.hv_thread_target = NULL;
371 #endif /* HYPERVISOR */
372
373 #if (DEVELOPMENT || DEBUG)
374 thread_template.t_page_creation_throttled_hard = 0;
375 thread_template.t_page_creation_throttled_soft = 0;
376 #endif /* DEVELOPMENT || DEBUG */
377 thread_template.t_page_creation_throttled = 0;
378 thread_template.t_page_creation_count = 0;
379 thread_template.t_page_creation_time = 0;
380
381 thread_template.affinity_set = NULL;
382
383 thread_template.syscalls_unix = 0;
384 thread_template.syscalls_mach = 0;
385
386 thread_template.t_ledger = LEDGER_NULL;
387 thread_template.t_threadledger = LEDGER_NULL;
388 thread_template.t_bankledger = LEDGER_NULL;
389 thread_template.t_deduct_bank_ledger_time = 0;
390
391 thread_template.requested_policy = (struct thread_requested_policy) {};
392 thread_template.effective_policy = (struct thread_effective_policy) {};
393
394 bzero(&thread_template.overrides, sizeof(thread_template.overrides));
395 thread_template.kevent_overrides = 0;
396
397 thread_template.iotier_override = THROTTLE_LEVEL_NONE;
398 thread_template.thread_io_stats = NULL;
399 #if CONFIG_EMBEDDED
400 thread_template.taskwatch = NULL;
401 #endif /* CONFIG_EMBEDDED */
402 thread_template.thread_callout_interrupt_wakeups = thread_template.thread_callout_platform_idle_wakeups = 0;
403
404 thread_template.thread_timer_wakeups_bin_1 = thread_template.thread_timer_wakeups_bin_2 = 0;
405 thread_template.callout_woken_from_icontext = thread_template.callout_woken_from_platform_idle = 0;
406 thread_template.guard_exc_fatal = 0;
407
408 thread_template.thread_tag = 0;
409
410 thread_template.ith_voucher_name = MACH_PORT_NULL;
411 thread_template.ith_voucher = IPC_VOUCHER_NULL;
412
413 thread_template.th_work_interval = NULL;
414
415 thread_template.decompressions = 0;
416 init_thread = thread_template;
417
418 /* fiddle with init thread to skip asserts in set_sched_pri */
419 init_thread.sched_pri = MAXPRI_KERNEL;
420
421 return &init_thread;
422 }
423
424 void
425 thread_machine_init_template(void)
426 {
427 machine_thread_template_init(&thread_template);
428 }
429
430 extern boolean_t allow_qos_policy_set;
431
432 void
433 thread_init(void)
434 {
435 thread_zone = zinit(
436 sizeof(struct thread),
437 thread_max * sizeof(struct thread),
438 THREAD_CHUNK * sizeof(struct thread),
439 "threads");
440
441 thread_qos_override_zone = zinit(
442 sizeof(struct thread_qos_override),
443 4 * thread_max * sizeof(struct thread_qos_override),
444 PAGE_SIZE,
445 "thread qos override");
446 zone_change(thread_qos_override_zone, Z_EXPAND, TRUE);
447 zone_change(thread_qos_override_zone, Z_COLLECT, TRUE);
448 zone_change(thread_qos_override_zone, Z_CALLERACCT, FALSE);
449 zone_change(thread_qos_override_zone, Z_NOENCRYPT, TRUE);
450
451 lck_grp_attr_setdefault(&thread_lck_grp_attr);
452 lck_grp_init(&thread_lck_grp, "thread", &thread_lck_grp_attr);
453 lck_attr_setdefault(&thread_lck_attr);
454
455 stack_init();
456
457 thread_policy_init();
458
459 /*
460 * Initialize any machine-dependent
461 * per-thread structures necessary.
462 */
463 machine_thread_init();
464
465 if (!PE_parse_boot_argn("cpumon_ustackshots_trigger_pct", &cpumon_ustackshots_trigger_pct,
466 sizeof(cpumon_ustackshots_trigger_pct))) {
467 cpumon_ustackshots_trigger_pct = CPUMON_USTACKSHOTS_TRIGGER_DEFAULT_PCT;
468 }
469
470 PE_parse_boot_argn("-qos-policy-allow", &allow_qos_policy_set, sizeof(allow_qos_policy_set));
471
472 init_thread_ledgers();
473 }
474
475 boolean_t
476 thread_is_active(thread_t thread)
477 {
478 return thread->active;
479 }
480
481 void
482 thread_corpse_continue(void)
483 {
484 thread_t thread = current_thread();
485
486 thread_terminate_internal(thread);
487
488 /*
489 * Handle the thread termination directly
490 * here instead of returning to userspace.
491 */
492 assert(thread->active == FALSE);
493 thread_ast_clear(thread, AST_APC);
494 thread_apc_ast(thread);
495
496 panic("thread_corpse_continue");
497 /*NOTREACHED*/
498 }
499
500 __dead2
501 static void
502 thread_terminate_continue(void)
503 {
504 panic("thread_terminate_continue");
505 /*NOTREACHED*/
506 }
507
508 /*
509 * thread_terminate_self:
510 */
511 void
512 thread_terminate_self(void)
513 {
514 thread_t thread = current_thread();
515 task_t task;
516 int threadcnt;
517
518 pal_thread_terminate_self(thread);
519
520 DTRACE_PROC(lwp__exit);
521
522 thread_mtx_lock(thread);
523
524 ipc_thread_disable(thread);
525
526 thread_mtx_unlock(thread);
527
528 thread_sched_call(thread, NULL);
529
530 spl_t s = splsched();
531 thread_lock(thread);
532
533 thread_depress_abort_locked(thread);
534
535 thread_unlock(thread);
536 splx(s);
537
538 #if CONFIG_EMBEDDED
539 thead_remove_taskwatch(thread);
540 #endif /* CONFIG_EMBEDDED */
541
542 work_interval_thread_terminate(thread);
543
544 thread_mtx_lock(thread);
545
546 thread_policy_reset(thread);
547
548 thread_mtx_unlock(thread);
549
550 bank_swap_thread_bank_ledger(thread, NULL);
551
552 if (kdebug_enable && bsd_hasthreadname(thread->uthread)) {
553 char threadname[MAXTHREADNAMESIZE];
554 bsd_getthreadname(thread->uthread, threadname);
555 kernel_debug_string_simple(TRACE_STRING_THREADNAME_PREV, threadname);
556 }
557
558 task = thread->task;
559 uthread_cleanup(task, thread->uthread, task->bsd_info);
560
561 if (kdebug_enable && task->bsd_info && !task_is_exec_copy(task)) {
562 /* trace out pid before we sign off */
563 long dbg_arg1 = 0;
564 long dbg_arg2 = 0;
565
566 kdbg_trace_data(thread->task->bsd_info, &dbg_arg1, &dbg_arg2);
567 KDBG_RELEASE(TRACE_DATA_THREAD_TERMINATE_PID, dbg_arg1, dbg_arg2);
568 }
569
570 /*
571 * After this subtraction, this thread should never access
572 * task->bsd_info unless it got 0 back from the os_atomic_dec. It
573 * could be racing with other threads to be the last thread in the
574 * process, and the last thread in the process will tear down the proc
575 * structure and zero-out task->bsd_info.
576 */
577 threadcnt = os_atomic_dec(&task->active_thread_count, relaxed);
578
579 /*
580 * If we are the last thread to terminate and the task is
581 * associated with a BSD process, perform BSD process exit.
582 */
583 if (threadcnt == 0 && task->bsd_info != NULL && !task_is_exec_copy(task)) {
584 mach_exception_data_type_t subcode = 0;
585 if (kdebug_enable) {
586 /* since we're the last thread in this process, trace out the command name too */
587 long args[4] = {};
588 kdbg_trace_string(thread->task->bsd_info, &args[0], &args[1], &args[2], &args[3]);
589 KDBG_RELEASE(TRACE_STRING_PROC_EXIT, args[0], args[1], args[2], args[3]);
590 }
591
592 /* Get the exit reason before proc_exit */
593 subcode = proc_encode_exit_exception_code(task->bsd_info);
594 proc_exit(task->bsd_info);
595 /*
596 * if there is crash info in task
597 * then do the deliver action since this is
598 * last thread for this task.
599 */
600 if (task->corpse_info) {
601 task_deliver_crash_notification(task, current_thread(), EXC_RESOURCE, subcode);
602 }
603 }
604
605 if (threadcnt == 0) {
606 task_lock(task);
607 if (task_is_a_corpse_fork(task)) {
608 thread_wakeup((event_t)&task->active_thread_count);
609 }
610 task_unlock(task);
611 }
612
613 uthread_cred_free(thread->uthread);
614
615 s = splsched();
616 thread_lock(thread);
617
618 /*
619 * Ensure that the depress timer is no longer enqueued,
620 * so the timer (stored in the thread) can be safely deallocated
621 *
622 * TODO: build timer_call_cancel_wait
623 */
624
625 assert((thread->sched_flags & TH_SFLAG_DEPRESSED_MASK) == 0);
626
627 uint32_t delay_us = 1;
628
629 while (thread->depress_timer_active > 0) {
630 thread_unlock(thread);
631 splx(s);
632
633 delay(delay_us++);
634
635 if (delay_us > USEC_PER_SEC) {
636 panic("depress timer failed to inactivate!"
637 "thread: %p depress_timer_active: %d",
638 thread, thread->depress_timer_active);
639 }
640
641 s = splsched();
642 thread_lock(thread);
643 }
644
645 /*
646 * Cancel wait timer, and wait for
647 * concurrent expirations.
648 */
649 if (thread->wait_timer_is_set) {
650 thread->wait_timer_is_set = FALSE;
651
652 if (timer_call_cancel(&thread->wait_timer)) {
653 thread->wait_timer_active--;
654 }
655 }
656
657 delay_us = 1;
658
659 while (thread->wait_timer_active > 0) {
660 thread_unlock(thread);
661 splx(s);
662
663 delay(delay_us++);
664
665 if (delay_us > USEC_PER_SEC) {
666 panic("wait timer failed to inactivate!"
667 "thread: %p wait_timer_active: %d",
668 thread, thread->wait_timer_active);
669 }
670
671 s = splsched();
672 thread_lock(thread);
673 }
674
675 /*
676 * If there is a reserved stack, release it.
677 */
678 if (thread->reserved_stack != 0) {
679 stack_free_reserved(thread);
680 thread->reserved_stack = 0;
681 }
682
683 /*
684 * Mark thread as terminating, and block.
685 */
686 thread->state |= TH_TERMINATE;
687 thread_mark_wait_locked(thread, THREAD_UNINT);
688
689 assert((thread->sched_flags & TH_SFLAG_WAITQ_PROMOTED) == 0);
690 assert((thread->sched_flags & TH_SFLAG_RW_PROMOTED) == 0);
691 assert((thread->sched_flags & TH_SFLAG_EXEC_PROMOTED) == 0);
692 assert((thread->sched_flags & TH_SFLAG_PROMOTED) == 0);
693 assert(thread->kern_promotion_schedpri == 0);
694 assert(thread->waiting_for_mutex == NULL);
695 assert(thread->rwlock_count == 0);
696
697 thread_unlock(thread);
698 /* splsched */
699
700 thread_block((thread_continue_t)thread_terminate_continue);
701 /*NOTREACHED*/
702 }
703
704 static bool
705 thread_ref_release(thread_t thread)
706 {
707 if (thread == THREAD_NULL) {
708 return false;
709 }
710
711 assert_thread_magic(thread);
712
713 return os_ref_release(&thread->ref_count) == 0;
714 }
715
716 /* Drop a thread refcount safely without triggering a zfree */
717 void
718 thread_deallocate_safe(thread_t thread)
719 {
720 if (__improbable(thread_ref_release(thread))) {
721 /* enqueue the thread for thread deallocate deamon to call thread_deallocate_complete */
722 thread_deallocate_enqueue(thread);
723 }
724 }
725
726 void
727 thread_deallocate(thread_t thread)
728 {
729 if (__improbable(thread_ref_release(thread))) {
730 thread_deallocate_complete(thread);
731 }
732 }
733
734 void
735 thread_deallocate_complete(
736 thread_t thread)
737 {
738 task_t task;
739
740 assert_thread_magic(thread);
741
742 assert(os_ref_get_count(&thread->ref_count) == 0);
743
744 if (!(thread->state & TH_TERMINATE2)) {
745 panic("thread_deallocate: thread not properly terminated\n");
746 }
747
748 assert(thread->runq == PROCESSOR_NULL);
749
750 #if KPC
751 kpc_thread_destroy(thread);
752 #endif
753
754 ipc_thread_terminate(thread);
755
756 proc_thread_qos_deallocate(thread);
757
758 task = thread->task;
759
760 #ifdef MACH_BSD
761 {
762 void *ut = thread->uthread;
763
764 thread->uthread = NULL;
765 uthread_zone_free(ut);
766 }
767 #endif /* MACH_BSD */
768
769 if (thread->t_ledger) {
770 ledger_dereference(thread->t_ledger);
771 }
772 if (thread->t_threadledger) {
773 ledger_dereference(thread->t_threadledger);
774 }
775
776 assert(thread->turnstile != TURNSTILE_NULL);
777 if (thread->turnstile) {
778 turnstile_deallocate(thread->turnstile);
779 }
780
781 if (IPC_VOUCHER_NULL != thread->ith_voucher) {
782 ipc_voucher_release(thread->ith_voucher);
783 }
784
785 if (thread->thread_io_stats) {
786 kfree(thread->thread_io_stats, sizeof(struct io_stat_info));
787 }
788
789 if (thread->kernel_stack != 0) {
790 stack_free(thread);
791 }
792
793 lck_mtx_destroy(&thread->mutex, &thread_lck_grp);
794 machine_thread_destroy(thread);
795
796 task_deallocate(task);
797
798 #if MACH_ASSERT
799 assert_thread_magic(thread);
800 thread->thread_magic = 0;
801 #endif /* MACH_ASSERT */
802
803 zfree(thread_zone, thread);
804 }
805
806 /*
807 * thread_inspect_deallocate:
808 *
809 * Drop a thread inspection reference.
810 */
811 void
812 thread_inspect_deallocate(
813 thread_inspect_t thread_inspect)
814 {
815 return thread_deallocate((thread_t)thread_inspect);
816 }
817
818 /*
819 * thread_exception_queue_invoke:
820 *
821 * Deliver EXC_{RESOURCE,GUARD} exception
822 */
823 static void
824 thread_exception_queue_invoke(mpsc_queue_chain_t elm,
825 __assert_only mpsc_daemon_queue_t dq)
826 {
827 struct thread_exception_elt *elt;
828 task_t task;
829 thread_t thread;
830 exception_type_t etype;
831
832 assert(dq == &thread_exception_queue);
833 elt = mpsc_queue_element(elm, struct thread_exception_elt, link);
834
835 etype = elt->exception_type;
836 task = elt->exception_task;
837 thread = elt->exception_thread;
838 assert_thread_magic(thread);
839
840 kfree(elt, sizeof(*elt));
841
842 /* wait for all the threads in the task to terminate */
843 task_lock(task);
844 task_wait_till_threads_terminate_locked(task);
845 task_unlock(task);
846
847 /* Consumes the task ref returned by task_generate_corpse_internal */
848 task_deallocate(task);
849 /* Consumes the thread ref returned by task_generate_corpse_internal */
850 thread_deallocate(thread);
851
852 /* Deliver the notification, also clears the corpse. */
853 task_deliver_crash_notification(task, thread, etype, 0);
854 }
855
856 /*
857 * thread_exception_enqueue:
858 *
859 * Enqueue a corpse port to be delivered an EXC_{RESOURCE,GUARD}.
860 */
861 void
862 thread_exception_enqueue(
863 task_t task,
864 thread_t thread,
865 exception_type_t etype)
866 {
867 assert(EXC_RESOURCE == etype || EXC_GUARD == etype);
868 struct thread_exception_elt *elt = kalloc(sizeof(*elt));
869 elt->exception_type = etype;
870 elt->exception_task = task;
871 elt->exception_thread = thread;
872
873 mpsc_daemon_enqueue(&thread_exception_queue, &elt->link,
874 MPSC_QUEUE_DISABLE_PREEMPTION);
875 }
876
877 /*
878 * thread_copy_resource_info
879 *
880 * Copy the resource info counters from source
881 * thread to destination thread.
882 */
883 void
884 thread_copy_resource_info(
885 thread_t dst_thread,
886 thread_t src_thread)
887 {
888 dst_thread->c_switch = src_thread->c_switch;
889 dst_thread->p_switch = src_thread->p_switch;
890 dst_thread->ps_switch = src_thread->ps_switch;
891 dst_thread->precise_user_kernel_time = src_thread->precise_user_kernel_time;
892 dst_thread->user_timer = src_thread->user_timer;
893 dst_thread->user_timer_save = src_thread->user_timer_save;
894 dst_thread->system_timer = src_thread->system_timer;
895 dst_thread->system_timer_save = src_thread->system_timer_save;
896 dst_thread->runnable_timer = src_thread->runnable_timer;
897 dst_thread->vtimer_user_save = src_thread->vtimer_user_save;
898 dst_thread->vtimer_prof_save = src_thread->vtimer_prof_save;
899 dst_thread->vtimer_rlim_save = src_thread->vtimer_rlim_save;
900 dst_thread->vtimer_qos_save = src_thread->vtimer_qos_save;
901 dst_thread->syscalls_unix = src_thread->syscalls_unix;
902 dst_thread->syscalls_mach = src_thread->syscalls_mach;
903 ledger_rollup(dst_thread->t_threadledger, src_thread->t_threadledger);
904 *dst_thread->thread_io_stats = *src_thread->thread_io_stats;
905 }
906
907 static void
908 thread_terminate_queue_invoke(mpsc_queue_chain_t e,
909 __assert_only mpsc_daemon_queue_t dq)
910 {
911 thread_t thread = mpsc_queue_element(e, struct thread, mpsc_links);
912 task_t task = thread->task;
913
914 assert(dq == &thread_terminate_queue);
915
916 task_lock(task);
917
918 /*
919 * if marked for crash reporting, skip reaping.
920 * The corpse delivery thread will clear bit and enqueue
921 * for reaping when done
922 *
923 * Note: the inspection field is set under the task lock
924 *
925 * FIXME[mad]: why enqueue for termination before `inspection` is false ?
926 */
927 if (__improbable(thread->inspection)) {
928 simple_lock(&crashed_threads_lock, &thread_lck_grp);
929 task_unlock(task);
930
931 enqueue_tail(&crashed_threads_queue, &thread->runq_links);
932 simple_unlock(&crashed_threads_lock);
933 return;
934 }
935
936
937 task->total_user_time += timer_grab(&thread->user_timer);
938 task->total_ptime += timer_grab(&thread->ptime);
939 task->total_runnable_time += timer_grab(&thread->runnable_timer);
940 if (thread->precise_user_kernel_time) {
941 task->total_system_time += timer_grab(&thread->system_timer);
942 } else {
943 task->total_user_time += timer_grab(&thread->system_timer);
944 }
945
946 task->c_switch += thread->c_switch;
947 task->p_switch += thread->p_switch;
948 task->ps_switch += thread->ps_switch;
949
950 task->syscalls_unix += thread->syscalls_unix;
951 task->syscalls_mach += thread->syscalls_mach;
952
953 task->task_timer_wakeups_bin_1 += thread->thread_timer_wakeups_bin_1;
954 task->task_timer_wakeups_bin_2 += thread->thread_timer_wakeups_bin_2;
955 task->task_gpu_ns += ml_gpu_stat(thread);
956 task->task_energy += ml_energy_stat(thread);
957 task->decompressions += thread->decompressions;
958
959 #if MONOTONIC
960 mt_terminate_update(task, thread);
961 #endif /* MONOTONIC */
962
963 thread_update_qos_cpu_time(thread);
964
965 queue_remove(&task->threads, thread, thread_t, task_threads);
966 task->thread_count--;
967
968 /*
969 * If the task is being halted, and there is only one thread
970 * left in the task after this one, then wakeup that thread.
971 */
972 if (task->thread_count == 1 && task->halting) {
973 thread_wakeup((event_t)&task->halting);
974 }
975
976 task_unlock(task);
977
978 lck_mtx_lock(&tasks_threads_lock);
979 queue_remove(&threads, thread, thread_t, threads);
980 threads_count--;
981 lck_mtx_unlock(&tasks_threads_lock);
982
983 thread_deallocate(thread);
984 }
985
986 static void
987 thread_deallocate_queue_invoke(mpsc_queue_chain_t e,
988 __assert_only mpsc_daemon_queue_t dq)
989 {
990 thread_t thread = mpsc_queue_element(e, struct thread, mpsc_links);
991
992 assert(dq == &thread_deallocate_queue);
993
994 thread_deallocate_complete(thread);
995 }
996
997 /*
998 * thread_terminate_enqueue:
999 *
1000 * Enqueue a terminating thread for final disposition.
1001 *
1002 * Called at splsched.
1003 */
1004 void
1005 thread_terminate_enqueue(
1006 thread_t thread)
1007 {
1008 KDBG_RELEASE(TRACE_DATA_THREAD_TERMINATE, thread->thread_id);
1009
1010 mpsc_daemon_enqueue(&thread_terminate_queue, &thread->mpsc_links,
1011 MPSC_QUEUE_DISABLE_PREEMPTION);
1012 }
1013
1014 /*
1015 * thread_deallocate_enqueue:
1016 *
1017 * Enqueue a thread for final deallocation.
1018 */
1019 static void
1020 thread_deallocate_enqueue(
1021 thread_t thread)
1022 {
1023 mpsc_daemon_enqueue(&thread_deallocate_queue, &thread->mpsc_links,
1024 MPSC_QUEUE_DISABLE_PREEMPTION);
1025 }
1026
1027 /*
1028 * thread_terminate_crashed_threads:
1029 * walk the list of crashed threads and put back set of threads
1030 * who are no longer being inspected.
1031 */
1032 void
1033 thread_terminate_crashed_threads(void)
1034 {
1035 thread_t th_remove;
1036
1037 simple_lock(&crashed_threads_lock, &thread_lck_grp);
1038 /*
1039 * loop through the crashed threads queue
1040 * to put any threads that are not being inspected anymore
1041 */
1042
1043 qe_foreach_element_safe(th_remove, &crashed_threads_queue, runq_links) {
1044 /* make sure current_thread is never in crashed queue */
1045 assert(th_remove != current_thread());
1046
1047 if (th_remove->inspection == FALSE) {
1048 remqueue(&th_remove->runq_links);
1049 mpsc_daemon_enqueue(&thread_terminate_queue, &th_remove->mpsc_links,
1050 MPSC_QUEUE_NONE);
1051 }
1052 }
1053
1054 simple_unlock(&crashed_threads_lock);
1055 }
1056
1057 /*
1058 * thread_stack_queue_invoke:
1059 *
1060 * Perform stack allocation as required due to
1061 * invoke failures.
1062 */
1063 static void
1064 thread_stack_queue_invoke(mpsc_queue_chain_t elm,
1065 __assert_only mpsc_daemon_queue_t dq)
1066 {
1067 thread_t thread = mpsc_queue_element(elm, struct thread, mpsc_links);
1068
1069 assert(dq == &thread_stack_queue);
1070
1071 /* allocate stack with interrupts enabled so that we can call into VM */
1072 stack_alloc(thread);
1073
1074 KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_STACK_WAIT) | DBG_FUNC_END, thread_tid(thread), 0, 0, 0, 0);
1075
1076 spl_t s = splsched();
1077 thread_lock(thread);
1078 thread_setrun(thread, SCHED_PREEMPT | SCHED_TAILQ);
1079 thread_unlock(thread);
1080 splx(s);
1081 }
1082
1083 /*
1084 * thread_stack_enqueue:
1085 *
1086 * Enqueue a thread for stack allocation.
1087 *
1088 * Called at splsched.
1089 */
1090 void
1091 thread_stack_enqueue(
1092 thread_t thread)
1093 {
1094 KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_STACK_WAIT) | DBG_FUNC_START, thread_tid(thread), 0, 0, 0, 0);
1095 assert_thread_magic(thread);
1096
1097 mpsc_daemon_enqueue(&thread_stack_queue, &thread->mpsc_links,
1098 MPSC_QUEUE_DISABLE_PREEMPTION);
1099 }
1100
1101 void
1102 thread_daemon_init(void)
1103 {
1104 kern_return_t result;
1105
1106 thread_deallocate_daemon_init();
1107
1108 thread_deallocate_daemon_register_queue(&thread_terminate_queue,
1109 thread_terminate_queue_invoke);
1110
1111 thread_deallocate_daemon_register_queue(&thread_deallocate_queue,
1112 thread_deallocate_queue_invoke);
1113
1114 simple_lock_init(&crashed_threads_lock, 0);
1115 queue_init(&crashed_threads_queue);
1116
1117 result = mpsc_daemon_queue_init_with_thread(&thread_stack_queue,
1118 thread_stack_queue_invoke, BASEPRI_PREEMPT_HIGH,
1119 "daemon.thread-stack");
1120 if (result != KERN_SUCCESS) {
1121 panic("thread_daemon_init: thread_stack_daemon");
1122 }
1123
1124 result = mpsc_daemon_queue_init_with_thread(&thread_exception_queue,
1125 thread_exception_queue_invoke, MINPRI_KERNEL,
1126 "daemon.thread-exception");
1127 if (result != KERN_SUCCESS) {
1128 panic("thread_daemon_init: thread_exception_daemon");
1129 }
1130 }
1131
1132 #define TH_OPTION_NONE 0x00
1133 #define TH_OPTION_NOCRED 0x01
1134 #define TH_OPTION_NOSUSP 0x02
1135 #define TH_OPTION_WORKQ 0x04
1136
1137 /*
1138 * Create a new thread.
1139 * Doesn't start the thread running.
1140 *
1141 * Task and tasks_threads_lock are returned locked on success.
1142 */
1143 static kern_return_t
1144 thread_create_internal(
1145 task_t parent_task,
1146 integer_t priority,
1147 thread_continue_t continuation,
1148 void *parameter,
1149 int options,
1150 thread_t *out_thread)
1151 {
1152 thread_t new_thread;
1153 static thread_t first_thread;
1154
1155 /*
1156 * Allocate a thread and initialize static fields
1157 */
1158 if (first_thread == THREAD_NULL) {
1159 new_thread = first_thread = current_thread();
1160 } else {
1161 new_thread = (thread_t)zalloc(thread_zone);
1162 }
1163 if (new_thread == THREAD_NULL) {
1164 return KERN_RESOURCE_SHORTAGE;
1165 }
1166
1167 if (new_thread != first_thread) {
1168 *new_thread = thread_template;
1169 }
1170
1171 os_ref_init_count(&new_thread->ref_count, &thread_refgrp, 2);
1172
1173 #ifdef MACH_BSD
1174 new_thread->uthread = uthread_alloc(parent_task, new_thread, (options & TH_OPTION_NOCRED) != 0);
1175 if (new_thread->uthread == NULL) {
1176 #if MACH_ASSERT
1177 new_thread->thread_magic = 0;
1178 #endif /* MACH_ASSERT */
1179
1180 zfree(thread_zone, new_thread);
1181 return KERN_RESOURCE_SHORTAGE;
1182 }
1183 #endif /* MACH_BSD */
1184
1185 if (machine_thread_create(new_thread, parent_task) != KERN_SUCCESS) {
1186 #ifdef MACH_BSD
1187 void *ut = new_thread->uthread;
1188
1189 new_thread->uthread = NULL;
1190 /* cred free may not be necessary */
1191 uthread_cleanup(parent_task, ut, parent_task->bsd_info);
1192 uthread_cred_free(ut);
1193 uthread_zone_free(ut);
1194 #endif /* MACH_BSD */
1195
1196 #if MACH_ASSERT
1197 new_thread->thread_magic = 0;
1198 #endif /* MACH_ASSERT */
1199
1200 zfree(thread_zone, new_thread);
1201 return KERN_FAILURE;
1202 }
1203
1204 new_thread->task = parent_task;
1205
1206 thread_lock_init(new_thread);
1207 wake_lock_init(new_thread);
1208
1209 lck_mtx_init(&new_thread->mutex, &thread_lck_grp, &thread_lck_attr);
1210
1211 ipc_thread_init(new_thread);
1212
1213 new_thread->continuation = continuation;
1214 new_thread->parameter = parameter;
1215 new_thread->inheritor_flags = TURNSTILE_UPDATE_FLAGS_NONE;
1216 priority_queue_init(&new_thread->sched_inheritor_queue,
1217 PRIORITY_QUEUE_BUILTIN_MAX_HEAP);
1218 priority_queue_init(&new_thread->base_inheritor_queue,
1219 PRIORITY_QUEUE_BUILTIN_MAX_HEAP);
1220 #if CONFIG_SCHED_CLUTCH
1221 priority_queue_entry_init(&new_thread->sched_clutchpri_link);
1222 #endif /* CONFIG_SCHED_CLUTCH */
1223
1224 /* Allocate I/O Statistics structure */
1225 new_thread->thread_io_stats = (io_stat_info_t)kalloc(sizeof(struct io_stat_info));
1226 assert(new_thread->thread_io_stats != NULL);
1227 bzero(new_thread->thread_io_stats, sizeof(struct io_stat_info));
1228
1229 #if KASAN
1230 kasan_init_thread(&new_thread->kasan_data);
1231 #endif
1232
1233 #if CONFIG_KSANCOV
1234 new_thread->ksancov_data = NULL;
1235 #endif
1236
1237 #if CONFIG_IOSCHED
1238 /* Clear out the I/O Scheduling info for AppleFSCompression */
1239 new_thread->decmp_upl = NULL;
1240 #endif /* CONFIG_IOSCHED */
1241
1242 #if DEVELOPMENT || DEBUG
1243 task_lock(parent_task);
1244 uint16_t thread_limit = parent_task->task_thread_limit;
1245 if (exc_resource_threads_enabled &&
1246 thread_limit > 0 &&
1247 parent_task->thread_count >= thread_limit &&
1248 !parent_task->task_has_crossed_thread_limit &&
1249 !(parent_task->t_flags & TF_CORPSE)) {
1250 int thread_count = parent_task->thread_count;
1251 parent_task->task_has_crossed_thread_limit = TRUE;
1252 task_unlock(parent_task);
1253 SENDING_NOTIFICATION__TASK_HAS_TOO_MANY_THREADS(parent_task, thread_count);
1254 } else {
1255 task_unlock(parent_task);
1256 }
1257 #endif
1258
1259 lck_mtx_lock(&tasks_threads_lock);
1260 task_lock(parent_task);
1261
1262 /*
1263 * Fail thread creation if parent task is being torn down or has too many threads
1264 * If the caller asked for TH_OPTION_NOSUSP, also fail if the parent task is suspended
1265 */
1266 if (parent_task->active == 0 || parent_task->halting ||
1267 (parent_task->suspend_count > 0 && (options & TH_OPTION_NOSUSP) != 0) ||
1268 (parent_task->thread_count >= task_threadmax && parent_task != kernel_task)) {
1269 task_unlock(parent_task);
1270 lck_mtx_unlock(&tasks_threads_lock);
1271
1272 #ifdef MACH_BSD
1273 {
1274 void *ut = new_thread->uthread;
1275
1276 new_thread->uthread = NULL;
1277 uthread_cleanup(parent_task, ut, parent_task->bsd_info);
1278 /* cred free may not be necessary */
1279 uthread_cred_free(ut);
1280 uthread_zone_free(ut);
1281 }
1282 #endif /* MACH_BSD */
1283 ipc_thread_disable(new_thread);
1284 ipc_thread_terminate(new_thread);
1285 kfree(new_thread->thread_io_stats, sizeof(struct io_stat_info));
1286 lck_mtx_destroy(&new_thread->mutex, &thread_lck_grp);
1287 machine_thread_destroy(new_thread);
1288 zfree(thread_zone, new_thread);
1289 return KERN_FAILURE;
1290 }
1291
1292 /* Protected by the tasks_threads_lock */
1293 new_thread->thread_id = ++thread_unique_id;
1294
1295 /* New threads inherit any default state on the task */
1296 machine_thread_inherit_taskwide(new_thread, parent_task);
1297
1298 task_reference_internal(parent_task);
1299
1300 if (new_thread->task->rusage_cpu_flags & TASK_RUSECPU_FLAGS_PERTHR_LIMIT) {
1301 /*
1302 * This task has a per-thread CPU limit; make sure this new thread
1303 * gets its limit set too, before it gets out of the kernel.
1304 */
1305 act_set_astledger(new_thread);
1306 }
1307
1308 /* Instantiate a thread ledger. Do not fail thread creation if ledger creation fails. */
1309 if ((new_thread->t_threadledger = ledger_instantiate(thread_ledger_template,
1310 LEDGER_CREATE_INACTIVE_ENTRIES)) != LEDGER_NULL) {
1311 ledger_entry_setactive(new_thread->t_threadledger, thread_ledgers.cpu_time);
1312 }
1313
1314 new_thread->t_bankledger = LEDGER_NULL;
1315 new_thread->t_deduct_bank_ledger_time = 0;
1316 new_thread->t_deduct_bank_ledger_energy = 0;
1317
1318 new_thread->t_ledger = new_thread->task->ledger;
1319 if (new_thread->t_ledger) {
1320 ledger_reference(new_thread->t_ledger);
1321 }
1322
1323 #if defined(CONFIG_SCHED_MULTIQ)
1324 /* Cache the task's sched_group */
1325 new_thread->sched_group = parent_task->sched_group;
1326 #endif /* defined(CONFIG_SCHED_MULTIQ) */
1327
1328 /* Cache the task's map */
1329 new_thread->map = parent_task->map;
1330
1331 timer_call_setup(&new_thread->wait_timer, thread_timer_expire, new_thread);
1332 timer_call_setup(&new_thread->depress_timer, thread_depress_expire, new_thread);
1333
1334 #if KPC
1335 kpc_thread_create(new_thread);
1336 #endif
1337
1338 /* Set the thread's scheduling parameters */
1339 new_thread->sched_mode = SCHED(initial_thread_sched_mode)(parent_task);
1340 new_thread->max_priority = parent_task->max_priority;
1341 new_thread->task_priority = parent_task->priority;
1342
1343
1344 int new_priority = (priority < 0) ? parent_task->priority: priority;
1345 new_priority = (priority < 0)? parent_task->priority: priority;
1346 if (new_priority > new_thread->max_priority) {
1347 new_priority = new_thread->max_priority;
1348 }
1349 #if CONFIG_EMBEDDED
1350 if (new_priority < MAXPRI_THROTTLE) {
1351 new_priority = MAXPRI_THROTTLE;
1352 }
1353 #endif /* CONFIG_EMBEDDED */
1354
1355 new_thread->importance = new_priority - new_thread->task_priority;
1356
1357 sched_set_thread_base_priority(new_thread, new_priority);
1358
1359 #if defined(CONFIG_SCHED_TIMESHARE_CORE)
1360 new_thread->sched_stamp = sched_tick;
1361 #if CONFIG_SCHED_CLUTCH
1362 new_thread->pri_shift = sched_clutch_thread_pri_shift(new_thread, new_thread->th_sched_bucket);
1363 #else /* CONFIG_SCHED_CLUTCH */
1364 new_thread->pri_shift = sched_pri_shifts[new_thread->th_sched_bucket];
1365 #endif /* CONFIG_SCHED_CLUTCH */
1366 #endif /* defined(CONFIG_SCHED_TIMESHARE_CORE) */
1367
1368 #if CONFIG_EMBEDDED
1369 if (parent_task->max_priority <= MAXPRI_THROTTLE) {
1370 sched_thread_mode_demote(new_thread, TH_SFLAG_THROTTLED);
1371 }
1372 #endif /* CONFIG_EMBEDDED */
1373
1374 thread_policy_create(new_thread);
1375
1376 /* Chain the thread onto the task's list */
1377 queue_enter(&parent_task->threads, new_thread, thread_t, task_threads);
1378 parent_task->thread_count++;
1379
1380 /* So terminating threads don't need to take the task lock to decrement */
1381 os_atomic_inc(&parent_task->active_thread_count, relaxed);
1382
1383 queue_enter(&threads, new_thread, thread_t, threads);
1384 threads_count++;
1385
1386 new_thread->active = TRUE;
1387 if (task_is_a_corpse_fork(parent_task)) {
1388 /* Set the inspection bit if the task is a corpse fork */
1389 new_thread->inspection = TRUE;
1390 } else {
1391 new_thread->inspection = FALSE;
1392 }
1393 new_thread->corpse_dup = FALSE;
1394 new_thread->turnstile = turnstile_alloc();
1395 *out_thread = new_thread;
1396
1397 if (kdebug_enable) {
1398 long args[4] = {};
1399
1400 kdbg_trace_data(parent_task->bsd_info, &args[1], &args[3]);
1401
1402 /*
1403 * Starting with 26604425, exec'ing creates a new task/thread.
1404 *
1405 * NEWTHREAD in the current process has two possible meanings:
1406 *
1407 * 1) Create a new thread for this process.
1408 * 2) Create a new thread for the future process this will become in an
1409 * exec.
1410 *
1411 * To disambiguate these, arg3 will be set to TRUE for case #2.
1412 *
1413 * The value we need to find (TPF_EXEC_COPY) is stable in the case of a
1414 * task exec'ing. The read of t_procflags does not take the proc_lock.
1415 */
1416 args[2] = task_is_exec_copy(parent_task) ? 1 : 0;
1417
1418 KDBG_RELEASE(TRACE_DATA_NEWTHREAD, (uintptr_t)thread_tid(new_thread),
1419 args[1], args[2], args[3]);
1420
1421 kdbg_trace_string(parent_task->bsd_info, &args[0], &args[1],
1422 &args[2], &args[3]);
1423 KDBG_RELEASE(TRACE_STRING_NEWTHREAD, args[0], args[1], args[2],
1424 args[3]);
1425 }
1426
1427 DTRACE_PROC1(lwp__create, thread_t, *out_thread);
1428
1429 return KERN_SUCCESS;
1430 }
1431
1432 static kern_return_t
1433 thread_create_internal2(
1434 task_t task,
1435 thread_t *new_thread,
1436 boolean_t from_user,
1437 thread_continue_t continuation)
1438 {
1439 kern_return_t result;
1440 thread_t thread;
1441
1442 if (task == TASK_NULL || task == kernel_task) {
1443 return KERN_INVALID_ARGUMENT;
1444 }
1445
1446 result = thread_create_internal(task, -1, continuation, NULL, TH_OPTION_NONE, &thread);
1447 if (result != KERN_SUCCESS) {
1448 return result;
1449 }
1450
1451 thread->user_stop_count = 1;
1452 thread_hold(thread);
1453 if (task->suspend_count > 0) {
1454 thread_hold(thread);
1455 }
1456
1457 if (from_user) {
1458 extmod_statistics_incr_thread_create(task);
1459 }
1460
1461 task_unlock(task);
1462 lck_mtx_unlock(&tasks_threads_lock);
1463
1464 *new_thread = thread;
1465
1466 return KERN_SUCCESS;
1467 }
1468
1469 /* No prototype, since task_server.h has the _from_user version if KERNEL_SERVER */
1470 kern_return_t
1471 thread_create(
1472 task_t task,
1473 thread_t *new_thread);
1474
1475 kern_return_t
1476 thread_create(
1477 task_t task,
1478 thread_t *new_thread)
1479 {
1480 return thread_create_internal2(task, new_thread, FALSE, (thread_continue_t)thread_bootstrap_return);
1481 }
1482
1483 kern_return_t
1484 thread_create_from_user(
1485 task_t task,
1486 thread_t *new_thread)
1487 {
1488 return thread_create_internal2(task, new_thread, TRUE, (thread_continue_t)thread_bootstrap_return);
1489 }
1490
1491 kern_return_t
1492 thread_create_with_continuation(
1493 task_t task,
1494 thread_t *new_thread,
1495 thread_continue_t continuation)
1496 {
1497 return thread_create_internal2(task, new_thread, FALSE, continuation);
1498 }
1499
1500 /*
1501 * Create a thread that is already started, but is waiting on an event
1502 */
1503 static kern_return_t
1504 thread_create_waiting_internal(
1505 task_t task,
1506 thread_continue_t continuation,
1507 event_t event,
1508 block_hint_t block_hint,
1509 int options,
1510 thread_t *new_thread)
1511 {
1512 kern_return_t result;
1513 thread_t thread;
1514
1515 if (task == TASK_NULL || task == kernel_task) {
1516 return KERN_INVALID_ARGUMENT;
1517 }
1518
1519 result = thread_create_internal(task, -1, continuation, NULL,
1520 options, &thread);
1521 if (result != KERN_SUCCESS) {
1522 return result;
1523 }
1524
1525 /* note no user_stop_count or thread_hold here */
1526
1527 if (task->suspend_count > 0) {
1528 thread_hold(thread);
1529 }
1530
1531 thread_mtx_lock(thread);
1532 thread_set_pending_block_hint(thread, block_hint);
1533 if (options & TH_OPTION_WORKQ) {
1534 thread->static_param = true;
1535 event = workq_thread_init_and_wq_lock(task, thread);
1536 }
1537 thread_start_in_assert_wait(thread, event, THREAD_INTERRUPTIBLE);
1538 thread_mtx_unlock(thread);
1539
1540 task_unlock(task);
1541 lck_mtx_unlock(&tasks_threads_lock);
1542
1543 *new_thread = thread;
1544
1545 return KERN_SUCCESS;
1546 }
1547
1548 kern_return_t
1549 thread_create_waiting(
1550 task_t task,
1551 thread_continue_t continuation,
1552 event_t event,
1553 thread_t *new_thread)
1554 {
1555 return thread_create_waiting_internal(task, continuation, event,
1556 kThreadWaitNone, TH_OPTION_NONE, new_thread);
1557 }
1558
1559
1560 static kern_return_t
1561 thread_create_running_internal2(
1562 task_t task,
1563 int flavor,
1564 thread_state_t new_state,
1565 mach_msg_type_number_t new_state_count,
1566 thread_t *new_thread,
1567 boolean_t from_user)
1568 {
1569 kern_return_t result;
1570 thread_t thread;
1571
1572 if (task == TASK_NULL || task == kernel_task) {
1573 return KERN_INVALID_ARGUMENT;
1574 }
1575
1576 result = thread_create_internal(task, -1,
1577 (thread_continue_t)thread_bootstrap_return, NULL,
1578 TH_OPTION_NONE, &thread);
1579 if (result != KERN_SUCCESS) {
1580 return result;
1581 }
1582
1583 if (task->suspend_count > 0) {
1584 thread_hold(thread);
1585 }
1586
1587 if (from_user) {
1588 result = machine_thread_state_convert_from_user(thread, flavor,
1589 new_state, new_state_count);
1590 }
1591 if (result == KERN_SUCCESS) {
1592 result = machine_thread_set_state(thread, flavor, new_state,
1593 new_state_count);
1594 }
1595 if (result != KERN_SUCCESS) {
1596 task_unlock(task);
1597 lck_mtx_unlock(&tasks_threads_lock);
1598
1599 thread_terminate(thread);
1600 thread_deallocate(thread);
1601 return result;
1602 }
1603
1604 thread_mtx_lock(thread);
1605 thread_start(thread);
1606 thread_mtx_unlock(thread);
1607
1608 if (from_user) {
1609 extmod_statistics_incr_thread_create(task);
1610 }
1611
1612 task_unlock(task);
1613 lck_mtx_unlock(&tasks_threads_lock);
1614
1615 *new_thread = thread;
1616
1617 return result;
1618 }
1619
1620 /* Prototype, see justification above */
1621 kern_return_t
1622 thread_create_running(
1623 task_t task,
1624 int flavor,
1625 thread_state_t new_state,
1626 mach_msg_type_number_t new_state_count,
1627 thread_t *new_thread);
1628
1629 kern_return_t
1630 thread_create_running(
1631 task_t task,
1632 int flavor,
1633 thread_state_t new_state,
1634 mach_msg_type_number_t new_state_count,
1635 thread_t *new_thread)
1636 {
1637 return thread_create_running_internal2(
1638 task, flavor, new_state, new_state_count,
1639 new_thread, FALSE);
1640 }
1641
1642 kern_return_t
1643 thread_create_running_from_user(
1644 task_t task,
1645 int flavor,
1646 thread_state_t new_state,
1647 mach_msg_type_number_t new_state_count,
1648 thread_t *new_thread)
1649 {
1650 return thread_create_running_internal2(
1651 task, flavor, new_state, new_state_count,
1652 new_thread, TRUE);
1653 }
1654
1655 kern_return_t
1656 thread_create_workq_waiting(
1657 task_t task,
1658 thread_continue_t continuation,
1659 thread_t *new_thread)
1660 {
1661 int options = TH_OPTION_NOCRED | TH_OPTION_NOSUSP | TH_OPTION_WORKQ;
1662 return thread_create_waiting_internal(task, continuation, NULL,
1663 kThreadWaitParkedWorkQueue, options, new_thread);
1664 }
1665
1666 /*
1667 * kernel_thread_create:
1668 *
1669 * Create a thread in the kernel task
1670 * to execute in kernel context.
1671 */
1672 kern_return_t
1673 kernel_thread_create(
1674 thread_continue_t continuation,
1675 void *parameter,
1676 integer_t priority,
1677 thread_t *new_thread)
1678 {
1679 kern_return_t result;
1680 thread_t thread;
1681 task_t task = kernel_task;
1682
1683 result = thread_create_internal(task, priority, continuation, parameter,
1684 TH_OPTION_NOCRED | TH_OPTION_NONE, &thread);
1685 if (result != KERN_SUCCESS) {
1686 return result;
1687 }
1688
1689 task_unlock(task);
1690 lck_mtx_unlock(&tasks_threads_lock);
1691
1692 stack_alloc(thread);
1693 assert(thread->kernel_stack != 0);
1694 #if CONFIG_EMBEDDED
1695 if (priority > BASEPRI_KERNEL)
1696 #endif
1697 thread->reserved_stack = thread->kernel_stack;
1698
1699 if (debug_task & 1) {
1700 kprintf("kernel_thread_create: thread = %p continuation = %p\n", thread, continuation);
1701 }
1702 *new_thread = thread;
1703
1704 return result;
1705 }
1706
1707 kern_return_t
1708 kernel_thread_start_priority(
1709 thread_continue_t continuation,
1710 void *parameter,
1711 integer_t priority,
1712 thread_t *new_thread)
1713 {
1714 kern_return_t result;
1715 thread_t thread;
1716
1717 result = kernel_thread_create(continuation, parameter, priority, &thread);
1718 if (result != KERN_SUCCESS) {
1719 return result;
1720 }
1721
1722 *new_thread = thread;
1723
1724 thread_mtx_lock(thread);
1725 thread_start(thread);
1726 thread_mtx_unlock(thread);
1727
1728 return result;
1729 }
1730
1731 kern_return_t
1732 kernel_thread_start(
1733 thread_continue_t continuation,
1734 void *parameter,
1735 thread_t *new_thread)
1736 {
1737 return kernel_thread_start_priority(continuation, parameter, -1, new_thread);
1738 }
1739
1740 /* Separated into helper function so it can be used by THREAD_BASIC_INFO and THREAD_EXTENDED_INFO */
1741 /* it is assumed that the thread is locked by the caller */
1742 static void
1743 retrieve_thread_basic_info(thread_t thread, thread_basic_info_t basic_info)
1744 {
1745 int state, flags;
1746
1747 /* fill in info */
1748
1749 thread_read_times(thread, &basic_info->user_time,
1750 &basic_info->system_time, NULL);
1751
1752 /*
1753 * Update lazy-evaluated scheduler info because someone wants it.
1754 */
1755 if (SCHED(can_update_priority)(thread)) {
1756 SCHED(update_priority)(thread);
1757 }
1758
1759 basic_info->sleep_time = 0;
1760
1761 /*
1762 * To calculate cpu_usage, first correct for timer rate,
1763 * then for 5/8 ageing. The correction factor [3/5] is
1764 * (1/(5/8) - 1).
1765 */
1766 basic_info->cpu_usage = 0;
1767 #if defined(CONFIG_SCHED_TIMESHARE_CORE)
1768 if (sched_tick_interval) {
1769 basic_info->cpu_usage = (integer_t)(((uint64_t)thread->cpu_usage
1770 * TH_USAGE_SCALE) / sched_tick_interval);
1771 basic_info->cpu_usage = (basic_info->cpu_usage * 3) / 5;
1772 }
1773 #endif
1774
1775 if (basic_info->cpu_usage > TH_USAGE_SCALE) {
1776 basic_info->cpu_usage = TH_USAGE_SCALE;
1777 }
1778
1779 basic_info->policy = ((thread->sched_mode == TH_MODE_TIMESHARE)?
1780 POLICY_TIMESHARE: POLICY_RR);
1781
1782 flags = 0;
1783 if (thread->options & TH_OPT_IDLE_THREAD) {
1784 flags |= TH_FLAGS_IDLE;
1785 }
1786
1787 if (thread->options & TH_OPT_GLOBAL_FORCED_IDLE) {
1788 flags |= TH_FLAGS_GLOBAL_FORCED_IDLE;
1789 }
1790
1791 if (!thread->kernel_stack) {
1792 flags |= TH_FLAGS_SWAPPED;
1793 }
1794
1795 state = 0;
1796 if (thread->state & TH_TERMINATE) {
1797 state = TH_STATE_HALTED;
1798 } else if (thread->state & TH_RUN) {
1799 state = TH_STATE_RUNNING;
1800 } else if (thread->state & TH_UNINT) {
1801 state = TH_STATE_UNINTERRUPTIBLE;
1802 } else if (thread->state & TH_SUSP) {
1803 state = TH_STATE_STOPPED;
1804 } else if (thread->state & TH_WAIT) {
1805 state = TH_STATE_WAITING;
1806 }
1807
1808 basic_info->run_state = state;
1809 basic_info->flags = flags;
1810
1811 basic_info->suspend_count = thread->user_stop_count;
1812
1813 return;
1814 }
1815
1816 kern_return_t
1817 thread_info_internal(
1818 thread_t thread,
1819 thread_flavor_t flavor,
1820 thread_info_t thread_info_out, /* ptr to OUT array */
1821 mach_msg_type_number_t *thread_info_count) /*IN/OUT*/
1822 {
1823 spl_t s;
1824
1825 if (thread == THREAD_NULL) {
1826 return KERN_INVALID_ARGUMENT;
1827 }
1828
1829 if (flavor == THREAD_BASIC_INFO) {
1830 if (*thread_info_count < THREAD_BASIC_INFO_COUNT) {
1831 return KERN_INVALID_ARGUMENT;
1832 }
1833
1834 s = splsched();
1835 thread_lock(thread);
1836
1837 retrieve_thread_basic_info(thread, (thread_basic_info_t) thread_info_out);
1838
1839 thread_unlock(thread);
1840 splx(s);
1841
1842 *thread_info_count = THREAD_BASIC_INFO_COUNT;
1843
1844 return KERN_SUCCESS;
1845 } else if (flavor == THREAD_IDENTIFIER_INFO) {
1846 thread_identifier_info_t identifier_info;
1847
1848 if (*thread_info_count < THREAD_IDENTIFIER_INFO_COUNT) {
1849 return KERN_INVALID_ARGUMENT;
1850 }
1851
1852 identifier_info = (thread_identifier_info_t) thread_info_out;
1853
1854 s = splsched();
1855 thread_lock(thread);
1856
1857 identifier_info->thread_id = thread->thread_id;
1858 identifier_info->thread_handle = thread->machine.cthread_self;
1859 identifier_info->dispatch_qaddr = thread_dispatchqaddr(thread);
1860
1861 thread_unlock(thread);
1862 splx(s);
1863 return KERN_SUCCESS;
1864 } else if (flavor == THREAD_SCHED_TIMESHARE_INFO) {
1865 policy_timeshare_info_t ts_info;
1866
1867 if (*thread_info_count < POLICY_TIMESHARE_INFO_COUNT) {
1868 return KERN_INVALID_ARGUMENT;
1869 }
1870
1871 ts_info = (policy_timeshare_info_t)thread_info_out;
1872
1873 s = splsched();
1874 thread_lock(thread);
1875
1876 if (thread->sched_mode != TH_MODE_TIMESHARE) {
1877 thread_unlock(thread);
1878 splx(s);
1879 return KERN_INVALID_POLICY;
1880 }
1881
1882 ts_info->depressed = (thread->sched_flags & TH_SFLAG_DEPRESSED_MASK) != 0;
1883 if (ts_info->depressed) {
1884 ts_info->base_priority = DEPRESSPRI;
1885 ts_info->depress_priority = thread->base_pri;
1886 } else {
1887 ts_info->base_priority = thread->base_pri;
1888 ts_info->depress_priority = -1;
1889 }
1890
1891 ts_info->cur_priority = thread->sched_pri;
1892 ts_info->max_priority = thread->max_priority;
1893
1894 thread_unlock(thread);
1895 splx(s);
1896
1897 *thread_info_count = POLICY_TIMESHARE_INFO_COUNT;
1898
1899 return KERN_SUCCESS;
1900 } else if (flavor == THREAD_SCHED_FIFO_INFO) {
1901 if (*thread_info_count < POLICY_FIFO_INFO_COUNT) {
1902 return KERN_INVALID_ARGUMENT;
1903 }
1904
1905 return KERN_INVALID_POLICY;
1906 } else if (flavor == THREAD_SCHED_RR_INFO) {
1907 policy_rr_info_t rr_info;
1908 uint32_t quantum_time;
1909 uint64_t quantum_ns;
1910
1911 if (*thread_info_count < POLICY_RR_INFO_COUNT) {
1912 return KERN_INVALID_ARGUMENT;
1913 }
1914
1915 rr_info = (policy_rr_info_t) thread_info_out;
1916
1917 s = splsched();
1918 thread_lock(thread);
1919
1920 if (thread->sched_mode == TH_MODE_TIMESHARE) {
1921 thread_unlock(thread);
1922 splx(s);
1923
1924 return KERN_INVALID_POLICY;
1925 }
1926
1927 rr_info->depressed = (thread->sched_flags & TH_SFLAG_DEPRESSED_MASK) != 0;
1928 if (rr_info->depressed) {
1929 rr_info->base_priority = DEPRESSPRI;
1930 rr_info->depress_priority = thread->base_pri;
1931 } else {
1932 rr_info->base_priority = thread->base_pri;
1933 rr_info->depress_priority = -1;
1934 }
1935
1936 quantum_time = SCHED(initial_quantum_size)(THREAD_NULL);
1937 absolutetime_to_nanoseconds(quantum_time, &quantum_ns);
1938
1939 rr_info->max_priority = thread->max_priority;
1940 rr_info->quantum = (uint32_t)(quantum_ns / 1000 / 1000);
1941
1942 thread_unlock(thread);
1943 splx(s);
1944
1945 *thread_info_count = POLICY_RR_INFO_COUNT;
1946
1947 return KERN_SUCCESS;
1948 } else if (flavor == THREAD_EXTENDED_INFO) {
1949 thread_basic_info_data_t basic_info;
1950 thread_extended_info_t extended_info = (thread_extended_info_t) thread_info_out;
1951
1952 if (*thread_info_count < THREAD_EXTENDED_INFO_COUNT) {
1953 return KERN_INVALID_ARGUMENT;
1954 }
1955
1956 s = splsched();
1957 thread_lock(thread);
1958
1959 /* NOTE: This mimics fill_taskthreadinfo(), which is the function used by proc_pidinfo() for
1960 * the PROC_PIDTHREADINFO flavor (which can't be used on corpses)
1961 */
1962 retrieve_thread_basic_info(thread, &basic_info);
1963 extended_info->pth_user_time = (((uint64_t)basic_info.user_time.seconds * NSEC_PER_SEC) + ((uint64_t)basic_info.user_time.microseconds * NSEC_PER_USEC));
1964 extended_info->pth_system_time = (((uint64_t)basic_info.system_time.seconds * NSEC_PER_SEC) + ((uint64_t)basic_info.system_time.microseconds * NSEC_PER_USEC));
1965
1966 extended_info->pth_cpu_usage = basic_info.cpu_usage;
1967 extended_info->pth_policy = basic_info.policy;
1968 extended_info->pth_run_state = basic_info.run_state;
1969 extended_info->pth_flags = basic_info.flags;
1970 extended_info->pth_sleep_time = basic_info.sleep_time;
1971 extended_info->pth_curpri = thread->sched_pri;
1972 extended_info->pth_priority = thread->base_pri;
1973 extended_info->pth_maxpriority = thread->max_priority;
1974
1975 bsd_getthreadname(thread->uthread, extended_info->pth_name);
1976
1977 thread_unlock(thread);
1978 splx(s);
1979
1980 *thread_info_count = THREAD_EXTENDED_INFO_COUNT;
1981
1982 return KERN_SUCCESS;
1983 } else if (flavor == THREAD_DEBUG_INFO_INTERNAL) {
1984 #if DEVELOPMENT || DEBUG
1985 thread_debug_info_internal_t dbg_info;
1986 if (*thread_info_count < THREAD_DEBUG_INFO_INTERNAL_COUNT) {
1987 return KERN_NOT_SUPPORTED;
1988 }
1989
1990 if (thread_info_out == NULL) {
1991 return KERN_INVALID_ARGUMENT;
1992 }
1993
1994 dbg_info = (thread_debug_info_internal_t) thread_info_out;
1995 dbg_info->page_creation_count = thread->t_page_creation_count;
1996
1997 *thread_info_count = THREAD_DEBUG_INFO_INTERNAL_COUNT;
1998 return KERN_SUCCESS;
1999 #endif /* DEVELOPMENT || DEBUG */
2000 return KERN_NOT_SUPPORTED;
2001 }
2002
2003 return KERN_INVALID_ARGUMENT;
2004 }
2005
2006 void
2007 thread_read_times(
2008 thread_t thread,
2009 time_value_t *user_time,
2010 time_value_t *system_time,
2011 time_value_t *runnable_time)
2012 {
2013 clock_sec_t secs;
2014 clock_usec_t usecs;
2015 uint64_t tval_user, tval_system;
2016
2017 tval_user = timer_grab(&thread->user_timer);
2018 tval_system = timer_grab(&thread->system_timer);
2019
2020 if (thread->precise_user_kernel_time) {
2021 absolutetime_to_microtime(tval_user, &secs, &usecs);
2022 user_time->seconds = (typeof(user_time->seconds))secs;
2023 user_time->microseconds = usecs;
2024
2025 absolutetime_to_microtime(tval_system, &secs, &usecs);
2026 system_time->seconds = (typeof(system_time->seconds))secs;
2027 system_time->microseconds = usecs;
2028 } else {
2029 /* system_timer may represent either sys or user */
2030 tval_user += tval_system;
2031 absolutetime_to_microtime(tval_user, &secs, &usecs);
2032 user_time->seconds = (typeof(user_time->seconds))secs;
2033 user_time->microseconds = usecs;
2034
2035 system_time->seconds = 0;
2036 system_time->microseconds = 0;
2037 }
2038
2039 if (runnable_time) {
2040 uint64_t tval_runnable = timer_grab(&thread->runnable_timer);
2041 absolutetime_to_microtime(tval_runnable, &secs, &usecs);
2042 runnable_time->seconds = (typeof(runnable_time->seconds))secs;
2043 runnable_time->microseconds = usecs;
2044 }
2045 }
2046
2047 uint64_t
2048 thread_get_runtime_self(void)
2049 {
2050 boolean_t interrupt_state;
2051 uint64_t runtime;
2052 thread_t thread = NULL;
2053 processor_t processor = NULL;
2054
2055 thread = current_thread();
2056
2057 /* Not interrupt safe, as the scheduler may otherwise update timer values underneath us */
2058 interrupt_state = ml_set_interrupts_enabled(FALSE);
2059 processor = current_processor();
2060 timer_update(PROCESSOR_DATA(processor, thread_timer), mach_absolute_time());
2061 runtime = (timer_grab(&thread->user_timer) + timer_grab(&thread->system_timer));
2062 ml_set_interrupts_enabled(interrupt_state);
2063
2064 return runtime;
2065 }
2066
2067 kern_return_t
2068 thread_assign(
2069 __unused thread_t thread,
2070 __unused processor_set_t new_pset)
2071 {
2072 return KERN_FAILURE;
2073 }
2074
2075 /*
2076 * thread_assign_default:
2077 *
2078 * Special version of thread_assign for assigning threads to default
2079 * processor set.
2080 */
2081 kern_return_t
2082 thread_assign_default(
2083 thread_t thread)
2084 {
2085 return thread_assign(thread, &pset0);
2086 }
2087
2088 /*
2089 * thread_get_assignment
2090 *
2091 * Return current assignment for this thread.
2092 */
2093 kern_return_t
2094 thread_get_assignment(
2095 thread_t thread,
2096 processor_set_t *pset)
2097 {
2098 if (thread == NULL) {
2099 return KERN_INVALID_ARGUMENT;
2100 }
2101
2102 *pset = &pset0;
2103
2104 return KERN_SUCCESS;
2105 }
2106
2107 /*
2108 * thread_wire_internal:
2109 *
2110 * Specify that the target thread must always be able
2111 * to run and to allocate memory.
2112 */
2113 kern_return_t
2114 thread_wire_internal(
2115 host_priv_t host_priv,
2116 thread_t thread,
2117 boolean_t wired,
2118 boolean_t *prev_state)
2119 {
2120 if (host_priv == NULL || thread != current_thread()) {
2121 return KERN_INVALID_ARGUMENT;
2122 }
2123
2124 assert(host_priv == &realhost);
2125
2126 if (prev_state) {
2127 *prev_state = (thread->options & TH_OPT_VMPRIV) != 0;
2128 }
2129
2130 if (wired) {
2131 if (!(thread->options & TH_OPT_VMPRIV)) {
2132 vm_page_free_reserve(1); /* XXX */
2133 }
2134 thread->options |= TH_OPT_VMPRIV;
2135 } else {
2136 if (thread->options & TH_OPT_VMPRIV) {
2137 vm_page_free_reserve(-1); /* XXX */
2138 }
2139 thread->options &= ~TH_OPT_VMPRIV;
2140 }
2141
2142 return KERN_SUCCESS;
2143 }
2144
2145
2146 /*
2147 * thread_wire:
2148 *
2149 * User-api wrapper for thread_wire_internal()
2150 */
2151 kern_return_t
2152 thread_wire(
2153 host_priv_t host_priv,
2154 thread_t thread,
2155 boolean_t wired)
2156 {
2157 return thread_wire_internal(host_priv, thread, wired, NULL);
2158 }
2159
2160
2161 boolean_t
2162 is_vm_privileged(void)
2163 {
2164 return current_thread()->options & TH_OPT_VMPRIV ? TRUE : FALSE;
2165 }
2166
2167 boolean_t
2168 set_vm_privilege(boolean_t privileged)
2169 {
2170 boolean_t was_vmpriv;
2171
2172 if (current_thread()->options & TH_OPT_VMPRIV) {
2173 was_vmpriv = TRUE;
2174 } else {
2175 was_vmpriv = FALSE;
2176 }
2177
2178 if (privileged != FALSE) {
2179 current_thread()->options |= TH_OPT_VMPRIV;
2180 } else {
2181 current_thread()->options &= ~TH_OPT_VMPRIV;
2182 }
2183
2184 return was_vmpriv;
2185 }
2186
2187 void
2188 set_thread_rwlock_boost(void)
2189 {
2190 current_thread()->rwlock_count++;
2191 }
2192
2193 void
2194 clear_thread_rwlock_boost(void)
2195 {
2196 thread_t thread = current_thread();
2197
2198 if ((thread->rwlock_count-- == 1) && (thread->sched_flags & TH_SFLAG_RW_PROMOTED)) {
2199 lck_rw_clear_promotion(thread, 0);
2200 }
2201 }
2202
2203 /*
2204 * XXX assuming current thread only, for now...
2205 */
2206 void
2207 thread_guard_violation(thread_t thread,
2208 mach_exception_data_type_t code, mach_exception_data_type_t subcode, boolean_t fatal)
2209 {
2210 assert(thread == current_thread());
2211
2212 /* Don't set up the AST for kernel threads; this check is needed to ensure
2213 * that the guard_exc_* fields in the thread structure are set only by the
2214 * current thread and therefore, don't require a lock.
2215 */
2216 if (thread->task == kernel_task) {
2217 return;
2218 }
2219
2220 assert(EXC_GUARD_DECODE_GUARD_TYPE(code));
2221
2222 /*
2223 * Use the saved state area of the thread structure
2224 * to store all info required to handle the AST when
2225 * returning to userspace. It's possible that there is
2226 * already a pending guard exception. If it's non-fatal,
2227 * it can only be over-written by a fatal exception code.
2228 */
2229 if (thread->guard_exc_info.code && (thread->guard_exc_fatal || !fatal)) {
2230 return;
2231 }
2232
2233 thread->guard_exc_info.code = code;
2234 thread->guard_exc_info.subcode = subcode;
2235 thread->guard_exc_fatal = fatal ? 1 : 0;
2236
2237 spl_t s = splsched();
2238 thread_ast_set(thread, AST_GUARD);
2239 ast_propagate(thread);
2240 splx(s);
2241 }
2242
2243 /*
2244 * guard_ast:
2245 *
2246 * Handle AST_GUARD for a thread. This routine looks at the
2247 * state saved in the thread structure to determine the cause
2248 * of this exception. Based on this value, it invokes the
2249 * appropriate routine which determines other exception related
2250 * info and raises the exception.
2251 */
2252 void
2253 guard_ast(thread_t t)
2254 {
2255 const mach_exception_data_type_t
2256 code = t->guard_exc_info.code,
2257 subcode = t->guard_exc_info.subcode;
2258
2259 t->guard_exc_info.code = 0;
2260 t->guard_exc_info.subcode = 0;
2261 t->guard_exc_fatal = 0;
2262
2263 switch (EXC_GUARD_DECODE_GUARD_TYPE(code)) {
2264 case GUARD_TYPE_NONE:
2265 /* lingering AST_GUARD on the processor? */
2266 break;
2267 case GUARD_TYPE_MACH_PORT:
2268 mach_port_guard_ast(t, code, subcode);
2269 break;
2270 case GUARD_TYPE_FD:
2271 fd_guard_ast(t, code, subcode);
2272 break;
2273 #if CONFIG_VNGUARD
2274 case GUARD_TYPE_VN:
2275 vn_guard_ast(t, code, subcode);
2276 break;
2277 #endif
2278 case GUARD_TYPE_VIRT_MEMORY:
2279 virt_memory_guard_ast(t, code, subcode);
2280 break;
2281 default:
2282 panic("guard_exc_info %llx %llx", code, subcode);
2283 }
2284 }
2285
2286 static void
2287 thread_cputime_callback(int warning, __unused const void *arg0, __unused const void *arg1)
2288 {
2289 if (warning == LEDGER_WARNING_ROSE_ABOVE) {
2290 #if CONFIG_TELEMETRY
2291 /*
2292 * This thread is in danger of violating the CPU usage monitor. Enable telemetry
2293 * on the entire task so there are micro-stackshots available if and when
2294 * EXC_RESOURCE is triggered. We could have chosen to enable micro-stackshots
2295 * for this thread only; but now that this task is suspect, knowing what all of
2296 * its threads are up to will be useful.
2297 */
2298 telemetry_task_ctl(current_task(), TF_CPUMON_WARNING, 1);
2299 #endif
2300 return;
2301 }
2302
2303 #if CONFIG_TELEMETRY
2304 /*
2305 * If the balance has dipped below the warning level (LEDGER_WARNING_DIPPED_BELOW) or
2306 * exceeded the limit, turn telemetry off for the task.
2307 */
2308 telemetry_task_ctl(current_task(), TF_CPUMON_WARNING, 0);
2309 #endif
2310
2311 if (warning == 0) {
2312 SENDING_NOTIFICATION__THIS_THREAD_IS_CONSUMING_TOO_MUCH_CPU();
2313 }
2314 }
2315
2316 void __attribute__((noinline))
2317 SENDING_NOTIFICATION__THIS_THREAD_IS_CONSUMING_TOO_MUCH_CPU(void)
2318 {
2319 int pid = 0;
2320 task_t task = current_task();
2321 thread_t thread = current_thread();
2322 uint64_t tid = thread->thread_id;
2323 const char *procname = "unknown";
2324 time_value_t thread_total_time = {0, 0};
2325 time_value_t thread_system_time;
2326 time_value_t thread_user_time;
2327 int action;
2328 uint8_t percentage;
2329 uint32_t usage_percent = 0;
2330 uint32_t interval_sec;
2331 uint64_t interval_ns;
2332 uint64_t balance_ns;
2333 boolean_t fatal = FALSE;
2334 boolean_t send_exc_resource = TRUE; /* in addition to RESOURCE_NOTIFY */
2335 kern_return_t kr;
2336
2337 #ifdef EXC_RESOURCE_MONITORS
2338 mach_exception_data_type_t code[EXCEPTION_CODE_MAX];
2339 #endif /* EXC_RESOURCE_MONITORS */
2340 struct ledger_entry_info lei;
2341
2342 assert(thread->t_threadledger != LEDGER_NULL);
2343
2344 /*
2345 * Extract the fatal bit and suspend the monitor (which clears the bit).
2346 */
2347 task_lock(task);
2348 if (task->rusage_cpu_flags & TASK_RUSECPU_FLAGS_FATAL_CPUMON) {
2349 fatal = TRUE;
2350 send_exc_resource = TRUE;
2351 }
2352 /* Only one thread can be here at a time. Whichever makes it through
2353 * first will successfully suspend the monitor and proceed to send the
2354 * notification. Other threads will get an error trying to suspend the
2355 * monitor and give up on sending the notification. In the first release,
2356 * the monitor won't be resumed for a number of seconds, but we may
2357 * eventually need to handle low-latency resume.
2358 */
2359 kr = task_suspend_cpumon(task);
2360 task_unlock(task);
2361 if (kr == KERN_INVALID_ARGUMENT) {
2362 return;
2363 }
2364
2365 #ifdef MACH_BSD
2366 pid = proc_selfpid();
2367 if (task->bsd_info != NULL) {
2368 procname = proc_name_address(task->bsd_info);
2369 }
2370 #endif
2371
2372 thread_get_cpulimit(&action, &percentage, &interval_ns);
2373
2374 interval_sec = (uint32_t)(interval_ns / NSEC_PER_SEC);
2375
2376 thread_read_times(thread, &thread_user_time, &thread_system_time, NULL);
2377 time_value_add(&thread_total_time, &thread_user_time);
2378 time_value_add(&thread_total_time, &thread_system_time);
2379 ledger_get_entry_info(thread->t_threadledger, thread_ledgers.cpu_time, &lei);
2380
2381 /* credit/debit/balance/limit are in absolute time units;
2382 * the refill info is in nanoseconds. */
2383 absolutetime_to_nanoseconds(lei.lei_balance, &balance_ns);
2384 if (lei.lei_last_refill > 0) {
2385 usage_percent = (uint32_t)((balance_ns * 100ULL) / lei.lei_last_refill);
2386 }
2387
2388 /* TODO: show task total runtime (via TASK_ABSOLUTETIME_INFO)? */
2389 printf("process %s[%d] thread %llu caught burning CPU! It used more than %d%% CPU over %u seconds\n",
2390 procname, pid, tid, percentage, interval_sec);
2391 printf(" (actual recent usage: %d%% over ~%llu seconds)\n",
2392 usage_percent, (lei.lei_last_refill + NSEC_PER_SEC / 2) / NSEC_PER_SEC);
2393 printf(" Thread lifetime cpu usage %d.%06ds, (%d.%06d user, %d.%06d sys)\n",
2394 thread_total_time.seconds, thread_total_time.microseconds,
2395 thread_user_time.seconds, thread_user_time.microseconds,
2396 thread_system_time.seconds, thread_system_time.microseconds);
2397 printf(" Ledger balance: %lld; mabs credit: %lld; mabs debit: %lld\n",
2398 lei.lei_balance, lei.lei_credit, lei.lei_debit);
2399 printf(" mabs limit: %llu; mabs period: %llu ns; last refill: %llu ns%s.\n",
2400 lei.lei_limit, lei.lei_refill_period, lei.lei_last_refill,
2401 (fatal ? " [fatal violation]" : ""));
2402
2403 /*
2404 * For now, send RESOURCE_NOTIFY in parallel with EXC_RESOURCE. Once
2405 * we have logging parity, we will stop sending EXC_RESOURCE (24508922).
2406 */
2407
2408 /* RESOURCE_NOTIFY MIG specifies nanoseconds of CPU time */
2409 lei.lei_balance = balance_ns;
2410 absolutetime_to_nanoseconds(lei.lei_limit, &lei.lei_limit);
2411 trace_resource_violation(RMON_CPUUSAGE_VIOLATED, &lei);
2412 kr = send_resource_violation(send_cpu_usage_violation, task, &lei,
2413 fatal ? kRNFatalLimitFlag : 0);
2414 if (kr) {
2415 printf("send_resource_violation(CPU usage, ...): error %#x\n", kr);
2416 }
2417
2418 #ifdef EXC_RESOURCE_MONITORS
2419 if (send_exc_resource) {
2420 if (disable_exc_resource) {
2421 printf("process %s[%d] thread %llu caught burning CPU! "
2422 "EXC_RESOURCE%s supressed by a boot-arg\n",
2423 procname, pid, tid, fatal ? " (and termination)" : "");
2424 return;
2425 }
2426
2427 if (audio_active) {
2428 printf("process %s[%d] thread %llu caught burning CPU! "
2429 "EXC_RESOURCE & termination supressed due to audio playback\n",
2430 procname, pid, tid);
2431 return;
2432 }
2433 }
2434
2435
2436 if (send_exc_resource) {
2437 code[0] = code[1] = 0;
2438 EXC_RESOURCE_ENCODE_TYPE(code[0], RESOURCE_TYPE_CPU);
2439 if (fatal) {
2440 EXC_RESOURCE_ENCODE_FLAVOR(code[0], FLAVOR_CPU_MONITOR_FATAL);
2441 } else {
2442 EXC_RESOURCE_ENCODE_FLAVOR(code[0], FLAVOR_CPU_MONITOR);
2443 }
2444 EXC_RESOURCE_CPUMONITOR_ENCODE_INTERVAL(code[0], interval_sec);
2445 EXC_RESOURCE_CPUMONITOR_ENCODE_PERCENTAGE(code[0], percentage);
2446 EXC_RESOURCE_CPUMONITOR_ENCODE_PERCENTAGE(code[1], usage_percent);
2447 exception_triage(EXC_RESOURCE, code, EXCEPTION_CODE_MAX);
2448 }
2449 #endif /* EXC_RESOURCE_MONITORS */
2450
2451 if (fatal) {
2452 #if CONFIG_JETSAM
2453 jetsam_on_ledger_cpulimit_exceeded();
2454 #else
2455 task_terminate_internal(task);
2456 #endif
2457 }
2458 }
2459
2460 #if DEVELOPMENT || DEBUG
2461 void __attribute__((noinline))
2462 SENDING_NOTIFICATION__TASK_HAS_TOO_MANY_THREADS(task_t task, int thread_count)
2463 {
2464 mach_exception_data_type_t code[EXCEPTION_CODE_MAX] = {0};
2465 int pid = task_pid(task);
2466 char procname[MAXCOMLEN + 1] = "unknown";
2467
2468 if (pid == 1) {
2469 /*
2470 * Cannot suspend launchd
2471 */
2472 return;
2473 }
2474
2475 proc_name(pid, procname, sizeof(procname));
2476
2477 if (disable_exc_resource) {
2478 printf("process %s[%d] crossed thread count high watermark (%d), EXC_RESOURCE "
2479 "supressed by a boot-arg. \n", procname, pid, thread_count);
2480 return;
2481 }
2482
2483 if (audio_active) {
2484 printf("process %s[%d] crossed thread count high watermark (%d), EXC_RESOURCE "
2485 "supressed due to audio playback.\n", procname, pid, thread_count);
2486 return;
2487 }
2488
2489 if (exc_via_corpse_forking == 0) {
2490 printf("process %s[%d] crossed thread count high watermark (%d), EXC_RESOURCE "
2491 "supressed due to corpse forking being disabled.\n", procname, pid,
2492 thread_count);
2493 return;
2494 }
2495
2496 printf("process %s[%d] crossed thread count high watermark (%d), sending "
2497 "EXC_RESOURCE\n", procname, pid, thread_count);
2498
2499 EXC_RESOURCE_ENCODE_TYPE(code[0], RESOURCE_TYPE_THREADS);
2500 EXC_RESOURCE_ENCODE_FLAVOR(code[0], FLAVOR_THREADS_HIGH_WATERMARK);
2501 EXC_RESOURCE_THREADS_ENCODE_THREADS(code[0], thread_count);
2502
2503 task_enqueue_exception_with_corpse(task, EXC_RESOURCE, code, EXCEPTION_CODE_MAX, NULL);
2504 }
2505 #endif /* DEVELOPMENT || DEBUG */
2506
2507 void
2508 thread_update_io_stats(thread_t thread, int size, int io_flags)
2509 {
2510 int io_tier;
2511
2512 if (thread->thread_io_stats == NULL || thread->task->task_io_stats == NULL) {
2513 return;
2514 }
2515
2516 if (io_flags & DKIO_READ) {
2517 UPDATE_IO_STATS(thread->thread_io_stats->disk_reads, size);
2518 UPDATE_IO_STATS_ATOMIC(thread->task->task_io_stats->disk_reads, size);
2519 }
2520
2521 if (io_flags & DKIO_META) {
2522 UPDATE_IO_STATS(thread->thread_io_stats->metadata, size);
2523 UPDATE_IO_STATS_ATOMIC(thread->task->task_io_stats->metadata, size);
2524 }
2525
2526 if (io_flags & DKIO_PAGING) {
2527 UPDATE_IO_STATS(thread->thread_io_stats->paging, size);
2528 UPDATE_IO_STATS_ATOMIC(thread->task->task_io_stats->paging, size);
2529 }
2530
2531 io_tier = ((io_flags & DKIO_TIER_MASK) >> DKIO_TIER_SHIFT);
2532 assert(io_tier < IO_NUM_PRIORITIES);
2533
2534 UPDATE_IO_STATS(thread->thread_io_stats->io_priority[io_tier], size);
2535 UPDATE_IO_STATS_ATOMIC(thread->task->task_io_stats->io_priority[io_tier], size);
2536
2537 /* Update Total I/O Counts */
2538 UPDATE_IO_STATS(thread->thread_io_stats->total_io, size);
2539 UPDATE_IO_STATS_ATOMIC(thread->task->task_io_stats->total_io, size);
2540
2541 if (!(io_flags & DKIO_READ)) {
2542 DTRACE_IO3(physical_writes, struct task *, thread->task, uint32_t, size, int, io_flags);
2543 ledger_credit(thread->task->ledger, task_ledgers.physical_writes, size);
2544 }
2545 }
2546
2547 static void
2548 init_thread_ledgers(void)
2549 {
2550 ledger_template_t t;
2551 int idx;
2552
2553 assert(thread_ledger_template == NULL);
2554
2555 if ((t = ledger_template_create("Per-thread ledger")) == NULL) {
2556 panic("couldn't create thread ledger template");
2557 }
2558
2559 if ((idx = ledger_entry_add(t, "cpu_time", "sched", "ns")) < 0) {
2560 panic("couldn't create cpu_time entry for thread ledger template");
2561 }
2562
2563 if (ledger_set_callback(t, idx, thread_cputime_callback, NULL, NULL) < 0) {
2564 panic("couldn't set thread ledger callback for cpu_time entry");
2565 }
2566
2567 thread_ledgers.cpu_time = idx;
2568
2569 ledger_template_complete(t);
2570 thread_ledger_template = t;
2571 }
2572
2573 /*
2574 * Returns currently applied CPU usage limit, or 0/0 if none is applied.
2575 */
2576 int
2577 thread_get_cpulimit(int *action, uint8_t *percentage, uint64_t *interval_ns)
2578 {
2579 int64_t abstime = 0;
2580 uint64_t limittime = 0;
2581 thread_t thread = current_thread();
2582
2583 *percentage = 0;
2584 *interval_ns = 0;
2585 *action = 0;
2586
2587 if (thread->t_threadledger == LEDGER_NULL) {
2588 /*
2589 * This thread has no per-thread ledger, so it can't possibly
2590 * have a CPU limit applied.
2591 */
2592 return KERN_SUCCESS;
2593 }
2594
2595 ledger_get_period(thread->t_threadledger, thread_ledgers.cpu_time, interval_ns);
2596 ledger_get_limit(thread->t_threadledger, thread_ledgers.cpu_time, &abstime);
2597
2598 if ((abstime == LEDGER_LIMIT_INFINITY) || (*interval_ns == 0)) {
2599 /*
2600 * This thread's CPU time ledger has no period or limit; so it
2601 * doesn't have a CPU limit applied.
2602 */
2603 return KERN_SUCCESS;
2604 }
2605
2606 /*
2607 * This calculation is the converse to the one in thread_set_cpulimit().
2608 */
2609 absolutetime_to_nanoseconds(abstime, &limittime);
2610 *percentage = (limittime * 100ULL) / *interval_ns;
2611 assert(*percentage <= 100);
2612
2613 if (thread->options & TH_OPT_PROC_CPULIMIT) {
2614 assert((thread->options & TH_OPT_PRVT_CPULIMIT) == 0);
2615
2616 *action = THREAD_CPULIMIT_BLOCK;
2617 } else if (thread->options & TH_OPT_PRVT_CPULIMIT) {
2618 assert((thread->options & TH_OPT_PROC_CPULIMIT) == 0);
2619
2620 *action = THREAD_CPULIMIT_EXCEPTION;
2621 } else {
2622 *action = THREAD_CPULIMIT_DISABLE;
2623 }
2624
2625 return KERN_SUCCESS;
2626 }
2627
2628 /*
2629 * Set CPU usage limit on a thread.
2630 *
2631 * Calling with percentage of 0 will unset the limit for this thread.
2632 */
2633 int
2634 thread_set_cpulimit(int action, uint8_t percentage, uint64_t interval_ns)
2635 {
2636 thread_t thread = current_thread();
2637 ledger_t l;
2638 uint64_t limittime = 0;
2639 uint64_t abstime = 0;
2640
2641 assert(percentage <= 100);
2642
2643 if (action == THREAD_CPULIMIT_DISABLE) {
2644 /*
2645 * Remove CPU limit, if any exists.
2646 */
2647 if (thread->t_threadledger != LEDGER_NULL) {
2648 l = thread->t_threadledger;
2649 ledger_set_limit(l, thread_ledgers.cpu_time, LEDGER_LIMIT_INFINITY, 0);
2650 ledger_set_action(l, thread_ledgers.cpu_time, LEDGER_ACTION_IGNORE);
2651 thread->options &= ~(TH_OPT_PROC_CPULIMIT | TH_OPT_PRVT_CPULIMIT);
2652 }
2653
2654 return 0;
2655 }
2656
2657 if (interval_ns < MINIMUM_CPULIMIT_INTERVAL_MS * NSEC_PER_MSEC) {
2658 return KERN_INVALID_ARGUMENT;
2659 }
2660
2661 l = thread->t_threadledger;
2662 if (l == LEDGER_NULL) {
2663 /*
2664 * This thread doesn't yet have a per-thread ledger; so create one with the CPU time entry active.
2665 */
2666 if ((l = ledger_instantiate(thread_ledger_template, LEDGER_CREATE_INACTIVE_ENTRIES)) == LEDGER_NULL) {
2667 return KERN_RESOURCE_SHORTAGE;
2668 }
2669
2670 /*
2671 * We are the first to create this thread's ledger, so only activate our entry.
2672 */
2673 ledger_entry_setactive(l, thread_ledgers.cpu_time);
2674 thread->t_threadledger = l;
2675 }
2676
2677 /*
2678 * The limit is specified as a percentage of CPU over an interval in nanoseconds.
2679 * Calculate the amount of CPU time that the thread needs to consume in order to hit the limit.
2680 */
2681 limittime = (interval_ns * percentage) / 100;
2682 nanoseconds_to_absolutetime(limittime, &abstime);
2683 ledger_set_limit(l, thread_ledgers.cpu_time, abstime, cpumon_ustackshots_trigger_pct);
2684 /*
2685 * Refill the thread's allotted CPU time every interval_ns nanoseconds.
2686 */
2687 ledger_set_period(l, thread_ledgers.cpu_time, interval_ns);
2688
2689 if (action == THREAD_CPULIMIT_EXCEPTION) {
2690 /*
2691 * We don't support programming the CPU usage monitor on a task if any of its
2692 * threads have a per-thread blocking CPU limit configured.
2693 */
2694 if (thread->options & TH_OPT_PRVT_CPULIMIT) {
2695 panic("CPU usage monitor activated, but blocking thread limit exists");
2696 }
2697
2698 /*
2699 * Make a note that this thread's CPU limit is being used for the task-wide CPU
2700 * usage monitor. We don't have to arm the callback which will trigger the
2701 * exception, because that was done for us in ledger_instantiate (because the
2702 * ledger template used has a default callback).
2703 */
2704 thread->options |= TH_OPT_PROC_CPULIMIT;
2705 } else {
2706 /*
2707 * We deliberately override any CPU limit imposed by a task-wide limit (eg
2708 * CPU usage monitor).
2709 */
2710 thread->options &= ~TH_OPT_PROC_CPULIMIT;
2711
2712 thread->options |= TH_OPT_PRVT_CPULIMIT;
2713 /* The per-thread ledger template by default has a callback for CPU time */
2714 ledger_disable_callback(l, thread_ledgers.cpu_time);
2715 ledger_set_action(l, thread_ledgers.cpu_time, LEDGER_ACTION_BLOCK);
2716 }
2717
2718 return 0;
2719 }
2720
2721 void
2722 thread_sched_call(
2723 thread_t thread,
2724 sched_call_t call)
2725 {
2726 assert((thread->state & TH_WAIT_REPORT) == 0);
2727 thread->sched_call = call;
2728 }
2729
2730 uint64_t
2731 thread_tid(
2732 thread_t thread)
2733 {
2734 return thread != THREAD_NULL? thread->thread_id: 0;
2735 }
2736
2737 uint16_t
2738 thread_set_tag(thread_t th, uint16_t tag)
2739 {
2740 return thread_set_tag_internal(th, tag);
2741 }
2742
2743 uint16_t
2744 thread_get_tag(thread_t th)
2745 {
2746 return thread_get_tag_internal(th);
2747 }
2748
2749 uint64_t
2750 thread_last_run_time(thread_t th)
2751 {
2752 return th->last_run_time;
2753 }
2754
2755 uint64_t
2756 thread_dispatchqaddr(
2757 thread_t thread)
2758 {
2759 uint64_t dispatchqueue_addr;
2760 uint64_t thread_handle;
2761
2762 if (thread == THREAD_NULL) {
2763 return 0;
2764 }
2765
2766 thread_handle = thread->machine.cthread_self;
2767 if (thread_handle == 0) {
2768 return 0;
2769 }
2770
2771 if (thread->inspection == TRUE) {
2772 dispatchqueue_addr = thread_handle + get_task_dispatchqueue_offset(thread->task);
2773 } else if (thread->task->bsd_info) {
2774 dispatchqueue_addr = thread_handle + get_dispatchqueue_offset_from_proc(thread->task->bsd_info);
2775 } else {
2776 dispatchqueue_addr = 0;
2777 }
2778
2779 return dispatchqueue_addr;
2780 }
2781
2782 uint64_t
2783 thread_rettokern_addr(
2784 thread_t thread)
2785 {
2786 uint64_t rettokern_addr;
2787 uint64_t rettokern_offset;
2788 uint64_t thread_handle;
2789
2790 if (thread == THREAD_NULL) {
2791 return 0;
2792 }
2793
2794 thread_handle = thread->machine.cthread_self;
2795 if (thread_handle == 0) {
2796 return 0;
2797 }
2798
2799 if (thread->task->bsd_info) {
2800 rettokern_offset = get_return_to_kernel_offset_from_proc(thread->task->bsd_info);
2801
2802 /* Return 0 if return to kernel offset is not initialized. */
2803 if (rettokern_offset == 0) {
2804 rettokern_addr = 0;
2805 } else {
2806 rettokern_addr = thread_handle + rettokern_offset;
2807 }
2808 } else {
2809 rettokern_addr = 0;
2810 }
2811
2812 return rettokern_addr;
2813 }
2814
2815 /*
2816 * Export routines to other components for things that are done as macros
2817 * within the osfmk component.
2818 */
2819
2820 #undef thread_mtx_lock
2821 void thread_mtx_lock(thread_t thread);
2822 void
2823 thread_mtx_lock(thread_t thread)
2824 {
2825 lck_mtx_lock(&thread->mutex);
2826 }
2827
2828 #undef thread_mtx_unlock
2829 void thread_mtx_unlock(thread_t thread);
2830 void
2831 thread_mtx_unlock(thread_t thread)
2832 {
2833 lck_mtx_unlock(&thread->mutex);
2834 }
2835
2836 #undef thread_reference
2837 void thread_reference(thread_t thread);
2838 void
2839 thread_reference(
2840 thread_t thread)
2841 {
2842 if (thread != THREAD_NULL) {
2843 thread_reference_internal(thread);
2844 }
2845 }
2846
2847 #undef thread_should_halt
2848
2849 boolean_t
2850 thread_should_halt(
2851 thread_t th)
2852 {
2853 return thread_should_halt_fast(th);
2854 }
2855
2856 /*
2857 * thread_set_voucher_name - reset the voucher port name bound to this thread
2858 *
2859 * Conditions: nothing locked
2860 */
2861
2862 kern_return_t
2863 thread_set_voucher_name(mach_port_name_t voucher_name)
2864 {
2865 thread_t thread = current_thread();
2866 ipc_voucher_t new_voucher = IPC_VOUCHER_NULL;
2867 ipc_voucher_t voucher;
2868 ledger_t bankledger = NULL;
2869 struct thread_group *banktg = NULL;
2870 uint32_t persona_id = 0;
2871
2872 if (MACH_PORT_DEAD == voucher_name) {
2873 return KERN_INVALID_RIGHT;
2874 }
2875
2876 /*
2877 * agressively convert to voucher reference
2878 */
2879 if (MACH_PORT_VALID(voucher_name)) {
2880 new_voucher = convert_port_name_to_voucher(voucher_name);
2881 if (IPC_VOUCHER_NULL == new_voucher) {
2882 return KERN_INVALID_ARGUMENT;
2883 }
2884 }
2885 bank_get_bank_ledger_thread_group_and_persona(new_voucher, &bankledger, &banktg, &persona_id);
2886
2887 thread_mtx_lock(thread);
2888 voucher = thread->ith_voucher;
2889 thread->ith_voucher_name = voucher_name;
2890 thread->ith_voucher = new_voucher;
2891 thread_mtx_unlock(thread);
2892
2893 bank_swap_thread_bank_ledger(thread, bankledger);
2894
2895 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
2896 MACHDBG_CODE(DBG_MACH_IPC, MACH_THREAD_SET_VOUCHER) | DBG_FUNC_NONE,
2897 (uintptr_t)thread_tid(thread),
2898 (uintptr_t)voucher_name,
2899 VM_KERNEL_ADDRPERM((uintptr_t)new_voucher),
2900 persona_id, 0);
2901
2902 if (IPC_VOUCHER_NULL != voucher) {
2903 ipc_voucher_release(voucher);
2904 }
2905
2906 return KERN_SUCCESS;
2907 }
2908
2909 /*
2910 * thread_get_mach_voucher - return a voucher reference for the specified thread voucher
2911 *
2912 * Conditions: nothing locked
2913 *
2914 * NOTE: At the moment, there is no distinction between the current and effective
2915 * vouchers because we only set them at the thread level currently.
2916 */
2917 kern_return_t
2918 thread_get_mach_voucher(
2919 thread_act_t thread,
2920 mach_voucher_selector_t __unused which,
2921 ipc_voucher_t *voucherp)
2922 {
2923 ipc_voucher_t voucher;
2924
2925 if (THREAD_NULL == thread) {
2926 return KERN_INVALID_ARGUMENT;
2927 }
2928
2929 thread_mtx_lock(thread);
2930 voucher = thread->ith_voucher;
2931
2932 if (IPC_VOUCHER_NULL != voucher) {
2933 ipc_voucher_reference(voucher);
2934 thread_mtx_unlock(thread);
2935 *voucherp = voucher;
2936 return KERN_SUCCESS;
2937 }
2938
2939 thread_mtx_unlock(thread);
2940
2941 *voucherp = IPC_VOUCHER_NULL;
2942 return KERN_SUCCESS;
2943 }
2944
2945 /*
2946 * thread_set_mach_voucher - set a voucher reference for the specified thread voucher
2947 *
2948 * Conditions: callers holds a reference on the voucher.
2949 * nothing locked.
2950 *
2951 * We grab another reference to the voucher and bind it to the thread.
2952 * The old voucher reference associated with the thread is
2953 * discarded.
2954 */
2955 kern_return_t
2956 thread_set_mach_voucher(
2957 thread_t thread,
2958 ipc_voucher_t voucher)
2959 {
2960 ipc_voucher_t old_voucher;
2961 ledger_t bankledger = NULL;
2962 struct thread_group *banktg = NULL;
2963 uint32_t persona_id = 0;
2964
2965 if (THREAD_NULL == thread) {
2966 return KERN_INVALID_ARGUMENT;
2967 }
2968
2969 if (thread != current_thread() && thread->started) {
2970 return KERN_INVALID_ARGUMENT;
2971 }
2972
2973 ipc_voucher_reference(voucher);
2974 bank_get_bank_ledger_thread_group_and_persona(voucher, &bankledger, &banktg, &persona_id);
2975
2976 thread_mtx_lock(thread);
2977 old_voucher = thread->ith_voucher;
2978 thread->ith_voucher = voucher;
2979 thread->ith_voucher_name = MACH_PORT_NULL;
2980 thread_mtx_unlock(thread);
2981
2982 bank_swap_thread_bank_ledger(thread, bankledger);
2983
2984 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
2985 MACHDBG_CODE(DBG_MACH_IPC, MACH_THREAD_SET_VOUCHER) | DBG_FUNC_NONE,
2986 (uintptr_t)thread_tid(thread),
2987 (uintptr_t)MACH_PORT_NULL,
2988 VM_KERNEL_ADDRPERM((uintptr_t)voucher),
2989 persona_id, 0);
2990
2991 ipc_voucher_release(old_voucher);
2992
2993 return KERN_SUCCESS;
2994 }
2995
2996 /*
2997 * thread_swap_mach_voucher - swap a voucher reference for the specified thread voucher
2998 *
2999 * Conditions: callers holds a reference on the new and presumed old voucher(s).
3000 * nothing locked.
3001 *
3002 * This function is no longer supported.
3003 */
3004 kern_return_t
3005 thread_swap_mach_voucher(
3006 __unused thread_t thread,
3007 __unused ipc_voucher_t new_voucher,
3008 ipc_voucher_t *in_out_old_voucher)
3009 {
3010 /*
3011 * Currently this function is only called from a MIG generated
3012 * routine which doesn't release the reference on the voucher
3013 * addressed by in_out_old_voucher. To avoid leaking this reference,
3014 * a call to release it has been added here.
3015 */
3016 ipc_voucher_release(*in_out_old_voucher);
3017 return KERN_NOT_SUPPORTED;
3018 }
3019
3020 /*
3021 * thread_get_current_voucher_origin_pid - get the pid of the originator of the current voucher.
3022 */
3023 kern_return_t
3024 thread_get_current_voucher_origin_pid(
3025 int32_t *pid)
3026 {
3027 uint32_t buf_size;
3028 kern_return_t kr;
3029 thread_t thread = current_thread();
3030
3031 buf_size = sizeof(*pid);
3032 kr = mach_voucher_attr_command(thread->ith_voucher,
3033 MACH_VOUCHER_ATTR_KEY_BANK,
3034 BANK_ORIGINATOR_PID,
3035 NULL,
3036 0,
3037 (mach_voucher_attr_content_t)pid,
3038 &buf_size);
3039
3040 return kr;
3041 }
3042
3043
3044 boolean_t
3045 thread_has_thread_name(thread_t th)
3046 {
3047 if ((th) && (th->uthread)) {
3048 return bsd_hasthreadname(th->uthread);
3049 }
3050
3051 /*
3052 * This is an odd case; clients may set the thread name based on the lack of
3053 * a name, but in this context there is no uthread to attach the name to.
3054 */
3055 return FALSE;
3056 }
3057
3058 void
3059 thread_set_thread_name(thread_t th, const char* name)
3060 {
3061 if ((th) && (th->uthread) && name) {
3062 bsd_setthreadname(th->uthread, name);
3063 }
3064 }
3065
3066 void
3067 thread_set_honor_qlimit(thread_t thread)
3068 {
3069 thread->options |= TH_OPT_HONOR_QLIMIT;
3070 }
3071
3072 void
3073 thread_clear_honor_qlimit(thread_t thread)
3074 {
3075 thread->options &= (~TH_OPT_HONOR_QLIMIT);
3076 }
3077
3078 /*
3079 * thread_enable_send_importance - set/clear the SEND_IMPORTANCE thread option bit.
3080 */
3081 void
3082 thread_enable_send_importance(thread_t thread, boolean_t enable)
3083 {
3084 if (enable == TRUE) {
3085 thread->options |= TH_OPT_SEND_IMPORTANCE;
3086 } else {
3087 thread->options &= ~TH_OPT_SEND_IMPORTANCE;
3088 }
3089 }
3090
3091 /*
3092 * thread_set_allocation_name - .
3093 */
3094
3095 kern_allocation_name_t
3096 thread_set_allocation_name(kern_allocation_name_t new_name)
3097 {
3098 kern_allocation_name_t ret;
3099 thread_kernel_state_t kstate = thread_get_kernel_state(current_thread());
3100 ret = kstate->allocation_name;
3101 // fifo
3102 if (!new_name || !kstate->allocation_name) {
3103 kstate->allocation_name = new_name;
3104 }
3105 return ret;
3106 }
3107
3108 void *
3109 thread_iokit_tls_get(uint32_t index)
3110 {
3111 assert(index < THREAD_SAVE_IOKIT_TLS_COUNT);
3112 return current_thread()->saved.iokit.tls[index];
3113 }
3114
3115 void
3116 thread_iokit_tls_set(uint32_t index, void * data)
3117 {
3118 assert(index < THREAD_SAVE_IOKIT_TLS_COUNT);
3119 current_thread()->saved.iokit.tls[index] = data;
3120 }
3121
3122 uint64_t
3123 thread_get_last_wait_duration(thread_t thread)
3124 {
3125 return thread->last_made_runnable_time - thread->last_run_time;
3126 }
3127
3128 integer_t
3129 thread_kern_get_pri(thread_t thr)
3130 {
3131 return thr->base_pri;
3132 }
3133
3134 void
3135 thread_kern_set_pri(thread_t thr, integer_t pri)
3136 {
3137 sched_set_kernel_thread_priority(thr, pri);
3138 }
3139
3140 integer_t
3141 thread_kern_get_kernel_maxpri(void)
3142 {
3143 return MAXPRI_KERNEL;
3144 }
3145
3146 #if CONFIG_DTRACE
3147 uint32_t
3148 dtrace_get_thread_predcache(thread_t thread)
3149 {
3150 if (thread != THREAD_NULL) {
3151 return thread->t_dtrace_predcache;
3152 } else {
3153 return 0;
3154 }
3155 }
3156
3157 int64_t
3158 dtrace_get_thread_vtime(thread_t thread)
3159 {
3160 if (thread != THREAD_NULL) {
3161 return thread->t_dtrace_vtime;
3162 } else {
3163 return 0;
3164 }
3165 }
3166
3167 int
3168 dtrace_get_thread_last_cpu_id(thread_t thread)
3169 {
3170 if ((thread != THREAD_NULL) && (thread->last_processor != PROCESSOR_NULL)) {
3171 return thread->last_processor->cpu_id;
3172 } else {
3173 return -1;
3174 }
3175 }
3176
3177 int64_t
3178 dtrace_get_thread_tracing(thread_t thread)
3179 {
3180 if (thread != THREAD_NULL) {
3181 return thread->t_dtrace_tracing;
3182 } else {
3183 return 0;
3184 }
3185 }
3186
3187 uint16_t
3188 dtrace_get_thread_inprobe(thread_t thread)
3189 {
3190 if (thread != THREAD_NULL) {
3191 return thread->t_dtrace_inprobe;
3192 } else {
3193 return 0;
3194 }
3195 }
3196
3197 vm_offset_t
3198 dtrace_get_kernel_stack(thread_t thread)
3199 {
3200 if (thread != THREAD_NULL) {
3201 return thread->kernel_stack;
3202 } else {
3203 return 0;
3204 }
3205 }
3206
3207 #if KASAN
3208 struct kasan_thread_data *
3209 kasan_get_thread_data(thread_t thread)
3210 {
3211 return &thread->kasan_data;
3212 }
3213 #endif
3214
3215 #if CONFIG_KSANCOV
3216 void **
3217 __sanitizer_get_thread_data(thread_t thread)
3218 {
3219 return &thread->ksancov_data;
3220 }
3221 #endif
3222
3223 int64_t
3224 dtrace_calc_thread_recent_vtime(thread_t thread)
3225 {
3226 if (thread != THREAD_NULL) {
3227 processor_t processor = current_processor();
3228 uint64_t abstime = mach_absolute_time();
3229 timer_t timer;
3230
3231 timer = PROCESSOR_DATA(processor, thread_timer);
3232
3233 return timer_grab(&(thread->system_timer)) + timer_grab(&(thread->user_timer)) +
3234 (abstime - timer->tstamp); /* XXX need interrupts off to prevent missed time? */
3235 } else {
3236 return 0;
3237 }
3238 }
3239
3240 void
3241 dtrace_set_thread_predcache(thread_t thread, uint32_t predcache)
3242 {
3243 if (thread != THREAD_NULL) {
3244 thread->t_dtrace_predcache = predcache;
3245 }
3246 }
3247
3248 void
3249 dtrace_set_thread_vtime(thread_t thread, int64_t vtime)
3250 {
3251 if (thread != THREAD_NULL) {
3252 thread->t_dtrace_vtime = vtime;
3253 }
3254 }
3255
3256 void
3257 dtrace_set_thread_tracing(thread_t thread, int64_t accum)
3258 {
3259 if (thread != THREAD_NULL) {
3260 thread->t_dtrace_tracing = accum;
3261 }
3262 }
3263
3264 void
3265 dtrace_set_thread_inprobe(thread_t thread, uint16_t inprobe)
3266 {
3267 if (thread != THREAD_NULL) {
3268 thread->t_dtrace_inprobe = inprobe;
3269 }
3270 }
3271
3272 vm_offset_t
3273 dtrace_set_thread_recover(thread_t thread, vm_offset_t recover)
3274 {
3275 vm_offset_t prev = 0;
3276
3277 if (thread != THREAD_NULL) {
3278 prev = thread->recover;
3279 thread->recover = recover;
3280 }
3281 return prev;
3282 }
3283
3284 vm_offset_t
3285 dtrace_sign_and_set_thread_recover(thread_t thread, vm_offset_t recover)
3286 {
3287 #if defined(HAS_APPLE_PAC)
3288 return dtrace_set_thread_recover(thread,
3289 (vm_address_t)ptrauth_sign_unauthenticated((void *)recover,
3290 ptrauth_key_function_pointer,
3291 ptrauth_blend_discriminator(&thread->recover, PAC_DISCRIMINATOR_RECOVER)));
3292 #else /* defined(HAS_APPLE_PAC) */
3293 return dtrace_set_thread_recover(thread, recover);
3294 #endif /* defined(HAS_APPLE_PAC) */
3295 }
3296
3297 void
3298 dtrace_thread_bootstrap(void)
3299 {
3300 task_t task = current_task();
3301
3302 if (task->thread_count == 1) {
3303 thread_t thread = current_thread();
3304 if (thread->t_dtrace_flags & TH_DTRACE_EXECSUCCESS) {
3305 thread->t_dtrace_flags &= ~TH_DTRACE_EXECSUCCESS;
3306 DTRACE_PROC(exec__success);
3307 KDBG(BSDDBG_CODE(DBG_BSD_PROC, BSD_PROC_EXEC),
3308 task_pid(task));
3309 }
3310 DTRACE_PROC(start);
3311 }
3312 DTRACE_PROC(lwp__start);
3313 }
3314
3315 void
3316 dtrace_thread_didexec(thread_t thread)
3317 {
3318 thread->t_dtrace_flags |= TH_DTRACE_EXECSUCCESS;
3319 }
3320 #endif /* CONFIG_DTRACE */
mirror of XNU