]> git.saurik.com Git - apple/xnu.git/blob - osfmk/kern/thread.c
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-6153.41.3.tar.gz
[apple/xnu.git] / osfmk / kern / thread.c
1 /*
2 * Copyright (c) 2000-2019 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_FREE_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56 /*
57 */
58 /*
59 * File: kern/thread.c
60 * Author: Avadis Tevanian, Jr., Michael Wayne Young, David Golub
61 * Date: 1986
62 *
63 * Thread management primitives implementation.
64 */
65 /*
66 * Copyright (c) 1993 The University of Utah and
67 * the Computer Systems Laboratory (CSL). All rights reserved.
68 *
69 * Permission to use, copy, modify and distribute this software and its
70 * documentation is hereby granted, provided that both the copyright
71 * notice and this permission notice appear in all copies of the
72 * software, derivative works or modified versions, and any portions
73 * thereof, and that both notices appear in supporting documentation.
74 *
75 * THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF THIS SOFTWARE IN ITS "AS
76 * IS" CONDITION. THE UNIVERSITY OF UTAH AND CSL DISCLAIM ANY LIABILITY OF
77 * ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
78 *
79 * CSL requests users of this software to return to csl-dist@cs.utah.edu any
80 * improvements that they make and grant CSL redistribution rights.
81 *
82 */
83
84 #include <mach/mach_types.h>
85 #include <mach/boolean.h>
86 #include <mach/policy.h>
87 #include <mach/thread_info.h>
88 #include <mach/thread_special_ports.h>
89 #include <mach/thread_status.h>
90 #include <mach/time_value.h>
91 #include <mach/vm_param.h>
92
93 #include <machine/thread.h>
94 #include <machine/pal_routines.h>
95 #include <machine/limits.h>
96
97 #include <kern/kern_types.h>
98 #include <kern/kalloc.h>
99 #include <kern/cpu_data.h>
100 #include <kern/counters.h>
101 #include <kern/extmod_statistics.h>
102 #include <kern/ipc_mig.h>
103 #include <kern/ipc_tt.h>
104 #include <kern/mach_param.h>
105 #include <kern/machine.h>
106 #include <kern/misc_protos.h>
107 #include <kern/processor.h>
108 #include <kern/queue.h>
109 #include <kern/sched.h>
110 #include <kern/sched_prim.h>
111 #include <kern/sync_lock.h>
112 #include <kern/syscall_subr.h>
113 #include <kern/task.h>
114 #include <kern/thread.h>
115 #include <kern/thread_group.h>
116 #include <kern/coalition.h>
117 #include <kern/host.h>
118 #include <kern/zalloc.h>
119 #include <kern/assert.h>
120 #include <kern/exc_resource.h>
121 #include <kern/exc_guard.h>
122 #include <kern/telemetry.h>
123 #include <kern/policy_internal.h>
124 #include <kern/turnstile.h>
125 #include <kern/sched_clutch.h>
126
127 #include <corpses/task_corpse.h>
128 #if KPC
129 #include <kern/kpc.h>
130 #endif
131
132 #if MONOTONIC
133 #include <kern/monotonic.h>
134 #include <machine/monotonic.h>
135 #endif /* MONOTONIC */
136
137 #include <ipc/ipc_kmsg.h>
138 #include <ipc/ipc_port.h>
139 #include <bank/bank_types.h>
140
141 #include <vm/vm_kern.h>
142 #include <vm/vm_pageout.h>
143
144 #include <sys/kdebug.h>
145 #include <sys/bsdtask_info.h>
146 #include <mach/sdt.h>
147 #include <san/kasan.h>
148 #if CONFIG_KSANCOV
149 #include <san/ksancov.h>
150 #endif
151
152 #include <stdatomic.h>
153
154 #if defined(HAS_APPLE_PAC)
155 #include <ptrauth.h>
156 #include <arm64/proc_reg.h>
157 #endif /* defined(HAS_APPLE_PAC) */
158
159 /*
160 * Exported interfaces
161 */
162 #include <mach/task_server.h>
163 #include <mach/thread_act_server.h>
164 #include <mach/mach_host_server.h>
165 #include <mach/host_priv_server.h>
166 #include <mach/mach_voucher_server.h>
167 #include <kern/policy_internal.h>
168
169 static struct zone *thread_zone;
170 static lck_grp_attr_t thread_lck_grp_attr;
171 lck_attr_t thread_lck_attr;
172 lck_grp_t thread_lck_grp;
173
174 struct zone *thread_qos_override_zone;
175
176 static struct mpsc_daemon_queue thread_stack_queue;
177 static struct mpsc_daemon_queue thread_terminate_queue;
178 static struct mpsc_daemon_queue thread_deallocate_queue;
179 static struct mpsc_daemon_queue thread_exception_queue;
180
181 decl_simple_lock_data(static, crashed_threads_lock);
182 static queue_head_t crashed_threads_queue;
183
184 struct thread_exception_elt {
185 struct mpsc_queue_chain link;
186 exception_type_t exception_type;
187 task_t exception_task;
188 thread_t exception_thread;
189 };
190
191 static struct thread thread_template, init_thread;
192 static void thread_deallocate_enqueue(thread_t thread);
193 static void thread_deallocate_complete(thread_t thread);
194
195 #ifdef MACH_BSD
196 extern void proc_exit(void *);
197 extern mach_exception_data_type_t proc_encode_exit_exception_code(void *);
198 extern uint64_t get_dispatchqueue_offset_from_proc(void *);
199 extern uint64_t get_return_to_kernel_offset_from_proc(void *p);
200 extern int proc_selfpid(void);
201 extern void proc_name(int, char*, int);
202 extern char * proc_name_address(void *p);
203 #endif /* MACH_BSD */
204
205 extern int disable_exc_resource;
206 extern int audio_active;
207 extern int debug_task;
208 int thread_max = CONFIG_THREAD_MAX; /* Max number of threads */
209 int task_threadmax = CONFIG_THREAD_MAX;
210
211 static uint64_t thread_unique_id = 100;
212
213 struct _thread_ledger_indices thread_ledgers = { .cpu_time = -1 };
214 static ledger_template_t thread_ledger_template = NULL;
215 static void init_thread_ledgers(void);
216
217 #if CONFIG_JETSAM
218 void jetsam_on_ledger_cpulimit_exceeded(void);
219 #endif
220
221 extern int task_thread_soft_limit;
222 extern int exc_via_corpse_forking;
223
224 #if DEVELOPMENT || DEBUG
225 extern int exc_resource_threads_enabled;
226 #endif /* DEVELOPMENT || DEBUG */
227
228 /*
229 * Level (in terms of percentage of the limit) at which the CPU usage monitor triggers telemetry.
230 *
231 * (ie when any thread's CPU consumption exceeds 70% of the limit, start taking user
232 * stacktraces, aka micro-stackshots)
233 */
234 #define CPUMON_USTACKSHOTS_TRIGGER_DEFAULT_PCT 70
235
236 int cpumon_ustackshots_trigger_pct; /* Percentage. Level at which we start gathering telemetry. */
237 void __attribute__((noinline)) SENDING_NOTIFICATION__THIS_THREAD_IS_CONSUMING_TOO_MUCH_CPU(void);
238 #if DEVELOPMENT || DEBUG
239 void __attribute__((noinline)) SENDING_NOTIFICATION__TASK_HAS_TOO_MANY_THREADS(task_t, int);
240 #endif /* DEVELOPMENT || DEBUG */
241
242 /*
243 * The smallest interval over which we support limiting CPU consumption is 1ms
244 */
245 #define MINIMUM_CPULIMIT_INTERVAL_MS 1
246
247 os_refgrp_decl(static, thread_refgrp, "thread", NULL);
248
249 void
250 thread_bootstrap(void)
251 {
252 /*
253 * Fill in a template thread for fast initialization.
254 */
255
256 #if MACH_ASSERT
257 thread_template.thread_magic = THREAD_MAGIC;
258 #endif /* MACH_ASSERT */
259
260 thread_template.runq = PROCESSOR_NULL;
261
262 thread_template.reason = AST_NONE;
263 thread_template.at_safe_point = FALSE;
264 thread_template.wait_event = NO_EVENT64;
265 thread_template.waitq = NULL;
266 thread_template.wait_result = THREAD_WAITING;
267 thread_template.options = THREAD_ABORTSAFE;
268 thread_template.state = TH_WAIT | TH_UNINT;
269 thread_template.wake_active = FALSE;
270 thread_template.continuation = THREAD_CONTINUE_NULL;
271 thread_template.parameter = NULL;
272
273 thread_template.importance = 0;
274 thread_template.sched_mode = TH_MODE_NONE;
275 thread_template.sched_flags = 0;
276 thread_template.saved_mode = TH_MODE_NONE;
277 thread_template.safe_release = 0;
278 thread_template.th_sched_bucket = TH_BUCKET_RUN;
279
280 thread_template.sfi_class = SFI_CLASS_UNSPECIFIED;
281 thread_template.sfi_wait_class = SFI_CLASS_UNSPECIFIED;
282
283 thread_template.active = 0;
284 thread_template.started = 0;
285 thread_template.static_param = 0;
286 thread_template.policy_reset = 0;
287
288 thread_template.base_pri = BASEPRI_DEFAULT;
289 thread_template.sched_pri = 0;
290 thread_template.max_priority = 0;
291 thread_template.task_priority = 0;
292 thread_template.rwlock_count = 0;
293 thread_template.waiting_for_mutex = NULL;
294
295
296 thread_template.realtime.deadline = UINT64_MAX;
297
298 thread_template.quantum_remaining = 0;
299 thread_template.last_run_time = 0;
300 thread_template.last_made_runnable_time = THREAD_NOT_RUNNABLE;
301 thread_template.last_basepri_change_time = THREAD_NOT_RUNNABLE;
302 thread_template.same_pri_latency = 0;
303
304 thread_template.computation_metered = 0;
305 thread_template.computation_epoch = 0;
306
307 #if defined(CONFIG_SCHED_TIMESHARE_CORE)
308 thread_template.sched_stamp = 0;
309 thread_template.pri_shift = INT8_MAX;
310 thread_template.sched_usage = 0;
311 thread_template.cpu_usage = thread_template.cpu_delta = 0;
312 #endif
313 thread_template.c_switch = thread_template.p_switch = thread_template.ps_switch = 0;
314
315 #if MONOTONIC
316 memset(&thread_template.t_monotonic, 0,
317 sizeof(thread_template.t_monotonic));
318 #endif /* MONOTONIC */
319
320 thread_template.bound_processor = PROCESSOR_NULL;
321 thread_template.last_processor = PROCESSOR_NULL;
322
323 thread_template.sched_call = NULL;
324
325 timer_init(&thread_template.user_timer);
326 timer_init(&thread_template.system_timer);
327 timer_init(&thread_template.ptime);
328 timer_init(&thread_template.runnable_timer);
329 thread_template.user_timer_save = 0;
330 thread_template.system_timer_save = 0;
331 thread_template.vtimer_user_save = 0;
332 thread_template.vtimer_prof_save = 0;
333 thread_template.vtimer_rlim_save = 0;
334 thread_template.vtimer_qos_save = 0;
335
336 #if CONFIG_SCHED_SFI
337 thread_template.wait_sfi_begin_time = 0;
338 #endif
339
340 thread_template.wait_timer_is_set = FALSE;
341 thread_template.wait_timer_active = 0;
342
343 thread_template.depress_timer_active = 0;
344
345 thread_template.recover = (vm_offset_t)NULL;
346
347 thread_template.map = VM_MAP_NULL;
348 #if DEVELOPMENT || DEBUG
349 thread_template.pmap_footprint_suspended = FALSE;
350 #endif /* DEVELOPMENT || DEBUG */
351
352 #if CONFIG_DTRACE
353 thread_template.t_dtrace_predcache = 0;
354 thread_template.t_dtrace_vtime = 0;
355 thread_template.t_dtrace_tracing = 0;
356 #endif /* CONFIG_DTRACE */
357
358 #if KPERF
359 thread_template.kperf_ast = 0;
360 thread_template.kperf_pet_gen = 0;
361 thread_template.kperf_c_switch = 0;
362 thread_template.kperf_pet_cnt = 0;
363 #endif
364
365 #if KPC
366 thread_template.kpc_buf = NULL;
367 #endif
368
369 #if HYPERVISOR
370 thread_template.hv_thread_target = NULL;
371 #endif /* HYPERVISOR */
372
373 #if (DEVELOPMENT || DEBUG)
374 thread_template.t_page_creation_throttled_hard = 0;
375 thread_template.t_page_creation_throttled_soft = 0;
376 #endif /* DEVELOPMENT || DEBUG */
377 thread_template.t_page_creation_throttled = 0;
378 thread_template.t_page_creation_count = 0;
379 thread_template.t_page_creation_time = 0;
380
381 thread_template.affinity_set = NULL;
382
383 thread_template.syscalls_unix = 0;
384 thread_template.syscalls_mach = 0;
385
386 thread_template.t_ledger = LEDGER_NULL;
387 thread_template.t_threadledger = LEDGER_NULL;
388 thread_template.t_bankledger = LEDGER_NULL;
389 thread_template.t_deduct_bank_ledger_time = 0;
390
391 thread_template.requested_policy = (struct thread_requested_policy) {};
392 thread_template.effective_policy = (struct thread_effective_policy) {};
393
394 bzero(&thread_template.overrides, sizeof(thread_template.overrides));
395 thread_template.kevent_overrides = 0;
396
397 thread_template.iotier_override = THROTTLE_LEVEL_NONE;
398 thread_template.thread_io_stats = NULL;
399 #if CONFIG_EMBEDDED
400 thread_template.taskwatch = NULL;
401 #endif /* CONFIG_EMBEDDED */
402 thread_template.thread_callout_interrupt_wakeups = thread_template.thread_callout_platform_idle_wakeups = 0;
403
404 thread_template.thread_timer_wakeups_bin_1 = thread_template.thread_timer_wakeups_bin_2 = 0;
405 thread_template.callout_woken_from_icontext = thread_template.callout_woken_from_platform_idle = 0;
406 thread_template.guard_exc_fatal = 0;
407
408 thread_template.thread_tag = 0;
409
410 thread_template.ith_voucher_name = MACH_PORT_NULL;
411 thread_template.ith_voucher = IPC_VOUCHER_NULL;
412
413 thread_template.th_work_interval = NULL;
414
415 thread_template.decompressions = 0;
416 init_thread = thread_template;
417
418 /* fiddle with init thread to skip asserts in set_sched_pri */
419 init_thread.sched_pri = MAXPRI_KERNEL;
420
421 machine_set_current_thread(&init_thread);
422 }
423
424 extern boolean_t allow_qos_policy_set;
425
426 void
427 thread_init(void)
428 {
429 thread_zone = zinit(
430 sizeof(struct thread),
431 thread_max * sizeof(struct thread),
432 THREAD_CHUNK * sizeof(struct thread),
433 "threads");
434
435 thread_qos_override_zone = zinit(
436 sizeof(struct thread_qos_override),
437 4 * thread_max * sizeof(struct thread_qos_override),
438 PAGE_SIZE,
439 "thread qos override");
440 zone_change(thread_qos_override_zone, Z_EXPAND, TRUE);
441 zone_change(thread_qos_override_zone, Z_COLLECT, TRUE);
442 zone_change(thread_qos_override_zone, Z_CALLERACCT, FALSE);
443 zone_change(thread_qos_override_zone, Z_NOENCRYPT, TRUE);
444
445 lck_grp_attr_setdefault(&thread_lck_grp_attr);
446 lck_grp_init(&thread_lck_grp, "thread", &thread_lck_grp_attr);
447 lck_attr_setdefault(&thread_lck_attr);
448
449 stack_init();
450
451 thread_policy_init();
452
453 /*
454 * Initialize any machine-dependent
455 * per-thread structures necessary.
456 */
457 machine_thread_init();
458
459 if (!PE_parse_boot_argn("cpumon_ustackshots_trigger_pct", &cpumon_ustackshots_trigger_pct,
460 sizeof(cpumon_ustackshots_trigger_pct))) {
461 cpumon_ustackshots_trigger_pct = CPUMON_USTACKSHOTS_TRIGGER_DEFAULT_PCT;
462 }
463
464 PE_parse_boot_argn("-qos-policy-allow", &allow_qos_policy_set, sizeof(allow_qos_policy_set));
465
466 init_thread_ledgers();
467 }
468
469 boolean_t
470 thread_is_active(thread_t thread)
471 {
472 return thread->active;
473 }
474
475 void
476 thread_corpse_continue(void)
477 {
478 thread_t thread = current_thread();
479
480 thread_terminate_internal(thread);
481
482 /*
483 * Handle the thread termination directly
484 * here instead of returning to userspace.
485 */
486 assert(thread->active == FALSE);
487 thread_ast_clear(thread, AST_APC);
488 thread_apc_ast(thread);
489
490 panic("thread_corpse_continue");
491 /*NOTREACHED*/
492 }
493
494 __dead2
495 static void
496 thread_terminate_continue(void)
497 {
498 panic("thread_terminate_continue");
499 /*NOTREACHED*/
500 }
501
502 /*
503 * thread_terminate_self:
504 */
505 void
506 thread_terminate_self(void)
507 {
508 thread_t thread = current_thread();
509 task_t task;
510 int threadcnt;
511
512 pal_thread_terminate_self(thread);
513
514 DTRACE_PROC(lwp__exit);
515
516 thread_mtx_lock(thread);
517
518 ipc_thread_disable(thread);
519
520 thread_mtx_unlock(thread);
521
522 thread_sched_call(thread, NULL);
523
524 spl_t s = splsched();
525 thread_lock(thread);
526
527 thread_depress_abort_locked(thread);
528
529 thread_unlock(thread);
530 splx(s);
531
532 #if CONFIG_EMBEDDED
533 thead_remove_taskwatch(thread);
534 #endif /* CONFIG_EMBEDDED */
535
536 work_interval_thread_terminate(thread);
537
538 thread_mtx_lock(thread);
539
540 thread_policy_reset(thread);
541
542 thread_mtx_unlock(thread);
543
544 bank_swap_thread_bank_ledger(thread, NULL);
545
546 if (kdebug_enable && bsd_hasthreadname(thread->uthread)) {
547 char threadname[MAXTHREADNAMESIZE];
548 bsd_getthreadname(thread->uthread, threadname);
549 kernel_debug_string_simple(TRACE_STRING_THREADNAME_PREV, threadname);
550 }
551
552 task = thread->task;
553 uthread_cleanup(task, thread->uthread, task->bsd_info);
554
555 if (kdebug_enable && task->bsd_info && !task_is_exec_copy(task)) {
556 /* trace out pid before we sign off */
557 long dbg_arg1 = 0;
558 long dbg_arg2 = 0;
559
560 kdbg_trace_data(thread->task->bsd_info, &dbg_arg1, &dbg_arg2);
561 KDBG_RELEASE(TRACE_DATA_THREAD_TERMINATE_PID, dbg_arg1, dbg_arg2);
562 }
563
564 /*
565 * After this subtraction, this thread should never access
566 * task->bsd_info unless it got 0 back from the os_atomic_dec. It
567 * could be racing with other threads to be the last thread in the
568 * process, and the last thread in the process will tear down the proc
569 * structure and zero-out task->bsd_info.
570 */
571 threadcnt = os_atomic_dec(&task->active_thread_count, relaxed);
572
573 /*
574 * If we are the last thread to terminate and the task is
575 * associated with a BSD process, perform BSD process exit.
576 */
577 if (threadcnt == 0 && task->bsd_info != NULL && !task_is_exec_copy(task)) {
578 mach_exception_data_type_t subcode = 0;
579 if (kdebug_enable) {
580 /* since we're the last thread in this process, trace out the command name too */
581 long args[4] = {};
582 kdbg_trace_string(thread->task->bsd_info, &args[0], &args[1], &args[2], &args[3]);
583 KDBG_RELEASE(TRACE_STRING_PROC_EXIT, args[0], args[1], args[2], args[3]);
584 }
585
586 /* Get the exit reason before proc_exit */
587 subcode = proc_encode_exit_exception_code(task->bsd_info);
588 proc_exit(task->bsd_info);
589 /*
590 * if there is crash info in task
591 * then do the deliver action since this is
592 * last thread for this task.
593 */
594 if (task->corpse_info) {
595 task_deliver_crash_notification(task, current_thread(), EXC_RESOURCE, subcode);
596 }
597 }
598
599 if (threadcnt == 0) {
600 task_lock(task);
601 if (task_is_a_corpse_fork(task)) {
602 thread_wakeup((event_t)&task->active_thread_count);
603 }
604 task_unlock(task);
605 }
606
607 uthread_cred_free(thread->uthread);
608
609 s = splsched();
610 thread_lock(thread);
611
612 /*
613 * Ensure that the depress timer is no longer enqueued,
614 * so the timer (stored in the thread) can be safely deallocated
615 *
616 * TODO: build timer_call_cancel_wait
617 */
618
619 assert((thread->sched_flags & TH_SFLAG_DEPRESSED_MASK) == 0);
620
621 uint32_t delay_us = 1;
622
623 while (thread->depress_timer_active > 0) {
624 thread_unlock(thread);
625 splx(s);
626
627 delay(delay_us++);
628
629 if (delay_us > USEC_PER_SEC) {
630 panic("depress timer failed to inactivate!"
631 "thread: %p depress_timer_active: %d",
632 thread, thread->depress_timer_active);
633 }
634
635 s = splsched();
636 thread_lock(thread);
637 }
638
639 /*
640 * Cancel wait timer, and wait for
641 * concurrent expirations.
642 */
643 if (thread->wait_timer_is_set) {
644 thread->wait_timer_is_set = FALSE;
645
646 if (timer_call_cancel(&thread->wait_timer)) {
647 thread->wait_timer_active--;
648 }
649 }
650
651 delay_us = 1;
652
653 while (thread->wait_timer_active > 0) {
654 thread_unlock(thread);
655 splx(s);
656
657 delay(delay_us++);
658
659 if (delay_us > USEC_PER_SEC) {
660 panic("wait timer failed to inactivate!"
661 "thread: %p wait_timer_active: %d",
662 thread, thread->wait_timer_active);
663 }
664
665 s = splsched();
666 thread_lock(thread);
667 }
668
669 /*
670 * If there is a reserved stack, release it.
671 */
672 if (thread->reserved_stack != 0) {
673 stack_free_reserved(thread);
674 thread->reserved_stack = 0;
675 }
676
677 /*
678 * Mark thread as terminating, and block.
679 */
680 thread->state |= TH_TERMINATE;
681 thread_mark_wait_locked(thread, THREAD_UNINT);
682
683 assert((thread->sched_flags & TH_SFLAG_WAITQ_PROMOTED) == 0);
684 assert((thread->sched_flags & TH_SFLAG_RW_PROMOTED) == 0);
685 assert((thread->sched_flags & TH_SFLAG_EXEC_PROMOTED) == 0);
686 assert((thread->sched_flags & TH_SFLAG_PROMOTED) == 0);
687 assert(thread->kern_promotion_schedpri == 0);
688 assert(thread->waiting_for_mutex == NULL);
689 assert(thread->rwlock_count == 0);
690
691 thread_unlock(thread);
692 /* splsched */
693
694 thread_block((thread_continue_t)thread_terminate_continue);
695 /*NOTREACHED*/
696 }
697
698 static bool
699 thread_ref_release(thread_t thread)
700 {
701 if (thread == THREAD_NULL) {
702 return false;
703 }
704
705 assert_thread_magic(thread);
706
707 return os_ref_release(&thread->ref_count) == 0;
708 }
709
710 /* Drop a thread refcount safely without triggering a zfree */
711 void
712 thread_deallocate_safe(thread_t thread)
713 {
714 if (__improbable(thread_ref_release(thread))) {
715 /* enqueue the thread for thread deallocate deamon to call thread_deallocate_complete */
716 thread_deallocate_enqueue(thread);
717 }
718 }
719
720 void
721 thread_deallocate(thread_t thread)
722 {
723 if (__improbable(thread_ref_release(thread))) {
724 thread_deallocate_complete(thread);
725 }
726 }
727
728 void
729 thread_deallocate_complete(
730 thread_t thread)
731 {
732 task_t task;
733
734 assert_thread_magic(thread);
735
736 assert(os_ref_get_count(&thread->ref_count) == 0);
737
738 if (!(thread->state & TH_TERMINATE2)) {
739 panic("thread_deallocate: thread not properly terminated\n");
740 }
741
742 assert(thread->runq == PROCESSOR_NULL);
743
744 #if KPC
745 kpc_thread_destroy(thread);
746 #endif
747
748 ipc_thread_terminate(thread);
749
750 proc_thread_qos_deallocate(thread);
751
752 task = thread->task;
753
754 #ifdef MACH_BSD
755 {
756 void *ut = thread->uthread;
757
758 thread->uthread = NULL;
759 uthread_zone_free(ut);
760 }
761 #endif /* MACH_BSD */
762
763 if (thread->t_ledger) {
764 ledger_dereference(thread->t_ledger);
765 }
766 if (thread->t_threadledger) {
767 ledger_dereference(thread->t_threadledger);
768 }
769
770 assert(thread->turnstile != TURNSTILE_NULL);
771 if (thread->turnstile) {
772 turnstile_deallocate(thread->turnstile);
773 }
774
775 if (IPC_VOUCHER_NULL != thread->ith_voucher) {
776 ipc_voucher_release(thread->ith_voucher);
777 }
778
779 if (thread->thread_io_stats) {
780 kfree(thread->thread_io_stats, sizeof(struct io_stat_info));
781 }
782
783 if (thread->kernel_stack != 0) {
784 stack_free(thread);
785 }
786
787 lck_mtx_destroy(&thread->mutex, &thread_lck_grp);
788 machine_thread_destroy(thread);
789
790 task_deallocate(task);
791
792 #if MACH_ASSERT
793 assert_thread_magic(thread);
794 thread->thread_magic = 0;
795 #endif /* MACH_ASSERT */
796
797 zfree(thread_zone, thread);
798 }
799
800 /*
801 * thread_inspect_deallocate:
802 *
803 * Drop a thread inspection reference.
804 */
805 void
806 thread_inspect_deallocate(
807 thread_inspect_t thread_inspect)
808 {
809 return thread_deallocate((thread_t)thread_inspect);
810 }
811
812 /*
813 * thread_exception_queue_invoke:
814 *
815 * Deliver EXC_{RESOURCE,GUARD} exception
816 */
817 static void
818 thread_exception_queue_invoke(mpsc_queue_chain_t elm,
819 __assert_only mpsc_daemon_queue_t dq)
820 {
821 struct thread_exception_elt *elt;
822 task_t task;
823 thread_t thread;
824 exception_type_t etype;
825
826 assert(dq == &thread_exception_queue);
827 elt = mpsc_queue_element(elm, struct thread_exception_elt, link);
828
829 etype = elt->exception_type;
830 task = elt->exception_task;
831 thread = elt->exception_thread;
832 assert_thread_magic(thread);
833
834 kfree(elt, sizeof(*elt));
835
836 /* wait for all the threads in the task to terminate */
837 task_lock(task);
838 task_wait_till_threads_terminate_locked(task);
839 task_unlock(task);
840
841 /* Consumes the task ref returned by task_generate_corpse_internal */
842 task_deallocate(task);
843 /* Consumes the thread ref returned by task_generate_corpse_internal */
844 thread_deallocate(thread);
845
846 /* Deliver the notification, also clears the corpse. */
847 task_deliver_crash_notification(task, thread, etype, 0);
848 }
849
850 /*
851 * thread_exception_enqueue:
852 *
853 * Enqueue a corpse port to be delivered an EXC_{RESOURCE,GUARD}.
854 */
855 void
856 thread_exception_enqueue(
857 task_t task,
858 thread_t thread,
859 exception_type_t etype)
860 {
861 assert(EXC_RESOURCE == etype || EXC_GUARD == etype);
862 struct thread_exception_elt *elt = kalloc(sizeof(*elt));
863 elt->exception_type = etype;
864 elt->exception_task = task;
865 elt->exception_thread = thread;
866
867 mpsc_daemon_enqueue(&thread_exception_queue, &elt->link,
868 MPSC_QUEUE_DISABLE_PREEMPTION);
869 }
870
871 /*
872 * thread_copy_resource_info
873 *
874 * Copy the resource info counters from source
875 * thread to destination thread.
876 */
877 void
878 thread_copy_resource_info(
879 thread_t dst_thread,
880 thread_t src_thread)
881 {
882 dst_thread->c_switch = src_thread->c_switch;
883 dst_thread->p_switch = src_thread->p_switch;
884 dst_thread->ps_switch = src_thread->ps_switch;
885 dst_thread->precise_user_kernel_time = src_thread->precise_user_kernel_time;
886 dst_thread->user_timer = src_thread->user_timer;
887 dst_thread->user_timer_save = src_thread->user_timer_save;
888 dst_thread->system_timer = src_thread->system_timer;
889 dst_thread->system_timer_save = src_thread->system_timer_save;
890 dst_thread->runnable_timer = src_thread->runnable_timer;
891 dst_thread->vtimer_user_save = src_thread->vtimer_user_save;
892 dst_thread->vtimer_prof_save = src_thread->vtimer_prof_save;
893 dst_thread->vtimer_rlim_save = src_thread->vtimer_rlim_save;
894 dst_thread->vtimer_qos_save = src_thread->vtimer_qos_save;
895 dst_thread->syscalls_unix = src_thread->syscalls_unix;
896 dst_thread->syscalls_mach = src_thread->syscalls_mach;
897 ledger_rollup(dst_thread->t_threadledger, src_thread->t_threadledger);
898 *dst_thread->thread_io_stats = *src_thread->thread_io_stats;
899 }
900
901 static void
902 thread_terminate_queue_invoke(mpsc_queue_chain_t e,
903 __assert_only mpsc_daemon_queue_t dq)
904 {
905 thread_t thread = mpsc_queue_element(e, struct thread, mpsc_links);
906 task_t task = thread->task;
907
908 assert(dq == &thread_terminate_queue);
909
910 task_lock(task);
911
912 /*
913 * if marked for crash reporting, skip reaping.
914 * The corpse delivery thread will clear bit and enqueue
915 * for reaping when done
916 *
917 * Note: the inspection field is set under the task lock
918 *
919 * FIXME[mad]: why enqueue for termination before `inspection` is false ?
920 */
921 if (__improbable(thread->inspection)) {
922 simple_lock(&crashed_threads_lock, &thread_lck_grp);
923 task_unlock(task);
924
925 enqueue_tail(&crashed_threads_queue, &thread->runq_links);
926 simple_unlock(&crashed_threads_lock);
927 return;
928 }
929
930
931 task->total_user_time += timer_grab(&thread->user_timer);
932 task->total_ptime += timer_grab(&thread->ptime);
933 task->total_runnable_time += timer_grab(&thread->runnable_timer);
934 if (thread->precise_user_kernel_time) {
935 task->total_system_time += timer_grab(&thread->system_timer);
936 } else {
937 task->total_user_time += timer_grab(&thread->system_timer);
938 }
939
940 task->c_switch += thread->c_switch;
941 task->p_switch += thread->p_switch;
942 task->ps_switch += thread->ps_switch;
943
944 task->syscalls_unix += thread->syscalls_unix;
945 task->syscalls_mach += thread->syscalls_mach;
946
947 task->task_timer_wakeups_bin_1 += thread->thread_timer_wakeups_bin_1;
948 task->task_timer_wakeups_bin_2 += thread->thread_timer_wakeups_bin_2;
949 task->task_gpu_ns += ml_gpu_stat(thread);
950 task->task_energy += ml_energy_stat(thread);
951 task->decompressions += thread->decompressions;
952
953 #if MONOTONIC
954 mt_terminate_update(task, thread);
955 #endif /* MONOTONIC */
956
957 thread_update_qos_cpu_time(thread);
958
959 queue_remove(&task->threads, thread, thread_t, task_threads);
960 task->thread_count--;
961
962 /*
963 * If the task is being halted, and there is only one thread
964 * left in the task after this one, then wakeup that thread.
965 */
966 if (task->thread_count == 1 && task->halting) {
967 thread_wakeup((event_t)&task->halting);
968 }
969
970 task_unlock(task);
971
972 lck_mtx_lock(&tasks_threads_lock);
973 queue_remove(&threads, thread, thread_t, threads);
974 threads_count--;
975 lck_mtx_unlock(&tasks_threads_lock);
976
977 thread_deallocate(thread);
978 }
979
980 static void
981 thread_deallocate_queue_invoke(mpsc_queue_chain_t e,
982 __assert_only mpsc_daemon_queue_t dq)
983 {
984 thread_t thread = mpsc_queue_element(e, struct thread, mpsc_links);
985
986 assert(dq == &thread_deallocate_queue);
987
988 thread_deallocate_complete(thread);
989 }
990
991 /*
992 * thread_terminate_enqueue:
993 *
994 * Enqueue a terminating thread for final disposition.
995 *
996 * Called at splsched.
997 */
998 void
999 thread_terminate_enqueue(
1000 thread_t thread)
1001 {
1002 KDBG_RELEASE(TRACE_DATA_THREAD_TERMINATE, thread->thread_id);
1003
1004 mpsc_daemon_enqueue(&thread_terminate_queue, &thread->mpsc_links,
1005 MPSC_QUEUE_DISABLE_PREEMPTION);
1006 }
1007
1008 /*
1009 * thread_deallocate_enqueue:
1010 *
1011 * Enqueue a thread for final deallocation.
1012 */
1013 static void
1014 thread_deallocate_enqueue(
1015 thread_t thread)
1016 {
1017 mpsc_daemon_enqueue(&thread_deallocate_queue, &thread->mpsc_links,
1018 MPSC_QUEUE_DISABLE_PREEMPTION);
1019 }
1020
1021 /*
1022 * thread_terminate_crashed_threads:
1023 * walk the list of crashed threads and put back set of threads
1024 * who are no longer being inspected.
1025 */
1026 void
1027 thread_terminate_crashed_threads(void)
1028 {
1029 thread_t th_remove;
1030
1031 simple_lock(&crashed_threads_lock, &thread_lck_grp);
1032 /*
1033 * loop through the crashed threads queue
1034 * to put any threads that are not being inspected anymore
1035 */
1036
1037 qe_foreach_element_safe(th_remove, &crashed_threads_queue, runq_links) {
1038 /* make sure current_thread is never in crashed queue */
1039 assert(th_remove != current_thread());
1040
1041 if (th_remove->inspection == FALSE) {
1042 remqueue(&th_remove->runq_links);
1043 mpsc_daemon_enqueue(&thread_terminate_queue, &th_remove->mpsc_links,
1044 MPSC_QUEUE_NONE);
1045 }
1046 }
1047
1048 simple_unlock(&crashed_threads_lock);
1049 }
1050
1051 /*
1052 * thread_stack_queue_invoke:
1053 *
1054 * Perform stack allocation as required due to
1055 * invoke failures.
1056 */
1057 static void
1058 thread_stack_queue_invoke(mpsc_queue_chain_t elm,
1059 __assert_only mpsc_daemon_queue_t dq)
1060 {
1061 thread_t thread = mpsc_queue_element(elm, struct thread, mpsc_links);
1062
1063 assert(dq == &thread_stack_queue);
1064
1065 /* allocate stack with interrupts enabled so that we can call into VM */
1066 stack_alloc(thread);
1067
1068 KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_STACK_WAIT) | DBG_FUNC_END, thread_tid(thread), 0, 0, 0, 0);
1069
1070 spl_t s = splsched();
1071 thread_lock(thread);
1072 thread_setrun(thread, SCHED_PREEMPT | SCHED_TAILQ);
1073 thread_unlock(thread);
1074 splx(s);
1075 }
1076
1077 /*
1078 * thread_stack_enqueue:
1079 *
1080 * Enqueue a thread for stack allocation.
1081 *
1082 * Called at splsched.
1083 */
1084 void
1085 thread_stack_enqueue(
1086 thread_t thread)
1087 {
1088 KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_STACK_WAIT) | DBG_FUNC_START, thread_tid(thread), 0, 0, 0, 0);
1089 assert_thread_magic(thread);
1090
1091 mpsc_daemon_enqueue(&thread_stack_queue, &thread->mpsc_links,
1092 MPSC_QUEUE_DISABLE_PREEMPTION);
1093 }
1094
1095 void
1096 thread_daemon_init(void)
1097 {
1098 kern_return_t result;
1099
1100 thread_deallocate_daemon_init();
1101
1102 thread_deallocate_daemon_register_queue(&thread_terminate_queue,
1103 thread_terminate_queue_invoke);
1104
1105 thread_deallocate_daemon_register_queue(&thread_deallocate_queue,
1106 thread_deallocate_queue_invoke);
1107
1108 simple_lock_init(&crashed_threads_lock, 0);
1109 queue_init(&crashed_threads_queue);
1110
1111 result = mpsc_daemon_queue_init_with_thread(&thread_stack_queue,
1112 thread_stack_queue_invoke, BASEPRI_PREEMPT_HIGH,
1113 "daemon.thread-stack");
1114 if (result != KERN_SUCCESS) {
1115 panic("thread_daemon_init: thread_stack_daemon");
1116 }
1117
1118 result = mpsc_daemon_queue_init_with_thread(&thread_exception_queue,
1119 thread_exception_queue_invoke, MINPRI_KERNEL,
1120 "daemon.thread-exception");
1121 if (result != KERN_SUCCESS) {
1122 panic("thread_daemon_init: thread_exception_daemon");
1123 }
1124 }
1125
1126 #define TH_OPTION_NONE 0x00
1127 #define TH_OPTION_NOCRED 0x01
1128 #define TH_OPTION_NOSUSP 0x02
1129 #define TH_OPTION_WORKQ 0x04
1130
1131 /*
1132 * Create a new thread.
1133 * Doesn't start the thread running.
1134 *
1135 * Task and tasks_threads_lock are returned locked on success.
1136 */
1137 static kern_return_t
1138 thread_create_internal(
1139 task_t parent_task,
1140 integer_t priority,
1141 thread_continue_t continuation,
1142 void *parameter,
1143 int options,
1144 thread_t *out_thread)
1145 {
1146 thread_t new_thread;
1147 static thread_t first_thread;
1148
1149 /*
1150 * Allocate a thread and initialize static fields
1151 */
1152 if (first_thread == THREAD_NULL) {
1153 new_thread = first_thread = current_thread();
1154 } else {
1155 new_thread = (thread_t)zalloc(thread_zone);
1156 }
1157 if (new_thread == THREAD_NULL) {
1158 return KERN_RESOURCE_SHORTAGE;
1159 }
1160
1161 if (new_thread != first_thread) {
1162 *new_thread = thread_template;
1163 }
1164
1165 os_ref_init_count(&new_thread->ref_count, &thread_refgrp, 2);
1166
1167 #ifdef MACH_BSD
1168 new_thread->uthread = uthread_alloc(parent_task, new_thread, (options & TH_OPTION_NOCRED) != 0);
1169 if (new_thread->uthread == NULL) {
1170 #if MACH_ASSERT
1171 new_thread->thread_magic = 0;
1172 #endif /* MACH_ASSERT */
1173
1174 zfree(thread_zone, new_thread);
1175 return KERN_RESOURCE_SHORTAGE;
1176 }
1177 #endif /* MACH_BSD */
1178
1179 if (machine_thread_create(new_thread, parent_task) != KERN_SUCCESS) {
1180 #ifdef MACH_BSD
1181 void *ut = new_thread->uthread;
1182
1183 new_thread->uthread = NULL;
1184 /* cred free may not be necessary */
1185 uthread_cleanup(parent_task, ut, parent_task->bsd_info);
1186 uthread_cred_free(ut);
1187 uthread_zone_free(ut);
1188 #endif /* MACH_BSD */
1189
1190 #if MACH_ASSERT
1191 new_thread->thread_magic = 0;
1192 #endif /* MACH_ASSERT */
1193
1194 zfree(thread_zone, new_thread);
1195 return KERN_FAILURE;
1196 }
1197
1198 new_thread->task = parent_task;
1199
1200 thread_lock_init(new_thread);
1201 wake_lock_init(new_thread);
1202
1203 lck_mtx_init(&new_thread->mutex, &thread_lck_grp, &thread_lck_attr);
1204
1205 ipc_thread_init(new_thread);
1206
1207 new_thread->continuation = continuation;
1208 new_thread->parameter = parameter;
1209 new_thread->inheritor_flags = TURNSTILE_UPDATE_FLAGS_NONE;
1210 priority_queue_init(&new_thread->sched_inheritor_queue,
1211 PRIORITY_QUEUE_BUILTIN_MAX_HEAP);
1212 priority_queue_init(&new_thread->base_inheritor_queue,
1213 PRIORITY_QUEUE_BUILTIN_MAX_HEAP);
1214 #if CONFIG_SCHED_CLUTCH
1215 priority_queue_entry_init(&new_thread->sched_clutchpri_link);
1216 #endif /* CONFIG_SCHED_CLUTCH */
1217
1218 /* Allocate I/O Statistics structure */
1219 new_thread->thread_io_stats = (io_stat_info_t)kalloc(sizeof(struct io_stat_info));
1220 assert(new_thread->thread_io_stats != NULL);
1221 bzero(new_thread->thread_io_stats, sizeof(struct io_stat_info));
1222
1223 #if KASAN
1224 kasan_init_thread(&new_thread->kasan_data);
1225 #endif
1226
1227 #if CONFIG_KSANCOV
1228 new_thread->ksancov_data = NULL;
1229 #endif
1230
1231 #if CONFIG_IOSCHED
1232 /* Clear out the I/O Scheduling info for AppleFSCompression */
1233 new_thread->decmp_upl = NULL;
1234 #endif /* CONFIG_IOSCHED */
1235
1236 #if DEVELOPMENT || DEBUG
1237 task_lock(parent_task);
1238 uint16_t thread_limit = parent_task->task_thread_limit;
1239 if (exc_resource_threads_enabled &&
1240 thread_limit > 0 &&
1241 parent_task->thread_count >= thread_limit &&
1242 !parent_task->task_has_crossed_thread_limit &&
1243 !(parent_task->t_flags & TF_CORPSE)) {
1244 int thread_count = parent_task->thread_count;
1245 parent_task->task_has_crossed_thread_limit = TRUE;
1246 task_unlock(parent_task);
1247 SENDING_NOTIFICATION__TASK_HAS_TOO_MANY_THREADS(parent_task, thread_count);
1248 } else {
1249 task_unlock(parent_task);
1250 }
1251 #endif
1252
1253 lck_mtx_lock(&tasks_threads_lock);
1254 task_lock(parent_task);
1255
1256 /*
1257 * Fail thread creation if parent task is being torn down or has too many threads
1258 * If the caller asked for TH_OPTION_NOSUSP, also fail if the parent task is suspended
1259 */
1260 if (parent_task->active == 0 || parent_task->halting ||
1261 (parent_task->suspend_count > 0 && (options & TH_OPTION_NOSUSP) != 0) ||
1262 (parent_task->thread_count >= task_threadmax && parent_task != kernel_task)) {
1263 task_unlock(parent_task);
1264 lck_mtx_unlock(&tasks_threads_lock);
1265
1266 #ifdef MACH_BSD
1267 {
1268 void *ut = new_thread->uthread;
1269
1270 new_thread->uthread = NULL;
1271 uthread_cleanup(parent_task, ut, parent_task->bsd_info);
1272 /* cred free may not be necessary */
1273 uthread_cred_free(ut);
1274 uthread_zone_free(ut);
1275 }
1276 #endif /* MACH_BSD */
1277 ipc_thread_disable(new_thread);
1278 ipc_thread_terminate(new_thread);
1279 kfree(new_thread->thread_io_stats, sizeof(struct io_stat_info));
1280 lck_mtx_destroy(&new_thread->mutex, &thread_lck_grp);
1281 machine_thread_destroy(new_thread);
1282 zfree(thread_zone, new_thread);
1283 return KERN_FAILURE;
1284 }
1285
1286 /* Protected by the tasks_threads_lock */
1287 new_thread->thread_id = ++thread_unique_id;
1288
1289 /* New threads inherit any default state on the task */
1290 machine_thread_inherit_taskwide(new_thread, parent_task);
1291
1292 task_reference_internal(parent_task);
1293
1294 if (new_thread->task->rusage_cpu_flags & TASK_RUSECPU_FLAGS_PERTHR_LIMIT) {
1295 /*
1296 * This task has a per-thread CPU limit; make sure this new thread
1297 * gets its limit set too, before it gets out of the kernel.
1298 */
1299 act_set_astledger(new_thread);
1300 }
1301
1302 /* Instantiate a thread ledger. Do not fail thread creation if ledger creation fails. */
1303 if ((new_thread->t_threadledger = ledger_instantiate(thread_ledger_template,
1304 LEDGER_CREATE_INACTIVE_ENTRIES)) != LEDGER_NULL) {
1305 ledger_entry_setactive(new_thread->t_threadledger, thread_ledgers.cpu_time);
1306 }
1307
1308 new_thread->t_bankledger = LEDGER_NULL;
1309 new_thread->t_deduct_bank_ledger_time = 0;
1310 new_thread->t_deduct_bank_ledger_energy = 0;
1311
1312 new_thread->t_ledger = new_thread->task->ledger;
1313 if (new_thread->t_ledger) {
1314 ledger_reference(new_thread->t_ledger);
1315 }
1316
1317 #if defined(CONFIG_SCHED_MULTIQ)
1318 /* Cache the task's sched_group */
1319 new_thread->sched_group = parent_task->sched_group;
1320 #endif /* defined(CONFIG_SCHED_MULTIQ) */
1321
1322 /* Cache the task's map */
1323 new_thread->map = parent_task->map;
1324
1325 timer_call_setup(&new_thread->wait_timer, thread_timer_expire, new_thread);
1326 timer_call_setup(&new_thread->depress_timer, thread_depress_expire, new_thread);
1327
1328 #if KPC
1329 kpc_thread_create(new_thread);
1330 #endif
1331
1332 /* Set the thread's scheduling parameters */
1333 new_thread->sched_mode = SCHED(initial_thread_sched_mode)(parent_task);
1334 new_thread->max_priority = parent_task->max_priority;
1335 new_thread->task_priority = parent_task->priority;
1336
1337
1338 int new_priority = (priority < 0) ? parent_task->priority: priority;
1339 new_priority = (priority < 0)? parent_task->priority: priority;
1340 if (new_priority > new_thread->max_priority) {
1341 new_priority = new_thread->max_priority;
1342 }
1343 #if CONFIG_EMBEDDED
1344 if (new_priority < MAXPRI_THROTTLE) {
1345 new_priority = MAXPRI_THROTTLE;
1346 }
1347 #endif /* CONFIG_EMBEDDED */
1348
1349 new_thread->importance = new_priority - new_thread->task_priority;
1350
1351 sched_set_thread_base_priority(new_thread, new_priority);
1352
1353 #if defined(CONFIG_SCHED_TIMESHARE_CORE)
1354 new_thread->sched_stamp = sched_tick;
1355 #if CONFIG_SCHED_CLUTCH
1356 new_thread->pri_shift = sched_clutch_thread_pri_shift(new_thread, new_thread->th_sched_bucket);
1357 #else /* CONFIG_SCHED_CLUTCH */
1358 new_thread->pri_shift = sched_pri_shifts[new_thread->th_sched_bucket];
1359 #endif /* CONFIG_SCHED_CLUTCH */
1360 #endif /* defined(CONFIG_SCHED_TIMESHARE_CORE) */
1361
1362 #if CONFIG_EMBEDDED
1363 if (parent_task->max_priority <= MAXPRI_THROTTLE) {
1364 sched_thread_mode_demote(new_thread, TH_SFLAG_THROTTLED);
1365 }
1366 #endif /* CONFIG_EMBEDDED */
1367
1368 thread_policy_create(new_thread);
1369
1370 /* Chain the thread onto the task's list */
1371 queue_enter(&parent_task->threads, new_thread, thread_t, task_threads);
1372 parent_task->thread_count++;
1373
1374 /* So terminating threads don't need to take the task lock to decrement */
1375 os_atomic_inc(&parent_task->active_thread_count, relaxed);
1376
1377 queue_enter(&threads, new_thread, thread_t, threads);
1378 threads_count++;
1379
1380 new_thread->active = TRUE;
1381 if (task_is_a_corpse_fork(parent_task)) {
1382 /* Set the inspection bit if the task is a corpse fork */
1383 new_thread->inspection = TRUE;
1384 } else {
1385 new_thread->inspection = FALSE;
1386 }
1387 new_thread->corpse_dup = FALSE;
1388 new_thread->turnstile = turnstile_alloc();
1389 *out_thread = new_thread;
1390
1391 if (kdebug_enable) {
1392 long args[4] = {};
1393
1394 kdbg_trace_data(parent_task->bsd_info, &args[1], &args[3]);
1395
1396 /*
1397 * Starting with 26604425, exec'ing creates a new task/thread.
1398 *
1399 * NEWTHREAD in the current process has two possible meanings:
1400 *
1401 * 1) Create a new thread for this process.
1402 * 2) Create a new thread for the future process this will become in an
1403 * exec.
1404 *
1405 * To disambiguate these, arg3 will be set to TRUE for case #2.
1406 *
1407 * The value we need to find (TPF_EXEC_COPY) is stable in the case of a
1408 * task exec'ing. The read of t_procflags does not take the proc_lock.
1409 */
1410 args[2] = task_is_exec_copy(parent_task) ? 1 : 0;
1411
1412 KDBG_RELEASE(TRACE_DATA_NEWTHREAD, (uintptr_t)thread_tid(new_thread),
1413 args[1], args[2], args[3]);
1414
1415 kdbg_trace_string(parent_task->bsd_info, &args[0], &args[1],
1416 &args[2], &args[3]);
1417 KDBG_RELEASE(TRACE_STRING_NEWTHREAD, args[0], args[1], args[2],
1418 args[3]);
1419 }
1420
1421 DTRACE_PROC1(lwp__create, thread_t, *out_thread);
1422
1423 return KERN_SUCCESS;
1424 }
1425
1426 static kern_return_t
1427 thread_create_internal2(
1428 task_t task,
1429 thread_t *new_thread,
1430 boolean_t from_user,
1431 thread_continue_t continuation)
1432 {
1433 kern_return_t result;
1434 thread_t thread;
1435
1436 if (task == TASK_NULL || task == kernel_task) {
1437 return KERN_INVALID_ARGUMENT;
1438 }
1439
1440 result = thread_create_internal(task, -1, continuation, NULL, TH_OPTION_NONE, &thread);
1441 if (result != KERN_SUCCESS) {
1442 return result;
1443 }
1444
1445 thread->user_stop_count = 1;
1446 thread_hold(thread);
1447 if (task->suspend_count > 0) {
1448 thread_hold(thread);
1449 }
1450
1451 if (from_user) {
1452 extmod_statistics_incr_thread_create(task);
1453 }
1454
1455 task_unlock(task);
1456 lck_mtx_unlock(&tasks_threads_lock);
1457
1458 *new_thread = thread;
1459
1460 return KERN_SUCCESS;
1461 }
1462
1463 /* No prototype, since task_server.h has the _from_user version if KERNEL_SERVER */
1464 kern_return_t
1465 thread_create(
1466 task_t task,
1467 thread_t *new_thread);
1468
1469 kern_return_t
1470 thread_create(
1471 task_t task,
1472 thread_t *new_thread)
1473 {
1474 return thread_create_internal2(task, new_thread, FALSE, (thread_continue_t)thread_bootstrap_return);
1475 }
1476
1477 kern_return_t
1478 thread_create_from_user(
1479 task_t task,
1480 thread_t *new_thread)
1481 {
1482 return thread_create_internal2(task, new_thread, TRUE, (thread_continue_t)thread_bootstrap_return);
1483 }
1484
1485 kern_return_t
1486 thread_create_with_continuation(
1487 task_t task,
1488 thread_t *new_thread,
1489 thread_continue_t continuation)
1490 {
1491 return thread_create_internal2(task, new_thread, FALSE, continuation);
1492 }
1493
1494 /*
1495 * Create a thread that is already started, but is waiting on an event
1496 */
1497 static kern_return_t
1498 thread_create_waiting_internal(
1499 task_t task,
1500 thread_continue_t continuation,
1501 event_t event,
1502 block_hint_t block_hint,
1503 int options,
1504 thread_t *new_thread)
1505 {
1506 kern_return_t result;
1507 thread_t thread;
1508
1509 if (task == TASK_NULL || task == kernel_task) {
1510 return KERN_INVALID_ARGUMENT;
1511 }
1512
1513 result = thread_create_internal(task, -1, continuation, NULL,
1514 options, &thread);
1515 if (result != KERN_SUCCESS) {
1516 return result;
1517 }
1518
1519 /* note no user_stop_count or thread_hold here */
1520
1521 if (task->suspend_count > 0) {
1522 thread_hold(thread);
1523 }
1524
1525 thread_mtx_lock(thread);
1526 thread_set_pending_block_hint(thread, block_hint);
1527 if (options & TH_OPTION_WORKQ) {
1528 thread->static_param = true;
1529 event = workq_thread_init_and_wq_lock(task, thread);
1530 }
1531 thread_start_in_assert_wait(thread, event, THREAD_INTERRUPTIBLE);
1532 thread_mtx_unlock(thread);
1533
1534 task_unlock(task);
1535 lck_mtx_unlock(&tasks_threads_lock);
1536
1537 *new_thread = thread;
1538
1539 return KERN_SUCCESS;
1540 }
1541
1542 kern_return_t
1543 thread_create_waiting(
1544 task_t task,
1545 thread_continue_t continuation,
1546 event_t event,
1547 thread_t *new_thread)
1548 {
1549 return thread_create_waiting_internal(task, continuation, event,
1550 kThreadWaitNone, TH_OPTION_NONE, new_thread);
1551 }
1552
1553
1554 static kern_return_t
1555 thread_create_running_internal2(
1556 task_t task,
1557 int flavor,
1558 thread_state_t new_state,
1559 mach_msg_type_number_t new_state_count,
1560 thread_t *new_thread,
1561 boolean_t from_user)
1562 {
1563 kern_return_t result;
1564 thread_t thread;
1565
1566 if (task == TASK_NULL || task == kernel_task) {
1567 return KERN_INVALID_ARGUMENT;
1568 }
1569
1570 result = thread_create_internal(task, -1,
1571 (thread_continue_t)thread_bootstrap_return, NULL,
1572 TH_OPTION_NONE, &thread);
1573 if (result != KERN_SUCCESS) {
1574 return result;
1575 }
1576
1577 if (task->suspend_count > 0) {
1578 thread_hold(thread);
1579 }
1580
1581 if (from_user) {
1582 result = machine_thread_state_convert_from_user(thread, flavor,
1583 new_state, new_state_count);
1584 }
1585 if (result == KERN_SUCCESS) {
1586 result = machine_thread_set_state(thread, flavor, new_state,
1587 new_state_count);
1588 }
1589 if (result != KERN_SUCCESS) {
1590 task_unlock(task);
1591 lck_mtx_unlock(&tasks_threads_lock);
1592
1593 thread_terminate(thread);
1594 thread_deallocate(thread);
1595 return result;
1596 }
1597
1598 thread_mtx_lock(thread);
1599 thread_start(thread);
1600 thread_mtx_unlock(thread);
1601
1602 if (from_user) {
1603 extmod_statistics_incr_thread_create(task);
1604 }
1605
1606 task_unlock(task);
1607 lck_mtx_unlock(&tasks_threads_lock);
1608
1609 *new_thread = thread;
1610
1611 return result;
1612 }
1613
1614 /* Prototype, see justification above */
1615 kern_return_t
1616 thread_create_running(
1617 task_t task,
1618 int flavor,
1619 thread_state_t new_state,
1620 mach_msg_type_number_t new_state_count,
1621 thread_t *new_thread);
1622
1623 kern_return_t
1624 thread_create_running(
1625 task_t task,
1626 int flavor,
1627 thread_state_t new_state,
1628 mach_msg_type_number_t new_state_count,
1629 thread_t *new_thread)
1630 {
1631 return thread_create_running_internal2(
1632 task, flavor, new_state, new_state_count,
1633 new_thread, FALSE);
1634 }
1635
1636 kern_return_t
1637 thread_create_running_from_user(
1638 task_t task,
1639 int flavor,
1640 thread_state_t new_state,
1641 mach_msg_type_number_t new_state_count,
1642 thread_t *new_thread)
1643 {
1644 return thread_create_running_internal2(
1645 task, flavor, new_state, new_state_count,
1646 new_thread, TRUE);
1647 }
1648
1649 kern_return_t
1650 thread_create_workq_waiting(
1651 task_t task,
1652 thread_continue_t continuation,
1653 thread_t *new_thread)
1654 {
1655 int options = TH_OPTION_NOCRED | TH_OPTION_NOSUSP | TH_OPTION_WORKQ;
1656 return thread_create_waiting_internal(task, continuation, NULL,
1657 kThreadWaitParkedWorkQueue, options, new_thread);
1658 }
1659
1660 /*
1661 * kernel_thread_create:
1662 *
1663 * Create a thread in the kernel task
1664 * to execute in kernel context.
1665 */
1666 kern_return_t
1667 kernel_thread_create(
1668 thread_continue_t continuation,
1669 void *parameter,
1670 integer_t priority,
1671 thread_t *new_thread)
1672 {
1673 kern_return_t result;
1674 thread_t thread;
1675 task_t task = kernel_task;
1676
1677 result = thread_create_internal(task, priority, continuation, parameter,
1678 TH_OPTION_NOCRED | TH_OPTION_NONE, &thread);
1679 if (result != KERN_SUCCESS) {
1680 return result;
1681 }
1682
1683 task_unlock(task);
1684 lck_mtx_unlock(&tasks_threads_lock);
1685
1686 stack_alloc(thread);
1687 assert(thread->kernel_stack != 0);
1688 #if CONFIG_EMBEDDED
1689 if (priority > BASEPRI_KERNEL)
1690 #endif
1691 thread->reserved_stack = thread->kernel_stack;
1692
1693 if (debug_task & 1) {
1694 kprintf("kernel_thread_create: thread = %p continuation = %p\n", thread, continuation);
1695 }
1696 *new_thread = thread;
1697
1698 return result;
1699 }
1700
1701 kern_return_t
1702 kernel_thread_start_priority(
1703 thread_continue_t continuation,
1704 void *parameter,
1705 integer_t priority,
1706 thread_t *new_thread)
1707 {
1708 kern_return_t result;
1709 thread_t thread;
1710
1711 result = kernel_thread_create(continuation, parameter, priority, &thread);
1712 if (result != KERN_SUCCESS) {
1713 return result;
1714 }
1715
1716 *new_thread = thread;
1717
1718 thread_mtx_lock(thread);
1719 thread_start(thread);
1720 thread_mtx_unlock(thread);
1721
1722 return result;
1723 }
1724
1725 kern_return_t
1726 kernel_thread_start(
1727 thread_continue_t continuation,
1728 void *parameter,
1729 thread_t *new_thread)
1730 {
1731 return kernel_thread_start_priority(continuation, parameter, -1, new_thread);
1732 }
1733
1734 /* Separated into helper function so it can be used by THREAD_BASIC_INFO and THREAD_EXTENDED_INFO */
1735 /* it is assumed that the thread is locked by the caller */
1736 static void
1737 retrieve_thread_basic_info(thread_t thread, thread_basic_info_t basic_info)
1738 {
1739 int state, flags;
1740
1741 /* fill in info */
1742
1743 thread_read_times(thread, &basic_info->user_time,
1744 &basic_info->system_time, NULL);
1745
1746 /*
1747 * Update lazy-evaluated scheduler info because someone wants it.
1748 */
1749 if (SCHED(can_update_priority)(thread)) {
1750 SCHED(update_priority)(thread);
1751 }
1752
1753 basic_info->sleep_time = 0;
1754
1755 /*
1756 * To calculate cpu_usage, first correct for timer rate,
1757 * then for 5/8 ageing. The correction factor [3/5] is
1758 * (1/(5/8) - 1).
1759 */
1760 basic_info->cpu_usage = 0;
1761 #if defined(CONFIG_SCHED_TIMESHARE_CORE)
1762 if (sched_tick_interval) {
1763 basic_info->cpu_usage = (integer_t)(((uint64_t)thread->cpu_usage
1764 * TH_USAGE_SCALE) / sched_tick_interval);
1765 basic_info->cpu_usage = (basic_info->cpu_usage * 3) / 5;
1766 }
1767 #endif
1768
1769 if (basic_info->cpu_usage > TH_USAGE_SCALE) {
1770 basic_info->cpu_usage = TH_USAGE_SCALE;
1771 }
1772
1773 basic_info->policy = ((thread->sched_mode == TH_MODE_TIMESHARE)?
1774 POLICY_TIMESHARE: POLICY_RR);
1775
1776 flags = 0;
1777 if (thread->options & TH_OPT_IDLE_THREAD) {
1778 flags |= TH_FLAGS_IDLE;
1779 }
1780
1781 if (thread->options & TH_OPT_GLOBAL_FORCED_IDLE) {
1782 flags |= TH_FLAGS_GLOBAL_FORCED_IDLE;
1783 }
1784
1785 if (!thread->kernel_stack) {
1786 flags |= TH_FLAGS_SWAPPED;
1787 }
1788
1789 state = 0;
1790 if (thread->state & TH_TERMINATE) {
1791 state = TH_STATE_HALTED;
1792 } else if (thread->state & TH_RUN) {
1793 state = TH_STATE_RUNNING;
1794 } else if (thread->state & TH_UNINT) {
1795 state = TH_STATE_UNINTERRUPTIBLE;
1796 } else if (thread->state & TH_SUSP) {
1797 state = TH_STATE_STOPPED;
1798 } else if (thread->state & TH_WAIT) {
1799 state = TH_STATE_WAITING;
1800 }
1801
1802 basic_info->run_state = state;
1803 basic_info->flags = flags;
1804
1805 basic_info->suspend_count = thread->user_stop_count;
1806
1807 return;
1808 }
1809
1810 kern_return_t
1811 thread_info_internal(
1812 thread_t thread,
1813 thread_flavor_t flavor,
1814 thread_info_t thread_info_out, /* ptr to OUT array */
1815 mach_msg_type_number_t *thread_info_count) /*IN/OUT*/
1816 {
1817 spl_t s;
1818
1819 if (thread == THREAD_NULL) {
1820 return KERN_INVALID_ARGUMENT;
1821 }
1822
1823 if (flavor == THREAD_BASIC_INFO) {
1824 if (*thread_info_count < THREAD_BASIC_INFO_COUNT) {
1825 return KERN_INVALID_ARGUMENT;
1826 }
1827
1828 s = splsched();
1829 thread_lock(thread);
1830
1831 retrieve_thread_basic_info(thread, (thread_basic_info_t) thread_info_out);
1832
1833 thread_unlock(thread);
1834 splx(s);
1835
1836 *thread_info_count = THREAD_BASIC_INFO_COUNT;
1837
1838 return KERN_SUCCESS;
1839 } else if (flavor == THREAD_IDENTIFIER_INFO) {
1840 thread_identifier_info_t identifier_info;
1841
1842 if (*thread_info_count < THREAD_IDENTIFIER_INFO_COUNT) {
1843 return KERN_INVALID_ARGUMENT;
1844 }
1845
1846 identifier_info = (thread_identifier_info_t) thread_info_out;
1847
1848 s = splsched();
1849 thread_lock(thread);
1850
1851 identifier_info->thread_id = thread->thread_id;
1852 identifier_info->thread_handle = thread->machine.cthread_self;
1853 identifier_info->dispatch_qaddr = thread_dispatchqaddr(thread);
1854
1855 thread_unlock(thread);
1856 splx(s);
1857 return KERN_SUCCESS;
1858 } else if (flavor == THREAD_SCHED_TIMESHARE_INFO) {
1859 policy_timeshare_info_t ts_info;
1860
1861 if (*thread_info_count < POLICY_TIMESHARE_INFO_COUNT) {
1862 return KERN_INVALID_ARGUMENT;
1863 }
1864
1865 ts_info = (policy_timeshare_info_t)thread_info_out;
1866
1867 s = splsched();
1868 thread_lock(thread);
1869
1870 if (thread->sched_mode != TH_MODE_TIMESHARE) {
1871 thread_unlock(thread);
1872 splx(s);
1873 return KERN_INVALID_POLICY;
1874 }
1875
1876 ts_info->depressed = (thread->sched_flags & TH_SFLAG_DEPRESSED_MASK) != 0;
1877 if (ts_info->depressed) {
1878 ts_info->base_priority = DEPRESSPRI;
1879 ts_info->depress_priority = thread->base_pri;
1880 } else {
1881 ts_info->base_priority = thread->base_pri;
1882 ts_info->depress_priority = -1;
1883 }
1884
1885 ts_info->cur_priority = thread->sched_pri;
1886 ts_info->max_priority = thread->max_priority;
1887
1888 thread_unlock(thread);
1889 splx(s);
1890
1891 *thread_info_count = POLICY_TIMESHARE_INFO_COUNT;
1892
1893 return KERN_SUCCESS;
1894 } else if (flavor == THREAD_SCHED_FIFO_INFO) {
1895 if (*thread_info_count < POLICY_FIFO_INFO_COUNT) {
1896 return KERN_INVALID_ARGUMENT;
1897 }
1898
1899 return KERN_INVALID_POLICY;
1900 } else if (flavor == THREAD_SCHED_RR_INFO) {
1901 policy_rr_info_t rr_info;
1902 uint32_t quantum_time;
1903 uint64_t quantum_ns;
1904
1905 if (*thread_info_count < POLICY_RR_INFO_COUNT) {
1906 return KERN_INVALID_ARGUMENT;
1907 }
1908
1909 rr_info = (policy_rr_info_t) thread_info_out;
1910
1911 s = splsched();
1912 thread_lock(thread);
1913
1914 if (thread->sched_mode == TH_MODE_TIMESHARE) {
1915 thread_unlock(thread);
1916 splx(s);
1917
1918 return KERN_INVALID_POLICY;
1919 }
1920
1921 rr_info->depressed = (thread->sched_flags & TH_SFLAG_DEPRESSED_MASK) != 0;
1922 if (rr_info->depressed) {
1923 rr_info->base_priority = DEPRESSPRI;
1924 rr_info->depress_priority = thread->base_pri;
1925 } else {
1926 rr_info->base_priority = thread->base_pri;
1927 rr_info->depress_priority = -1;
1928 }
1929
1930 quantum_time = SCHED(initial_quantum_size)(THREAD_NULL);
1931 absolutetime_to_nanoseconds(quantum_time, &quantum_ns);
1932
1933 rr_info->max_priority = thread->max_priority;
1934 rr_info->quantum = (uint32_t)(quantum_ns / 1000 / 1000);
1935
1936 thread_unlock(thread);
1937 splx(s);
1938
1939 *thread_info_count = POLICY_RR_INFO_COUNT;
1940
1941 return KERN_SUCCESS;
1942 } else if (flavor == THREAD_EXTENDED_INFO) {
1943 thread_basic_info_data_t basic_info;
1944 thread_extended_info_t extended_info = (thread_extended_info_t) thread_info_out;
1945
1946 if (*thread_info_count < THREAD_EXTENDED_INFO_COUNT) {
1947 return KERN_INVALID_ARGUMENT;
1948 }
1949
1950 s = splsched();
1951 thread_lock(thread);
1952
1953 /* NOTE: This mimics fill_taskthreadinfo(), which is the function used by proc_pidinfo() for
1954 * the PROC_PIDTHREADINFO flavor (which can't be used on corpses)
1955 */
1956 retrieve_thread_basic_info(thread, &basic_info);
1957 extended_info->pth_user_time = (((uint64_t)basic_info.user_time.seconds * NSEC_PER_SEC) + ((uint64_t)basic_info.user_time.microseconds * NSEC_PER_USEC));
1958 extended_info->pth_system_time = (((uint64_t)basic_info.system_time.seconds * NSEC_PER_SEC) + ((uint64_t)basic_info.system_time.microseconds * NSEC_PER_USEC));
1959
1960 extended_info->pth_cpu_usage = basic_info.cpu_usage;
1961 extended_info->pth_policy = basic_info.policy;
1962 extended_info->pth_run_state = basic_info.run_state;
1963 extended_info->pth_flags = basic_info.flags;
1964 extended_info->pth_sleep_time = basic_info.sleep_time;
1965 extended_info->pth_curpri = thread->sched_pri;
1966 extended_info->pth_priority = thread->base_pri;
1967 extended_info->pth_maxpriority = thread->max_priority;
1968
1969 bsd_getthreadname(thread->uthread, extended_info->pth_name);
1970
1971 thread_unlock(thread);
1972 splx(s);
1973
1974 *thread_info_count = THREAD_EXTENDED_INFO_COUNT;
1975
1976 return KERN_SUCCESS;
1977 } else if (flavor == THREAD_DEBUG_INFO_INTERNAL) {
1978 #if DEVELOPMENT || DEBUG
1979 thread_debug_info_internal_t dbg_info;
1980 if (*thread_info_count < THREAD_DEBUG_INFO_INTERNAL_COUNT) {
1981 return KERN_NOT_SUPPORTED;
1982 }
1983
1984 if (thread_info_out == NULL) {
1985 return KERN_INVALID_ARGUMENT;
1986 }
1987
1988 dbg_info = (thread_debug_info_internal_t) thread_info_out;
1989 dbg_info->page_creation_count = thread->t_page_creation_count;
1990
1991 *thread_info_count = THREAD_DEBUG_INFO_INTERNAL_COUNT;
1992 return KERN_SUCCESS;
1993 #endif /* DEVELOPMENT || DEBUG */
1994 return KERN_NOT_SUPPORTED;
1995 }
1996
1997 return KERN_INVALID_ARGUMENT;
1998 }
1999
2000 void
2001 thread_read_times(
2002 thread_t thread,
2003 time_value_t *user_time,
2004 time_value_t *system_time,
2005 time_value_t *runnable_time)
2006 {
2007 clock_sec_t secs;
2008 clock_usec_t usecs;
2009 uint64_t tval_user, tval_system;
2010
2011 tval_user = timer_grab(&thread->user_timer);
2012 tval_system = timer_grab(&thread->system_timer);
2013
2014 if (thread->precise_user_kernel_time) {
2015 absolutetime_to_microtime(tval_user, &secs, &usecs);
2016 user_time->seconds = (typeof(user_time->seconds))secs;
2017 user_time->microseconds = usecs;
2018
2019 absolutetime_to_microtime(tval_system, &secs, &usecs);
2020 system_time->seconds = (typeof(system_time->seconds))secs;
2021 system_time->microseconds = usecs;
2022 } else {
2023 /* system_timer may represent either sys or user */
2024 tval_user += tval_system;
2025 absolutetime_to_microtime(tval_user, &secs, &usecs);
2026 user_time->seconds = (typeof(user_time->seconds))secs;
2027 user_time->microseconds = usecs;
2028
2029 system_time->seconds = 0;
2030 system_time->microseconds = 0;
2031 }
2032
2033 if (runnable_time) {
2034 uint64_t tval_runnable = timer_grab(&thread->runnable_timer);
2035 absolutetime_to_microtime(tval_runnable, &secs, &usecs);
2036 runnable_time->seconds = (typeof(runnable_time->seconds))secs;
2037 runnable_time->microseconds = usecs;
2038 }
2039 }
2040
2041 uint64_t
2042 thread_get_runtime_self(void)
2043 {
2044 boolean_t interrupt_state;
2045 uint64_t runtime;
2046 thread_t thread = NULL;
2047 processor_t processor = NULL;
2048
2049 thread = current_thread();
2050
2051 /* Not interrupt safe, as the scheduler may otherwise update timer values underneath us */
2052 interrupt_state = ml_set_interrupts_enabled(FALSE);
2053 processor = current_processor();
2054 timer_update(PROCESSOR_DATA(processor, thread_timer), mach_absolute_time());
2055 runtime = (timer_grab(&thread->user_timer) + timer_grab(&thread->system_timer));
2056 ml_set_interrupts_enabled(interrupt_state);
2057
2058 return runtime;
2059 }
2060
2061 kern_return_t
2062 thread_assign(
2063 __unused thread_t thread,
2064 __unused processor_set_t new_pset)
2065 {
2066 return KERN_FAILURE;
2067 }
2068
2069 /*
2070 * thread_assign_default:
2071 *
2072 * Special version of thread_assign for assigning threads to default
2073 * processor set.
2074 */
2075 kern_return_t
2076 thread_assign_default(
2077 thread_t thread)
2078 {
2079 return thread_assign(thread, &pset0);
2080 }
2081
2082 /*
2083 * thread_get_assignment
2084 *
2085 * Return current assignment for this thread.
2086 */
2087 kern_return_t
2088 thread_get_assignment(
2089 thread_t thread,
2090 processor_set_t *pset)
2091 {
2092 if (thread == NULL) {
2093 return KERN_INVALID_ARGUMENT;
2094 }
2095
2096 *pset = &pset0;
2097
2098 return KERN_SUCCESS;
2099 }
2100
2101 /*
2102 * thread_wire_internal:
2103 *
2104 * Specify that the target thread must always be able
2105 * to run and to allocate memory.
2106 */
2107 kern_return_t
2108 thread_wire_internal(
2109 host_priv_t host_priv,
2110 thread_t thread,
2111 boolean_t wired,
2112 boolean_t *prev_state)
2113 {
2114 if (host_priv == NULL || thread != current_thread()) {
2115 return KERN_INVALID_ARGUMENT;
2116 }
2117
2118 assert(host_priv == &realhost);
2119
2120 if (prev_state) {
2121 *prev_state = (thread->options & TH_OPT_VMPRIV) != 0;
2122 }
2123
2124 if (wired) {
2125 if (!(thread->options & TH_OPT_VMPRIV)) {
2126 vm_page_free_reserve(1); /* XXX */
2127 }
2128 thread->options |= TH_OPT_VMPRIV;
2129 } else {
2130 if (thread->options & TH_OPT_VMPRIV) {
2131 vm_page_free_reserve(-1); /* XXX */
2132 }
2133 thread->options &= ~TH_OPT_VMPRIV;
2134 }
2135
2136 return KERN_SUCCESS;
2137 }
2138
2139
2140 /*
2141 * thread_wire:
2142 *
2143 * User-api wrapper for thread_wire_internal()
2144 */
2145 kern_return_t
2146 thread_wire(
2147 host_priv_t host_priv,
2148 thread_t thread,
2149 boolean_t wired)
2150 {
2151 return thread_wire_internal(host_priv, thread, wired, NULL);
2152 }
2153
2154
2155 boolean_t
2156 is_vm_privileged(void)
2157 {
2158 return current_thread()->options & TH_OPT_VMPRIV ? TRUE : FALSE;
2159 }
2160
2161 boolean_t
2162 set_vm_privilege(boolean_t privileged)
2163 {
2164 boolean_t was_vmpriv;
2165
2166 if (current_thread()->options & TH_OPT_VMPRIV) {
2167 was_vmpriv = TRUE;
2168 } else {
2169 was_vmpriv = FALSE;
2170 }
2171
2172 if (privileged != FALSE) {
2173 current_thread()->options |= TH_OPT_VMPRIV;
2174 } else {
2175 current_thread()->options &= ~TH_OPT_VMPRIV;
2176 }
2177
2178 return was_vmpriv;
2179 }
2180
2181 void
2182 set_thread_rwlock_boost(void)
2183 {
2184 current_thread()->rwlock_count++;
2185 }
2186
2187 void
2188 clear_thread_rwlock_boost(void)
2189 {
2190 thread_t thread = current_thread();
2191
2192 if ((thread->rwlock_count-- == 1) && (thread->sched_flags & TH_SFLAG_RW_PROMOTED)) {
2193 lck_rw_clear_promotion(thread, 0);
2194 }
2195 }
2196
2197 /*
2198 * XXX assuming current thread only, for now...
2199 */
2200 void
2201 thread_guard_violation(thread_t thread,
2202 mach_exception_data_type_t code, mach_exception_data_type_t subcode, boolean_t fatal)
2203 {
2204 assert(thread == current_thread());
2205
2206 /* Don't set up the AST for kernel threads; this check is needed to ensure
2207 * that the guard_exc_* fields in the thread structure are set only by the
2208 * current thread and therefore, don't require a lock.
2209 */
2210 if (thread->task == kernel_task) {
2211 return;
2212 }
2213
2214 assert(EXC_GUARD_DECODE_GUARD_TYPE(code));
2215
2216 /*
2217 * Use the saved state area of the thread structure
2218 * to store all info required to handle the AST when
2219 * returning to userspace. It's possible that there is
2220 * already a pending guard exception. If it's non-fatal,
2221 * it can only be over-written by a fatal exception code.
2222 */
2223 if (thread->guard_exc_info.code && (thread->guard_exc_fatal || !fatal)) {
2224 return;
2225 }
2226
2227 thread->guard_exc_info.code = code;
2228 thread->guard_exc_info.subcode = subcode;
2229 thread->guard_exc_fatal = fatal ? 1 : 0;
2230
2231 spl_t s = splsched();
2232 thread_ast_set(thread, AST_GUARD);
2233 ast_propagate(thread);
2234 splx(s);
2235 }
2236
2237 /*
2238 * guard_ast:
2239 *
2240 * Handle AST_GUARD for a thread. This routine looks at the
2241 * state saved in the thread structure to determine the cause
2242 * of this exception. Based on this value, it invokes the
2243 * appropriate routine which determines other exception related
2244 * info and raises the exception.
2245 */
2246 void
2247 guard_ast(thread_t t)
2248 {
2249 const mach_exception_data_type_t
2250 code = t->guard_exc_info.code,
2251 subcode = t->guard_exc_info.subcode;
2252
2253 t->guard_exc_info.code = 0;
2254 t->guard_exc_info.subcode = 0;
2255 t->guard_exc_fatal = 0;
2256
2257 switch (EXC_GUARD_DECODE_GUARD_TYPE(code)) {
2258 case GUARD_TYPE_NONE:
2259 /* lingering AST_GUARD on the processor? */
2260 break;
2261 case GUARD_TYPE_MACH_PORT:
2262 mach_port_guard_ast(t, code, subcode);
2263 break;
2264 case GUARD_TYPE_FD:
2265 fd_guard_ast(t, code, subcode);
2266 break;
2267 #if CONFIG_VNGUARD
2268 case GUARD_TYPE_VN:
2269 vn_guard_ast(t, code, subcode);
2270 break;
2271 #endif
2272 case GUARD_TYPE_VIRT_MEMORY:
2273 virt_memory_guard_ast(t, code, subcode);
2274 break;
2275 default:
2276 panic("guard_exc_info %llx %llx", code, subcode);
2277 }
2278 }
2279
2280 static void
2281 thread_cputime_callback(int warning, __unused const void *arg0, __unused const void *arg1)
2282 {
2283 if (warning == LEDGER_WARNING_ROSE_ABOVE) {
2284 #if CONFIG_TELEMETRY
2285 /*
2286 * This thread is in danger of violating the CPU usage monitor. Enable telemetry
2287 * on the entire task so there are micro-stackshots available if and when
2288 * EXC_RESOURCE is triggered. We could have chosen to enable micro-stackshots
2289 * for this thread only; but now that this task is suspect, knowing what all of
2290 * its threads are up to will be useful.
2291 */
2292 telemetry_task_ctl(current_task(), TF_CPUMON_WARNING, 1);
2293 #endif
2294 return;
2295 }
2296
2297 #if CONFIG_TELEMETRY
2298 /*
2299 * If the balance has dipped below the warning level (LEDGER_WARNING_DIPPED_BELOW) or
2300 * exceeded the limit, turn telemetry off for the task.
2301 */
2302 telemetry_task_ctl(current_task(), TF_CPUMON_WARNING, 0);
2303 #endif
2304
2305 if (warning == 0) {
2306 SENDING_NOTIFICATION__THIS_THREAD_IS_CONSUMING_TOO_MUCH_CPU();
2307 }
2308 }
2309
2310 void __attribute__((noinline))
2311 SENDING_NOTIFICATION__THIS_THREAD_IS_CONSUMING_TOO_MUCH_CPU(void)
2312 {
2313 int pid = 0;
2314 task_t task = current_task();
2315 thread_t thread = current_thread();
2316 uint64_t tid = thread->thread_id;
2317 const char *procname = "unknown";
2318 time_value_t thread_total_time = {0, 0};
2319 time_value_t thread_system_time;
2320 time_value_t thread_user_time;
2321 int action;
2322 uint8_t percentage;
2323 uint32_t usage_percent = 0;
2324 uint32_t interval_sec;
2325 uint64_t interval_ns;
2326 uint64_t balance_ns;
2327 boolean_t fatal = FALSE;
2328 boolean_t send_exc_resource = TRUE; /* in addition to RESOURCE_NOTIFY */
2329 kern_return_t kr;
2330
2331 #ifdef EXC_RESOURCE_MONITORS
2332 mach_exception_data_type_t code[EXCEPTION_CODE_MAX];
2333 #endif /* EXC_RESOURCE_MONITORS */
2334 struct ledger_entry_info lei;
2335
2336 assert(thread->t_threadledger != LEDGER_NULL);
2337
2338 /*
2339 * Extract the fatal bit and suspend the monitor (which clears the bit).
2340 */
2341 task_lock(task);
2342 if (task->rusage_cpu_flags & TASK_RUSECPU_FLAGS_FATAL_CPUMON) {
2343 fatal = TRUE;
2344 send_exc_resource = TRUE;
2345 }
2346 /* Only one thread can be here at a time. Whichever makes it through
2347 * first will successfully suspend the monitor and proceed to send the
2348 * notification. Other threads will get an error trying to suspend the
2349 * monitor and give up on sending the notification. In the first release,
2350 * the monitor won't be resumed for a number of seconds, but we may
2351 * eventually need to handle low-latency resume.
2352 */
2353 kr = task_suspend_cpumon(task);
2354 task_unlock(task);
2355 if (kr == KERN_INVALID_ARGUMENT) {
2356 return;
2357 }
2358
2359 #ifdef MACH_BSD
2360 pid = proc_selfpid();
2361 if (task->bsd_info != NULL) {
2362 procname = proc_name_address(task->bsd_info);
2363 }
2364 #endif
2365
2366 thread_get_cpulimit(&action, &percentage, &interval_ns);
2367
2368 interval_sec = (uint32_t)(interval_ns / NSEC_PER_SEC);
2369
2370 thread_read_times(thread, &thread_user_time, &thread_system_time, NULL);
2371 time_value_add(&thread_total_time, &thread_user_time);
2372 time_value_add(&thread_total_time, &thread_system_time);
2373 ledger_get_entry_info(thread->t_threadledger, thread_ledgers.cpu_time, &lei);
2374
2375 /* credit/debit/balance/limit are in absolute time units;
2376 * the refill info is in nanoseconds. */
2377 absolutetime_to_nanoseconds(lei.lei_balance, &balance_ns);
2378 if (lei.lei_last_refill > 0) {
2379 usage_percent = (uint32_t)((balance_ns * 100ULL) / lei.lei_last_refill);
2380 }
2381
2382 /* TODO: show task total runtime (via TASK_ABSOLUTETIME_INFO)? */
2383 printf("process %s[%d] thread %llu caught burning CPU! It used more than %d%% CPU over %u seconds\n",
2384 procname, pid, tid, percentage, interval_sec);
2385 printf(" (actual recent usage: %d%% over ~%llu seconds)\n",
2386 usage_percent, (lei.lei_last_refill + NSEC_PER_SEC / 2) / NSEC_PER_SEC);
2387 printf(" Thread lifetime cpu usage %d.%06ds, (%d.%06d user, %d.%06d sys)\n",
2388 thread_total_time.seconds, thread_total_time.microseconds,
2389 thread_user_time.seconds, thread_user_time.microseconds,
2390 thread_system_time.seconds, thread_system_time.microseconds);
2391 printf(" Ledger balance: %lld; mabs credit: %lld; mabs debit: %lld\n",
2392 lei.lei_balance, lei.lei_credit, lei.lei_debit);
2393 printf(" mabs limit: %llu; mabs period: %llu ns; last refill: %llu ns%s.\n",
2394 lei.lei_limit, lei.lei_refill_period, lei.lei_last_refill,
2395 (fatal ? " [fatal violation]" : ""));
2396
2397 /*
2398 * For now, send RESOURCE_NOTIFY in parallel with EXC_RESOURCE. Once
2399 * we have logging parity, we will stop sending EXC_RESOURCE (24508922).
2400 */
2401
2402 /* RESOURCE_NOTIFY MIG specifies nanoseconds of CPU time */
2403 lei.lei_balance = balance_ns;
2404 absolutetime_to_nanoseconds(lei.lei_limit, &lei.lei_limit);
2405 trace_resource_violation(RMON_CPUUSAGE_VIOLATED, &lei);
2406 kr = send_resource_violation(send_cpu_usage_violation, task, &lei,
2407 fatal ? kRNFatalLimitFlag : 0);
2408 if (kr) {
2409 printf("send_resource_violation(CPU usage, ...): error %#x\n", kr);
2410 }
2411
2412 #ifdef EXC_RESOURCE_MONITORS
2413 if (send_exc_resource) {
2414 if (disable_exc_resource) {
2415 printf("process %s[%d] thread %llu caught burning CPU! "
2416 "EXC_RESOURCE%s supressed by a boot-arg\n",
2417 procname, pid, tid, fatal ? " (and termination)" : "");
2418 return;
2419 }
2420
2421 if (audio_active) {
2422 printf("process %s[%d] thread %llu caught burning CPU! "
2423 "EXC_RESOURCE & termination supressed due to audio playback\n",
2424 procname, pid, tid);
2425 return;
2426 }
2427 }
2428
2429
2430 if (send_exc_resource) {
2431 code[0] = code[1] = 0;
2432 EXC_RESOURCE_ENCODE_TYPE(code[0], RESOURCE_TYPE_CPU);
2433 if (fatal) {
2434 EXC_RESOURCE_ENCODE_FLAVOR(code[0], FLAVOR_CPU_MONITOR_FATAL);
2435 } else {
2436 EXC_RESOURCE_ENCODE_FLAVOR(code[0], FLAVOR_CPU_MONITOR);
2437 }
2438 EXC_RESOURCE_CPUMONITOR_ENCODE_INTERVAL(code[0], interval_sec);
2439 EXC_RESOURCE_CPUMONITOR_ENCODE_PERCENTAGE(code[0], percentage);
2440 EXC_RESOURCE_CPUMONITOR_ENCODE_PERCENTAGE(code[1], usage_percent);
2441 exception_triage(EXC_RESOURCE, code, EXCEPTION_CODE_MAX);
2442 }
2443 #endif /* EXC_RESOURCE_MONITORS */
2444
2445 if (fatal) {
2446 #if CONFIG_JETSAM
2447 jetsam_on_ledger_cpulimit_exceeded();
2448 #else
2449 task_terminate_internal(task);
2450 #endif
2451 }
2452 }
2453
2454 #if DEVELOPMENT || DEBUG
2455 void __attribute__((noinline))
2456 SENDING_NOTIFICATION__TASK_HAS_TOO_MANY_THREADS(task_t task, int thread_count)
2457 {
2458 mach_exception_data_type_t code[EXCEPTION_CODE_MAX] = {0};
2459 int pid = task_pid(task);
2460 char procname[MAXCOMLEN + 1] = "unknown";
2461
2462 if (pid == 1) {
2463 /*
2464 * Cannot suspend launchd
2465 */
2466 return;
2467 }
2468
2469 proc_name(pid, procname, sizeof(procname));
2470
2471 if (disable_exc_resource) {
2472 printf("process %s[%d] crossed thread count high watermark (%d), EXC_RESOURCE "
2473 "supressed by a boot-arg. \n", procname, pid, thread_count);
2474 return;
2475 }
2476
2477 if (audio_active) {
2478 printf("process %s[%d] crossed thread count high watermark (%d), EXC_RESOURCE "
2479 "supressed due to audio playback.\n", procname, pid, thread_count);
2480 return;
2481 }
2482
2483 if (exc_via_corpse_forking == 0) {
2484 printf("process %s[%d] crossed thread count high watermark (%d), EXC_RESOURCE "
2485 "supressed due to corpse forking being disabled.\n", procname, pid,
2486 thread_count);
2487 return;
2488 }
2489
2490 printf("process %s[%d] crossed thread count high watermark (%d), sending "
2491 "EXC_RESOURCE\n", procname, pid, thread_count);
2492
2493 EXC_RESOURCE_ENCODE_TYPE(code[0], RESOURCE_TYPE_THREADS);
2494 EXC_RESOURCE_ENCODE_FLAVOR(code[0], FLAVOR_THREADS_HIGH_WATERMARK);
2495 EXC_RESOURCE_THREADS_ENCODE_THREADS(code[0], thread_count);
2496
2497 task_enqueue_exception_with_corpse(task, EXC_RESOURCE, code, EXCEPTION_CODE_MAX, NULL);
2498 }
2499 #endif /* DEVELOPMENT || DEBUG */
2500
2501 void
2502 thread_update_io_stats(thread_t thread, int size, int io_flags)
2503 {
2504 int io_tier;
2505
2506 if (thread->thread_io_stats == NULL || thread->task->task_io_stats == NULL) {
2507 return;
2508 }
2509
2510 if (io_flags & DKIO_READ) {
2511 UPDATE_IO_STATS(thread->thread_io_stats->disk_reads, size);
2512 UPDATE_IO_STATS_ATOMIC(thread->task->task_io_stats->disk_reads, size);
2513 }
2514
2515 if (io_flags & DKIO_META) {
2516 UPDATE_IO_STATS(thread->thread_io_stats->metadata, size);
2517 UPDATE_IO_STATS_ATOMIC(thread->task->task_io_stats->metadata, size);
2518 }
2519
2520 if (io_flags & DKIO_PAGING) {
2521 UPDATE_IO_STATS(thread->thread_io_stats->paging, size);
2522 UPDATE_IO_STATS_ATOMIC(thread->task->task_io_stats->paging, size);
2523 }
2524
2525 io_tier = ((io_flags & DKIO_TIER_MASK) >> DKIO_TIER_SHIFT);
2526 assert(io_tier < IO_NUM_PRIORITIES);
2527
2528 UPDATE_IO_STATS(thread->thread_io_stats->io_priority[io_tier], size);
2529 UPDATE_IO_STATS_ATOMIC(thread->task->task_io_stats->io_priority[io_tier], size);
2530
2531 /* Update Total I/O Counts */
2532 UPDATE_IO_STATS(thread->thread_io_stats->total_io, size);
2533 UPDATE_IO_STATS_ATOMIC(thread->task->task_io_stats->total_io, size);
2534
2535 if (!(io_flags & DKIO_READ)) {
2536 DTRACE_IO3(physical_writes, struct task *, thread->task, uint32_t, size, int, io_flags);
2537 ledger_credit(thread->task->ledger, task_ledgers.physical_writes, size);
2538 }
2539 }
2540
2541 static void
2542 init_thread_ledgers(void)
2543 {
2544 ledger_template_t t;
2545 int idx;
2546
2547 assert(thread_ledger_template == NULL);
2548
2549 if ((t = ledger_template_create("Per-thread ledger")) == NULL) {
2550 panic("couldn't create thread ledger template");
2551 }
2552
2553 if ((idx = ledger_entry_add(t, "cpu_time", "sched", "ns")) < 0) {
2554 panic("couldn't create cpu_time entry for thread ledger template");
2555 }
2556
2557 if (ledger_set_callback(t, idx, thread_cputime_callback, NULL, NULL) < 0) {
2558 panic("couldn't set thread ledger callback for cpu_time entry");
2559 }
2560
2561 thread_ledgers.cpu_time = idx;
2562
2563 ledger_template_complete(t);
2564 thread_ledger_template = t;
2565 }
2566
2567 /*
2568 * Returns currently applied CPU usage limit, or 0/0 if none is applied.
2569 */
2570 int
2571 thread_get_cpulimit(int *action, uint8_t *percentage, uint64_t *interval_ns)
2572 {
2573 int64_t abstime = 0;
2574 uint64_t limittime = 0;
2575 thread_t thread = current_thread();
2576
2577 *percentage = 0;
2578 *interval_ns = 0;
2579 *action = 0;
2580
2581 if (thread->t_threadledger == LEDGER_NULL) {
2582 /*
2583 * This thread has no per-thread ledger, so it can't possibly
2584 * have a CPU limit applied.
2585 */
2586 return KERN_SUCCESS;
2587 }
2588
2589 ledger_get_period(thread->t_threadledger, thread_ledgers.cpu_time, interval_ns);
2590 ledger_get_limit(thread->t_threadledger, thread_ledgers.cpu_time, &abstime);
2591
2592 if ((abstime == LEDGER_LIMIT_INFINITY) || (*interval_ns == 0)) {
2593 /*
2594 * This thread's CPU time ledger has no period or limit; so it
2595 * doesn't have a CPU limit applied.
2596 */
2597 return KERN_SUCCESS;
2598 }
2599
2600 /*
2601 * This calculation is the converse to the one in thread_set_cpulimit().
2602 */
2603 absolutetime_to_nanoseconds(abstime, &limittime);
2604 *percentage = (limittime * 100ULL) / *interval_ns;
2605 assert(*percentage <= 100);
2606
2607 if (thread->options & TH_OPT_PROC_CPULIMIT) {
2608 assert((thread->options & TH_OPT_PRVT_CPULIMIT) == 0);
2609
2610 *action = THREAD_CPULIMIT_BLOCK;
2611 } else if (thread->options & TH_OPT_PRVT_CPULIMIT) {
2612 assert((thread->options & TH_OPT_PROC_CPULIMIT) == 0);
2613
2614 *action = THREAD_CPULIMIT_EXCEPTION;
2615 } else {
2616 *action = THREAD_CPULIMIT_DISABLE;
2617 }
2618
2619 return KERN_SUCCESS;
2620 }
2621
2622 /*
2623 * Set CPU usage limit on a thread.
2624 *
2625 * Calling with percentage of 0 will unset the limit for this thread.
2626 */
2627 int
2628 thread_set_cpulimit(int action, uint8_t percentage, uint64_t interval_ns)
2629 {
2630 thread_t thread = current_thread();
2631 ledger_t l;
2632 uint64_t limittime = 0;
2633 uint64_t abstime = 0;
2634
2635 assert(percentage <= 100);
2636
2637 if (action == THREAD_CPULIMIT_DISABLE) {
2638 /*
2639 * Remove CPU limit, if any exists.
2640 */
2641 if (thread->t_threadledger != LEDGER_NULL) {
2642 l = thread->t_threadledger;
2643 ledger_set_limit(l, thread_ledgers.cpu_time, LEDGER_LIMIT_INFINITY, 0);
2644 ledger_set_action(l, thread_ledgers.cpu_time, LEDGER_ACTION_IGNORE);
2645 thread->options &= ~(TH_OPT_PROC_CPULIMIT | TH_OPT_PRVT_CPULIMIT);
2646 }
2647
2648 return 0;
2649 }
2650
2651 if (interval_ns < MINIMUM_CPULIMIT_INTERVAL_MS * NSEC_PER_MSEC) {
2652 return KERN_INVALID_ARGUMENT;
2653 }
2654
2655 l = thread->t_threadledger;
2656 if (l == LEDGER_NULL) {
2657 /*
2658 * This thread doesn't yet have a per-thread ledger; so create one with the CPU time entry active.
2659 */
2660 if ((l = ledger_instantiate(thread_ledger_template, LEDGER_CREATE_INACTIVE_ENTRIES)) == LEDGER_NULL) {
2661 return KERN_RESOURCE_SHORTAGE;
2662 }
2663
2664 /*
2665 * We are the first to create this thread's ledger, so only activate our entry.
2666 */
2667 ledger_entry_setactive(l, thread_ledgers.cpu_time);
2668 thread->t_threadledger = l;
2669 }
2670
2671 /*
2672 * The limit is specified as a percentage of CPU over an interval in nanoseconds.
2673 * Calculate the amount of CPU time that the thread needs to consume in order to hit the limit.
2674 */
2675 limittime = (interval_ns * percentage) / 100;
2676 nanoseconds_to_absolutetime(limittime, &abstime);
2677 ledger_set_limit(l, thread_ledgers.cpu_time, abstime, cpumon_ustackshots_trigger_pct);
2678 /*
2679 * Refill the thread's allotted CPU time every interval_ns nanoseconds.
2680 */
2681 ledger_set_period(l, thread_ledgers.cpu_time, interval_ns);
2682
2683 if (action == THREAD_CPULIMIT_EXCEPTION) {
2684 /*
2685 * We don't support programming the CPU usage monitor on a task if any of its
2686 * threads have a per-thread blocking CPU limit configured.
2687 */
2688 if (thread->options & TH_OPT_PRVT_CPULIMIT) {
2689 panic("CPU usage monitor activated, but blocking thread limit exists");
2690 }
2691
2692 /*
2693 * Make a note that this thread's CPU limit is being used for the task-wide CPU
2694 * usage monitor. We don't have to arm the callback which will trigger the
2695 * exception, because that was done for us in ledger_instantiate (because the
2696 * ledger template used has a default callback).
2697 */
2698 thread->options |= TH_OPT_PROC_CPULIMIT;
2699 } else {
2700 /*
2701 * We deliberately override any CPU limit imposed by a task-wide limit (eg
2702 * CPU usage monitor).
2703 */
2704 thread->options &= ~TH_OPT_PROC_CPULIMIT;
2705
2706 thread->options |= TH_OPT_PRVT_CPULIMIT;
2707 /* The per-thread ledger template by default has a callback for CPU time */
2708 ledger_disable_callback(l, thread_ledgers.cpu_time);
2709 ledger_set_action(l, thread_ledgers.cpu_time, LEDGER_ACTION_BLOCK);
2710 }
2711
2712 return 0;
2713 }
2714
2715 void
2716 thread_sched_call(
2717 thread_t thread,
2718 sched_call_t call)
2719 {
2720 assert((thread->state & TH_WAIT_REPORT) == 0);
2721 thread->sched_call = call;
2722 }
2723
2724 uint64_t
2725 thread_tid(
2726 thread_t thread)
2727 {
2728 return thread != THREAD_NULL? thread->thread_id: 0;
2729 }
2730
2731 uint16_t
2732 thread_set_tag(thread_t th, uint16_t tag)
2733 {
2734 return thread_set_tag_internal(th, tag);
2735 }
2736
2737 uint16_t
2738 thread_get_tag(thread_t th)
2739 {
2740 return thread_get_tag_internal(th);
2741 }
2742
2743 uint64_t
2744 thread_last_run_time(thread_t th)
2745 {
2746 return th->last_run_time;
2747 }
2748
2749 uint64_t
2750 thread_dispatchqaddr(
2751 thread_t thread)
2752 {
2753 uint64_t dispatchqueue_addr;
2754 uint64_t thread_handle;
2755
2756 if (thread == THREAD_NULL) {
2757 return 0;
2758 }
2759
2760 thread_handle = thread->machine.cthread_self;
2761 if (thread_handle == 0) {
2762 return 0;
2763 }
2764
2765 if (thread->inspection == TRUE) {
2766 dispatchqueue_addr = thread_handle + get_task_dispatchqueue_offset(thread->task);
2767 } else if (thread->task->bsd_info) {
2768 dispatchqueue_addr = thread_handle + get_dispatchqueue_offset_from_proc(thread->task->bsd_info);
2769 } else {
2770 dispatchqueue_addr = 0;
2771 }
2772
2773 return dispatchqueue_addr;
2774 }
2775
2776 uint64_t
2777 thread_rettokern_addr(
2778 thread_t thread)
2779 {
2780 uint64_t rettokern_addr;
2781 uint64_t rettokern_offset;
2782 uint64_t thread_handle;
2783
2784 if (thread == THREAD_NULL) {
2785 return 0;
2786 }
2787
2788 thread_handle = thread->machine.cthread_self;
2789 if (thread_handle == 0) {
2790 return 0;
2791 }
2792
2793 if (thread->task->bsd_info) {
2794 rettokern_offset = get_return_to_kernel_offset_from_proc(thread->task->bsd_info);
2795
2796 /* Return 0 if return to kernel offset is not initialized. */
2797 if (rettokern_offset == 0) {
2798 rettokern_addr = 0;
2799 } else {
2800 rettokern_addr = thread_handle + rettokern_offset;
2801 }
2802 } else {
2803 rettokern_addr = 0;
2804 }
2805
2806 return rettokern_addr;
2807 }
2808
2809 /*
2810 * Export routines to other components for things that are done as macros
2811 * within the osfmk component.
2812 */
2813
2814 #undef thread_mtx_lock
2815 void thread_mtx_lock(thread_t thread);
2816 void
2817 thread_mtx_lock(thread_t thread)
2818 {
2819 lck_mtx_lock(&thread->mutex);
2820 }
2821
2822 #undef thread_mtx_unlock
2823 void thread_mtx_unlock(thread_t thread);
2824 void
2825 thread_mtx_unlock(thread_t thread)
2826 {
2827 lck_mtx_unlock(&thread->mutex);
2828 }
2829
2830 #undef thread_reference
2831 void thread_reference(thread_t thread);
2832 void
2833 thread_reference(
2834 thread_t thread)
2835 {
2836 if (thread != THREAD_NULL) {
2837 thread_reference_internal(thread);
2838 }
2839 }
2840
2841 #undef thread_should_halt
2842
2843 boolean_t
2844 thread_should_halt(
2845 thread_t th)
2846 {
2847 return thread_should_halt_fast(th);
2848 }
2849
2850 /*
2851 * thread_set_voucher_name - reset the voucher port name bound to this thread
2852 *
2853 * Conditions: nothing locked
2854 */
2855
2856 kern_return_t
2857 thread_set_voucher_name(mach_port_name_t voucher_name)
2858 {
2859 thread_t thread = current_thread();
2860 ipc_voucher_t new_voucher = IPC_VOUCHER_NULL;
2861 ipc_voucher_t voucher;
2862 ledger_t bankledger = NULL;
2863 struct thread_group *banktg = NULL;
2864 uint32_t persona_id = 0;
2865
2866 if (MACH_PORT_DEAD == voucher_name) {
2867 return KERN_INVALID_RIGHT;
2868 }
2869
2870 /*
2871 * agressively convert to voucher reference
2872 */
2873 if (MACH_PORT_VALID(voucher_name)) {
2874 new_voucher = convert_port_name_to_voucher(voucher_name);
2875 if (IPC_VOUCHER_NULL == new_voucher) {
2876 return KERN_INVALID_ARGUMENT;
2877 }
2878 }
2879 bank_get_bank_ledger_thread_group_and_persona(new_voucher, &bankledger, &banktg, &persona_id);
2880
2881 thread_mtx_lock(thread);
2882 voucher = thread->ith_voucher;
2883 thread->ith_voucher_name = voucher_name;
2884 thread->ith_voucher = new_voucher;
2885 thread_mtx_unlock(thread);
2886
2887 bank_swap_thread_bank_ledger(thread, bankledger);
2888
2889 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
2890 MACHDBG_CODE(DBG_MACH_IPC, MACH_THREAD_SET_VOUCHER) | DBG_FUNC_NONE,
2891 (uintptr_t)thread_tid(thread),
2892 (uintptr_t)voucher_name,
2893 VM_KERNEL_ADDRPERM((uintptr_t)new_voucher),
2894 persona_id, 0);
2895
2896 if (IPC_VOUCHER_NULL != voucher) {
2897 ipc_voucher_release(voucher);
2898 }
2899
2900 return KERN_SUCCESS;
2901 }
2902
2903 /*
2904 * thread_get_mach_voucher - return a voucher reference for the specified thread voucher
2905 *
2906 * Conditions: nothing locked
2907 *
2908 * NOTE: At the moment, there is no distinction between the current and effective
2909 * vouchers because we only set them at the thread level currently.
2910 */
2911 kern_return_t
2912 thread_get_mach_voucher(
2913 thread_act_t thread,
2914 mach_voucher_selector_t __unused which,
2915 ipc_voucher_t *voucherp)
2916 {
2917 ipc_voucher_t voucher;
2918
2919 if (THREAD_NULL == thread) {
2920 return KERN_INVALID_ARGUMENT;
2921 }
2922
2923 thread_mtx_lock(thread);
2924 voucher = thread->ith_voucher;
2925
2926 if (IPC_VOUCHER_NULL != voucher) {
2927 ipc_voucher_reference(voucher);
2928 thread_mtx_unlock(thread);
2929 *voucherp = voucher;
2930 return KERN_SUCCESS;
2931 }
2932
2933 thread_mtx_unlock(thread);
2934
2935 *voucherp = IPC_VOUCHER_NULL;
2936 return KERN_SUCCESS;
2937 }
2938
2939 /*
2940 * thread_set_mach_voucher - set a voucher reference for the specified thread voucher
2941 *
2942 * Conditions: callers holds a reference on the voucher.
2943 * nothing locked.
2944 *
2945 * We grab another reference to the voucher and bind it to the thread.
2946 * The old voucher reference associated with the thread is
2947 * discarded.
2948 */
2949 kern_return_t
2950 thread_set_mach_voucher(
2951 thread_t thread,
2952 ipc_voucher_t voucher)
2953 {
2954 ipc_voucher_t old_voucher;
2955 ledger_t bankledger = NULL;
2956 struct thread_group *banktg = NULL;
2957 uint32_t persona_id = 0;
2958
2959 if (THREAD_NULL == thread) {
2960 return KERN_INVALID_ARGUMENT;
2961 }
2962
2963 if (thread != current_thread() && thread->started) {
2964 return KERN_INVALID_ARGUMENT;
2965 }
2966
2967 ipc_voucher_reference(voucher);
2968 bank_get_bank_ledger_thread_group_and_persona(voucher, &bankledger, &banktg, &persona_id);
2969
2970 thread_mtx_lock(thread);
2971 old_voucher = thread->ith_voucher;
2972 thread->ith_voucher = voucher;
2973 thread->ith_voucher_name = MACH_PORT_NULL;
2974 thread_mtx_unlock(thread);
2975
2976 bank_swap_thread_bank_ledger(thread, bankledger);
2977
2978 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
2979 MACHDBG_CODE(DBG_MACH_IPC, MACH_THREAD_SET_VOUCHER) | DBG_FUNC_NONE,
2980 (uintptr_t)thread_tid(thread),
2981 (uintptr_t)MACH_PORT_NULL,
2982 VM_KERNEL_ADDRPERM((uintptr_t)voucher),
2983 persona_id, 0);
2984
2985 ipc_voucher_release(old_voucher);
2986
2987 return KERN_SUCCESS;
2988 }
2989
2990 /*
2991 * thread_swap_mach_voucher - swap a voucher reference for the specified thread voucher
2992 *
2993 * Conditions: callers holds a reference on the new and presumed old voucher(s).
2994 * nothing locked.
2995 *
2996 * This function is no longer supported.
2997 */
2998 kern_return_t
2999 thread_swap_mach_voucher(
3000 __unused thread_t thread,
3001 __unused ipc_voucher_t new_voucher,
3002 ipc_voucher_t *in_out_old_voucher)
3003 {
3004 /*
3005 * Currently this function is only called from a MIG generated
3006 * routine which doesn't release the reference on the voucher
3007 * addressed by in_out_old_voucher. To avoid leaking this reference,
3008 * a call to release it has been added here.
3009 */
3010 ipc_voucher_release(*in_out_old_voucher);
3011 return KERN_NOT_SUPPORTED;
3012 }
3013
3014 /*
3015 * thread_get_current_voucher_origin_pid - get the pid of the originator of the current voucher.
3016 */
3017 kern_return_t
3018 thread_get_current_voucher_origin_pid(
3019 int32_t *pid)
3020 {
3021 uint32_t buf_size;
3022 kern_return_t kr;
3023 thread_t thread = current_thread();
3024
3025 buf_size = sizeof(*pid);
3026 kr = mach_voucher_attr_command(thread->ith_voucher,
3027 MACH_VOUCHER_ATTR_KEY_BANK,
3028 BANK_ORIGINATOR_PID,
3029 NULL,
3030 0,
3031 (mach_voucher_attr_content_t)pid,
3032 &buf_size);
3033
3034 return kr;
3035 }
3036
3037
3038 boolean_t
3039 thread_has_thread_name(thread_t th)
3040 {
3041 if ((th) && (th->uthread)) {
3042 return bsd_hasthreadname(th->uthread);
3043 }
3044
3045 /*
3046 * This is an odd case; clients may set the thread name based on the lack of
3047 * a name, but in this context there is no uthread to attach the name to.
3048 */
3049 return FALSE;
3050 }
3051
3052 void
3053 thread_set_thread_name(thread_t th, const char* name)
3054 {
3055 if ((th) && (th->uthread) && name) {
3056 bsd_setthreadname(th->uthread, name);
3057 }
3058 }
3059
3060 void
3061 thread_set_honor_qlimit(thread_t thread)
3062 {
3063 thread->options |= TH_OPT_HONOR_QLIMIT;
3064 }
3065
3066 void
3067 thread_clear_honor_qlimit(thread_t thread)
3068 {
3069 thread->options &= (~TH_OPT_HONOR_QLIMIT);
3070 }
3071
3072 /*
3073 * thread_enable_send_importance - set/clear the SEND_IMPORTANCE thread option bit.
3074 */
3075 void
3076 thread_enable_send_importance(thread_t thread, boolean_t enable)
3077 {
3078 if (enable == TRUE) {
3079 thread->options |= TH_OPT_SEND_IMPORTANCE;
3080 } else {
3081 thread->options &= ~TH_OPT_SEND_IMPORTANCE;
3082 }
3083 }
3084
3085 /*
3086 * thread_set_allocation_name - .
3087 */
3088
3089 kern_allocation_name_t
3090 thread_set_allocation_name(kern_allocation_name_t new_name)
3091 {
3092 kern_allocation_name_t ret;
3093 thread_kernel_state_t kstate = thread_get_kernel_state(current_thread());
3094 ret = kstate->allocation_name;
3095 // fifo
3096 if (!new_name || !kstate->allocation_name) {
3097 kstate->allocation_name = new_name;
3098 }
3099 return ret;
3100 }
3101
3102 void *
3103 thread_iokit_tls_get(uint32_t index)
3104 {
3105 assert(index < THREAD_SAVE_IOKIT_TLS_COUNT);
3106 return current_thread()->saved.iokit.tls[index];
3107 }
3108
3109 void
3110 thread_iokit_tls_set(uint32_t index, void * data)
3111 {
3112 assert(index < THREAD_SAVE_IOKIT_TLS_COUNT);
3113 current_thread()->saved.iokit.tls[index] = data;
3114 }
3115
3116 uint64_t
3117 thread_get_last_wait_duration(thread_t thread)
3118 {
3119 return thread->last_made_runnable_time - thread->last_run_time;
3120 }
3121
3122 integer_t
3123 thread_kern_get_pri(thread_t thr)
3124 {
3125 return thr->base_pri;
3126 }
3127
3128 void
3129 thread_kern_set_pri(thread_t thr, integer_t pri)
3130 {
3131 sched_set_kernel_thread_priority(thr, pri);
3132 }
3133
3134 integer_t
3135 thread_kern_get_kernel_maxpri(void)
3136 {
3137 return MAXPRI_KERNEL;
3138 }
3139
3140 #if CONFIG_DTRACE
3141 uint32_t
3142 dtrace_get_thread_predcache(thread_t thread)
3143 {
3144 if (thread != THREAD_NULL) {
3145 return thread->t_dtrace_predcache;
3146 } else {
3147 return 0;
3148 }
3149 }
3150
3151 int64_t
3152 dtrace_get_thread_vtime(thread_t thread)
3153 {
3154 if (thread != THREAD_NULL) {
3155 return thread->t_dtrace_vtime;
3156 } else {
3157 return 0;
3158 }
3159 }
3160
3161 int
3162 dtrace_get_thread_last_cpu_id(thread_t thread)
3163 {
3164 if ((thread != THREAD_NULL) && (thread->last_processor != PROCESSOR_NULL)) {
3165 return thread->last_processor->cpu_id;
3166 } else {
3167 return -1;
3168 }
3169 }
3170
3171 int64_t
3172 dtrace_get_thread_tracing(thread_t thread)
3173 {
3174 if (thread != THREAD_NULL) {
3175 return thread->t_dtrace_tracing;
3176 } else {
3177 return 0;
3178 }
3179 }
3180
3181 uint16_t
3182 dtrace_get_thread_inprobe(thread_t thread)
3183 {
3184 if (thread != THREAD_NULL) {
3185 return thread->t_dtrace_inprobe;
3186 } else {
3187 return 0;
3188 }
3189 }
3190
3191 vm_offset_t
3192 dtrace_get_kernel_stack(thread_t thread)
3193 {
3194 if (thread != THREAD_NULL) {
3195 return thread->kernel_stack;
3196 } else {
3197 return 0;
3198 }
3199 }
3200
3201 #if KASAN
3202 struct kasan_thread_data *
3203 kasan_get_thread_data(thread_t thread)
3204 {
3205 return &thread->kasan_data;
3206 }
3207 #endif
3208
3209 #if CONFIG_KSANCOV
3210 void **
3211 __sanitizer_get_thread_data(thread_t thread)
3212 {
3213 return &thread->ksancov_data;
3214 }
3215 #endif
3216
3217 int64_t
3218 dtrace_calc_thread_recent_vtime(thread_t thread)
3219 {
3220 if (thread != THREAD_NULL) {
3221 processor_t processor = current_processor();
3222 uint64_t abstime = mach_absolute_time();
3223 timer_t timer;
3224
3225 timer = PROCESSOR_DATA(processor, thread_timer);
3226
3227 return timer_grab(&(thread->system_timer)) + timer_grab(&(thread->user_timer)) +
3228 (abstime - timer->tstamp); /* XXX need interrupts off to prevent missed time? */
3229 } else {
3230 return 0;
3231 }
3232 }
3233
3234 void
3235 dtrace_set_thread_predcache(thread_t thread, uint32_t predcache)
3236 {
3237 if (thread != THREAD_NULL) {
3238 thread->t_dtrace_predcache = predcache;
3239 }
3240 }
3241
3242 void
3243 dtrace_set_thread_vtime(thread_t thread, int64_t vtime)
3244 {
3245 if (thread != THREAD_NULL) {
3246 thread->t_dtrace_vtime = vtime;
3247 }
3248 }
3249
3250 void
3251 dtrace_set_thread_tracing(thread_t thread, int64_t accum)
3252 {
3253 if (thread != THREAD_NULL) {
3254 thread->t_dtrace_tracing = accum;
3255 }
3256 }
3257
3258 void
3259 dtrace_set_thread_inprobe(thread_t thread, uint16_t inprobe)
3260 {
3261 if (thread != THREAD_NULL) {
3262 thread->t_dtrace_inprobe = inprobe;
3263 }
3264 }
3265
3266 vm_offset_t
3267 dtrace_set_thread_recover(thread_t thread, vm_offset_t recover)
3268 {
3269 vm_offset_t prev = 0;
3270
3271 if (thread != THREAD_NULL) {
3272 prev = thread->recover;
3273 thread->recover = recover;
3274 }
3275 return prev;
3276 }
3277
3278 vm_offset_t
3279 dtrace_sign_and_set_thread_recover(thread_t thread, vm_offset_t recover)
3280 {
3281 #if defined(HAS_APPLE_PAC)
3282 return dtrace_set_thread_recover(thread,
3283 (vm_address_t)ptrauth_sign_unauthenticated((void *)recover,
3284 ptrauth_key_function_pointer,
3285 ptrauth_blend_discriminator(&thread->recover, PAC_DISCRIMINATOR_RECOVER)));
3286 #else /* defined(HAS_APPLE_PAC) */
3287 return dtrace_set_thread_recover(thread, recover);
3288 #endif /* defined(HAS_APPLE_PAC) */
3289 }
3290
3291 void
3292 dtrace_thread_bootstrap(void)
3293 {
3294 task_t task = current_task();
3295
3296 if (task->thread_count == 1) {
3297 thread_t thread = current_thread();
3298 if (thread->t_dtrace_flags & TH_DTRACE_EXECSUCCESS) {
3299 thread->t_dtrace_flags &= ~TH_DTRACE_EXECSUCCESS;
3300 DTRACE_PROC(exec__success);
3301 KDBG(BSDDBG_CODE(DBG_BSD_PROC, BSD_PROC_EXEC),
3302 task_pid(task));
3303 }
3304 DTRACE_PROC(start);
3305 }
3306 DTRACE_PROC(lwp__start);
3307 }
3308
3309 void
3310 dtrace_thread_didexec(thread_t thread)
3311 {
3312 thread->t_dtrace_flags |= TH_DTRACE_EXECSUCCESS;
3313 }
3314 #endif /* CONFIG_DTRACE */
mirror of XNU