]> git.saurik.com Git - apple/xnu.git/blob - osfmk/kern/thread.c
projects / apple / xnu.git / blob
? search:


8b7d863b7a8e08f0ef4c867b4ce38e567009775d
[apple/xnu.git] / osfmk / kern / thread.c
1 /*
2 * Copyright (c) 2000-2014 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_FREE_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56 /*
57 */
58 /*
59 * File: kern/thread.c
60 * Author: Avadis Tevanian, Jr., Michael Wayne Young, David Golub
61 * Date: 1986
62 *
63 * Thread management primitives implementation.
64 */
65 /*
66 * Copyright (c) 1993 The University of Utah and
67 * the Computer Systems Laboratory (CSL). All rights reserved.
68 *
69 * Permission to use, copy, modify and distribute this software and its
70 * documentation is hereby granted, provided that both the copyright
71 * notice and this permission notice appear in all copies of the
72 * software, derivative works or modified versions, and any portions
73 * thereof, and that both notices appear in supporting documentation.
74 *
75 * THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF THIS SOFTWARE IN ITS "AS
76 * IS" CONDITION. THE UNIVERSITY OF UTAH AND CSL DISCLAIM ANY LIABILITY OF
77 * ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
78 *
79 * CSL requests users of this software to return to csl-dist@cs.utah.edu any
80 * improvements that they make and grant CSL redistribution rights.
81 *
82 */
83
84 #include <mach/mach_types.h>
85 #include <mach/boolean.h>
86 #include <mach/policy.h>
87 #include <mach/thread_info.h>
88 #include <mach/thread_special_ports.h>
89 #include <mach/thread_status.h>
90 #include <mach/time_value.h>
91 #include <mach/vm_param.h>
92
93 #include <machine/thread.h>
94 #include <machine/pal_routines.h>
95 #include <machine/limits.h>
96
97 #include <kern/kern_types.h>
98 #include <kern/kalloc.h>
99 #include <kern/cpu_data.h>
100 #include <kern/counters.h>
101 #include <kern/extmod_statistics.h>
102 #include <kern/ipc_mig.h>
103 #include <kern/ipc_tt.h>
104 #include <kern/mach_param.h>
105 #include <kern/machine.h>
106 #include <kern/misc_protos.h>
107 #include <kern/processor.h>
108 #include <kern/queue.h>
109 #include <kern/sched.h>
110 #include <kern/sched_prim.h>
111 #include <kern/sync_lock.h>
112 #include <kern/syscall_subr.h>
113 #include <kern/task.h>
114 #include <kern/thread.h>
115 #include <kern/host.h>
116 #include <kern/zalloc.h>
117 #include <kern/assert.h>
118 #include <kern/exc_resource.h>
119 #include <kern/telemetry.h>
120 #if KPC
121 #include <kern/kpc.h>
122 #endif
123
124 #include <ipc/ipc_kmsg.h>
125 #include <ipc/ipc_port.h>
126 #include <bank/bank_types.h>
127
128 #include <vm/vm_kern.h>
129 #include <vm/vm_pageout.h>
130
131 #include <sys/kdebug.h>
132
133 #include <mach/sdt.h>
134
135 /*
136 * Exported interfaces
137 */
138 #include <mach/task_server.h>
139 #include <mach/thread_act_server.h>
140 #include <mach/mach_host_server.h>
141 #include <mach/host_priv_server.h>
142 #include <mach/mach_voucher_server.h>
143
144 static struct zone *thread_zone;
145 static lck_grp_attr_t thread_lck_grp_attr;
146 lck_attr_t thread_lck_attr;
147 lck_grp_t thread_lck_grp;
148
149 struct zone *thread_qos_override_zone;
150
151 decl_simple_lock_data(static,thread_stack_lock)
152 static queue_head_t thread_stack_queue;
153
154 decl_simple_lock_data(static,thread_terminate_lock)
155 static queue_head_t thread_terminate_queue;
156
157 static struct thread thread_template, init_thread;
158
159 static void sched_call_null(
160 int type,
161 thread_t thread);
162
163 #ifdef MACH_BSD
164 extern void proc_exit(void *);
165 extern uint64_t get_dispatchqueue_offset_from_proc(void *);
166 extern int proc_selfpid(void);
167 extern char * proc_name_address(void *p);
168 #endif /* MACH_BSD */
169
170 extern int disable_exc_resource;
171 extern int audio_active;
172 extern int debug_task;
173 int thread_max = CONFIG_THREAD_MAX; /* Max number of threads */
174 int task_threadmax = CONFIG_THREAD_MAX;
175
176 static uint64_t thread_unique_id = 100;
177
178 struct _thread_ledger_indices thread_ledgers = { -1 };
179 static ledger_template_t thread_ledger_template = NULL;
180 void init_thread_ledgers(void);
181 int task_disable_cpumon(task_t task);
182
183 #if CONFIG_JETSAM
184 void jetsam_on_ledger_cpulimit_exceeded(void);
185 #endif
186
187 /*
188 * Level (in terms of percentage of the limit) at which the CPU usage monitor triggers telemetry.
189 *
190 * (ie when any thread's CPU consumption exceeds 70% of the limit, start taking user
191 * stacktraces, aka micro-stackshots)
192 */
193 #define CPUMON_USTACKSHOTS_TRIGGER_DEFAULT_PCT 70
194
195 int cpumon_ustackshots_trigger_pct; /* Percentage. Level at which we start gathering telemetry. */
196 void __attribute__((noinline)) THIS_THREAD_IS_CONSUMING_TOO_MUCH_CPU__SENDING_EXC_RESOURCE(void);
197
198 /*
199 * The smallest interval over which we support limiting CPU consumption is 1ms
200 */
201 #define MINIMUM_CPULIMIT_INTERVAL_MS 1
202
203 void
204 thread_bootstrap(void)
205 {
206 /*
207 * Fill in a template thread for fast initialization.
208 */
209
210 thread_template.runq = PROCESSOR_NULL;
211
212 thread_template.ref_count = 2;
213
214 thread_template.reason = AST_NONE;
215 thread_template.at_safe_point = FALSE;
216 thread_template.wait_event = NO_EVENT64;
217 thread_template.wait_queue = WAIT_QUEUE_NULL;
218 thread_template.wait_result = THREAD_WAITING;
219 thread_template.options = THREAD_ABORTSAFE;
220 thread_template.state = TH_WAIT | TH_UNINT;
221 thread_template.wake_active = FALSE;
222 thread_template.continuation = THREAD_CONTINUE_NULL;
223 thread_template.parameter = NULL;
224
225 thread_template.importance = 0;
226 thread_template.sched_mode = TH_MODE_NONE;
227 thread_template.sched_flags = 0;
228 thread_template.saved_mode = TH_MODE_NONE;
229 thread_template.safe_release = 0;
230
231 thread_template.sfi_class = SFI_CLASS_UNSPECIFIED;
232 thread_template.sfi_wait_class = SFI_CLASS_UNSPECIFIED;
233
234 thread_template.active = 0;
235 thread_template.started = 0;
236 thread_template.static_param = 0;
237 thread_template.policy_reset = 0;
238
239 thread_template.priority = 0;
240 thread_template.sched_pri = 0;
241 thread_template.max_priority = 0;
242 thread_template.task_priority = 0;
243 thread_template.promotions = 0;
244 thread_template.pending_promoter_index = 0;
245 thread_template.pending_promoter[0] =
246 thread_template.pending_promoter[1] = NULL;
247 thread_template.rwlock_count = 0;
248
249 #if MACH_ASSERT
250 thread_template.SHARE_COUNT = 0;
251 thread_template.BG_COUNT = 0;
252 #endif /* MACH_ASSERT */
253
254 thread_template.realtime.deadline = UINT64_MAX;
255
256 thread_template.quantum_remaining = 0;
257 thread_template.last_run_time = 0;
258
259 thread_template.computation_metered = 0;
260 thread_template.computation_epoch = 0;
261
262 #if defined(CONFIG_SCHED_TIMESHARE_CORE)
263 thread_template.sched_stamp = 0;
264 thread_template.pri_shift = INT8_MAX;
265 thread_template.sched_usage = 0;
266 thread_template.cpu_usage = thread_template.cpu_delta = 0;
267 #endif
268 thread_template.c_switch = thread_template.p_switch = thread_template.ps_switch = 0;
269
270 thread_template.bound_processor = PROCESSOR_NULL;
271 thread_template.last_processor = PROCESSOR_NULL;
272
273 thread_template.sched_call = sched_call_null;
274
275 timer_init(&thread_template.user_timer);
276 timer_init(&thread_template.system_timer);
277 thread_template.user_timer_save = 0;
278 thread_template.system_timer_save = 0;
279 thread_template.vtimer_user_save = 0;
280 thread_template.vtimer_prof_save = 0;
281 thread_template.vtimer_rlim_save = 0;
282
283 thread_template.wait_sfi_begin_time = 0;
284
285 thread_template.wait_timer_is_set = FALSE;
286 thread_template.wait_timer_active = 0;
287
288 thread_template.depress_timer_active = 0;
289
290 thread_template.special_handler.handler = special_handler;
291 thread_template.special_handler.next = NULL;
292
293 thread_template.recover = (vm_offset_t)NULL;
294
295 thread_template.map = VM_MAP_NULL;
296
297 #if CONFIG_DTRACE
298 thread_template.t_dtrace_predcache = 0;
299 thread_template.t_dtrace_vtime = 0;
300 thread_template.t_dtrace_tracing = 0;
301 #endif /* CONFIG_DTRACE */
302
303 #if KPC
304 thread_template.kpc_buf = NULL;
305 #endif
306
307 #if HYPERVISOR
308 thread_template.hv_thread_target = NULL;
309 #endif /* HYPERVISOR */
310
311 thread_template.t_chud = 0;
312
313 #if (DEVELOPMENT || DEBUG)
314 thread_template.t_page_creation_throttled_hard = 0;
315 thread_template.t_page_creation_throttled_soft = 0;
316 #endif /* DEVELOPMENT || DEBUG */
317 thread_template.t_page_creation_throttled = 0;
318 thread_template.t_page_creation_count = 0;
319 thread_template.t_page_creation_time = 0;
320
321 thread_template.affinity_set = NULL;
322
323 thread_template.syscalls_unix = 0;
324 thread_template.syscalls_mach = 0;
325
326 thread_template.t_ledger = LEDGER_NULL;
327 thread_template.t_threadledger = LEDGER_NULL;
328 #ifdef CONFIG_BANK
329 thread_template.t_bankledger = LEDGER_NULL;
330 thread_template.t_deduct_bank_ledger_time = 0;
331 #endif
332
333 thread_template.requested_policy = default_task_requested_policy;
334 thread_template.effective_policy = default_task_effective_policy;
335 thread_template.pended_policy = default_task_pended_policy;
336
337 bzero(&thread_template.overrides, sizeof(thread_template.overrides));
338
339 thread_template.iotier_override = THROTTLE_LEVEL_NONE;
340 thread_template.thread_io_stats = NULL;
341 thread_template.thread_callout_interrupt_wakeups = thread_template.thread_callout_platform_idle_wakeups = 0;
342
343 thread_template.thread_timer_wakeups_bin_1 = thread_template.thread_timer_wakeups_bin_2 = 0;
344 thread_template.callout_woken_from_icontext = thread_template.callout_woken_from_platform_idle = 0;
345
346 thread_template.thread_tag = 0;
347
348 thread_template.ith_voucher_name = MACH_PORT_NULL;
349 thread_template.ith_voucher = IPC_VOUCHER_NULL;
350
351 init_thread = thread_template;
352 machine_set_current_thread(&init_thread);
353 }
354
355 extern boolean_t allow_qos_policy_set;
356
357 void
358 thread_init(void)
359 {
360 thread_zone = zinit(
361 sizeof(struct thread),
362 thread_max * sizeof(struct thread),
363 THREAD_CHUNK * sizeof(struct thread),
364 "threads");
365
366 thread_qos_override_zone = zinit(
367 sizeof(struct thread_qos_override),
368 4 * thread_max * sizeof(struct thread_qos_override),
369 PAGE_SIZE,
370 "thread qos override");
371 zone_change(thread_qos_override_zone, Z_EXPAND, TRUE);
372 zone_change(thread_qos_override_zone, Z_COLLECT, TRUE);
373 zone_change(thread_qos_override_zone, Z_CALLERACCT, FALSE);
374 zone_change(thread_qos_override_zone, Z_NOENCRYPT, TRUE);
375
376 lck_grp_attr_setdefault(&thread_lck_grp_attr);
377 lck_grp_init(&thread_lck_grp, "thread", &thread_lck_grp_attr);
378 lck_attr_setdefault(&thread_lck_attr);
379
380 stack_init();
381
382 /*
383 * Initialize any machine-dependent
384 * per-thread structures necessary.
385 */
386 machine_thread_init();
387
388 if (!PE_parse_boot_argn("cpumon_ustackshots_trigger_pct", &cpumon_ustackshots_trigger_pct,
389 sizeof (cpumon_ustackshots_trigger_pct))) {
390 cpumon_ustackshots_trigger_pct = CPUMON_USTACKSHOTS_TRIGGER_DEFAULT_PCT;
391 }
392
393 PE_parse_boot_argn("-qos-policy-allow", &allow_qos_policy_set, sizeof(allow_qos_policy_set));
394
395 init_thread_ledgers();
396 }
397
398 static void
399 thread_terminate_continue(void)
400 {
401 panic("thread_terminate_continue");
402 /*NOTREACHED*/
403 }
404
405 /*
406 * thread_terminate_self:
407 */
408 void
409 thread_terminate_self(void)
410 {
411 thread_t thread = current_thread();
412
413 task_t task;
414 spl_t s;
415 int threadcnt;
416
417 pal_thread_terminate_self(thread);
418
419 DTRACE_PROC(lwp__exit);
420
421 thread_mtx_lock(thread);
422
423 ipc_thread_disable(thread);
424
425 thread_mtx_unlock(thread);
426
427 s = splsched();
428 thread_lock(thread);
429
430 assert_thread_sched_count(thread);
431
432 /*
433 * Cancel priority depression, wait for concurrent expirations
434 * on other processors.
435 */
436 if (thread->sched_flags & TH_SFLAG_DEPRESSED_MASK) {
437 thread->sched_flags &= ~TH_SFLAG_DEPRESSED_MASK;
438
439 /* If our priority was low because of a depressed yield, restore it in case we block below */
440 set_sched_pri(thread, thread->priority);
441
442 if (timer_call_cancel(&thread->depress_timer))
443 thread->depress_timer_active--;
444 }
445
446 while (thread->depress_timer_active > 0) {
447 thread_unlock(thread);
448 splx(s);
449
450 delay(1);
451
452 s = splsched();
453 thread_lock(thread);
454 }
455
456 thread_sched_call(thread, NULL);
457
458 thread_unlock(thread);
459 splx(s);
460
461
462 thread_mtx_lock(thread);
463
464 thread_policy_reset(thread);
465
466 thread_mtx_unlock(thread);
467
468 task = thread->task;
469 uthread_cleanup(task, thread->uthread, task->bsd_info);
470 threadcnt = hw_atomic_sub(&task->active_thread_count, 1);
471
472 /*
473 * If we are the last thread to terminate and the task is
474 * associated with a BSD process, perform BSD process exit.
475 */
476 if (threadcnt == 0 && task->bsd_info != NULL)
477 proc_exit(task->bsd_info);
478
479 uthread_cred_free(thread->uthread);
480
481 s = splsched();
482 thread_lock(thread);
483
484 /*
485 * Cancel wait timer, and wait for
486 * concurrent expirations.
487 */
488 if (thread->wait_timer_is_set) {
489 thread->wait_timer_is_set = FALSE;
490
491 if (timer_call_cancel(&thread->wait_timer))
492 thread->wait_timer_active--;
493 }
494
495 while (thread->wait_timer_active > 0) {
496 thread_unlock(thread);
497 splx(s);
498
499 delay(1);
500
501 s = splsched();
502 thread_lock(thread);
503 }
504
505 /*
506 * If there is a reserved stack, release it.
507 */
508 if (thread->reserved_stack != 0) {
509 stack_free_reserved(thread);
510 thread->reserved_stack = 0;
511 }
512
513 /*
514 * Mark thread as terminating, and block.
515 */
516 thread->state |= TH_TERMINATE;
517 thread_mark_wait_locked(thread, THREAD_UNINT);
518 assert(thread->promotions == 0);
519 assert(thread->rwlock_count == 0);
520 thread_unlock(thread);
521 /* splsched */
522
523 thread_block((thread_continue_t)thread_terminate_continue);
524 /*NOTREACHED*/
525 }
526
527 void
528 thread_deallocate(
529 thread_t thread)
530 {
531 task_t task;
532
533 if (thread == THREAD_NULL)
534 return;
535
536 if (thread_deallocate_internal(thread) > 0)
537 return;
538
539 if(!(thread->state & TH_TERMINATE2))
540 panic("thread_deallocate: thread not properly terminated\n");
541
542 #if KPC
543 kpc_thread_destroy(thread);
544 #endif
545
546 ipc_thread_terminate(thread);
547
548 proc_thread_qos_deallocate(thread);
549
550 task = thread->task;
551
552 #ifdef MACH_BSD
553 {
554 void *ut = thread->uthread;
555
556 thread->uthread = NULL;
557 uthread_zone_free(ut);
558 }
559 #endif /* MACH_BSD */
560
561 if (thread->t_ledger)
562 ledger_dereference(thread->t_ledger);
563 if (thread->t_threadledger)
564 ledger_dereference(thread->t_threadledger);
565
566 if (IPC_VOUCHER_NULL != thread->ith_voucher)
567 ipc_voucher_release(thread->ith_voucher);
568
569 if (thread->thread_io_stats)
570 kfree(thread->thread_io_stats, sizeof(struct io_stat_info));
571
572 if (thread->kernel_stack != 0)
573 stack_free(thread);
574
575 lck_mtx_destroy(&thread->mutex, &thread_lck_grp);
576 machine_thread_destroy(thread);
577
578 task_deallocate(task);
579
580 zfree(thread_zone, thread);
581 }
582
583 /*
584 * thread_terminate_daemon:
585 *
586 * Perform final clean up for terminating threads.
587 */
588 static void
589 thread_terminate_daemon(void)
590 {
591 thread_t self, thread;
592 task_t task;
593
594 self = current_thread();
595 self->options |= TH_OPT_SYSTEM_CRITICAL;
596
597 (void)splsched();
598 simple_lock(&thread_terminate_lock);
599
600 while ((thread = (thread_t)dequeue_head(&thread_terminate_queue)) != THREAD_NULL) {
601 simple_unlock(&thread_terminate_lock);
602 (void)spllo();
603
604 assert(thread->SHARE_COUNT == 0);
605 assert(thread->BG_COUNT == 0);
606
607 task = thread->task;
608
609 task_lock(task);
610 task->total_user_time += timer_grab(&thread->user_timer);
611 if (thread->precise_user_kernel_time) {
612 task->total_system_time += timer_grab(&thread->system_timer);
613 } else {
614 task->total_user_time += timer_grab(&thread->system_timer);
615 }
616
617 task->c_switch += thread->c_switch;
618 task->p_switch += thread->p_switch;
619 task->ps_switch += thread->ps_switch;
620
621 task->syscalls_unix += thread->syscalls_unix;
622 task->syscalls_mach += thread->syscalls_mach;
623
624 task->task_timer_wakeups_bin_1 += thread->thread_timer_wakeups_bin_1;
625 task->task_timer_wakeups_bin_2 += thread->thread_timer_wakeups_bin_2;
626 task->task_gpu_ns += ml_gpu_stat(thread);
627
628 thread_update_qos_cpu_time(thread, FALSE);
629 queue_remove(&task->threads, thread, thread_t, task_threads);
630 task->thread_count--;
631
632 /*
633 * If the task is being halted, and there is only one thread
634 * left in the task after this one, then wakeup that thread.
635 */
636 if (task->thread_count == 1 && task->halting)
637 thread_wakeup((event_t)&task->halting);
638
639 task_unlock(task);
640
641 lck_mtx_lock(&tasks_threads_lock);
642 queue_remove(&threads, thread, thread_t, threads);
643 threads_count--;
644 lck_mtx_unlock(&tasks_threads_lock);
645
646 thread_deallocate(thread);
647
648 (void)splsched();
649 simple_lock(&thread_terminate_lock);
650 }
651
652 assert_wait((event_t)&thread_terminate_queue, THREAD_UNINT);
653 simple_unlock(&thread_terminate_lock);
654 /* splsched */
655
656 self->options &= ~TH_OPT_SYSTEM_CRITICAL;
657 thread_block((thread_continue_t)thread_terminate_daemon);
658 /*NOTREACHED*/
659 }
660
661 /*
662 * thread_terminate_enqueue:
663 *
664 * Enqueue a terminating thread for final disposition.
665 *
666 * Called at splsched.
667 */
668 void
669 thread_terminate_enqueue(
670 thread_t thread)
671 {
672 KERNEL_DEBUG_CONSTANT(TRACE_DATA_THREAD_TERMINATE | DBG_FUNC_NONE, thread->thread_id, 0, 0, 0, 0);
673
674 simple_lock(&thread_terminate_lock);
675 enqueue_tail(&thread_terminate_queue, (queue_entry_t)thread);
676 simple_unlock(&thread_terminate_lock);
677
678 thread_wakeup((event_t)&thread_terminate_queue);
679 }
680
681 /*
682 * thread_stack_daemon:
683 *
684 * Perform stack allocation as required due to
685 * invoke failures.
686 */
687 static void
688 thread_stack_daemon(void)
689 {
690 thread_t thread;
691 spl_t s;
692
693 s = splsched();
694 simple_lock(&thread_stack_lock);
695
696 while ((thread = (thread_t)dequeue_head(&thread_stack_queue)) != THREAD_NULL) {
697 simple_unlock(&thread_stack_lock);
698 splx(s);
699
700 /* allocate stack with interrupts enabled so that we can call into VM */
701 stack_alloc(thread);
702
703 s = splsched();
704 thread_lock(thread);
705 thread_setrun(thread, SCHED_PREEMPT | SCHED_TAILQ);
706 thread_unlock(thread);
707
708 simple_lock(&thread_stack_lock);
709 }
710
711 assert_wait((event_t)&thread_stack_queue, THREAD_UNINT);
712 simple_unlock(&thread_stack_lock);
713 splx(s);
714
715 thread_block((thread_continue_t)thread_stack_daemon);
716 /*NOTREACHED*/
717 }
718
719 /*
720 * thread_stack_enqueue:
721 *
722 * Enqueue a thread for stack allocation.
723 *
724 * Called at splsched.
725 */
726 void
727 thread_stack_enqueue(
728 thread_t thread)
729 {
730 simple_lock(&thread_stack_lock);
731 enqueue_tail(&thread_stack_queue, (queue_entry_t)thread);
732 simple_unlock(&thread_stack_lock);
733
734 thread_wakeup((event_t)&thread_stack_queue);
735 }
736
737 void
738 thread_daemon_init(void)
739 {
740 kern_return_t result;
741 thread_t thread = NULL;
742
743 simple_lock_init(&thread_terminate_lock, 0);
744 queue_init(&thread_terminate_queue);
745
746 result = kernel_thread_start_priority((thread_continue_t)thread_terminate_daemon, NULL, MINPRI_KERNEL, &thread);
747 if (result != KERN_SUCCESS)
748 panic("thread_daemon_init: thread_terminate_daemon");
749
750 thread_deallocate(thread);
751
752 simple_lock_init(&thread_stack_lock, 0);
753 queue_init(&thread_stack_queue);
754
755 result = kernel_thread_start_priority((thread_continue_t)thread_stack_daemon, NULL, BASEPRI_PREEMPT, &thread);
756 if (result != KERN_SUCCESS)
757 panic("thread_daemon_init: thread_stack_daemon");
758
759 thread_deallocate(thread);
760 }
761
762 /*
763 * Create a new thread.
764 * Doesn't start the thread running.
765 *
766 * Task and tasks_threads_lock are returned locked on success.
767 */
768 static kern_return_t
769 thread_create_internal(
770 task_t parent_task,
771 integer_t priority,
772 thread_continue_t continuation,
773 int options,
774 #define TH_OPTION_NONE 0x00
775 #define TH_OPTION_NOCRED 0x01
776 #define TH_OPTION_NOSUSP 0x02
777 thread_t *out_thread)
778 {
779 thread_t new_thread;
780 static thread_t first_thread;
781
782 /*
783 * Allocate a thread and initialize static fields
784 */
785 if (first_thread == THREAD_NULL)
786 new_thread = first_thread = current_thread();
787 else
788 new_thread = (thread_t)zalloc(thread_zone);
789 if (new_thread == THREAD_NULL)
790 return (KERN_RESOURCE_SHORTAGE);
791
792 if (new_thread != first_thread)
793 *new_thread = thread_template;
794
795 #ifdef MACH_BSD
796 new_thread->uthread = uthread_alloc(parent_task, new_thread, (options & TH_OPTION_NOCRED) != 0);
797 if (new_thread->uthread == NULL) {
798 zfree(thread_zone, new_thread);
799 return (KERN_RESOURCE_SHORTAGE);
800 }
801 #endif /* MACH_BSD */
802
803 if (machine_thread_create(new_thread, parent_task) != KERN_SUCCESS) {
804 #ifdef MACH_BSD
805 void *ut = new_thread->uthread;
806
807 new_thread->uthread = NULL;
808 /* cred free may not be necessary */
809 uthread_cleanup(parent_task, ut, parent_task->bsd_info);
810 uthread_cred_free(ut);
811 uthread_zone_free(ut);
812 #endif /* MACH_BSD */
813
814 zfree(thread_zone, new_thread);
815 return (KERN_FAILURE);
816 }
817
818 new_thread->task = parent_task;
819
820 thread_lock_init(new_thread);
821 wake_lock_init(new_thread);
822
823 lck_mtx_init(&new_thread->mutex, &thread_lck_grp, &thread_lck_attr);
824
825 ipc_thread_init(new_thread);
826
827 new_thread->continuation = continuation;
828
829 /* Allocate I/O Statistics structure */
830 new_thread->thread_io_stats = (io_stat_info_t)kalloc(sizeof(struct io_stat_info));
831 assert(new_thread->thread_io_stats != NULL);
832 bzero(new_thread->thread_io_stats, sizeof(struct io_stat_info));
833
834 #if CONFIG_IOSCHED
835 /* Clear out the I/O Scheduling info for AppleFSCompression */
836 new_thread->decmp_upl = NULL;
837 #endif /* CONFIG_IOSCHED */
838
839 lck_mtx_lock(&tasks_threads_lock);
840 task_lock(parent_task);
841
842 if ( !parent_task->active || parent_task->halting ||
843 ((options & TH_OPTION_NOSUSP) != 0 &&
844 parent_task->suspend_count > 0) ||
845 (parent_task->thread_count >= task_threadmax &&
846 parent_task != kernel_task) ) {
847 task_unlock(parent_task);
848 lck_mtx_unlock(&tasks_threads_lock);
849
850 #ifdef MACH_BSD
851 {
852 void *ut = new_thread->uthread;
853
854 new_thread->uthread = NULL;
855 uthread_cleanup(parent_task, ut, parent_task->bsd_info);
856 /* cred free may not be necessary */
857 uthread_cred_free(ut);
858 uthread_zone_free(ut);
859 }
860 #endif /* MACH_BSD */
861 ipc_thread_disable(new_thread);
862 ipc_thread_terminate(new_thread);
863 kfree(new_thread->thread_io_stats, sizeof(struct io_stat_info));
864 lck_mtx_destroy(&new_thread->mutex, &thread_lck_grp);
865 machine_thread_destroy(new_thread);
866 zfree(thread_zone, new_thread);
867 return (KERN_FAILURE);
868 }
869
870 /* New threads inherit any default state on the task */
871 machine_thread_inherit_taskwide(new_thread, parent_task);
872
873 task_reference_internal(parent_task);
874
875 if (new_thread->task->rusage_cpu_flags & TASK_RUSECPU_FLAGS_PERTHR_LIMIT) {
876 /*
877 * This task has a per-thread CPU limit; make sure this new thread
878 * gets its limit set too, before it gets out of the kernel.
879 */
880 set_astledger(new_thread);
881 }
882
883 /* Instantiate a thread ledger. Do not fail thread creation if ledger creation fails. */
884 if ((new_thread->t_threadledger = ledger_instantiate(thread_ledger_template,
885 LEDGER_CREATE_INACTIVE_ENTRIES)) != LEDGER_NULL) {
886
887 ledger_entry_setactive(new_thread->t_threadledger, thread_ledgers.cpu_time);
888 }
889
890 new_thread->cpu_time_last_qos = 0;
891 #ifdef CONFIG_BANK
892 new_thread->t_bankledger = LEDGER_NULL;
893 new_thread->t_deduct_bank_ledger_time = 0;
894 #endif
895
896 new_thread->t_ledger = new_thread->task->ledger;
897 if (new_thread->t_ledger)
898 ledger_reference(new_thread->t_ledger);
899
900 #if defined(CONFIG_SCHED_MULTIQ)
901 /* Cache the task's sched_group */
902 new_thread->sched_group = parent_task->sched_group;
903 #endif /* defined(CONFIG_SCHED_MULTIQ) */
904
905 /* Cache the task's map */
906 new_thread->map = parent_task->map;
907
908 timer_call_setup(&new_thread->wait_timer, thread_timer_expire, new_thread);
909 timer_call_setup(&new_thread->depress_timer, thread_depress_expire, new_thread);
910
911 #if CONFIG_COUNTERS
912 /*
913 * If parent task has any reservations, they need to be propagated to this
914 * thread.
915 */
916 new_thread->t_chud = (TASK_PMC_FLAG == (parent_task->t_chud & TASK_PMC_FLAG)) ?
917 THREAD_PMC_FLAG : 0U;
918 #endif
919 #if KPC
920 kpc_thread_create(new_thread);
921 #endif
922
923 /* Only need to update policies pushed from task to thread */
924 new_thread->requested_policy.bg_iotier = parent_task->effective_policy.bg_iotier;
925 new_thread->requested_policy.terminated = parent_task->effective_policy.terminated;
926
927 /* Set the thread's scheduling parameters */
928 new_thread->sched_mode = SCHED(initial_thread_sched_mode)(parent_task);
929 new_thread->sched_flags = 0;
930 new_thread->max_priority = parent_task->max_priority;
931 new_thread->task_priority = parent_task->priority;
932 new_thread->priority = (priority < 0)? parent_task->priority: priority;
933 if (new_thread->priority > new_thread->max_priority)
934 new_thread->priority = new_thread->max_priority;
935 new_thread->importance = new_thread->priority - new_thread->task_priority;
936 new_thread->saved_importance = new_thread->importance;
937
938 #if defined(CONFIG_SCHED_TIMESHARE_CORE)
939 new_thread->sched_stamp = sched_tick;
940 new_thread->pri_shift = sched_pri_shift;
941 #endif /* defined(CONFIG_SCHED_TIMESHARE_CORE) */
942
943 if (parent_task->max_priority <= MAXPRI_THROTTLE) {
944 sched_set_thread_throttled(new_thread, TRUE);
945 }
946
947 SCHED(compute_priority)(new_thread, FALSE);
948
949 thread_policy_create(new_thread);
950
951 /* Chain the thread onto the task's list */
952 queue_enter(&parent_task->threads, new_thread, thread_t, task_threads);
953 parent_task->thread_count++;
954
955 /* So terminating threads don't need to take the task lock to decrement */
956 hw_atomic_add(&parent_task->active_thread_count, 1);
957
958 /* Protected by the tasks_threads_lock */
959 new_thread->thread_id = ++thread_unique_id;
960
961 queue_enter(&threads, new_thread, thread_t, threads);
962 threads_count++;
963
964 new_thread->active = TRUE;
965
966 *out_thread = new_thread;
967
968 {
969 long dbg_arg1, dbg_arg2, dbg_arg3, dbg_arg4;
970
971 kdbg_trace_data(parent_task->bsd_info, &dbg_arg2);
972
973 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
974 TRACEDBG_CODE(DBG_TRACE_DATA, 1) | DBG_FUNC_NONE,
975 (vm_address_t)(uintptr_t)thread_tid(new_thread), dbg_arg2, 0, 0, 0);
976
977 kdbg_trace_string(parent_task->bsd_info,
978 &dbg_arg1, &dbg_arg2, &dbg_arg3, &dbg_arg4);
979
980 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
981 TRACEDBG_CODE(DBG_TRACE_STRING, 1) | DBG_FUNC_NONE,
982 dbg_arg1, dbg_arg2, dbg_arg3, dbg_arg4, 0);
983 }
984
985 DTRACE_PROC1(lwp__create, thread_t, *out_thread);
986
987 return (KERN_SUCCESS);
988 }
989
990 static kern_return_t
991 thread_create_internal2(
992 task_t task,
993 thread_t *new_thread,
994 boolean_t from_user)
995 {
996 kern_return_t result;
997 thread_t thread;
998
999 if (task == TASK_NULL || task == kernel_task)
1000 return (KERN_INVALID_ARGUMENT);
1001
1002 result = thread_create_internal(task, -1, (thread_continue_t)thread_bootstrap_return, TH_OPTION_NONE, &thread);
1003 if (result != KERN_SUCCESS)
1004 return (result);
1005
1006 thread->user_stop_count = 1;
1007 thread_hold(thread);
1008 if (task->suspend_count > 0)
1009 thread_hold(thread);
1010
1011 if (from_user)
1012 extmod_statistics_incr_thread_create(task);
1013
1014 task_unlock(task);
1015 lck_mtx_unlock(&tasks_threads_lock);
1016
1017 *new_thread = thread;
1018
1019 return (KERN_SUCCESS);
1020 }
1021
1022 /* No prototype, since task_server.h has the _from_user version if KERNEL_SERVER */
1023 kern_return_t
1024 thread_create(
1025 task_t task,
1026 thread_t *new_thread);
1027
1028 kern_return_t
1029 thread_create(
1030 task_t task,
1031 thread_t *new_thread)
1032 {
1033 return thread_create_internal2(task, new_thread, FALSE);
1034 }
1035
1036 kern_return_t
1037 thread_create_from_user(
1038 task_t task,
1039 thread_t *new_thread)
1040 {
1041 return thread_create_internal2(task, new_thread, TRUE);
1042 }
1043
1044 static kern_return_t
1045 thread_create_running_internal2(
1046 register task_t task,
1047 int flavor,
1048 thread_state_t new_state,
1049 mach_msg_type_number_t new_state_count,
1050 thread_t *new_thread,
1051 boolean_t from_user)
1052 {
1053 register kern_return_t result;
1054 thread_t thread;
1055
1056 if (task == TASK_NULL || task == kernel_task)
1057 return (KERN_INVALID_ARGUMENT);
1058
1059 result = thread_create_internal(task, -1, (thread_continue_t)thread_bootstrap_return, TH_OPTION_NONE, &thread);
1060 if (result != KERN_SUCCESS)
1061 return (result);
1062
1063 result = machine_thread_set_state(
1064 thread, flavor, new_state, new_state_count);
1065 if (result != KERN_SUCCESS) {
1066 task_unlock(task);
1067 lck_mtx_unlock(&tasks_threads_lock);
1068
1069 thread_terminate(thread);
1070 thread_deallocate(thread);
1071 return (result);
1072 }
1073
1074 thread_mtx_lock(thread);
1075 thread_start_internal(thread);
1076 thread_mtx_unlock(thread);
1077
1078 if (from_user)
1079 extmod_statistics_incr_thread_create(task);
1080
1081 task_unlock(task);
1082 lck_mtx_unlock(&tasks_threads_lock);
1083
1084 *new_thread = thread;
1085
1086 return (result);
1087 }
1088
1089 /* Prototype, see justification above */
1090 kern_return_t
1091 thread_create_running(
1092 register task_t task,
1093 int flavor,
1094 thread_state_t new_state,
1095 mach_msg_type_number_t new_state_count,
1096 thread_t *new_thread);
1097
1098 kern_return_t
1099 thread_create_running(
1100 register task_t task,
1101 int flavor,
1102 thread_state_t new_state,
1103 mach_msg_type_number_t new_state_count,
1104 thread_t *new_thread)
1105 {
1106 return thread_create_running_internal2(
1107 task, flavor, new_state, new_state_count,
1108 new_thread, FALSE);
1109 }
1110
1111 kern_return_t
1112 thread_create_running_from_user(
1113 register task_t task,
1114 int flavor,
1115 thread_state_t new_state,
1116 mach_msg_type_number_t new_state_count,
1117 thread_t *new_thread)
1118 {
1119 return thread_create_running_internal2(
1120 task, flavor, new_state, new_state_count,
1121 new_thread, TRUE);
1122 }
1123
1124 kern_return_t
1125 thread_create_workq(
1126 task_t task,
1127 thread_continue_t thread_return,
1128 thread_t *new_thread)
1129 {
1130 kern_return_t result;
1131 thread_t thread;
1132
1133 if (task == TASK_NULL || task == kernel_task)
1134 return (KERN_INVALID_ARGUMENT);
1135
1136 result = thread_create_internal(task, -1, thread_return, TH_OPTION_NOCRED | TH_OPTION_NOSUSP, &thread);
1137 if (result != KERN_SUCCESS)
1138 return (result);
1139
1140 thread->user_stop_count = 1;
1141 thread_hold(thread);
1142 if (task->suspend_count > 0)
1143 thread_hold(thread);
1144
1145 task_unlock(task);
1146 lck_mtx_unlock(&tasks_threads_lock);
1147
1148 *new_thread = thread;
1149
1150 return (KERN_SUCCESS);
1151 }
1152
1153 /*
1154 * kernel_thread_create:
1155 *
1156 * Create a thread in the kernel task
1157 * to execute in kernel context.
1158 */
1159 kern_return_t
1160 kernel_thread_create(
1161 thread_continue_t continuation,
1162 void *parameter,
1163 integer_t priority,
1164 thread_t *new_thread)
1165 {
1166 kern_return_t result;
1167 thread_t thread;
1168 task_t task = kernel_task;
1169
1170 result = thread_create_internal(task, priority, continuation, TH_OPTION_NONE, &thread);
1171 if (result != KERN_SUCCESS)
1172 return (result);
1173
1174 task_unlock(task);
1175 lck_mtx_unlock(&tasks_threads_lock);
1176
1177 stack_alloc(thread);
1178 assert(thread->kernel_stack != 0);
1179 thread->reserved_stack = thread->kernel_stack;
1180
1181 thread->parameter = parameter;
1182
1183 if(debug_task & 1)
1184 kprintf("kernel_thread_create: thread = %p continuation = %p\n", thread, continuation);
1185 *new_thread = thread;
1186
1187 return (result);
1188 }
1189
1190 kern_return_t
1191 kernel_thread_start_priority(
1192 thread_continue_t continuation,
1193 void *parameter,
1194 integer_t priority,
1195 thread_t *new_thread)
1196 {
1197 kern_return_t result;
1198 thread_t thread;
1199
1200 result = kernel_thread_create(continuation, parameter, priority, &thread);
1201 if (result != KERN_SUCCESS)
1202 return (result);
1203
1204 *new_thread = thread;
1205
1206 thread_mtx_lock(thread);
1207 thread_start_internal(thread);
1208 thread_mtx_unlock(thread);
1209
1210 return (result);
1211 }
1212
1213 kern_return_t
1214 kernel_thread_start(
1215 thread_continue_t continuation,
1216 void *parameter,
1217 thread_t *new_thread)
1218 {
1219 return kernel_thread_start_priority(continuation, parameter, -1, new_thread);
1220 }
1221
1222
1223 kern_return_t
1224 thread_info_internal(
1225 register thread_t thread,
1226 thread_flavor_t flavor,
1227 thread_info_t thread_info_out, /* ptr to OUT array */
1228 mach_msg_type_number_t *thread_info_count) /*IN/OUT*/
1229 {
1230 int state, flags;
1231 spl_t s;
1232
1233 if (thread == THREAD_NULL)
1234 return (KERN_INVALID_ARGUMENT);
1235
1236 if (flavor == THREAD_BASIC_INFO) {
1237 register thread_basic_info_t basic_info;
1238
1239 if (*thread_info_count < THREAD_BASIC_INFO_COUNT)
1240 return (KERN_INVALID_ARGUMENT);
1241
1242 basic_info = (thread_basic_info_t) thread_info_out;
1243
1244 s = splsched();
1245 thread_lock(thread);
1246
1247 /* fill in info */
1248
1249 thread_read_times(thread, &basic_info->user_time,
1250 &basic_info->system_time);
1251
1252 /*
1253 * Update lazy-evaluated scheduler info because someone wants it.
1254 */
1255 if (SCHED(can_update_priority)(thread))
1256 SCHED(update_priority)(thread);
1257
1258 basic_info->sleep_time = 0;
1259
1260 /*
1261 * To calculate cpu_usage, first correct for timer rate,
1262 * then for 5/8 ageing. The correction factor [3/5] is
1263 * (1/(5/8) - 1).
1264 */
1265 basic_info->cpu_usage = 0;
1266 #if defined(CONFIG_SCHED_TIMESHARE_CORE)
1267 if (sched_tick_interval) {
1268 basic_info->cpu_usage = (integer_t)(((uint64_t)thread->cpu_usage
1269 * TH_USAGE_SCALE) / sched_tick_interval);
1270 basic_info->cpu_usage = (basic_info->cpu_usage * 3) / 5;
1271 }
1272 #endif
1273
1274 if (basic_info->cpu_usage > TH_USAGE_SCALE)
1275 basic_info->cpu_usage = TH_USAGE_SCALE;
1276
1277 basic_info->policy = ((thread->sched_mode == TH_MODE_TIMESHARE)?
1278 POLICY_TIMESHARE: POLICY_RR);
1279
1280 flags = 0;
1281 if (thread->options & TH_OPT_IDLE_THREAD)
1282 flags |= TH_FLAGS_IDLE;
1283
1284 if (!thread->kernel_stack)
1285 flags |= TH_FLAGS_SWAPPED;
1286
1287 state = 0;
1288 if (thread->state & TH_TERMINATE)
1289 state = TH_STATE_HALTED;
1290 else
1291 if (thread->state & TH_RUN)
1292 state = TH_STATE_RUNNING;
1293 else
1294 if (thread->state & TH_UNINT)
1295 state = TH_STATE_UNINTERRUPTIBLE;
1296 else
1297 if (thread->state & TH_SUSP)
1298 state = TH_STATE_STOPPED;
1299 else
1300 if (thread->state & TH_WAIT)
1301 state = TH_STATE_WAITING;
1302
1303 basic_info->run_state = state;
1304 basic_info->flags = flags;
1305
1306 basic_info->suspend_count = thread->user_stop_count;
1307
1308 thread_unlock(thread);
1309 splx(s);
1310
1311 *thread_info_count = THREAD_BASIC_INFO_COUNT;
1312
1313 return (KERN_SUCCESS);
1314 }
1315 else
1316 if (flavor == THREAD_IDENTIFIER_INFO) {
1317 register thread_identifier_info_t identifier_info;
1318
1319 if (*thread_info_count < THREAD_IDENTIFIER_INFO_COUNT)
1320 return (KERN_INVALID_ARGUMENT);
1321
1322 identifier_info = (thread_identifier_info_t) thread_info_out;
1323
1324 s = splsched();
1325 thread_lock(thread);
1326
1327 identifier_info->thread_id = thread->thread_id;
1328 identifier_info->thread_handle = thread->machine.cthread_self;
1329 if(thread->task->bsd_info) {
1330 identifier_info->dispatch_qaddr = identifier_info->thread_handle + get_dispatchqueue_offset_from_proc(thread->task->bsd_info);
1331 } else {
1332 thread_unlock(thread);
1333 splx(s);
1334 return KERN_INVALID_ARGUMENT;
1335 }
1336
1337 thread_unlock(thread);
1338 splx(s);
1339 return KERN_SUCCESS;
1340 }
1341 else
1342 if (flavor == THREAD_SCHED_TIMESHARE_INFO) {
1343 policy_timeshare_info_t ts_info;
1344
1345 if (*thread_info_count < POLICY_TIMESHARE_INFO_COUNT)
1346 return (KERN_INVALID_ARGUMENT);
1347
1348 ts_info = (policy_timeshare_info_t)thread_info_out;
1349
1350 s = splsched();
1351 thread_lock(thread);
1352
1353 if (thread->sched_mode != TH_MODE_TIMESHARE) {
1354 thread_unlock(thread);
1355 splx(s);
1356
1357 return (KERN_INVALID_POLICY);
1358 }
1359
1360 ts_info->depressed = (thread->sched_flags & TH_SFLAG_DEPRESSED_MASK) != 0;
1361 if (ts_info->depressed) {
1362 ts_info->base_priority = DEPRESSPRI;
1363 ts_info->depress_priority = thread->priority;
1364 }
1365 else {
1366 ts_info->base_priority = thread->priority;
1367 ts_info->depress_priority = -1;
1368 }
1369
1370 ts_info->cur_priority = thread->sched_pri;
1371 ts_info->max_priority = thread->max_priority;
1372
1373 thread_unlock(thread);
1374 splx(s);
1375
1376 *thread_info_count = POLICY_TIMESHARE_INFO_COUNT;
1377
1378 return (KERN_SUCCESS);
1379 }
1380 else
1381 if (flavor == THREAD_SCHED_FIFO_INFO) {
1382 if (*thread_info_count < POLICY_FIFO_INFO_COUNT)
1383 return (KERN_INVALID_ARGUMENT);
1384
1385 return (KERN_INVALID_POLICY);
1386 }
1387 else
1388 if (flavor == THREAD_SCHED_RR_INFO) {
1389 policy_rr_info_t rr_info;
1390 uint32_t quantum_time;
1391 uint64_t quantum_ns;
1392
1393 if (*thread_info_count < POLICY_RR_INFO_COUNT)
1394 return (KERN_INVALID_ARGUMENT);
1395
1396 rr_info = (policy_rr_info_t) thread_info_out;
1397
1398 s = splsched();
1399 thread_lock(thread);
1400
1401 if (thread->sched_mode == TH_MODE_TIMESHARE) {
1402 thread_unlock(thread);
1403 splx(s);
1404
1405 return (KERN_INVALID_POLICY);
1406 }
1407
1408 rr_info->depressed = (thread->sched_flags & TH_SFLAG_DEPRESSED_MASK) != 0;
1409 if (rr_info->depressed) {
1410 rr_info->base_priority = DEPRESSPRI;
1411 rr_info->depress_priority = thread->priority;
1412 }
1413 else {
1414 rr_info->base_priority = thread->priority;
1415 rr_info->depress_priority = -1;
1416 }
1417
1418 quantum_time = SCHED(initial_quantum_size)(THREAD_NULL);
1419 absolutetime_to_nanoseconds(quantum_time, &quantum_ns);
1420
1421 rr_info->max_priority = thread->max_priority;
1422 rr_info->quantum = (uint32_t)(quantum_ns / 1000 / 1000);
1423
1424 thread_unlock(thread);
1425 splx(s);
1426
1427 *thread_info_count = POLICY_RR_INFO_COUNT;
1428
1429 return (KERN_SUCCESS);
1430 }
1431
1432 return (KERN_INVALID_ARGUMENT);
1433 }
1434
1435 void
1436 thread_read_times(
1437 thread_t thread,
1438 time_value_t *user_time,
1439 time_value_t *system_time)
1440 {
1441 clock_sec_t secs;
1442 clock_usec_t usecs;
1443 uint64_t tval_user, tval_system;
1444
1445 tval_user = timer_grab(&thread->user_timer);
1446 tval_system = timer_grab(&thread->system_timer);
1447
1448 if (thread->precise_user_kernel_time) {
1449 absolutetime_to_microtime(tval_user, &secs, &usecs);
1450 user_time->seconds = (typeof(user_time->seconds))secs;
1451 user_time->microseconds = usecs;
1452
1453 absolutetime_to_microtime(tval_system, &secs, &usecs);
1454 system_time->seconds = (typeof(system_time->seconds))secs;
1455 system_time->microseconds = usecs;
1456 } else {
1457 /* system_timer may represent either sys or user */
1458 tval_user += tval_system;
1459 absolutetime_to_microtime(tval_user, &secs, &usecs);
1460 user_time->seconds = (typeof(user_time->seconds))secs;
1461 user_time->microseconds = usecs;
1462
1463 system_time->seconds = 0;
1464 system_time->microseconds = 0;
1465 }
1466 }
1467
1468 uint64_t thread_get_runtime_self(void)
1469 {
1470 boolean_t interrupt_state;
1471 uint64_t runtime;
1472 thread_t thread = NULL;
1473 processor_t processor = NULL;
1474
1475 thread = current_thread();
1476
1477 /* Not interrupt safe, as the scheduler may otherwise update timer values underneath us */
1478 interrupt_state = ml_set_interrupts_enabled(FALSE);
1479 processor = current_processor();
1480 timer_switch(PROCESSOR_DATA(processor, thread_timer), mach_absolute_time(), PROCESSOR_DATA(processor, thread_timer));
1481 runtime = (timer_grab(&thread->user_timer) + timer_grab(&thread->system_timer));
1482 ml_set_interrupts_enabled(interrupt_state);
1483
1484 return runtime;
1485 }
1486
1487 kern_return_t
1488 thread_assign(
1489 __unused thread_t thread,
1490 __unused processor_set_t new_pset)
1491 {
1492 return (KERN_FAILURE);
1493 }
1494
1495 /*
1496 * thread_assign_default:
1497 *
1498 * Special version of thread_assign for assigning threads to default
1499 * processor set.
1500 */
1501 kern_return_t
1502 thread_assign_default(
1503 thread_t thread)
1504 {
1505 return (thread_assign(thread, &pset0));
1506 }
1507
1508 /*
1509 * thread_get_assignment
1510 *
1511 * Return current assignment for this thread.
1512 */
1513 kern_return_t
1514 thread_get_assignment(
1515 thread_t thread,
1516 processor_set_t *pset)
1517 {
1518 if (thread == NULL)
1519 return (KERN_INVALID_ARGUMENT);
1520
1521 *pset = &pset0;
1522
1523 return (KERN_SUCCESS);
1524 }
1525
1526 /*
1527 * thread_wire_internal:
1528 *
1529 * Specify that the target thread must always be able
1530 * to run and to allocate memory.
1531 */
1532 kern_return_t
1533 thread_wire_internal(
1534 host_priv_t host_priv,
1535 thread_t thread,
1536 boolean_t wired,
1537 boolean_t *prev_state)
1538 {
1539 if (host_priv == NULL || thread != current_thread())
1540 return (KERN_INVALID_ARGUMENT);
1541
1542 assert(host_priv == &realhost);
1543
1544 if (prev_state)
1545 *prev_state = (thread->options & TH_OPT_VMPRIV) != 0;
1546
1547 if (wired) {
1548 if (!(thread->options & TH_OPT_VMPRIV))
1549 vm_page_free_reserve(1); /* XXX */
1550 thread->options |= TH_OPT_VMPRIV;
1551 }
1552 else {
1553 if (thread->options & TH_OPT_VMPRIV)
1554 vm_page_free_reserve(-1); /* XXX */
1555 thread->options &= ~TH_OPT_VMPRIV;
1556 }
1557
1558 return (KERN_SUCCESS);
1559 }
1560
1561
1562 /*
1563 * thread_wire:
1564 *
1565 * User-api wrapper for thread_wire_internal()
1566 */
1567 kern_return_t
1568 thread_wire(
1569 host_priv_t host_priv,
1570 thread_t thread,
1571 boolean_t wired)
1572 {
1573 return (thread_wire_internal(host_priv, thread, wired, NULL));
1574 }
1575
1576
1577 boolean_t
1578 set_vm_privilege(boolean_t privileged)
1579 {
1580 boolean_t was_vmpriv;
1581
1582 if (current_thread()->options & TH_OPT_VMPRIV)
1583 was_vmpriv = TRUE;
1584 else
1585 was_vmpriv = FALSE;
1586
1587 if (privileged != FALSE)
1588 current_thread()->options |= TH_OPT_VMPRIV;
1589 else
1590 current_thread()->options &= ~TH_OPT_VMPRIV;
1591
1592 return (was_vmpriv);
1593 }
1594
1595
1596 /*
1597 * XXX assuming current thread only, for now...
1598 */
1599 void
1600 thread_guard_violation(thread_t thread, unsigned type)
1601 {
1602 assert(thread == current_thread());
1603
1604 spl_t s = splsched();
1605 /*
1606 * Use the saved state area of the thread structure
1607 * to store all info required to handle the AST when
1608 * returning to userspace
1609 */
1610 thread->guard_exc_info.type = type;
1611 thread_ast_set(thread, AST_GUARD);
1612 ast_propagate(thread->ast);
1613
1614 splx(s);
1615 }
1616
1617 /*
1618 * guard_ast:
1619 *
1620 * Handle AST_GUARD for a thread. This routine looks at the
1621 * state saved in the thread structure to determine the cause
1622 * of this exception. Based on this value, it invokes the
1623 * appropriate routine which determines other exception related
1624 * info and raises the exception.
1625 */
1626 void
1627 guard_ast(thread_t thread)
1628 {
1629 if (thread->guard_exc_info.type == GUARD_TYPE_MACH_PORT)
1630 mach_port_guard_ast(thread);
1631 else
1632 fd_guard_ast(thread);
1633 }
1634
1635 static void
1636 thread_cputime_callback(int warning, __unused const void *arg0, __unused const void *arg1)
1637 {
1638 if (warning == LEDGER_WARNING_ROSE_ABOVE) {
1639 #if CONFIG_TELEMETRY
1640 /*
1641 * This thread is in danger of violating the CPU usage monitor. Enable telemetry
1642 * on the entire task so there are micro-stackshots available if and when
1643 * EXC_RESOURCE is triggered. We could have chosen to enable micro-stackshots
1644 * for this thread only; but now that this task is suspect, knowing what all of
1645 * its threads are up to will be useful.
1646 */
1647 telemetry_task_ctl(current_task(), TF_CPUMON_WARNING, 1);
1648 #endif
1649 return;
1650 }
1651
1652 #if CONFIG_TELEMETRY
1653 /*
1654 * If the balance has dipped below the warning level (LEDGER_WARNING_DIPPED_BELOW) or
1655 * exceeded the limit, turn telemetry off for the task.
1656 */
1657 telemetry_task_ctl(current_task(), TF_CPUMON_WARNING, 0);
1658 #endif
1659
1660 if (warning == 0) {
1661 THIS_THREAD_IS_CONSUMING_TOO_MUCH_CPU__SENDING_EXC_RESOURCE();
1662 }
1663 }
1664
1665 void __attribute__((noinline))
1666 THIS_THREAD_IS_CONSUMING_TOO_MUCH_CPU__SENDING_EXC_RESOURCE(void)
1667 {
1668 int pid = 0;
1669 task_t task = current_task();
1670 thread_t thread = current_thread();
1671 uint64_t tid = thread->thread_id;
1672 char *procname = (char *) "unknown";
1673 time_value_t thread_total_time = {0, 0};
1674 time_value_t thread_system_time;
1675 time_value_t thread_user_time;
1676 int action;
1677 uint8_t percentage;
1678 uint32_t limit_percent;
1679 uint32_t usage_percent;
1680 uint32_t interval_sec;
1681 uint64_t interval_ns;
1682 uint64_t balance_ns;
1683 boolean_t fatal = FALSE;
1684
1685 mach_exception_data_type_t code[EXCEPTION_CODE_MAX];
1686 struct ledger_entry_info lei;
1687
1688 assert(thread->t_threadledger != LEDGER_NULL);
1689
1690 /*
1691 * Now that a thread has tripped the monitor, disable it for the entire task.
1692 */
1693 task_lock(task);
1694 if ((task->rusage_cpu_flags & TASK_RUSECPU_FLAGS_PERTHR_LIMIT) == 0) {
1695 /*
1696 * The CPU usage monitor has been disabled on our task, so some other
1697 * thread must have gotten here first. We only send one exception per
1698 * task lifetime, so there's nothing left for us to do here.
1699 */
1700 task_unlock(task);
1701 return;
1702 }
1703 if (task->rusage_cpu_flags & TASK_RUSECPU_FLAGS_FATAL_CPUMON) {
1704 fatal = TRUE;
1705 }
1706 task_disable_cpumon(task);
1707 task_unlock(task);
1708
1709 #ifdef MACH_BSD
1710 pid = proc_selfpid();
1711 if (task->bsd_info != NULL)
1712 procname = proc_name_address(task->bsd_info);
1713 #endif
1714
1715 thread_get_cpulimit(&action, &percentage, &interval_ns);
1716
1717 interval_sec = (uint32_t)(interval_ns / NSEC_PER_SEC);
1718
1719 thread_read_times(thread, &thread_user_time, &thread_system_time);
1720 time_value_add(&thread_total_time, &thread_user_time);
1721 time_value_add(&thread_total_time, &thread_system_time);
1722
1723 ledger_get_entry_info(thread->t_threadledger, thread_ledgers.cpu_time, &lei);
1724
1725 absolutetime_to_nanoseconds(lei.lei_balance, &balance_ns);
1726 usage_percent = (uint32_t) ((balance_ns * 100ULL) / lei.lei_last_refill);
1727
1728 /* Show refill period in the same units as balance, limit, etc */
1729 nanoseconds_to_absolutetime(lei.lei_refill_period, &lei.lei_refill_period);
1730
1731 limit_percent = (uint32_t) ((lei.lei_limit * 100ULL) / lei.lei_refill_period);
1732
1733 /* TODO: show task total runtime as well? see TASK_ABSOLUTETIME_INFO */
1734
1735 if (disable_exc_resource) {
1736 printf("process %s[%d] thread %llu caught burning CPU!; EXC_RESOURCE "
1737 "supressed by a boot-arg\n", procname, pid, tid);
1738 return;
1739 }
1740
1741 if (audio_active) {
1742 printf("process %s[%d] thread %llu caught burning CPU!; EXC_RESOURCE "
1743 "supressed due to audio playback\n", procname, pid, tid);
1744 return;
1745 }
1746 printf("process %s[%d] thread %llu caught burning CPU! "
1747 "It used more than %d%% CPU (Actual recent usage: %d%%) over %d seconds. "
1748 "thread lifetime cpu usage %d.%06d seconds, (%d.%06d user, %d.%06d system) "
1749 "ledger info: balance: %lld credit: %lld debit: %lld limit: %llu (%d%%) "
1750 "period: %llu time since last refill (ns): %llu %s\n",
1751 procname, pid, tid,
1752 percentage, usage_percent, interval_sec,
1753 thread_total_time.seconds, thread_total_time.microseconds,
1754 thread_user_time.seconds, thread_user_time.microseconds,
1755 thread_system_time.seconds, thread_system_time.microseconds,
1756 lei.lei_balance,
1757 lei.lei_credit, lei.lei_debit,
1758 lei.lei_limit, limit_percent,
1759 lei.lei_refill_period, lei.lei_last_refill,
1760 (fatal ? "[fatal violation]" : ""));
1761
1762
1763 code[0] = code[1] = 0;
1764 EXC_RESOURCE_ENCODE_TYPE(code[0], RESOURCE_TYPE_CPU);
1765 if (fatal) {
1766 EXC_RESOURCE_ENCODE_FLAVOR(code[0], FLAVOR_CPU_MONITOR_FATAL);
1767 }else {
1768 EXC_RESOURCE_ENCODE_FLAVOR(code[0], FLAVOR_CPU_MONITOR);
1769 }
1770 EXC_RESOURCE_CPUMONITOR_ENCODE_INTERVAL(code[0], interval_sec);
1771 EXC_RESOURCE_CPUMONITOR_ENCODE_PERCENTAGE(code[0], limit_percent);
1772 EXC_RESOURCE_CPUMONITOR_ENCODE_PERCENTAGE(code[1], usage_percent);
1773 exception_triage(EXC_RESOURCE, code, EXCEPTION_CODE_MAX);
1774
1775 if (fatal) {
1776 #if CONFIG_JETSAM
1777 jetsam_on_ledger_cpulimit_exceeded();
1778 #else
1779 task_terminate_internal(task);
1780 #endif
1781 }
1782 }
1783
1784 #define UPDATE_IO_STATS(info, size) \
1785 { \
1786 info.count++; \
1787 info.size += size; \
1788 }
1789
1790 #define UPDATE_IO_STATS_ATOMIC(info, size) \
1791 { \
1792 OSIncrementAtomic64((SInt64 *)&(info.count)); \
1793 OSAddAtomic64(size, (SInt64 *)&(info.size)); \
1794 }
1795
1796 void thread_update_io_stats(thread_t thread, int size, int io_flags)
1797 {
1798 int io_tier;
1799
1800 if (thread->thread_io_stats == NULL || thread->task->task_io_stats == NULL)
1801 return;
1802
1803 if (io_flags & DKIO_READ) {
1804 UPDATE_IO_STATS(thread->thread_io_stats->disk_reads, size);
1805 UPDATE_IO_STATS_ATOMIC(thread->task->task_io_stats->disk_reads, size);
1806 }
1807
1808 if (io_flags & DKIO_META) {
1809 UPDATE_IO_STATS(thread->thread_io_stats->metadata, size);
1810 UPDATE_IO_STATS_ATOMIC(thread->task->task_io_stats->metadata, size);
1811 }
1812
1813 if (io_flags & DKIO_PAGING) {
1814 UPDATE_IO_STATS(thread->thread_io_stats->paging, size);
1815 UPDATE_IO_STATS_ATOMIC(thread->task->task_io_stats->paging, size);
1816 }
1817
1818 io_tier = ((io_flags & DKIO_TIER_MASK) >> DKIO_TIER_SHIFT);
1819 assert (io_tier < IO_NUM_PRIORITIES);
1820
1821 UPDATE_IO_STATS(thread->thread_io_stats->io_priority[io_tier], size);
1822 UPDATE_IO_STATS_ATOMIC(thread->task->task_io_stats->io_priority[io_tier], size);
1823
1824 /* Update Total I/O Counts */
1825 UPDATE_IO_STATS(thread->thread_io_stats->total_io, size);
1826 UPDATE_IO_STATS_ATOMIC(thread->task->task_io_stats->total_io, size);
1827
1828 }
1829
1830 void
1831 init_thread_ledgers(void) {
1832 ledger_template_t t;
1833 int idx;
1834
1835 assert(thread_ledger_template == NULL);
1836
1837 if ((t = ledger_template_create("Per-thread ledger")) == NULL)
1838 panic("couldn't create thread ledger template");
1839
1840 if ((idx = ledger_entry_add(t, "cpu_time", "sched", "ns")) < 0) {
1841 panic("couldn't create cpu_time entry for thread ledger template");
1842 }
1843
1844 if (ledger_set_callback(t, idx, thread_cputime_callback, NULL, NULL) < 0) {
1845 panic("couldn't set thread ledger callback for cpu_time entry");
1846 }
1847
1848 thread_ledgers.cpu_time = idx;
1849
1850 thread_ledger_template = t;
1851 }
1852
1853 /*
1854 * Returns currently applied CPU usage limit, or 0/0 if none is applied.
1855 */
1856 int
1857 thread_get_cpulimit(int *action, uint8_t *percentage, uint64_t *interval_ns)
1858 {
1859 int64_t abstime = 0;
1860 uint64_t limittime = 0;
1861 thread_t thread = current_thread();
1862
1863 *percentage = 0;
1864 *interval_ns = 0;
1865 *action = 0;
1866
1867 if (thread->t_threadledger == LEDGER_NULL) {
1868 /*
1869 * This thread has no per-thread ledger, so it can't possibly
1870 * have a CPU limit applied.
1871 */
1872 return (KERN_SUCCESS);
1873 }
1874
1875 ledger_get_period(thread->t_threadledger, thread_ledgers.cpu_time, interval_ns);
1876 ledger_get_limit(thread->t_threadledger, thread_ledgers.cpu_time, &abstime);
1877
1878 if ((abstime == LEDGER_LIMIT_INFINITY) || (*interval_ns == 0)) {
1879 /*
1880 * This thread's CPU time ledger has no period or limit; so it
1881 * doesn't have a CPU limit applied.
1882 */
1883 return (KERN_SUCCESS);
1884 }
1885
1886 /*
1887 * This calculation is the converse to the one in thread_set_cpulimit().
1888 */
1889 absolutetime_to_nanoseconds(abstime, &limittime);
1890 *percentage = (limittime * 100ULL) / *interval_ns;
1891 assert(*percentage <= 100);
1892
1893 if (thread->options & TH_OPT_PROC_CPULIMIT) {
1894 assert((thread->options & TH_OPT_PRVT_CPULIMIT) == 0);
1895
1896 *action = THREAD_CPULIMIT_BLOCK;
1897 } else if (thread->options & TH_OPT_PRVT_CPULIMIT) {
1898 assert((thread->options & TH_OPT_PROC_CPULIMIT) == 0);
1899
1900 *action = THREAD_CPULIMIT_EXCEPTION;
1901 } else {
1902 *action = THREAD_CPULIMIT_DISABLE;
1903 }
1904
1905 return (KERN_SUCCESS);
1906 }
1907
1908 /*
1909 * Set CPU usage limit on a thread.
1910 *
1911 * Calling with percentage of 0 will unset the limit for this thread.
1912 */
1913 int
1914 thread_set_cpulimit(int action, uint8_t percentage, uint64_t interval_ns)
1915 {
1916 thread_t thread = current_thread();
1917 ledger_t l;
1918 uint64_t limittime = 0;
1919 uint64_t abstime = 0;
1920
1921 assert(percentage <= 100);
1922
1923 if (action == THREAD_CPULIMIT_DISABLE) {
1924 /*
1925 * Remove CPU limit, if any exists.
1926 */
1927 if (thread->t_threadledger != LEDGER_NULL) {
1928 l = thread->t_threadledger;
1929 ledger_set_limit(l, thread_ledgers.cpu_time, LEDGER_LIMIT_INFINITY, 0);
1930 ledger_set_action(l, thread_ledgers.cpu_time, LEDGER_ACTION_IGNORE);
1931 thread->options &= ~(TH_OPT_PROC_CPULIMIT | TH_OPT_PRVT_CPULIMIT);
1932 }
1933
1934 return (0);
1935 }
1936
1937 if (interval_ns < MINIMUM_CPULIMIT_INTERVAL_MS * NSEC_PER_MSEC) {
1938 return (KERN_INVALID_ARGUMENT);
1939 }
1940
1941 l = thread->t_threadledger;
1942 if (l == LEDGER_NULL) {
1943 /*
1944 * This thread doesn't yet have a per-thread ledger; so create one with the CPU time entry active.
1945 */
1946 if ((l = ledger_instantiate(thread_ledger_template, LEDGER_CREATE_INACTIVE_ENTRIES)) == LEDGER_NULL)
1947 return (KERN_RESOURCE_SHORTAGE);
1948
1949 /*
1950 * We are the first to create this thread's ledger, so only activate our entry.
1951 */
1952 ledger_entry_setactive(l, thread_ledgers.cpu_time);
1953 thread->t_threadledger = l;
1954 }
1955
1956 /*
1957 * The limit is specified as a percentage of CPU over an interval in nanoseconds.
1958 * Calculate the amount of CPU time that the thread needs to consume in order to hit the limit.
1959 */
1960 limittime = (interval_ns * percentage) / 100;
1961 nanoseconds_to_absolutetime(limittime, &abstime);
1962 ledger_set_limit(l, thread_ledgers.cpu_time, abstime, cpumon_ustackshots_trigger_pct);
1963 /*
1964 * Refill the thread's allotted CPU time every interval_ns nanoseconds.
1965 */
1966 ledger_set_period(l, thread_ledgers.cpu_time, interval_ns);
1967
1968 if (action == THREAD_CPULIMIT_EXCEPTION) {
1969 /*
1970 * We don't support programming the CPU usage monitor on a task if any of its
1971 * threads have a per-thread blocking CPU limit configured.
1972 */
1973 if (thread->options & TH_OPT_PRVT_CPULIMIT) {
1974 panic("CPU usage monitor activated, but blocking thread limit exists");
1975 }
1976
1977 /*
1978 * Make a note that this thread's CPU limit is being used for the task-wide CPU
1979 * usage monitor. We don't have to arm the callback which will trigger the
1980 * exception, because that was done for us in ledger_instantiate (because the
1981 * ledger template used has a default callback).
1982 */
1983 thread->options |= TH_OPT_PROC_CPULIMIT;
1984 } else {
1985 /*
1986 * We deliberately override any CPU limit imposed by a task-wide limit (eg
1987 * CPU usage monitor).
1988 */
1989 thread->options &= ~TH_OPT_PROC_CPULIMIT;
1990
1991 thread->options |= TH_OPT_PRVT_CPULIMIT;
1992 /* The per-thread ledger template by default has a callback for CPU time */
1993 ledger_disable_callback(l, thread_ledgers.cpu_time);
1994 ledger_set_action(l, thread_ledgers.cpu_time, LEDGER_ACTION_BLOCK);
1995 }
1996
1997 return (0);
1998 }
1999
2000 static void
2001 sched_call_null(
2002 __unused int type,
2003 __unused thread_t thread)
2004 {
2005 return;
2006 }
2007
2008 void
2009 thread_sched_call(
2010 thread_t thread,
2011 sched_call_t call)
2012 {
2013 thread->sched_call = (call != NULL)? call: sched_call_null;
2014 }
2015
2016 void
2017 thread_static_param(
2018 thread_t thread,
2019 boolean_t state)
2020 {
2021 thread_mtx_lock(thread);
2022 thread->static_param = state;
2023 thread_mtx_unlock(thread);
2024 }
2025
2026 uint64_t
2027 thread_tid(
2028 thread_t thread)
2029 {
2030 return (thread != THREAD_NULL? thread->thread_id: 0);
2031 }
2032
2033 uint16_t thread_set_tag(thread_t th, uint16_t tag) {
2034 return thread_set_tag_internal(th, tag);
2035 }
2036 uint16_t thread_get_tag(thread_t th) {
2037 return thread_get_tag_internal(th);
2038 }
2039
2040 uint64_t
2041 thread_dispatchqaddr(
2042 thread_t thread)
2043 {
2044 uint64_t dispatchqueue_addr = 0;
2045 uint64_t thread_handle = 0;
2046
2047 if (thread != THREAD_NULL) {
2048 thread_handle = thread->machine.cthread_self;
2049
2050 if (thread->task->bsd_info)
2051 dispatchqueue_addr = thread_handle + get_dispatchqueue_offset_from_proc(thread->task->bsd_info);
2052 }
2053
2054 return (dispatchqueue_addr);
2055 }
2056
2057 /*
2058 * Export routines to other components for things that are done as macros
2059 * within the osfmk component.
2060 */
2061
2062 #undef thread_reference
2063 void thread_reference(thread_t thread);
2064 void
2065 thread_reference(
2066 thread_t thread)
2067 {
2068 if (thread != THREAD_NULL)
2069 thread_reference_internal(thread);
2070 }
2071
2072 #undef thread_should_halt
2073
2074 boolean_t
2075 thread_should_halt(
2076 thread_t th)
2077 {
2078 return (thread_should_halt_fast(th));
2079 }
2080
2081 /*
2082 * thread_set_voucher_name - reset the voucher port name bound to this thread
2083 *
2084 * Conditions: nothing locked
2085 *
2086 * If we already converted the previous name to a cached voucher
2087 * reference, then we discard that reference here. The next lookup
2088 * will cache it again.
2089 */
2090
2091 kern_return_t
2092 thread_set_voucher_name(mach_port_name_t voucher_name)
2093 {
2094 thread_t thread = current_thread();
2095 ipc_voucher_t new_voucher = IPC_VOUCHER_NULL;
2096 ipc_voucher_t voucher;
2097 #ifdef CONFIG_BANK
2098 ledger_t bankledger = NULL;
2099 #endif
2100
2101 if (MACH_PORT_DEAD == voucher_name)
2102 return KERN_INVALID_RIGHT;
2103
2104 /*
2105 * agressively convert to voucher reference
2106 */
2107 if (MACH_PORT_VALID(voucher_name)) {
2108 new_voucher = convert_port_name_to_voucher(voucher_name);
2109 if (IPC_VOUCHER_NULL == new_voucher)
2110 return KERN_INVALID_ARGUMENT;
2111 }
2112 #ifdef CONFIG_BANK
2113 bankledger = bank_get_voucher_ledger(new_voucher);
2114 #endif
2115
2116 thread_mtx_lock(thread);
2117 voucher = thread->ith_voucher;
2118 thread->ith_voucher_name = voucher_name;
2119 thread->ith_voucher = new_voucher;
2120 #ifdef CONFIG_BANK
2121 bank_swap_thread_bank_ledger(thread, bankledger);
2122 #endif
2123 thread_mtx_unlock(thread);
2124
2125 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
2126 MACHDBG_CODE(DBG_MACH_IPC,MACH_THREAD_SET_VOUCHER) | DBG_FUNC_NONE,
2127 (uintptr_t)thread_tid(thread),
2128 (uintptr_t)voucher_name,
2129 VM_KERNEL_ADDRPERM((uintptr_t)new_voucher),
2130 1, 0);
2131
2132 if (IPC_VOUCHER_NULL != voucher)
2133 ipc_voucher_release(voucher);
2134
2135 return KERN_SUCCESS;
2136 }
2137
2138 /*
2139 * thread_get_mach_voucher - return a voucher reference for the specified thread voucher
2140 *
2141 * Conditions: nothing locked
2142 *
2143 * A reference to the voucher may be lazily pending, if someone set the voucher name
2144 * but nobody has done a lookup yet. In that case, we'll have to do the equivalent
2145 * lookup here.
2146 *
2147 * NOTE: At the moment, there is no distinction between the current and effective
2148 * vouchers because we only set them at the thread level currently.
2149 */
2150 kern_return_t
2151 thread_get_mach_voucher(
2152 thread_act_t thread,
2153 mach_voucher_selector_t __unused which,
2154 ipc_voucher_t *voucherp)
2155 {
2156 ipc_voucher_t voucher;
2157 mach_port_name_t voucher_name;
2158
2159 if (THREAD_NULL == thread)
2160 return KERN_INVALID_ARGUMENT;
2161
2162 thread_mtx_lock(thread);
2163 voucher = thread->ith_voucher;
2164
2165 /* if already cached, just return a ref */
2166 if (IPC_VOUCHER_NULL != voucher) {
2167 ipc_voucher_reference(voucher);
2168 thread_mtx_unlock(thread);
2169 *voucherp = voucher;
2170 return KERN_SUCCESS;
2171 }
2172
2173 voucher_name = thread->ith_voucher_name;
2174
2175 /* convert the name to a port, then voucher reference */
2176 if (MACH_PORT_VALID(voucher_name)) {
2177 ipc_port_t port;
2178
2179 if (KERN_SUCCESS !=
2180 ipc_object_copyin(thread->task->itk_space, voucher_name,
2181 MACH_MSG_TYPE_COPY_SEND, (ipc_object_t *)&port)) {
2182 thread->ith_voucher_name = MACH_PORT_NULL;
2183 thread_mtx_unlock(thread);
2184 *voucherp = IPC_VOUCHER_NULL;
2185 return KERN_SUCCESS;
2186 }
2187
2188 /* convert to a voucher ref to return, and cache a ref on thread */
2189 voucher = convert_port_to_voucher(port);
2190 ipc_voucher_reference(voucher);
2191 thread->ith_voucher = voucher;
2192 thread_mtx_unlock(thread);
2193
2194 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
2195 MACHDBG_CODE(DBG_MACH_IPC,MACH_THREAD_SET_VOUCHER) | DBG_FUNC_NONE,
2196 (uintptr_t)thread_tid(thread),
2197 (uintptr_t)port,
2198 VM_KERNEL_ADDRPERM((uintptr_t)voucher),
2199 2, 0);
2200
2201
2202 ipc_port_release_send(port);
2203 } else
2204 thread_mtx_unlock(thread);
2205
2206 *voucherp = voucher;
2207 return KERN_SUCCESS;
2208 }
2209
2210 /*
2211 * thread_set_mach_voucher - set a voucher reference for the specified thread voucher
2212 *
2213 * Conditions: callers holds a reference on the voucher.
2214 * nothing locked.
2215 *
2216 * We grab another reference to the voucher and bind it to the thread. Any lazy
2217 * binding is erased. The old voucher reference associated with the thread is
2218 * discarded.
2219 */
2220 kern_return_t
2221 thread_set_mach_voucher(
2222 thread_t thread,
2223 ipc_voucher_t voucher)
2224 {
2225 ipc_voucher_t old_voucher;
2226 #ifdef CONFIG_BANK
2227 ledger_t bankledger = NULL;
2228 #endif
2229
2230 if (THREAD_NULL == thread)
2231 return KERN_INVALID_ARGUMENT;
2232
2233 if (thread != current_thread() || thread->started)
2234 return KERN_INVALID_ARGUMENT;
2235
2236
2237 ipc_voucher_reference(voucher);
2238 #ifdef CONFIG_BANK
2239 bankledger = bank_get_voucher_ledger(voucher);
2240 #endif
2241 thread_mtx_lock(thread);
2242 old_voucher = thread->ith_voucher;
2243 thread->ith_voucher = voucher;
2244 thread->ith_voucher_name = MACH_PORT_NULL;
2245 #ifdef CONFIG_BANK
2246 bank_swap_thread_bank_ledger(thread, bankledger);
2247 #endif
2248 thread_mtx_unlock(thread);
2249
2250 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
2251 MACHDBG_CODE(DBG_MACH_IPC,MACH_THREAD_SET_VOUCHER) | DBG_FUNC_NONE,
2252 (uintptr_t)thread_tid(thread),
2253 (uintptr_t)MACH_PORT_NULL,
2254 VM_KERNEL_ADDRPERM((uintptr_t)voucher),
2255 3, 0);
2256
2257 ipc_voucher_release(old_voucher);
2258
2259 return KERN_SUCCESS;
2260 }
2261
2262 /*
2263 * thread_swap_mach_voucher - swap a voucher reference for the specified thread voucher
2264 *
2265 * Conditions: callers holds a reference on the new and presumed old voucher(s).
2266 * nothing locked.
2267 *
2268 * If the old voucher is still the same as passed in, replace it with new voucher
2269 * and discard the old (and the reference passed in). Otherwise, discard the new
2270 * and return an updated old voucher.
2271 */
2272 kern_return_t
2273 thread_swap_mach_voucher(
2274 thread_t thread,
2275 ipc_voucher_t new_voucher,
2276 ipc_voucher_t *in_out_old_voucher)
2277 {
2278 mach_port_name_t old_voucher_name;
2279 ipc_voucher_t old_voucher;
2280 #ifdef CONFIG_BANK
2281 ledger_t bankledger = NULL;
2282 #endif
2283
2284 if (THREAD_NULL == thread)
2285 return KERN_INVALID_TASK;
2286
2287 if (thread != current_thread() || thread->started)
2288 return KERN_INVALID_ARGUMENT;
2289
2290 #ifdef CONFIG_BANK
2291 bankledger = bank_get_voucher_ledger(new_voucher);
2292 #endif
2293
2294 thread_mtx_lock(thread);
2295
2296 old_voucher = thread->ith_voucher;
2297
2298 if (IPC_VOUCHER_NULL == old_voucher) {
2299 old_voucher_name = thread->ith_voucher_name;
2300
2301 /* perform lazy binding if needed */
2302 if (MACH_PORT_VALID(old_voucher_name)) {
2303 old_voucher = convert_port_name_to_voucher(old_voucher_name);
2304 thread->ith_voucher_name = MACH_PORT_NULL;
2305 thread->ith_voucher = old_voucher;
2306
2307 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
2308 MACHDBG_CODE(DBG_MACH_IPC,MACH_THREAD_SET_VOUCHER) | DBG_FUNC_NONE,
2309 (uintptr_t)thread_tid(thread),
2310 (uintptr_t)old_voucher_name,
2311 VM_KERNEL_ADDRPERM((uintptr_t)old_voucher),
2312 4, 0);
2313
2314 }
2315 }
2316
2317 /* swap in new voucher, if old voucher matches the one supplied */
2318 if (old_voucher == *in_out_old_voucher) {
2319 ipc_voucher_reference(new_voucher);
2320 thread->ith_voucher = new_voucher;
2321 thread->ith_voucher_name = MACH_PORT_NULL;
2322 #ifdef CONFIG_BANK
2323 bank_swap_thread_bank_ledger(thread, bankledger);
2324 #endif
2325 thread_mtx_unlock(thread);
2326
2327 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
2328 MACHDBG_CODE(DBG_MACH_IPC,MACH_THREAD_SET_VOUCHER) | DBG_FUNC_NONE,
2329 (uintptr_t)thread_tid(thread),
2330 (uintptr_t)MACH_PORT_NULL,
2331 VM_KERNEL_ADDRPERM((uintptr_t)new_voucher),
2332 5, 0);
2333
2334 ipc_voucher_release(old_voucher);
2335
2336 *in_out_old_voucher = IPC_VOUCHER_NULL;
2337 return KERN_SUCCESS;
2338 }
2339
2340 /* Otherwise, just return old voucher reference */
2341 ipc_voucher_reference(old_voucher);
2342 thread_mtx_unlock(thread);
2343 *in_out_old_voucher = old_voucher;
2344 return KERN_SUCCESS;
2345 }
2346
2347 /*
2348 * thread_get_current_voucher_origin_pid - get the pid of the originator of the current voucher.
2349 */
2350 kern_return_t
2351 thread_get_current_voucher_origin_pid(
2352 int32_t *pid)
2353 {
2354 uint32_t buf_size;
2355 kern_return_t kr;
2356 thread_t thread = current_thread();
2357
2358 buf_size = sizeof(*pid);
2359 kr = mach_voucher_attr_command(thread->ith_voucher,
2360 MACH_VOUCHER_ATTR_KEY_BANK,
2361 BANK_ORIGINATOR_PID,
2362 NULL,
2363 0,
2364 (mach_voucher_attr_content_t)pid,
2365 &buf_size);
2366
2367 return kr;
2368 }
2369
2370 #if CONFIG_DTRACE
2371 uint32_t dtrace_get_thread_predcache(thread_t thread)
2372 {
2373 if (thread != THREAD_NULL)
2374 return thread->t_dtrace_predcache;
2375 else
2376 return 0;
2377 }
2378
2379 int64_t dtrace_get_thread_vtime(thread_t thread)
2380 {
2381 if (thread != THREAD_NULL)
2382 return thread->t_dtrace_vtime;
2383 else
2384 return 0;
2385 }
2386
2387 int64_t dtrace_get_thread_tracing(thread_t thread)
2388 {
2389 if (thread != THREAD_NULL)
2390 return thread->t_dtrace_tracing;
2391 else
2392 return 0;
2393 }
2394
2395 boolean_t dtrace_get_thread_reentering(thread_t thread)
2396 {
2397 if (thread != THREAD_NULL)
2398 return (thread->options & TH_OPT_DTRACE) ? TRUE : FALSE;
2399 else
2400 return 0;
2401 }
2402
2403 vm_offset_t dtrace_get_kernel_stack(thread_t thread)
2404 {
2405 if (thread != THREAD_NULL)
2406 return thread->kernel_stack;
2407 else
2408 return 0;
2409 }
2410
2411 int64_t dtrace_calc_thread_recent_vtime(thread_t thread)
2412 {
2413 if (thread != THREAD_NULL) {
2414 processor_t processor = current_processor();
2415 uint64_t abstime = mach_absolute_time();
2416 timer_t timer;
2417
2418 timer = PROCESSOR_DATA(processor, thread_timer);
2419
2420 return timer_grab(&(thread->system_timer)) + timer_grab(&(thread->user_timer)) +
2421 (abstime - timer->tstamp); /* XXX need interrupts off to prevent missed time? */
2422 } else
2423 return 0;
2424 }
2425
2426 void dtrace_set_thread_predcache(thread_t thread, uint32_t predcache)
2427 {
2428 if (thread != THREAD_NULL)
2429 thread->t_dtrace_predcache = predcache;
2430 }
2431
2432 void dtrace_set_thread_vtime(thread_t thread, int64_t vtime)
2433 {
2434 if (thread != THREAD_NULL)
2435 thread->t_dtrace_vtime = vtime;
2436 }
2437
2438 void dtrace_set_thread_tracing(thread_t thread, int64_t accum)
2439 {
2440 if (thread != THREAD_NULL)
2441 thread->t_dtrace_tracing = accum;
2442 }
2443
2444 void dtrace_set_thread_reentering(thread_t thread, boolean_t vbool)
2445 {
2446 if (thread != THREAD_NULL) {
2447 if (vbool)
2448 thread->options |= TH_OPT_DTRACE;
2449 else
2450 thread->options &= (~TH_OPT_DTRACE);
2451 }
2452 }
2453
2454 vm_offset_t dtrace_set_thread_recover(thread_t thread, vm_offset_t recover)
2455 {
2456 vm_offset_t prev = 0;
2457
2458 if (thread != THREAD_NULL) {
2459 prev = thread->recover;
2460 thread->recover = recover;
2461 }
2462 return prev;
2463 }
2464
2465 void dtrace_thread_bootstrap(void)
2466 {
2467 task_t task = current_task();
2468
2469 if (task->thread_count == 1) {
2470 thread_t thread = current_thread();
2471 if (thread->t_dtrace_flags & TH_DTRACE_EXECSUCCESS) {
2472 thread->t_dtrace_flags &= ~TH_DTRACE_EXECSUCCESS;
2473 DTRACE_PROC(exec__success);
2474 }
2475 DTRACE_PROC(start);
2476 }
2477 DTRACE_PROC(lwp__start);
2478
2479 }
2480
2481 void
2482 dtrace_thread_didexec(thread_t thread)
2483 {
2484 thread->t_dtrace_flags |= TH_DTRACE_EXECSUCCESS;
2485 }
2486 #endif /* CONFIG_DTRACE */
mirror of XNU