]> git.saurik.com Git - apple/xnu.git/blob - osfmk/kern/thread.c
projects / apple / xnu.git / blob
? search:


2738d385001223c9161c64b61e30fcb4068189fb
[apple/xnu.git] / osfmk / kern / thread.c
1 /*
2 * Copyright (c) 2000-2010 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_FREE_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56 /*
57 */
58 /*
59 * File: kern/thread.c
60 * Author: Avadis Tevanian, Jr., Michael Wayne Young, David Golub
61 * Date: 1986
62 *
63 * Thread management primitives implementation.
64 */
65 /*
66 * Copyright (c) 1993 The University of Utah and
67 * the Computer Systems Laboratory (CSL). All rights reserved.
68 *
69 * Permission to use, copy, modify and distribute this software and its
70 * documentation is hereby granted, provided that both the copyright
71 * notice and this permission notice appear in all copies of the
72 * software, derivative works or modified versions, and any portions
73 * thereof, and that both notices appear in supporting documentation.
74 *
75 * THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF THIS SOFTWARE IN ITS "AS
76 * IS" CONDITION. THE UNIVERSITY OF UTAH AND CSL DISCLAIM ANY LIABILITY OF
77 * ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
78 *
79 * CSL requests users of this software to return to csl-dist@cs.utah.edu any
80 * improvements that they make and grant CSL redistribution rights.
81 *
82 */
83
84 #include <mach/mach_types.h>
85 #include <mach/boolean.h>
86 #include <mach/policy.h>
87 #include <mach/thread_info.h>
88 #include <mach/thread_special_ports.h>
89 #include <mach/thread_status.h>
90 #include <mach/time_value.h>
91 #include <mach/vm_param.h>
92
93 #include <machine/thread.h>
94 #include <machine/pal_routines.h>
95 #include <machine/limits.h>
96
97 #include <kern/kern_types.h>
98 #include <kern/kalloc.h>
99 #include <kern/cpu_data.h>
100 #include <kern/counters.h>
101 #include <kern/extmod_statistics.h>
102 #include <kern/ipc_mig.h>
103 #include <kern/ipc_tt.h>
104 #include <kern/mach_param.h>
105 #include <kern/machine.h>
106 #include <kern/misc_protos.h>
107 #include <kern/processor.h>
108 #include <kern/queue.h>
109 #include <kern/sched.h>
110 #include <kern/sched_prim.h>
111 #include <kern/sync_lock.h>
112 #include <kern/syscall_subr.h>
113 #include <kern/task.h>
114 #include <kern/thread.h>
115 #include <kern/host.h>
116 #include <kern/zalloc.h>
117 #include <kern/assert.h>
118
119 #include <ipc/ipc_kmsg.h>
120 #include <ipc/ipc_port.h>
121
122 #include <vm/vm_kern.h>
123 #include <vm/vm_pageout.h>
124
125 #include <sys/kdebug.h>
126
127 #include <mach/sdt.h>
128
129 /*
130 * Exported interfaces
131 */
132 #include <mach/task_server.h>
133 #include <mach/thread_act_server.h>
134 #include <mach/mach_host_server.h>
135 #include <mach/host_priv_server.h>
136
137 static struct zone *thread_zone;
138 static lck_grp_attr_t thread_lck_grp_attr;
139 lck_attr_t thread_lck_attr;
140 lck_grp_t thread_lck_grp;
141
142 decl_simple_lock_data(static,thread_stack_lock)
143 static queue_head_t thread_stack_queue;
144
145 decl_simple_lock_data(static,thread_terminate_lock)
146 static queue_head_t thread_terminate_queue;
147
148 static struct thread thread_template, init_thread;
149
150 static void sched_call_null(
151 int type,
152 thread_t thread);
153
154 #ifdef MACH_BSD
155 extern void proc_exit(void *);
156 extern uint64_t get_dispatchqueue_offset_from_proc(void *);
157 #endif /* MACH_BSD */
158
159 extern int debug_task;
160 int thread_max = CONFIG_THREAD_MAX; /* Max number of threads */
161 int task_threadmax = CONFIG_THREAD_MAX;
162
163 static uint64_t thread_unique_id = 0;
164
165 struct _thread_ledger_indices thread_ledgers = { -1 };
166 static ledger_template_t thread_ledger_template = NULL;
167 void init_thread_ledgers(void);
168
169 void
170 thread_bootstrap(void)
171 {
172 /*
173 * Fill in a template thread for fast initialization.
174 */
175
176 thread_template.runq = PROCESSOR_NULL;
177
178 thread_template.ref_count = 2;
179
180 thread_template.reason = AST_NONE;
181 thread_template.at_safe_point = FALSE;
182 thread_template.wait_event = NO_EVENT64;
183 thread_template.wait_queue = WAIT_QUEUE_NULL;
184 thread_template.wait_result = THREAD_WAITING;
185 thread_template.options = THREAD_ABORTSAFE;
186 thread_template.state = TH_WAIT | TH_UNINT;
187 thread_template.wake_active = FALSE;
188 thread_template.continuation = THREAD_CONTINUE_NULL;
189 thread_template.parameter = NULL;
190
191 thread_template.importance = 0;
192 thread_template.sched_mode = TH_MODE_NONE;
193 thread_template.sched_flags = 0;
194 thread_template.saved_mode = TH_MODE_NONE;
195 thread_template.safe_release = 0;
196
197 thread_template.priority = 0;
198 thread_template.sched_pri = 0;
199 thread_template.max_priority = 0;
200 thread_template.task_priority = 0;
201 thread_template.promotions = 0;
202 thread_template.pending_promoter_index = 0;
203 thread_template.pending_promoter[0] =
204 thread_template.pending_promoter[1] = NULL;
205
206 thread_template.realtime.deadline = UINT64_MAX;
207
208 thread_template.current_quantum = 0;
209 thread_template.last_run_time = 0;
210 thread_template.last_quantum_refill_time = 0;
211
212 thread_template.computation_metered = 0;
213 thread_template.computation_epoch = 0;
214
215 #if defined(CONFIG_SCHED_TRADITIONAL)
216 thread_template.sched_stamp = 0;
217 thread_template.pri_shift = INT8_MAX;
218 thread_template.sched_usage = 0;
219 thread_template.cpu_usage = thread_template.cpu_delta = 0;
220 #endif
221 thread_template.c_switch = thread_template.p_switch = thread_template.ps_switch = 0;
222
223 thread_template.bound_processor = PROCESSOR_NULL;
224 thread_template.last_processor = PROCESSOR_NULL;
225
226 thread_template.sched_call = sched_call_null;
227
228 timer_init(&thread_template.user_timer);
229 timer_init(&thread_template.system_timer);
230 thread_template.user_timer_save = 0;
231 thread_template.system_timer_save = 0;
232 thread_template.vtimer_user_save = 0;
233 thread_template.vtimer_prof_save = 0;
234 thread_template.vtimer_rlim_save = 0;
235
236 thread_template.wait_timer_is_set = FALSE;
237 thread_template.wait_timer_active = 0;
238
239 thread_template.depress_timer_active = 0;
240
241 thread_template.special_handler.handler = special_handler;
242 thread_template.special_handler.next = NULL;
243
244 thread_template.funnel_lock = THR_FUNNEL_NULL;
245 thread_template.funnel_state = 0;
246 thread_template.recover = (vm_offset_t)NULL;
247
248 thread_template.map = VM_MAP_NULL;
249
250 #if CONFIG_DTRACE
251 thread_template.t_dtrace_predcache = 0;
252 thread_template.t_dtrace_vtime = 0;
253 thread_template.t_dtrace_tracing = 0;
254 #endif /* CONFIG_DTRACE */
255
256 thread_template.t_chud = 0;
257 thread_template.t_page_creation_count = 0;
258 thread_template.t_page_creation_time = 0;
259
260 thread_template.affinity_set = NULL;
261
262 thread_template.syscalls_unix = 0;
263 thread_template.syscalls_mach = 0;
264
265 thread_template.t_ledger = LEDGER_NULL;
266 thread_template.t_threadledger = LEDGER_NULL;
267
268 thread_template.appliedstate = default_task_null_policy;
269 thread_template.ext_appliedstate = default_task_null_policy;
270 thread_template.policystate = default_task_proc_policy;
271 thread_template.ext_policystate = default_task_proc_policy;
272 #if CONFIG_EMBEDDED
273 thread_template.taskwatch = NULL;
274 thread_template.saved_importance = 0;
275 #endif /* CONFIG_EMBEDDED */
276
277 init_thread = thread_template;
278 machine_set_current_thread(&init_thread);
279 }
280
281 void
282 thread_init(void)
283 {
284 thread_zone = zinit(
285 sizeof(struct thread),
286 thread_max * sizeof(struct thread),
287 THREAD_CHUNK * sizeof(struct thread),
288 "threads");
289
290 lck_grp_attr_setdefault(&thread_lck_grp_attr);
291 lck_grp_init(&thread_lck_grp, "thread", &thread_lck_grp_attr);
292 lck_attr_setdefault(&thread_lck_attr);
293
294 stack_init();
295
296 /*
297 * Initialize any machine-dependent
298 * per-thread structures necessary.
299 */
300 machine_thread_init();
301
302 init_thread_ledgers();
303 }
304
305 static void
306 thread_terminate_continue(void)
307 {
308 panic("thread_terminate_continue");
309 /*NOTREACHED*/
310 }
311
312 /*
313 * thread_terminate_self:
314 */
315 void
316 thread_terminate_self(void)
317 {
318 thread_t thread = current_thread();
319
320 task_t task;
321 spl_t s;
322 int threadcnt;
323
324 pal_thread_terminate_self(thread);
325
326 DTRACE_PROC(lwp__exit);
327
328 thread_mtx_lock(thread);
329
330 ulock_release_all(thread);
331
332 ipc_thread_disable(thread);
333
334 thread_mtx_unlock(thread);
335
336 s = splsched();
337 thread_lock(thread);
338
339 /*
340 * Cancel priority depression, wait for concurrent expirations
341 * on other processors.
342 */
343 if (thread->sched_flags & TH_SFLAG_DEPRESSED_MASK) {
344 thread->sched_flags &= ~TH_SFLAG_DEPRESSED_MASK;
345
346 if (timer_call_cancel(&thread->depress_timer))
347 thread->depress_timer_active--;
348 }
349
350 while (thread->depress_timer_active > 0) {
351 thread_unlock(thread);
352 splx(s);
353
354 delay(1);
355
356 s = splsched();
357 thread_lock(thread);
358 }
359
360 thread_sched_call(thread, NULL);
361
362 thread_unlock(thread);
363 splx(s);
364
365 thread_policy_reset(thread);
366
367 #if CONFIG_EMBEDDED
368 thead_remove_taskwatch(thread);
369 #endif /* CONFIG_EMBEDDED */
370
371 task = thread->task;
372 uthread_cleanup(task, thread->uthread, task->bsd_info);
373 threadcnt = hw_atomic_sub(&task->active_thread_count, 1);
374
375 /*
376 * If we are the last thread to terminate and the task is
377 * associated with a BSD process, perform BSD process exit.
378 */
379 if (threadcnt == 0 && task->bsd_info != NULL)
380 proc_exit(task->bsd_info);
381
382 uthread_cred_free(thread->uthread);
383
384 s = splsched();
385 thread_lock(thread);
386
387 /*
388 * Cancel wait timer, and wait for
389 * concurrent expirations.
390 */
391 if (thread->wait_timer_is_set) {
392 thread->wait_timer_is_set = FALSE;
393
394 if (timer_call_cancel(&thread->wait_timer))
395 thread->wait_timer_active--;
396 }
397
398 while (thread->wait_timer_active > 0) {
399 thread_unlock(thread);
400 splx(s);
401
402 delay(1);
403
404 s = splsched();
405 thread_lock(thread);
406 }
407
408 /*
409 * If there is a reserved stack, release it.
410 */
411 if (thread->reserved_stack != 0) {
412 stack_free_reserved(thread);
413 thread->reserved_stack = 0;
414 }
415
416 /*
417 * Mark thread as terminating, and block.
418 */
419 thread->state |= TH_TERMINATE;
420 thread_mark_wait_locked(thread, THREAD_UNINT);
421 assert(thread->promotions == 0);
422 thread_unlock(thread);
423 /* splsched */
424
425 thread_block((thread_continue_t)thread_terminate_continue);
426 /*NOTREACHED*/
427 }
428
429 void
430 thread_deallocate(
431 thread_t thread)
432 {
433 task_t task;
434
435 if (thread == THREAD_NULL)
436 return;
437
438 if (thread_deallocate_internal(thread) > 0)
439 return;
440
441
442 ipc_thread_terminate(thread);
443
444 task = thread->task;
445
446 #ifdef MACH_BSD
447 {
448 void *ut = thread->uthread;
449
450 thread->uthread = NULL;
451 uthread_zone_free(ut);
452 }
453 #endif /* MACH_BSD */
454
455 if (thread->t_ledger)
456 ledger_dereference(thread->t_ledger);
457 if (thread->t_threadledger)
458 ledger_dereference(thread->t_threadledger);
459
460 if (thread->kernel_stack != 0)
461 stack_free(thread);
462
463 lck_mtx_destroy(&thread->mutex, &thread_lck_grp);
464 machine_thread_destroy(thread);
465
466 task_deallocate(task);
467
468 zfree(thread_zone, thread);
469 }
470
471 /*
472 * thread_terminate_daemon:
473 *
474 * Perform final clean up for terminating threads.
475 */
476 static void
477 thread_terminate_daemon(void)
478 {
479 thread_t self, thread;
480 task_t task;
481
482 self = current_thread();
483 self->options |= TH_OPT_SYSTEM_CRITICAL;
484
485 (void)splsched();
486 simple_lock(&thread_terminate_lock);
487
488 while ((thread = (thread_t)dequeue_head(&thread_terminate_queue)) != THREAD_NULL) {
489 simple_unlock(&thread_terminate_lock);
490 (void)spllo();
491
492 task = thread->task;
493
494 task_lock(task);
495 task->total_user_time += timer_grab(&thread->user_timer);
496 if (thread->precise_user_kernel_time) {
497 task->total_system_time += timer_grab(&thread->system_timer);
498 } else {
499 task->total_user_time += timer_grab(&thread->system_timer);
500 }
501
502 task->c_switch += thread->c_switch;
503 task->p_switch += thread->p_switch;
504 task->ps_switch += thread->ps_switch;
505
506 task->syscalls_unix += thread->syscalls_unix;
507 task->syscalls_mach += thread->syscalls_mach;
508
509 queue_remove(&task->threads, thread, thread_t, task_threads);
510 task->thread_count--;
511
512 /*
513 * If the task is being halted, and there is only one thread
514 * left in the task after this one, then wakeup that thread.
515 */
516 if (task->thread_count == 1 && task->halting)
517 thread_wakeup((event_t)&task->halting);
518
519 task_unlock(task);
520
521 lck_mtx_lock(&tasks_threads_lock);
522 queue_remove(&threads, thread, thread_t, threads);
523 threads_count--;
524 lck_mtx_unlock(&tasks_threads_lock);
525
526 thread_deallocate(thread);
527
528 (void)splsched();
529 simple_lock(&thread_terminate_lock);
530 }
531
532 assert_wait((event_t)&thread_terminate_queue, THREAD_UNINT);
533 simple_unlock(&thread_terminate_lock);
534 /* splsched */
535
536 self->options &= ~TH_OPT_SYSTEM_CRITICAL;
537 thread_block((thread_continue_t)thread_terminate_daemon);
538 /*NOTREACHED*/
539 }
540
541 /*
542 * thread_terminate_enqueue:
543 *
544 * Enqueue a terminating thread for final disposition.
545 *
546 * Called at splsched.
547 */
548 void
549 thread_terminate_enqueue(
550 thread_t thread)
551 {
552 simple_lock(&thread_terminate_lock);
553 enqueue_tail(&thread_terminate_queue, (queue_entry_t)thread);
554 simple_unlock(&thread_terminate_lock);
555
556 thread_wakeup((event_t)&thread_terminate_queue);
557 }
558
559 /*
560 * thread_stack_daemon:
561 *
562 * Perform stack allocation as required due to
563 * invoke failures.
564 */
565 static void
566 thread_stack_daemon(void)
567 {
568 thread_t thread;
569
570 simple_lock(&thread_stack_lock);
571
572 while ((thread = (thread_t)dequeue_head(&thread_stack_queue)) != THREAD_NULL) {
573 simple_unlock(&thread_stack_lock);
574
575 stack_alloc(thread);
576
577 (void)splsched();
578 thread_lock(thread);
579 thread_setrun(thread, SCHED_PREEMPT | SCHED_TAILQ);
580 thread_unlock(thread);
581 (void)spllo();
582
583 simple_lock(&thread_stack_lock);
584 }
585
586 assert_wait((event_t)&thread_stack_queue, THREAD_UNINT);
587 simple_unlock(&thread_stack_lock);
588
589 thread_block((thread_continue_t)thread_stack_daemon);
590 /*NOTREACHED*/
591 }
592
593 /*
594 * thread_stack_enqueue:
595 *
596 * Enqueue a thread for stack allocation.
597 *
598 * Called at splsched.
599 */
600 void
601 thread_stack_enqueue(
602 thread_t thread)
603 {
604 simple_lock(&thread_stack_lock);
605 enqueue_tail(&thread_stack_queue, (queue_entry_t)thread);
606 simple_unlock(&thread_stack_lock);
607
608 thread_wakeup((event_t)&thread_stack_queue);
609 }
610
611 void
612 thread_daemon_init(void)
613 {
614 kern_return_t result;
615 thread_t thread = NULL;
616
617 simple_lock_init(&thread_terminate_lock, 0);
618 queue_init(&thread_terminate_queue);
619
620 result = kernel_thread_start_priority((thread_continue_t)thread_terminate_daemon, NULL, MINPRI_KERNEL, &thread);
621 if (result != KERN_SUCCESS)
622 panic("thread_daemon_init: thread_terminate_daemon");
623
624 thread_deallocate(thread);
625
626 simple_lock_init(&thread_stack_lock, 0);
627 queue_init(&thread_stack_queue);
628
629 result = kernel_thread_start_priority((thread_continue_t)thread_stack_daemon, NULL, BASEPRI_PREEMPT, &thread);
630 if (result != KERN_SUCCESS)
631 panic("thread_daemon_init: thread_stack_daemon");
632
633 thread_deallocate(thread);
634 }
635
636 /*
637 * Create a new thread.
638 * Doesn't start the thread running.
639 */
640 static kern_return_t
641 thread_create_internal(
642 task_t parent_task,
643 integer_t priority,
644 thread_continue_t continuation,
645 int options,
646 #define TH_OPTION_NONE 0x00
647 #define TH_OPTION_NOCRED 0x01
648 #define TH_OPTION_NOSUSP 0x02
649 thread_t *out_thread)
650 {
651 thread_t new_thread;
652 static thread_t first_thread;
653
654 /*
655 * Allocate a thread and initialize static fields
656 */
657 if (first_thread == THREAD_NULL)
658 new_thread = first_thread = current_thread();
659 else
660 new_thread = (thread_t)zalloc(thread_zone);
661 if (new_thread == THREAD_NULL)
662 return (KERN_RESOURCE_SHORTAGE);
663
664 if (new_thread != first_thread)
665 *new_thread = thread_template;
666
667 #ifdef MACH_BSD
668 new_thread->uthread = uthread_alloc(parent_task, new_thread, (options & TH_OPTION_NOCRED) != 0);
669 if (new_thread->uthread == NULL) {
670 zfree(thread_zone, new_thread);
671 return (KERN_RESOURCE_SHORTAGE);
672 }
673 #endif /* MACH_BSD */
674
675 if (machine_thread_create(new_thread, parent_task) != KERN_SUCCESS) {
676 #ifdef MACH_BSD
677 void *ut = new_thread->uthread;
678
679 new_thread->uthread = NULL;
680 /* cred free may not be necessary */
681 uthread_cleanup(parent_task, ut, parent_task->bsd_info);
682 uthread_cred_free(ut);
683 uthread_zone_free(ut);
684 #endif /* MACH_BSD */
685
686 zfree(thread_zone, new_thread);
687 return (KERN_FAILURE);
688 }
689
690 new_thread->task = parent_task;
691
692 thread_lock_init(new_thread);
693 wake_lock_init(new_thread);
694
695 lck_mtx_init(&new_thread->mutex, &thread_lck_grp, &thread_lck_attr);
696
697 ipc_thread_init(new_thread);
698 queue_init(&new_thread->held_ulocks);
699
700 new_thread->continuation = continuation;
701
702 lck_mtx_lock(&tasks_threads_lock);
703 task_lock(parent_task);
704
705 if ( !parent_task->active || parent_task->halting ||
706 ((options & TH_OPTION_NOSUSP) != 0 &&
707 parent_task->suspend_count > 0) ||
708 (parent_task->thread_count >= task_threadmax &&
709 parent_task != kernel_task) ) {
710 task_unlock(parent_task);
711 lck_mtx_unlock(&tasks_threads_lock);
712
713 #ifdef MACH_BSD
714 {
715 void *ut = new_thread->uthread;
716
717 new_thread->uthread = NULL;
718 uthread_cleanup(parent_task, ut, parent_task->bsd_info);
719 /* cred free may not be necessary */
720 uthread_cred_free(ut);
721 uthread_zone_free(ut);
722 }
723 #endif /* MACH_BSD */
724 ipc_thread_disable(new_thread);
725 ipc_thread_terminate(new_thread);
726 lck_mtx_destroy(&new_thread->mutex, &thread_lck_grp);
727 machine_thread_destroy(new_thread);
728 zfree(thread_zone, new_thread);
729 return (KERN_FAILURE);
730 }
731
732 /* New threads inherit any default state on the task */
733 machine_thread_inherit_taskwide(new_thread, parent_task);
734
735 task_reference_internal(parent_task);
736
737 if (new_thread->task->rusage_cpu_flags & TASK_RUSECPU_FLAGS_PERTHR_LIMIT) {
738 /*
739 * This task has a per-thread CPU limit; make sure this new thread
740 * gets its limit set too, before it gets out of the kernel.
741 */
742 set_astledger(new_thread);
743 }
744 new_thread->t_threadledger = LEDGER_NULL; /* per thread ledger is not inherited */
745 new_thread->t_ledger = new_thread->task->ledger;
746 if (new_thread->t_ledger)
747 ledger_reference(new_thread->t_ledger);
748
749 /* Cache the task's map */
750 new_thread->map = parent_task->map;
751
752 /* Chain the thread onto the task's list */
753 queue_enter(&parent_task->threads, new_thread, thread_t, task_threads);
754 parent_task->thread_count++;
755
756 /* So terminating threads don't need to take the task lock to decrement */
757 hw_atomic_add(&parent_task->active_thread_count, 1);
758
759 /* Protected by the tasks_threads_lock */
760 new_thread->thread_id = ++thread_unique_id;
761
762 queue_enter(&threads, new_thread, thread_t, threads);
763 threads_count++;
764
765 timer_call_setup(&new_thread->wait_timer, thread_timer_expire, new_thread);
766 timer_call_setup(&new_thread->depress_timer, thread_depress_expire, new_thread);
767
768 #if CONFIG_COUNTERS
769 /*
770 * If parent task has any reservations, they need to be propagated to this
771 * thread.
772 */
773 new_thread->t_chud = (TASK_PMC_FLAG == (parent_task->t_chud & TASK_PMC_FLAG)) ?
774 THREAD_PMC_FLAG : 0U;
775 #endif
776
777 /* Set the thread's scheduling parameters */
778 new_thread->sched_mode = SCHED(initial_thread_sched_mode)(parent_task);
779 new_thread->sched_flags = 0;
780 new_thread->max_priority = parent_task->max_priority;
781 new_thread->task_priority = parent_task->priority;
782 new_thread->priority = (priority < 0)? parent_task->priority: priority;
783 if (new_thread->priority > new_thread->max_priority)
784 new_thread->priority = new_thread->max_priority;
785 #if CONFIG_EMBEDDED
786 if (new_thread->priority < MAXPRI_THROTTLE) {
787 new_thread->priority = MAXPRI_THROTTLE;
788 }
789 #endif /* CONFIG_EMBEDDED */
790 new_thread->importance =
791 new_thread->priority - new_thread->task_priority;
792 #if CONFIG_EMBEDDED
793 new_thread->saved_importance = new_thread->importance;
794 /* apple ios daemon starts all threads in darwin background */
795 if (parent_task->ext_appliedstate.apptype == PROC_POLICY_IOS_APPLE_DAEMON) {
796 /* Cannot use generic routines here so apply darwin bacground directly */
797 new_thread->policystate.hw_bg = TASK_POLICY_BACKGROUND_ATTRIBUTE_ALL;
798 /* set thread self backgrounding */
799 new_thread->appliedstate.hw_bg = new_thread->policystate.hw_bg;
800 /* priority will get recomputed suitably bit later */
801 new_thread->importance = INT_MIN;
802 /* to avoid changes to many pri compute routines, set the effect of those here */
803 new_thread->priority = MAXPRI_THROTTLE;
804 }
805 #endif /* CONFIG_EMBEDDED */
806
807 #if defined(CONFIG_SCHED_TRADITIONAL)
808 new_thread->sched_stamp = sched_tick;
809 new_thread->pri_shift = sched_pri_shift;
810 #endif
811 SCHED(compute_priority)(new_thread, FALSE);
812
813 new_thread->active = TRUE;
814
815 *out_thread = new_thread;
816
817 {
818 long dbg_arg1, dbg_arg2, dbg_arg3, dbg_arg4;
819
820 kdbg_trace_data(parent_task->bsd_info, &dbg_arg2);
821
822 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
823 TRACEDBG_CODE(DBG_TRACE_DATA, 1) | DBG_FUNC_NONE,
824 (vm_address_t)(uintptr_t)thread_tid(new_thread), dbg_arg2, 0, 0, 0);
825
826 kdbg_trace_string(parent_task->bsd_info,
827 &dbg_arg1, &dbg_arg2, &dbg_arg3, &dbg_arg4);
828
829 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
830 TRACEDBG_CODE(DBG_TRACE_STRING, 1) | DBG_FUNC_NONE,
831 dbg_arg1, dbg_arg2, dbg_arg3, dbg_arg4, 0);
832 }
833
834 DTRACE_PROC1(lwp__create, thread_t, *out_thread);
835
836 return (KERN_SUCCESS);
837 }
838
839 static kern_return_t
840 thread_create_internal2(
841 task_t task,
842 thread_t *new_thread,
843 boolean_t from_user)
844 {
845 kern_return_t result;
846 thread_t thread;
847
848 if (task == TASK_NULL || task == kernel_task)
849 return (KERN_INVALID_ARGUMENT);
850
851 result = thread_create_internal(task, -1, (thread_continue_t)thread_bootstrap_return, TH_OPTION_NONE, &thread);
852 if (result != KERN_SUCCESS)
853 return (result);
854
855 thread->user_stop_count = 1;
856 thread_hold(thread);
857 if (task->suspend_count > 0)
858 thread_hold(thread);
859
860 if (from_user)
861 extmod_statistics_incr_thread_create(task);
862
863 task_unlock(task);
864 lck_mtx_unlock(&tasks_threads_lock);
865
866 *new_thread = thread;
867
868 return (KERN_SUCCESS);
869 }
870
871 /* No prototype, since task_server.h has the _from_user version if KERNEL_SERVER */
872 kern_return_t
873 thread_create(
874 task_t task,
875 thread_t *new_thread);
876
877 kern_return_t
878 thread_create(
879 task_t task,
880 thread_t *new_thread)
881 {
882 return thread_create_internal2(task, new_thread, FALSE);
883 }
884
885 kern_return_t
886 thread_create_from_user(
887 task_t task,
888 thread_t *new_thread)
889 {
890 return thread_create_internal2(task, new_thread, TRUE);
891 }
892
893 static kern_return_t
894 thread_create_running_internal2(
895 register task_t task,
896 int flavor,
897 thread_state_t new_state,
898 mach_msg_type_number_t new_state_count,
899 thread_t *new_thread,
900 boolean_t from_user)
901 {
902 register kern_return_t result;
903 thread_t thread;
904
905 if (task == TASK_NULL || task == kernel_task)
906 return (KERN_INVALID_ARGUMENT);
907
908 result = thread_create_internal(task, -1, (thread_continue_t)thread_bootstrap_return, TH_OPTION_NONE, &thread);
909 if (result != KERN_SUCCESS)
910 return (result);
911
912 result = machine_thread_set_state(
913 thread, flavor, new_state, new_state_count);
914 if (result != KERN_SUCCESS) {
915 task_unlock(task);
916 lck_mtx_unlock(&tasks_threads_lock);
917
918 thread_terminate(thread);
919 thread_deallocate(thread);
920 return (result);
921 }
922
923 thread_mtx_lock(thread);
924 thread_start_internal(thread);
925 thread_mtx_unlock(thread);
926
927 if (from_user)
928 extmod_statistics_incr_thread_create(task);
929
930 task_unlock(task);
931 lck_mtx_unlock(&tasks_threads_lock);
932
933 *new_thread = thread;
934
935 return (result);
936 }
937
938 /* Prototype, see justification above */
939 kern_return_t
940 thread_create_running(
941 register task_t task,
942 int flavor,
943 thread_state_t new_state,
944 mach_msg_type_number_t new_state_count,
945 thread_t *new_thread);
946
947 kern_return_t
948 thread_create_running(
949 register task_t task,
950 int flavor,
951 thread_state_t new_state,
952 mach_msg_type_number_t new_state_count,
953 thread_t *new_thread)
954 {
955 return thread_create_running_internal2(
956 task, flavor, new_state, new_state_count,
957 new_thread, FALSE);
958 }
959
960 kern_return_t
961 thread_create_running_from_user(
962 register task_t task,
963 int flavor,
964 thread_state_t new_state,
965 mach_msg_type_number_t new_state_count,
966 thread_t *new_thread)
967 {
968 return thread_create_running_internal2(
969 task, flavor, new_state, new_state_count,
970 new_thread, TRUE);
971 }
972
973 kern_return_t
974 thread_create_workq(
975 task_t task,
976 thread_continue_t thread_return,
977 thread_t *new_thread)
978 {
979 kern_return_t result;
980 thread_t thread;
981
982 if (task == TASK_NULL || task == kernel_task)
983 return (KERN_INVALID_ARGUMENT);
984
985 result = thread_create_internal(task, -1, thread_return, TH_OPTION_NOCRED | TH_OPTION_NOSUSP, &thread);
986 if (result != KERN_SUCCESS)
987 return (result);
988
989 thread->user_stop_count = 1;
990 thread_hold(thread);
991 if (task->suspend_count > 0)
992 thread_hold(thread);
993
994 task_unlock(task);
995 lck_mtx_unlock(&tasks_threads_lock);
996
997 *new_thread = thread;
998
999 return (KERN_SUCCESS);
1000 }
1001
1002 /*
1003 * kernel_thread_create:
1004 *
1005 * Create a thread in the kernel task
1006 * to execute in kernel context.
1007 */
1008 kern_return_t
1009 kernel_thread_create(
1010 thread_continue_t continuation,
1011 void *parameter,
1012 integer_t priority,
1013 thread_t *new_thread)
1014 {
1015 kern_return_t result;
1016 thread_t thread;
1017 task_t task = kernel_task;
1018
1019 result = thread_create_internal(task, priority, continuation, TH_OPTION_NONE, &thread);
1020 if (result != KERN_SUCCESS)
1021 return (result);
1022
1023 task_unlock(task);
1024 lck_mtx_unlock(&tasks_threads_lock);
1025
1026 stack_alloc(thread);
1027 assert(thread->kernel_stack != 0);
1028 #if CONFIG_EMBEDDED
1029 if (priority > BASEPRI_KERNEL)
1030 #endif
1031 thread->reserved_stack = thread->kernel_stack;
1032
1033 thread->parameter = parameter;
1034
1035 if(debug_task & 1)
1036 kprintf("kernel_thread_create: thread = %p continuation = %p\n", thread, continuation);
1037 *new_thread = thread;
1038
1039 return (result);
1040 }
1041
1042 kern_return_t
1043 kernel_thread_start_priority(
1044 thread_continue_t continuation,
1045 void *parameter,
1046 integer_t priority,
1047 thread_t *new_thread)
1048 {
1049 kern_return_t result;
1050 thread_t thread;
1051
1052 result = kernel_thread_create(continuation, parameter, priority, &thread);
1053 if (result != KERN_SUCCESS)
1054 return (result);
1055
1056 *new_thread = thread;
1057
1058 thread_mtx_lock(thread);
1059 thread_start_internal(thread);
1060 thread_mtx_unlock(thread);
1061
1062 return (result);
1063 }
1064
1065 kern_return_t
1066 kernel_thread_start(
1067 thread_continue_t continuation,
1068 void *parameter,
1069 thread_t *new_thread)
1070 {
1071 return kernel_thread_start_priority(continuation, parameter, -1, new_thread);
1072 }
1073
1074 #if defined(__i386__)
1075
1076 thread_t
1077 kernel_thread(
1078 task_t task,
1079 void (*start)(void))
1080 {
1081 kern_return_t result;
1082 thread_t thread;
1083
1084 if (task != kernel_task)
1085 panic("kernel_thread");
1086
1087 result = kernel_thread_start_priority((thread_continue_t)start, NULL, -1, &thread);
1088 if (result != KERN_SUCCESS)
1089 return (THREAD_NULL);
1090
1091 thread_deallocate(thread);
1092
1093 return (thread);
1094 }
1095
1096 #endif /* defined(__i386__) */
1097
1098 kern_return_t
1099 thread_info_internal(
1100 register thread_t thread,
1101 thread_flavor_t flavor,
1102 thread_info_t thread_info_out, /* ptr to OUT array */
1103 mach_msg_type_number_t *thread_info_count) /*IN/OUT*/
1104 {
1105 int state, flags;
1106 spl_t s;
1107
1108 if (thread == THREAD_NULL)
1109 return (KERN_INVALID_ARGUMENT);
1110
1111 if (flavor == THREAD_BASIC_INFO) {
1112 register thread_basic_info_t basic_info;
1113
1114 if (*thread_info_count < THREAD_BASIC_INFO_COUNT)
1115 return (KERN_INVALID_ARGUMENT);
1116
1117 basic_info = (thread_basic_info_t) thread_info_out;
1118
1119 s = splsched();
1120 thread_lock(thread);
1121
1122 /* fill in info */
1123
1124 thread_read_times(thread, &basic_info->user_time,
1125 &basic_info->system_time);
1126
1127 /*
1128 * Update lazy-evaluated scheduler info because someone wants it.
1129 */
1130 if (SCHED(can_update_priority)(thread))
1131 SCHED(update_priority)(thread);
1132
1133 basic_info->sleep_time = 0;
1134
1135 /*
1136 * To calculate cpu_usage, first correct for timer rate,
1137 * then for 5/8 ageing. The correction factor [3/5] is
1138 * (1/(5/8) - 1).
1139 */
1140 basic_info->cpu_usage = 0;
1141 #if defined(CONFIG_SCHED_TRADITIONAL)
1142 if (sched_tick_interval) {
1143 basic_info->cpu_usage = (integer_t)(((uint64_t)thread->cpu_usage
1144 * TH_USAGE_SCALE) / sched_tick_interval);
1145 basic_info->cpu_usage = (basic_info->cpu_usage * 3) / 5;
1146 }
1147 #endif
1148
1149 if (basic_info->cpu_usage > TH_USAGE_SCALE)
1150 basic_info->cpu_usage = TH_USAGE_SCALE;
1151
1152 basic_info->policy = ((thread->sched_mode == TH_MODE_TIMESHARE)?
1153 POLICY_TIMESHARE: POLICY_RR);
1154
1155 flags = 0;
1156 if (thread->bound_processor != PROCESSOR_NULL && thread->bound_processor->idle_thread == thread)
1157 flags |= TH_FLAGS_IDLE;
1158
1159 if (!thread->kernel_stack)
1160 flags |= TH_FLAGS_SWAPPED;
1161
1162 state = 0;
1163 if (thread->state & TH_TERMINATE)
1164 state = TH_STATE_HALTED;
1165 else
1166 if (thread->state & TH_RUN)
1167 state = TH_STATE_RUNNING;
1168 else
1169 if (thread->state & TH_UNINT)
1170 state = TH_STATE_UNINTERRUPTIBLE;
1171 else
1172 if (thread->state & TH_SUSP)
1173 state = TH_STATE_STOPPED;
1174 else
1175 if (thread->state & TH_WAIT)
1176 state = TH_STATE_WAITING;
1177
1178 basic_info->run_state = state;
1179 basic_info->flags = flags;
1180
1181 basic_info->suspend_count = thread->user_stop_count;
1182
1183 thread_unlock(thread);
1184 splx(s);
1185
1186 *thread_info_count = THREAD_BASIC_INFO_COUNT;
1187
1188 return (KERN_SUCCESS);
1189 }
1190 else
1191 if (flavor == THREAD_IDENTIFIER_INFO) {
1192 register thread_identifier_info_t identifier_info;
1193
1194 if (*thread_info_count < THREAD_IDENTIFIER_INFO_COUNT)
1195 return (KERN_INVALID_ARGUMENT);
1196
1197 identifier_info = (thread_identifier_info_t) thread_info_out;
1198
1199 s = splsched();
1200 thread_lock(thread);
1201
1202 identifier_info->thread_id = thread->thread_id;
1203 identifier_info->thread_handle = thread->machine.cthread_self;
1204 if(thread->task->bsd_info) {
1205 identifier_info->dispatch_qaddr = identifier_info->thread_handle + get_dispatchqueue_offset_from_proc(thread->task->bsd_info);
1206 } else {
1207 thread_unlock(thread);
1208 splx(s);
1209 return KERN_INVALID_ARGUMENT;
1210 }
1211
1212 thread_unlock(thread);
1213 splx(s);
1214 return KERN_SUCCESS;
1215 }
1216 else
1217 if (flavor == THREAD_SCHED_TIMESHARE_INFO) {
1218 policy_timeshare_info_t ts_info;
1219
1220 if (*thread_info_count < POLICY_TIMESHARE_INFO_COUNT)
1221 return (KERN_INVALID_ARGUMENT);
1222
1223 ts_info = (policy_timeshare_info_t)thread_info_out;
1224
1225 s = splsched();
1226 thread_lock(thread);
1227
1228 if (thread->sched_mode != TH_MODE_TIMESHARE) {
1229 thread_unlock(thread);
1230 splx(s);
1231
1232 return (KERN_INVALID_POLICY);
1233 }
1234
1235 ts_info->depressed = (thread->sched_flags & TH_SFLAG_DEPRESSED_MASK) != 0;
1236 if (ts_info->depressed) {
1237 ts_info->base_priority = DEPRESSPRI;
1238 ts_info->depress_priority = thread->priority;
1239 }
1240 else {
1241 ts_info->base_priority = thread->priority;
1242 ts_info->depress_priority = -1;
1243 }
1244
1245 ts_info->cur_priority = thread->sched_pri;
1246 ts_info->max_priority = thread->max_priority;
1247
1248 thread_unlock(thread);
1249 splx(s);
1250
1251 *thread_info_count = POLICY_TIMESHARE_INFO_COUNT;
1252
1253 return (KERN_SUCCESS);
1254 }
1255 else
1256 if (flavor == THREAD_SCHED_FIFO_INFO) {
1257 if (*thread_info_count < POLICY_FIFO_INFO_COUNT)
1258 return (KERN_INVALID_ARGUMENT);
1259
1260 return (KERN_INVALID_POLICY);
1261 }
1262 else
1263 if (flavor == THREAD_SCHED_RR_INFO) {
1264 policy_rr_info_t rr_info;
1265 uint32_t quantum_time;
1266 uint64_t quantum_ns;
1267
1268 if (*thread_info_count < POLICY_RR_INFO_COUNT)
1269 return (KERN_INVALID_ARGUMENT);
1270
1271 rr_info = (policy_rr_info_t) thread_info_out;
1272
1273 s = splsched();
1274 thread_lock(thread);
1275
1276 if (thread->sched_mode == TH_MODE_TIMESHARE) {
1277 thread_unlock(thread);
1278 splx(s);
1279
1280 return (KERN_INVALID_POLICY);
1281 }
1282
1283 rr_info->depressed = (thread->sched_flags & TH_SFLAG_DEPRESSED_MASK) != 0;
1284 if (rr_info->depressed) {
1285 rr_info->base_priority = DEPRESSPRI;
1286 rr_info->depress_priority = thread->priority;
1287 }
1288 else {
1289 rr_info->base_priority = thread->priority;
1290 rr_info->depress_priority = -1;
1291 }
1292
1293 quantum_time = SCHED(initial_quantum_size)(THREAD_NULL);
1294 absolutetime_to_nanoseconds(quantum_time, &quantum_ns);
1295
1296 rr_info->max_priority = thread->max_priority;
1297 rr_info->quantum = (uint32_t)(quantum_ns / 1000 / 1000);
1298
1299 thread_unlock(thread);
1300 splx(s);
1301
1302 *thread_info_count = POLICY_RR_INFO_COUNT;
1303
1304 return (KERN_SUCCESS);
1305 }
1306
1307 return (KERN_INVALID_ARGUMENT);
1308 }
1309
1310 void
1311 thread_read_times(
1312 thread_t thread,
1313 time_value_t *user_time,
1314 time_value_t *system_time)
1315 {
1316 clock_sec_t secs;
1317 clock_usec_t usecs;
1318 uint64_t tval_user, tval_system;
1319
1320 tval_user = timer_grab(&thread->user_timer);
1321 tval_system = timer_grab(&thread->system_timer);
1322
1323 if (thread->precise_user_kernel_time) {
1324 absolutetime_to_microtime(tval_user, &secs, &usecs);
1325 user_time->seconds = (typeof(user_time->seconds))secs;
1326 user_time->microseconds = usecs;
1327
1328 absolutetime_to_microtime(tval_system, &secs, &usecs);
1329 system_time->seconds = (typeof(system_time->seconds))secs;
1330 system_time->microseconds = usecs;
1331 } else {
1332 /* system_timer may represent either sys or user */
1333 tval_user += tval_system;
1334 absolutetime_to_microtime(tval_user, &secs, &usecs);
1335 user_time->seconds = (typeof(user_time->seconds))secs;
1336 user_time->microseconds = usecs;
1337
1338 system_time->seconds = 0;
1339 system_time->microseconds = 0;
1340 }
1341 }
1342
1343 kern_return_t
1344 thread_assign(
1345 __unused thread_t thread,
1346 __unused processor_set_t new_pset)
1347 {
1348 return (KERN_FAILURE);
1349 }
1350
1351 /*
1352 * thread_assign_default:
1353 *
1354 * Special version of thread_assign for assigning threads to default
1355 * processor set.
1356 */
1357 kern_return_t
1358 thread_assign_default(
1359 thread_t thread)
1360 {
1361 return (thread_assign(thread, &pset0));
1362 }
1363
1364 /*
1365 * thread_get_assignment
1366 *
1367 * Return current assignment for this thread.
1368 */
1369 kern_return_t
1370 thread_get_assignment(
1371 thread_t thread,
1372 processor_set_t *pset)
1373 {
1374 if (thread == NULL)
1375 return (KERN_INVALID_ARGUMENT);
1376
1377 *pset = &pset0;
1378
1379 return (KERN_SUCCESS);
1380 }
1381
1382 /*
1383 * thread_wire_internal:
1384 *
1385 * Specify that the target thread must always be able
1386 * to run and to allocate memory.
1387 */
1388 kern_return_t
1389 thread_wire_internal(
1390 host_priv_t host_priv,
1391 thread_t thread,
1392 boolean_t wired,
1393 boolean_t *prev_state)
1394 {
1395 if (host_priv == NULL || thread != current_thread())
1396 return (KERN_INVALID_ARGUMENT);
1397
1398 assert(host_priv == &realhost);
1399
1400 if (prev_state)
1401 *prev_state = (thread->options & TH_OPT_VMPRIV) != 0;
1402
1403 if (wired) {
1404 if (!(thread->options & TH_OPT_VMPRIV))
1405 vm_page_free_reserve(1); /* XXX */
1406 thread->options |= TH_OPT_VMPRIV;
1407 }
1408 else {
1409 if (thread->options & TH_OPT_VMPRIV)
1410 vm_page_free_reserve(-1); /* XXX */
1411 thread->options &= ~TH_OPT_VMPRIV;
1412 }
1413
1414 return (KERN_SUCCESS);
1415 }
1416
1417
1418 /*
1419 * thread_wire:
1420 *
1421 * User-api wrapper for thread_wire_internal()
1422 */
1423 kern_return_t
1424 thread_wire(
1425 host_priv_t host_priv,
1426 thread_t thread,
1427 boolean_t wired)
1428 {
1429 return (thread_wire_internal(host_priv, thread, wired, NULL));
1430 }
1431
1432 static void
1433 thread_resource_exception(const void *arg0, __unused const void *arg1)
1434 {
1435 thread_t thread = current_thread();
1436 int code = (int)((uintptr_t)arg0 & ((int)-1));
1437
1438 assert(thread->t_threadledger != LEDGER_NULL);
1439
1440 /*
1441 * Disable the exception notification so we don't overwhelm
1442 * the listener with an endless stream of redundant exceptions.
1443 */
1444 ledger_set_action(thread->t_threadledger, thread_ledgers.cpu_time,
1445 LEDGER_ACTION_IGNORE);
1446 ledger_disable_callback(thread->t_threadledger, thread_ledgers.cpu_time);
1447
1448 /* XXX code should eventually be a user-exported namespace of resources */
1449 (void) task_exception_notify(EXC_RESOURCE, code, 0);
1450 }
1451
1452 void
1453 init_thread_ledgers(void) {
1454 ledger_template_t t;
1455 int idx;
1456
1457 assert(thread_ledger_template == NULL);
1458
1459 if ((t = ledger_template_create("Per-thread ledger")) == NULL)
1460 panic("couldn't create thread ledger template");
1461
1462 if ((idx = ledger_entry_add(t, "cpu_time", "sched", "ns")) < 0) {
1463 panic("couldn't create cpu_time entry for thread ledger template");
1464 }
1465
1466 if (ledger_set_callback(t, idx, thread_resource_exception,
1467 (void *)(uintptr_t)idx, NULL) < 0) {
1468 panic("couldn't set thread ledger callback for cpu_time entry");
1469 }
1470
1471 thread_ledgers.cpu_time = idx;
1472 thread_ledger_template = t;
1473 }
1474
1475 /*
1476 * Set CPU usage limit on a thread.
1477 *
1478 * Calling with percentage of 0 will unset the limit for this thread.
1479 */
1480
1481 int
1482 thread_set_cpulimit(int action, uint8_t percentage, uint64_t interval_ns)
1483 {
1484 thread_t thread = current_thread();
1485 ledger_t l;
1486 uint64_t limittime = 0;
1487 uint64_t abstime = 0;
1488
1489 assert(percentage <= 100);
1490
1491 if (percentage == 0) {
1492 /*
1493 * Remove CPU limit, if any exists.
1494 */
1495 if (thread->t_threadledger != LEDGER_NULL) {
1496 /*
1497 * The only way to get a per-thread ledger is via CPU limits.
1498 */
1499 assert(thread->options & (TH_OPT_PROC_CPULIMIT | TH_OPT_PRVT_CPULIMIT));
1500 ledger_dereference(thread->t_threadledger);
1501 thread->t_threadledger = LEDGER_NULL;
1502 thread->options &= ~(TH_OPT_PROC_CPULIMIT | TH_OPT_PRVT_CPULIMIT);
1503 }
1504
1505 return (0);
1506 }
1507
1508 l = thread->t_threadledger;
1509 if (l == LEDGER_NULL) {
1510 /*
1511 * This thread doesn't yet have a per-thread ledger; so create one with the CPU time entry active.
1512 */
1513 if ((l = ledger_instantiate(thread_ledger_template, LEDGER_CREATE_INACTIVE_ENTRIES)) == LEDGER_NULL)
1514 return (KERN_RESOURCE_SHORTAGE);
1515
1516 /*
1517 * We are the first to create this thread's ledger, so only activate our entry.
1518 */
1519 ledger_entry_setactive(l, thread_ledgers.cpu_time);
1520 thread->t_threadledger = l;
1521 }
1522
1523 /*
1524 * The limit is specified as a percentage of CPU over an interval in nanoseconds.
1525 * Calculate the amount of CPU time that the thread needs to consume in order to hit the limit.
1526 */
1527 limittime = (interval_ns * percentage) / 100;
1528 nanoseconds_to_absolutetime(limittime, &abstime);
1529 ledger_set_limit(l, thread_ledgers.cpu_time, abstime);
1530 /*
1531 * Refill the thread's allotted CPU time every interval_ns nanoseconds.
1532 */
1533 ledger_set_period(l, thread_ledgers.cpu_time, interval_ns);
1534
1535 /*
1536 * Ledgers supports multiple actions for one ledger entry, so we do too.
1537 */
1538 if (action == THREAD_CPULIMIT_EXCEPTION) {
1539 thread->options |= TH_OPT_PROC_CPULIMIT;
1540 ledger_set_action(l, thread_ledgers.cpu_time, LEDGER_ACTION_EXCEPTION);
1541 }
1542
1543 if (action == THREAD_CPULIMIT_BLOCK) {
1544 thread->options |= TH_OPT_PRVT_CPULIMIT;
1545 /* The per-thread ledger template by default has a callback for CPU time */
1546 ledger_disable_callback(l, thread_ledgers.cpu_time);
1547 ledger_set_action(l, thread_ledgers.cpu_time, LEDGER_ACTION_BLOCK);
1548 }
1549
1550 thread->t_threadledger = l;
1551 return (0);
1552 }
1553
1554 int split_funnel_off = 0;
1555 lck_grp_t *funnel_lck_grp = LCK_GRP_NULL;
1556 lck_grp_attr_t *funnel_lck_grp_attr;
1557 lck_attr_t *funnel_lck_attr;
1558
1559 funnel_t *
1560 funnel_alloc(
1561 int type)
1562 {
1563 lck_mtx_t *m;
1564 funnel_t *fnl;
1565
1566 if (funnel_lck_grp == LCK_GRP_NULL) {
1567 funnel_lck_grp_attr = lck_grp_attr_alloc_init();
1568
1569 funnel_lck_grp = lck_grp_alloc_init("Funnel", funnel_lck_grp_attr);
1570
1571 funnel_lck_attr = lck_attr_alloc_init();
1572 }
1573 if ((fnl = (funnel_t *)kalloc(sizeof(funnel_t))) != 0){
1574 bzero((void *)fnl, sizeof(funnel_t));
1575 if ((m = lck_mtx_alloc_init(funnel_lck_grp, funnel_lck_attr)) == (lck_mtx_t *)NULL) {
1576 kfree(fnl, sizeof(funnel_t));
1577 return(THR_FUNNEL_NULL);
1578 }
1579 fnl->fnl_mutex = m;
1580 fnl->fnl_type = type;
1581 }
1582 return(fnl);
1583 }
1584
1585 void
1586 funnel_free(
1587 funnel_t * fnl)
1588 {
1589 lck_mtx_free(fnl->fnl_mutex, funnel_lck_grp);
1590 if (fnl->fnl_oldmutex)
1591 lck_mtx_free(fnl->fnl_oldmutex, funnel_lck_grp);
1592 kfree(fnl, sizeof(funnel_t));
1593 }
1594
1595 void
1596 funnel_lock(
1597 funnel_t * fnl)
1598 {
1599 lck_mtx_lock(fnl->fnl_mutex);
1600 fnl->fnl_mtxholder = current_thread();
1601 }
1602
1603 void
1604 funnel_unlock(
1605 funnel_t * fnl)
1606 {
1607 lck_mtx_unlock(fnl->fnl_mutex);
1608 fnl->fnl_mtxholder = NULL;
1609 fnl->fnl_mtxrelease = current_thread();
1610 }
1611
1612 funnel_t *
1613 thread_funnel_get(
1614 void)
1615 {
1616 thread_t th = current_thread();
1617
1618 if (th->funnel_state & TH_FN_OWNED) {
1619 return(th->funnel_lock);
1620 }
1621 return(THR_FUNNEL_NULL);
1622 }
1623
1624 boolean_t
1625 thread_funnel_set(
1626 funnel_t * fnl,
1627 boolean_t funneled)
1628 {
1629 thread_t cur_thread;
1630 boolean_t funnel_state_prev;
1631 boolean_t intr;
1632
1633 cur_thread = current_thread();
1634 funnel_state_prev = ((cur_thread->funnel_state & TH_FN_OWNED) == TH_FN_OWNED);
1635
1636 if (funnel_state_prev != funneled) {
1637 intr = ml_set_interrupts_enabled(FALSE);
1638
1639 if (funneled == TRUE) {
1640 if (cur_thread->funnel_lock)
1641 panic("Funnel lock called when holding one %p", cur_thread->funnel_lock);
1642 KERNEL_DEBUG(0x6032428 | DBG_FUNC_NONE,
1643 fnl, 1, 0, 0, 0);
1644 funnel_lock(fnl);
1645 KERNEL_DEBUG(0x6032434 | DBG_FUNC_NONE,
1646 fnl, 1, 0, 0, 0);
1647 cur_thread->funnel_state |= TH_FN_OWNED;
1648 cur_thread->funnel_lock = fnl;
1649 } else {
1650 if(cur_thread->funnel_lock->fnl_mutex != fnl->fnl_mutex)
1651 panic("Funnel unlock when not holding funnel");
1652 cur_thread->funnel_state &= ~TH_FN_OWNED;
1653 KERNEL_DEBUG(0x603242c | DBG_FUNC_NONE,
1654 fnl, 1, 0, 0, 0);
1655
1656 cur_thread->funnel_lock = THR_FUNNEL_NULL;
1657 funnel_unlock(fnl);
1658 }
1659 (void)ml_set_interrupts_enabled(intr);
1660 } else {
1661 /* if we are trying to acquire funnel recursively
1662 * check for funnel to be held already
1663 */
1664 if (funneled && (fnl->fnl_mutex != cur_thread->funnel_lock->fnl_mutex)) {
1665 panic("thread_funnel_set: already holding a different funnel");
1666 }
1667 }
1668 return(funnel_state_prev);
1669 }
1670
1671 static void
1672 sched_call_null(
1673 __unused int type,
1674 __unused thread_t thread)
1675 {
1676 return;
1677 }
1678
1679 void
1680 thread_sched_call(
1681 thread_t thread,
1682 sched_call_t call)
1683 {
1684 thread->sched_call = (call != NULL)? call: sched_call_null;
1685 }
1686
1687 void
1688 thread_static_param(
1689 thread_t thread,
1690 boolean_t state)
1691 {
1692 thread_mtx_lock(thread);
1693 thread->static_param = state;
1694 thread_mtx_unlock(thread);
1695 }
1696
1697 uint64_t
1698 thread_tid(
1699 thread_t thread)
1700 {
1701 return (thread != THREAD_NULL? thread->thread_id: 0);
1702 }
1703
1704 uint64_t
1705 thread_dispatchqaddr(
1706 thread_t thread)
1707 {
1708 uint64_t dispatchqueue_addr = 0;
1709 uint64_t thread_handle = 0;
1710
1711 if (thread != THREAD_NULL) {
1712 thread_handle = thread->machine.cthread_self;
1713
1714 if (thread->task->bsd_info)
1715 dispatchqueue_addr = thread_handle + get_dispatchqueue_offset_from_proc(thread->task->bsd_info);
1716 }
1717
1718 return (dispatchqueue_addr);
1719 }
1720
1721 /*
1722 * Export routines to other components for things that are done as macros
1723 * within the osfmk component.
1724 */
1725
1726 #undef thread_reference
1727 void thread_reference(thread_t thread);
1728 void
1729 thread_reference(
1730 thread_t thread)
1731 {
1732 if (thread != THREAD_NULL)
1733 thread_reference_internal(thread);
1734 }
1735
1736 #undef thread_should_halt
1737
1738 boolean_t
1739 thread_should_halt(
1740 thread_t th)
1741 {
1742 return (thread_should_halt_fast(th));
1743 }
1744
1745 #if CONFIG_DTRACE
1746 uint32_t dtrace_get_thread_predcache(thread_t thread)
1747 {
1748 if (thread != THREAD_NULL)
1749 return thread->t_dtrace_predcache;
1750 else
1751 return 0;
1752 }
1753
1754 int64_t dtrace_get_thread_vtime(thread_t thread)
1755 {
1756 if (thread != THREAD_NULL)
1757 return thread->t_dtrace_vtime;
1758 else
1759 return 0;
1760 }
1761
1762 int64_t dtrace_get_thread_tracing(thread_t thread)
1763 {
1764 if (thread != THREAD_NULL)
1765 return thread->t_dtrace_tracing;
1766 else
1767 return 0;
1768 }
1769
1770 boolean_t dtrace_get_thread_reentering(thread_t thread)
1771 {
1772 if (thread != THREAD_NULL)
1773 return (thread->options & TH_OPT_DTRACE) ? TRUE : FALSE;
1774 else
1775 return 0;
1776 }
1777
1778 vm_offset_t dtrace_get_kernel_stack(thread_t thread)
1779 {
1780 if (thread != THREAD_NULL)
1781 return thread->kernel_stack;
1782 else
1783 return 0;
1784 }
1785
1786 int64_t dtrace_calc_thread_recent_vtime(thread_t thread)
1787 {
1788 if (thread != THREAD_NULL) {
1789 processor_t processor = current_processor();
1790 uint64_t abstime = mach_absolute_time();
1791 timer_t timer;
1792
1793 timer = PROCESSOR_DATA(processor, thread_timer);
1794
1795 return timer_grab(&(thread->system_timer)) + timer_grab(&(thread->user_timer)) +
1796 (abstime - timer->tstamp); /* XXX need interrupts off to prevent missed time? */
1797 } else
1798 return 0;
1799 }
1800
1801 void dtrace_set_thread_predcache(thread_t thread, uint32_t predcache)
1802 {
1803 if (thread != THREAD_NULL)
1804 thread->t_dtrace_predcache = predcache;
1805 }
1806
1807 void dtrace_set_thread_vtime(thread_t thread, int64_t vtime)
1808 {
1809 if (thread != THREAD_NULL)
1810 thread->t_dtrace_vtime = vtime;
1811 }
1812
1813 void dtrace_set_thread_tracing(thread_t thread, int64_t accum)
1814 {
1815 if (thread != THREAD_NULL)
1816 thread->t_dtrace_tracing = accum;
1817 }
1818
1819 void dtrace_set_thread_reentering(thread_t thread, boolean_t vbool)
1820 {
1821 if (thread != THREAD_NULL) {
1822 if (vbool)
1823 thread->options |= TH_OPT_DTRACE;
1824 else
1825 thread->options &= (~TH_OPT_DTRACE);
1826 }
1827 }
1828
1829 vm_offset_t dtrace_set_thread_recover(thread_t thread, vm_offset_t recover)
1830 {
1831 vm_offset_t prev = 0;
1832
1833 if (thread != THREAD_NULL) {
1834 prev = thread->recover;
1835 thread->recover = recover;
1836 }
1837 return prev;
1838 }
1839
1840 void dtrace_thread_bootstrap(void)
1841 {
1842 task_t task = current_task();
1843 if(task->thread_count == 1) {
1844 DTRACE_PROC(start);
1845 }
1846 DTRACE_PROC(lwp__start);
1847
1848 }
1849 #endif /* CONFIG_DTRACE */
mirror of XNU