2 * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
4 * @APPLE_LICENSE_HEADER_START@
6 * The contents of this file constitute Original Code as defined in and
7 * are subject to the Apple Public Source License Version 1.1 (the
8 * "License"). You may not use this file except in compliance with the
9 * License. Please obtain a copy of the License at
10 * http://www.apple.com/publicsource and read it before using this file.
12 * This Original Code and all software distributed under the License are
13 * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
14 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
15 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
17 * License for the specific language governing rights and limitations
20 * @APPLE_LICENSE_HEADER_END@
23 * @OSF_FREE_COPYRIGHT@
26 * Mach Operating System
27 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
28 * All Rights Reserved.
30 * Permission to use, copy, modify and distribute this software and its
31 * documentation is hereby granted, provided that both the copyright
32 * notice and this permission notice appear in all copies of the
33 * software, derivative works or modified versions, and any portions
34 * thereof, and that both notices appear in supporting documentation.
36 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
37 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
38 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
40 * Carnegie Mellon requests users of this software to return to
42 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
43 * School of Computer Science
44 * Carnegie Mellon University
45 * Pittsburgh PA 15213-3890
47 * any improvements or extensions that they make and grant Carnegie Mellon
48 * the rights to redistribute these changes.
54 * Author: Avadis Tevanian, Jr., Michael Wayne Young, David Golub
57 * Thread/thread_shuttle management primitives implementation.
60 * Copyright (c) 1993 The University of Utah and
61 * the Computer Systems Laboratory (CSL). All rights reserved.
63 * Permission to use, copy, modify and distribute this software and its
64 * documentation is hereby granted, provided that both the copyright
65 * notice and this permission notice appear in all copies of the
66 * software, derivative works or modified versions, and any portions
67 * thereof, and that both notices appear in supporting documentation.
69 * THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF THIS SOFTWARE IN ITS "AS
70 * IS" CONDITION. THE UNIVERSITY OF UTAH AND CSL DISCLAIM ANY LIABILITY OF
71 * ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
73 * CSL requests users of this software to return to csl-dist@cs.utah.edu any
74 * improvements that they make and grant CSL redistribution rights.
79 #include <mach_host.h>
80 #include <simple_clock.h>
81 #include <mach_debug.h>
82 #include <mach_prof.h>
84 #include <mach/boolean.h>
85 #include <mach/policy.h>
86 #include <mach/thread_info.h>
87 #include <mach/thread_special_ports.h>
88 #include <mach/thread_status.h>
89 #include <mach/time_value.h>
90 #include <mach/vm_param.h>
92 #include <kern/cpu_data.h>
93 #include <kern/counters.h>
94 #include <kern/etap_macros.h>
95 #include <kern/ipc_mig.h>
96 #include <kern/ipc_tt.h>
97 #include <kern/mach_param.h>
98 #include <kern/machine.h>
99 #include <kern/misc_protos.h>
100 #include <kern/processor.h>
101 #include <kern/queue.h>
102 #include <kern/sched.h>
103 #include <kern/sched_prim.h>
104 #include <kern/mk_sp.h> /*** ??? fix so this can be removed ***/
105 #include <kern/task.h>
106 #include <kern/thread.h>
107 #include <kern/thread_act.h>
108 #include <kern/thread_swap.h>
109 #include <kern/host.h>
110 #include <kern/zalloc.h>
111 #include <vm/vm_kern.h>
112 #include <ipc/ipc_kmsg.h>
113 #include <ipc/ipc_port.h>
114 #include <machine/thread.h> /* for MACHINE_STACK */
115 #include <kern/profile.h>
116 #include <kern/assert.h>
117 #include <sys/kdebug.h>
120 * Exported interfaces
123 #include <mach/thread_act_server.h>
124 #include <mach/mach_host_server.h>
126 static struct zone
*thread_zone
;
128 static queue_head_t reaper_queue
;
129 decl_simple_lock_data(static,reaper_lock
)
134 static struct thread thread_template
, init_thread
;
139 extern void stack_statistics(
140 unsigned int *totalp
,
141 vm_size_t
*maxusagep
);
142 #endif /* MACHINE_STACK */
143 #endif /* MACH_DEBUG */
147 * Machine-dependent code must define:
156 #else /* MACHINE_STACK */
158 * We allocate stacks from generic kernel VM.
159 * Machine-dependent code must define:
160 * machine_kernel_stack_init
162 * The stack_free_list can only be accessed at splsched,
163 * because stack_alloc_try/thread_invoke operate at splsched.
166 decl_simple_lock_data(static,stack_lock_data
)
167 #define stack_lock() simple_lock(&stack_lock_data)
168 #define stack_unlock() simple_unlock(&stack_lock_data)
170 static vm_map_t stack_map
;
171 static vm_offset_t stack_free_list
;
173 static vm_offset_t stack_free_cache
[NCPUS
];
175 unsigned int stack_free_max
= 0;
176 unsigned int stack_free_count
= 0; /* splsched only */
177 unsigned int stack_free_limit
= 1; /* Arbitrary */
179 unsigned int stack_cache_hits
= 0; /* debugging */
181 unsigned int stack_alloc_hits
= 0; /* debugging */
182 unsigned int stack_alloc_misses
= 0; /* debugging */
184 unsigned int stack_alloc_total
= 0;
185 unsigned int stack_alloc_hiwater
= 0;
186 unsigned int stack_alloc_bndry
= 0;
190 * The next field is at the base of the stack,
191 * so the low end is left unsullied.
194 #define stack_next(stack) (*((vm_offset_t *)((stack) + KERNEL_STACK_SIZE) - 1))
199 * Allocate a kernel stack for a thread.
205 void (*start_pos
)(thread_t
))
207 vm_offset_t stack
= thread
->kernel_stack
;
215 stack
= stack_free_list
;
217 stack_free_list
= stack_next(stack
);
224 machine_stack_attach(thread
, stack
, start_pos
);
228 if (kernel_memory_allocate(
230 KERNEL_STACK_SIZE
, stack_alloc_bndry
- 1,
231 KMA_KOBJECT
) != KERN_SUCCESS
)
232 panic("stack_alloc: no space left for stack maps");
235 if (stack_alloc_total
> stack_alloc_hiwater
)
236 stack_alloc_hiwater
= stack_alloc_total
;
238 machine_stack_attach(thread
, stack
, start_pos
);
245 * Free a kernel stack.
252 vm_offset_t stack
= machine_stack_detach(thread
);
255 if (stack
!= thread
->reserved_stack
) {
256 spl_t s
= splsched();
259 cache
= &stack_free_cache
[cpu_number()];
268 stack_next(stack
) = stack_free_list
;
269 stack_free_list
= stack
;
270 if (++stack_free_count
> stack_free_max
)
271 stack_free_max
= stack_free_count
;
281 spl_t s
= splsched();
284 cache
= &stack_free_cache
[cpu_number()];
293 stack_next(stack
) = stack_free_list
;
294 stack_free_list
= stack
;
295 if (++stack_free_count
> stack_free_max
)
296 stack_free_max
= stack_free_count
;
304 * Free excess kernel stacks.
311 spl_t s
= splsched();
314 while (stack_free_count
> stack_free_limit
) {
315 vm_offset_t stack
= stack_free_list
;
317 stack_free_list
= stack_next(stack
);
323 stack_map
, stack
, stack
+ KERNEL_STACK_SIZE
,
324 VM_MAP_REMOVE_KUNWIRE
) != KERN_SUCCESS
)
325 panic("stack_collect: vm_map_remove failed");
338 * Non-blocking attempt to allocate a kernel stack.
339 * Called at splsched with the thread locked.
342 boolean_t
stack_alloc_try(
344 void (*start
)(thread_t
))
346 register vm_offset_t stack
, *cache
;
348 cache
= &stack_free_cache
[cpu_number()];
349 if (stack
= *cache
) {
351 machine_stack_attach(thread
, stack
, start
);
358 stack
= stack_free_list
;
359 if (stack
!= (vm_offset_t
)0) {
360 stack_free_list
= stack_next(stack
);
366 stack
= thread
->reserved_stack
;
369 machine_stack_attach(thread
, stack
, start
);
375 stack_alloc_misses
++;
385 * Return statistics on cached kernel stacks.
386 * *maxusagep must be initialized by the caller.
391 unsigned int *totalp
,
392 vm_size_t
*maxusagep
)
399 *totalp
= stack_free_count
;
405 #endif /* MACH_DEBUG */
407 #endif /* MACHINE_STACK */
410 stack_fake_zone_info(int *count
, vm_size_t
*cur_size
, vm_size_t
*max_size
, vm_size_t
*elem_size
,
411 vm_size_t
*alloc_size
, int *collectable
, int *exhaustable
)
413 *count
= stack_alloc_total
- stack_free_count
;
414 *cur_size
= KERNEL_STACK_SIZE
* stack_alloc_total
;
415 *max_size
= KERNEL_STACK_SIZE
* stack_alloc_hiwater
;
416 *elem_size
= KERNEL_STACK_SIZE
;
417 *alloc_size
= KERNEL_STACK_SIZE
;
424 register thread_t thread
)
430 thread_bootstrap(void)
433 * Fill in a template thread for fast initialization.
436 thread_template
.runq
= RUN_QUEUE_NULL
;
438 thread_template
.ref_count
= 1;
440 thread_template
.reason
= AST_NONE
;
441 thread_template
.at_safe_point
= FALSE
;
442 thread_template
.wait_event
= NO_EVENT64
;
443 thread_template
.wait_queue
= WAIT_QUEUE_NULL
;
444 thread_template
.wait_result
= THREAD_WAITING
;
445 thread_template
.interrupt_level
= THREAD_ABORTSAFE
;
446 thread_template
.state
= TH_STACK_HANDOFF
| TH_WAIT
| TH_UNINT
;
447 thread_template
.wake_active
= FALSE
;
448 thread_template
.active_callout
= FALSE
;
449 thread_template
.continuation
= (void (*)(void))0;
450 thread_template
.top_act
= THR_ACT_NULL
;
452 thread_template
.importance
= 0;
453 thread_template
.sched_mode
= 0;
454 thread_template
.safe_mode
= 0;
456 thread_template
.priority
= 0;
457 thread_template
.sched_pri
= 0;
458 thread_template
.max_priority
= 0;
459 thread_template
.task_priority
= 0;
460 thread_template
.promotions
= 0;
461 thread_template
.pending_promoter_index
= 0;
462 thread_template
.pending_promoter
[0] =
463 thread_template
.pending_promoter
[1] = NULL
;
465 thread_template
.realtime
.deadline
= UINT64_MAX
;
467 thread_template
.current_quantum
= 0;
469 thread_template
.computation_metered
= 0;
470 thread_template
.computation_epoch
= 0;
472 thread_template
.cpu_usage
= 0;
473 thread_template
.cpu_delta
= 0;
474 thread_template
.sched_usage
= 0;
475 thread_template
.sched_delta
= 0;
476 thread_template
.sched_stamp
= 0;
477 thread_template
.sleep_stamp
= 0;
478 thread_template
.safe_release
= 0;
480 thread_template
.bound_processor
= PROCESSOR_NULL
;
481 thread_template
.last_processor
= PROCESSOR_NULL
;
482 thread_template
.last_switch
= 0;
484 thread_template
.vm_privilege
= FALSE
;
486 timer_init(&(thread_template
.user_timer
));
487 timer_init(&(thread_template
.system_timer
));
488 thread_template
.user_timer_save
.low
= 0;
489 thread_template
.user_timer_save
.high
= 0;
490 thread_template
.system_timer_save
.low
= 0;
491 thread_template
.system_timer_save
.high
= 0;
493 thread_template
.processor_set
= PROCESSOR_SET_NULL
;
495 thread_template
.act_ref_count
= 2;
497 thread_template
.special_handler
.handler
= special_handler
;
498 thread_template
.special_handler
.next
= 0;
501 thread_template
.may_assign
= TRUE
;
502 thread_template
.assign_active
= FALSE
;
503 #endif /* MACH_HOST */
504 thread_template
.funnel_lock
= THR_FUNNEL_NULL
;
505 thread_template
.funnel_state
= 0;
507 thread_template
.mutex_count
= 0;
508 #endif /* MACH_LDEBUG */
510 init_thread
= thread_template
;
512 init_thread
.top_act
= &init_thread
;
513 init_thread
.thread
= &init_thread
;
514 machine_thread_set_current(&init_thread
);
524 sizeof(struct thread
),
525 THREAD_MAX
* sizeof(struct thread
),
526 THREAD_CHUNK
* sizeof(struct thread
),
530 * Initialize other data structures used in
534 queue_init(&reaper_queue
);
535 simple_lock_init(&reaper_lock
, ETAP_THREAD_REAPER
);
537 #ifndef MACHINE_STACK
538 simple_lock_init(&stack_lock_data
, ETAP_THREAD_STACK
); /* Initialize the stack lock */
540 if (KERNEL_STACK_SIZE
< round_page_32(KERNEL_STACK_SIZE
)) { /* Kernel stacks must be multiples of pages */
541 panic("thread_init: kernel stack size (%08X) must be a multiple of page size (%08X)\n",
542 KERNEL_STACK_SIZE
, PAGE_SIZE
);
545 for(stack_alloc_bndry
= PAGE_SIZE
; stack_alloc_bndry
<= KERNEL_STACK_SIZE
; stack_alloc_bndry
<<= 1); /* Find next power of 2 above stack size */
547 ret
= kmem_suballoc(kernel_map
, /* Suballocate from the kernel map */
550 (stack_alloc_bndry
* (2*THREAD_MAX
+ 64)), /* Allocate enough for all of it */
551 FALSE
, /* Say not pageable so that it is wired */
552 TRUE
, /* Allocate from anywhere */
553 &stack_map
); /* Allocate a submap */
555 if(ret
!= KERN_SUCCESS
) { /* Did we get one? */
556 panic("thread_init: kmem_suballoc for stacks failed - ret = %d\n", ret
); /* Die */
558 stack
= vm_map_min(stack_map
); /* Make sure we skip the first hunk */
559 ret
= vm_map_enter(stack_map
, &stack
, PAGE_SIZE
, 0, /* Make sure there is nothing at the start */
560 0, /* Force it at start */
561 VM_OBJECT_NULL
, 0, /* No object yet */
563 VM_PROT_NONE
, /* Allow no access */
564 VM_PROT_NONE
, /* Allow no access */
565 VM_INHERIT_DEFAULT
); /* Just be normal */
567 if(ret
!= KERN_SUCCESS
) { /* Did it work? */
568 panic("thread_init: dummy alignment allocation failed; ret = %d\n", ret
);
571 #endif /* MACHINE_STACK */
574 * Initialize any machine-dependent
575 * per-thread structures necessary.
577 machine_thread_init();
581 * Called at splsched.
584 thread_reaper_enqueue(
587 simple_lock(&reaper_lock
);
588 enqueue_tail(&reaper_queue
, (queue_entry_t
)thread
);
589 simple_unlock(&reaper_lock
);
591 thread_wakeup((event_t
)&reaper_queue
);
595 thread_termination_continue(void)
597 panic("thread_termination_continue");
602 * Routine: thread_terminate_self
604 * This routine is called by a thread which has unwound from
605 * its current RPC and kernel contexts and found that it's
606 * root activation has been marked for extinction. This lets
607 * it clean up the last few things that can only be cleaned
608 * up in this context and then impale itself on the reaper
611 * When the reaper gets the thread, it will deallocate the
612 * thread_act's reference on itself, which in turn will release
613 * its own reference on this thread. By doing things in that
614 * order, a thread_act will always have a valid thread - but the
615 * thread may persist beyond having a thread_act (but must never
619 thread_terminate_self(void)
621 thread_act_t thr_act
= current_act();
623 task_t task
= thr_act
->task
;
628 * We should be at the base of the inheritance chain.
630 thread
= act_lock_thread(thr_act
);
631 assert(thr_act
->thread
== thread
);
633 /* This will allow no more control ops on this thr_act. */
634 ipc_thr_act_disable(thr_act
);
636 /* Clean-up any ulocks that are still owned by the thread
637 * activation (acquired but not released or handed-off).
639 act_ulock_release_all(thr_act
);
641 act_unlock_thread(thr_act
);
643 _mk_sp_thread_depress_abort(thread
, TRUE
);
646 * Check to see if this is the last active activation. By
647 * this we mean the last activation to call thread_terminate_self.
648 * If so, and the task is associated with a BSD process, we
649 * need to call BSD and let them clean up.
651 active_acts
= hw_atomic_sub(&task
->active_thread_count
, 1);
653 if (active_acts
== 0 && task
->bsd_info
)
654 proc_exit(task
->bsd_info
);
656 /* JMM - for now, no migration */
657 assert(!thr_act
->lower
);
659 thread_timer_terminate();
661 ipc_thread_terminate(thread
);
665 thread
->state
|= TH_TERMINATE
;
666 assert((thread
->state
& TH_UNINT
) == 0);
667 thread_mark_wait_locked(thread
, THREAD_UNINT
);
668 assert(thread
->promotions
== 0);
669 thread_unlock(thread
);
672 ETAP_SET_REASON(thread
, BLOCKED_ON_TERMINATION
);
673 thread_block(thread_termination_continue
);
678 * Create a new thread.
679 * Doesn't start the thread running.
682 thread_create_internal(
686 thread_t
*out_thread
)
689 processor_set_t pset
;
690 static thread_t first_thread
;
693 * Allocate a thread and initialize static fields
695 if (first_thread
== NULL
)
696 new_thread
= first_thread
= current_act();
698 new_thread
= (thread_t
)zalloc(thread_zone
);
699 if (new_thread
== NULL
)
700 return (KERN_RESOURCE_SHORTAGE
);
702 if (new_thread
!= first_thread
)
703 *new_thread
= thread_template
;
707 extern void *uthread_alloc(task_t
, thread_act_t
);
709 new_thread
->uthread
= uthread_alloc(parent_task
, new_thread
);
710 if (new_thread
->uthread
== NULL
) {
711 zfree(thread_zone
, (vm_offset_t
)new_thread
);
712 return (KERN_RESOURCE_SHORTAGE
);
715 #endif /* MACH_BSD */
717 if (machine_thread_create(new_thread
, parent_task
) != KERN_SUCCESS
) {
720 extern void uthread_free(task_t
, void *, void *, void *);
721 void *ut
= new_thread
->uthread
;
723 new_thread
->uthread
= NULL
;
724 uthread_free(parent_task
, (void *)new_thread
, ut
, parent_task
->bsd_info
);
726 #endif /* MACH_BSD */
727 zfree(thread_zone
, (vm_offset_t
)new_thread
);
728 return (KERN_FAILURE
);
731 new_thread
->task
= parent_task
;
733 thread_lock_init(new_thread
);
734 wake_lock_init(new_thread
);
736 mutex_init(&new_thread
->lock
, ETAP_THREAD_ACT
);
738 ipc_thr_act_init(parent_task
, new_thread
);
740 ipc_thread_init(new_thread
);
741 queue_init(&new_thread
->held_ulocks
);
742 act_prof_init(new_thread
, parent_task
);
744 new_thread
->continuation
= start
;
745 new_thread
->sleep_stamp
= sched_tick
;
747 pset
= parent_task
->processor_set
;
748 assert(pset
== &default_pset
);
751 task_lock(parent_task
);
752 assert(parent_task
->processor_set
== pset
);
754 if ( !parent_task
->active
||
755 (parent_task
->thread_count
>= THREAD_MAX
&&
756 parent_task
!= kernel_task
)) {
757 task_unlock(parent_task
);
762 extern void uthread_free(task_t
, void *, void *, void *);
763 void *ut
= new_thread
->uthread
;
765 new_thread
->uthread
= NULL
;
766 uthread_free(parent_task
, (void *)new_thread
, ut
, parent_task
->bsd_info
);
768 #endif /* MACH_BSD */
769 act_prof_deallocate(new_thread
);
770 ipc_thr_act_terminate(new_thread
);
771 machine_thread_destroy(new_thread
);
772 zfree(thread_zone
, (vm_offset_t
) new_thread
);
773 return (KERN_FAILURE
);
776 act_attach(new_thread
, new_thread
);
778 task_reference_locked(parent_task
);
780 /* Cache the task's map */
781 new_thread
->map
= parent_task
->map
;
783 /* Chain the thread onto the task's list */
784 queue_enter(&parent_task
->threads
, new_thread
, thread_act_t
, task_threads
);
785 parent_task
->thread_count
++;
786 parent_task
->res_thread_count
++;
788 /* So terminating threads don't need to take the task lock to decrement */
789 hw_atomic_add(&parent_task
->active_thread_count
, 1);
791 /* Associate the thread with the processor set */
792 pset_add_thread(pset
, new_thread
);
794 thread_timer_setup(new_thread
);
796 /* Set the thread's scheduling parameters */
797 if (parent_task
!= kernel_task
)
798 new_thread
->sched_mode
|= TH_MODE_TIMESHARE
;
799 new_thread
->max_priority
= parent_task
->max_priority
;
800 new_thread
->task_priority
= parent_task
->priority
;
801 new_thread
->priority
= (priority
< 0)? parent_task
->priority
: priority
;
802 if (new_thread
->priority
> new_thread
->max_priority
)
803 new_thread
->priority
= new_thread
->max_priority
;
804 new_thread
->importance
=
805 new_thread
->priority
- new_thread
->task_priority
;
806 new_thread
->sched_stamp
= sched_tick
;
807 compute_priority(new_thread
, FALSE
);
809 #if ETAP_EVENT_MONITOR
810 new_thread
->etap_reason
= 0;
811 new_thread
->etap_trace
= FALSE
;
812 #endif /* ETAP_EVENT_MONITOR */
814 new_thread
->active
= TRUE
;
816 *out_thread
= new_thread
;
819 long dbg_arg1
, dbg_arg2
, dbg_arg3
, dbg_arg4
;
821 kdbg_trace_data(parent_task
->bsd_info
, &dbg_arg2
);
823 KERNEL_DEBUG_CONSTANT(
824 TRACEDBG_CODE(DBG_TRACE_DATA
, 1) | DBG_FUNC_NONE
,
825 (vm_address_t
)new_thread
, dbg_arg2
, 0, 0, 0);
827 kdbg_trace_string(parent_task
->bsd_info
,
828 &dbg_arg1
, &dbg_arg2
, &dbg_arg3
, &dbg_arg4
);
830 KERNEL_DEBUG_CONSTANT(
831 TRACEDBG_CODE(DBG_TRACE_STRING
, 1) | DBG_FUNC_NONE
,
832 dbg_arg1
, dbg_arg2
, dbg_arg3
, dbg_arg4
, 0);
835 return (KERN_SUCCESS
);
838 extern void thread_bootstrap_return(void);
843 thread_act_t
*new_thread
)
845 kern_return_t result
;
848 if (task
== TASK_NULL
|| task
== kernel_task
)
849 return (KERN_INVALID_ARGUMENT
);
851 result
= thread_create_internal(task
, -1, thread_bootstrap_return
, &thread
);
852 if (result
!= KERN_SUCCESS
)
855 thread
->user_stop_count
= 1;
857 if (task
->suspend_count
> 0)
860 pset_unlock(task
->processor_set
);
863 *new_thread
= thread
;
865 return (KERN_SUCCESS
);
869 thread_create_running(
870 register task_t task
,
872 thread_state_t new_state
,
873 mach_msg_type_number_t new_state_count
,
874 thread_act_t
*new_thread
)
876 register kern_return_t result
;
879 if (task
== TASK_NULL
|| task
== kernel_task
)
880 return (KERN_INVALID_ARGUMENT
);
882 result
= thread_create_internal(task
, -1, thread_bootstrap_return
, &thread
);
883 if (result
!= KERN_SUCCESS
)
886 result
= machine_thread_set_state(thread
, flavor
, new_state
, new_state_count
);
887 if (result
!= KERN_SUCCESS
) {
888 pset_unlock(task
->processor_set
);
891 thread_terminate(thread
);
892 act_deallocate(thread
);
897 clear_wait(thread
, THREAD_AWAKENED
);
898 thread
->started
= TRUE
;
900 pset_unlock(task
->processor_set
);
903 *new_thread
= thread
;
911 * Create a thread in the kernel task
912 * to execute in kernel context.
915 kernel_thread_create(
919 kern_return_t result
;
920 task_t task
= kernel_task
;
923 result
= thread_create_internal(task
, priority
, start
, &thread
);
924 if (result
!= KERN_SUCCESS
)
925 return (THREAD_NULL
);
927 pset_unlock(task
->processor_set
);
930 thread_doswapin(thread
);
931 assert(thread
->kernel_stack
!= 0);
932 thread
->reserved_stack
= thread
->kernel_stack
;
934 act_deallocate(thread
);
940 kernel_thread_with_priority(
946 thread
= kernel_thread_create(start
, priority
);
947 if (thread
== THREAD_NULL
)
948 return (THREAD_NULL
);
951 clear_wait(thread
, THREAD_AWAKENED
);
952 thread
->started
= TRUE
;
956 thread_bind(thread
, master_processor
);
966 if (task
!= kernel_task
)
967 panic("kernel_thread");
969 return kernel_thread_with_priority(start
, -1);
972 unsigned int c_weird_pset_ref_exit
= 0; /* pset code raced us */
975 /* Preclude thread processor set assignement */
976 #define thread_freeze(thread) assert((thread)->processor_set == &default_pset)
978 /* Allow thread processor set assignement */
979 #define thread_unfreeze(thread) assert((thread)->processor_set == &default_pset)
981 #endif /* MACH_HOST */
988 processor_set_t pset
;
992 if (thread
== THREAD_NULL
)
996 * First, check for new count > 0 (the common case).
997 * Only the thread needs to be locked.
1000 thread_lock(thread
);
1001 refs
= --thread
->ref_count
;
1002 thread_unlock(thread
);
1008 if (thread
== current_thread())
1009 panic("thread_deallocate");
1012 * There is a dangling pointer to the thread from the
1013 * processor_set. To clean it up, we freeze the thread
1014 * in the pset (because pset destruction can cause even
1015 * reference-less threads to be reassigned to the default
1016 * pset) and then remove it.
1020 thread_freeze(thread
);
1023 pset
= thread
->processor_set
;
1025 pset_remove_thread(pset
, thread
);
1029 thread_unfreeze(thread
);
1032 pset_deallocate(pset
);
1034 if (thread
->reserved_stack
!= 0) {
1035 if (thread
->reserved_stack
!= thread
->kernel_stack
)
1036 stack_free_stack(thread
->reserved_stack
);
1037 thread
->reserved_stack
= 0;
1040 if (thread
->kernel_stack
!= 0)
1043 machine_thread_destroy(thread
);
1045 zfree(thread_zone
, (vm_offset_t
) thread
);
1054 if (thread
== THREAD_NULL
)
1058 thread_lock(thread
);
1059 thread_reference_locked(thread
);
1060 thread_unlock(thread
);
1065 * Called with "appropriate" thread-related locks held on
1066 * thread and its top_act for synchrony with RPC (see
1067 * act_lock_thread()).
1070 thread_info_shuttle(
1071 register thread_act_t thr_act
,
1072 thread_flavor_t flavor
,
1073 thread_info_t thread_info_out
, /* ptr to OUT array */
1074 mach_msg_type_number_t
*thread_info_count
) /*IN/OUT*/
1076 register thread_t thread
= thr_act
->thread
;
1080 if (thread
== THREAD_NULL
)
1081 return (KERN_INVALID_ARGUMENT
);
1083 if (flavor
== THREAD_BASIC_INFO
) {
1084 register thread_basic_info_t basic_info
;
1086 if (*thread_info_count
< THREAD_BASIC_INFO_COUNT
)
1087 return (KERN_INVALID_ARGUMENT
);
1089 basic_info
= (thread_basic_info_t
) thread_info_out
;
1092 thread_lock(thread
);
1096 thread_read_times(thread
, &basic_info
->user_time
,
1097 &basic_info
->system_time
);
1100 * Update lazy-evaluated scheduler info because someone wants it.
1102 if (thread
->sched_stamp
!= sched_tick
)
1103 update_priority(thread
);
1105 basic_info
->sleep_time
= 0;
1108 * To calculate cpu_usage, first correct for timer rate,
1109 * then for 5/8 ageing. The correction factor [3/5] is
1112 basic_info
->cpu_usage
= (thread
->cpu_usage
<< SCHED_TICK_SHIFT
) /
1113 (TIMER_RATE
/ TH_USAGE_SCALE
);
1114 basic_info
->cpu_usage
= (basic_info
->cpu_usage
* 3) / 5;
1117 * Clock drift compensation.
1119 basic_info
->cpu_usage
= (basic_info
->cpu_usage
* 1000000) / sched_usec
;
1120 #endif /* SIMPLE_CLOCK */
1122 basic_info
->policy
= ((thread
->sched_mode
& TH_MODE_TIMESHARE
)?
1123 POLICY_TIMESHARE
: POLICY_RR
);
1126 if (thread
->state
& TH_IDLE
)
1127 flags
|= TH_FLAGS_IDLE
;
1129 if (thread
->state
& TH_STACK_HANDOFF
)
1130 flags
|= TH_FLAGS_SWAPPED
;
1133 if (thread
->state
& TH_TERMINATE
)
1134 state
= TH_STATE_HALTED
;
1136 if (thread
->state
& TH_RUN
)
1137 state
= TH_STATE_RUNNING
;
1139 if (thread
->state
& TH_UNINT
)
1140 state
= TH_STATE_UNINTERRUPTIBLE
;
1142 if (thread
->state
& TH_SUSP
)
1143 state
= TH_STATE_STOPPED
;
1145 if (thread
->state
& TH_WAIT
)
1146 state
= TH_STATE_WAITING
;
1148 basic_info
->run_state
= state
;
1149 basic_info
->flags
= flags
;
1151 basic_info
->suspend_count
= thr_act
->user_stop_count
;
1153 thread_unlock(thread
);
1156 *thread_info_count
= THREAD_BASIC_INFO_COUNT
;
1158 return (KERN_SUCCESS
);
1161 if (flavor
== THREAD_SCHED_TIMESHARE_INFO
) {
1162 policy_timeshare_info_t ts_info
;
1164 if (*thread_info_count
< POLICY_TIMESHARE_INFO_COUNT
)
1165 return (KERN_INVALID_ARGUMENT
);
1167 ts_info
= (policy_timeshare_info_t
)thread_info_out
;
1170 thread_lock(thread
);
1172 if (!(thread
->sched_mode
& TH_MODE_TIMESHARE
)) {
1173 thread_unlock(thread
);
1176 return (KERN_INVALID_POLICY
);
1179 ts_info
->depressed
= (thread
->sched_mode
& TH_MODE_ISDEPRESSED
) != 0;
1180 if (ts_info
->depressed
) {
1181 ts_info
->base_priority
= DEPRESSPRI
;
1182 ts_info
->depress_priority
= thread
->priority
;
1185 ts_info
->base_priority
= thread
->priority
;
1186 ts_info
->depress_priority
= -1;
1189 ts_info
->cur_priority
= thread
->sched_pri
;
1190 ts_info
->max_priority
= thread
->max_priority
;
1192 thread_unlock(thread
);
1195 *thread_info_count
= POLICY_TIMESHARE_INFO_COUNT
;
1197 return (KERN_SUCCESS
);
1200 if (flavor
== THREAD_SCHED_FIFO_INFO
) {
1201 if (*thread_info_count
< POLICY_FIFO_INFO_COUNT
)
1202 return (KERN_INVALID_ARGUMENT
);
1204 return (KERN_INVALID_POLICY
);
1207 if (flavor
== THREAD_SCHED_RR_INFO
) {
1208 policy_rr_info_t rr_info
;
1210 if (*thread_info_count
< POLICY_RR_INFO_COUNT
)
1211 return (KERN_INVALID_ARGUMENT
);
1213 rr_info
= (policy_rr_info_t
) thread_info_out
;
1216 thread_lock(thread
);
1218 if (thread
->sched_mode
& TH_MODE_TIMESHARE
) {
1219 thread_unlock(thread
);
1222 return (KERN_INVALID_POLICY
);
1225 rr_info
->depressed
= (thread
->sched_mode
& TH_MODE_ISDEPRESSED
) != 0;
1226 if (rr_info
->depressed
) {
1227 rr_info
->base_priority
= DEPRESSPRI
;
1228 rr_info
->depress_priority
= thread
->priority
;
1231 rr_info
->base_priority
= thread
->priority
;
1232 rr_info
->depress_priority
= -1;
1235 rr_info
->max_priority
= thread
->max_priority
;
1236 rr_info
->quantum
= std_quantum_us
/ 1000;
1238 thread_unlock(thread
);
1241 *thread_info_count
= POLICY_RR_INFO_COUNT
;
1243 return (KERN_SUCCESS
);
1246 return (KERN_INVALID_ARGUMENT
);
1251 register thread_t thread
)
1253 thread_act_t thr_act
;
1256 thr_act
= thread_lock_act(thread
);
1257 assert(thr_act
&& thr_act
->thread
== thread
);
1259 act_reference_locked(thr_act
);
1262 * Replace `act_unlock_thread()' with individual
1263 * calls. (`act_detach()' can change fields used
1264 * to determine which locks are held, confusing
1265 * `act_unlock_thread()'.)
1267 act_unlock(thr_act
);
1269 /* Remove the reference held by a rooted thread */
1270 act_deallocate(thr_act
);
1272 /* Remove the reference held by the thread: */
1273 act_deallocate(thr_act
);
1279 * This kernel thread runs forever looking for terminating
1280 * threads, releasing their "self" references.
1283 reaper_thread_continue(void)
1285 register thread_t thread
;
1288 simple_lock(&reaper_lock
);
1290 while ((thread
= (thread_t
) dequeue_head(&reaper_queue
)) != THREAD_NULL
) {
1291 simple_unlock(&reaper_lock
);
1294 thread_doreap(thread
);
1297 simple_lock(&reaper_lock
);
1300 assert_wait((event_t
)&reaper_queue
, THREAD_UNINT
);
1301 simple_unlock(&reaper_lock
);
1304 thread_block(reaper_thread_continue
);
1311 reaper_thread_continue();
1316 thread_reaper_init(void)
1318 kernel_thread_with_priority(reaper_thread
, MINPRI_KERNEL
);
1323 thread_act_t thr_act
,
1324 processor_set_t new_pset
)
1326 return(KERN_FAILURE
);
1330 * thread_assign_default:
1332 * Special version of thread_assign for assigning threads to default
1336 thread_assign_default(
1337 thread_act_t thr_act
)
1339 return (thread_assign(thr_act
, &default_pset
));
1343 * thread_get_assignment
1345 * Return current assignment for this thread.
1348 thread_get_assignment(
1349 thread_act_t thr_act
,
1350 processor_set_t
*pset
)
1354 if (thr_act
== THR_ACT_NULL
)
1355 return(KERN_INVALID_ARGUMENT
);
1356 thread
= act_lock_thread(thr_act
);
1357 if (thread
== THREAD_NULL
) {
1358 act_unlock_thread(thr_act
);
1359 return(KERN_INVALID_ARGUMENT
);
1361 *pset
= thread
->processor_set
;
1362 act_unlock_thread(thr_act
);
1363 pset_reference(*pset
);
1364 return(KERN_SUCCESS
);
1368 * thread_wire_internal:
1370 * Specify that the target thread must always be able
1371 * to run and to allocate memory.
1374 thread_wire_internal(
1375 host_priv_t host_priv
,
1376 thread_act_t thr_act
,
1378 boolean_t
*prev_state
)
1382 extern void vm_page_free_reserve(int pages
);
1384 if (thr_act
== THR_ACT_NULL
|| host_priv
== HOST_PRIV_NULL
)
1385 return (KERN_INVALID_ARGUMENT
);
1387 assert(host_priv
== &realhost
);
1389 thread
= act_lock_thread(thr_act
);
1390 if (thread
==THREAD_NULL
) {
1391 act_unlock_thread(thr_act
);
1392 return(KERN_INVALID_ARGUMENT
);
1396 * This implementation only works for the current thread.
1398 if (thr_act
!= current_act())
1399 return KERN_INVALID_ARGUMENT
;
1402 thread_lock(thread
);
1405 *prev_state
= thread
->vm_privilege
;
1409 if (thread
->vm_privilege
== FALSE
)
1410 vm_page_free_reserve(1); /* XXX */
1411 thread
->vm_privilege
= TRUE
;
1413 if (thread
->vm_privilege
== TRUE
)
1414 vm_page_free_reserve(-1); /* XXX */
1415 thread
->vm_privilege
= FALSE
;
1418 thread_unlock(thread
);
1420 act_unlock_thread(thr_act
);
1422 return KERN_SUCCESS
;
1429 * User-api wrapper for thread_wire_internal()
1433 host_priv_t host_priv
,
1434 thread_act_t thr_act
,
1438 return thread_wire_internal(host_priv
, thr_act
, wired
, NULL
);
1444 vm_size_t
*reservedp
,
1445 unsigned int *totalp
,
1447 vm_size_t
*residentp
,
1448 vm_size_t
*maxusagep
,
1449 vm_offset_t
*maxstackp
)
1452 return KERN_NOT_SUPPORTED
;
1457 if (host
== HOST_NULL
)
1458 return KERN_INVALID_HOST
;
1462 stack_statistics(&total
, &maxusage
);
1466 *spacep
= *residentp
= total
* round_page_32(KERNEL_STACK_SIZE
);
1467 *maxusagep
= maxusage
;
1469 return KERN_SUCCESS
;
1471 #endif /* MACH_DEBUG */
1475 * Return info on stack usage for threads in a specific processor set
1478 processor_set_stack_usage(
1479 processor_set_t pset
,
1480 unsigned int *totalp
,
1482 vm_size_t
*residentp
,
1483 vm_size_t
*maxusagep
,
1484 vm_offset_t
*maxstackp
)
1487 return KERN_NOT_SUPPORTED
;
1491 vm_offset_t maxstack
;
1493 register thread_t
*threads
;
1494 register thread_t thread
;
1496 unsigned int actual
; /* this many things */
1499 vm_size_t size
, size_needed
;
1504 if (pset
== PROCESSOR_SET_NULL
)
1505 return KERN_INVALID_ARGUMENT
;
1511 if (!pset
->active
) {
1513 return KERN_INVALID_ARGUMENT
;
1516 actual
= pset
->thread_count
;
1518 /* do we have the memory we need? */
1520 size_needed
= actual
* sizeof(thread_t
);
1521 if (size_needed
<= size
)
1524 /* unlock the pset and allocate more memory */
1530 assert(size_needed
> 0);
1533 addr
= kalloc(size
);
1535 return KERN_RESOURCE_SHORTAGE
;
1538 /* OK, have memory and the processor_set is locked & active */
1540 threads
= (thread_t
*) addr
;
1541 for (i
= 0, thread
= (thread_t
) queue_first(&pset
->threads
);
1542 !queue_end(&pset
->threads
, (queue_entry_t
) thread
);
1543 thread
= (thread_t
) queue_next(&thread
->pset_threads
)) {
1544 thread_lock(thread
);
1545 if (thread
->ref_count
> 0) {
1546 thread_reference_locked(thread
);
1547 threads
[i
++] = thread
;
1549 thread_unlock(thread
);
1552 assert(i
<= actual
);
1554 /* can unlock processor set now that we have the thread refs */
1557 /* calculate maxusage and free thread references */
1563 thread_t thread
= threads
[--i
];
1565 if (thread
->kernel_stack
!= 0)
1568 thread_deallocate(thread
);
1575 *residentp
= *spacep
= total
* round_page_32(KERNEL_STACK_SIZE
);
1576 *maxusagep
= maxusage
;
1577 *maxstackp
= maxstack
;
1578 return KERN_SUCCESS
;
1580 #endif /* MACH_DEBUG */
1583 int split_funnel_off
= 0;
1590 if ((fnl
= (funnel_t
*)kalloc(sizeof(funnel_t
))) != 0){
1591 bzero((void *)fnl
, sizeof(funnel_t
));
1592 if ((m
= mutex_alloc(0)) == (mutex_t
*)NULL
) {
1593 kfree((vm_offset_t
)fnl
, sizeof(funnel_t
));
1594 return(THR_FUNNEL_NULL
);
1597 fnl
->fnl_type
= type
;
1606 mutex_free(fnl
->fnl_mutex
);
1607 if (fnl
->fnl_oldmutex
)
1608 mutex_free(fnl
->fnl_oldmutex
);
1609 kfree((vm_offset_t
)fnl
, sizeof(funnel_t
));
1621 fnl
->fnl_mtxholder
= current_thread();
1622 if (split_funnel_off
&& (m
!= fnl
->fnl_mutex
)) {
1633 mutex_unlock(fnl
->fnl_mutex
);
1634 fnl
->fnl_mtxrelease
= current_thread();
1637 int refunnel_hint_enabled
= 0;
1642 wait_result_t wresult
)
1644 if ( !(thread
->funnel_state
& TH_FN_REFUNNEL
) ||
1645 wresult
!= THREAD_AWAKENED
)
1648 if (!refunnel_hint_enabled
)
1651 return (mutex_preblock(thread
->funnel_lock
->fnl_mutex
, thread
));
1658 thread_t th
= current_thread();
1660 if (th
->funnel_state
& TH_FN_OWNED
) {
1661 return(th
->funnel_lock
);
1663 return(THR_FUNNEL_NULL
);
1671 thread_t cur_thread
;
1672 boolean_t funnel_state_prev
;
1675 cur_thread
= current_thread();
1676 funnel_state_prev
= ((cur_thread
->funnel_state
& TH_FN_OWNED
) == TH_FN_OWNED
);
1678 if (funnel_state_prev
!= funneled
) {
1679 intr
= ml_set_interrupts_enabled(FALSE
);
1681 if (funneled
== TRUE
) {
1682 if (cur_thread
->funnel_lock
)
1683 panic("Funnel lock called when holding one %x", cur_thread
->funnel_lock
);
1684 KERNEL_DEBUG(0x6032428 | DBG_FUNC_NONE
,
1687 KERNEL_DEBUG(0x6032434 | DBG_FUNC_NONE
,
1689 cur_thread
->funnel_state
|= TH_FN_OWNED
;
1690 cur_thread
->funnel_lock
= fnl
;
1692 if(cur_thread
->funnel_lock
->fnl_mutex
!= fnl
->fnl_mutex
)
1693 panic("Funnel unlock when not holding funnel");
1694 cur_thread
->funnel_state
&= ~TH_FN_OWNED
;
1695 KERNEL_DEBUG(0x603242c | DBG_FUNC_NONE
,
1698 cur_thread
->funnel_lock
= THR_FUNNEL_NULL
;
1701 (void)ml_set_interrupts_enabled(intr
);
1703 /* if we are trying to acquire funnel recursively
1704 * check for funnel to be held already
1706 if (funneled
&& (fnl
->fnl_mutex
!= cur_thread
->funnel_lock
->fnl_mutex
)) {
1707 panic("thread_funnel_set: already holding a different funnel");
1710 return(funnel_state_prev
);
1714 thread_funnel_merge(
1716 funnel_t
* otherfnl
)
1721 extern int disable_funnel
;
1723 if ((gfnl
= thread_funnel_get()) == THR_FUNNEL_NULL
)
1724 panic("thread_funnel_merge called with no funnels held");
1726 if (gfnl
->fnl_type
!= 1)
1727 panic("thread_funnel_merge called from non kernel funnel");
1730 panic("thread_funnel_merge incorrect invocation");
1732 if (disable_funnel
|| split_funnel_off
)
1733 return (KERN_FAILURE
);
1736 otherm
= otherfnl
->fnl_mutex
;
1738 /* Acquire other funnel mutex */
1740 split_funnel_off
= 1;
1742 otherfnl
->fnl_mutex
= m
;
1743 otherfnl
->fnl_type
= fnl
->fnl_type
;
1744 otherfnl
->fnl_oldmutex
= otherm
; /* save this for future use */
1746 mutex_unlock(otherm
);
1747 return(KERN_SUCCESS
);
1751 thread_set_cont_arg(
1754 thread_t self
= current_thread();
1756 self
->saved
.misc
= arg
;
1760 thread_get_cont_arg(void)
1762 thread_t self
= current_thread();
1764 return (self
->saved
.misc
);
1768 * Export routines to other components for things that are done as macros
1769 * within the osfmk component.
1771 #undef thread_should_halt
1776 return(thread_should_halt_fast(th
));
1779 vm_offset_t
min_valid_stack_address(void)
1781 return vm_map_min(stack_map
);
1784 vm_offset_t
max_valid_stack_address(void)
1786 return vm_map_max(stack_map
);