2 * Copyright (c) 2000-2016 Apple Inc. All rights reserved.
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
29 * @OSF_FREE_COPYRIGHT@
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
34 * All Rights Reserved.
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
46 * Carnegie Mellon requests users of this software to return to
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
60 * Author: Avadis Tevanian, Jr.
62 * This file contains the structure definitions for threads.
66 * Copyright (c) 1993 The University of Utah and
67 * the Computer Systems Laboratory (CSL). All rights reserved.
69 * Permission to use, copy, modify and distribute this software and its
70 * documentation is hereby granted, provided that both the copyright
71 * notice and this permission notice appear in all copies of the
72 * software, derivative works or modified versions, and any portions
73 * thereof, and that both notices appear in supporting documentation.
75 * THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF THIS SOFTWARE IN ITS "AS
76 * IS" CONDITION. THE UNIVERSITY OF UTAH AND CSL DISCLAIM ANY LIABILITY OF
77 * ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
79 * CSL requests users of this software to return to csl-dist@cs.utah.edu any
80 * improvements that they make and grant CSL redistribution rights.
84 #ifndef _KERN_THREAD_H_
85 #define _KERN_THREAD_H_
87 #include <mach/kern_return.h>
88 #include <mach/mach_types.h>
89 #include <mach/message.h>
90 #include <mach/boolean.h>
91 #include <mach/vm_param.h>
92 #include <mach/thread_info.h>
93 #include <mach/thread_status.h>
94 #include <mach/exception_types.h>
96 #include <kern/kern_types.h>
97 #include <vm/vm_kern.h>
99 #include <sys/cdefs.h>
101 #ifdef MACH_KERNEL_PRIVATE
103 #include <mach_assert.h>
104 #include <mach_ldebug.h>
106 #include <ipc/ipc_types.h>
108 #include <mach/port.h>
109 #include <kern/cpu_number.h>
110 #include <kern/smp.h>
111 #include <kern/queue.h>
113 #include <kern/timer.h>
114 #include <kern/simple_lock.h>
115 #include <kern/locks.h>
116 #include <kern/sched.h>
117 #include <kern/sched_prim.h>
118 #include <mach/sfi_class.h>
119 #include <kern/thread_call.h>
120 #include <kern/thread_group.h>
121 #include <kern/timer_call.h>
122 #include <kern/task.h>
123 #include <kern/exception.h>
124 #include <kern/affinity.h>
125 #include <kern/debug.h>
126 #include <kern/block_hint.h>
127 #include <kern/turnstile.h>
129 #include <kern/waitq.h>
130 #include <san/kasan.h>
131 #include <os/refcnt.h>
133 #include <ipc/ipc_kmsg.h>
135 #include <machine/cpu_data.h>
136 #include <machine/thread.h>
138 #ifdef XNU_KERNEL_PRIVATE
139 /* priority queue static asserts fail for __ARM64_ARCH_8_32__ kext builds */
140 #include <kern/priority_queue.h>
141 #endif /* XNU_KERNEL_PRIVATE */
144 #include <stdatomic.h>
145 #include <machine/monotonic.h>
146 #endif /* MONOTONIC */
149 /* Taskwatch related. TODO: find this a better home */
150 typedef struct task_watcher task_watch_t
;
151 #endif /* CONFIG_EMBEDDED */
156 #define THREAD_MAGIC 0x1234ABCDDCBA4321ULL
157 /* Ensure nothing uses &thread as a queue entry */
158 uint64_t thread_magic
;
159 #endif /* MACH_ASSERT */
162 * NOTE: The runq field in the thread structure has an unusual
163 * locking protocol. If its value is PROCESSOR_NULL, then it is
164 * locked by the thread_lock, but if its value is something else
165 * then it is locked by the associated run queue lock. It is
166 * set to PROCESSOR_NULL without holding the thread lock, but the
167 * transition from PROCESSOR_NULL to non-null must be done
168 * under the thread lock and the run queue lock.
170 * New waitq APIs allow the 'links' and 'runq' fields to be
171 * anywhere in the thread structure.
174 queue_chain_t runq_links
; /* run queue links */
175 queue_chain_t wait_links
; /* wait queue links */
176 struct priority_queue_entry wait_prioq_links
; /* priority ordered waitq links */
179 processor_t runq
; /* run queue assignment */
181 event64_t wait_event
; /* wait queue event */
182 struct waitq
*waitq
; /* wait queue this thread is enqueued on */
183 struct turnstile
*turnstile
; /* thread's turnstile, protected by primitives interlock */
184 void *inheritor
; /* inheritor of the primitive the thread will block on */
185 struct priority_queue inheritor_queue
; /* Inheritor queue */
187 /* Data updated during assert_wait/thread_wakeup */
189 decl_simple_lock_data(,sched_lock
) /* scheduling lock (thread_lock()) */
190 decl_simple_lock_data(,wake_lock
) /* for thread stop / wait (wake_lock()) */
192 integer_t options
; /* options set by thread itself */
193 #define TH_OPT_INTMASK 0x0003 /* interrupt / abort level */
194 #define TH_OPT_VMPRIV 0x0004 /* may allocate reserved memory */
195 #define TH_OPT_DTRACE 0x0008 /* executing under dtrace_probe */
196 #define TH_OPT_SYSTEM_CRITICAL 0x0010 /* Thread must always be allowed to run - even under heavy load */
197 #define TH_OPT_PROC_CPULIMIT 0x0020 /* Thread has a task-wide CPU limit applied to it */
198 #define TH_OPT_PRVT_CPULIMIT 0x0040 /* Thread has a thread-private CPU limit applied to it */
199 #define TH_OPT_IDLE_THREAD 0x0080 /* Thread is a per-processor idle thread */
200 #define TH_OPT_GLOBAL_FORCED_IDLE 0x0100 /* Thread performs forced idle for thermal control */
201 #define TH_OPT_SCHED_VM_GROUP 0x0200 /* Thread belongs to special scheduler VM group */
202 #define TH_OPT_HONOR_QLIMIT 0x0400 /* Thread will honor qlimit while sending mach_msg, regardless of MACH_SEND_ALWAYS */
203 #define TH_OPT_SEND_IMPORTANCE 0x0800 /* Thread will allow importance donation from kernel rpc */
204 #define TH_OPT_ZONE_GC 0x1000 /* zone_gc() called on this thread */
206 boolean_t wake_active
; /* wake event on stop */
207 int at_safe_point
; /* thread_abort_safely allowed */
208 ast_t reason
; /* why we blocked */
209 uint32_t quantum_remaining
;
210 wait_result_t wait_result
; /* outcome of wait -
211 * may be examined by this thread
213 thread_continue_t continuation
; /* continue here next dispatch */
214 void *parameter
; /* continuation parameter */
216 /* Data updated/used in thread_invoke */
217 vm_offset_t kernel_stack
; /* current kernel stack */
218 vm_offset_t reserved_stack
; /* reserved kernel stack */
221 struct kasan_thread_data kasan_data
;
227 * Thread states [bits or'ed]
229 #define TH_WAIT 0x01 /* queued for waiting */
230 #define TH_SUSP 0x02 /* stopped or requested to stop */
231 #define TH_RUN 0x04 /* running or on runq */
232 #define TH_UNINT 0x08 /* waiting uninteruptibly */
233 #define TH_TERMINATE 0x10 /* halted at termination */
234 #define TH_TERMINATE2 0x20 /* added to termination queue */
235 #define TH_WAIT_REPORT 0x40 /* the wait is using the sched_call,
236 only set if TH_WAIT is also set */
237 #define TH_IDLE 0x80 /* idling processor */
239 /* Scheduling information */
240 sched_mode_t sched_mode
; /* scheduling mode */
241 sched_mode_t saved_mode
; /* saved mode during forced mode demotion */
243 /* This thread's contribution to global sched counters */
244 sched_bucket_t th_sched_bucket
;
246 sfi_class_id_t sfi_class
; /* SFI class (XXX Updated on CSW/QE/AST) */
247 sfi_class_id_t sfi_wait_class
; /* Currently in SFI wait for this class, protected by sfi_lock */
250 uint32_t sched_flags
; /* current flag bits */
251 /* TH_SFLAG_FAIRSHARE_TRIPPED (unused) 0x0001 */
252 #define TH_SFLAG_FAILSAFE 0x0002 /* fail-safe has tripped */
253 #define TH_SFLAG_THROTTLED 0x0004 /* throttled thread forced to timeshare mode (may be applied in addition to failsafe) */
254 #define TH_SFLAG_DEMOTED_MASK (TH_SFLAG_THROTTLED | TH_SFLAG_FAILSAFE) /* saved_mode contains previous sched_mode */
256 #define TH_SFLAG_PROMOTED 0x0008 /* sched pri has been promoted by kernel mutex priority promotion */
257 #define TH_SFLAG_ABORT 0x0010 /* abort interruptible waits */
258 #define TH_SFLAG_ABORTSAFELY 0x0020 /* ... but only those at safe point */
259 #define TH_SFLAG_ABORTED_MASK (TH_SFLAG_ABORT | TH_SFLAG_ABORTSAFELY)
260 #define TH_SFLAG_DEPRESS 0x0040 /* normal depress yield */
261 #define TH_SFLAG_POLLDEPRESS 0x0080 /* polled depress yield */
262 #define TH_SFLAG_DEPRESSED_MASK (TH_SFLAG_DEPRESS | TH_SFLAG_POLLDEPRESS)
263 /* unused TH_SFLAG_PRI_UPDATE 0x0100 */
264 #define TH_SFLAG_EAGERPREEMPT 0x0200 /* Any preemption of this thread should be treated as if AST_URGENT applied */
265 #define TH_SFLAG_RW_PROMOTED 0x0400 /* promote reason: blocking with RW lock held */
266 /* unused TH_SFLAG_THROTTLE_DEMOTED 0x0800 */
267 #define TH_SFLAG_WAITQ_PROMOTED 0x1000 /* promote reason: waitq wakeup (generally for IPC receive) */
270 #define TH_SFLAG_EXEC_PROMOTED 0x8000 /* promote reason: thread is in an exec */
272 /* 'promote reasons' that request a priority floor only, not a custom priority */
273 #define TH_SFLAG_PROMOTE_REASON_MASK (TH_SFLAG_RW_PROMOTED | TH_SFLAG_WAITQ_PROMOTED | TH_SFLAG_EXEC_PROMOTED)
275 #define TH_SFLAG_RW_PROMOTED_BIT (10) /* 0x400 */
277 int16_t sched_pri
; /* scheduled (current) priority */
278 int16_t base_pri
; /* base priority */
279 int16_t max_priority
; /* copy of max base priority */
280 int16_t task_priority
; /* copy of task base priority */
281 int16_t promotion_priority
; /* priority thread is currently promoted to */
283 #if defined(CONFIG_SCHED_GRRR)
285 uint16_t grrr_deficit
; /* fixed point (1/1000th quantum) fractional deficit */
289 int16_t promotions
; /* level of promotion */
290 int iotier_override
; /* atomic operations to set, cleared on ret to user */
291 struct os_refcnt ref_count
; /* number of references to me */
293 lck_mtx_t
* waiting_for_mutex
; /* points to mutex we're waiting for until we acquire it */
295 uint32_t rwlock_count
; /* Number of lck_rw_t locks held by thread */
297 integer_t importance
; /* task-relative importance */
298 uint32_t was_promoted_on_wakeup
; /* thread promoted on wakeup to acquire mutex */
300 /* Priority depression expiration */
301 integer_t depress_timer_active
;
302 timer_call_data_t depress_timer
;
303 /* real-time parameters */
304 struct { /* see mach/thread_policy.h */
306 uint32_t computation
;
308 boolean_t preemptible
;
312 uint64_t last_run_time
; /* time when thread was switched away from */
313 uint64_t last_made_runnable_time
; /* time when thread was unblocked or preempted */
314 uint64_t last_basepri_change_time
; /* time when thread was last changed in basepri while runnable */
315 uint64_t same_pri_latency
;
316 #define THREAD_NOT_RUNNABLE (~0ULL)
319 #if defined(CONFIG_SCHED_MULTIQ)
320 sched_group_t sched_group
;
321 #endif /* defined(CONFIG_SCHED_MULTIQ) */
323 /* Data used during setrun/dispatch */
324 timer_data_t system_timer
; /* system mode timer */
325 processor_t bound_processor
; /* bound to a processor? */
326 processor_t last_processor
; /* processor last dispatched on */
327 processor_t chosen_processor
; /* Where we want to run this thread */
329 /* Fail-safe computation since last unblock or qualifying yield */
330 uint64_t computation_metered
;
331 uint64_t computation_epoch
;
332 uint64_t safe_release
; /* when to release fail-safe */
334 /* Call out from scheduler */
338 #if defined(CONFIG_SCHED_PROTO)
339 uint32_t runqueue_generation
; /* last time runqueue was drained */
342 /* Statistics and timesharing calculations */
343 #if defined(CONFIG_SCHED_TIMESHARE_CORE)
344 natural_t sched_stamp
; /* last scheduler tick */
345 natural_t sched_usage
; /* timesharing cpu usage [sched] */
346 natural_t pri_shift
; /* usage -> priority from pset */
347 natural_t cpu_usage
; /* instrumented cpu usage [%cpu] */
348 natural_t cpu_delta
; /* accumulated cpu_usage delta */
349 #endif /* CONFIG_SCHED_TIMESHARE_CORE */
351 uint32_t c_switch
; /* total context switches */
352 uint32_t p_switch
; /* total processor switches */
353 uint32_t ps_switch
; /* total pset switches */
355 integer_t mutex_count
; /* total count of locks held */
356 /* Timing data structures */
357 int precise_user_kernel_time
; /* precise user/kernel enabled for this thread */
358 timer_data_t user_timer
; /* user mode timer */
359 uint64_t user_timer_save
; /* saved user timer value */
360 uint64_t system_timer_save
; /* saved system timer value */
361 uint64_t vtimer_user_save
; /* saved values for vtimers */
362 uint64_t vtimer_prof_save
;
363 uint64_t vtimer_rlim_save
;
364 uint64_t vtimer_qos_save
;
366 timer_data_t ptime
; /* time executing in P mode */
367 timer_data_t runnable_timer
; /* time the thread is runnable (including running) */
370 /* Timing for wait state */
371 uint64_t wait_sfi_begin_time
; /* start time for thread waiting in SFI */
374 /* Timed wait expiration */
375 timer_call_data_t wait_timer
;
376 integer_t wait_timer_active
;
377 boolean_t wait_timer_is_set
;
381 * Processor/cache affinity
382 * - affinity_threads links task threads with the same affinity set
384 affinity_set_t affinity_set
;
385 queue_chain_t affinity_threads
;
387 /* Various bits of state to stash across a continuation, exclusive to the current thread block point */
390 mach_msg_return_t state
; /* receive state */
391 mach_port_seqno_t seqno
; /* seqno of recvd message */
392 ipc_object_t object
; /* object received on */
393 mach_vm_address_t msg_addr
; /* receive buffer pointer */
394 mach_msg_size_t rsize
; /* max size for recvd msg */
395 mach_msg_size_t msize
; /* actual size for recvd msg */
396 mach_msg_option_t option
; /* options for receive */
397 mach_port_name_t receiver_name
; /* the receive port name */
398 struct knote
*knote
; /* knote fired for rcv */
400 struct ipc_kmsg
*kmsg
; /* received message */
401 struct ipc_mqueue
*peekq
; /* mqueue to peek at */
403 mach_msg_priority_t qos
; /* received message qos */
404 mach_msg_priority_t oqos
; /* override qos for message */
407 mach_msg_continue_t continuation
;
410 struct semaphore
*waitsemaphore
; /* semaphore ref */
411 struct semaphore
*signalsemaphore
; /* semaphore ref */
412 int options
; /* semaphore options */
413 kern_return_t result
; /* primary result */
414 mach_msg_continue_t continuation
;
418 /* Only user threads can cause guard exceptions, only kernel threads can be thread call threads */
420 /* Group and call this thread is working on behalf of */
422 struct thread_call_group
* thc_group
;
423 struct thread_call
* thc_call
; /* debug only, may be deallocated */
426 /* Structure to save information about guard exception */
428 mach_exception_code_t code
;
429 mach_exception_subcode_t subcode
;
433 /* Kernel holds on this thread */
434 int16_t suspend_count
;
435 /* User level suspensions */
436 int16_t user_stop_count
;
438 /* IPC data structures */
439 #if IMPORTANCE_INHERITANCE
440 natural_t ith_assertions
; /* assertions pending drop */
442 struct ipc_kmsg_queue ith_messages
; /* messages to reap */
443 mach_port_t ith_rpc_reply
; /* reply port for kernel RPCs */
445 /* Ast/Halt data structures */
446 vm_offset_t recover
; /* page fault recover(copyin/out) */
448 queue_chain_t threads
; /* global list of all threads */
451 queue_chain_t task_threads
;
453 /* Task membership */
456 #if DEVELOPMENT || DEBUG
457 boolean_t pmap_footprint_suspended
;
458 #endif /* DEVELOPMENT || DEBUG */
460 decl_lck_mtx_data(,mutex
)
463 /* Pending thread ast(s) */
466 /* Miscellaneous bits guarded by mutex */
468 active
:1, /* Thread is active and has not been terminated */
469 started
:1, /* Thread has been started after creation */
470 static_param
:1, /* Disallow policy parameter changes */
471 inspection
:1, /* TRUE when task is being inspected by crash reporter */
472 policy_reset
:1, /* Disallow policy parameter changes on terminating threads */
473 suspend_parked
:1, /* thread parked in thread_suspended */
474 corpse_dup
:1, /* TRUE when thread is an inactive duplicate in a corpse */
477 /* Ports associated with this thread */
478 struct ipc_port
*ith_self
; /* not a right, doesn't hold ref */
479 struct ipc_port
*ith_sself
; /* a send right */
480 struct ipc_port
*ith_special_reply_port
; /* ref to special reply port */
481 struct exception_action
*exc_actions
;
488 uint32_t t_dtrace_flags
; /* DTrace thread states */
489 #define TH_DTRACE_EXECSUCCESS 0x01
490 uint32_t t_dtrace_predcache
;/* DTrace per thread predicate value hint */
491 int64_t t_dtrace_tracing
; /* Thread time under dtrace_probe() */
492 int64_t t_dtrace_vtime
;
495 clock_sec_t t_page_creation_time
;
496 uint32_t t_page_creation_count
;
497 uint32_t t_page_creation_throttled
;
498 #if (DEVELOPMENT || DEBUG)
499 uint64_t t_page_creation_throttled_hard
;
500 uint64_t t_page_creation_throttled_soft
;
501 #endif /* DEVELOPMENT || DEBUG */
504 /* The high 7 bits are the number of frames to sample of a user callstack. */
505 #define T_KPERF_CALLSTACK_DEPTH_OFFSET (25)
506 #define T_KPERF_SET_CALLSTACK_DEPTH(DEPTH) (((uint32_t)(DEPTH)) << T_KPERF_CALLSTACK_DEPTH_OFFSET)
507 #define T_KPERF_GET_CALLSTACK_DEPTH(FLAGS) ((FLAGS) >> T_KPERF_CALLSTACK_DEPTH_OFFSET)
510 #define T_KPERF_AST_CALLSTACK (1U << 0) /* dump a callstack on thread's next AST */
511 #define T_KPERF_AST_DISPATCH (1U << 1) /* dump a name on thread's next AST */
512 #define T_KPC_ALLOC (1U << 2) /* thread needs a kpc_buf allocated */
513 /* only go up to T_KPERF_CALLSTACK_DEPTH_OFFSET - 1 */
516 uint32_t kperf_flags
;
517 uint32_t kperf_pet_gen
; /* last generation of PET that sampled this thread*/
518 uint32_t kperf_c_switch
; /* last dispatch detection */
519 uint32_t kperf_pet_cnt
; /* how many times a thread has been sampled by PET */
523 /* accumulated performance counters for this thread */
528 /* hypervisor virtual CPU object associated with this thread */
529 void *hv_thread_target
;
530 #endif /* HYPERVISOR */
532 uint64_t thread_id
; /*system wide unique thread-id*/
534 /* Statistics accumulated per-thread and aggregated per-task */
535 uint32_t syscalls_unix
;
536 uint32_t syscalls_mach
;
538 ledger_t t_threadledger
; /* per thread ledger */
539 ledger_t t_bankledger
; /* ledger to charge someone */
540 uint64_t t_deduct_bank_ledger_time
; /* cpu time to be deducted from bank ledger */
541 uint64_t t_deduct_bank_ledger_energy
; /* energy to be deducted from bank ledger */
544 struct mt_thread t_monotonic
;
545 #endif /* MONOTONIC */
547 /*** Machine-dependent state ***/
548 struct machine_thread machine
;
550 /* policy is protected by the thread mutex */
551 struct thread_requested_policy requested_policy
;
552 struct thread_effective_policy effective_policy
;
554 /* usynch override is protected by the task lock, eventually will be thread mutex */
555 struct thread_qos_override
{
556 struct thread_qos_override
*override_next
;
557 uint32_t override_contended_resource_count
;
558 int16_t override_qos
;
559 int16_t override_resource_type
;
560 user_addr_t override_resource
;
563 uint32_t ipc_overrides
;
564 _Atomic
uint32_t kqwl_owning_count
;
565 uint32_t sync_ipc_overrides
;
566 uint16_t user_promotion_basepri
;
567 _Atomic
uint16_t kevent_ast_bits
;
569 io_stat_info_t thread_io_stats
; /* per-thread I/O statistics */
572 task_watch_t
* taskwatch
; /* task watch */
573 #endif /* CONFIG_EMBEDDED */
575 uint32_t thread_callout_interrupt_wakeups
;
576 uint32_t thread_callout_platform_idle_wakeups
;
577 uint32_t thread_timer_wakeups_bin_1
;
578 uint32_t thread_timer_wakeups_bin_2
;
580 uint16_t callout_woken_from_icontext
:1,
581 callout_woken_from_platform_idle
:1,
582 callout_woke_thread
:1,
583 thread_bitfield_unused
:13;
585 mach_port_name_t ith_voucher_name
;
586 ipc_voucher_t ith_voucher
;
589 #endif /* CONFIG_IOSCHED */
591 /* work interval (if any) associated with the thread. Uses thread mutex */
592 struct work_interval
*th_work_interval
;
594 #if SCHED_TRACE_THREAD_WAKEUPS
595 uintptr_t thread_wakeup_bt
[64];
597 turnstile_update_flags_t inheritor_flags
; /* inheritor flags for inheritor field */
598 block_hint_t pending_block_hint
;
599 block_hint_t block_hint
; /* What type of primitive last caused us to block. */
602 #define ith_state saved.receive.state
603 #define ith_object saved.receive.object
604 #define ith_msg_addr saved.receive.msg_addr
605 #define ith_rsize saved.receive.rsize
606 #define ith_msize saved.receive.msize
607 #define ith_option saved.receive.option
608 #define ith_receiver_name saved.receive.receiver_name
609 #define ith_continuation saved.receive.continuation
610 #define ith_kmsg saved.receive.kmsg
611 #define ith_peekq saved.receive.peekq
612 #define ith_knote saved.receive.knote
613 #define ith_qos saved.receive.received_qos.qos
614 #define ith_qos_override saved.receive.received_qos.oqos
615 #define ith_seqno saved.receive.seqno
617 #define sth_waitsemaphore saved.sema.waitsemaphore
618 #define sth_signalsemaphore saved.sema.signalsemaphore
619 #define sth_options saved.sema.options
620 #define sth_result saved.sema.result
621 #define sth_continuation saved.sema.continuation
623 #define ITH_KNOTE_NULL ((void *)NULL)
624 #define ITH_KNOTE_PSEUDO ((void *)0xdeadbeef)
626 * The ith_knote is used during message delivery, and can safely be interpreted
627 * only when used for one of these codepaths, which the test for the msgt_name
628 * being RECEIVE or SEND_ONCE is about.
630 #define ITH_KNOTE_VALID(kn, msgt_name) \
631 (((kn) != ITH_KNOTE_NULL && (kn) != ITH_KNOTE_PSEUDO) && \
632 ((msgt_name) == MACH_MSG_TYPE_PORT_RECEIVE || \
633 (msgt_name) == MACH_MSG_TYPE_PORT_SEND_ONCE))
636 #define assert_thread_magic(thread) assertf((thread)->thread_magic == THREAD_MAGIC, \
637 "bad thread magic 0x%llx for thread %p, expected 0x%llx", \
638 (thread)->thread_magic, (thread), THREAD_MAGIC)
640 #define assert_thread_magic(thread) do { (void)(thread); } while (0)
643 extern void thread_bootstrap(void);
645 extern void thread_init(void);
647 extern void thread_daemon_init(void);
649 #define thread_reference_internal(thread) \
650 os_ref_retain(&(thread)->ref_count);
652 #define thread_reference(thread) \
654 if ((thread) != THREAD_NULL) \
655 thread_reference_internal(thread); \
658 extern void thread_deallocate(
661 extern void thread_deallocate_safe(
664 extern void thread_inspect_deallocate(
665 thread_inspect_t thread
);
667 extern void thread_terminate_self(void);
669 extern kern_return_t
thread_terminate_internal(
672 extern void thread_start(
673 thread_t thread
) __attribute__ ((noinline
));
675 extern void thread_start_in_assert_wait(
678 wait_interrupt_t interruptible
) __attribute__ ((noinline
));
680 extern void thread_terminate_enqueue(
683 extern void thread_exception_enqueue(
686 exception_type_t etype
);
688 extern void thread_copy_resource_info(
690 thread_t src_thread
);
692 extern void thread_terminate_crashed_threads(void);
694 extern void turnstile_deallocate_enqueue(
695 struct turnstile
*turnstile
);
697 extern void thread_stack_enqueue(
700 extern void thread_hold(
703 extern void thread_release(
706 extern void thread_corpse_continue(void);
708 extern boolean_t
thread_is_active(thread_t thread
);
710 /* Locking for scheduler state, always acquired with interrupts disabled (splsched()) */
712 #define thread_lock_init(th) simple_lock_init(&(th)->sched_lock, 0)
713 #define thread_lock(th) simple_lock(&(th)->sched_lock)
714 #define thread_unlock(th) simple_unlock(&(th)->sched_lock)
716 #define wake_lock_init(th) simple_lock_init(&(th)->wake_lock, 0)
717 #define wake_lock(th) simple_lock(&(th)->wake_lock)
718 #define wake_unlock(th) simple_unlock(&(th)->wake_lock)
720 #define thread_lock_init(th) do { (void)th; } while(0)
721 #define thread_lock(th) do { (void)th; } while(0)
722 #define thread_unlock(th) do { (void)th; } while(0)
724 #define wake_lock_init(th) do { (void)th; } while(0)
725 #define wake_lock(th) do { (void)th; } while(0)
726 #define wake_unlock(th) do { (void)th; } while(0)
729 #define thread_should_halt_fast(thread) (!(thread)->active)
731 extern void stack_alloc(
734 extern void stack_handoff(
738 extern void stack_free(
741 extern void stack_free_reserved(
744 extern boolean_t
stack_alloc_try(
747 extern void stack_collect(void);
749 extern void stack_init(void);
752 extern kern_return_t
thread_info_internal(
754 thread_flavor_t flavor
,
755 thread_info_t thread_info_out
,
756 mach_msg_type_number_t
*thread_info_count
);
760 extern kern_return_t
kernel_thread_create(
761 thread_continue_t continuation
,
764 thread_t
*new_thread
);
766 extern kern_return_t
kernel_thread_start_priority(
767 thread_continue_t continuation
,
770 thread_t
*new_thread
);
772 extern void machine_stack_attach(
776 extern vm_offset_t
machine_stack_detach(
779 extern void machine_stack_handoff(
783 extern thread_t
machine_switch_context(
785 thread_continue_t continuation
,
786 thread_t new_thread
);
788 extern void machine_load_context(
789 thread_t thread
) __attribute__((noreturn
));
792 extern kern_return_t
machine_thread_state_initialize(
795 extern kern_return_t
machine_thread_set_state(
797 thread_flavor_t flavor
,
798 thread_state_t state
,
799 mach_msg_type_number_t count
);
801 extern kern_return_t
machine_thread_get_state(
803 thread_flavor_t flavor
,
804 thread_state_t state
,
805 mach_msg_type_number_t
*count
);
807 extern kern_return_t
machine_thread_state_convert_from_user(
809 thread_flavor_t flavor
,
810 thread_state_t tstate
,
811 mach_msg_type_number_t count
);
813 extern kern_return_t
machine_thread_state_convert_to_user(
815 thread_flavor_t flavor
,
816 thread_state_t tstate
,
817 mach_msg_type_number_t
*count
);
819 extern kern_return_t
machine_thread_dup(
822 boolean_t is_corpse
);
824 extern void machine_thread_init(void);
826 extern kern_return_t
machine_thread_create(
829 extern void machine_thread_switch_addrmode(
832 extern void machine_thread_destroy(
835 extern void machine_set_current_thread(
838 extern kern_return_t
machine_thread_get_kern_state(
840 thread_flavor_t flavor
,
841 thread_state_t tstate
,
842 mach_msg_type_number_t
*count
);
844 extern kern_return_t
machine_thread_inherit_taskwide(
848 extern kern_return_t
machine_thread_set_tsd_base(
850 mach_vm_offset_t tsd_base
);
852 #define thread_mtx_lock(thread) lck_mtx_lock(&(thread)->mutex)
853 #define thread_mtx_try(thread) lck_mtx_try_lock(&(thread)->mutex)
854 #define thread_mtx_unlock(thread) lck_mtx_unlock(&(thread)->mutex)
856 extern void thread_apc_ast(thread_t thread
);
858 extern void thread_update_qos_cpu_time(thread_t thread
);
860 void act_machine_sv_free(thread_t
, int);
862 vm_offset_t
min_valid_stack_address(void);
863 vm_offset_t
max_valid_stack_address(void);
865 static inline uint16_t thread_set_tag_internal(thread_t thread
, uint16_t tag
) {
866 return __sync_fetch_and_or(&thread
->thread_tag
, tag
);
869 static inline uint16_t thread_get_tag_internal(thread_t thread
) {
870 return thread
->thread_tag
;
874 extern void thread_set_options(uint32_t thopt
);
877 #else /* MACH_KERNEL_PRIVATE */
881 extern void thread_mtx_lock(thread_t thread
);
883 extern void thread_mtx_unlock(thread_t thread
);
885 extern thread_t
current_thread(void);
887 extern void thread_reference(
890 extern void thread_deallocate(
895 #endif /* MACH_KERNEL_PRIVATE */
897 #ifdef KERNEL_PRIVATE
901 extern void thread_starts_owning_workloop(
904 extern void thread_ends_owning_workloop(
907 extern uint32_t thread_owned_workloops_count(
911 extern uint64_t thread_dispatchqaddr(
914 extern uint64_t thread_rettokern_addr(
919 #endif /* KERNEL_PRIVATE */
924 extern uint64_t thread_tid(thread_t thread
);
932 #ifdef XNU_KERNEL_PRIVATE
935 * Thread tags; for easy identification.
937 #define THREAD_TAG_MAINTHREAD 0x1
938 #define THREAD_TAG_CALLOUT 0x2
939 #define THREAD_TAG_IOWORKLOOP 0x4
941 #define THREAD_TAG_PTHREAD 0x10
942 #define THREAD_TAG_WORKQUEUE 0x20
944 uint16_t thread_set_tag(thread_t
, uint16_t);
945 uint16_t thread_get_tag(thread_t
);
946 uint64_t thread_last_run_time(thread_t
);
948 extern kern_return_t
thread_state_initialize(
951 extern kern_return_t
thread_setstatus(
954 thread_state_t tstate
,
955 mach_msg_type_number_t count
);
957 extern kern_return_t
thread_setstatus_from_user(
960 thread_state_t tstate
,
961 mach_msg_type_number_t count
);
963 extern kern_return_t
thread_getstatus(
966 thread_state_t tstate
,
967 mach_msg_type_number_t
*count
);
969 extern kern_return_t
thread_getstatus_to_user(
972 thread_state_t tstate
,
973 mach_msg_type_number_t
*count
);
975 extern kern_return_t
thread_create_with_continuation(
977 thread_t
*new_thread
,
978 thread_continue_t continuation
);
980 extern kern_return_t
thread_create_waiting(task_t task
,
981 thread_continue_t continuation
,
983 thread_t
*new_thread
);
985 extern kern_return_t
thread_create_workq_waiting(
987 thread_continue_t thread_return
,
988 thread_t
*new_thread
);
990 extern void thread_yield_internal(
991 mach_msg_timeout_t interval
);
993 extern void thread_yield_to_preemption(void);
996 * Thread-private CPU limits: apply a private CPU limit to this thread only. Available actions are:
998 * 1) Block. Prevent CPU consumption of the thread from exceeding the limit.
999 * 2) Exception. Generate a resource consumption exception when the limit is exceeded.
1000 * 3) Disable. Remove any existing CPU limit.
1002 #define THREAD_CPULIMIT_BLOCK 0x1
1003 #define THREAD_CPULIMIT_EXCEPTION 0x2
1004 #define THREAD_CPULIMIT_DISABLE 0x3
1006 struct _thread_ledger_indices
{
1010 extern struct _thread_ledger_indices thread_ledgers
;
1012 extern int thread_get_cpulimit(int *action
, uint8_t *percentage
, uint64_t *interval_ns
);
1013 extern int thread_set_cpulimit(int action
, uint8_t percentage
, uint64_t interval_ns
);
1015 extern void thread_read_times(
1017 time_value_t
*user_time
,
1018 time_value_t
*system_time
,
1019 time_value_t
*runnable_time
);
1021 extern uint64_t thread_get_runtime_self(void);
1023 extern void thread_setuserstack(
1025 mach_vm_offset_t user_stack
);
1027 extern uint64_t thread_adjuserstack(
1031 extern void thread_setentrypoint(
1033 mach_vm_offset_t entry
);
1035 extern kern_return_t
thread_set_tsd_base(
1037 mach_vm_offset_t tsd_base
);
1039 extern kern_return_t
thread_setsinglestep(
1043 extern kern_return_t
thread_userstack(
1052 extern kern_return_t
thread_entrypoint(
1057 mach_vm_offset_t
*);
1059 extern kern_return_t
thread_userstackdefault(
1063 extern kern_return_t
thread_wire_internal(
1064 host_priv_t host_priv
,
1067 boolean_t
*prev_state
);
1070 extern kern_return_t
thread_dup(thread_t
);
1072 extern kern_return_t
thread_dup2(thread_t
, thread_t
);
1074 #if !defined(_SCHED_CALL_T_DEFINED)
1075 #define _SCHED_CALL_T_DEFINED
1076 typedef void (*sched_call_t
)(
1081 #define SCHED_CALL_BLOCK 0x1
1082 #define SCHED_CALL_UNBLOCK 0x2
1084 extern void thread_sched_call(
1088 extern boolean_t
thread_is_static_param(
1091 extern task_t
get_threadtask(thread_t
);
1094 * Thread is running within a 64-bit address space.
1096 #define thread_is_64bit_addr(thd) \
1097 task_has_64Bit_addr(get_threadtask(thd))
1100 * Thread is using 64-bit machine state.
1102 #define thread_is_64bit_data(thd) \
1103 task_has_64Bit_data(get_threadtask(thd))
1105 extern void *get_bsdthread_info(thread_t
);
1106 extern void set_bsdthread_info(thread_t
, void *);
1107 extern void *uthread_alloc(task_t
, thread_t
, int);
1108 extern event_t
workq_thread_init_and_wq_lock(task_t
, thread_t
); // bsd/pthread/
1109 extern void uthread_cleanup_name(void *uthread
);
1110 extern void uthread_cleanup(task_t
, void *, void *);
1111 extern void uthread_zone_free(void *);
1112 extern void uthread_cred_free(void *);
1114 extern void uthread_reset_proc_refcount(void *);
1116 extern int uthread_get_proc_refcount(void *);
1117 extern int proc_ref_tracking_disabled
;
1120 extern boolean_t
thread_should_halt(
1123 extern boolean_t
thread_should_abort(
1126 extern int is_64signalregset(void);
1128 extern void act_set_kperf(thread_t
);
1129 extern void act_set_astledger(thread_t thread
);
1130 extern void act_set_astledger_async(thread_t thread
);
1131 extern void act_set_io_telemetry_ast(thread_t
);
1133 extern uint32_t dtrace_get_thread_predcache(thread_t
);
1134 extern int64_t dtrace_get_thread_vtime(thread_t
);
1135 extern int64_t dtrace_get_thread_tracing(thread_t
);
1136 extern boolean_t
dtrace_get_thread_reentering(thread_t
);
1137 extern int dtrace_get_thread_last_cpu_id(thread_t
);
1138 extern vm_offset_t
dtrace_get_kernel_stack(thread_t
);
1139 extern void dtrace_set_thread_predcache(thread_t
, uint32_t);
1140 extern void dtrace_set_thread_vtime(thread_t
, int64_t);
1141 extern void dtrace_set_thread_tracing(thread_t
, int64_t);
1142 extern void dtrace_set_thread_reentering(thread_t
, boolean_t
);
1143 extern vm_offset_t
dtrace_set_thread_recover(thread_t
, vm_offset_t
);
1144 extern void dtrace_thread_bootstrap(void);
1145 extern void dtrace_thread_didexec(thread_t
);
1147 extern int64_t dtrace_calc_thread_recent_vtime(thread_t
);
1150 extern kern_return_t
thread_set_wq_state32(
1152 thread_state_t tstate
);
1154 extern kern_return_t
thread_set_wq_state64(
1156 thread_state_t tstate
);
1158 extern vm_offset_t kernel_stack_mask
;
1159 extern vm_offset_t kernel_stack_size
;
1160 extern vm_offset_t kernel_stack_depth_max
;
1162 extern void guard_ast(thread_t
);
1163 extern void fd_guard_ast(thread_t
,
1164 mach_exception_code_t
, mach_exception_subcode_t
);
1166 extern void vn_guard_ast(thread_t
,
1167 mach_exception_code_t
, mach_exception_subcode_t
);
1169 extern void mach_port_guard_ast(thread_t
,
1170 mach_exception_code_t
, mach_exception_subcode_t
);
1171 extern void virt_memory_guard_ast(thread_t
,
1172 mach_exception_code_t
, mach_exception_subcode_t
);
1173 extern void thread_guard_violation(thread_t
,
1174 mach_exception_code_t
, mach_exception_subcode_t
);
1175 extern void thread_update_io_stats(thread_t
, int size
, int io_flags
);
1177 extern kern_return_t
thread_set_voucher_name(mach_port_name_t name
);
1178 extern kern_return_t
thread_get_current_voucher_origin_pid(int32_t *pid
);
1180 extern void set_thread_rwlock_boost(void);
1181 extern void clear_thread_rwlock_boost(void);
1183 /*! @function thread_has_thread_name
1184 @abstract Checks if a thread has a name.
1185 @discussion This function takes one input, a thread, and returns a boolean value indicating if that thread already has a name associated with it.
1186 @param th The thread to inspect.
1187 @result TRUE if the thread has a name, FALSE otherwise.
1189 extern boolean_t
thread_has_thread_name(thread_t th
);
1191 /*! @function thread_set_thread_name
1192 @abstract Set a thread's name.
1193 @discussion This function takes two input parameters: a thread to name, and the name to apply to the thread. The name will be attached to the thread in order to better identify the thread.
1194 @param th The thread to be named.
1195 @param name The name to apply to the thread.
1197 extern void thread_set_thread_name(thread_t th
, const char* name
);
1199 extern void thread_enable_send_importance(thread_t thread
, boolean_t enable
);
1202 * Translate signal context data pointer to userspace representation
1205 extern kern_return_t
machine_thread_siguctx_pointer_convert_to_user(
1207 user_addr_t
*uctxp
);
1210 * Translate array of function pointer syscall arguments from userspace representation
1213 extern kern_return_t
machine_thread_function_pointers_convert_from_user(
1218 /* Get a backtrace for a threads kernel or user stack (user_p), with pc and optionally
1219 * frame pointer (getfp). Returns bytes added to buffer, and kThreadTruncatedBT in
1220 * thread_trace_flags if a user page is not present after kdp_lightweight_fault() is
1224 extern int machine_trace_thread(
1231 uint32_t *thread_trace_flags
);
1233 extern int machine_trace_thread64(thread_t thread
,
1239 uint32_t *thread_trace_flags
,
1243 * Get the duration of the given thread's last wait.
1245 uint64_t thread_get_last_wait_duration(thread_t thread
);
1247 #endif /* XNU_KERNEL_PRIVATE */
1250 /*! @function kernel_thread_start
1251 @abstract Create a kernel thread.
1252 @discussion This function takes three input parameters, namely reference to the function that the thread should execute, caller specified data and a reference which is used to return the newly created kernel thread. The function returns KERN_SUCCESS on success or an appropriate kernel code type indicating the error. It may be noted that the caller is responsible for explicitly releasing the reference to the created thread when no longer needed. This should be done by calling thread_deallocate(new_thread).
1253 @param continuation A C-function pointer where the thread will begin execution.
1254 @param parameter Caller specified data to be passed to the new thread.
1255 @param new_thread Reference to the new thread is returned in this parameter.
1256 @result Returns KERN_SUCCESS on success or an appropriate kernel code type.
1259 extern kern_return_t
kernel_thread_start(
1260 thread_continue_t continuation
,
1262 thread_t
*new_thread
);
1264 #ifdef KERNEL_PRIVATE
1265 void thread_set_eager_preempt(thread_t thread
);
1266 void thread_clear_eager_preempt(thread_t thread
);
1267 void thread_set_honor_qlimit(thread_t thread
);
1268 void thread_clear_honor_qlimit(thread_t thread
);
1269 extern ipc_port_t
convert_thread_to_port(thread_t
);
1270 extern ipc_port_t
convert_thread_inspect_to_port(thread_inspect_t
);
1271 extern boolean_t
is_vm_privileged(void);
1272 extern boolean_t
set_vm_privilege(boolean_t
);
1273 extern kern_allocation_name_t
thread_set_allocation_name(kern_allocation_name_t new_name
);
1274 #endif /* KERNEL_PRIVATE */
1278 #endif /* _KERN_THREAD_H_ */