osfmk/kern/thread.h

   1 /*
   2  * Copyright (c) 2000-2012 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 /*
  29  * @OSF_FREE_COPYRIGHT@
  30  */
  31 /*
  32  * Mach Operating System
  33  * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
  34  * All Rights Reserved.
  35  *
  36  * Permission to use, copy, modify and distribute this software and its
  37  * documentation is hereby granted, provided that both the copyright
  38  * notice and this permission notice appear in all copies of the
  39  * software, derivative works or modified versions, and any portions
  40  * thereof, and that both notices appear in supporting documentation.
  41  *
  42  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  43  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
  44  * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
  45  *
  46  * Carnegie Mellon requests users of this software to return to
  47  *
  48  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  49  *  School of Computer Science
  50  *  Carnegie Mellon University
  51  *  Pittsburgh PA 15213-3890
  52  *
  53  * any improvements or extensions that they make and grant Carnegie Mellon
  54  * the rights to redistribute these changes.
  55  */
  56 /*
  57  */
  58 /*
  59  *      File:   thread.h
  60  *      Author: Avadis Tevanian, Jr.
  61  *
  62  *      This file contains the structure definitions for threads.
  63  *
  64  */
  65 /*
  66  * Copyright (c) 1993 The University of Utah and
  67  * the Computer Systems Laboratory (CSL).  All rights reserved.
  68  *
  69  * Permission to use, copy, modify and distribute this software and its
  70  * documentation is hereby granted, provided that both the copyright
  71  * notice and this permission notice appear in all copies of the
  72  * software, derivative works or modified versions, and any portions
  73  * thereof, and that both notices appear in supporting documentation.
  74  *
  75  * THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF THIS SOFTWARE IN ITS "AS
  76  * IS" CONDITION.  THE UNIVERSITY OF UTAH AND CSL DISCLAIM ANY LIABILITY OF
  77  * ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
  78  *
  79  * CSL requests users of this software to return to csl-dist@cs.utah.edu any
  80  * improvements that they make and grant CSL redistribution rights.
  81  *
  82  */
  83
  84 #ifndef _KERN_THREAD_H_
  85 #define _KERN_THREAD_H_
  86
  87 #include <mach/kern_return.h>
  88 #include <mach/mach_types.h>
  89 #include <mach/message.h>
  90 #include <mach/boolean.h>
  91 #include <mach/vm_param.h>
  92 #include <mach/thread_info.h>
  93 #include <mach/thread_status.h>
  94 #include <mach/exception_types.h>
  95
  96 #include <kern/kern_types.h>
  97
  98 #include <sys/cdefs.h>
  99
 100 #ifdef  MACH_KERNEL_PRIVATE
 101
 102 #include <cputypes.h>
 103
 104 #include <mach_assert.h>
 105 #include <mach_ldebug.h>
 106
 107 #include <ipc/ipc_types.h>
 108
 109 #include <mach/port.h>
 110 #include <kern/cpu_number.h>
 111 #include <kern/queue.h>
 112 #include <kern/timer.h>
 113 #include <kern/lock.h>
 114 #include <kern/locks.h>
 115 #include <kern/sched.h>
 116 #include <kern/sched_prim.h>
 117 #include <kern/thread_call.h>
 118 #include <kern/timer_call.h>
 119 #include <kern/task.h>
 120 #include <kern/exception.h>
 121 #include <kern/affinity.h>
 122
 123 #include <ipc/ipc_kmsg.h>
 124
 125 #include <machine/cpu_data.h>
 126 #include <machine/thread.h>
 127
 128 struct thread {
 129         /*
 130          *      NOTE:   The runq field in the thread structure has an unusual
 131          *      locking protocol.  If its value is PROCESSOR_NULL, then it is
 132          *      locked by the thread_lock, but if its value is something else
 133          *      then it is locked by the associated run queue lock.
 134          *
 135          *      When the thread is on a wait queue, these first three fields
 136          *      are treated as an unofficial union with a wait_queue_element.
 137          *      If you change these, you must change that definition as well
 138          *      (kern/wait_queue.h).
 139          */
 140         /* Items examined often, modified infrequently */
 141         queue_chain_t   links;                          /* run/wait queue links */
 142         processor_t             runq;                           /* run queue assignment */
 143         wait_queue_t    wait_queue;                     /* wait queue we are currently on */
 144         event64_t               wait_event;                     /* wait queue event */
 145         /* Data updated during assert_wait/thread_wakeup */
 146         decl_simple_lock_data(,sched_lock)      /* scheduling lock (thread_lock()) */
 147         decl_simple_lock_data(,wake_lock)       /* for thread stop / wait (wake_lock()) */
 148         integer_t               options;                        /* options set by thread itself */
 149 #define TH_OPT_INTMASK          0x0003          /* interrupt / abort level */
 150 #define TH_OPT_VMPRIV           0x0004          /* may allocate reserved memory */
 151 #define TH_OPT_DTRACE           0x0008          /* executing under dtrace_probe */
 152 #define TH_OPT_SYSTEM_CRITICAL  0x0010          /* Thread must always be allowed to run - even under heavy load */
 153 #define TH_OPT_PROC_CPULIMIT    0x0020          /* Thread has a task-wide CPU limit applied to it */
 154 #define TH_OPT_PRVT_CPULIMIT    0x0040          /* Thread has a thread-private CPU limit applied to it */
 155 #define TH_OPT_IDLE_THREAD              0x0080          /* Thread is a per-processor idle thread */
 156
 157         boolean_t                       wake_active;    /* wake event on stop */
 158         int                                     at_safe_point;  /* thread_abort_safely allowed */
 159         ast_t                           reason;                 /* why we blocked */
 160         thread_continue_t       continuation;   /* continue here next dispatch */
 161         void                            *parameter;             /* continuation parameter */
 162         wait_result_t           wait_result;    /* outcome of wait -
 163                                                                                  * may be examined by this thread
 164                                                                                  * WITHOUT locking */
 165
 166         /* Data updated/used in thread_invoke */
 167     int                             funnel_state;
 168     struct funnel_lock  *funnel_lock;           /* Non-reentrancy funnel */
 169 #define TH_FN_OWNED                     0x1                             /* we own the funnel */
 170 #define TH_FN_REFUNNEL          0x2                             /* re-acquire funnel on dispatch */
 171
 172         vm_offset_t             kernel_stack;           /* current kernel stack */
 173         vm_offset_t                     reserved_stack;         /* reserved kernel stack */
 174
 175         /* Thread state: */
 176         int                                     state;
 177 /*
 178  *      Thread states [bits or'ed]
 179  */
 180 #define TH_WAIT                 0x01                    /* queued for waiting */
 181 #define TH_SUSP                 0x02                    /* stopped or requested to stop */
 182 #define TH_RUN                  0x04                    /* running or on runq */
 183 #define TH_UNINT                0x08                    /* waiting uninteruptibly */
 184 #define TH_TERMINATE    0x10                    /* halted at termination */
 185 #define TH_TERMINATE2   0x20                    /* added to termination queue */
 186
 187 #define TH_IDLE                 0x80                    /* idling processor */
 188
 189         /* Scheduling information */
 190         sched_mode_t                    sched_mode;             /* scheduling mode */
 191         sched_mode_t                    saved_mode;             /* saved mode during forced mode demotion */
 192
 193         unsigned int                    sched_flags;            /* current flag bits */
 194 #define TH_SFLAG_FAIRSHARE_TRIPPED      0x0001          /* fairshare scheduling activated */
 195 #define TH_SFLAG_FAILSAFE               0x0002          /* fail-safe has tripped */
 196 #define TH_SFLAG_THROTTLED              0x0004      /* owner task in throttled state */
 197 #define TH_SFLAG_DEMOTED_MASK      (TH_SFLAG_THROTTLED | TH_SFLAG_FAILSAFE | TH_SFLAG_FAIRSHARE_TRIPPED)
 198
 199 #define TH_SFLAG_PROMOTED               0x0008          /* sched pri has been promoted */
 200 #define TH_SFLAG_ABORT                  0x0010          /* abort interruptible waits */
 201 #define TH_SFLAG_ABORTSAFELY            0x0020          /* ... but only those at safe point */
 202 #define TH_SFLAG_ABORTED_MASK           (TH_SFLAG_ABORT | TH_SFLAG_ABORTSAFELY)
 203 #define TH_SFLAG_DEPRESS                0x0040          /* normal depress yield */
 204 #define TH_SFLAG_POLLDEPRESS            0x0080          /* polled depress yield */
 205 #define TH_SFLAG_DEPRESSED_MASK         (TH_SFLAG_DEPRESS | TH_SFLAG_POLLDEPRESS)
 206 #define TH_SFLAG_PRI_UPDATE             0x0100          /* Updating priority */
 207 #define TH_SFLAG_EAGERPREEMPT           0x0200          /* Any preemption of this thread should be treated as if AST_URGENT applied */
 208 #define TH_SFLAG_RW_PROMOTED            0x0400          /* sched pri has been promoted due to blocking with RW lock held */
 209 #define TH_SFLAG_PROMOTED_MASK          (TH_SFLAG_PROMOTED | TH_SFLAG_RW_PROMOTED)
 210
 211 #define TH_SFLAG_RW_PROMOTED_BIT        (10)    /* 0x400 */
 212
 213 /*
 214  * A thread can either be completely unthrottled, about to be throttled,
 215  * throttled (TH_SFLAG_THROTTLED), or about to be unthrottled
 216  */
 217 #define TH_SFLAG_PENDING_THROTTLE_DEMOTION      0x1000  /* Pending sched_mode demotion */
 218 #define TH_SFLAG_PENDING_THROTTLE_PROMOTION     0x2000  /* Pending sched_mode promition */
 219 #define TH_SFLAG_PENDING_THROTTLE_MASK          (TH_SFLAG_PENDING_THROTTLE_DEMOTION | TH_SFLAG_PENDING_THROTTLE_PROMOTION)
 220
 221         int16_t                         sched_pri;                      /* scheduled (current) priority */
 222         int16_t                         priority;                       /* base priority */
 223         int16_t                         max_priority;           /* max base priority */
 224         int16_t                         task_priority;          /* copy of task base priority */
 225 #if defined(CONFIG_SCHED_GRRR)
 226 #if 0
 227         uint16_t                        grrr_deficit;           /* fixed point (1/1000th quantum) fractional deficit */
 228 #endif
 229 #endif
 230
 231         int16_t                         promotions;                     /* level of promotion */
 232         int16_t                         pending_promoter_index;
 233         uint32_t                        ref_count;              /* number of references to me */
 234         void                            *pending_promoter[2];
 235
 236         uint32_t                        rwlock_count;   /* Number of lck_rw_t locks held by thread */
 237
 238         integer_t                       importance;                     /* task-relative importance */
 239         /* Priority depression expiration */
 240         integer_t                       depress_timer_active;
 241         timer_call_data_t       depress_timer;
 242                                                                                 /* real-time parameters */
 243         struct {                                                                /* see mach/thread_policy.h */
 244                 uint32_t                        period;
 245                 uint32_t                        computation;
 246                 uint32_t                        constraint;
 247                 boolean_t                       preemptible;
 248                 uint64_t                        deadline;
 249         }                                       realtime;
 250
 251         uint32_t                        was_promoted_on_wakeup;
 252         uint32_t                        current_quantum;        /* duration of current quantum */
 253         uint64_t                        last_run_time;          /* time when thread was switched away from */
 254         uint64_t                        last_quantum_refill_time;       /* time when current_quantum was refilled after expiration */
 255
 256   /* Data used during setrun/dispatch */
 257         timer_data_t            system_timer;           /* system mode timer */
 258         processor_t                     bound_processor;        /* bound to a processor? */
 259         processor_t                     last_processor;         /* processor last dispatched on */
 260         processor_t                     chosen_processor;       /* Where we want to run this thread */
 261
 262         /* Fail-safe computation since last unblock or qualifying yield */
 263         uint64_t                        computation_metered;
 264         uint64_t                        computation_epoch;
 265         uint64_t                        safe_release;   /* when to release fail-safe */
 266
 267         /* Call out from scheduler */
 268         void                            (*sched_call)(
 269                                                         int                     type,
 270                                                         thread_t        thread);
 271 #if defined(CONFIG_SCHED_PROTO)
 272         uint32_t                        runqueue_generation;    /* last time runqueue was drained */
 273 #endif
 274
 275         /* Statistics and timesharing calculations */
 276 #if defined(CONFIG_SCHED_TRADITIONAL)
 277         natural_t                       sched_stamp;    /* last scheduler tick */
 278         natural_t                       sched_usage;    /* timesharing cpu usage [sched] */
 279         natural_t                       pri_shift;              /* usage -> priority from pset */
 280         natural_t                       cpu_usage;              /* instrumented cpu usage [%cpu] */
 281         natural_t                       cpu_delta;              /* accumulated cpu_usage delta */
 282 #endif
 283         uint32_t                        c_switch;               /* total context switches */
 284         uint32_t                        p_switch;               /* total processor switches */
 285         uint32_t                        ps_switch;              /* total pset switches */
 286
 287         /* Timing data structures */
 288         int                                     precise_user_kernel_time; /* precise user/kernel enabled for this thread */
 289         timer_data_t            user_timer;                     /* user mode timer */
 290         uint64_t                        user_timer_save;        /* saved user timer value */
 291         uint64_t                        system_timer_save;      /* saved system timer value */
 292         uint64_t                        vtimer_user_save;       /* saved values for vtimers */
 293         uint64_t                        vtimer_prof_save;
 294         uint64_t                        vtimer_rlim_save;
 295
 296         /* Timed wait expiration */
 297         timer_call_data_t       wait_timer;
 298         integer_t                       wait_timer_active;
 299         boolean_t                       wait_timer_is_set;
 300
 301
 302         /*
 303          * Processor/cache affinity
 304          * - affinity_threads links task threads with the same affinity set
 305          */
 306         affinity_set_t                  affinity_set;
 307         queue_chain_t                   affinity_threads;
 308
 309         /* Various bits of stashed state */
 310         union {
 311                 struct {
 312                         mach_msg_return_t       state;          /* receive state */
 313                         mach_port_seqno_t       seqno;          /* seqno of recvd message */
 314                         ipc_object_t            object;         /* object received on */
 315                         mach_vm_address_t       msg_addr;       /* receive buffer pointer */
 316                         mach_msg_size_t         msize;          /* max size for recvd msg */
 317                         mach_msg_option_t       option;         /* options for receive */
 318                         mach_msg_size_t         slist_size;     /* scatter list size */
 319                         mach_port_name_t        receiver_name;  /* the receive port name */
 320                         struct ipc_kmsg         *kmsg;          /* received message */
 321                         mach_msg_continue_t     continuation;
 322                 } receive;
 323                 struct {
 324                         struct semaphore        *waitsemaphore;         /* semaphore ref */
 325                         struct semaphore        *signalsemaphore;       /* semaphore ref */
 326                         int                                     options;                        /* semaphore options */
 327                         kern_return_t           result;                         /* primary result */
 328                         mach_msg_continue_t continuation;
 329                 } sema;
 330                 struct {
 331                         int                                     option;         /* switch option */
 332                         boolean_t                               reenable_workq_callback;        /* on entry, callbacks were suspended */
 333                 } swtch;
 334                 int                                             misc;           /* catch-all for other state */
 335         } saved;
 336
 337         /* Structure to save information about guard exception */
 338         struct {
 339                 unsigned                                type;           /* EXC_GUARD reason/type */
 340                 mach_exception_data_type_t              code;           /* Exception code */
 341                 mach_exception_data_type_t              subcode;        /* Exception sub-code */
 342         } guard_exc_info;
 343
 344
 345         /* IPC data structures */
 346 #if IMPORTANCE_INHERITANCE
 347         natural_t ith_assertions;                       /* assertions pending drop */
 348 #endif
 349         struct ipc_kmsg_queue ith_messages;             /* messages to reap */
 350         mach_port_t ith_rpc_reply;                      /* reply port for kernel RPCs */
 351
 352         /* Ast/Halt data structures */
 353         vm_offset_t                                     recover;                /* page fault recover(copyin/out) */
 354
 355         queue_chain_t                           threads;                /* global list of all threads */
 356
 357         /* Activation */
 358                 queue_chain_t                   task_threads;
 359
 360                 /*** Machine-dependent state ***/
 361                 struct machine_thread   machine;
 362
 363                 /* Task membership */
 364                 struct task                             *task;
 365                 vm_map_t                                map;
 366
 367                 decl_lck_mtx_data(,mutex)
 368
 369
 370                 /* Pending thread ast(s) */
 371                 ast_t                                   ast;
 372
 373                 /* Miscellaneous bits guarded by mutex */
 374                 uint32_t
 375                         active:1,                               /* Thread is active and has not been terminated */
 376                         started:1,                              /* Thread has been started after creation */
 377                         static_param:1,                 /* Disallow policy parameter changes */
 378                         :0;
 379
 380                 /* Return Handers */
 381                 struct ReturnHandler {
 382                         struct ReturnHandler    *next;
 383                         void            (*handler)(
 384                                                         struct ReturnHandler            *rh,
 385                                                         struct thread                           *thread);
 386                 } *handlers, special_handler;
 387
 388                 /* Ports associated with this thread */
 389                 struct ipc_port                 *ith_self;              /* not a right, doesn't hold ref */
 390                 struct ipc_port                 *ith_sself;             /* a send right */
 391                 struct exception_action *exc_actions;
 392
 393 #ifdef  MACH_BSD
 394                 void                                    *uthread;
 395 #endif
 396
 397 #if CONFIG_DTRACE
 398                 uint32_t t_dtrace_flags;        /* DTrace thread states */
 399 #define TH_DTRACE_EXECSUCCESS   0x01
 400                 uint32_t t_dtrace_predcache;/* DTrace per thread predicate value hint */
 401                 int64_t t_dtrace_tracing;       /* Thread time under dtrace_probe() */
 402                 int64_t t_dtrace_vtime;
 403 #endif
 404
 405                 clock_sec_t t_page_creation_time;
 406                 uint32_t    t_page_creation_count;
 407
 408 #define T_CHUD_MARKED           0x01          /* this thread is marked by CHUD */
 409 #define T_IN_CHUD               0x02          /* this thread is already in a CHUD handler */
 410 #define THREAD_PMC_FLAG         0x04          /* Bit in "t_chud" signifying PMC interest */
 411 #define T_AST_CALLSTACK         0x08          /* Thread scheduled to dump a
 412                                                * callstack on its next
 413                                                * AST */
 414 #define T_AST_NAME              0x10          /* Thread scheduled to dump
 415                                                * its name on its next
 416                                                * AST */
 417 #define T_NAME_DONE             0x20          /* Thread has previously
 418                                                * recorded its name */
 419 #define T_KPC_ALLOC             0x40          /* Thread needs a kpc_buf */
 420
 421                 uint32_t t_chud;        /* CHUD flags, used for Shark */
 422                 uint32_t chud_c_switch; /* last dispatch detection */
 423
 424                 integer_t mutex_count;  /* total count of locks held */
 425
 426 #ifdef KPC
 427         /* accumulated performance counters for this thread */
 428         uint64_t *kpc_buf;
 429 #endif
 430
 431 #ifdef KPERF
 432         /* count of how many times a thread has been sampled since it was last scheduled */
 433         uint64_t kperf_pet_cnt;
 434 #endif
 435
 436                 uint64_t thread_id;     /*system wide unique thread-id*/
 437
 438         /* Statistics accumulated per-thread and aggregated per-task */
 439         uint32_t                syscalls_unix;
 440         uint32_t                syscalls_mach;
 441         ledger_t                t_ledger;
 442         ledger_t                t_threadledger; /* per thread ledger */
 443
 444         /* policy is protected by the task lock */
 445         struct task_requested_policy     requested_policy;
 446         struct task_effective_policy     effective_policy;
 447         struct task_pended_policy        pended_policy;
 448
 449         int     iotier_override; /* atomic operations to set, cleared on ret to user */
 450
 451
 452         integer_t               saved_importance;               /* saved task-relative importance */
 453
 454         uint32_t                        thread_callout_interrupt_wakeups;
 455         uint32_t                        thread_callout_platform_idle_wakeups;
 456         uint32_t                        thread_timer_wakeups_bin_1;
 457         uint32_t                        thread_timer_wakeups_bin_2;
 458         uint16_t                        thread_tag;
 459         uint16_t                        callout_woken_from_icontext:1,
 460                                         callout_woken_from_platform_idle:1,
 461                                         callout_woke_thread:1,
 462                                         thread_bitfield_unused:13;
 463         /* Kernel holds on this thread  */
 464         int16_t                                         suspend_count;
 465         /* User level suspensions */
 466         int16_t                                         user_stop_count;
 467 };
 468
 469 #define ith_state               saved.receive.state
 470 #define ith_object              saved.receive.object
 471 #define ith_msg_addr                    saved.receive.msg_addr
 472 #define ith_msize               saved.receive.msize
 473 #define ith_option              saved.receive.option
 474 #define ith_scatter_list_size   saved.receive.slist_size
 475 #define ith_receiver_name       saved.receive.receiver_name
 476 #define ith_continuation        saved.receive.continuation
 477 #define ith_kmsg                saved.receive.kmsg
 478 #define ith_seqno               saved.receive.seqno
 479
 480 #define sth_waitsemaphore       saved.sema.waitsemaphore
 481 #define sth_signalsemaphore     saved.sema.signalsemaphore
 482 #define sth_options             saved.sema.options
 483 #define sth_result              saved.sema.result
 484 #define sth_continuation        saved.sema.continuation
 485
 486 extern void                     thread_bootstrap(void);
 487
 488 extern void                     thread_init(void);
 489
 490 extern void                     thread_daemon_init(void);
 491
 492 #define thread_reference_internal(thread)       \
 493                         (void)hw_atomic_add(&(thread)->ref_count, 1)
 494
 495 #define thread_deallocate_internal(thread)      \
 496                         hw_atomic_sub(&(thread)->ref_count, 1)
 497
 498 #define thread_reference(thread)                                        \
 499 MACRO_BEGIN                                                                                     \
 500         if ((thread) != THREAD_NULL)                                    \
 501                 thread_reference_internal(thread);              \
 502 MACRO_END
 503
 504 extern void                     thread_deallocate(
 505                                                 thread_t                thread);
 506
 507 extern void                     thread_terminate_self(void);
 508
 509 extern kern_return_t    thread_terminate_internal(
 510                                                         thread_t                thread);
 511
 512 extern void                     thread_start_internal(
 513                                                         thread_t                        thread) __attribute__ ((noinline));
 514
 515 extern void                     thread_terminate_enqueue(
 516                                                 thread_t                thread);
 517
 518 extern void                     thread_stack_enqueue(
 519                                                 thread_t                thread);
 520
 521 extern void                     thread_hold(
 522                                                 thread_t        thread);
 523
 524 extern void                     thread_release(
 525                                                 thread_t        thread);
 526
 527
 528 #define thread_lock_init(th)    simple_lock_init(&(th)->sched_lock, 0)
 529 #define thread_lock(th)                 simple_lock(&(th)->sched_lock)
 530 #define thread_unlock(th)               simple_unlock(&(th)->sched_lock)
 531
 532 #define wake_lock_init(th)              simple_lock_init(&(th)->wake_lock, 0)
 533 #define wake_lock(th)                   simple_lock(&(th)->wake_lock)
 534 #define wake_unlock(th)                 simple_unlock(&(th)->wake_lock)
 535
 536 #define thread_should_halt_fast(thread)         (!(thread)->active)
 537
 538 extern void                             stack_alloc(
 539                                                         thread_t                thread);
 540
 541 extern void                     stack_handoff(
 542                                                         thread_t                from,
 543                                                         thread_t                to);
 544
 545 extern void                             stack_free(
 546                                                         thread_t                thread);
 547
 548 extern void                             stack_free_reserved(
 549                                                         thread_t                thread);
 550
 551 extern boolean_t                stack_alloc_try(
 552                                                         thread_t            thread);
 553
 554 extern void                             stack_collect(void);
 555
 556 extern void                             stack_init(void);
 557
 558
 559 extern kern_return_t    thread_info_internal(
 560                                                         thread_t                                thread,
 561                                                         thread_flavor_t                 flavor,
 562                                                         thread_info_t                   thread_info_out,
 563                                                         mach_msg_type_number_t  *thread_info_count);
 564
 565 extern void                             thread_task_priority(
 566                                                         thread_t                thread,
 567                                                         integer_t               priority,
 568                                                         integer_t               max_priority);
 569
 570 extern void                             thread_policy_reset(
 571                                                         thread_t                thread);
 572
 573 extern kern_return_t    kernel_thread_create(
 574                                                         thread_continue_t       continuation,
 575                                                         void                            *parameter,
 576                                                         integer_t                       priority,
 577                                                         thread_t                        *new_thread);
 578
 579 extern kern_return_t    kernel_thread_start_priority(
 580                                                         thread_continue_t       continuation,
 581                                                         void                            *parameter,
 582                                                         integer_t                       priority,
 583                                                         thread_t                        *new_thread);
 584
 585 extern void                             machine_stack_attach(
 586                                                         thread_t                thread,
 587                                                         vm_offset_t             stack);
 588
 589 extern vm_offset_t              machine_stack_detach(
 590                                                         thread_t                thread);
 591
 592 extern void                             machine_stack_handoff(
 593                                                         thread_t                old,
 594                                                         thread_t                new);
 595
 596 extern thread_t                 machine_switch_context(
 597                                                         thread_t                        old_thread,
 598                                                         thread_continue_t       continuation,
 599                                                         thread_t                        new_thread);
 600
 601 extern void                             machine_load_context(
 602                                                         thread_t                thread);
 603
 604 extern kern_return_t    machine_thread_state_initialize(
 605                                                         thread_t                                thread);
 606
 607 extern kern_return_t    machine_thread_set_state(
 608                                                         thread_t                                thread,
 609                                                         thread_flavor_t                 flavor,
 610                                                         thread_state_t                  state,
 611                                                         mach_msg_type_number_t  count);
 612
 613 extern kern_return_t    machine_thread_get_state(
 614                                                         thread_t                                thread,
 615                                                         thread_flavor_t                 flavor,
 616                                                         thread_state_t                  state,
 617                                                         mach_msg_type_number_t  *count);
 618
 619 extern kern_return_t    machine_thread_dup(
 620                                                         thread_t                self,
 621                                                         thread_t                target);
 622
 623 extern void                             machine_thread_init(void);
 624
 625 extern kern_return_t    machine_thread_create(
 626                                                         thread_t                thread,
 627                                                         task_t                  task);
 628 extern void             machine_thread_switch_addrmode(
 629                                                        thread_t                 thread);
 630
 631 extern void                 machine_thread_destroy(
 632                                                         thread_t                thread);
 633
 634 extern void                             machine_set_current_thread(
 635                                                         thread_t                        thread);
 636
 637 extern kern_return_t    machine_thread_get_kern_state(
 638                                                         thread_t                                thread,
 639                                                         thread_flavor_t                 flavor,
 640                                                         thread_state_t                  tstate,
 641                                                         mach_msg_type_number_t  *count);
 642
 643 extern kern_return_t    machine_thread_inherit_taskwide(
 644                                                         thread_t                thread,
 645                                                         task_t                  parent_task);
 646
 647 /*
 648  * XXX Funnel locks XXX
 649  */
 650
 651 struct funnel_lock {
 652         int                     fnl_type;                       /* funnel type */
 653         lck_mtx_t       *fnl_mutex;                     /* underlying mutex for the funnel */
 654         void *          fnl_mtxholder;          /* thread (last)holdng mutex */
 655         void *          fnl_mtxrelease;         /* thread (last)releasing mutex */
 656         lck_mtx_t       *fnl_oldmutex;          /* Mutex before collapsing split funnel */
 657 };
 658
 659 typedef struct ReturnHandler            ReturnHandler;
 660
 661 #define thread_mtx_lock(thread)                 lck_mtx_lock(&(thread)->mutex)
 662 #define thread_mtx_try(thread)                  lck_mtx_try_lock(&(thread)->mutex)
 663 #define thread_mtx_unlock(thread)               lck_mtx_unlock(&(thread)->mutex)
 664
 665 extern void                     act_execute_returnhandlers(void);
 666
 667 extern void                     install_special_handler(
 668                                                 thread_t                thread);
 669
 670 extern void                     special_handler(
 671                                                 ReturnHandler   *rh,
 672                                                 thread_t                thread);
 673
 674 void act_machine_sv_free(thread_t, int);
 675
 676 vm_offset_t                     min_valid_stack_address(void);
 677 vm_offset_t                     max_valid_stack_address(void);
 678
 679 extern void             funnel_lock(
 680                                                 struct funnel_lock      *lock);
 681
 682 extern void             funnel_unlock(
 683                                                 struct funnel_lock      *lock);
 684
 685
 686 static inline uint16_t  thread_set_tag_internal(thread_t        thread, uint16_t tag) {
 687         return __sync_fetch_and_or(&thread->thread_tag, tag);
 688 }
 689
 690 static inline uint16_t  thread_get_tag_internal(thread_t        thread) {
 691         return thread->thread_tag;
 692 }
 693
 694 #else   /* MACH_KERNEL_PRIVATE */
 695
 696 __BEGIN_DECLS
 697
 698 extern thread_t         current_thread(void);
 699
 700 extern void                     thread_reference(
 701                                                 thread_t        thread);
 702
 703 extern void                     thread_deallocate(
 704                                                 thread_t        thread);
 705
 706 __END_DECLS
 707
 708 #endif  /* MACH_KERNEL_PRIVATE */
 709
 710 #ifdef  KERNEL_PRIVATE
 711
 712 __BEGIN_DECLS
 713
 714
 715 extern uint64_t                 thread_tid(
 716                                                 thread_t thread);
 717
 718 extern uint64_t                 thread_dispatchqaddr(
 719                                                 thread_t thread);
 720
 721 __END_DECLS
 722
 723 #endif  /* KERNEL_PRIVATE */
 724
 725 __BEGIN_DECLS
 726
 727 #ifdef  XNU_KERNEL_PRIVATE
 728
 729 /*
 730  * Thread tags; for easy identification.
 731  */
 732 #define THREAD_TAG_MAINTHREAD 0x1
 733 #define THREAD_TAG_CALLOUT 0x2
 734 #define THREAD_TAG_IOWORKLOOP 0x4
 735
 736 uint16_t        thread_set_tag(thread_t, uint16_t);
 737 uint16_t        thread_get_tag(thread_t);
 738
 739
 740 extern kern_return_t    thread_state_initialize(
 741                                                         thread_t                                thread);
 742
 743 extern kern_return_t    thread_setstatus(
 744                                                         thread_t                                thread,
 745                                                         int                                             flavor,
 746                                                         thread_state_t                  tstate,
 747                                                         mach_msg_type_number_t  count);
 748
 749 extern kern_return_t    thread_getstatus(
 750                                                         thread_t                                thread,
 751                                                         int                                             flavor,
 752                                                         thread_state_t                  tstate,
 753                                                         mach_msg_type_number_t  *count);
 754
 755 extern kern_return_t    thread_create_workq(
 756                                                         task_t                  task,
 757                                                         thread_continue_t       thread_return,
 758                                                         thread_t                *new_thread);
 759
 760 extern  void    thread_yield_internal(
 761         mach_msg_timeout_t      interval);
 762
 763 /*
 764  * Thread-private CPU limits: apply a private CPU limit to this thread only. Available actions are:
 765  *
 766  * 1) Block. Prevent CPU consumption of the thread from exceeding the limit.
 767  * 2) Exception. Generate a resource consumption exception when the limit is exceeded.
 768  * 3) Disable. Remove any existing CPU limit.
 769  */
 770 #define THREAD_CPULIMIT_BLOCK           0x1
 771 #define THREAD_CPULIMIT_EXCEPTION       0x2
 772 #define THREAD_CPULIMIT_DISABLE         0x3
 773
 774 struct _thread_ledger_indices {
 775         int cpu_time;
 776 };
 777
 778 extern struct _thread_ledger_indices thread_ledgers;
 779
 780 extern int thread_get_cpulimit(int *action, uint8_t *percentage, uint64_t *interval_ns);
 781 extern int thread_set_cpulimit(int action, uint8_t percentage, uint64_t interval_ns);
 782
 783 typedef struct funnel_lock              funnel_t;
 784
 785 #define THR_FUNNEL_NULL (funnel_t *)0
 786
 787 extern funnel_t          *funnel_alloc(
 788                                                 int                     type);
 789
 790 extern void                     funnel_free(
 791                                                 funnel_t        *lock);
 792
 793 extern funnel_t         *thread_funnel_get(void);
 794
 795 extern boolean_t        thread_funnel_set(
 796                                                 funnel_t        *lock,
 797                                                 boolean_t        funneled);
 798
 799 extern void                     thread_read_times(
 800                                                 thread_t                thread,
 801                                                 time_value_t    *user_time,
 802                                                 time_value_t    *system_time);
 803
 804 extern void                     thread_setuserstack(
 805                                                 thread_t                thread,
 806                                                 mach_vm_offset_t        user_stack);
 807
 808 extern uint64_t         thread_adjuserstack(
 809                                                 thread_t                thread,
 810                                                 int                             adjust);
 811
 812 extern void                     thread_setentrypoint(
 813                                                 thread_t                thread,
 814                                                 mach_vm_offset_t        entry);
 815
 816 extern kern_return_t    thread_setsinglestep(
 817                                                 thread_t                thread,
 818                                                 int                     on);
 819
 820 extern kern_return_t    thread_userstack(
 821                                                 thread_t,
 822                                                 int,
 823                                                 thread_state_t,
 824                                                 unsigned int,
 825                                                 mach_vm_offset_t *,
 826                                                 int *);
 827
 828 extern kern_return_t    thread_entrypoint(
 829                                                 thread_t,
 830                                                 int,
 831                                                 thread_state_t,
 832                                                 unsigned int,
 833                                                 mach_vm_offset_t *);
 834
 835 extern kern_return_t    thread_userstackdefault(
 836                                                 thread_t,
 837                                                 mach_vm_offset_t *);
 838
 839 extern kern_return_t    thread_wire_internal(
 840                                                         host_priv_t             host_priv,
 841                                                         thread_t                thread,
 842                                                         boolean_t               wired,
 843                                                         boolean_t               *prev_state);
 844
 845 extern kern_return_t    thread_dup(thread_t);
 846
 847 typedef void    (*sched_call_t)(
 848                                         int                             type,
 849                                         thread_t                thread);
 850
 851 #define SCHED_CALL_BLOCK                0x1
 852 #define SCHED_CALL_UNBLOCK              0x2
 853
 854 extern void             thread_sched_call(
 855                                         thread_t                thread,
 856                                         sched_call_t    call);
 857
 858 extern void             thread_static_param(
 859                                         thread_t                thread,
 860                                         boolean_t               state);
 861
 862 extern kern_return_t    thread_policy_set_internal(
 863                                         thread_t                thread,
 864                                         thread_policy_flavor_t  flavor,
 865                                         thread_policy_t         policy_info,
 866                                         mach_msg_type_number_t  count);
 867
 868
 869 extern task_t   get_threadtask(thread_t);
 870 #define thread_is_64bit(thd)    \
 871         task_has_64BitAddr(get_threadtask(thd))
 872
 873
 874 extern void             *get_bsdthread_info(thread_t);
 875 extern void             set_bsdthread_info(thread_t, void *);
 876 extern void             *uthread_alloc(task_t, thread_t, int);
 877 extern void             uthread_cleanup(task_t, void *, void *);
 878 extern void             uthread_zone_free(void *);
 879 extern void             uthread_cred_free(void *);
 880
 881 extern boolean_t        thread_should_halt(
 882                                                 thread_t                thread);
 883
 884 extern boolean_t        thread_should_abort(
 885                                                 thread_t);
 886
 887 extern int is_64signalregset(void);
 888
 889 void act_set_apc(thread_t);
 890 void act_set_kperf(thread_t);
 891
 892 extern uint32_t dtrace_get_thread_predcache(thread_t);
 893 extern int64_t dtrace_get_thread_vtime(thread_t);
 894 extern int64_t dtrace_get_thread_tracing(thread_t);
 895 extern boolean_t dtrace_get_thread_reentering(thread_t);
 896 extern vm_offset_t dtrace_get_kernel_stack(thread_t);
 897 extern void dtrace_set_thread_predcache(thread_t, uint32_t);
 898 extern void dtrace_set_thread_vtime(thread_t, int64_t);
 899 extern void dtrace_set_thread_tracing(thread_t, int64_t);
 900 extern void dtrace_set_thread_reentering(thread_t, boolean_t);
 901 extern vm_offset_t dtrace_set_thread_recover(thread_t, vm_offset_t);
 902 extern void dtrace_thread_bootstrap(void);
 903 extern void dtrace_thread_didexec(thread_t);
 904
 905 extern int64_t dtrace_calc_thread_recent_vtime(thread_t);
 906
 907
 908 extern kern_return_t    thread_set_wq_state32(
 909                                               thread_t          thread,
 910                                               thread_state_t    tstate);
 911
 912 extern kern_return_t    thread_set_wq_state64(
 913                                               thread_t          thread,
 914                                               thread_state_t    tstate);
 915
 916 extern vm_offset_t      kernel_stack_mask;
 917 extern vm_offset_t      kernel_stack_size;
 918 extern vm_offset_t      kernel_stack_depth_max;
 919
 920 void guard_ast(thread_t thread);
 921 extern void fd_guard_ast(thread_t thread);
 922 extern void mach_port_guard_ast(thread_t thread);
 923 extern void thread_guard_violation(thread_t thread, unsigned type);
 924
 925 #endif  /* XNU_KERNEL_PRIVATE */
 926
 927 /*! @function kernel_thread_start
 928     @abstract Create a kernel thread.
 929     @discussion This function takes three input parameters, namely reference to the function that the thread should execute, caller specified data and a reference which is used to return the newly created kernel thread. The function returns KERN_SUCCESS on success or an appropriate kernel code type indicating the error. It may be noted that the caller is responsible for explicitly releasing the reference to the created thread when no longer needed. This should be done by calling thread_deallocate(new_thread).
 930     @param continuation A C-function pointer where the thread will begin execution.
 931     @param parameter Caller specified data to be passed to the new thread.
 932     @param new_thread Reference to the new thread is returned in this parameter.
 933     @result Returns KERN_SUCCESS on success or an appropriate kernel code type.
 934 */
 935
 936 extern kern_return_t    kernel_thread_start(
 937                                                         thread_continue_t       continuation,
 938                                                         void                            *parameter,
 939                                                         thread_t                        *new_thread);
 940 #ifdef KERNEL_PRIVATE
 941 void thread_set_eager_preempt(thread_t thread);
 942 void thread_clear_eager_preempt(thread_t thread);
 943 extern ipc_port_t convert_thread_to_port(thread_t);
 944 #endif /* KERNEL_PRIVATE */
 945
 946 __END_DECLS
 947
 948 #endif  /* _KERN_THREAD_H_ */