osfmk/kern/thread.h

   1 /*
   2  * Copyright (c) 2000-2014 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 /*
  29  * @OSF_FREE_COPYRIGHT@
  30  */
  31 /*
  32  * Mach Operating System
  33  * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
  34  * All Rights Reserved.
  35  *
  36  * Permission to use, copy, modify and distribute this software and its
  37  * documentation is hereby granted, provided that both the copyright
  38  * notice and this permission notice appear in all copies of the
  39  * software, derivative works or modified versions, and any portions
  40  * thereof, and that both notices appear in supporting documentation.
  41  *
  42  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  43  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
  44  * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
  45  *
  46  * Carnegie Mellon requests users of this software to return to
  47  *
  48  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  49  *  School of Computer Science
  50  *  Carnegie Mellon University
  51  *  Pittsburgh PA 15213-3890
  52  *
  53  * any improvements or extensions that they make and grant Carnegie Mellon
  54  * the rights to redistribute these changes.
  55  */
  56 /*
  57  */
  58 /*
  59  *      File:   thread.h
  60  *      Author: Avadis Tevanian, Jr.
  61  *
  62  *      This file contains the structure definitions for threads.
  63  *
  64  */
  65 /*
  66  * Copyright (c) 1993 The University of Utah and
  67  * the Computer Systems Laboratory (CSL).  All rights reserved.
  68  *
  69  * Permission to use, copy, modify and distribute this software and its
  70  * documentation is hereby granted, provided that both the copyright
  71  * notice and this permission notice appear in all copies of the
  72  * software, derivative works or modified versions, and any portions
  73  * thereof, and that both notices appear in supporting documentation.
  74  *
  75  * THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF THIS SOFTWARE IN ITS "AS
  76  * IS" CONDITION.  THE UNIVERSITY OF UTAH AND CSL DISCLAIM ANY LIABILITY OF
  77  * ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
  78  *
  79  * CSL requests users of this software to return to csl-dist@cs.utah.edu any
  80  * improvements that they make and grant CSL redistribution rights.
  81  *
  82  */
  83
  84 #ifndef _KERN_THREAD_H_
  85 #define _KERN_THREAD_H_
  86
  87 #include <mach/kern_return.h>
  88 #include <mach/mach_types.h>
  89 #include <mach/message.h>
  90 #include <mach/boolean.h>
  91 #include <mach/vm_param.h>
  92 #include <mach/thread_info.h>
  93 #include <mach/thread_status.h>
  94 #include <mach/exception_types.h>
  95
  96 #include <kern/kern_types.h>
  97
  98 #include <sys/cdefs.h>
  99
 100 #ifdef  MACH_KERNEL_PRIVATE
 101
 102 #include <mach_assert.h>
 103 #include <mach_ldebug.h>
 104
 105 #include <ipc/ipc_types.h>
 106
 107 #include <mach/port.h>
 108 #include <kern/cpu_number.h>
 109 #include <kern/queue.h>
 110 #include <kern/timer.h>
 111 #include <kern/simple_lock.h>
 112 #include <kern/locks.h>
 113 #include <kern/sched.h>
 114 #include <kern/sched_prim.h>
 115 #include <mach/sfi_class.h>
 116 #include <kern/thread_call.h>
 117 #include <kern/timer_call.h>
 118 #include <kern/task.h>
 119 #include <kern/exception.h>
 120 #include <kern/affinity.h>
 121
 122 #include <ipc/ipc_kmsg.h>
 123
 124 #include <machine/cpu_data.h>
 125 #include <machine/thread.h>
 126
 127 struct thread {
 128         /*
 129          *      NOTE:   The runq field in the thread structure has an unusual
 130          *      locking protocol.  If its value is PROCESSOR_NULL, then it is
 131          *      locked by the thread_lock, but if its value is something else
 132          *      then it is locked by the associated run queue lock. It is
 133          *      set to PROCESSOR_NULL without holding the thread lock, but the
 134          *      transition from PROCESSOR_NULL to non-null must be done
 135          *      under the thread lock and the run queue lock.
 136          *
 137          *      When the thread is on a wait queue, these first three fields
 138          *      are treated as an unofficial union with a wait_queue_element.
 139          *      If you change these, you must change that definition as well
 140          *      (kern/wait_queue.h).
 141          */
 142         /* Items examined often, modified infrequently */
 143         queue_chain_t   links;                          /* run/wait queue links */
 144         processor_t             runq;                           /* run queue assignment */
 145         wait_queue_t    wait_queue;                     /* wait queue we are currently on */
 146         event64_t               wait_event;                     /* wait queue event */
 147         /* Data updated during assert_wait/thread_wakeup */
 148         decl_simple_lock_data(,sched_lock)      /* scheduling lock (thread_lock()) */
 149         decl_simple_lock_data(,wake_lock)       /* for thread stop / wait (wake_lock()) */
 150         integer_t               options;                        /* options set by thread itself */
 151 #define TH_OPT_INTMASK          0x0003          /* interrupt / abort level */
 152 #define TH_OPT_VMPRIV           0x0004          /* may allocate reserved memory */
 153 #define TH_OPT_DTRACE           0x0008          /* executing under dtrace_probe */
 154 #define TH_OPT_SYSTEM_CRITICAL  0x0010          /* Thread must always be allowed to run - even under heavy load */
 155 #define TH_OPT_PROC_CPULIMIT    0x0020          /* Thread has a task-wide CPU limit applied to it */
 156 #define TH_OPT_PRVT_CPULIMIT    0x0040          /* Thread has a thread-private CPU limit applied to it */
 157 #define TH_OPT_IDLE_THREAD              0x0080          /* Thread is a per-processor idle thread */
 158
 159         boolean_t                       wake_active;    /* wake event on stop */
 160         int                                     at_safe_point;  /* thread_abort_safely allowed */
 161         ast_t                           reason;                 /* why we blocked */
 162         thread_continue_t       continuation;   /* continue here next dispatch */
 163         void                            *parameter;             /* continuation parameter */
 164         wait_result_t           wait_result;    /* outcome of wait -
 165                                                                                  * may be examined by this thread
 166                                                                                  * WITHOUT locking */
 167
 168         /* Data updated/used in thread_invoke */
 169         vm_offset_t             kernel_stack;           /* current kernel stack */
 170         vm_offset_t                     reserved_stack;         /* reserved kernel stack */
 171
 172         /* Thread state: */
 173         int                                     state;
 174 /*
 175  *      Thread states [bits or'ed]
 176  */
 177 #define TH_WAIT                 0x01                    /* queued for waiting */
 178 #define TH_SUSP                 0x02                    /* stopped or requested to stop */
 179 #define TH_RUN                  0x04                    /* running or on runq */
 180 #define TH_UNINT                0x08                    /* waiting uninteruptibly */
 181 #define TH_TERMINATE    0x10                    /* halted at termination */
 182 #define TH_TERMINATE2   0x20                    /* added to termination queue */
 183
 184 #define TH_IDLE                 0x80                    /* idling processor */
 185
 186         /* Scheduling information */
 187         sched_mode_t                    sched_mode;             /* scheduling mode */
 188         sched_mode_t                    saved_mode;             /* saved mode during forced mode demotion */
 189
 190         sfi_class_id_t                  sfi_class;              /* SFI class (XXX Updated on CSW/QE/AST) */
 191         sfi_class_id_t                  sfi_wait_class; /* Currently in SFI wait for this class, protected by sfi_lock */
 192
 193         uint32_t                        sched_flags;            /* current flag bits */
 194 #define TH_SFLAG_FAIRSHARE_TRIPPED      0x0001          /* fairshare scheduling activated */
 195 #define TH_SFLAG_FAILSAFE               0x0002          /* fail-safe has tripped */
 196 #define TH_SFLAG_THROTTLED              0x0004          /* thread treated as background for scheduler decay purposes */
 197 #define TH_SFLAG_DEMOTED_MASK      (TH_SFLAG_THROTTLE_DEMOTED | TH_SFLAG_FAILSAFE | TH_SFLAG_FAIRSHARE_TRIPPED) /* saved_mode contains previous sched_mode */
 198
 199 #define TH_SFLAG_PROMOTED               0x0008          /* sched pri has been promoted */
 200 #define TH_SFLAG_ABORT                  0x0010          /* abort interruptible waits */
 201 #define TH_SFLAG_ABORTSAFELY            0x0020          /* ... but only those at safe point */
 202 #define TH_SFLAG_ABORTED_MASK           (TH_SFLAG_ABORT | TH_SFLAG_ABORTSAFELY)
 203 #define TH_SFLAG_DEPRESS                0x0040          /* normal depress yield */
 204 #define TH_SFLAG_POLLDEPRESS            0x0080          /* polled depress yield */
 205 #define TH_SFLAG_DEPRESSED_MASK         (TH_SFLAG_DEPRESS | TH_SFLAG_POLLDEPRESS)
 206 #define TH_SFLAG_PRI_UPDATE             0x0100          /* Updating priority */
 207 #define TH_SFLAG_EAGERPREEMPT           0x0200          /* Any preemption of this thread should be treated as if AST_URGENT applied */
 208 #define TH_SFLAG_RW_PROMOTED            0x0400          /* sched pri has been promoted due to blocking with RW lock held */
 209 #define TH_SFLAG_PROMOTED_MASK          (TH_SFLAG_PROMOTED | TH_SFLAG_RW_PROMOTED)
 210 #define TH_SFLAG_THROTTLE_DEMOTED       0x0800          /* throttled thread forced to timeshare mode (may be applied in addition to failsafe) */
 211
 212 #define TH_SFLAG_RW_PROMOTED_BIT        (10)    /* 0x400 */
 213
 214         int16_t                         sched_pri;                      /* scheduled (current) priority */
 215         int16_t                         priority;                       /* base priority */
 216         int16_t                         max_priority;           /* copy of max base priority */
 217         int16_t                         task_priority;          /* copy of task base priority */
 218 #if defined(CONFIG_SCHED_GRRR)
 219 #if 0
 220         uint16_t                        grrr_deficit;           /* fixed point (1/1000th quantum) fractional deficit */
 221 #endif
 222 #endif
 223
 224         int16_t                         promotions;                     /* level of promotion */
 225         int16_t                         pending_promoter_index;
 226         uint32_t                        ref_count;              /* number of references to me */
 227         void                            *pending_promoter[2];
 228
 229         uint32_t                        rwlock_count;   /* Number of lck_rw_t locks held by thread */
 230
 231 #if MACH_ASSERT
 232         uint32_t                        SHARE_COUNT, BG_COUNT; /* This thread's contribution to global sched counters (temporary debugging) */
 233 #endif /* MACH_ASSERT */
 234
 235         integer_t                       importance;                     /* task-relative importance */
 236
 237         /* Priority depression expiration */
 238         integer_t                       depress_timer_active;
 239         timer_call_data_t       depress_timer;
 240                                                                                 /* real-time parameters */
 241         struct {                                                                /* see mach/thread_policy.h */
 242                 uint32_t                        period;
 243                 uint32_t                        computation;
 244                 uint32_t                        constraint;
 245                 boolean_t                       preemptible;
 246                 uint64_t                        deadline;
 247         }                                       realtime;
 248
 249         uint32_t                        was_promoted_on_wakeup;
 250         uint64_t                        last_run_time;          /* time when thread was switched away from */
 251         uint32_t                        quantum_remaining;                      /* duration of current quantum remaining */
 252
 253 #if defined(CONFIG_SCHED_MULTIQ)
 254         sched_group_t                   sched_group;
 255 #endif /* defined(CONFIG_SCHED_MULTIQ) */
 256
 257   /* Data used during setrun/dispatch */
 258         timer_data_t            system_timer;           /* system mode timer */
 259         processor_t                     bound_processor;        /* bound to a processor? */
 260         processor_t                     last_processor;         /* processor last dispatched on */
 261         processor_t                     chosen_processor;       /* Where we want to run this thread */
 262
 263         /* Fail-safe computation since last unblock or qualifying yield */
 264         uint64_t                        computation_metered;
 265         uint64_t                        computation_epoch;
 266         uint64_t                        safe_release;   /* when to release fail-safe */
 267
 268         /* Call out from scheduler */
 269         void                            (*sched_call)(
 270                                                         int                     type,
 271                                                         thread_t        thread);
 272 #if defined(CONFIG_SCHED_PROTO)
 273         uint32_t                        runqueue_generation;    /* last time runqueue was drained */
 274 #endif
 275
 276         /* Statistics and timesharing calculations */
 277 #if defined(CONFIG_SCHED_TIMESHARE_CORE)
 278         natural_t                       sched_stamp;    /* last scheduler tick */
 279         natural_t                       sched_usage;    /* timesharing cpu usage [sched] */
 280         natural_t                       pri_shift;              /* usage -> priority from pset */
 281         natural_t                       cpu_usage;              /* instrumented cpu usage [%cpu] */
 282         natural_t                       cpu_delta;              /* accumulated cpu_usage delta */
 283 #endif /* CONFIG_SCHED_TIMESHARE_CORE */
 284
 285         uint32_t                        c_switch;               /* total context switches */
 286         uint32_t                        p_switch;               /* total processor switches */
 287         uint32_t                        ps_switch;              /* total pset switches */
 288
 289         /* Timing data structures */
 290         int                                     precise_user_kernel_time; /* precise user/kernel enabled for this thread */
 291         timer_data_t            user_timer;                     /* user mode timer */
 292         uint64_t                        user_timer_save;        /* saved user timer value */
 293         uint64_t                        system_timer_save;      /* saved system timer value */
 294         uint64_t                        vtimer_user_save;       /* saved values for vtimers */
 295         uint64_t                        vtimer_prof_save;
 296         uint64_t                        vtimer_rlim_save;
 297
 298         /* Timing for wait state */
 299         uint64_t                wait_sfi_begin_time;    /* start time for thread waiting in SFI */
 300
 301         /* Timed wait expiration */
 302         timer_call_data_t       wait_timer;
 303         integer_t                       wait_timer_active;
 304         boolean_t                       wait_timer_is_set;
 305
 306
 307         /*
 308          * Processor/cache affinity
 309          * - affinity_threads links task threads with the same affinity set
 310          */
 311         affinity_set_t                  affinity_set;
 312         queue_chain_t                   affinity_threads;
 313
 314         /* Various bits of stashed state */
 315         union {
 316                 struct {
 317                         mach_msg_return_t       state;          /* receive state */
 318                         mach_port_seqno_t       seqno;          /* seqno of recvd message */
 319                         ipc_object_t            object;         /* object received on */
 320                         mach_vm_address_t       msg_addr;       /* receive buffer pointer */
 321                         mach_msg_size_t         msize;          /* max size for recvd msg */
 322                         mach_msg_option_t       option;         /* options for receive */
 323                         mach_msg_size_t         slist_size;     /* scatter list size */
 324                         mach_port_name_t        receiver_name;  /* the receive port name */
 325                         struct ipc_kmsg         *kmsg;          /* received message */
 326                         mach_msg_continue_t     continuation;
 327                 } receive;
 328                 struct {
 329                         struct semaphore        *waitsemaphore;         /* semaphore ref */
 330                         struct semaphore        *signalsemaphore;       /* semaphore ref */
 331                         int                                     options;                        /* semaphore options */
 332                         kern_return_t           result;                         /* primary result */
 333                         mach_msg_continue_t continuation;
 334                 } sema;
 335                 struct {
 336                         int                                     option;         /* switch option */
 337                         boolean_t                               reenable_workq_callback;        /* on entry, callbacks were suspended */
 338                 } swtch;
 339                 int                                             misc;           /* catch-all for other state */
 340         } saved;
 341
 342         /* Structure to save information about guard exception */
 343         struct {
 344                 unsigned                                type;           /* EXC_GUARD reason/type */
 345                 mach_exception_data_type_t              code;           /* Exception code */
 346                 mach_exception_data_type_t              subcode;        /* Exception sub-code */
 347         } guard_exc_info;
 348
 349
 350         /* IPC data structures */
 351 #if IMPORTANCE_INHERITANCE
 352         natural_t ith_assertions;                       /* assertions pending drop */
 353 #endif
 354         struct ipc_kmsg_queue ith_messages;             /* messages to reap */
 355         mach_port_t ith_rpc_reply;                      /* reply port for kernel RPCs */
 356
 357         /* Ast/Halt data structures */
 358         vm_offset_t                                     recover;                /* page fault recover(copyin/out) */
 359
 360         queue_chain_t                           threads;                /* global list of all threads */
 361
 362         /* Activation */
 363                 queue_chain_t                   task_threads;
 364
 365                 /*** Machine-dependent state ***/
 366                 struct machine_thread   machine;
 367
 368                 /* Task membership */
 369                 struct task                             *task;
 370                 vm_map_t                                map;
 371
 372                 decl_lck_mtx_data(,mutex)
 373
 374
 375                 /* Pending thread ast(s) */
 376                 ast_t                                   ast;
 377
 378                 /* Miscellaneous bits guarded by mutex */
 379                 uint32_t
 380                         active:1,                               /* Thread is active and has not been terminated */
 381                         started:1,                              /* Thread has been started after creation */
 382                         static_param:1,                 /* Disallow policy parameter changes */
 383                         policy_reset:1,                 /* Disallow policy parameter changes on terminating threads */
 384                         :0;
 385
 386                 /* Return Handers */
 387                 struct ReturnHandler {
 388                         struct ReturnHandler    *next;
 389                         void            (*handler)(
 390                                                         struct ReturnHandler            *rh,
 391                                                         struct thread                           *thread);
 392                 } *handlers, special_handler;
 393
 394                 /* Ports associated with this thread */
 395                 struct ipc_port                 *ith_self;              /* not a right, doesn't hold ref */
 396                 struct ipc_port                 *ith_sself;             /* a send right */
 397                 struct exception_action *exc_actions;
 398
 399 #ifdef  MACH_BSD
 400                 void                                    *uthread;
 401 #endif
 402
 403 #if CONFIG_DTRACE
 404                 uint32_t t_dtrace_flags;        /* DTrace thread states */
 405 #define TH_DTRACE_EXECSUCCESS   0x01
 406                 uint32_t t_dtrace_predcache;/* DTrace per thread predicate value hint */
 407                 int64_t t_dtrace_tracing;       /* Thread time under dtrace_probe() */
 408                 int64_t t_dtrace_vtime;
 409 #endif
 410
 411                 clock_sec_t t_page_creation_time;
 412                 uint32_t    t_page_creation_count;
 413
 414 #define T_CHUD_MARKED           0x01          /* this thread is marked by CHUD */
 415 #define T_IN_CHUD               0x02          /* this thread is already in a CHUD handler */
 416 #define THREAD_PMC_FLAG         0x04          /* Bit in "t_chud" signifying PMC interest */
 417 #define T_AST_CALLSTACK         0x08          /* Thread scheduled to dump a
 418                                                * callstack on its next
 419                                                * AST */
 420 #define T_AST_NAME              0x10          /* Thread scheduled to dump
 421                                                * its name on its next
 422                                                * AST */
 423 #define T_NAME_DONE             0x20          /* Thread has previously
 424                                                * recorded its name */
 425 #define T_KPC_ALLOC             0x40          /* Thread needs a kpc_buf */
 426
 427                 uint32_t t_chud;        /* CHUD flags, used for Shark */
 428                 uint32_t chud_c_switch; /* last dispatch detection */
 429
 430                 integer_t mutex_count;  /* total count of locks held */
 431
 432 #ifdef KPC
 433         /* accumulated performance counters for this thread */
 434         uint64_t *kpc_buf;
 435 #endif
 436
 437 #ifdef KPERF
 438         /* count of how many times a thread has been sampled since it was last scheduled */
 439         uint64_t kperf_pet_cnt;
 440 #endif
 441
 442 #if HYPERVISOR
 443         /* hypervisor virtual CPU object associated with this thread */
 444         void *hv_thread_target;
 445 #endif /* HYPERVISOR */
 446
 447                 uint64_t thread_id;     /*system wide unique thread-id*/
 448
 449         /* Statistics accumulated per-thread and aggregated per-task */
 450         uint32_t                syscalls_unix;
 451         uint32_t                syscalls_mach;
 452         ledger_t                t_ledger;
 453         ledger_t                t_threadledger; /* per thread ledger */
 454         uint64_t                cpu_time_last_qos;
 455 #ifdef CONFIG_BANK
 456         ledger_t                t_bankledger;              /* ledger to charge someone */
 457         uint64_t                t_deduct_bank_ledger_time; /* cpu time to be deducted from bank ledger */
 458 #endif
 459
 460         /* policy is protected by the task lock */
 461         struct task_requested_policy     requested_policy;
 462         struct task_effective_policy     effective_policy;
 463         struct task_pended_policy        pended_policy;
 464
 465         /* usynch override is protected by the task lock, eventually will be thread mutex */
 466         int             usynch_override_contended_resource_count;
 467
 468         int     iotier_override; /* atomic operations to set, cleared on ret to user */
 469         io_stat_info_t                  thread_io_stats; /* per-thread I/O statistics */
 470
 471
 472         integer_t               saved_importance;               /* saved task-relative importance */
 473
 474         uint32_t                        thread_callout_interrupt_wakeups;
 475         uint32_t                        thread_callout_platform_idle_wakeups;
 476         uint32_t                        thread_timer_wakeups_bin_1;
 477         uint32_t                        thread_timer_wakeups_bin_2;
 478         uint16_t                        thread_tag;
 479         uint16_t                        callout_woken_from_icontext:1,
 480                                         callout_woken_from_platform_idle:1,
 481                                         callout_woke_thread:1,
 482                                         thread_bitfield_unused:13;
 483         /* Kernel holds on this thread  */
 484         int16_t                                         suspend_count;
 485         /* User level suspensions */
 486         int16_t                                         user_stop_count;
 487
 488         mach_port_name_t                ith_voucher_name;
 489         ipc_voucher_t                   ith_voucher;
 490 #if CONFIG_IOSCHED
 491         void                            *decmp_upl;
 492 #endif /* CONFIG_IOSCHED */
 493 };
 494
 495 #define ith_state               saved.receive.state
 496 #define ith_object              saved.receive.object
 497 #define ith_msg_addr                    saved.receive.msg_addr
 498 #define ith_msize               saved.receive.msize
 499 #define ith_option              saved.receive.option
 500 #define ith_scatter_list_size   saved.receive.slist_size
 501 #define ith_receiver_name       saved.receive.receiver_name
 502 #define ith_continuation        saved.receive.continuation
 503 #define ith_kmsg                saved.receive.kmsg
 504 #define ith_seqno               saved.receive.seqno
 505
 506 #define sth_waitsemaphore       saved.sema.waitsemaphore
 507 #define sth_signalsemaphore     saved.sema.signalsemaphore
 508 #define sth_options             saved.sema.options
 509 #define sth_result              saved.sema.result
 510 #define sth_continuation        saved.sema.continuation
 511
 512 extern void                     thread_bootstrap(void);
 513
 514 extern void                     thread_init(void);
 515
 516 extern void                     thread_daemon_init(void);
 517
 518 #define thread_reference_internal(thread)       \
 519                         (void)hw_atomic_add(&(thread)->ref_count, 1)
 520
 521 #define thread_deallocate_internal(thread)      \
 522                         hw_atomic_sub(&(thread)->ref_count, 1)
 523
 524 #define thread_reference(thread)                                        \
 525 MACRO_BEGIN                                                                                     \
 526         if ((thread) != THREAD_NULL)                                    \
 527                 thread_reference_internal(thread);              \
 528 MACRO_END
 529
 530 extern void                     thread_deallocate(
 531                                                 thread_t                thread);
 532
 533 extern void                     thread_terminate_self(void);
 534
 535 extern kern_return_t    thread_terminate_internal(
 536                                                         thread_t                thread);
 537
 538 extern void                     thread_start_internal(
 539                                                         thread_t                        thread) __attribute__ ((noinline));
 540
 541 extern void                     thread_terminate_enqueue(
 542                                                 thread_t                thread);
 543
 544 extern void                     thread_stack_enqueue(
 545                                                 thread_t                thread);
 546
 547 extern void                     thread_hold(
 548                                                 thread_t        thread);
 549
 550 extern void                     thread_release(
 551                                                 thread_t        thread);
 552
 553
 554 #define thread_lock_init(th)    simple_lock_init(&(th)->sched_lock, 0)
 555 #define thread_lock(th)                 simple_lock(&(th)->sched_lock)
 556 #define thread_unlock(th)               simple_unlock(&(th)->sched_lock)
 557
 558 #define wake_lock_init(th)              simple_lock_init(&(th)->wake_lock, 0)
 559 #define wake_lock(th)                   simple_lock(&(th)->wake_lock)
 560 #define wake_unlock(th)                 simple_unlock(&(th)->wake_lock)
 561
 562 #define thread_should_halt_fast(thread)         (!(thread)->active)
 563
 564 extern void                             stack_alloc(
 565                                                         thread_t                thread);
 566
 567 extern void                     stack_handoff(
 568                                                         thread_t                from,
 569                                                         thread_t                to);
 570
 571 extern void                             stack_free(
 572                                                         thread_t                thread);
 573
 574 extern void                             stack_free_reserved(
 575                                                         thread_t                thread);
 576
 577 extern boolean_t                stack_alloc_try(
 578                                                         thread_t            thread);
 579
 580 extern void                             stack_collect(void);
 581
 582 extern void                             stack_init(void);
 583
 584
 585 extern kern_return_t    thread_info_internal(
 586                                                         thread_t                                thread,
 587                                                         thread_flavor_t                 flavor,
 588                                                         thread_info_t                   thread_info_out,
 589                                                         mach_msg_type_number_t  *thread_info_count);
 590
 591 extern void                             thread_task_priority(
 592                                                         thread_t                thread,
 593                                                         integer_t               priority,
 594                                                         integer_t               max_priority);
 595
 596 extern kern_return_t                    thread_set_mode_and_absolute_pri(
 597                                                         thread_t       thread,
 598                                                         integer_t      policy,
 599                                                         integer_t      priority);
 600
 601 extern void                             thread_policy_reset(
 602                                                         thread_t                thread);
 603
 604 extern kern_return_t    kernel_thread_create(
 605                                                         thread_continue_t       continuation,
 606                                                         void                            *parameter,
 607                                                         integer_t                       priority,
 608                                                         thread_t                        *new_thread);
 609
 610 extern kern_return_t    kernel_thread_start_priority(
 611                                                         thread_continue_t       continuation,
 612                                                         void                            *parameter,
 613                                                         integer_t                       priority,
 614                                                         thread_t                        *new_thread);
 615
 616 extern void                             machine_stack_attach(
 617                                                         thread_t                thread,
 618                                                         vm_offset_t             stack);
 619
 620 extern vm_offset_t              machine_stack_detach(
 621                                                         thread_t                thread);
 622
 623 extern void                             machine_stack_handoff(
 624                                                         thread_t                old,
 625                                                         thread_t                new);
 626
 627 extern thread_t                 machine_switch_context(
 628                                                         thread_t                        old_thread,
 629                                                         thread_continue_t       continuation,
 630                                                         thread_t                        new_thread);
 631
 632 extern void                             machine_load_context(
 633                                                         thread_t                thread);
 634
 635 extern kern_return_t    machine_thread_state_initialize(
 636                                                         thread_t                                thread);
 637
 638 extern kern_return_t    machine_thread_set_state(
 639                                                         thread_t                                thread,
 640                                                         thread_flavor_t                 flavor,
 641                                                         thread_state_t                  state,
 642                                                         mach_msg_type_number_t  count);
 643
 644 extern kern_return_t    machine_thread_get_state(
 645                                                         thread_t                                thread,
 646                                                         thread_flavor_t                 flavor,
 647                                                         thread_state_t                  state,
 648                                                         mach_msg_type_number_t  *count);
 649
 650 extern kern_return_t    machine_thread_dup(
 651                                                         thread_t                self,
 652                                                         thread_t                target);
 653
 654 extern void                             machine_thread_init(void);
 655
 656 extern kern_return_t    machine_thread_create(
 657                                                         thread_t                thread,
 658                                                         task_t                  task);
 659 extern void             machine_thread_switch_addrmode(
 660                                                        thread_t                 thread);
 661
 662 extern void                 machine_thread_destroy(
 663                                                         thread_t                thread);
 664
 665 extern void                             machine_set_current_thread(
 666                                                         thread_t                        thread);
 667
 668 extern kern_return_t    machine_thread_get_kern_state(
 669                                                         thread_t                                thread,
 670                                                         thread_flavor_t                 flavor,
 671                                                         thread_state_t                  tstate,
 672                                                         mach_msg_type_number_t  *count);
 673
 674 extern kern_return_t    machine_thread_inherit_taskwide(
 675                                                         thread_t                thread,
 676                                                         task_t                  parent_task);
 677
 678 extern kern_return_t    machine_thread_set_tsd_base(
 679                                                         thread_t                                thread,
 680                                                         mach_vm_offset_t                tsd_base);
 681
 682 typedef struct ReturnHandler            ReturnHandler;
 683
 684 #define thread_mtx_lock(thread)                 lck_mtx_lock(&(thread)->mutex)
 685 #define thread_mtx_try(thread)                  lck_mtx_try_lock(&(thread)->mutex)
 686 #define thread_mtx_unlock(thread)               lck_mtx_unlock(&(thread)->mutex)
 687
 688 extern void                     act_execute_returnhandlers(void);
 689
 690 extern void                     install_special_handler(
 691                                                 thread_t                thread);
 692
 693 extern void                     special_handler(
 694                                                 ReturnHandler   *rh,
 695                                                 thread_t                thread);
 696
 697 extern void
 698 thread_update_qos_cpu_time(
 699                         thread_t thread,
 700                         boolean_t lock_needed);
 701
 702 void act_machine_sv_free(thread_t, int);
 703
 704 vm_offset_t                     min_valid_stack_address(void);
 705 vm_offset_t                     max_valid_stack_address(void);
 706
 707 static inline uint16_t  thread_set_tag_internal(thread_t        thread, uint16_t tag) {
 708         return __sync_fetch_and_or(&thread->thread_tag, tag);
 709 }
 710
 711 static inline uint16_t  thread_get_tag_internal(thread_t        thread) {
 712         return thread->thread_tag;
 713 }
 714
 715 typedef struct {
 716         int             qos_pri[THREAD_QOS_LAST];
 717         int             qos_iotier[THREAD_QOS_LAST];
 718         uint32_t        qos_through_qos[THREAD_QOS_LAST];
 719         uint32_t        qos_latency_qos[THREAD_QOS_LAST];
 720 } qos_policy_params_t;
 721
 722 #else   /* MACH_KERNEL_PRIVATE */
 723
 724 __BEGIN_DECLS
 725
 726 extern thread_t         current_thread(void);
 727
 728 extern void                     thread_reference(
 729                                                 thread_t        thread);
 730
 731 extern void                     thread_deallocate(
 732                                                 thread_t        thread);
 733
 734 __END_DECLS
 735
 736 #endif  /* MACH_KERNEL_PRIVATE */
 737
 738 #ifdef  KERNEL_PRIVATE
 739
 740 __BEGIN_DECLS
 741
 742 extern uint64_t                 thread_dispatchqaddr(
 743                                                 thread_t thread);
 744
 745 __END_DECLS
 746
 747 #endif  /* KERNEL_PRIVATE */
 748
 749 #ifdef KERNEL
 750 __BEGIN_DECLS
 751
 752 extern uint64_t                 thread_tid(thread_t thread);
 753
 754 __END_DECLS
 755
 756 #endif  /* KERNEL */
 757
 758 __BEGIN_DECLS
 759
 760 #ifdef  XNU_KERNEL_PRIVATE
 761
 762 /*
 763  * Thread tags; for easy identification.
 764  */
 765 #define THREAD_TAG_MAINTHREAD 0x1
 766 #define THREAD_TAG_CALLOUT 0x2
 767 #define THREAD_TAG_IOWORKLOOP 0x4
 768
 769 uint16_t        thread_set_tag(thread_t, uint16_t);
 770 uint16_t        thread_get_tag(thread_t);
 771
 772
 773 extern kern_return_t    thread_state_initialize(
 774                                                         thread_t                                thread);
 775
 776 extern kern_return_t    thread_setstatus(
 777                                                         thread_t                                thread,
 778                                                         int                                             flavor,
 779                                                         thread_state_t                  tstate,
 780                                                         mach_msg_type_number_t  count);
 781
 782 extern kern_return_t    thread_getstatus(
 783                                                         thread_t                                thread,
 784                                                         int                                             flavor,
 785                                                         thread_state_t                  tstate,
 786                                                         mach_msg_type_number_t  *count);
 787
 788 extern kern_return_t    thread_create_workq(
 789                                                         task_t                  task,
 790                                                         thread_continue_t       thread_return,
 791                                                         thread_t                *new_thread);
 792
 793 extern  void    thread_yield_internal(
 794         mach_msg_timeout_t      interval);
 795
 796 /*
 797  * Thread-private CPU limits: apply a private CPU limit to this thread only. Available actions are:
 798  *
 799  * 1) Block. Prevent CPU consumption of the thread from exceeding the limit.
 800  * 2) Exception. Generate a resource consumption exception when the limit is exceeded.
 801  * 3) Disable. Remove any existing CPU limit.
 802  */
 803 #define THREAD_CPULIMIT_BLOCK           0x1
 804 #define THREAD_CPULIMIT_EXCEPTION       0x2
 805 #define THREAD_CPULIMIT_DISABLE         0x3
 806
 807 struct _thread_ledger_indices {
 808         int cpu_time;
 809 };
 810
 811 extern struct _thread_ledger_indices thread_ledgers;
 812
 813 extern int thread_get_cpulimit(int *action, uint8_t *percentage, uint64_t *interval_ns);
 814 extern int thread_set_cpulimit(int action, uint8_t percentage, uint64_t interval_ns);
 815
 816 extern void                     thread_read_times(
 817                                                 thread_t                thread,
 818                                                 time_value_t    *user_time,
 819                                                 time_value_t    *system_time);
 820
 821 extern uint64_t         thread_get_runtime_self(void);
 822
 823 extern void                     thread_setuserstack(
 824                                                 thread_t                thread,
 825                                                 mach_vm_offset_t        user_stack);
 826
 827 extern uint64_t         thread_adjuserstack(
 828                                                 thread_t                thread,
 829                                                 int                             adjust);
 830
 831 extern void                     thread_setentrypoint(
 832                                                 thread_t                thread,
 833                                                 mach_vm_offset_t        entry);
 834
 835 extern kern_return_t    thread_set_tsd_base(
 836                                                         thread_t        thread,
 837                                                         mach_vm_offset_t tsd_base);
 838
 839 extern kern_return_t    thread_setsinglestep(
 840                                                 thread_t                thread,
 841                                                 int                     on);
 842
 843 extern kern_return_t    thread_userstack(
 844                                                 thread_t,
 845                                                 int,
 846                                                 thread_state_t,
 847                                                 unsigned int,
 848                                                 mach_vm_offset_t *,
 849                                                 int *);
 850
 851 extern kern_return_t    thread_entrypoint(
 852                                                 thread_t,
 853                                                 int,
 854                                                 thread_state_t,
 855                                                 unsigned int,
 856                                                 mach_vm_offset_t *);
 857
 858 extern kern_return_t    thread_userstackdefault(
 859                                                 thread_t,
 860                                                 mach_vm_offset_t *);
 861
 862 extern kern_return_t    thread_wire_internal(
 863                                                         host_priv_t             host_priv,
 864                                                         thread_t                thread,
 865                                                         boolean_t               wired,
 866                                                         boolean_t               *prev_state);
 867
 868
 869 extern kern_return_t    thread_dup(thread_t);
 870
 871 typedef void    (*sched_call_t)(
 872                                         int                             type,
 873                                         thread_t                thread);
 874
 875 #define SCHED_CALL_BLOCK                0x1
 876 #define SCHED_CALL_UNBLOCK              0x2
 877
 878 extern void             thread_sched_call(
 879                                         thread_t                thread,
 880                                         sched_call_t    call);
 881
 882 extern void             thread_static_param(
 883                                         thread_t                thread,
 884                                         boolean_t               state);
 885
 886 extern boolean_t        thread_is_static_param(
 887                                         thread_t                thread);
 888
 889 extern kern_return_t    thread_policy_set_internal(
 890                                         thread_t                thread,
 891                                         thread_policy_flavor_t  flavor,
 892                                         thread_policy_t         policy_info,
 893                                         mach_msg_type_number_t  count);
 894
 895 extern boolean_t thread_has_qos_policy(thread_t thread);
 896
 897 extern kern_return_t thread_remove_qos_policy(thread_t thread);
 898
 899 extern task_t   get_threadtask(thread_t);
 900 #define thread_is_64bit(thd)    \
 901         task_has_64BitAddr(get_threadtask(thd))
 902
 903
 904 extern void             *get_bsdthread_info(thread_t);
 905 extern void             set_bsdthread_info(thread_t, void *);
 906 extern void             *uthread_alloc(task_t, thread_t, int);
 907 extern void             uthread_cleanup(task_t, void *, void *);
 908 extern void             uthread_zone_free(void *);
 909 extern void             uthread_cred_free(void *);
 910
 911 extern boolean_t        thread_should_halt(
 912                                                 thread_t                thread);
 913
 914 extern boolean_t        thread_should_abort(
 915                                                 thread_t);
 916
 917 extern int is_64signalregset(void);
 918
 919 void act_set_apc(thread_t);
 920 void act_set_kperf(thread_t);
 921
 922 extern uint32_t dtrace_get_thread_predcache(thread_t);
 923 extern int64_t dtrace_get_thread_vtime(thread_t);
 924 extern int64_t dtrace_get_thread_tracing(thread_t);
 925 extern boolean_t dtrace_get_thread_reentering(thread_t);
 926 extern vm_offset_t dtrace_get_kernel_stack(thread_t);
 927 extern void dtrace_set_thread_predcache(thread_t, uint32_t);
 928 extern void dtrace_set_thread_vtime(thread_t, int64_t);
 929 extern void dtrace_set_thread_tracing(thread_t, int64_t);
 930 extern void dtrace_set_thread_reentering(thread_t, boolean_t);
 931 extern vm_offset_t dtrace_set_thread_recover(thread_t, vm_offset_t);
 932 extern void dtrace_thread_bootstrap(void);
 933 extern void dtrace_thread_didexec(thread_t);
 934
 935 extern int64_t dtrace_calc_thread_recent_vtime(thread_t);
 936
 937
 938 extern kern_return_t    thread_set_wq_state32(
 939                                               thread_t          thread,
 940                                               thread_state_t    tstate);
 941
 942 extern kern_return_t    thread_set_wq_state64(
 943                                               thread_t          thread,
 944                                               thread_state_t    tstate);
 945
 946 extern vm_offset_t      kernel_stack_mask;
 947 extern vm_offset_t      kernel_stack_size;
 948 extern vm_offset_t      kernel_stack_depth_max;
 949
 950 void guard_ast(thread_t thread);
 951 extern void fd_guard_ast(thread_t thread);
 952 extern void mach_port_guard_ast(thread_t thread);
 953 extern void thread_guard_violation(thread_t thread, unsigned type);
 954 extern void thread_update_io_stats(thread_t thread, int size, int io_flags);
 955
 956 extern kern_return_t    thread_set_voucher_name(mach_port_name_t name);
 957 extern kern_return_t thread_get_current_voucher_origin_pid(int32_t *pid);
 958
 959 #endif  /* XNU_KERNEL_PRIVATE */
 960
 961
 962 /*! @function kernel_thread_start
 963     @abstract Create a kernel thread.
 964     @discussion This function takes three input parameters, namely reference to the function that the thread should execute, caller specified data and a reference which is used to return the newly created kernel thread. The function returns KERN_SUCCESS on success or an appropriate kernel code type indicating the error. It may be noted that the caller is responsible for explicitly releasing the reference to the created thread when no longer needed. This should be done by calling thread_deallocate(new_thread).
 965     @param continuation A C-function pointer where the thread will begin execution.
 966     @param parameter Caller specified data to be passed to the new thread.
 967     @param new_thread Reference to the new thread is returned in this parameter.
 968     @result Returns KERN_SUCCESS on success or an appropriate kernel code type.
 969 */
 970
 971 extern kern_return_t    kernel_thread_start(
 972                                                         thread_continue_t       continuation,
 973                                                         void                            *parameter,
 974                                                         thread_t                        *new_thread);
 975 #ifdef KERNEL_PRIVATE
 976 void thread_set_eager_preempt(thread_t thread);
 977 void thread_clear_eager_preempt(thread_t thread);
 978 extern ipc_port_t convert_thread_to_port(thread_t);
 979 extern boolean_t set_vm_privilege(boolean_t);
 980 #endif /* KERNEL_PRIVATE */
 981
 982 __END_DECLS
 983
 984 #endif  /* _KERN_THREAD_H_ */