xnu-4903.270.47.tar.gz

[apple/xnu.git] / osfmk / kern / thread_act.c

diff --git a/osfmk/kern/thread_act.c b/osfmk/kern/thread_act.c
index 6c1b2e37959439466f37e3f0e791eeb8df68fc01..c93dda8e30ddad37935ad1d6390f52e418ac5fe8 100644 (file)
--- a/osfmk/kern/thread_act.c
+++ b/osfmk/kern/thread_act.c
@@ -1,8 +1,8 @@
 /*
 /*

- * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
+ * Copyright (c) 2000-2016 Apple Inc. All rights reserved.

  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@

- * 
+ *

  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in
  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in


@@ -11,10 +11,10 @@
  * unlawful or unlicensed copies of an Apple operating system, or to
  * circumvent, violate, or enable the circumvention or violation of, any
  * terms of an Apple operating system software license agreement.
  * unlawful or unlicensed copies of an Apple operating system, or to
  * circumvent, violate, or enable the circumvention or violation of, any
  * terms of an Apple operating system software license agreement.

- * 
+ *

  * Please obtain a copy of the License at
  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  * Please obtain a copy of the License at
  * http://www.opensource.apple.com/apsl/ and read it before using this file.

- * 
+ *

  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,


@@ -22,7 +22,7 @@
  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  * Please see the License for the specific language governing rights and
  * limitations under the License.
  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  * Please see the License for the specific language governing rights and
  * limitations under the License.

- * 
+ *

  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*


@@ -49,10 +49,9 @@
  *
  *     Thread management routines
  */
  *
  *     Thread management routines
  */

+

 #include <mach/mach_types.h>
 #include <mach/kern_return.h>
 #include <mach/mach_types.h>
 #include <mach/kern_return.h>

-#include <mach/alert.h>
-#include <mach/rpc.h>

 #include <mach/thread_act_server.h>
 
 #include <kern/kern_types.h>
 #include <mach/thread_act_server.h>
 
 #include <kern/kern_types.h>


@@ -76,32 +75,79 @@
 #include <kern/timer.h>
 #include <kern/affinity.h>
 
 #include <kern/timer.h>
 #include <kern/affinity.h>
 

-#include <mach/rpc.h>
+#include <stdatomic.h>

 
 #include <security/mac_mach_internal.h>
 
 
 #include <security/mac_mach_internal.h>
 

-void                   act_abort(thread_t);
-void                   install_special_handler_locked(thread_t);
-void                   special_handler_continue(void);
+static void act_abort(thread_t thread);
+
+static void thread_suspended(void *arg, wait_result_t result);
+static void thread_set_apc_ast(thread_t thread);
+static void thread_set_apc_ast_locked(thread_t thread);

 
 /*
  * Internal routine to mark a thread as started.
  * Always called with the thread mutex locked.
 
 /*
  * Internal routine to mark a thread as started.
  * Always called with the thread mutex locked.

- *
- * Note: function intentionally declared with the noinline attribute to
- * prevent multiple declaration of probe symbols in this file; we would
- * prefer "#pragma noinline", but gcc does not support it.
- * PR-6385749 -- the lwp-start probe should fire from within the context
- * of the newly created thread.  Commented out for now, in case we
- * turn it into a dead code probe.

  */
 void
  */
 void

-thread_start_internal(
-       thread_t                        thread)
+thread_start(
+       thread_t                        thread)

 {
        clear_wait(thread, THREAD_AWAKENED);
        thread->started = TRUE;
 {
        clear_wait(thread, THREAD_AWAKENED);
        thread->started = TRUE;

-       // DTRACE_PROC1(lwp__start, thread_t, thread);
+}
+
+/*
+ * Internal routine to mark a thread as waiting
+ * right after it has been created.  The caller
+ * is responsible to call wakeup()/thread_wakeup()
+ * or thread_terminate() to get it going.
+ *
+ * Always called with the thread mutex locked.
+ *
+ * Task and task_threads mutexes also held
+ * (so nobody can set the thread running before
+ * this point)
+ *
+ * Converts TH_UNINT wait to THREAD_INTERRUPTIBLE
+ * to allow termination from this point forward.
+ */
+void
+thread_start_in_assert_wait(
+       thread_t                        thread,
+       event_t             event,
+       wait_interrupt_t    interruptible)
+{
+       struct waitq *waitq = assert_wait_queue(event);
+       wait_result_t wait_result;
+       spl_t spl;
+
+       spl = splsched();
+       waitq_lock(waitq);
+
+       /* clear out startup condition (safe because thread not started yet) */
+       thread_lock(thread);
+       assert(!thread->started);
+       assert((thread->state & (TH_WAIT | TH_UNINT)) == (TH_WAIT | TH_UNINT));
+       thread->state &= ~(TH_WAIT | TH_UNINT);
+       thread_unlock(thread);
+
+       /* assert wait interruptibly forever */
+       wait_result = waitq_assert_wait64_locked(waitq, CAST_EVENT64_T(event),
+           interruptible,
+           TIMEOUT_URGENCY_SYS_NORMAL,
+           TIMEOUT_WAIT_FOREVER,
+           TIMEOUT_NO_LEEWAY,
+           thread);
+       assert(wait_result == THREAD_WAITING);
+
+       /* mark thread started while we still hold the waitq lock */
+       thread_lock(thread);
+       thread->started = TRUE;
+       thread_unlock(thread);
+
+       waitq_unlock(waitq);
+       splx(spl);

 }
 
 /*
 }
 
 /*


@@ -110,9 +156,9 @@ thread_start_internal(
  */
 kern_return_t
 thread_terminate_internal(
  */
 kern_return_t
 thread_terminate_internal(

-       thread_t                        thread)
+       thread_t                        thread)

 {
 {

-       kern_return_t           result = KERN_SUCCESS;
+       kern_return_t           result = KERN_SUCCESS;

 
        thread_mtx_lock(thread);
 
 
        thread_mtx_lock(thread);
 


@@ -121,24 +167,26 @@ thread_terminate_internal(
 
                act_abort(thread);
 
 
                act_abort(thread);
 

-               if (thread->started)
+               if (thread->started) {

                        clear_wait(thread, THREAD_INTERRUPTED);
                        clear_wait(thread, THREAD_INTERRUPTED);

-               else {
-                       thread_start_internal(thread);
+               } else {
+                       thread_start(thread);

                }
                }

-       }
-       else
+       } else {

                result = KERN_TERMINATED;
                result = KERN_TERMINATED;

+       }

 
 

-       if (thread->affinity_set != NULL)
+       if (thread->affinity_set != NULL) {

                thread_affinity_terminate(thread);
                thread_affinity_terminate(thread);

+       }

 
        thread_mtx_unlock(thread);
 
 
        thread_mtx_unlock(thread);
 

-       if (thread != current_thread() && result == KERN_SUCCESS)
+       if (thread != current_thread() && result == KERN_SUCCESS) {

                thread_wait(thread, FALSE);
                thread_wait(thread, FALSE);

+       }

 
 

-       return (result);
+       return result;

 }
 
 /*
 }
 
 /*


@@ -146,32 +194,34 @@ thread_terminate_internal(
  */
 kern_return_t
 thread_terminate(
  */
 kern_return_t
 thread_terminate(

-       thread_t                thread)
+       thread_t                thread)

 {
 {

-       kern_return_t   result;
-
-       if (thread == THREAD_NULL)
-               return (KERN_INVALID_ARGUMENT);
+       if (thread == THREAD_NULL) {
+               return KERN_INVALID_ARGUMENT;
+       }

 
 

-       if (    thread->task == kernel_task             &&
-                       thread != current_thread()                      )
-               return (KERN_FAILURE);
+       /* Kernel threads can't be terminated without their own cooperation */
+       if (thread->task == kernel_task && thread != current_thread()) {
+               return KERN_FAILURE;
+       }

 
 

-       result = thread_terminate_internal(thread);
+       kern_return_t result = thread_terminate_internal(thread);

 
        /*
 
        /*

-        * If a kernel thread is terminating itself, force an AST here.
-        * Kernel threads don't normally pass through the AST checking
-        * code - and all threads finish their own termination in the
-        * special handler APC.
+        * If a kernel thread is terminating itself, force handle the APC_AST here.
+        * Kernel threads don't pass through the return-to-user AST checking code,
+        * but all threads must finish their own termination in thread_apc_ast.

         */
        if (thread->task == kernel_task) {
         */
        if (thread->task == kernel_task) {

-               ml_set_interrupts_enabled(FALSE);
-               ast_taken(AST_APC, TRUE);
+               assert(thread->active == FALSE);
+               thread_ast_clear(thread, AST_APC);
+               thread_apc_ast(thread);
+

                panic("thread_terminate");
                panic("thread_terminate");

+               /* NOTREACHED */

        }
 
        }
 

-       return (result);
+       return result;

 }
 
 /*
 }
 
 /*


@@ -182,13 +232,11 @@ thread_terminate(
  * Called with thread mutex held.
  */
 void
  * Called with thread mutex held.
  */
 void

-thread_hold(
-       register thread_t       thread)
+thread_hold(thread_t thread)

 {
        if (thread->suspend_count++ == 0) {
 {
        if (thread->suspend_count++ == 0) {

-               install_special_handler(thread);
-               if (thread->started)
-                       thread_wakeup_one(&thread->suspend_count);
+               thread_set_apc_ast(thread);
+               assert(thread->suspend_parked == FALSE);

        }
 }
 
        }
 }
 


@@ -196,192 +244,198 @@ thread_hold(
  * Decrement internal suspension count, setting thread
  * runnable when count falls to zero.
  *
  * Decrement internal suspension count, setting thread
  * runnable when count falls to zero.
  *

+ * Because the wait is abortsafe, we can't be guaranteed that the thread
+ * is currently actually waiting even if suspend_parked is set.
+ *

  * Called with thread mutex held.
  */
 void
  * Called with thread mutex held.
  */
 void

-thread_release(
-       register thread_t       thread)
-{
-       if (    thread->suspend_count > 0               &&
-                       --thread->suspend_count == 0    ) {
-               if (thread->started)
-                       thread_wakeup_one(&thread->suspend_count);
-               else {
-                       thread_start_internal(thread);
+thread_release(thread_t thread)
+{
+       assertf(thread->suspend_count > 0, "thread %p over-resumed", thread);
+
+       /* fail-safe on non-assert builds */
+       if (thread->suspend_count == 0) {
+               return;
+       }
+
+       if (--thread->suspend_count == 0) {
+               if (!thread->started) {
+                       thread_start(thread);
+               } else if (thread->suspend_parked) {
+                       thread->suspend_parked = FALSE;
+                       thread_wakeup_thread(&thread->suspend_count, thread);

                }
        }
 }
 
 kern_return_t
                }
        }
 }
 
 kern_return_t

-thread_suspend(
-       register thread_t       thread)
+thread_suspend(thread_t thread)

 {
 {

-       thread_t                        self = current_thread();
-       kern_return_t           result = KERN_SUCCESS;
+       kern_return_t result = KERN_SUCCESS;

 
 

-       if (thread == THREAD_NULL || thread->task == kernel_task)
-               return (KERN_INVALID_ARGUMENT);
+       if (thread == THREAD_NULL || thread->task == kernel_task) {
+               return KERN_INVALID_ARGUMENT;
+       }

 
        thread_mtx_lock(thread);
 
        if (thread->active) {
 
        thread_mtx_lock(thread);
 
        if (thread->active) {

-               if (    thread->user_stop_count++ == 0          &&
-                               thread->suspend_count++ == 0            ) {
-                       install_special_handler(thread);
-                       if (thread != self)
-                               thread_wakeup_one(&thread->suspend_count);
+               if (thread->user_stop_count++ == 0) {
+                       thread_hold(thread);

                }
                }

-       }
-       else
+       } else {

                result = KERN_TERMINATED;
                result = KERN_TERMINATED;

+       }

 
        thread_mtx_unlock(thread);
 
 
        thread_mtx_unlock(thread);
 

-       if (thread != self && result == KERN_SUCCESS)
+       if (thread != current_thread() && result == KERN_SUCCESS) {

                thread_wait(thread, FALSE);
                thread_wait(thread, FALSE);

+       }

 
 

-       return (result);
+       return result;

 }
 
 kern_return_t
 }
 
 kern_return_t

-thread_resume(
-       register thread_t       thread)
+thread_resume(thread_t thread)

 {
 {

-       kern_return_t           result = KERN_SUCCESS;
+       kern_return_t result = KERN_SUCCESS;

 
 

-       if (thread == THREAD_NULL || thread->task == kernel_task)
-               return (KERN_INVALID_ARGUMENT);
+       if (thread == THREAD_NULL || thread->task == kernel_task) {
+               return KERN_INVALID_ARGUMENT;
+       }

 
        thread_mtx_lock(thread);
 
        if (thread->active) {
                if (thread->user_stop_count > 0) {
 
        thread_mtx_lock(thread);
 
        if (thread->active) {
                if (thread->user_stop_count > 0) {

-                       if (    --thread->user_stop_count == 0          &&
-                                       --thread->suspend_count == 0            ) {
-                               if (thread->started)
-                                       thread_wakeup_one(&thread->suspend_count);
-                               else {
-                                       thread_start_internal(thread);
-                               }
+                       if (--thread->user_stop_count == 0) {
+                               thread_release(thread);

                        }
                        }

-               }
-               else
+               } else {

                        result = KERN_FAILURE;
                        result = KERN_FAILURE;

-       }
-       else
+               }
+       } else {

                result = KERN_TERMINATED;
                result = KERN_TERMINATED;

+       }

 
        thread_mtx_unlock(thread);
 
 
        thread_mtx_unlock(thread);
 

-       return (result);
+       return result;

 }
 
 /*
 }
 
 /*

- *     thread_depress_abort:
+ *     thread_depress_abort_from_user:

  *
  *     Prematurely abort priority depression if there is one.
  */
 kern_return_t
  *
  *     Prematurely abort priority depression if there is one.
  */
 kern_return_t

-thread_depress_abort(
-       register thread_t       thread)
+thread_depress_abort_from_user(thread_t thread)

 {
 {

-       kern_return_t           result;
+       kern_return_t result;

 
 

-    if (thread == THREAD_NULL)
-               return (KERN_INVALID_ARGUMENT);
+       if (thread == THREAD_NULL) {
+               return KERN_INVALID_ARGUMENT;
+       }

 
 

-    thread_mtx_lock(thread);
+       thread_mtx_lock(thread);

 
 

-       if (thread->active)
-               result = thread_depress_abort_internal(thread);
-       else
+       if (thread->active) {
+               result = thread_depress_abort(thread);
+       } else {

                result = KERN_TERMINATED;
                result = KERN_TERMINATED;

+       }

 
 

-    thread_mtx_unlock(thread);
+       thread_mtx_unlock(thread);

 
 

-       return (result);
+       return result;

 }
 
 
 /*
 }
 
 
 /*

- * Indicate that the activation should run its
- * special handler to detect a condition.
+ * Indicate that the thread should run the AST_APC callback
+ * to detect an abort condition.

  *
  * Called with thread mutex held.
  */
  *
  * Called with thread mutex held.
  */

-void
+static void

 act_abort(
 act_abort(

-       thread_t        thread)
+       thread_t        thread)

 {
 {

-       spl_t           s = splsched();
+       spl_t           s = splsched();

 
        thread_lock(thread);
 
        if (!(thread->sched_flags & TH_SFLAG_ABORT)) {
                thread->sched_flags |= TH_SFLAG_ABORT;
 
        thread_lock(thread);
 
        if (!(thread->sched_flags & TH_SFLAG_ABORT)) {
                thread->sched_flags |= TH_SFLAG_ABORT;

-               install_special_handler_locked(thread);
-       }
-       else
+               thread_set_apc_ast_locked(thread);
+               thread_depress_abort_locked(thread);
+       } else {

                thread->sched_flags &= ~TH_SFLAG_ABORTSAFELY;
                thread->sched_flags &= ~TH_SFLAG_ABORTSAFELY;

+       }

 
        thread_unlock(thread);
        splx(s);
 }
 
        thread_unlock(thread);
        splx(s);
 }

-       
+

 kern_return_t
 thread_abort(
 kern_return_t
 thread_abort(

-       register thread_t       thread)
+       thread_t        thread)

 {
 {

-       kern_return_t   result = KERN_SUCCESS;
+       kern_return_t   result = KERN_SUCCESS;

 
 

-       if (thread == THREAD_NULL)
-               return (KERN_INVALID_ARGUMENT);
+       if (thread == THREAD_NULL) {
+               return KERN_INVALID_ARGUMENT;
+       }

 
        thread_mtx_lock(thread);
 
        if (thread->active) {
                act_abort(thread);
                clear_wait(thread, THREAD_INTERRUPTED);
 
        thread_mtx_lock(thread);
 
        if (thread->active) {
                act_abort(thread);
                clear_wait(thread, THREAD_INTERRUPTED);

-       }
-       else
+       } else {

                result = KERN_TERMINATED;
                result = KERN_TERMINATED;

+       }

 
        thread_mtx_unlock(thread);
 
 
        thread_mtx_unlock(thread);
 

-       return (result);
+       return result;

 }
 
 kern_return_t
 thread_abort_safely(
 }
 
 kern_return_t
 thread_abort_safely(

-       thread_t                thread)
+       thread_t                thread)

 {
 {

-       kern_return_t   result = KERN_SUCCESS;
+       kern_return_t   result = KERN_SUCCESS;

 
 

-       if (thread == THREAD_NULL)
-               return (KERN_INVALID_ARGUMENT);
+       if (thread == THREAD_NULL) {
+               return KERN_INVALID_ARGUMENT;
+       }

 
        thread_mtx_lock(thread);
 
        if (thread->active) {
 
        thread_mtx_lock(thread);
 
        if (thread->active) {

-               spl_t           s = splsched();
+               spl_t           s = splsched();

 
                thread_lock(thread);
                if (!thread->at_safe_point ||
 
                thread_lock(thread);
                if (!thread->at_safe_point ||

-                               clear_wait_internal(thread, THREAD_INTERRUPTED) != KERN_SUCCESS) {
+                   clear_wait_internal(thread, THREAD_INTERRUPTED) != KERN_SUCCESS) {

                        if (!(thread->sched_flags & TH_SFLAG_ABORT)) {
                                thread->sched_flags |= TH_SFLAG_ABORTED_MASK;
                        if (!(thread->sched_flags & TH_SFLAG_ABORT)) {
                                thread->sched_flags |= TH_SFLAG_ABORTED_MASK;

-                               install_special_handler_locked(thread);
+                               thread_set_apc_ast_locked(thread);
+                               thread_depress_abort_locked(thread);

                        }
                }
                thread_unlock(thread);
                splx(s);
                        }
                }
                thread_unlock(thread);
                splx(s);

-       }
-       else
+       } else {

                result = KERN_TERMINATED;
                result = KERN_TERMINATED;

-               
+       }
+

        thread_mtx_unlock(thread);
 
        thread_mtx_unlock(thread);
 

-       return (result);
+       return result;

 }
 
 /*** backward compatibility hacks ***/
 }
 
 /*** backward compatibility hacks ***/


@@ -391,40 +445,44 @@ thread_abort_safely(
 
 kern_return_t
 thread_info(
 
 kern_return_t
 thread_info(

-       thread_t                        thread,
-       thread_flavor_t                 flavor,
-       thread_info_t                   thread_info_out,
-       mach_msg_type_number_t  *thread_info_count)
+       thread_t                        thread,
+       thread_flavor_t                 flavor,
+       thread_info_t                   thread_info_out,
+       mach_msg_type_number_t  *thread_info_count)

 {
 {

-       kern_return_t                   result;
+       kern_return_t                   result;

 
 

-       if (thread == THREAD_NULL)
-               return (KERN_INVALID_ARGUMENT);
+       if (thread == THREAD_NULL) {
+               return KERN_INVALID_ARGUMENT;
+       }

 
        thread_mtx_lock(thread);
 
 
        thread_mtx_lock(thread);
 

-       if (thread->active || thread->inspection)
+       if (thread->active || thread->inspection) {

                result = thread_info_internal(
                result = thread_info_internal(

-                                               thread, flavor, thread_info_out, thread_info_count);
-       else
+                       thread, flavor, thread_info_out, thread_info_count);
+       } else {

                result = KERN_TERMINATED;
                result = KERN_TERMINATED;

+       }

 
        thread_mtx_unlock(thread);
 
 
        thread_mtx_unlock(thread);
 

-       return (result);
+       return result;

 }
 
 }
 

-kern_return_t
-thread_get_state(
-       register thread_t               thread,
-       int                                             flavor,
-       thread_state_t                  state,                  /* pointer to OUT array */
-       mach_msg_type_number_t  *state_count)   /*IN/OUT*/
+static inline kern_return_t
+thread_get_state_internal(
+       thread_t                thread,
+       int                                             flavor,
+       thread_state_t                  state,                  /* pointer to OUT array */
+       mach_msg_type_number_t  *state_count,   /*IN/OUT*/
+       boolean_t                               to_user)

 {
 {

-       kern_return_t           result = KERN_SUCCESS;
+       kern_return_t           result = KERN_SUCCESS;

 
 

-       if (thread == THREAD_NULL)
-               return (KERN_INVALID_ARGUMENT);
+       if (thread == THREAD_NULL) {
+               return KERN_INVALID_ARGUMENT;
+       }

 
        thread_mtx_lock(thread);
 
 
        thread_mtx_lock(thread);
 


@@ -437,53 +495,92 @@ thread_get_state(
                        if (thread_stop(thread, FALSE)) {
                                thread_mtx_lock(thread);
                                result = machine_thread_get_state(
                        if (thread_stop(thread, FALSE)) {
                                thread_mtx_lock(thread);
                                result = machine_thread_get_state(

-                                                                               thread, flavor, state, state_count);
+                                       thread, flavor, state, state_count);

                                thread_unstop(thread);
                                thread_unstop(thread);

-                       }
-                       else {
+                       } else {

                                thread_mtx_lock(thread);
                                result = KERN_ABORTED;
                        }
 
                        thread_release(thread);
                                thread_mtx_lock(thread);
                                result = KERN_ABORTED;
                        }
 
                        thread_release(thread);

-               }
-               else
+               } else {

                        result = machine_thread_get_state(
                        result = machine_thread_get_state(

-                                                                       thread, flavor, state, state_count);
-       }
-       else if (thread->inspection)
-       {
+                               thread, flavor, state, state_count);
+               }
+       } else if (thread->inspection) {

                result = machine_thread_get_state(
                result = machine_thread_get_state(

-                                                                       thread, flavor, state, state_count);
-       }
-       else
+                       thread, flavor, state, state_count);
+       } else {

                result = KERN_TERMINATED;
                result = KERN_TERMINATED;

+       }
+
+       if (to_user && result == KERN_SUCCESS) {
+               result = machine_thread_state_convert_to_user(thread, flavor, state,
+                   state_count);
+       }

 
        thread_mtx_unlock(thread);
 
 
        thread_mtx_unlock(thread);
 

-       return (result);
+       return result;
+}
+
+/* No prototype, since thread_act_server.h has the _to_user version if KERNEL_SERVER */
+
+kern_return_t
+thread_get_state(
+       thread_t                thread,
+       int                                             flavor,
+       thread_state_t                  state,
+       mach_msg_type_number_t  *state_count);
+
+kern_return_t
+thread_get_state(
+       thread_t                thread,
+       int                                             flavor,
+       thread_state_t                  state,                  /* pointer to OUT array */
+       mach_msg_type_number_t  *state_count)   /*IN/OUT*/
+{
+       return thread_get_state_internal(thread, flavor, state, state_count, FALSE);
+}
+
+kern_return_t
+thread_get_state_to_user(
+       thread_t                thread,
+       int                                             flavor,
+       thread_state_t                  state,                  /* pointer to OUT array */
+       mach_msg_type_number_t  *state_count)   /*IN/OUT*/
+{
+       return thread_get_state_internal(thread, flavor, state, state_count, TRUE);

 }
 
 /*
  *     Change thread's machine-dependent state.  Called with nothing
  *     locked.  Returns same way.
  */
 }
 
 /*
  *     Change thread's machine-dependent state.  Called with nothing
  *     locked.  Returns same way.
  */

-static kern_return_t
+static inline kern_return_t

 thread_set_state_internal(
 thread_set_state_internal(

-       register thread_t               thread,
-       int                                             flavor,
-       thread_state_t                  state,
-       mach_msg_type_number_t  state_count,
-       boolean_t                               from_user)
+       thread_t                thread,
+       int                                             flavor,
+       thread_state_t                  state,
+       mach_msg_type_number_t  state_count,
+       boolean_t                               from_user)

 {
 {

-       kern_return_t           result = KERN_SUCCESS;
+       kern_return_t           result = KERN_SUCCESS;

 
 

-       if (thread == THREAD_NULL)
-               return (KERN_INVALID_ARGUMENT);
+       if (thread == THREAD_NULL) {
+               return KERN_INVALID_ARGUMENT;
+       }

 
        thread_mtx_lock(thread);
 
        if (thread->active) {
 
        thread_mtx_lock(thread);
 
        if (thread->active) {

+               if (from_user) {
+                       result = machine_thread_state_convert_from_user(thread, flavor,
+                           state, state_count);
+                       if (result != KERN_SUCCESS) {
+                               goto out;
+                       }
+               }

                if (thread != current_thread()) {
                        thread_hold(thread);
 
                if (thread != current_thread()) {
                        thread_hold(thread);
 


@@ -492,59 +589,60 @@ thread_set_state_internal(
                        if (thread_stop(thread, TRUE)) {
                                thread_mtx_lock(thread);
                                result = machine_thread_set_state(
                        if (thread_stop(thread, TRUE)) {
                                thread_mtx_lock(thread);
                                result = machine_thread_set_state(

-                                                                               thread, flavor, state, state_count);
+                                       thread, flavor, state, state_count);

                                thread_unstop(thread);
                                thread_unstop(thread);

-                       }
-                       else {
+                       } else {

                                thread_mtx_lock(thread);
                                result = KERN_ABORTED;
                        }
 
                        thread_release(thread);
                                thread_mtx_lock(thread);
                                result = KERN_ABORTED;
                        }
 
                        thread_release(thread);

-               }
-               else
+               } else {

                        result = machine_thread_set_state(
                        result = machine_thread_set_state(

-                                                                       thread, flavor, state, state_count);
-       }
-       else
+                               thread, flavor, state, state_count);
+               }
+       } else {

                result = KERN_TERMINATED;
                result = KERN_TERMINATED;

+       }

 
 

-       if ((result == KERN_SUCCESS) && from_user)
+       if ((result == KERN_SUCCESS) && from_user) {

                extmod_statistics_incr_thread_set_state(thread);
                extmod_statistics_incr_thread_set_state(thread);

+       }

 
 

+out:

        thread_mtx_unlock(thread);
 
        thread_mtx_unlock(thread);
 

-       return (result);
+       return result;

 }
 
 }
 

-/* No prototype, since thread_act_server.h has the _from_user version if KERNEL_SERVER */ 
+/* No prototype, since thread_act_server.h has the _from_user version if KERNEL_SERVER */

 kern_return_t
 thread_set_state(
 kern_return_t
 thread_set_state(

-       register thread_t               thread,
-       int                                             flavor,
-       thread_state_t                  state,
-       mach_msg_type_number_t  state_count);
+       thread_t                thread,
+       int                                             flavor,
+       thread_state_t                  state,
+       mach_msg_type_number_t  state_count);

 
 kern_return_t
 thread_set_state(
 
 kern_return_t
 thread_set_state(

-       register thread_t               thread,
-       int                                             flavor,
-       thread_state_t                  state,
-       mach_msg_type_number_t  state_count)
+       thread_t                thread,
+       int                                             flavor,
+       thread_state_t                  state,
+       mach_msg_type_number_t  state_count)

 {
        return thread_set_state_internal(thread, flavor, state, state_count, FALSE);
 }
 {
        return thread_set_state_internal(thread, flavor, state, state_count, FALSE);
 }

- 
+

 kern_return_t
 thread_set_state_from_user(
 kern_return_t
 thread_set_state_from_user(

-       register thread_t               thread,
-       int                                             flavor,
-       thread_state_t                  state,
-       mach_msg_type_number_t  state_count)
+       thread_t                thread,
+       int                                             flavor,
+       thread_state_t                  state,
+       mach_msg_type_number_t  state_count)

 {
        return thread_set_state_internal(thread, flavor, state, state_count, TRUE);
 }
 {
        return thread_set_state_internal(thread, flavor, state, state_count, TRUE);
 }

- 
+

 /*
  * Kernel-internal "thread" interfaces used outside this file:
  */
 /*
  * Kernel-internal "thread" interfaces used outside this file:
  */


@@ -554,12 +652,13 @@ thread_set_state_from_user(
  */
 kern_return_t
 thread_state_initialize(
  */
 kern_return_t
 thread_state_initialize(

-       register thread_t               thread)
+       thread_t                thread)

 {
 {

-       kern_return_t           result = KERN_SUCCESS;
+       kern_return_t           result = KERN_SUCCESS;

 
 

-       if (thread == THREAD_NULL)
-               return (KERN_INVALID_ARGUMENT);
+       if (thread == THREAD_NULL) {
+               return KERN_INVALID_ARGUMENT;
+       }

 
        thread_mtx_lock(thread);
 
 
        thread_mtx_lock(thread);
 


@@ -573,35 +672,35 @@ thread_state_initialize(
                                thread_mtx_lock(thread);
                                result = machine_thread_state_initialize( thread );
                                thread_unstop(thread);
                                thread_mtx_lock(thread);
                                result = machine_thread_state_initialize( thread );
                                thread_unstop(thread);

-                       }
-                       else {
+                       } else {

                                thread_mtx_lock(thread);
                                result = KERN_ABORTED;
                        }
 
                        thread_release(thread);
                                thread_mtx_lock(thread);
                                result = KERN_ABORTED;
                        }
 
                        thread_release(thread);

-               }
-               else
+               } else {

                        result = machine_thread_state_initialize( thread );
                        result = machine_thread_state_initialize( thread );

-       }
-       else
+               }
+       } else {

                result = KERN_TERMINATED;
                result = KERN_TERMINATED;

+       }

 
        thread_mtx_unlock(thread);
 
 
        thread_mtx_unlock(thread);
 

-       return (result);
+       return result;

 }
 
 
 kern_return_t
 thread_dup(
 }
 
 
 kern_return_t
 thread_dup(

-       register thread_t       target)
+       thread_t        target)

 {
 {

-       thread_t                        self = current_thread();
-       kern_return_t           result = KERN_SUCCESS;
+       thread_t                        self = current_thread();
+       kern_return_t           result = KERN_SUCCESS;

 
 

-       if (target == THREAD_NULL || target == self)
-               return (KERN_INVALID_ARGUMENT);
+       if (target == THREAD_NULL || target == self) {
+               return KERN_INVALID_ARGUMENT;
+       }

 
        thread_mtx_lock(target);
 
 
        thread_mtx_lock(target);
 


@@ -612,27 +711,77 @@ thread_dup(
 
                if (thread_stop(target, TRUE)) {
                        thread_mtx_lock(target);
 
                if (thread_stop(target, TRUE)) {
                        thread_mtx_lock(target);

-                       result = machine_thread_dup(self, target);
-                       if (self->affinity_set != AFFINITY_SET_NULL)
+                       result = machine_thread_dup(self, target, FALSE);
+
+                       if (self->affinity_set != AFFINITY_SET_NULL) {

                                thread_affinity_dup(self, target);
                                thread_affinity_dup(self, target);

+                       }

                        thread_unstop(target);
                        thread_unstop(target);

-               }
-               else {
+               } else {

                        thread_mtx_lock(target);
                        result = KERN_ABORTED;
                }
 
                thread_release(target);
                        thread_mtx_lock(target);
                        result = KERN_ABORTED;
                }
 
                thread_release(target);

-       }
-       else
+       } else {

                result = KERN_TERMINATED;
                result = KERN_TERMINATED;

+       }

 
        thread_mtx_unlock(target);
 
 
        thread_mtx_unlock(target);
 

-       return (result);
+       return result;

 }
 
 
 }
 
 

+kern_return_t
+thread_dup2(
+       thread_t        source,
+       thread_t        target)
+{
+       kern_return_t           result = KERN_SUCCESS;
+       uint32_t                active = 0;
+
+       if (source == THREAD_NULL || target == THREAD_NULL || target == source) {
+               return KERN_INVALID_ARGUMENT;
+       }
+
+       thread_mtx_lock(source);
+       active = source->active;
+       thread_mtx_unlock(source);
+
+       if (!active) {
+               return KERN_TERMINATED;
+       }
+
+       thread_mtx_lock(target);
+
+       if (target->active || target->inspection) {
+               thread_hold(target);
+
+               thread_mtx_unlock(target);
+
+               if (thread_stop(target, TRUE)) {
+                       thread_mtx_lock(target);
+                       result = machine_thread_dup(source, target, TRUE);
+                       if (source->affinity_set != AFFINITY_SET_NULL) {
+                               thread_affinity_dup(source, target);
+                       }
+                       thread_unstop(target);
+               } else {
+                       thread_mtx_lock(target);
+                       result = KERN_ABORTED;
+               }
+
+               thread_release(target);
+       } else {
+               result = KERN_TERMINATED;
+       }
+
+       thread_mtx_unlock(target);
+
+       return result;
+}
+

 /*
  *     thread_setstatus:
  *
 /*
  *     thread_setstatus:
  *


@@ -641,13 +790,22 @@ thread_dup(
  */
 kern_return_t
 thread_setstatus(
  */
 kern_return_t
 thread_setstatus(

-       register thread_t               thread,
-       int                                             flavor,
-       thread_state_t                  tstate,
-       mach_msg_type_number_t  count)
+       thread_t                thread,
+       int                                             flavor,
+       thread_state_t                  tstate,
+       mach_msg_type_number_t  count)

 {
 {

+       return thread_set_state(thread, flavor, tstate, count);
+}

 
 

-       return (thread_set_state(thread, flavor, tstate, count));
+kern_return_t
+thread_setstatus_from_user(
+       thread_t                thread,
+       int                                             flavor,
+       thread_state_t                  tstate,
+       mach_msg_type_number_t  count)
+{
+       return thread_set_state_from_user(thread, flavor, tstate, count);

 }
 
 /*
 }
 
 /*


@@ -657,12 +815,22 @@ thread_setstatus(
  */
 kern_return_t
 thread_getstatus(
  */
 kern_return_t
 thread_getstatus(

-       register thread_t               thread,
-       int                                             flavor,
-       thread_state_t                  tstate,
-       mach_msg_type_number_t  *count)
+       thread_t                thread,
+       int                                             flavor,
+       thread_state_t                  tstate,
+       mach_msg_type_number_t  *count)
+{
+       return thread_get_state(thread, flavor, tstate, count);
+}
+
+kern_return_t
+thread_getstatus_to_user(
+       thread_t                thread,
+       int                                             flavor,
+       thread_state_t                  tstate,
+       mach_msg_type_number_t  *count)

 {
 {

-       return (thread_get_state(thread, flavor, tstate, count));
+       return thread_get_state_to_user(thread, flavor, tstate, count);

 }
 
 /*
 }
 
 /*


@@ -671,13 +839,14 @@ thread_getstatus(
  */
 kern_return_t
 thread_set_tsd_base(
  */
 kern_return_t
 thread_set_tsd_base(

-       thread_t                        thread,
-       mach_vm_offset_t        tsd_base)
+       thread_t                        thread,
+       mach_vm_offset_t        tsd_base)

 {
 {

-       kern_return_t           result = KERN_SUCCESS;
+       kern_return_t           result = KERN_SUCCESS;

 
 

-       if (thread == THREAD_NULL)
-               return (KERN_INVALID_ARGUMENT);
+       if (thread == THREAD_NULL) {
+               return KERN_INVALID_ARGUMENT;
+       }

 
        thread_mtx_lock(thread);
 
 
        thread_mtx_lock(thread);
 


@@ -691,76 +860,66 @@ thread_set_tsd_base(
                                thread_mtx_lock(thread);
                                result = machine_thread_set_tsd_base(thread, tsd_base);
                                thread_unstop(thread);
                                thread_mtx_lock(thread);
                                result = machine_thread_set_tsd_base(thread, tsd_base);
                                thread_unstop(thread);

-                       }
-                       else {
+                       } else {

                                thread_mtx_lock(thread);
                                result = KERN_ABORTED;
                        }
 
                        thread_release(thread);
                                thread_mtx_lock(thread);
                                result = KERN_ABORTED;
                        }
 
                        thread_release(thread);

-               }
-               else
+               } else {

                        result = machine_thread_set_tsd_base(thread, tsd_base);
                        result = machine_thread_set_tsd_base(thread, tsd_base);

-       }
-       else
+               }
+       } else {

                result = KERN_TERMINATED;
                result = KERN_TERMINATED;

+       }

 
        thread_mtx_unlock(thread);
 
 
        thread_mtx_unlock(thread);
 

-       return (result);
+       return result;

 }
 
 /*
 }
 
 /*

- * install_special_handler:
+ * thread_set_apc_ast:

  *
  *

- *     Install the special returnhandler that handles suspension and
- *     termination, if it hasn't been installed already.
+ * Register the AST_APC callback that handles suspension and
+ * termination, if it hasn't been installed already.

  *
  *

- *     Called with the thread mutex held.
+ * Called with the thread mutex held.

  */
  */

-void
-install_special_handler(
-       thread_t                thread)
+static void
+thread_set_apc_ast(thread_t thread)

 {
 {

-       spl_t           s = splsched();
+       spl_t s = splsched();

 
        thread_lock(thread);
 
        thread_lock(thread);

-       install_special_handler_locked(thread);
+       thread_set_apc_ast_locked(thread);

        thread_unlock(thread);
        thread_unlock(thread);

+

        splx(s);
 }
 
 /*
        splx(s);
 }
 
 /*

- * install_special_handler_locked:
+ * thread_set_apc_ast_locked:

  *
  *

- *     Do the work of installing the special_handler.
+ * Do the work of registering for the AST_APC callback.

  *
  *

- *     Called with the thread mutex and scheduling lock held.
+ * Called with the thread mutex and scheduling lock held.

  */
  */

-void
-install_special_handler_locked(
-       thread_t                                thread)
+static void
+thread_set_apc_ast_locked(thread_t thread)

 {
 {

-       
-       /*
-        * Temporarily undepress, so target has
-        * a chance to do locking required to
-        * block itself in special_handler().
-        */
-       if (thread->sched_flags & TH_SFLAG_DEPRESSED_MASK)
-               thread_recompute_sched_pri(thread, TRUE);
-

        thread_ast_set(thread, AST_APC);
 
        thread_ast_set(thread, AST_APC);
 

-       if (thread == current_thread())
-               ast_propagate(thread->ast);
-       else {
-               processor_t             processor = thread->last_processor;
+       if (thread == current_thread()) {
+               ast_propagate(thread);
+       } else {
+               processor_t processor = thread->last_processor;

 
 

-               if (    processor != PROCESSOR_NULL                                     &&
-                               processor->state == PROCESSOR_RUNNING           &&
-                               processor->active_thread == thread                      )
+               if (processor != PROCESSOR_NULL &&
+                   processor->state == PROCESSOR_RUNNING &&
+                   processor->active_thread == thread) {

                        cause_ast_check(processor);
                        cause_ast_check(processor);

+               }

        }
 }
 
        }
 }
 


@@ -770,35 +929,28 @@ install_special_handler_locked(
  */
 
 /*
  */
 
 /*

- * special_handler_continue
+ * thread_suspended

  *
  *

- * Continuation routine for the special handler blocks.  It checks
+ * Continuation routine for thread suspension.  It checks

  * to see whether there has been any new suspensions.  If so, it
  * to see whether there has been any new suspensions.  If so, it

- * installs the special handler again.  Otherwise, it checks to see
- * if the current depression needs to be re-instated (it may have
- * been temporarily removed in order to get to this point in a hurry).
+ * installs the AST_APC handler again.

  */
  */

-void
-special_handler_continue(void)
+__attribute__((noreturn))
+static void
+thread_suspended(__unused void *parameter, wait_result_t result)

 {
 {

-       thread_t                thread = current_thread();
+       thread_t thread = current_thread();

 
        thread_mtx_lock(thread);
 
 
        thread_mtx_lock(thread);
 

-       if (thread->suspend_count > 0)
-               install_special_handler(thread);
-       else {
-               spl_t                   s = splsched();
-
-               thread_lock(thread);
-               if (thread->sched_flags & TH_SFLAG_DEPRESSED_MASK) {
-                       processor_t             myprocessor = thread->last_processor;
+       if (result == THREAD_INTERRUPTED) {
+               thread->suspend_parked = FALSE;
+       } else {
+               assert(thread->suspend_parked == FALSE);
+       }

 
 

-                       thread->sched_pri = DEPRESSPRI;
-                       myprocessor->current_pri = thread->sched_pri;
-               }
-               thread_unlock(thread);
-               splx(s);
+       if (thread->suspend_count > 0) {
+               thread_set_apc_ast(thread);

        }
 
        thread_mtx_unlock(thread);
        }
 
        thread_mtx_unlock(thread);


@@ -808,134 +960,194 @@ special_handler_continue(void)
 }
 
 /*
 }
 
 /*

- * special_handler     - handles suspension, termination.  Called
- * with nothing locked.  Returns (if it returns) the same way.
+ * thread_apc_ast - handles AST_APC and drives thread suspension and termination.
+ * Called with nothing locked.  Returns (if it returns) the same way.

  */
 void
  */
 void

-special_handler(
-       thread_t                                thread)
+thread_apc_ast(thread_t thread)

 {
 {

-       spl_t           s;
-

        thread_mtx_lock(thread);
 
        thread_mtx_lock(thread);
 

-       s = splsched();
+       assert(thread->suspend_parked == FALSE);
+
+       spl_t s = splsched();

        thread_lock(thread);
        thread_lock(thread);

+
+       /* TH_SFLAG_POLLDEPRESS is OK to have here */
+       assert((thread->sched_flags & TH_SFLAG_DEPRESS) == 0);
+

        thread->sched_flags &= ~TH_SFLAG_ABORTED_MASK;
        thread_unlock(thread);
        splx(s);
 
        thread->sched_flags &= ~TH_SFLAG_ABORTED_MASK;
        thread_unlock(thread);
        splx(s);
 

-       /*
-        * If we're suspended, go to sleep and wait for someone to wake us up.
-        */
-       if (thread->active) {
-               if (thread->suspend_count > 0) {
-                       assert_wait(&thread->suspend_count, THREAD_ABORTSAFE);
-                       thread_mtx_unlock(thread);
-                       thread_block((thread_continue_t)special_handler_continue);
-                       /*NOTREACHED*/
-               }
-       }
-       else {
+       if (!thread->active) {
+               /* Thread is ready to terminate, time to tear it down */

                thread_mtx_unlock(thread);
 
                thread_terminate_self();
                /*NOTREACHED*/
        }
 
                thread_mtx_unlock(thread);
 
                thread_terminate_self();
                /*NOTREACHED*/
        }
 

+       /* If we're suspended, go to sleep and wait for someone to wake us up. */
+       if (thread->suspend_count > 0) {
+               thread->suspend_parked = TRUE;
+               assert_wait(&thread->suspend_count,
+                   THREAD_ABORTSAFE | THREAD_WAIT_NOREPORT_USER);
+               thread_mtx_unlock(thread);
+
+               thread_block(thread_suspended);
+               /*NOTREACHED*/
+       }
+

        thread_mtx_unlock(thread);
 }
 
 /* Prototype, see justification above */
 kern_return_t
 act_set_state(
        thread_mtx_unlock(thread);
 }
 
 /* Prototype, see justification above */
 kern_return_t
 act_set_state(

-       thread_t                                thread,
-       int                                             flavor,
-       thread_state_t                  state,
-       mach_msg_type_number_t  count);
+       thread_t                                thread,
+       int                                             flavor,
+       thread_state_t                  state,
+       mach_msg_type_number_t  count);

 
 kern_return_t
 act_set_state(
 
 kern_return_t
 act_set_state(

-       thread_t                                thread,
-       int                                             flavor,
-       thread_state_t                  state,
-       mach_msg_type_number_t  count)
+       thread_t                                thread,
+       int                                             flavor,
+       thread_state_t                  state,
+       mach_msg_type_number_t  count)

 {
 {

-    if (thread == current_thread())
-           return (KERN_INVALID_ARGUMENT);
+       if (thread == current_thread()) {
+               return KERN_INVALID_ARGUMENT;
+       }

 
 

-    return (thread_set_state(thread, flavor, state, count));
-    
+       return thread_set_state(thread, flavor, state, count);

 }
 
 kern_return_t
 act_set_state_from_user(
 }
 
 kern_return_t
 act_set_state_from_user(

-       thread_t                                thread,
-       int                                             flavor,
-       thread_state_t                  state,
-       mach_msg_type_number_t  count)
+       thread_t                                thread,
+       int                                             flavor,
+       thread_state_t                  state,
+       mach_msg_type_number_t  count)

 {
 {

-    if (thread == current_thread())
-           return (KERN_INVALID_ARGUMENT);
+       if (thread == current_thread()) {
+               return KERN_INVALID_ARGUMENT;
+       }

 
 

-    return (thread_set_state_from_user(thread, flavor, state, count));
-    
+       return thread_set_state_from_user(thread, flavor, state, count);

 }
 
 }
 

+/* Prototype, see justification above */
+kern_return_t
+act_get_state(
+       thread_t                                thread,
+       int                                             flavor,
+       thread_state_t                  state,
+       mach_msg_type_number_t  *count);
+

 kern_return_t
 act_get_state(
 kern_return_t
 act_get_state(

-       thread_t                                thread,
-       int                                             flavor,
-       thread_state_t                  state,
-       mach_msg_type_number_t  *count)
+       thread_t                                thread,
+       int                                             flavor,
+       thread_state_t                  state,
+       mach_msg_type_number_t  *count)

 {
 {

-    if (thread == current_thread())
-           return (KERN_INVALID_ARGUMENT);
+       if (thread == current_thread()) {
+               return KERN_INVALID_ARGUMENT;
+       }

 
 

-    return (thread_get_state(thread, flavor, state, count));
+       return thread_get_state(thread, flavor, state, count);
+}
+
+kern_return_t
+act_get_state_to_user(
+       thread_t                                thread,
+       int                                             flavor,
+       thread_state_t                  state,
+       mach_msg_type_number_t  *count)
+{
+       if (thread == current_thread()) {
+               return KERN_INVALID_ARGUMENT;
+       }
+
+       return thread_get_state_to_user(thread, flavor, state, count);

 }
 
 static void
 act_set_ast(
 }
 
 static void
 act_set_ast(

-           thread_t thread,
-           ast_t ast)
+       thread_t thread,
+       ast_t ast)

 {
        spl_t s = splsched();
 
        if (thread == current_thread()) {
                thread_ast_set(thread, ast);
 {
        spl_t s = splsched();
 
        if (thread == current_thread()) {
                thread_ast_set(thread, ast);

-               ast_propagate(thread->ast);
+               ast_propagate(thread);

        } else {
                processor_t processor;
 
                thread_lock(thread);
                thread_ast_set(thread, ast);
                processor = thread->last_processor;
        } else {
                processor_t processor;
 
                thread_lock(thread);
                thread_ast_set(thread, ast);
                processor = thread->last_processor;

-               if ( processor != PROCESSOR_NULL            &&
-                    processor->state == PROCESSOR_RUNNING  &&
-                    processor->active_thread == thread     )
+               if (processor != PROCESSOR_NULL &&
+                   processor->state == PROCESSOR_RUNNING &&
+                   processor->active_thread == thread) {

                        cause_ast_check(processor);
                        cause_ast_check(processor);

+               }

                thread_unlock(thread);
        }
 
        splx(s);
 }
 
                thread_unlock(thread);
        }
 
        splx(s);
 }
 

+/*
+ * set AST on thread without causing an AST check
+ * and without taking the thread lock
+ *
+ * If thread is not the current thread, then it may take
+ * up until the next context switch or quantum expiration
+ * on that thread for it to notice the AST.
+ */
+static void
+act_set_ast_async(thread_t  thread,
+    ast_t     ast)
+{
+       thread_ast_set(thread, ast);
+
+       if (thread == current_thread()) {
+               spl_t s = splsched();
+               ast_propagate(thread);
+               splx(s);
+       }
+}
+

 void
 act_set_astbsd(
 void
 act_set_astbsd(

-       thread_t        thread)
+       thread_t        thread)

 {
        act_set_ast( thread, AST_BSD );
 }
 
 {
        act_set_ast( thread, AST_BSD );
 }
 

+void
+act_set_astkevent(thread_t thread, uint16_t bits)
+{
+       atomic_fetch_or(&thread->kevent_ast_bits, bits);
+
+       /* kevent AST shouldn't send immediate IPIs */
+       act_set_ast_async(thread, AST_KEVENT);
+}
+

 void
 act_set_kperf(
 void
 act_set_kperf(

-       thread_t        thread)
+       thread_t        thread)

 {
        /* safety check */
 {
        /* safety check */

-       if (thread != current_thread())
-               if( !ml_get_interrupts_enabled() )
+       if (thread != current_thread()) {
+               if (!ml_get_interrupts_enabled()) {

                        panic("unsafe act_set_kperf operation");
                        panic("unsafe act_set_kperf operation");

+               }
+       }

 
        act_set_ast( thread, AST_KPERF );
 }
 
        act_set_ast( thread, AST_KPERF );
 }


@@ -943,16 +1155,34 @@ act_set_kperf(
 #if CONFIG_MACF
 void
 act_set_astmacf(
 #if CONFIG_MACF
 void
 act_set_astmacf(

-       thread_t        thread)
+       thread_t        thread)

 {
        act_set_ast( thread, AST_MACF);
 }
 #endif
 
 void
 {
        act_set_ast( thread, AST_MACF);
 }
 #endif
 
 void

-set_astledger(thread_t thread)
+act_set_astledger(thread_t thread)

 {
        act_set_ast(thread, AST_LEDGER);
 }
 
 {
        act_set_ast(thread, AST_LEDGER);
 }
 

+/*
+ * The ledger AST may need to be set while already holding
+ * the thread lock.  This routine skips sending the IPI,
+ * allowing us to avoid the lock hold.
+ *
+ * However, it means the targeted thread must context switch
+ * to recognize the ledger AST.
+ */
+void
+act_set_astledger_async(thread_t thread)
+{
+       act_set_ast_async(thread, AST_LEDGER);
+}

 
 

+void
+act_set_io_telemetry_ast(thread_t thread)
+{
+       act_set_ast(thread, AST_TELEMETRY_IO);
+}