xnu-3248.30.4.tar.gz

[apple/xnu.git] / osfmk / kern / ast.c

diff --git a/osfmk/kern/ast.c b/osfmk/kern/ast.c
index f0b75b59a1ee4020e9028af80e2b3a47c6e5dcd5..f2ceba343763afd22146f7933b61a4e7563de598 100644 (file)
--- a/osfmk/kern/ast.c
+++ b/osfmk/kern/ast.c
@@ -1,5 +1,5 @@
 /*
 /*

- * Copyright (c) 2000-2008 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2012 Apple Inc. All rights reserved.

  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 


@@ -65,9 +65,6 @@
  *
  */
 
  *
  */
 

-#include <cputypes.h>
-#include <platforms.h>
-

 #include <kern/ast.h>
 #include <kern/counters.h>
 #include <kern/cpu_number.h>
 #include <kern/ast.h>
 #include <kern/counters.h>
 #include <kern/cpu_number.h>


@@ -77,15 +74,27 @@
 #include <kern/thread.h>
 #include <kern/processor.h>
 #include <kern/spl.h>
 #include <kern/thread.h>
 #include <kern/processor.h>
 #include <kern/spl.h>

-#include <kern/wait_queue.h>
+#include <kern/sfi.h>
+#if CONFIG_TELEMETRY
+#include <kern/telemetry.h>
+#endif
+#include <kern/waitq.h>
+#include <kern/ledger.h>

 #include <mach/policy.h>
 #include <machine/trap.h> // for CHUD AST hook
 #include <mach/policy.h>
 #include <machine/trap.h> // for CHUD AST hook

+#include <machine/pal_routines.h>
+#include <security/mac_mach_internal.h> // for MACF AST hook
+
+volatile perfASTCallback perfASTHook;
+

 
 void
 ast_init(void)
 {
 }
 
 
 void
 ast_init(void)
 {
 }
 

+extern void chudxnu_thread_ast(thread_t); // XXX this should probably be in a header...
+

 /*
  * Called at splsched.
  */
 /*
  * Called at splsched.
  */


@@ -98,21 +107,21 @@ ast_taken(
        boolean_t               preempt_trap = (reasons == AST_PREEMPTION);
        ast_t                   *myast = ast_pending();
        thread_t                thread = current_thread();
        boolean_t               preempt_trap = (reasons == AST_PREEMPTION);
        ast_t                   *myast = ast_pending();
        thread_t                thread = current_thread();

+       perfASTCallback perf_hook = perfASTHook;

 
        /*
         * CHUD hook - all threads including idle processor threads
         */
 
        /*
         * CHUD hook - all threads including idle processor threads
         */

-       if(perfASTHook) {
-               if(*myast & AST_CHUD_ALL) {
-                       perfASTHook(0, NULL, 0, 0);
+       if (perf_hook) {
+               if (*myast & AST_CHUD_ALL) {
+                       (*perf_hook)(reasons, myast);

                        
                        

-                       if(*myast == AST_NONE) {
-                               return; // nothing left to do
-                       }
+                       if (*myast == AST_NONE)
+                               return;

                }
                }

-       } else {
-               *myast &= ~AST_CHUD_ALL;

        }
        }

+       else
+               *myast &= ~AST_CHUD_ALL;

 
        reasons &= *myast;
        *myast &= ~reasons;
 
        reasons &= *myast;
        *myast &= ~reasons;


@@ -126,11 +135,11 @@ ast_taken(
                 * Check for urgent preemption.
                 */
                if (    (reasons & AST_URGENT)                          &&
                 * Check for urgent preemption.
                 */
                if (    (reasons & AST_URGENT)                          &&

-                               wait_queue_assert_possible(thread)              ) {
+                               waitq_wait_possible(thread)             ) {

                        if (reasons & AST_PREEMPT) {
                                counter(c_ast_taken_block++);
                                thread_block_reason(THREAD_CONTINUE_NULL, NULL,
                        if (reasons & AST_PREEMPT) {
                                counter(c_ast_taken_block++);
                                thread_block_reason(THREAD_CONTINUE_NULL, NULL,

-                                                                               AST_PREEMPT | AST_URGENT);
+                                                                               reasons & AST_PREEMPTION);

                        }
 
                        reasons &= ~AST_PREEMPTION;
                        }
 
                        reasons &= ~AST_PREEMPTION;


@@ -152,30 +161,75 @@ ast_taken(
                                bsd_ast(thread);
                        }
 #endif
                                bsd_ast(thread);
                        }
 #endif

-
+#if CONFIG_MACF
+                       /*
+                        * Handle MACF hook.
+                        */
+                       if (reasons & AST_MACF) {
+                               thread_ast_clear(thread, AST_MACF);
+                               mac_thread_userret(thread);
+                       }
+#endif

                        /* 
                         * Thread APC hook.
                         */
                        /* 
                         * Thread APC hook.
                         */

-                       if (reasons & AST_APC)
-                               act_execute_returnhandlers();
+                       if (reasons & AST_APC) {
+                               thread_ast_clear(thread, AST_APC);
+                               special_handler(thread);
+                       }
+                       
+                       if (reasons & AST_GUARD) {
+                               thread_ast_clear(thread, AST_GUARD);
+                               guard_ast(thread);
+                       }
+                       
+                       if (reasons & AST_LEDGER) {
+                               thread_ast_clear(thread, AST_LEDGER);
+                               ledger_ast(thread);
+                       }
+
+                       /*
+                        * Kernel Profiling Hook
+                        */
+                       if (reasons & AST_KPERF) {
+                               thread_ast_clear(thread, AST_KPERF);
+                               chudxnu_thread_ast(thread);
+                       }
+
+#if CONFIG_TELEMETRY
+                       if (reasons & AST_TELEMETRY_ALL) {
+                               boolean_t interrupted_userspace = FALSE;
+                               boolean_t is_windowed = FALSE;
+
+                               assert((reasons & AST_TELEMETRY_ALL) != AST_TELEMETRY_ALL); /* only one is valid at a time */
+                               interrupted_userspace = (reasons & AST_TELEMETRY_USER) ? TRUE : FALSE;
+                               is_windowed = ((reasons & AST_TELEMETRY_WINDOWED) ? TRUE : FALSE);
+                               thread_ast_clear(thread, AST_TELEMETRY_ALL);
+                               telemetry_ast(thread, interrupted_userspace, is_windowed);
+                       }
+#endif

 
                        ml_set_interrupts_enabled(FALSE);
 
 
                        ml_set_interrupts_enabled(FALSE);
 

-                       /* 
-                        * Check for preemption.
+#if CONFIG_SCHED_SFI
+                       if (reasons & AST_SFI) {
+                               sfi_ast(thread);
+                       }
+#endif
+
+                       /*
+                        * Check for preemption. Conditions may have changed from when the AST_PREEMPT was originally set.

                         */
                         */

-                       if (reasons & AST_PREEMPT) {
-                               processor_t             myprocessor = current_processor();
+                       thread_lock(thread);
+                       if (reasons & AST_PREEMPT)
+                               reasons = csw_check(current_processor(), reasons & AST_QUANTUM);
+                       thread_unlock(thread);

 
 

-                               if (csw_needed(thread, myprocessor))
-                                       reasons = AST_PREEMPT;
-                               else
-                                       reasons = AST_NONE;
-                       }
-                       if (    (reasons & AST_PREEMPT)                         &&
-                                       wait_queue_assert_possible(thread)              ) {             
+                       assert(waitq_wait_possible(thread));
+
+                       if (reasons & AST_PREEMPT) {

                                counter(c_ast_taken_block++);
                                counter(c_ast_taken_block++);

-                               thread_block_reason((thread_continue_t)thread_exception_return, NULL, AST_PREEMPT);
+                               thread_block_reason((thread_continue_t)thread_exception_return, NULL, reasons & AST_PREEMPTION);

                        }
                }
        }
                        }
                }
        }


@@ -188,24 +242,71 @@ ast_taken(
  */
 void
 ast_check(
  */
 void
 ast_check(

-       processor_t             processor)
+       processor_t processor)

 {
 {

-       thread_t                        thread = processor->active_thread;
+       thread_t thread = processor->active_thread;

 
 

-       processor->current_pri = thread->sched_pri;
-       if (    processor->state == PROCESSOR_RUNNING           ||
-                       processor->state == PROCESSOR_SHUTDOWN          ) {
-               ast_t                   preempt;
+       if (processor->state == PROCESSOR_RUNNING ||
+           processor->state == PROCESSOR_SHUTDOWN) {
+               ast_t preempt;

 
                /*
                 *      Propagate thread ast to processor.
                 */
 
                /*
                 *      Propagate thread ast to processor.
                 */

+               pal_ast_check(thread);
+

                ast_propagate(thread->ast);
 
                /*
                 *      Context switch check.
                 */
                ast_propagate(thread->ast);
 
                /*
                 *      Context switch check.
                 */

-               if ((preempt = csw_check(thread, processor)) != AST_NONE)
+               thread_lock(thread);
+
+               processor->current_pri = thread->sched_pri;
+               processor->current_thmode = thread->sched_mode;
+               processor->current_sfi_class = thread->sfi_class = sfi_thread_classify(thread);
+
+               if ((preempt = csw_check(processor, AST_NONE)) != AST_NONE)

                        ast_on(preempt);
                        ast_on(preempt);

+
+               thread_unlock(thread);

        }
 }
        }
 }

+
+/*
+ * Set AST flags on current processor
+ * Called at splsched
+ */
+void
+ast_on(ast_t reasons)
+{
+       ast_t *pending_ast = ast_pending();
+
+       *pending_ast |= reasons;
+}
+
+/*
+ * Clear AST flags on current processor
+ * Called at splsched
+ */
+void
+ast_off(ast_t reasons)
+{
+       ast_t *pending_ast = ast_pending();
+
+       *pending_ast &= ~reasons;
+}
+
+/*
+ * Re-set current processor's per-thread AST flags to those set on thread
+ * Called at splsched
+ */
+void
+ast_context(thread_t thread)
+{
+       ast_t *pending_ast = ast_pending();
+
+       *pending_ast = ((*pending_ast & ~AST_PER_THREAD) | thread->ast);
+}
+
+