xnu-3248.30.4.tar.gz

[apple/xnu.git] / osfmk / kern / ast.c

diff --git a/osfmk/kern/ast.c b/osfmk/kern/ast.c
index 848c13caf68ee2344aa2bcd00b4667c43fb351ad..f2ceba343763afd22146f7933b61a4e7563de598 100644 (file)
--- a/osfmk/kern/ast.c
+++ b/osfmk/kern/ast.c
@@ -1,23 +1,29 @@
 /*
 /*

- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2012 Apple Inc. All rights reserved.

  *
  *

- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@

  * 
  * 

- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.

  * 
  * 

- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER

  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,

- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.

  * 
  * 

- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@

  */
 /*
  * @OSF_COPYRIGHT@
  */
 /*
  * @OSF_COPYRIGHT@


@@ -59,10 +65,6 @@
  *
  */
 
  *
  */
 

-#include <cputypes.h>
-#include <cpus.h>
-#include <platforms.h>
-

 #include <kern/ast.h>
 #include <kern/counters.h>
 #include <kern/cpu_number.h>
 #include <kern/ast.h>
 #include <kern/counters.h>
 #include <kern/cpu_number.h>


@@ -70,112 +72,169 @@
 #include <kern/queue.h>
 #include <kern/sched_prim.h>
 #include <kern/thread.h>
 #include <kern/queue.h>
 #include <kern/sched_prim.h>
 #include <kern/thread.h>

-#include <kern/thread_act.h>
-#include <kern/thread_swap.h>

 #include <kern/processor.h>
 #include <kern/spl.h>
 #include <kern/processor.h>
 #include <kern/spl.h>

+#include <kern/sfi.h>
+#if CONFIG_TELEMETRY
+#include <kern/telemetry.h>
+#endif
+#include <kern/waitq.h>
+#include <kern/ledger.h>

 #include <mach/policy.h>
 #include <mach/policy.h>

+#include <machine/trap.h> // for CHUD AST hook
+#include <machine/pal_routines.h>
+#include <security/mac_mach_internal.h> // for MACF AST hook
+
+volatile perfASTCallback perfASTHook;

 
 

-volatile ast_t need_ast[NCPUS];

 
 void
 ast_init(void)
 {
 
 void
 ast_init(void)
 {

-#ifndef        MACHINE_AST
-       register int i;
-
-       for (i=0; i<NCPUS; i++) {
-               need_ast[i] = AST_NONE;
-       }
-#endif /* MACHINE_AST */

 }
 
 }
 

+extern void chudxnu_thread_ast(thread_t); // XXX this should probably be in a header...
+
+/*
+ * Called at splsched.
+ */

 void
 ast_taken(
 void
 ast_taken(

-       ast_t                   reasons,
-       boolean_t               enable
+       ast_t           reasons,
+       boolean_t       enable

 )
 {
 )
 {

-       register int                    mycpu;
-       register processor_t    myprocessor;
-       register thread_t               self = current_thread();
-       boolean_t                               preempt_trap = (reasons == AST_PREEMPT);
-
-       disable_preemption();
-       mycpu = cpu_number();
-       reasons &= need_ast[mycpu];
-       need_ast[mycpu] &= ~reasons;
-       enable_preemption();
+       boolean_t               preempt_trap = (reasons == AST_PREEMPTION);
+       ast_t                   *myast = ast_pending();
+       thread_t                thread = current_thread();
+       perfASTCallback perf_hook = perfASTHook;

 
        /*
 
        /*

-        * No ast for an idle thread
+        * CHUD hook - all threads including idle processor threads

         */
         */

-       if (self->state & TH_IDLE)
-               goto enable_and_return;
+       if (perf_hook) {
+               if (*myast & AST_CHUD_ALL) {
+                       (*perf_hook)(reasons, myast);
+                       
+                       if (*myast == AST_NONE)
+                               return;
+               }
+       }
+       else
+               *myast &= ~AST_CHUD_ALL;
+
+       reasons &= *myast;
+       *myast &= ~reasons;

 
        /*
 
        /*

-        * Check for urgent preemption
+        * Handle ASTs for all threads
+        * except idle processor threads.

         */
         */

-       if ((reasons & AST_URGENT) && wait_queue_assert_possible(self)) {
-               if (reasons & AST_BLOCK) {
-                       counter(c_ast_taken_block++);
-                       thread_block_reason((void (*)(void))0, AST_BLOCK);
+       if (!(thread->state & TH_IDLE)) {
+               /*
+                * Check for urgent preemption.
+                */
+               if (    (reasons & AST_URGENT)                          &&
+                               waitq_wait_possible(thread)             ) {
+                       if (reasons & AST_PREEMPT) {
+                               counter(c_ast_taken_block++);
+                               thread_block_reason(THREAD_CONTINUE_NULL, NULL,
+                                                                               reasons & AST_PREEMPTION);
+                       }
+
+                       reasons &= ~AST_PREEMPTION;

                }
 
                }
 

-               reasons &= ~AST_PREEMPT;
-               if (reasons == 0)
-                       goto enable_and_return;
-       }
+               /*
+                * The kernel preempt traps
+                * skip all other ASTs.
+                */
+               if (!preempt_trap) {
+                       ml_set_interrupts_enabled(enable);

 
 

-       if (preempt_trap)
-               goto enable_and_return;
+#ifdef MACH_BSD
+                       /*
+                        * Handle BSD hook.
+                        */
+                       if (reasons & AST_BSD) {
+                               thread_ast_clear(thread, AST_BSD);
+                               bsd_ast(thread);
+                       }
+#endif
+#if CONFIG_MACF
+                       /*
+                        * Handle MACF hook.
+                        */
+                       if (reasons & AST_MACF) {
+                               thread_ast_clear(thread, AST_MACF);
+                               mac_thread_userret(thread);
+                       }
+#endif
+                       /* 
+                        * Thread APC hook.
+                        */
+                       if (reasons & AST_APC) {
+                               thread_ast_clear(thread, AST_APC);
+                               special_handler(thread);
+                       }
+                       
+                       if (reasons & AST_GUARD) {
+                               thread_ast_clear(thread, AST_GUARD);
+                               guard_ast(thread);
+                       }
+                       
+                       if (reasons & AST_LEDGER) {
+                               thread_ast_clear(thread, AST_LEDGER);
+                               ledger_ast(thread);
+                       }

 
 

-       ml_set_interrupts_enabled(enable);
+                       /*
+                        * Kernel Profiling Hook
+                        */
+                       if (reasons & AST_KPERF) {
+                               thread_ast_clear(thread, AST_KPERF);
+                               chudxnu_thread_ast(thread);
+                       }

 
 

-#ifdef MACH_BSD
-       /*
-        * Check for BSD hook
-        */
-       if (reasons & AST_BSD) {
-               extern void             bsd_ast(thread_act_t    act);
-               thread_act_t    act = self->top_act;
+#if CONFIG_TELEMETRY
+                       if (reasons & AST_TELEMETRY_ALL) {
+                               boolean_t interrupted_userspace = FALSE;
+                               boolean_t is_windowed = FALSE;

 
 

-               thread_ast_clear(act, AST_BSD);
-               bsd_ast(act);
-       }
+                               assert((reasons & AST_TELEMETRY_ALL) != AST_TELEMETRY_ALL); /* only one is valid at a time */
+                               interrupted_userspace = (reasons & AST_TELEMETRY_USER) ? TRUE : FALSE;
+                               is_windowed = ((reasons & AST_TELEMETRY_WINDOWED) ? TRUE : FALSE);
+                               thread_ast_clear(thread, AST_TELEMETRY_ALL);
+                               telemetry_ast(thread, interrupted_userspace, is_windowed);
+                       }

 #endif
 
 #endif
 

-       /* 
-        * migration APC hook 
-        */
-       if (reasons & AST_APC) {
-               act_execute_returnhandlers();
-       }
+                       ml_set_interrupts_enabled(FALSE);

 
 

-       /* 
-        * Check for normal preemption
-        */
-       reasons &= AST_BLOCK;
-    if (reasons == 0) {
-        disable_preemption();
-        myprocessor = current_processor();
-        if (csw_needed(self, myprocessor))
-            reasons = AST_BLOCK;
-        enable_preemption();
-    }
-       if (    (reasons & AST_BLOCK)                           &&
-                       wait_queue_assert_possible(self)                ) {             
-               counter(c_ast_taken_block++);
-               thread_block_reason(thread_exception_return, AST_BLOCK);
-       }
+#if CONFIG_SCHED_SFI
+                       if (reasons & AST_SFI) {
+                               sfi_ast(thread);
+                       }
+#endif

 
 

-       goto just_return;
+                       /*
+                        * Check for preemption. Conditions may have changed from when the AST_PREEMPT was originally set.
+                        */
+                       thread_lock(thread);
+                       if (reasons & AST_PREEMPT)
+                               reasons = csw_check(current_processor(), reasons & AST_QUANTUM);
+                       thread_unlock(thread);

 
 

-enable_and_return:
-    ml_set_interrupts_enabled(enable);
+                       assert(waitq_wait_possible(thread));

 
 

-just_return:
-       return;
+                       if (reasons & AST_PREEMPT) {
+                               counter(c_ast_taken_block++);
+                               thread_block_reason((thread_continue_t)thread_exception_return, NULL, reasons & AST_PREEMPTION);
+                       }
+               }
+       }
+
+       ml_set_interrupts_enabled(enable);

 }
 
 /*
 }
 
 /*


@@ -183,35 +242,71 @@ just_return:
  */
 void
 ast_check(
  */
 void
 ast_check(

-       processor_t             processor)
+       processor_t processor)

 {
 {

-       register thread_t               self = processor->cpu_data->active_thread;
+       thread_t thread = processor->active_thread;

 
 

-       processor->current_pri = self->sched_pri;
-       if (processor->state == PROCESSOR_RUNNING) {
-               register ast_t          preempt;
-processor_running:
+       if (processor->state == PROCESSOR_RUNNING ||
+           processor->state == PROCESSOR_SHUTDOWN) {
+               ast_t preempt;

 
                /*
                 *      Propagate thread ast to processor.
                 */
 
                /*
                 *      Propagate thread ast to processor.
                 */

-               ast_propagate(self->top_act->ast);
+               pal_ast_check(thread);
+
+               ast_propagate(thread->ast);

 
                /*
                 *      Context switch check.
                 */
 
                /*
                 *      Context switch check.
                 */

-               if ((preempt = csw_check(self, processor)) != AST_NONE)
+               thread_lock(thread);
+
+               processor->current_pri = thread->sched_pri;
+               processor->current_thmode = thread->sched_mode;
+               processor->current_sfi_class = thread->sfi_class = sfi_thread_classify(thread);
+
+               if ((preempt = csw_check(processor, AST_NONE)) != AST_NONE)

                        ast_on(preempt);
                        ast_on(preempt);

+
+               thread_unlock(thread);

        }
        }

-       else
-       if (    processor->state == PROCESSOR_DISPATCHING       ||
-                       processor->state == PROCESSOR_IDLE                      ) {
-               return;
-       }
-       else
-       if (processor->state == PROCESSOR_SHUTDOWN)
-               goto processor_running;
-       else
-       if (processor->state == PROCESSOR_ASSIGN)
-               ast_on(AST_BLOCK);

 }
 }

+
+/*
+ * Set AST flags on current processor
+ * Called at splsched
+ */
+void
+ast_on(ast_t reasons)
+{
+       ast_t *pending_ast = ast_pending();
+
+       *pending_ast |= reasons;
+}
+
+/*
+ * Clear AST flags on current processor
+ * Called at splsched
+ */
+void
+ast_off(ast_t reasons)
+{
+       ast_t *pending_ast = ast_pending();
+
+       *pending_ast &= ~reasons;
+}
+
+/*
+ * Re-set current processor's per-thread AST flags to those set on thread
+ * Called at splsched
+ */
+void
+ast_context(thread_t thread)
+{
+       ast_t *pending_ast = ast_pending();
+
+       *pending_ast = ((*pending_ast & ~AST_PER_THREAD) | thread->ast);
+}
+
+