X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/2d21ac55c334faf3a56e5634905ed6987fc787d4..c7d2c2c6ee645e10cbccdd01c6191873ec77239d:/osfmk/kern/ast.c?ds=sidebyside diff --git a/osfmk/kern/ast.c b/osfmk/kern/ast.c index 8aaff637f..f2ceba343 100644 --- a/osfmk/kern/ast.c +++ b/osfmk/kern/ast.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved. + * Copyright (c) 2000-2012 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -65,9 +65,6 @@ * */ -#include -#include - #include #include #include @@ -77,15 +74,27 @@ #include #include #include -#include +#include +#if CONFIG_TELEMETRY +#include +#endif +#include +#include #include #include // for CHUD AST hook +#include +#include // for MACF AST hook + +volatile perfASTCallback perfASTHook; + void ast_init(void) { } +extern void chudxnu_thread_ast(thread_t); // XXX this should probably be in a header... + /* * Called at splsched. */ @@ -98,21 +107,21 @@ ast_taken( boolean_t preempt_trap = (reasons == AST_PREEMPTION); ast_t *myast = ast_pending(); thread_t thread = current_thread(); + perfASTCallback perf_hook = perfASTHook; /* * CHUD hook - all threads including idle processor threads */ - if(perfASTHook) { - if(*myast & AST_CHUD_ALL) { - perfASTHook(0, NULL, 0, 0); + if (perf_hook) { + if (*myast & AST_CHUD_ALL) { + (*perf_hook)(reasons, myast); - if(*myast == AST_NONE) { - return; // nothing left to do - } + if (*myast == AST_NONE) + return; } - } else { - *myast &= ~AST_CHUD_ALL; } + else + *myast &= ~AST_CHUD_ALL; reasons &= *myast; *myast &= ~reasons; @@ -126,11 +135,11 @@ ast_taken( * Check for urgent preemption. */ if ( (reasons & AST_URGENT) && - wait_queue_assert_possible(thread) ) { + waitq_wait_possible(thread) ) { if (reasons & AST_PREEMPT) { counter(c_ast_taken_block++); thread_block_reason(THREAD_CONTINUE_NULL, NULL, - AST_PREEMPT | AST_URGENT); + reasons & AST_PREEMPTION); } reasons &= ~AST_PREEMPTION; @@ -152,30 +161,75 @@ ast_taken( bsd_ast(thread); } #endif - +#if CONFIG_MACF + /* + * Handle MACF hook. + */ + if (reasons & AST_MACF) { + thread_ast_clear(thread, AST_MACF); + mac_thread_userret(thread); + } +#endif /* * Thread APC hook. */ - if (reasons & AST_APC) - act_execute_returnhandlers(); + if (reasons & AST_APC) { + thread_ast_clear(thread, AST_APC); + special_handler(thread); + } + + if (reasons & AST_GUARD) { + thread_ast_clear(thread, AST_GUARD); + guard_ast(thread); + } + + if (reasons & AST_LEDGER) { + thread_ast_clear(thread, AST_LEDGER); + ledger_ast(thread); + } + + /* + * Kernel Profiling Hook + */ + if (reasons & AST_KPERF) { + thread_ast_clear(thread, AST_KPERF); + chudxnu_thread_ast(thread); + } + +#if CONFIG_TELEMETRY + if (reasons & AST_TELEMETRY_ALL) { + boolean_t interrupted_userspace = FALSE; + boolean_t is_windowed = FALSE; + + assert((reasons & AST_TELEMETRY_ALL) != AST_TELEMETRY_ALL); /* only one is valid at a time */ + interrupted_userspace = (reasons & AST_TELEMETRY_USER) ? TRUE : FALSE; + is_windowed = ((reasons & AST_TELEMETRY_WINDOWED) ? TRUE : FALSE); + thread_ast_clear(thread, AST_TELEMETRY_ALL); + telemetry_ast(thread, interrupted_userspace, is_windowed); + } +#endif ml_set_interrupts_enabled(FALSE); - /* - * Check for preemption. +#if CONFIG_SCHED_SFI + if (reasons & AST_SFI) { + sfi_ast(thread); + } +#endif + + /* + * Check for preemption. Conditions may have changed from when the AST_PREEMPT was originally set. */ - if (reasons & AST_PREEMPT) { - processor_t myprocessor = current_processor(); + thread_lock(thread); + if (reasons & AST_PREEMPT) + reasons = csw_check(current_processor(), reasons & AST_QUANTUM); + thread_unlock(thread); - if (csw_needed(thread, myprocessor)) - reasons = AST_PREEMPT; - else - reasons = AST_NONE; - } - if ( (reasons & AST_PREEMPT) && - wait_queue_assert_possible(thread) ) { + assert(waitq_wait_possible(thread)); + + if (reasons & AST_PREEMPT) { counter(c_ast_taken_block++); - thread_block_reason((thread_continue_t)thread_exception_return, NULL, AST_PREEMPT); + thread_block_reason((thread_continue_t)thread_exception_return, NULL, reasons & AST_PREEMPTION); } } } @@ -188,22 +242,71 @@ ast_taken( */ void ast_check( - processor_t processor) + processor_t processor) { - if ( processor->state == PROCESSOR_RUNNING || - processor->state == PROCESSOR_SHUTDOWN ) { - thread_t thread = processor->active_thread; - ast_t preempt; + thread_t thread = processor->active_thread; + + if (processor->state == PROCESSOR_RUNNING || + processor->state == PROCESSOR_SHUTDOWN) { + ast_t preempt; /* * Propagate thread ast to processor. */ + pal_ast_check(thread); + ast_propagate(thread->ast); /* * Context switch check. */ - if ((preempt = csw_check(thread, processor)) != AST_NONE) + thread_lock(thread); + + processor->current_pri = thread->sched_pri; + processor->current_thmode = thread->sched_mode; + processor->current_sfi_class = thread->sfi_class = sfi_thread_classify(thread); + + if ((preempt = csw_check(processor, AST_NONE)) != AST_NONE) ast_on(preempt); + + thread_unlock(thread); } } + +/* + * Set AST flags on current processor + * Called at splsched + */ +void +ast_on(ast_t reasons) +{ + ast_t *pending_ast = ast_pending(); + + *pending_ast |= reasons; +} + +/* + * Clear AST flags on current processor + * Called at splsched + */ +void +ast_off(ast_t reasons) +{ + ast_t *pending_ast = ast_pending(); + + *pending_ast &= ~reasons; +} + +/* + * Re-set current processor's per-thread AST flags to those set on thread + * Called at splsched + */ +void +ast_context(thread_t thread) +{ + ast_t *pending_ast = ast_pending(); + + *pending_ast = ((*pending_ast & ~AST_PER_THREAD) | thread->ast); +} + +