X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/8ad349bb6ed4a0be06e34c92be0d98b92e078db4..c7d2c2c6ee645e10cbccdd01c6191873ec77239d:/osfmk/kern/ast.c?ds=inline diff --git a/osfmk/kern/ast.c b/osfmk/kern/ast.c index 1f7d743f1..f2ceba343 100644 --- a/osfmk/kern/ast.c +++ b/osfmk/kern/ast.c @@ -1,31 +1,29 @@ /* - * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved. + * Copyright (c) 2000-2012 Apple Inc. All rights reserved. * - * @APPLE_LICENSE_OSREFERENCE_HEADER_START@ + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. The rights granted to you under the - * License may not be used to create, or enable the creation or - * redistribution of, unlawful or unlicensed copies of an Apple operating - * system, or to circumvent, violate, or enable the circumvention or - * violation of, any terms of an Apple operating system software license - * agreement. - * - * Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this - * file. - * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and * limitations under the License. - * - * @APPLE_LICENSE_OSREFERENCE_HEADER_END@ + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ /* * @OSF_COPYRIGHT@ @@ -67,9 +65,6 @@ * */ -#include -#include - #include #include #include @@ -79,18 +74,27 @@ #include #include #include -#include +#include +#if CONFIG_TELEMETRY +#include +#endif +#include +#include #include +#include // for CHUD AST hook +#include +#include // for MACF AST hook + +volatile perfASTCallback perfASTHook; -#ifdef __ppc__ -#include // for CHUD AST hook -#endif void ast_init(void) { } +extern void chudxnu_thread_ast(thread_t); // XXX this should probably be in a header... + /* * Called at splsched. */ @@ -103,23 +107,21 @@ ast_taken( boolean_t preempt_trap = (reasons == AST_PREEMPTION); ast_t *myast = ast_pending(); thread_t thread = current_thread(); + perfASTCallback perf_hook = perfASTHook; -#ifdef __ppc__ /* * CHUD hook - all threads including idle processor threads */ - if(perfASTHook) { - if(*myast & AST_PPC_CHUD_ALL) { - perfASTHook(0, NULL, 0, 0); + if (perf_hook) { + if (*myast & AST_CHUD_ALL) { + (*perf_hook)(reasons, myast); - if(*myast == AST_NONE) { - return; // nothing left to do - } + if (*myast == AST_NONE) + return; } - } else { - *myast &= ~AST_PPC_CHUD_ALL; } -#endif + else + *myast &= ~AST_CHUD_ALL; reasons &= *myast; *myast &= ~reasons; @@ -133,11 +135,11 @@ ast_taken( * Check for urgent preemption. */ if ( (reasons & AST_URGENT) && - wait_queue_assert_possible(thread) ) { + waitq_wait_possible(thread) ) { if (reasons & AST_PREEMPT) { counter(c_ast_taken_block++); thread_block_reason(THREAD_CONTINUE_NULL, NULL, - AST_PREEMPT | AST_URGENT); + reasons & AST_PREEMPTION); } reasons &= ~AST_PREEMPTION; @@ -159,30 +161,75 @@ ast_taken( bsd_ast(thread); } #endif - +#if CONFIG_MACF + /* + * Handle MACF hook. + */ + if (reasons & AST_MACF) { + thread_ast_clear(thread, AST_MACF); + mac_thread_userret(thread); + } +#endif /* * Thread APC hook. */ - if (reasons & AST_APC) - act_execute_returnhandlers(); + if (reasons & AST_APC) { + thread_ast_clear(thread, AST_APC); + special_handler(thread); + } + + if (reasons & AST_GUARD) { + thread_ast_clear(thread, AST_GUARD); + guard_ast(thread); + } + + if (reasons & AST_LEDGER) { + thread_ast_clear(thread, AST_LEDGER); + ledger_ast(thread); + } + + /* + * Kernel Profiling Hook + */ + if (reasons & AST_KPERF) { + thread_ast_clear(thread, AST_KPERF); + chudxnu_thread_ast(thread); + } + +#if CONFIG_TELEMETRY + if (reasons & AST_TELEMETRY_ALL) { + boolean_t interrupted_userspace = FALSE; + boolean_t is_windowed = FALSE; + + assert((reasons & AST_TELEMETRY_ALL) != AST_TELEMETRY_ALL); /* only one is valid at a time */ + interrupted_userspace = (reasons & AST_TELEMETRY_USER) ? TRUE : FALSE; + is_windowed = ((reasons & AST_TELEMETRY_WINDOWED) ? TRUE : FALSE); + thread_ast_clear(thread, AST_TELEMETRY_ALL); + telemetry_ast(thread, interrupted_userspace, is_windowed); + } +#endif ml_set_interrupts_enabled(FALSE); - /* - * Check for preemption. +#if CONFIG_SCHED_SFI + if (reasons & AST_SFI) { + sfi_ast(thread); + } +#endif + + /* + * Check for preemption. Conditions may have changed from when the AST_PREEMPT was originally set. */ - if (reasons & AST_PREEMPT) { - processor_t myprocessor = current_processor(); + thread_lock(thread); + if (reasons & AST_PREEMPT) + reasons = csw_check(current_processor(), reasons & AST_QUANTUM); + thread_unlock(thread); - if (csw_needed(thread, myprocessor)) - reasons = AST_PREEMPT; - else - reasons = AST_NONE; - } - if ( (reasons & AST_PREEMPT) && - wait_queue_assert_possible(thread) ) { + assert(waitq_wait_possible(thread)); + + if (reasons & AST_PREEMPT) { counter(c_ast_taken_block++); - thread_block_reason((thread_continue_t)thread_exception_return, NULL, AST_PREEMPT); + thread_block_reason((thread_continue_t)thread_exception_return, NULL, reasons & AST_PREEMPTION); } } } @@ -195,24 +242,71 @@ ast_taken( */ void ast_check( - processor_t processor) + processor_t processor) { - register thread_t thread = processor->active_thread; + thread_t thread = processor->active_thread; - processor->current_pri = thread->sched_pri; - if ( processor->state == PROCESSOR_RUNNING || - processor->state == PROCESSOR_SHUTDOWN ) { - register ast_t preempt; + if (processor->state == PROCESSOR_RUNNING || + processor->state == PROCESSOR_SHUTDOWN) { + ast_t preempt; /* * Propagate thread ast to processor. */ + pal_ast_check(thread); + ast_propagate(thread->ast); /* * Context switch check. */ - if ((preempt = csw_check(thread, processor)) != AST_NONE) + thread_lock(thread); + + processor->current_pri = thread->sched_pri; + processor->current_thmode = thread->sched_mode; + processor->current_sfi_class = thread->sfi_class = sfi_thread_classify(thread); + + if ((preempt = csw_check(processor, AST_NONE)) != AST_NONE) ast_on(preempt); + + thread_unlock(thread); } } + +/* + * Set AST flags on current processor + * Called at splsched + */ +void +ast_on(ast_t reasons) +{ + ast_t *pending_ast = ast_pending(); + + *pending_ast |= reasons; +} + +/* + * Clear AST flags on current processor + * Called at splsched + */ +void +ast_off(ast_t reasons) +{ + ast_t *pending_ast = ast_pending(); + + *pending_ast &= ~reasons; +} + +/* + * Re-set current processor's per-thread AST flags to those set on thread + * Called at splsched + */ +void +ast_context(thread_t thread) +{ + ast_t *pending_ast = ast_pending(); + + *pending_ast = ((*pending_ast & ~AST_PER_THREAD) | thread->ast); +} + +