[apple/xnu.git] / osfmk / chud / i386 / chud_osfmk_callback_i386.c

/*
 * Copyright (c) 2003-2007 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 * 
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */

#include <stdint.h>
#include <mach/boolean.h>
#include <mach/mach_types.h>

#include <kern/kern_types.h>
#include <kern/processor.h>
#include <kern/timer_call.h>
#include <kern/thread_call.h>
#include <kern/kalloc.h>
#include <kern/thread.h>

#include <machine/machine_routines.h>
#include <machine/cpu_data.h>
#include <machine/trap.h>

#include <chud/chud_xnu.h>
#include <chud/chud_xnu_private.h>
#include <chud/chud_thread.h>

#include <i386/misc_protos.h>
#include <i386/mp.h>
#include <i386/machine_cpu.h>

#include <sys/kdebug.h>
#define CHUD_TIMER_CALLBACK_CANCEL	0
#define CHUD_TIMER_CALLBACK_ENTER	1
#define CHUD_TIMER_CALLBACK		2
#define CHUD_AST_SEND			3
#define CHUD_AST_CALLBACK		4
#define CHUD_CPUSIG_SEND		5
#define CHUD_CPUSIG_CALLBACK		6

__private_extern__
void chudxnu_cancel_all_callbacks(void)
{
    chudxnu_cpusig_callback_cancel();
    chudxnu_cpu_timer_callback_cancel_all();
    chudxnu_interrupt_callback_cancel();
    chudxnu_perfmon_ast_callback_cancel();
    chudxnu_kdebug_callback_cancel();
    chudxnu_trap_callback_cancel();
	chudxnu_syscall_callback_cancel();
	chudxnu_dtrace_callback_cancel();
}

static chudcpu_data_t chudcpu_boot_cpu;
void *
chudxnu_cpu_alloc(boolean_t boot_processor)
{
	chudcpu_data_t	*chud_proc_info;

	if (boot_processor) {
		chud_proc_info = &chudcpu_boot_cpu;
	} else {
		chud_proc_info = (chudcpu_data_t *)
					kalloc(sizeof(chudcpu_data_t));
		if (chud_proc_info == (chudcpu_data_t *)NULL) {
			return (void *)NULL;
		}
	}
	bzero((char *)chud_proc_info, sizeof(chudcpu_data_t));
	chud_proc_info->t_deadline = 0xFFFFFFFFFFFFFFFFULL;
	mpqueue_init(&chud_proc_info->cpu_request_queue);


	return (void *)chud_proc_info;
}

void
chudxnu_cpu_free(void *cp)
{
	if (cp == NULL || cp == (void *)&chudcpu_boot_cpu) {
		return;
	} else {
		kfree(cp,sizeof(chudcpu_data_t));
	}
}

static void
chudxnu_private_cpu_timer_callback(
	timer_call_param_t param0,
	timer_call_param_t param1)
{
#pragma unused (param0)
#pragma unused (param1)
	chudcpu_data_t			*chud_proc_info;
	boolean_t			oldlevel;
	x86_thread_state_t 		state;
	mach_msg_type_number_t		count;
	chudxnu_cpu_timer_callback_func_t fn;

	oldlevel = ml_set_interrupts_enabled(FALSE);
	chud_proc_info = (chudcpu_data_t *)(current_cpu_datap()->cpu_chud);

	count = x86_THREAD_STATE_COUNT;
	if (chudxnu_thread_get_state(current_thread(),
				     x86_THREAD_STATE,
				     (thread_state_t)&state,
				     &count,
				     FALSE) == KERN_SUCCESS) {
			fn = chud_proc_info->cpu_timer_callback_fn;
       		if (fn) {
			KERNEL_DEBUG_CONSTANT(
				MACHDBG_CODE(DBG_MACH_CHUD,
					CHUD_TIMER_CALLBACK) | DBG_FUNC_NONE,
				(uint32_t)fn, 0,0,0,0);
				//state.eip, state.cs, 0, 0);
       			(fn)(
				x86_THREAD_STATE,
				(thread_state_t)&state,
				count);
       		} 
	} 
	
	ml_set_interrupts_enabled(oldlevel);
}

__private_extern__ kern_return_t
chudxnu_cpu_timer_callback_enter(
	chudxnu_cpu_timer_callback_func_t	func,
	uint32_t				time,
	uint32_t				units)
{
	chudcpu_data_t	*chud_proc_info;
	boolean_t	oldlevel;

	oldlevel = ml_set_interrupts_enabled(FALSE);
	chud_proc_info = (chudcpu_data_t *)(current_cpu_datap()->cpu_chud);

	// cancel any existing callback for this cpu
	timer_call_cancel(&(chud_proc_info->cpu_timer_call));

	chud_proc_info->cpu_timer_callback_fn = func;

	clock_interval_to_deadline(time, units, &(chud_proc_info->t_deadline));
	timer_call_setup(&(chud_proc_info->cpu_timer_call),
			 chudxnu_private_cpu_timer_callback, NULL);
	timer_call_enter(&(chud_proc_info->cpu_timer_call),
			 chud_proc_info->t_deadline);

	KERNEL_DEBUG_CONSTANT(
		MACHDBG_CODE(DBG_MACH_CHUD,
			     CHUD_TIMER_CALLBACK_ENTER) | DBG_FUNC_NONE,
		(uint32_t) func, time, units, 0, 0);

	ml_set_interrupts_enabled(oldlevel);
	return KERN_SUCCESS;
}

__private_extern__ kern_return_t
chudxnu_cpu_timer_callback_cancel(void)
{
	chudcpu_data_t	*chud_proc_info;
	boolean_t	oldlevel;

	oldlevel = ml_set_interrupts_enabled(FALSE);
	chud_proc_info = (chudcpu_data_t *)(current_cpu_datap()->cpu_chud);

	timer_call_cancel(&(chud_proc_info->cpu_timer_call));

	KERNEL_DEBUG_CONSTANT(
		MACHDBG_CODE(DBG_MACH_CHUD,
			     CHUD_TIMER_CALLBACK_CANCEL) | DBG_FUNC_NONE,
		0, 0, 0, 0, 0);

	// set to max value:
	chud_proc_info->t_deadline |= ~(chud_proc_info->t_deadline);
	chud_proc_info->cpu_timer_callback_fn = NULL;

	ml_set_interrupts_enabled(oldlevel);
 	return KERN_SUCCESS;
}

__private_extern__ kern_return_t
chudxnu_cpu_timer_callback_cancel_all(void)
{
	unsigned int	cpu;
	chudcpu_data_t	*chud_proc_info;

	for(cpu=0; cpu < real_ncpus; cpu++) {
		chud_proc_info = (chudcpu_data_t *) cpu_data_ptr[cpu]->cpu_chud;
		if (chud_proc_info == NULL)
			continue;
		timer_call_cancel(&(chud_proc_info->cpu_timer_call));
		chud_proc_info->t_deadline |= ~(chud_proc_info->t_deadline);
		chud_proc_info->cpu_timer_callback_fn = NULL;
	}
	return KERN_SUCCESS;
}

#pragma mark **** trap ****
static chudxnu_trap_callback_func_t trap_callback_fn = NULL;

static kern_return_t
chudxnu_private_trap_callback(
	int trapno,
	void			*regs,
	int			unused1,
	int			unused2)
{
#pragma unused (regs)
#pragma unused (unused1)
#pragma unused (unused2)
	kern_return_t retval = KERN_FAILURE;
	chudxnu_trap_callback_func_t fn = trap_callback_fn;

	if(fn) {
		boolean_t oldlevel;
		x86_thread_state_t state;
		mach_msg_type_number_t count;
		thread_t thread = current_thread();
		
		oldlevel = ml_set_interrupts_enabled(FALSE);
		
		/* prevent reentry into CHUD when dtracing */
		if(thread->t_chud & T_IN_CHUD) {
			/* restore interrupts */
			ml_set_interrupts_enabled(oldlevel);

			return KERN_FAILURE;	// not handled - pass off to dtrace
		}

		/* update the chud state bits */
		thread->t_chud |= T_IN_CHUD;

		count = x86_THREAD_STATE_COUNT;
		
		if(chudxnu_thread_get_state(thread,
				x86_THREAD_STATE,
				(thread_state_t)&state,
				&count,
				FALSE) == KERN_SUCCESS) {
		  
					retval = (fn)(
						trapno,
						x86_THREAD_STATE,
						(thread_state_t)&state,
						count);
		}

		/* no longer in CHUD */
		thread->t_chud &= ~(T_IN_CHUD);

		ml_set_interrupts_enabled(oldlevel);
	}

	return retval;
}

__private_extern__ kern_return_t
chudxnu_trap_callback_enter(chudxnu_trap_callback_func_t func)
{
    trap_callback_fn = func;
    perfTrapHook = chudxnu_private_trap_callback;
    return KERN_SUCCESS;
}

__private_extern__ kern_return_t
chudxnu_trap_callback_cancel(void)
{
    trap_callback_fn = NULL;
        perfTrapHook = NULL;
    return KERN_SUCCESS;
}

#pragma mark **** ast ****
static
chudxnu_perfmon_ast_callback_func_t perfmon_ast_callback_fn = NULL;

static kern_return_t
chudxnu_private_chud_ast_callback(
	int			trapno,
	void			*regs,
	int			unused1,
	int			unused2)
{
#pragma unused (trapno)
#pragma unused (regs)
#pragma unused (unused1)
#pragma unused (unused2)
	boolean_t	oldlevel = ml_set_interrupts_enabled(FALSE);
	ast_t		*myast = ast_pending();
	kern_return_t	retval = KERN_FAILURE;
	chudxnu_perfmon_ast_callback_func_t fn = perfmon_ast_callback_fn;
    
	if (*myast & AST_CHUD_URGENT) {
		*myast &= ~(AST_CHUD_URGENT | AST_CHUD);
		if ((*myast & AST_PREEMPTION) != AST_PREEMPTION)
			*myast &= ~(AST_URGENT);
		retval = KERN_SUCCESS;
	} else if (*myast & AST_CHUD) {
		*myast &= ~(AST_CHUD);
		retval = KERN_SUCCESS;
	}

	if (fn) {
		x86_thread_state_t state;
		mach_msg_type_number_t count;
		count = x86_THREAD_STATE_COUNT;

		if (chudxnu_thread_get_state(
			current_thread(),
			x86_THREAD_STATE,
			(thread_state_t) &state, &count,
			TRUE) == KERN_SUCCESS) {

			KERNEL_DEBUG_CONSTANT(
				MACHDBG_CODE(DBG_MACH_CHUD,
				    CHUD_AST_CALLBACK) | DBG_FUNC_NONE,
				(uint32_t) fn, 0, 0, 0, 0);

			(fn)(
				x86_THREAD_STATE,
				(thread_state_t) &state,
				count);
		}
	}
    
	ml_set_interrupts_enabled(oldlevel);
	return retval;
}

__private_extern__ kern_return_t
chudxnu_perfmon_ast_callback_enter(chudxnu_perfmon_ast_callback_func_t func)
{
	perfmon_ast_callback_fn = func;
	perfASTHook = chudxnu_private_chud_ast_callback;
	return KERN_SUCCESS;
}

__private_extern__ kern_return_t
chudxnu_perfmon_ast_callback_cancel(void)
{
    perfmon_ast_callback_fn = NULL;
    perfASTHook = NULL;
    return KERN_SUCCESS;
}

__private_extern__ kern_return_t
chudxnu_perfmon_ast_send_urgent(boolean_t urgent)
{
    boolean_t oldlevel = ml_set_interrupts_enabled(FALSE);
	ast_t *myast = ast_pending();

    if(urgent) {
        *myast |= (AST_CHUD_URGENT | AST_URGENT);
    } else {
        *myast |= (AST_CHUD);
    }

    KERNEL_DEBUG_CONSTANT(
	MACHDBG_CODE(DBG_MACH_CHUD, CHUD_AST_SEND) | DBG_FUNC_NONE,
	urgent, 0, 0, 0, 0);

    ml_set_interrupts_enabled(oldlevel);
    return KERN_SUCCESS;
}

__private_extern__ kern_return_t
chudxnu_perfmon_ast_send(void)
{
    return chudxnu_perfmon_ast_send_urgent(TRUE);
}

#pragma mark **** interrupt ****
static chudxnu_interrupt_callback_func_t interrupt_callback_fn = NULL;

static void
chudxnu_private_interrupt_callback(void *foo)
{
#pragma unused (foo)
	chudxnu_interrupt_callback_func_t fn = interrupt_callback_fn;

	if(fn) {
		boolean_t			oldlevel;
		x86_thread_state_t		state;
		mach_msg_type_number_t		count;

		oldlevel = ml_set_interrupts_enabled(FALSE);

		count = x86_THREAD_STATE_COUNT;
		if(chudxnu_thread_get_state(current_thread(),
					    x86_THREAD_STATE,
					    (thread_state_t)&state,
					    &count,
					    FALSE) == KERN_SUCCESS) {
			(fn)(
				X86_INTERRUPT_PERFMON,
				x86_THREAD_STATE,
				(thread_state_t)&state,
				count);
		}
		ml_set_interrupts_enabled(oldlevel);
	}
}

__private_extern__ kern_return_t
chudxnu_interrupt_callback_enter(chudxnu_interrupt_callback_func_t func)
{
    interrupt_callback_fn = func;
    lapic_set_pmi_func((i386_intr_func_t)chudxnu_private_interrupt_callback);
    return KERN_SUCCESS;
}

__private_extern__ kern_return_t
chudxnu_interrupt_callback_cancel(void)
{
    interrupt_callback_fn = NULL;
    lapic_set_pmi_func(NULL);
    return KERN_SUCCESS;
}

#pragma mark **** cpu signal ****
static chudxnu_cpusig_callback_func_t cpusig_callback_fn = NULL;

static          kern_return_t
chudxnu_private_cpu_signal_handler(int request)
{
	chudxnu_cpusig_callback_func_t fn = cpusig_callback_fn;
	
	if (fn) {
	x86_thread_state_t  state;
		mach_msg_type_number_t count = x86_THREAD_STATE_COUNT;

		if (chudxnu_thread_get_state(current_thread(),
					     x86_THREAD_STATE,
					     (thread_state_t) &state, &count,
					     FALSE) == KERN_SUCCESS) {
			KERNEL_DEBUG_CONSTANT(
				MACHDBG_CODE(DBG_MACH_CHUD,
					CHUD_CPUSIG_CALLBACK) | DBG_FUNC_NONE,
				(uint32_t)fn, request, 0, 0, 0);
			return (fn)(
					request, x86_THREAD_STATE,
					(thread_state_t) &state, count);
		} else {
			return KERN_FAILURE;
		}
	}
	return KERN_SUCCESS; //ignored
}
/*
 * chudxnu_cpu_signal_handler() is called from the IPI handler
 * when a CHUD signal arrives from another processor.
 */
__private_extern__ void
chudxnu_cpu_signal_handler(void)
{
	chudcpu_signal_request_t	*reqp;
	chudcpu_data_t			*chudinfop;

	chudinfop = (chudcpu_data_t *) current_cpu_datap()->cpu_chud;

	mpdequeue_head(&(chudinfop->cpu_request_queue),
		       (queue_entry_t *) &reqp);
	while (reqp != NULL) {
		chudxnu_private_cpu_signal_handler(reqp->req_code);
		reqp->req_sync = 0;
		mpdequeue_head(&(chudinfop->cpu_request_queue),
			       (queue_entry_t *) &reqp);
	}
}

__private_extern__ kern_return_t
chudxnu_cpusig_callback_enter(chudxnu_cpusig_callback_func_t func)
{
	cpusig_callback_fn = func;
	return KERN_SUCCESS;
}

__private_extern__ kern_return_t
chudxnu_cpusig_callback_cancel(void)
{
	cpusig_callback_fn = NULL;
	return KERN_SUCCESS;
}

__private_extern__ kern_return_t
chudxnu_cpusig_send(int otherCPU, uint32_t request_code)
{
	int				thisCPU;
	kern_return_t			retval = KERN_FAILURE;
	chudcpu_signal_request_t	request;
	uint64_t			deadline;
	chudcpu_data_t			*target_chudp;
	boolean_t old_level;

	disable_preemption();
	// force interrupts on for a cross CPU signal.
	old_level = chudxnu_set_interrupts_enabled(TRUE);
	thisCPU = cpu_number();

	if ((unsigned) otherCPU < real_ncpus &&
	    thisCPU != otherCPU &&
	    cpu_data_ptr[otherCPU]->cpu_running) {

		target_chudp = (chudcpu_data_t *)
					cpu_data_ptr[otherCPU]->cpu_chud;

		/* Fill out request */
		request.req_sync = 0xFFFFFFFF;		/* set sync flag */
		//request.req_type = CPRQchud;		/* set request type */
		request.req_code = request_code;	/* set request */

		KERNEL_DEBUG_CONSTANT(
			MACHDBG_CODE(DBG_MACH_CHUD,
				     CHUD_CPUSIG_SEND) | DBG_FUNC_NONE,
			otherCPU, request_code, 0, 0, 0);

		/*
		 * Insert the new request in the target cpu's request queue
		 * and signal target cpu.
		 */
		mpenqueue_tail(&target_chudp->cpu_request_queue,
			       &request.req_entry);
		i386_signal_cpu(otherCPU, MP_CHUD, ASYNC);

		/* Wait for response or timeout */
		deadline = mach_absolute_time() + LockTimeOut;
		while (request.req_sync != 0) {
			if (mach_absolute_time() > deadline) {
				panic("chudxnu_cpusig_send(%d,%d) timed out\n",
					otherCPU, request_code);
			}
			cpu_pause();
		}
		retval = KERN_SUCCESS;
	} else {
		retval = KERN_INVALID_ARGUMENT;
	}

	chudxnu_set_interrupts_enabled(old_level);
	enable_preemption();
	return retval;
}
Commit	Line	Data
0c530ab8	1	/*
2d21ac55	2	* Copyright (c) 2003-2007 Apple Inc. All rights reserved.
0c530ab8	3	*
2d21ac55 A	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
	5	*
	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
	14	*
	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
0c530ab8 A	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
0c530ab8 A	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
2d21ac55 A	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
	25	*
	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
0c530ab8 A	27	*/
	28
	29	#include <stdint.h>
	30	#include <mach/boolean.h>
	31	#include <mach/mach_types.h>
	32
	33	#include <kern/kern_types.h>
	34	#include <kern/processor.h>
	35	#include <kern/timer_call.h>
	36	#include <kern/thread_call.h>
	37	#include <kern/kalloc.h>
	38	#include <kern/thread.h>
	39
	40	#include <machine/machine_routines.h>
	41	#include <machine/cpu_data.h>
	42	#include <machine/trap.h>
	43
	44	#include <chud/chud_xnu.h>
	45	#include <chud/chud_xnu_private.h>
2d21ac55	46	#include <chud/chud_thread.h>
0c530ab8 A	47
	48	#include <i386/misc_protos.h>
	49	#include <i386/mp.h>
	50	#include <i386/machine_cpu.h>
	51
	52	#include <sys/kdebug.h>
	53	#define CHUD_TIMER_CALLBACK_CANCEL 0
	54	#define CHUD_TIMER_CALLBACK_ENTER 1
	55	#define CHUD_TIMER_CALLBACK 2
	56	#define CHUD_AST_SEND 3
	57	#define CHUD_AST_CALLBACK 4
	58	#define CHUD_CPUSIG_SEND 5
	59	#define CHUD_CPUSIG_CALLBACK 6
	60
	61	__private_extern__
	62	void chudxnu_cancel_all_callbacks(void)
	63	{
	64	chudxnu_cpusig_callback_cancel();
	65	chudxnu_cpu_timer_callback_cancel_all();
	66	chudxnu_interrupt_callback_cancel();
	67	chudxnu_perfmon_ast_callback_cancel();
	68	chudxnu_kdebug_callback_cancel();
0c530ab8	69	chudxnu_trap_callback_cancel();
0c530ab8	70	chudxnu_syscall_callback_cancel();
2d21ac55	71	chudxnu_dtrace_callback_cancel();
0c530ab8 A	72	}
	73
	74	static chudcpu_data_t chudcpu_boot_cpu;
	75	void *
	76	chudxnu_cpu_alloc(boolean_t boot_processor)
	77	{
	78	chudcpu_data_t *chud_proc_info;
	79
0c530ab8 A	80	if (boot_processor) {
	81	chud_proc_info = &chudcpu_boot_cpu;
	82	} else {
	83	chud_proc_info = (chudcpu_data_t *)
	84	kalloc(sizeof(chudcpu_data_t));
	85	if (chud_proc_info == (chudcpu_data_t *)NULL) {
	86	return (void *)NULL;
	87	}
	88	}
	89	bzero((char *)chud_proc_info, sizeof(chudcpu_data_t));
	90	chud_proc_info->t_deadline = 0xFFFFFFFFFFFFFFFFULL;
	91	mpqueue_init(&chud_proc_info->cpu_request_queue);
	92
	93
	94	return (void *)chud_proc_info;
	95	}
	96
	97	void
	98	chudxnu_cpu_free(void *cp)
	99	{
	100	if (cp == NULL \|\| cp == (void *)&chudcpu_boot_cpu) {
	101	return;
	102	} else {
	103	kfree(cp,sizeof(chudcpu_data_t));
	104	}
	105	}
	106
	107	static void
	108	chudxnu_private_cpu_timer_callback(
	109	timer_call_param_t param0,
	110	timer_call_param_t param1)
	111	{
	112	#pragma unused (param0)
	113	#pragma unused (param1)
	114	chudcpu_data_t *chud_proc_info;
	115	boolean_t oldlevel;
	116	x86_thread_state_t state;
	117	mach_msg_type_number_t count;
	118	chudxnu_cpu_timer_callback_func_t fn;
	119
	120	oldlevel = ml_set_interrupts_enabled(FALSE);
	121	chud_proc_info = (chudcpu_data_t *)(current_cpu_datap()->cpu_chud);
	122
	123	count = x86_THREAD_STATE_COUNT;
	124	if (chudxnu_thread_get_state(current_thread(),
	125	x86_THREAD_STATE,
	126	(thread_state_t)&state,
	127	&count,
	128	FALSE) == KERN_SUCCESS) {
	129	fn = chud_proc_info->cpu_timer_callback_fn;
	130	if (fn) {
	131	KERNEL_DEBUG_CONSTANT(
	132	MACHDBG_CODE(DBG_MACH_CHUD,
	133	CHUD_TIMER_CALLBACK) \| DBG_FUNC_NONE,
	134	(uint32_t)fn, 0,0,0,0);
	135	//state.eip, state.cs, 0, 0);
	136	(fn)(
	137	x86_THREAD_STATE,
	138	(thread_state_t)&state,
	139	count);
	140	}
	141	}
	142
	143	ml_set_interrupts_enabled(oldlevel);
144	}
145
146	__private_extern__ kern_return_t
147	chudxnu_cpu_timer_callback_enter(
148	chudxnu_cpu_timer_callback_func_t func,
149	uint32_t time,
150	uint32_t units)
151	{
152	chudcpu_data_t *chud_proc_info;
153	boolean_t oldlevel;
154
155	oldlevel = ml_set_interrupts_enabled(FALSE);
156	chud_proc_info = (chudcpu_data_t *)(current_cpu_datap()->cpu_chud);
157
158	// cancel any existing callback for this cpu
159	timer_call_cancel(&(chud_proc_info->cpu_timer_call));
160
161	chud_proc_info->cpu_timer_callback_fn = func;
162
163	clock_interval_to_deadline(time, units, &(chud_proc_info->t_deadline));
164	timer_call_setup(&(chud_proc_info->cpu_timer_call),
165	chudxnu_private_cpu_timer_callback, NULL);
166	timer_call_enter(&(chud_proc_info->cpu_timer_call),
167	chud_proc_info->t_deadline);
168
169	KERNEL_DEBUG_CONSTANT(
170	MACHDBG_CODE(DBG_MACH_CHUD,
171	CHUD_TIMER_CALLBACK_ENTER) \| DBG_FUNC_NONE,
172	(uint32_t) func, time, units, 0, 0);
173
174	ml_set_interrupts_enabled(oldlevel);
175	return KERN_SUCCESS;
176	}
177
178	__private_extern__ kern_return_t
179	chudxnu_cpu_timer_callback_cancel(void)
180	{
181	chudcpu_data_t *chud_proc_info;
182	boolean_t oldlevel;
183
184	oldlevel = ml_set_interrupts_enabled(FALSE);
185	chud_proc_info = (chudcpu_data_t *)(current_cpu_datap()->cpu_chud);
186
187	timer_call_cancel(&(chud_proc_info->cpu_timer_call));
188
189	KERNEL_DEBUG_CONSTANT(
190	MACHDBG_CODE(DBG_MACH_CHUD,
191	CHUD_TIMER_CALLBACK_CANCEL) \| DBG_FUNC_NONE,
192	0, 0, 0, 0, 0);
193
194	// set to max value:
195	chud_proc_info->t_deadline \|= ~(chud_proc_info->t_deadline);
196	chud_proc_info->cpu_timer_callback_fn = NULL;
197
198	ml_set_interrupts_enabled(oldlevel);
199	return KERN_SUCCESS;
200	}
201
202	__private_extern__ kern_return_t
203	chudxnu_cpu_timer_callback_cancel_all(void)
204	{
205	unsigned int cpu;
206	chudcpu_data_t *chud_proc_info;
207
208	for(cpu=0; cpu < real_ncpus; cpu++) {
209	chud_proc_info = (chudcpu_data_t *) cpu_data_ptr[cpu]->cpu_chud;
210	if (chud_proc_info == NULL)
211	continue;
212	timer_call_cancel(&(chud_proc_info->cpu_timer_call));
213	chud_proc_info->t_deadline \|= ~(chud_proc_info->t_deadline);
214	chud_proc_info->cpu_timer_callback_fn = NULL;
215	}
216	return KERN_SUCCESS;
217	}
218
219	#pragma mark ** trap **
220	static chudxnu_trap_callback_func_t trap_callback_fn = NULL;
221
222	static kern_return_t
223	chudxnu_private_trap_callback(
2d21ac55	224	int trapno,
0c530ab8 A	225	void *regs,
	226	int unused1,
	227	int unused2)
	228	{
	229	#pragma unused (regs)
	230	#pragma unused (unused1)
	231	#pragma unused (unused2)
2d21ac55	232	kern_return_t retval = KERN_FAILURE;
0c530ab8 A	233	chudxnu_trap_callback_func_t fn = trap_callback_fn;
0c530ab8 A	234
2d21ac55 A	235	if(fn) {
	236	boolean_t oldlevel;
	237	x86_thread_state_t state;
	238	mach_msg_type_number_t count;
	239	thread_t thread = current_thread();
	240
0c530ab8	241	oldlevel = ml_set_interrupts_enabled(FALSE);
2d21ac55 A	242
	243	/* prevent reentry into CHUD when dtracing */
	244	if(thread->t_chud & T_IN_CHUD) {
	245	/* restore interrupts */
	246	ml_set_interrupts_enabled(oldlevel);
	247
	248	return KERN_FAILURE; // not handled - pass off to dtrace
	249	}
	250
	251	/* update the chud state bits */
	252	thread->t_chud \|= T_IN_CHUD;
0c530ab8 A	253
0c530ab8 A	254	count = x86_THREAD_STATE_COUNT;
2d21ac55 A	255
2d21ac55 A	256	if(chudxnu_thread_get_state(thread,
0c530ab8 A	257	x86_THREAD_STATE,
	258	(thread_state_t)&state,
	259	&count,
	260	FALSE) == KERN_SUCCESS) {
2d21ac55 A	261
	262	retval = (fn)(
	263	trapno,
	264	x86_THREAD_STATE,
	265	(thread_state_t)&state,
	266	count);
0c530ab8	267	}
2d21ac55 A	268
	269	/* no longer in CHUD */
	270	thread->t_chud &= ~(T_IN_CHUD);
	271
	272	ml_set_interrupts_enabled(oldlevel);
0c530ab8 A	273	}
0c530ab8 A	274
2d21ac55	275	return retval;
0c530ab8 A	276	}
	277
	278	__private_extern__ kern_return_t
	279	chudxnu_trap_callback_enter(chudxnu_trap_callback_func_t func)
	280	{
	281	trap_callback_fn = func;
	282	perfTrapHook = chudxnu_private_trap_callback;
	283	return KERN_SUCCESS;
	284	}
	285
	286	__private_extern__ kern_return_t
	287	chudxnu_trap_callback_cancel(void)
	288	{
	289	trap_callback_fn = NULL;
	290	perfTrapHook = NULL;
	291	return KERN_SUCCESS;
	292	}
	293
	294	#pragma mark ** ast **
	295	static
	296	chudxnu_perfmon_ast_callback_func_t perfmon_ast_callback_fn = NULL;
	297
	298	static kern_return_t
	299	chudxnu_private_chud_ast_callback(
	300	int trapno,
	301	void *regs,
	302	int unused1,
	303	int unused2)
	304	{
	305	#pragma unused (trapno)
	306	#pragma unused (regs)
	307	#pragma unused (unused1)
	308	#pragma unused (unused2)
	309	boolean_t oldlevel = ml_set_interrupts_enabled(FALSE);
	310	ast_t *myast = ast_pending();
	311	kern_return_t retval = KERN_FAILURE;
	312	chudxnu_perfmon_ast_callback_func_t fn = perfmon_ast_callback_fn;
	313
	314	if (*myast & AST_CHUD_URGENT) {
	315	*myast &= ~(AST_CHUD_URGENT \| AST_CHUD);
	316	if ((*myast & AST_PREEMPTION) != AST_PREEMPTION)
	317	*myast &= ~(AST_URGENT);
	318	retval = KERN_SUCCESS;
	319	} else if (*myast & AST_CHUD) {
	320	*myast &= ~(AST_CHUD);
	321	retval = KERN_SUCCESS;
	322	}
	323
	324	if (fn) {
	325	x86_thread_state_t state;
	326	mach_msg_type_number_t count;
	327	count = x86_THREAD_STATE_COUNT;
2d21ac55	328
0c530ab8 A	329	if (chudxnu_thread_get_state(
	330	current_thread(),
	331	x86_THREAD_STATE,
	332	(thread_state_t) &state, &count,
	333	TRUE) == KERN_SUCCESS) {
	334
	335	KERNEL_DEBUG_CONSTANT(
	336	MACHDBG_CODE(DBG_MACH_CHUD,
	337	CHUD_AST_CALLBACK) \| DBG_FUNC_NONE,
	338	(uint32_t) fn, 0, 0, 0, 0);
	339
	340	(fn)(
	341	x86_THREAD_STATE,
	342	(thread_state_t) &state,
	343	count);
	344	}
	345	}
	346
	347	ml_set_interrupts_enabled(oldlevel);
	348	return retval;
	349	}
	350
	351	__private_extern__ kern_return_t
	352	chudxnu_perfmon_ast_callback_enter(chudxnu_perfmon_ast_callback_func_t func)
	353	{
	354	perfmon_ast_callback_fn = func;
	355	perfASTHook = chudxnu_private_chud_ast_callback;
	356	return KERN_SUCCESS;
	357	}
	358
	359	__private_extern__ kern_return_t
	360	chudxnu_perfmon_ast_callback_cancel(void)
	361	{
	362	perfmon_ast_callback_fn = NULL;
	363	perfASTHook = NULL;
	364	return KERN_SUCCESS;
	365	}
	366
	367	__private_extern__ kern_return_t
	368	chudxnu_perfmon_ast_send_urgent(boolean_t urgent)
	369	{
	370	boolean_t oldlevel = ml_set_interrupts_enabled(FALSE);
	371	ast_t *myast = ast_pending();
	372
	373	if(urgent) {
	374	*myast \|= (AST_CHUD_URGENT \| AST_URGENT);
	375	} else {
	376	*myast \|= (AST_CHUD);
	377	}
	378
	379	KERNEL_DEBUG_CONSTANT(
	380	MACHDBG_CODE(DBG_MACH_CHUD, CHUD_AST_SEND) \| DBG_FUNC_NONE,
	381	urgent, 0, 0, 0, 0);
	382
	383	ml_set_interrupts_enabled(oldlevel);
	384	return KERN_SUCCESS;
	385	}
	386
	387	__private_extern__ kern_return_t
	388	chudxnu_perfmon_ast_send(void)
	389	{
	390	return chudxnu_perfmon_ast_send_urgent(TRUE);
	391	}
	392
393	#pragma mark ** interrupt **
394	static chudxnu_interrupt_callback_func_t interrupt_callback_fn = NULL;
395
396	static void
397	chudxnu_private_interrupt_callback(void *foo)
398	{
399	#pragma unused (foo)
400	chudxnu_interrupt_callback_func_t fn = interrupt_callback_fn;
401
402	if(fn) {
403	boolean_t oldlevel;
404	x86_thread_state_t state;
405	mach_msg_type_number_t count;
406
407	oldlevel = ml_set_interrupts_enabled(FALSE);
408
409	count = x86_THREAD_STATE_COUNT;
410	if(chudxnu_thread_get_state(current_thread(),
411	x86_THREAD_STATE,
412	(thread_state_t)&state,
413	&count,
414	FALSE) == KERN_SUCCESS) {
415	(fn)(
416	X86_INTERRUPT_PERFMON,
417	x86_THREAD_STATE,
418	(thread_state_t)&state,
419	count);
420	}
421	ml_set_interrupts_enabled(oldlevel);
422	}
423	}
424
425	__private_extern__ kern_return_t
426	chudxnu_interrupt_callback_enter(chudxnu_interrupt_callback_func_t func)
427	{
428	interrupt_callback_fn = func;
429	lapic_set_pmi_func((i386_intr_func_t)chudxnu_private_interrupt_callback);
430	return KERN_SUCCESS;
431	}
432
433	__private_extern__ kern_return_t
434	chudxnu_interrupt_callback_cancel(void)
435	{
436	interrupt_callback_fn = NULL;
437	lapic_set_pmi_func(NULL);
438	return KERN_SUCCESS;
439	}
440
441	#pragma mark ** cpu signal **
442	static chudxnu_cpusig_callback_func_t cpusig_callback_fn = NULL;
443
444	static kern_return_t
445	chudxnu_private_cpu_signal_handler(int request)
446	{
447	chudxnu_cpusig_callback_func_t fn = cpusig_callback_fn;
448
449	if (fn) {
450	x86_thread_state_t state;
451	mach_msg_type_number_t count = x86_THREAD_STATE_COUNT;
452
453	if (chudxnu_thread_get_state(current_thread(),
454	x86_THREAD_STATE,
455	(thread_state_t) &state, &count,
456	FALSE) == KERN_SUCCESS) {
457	KERNEL_DEBUG_CONSTANT(
458	MACHDBG_CODE(DBG_MACH_CHUD,
459	CHUD_CPUSIG_CALLBACK) \| DBG_FUNC_NONE,
460	(uint32_t)fn, request, 0, 0, 0);
461	return (fn)(
462	request, x86_THREAD_STATE,
463	(thread_state_t) &state, count);
464	} else {
465	return KERN_FAILURE;
466	}
467	}
468	return KERN_SUCCESS; //ignored
469	}
470	/*
471	* chudxnu_cpu_signal_handler() is called from the IPI handler
472	* when a CHUD signal arrives from another processor.
473	*/
474	__private_extern__ void
475	chudxnu_cpu_signal_handler(void)
476	{
477	chudcpu_signal_request_t *reqp;
478	chudcpu_data_t *chudinfop;
479
480	chudinfop = (chudcpu_data_t *) current_cpu_datap()->cpu_chud;
481
482	mpdequeue_head(&(chudinfop->cpu_request_queue),
483	(queue_entry_t *) &reqp);
484	while (reqp != NULL) {
485	chudxnu_private_cpu_signal_handler(reqp->req_code);
486	reqp->req_sync = 0;
487	mpdequeue_head(&(chudinfop->cpu_request_queue),
488	(queue_entry_t *) &reqp);
489	}
490	}
491
492	__private_extern__ kern_return_t
493	chudxnu_cpusig_callback_enter(chudxnu_cpusig_callback_func_t func)
494	{
495	cpusig_callback_fn = func;
496	return KERN_SUCCESS;
497	}
498
499	__private_extern__ kern_return_t
500	chudxnu_cpusig_callback_cancel(void)
501	{
502	cpusig_callback_fn = NULL;
503	return KERN_SUCCESS;
504	}
505
506	__private_extern__ kern_return_t
507	chudxnu_cpusig_send(int otherCPU, uint32_t request_code)
508	{
509	int thisCPU;
510	kern_return_t retval = KERN_FAILURE;
511	chudcpu_signal_request_t request;
512	uint64_t deadline;
513	chudcpu_data_t *target_chudp;
514	boolean_t old_level;
515
516	disable_preemption();
517	// force interrupts on for a cross CPU signal.
518	old_level = chudxnu_set_interrupts_enabled(TRUE);
519	thisCPU = cpu_number();
520
521	if ((unsigned) otherCPU < real_ncpus &&
522	thisCPU != otherCPU &&
523	cpu_data_ptr[otherCPU]->cpu_running) {
524
525	target_chudp = (chudcpu_data_t *)
526	cpu_data_ptr[otherCPU]->cpu_chud;
527
528	/* Fill out request */
529	request.req_sync = 0xFFFFFFFF; /* set sync flag */
530	//request.req_type = CPRQchud; /* set request type */
531	request.req_code = request_code; /* set request */
532
533	KERNEL_DEBUG_CONSTANT(
534	MACHDBG_CODE(DBG_MACH_CHUD,
535	CHUD_CPUSIG_SEND) \| DBG_FUNC_NONE,
536	otherCPU, request_code, 0, 0, 0);
537
538	/*
539	* Insert the new request in the target cpu's request queue
540	* and signal target cpu.
541	*/
542	mpenqueue_tail(&target_chudp->cpu_request_queue,
543	&request.req_entry);
544	i386_signal_cpu(otherCPU, MP_CHUD, ASYNC);
545
546	/* Wait for response or timeout */
547	deadline = mach_absolute_time() + LockTimeOut;
548	while (request.req_sync != 0) {
549	if (mach_absolute_time() > deadline) {
550	panic("chudxnu_cpusig_send(%d,%d) timed out\n",
551	otherCPU, request_code);
552	}
553	cpu_pause();
554	}
555	retval = KERN_SUCCESS;
556	} else {
557	retval = KERN_INVALID_ARGUMENT;
558	}
559
560	chudxnu_set_interrupts_enabled(old_level);
561	enable_preemption();
562	return retval;
563	}
564