[apple/xnu.git] / osfmk / chud / i386 / chud_osfmk_callback_i386.c

/*
 * Copyright (c) 2003-2009 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 * 
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */

#include <stdint.h>
#include <mach/boolean.h>
#include <mach/mach_types.h>

#include <kern/kern_types.h>
#include <kern/processor.h>
#include <kern/timer_call.h>
#include <kern/thread_call.h>
#include <kern/kalloc.h>
#include <kern/thread.h>

#include <libkern/OSAtomic.h>

#include <machine/machine_routines.h>
#include <machine/cpu_data.h>
#include <machine/trap.h>

#include <chud/chud_xnu.h>
#include <chud/chud_xnu_private.h>

#include <i386/misc_protos.h>
#include <i386/lapic.h>
#include <i386/mp.h>
#include <i386/machine_cpu.h>

#include <sys/kdebug.h>
#define CHUD_TIMER_CALLBACK_CANCEL	0
#define CHUD_TIMER_CALLBACK_ENTER	1
#define CHUD_TIMER_CALLBACK		2
#define CHUD_AST_SEND			3
#define CHUD_AST_CALLBACK		4
#define CHUD_CPUSIG_SEND		5
#define CHUD_CPUSIG_CALLBACK		6

__private_extern__
void chudxnu_cancel_all_callbacks(void)
{
    chudxnu_cpusig_callback_cancel();
    chudxnu_cpu_timer_callback_cancel_all();
    chudxnu_interrupt_callback_cancel();
    chudxnu_perfmon_ast_callback_cancel();
    chudxnu_kdebug_callback_cancel();
    chudxnu_trap_callback_cancel();
	chudxnu_syscall_callback_cancel();
	chudxnu_dtrace_callback_cancel();
}

static chudcpu_data_t chudcpu_boot_cpu;
void *
chudxnu_cpu_alloc(boolean_t boot_processor)
{
	chudcpu_data_t	*chud_proc_info;

	if (boot_processor) {
		chud_proc_info = &chudcpu_boot_cpu;
	} else {
		chud_proc_info = (chudcpu_data_t *)
					kalloc(sizeof(chudcpu_data_t));
		if (chud_proc_info == (chudcpu_data_t *)NULL) {
			return (void *)NULL;
		}
	}
	bzero((char *)chud_proc_info, sizeof(chudcpu_data_t));
	chud_proc_info->t_deadline = 0xFFFFFFFFFFFFFFFFULL;
	mpqueue_init(&chud_proc_info->cpu_request_queue);


	return (void *)chud_proc_info;
}

void
chudxnu_cpu_free(void *cp)
{
	if (cp == NULL || cp == (void *)&chudcpu_boot_cpu) {
		return;
	} else {
		kfree(cp,sizeof(chudcpu_data_t));
	}
}

static void
chudxnu_private_cpu_timer_callback(
	timer_call_param_t param0,
	timer_call_param_t param1)
{
#pragma unused (param0)
#pragma unused (param1)
	chudcpu_data_t			*chud_proc_info;
	boolean_t			oldlevel;
	x86_thread_state_t 		state;
	mach_msg_type_number_t		count;
	chudxnu_cpu_timer_callback_func_t fn;

	oldlevel = ml_set_interrupts_enabled(FALSE);
	chud_proc_info = (chudcpu_data_t *)(current_cpu_datap()->cpu_chud);

	count = x86_THREAD_STATE_COUNT;
	if (chudxnu_thread_get_state(current_thread(),
				     x86_THREAD_STATE,
				     (thread_state_t)&state,
				     &count,
				     FALSE) == KERN_SUCCESS) {
			fn = chud_proc_info->cpu_timer_callback_fn;
       		if (fn) {
       			(fn)(
				x86_THREAD_STATE,
				(thread_state_t)&state,
				count);
       		} 
	} 
	
	ml_set_interrupts_enabled(oldlevel);
}

__private_extern__ kern_return_t
chudxnu_cpu_timer_callback_enter(
	chudxnu_cpu_timer_callback_func_t	func,
	uint32_t				time,
	uint32_t				units)
{
	chudcpu_data_t	*chud_proc_info;
	boolean_t	oldlevel;

	oldlevel = ml_set_interrupts_enabled(FALSE);
	chud_proc_info = (chudcpu_data_t *)(current_cpu_datap()->cpu_chud);

	// cancel any existing callback for this cpu
	timer_call_cancel(&(chud_proc_info->cpu_timer_call));

	chud_proc_info->cpu_timer_callback_fn = func;

	clock_interval_to_deadline(time, units, &(chud_proc_info->t_deadline));
	timer_call_setup(&(chud_proc_info->cpu_timer_call),
			 chudxnu_private_cpu_timer_callback, NULL);
	timer_call_enter(&(chud_proc_info->cpu_timer_call),
			 chud_proc_info->t_deadline);

	ml_set_interrupts_enabled(oldlevel);
	return KERN_SUCCESS;
}

__private_extern__ kern_return_t
chudxnu_cpu_timer_callback_cancel(void)
{
	chudcpu_data_t	*chud_proc_info;
	boolean_t	oldlevel;

	oldlevel = ml_set_interrupts_enabled(FALSE);
	chud_proc_info = (chudcpu_data_t *)(current_cpu_datap()->cpu_chud);

	timer_call_cancel(&(chud_proc_info->cpu_timer_call));

	// set to max value:
	chud_proc_info->t_deadline |= ~(chud_proc_info->t_deadline);
	chud_proc_info->cpu_timer_callback_fn = NULL;

	ml_set_interrupts_enabled(oldlevel);
 	return KERN_SUCCESS;
}

__private_extern__ kern_return_t
chudxnu_cpu_timer_callback_cancel_all(void)
{
	unsigned int	cpu;
	chudcpu_data_t	*chud_proc_info;

	for(cpu=0; cpu < real_ncpus; cpu++) {
		chud_proc_info = (chudcpu_data_t *) cpu_data_ptr[cpu]->cpu_chud;
		if (chud_proc_info == NULL)
			continue;
		timer_call_cancel(&(chud_proc_info->cpu_timer_call));
		chud_proc_info->t_deadline |= ~(chud_proc_info->t_deadline);
		chud_proc_info->cpu_timer_callback_fn = NULL;
	}
	return KERN_SUCCESS;
}

#if 0
#pragma mark **** trap ****
#endif
static kern_return_t chud_null_trap(uint32_t trapentry, thread_flavor_t flavor,
	thread_state_t tstate,  mach_msg_type_number_t count);
static chudxnu_trap_callback_func_t trap_callback_fn = chud_null_trap;

static kern_return_t chud_null_trap(uint32_t trapentry __unused, thread_flavor_t flavor __unused,
	thread_state_t tstate __unused,  mach_msg_type_number_t count __unused) {
	return KERN_FAILURE;
}

static kern_return_t
chudxnu_private_trap_callback(
	int trapno,
	void			*regs,
	int			unused1,
	int			unused2)
{
#pragma unused (regs)
#pragma unused (unused1)
#pragma unused (unused2)
	kern_return_t retval = KERN_FAILURE;
	chudxnu_trap_callback_func_t fn = trap_callback_fn;

	if(fn) {
		boolean_t oldlevel;
		x86_thread_state_t state;
		mach_msg_type_number_t count;
		thread_t thread = current_thread();
		
		oldlevel = ml_set_interrupts_enabled(FALSE);
		
		/* prevent reentry into CHUD when dtracing */
		if(thread->t_chud & T_IN_CHUD) {
			/* restore interrupts */
			ml_set_interrupts_enabled(oldlevel);

			return KERN_FAILURE;	// not handled - pass off to dtrace
		}

		/* update the chud state bits */
		thread->t_chud |= T_IN_CHUD;

		count = x86_THREAD_STATE_COUNT;
		
		if(chudxnu_thread_get_state(thread,
				x86_THREAD_STATE,
				(thread_state_t)&state,
				&count,
				FALSE) == KERN_SUCCESS) {
		  
					retval = (fn)(
						trapno,
						x86_THREAD_STATE,
						(thread_state_t)&state,
						count);
		}

		/* no longer in CHUD */
		thread->t_chud &= ~(T_IN_CHUD);

		ml_set_interrupts_enabled(oldlevel);
	}

	return retval;
}

__private_extern__ kern_return_t
chudxnu_trap_callback_enter(chudxnu_trap_callback_func_t func)
{
	if(OSCompareAndSwapPtr(NULL, chudxnu_private_trap_callback, 
		(void * volatile *)&perfTrapHook)) {

		chudxnu_trap_callback_func_t old = trap_callback_fn;
		while(!OSCompareAndSwapPtr(old, func, 
			(void * volatile *)&trap_callback_fn)) {
			old = trap_callback_fn;
		}
		return KERN_SUCCESS;
	}
	return KERN_FAILURE;
}

__private_extern__ kern_return_t
chudxnu_trap_callback_cancel(void)
{
	if(OSCompareAndSwapPtr(chudxnu_private_trap_callback,  NULL,
		(void * volatile *)&perfTrapHook)) {

		chudxnu_trap_callback_func_t old = trap_callback_fn;
		while(!OSCompareAndSwapPtr(old, chud_null_trap, 
			(void * volatile *)&trap_callback_fn)) {
			old = trap_callback_fn;
		}
		return KERN_SUCCESS;
	}
	return KERN_FAILURE;
}

#if 0
#pragma mark **** ast ****
#endif
static kern_return_t chud_null_ast(thread_flavor_t flavor, thread_state_t tstate,  
	mach_msg_type_number_t count);
static chudxnu_perfmon_ast_callback_func_t perfmon_ast_callback_fn = chud_null_ast;

static kern_return_t chud_null_ast(thread_flavor_t flavor __unused,
	thread_state_t tstate __unused,  mach_msg_type_number_t count __unused) {
	return KERN_FAILURE;
}

static kern_return_t
chudxnu_private_chud_ast_callback(
	int			trapno,
	void			*regs,
	int			unused1,
	int			unused2)
{
#pragma unused (trapno)
#pragma unused (regs)
#pragma unused (unused1)
#pragma unused (unused2)
	boolean_t	oldlevel = ml_set_interrupts_enabled(FALSE);
	ast_t		*myast = ast_pending();
	kern_return_t	retval = KERN_FAILURE;
	chudxnu_perfmon_ast_callback_func_t fn = perfmon_ast_callback_fn;
    
	if (*myast & AST_CHUD_URGENT) {
		*myast &= ~(AST_CHUD_URGENT | AST_CHUD);
		if ((*myast & AST_PREEMPTION) != AST_PREEMPTION)
			*myast &= ~(AST_URGENT);
		retval = KERN_SUCCESS;
	} else if (*myast & AST_CHUD) {
		*myast &= ~(AST_CHUD);
		retval = KERN_SUCCESS;
	}

	if (fn) {
		x86_thread_state_t state;
		mach_msg_type_number_t count;
		count = x86_THREAD_STATE_COUNT;

		if (chudxnu_thread_get_state(
			current_thread(),
			x86_THREAD_STATE,
			(thread_state_t) &state, &count,
			TRUE) == KERN_SUCCESS) {

			(fn)(
				x86_THREAD_STATE,
				(thread_state_t) &state,
				count);
		}
	}
    
	ml_set_interrupts_enabled(oldlevel);
	return retval;
}

__private_extern__ kern_return_t
chudxnu_perfmon_ast_callback_enter(chudxnu_perfmon_ast_callback_func_t func)
{
	if(OSCompareAndSwapPtr(NULL, chudxnu_private_chud_ast_callback,
		(void * volatile *)&perfASTHook)) {
		chudxnu_perfmon_ast_callback_func_t old = perfmon_ast_callback_fn;

		while(!OSCompareAndSwapPtr(old, func,
			(void * volatile *)&perfmon_ast_callback_fn)) {
			old = perfmon_ast_callback_fn;
		}

		return KERN_SUCCESS;
	}
	return KERN_FAILURE;
}

__private_extern__ kern_return_t
chudxnu_perfmon_ast_callback_cancel(void)
{
	if(OSCompareAndSwapPtr(chudxnu_private_chud_ast_callback, NULL,
		(void * volatile *)&perfASTHook)) {
		chudxnu_perfmon_ast_callback_func_t old = perfmon_ast_callback_fn;

		while(!OSCompareAndSwapPtr(old, chud_null_ast,
			(void * volatile *)&perfmon_ast_callback_fn)) {
			old = perfmon_ast_callback_fn;
		}

		return KERN_SUCCESS;
	}
	return KERN_FAILURE;
}

__private_extern__ kern_return_t
chudxnu_perfmon_ast_send_urgent(boolean_t urgent)
{
    boolean_t oldlevel = ml_set_interrupts_enabled(FALSE);
	ast_t *myast = ast_pending();

    if(urgent) {
        *myast |= (AST_CHUD_URGENT | AST_URGENT);
    } else {
        *myast |= (AST_CHUD);
    }

    ml_set_interrupts_enabled(oldlevel);
    return KERN_SUCCESS;
}

#if 0
#pragma mark **** interrupt ****
#endif
static kern_return_t chud_null_int(uint32_t trapentry, thread_flavor_t flavor, 
	thread_state_t tstate,  mach_msg_type_number_t count);
static chudxnu_interrupt_callback_func_t interrupt_callback_fn = chud_null_int;

static kern_return_t chud_null_int(uint32_t trapentry __unused, thread_flavor_t flavor __unused,
	thread_state_t tstate __unused,  mach_msg_type_number_t count __unused) {
	return KERN_FAILURE;
}

static void
chudxnu_private_interrupt_callback(void *foo)
{
#pragma unused (foo)
	chudxnu_interrupt_callback_func_t fn = interrupt_callback_fn;

	if(fn) {
		boolean_t			oldlevel;
		x86_thread_state_t		state;
		mach_msg_type_number_t		count;

		oldlevel = ml_set_interrupts_enabled(FALSE);

		count = x86_THREAD_STATE_COUNT;
		if(chudxnu_thread_get_state(current_thread(),
					    x86_THREAD_STATE,
					    (thread_state_t)&state,
					    &count,
					    FALSE) == KERN_SUCCESS) {
			(fn)(
				X86_INTERRUPT_PERFMON,
				x86_THREAD_STATE,
				(thread_state_t)&state,
				count);
		}
		ml_set_interrupts_enabled(oldlevel);
	}
}

__private_extern__ kern_return_t
chudxnu_interrupt_callback_enter(chudxnu_interrupt_callback_func_t func)
{
	if(OSCompareAndSwapPtr(chud_null_int, func, 
		(void * volatile *)&interrupt_callback_fn)) {
		lapic_set_pmi_func((i386_intr_func_t)chudxnu_private_interrupt_callback);

		return KERN_SUCCESS;
	}
    return KERN_FAILURE;
}

__private_extern__ kern_return_t
chudxnu_interrupt_callback_cancel(void)
{
	chudxnu_interrupt_callback_func_t old = interrupt_callback_fn;

	while(!OSCompareAndSwapPtr(old, chud_null_int,
		(void * volatile *)&interrupt_callback_fn)) {
		old = interrupt_callback_fn;
	}

    lapic_set_pmi_func(NULL);
    return KERN_SUCCESS;
}

#if 0
#pragma mark **** cpu signal ****
#endif
static chudxnu_cpusig_callback_func_t cpusig_callback_fn = NULL;

static          kern_return_t
chudxnu_private_cpu_signal_handler(int request)
{
	chudxnu_cpusig_callback_func_t fn = cpusig_callback_fn;
	
	if (fn) {
	x86_thread_state_t  state;
		mach_msg_type_number_t count = x86_THREAD_STATE_COUNT;

		if (chudxnu_thread_get_state(current_thread(),
					     x86_THREAD_STATE,
					     (thread_state_t) &state, &count,
					     FALSE) == KERN_SUCCESS) {
			return (fn)(
					request, x86_THREAD_STATE,
					(thread_state_t) &state, count);
		} else {
			return KERN_FAILURE;
		}
	}
	return KERN_SUCCESS; //ignored
}
/*
 * chudxnu_cpu_signal_handler() is called from the IPI handler
 * when a CHUD signal arrives from another processor.
 */
__private_extern__ void
chudxnu_cpu_signal_handler(void)
{
	chudcpu_signal_request_t	*reqp;
	chudcpu_data_t			*chudinfop;

	chudinfop = (chudcpu_data_t *) current_cpu_datap()->cpu_chud;

	mpdequeue_head(&(chudinfop->cpu_request_queue),
		       (queue_entry_t *) &reqp);
	while (reqp != NULL) {
		chudxnu_private_cpu_signal_handler(reqp->req_code);
		reqp->req_sync = 0;
		mpdequeue_head(&(chudinfop->cpu_request_queue),
			       (queue_entry_t *) &reqp);
	}
}

__private_extern__ kern_return_t
chudxnu_cpusig_callback_enter(chudxnu_cpusig_callback_func_t func)
{
	if(OSCompareAndSwapPtr(NULL, func, 
		(void * volatile *)&cpusig_callback_fn)) {
		return KERN_SUCCESS;
	}
	return KERN_FAILURE;
}

__private_extern__ kern_return_t
chudxnu_cpusig_callback_cancel(void)
{
	chudxnu_cpusig_callback_func_t old = cpusig_callback_fn;

	while(!OSCompareAndSwapPtr(old, NULL,
		(void * volatile *)&cpusig_callback_fn)) {
		old = cpusig_callback_fn;
	}

	return KERN_SUCCESS;
}

__private_extern__ kern_return_t
chudxnu_cpusig_send(int otherCPU, uint32_t request_code)
{
	int				thisCPU;
	kern_return_t			retval = KERN_FAILURE;
	chudcpu_signal_request_t	request;
	uint64_t			deadline;
	chudcpu_data_t			*target_chudp;
	boolean_t old_level;

	disable_preemption();
	// force interrupts on for a cross CPU signal.
	old_level = chudxnu_set_interrupts_enabled(TRUE);
	thisCPU = cpu_number();

	if ((unsigned) otherCPU < real_ncpus &&
	    thisCPU != otherCPU &&
	    cpu_data_ptr[otherCPU]->cpu_running) {

		target_chudp = (chudcpu_data_t *)
					cpu_data_ptr[otherCPU]->cpu_chud;

		/* Fill out request */
		request.req_sync = 0xFFFFFFFF;		/* set sync flag */
		//request.req_type = CPRQchud;		/* set request type */
		request.req_code = request_code;	/* set request */

		/*
		 * Insert the new request in the target cpu's request queue
		 * and signal target cpu.
		 */
		mpenqueue_tail(&target_chudp->cpu_request_queue,
			       &request.req_entry);
		i386_signal_cpu(otherCPU, MP_CHUD, ASYNC);

		/* Wait for response or timeout */
		deadline = mach_absolute_time() + LockTimeOut;
		while (request.req_sync != 0) {
			if (mach_absolute_time() > deadline) {
				panic("chudxnu_cpusig_send(%d,%d) timed out\n",
					otherCPU, request_code);
			}
			cpu_pause();
		}
		retval = KERN_SUCCESS;
	} else {
		retval = KERN_INVALID_ARGUMENT;
	}

	chudxnu_set_interrupts_enabled(old_level);
	enable_preemption();
	return retval;
}
Commit	Line	Data
0c530ab8	1	/*
b0d623f7	2	* Copyright (c) 2003-2009 Apple Inc. All rights reserved.
0c530ab8	3	*
2d21ac55 A	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
	5	*
	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
	14	*
	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
0c530ab8 A	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
0c530ab8 A	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
2d21ac55 A	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
	25	*
	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
0c530ab8 A	27	*/
	28
	29	#include <stdint.h>
	30	#include <mach/boolean.h>
	31	#include <mach/mach_types.h>
	32
	33	#include <kern/kern_types.h>
	34	#include <kern/processor.h>
	35	#include <kern/timer_call.h>
	36	#include <kern/thread_call.h>
	37	#include <kern/kalloc.h>
	38	#include <kern/thread.h>
	39
b0d623f7 A	40	#include <libkern/OSAtomic.h>
b0d623f7 A	41
0c530ab8 A	42	#include <machine/machine_routines.h>
	43	#include <machine/cpu_data.h>
	44	#include <machine/trap.h>
	45
	46	#include <chud/chud_xnu.h>
	47	#include <chud/chud_xnu_private.h>
	48
	49	#include <i386/misc_protos.h>
593a1d5f	50	#include <i386/lapic.h>
0c530ab8 A	51	#include <i386/mp.h>
	52	#include <i386/machine_cpu.h>
	53
	54	#include <sys/kdebug.h>
	55	#define CHUD_TIMER_CALLBACK_CANCEL 0
	56	#define CHUD_TIMER_CALLBACK_ENTER 1
	57	#define CHUD_TIMER_CALLBACK 2
	58	#define CHUD_AST_SEND 3
	59	#define CHUD_AST_CALLBACK 4
	60	#define CHUD_CPUSIG_SEND 5
	61	#define CHUD_CPUSIG_CALLBACK 6
	62
	63	__private_extern__
	64	void chudxnu_cancel_all_callbacks(void)
	65	{
	66	chudxnu_cpusig_callback_cancel();
	67	chudxnu_cpu_timer_callback_cancel_all();
	68	chudxnu_interrupt_callback_cancel();
	69	chudxnu_perfmon_ast_callback_cancel();
	70	chudxnu_kdebug_callback_cancel();
0c530ab8	71	chudxnu_trap_callback_cancel();
0c530ab8	72	chudxnu_syscall_callback_cancel();
2d21ac55	73	chudxnu_dtrace_callback_cancel();
0c530ab8 A	74	}
	75
	76	static chudcpu_data_t chudcpu_boot_cpu;
	77	void *
	78	chudxnu_cpu_alloc(boolean_t boot_processor)
	79	{
	80	chudcpu_data_t *chud_proc_info;
	81
0c530ab8 A	82	if (boot_processor) {
	83	chud_proc_info = &chudcpu_boot_cpu;
	84	} else {
	85	chud_proc_info = (chudcpu_data_t *)
	86	kalloc(sizeof(chudcpu_data_t));
	87	if (chud_proc_info == (chudcpu_data_t *)NULL) {
	88	return (void *)NULL;
	89	}
	90	}
	91	bzero((char *)chud_proc_info, sizeof(chudcpu_data_t));
	92	chud_proc_info->t_deadline = 0xFFFFFFFFFFFFFFFFULL;
	93	mpqueue_init(&chud_proc_info->cpu_request_queue);
	94
	95
	96	return (void *)chud_proc_info;
	97	}
	98
	99	void
	100	chudxnu_cpu_free(void *cp)
	101	{
	102	if (cp == NULL \|\| cp == (void *)&chudcpu_boot_cpu) {
	103	return;
	104	} else {
	105	kfree(cp,sizeof(chudcpu_data_t));
	106	}
	107	}
	108
	109	static void
	110	chudxnu_private_cpu_timer_callback(
	111	timer_call_param_t param0,
	112	timer_call_param_t param1)
	113	{
	114	#pragma unused (param0)
	115	#pragma unused (param1)
	116	chudcpu_data_t *chud_proc_info;
	117	boolean_t oldlevel;
	118	x86_thread_state_t state;
	119	mach_msg_type_number_t count;
	120	chudxnu_cpu_timer_callback_func_t fn;
	121
	122	oldlevel = ml_set_interrupts_enabled(FALSE);
	123	chud_proc_info = (chudcpu_data_t *)(current_cpu_datap()->cpu_chud);
	124
	125	count = x86_THREAD_STATE_COUNT;
	126	if (chudxnu_thread_get_state(current_thread(),
	127	x86_THREAD_STATE,
	128	(thread_state_t)&state,
	129	&count,
	130	FALSE) == KERN_SUCCESS) {
	131	fn = chud_proc_info->cpu_timer_callback_fn;
	132	if (fn) {
0c530ab8 A	133	(fn)(
	134	x86_THREAD_STATE,
	135	(thread_state_t)&state,
	136	count);
	137	}
	138	}
	139
	140	ml_set_interrupts_enabled(oldlevel);
	141	}
	142
	143	__private_extern__ kern_return_t
	144	chudxnu_cpu_timer_callback_enter(
	145	chudxnu_cpu_timer_callback_func_t func,
	146	uint32_t time,
	147	uint32_t units)
	148	{
	149	chudcpu_data_t *chud_proc_info;
	150	boolean_t oldlevel;
	151
	152	oldlevel = ml_set_interrupts_enabled(FALSE);
	153	chud_proc_info = (chudcpu_data_t *)(current_cpu_datap()->cpu_chud);
	154
	155	// cancel any existing callback for this cpu
	156	timer_call_cancel(&(chud_proc_info->cpu_timer_call));
	157
	158	chud_proc_info->cpu_timer_callback_fn = func;
	159
	160	clock_interval_to_deadline(time, units, &(chud_proc_info->t_deadline));
	161	timer_call_setup(&(chud_proc_info->cpu_timer_call),
	162	chudxnu_private_cpu_timer_callback, NULL);
	163	timer_call_enter(&(chud_proc_info->cpu_timer_call),
	164	chud_proc_info->t_deadline);
	165
0c530ab8 A	166	ml_set_interrupts_enabled(oldlevel);
	167	return KERN_SUCCESS;
	168	}
	169
	170	__private_extern__ kern_return_t
	171	chudxnu_cpu_timer_callback_cancel(void)
	172	{
	173	chudcpu_data_t *chud_proc_info;
	174	boolean_t oldlevel;
	175
	176	oldlevel = ml_set_interrupts_enabled(FALSE);
	177	chud_proc_info = (chudcpu_data_t *)(current_cpu_datap()->cpu_chud);
	178
	179	timer_call_cancel(&(chud_proc_info->cpu_timer_call));
	180
0c530ab8 A	181	// set to max value:
	182	chud_proc_info->t_deadline \|= ~(chud_proc_info->t_deadline);
	183	chud_proc_info->cpu_timer_callback_fn = NULL;
	184
	185	ml_set_interrupts_enabled(oldlevel);
	186	return KERN_SUCCESS;
	187	}
	188
	189	__private_extern__ kern_return_t
	190	chudxnu_cpu_timer_callback_cancel_all(void)
	191	{
	192	unsigned int cpu;
	193	chudcpu_data_t *chud_proc_info;
	194
	195	for(cpu=0; cpu < real_ncpus; cpu++) {
	196	chud_proc_info = (chudcpu_data_t *) cpu_data_ptr[cpu]->cpu_chud;
	197	if (chud_proc_info == NULL)
	198	continue;
	199	timer_call_cancel(&(chud_proc_info->cpu_timer_call));
	200	chud_proc_info->t_deadline \|= ~(chud_proc_info->t_deadline);
	201	chud_proc_info->cpu_timer_callback_fn = NULL;
	202	}
	203	return KERN_SUCCESS;
	204	}
	205
b0d623f7	206	#if 0
0c530ab8	207	#pragma mark ** trap **
b0d623f7 A	208	#endif
	209	static kern_return_t chud_null_trap(uint32_t trapentry, thread_flavor_t flavor,
	210	thread_state_t tstate, mach_msg_type_number_t count);
	211	static chudxnu_trap_callback_func_t trap_callback_fn = chud_null_trap;
	212
	213	static kern_return_t chud_null_trap(uint32_t trapentry __unused, thread_flavor_t flavor __unused,
	214	thread_state_t tstate __unused, mach_msg_type_number_t count __unused) {
	215	return KERN_FAILURE;
	216	}
0c530ab8 A	217
	218	static kern_return_t
	219	chudxnu_private_trap_callback(
2d21ac55	220	int trapno,
0c530ab8 A	221	void *regs,
	222	int unused1,
	223	int unused2)
	224	{
	225	#pragma unused (regs)
	226	#pragma unused (unused1)
	227	#pragma unused (unused2)
2d21ac55	228	kern_return_t retval = KERN_FAILURE;
0c530ab8 A	229	chudxnu_trap_callback_func_t fn = trap_callback_fn;
0c530ab8 A	230
2d21ac55 A	231	if(fn) {
	232	boolean_t oldlevel;
	233	x86_thread_state_t state;
	234	mach_msg_type_number_t count;
	235	thread_t thread = current_thread();
	236
0c530ab8	237	oldlevel = ml_set_interrupts_enabled(FALSE);
2d21ac55 A	238
	239	/* prevent reentry into CHUD when dtracing */
	240	if(thread->t_chud & T_IN_CHUD) {
	241	/* restore interrupts */
	242	ml_set_interrupts_enabled(oldlevel);
	243
	244	return KERN_FAILURE; // not handled - pass off to dtrace
	245	}
	246
	247	/* update the chud state bits */
	248	thread->t_chud \|= T_IN_CHUD;
0c530ab8 A	249
0c530ab8 A	250	count = x86_THREAD_STATE_COUNT;
2d21ac55 A	251
2d21ac55 A	252	if(chudxnu_thread_get_state(thread,
0c530ab8 A	253	x86_THREAD_STATE,
	254	(thread_state_t)&state,
	255	&count,
	256	FALSE) == KERN_SUCCESS) {
2d21ac55 A	257
	258	retval = (fn)(
	259	trapno,
	260	x86_THREAD_STATE,
	261	(thread_state_t)&state,
	262	count);
0c530ab8	263	}
2d21ac55 A	264
	265	/* no longer in CHUD */
	266	thread->t_chud &= ~(T_IN_CHUD);
	267
	268	ml_set_interrupts_enabled(oldlevel);
0c530ab8 A	269	}
0c530ab8 A	270
2d21ac55	271	return retval;
0c530ab8 A	272	}
	273
	274	__private_extern__ kern_return_t
	275	chudxnu_trap_callback_enter(chudxnu_trap_callback_func_t func)
	276	{
b0d623f7 A	277	if(OSCompareAndSwapPtr(NULL, chudxnu_private_trap_callback,
	278	(void * volatile *)&perfTrapHook)) {
	279
	280	chudxnu_trap_callback_func_t old = trap_callback_fn;
	281	while(!OSCompareAndSwapPtr(old, func,
	282	(void * volatile *)&trap_callback_fn)) {
	283	old = trap_callback_fn;
	284	}
	285	return KERN_SUCCESS;
	286	}
	287	return KERN_FAILURE;
0c530ab8 A	288	}
	289
	290	__private_extern__ kern_return_t
	291	chudxnu_trap_callback_cancel(void)
	292	{
b0d623f7 A	293	if(OSCompareAndSwapPtr(chudxnu_private_trap_callback, NULL,
	294	(void * volatile *)&perfTrapHook)) {
	295
	296	chudxnu_trap_callback_func_t old = trap_callback_fn;
	297	while(!OSCompareAndSwapPtr(old, chud_null_trap,
	298	(void * volatile *)&trap_callback_fn)) {
	299	old = trap_callback_fn;
	300	}
	301	return KERN_SUCCESS;
	302	}
	303	return KERN_FAILURE;
0c530ab8 A	304	}
0c530ab8 A	305
b0d623f7	306	#if 0
0c530ab8	307	#pragma mark ** ast **
b0d623f7 A	308	#endif
	309	static kern_return_t chud_null_ast(thread_flavor_t flavor, thread_state_t tstate,
	310	mach_msg_type_number_t count);
	311	static chudxnu_perfmon_ast_callback_func_t perfmon_ast_callback_fn = chud_null_ast;
	312
	313	static kern_return_t chud_null_ast(thread_flavor_t flavor __unused,
	314	thread_state_t tstate __unused, mach_msg_type_number_t count __unused) {
	315	return KERN_FAILURE;
	316	}
0c530ab8 A	317
	318	static kern_return_t
	319	chudxnu_private_chud_ast_callback(
	320	int trapno,
	321	void *regs,
	322	int unused1,
	323	int unused2)
	324	{
	325	#pragma unused (trapno)
	326	#pragma unused (regs)
	327	#pragma unused (unused1)
	328	#pragma unused (unused2)
	329	boolean_t oldlevel = ml_set_interrupts_enabled(FALSE);
	330	ast_t *myast = ast_pending();
	331	kern_return_t retval = KERN_FAILURE;
	332	chudxnu_perfmon_ast_callback_func_t fn = perfmon_ast_callback_fn;
	333
	334	if (*myast & AST_CHUD_URGENT) {
	335	*myast &= ~(AST_CHUD_URGENT \| AST_CHUD);
	336	if ((*myast & AST_PREEMPTION) != AST_PREEMPTION)
	337	*myast &= ~(AST_URGENT);
	338	retval = KERN_SUCCESS;
	339	} else if (*myast & AST_CHUD) {
	340	*myast &= ~(AST_CHUD);
	341	retval = KERN_SUCCESS;
	342	}
	343
	344	if (fn) {
	345	x86_thread_state_t state;
	346	mach_msg_type_number_t count;
	347	count = x86_THREAD_STATE_COUNT;
2d21ac55	348
0c530ab8 A	349	if (chudxnu_thread_get_state(
	350	current_thread(),
	351	x86_THREAD_STATE,
	352	(thread_state_t) &state, &count,
	353	TRUE) == KERN_SUCCESS) {
	354
0c530ab8 A	355	(fn)(
	356	x86_THREAD_STATE,
	357	(thread_state_t) &state,
	358	count);
	359	}
	360	}
	361
	362	ml_set_interrupts_enabled(oldlevel);
	363	return retval;
	364	}
	365
	366	__private_extern__ kern_return_t
	367	chudxnu_perfmon_ast_callback_enter(chudxnu_perfmon_ast_callback_func_t func)
	368	{
b0d623f7 A	369	if(OSCompareAndSwapPtr(NULL, chudxnu_private_chud_ast_callback,
	370	(void * volatile *)&perfASTHook)) {
	371	chudxnu_perfmon_ast_callback_func_t old = perfmon_ast_callback_fn;
	372
	373	while(!OSCompareAndSwapPtr(old, func,
	374	(void * volatile *)&perfmon_ast_callback_fn)) {
	375	old = perfmon_ast_callback_fn;
	376	}
	377
	378	return KERN_SUCCESS;
	379	}
	380	return KERN_FAILURE;
0c530ab8 A	381	}
	382
	383	__private_extern__ kern_return_t
	384	chudxnu_perfmon_ast_callback_cancel(void)
	385	{
b0d623f7 A	386	if(OSCompareAndSwapPtr(chudxnu_private_chud_ast_callback, NULL,
	387	(void * volatile *)&perfASTHook)) {
	388	chudxnu_perfmon_ast_callback_func_t old = perfmon_ast_callback_fn;
	389
	390	while(!OSCompareAndSwapPtr(old, chud_null_ast,
	391	(void * volatile *)&perfmon_ast_callback_fn)) {
	392	old = perfmon_ast_callback_fn;
	393	}
	394
	395	return KERN_SUCCESS;
	396	}
	397	return KERN_FAILURE;
0c530ab8 A	398	}
	399
	400	__private_extern__ kern_return_t
	401	chudxnu_perfmon_ast_send_urgent(boolean_t urgent)
	402	{
	403	boolean_t oldlevel = ml_set_interrupts_enabled(FALSE);
	404	ast_t *myast = ast_pending();
	405
	406	if(urgent) {
	407	*myast \|= (AST_CHUD_URGENT \| AST_URGENT);
	408	} else {
	409	*myast \|= (AST_CHUD);
	410	}
	411
0c530ab8 A	412	ml_set_interrupts_enabled(oldlevel);
	413	return KERN_SUCCESS;
	414	}
	415
b0d623f7	416	#if 0
0c530ab8	417	#pragma mark ** interrupt **
b0d623f7 A	418	#endif
	419	static kern_return_t chud_null_int(uint32_t trapentry, thread_flavor_t flavor,
	420	thread_state_t tstate, mach_msg_type_number_t count);
	421	static chudxnu_interrupt_callback_func_t interrupt_callback_fn = chud_null_int;
	422
	423	static kern_return_t chud_null_int(uint32_t trapentry __unused, thread_flavor_t flavor __unused,
	424	thread_state_t tstate __unused, mach_msg_type_number_t count __unused) {
	425	return KERN_FAILURE;
	426	}
0c530ab8 A	427
	428	static void
	429	chudxnu_private_interrupt_callback(void *foo)
	430	{
	431	#pragma unused (foo)
	432	chudxnu_interrupt_callback_func_t fn = interrupt_callback_fn;
	433
	434	if(fn) {
	435	boolean_t oldlevel;
	436	x86_thread_state_t state;
	437	mach_msg_type_number_t count;
	438
	439	oldlevel = ml_set_interrupts_enabled(FALSE);
	440
	441	count = x86_THREAD_STATE_COUNT;
	442	if(chudxnu_thread_get_state(current_thread(),
	443	x86_THREAD_STATE,
	444	(thread_state_t)&state,
	445	&count,
	446	FALSE) == KERN_SUCCESS) {
	447	(fn)(
	448	X86_INTERRUPT_PERFMON,
	449	x86_THREAD_STATE,
	450	(thread_state_t)&state,
	451	count);
	452	}
	453	ml_set_interrupts_enabled(oldlevel);
	454	}
	455	}
	456
	457	__private_extern__ kern_return_t
	458	chudxnu_interrupt_callback_enter(chudxnu_interrupt_callback_func_t func)
	459	{
b0d623f7 A	460	if(OSCompareAndSwapPtr(chud_null_int, func,
	461	(void * volatile *)&interrupt_callback_fn)) {
	462	lapic_set_pmi_func((i386_intr_func_t)chudxnu_private_interrupt_callback);
	463
	464	return KERN_SUCCESS;
	465	}
	466	return KERN_FAILURE;
0c530ab8 A	467	}
	468
	469	__private_extern__ kern_return_t
	470	chudxnu_interrupt_callback_cancel(void)
	471	{
b0d623f7 A	472	chudxnu_interrupt_callback_func_t old = interrupt_callback_fn;
	473
	474	while(!OSCompareAndSwapPtr(old, chud_null_int,
	475	(void * volatile *)&interrupt_callback_fn)) {
	476	old = interrupt_callback_fn;
	477	}
	478
0c530ab8 A	479	lapic_set_pmi_func(NULL);
	480	return KERN_SUCCESS;
	481	}
	482
b0d623f7	483	#if 0
0c530ab8	484	#pragma mark ** cpu signal **
b0d623f7	485	#endif
0c530ab8 A	486	static chudxnu_cpusig_callback_func_t cpusig_callback_fn = NULL;
	487
	488	static kern_return_t
	489	chudxnu_private_cpu_signal_handler(int request)
	490	{
	491	chudxnu_cpusig_callback_func_t fn = cpusig_callback_fn;
	492
	493	if (fn) {
	494	x86_thread_state_t state;
	495	mach_msg_type_number_t count = x86_THREAD_STATE_COUNT;
	496
	497	if (chudxnu_thread_get_state(current_thread(),
	498	x86_THREAD_STATE,
	499	(thread_state_t) &state, &count,
	500	FALSE) == KERN_SUCCESS) {
0c530ab8 A	501	return (fn)(
	502	request, x86_THREAD_STATE,
	503	(thread_state_t) &state, count);
	504	} else {
	505	return KERN_FAILURE;
	506	}
	507	}
	508	return KERN_SUCCESS; //ignored
	509	}
	510	/*
	511	* chudxnu_cpu_signal_handler() is called from the IPI handler
	512	* when a CHUD signal arrives from another processor.
	513	*/
	514	__private_extern__ void
	515	chudxnu_cpu_signal_handler(void)
	516	{
	517	chudcpu_signal_request_t *reqp;
	518	chudcpu_data_t *chudinfop;
	519
	520	chudinfop = (chudcpu_data_t *) current_cpu_datap()->cpu_chud;
	521
	522	mpdequeue_head(&(chudinfop->cpu_request_queue),
	523	(queue_entry_t *) &reqp);
	524	while (reqp != NULL) {
	525	chudxnu_private_cpu_signal_handler(reqp->req_code);
	526	reqp->req_sync = 0;
	527	mpdequeue_head(&(chudinfop->cpu_request_queue),
	528	(queue_entry_t *) &reqp);
	529	}
	530	}
	531
	532	__private_extern__ kern_return_t
	533	chudxnu_cpusig_callback_enter(chudxnu_cpusig_callback_func_t func)
	534	{
b0d623f7 A	535	if(OSCompareAndSwapPtr(NULL, func,
	536	(void * volatile *)&cpusig_callback_fn)) {
	537	return KERN_SUCCESS;
	538	}
	539	return KERN_FAILURE;
0c530ab8 A	540	}
	541
	542	__private_extern__ kern_return_t
	543	chudxnu_cpusig_callback_cancel(void)
	544	{
b0d623f7 A	545	chudxnu_cpusig_callback_func_t old = cpusig_callback_fn;
	546
	547	while(!OSCompareAndSwapPtr(old, NULL,
	548	(void * volatile *)&cpusig_callback_fn)) {
	549	old = cpusig_callback_fn;
	550	}
	551
0c530ab8 A	552	return KERN_SUCCESS;
	553	}
	554
	555	__private_extern__ kern_return_t
	556	chudxnu_cpusig_send(int otherCPU, uint32_t request_code)
	557	{
	558	int thisCPU;
	559	kern_return_t retval = KERN_FAILURE;
	560	chudcpu_signal_request_t request;
	561	uint64_t deadline;
	562	chudcpu_data_t *target_chudp;
	563	boolean_t old_level;
	564
	565	disable_preemption();
	566	// force interrupts on for a cross CPU signal.
	567	old_level = chudxnu_set_interrupts_enabled(TRUE);
	568	thisCPU = cpu_number();
	569
	570	if ((unsigned) otherCPU < real_ncpus &&
	571	thisCPU != otherCPU &&
	572	cpu_data_ptr[otherCPU]->cpu_running) {
	573
	574	target_chudp = (chudcpu_data_t *)
	575	cpu_data_ptr[otherCPU]->cpu_chud;
	576
	577	/* Fill out request */
	578	request.req_sync = 0xFFFFFFFF; /* set sync flag */
	579	//request.req_type = CPRQchud; /* set request type */
	580	request.req_code = request_code; /* set request */
	581
0c530ab8 A	582	/*
	583	* Insert the new request in the target cpu's request queue
	584	* and signal target cpu.
	585	*/
	586	mpenqueue_tail(&target_chudp->cpu_request_queue,
	587	&request.req_entry);
	588	i386_signal_cpu(otherCPU, MP_CHUD, ASYNC);
	589
	590	/* Wait for response or timeout */
	591	deadline = mach_absolute_time() + LockTimeOut;
	592	while (request.req_sync != 0) {
	593	if (mach_absolute_time() > deadline) {
	594	panic("chudxnu_cpusig_send(%d,%d) timed out\n",
	595	otherCPU, request_code);
	596	}
	597	cpu_pause();
	598	}
	599	retval = KERN_SUCCESS;
	600	} else {
	601	retval = KERN_INVALID_ARGUMENT;
	602	}
	603
	604	chudxnu_set_interrupts_enabled(old_level);
	605	enable_preemption();
	606	return retval;
	607	}