2 * Copyright (c) 2003-2009 Apple Inc. All rights reserved.
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
30 #include <mach/boolean.h>
31 #include <mach/mach_types.h>
33 #include <kern/kern_types.h>
34 #include <kern/processor.h>
35 #include <kern/timer_call.h>
36 #include <kern/thread_call.h>
37 #include <kern/kalloc.h>
38 #include <kern/thread.h>
40 #include <libkern/OSAtomic.h>
42 #include <machine/machine_routines.h>
43 #include <machine/cpu_data.h>
44 #include <machine/trap.h>
46 #include <chud/chud_xnu.h>
47 #include <chud/chud_xnu_private.h>
49 #include <i386/misc_protos.h>
50 #include <i386/lapic.h>
52 #include <i386/machine_cpu.h>
54 #include <sys/kdebug.h>
55 #define CHUD_TIMER_CALLBACK_CANCEL 0
56 #define CHUD_TIMER_CALLBACK_ENTER 1
57 #define CHUD_TIMER_CALLBACK 2
58 #define CHUD_AST_SEND 3
59 #define CHUD_AST_CALLBACK 4
60 #define CHUD_CPUSIG_SEND 5
61 #define CHUD_CPUSIG_CALLBACK 6
64 void chudxnu_cancel_all_callbacks(void)
66 chudxnu_cpusig_callback_cancel();
67 chudxnu_cpu_timer_callback_cancel_all();
68 chudxnu_interrupt_callback_cancel();
69 chudxnu_perfmon_ast_callback_cancel();
70 chudxnu_kdebug_callback_cancel();
71 chudxnu_trap_callback_cancel();
72 chudxnu_syscall_callback_cancel();
73 chudxnu_dtrace_callback_cancel();
76 static lck_grp_t chud_request_lck_grp
;
77 static lck_grp_attr_t chud_request_lck_grp_attr
;
78 static lck_attr_t chud_request_lck_attr
;
81 static chudcpu_data_t chudcpu_boot_cpu
;
83 chudxnu_cpu_alloc(boolean_t boot_processor
)
85 chudcpu_data_t
*chud_proc_info
;
88 chud_proc_info
= &chudcpu_boot_cpu
;
90 lck_attr_setdefault(&chud_request_lck_attr
);
91 lck_grp_attr_setdefault(&chud_request_lck_grp_attr
);
92 lck_grp_init(&chud_request_lck_grp
, "chud_request", &chud_request_lck_grp_attr
);
95 chud_proc_info
= (chudcpu_data_t
*)
96 kalloc(sizeof(chudcpu_data_t
));
97 if (chud_proc_info
== (chudcpu_data_t
*)NULL
) {
101 bzero((char *)chud_proc_info
, sizeof(chudcpu_data_t
));
102 chud_proc_info
->t_deadline
= 0xFFFFFFFFFFFFFFFFULL
;
104 mpqueue_init(&chud_proc_info
->cpu_request_queue
, &chud_request_lck_grp
, &chud_request_lck_attr
);
107 return (void *)chud_proc_info
;
111 chudxnu_cpu_free(void *cp
)
113 if (cp
== NULL
|| cp
== (void *)&chudcpu_boot_cpu
) {
116 kfree(cp
,sizeof(chudcpu_data_t
));
121 chudxnu_private_cpu_timer_callback(
122 timer_call_param_t param0
,
123 timer_call_param_t param1
)
125 #pragma unused (param0)
126 #pragma unused (param1)
127 chudcpu_data_t
*chud_proc_info
;
129 x86_thread_state_t state
;
130 mach_msg_type_number_t count
;
131 chudxnu_cpu_timer_callback_func_t fn
;
133 oldlevel
= ml_set_interrupts_enabled(FALSE
);
134 chud_proc_info
= (chudcpu_data_t
*)(current_cpu_datap()->cpu_chud
);
136 count
= x86_THREAD_STATE_COUNT
;
137 if (chudxnu_thread_get_state(current_thread(),
139 (thread_state_t
)&state
,
141 FALSE
) == KERN_SUCCESS
) {
142 fn
= chud_proc_info
->cpu_timer_callback_fn
;
146 (thread_state_t
)&state
,
151 ml_set_interrupts_enabled(oldlevel
);
154 __private_extern__ kern_return_t
155 chudxnu_cpu_timer_callback_enter(
156 chudxnu_cpu_timer_callback_func_t func
,
160 chudcpu_data_t
*chud_proc_info
;
163 oldlevel
= ml_set_interrupts_enabled(FALSE
);
164 chud_proc_info
= (chudcpu_data_t
*)(current_cpu_datap()->cpu_chud
);
166 // cancel any existing callback for this cpu
167 timer_call_cancel(&(chud_proc_info
->cpu_timer_call
));
169 chud_proc_info
->cpu_timer_callback_fn
= func
;
171 clock_interval_to_deadline(time
, units
, &(chud_proc_info
->t_deadline
));
172 timer_call_setup(&(chud_proc_info
->cpu_timer_call
),
173 chudxnu_private_cpu_timer_callback
, NULL
);
174 timer_call_enter(&(chud_proc_info
->cpu_timer_call
),
175 chud_proc_info
->t_deadline
,
176 TIMER_CALL_CRITICAL
|TIMER_CALL_LOCAL
);
178 ml_set_interrupts_enabled(oldlevel
);
182 __private_extern__ kern_return_t
183 chudxnu_cpu_timer_callback_cancel(void)
185 chudcpu_data_t
*chud_proc_info
;
188 oldlevel
= ml_set_interrupts_enabled(FALSE
);
189 chud_proc_info
= (chudcpu_data_t
*)(current_cpu_datap()->cpu_chud
);
191 timer_call_cancel(&(chud_proc_info
->cpu_timer_call
));
194 chud_proc_info
->t_deadline
|= ~(chud_proc_info
->t_deadline
);
195 chud_proc_info
->cpu_timer_callback_fn
= NULL
;
197 ml_set_interrupts_enabled(oldlevel
);
201 __private_extern__ kern_return_t
202 chudxnu_cpu_timer_callback_cancel_all(void)
205 chudcpu_data_t
*chud_proc_info
;
207 for(cpu
=0; cpu
< real_ncpus
; cpu
++) {
208 chud_proc_info
= (chudcpu_data_t
*) cpu_data_ptr
[cpu
]->cpu_chud
;
209 if (chud_proc_info
== NULL
)
211 timer_call_cancel(&(chud_proc_info
->cpu_timer_call
));
212 chud_proc_info
->t_deadline
|= ~(chud_proc_info
->t_deadline
);
213 chud_proc_info
->cpu_timer_callback_fn
= NULL
;
219 #pragma mark **** trap ****
221 static kern_return_t
chud_null_trap(uint32_t trapentry
, thread_flavor_t flavor
,
222 thread_state_t tstate
, mach_msg_type_number_t count
);
223 static chudxnu_trap_callback_func_t trap_callback_fn
= chud_null_trap
;
225 static kern_return_t
chud_null_trap(uint32_t trapentry __unused
, thread_flavor_t flavor __unused
,
226 thread_state_t tstate __unused
, mach_msg_type_number_t count __unused
) {
231 chudxnu_private_trap_callback(
237 #pragma unused (regs)
238 #pragma unused (unused1)
239 #pragma unused (unused2)
240 kern_return_t retval
= KERN_FAILURE
;
241 chudxnu_trap_callback_func_t fn
= trap_callback_fn
;
245 x86_thread_state_t state
;
246 mach_msg_type_number_t count
;
247 thread_t thread
= current_thread();
249 oldlevel
= ml_set_interrupts_enabled(FALSE
);
251 /* prevent reentry into CHUD when dtracing */
252 if(thread
->t_chud
& T_IN_CHUD
) {
253 /* restore interrupts */
254 ml_set_interrupts_enabled(oldlevel
);
256 return KERN_FAILURE
; // not handled - pass off to dtrace
259 /* update the chud state bits */
260 thread
->t_chud
|= T_IN_CHUD
;
262 count
= x86_THREAD_STATE_COUNT
;
264 if(chudxnu_thread_get_state(thread
,
266 (thread_state_t
)&state
,
268 FALSE
) == KERN_SUCCESS
) {
273 (thread_state_t
)&state
,
277 /* no longer in CHUD */
278 thread
->t_chud
&= ~(T_IN_CHUD
);
280 ml_set_interrupts_enabled(oldlevel
);
286 __private_extern__ kern_return_t
287 chudxnu_trap_callback_enter(chudxnu_trap_callback_func_t func
)
289 if(OSCompareAndSwapPtr(NULL
, chudxnu_private_trap_callback
,
290 (void * volatile *)&perfTrapHook
)) {
292 chudxnu_trap_callback_func_t old
= trap_callback_fn
;
293 while(!OSCompareAndSwapPtr(old
, func
,
294 (void * volatile *)&trap_callback_fn
)) {
295 old
= trap_callback_fn
;
302 __private_extern__ kern_return_t
303 chudxnu_trap_callback_cancel(void)
305 if(OSCompareAndSwapPtr(chudxnu_private_trap_callback
, NULL
,
306 (void * volatile *)&perfTrapHook
)) {
308 chudxnu_trap_callback_func_t old
= trap_callback_fn
;
309 while(!OSCompareAndSwapPtr(old
, chud_null_trap
,
310 (void * volatile *)&trap_callback_fn
)) {
311 old
= trap_callback_fn
;
319 #pragma mark **** ast ****
321 static kern_return_t
chud_null_ast(thread_flavor_t flavor
, thread_state_t tstate
,
322 mach_msg_type_number_t count
);
323 static chudxnu_perfmon_ast_callback_func_t perfmon_ast_callback_fn
= chud_null_ast
;
325 static kern_return_t
chud_null_ast(thread_flavor_t flavor __unused
,
326 thread_state_t tstate __unused
, mach_msg_type_number_t count __unused
) {
331 chudxnu_private_chud_ast_callback(ast_t reasons
, ast_t
*myast
)
333 boolean_t oldlevel
= ml_set_interrupts_enabled(FALSE
);
334 kern_return_t retval
= KERN_FAILURE
;
335 chudxnu_perfmon_ast_callback_func_t fn
= perfmon_ast_callback_fn
;
338 if ((*myast
& AST_CHUD_URGENT
) && (reasons
& (AST_URGENT
| AST_CHUD_URGENT
))) { // Only execute urgent callbacks if reasons specifies an urgent context.
339 *myast
&= ~AST_CHUD_URGENT
;
341 if (AST_URGENT
== *myast
) { // If the only flag left is AST_URGENT, we can clear it; we know that we set it, but if there are also other bits set in reasons then someone else might still need AST_URGENT, so we'll leave it set. The normal machinery in ast_taken will ensure it gets cleared eventually, as necessary.
345 retval
= KERN_SUCCESS
;
348 if ((*myast
& AST_CHUD
) && (reasons
& AST_CHUD
)) { // Only execute non-urgent callbacks if reasons actually specifies AST_CHUD. This implies non-urgent callbacks since the only time this'll happen is if someone either calls ast_taken with AST_CHUD explicitly (not done at time of writing, but possible) or with AST_ALL, which of course includes AST_CHUD.
350 retval
= KERN_SUCCESS
;
353 if (KERN_SUCCESS
== retval
) {
354 x86_thread_state_t state
;
355 mach_msg_type_number_t count
= x86_THREAD_STATE_COUNT
;
356 thread_t thread
= current_thread();
358 if (KERN_SUCCESS
== chudxnu_thread_get_state(thread
,
360 (thread_state_t
)&state
,
362 (thread
->task
!= kernel_task
))) {
363 (fn
)(x86_THREAD_STATE
, (thread_state_t
)&state
, count
);
368 ml_set_interrupts_enabled(oldlevel
);
372 __private_extern__ kern_return_t
373 chudxnu_perfmon_ast_callback_enter(chudxnu_perfmon_ast_callback_func_t func
)
375 if(OSCompareAndSwapPtr(NULL
, chudxnu_private_chud_ast_callback
,
376 (void * volatile *)&perfASTHook
)) {
377 chudxnu_perfmon_ast_callback_func_t old
= perfmon_ast_callback_fn
;
379 while(!OSCompareAndSwapPtr(old
, func
,
380 (void * volatile *)&perfmon_ast_callback_fn
)) {
381 old
= perfmon_ast_callback_fn
;
389 __private_extern__ kern_return_t
390 chudxnu_perfmon_ast_callback_cancel(void)
392 if(OSCompareAndSwapPtr(chudxnu_private_chud_ast_callback
, NULL
,
393 (void * volatile *)&perfASTHook
)) {
394 chudxnu_perfmon_ast_callback_func_t old
= perfmon_ast_callback_fn
;
396 while(!OSCompareAndSwapPtr(old
, chud_null_ast
,
397 (void * volatile *)&perfmon_ast_callback_fn
)) {
398 old
= perfmon_ast_callback_fn
;
406 __private_extern__ kern_return_t
407 chudxnu_perfmon_ast_send_urgent(boolean_t urgent
)
409 boolean_t oldlevel
= ml_set_interrupts_enabled(FALSE
);
410 ast_t
*myast
= ast_pending();
413 *myast
|= (AST_CHUD_URGENT
| AST_URGENT
);
415 *myast
|= (AST_CHUD
);
418 ml_set_interrupts_enabled(oldlevel
);
423 #pragma mark **** interrupt ****
425 static kern_return_t
chud_null_int(uint32_t trapentry
, thread_flavor_t flavor
,
426 thread_state_t tstate
, mach_msg_type_number_t count
);
427 static chudxnu_interrupt_callback_func_t interrupt_callback_fn
= chud_null_int
;
429 static kern_return_t
chud_null_int(uint32_t trapentry __unused
, thread_flavor_t flavor __unused
,
430 thread_state_t tstate __unused
, mach_msg_type_number_t count __unused
) {
435 chudxnu_private_interrupt_callback(void *foo
) __attribute__((used
));
438 chudxnu_private_interrupt_callback(void *foo
)
441 chudxnu_interrupt_callback_func_t fn
= interrupt_callback_fn
;
445 x86_thread_state_t state
;
446 mach_msg_type_number_t count
;
448 oldlevel
= ml_set_interrupts_enabled(FALSE
);
450 count
= x86_THREAD_STATE_COUNT
;
451 if(chudxnu_thread_get_state(current_thread(),
453 (thread_state_t
)&state
,
455 FALSE
) == KERN_SUCCESS
) {
457 X86_INTERRUPT_PERFMON
,
459 (thread_state_t
)&state
,
462 ml_set_interrupts_enabled(oldlevel
);
466 __private_extern__ kern_return_t
467 chudxnu_interrupt_callback_enter(chudxnu_interrupt_callback_func_t func
)
469 if(OSCompareAndSwapPtr(chud_null_int
, func
,
470 (void * volatile *)&interrupt_callback_fn
)) {
471 lapic_set_pmi_func((i386_intr_func_t
)chudxnu_private_interrupt_callback
);
477 __private_extern__ kern_return_t
478 chudxnu_interrupt_callback_cancel(void)
480 chudxnu_interrupt_callback_func_t old
= interrupt_callback_fn
;
482 while(!OSCompareAndSwapPtr(old
, chud_null_int
,
483 (void * volatile *)&interrupt_callback_fn
)) {
484 old
= interrupt_callback_fn
;
487 lapic_set_pmi_func(NULL
);
492 #pragma mark **** cpu signal ****
494 static chudxnu_cpusig_callback_func_t cpusig_callback_fn
= NULL
;
497 chudxnu_private_cpu_signal_handler(int request
)
499 chudxnu_cpusig_callback_func_t fn
= cpusig_callback_fn
;
502 x86_thread_state_t state
;
503 mach_msg_type_number_t count
= x86_THREAD_STATE_COUNT
;
505 if (chudxnu_thread_get_state(current_thread(),
507 (thread_state_t
) &state
, &count
,
508 FALSE
) == KERN_SUCCESS
) {
510 request
, x86_THREAD_STATE
,
511 (thread_state_t
) &state
, count
);
516 return KERN_SUCCESS
; //ignored
519 * chudxnu_cpu_signal_handler() is called from the IPI handler
520 * when a CHUD signal arrives from another processor.
522 __private_extern__
void
523 chudxnu_cpu_signal_handler(void)
525 chudcpu_signal_request_t
*reqp
;
526 chudcpu_data_t
*chudinfop
;
528 chudinfop
= (chudcpu_data_t
*) current_cpu_datap()->cpu_chud
;
530 mpdequeue_head(&(chudinfop
->cpu_request_queue
),
531 (queue_entry_t
*) &reqp
);
532 while (reqp
!= NULL
) {
533 chudxnu_private_cpu_signal_handler(reqp
->req_code
);
535 mpdequeue_head(&(chudinfop
->cpu_request_queue
),
536 (queue_entry_t
*) &reqp
);
540 __private_extern__ kern_return_t
541 chudxnu_cpusig_callback_enter(chudxnu_cpusig_callback_func_t func
)
543 if(OSCompareAndSwapPtr(NULL
, func
,
544 (void * volatile *)&cpusig_callback_fn
)) {
550 __private_extern__ kern_return_t
551 chudxnu_cpusig_callback_cancel(void)
553 chudxnu_cpusig_callback_func_t old
= cpusig_callback_fn
;
555 while(!OSCompareAndSwapPtr(old
, NULL
,
556 (void * volatile *)&cpusig_callback_fn
)) {
557 old
= cpusig_callback_fn
;
563 __private_extern__ kern_return_t
564 chudxnu_cpusig_send(int otherCPU
, uint32_t request_code
)
567 kern_return_t retval
= KERN_FAILURE
;
568 chudcpu_signal_request_t request
;
570 chudcpu_data_t
*target_chudp
;
573 disable_preemption();
574 // force interrupts on for a cross CPU signal.
575 old_level
= chudxnu_set_interrupts_enabled(TRUE
);
576 thisCPU
= cpu_number();
578 if ((unsigned) otherCPU
< real_ncpus
&&
579 thisCPU
!= otherCPU
&&
580 cpu_data_ptr
[otherCPU
]->cpu_running
) {
582 target_chudp
= (chudcpu_data_t
*)
583 cpu_data_ptr
[otherCPU
]->cpu_chud
;
585 /* Fill out request */
586 request
.req_sync
= 0xFFFFFFFF; /* set sync flag */
587 //request.req_type = CPRQchud; /* set request type */
588 request
.req_code
= request_code
; /* set request */
591 * Insert the new request in the target cpu's request queue
592 * and signal target cpu.
594 mpenqueue_tail(&target_chudp
->cpu_request_queue
,
596 i386_signal_cpu(otherCPU
, MP_CHUD
, ASYNC
);
598 /* Wait for response or timeout */
599 deadline
= mach_absolute_time() + LockTimeOut
;
600 while (request
.req_sync
!= 0) {
601 if (mach_absolute_time() > deadline
) {
602 panic("chudxnu_cpusig_send(%d,%d) timed out\n",
603 otherCPU
, request_code
);
607 retval
= KERN_SUCCESS
;
609 retval
= KERN_INVALID_ARGUMENT
;
612 chudxnu_set_interrupts_enabled(old_level
);