2 * Copyright (c) 2011 Apple Computer, Inc. All rights reserved.
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
31 #include <mach/mach_types.h>
32 #include <kern/cpu_data.h> /* current_thread() */
33 #include <kern/kalloc.h>
34 #include <stdatomic.h>
35 #include <sys/errno.h>
37 #include <sys/ktrace.h>
39 #include <machine/machine_routines.h>
40 #if defined(__x86_64__)
42 #endif /* defined(__x86_64__) */
44 #include <kperf/kperf.h>
45 #include <kperf/buffer.h>
46 #include <kperf/context.h>
47 #include <kperf/action.h>
48 #include <kperf/kperf_timer.h>
49 #include <kperf/kperf_arch.h>
50 #include <kperf/pet.h>
51 #include <kperf/sample.h>
53 /* the list of timers */
54 struct kperf_timer
*kperf_timerv
= NULL
;
55 unsigned int kperf_timerc
= 0;
57 static unsigned int pet_timer_id
= 999;
59 #define KPERF_TMR_ACTION_MASK (0xff)
60 #define KPERF_TMR_ACTION(action_state) ((action_state) & KPERF_TMR_ACTION_MASK)
61 #define KPERF_TMR_ACTIVE (0x100)
63 /* maximum number of timers we can construct */
64 #define TIMER_MAX (16)
66 static uint64_t min_period_abstime
;
67 static uint64_t min_period_bg_abstime
;
68 static uint64_t min_period_pet_abstime
;
69 static uint64_t min_period_pet_bg_abstime
;
72 kperf_timer_min_period_abstime(void)
74 if (ktrace_background_active()) {
75 return min_period_bg_abstime
;
77 return min_period_abstime
;
82 kperf_timer_min_pet_period_abstime(void)
84 if (ktrace_background_active()) {
85 return min_period_pet_bg_abstime
;
87 return min_period_pet_abstime
;
92 kperf_timer_schedule(struct kperf_timer
*timer
, uint64_t now
)
94 BUF_INFO(PERF_TM_SCHED
, timer
->period
);
96 /* if we re-programmed the timer to zero, just drop it */
97 if (timer
->period
== 0) {
101 /* calculate deadline */
102 uint64_t deadline
= now
+ timer
->period
;
104 /* re-schedule the timer, making sure we don't apply slop */
105 timer_call_enter(&timer
->tcall
, deadline
, TIMER_CALL_SYS_CRITICAL
);
109 kperf_sample_cpu(struct kperf_timer
*timer
, bool system_sample
,
112 assert(timer
!= NULL
);
114 /* Always cut a tracepoint to show a sample event occurred */
115 BUF_DATA(PERF_TM_HNDLR
| DBG_FUNC_START
, 0);
117 int ncpu
= cpu_number();
119 struct kperf_sample
*intbuf
= kperf_intr_sample_buffer();
120 #if DEVELOPMENT || DEBUG
121 intbuf
->sample_time
= mach_absolute_time();
122 #endif /* DEVELOPMENT || DEBUG */
124 /* On a timer, we can see the "real" current thread */
125 thread_t thread
= current_thread();
126 task_t task
= get_threadtask(thread
);
127 struct kperf_context ctx
= {
128 .cur_thread
= thread
,
130 .cur_pid
= task_pid(task
),
131 .trigger_type
= TRIGGER_TYPE_TIMER
,
132 .trigger_id
= (unsigned int)(timer
- kperf_timerv
),
135 if (ctx
.trigger_id
== pet_timer_id
&& ncpu
< machine_info
.logical_cpu_max
) {
136 kperf_tid_on_cpus
[ncpu
] = thread_tid(ctx
.cur_thread
);
139 /* make sure sampling is on */
140 unsigned int status
= kperf_sampling_status();
141 if (status
== KPERF_SAMPLING_OFF
) {
142 BUF_INFO(PERF_TM_HNDLR
| DBG_FUNC_END
, SAMPLE_OFF
);
144 } else if (status
== KPERF_SAMPLING_SHUTDOWN
) {
145 BUF_INFO(PERF_TM_HNDLR
| DBG_FUNC_END
, SAMPLE_SHUTDOWN
);
149 /* call the action -- kernel-only from interrupt, pend user */
150 int r
= kperf_sample(intbuf
, &ctx
, timer
->actionid
,
151 SAMPLE_FLAG_PEND_USER
| (system_sample
? SAMPLE_FLAG_SYSTEM
: 0) |
152 (only_system
? SAMPLE_FLAG_ONLY_SYSTEM
: 0));
154 /* end tracepoint is informational */
155 BUF_INFO(PERF_TM_HNDLR
| DBG_FUNC_END
, r
);
157 (void)atomic_fetch_and_explicit(&timer
->pending_cpus
,
158 ~(UINT64_C(1) << ncpu
), memory_order_relaxed
);
162 kperf_ipi_handler(void *param
)
164 kperf_sample_cpu((struct kperf_timer
*)param
, false, false);
168 kperf_timer_handler(void *param0
, __unused
void *param1
)
170 struct kperf_timer
*timer
= param0
;
171 unsigned int ntimer
= (unsigned int)(timer
- kperf_timerv
);
172 unsigned int ncpus
= machine_info
.logical_cpu_max
;
173 bool system_only_self
= true;
175 uint32_t action_state
= atomic_fetch_or(&timer
->action_state
,
178 uint32_t actionid
= KPERF_TMR_ACTION(action_state
);
183 #if DEVELOPMENT || DEBUG
184 timer
->fire_time
= mach_absolute_time();
185 #endif /* DEVELOPMENT || DEBUG */
187 /* along the lines of do not ipi if we are all shutting down */
188 if (kperf_sampling_status() == KPERF_SAMPLING_SHUTDOWN
) {
192 BUF_DATA(PERF_TM_FIRE
, ntimer
, ntimer
== pet_timer_id
, timer
->period
,
195 if (ntimer
== pet_timer_id
) {
196 kperf_pet_fire_before();
198 /* clean-up the thread-on-CPUs cache */
199 bzero(kperf_tid_on_cpus
, ncpus
* sizeof(*kperf_tid_on_cpus
));
203 * IPI other cores only if the action has non-system samplers.
205 if (kperf_action_has_non_system(actionid
)) {
207 * If the core that's handling the timer is not scheduling
208 * threads, only run system samplers.
210 system_only_self
= kperf_mp_broadcast_other_running(timer
);
212 kperf_sample_cpu(timer
, true, system_only_self
);
214 /* release the pet thread? */
215 if (ntimer
== pet_timer_id
) {
216 /* PET mode is responsible for rearming the timer */
217 kperf_pet_fire_after();
220 * FIXME: Get the current time from elsewhere. The next
221 * timer's period now includes the time taken to reach this
222 * point. This causes a bias towards longer sampling periods
225 kperf_timer_schedule(timer
, mach_absolute_time());
229 atomic_fetch_and(&timer
->action_state
, ~KPERF_TMR_ACTIVE
);
232 /* program the timer from the PET thread */
234 kperf_timer_pet_rearm(uint64_t elapsed_ticks
)
236 struct kperf_timer
*timer
= NULL
;
241 * If the pet_timer_id is invalid, it has been disabled, so this should
244 if (pet_timer_id
>= kperf_timerc
) {
248 unsigned int status
= kperf_sampling_status();
249 /* do not reprogram the timer if it has been shutdown or sampling is off */
250 if (status
== KPERF_SAMPLING_OFF
) {
251 BUF_INFO(PERF_PET_END
, SAMPLE_OFF
);
253 } else if (status
== KPERF_SAMPLING_SHUTDOWN
) {
254 BUF_INFO(PERF_PET_END
, SAMPLE_SHUTDOWN
);
258 timer
= &(kperf_timerv
[pet_timer_id
]);
260 /* if we re-programmed the timer to zero, just drop it */
261 if (!timer
->period
) {
265 /* subtract the time the pet sample took being careful not to underflow */
266 if (timer
->period
> elapsed_ticks
) {
267 period
= timer
->period
- elapsed_ticks
;
270 /* make sure we don't set the next PET sample to happen too soon */
271 if (period
< min_period_pet_abstime
) {
272 period
= min_period_pet_abstime
;
275 /* we probably took so long in the PET thread, it makes sense to take
278 deadline
= mach_absolute_time() + period
;
280 BUF_INFO(PERF_PET_SCHED
, timer
->period
, period
, elapsed_ticks
, deadline
);
282 /* re-schedule the timer, making sure we don't apply slop */
283 timer_call_enter(&timer
->tcall
, deadline
, TIMER_CALL_SYS_CRITICAL
);
288 /* turn on all the timers */
292 /* get the PET thread going */
293 if (pet_timer_id
< kperf_timerc
) {
294 kperf_pet_config(kperf_timerv
[pet_timer_id
].actionid
);
297 uint64_t now
= mach_absolute_time();
299 for (unsigned int i
= 0; i
< kperf_timerc
; i
++) {
300 struct kperf_timer
*timer
= &kperf_timerv
[i
];
301 if (timer
->period
== 0) {
305 atomic_store(&timer
->action_state
,
306 timer
->actionid
& KPERF_TMR_ACTION_MASK
);
307 kperf_timer_schedule(timer
, now
);
312 kperf_timer_stop(void)
315 * Determine which timers are running and store them in a bitset, while
316 * cancelling their timer call.
318 uint64_t running_timers
= 0;
319 for (unsigned int i
= 0; i
< kperf_timerc
; i
++) {
320 struct kperf_timer
*timer
= &kperf_timerv
[i
];
321 if (timer
->period
== 0) {
325 uint32_t action_state
= atomic_fetch_and(&timer
->action_state
,
326 ~KPERF_TMR_ACTION_MASK
);
327 if (action_state
& KPERF_TMR_ACTIVE
) {
328 bit_set(running_timers
, i
);
331 timer_call_cancel(&timer
->tcall
);
335 * Wait for any running timers to finish their critical sections.
337 for (unsigned int i
= lsb_first(running_timers
); i
< kperf_timerc
;
338 i
= lsb_next(running_timers
, i
)) {
339 while (atomic_load(&kperf_timerv
[i
].action_state
) != 0) {
344 if (pet_timer_id
< kperf_timerc
) {
345 /* wait for PET to stop, too */
351 kperf_timer_get_petid(void)
357 kperf_timer_set_petid(unsigned int timerid
)
359 if (timerid
< kperf_timerc
) {
362 min_period
= kperf_timer_min_pet_period_abstime();
363 if (kperf_timerv
[timerid
].period
< min_period
) {
364 kperf_timerv
[timerid
].period
= min_period
;
366 kperf_pet_config(kperf_timerv
[timerid
].actionid
);
368 /* clear the PET trigger if it's a bogus ID */
372 pet_timer_id
= timerid
;
378 kperf_timer_get_period(unsigned int timerid
, uint64_t *period_abstime
)
380 if (timerid
>= kperf_timerc
) {
384 *period_abstime
= kperf_timerv
[timerid
].period
;
389 kperf_timer_set_period(unsigned int timerid
, uint64_t period_abstime
)
393 if (timerid
>= kperf_timerc
) {
397 if (pet_timer_id
== timerid
) {
398 min_period
= kperf_timer_min_pet_period_abstime();
400 min_period
= kperf_timer_min_period_abstime();
403 if (period_abstime
> 0 && period_abstime
< min_period
) {
404 period_abstime
= min_period
;
407 kperf_timerv
[timerid
].period
= period_abstime
;
409 /* FIXME: re-program running timers? */
415 kperf_timer_get_action(unsigned int timerid
, uint32_t *action
)
417 if (timerid
>= kperf_timerc
) {
421 *action
= kperf_timerv
[timerid
].actionid
;
426 kperf_timer_set_action(unsigned int timerid
, uint32_t action
)
428 if (timerid
>= kperf_timerc
) {
432 kperf_timerv
[timerid
].actionid
= action
;
437 kperf_timer_get_count(void)
443 kperf_timer_reset(void)
445 kperf_timer_set_petid(999);
446 kperf_set_pet_idle_rate(KPERF_PET_DEFAULT_IDLE_RATE
);
447 kperf_set_lightweight_pet(0);
448 for (unsigned int i
= 0; i
< kperf_timerc
; i
++) {
449 kperf_timerv
[i
].period
= 0;
450 kperf_timerv
[i
].actionid
= 0;
451 atomic_store_explicit(&kperf_timerv
[i
].pending_cpus
, 0, memory_order_relaxed
);
456 kperf_timer_set_count(unsigned int count
)
458 struct kperf_timer
*new_timerv
= NULL
, *old_timerv
= NULL
;
459 unsigned int old_count
;
461 if (min_period_abstime
== 0) {
462 nanoseconds_to_absolutetime(KP_MIN_PERIOD_NS
, &min_period_abstime
);
463 nanoseconds_to_absolutetime(KP_MIN_PERIOD_BG_NS
, &min_period_bg_abstime
);
464 nanoseconds_to_absolutetime(KP_MIN_PERIOD_PET_NS
, &min_period_pet_abstime
);
465 nanoseconds_to_absolutetime(KP_MIN_PERIOD_PET_BG_NS
,
466 &min_period_pet_bg_abstime
);
467 assert(min_period_abstime
> 0);
470 if (count
== kperf_timerc
) {
473 if (count
> TIMER_MAX
) {
477 /* TODO: allow shrinking? */
478 if (count
< kperf_timerc
) {
483 * Make sure kperf is initialized when creating the array for the first
486 if (kperf_timerc
== 0) {
490 if ((r
= kperf_init())) {
496 * Shut down any running timers since we will be messing with the timer
501 /* create a new array */
502 new_timerv
= kalloc_tag(count
* sizeof(struct kperf_timer
),
503 VM_KERN_MEMORY_DIAG
);
504 if (new_timerv
== NULL
) {
507 old_timerv
= kperf_timerv
;
508 old_count
= kperf_timerc
;
510 if (old_timerv
!= NULL
) {
511 bcopy(kperf_timerv
, new_timerv
,
512 kperf_timerc
* sizeof(struct kperf_timer
));
515 /* zero the new entries */
516 bzero(&(new_timerv
[kperf_timerc
]),
517 (count
- old_count
) * sizeof(struct kperf_timer
));
519 /* (re-)setup the timer call info for all entries */
520 for (unsigned int i
= 0; i
< count
; i
++) {
521 timer_call_setup(&new_timerv
[i
].tcall
, kperf_timer_handler
, &new_timerv
[i
]);
524 kperf_timerv
= new_timerv
;
525 kperf_timerc
= count
;
527 if (old_timerv
!= NULL
) {
528 kfree(old_timerv
, old_count
* sizeof(struct kperf_timer
));