]> git.saurik.com Git - apple/xnu.git/blame - bsd/kern/kdebug.c
projects / apple / xnu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
xnu-2782.1.97.tar.gz
[apple/xnu.git] / bsd / kern / kdebug.c
CommitLineData
1c79356b 1/*
39236c6e 2 * Copyright (c) 2000-2013 Apple Inc. All rights reserved.
1c79356b 3 *
91447636 4 * @Apple_LICENSE_HEADER_START@
1c79356b 5 *
e5568f75
A		 6 * The contents of this file constitute Original Code as defined in and
7 * are subject to the Apple Public Source License Version 1.1 (the
8 * "License"). You may not use this file except in compliance with the
9 * License. Please obtain a copy of the License at
10 * http://www.apple.com/publicsource and read it before using this file.
1c79356b 11 *
e5568f75
A		 12 * This Original Code and all software distributed under the License are
13 * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
1c79356b
A		 14 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
15 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
e5568f75
A		 16 * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
17 * License for the specific language governing rights and limitations
18 * under the License.
1c79356b 19 *
2d21ac55 20 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
1c79356b
A		 21 */
22
6d2010ae 23
1c79356b
A		 24#include <machine/spl.h>
25
91447636
A		 26#include <sys/errno.h>
27#include <sys/param.h>
0c530ab8 28#include <sys/systm.h>
91447636
A		 29#include <sys/proc_internal.h>
30#include <sys/vm.h>
31#include <sys/sysctl.h>
32#include <sys/kdebug.h>
33#include <sys/sysproto.h>
6d2010ae 34#include <sys/bsdtask_info.h>
fe8ab488 35#include <sys/random.h>
91447636 36
1c79356b
A		 37#define HZ 100
38#include <mach/clock_types.h>
39#include <mach/mach_types.h>
55e303ae 40#include <mach/mach_time.h>
1c79356b
A		 41#include <machine/machine_routines.h>
42
b0d623f7 43#if defined(__i386__) || defined(__x86_64__)
6d2010ae
A		 44#include <i386/rtclock_protos.h>
45#include <i386/mp.h>
46#include <i386/machine_routines.h>
b0d623f7 47#endif
6d2010ae
A		 48
49#include <kern/clock.h>
50
1c79356b
A		 51#include <kern/thread.h>
52#include <kern/task.h>
2d21ac55 53#include <kern/debug.h>
6d2010ae
A		 54#include <kern/kalloc.h>
55#include <kern/cpu_data.h>
d41d1dae 56#include <kern/assert.h>
39236c6e 57#include <kern/telemetry.h>
1c79356b
A		 58#include <vm/vm_kern.h>
59#include <sys/lock.h>
60
0c530ab8 61#include <sys/malloc.h>
b0d623f7 62#include <sys/mcache.h>
0c530ab8
A		 63#include <sys/kauth.h>
64
b0d623f7
A		 65#include <sys/vnode.h>
66#include <sys/vnode_internal.h>
67#include <sys/fcntl.h>
6d2010ae
A		 68#include <sys/file_internal.h>
69#include <sys/ubc.h>
316670eb 70#include <sys/param.h> /* for isset() */
b0d623f7 71
0c530ab8
A		 72#include <mach/mach_host.h> /* for host_info() */
73#include <libkern/OSAtomic.h>
74
6d2010ae
A		 75#include <machine/pal_routines.h>
76
39236c6e
A		 77/*
78 * IOP(s)
79 *
80 * https://coreoswiki.apple.com/wiki/pages/U6z3i0q9/Consistent_Logging_Implementers_Guide.html
81 *
82 * IOP(s) are auxiliary cores that want to participate in kdebug event logging.
83 * They are registered dynamically. Each is assigned a cpu_id at registration.
84 *
85 * NOTE: IOP trace events may not use the same clock hardware as "normal"
86 * cpus. There is an effort made to synchronize the IOP timebase with the
87 * AP, but it should be understood that there may be discrepancies.
88 *
89 * Once registered, an IOP is permanent, it cannot be unloaded/unregistered.
90 * The current implementation depends on this for thread safety.
91 *
92 * New registrations occur by allocating an kd_iop struct and assigning
93 * a provisional cpu_id of list_head->cpu_id + 1. Then a CAS to claim the
94 * list_head pointer resolves any races.
95 *
96 * You may safely walk the kd_iops list at any time, without holding locks.
97 *
98 * When allocating buffers, the current kd_iops head is captured. Any operations
99 * that depend on the buffer state (such as flushing IOP traces on reads,
100 * etc.) should use the captured list head. This will allow registrations to
101 * take place while trace is in use.
102 */
103
104typedef struct kd_iop {
105 kd_callback_t callback;
106 uint32_t cpu_id;
107 uint64_t last_timestamp; /* Prevent timer rollback */
108 struct kd_iop* next;
109} kd_iop_t;
110
111static kd_iop_t* kd_iops = NULL;
112
0c530ab8
A		 113/* XXX should have prototypes, but Mach does not provide one */
114void task_act_iterate_wth_args(task_t, void(*)(thread_t, void *), void *);
115int cpu_number(void); /* XXX <machine/...> include path broken */
116
117/* XXX should probably be static, but it's debugging code... */
b0d623f7 118int kdbg_read(user_addr_t, size_t *, vnode_t, vfs_context_t);
0c530ab8
A		 119void kdbg_control_chud(int, void *);
120int kdbg_control(int *, u_int, user_addr_t, size_t *);
39236c6e
A		 121int kdbg_readcpumap(user_addr_t, size_t *);
122int kdbg_readcurcpumap(user_addr_t, size_t *);
123int kdbg_readthrmap(user_addr_t, size_t *, vnode_t, vfs_context_t);
124int kdbg_readcurthrmap(user_addr_t, size_t *);
0c530ab8
A		 125int kdbg_getreg(kd_regtype *);
126int kdbg_setreg(kd_regtype *);
127int kdbg_setrtcdec(kd_regtype *);
128int kdbg_setpidex(kd_regtype *);
129int kdbg_setpid(kd_regtype *);
39236c6e 130void kdbg_thrmap_init(void);
6d2010ae
A		 131int kdbg_reinit(boolean_t);
132int kdbg_bootstrap(boolean_t);
0c530ab8 133
39236c6e
A		 134int kdbg_cpumap_init_internal(kd_iop_t* iops, uint32_t cpu_count, uint8_t** cpumap, uint32_t* cpumap_size);
135kd_threadmap* kdbg_thrmap_init_internal(unsigned int count, unsigned int *mapsize, unsigned int *mapcount);
136
316670eb
A		 137static int kdbg_enable_typefilter(void);
138static int kdbg_disable_typefilter(void);
139
6d2010ae 140static int create_buffers(boolean_t);
0c530ab8
A		 141static void delete_buffers(void);
142
2d21ac55
A		 143extern void IOSleep(int);
144
9bccf70c
A		 145/* trace enable status */
146unsigned int kdebug_enable = 0;
147
fe8ab488
A		 148/* A static buffer to record events prior to the start of regular logging */
149#define KD_EARLY_BUFFER_MAX 64
150static kd_buf kd_early_buffer[KD_EARLY_BUFFER_MAX];
151static int kd_early_index = 0;
152static boolean_t kd_early_overflow = FALSE;
6d2010ae 153
91447636
A		 154#define SLOW_NOLOG 0x01
155#define SLOW_CHECKS 0x02
fe8ab488 156#define SLOW_ENTROPY 0x04 /* Obsolescent */
6d2010ae 157#define SLOW_CHUD 0x08
91447636 158
b0d623f7
A		 159#define EVENTS_PER_STORAGE_UNIT 2048
160#define MIN_STORAGE_UNITS_PER_CPU 4
161
6d2010ae
A		 162#define POINTER_FROM_KDS_PTR(x) (&kd_bufs[x.buffer_index].kdsb_addr[x.offset])
163
6d2010ae
A		 164union kds_ptr {
165 struct {
166 uint32_t buffer_index:21;
167 uint16_t offset:11;
168 };
169 uint32_t raw;
170};
171
b0d623f7 172struct kd_storage {
6d2010ae
A		 173 union kds_ptr kds_next;
174 uint32_t kds_bufindx;
175 uint32_t kds_bufcnt;
176 uint32_t kds_readlast;
177 boolean_t kds_lostevents;
178 uint64_t kds_timestamp;
0c530ab8 179
b0d623f7 180 kd_buf kds_records[EVENTS_PER_STORAGE_UNIT];
0c530ab8
A		 181};
182
b0d623f7
A		 183#define MAX_BUFFER_SIZE (1024 * 1024 * 128)
184#define N_STORAGE_UNITS_PER_BUFFER (MAX_BUFFER_SIZE / sizeof(struct kd_storage))
185
b0d623f7
A		 186struct kd_storage_buffers {
187 struct kd_storage *kdsb_addr;
188 uint32_t kdsb_size;
189};
190
6d2010ae 191#define KDS_PTR_NULL 0xffffffff
b0d623f7
A		 192struct kd_storage_buffers *kd_bufs = NULL;
193int n_storage_units = 0;
194int n_storage_buffers = 0;
6d2010ae
A		 195int n_storage_threshold = 0;
196int kds_waiter = 0;
b0d623f7 197
6d2010ae 198#pragma pack(0)
b0d623f7 199struct kd_bufinfo {
6d2010ae
A		 200 union kds_ptr kd_list_head;
201 union kds_ptr kd_list_tail;
202 boolean_t kd_lostevents;
203 uint32_t _pad;
204 uint64_t kd_prev_timebase;
205 uint32_t num_bufs;
39236c6e 206} __attribute__(( aligned(MAX_CPU_CACHE_LINE_SIZE) ));
b0d623f7 207
6d2010ae
A		 208struct kd_ctrl_page_t {
209 union kds_ptr kds_free_list;
210 uint32_t enabled :1;
211 uint32_t _pad0 :31;
212 int kds_inuse_count;
213 uint32_t kdebug_flags;
214 uint32_t kdebug_slowcheck;
39236c6e
A		 215 /*
216 * The number of kd_bufinfo structs allocated may not match the current
217 * number of active cpus. We capture the iops list head at initialization
218 * which we could use to calculate the number of cpus we allocated data for,
219 * unless it happens to be null. To avoid that case, we explicitly also
220 * capture a cpu count.
221 */
222 kd_iop_t* kdebug_iops;
223 uint32_t kdebug_cpus;
224} kd_ctrl_page = { .kds_free_list = {.raw = KDS_PTR_NULL}, .kdebug_slowcheck = SLOW_NOLOG };
225
6d2010ae
A		 226#pragma pack()
227
0c530ab8
A		 228struct kd_bufinfo *kdbip = NULL;
229
6d2010ae 230#define KDCOPYBUF_COUNT 8192
0c530ab8
A		 231#define KDCOPYBUF_SIZE (KDCOPYBUF_COUNT * sizeof(kd_buf))
232kd_buf *kdcopybuf = NULL;
233
316670eb
A		 234boolean_t kdlog_bg_trace = FALSE;
235boolean_t kdlog_bg_trace_running = FALSE;
236unsigned int bg_nkdbufs = 0;
237
238unsigned int nkdbufs = 0;
1c79356b
A		 239unsigned int kdlog_beg=0;
240unsigned int kdlog_end=0;
241unsigned int kdlog_value1=0;
242unsigned int kdlog_value2=0;
243unsigned int kdlog_value3=0;
244unsigned int kdlog_value4=0;
245
6d2010ae 246static lck_spin_t * kdw_spin_lock;
b0d623f7 247static lck_spin_t * kds_spin_lock;
0c530ab8
A		 248static lck_mtx_t * kd_trace_mtx_sysctl;
249static lck_grp_t * kd_trace_mtx_sysctl_grp;
250static lck_attr_t * kd_trace_mtx_sysctl_attr;
251static lck_grp_attr_t *kd_trace_mtx_sysctl_grp_attr;
252
253static lck_grp_t *stackshot_subsys_lck_grp;
254static lck_grp_attr_t *stackshot_subsys_lck_grp_attr;
255static lck_attr_t *stackshot_subsys_lck_attr;
256static lck_mtx_t stackshot_subsys_mutex;
257
258void *stackshot_snapbuf = NULL;
259
260int
b7266188 261stack_snapshot2(pid_t pid, user_addr_t tracebuf, uint32_t tracebuf_size, uint32_t flags, uint32_t dispatch_offset, int32_t *retval);
91447636 262
39236c6e
A		 263int
264stack_snapshot_from_kernel(pid_t pid, void *buf, uint32_t size, uint32_t flags, unsigned *bytesTraced);
0c530ab8 265extern void
b7266188 266kdp_snapshot_preflight(int pid, void *tracebuf, uint32_t tracebuf_size, uint32_t flags, uint32_t dispatch_offset);
91447636 267
0c530ab8
A		 268extern int
269kdp_stack_snapshot_geterror(void);
270extern unsigned int
271kdp_stack_snapshot_bytes_traced(void);
1c79356b
A		 272
273kd_threadmap *kd_mapptr = 0;
274unsigned int kd_mapsize = 0;
275unsigned int kd_mapcount = 0;
b0d623f7
A		 276
277off_t RAW_file_offset = 0;
6d2010ae
A		 278int RAW_file_written = 0;
279
280#define RAW_FLUSH_SIZE (2 * 1024 * 1024)
281
1c79356b
A		 282pid_t global_state_pid = -1; /* Used to control exclusive use of kd_buffer */
283
6d2010ae
A		 284#define DBG_FUNC_MASK 0xfffffffc
285
316670eb
A		 286/* TODO: move to kdebug.h */
287#define CLASS_MASK 0xff000000
288#define CLASS_OFFSET 24
289#define SUBCLASS_MASK 0x00ff0000
290#define SUBCLASS_OFFSET 16
291#define CSC_MASK 0xffff0000 /* class and subclass mask */
292#define CSC_OFFSET SUBCLASS_OFFSET
293
294#define EXTRACT_CLASS(debugid) ( (uint8_t) ( ((debugid) & CLASS_MASK ) >> CLASS_OFFSET ) )
295#define EXTRACT_SUBCLASS(debugid) ( (uint8_t) ( ((debugid) & SUBCLASS_MASK) >> SUBCLASS_OFFSET ) )
296#define EXTRACT_CSC(debugid) ( (uint16_t)( ((debugid) & CSC_MASK ) >> CSC_OFFSET ) )
297
6d2010ae
A		 298#define INTERRUPT 0x01050000
299#define MACH_vmfault 0x01300008
300#define BSC_SysCall 0x040c0000
301#define MACH_SysCall 0x010c0000
302#define DBG_SCALL_MASK 0xffff0000
303
1c79356b 304
9bccf70c
A		 305/* task to string structure */
306struct tts
307{
0c530ab8 308 task_t task; /* from procs task */
55e303ae 309 pid_t pid; /* from procs p_pid */
9bccf70c
A		 310 char task_comm[20]; /* from procs p_comm */
311};
312
313typedef struct tts tts_t;
314
1c79356b
A		 315struct krt
316{
6d2010ae
A		 317 kd_threadmap *map; /* pointer to the map buffer */
318 int count;
319 int maxcount;
320 struct tts *atts;
1c79356b
A		 321};
322
323typedef struct krt krt_t;
324
9bccf70c 325/* This is for the CHUD toolkit call */
b0d623f7
A		 326typedef void (*kd_chudhook_fn) (uint32_t debugid, uintptr_t arg1,
327 uintptr_t arg2, uintptr_t arg3,
328 uintptr_t arg4, uintptr_t arg5);
9bccf70c 329
6d2010ae 330volatile kd_chudhook_fn kdebug_chudhook = 0; /* pointer to CHUD toolkit function */
9bccf70c 331
39236c6e 332__private_extern__ void stackshot_lock_init( void );
91447636 333
316670eb
A		 334static uint8_t *type_filter_bitmap;
335
fe8ab488
A		 336/*
337 * This allows kperf to swap out the global state pid when kperf ownership is
338 * passed from one process to another. It checks the old global state pid so
339 * that kperf can't accidentally steal control of trace when a non-kperf trace user has
340 * control of trace.
341 */
342void
343kdbg_swap_global_state_pid(pid_t old_pid, pid_t new_pid);
344
345void
346kdbg_swap_global_state_pid(pid_t old_pid, pid_t new_pid)
347{
348 if (!(kd_ctrl_page.kdebug_flags & KDBG_LOCKINIT))
349 return;
350
351 lck_mtx_lock(kd_trace_mtx_sysctl);
352
353 if (old_pid == global_state_pid)
354 global_state_pid = new_pid;
355
356 lck_mtx_unlock(kd_trace_mtx_sysctl);
357}
358
39236c6e
A		 359static uint32_t
360kdbg_cpu_count(boolean_t early_trace)
361{
362 if (early_trace) {
363 /*
364 * we've started tracing before the IOKit has even
365 * started running... just use the static max value
366 */
367 return max_ncpus;
368 }
369
370 host_basic_info_data_t hinfo;
371 mach_msg_type_number_t count = HOST_BASIC_INFO_COUNT;
372 host_info((host_t)1 /* BSD_HOST */, HOST_BASIC_INFO, (host_info_t)&hinfo, &count);
373 assert(hinfo.logical_cpu_max > 0);
374 return hinfo.logical_cpu_max;
375}
376
377#if MACH_ASSERT
39236c6e
A		 378#endif /* MACH_ASSERT */
379
380static void
381kdbg_iop_list_callback(kd_iop_t* iop, kd_callback_type type, void* arg)
382{
383 while (iop) {
384 iop->callback.func(iop->callback.context, type, arg);
385 iop = iop->next;
386 }
387}
388
6d2010ae 389static void
316670eb 390kdbg_set_tracing_enabled(boolean_t enabled, uint32_t trace_type)
1c79356b 391{
6d2010ae
A		 392 int s = ml_set_interrupts_enabled(FALSE);
393 lck_spin_lock(kds_spin_lock);
394
395 if (enabled) {
316670eb 396 kdebug_enable |= trace_type;
6d2010ae
A		 397 kd_ctrl_page.kdebug_slowcheck &= ~SLOW_NOLOG;
398 kd_ctrl_page.enabled = 1;
399 } else {
316670eb 400 kdebug_enable &= ~(KDEBUG_ENABLE_TRACE|KDEBUG_ENABLE_PPT);
6d2010ae
A		 401 kd_ctrl_page.kdebug_slowcheck |= SLOW_NOLOG;
402 kd_ctrl_page.enabled = 0;
403 }
404 lck_spin_unlock(kds_spin_lock);
405 ml_set_interrupts_enabled(s);
39236c6e
A		 406
407 if (enabled) {
408 kdbg_iop_list_callback(kd_ctrl_page.kdebug_iops, KD_CALLBACK_KDEBUG_ENABLED, NULL);
409 } else {
410 /*
411 * If you do not flush the IOP trace buffers, they can linger
412 * for a considerable period; consider code which disables and
413 * deallocates without a final sync flush.
414 */
415 kdbg_iop_list_callback(kd_ctrl_page.kdebug_iops, KD_CALLBACK_KDEBUG_DISABLED, NULL);
416 kdbg_iop_list_callback(kd_ctrl_page.kdebug_iops, KD_CALLBACK_SYNC_FLUSH, NULL);
417 }
1c79356b
A		 418}
419
6d2010ae
A		 420static void
421kdbg_set_flags(int slowflag, int enableflag, boolean_t enabled)
422{
423 int s = ml_set_interrupts_enabled(FALSE);
424 lck_spin_lock(kds_spin_lock);
425
426 if (enabled) {
427 kd_ctrl_page.kdebug_slowcheck |= slowflag;
428 kdebug_enable |= enableflag;
429 } else {
430 kd_ctrl_page.kdebug_slowcheck &= ~slowflag;
431 kdebug_enable &= ~enableflag;
432 }
39236c6e 433
6d2010ae
A		 434 lck_spin_unlock(kds_spin_lock);
435 ml_set_interrupts_enabled(s);
436}
437
6d2010ae
A		 438void
439disable_wrap(uint32_t *old_slowcheck, uint32_t *old_flags)
440{
441 int s = ml_set_interrupts_enabled(FALSE);
442 lck_spin_lock(kds_spin_lock);
443
444 *old_slowcheck = kd_ctrl_page.kdebug_slowcheck;
445 *old_flags = kd_ctrl_page.kdebug_flags;
446
447 kd_ctrl_page.kdebug_flags &= ~KDBG_WRAPPED;
448 kd_ctrl_page.kdebug_flags |= KDBG_NOWRAP;
449
450 lck_spin_unlock(kds_spin_lock);
451 ml_set_interrupts_enabled(s);
452}
453
454void
455enable_wrap(uint32_t old_slowcheck, boolean_t lostevents)
456{
457 int s = ml_set_interrupts_enabled(FALSE);
458 lck_spin_lock(kds_spin_lock);
459
460 kd_ctrl_page.kdebug_flags &= ~KDBG_NOWRAP;
461
462 if ( !(old_slowcheck & SLOW_NOLOG))
463 kd_ctrl_page.kdebug_slowcheck &= ~SLOW_NOLOG;
464
465 if (lostevents == TRUE)
466 kd_ctrl_page.kdebug_flags |= KDBG_WRAPPED;
467
468 lck_spin_unlock(kds_spin_lock);
469 ml_set_interrupts_enabled(s);
470}
471
0c530ab8 472static int
6d2010ae 473create_buffers(boolean_t early_trace)
0c530ab8 474{
b0d623f7
A		 475 int i;
476 int p_buffer_size;
477 int f_buffer_size;
478 int f_buffers;
479 int error = 0;
480
39236c6e
A		 481 /*
482 * For the duration of this allocation, trace code will only reference
483 * kdebug_iops. Any iops registered after this enabling will not be
484 * messaged until the buffers are reallocated.
485 *
486 * TLDR; Must read kd_iops once and only once!
487 */
488 kd_ctrl_page.kdebug_iops = kd_iops;
6d2010ae 489
39236c6e
A		 490
491 /*
492 * If the list is valid, it is sorted, newest -> oldest. Each iop entry
493 * has a cpu_id of "the older entry + 1", so the highest cpu_id will
494 * be the list head + 1.
495 */
6d2010ae 496
39236c6e 497 kd_ctrl_page.kdebug_cpus = kd_ctrl_page.kdebug_iops ? kd_ctrl_page.kdebug_iops->cpu_id + 1 : kdbg_cpu_count(early_trace);
6d2010ae 498
39236c6e
A		 499 if (kmem_alloc(kernel_map, (vm_offset_t *)&kdbip, sizeof(struct kd_bufinfo) * kd_ctrl_page.kdebug_cpus) != KERN_SUCCESS) {
500 error = ENOSPC;
501 goto out;
6d2010ae 502 }
6d2010ae 503
39236c6e
A		 504 if (nkdbufs < (kd_ctrl_page.kdebug_cpus * EVENTS_PER_STORAGE_UNIT * MIN_STORAGE_UNITS_PER_CPU))
505 n_storage_units = kd_ctrl_page.kdebug_cpus * MIN_STORAGE_UNITS_PER_CPU;
b0d623f7
A		 506 else
507 n_storage_units = nkdbufs / EVENTS_PER_STORAGE_UNIT;
0c530ab8 508
b0d623f7 509 nkdbufs = n_storage_units * EVENTS_PER_STORAGE_UNIT;
2d21ac55 510
b0d623f7
A		 511 f_buffers = n_storage_units / N_STORAGE_UNITS_PER_BUFFER;
512 n_storage_buffers = f_buffers;
0c530ab8 513
b0d623f7
A		 514 f_buffer_size = N_STORAGE_UNITS_PER_BUFFER * sizeof(struct kd_storage);
515 p_buffer_size = (n_storage_units % N_STORAGE_UNITS_PER_BUFFER) * sizeof(struct kd_storage);
516
517 if (p_buffer_size)
518 n_storage_buffers++;
519
520 kd_bufs = NULL;
0c530ab8
A		 521
522 if (kdcopybuf == 0) {
b0d623f7
A		 523 if (kmem_alloc(kernel_map, (vm_offset_t *)&kdcopybuf, (vm_size_t)KDCOPYBUF_SIZE) != KERN_SUCCESS) {
524 error = ENOSPC;
525 goto out;
526 }
0c530ab8 527 }
b0d623f7
A		 528 if (kmem_alloc(kernel_map, (vm_offset_t *)&kd_bufs, (vm_size_t)(n_storage_buffers * sizeof(struct kd_storage_buffers))) != KERN_SUCCESS) {
529 error = ENOSPC;
530 goto out;
0c530ab8 531 }
b0d623f7 532 bzero(kd_bufs, n_storage_buffers * sizeof(struct kd_storage_buffers));
0c530ab8 533
b0d623f7
A		 534 for (i = 0; i < f_buffers; i++) {
535 if (kmem_alloc(kernel_map, (vm_offset_t *)&kd_bufs[i].kdsb_addr, (vm_size_t)f_buffer_size) != KERN_SUCCESS) {
536 error = ENOSPC;
537 goto out;
538 }
6d2010ae
A		 539 bzero(kd_bufs[i].kdsb_addr, f_buffer_size);
540
b0d623f7 541 kd_bufs[i].kdsb_size = f_buffer_size;
0c530ab8 542 }
b0d623f7
A		 543 if (p_buffer_size) {
544 if (kmem_alloc(kernel_map, (vm_offset_t *)&kd_bufs[i].kdsb_addr, (vm_size_t)p_buffer_size) != KERN_SUCCESS) {
545 error = ENOSPC;
546 goto out;
547 }
6d2010ae
A		 548 bzero(kd_bufs[i].kdsb_addr, p_buffer_size);
549
b0d623f7
A		 550 kd_bufs[i].kdsb_size = p_buffer_size;
551 }
6d2010ae 552 n_storage_units = 0;
b0d623f7
A		 553
554 for (i = 0; i < n_storage_buffers; i++) {
555 struct kd_storage *kds;
556 int n_elements;
557 int n;
558
559 n_elements = kd_bufs[i].kdsb_size / sizeof(struct kd_storage);
560 kds = kd_bufs[i].kdsb_addr;
561
562 for (n = 0; n < n_elements; n++) {
6d2010ae
A		 563 kds[n].kds_next.buffer_index = kd_ctrl_page.kds_free_list.buffer_index;
564 kds[n].kds_next.offset = kd_ctrl_page.kds_free_list.offset;
b0d623f7 565
6d2010ae
A		 566 kd_ctrl_page.kds_free_list.buffer_index = i;
567 kd_ctrl_page.kds_free_list.offset = n;
b0d623f7 568 }
6d2010ae 569 n_storage_units += n_elements;
0c530ab8 570 }
6d2010ae 571
39236c6e 572 bzero((char *)kdbip, sizeof(struct kd_bufinfo) * kd_ctrl_page.kdebug_cpus);
b0d623f7 573
39236c6e 574 for (i = 0; i < (int)kd_ctrl_page.kdebug_cpus; i++) {
6d2010ae
A		 575 kdbip[i].kd_list_head.raw = KDS_PTR_NULL;
576 kdbip[i].kd_list_tail.raw = KDS_PTR_NULL;
577 kdbip[i].kd_lostevents = FALSE;
578 kdbip[i].num_bufs = 0;
579 }
39236c6e 580
6d2010ae
A		 581 kd_ctrl_page.kdebug_flags |= KDBG_BUFINIT;
582
583 kd_ctrl_page.kds_inuse_count = 0;
584 n_storage_threshold = n_storage_units / 2;
b0d623f7
A		 585out:
586 if (error)
587 delete_buffers();
0c530ab8 588
b0d623f7 589 return(error);
0c530ab8
A		 590}
591
0c530ab8
A		 592static void
593delete_buffers(void)
4452a7af 594{
39236c6e 595 int i;
b0d623f7
A		 596
597 if (kd_bufs) {
598 for (i = 0; i < n_storage_buffers; i++) {
6d2010ae 599 if (kd_bufs[i].kdsb_addr) {
b0d623f7 600 kmem_free(kernel_map, (vm_offset_t)kd_bufs[i].kdsb_addr, (vm_size_t)kd_bufs[i].kdsb_size);
6d2010ae 601 }
b0d623f7
A		 602 }
603 kmem_free(kernel_map, (vm_offset_t)kd_bufs, (vm_size_t)(n_storage_buffers * sizeof(struct kd_storage_buffers)));
0c530ab8 604
b0d623f7
A		 605 kd_bufs = NULL;
606 n_storage_buffers = 0;
0c530ab8
A		 607 }
608 if (kdcopybuf) {
609 kmem_free(kernel_map, (vm_offset_t)kdcopybuf, KDCOPYBUF_SIZE);
b0d623f7 610
0c530ab8
A		 611 kdcopybuf = NULL;
612 }
6d2010ae 613 kd_ctrl_page.kds_free_list.raw = KDS_PTR_NULL;
b0d623f7 614
6d2010ae 615 if (kdbip) {
39236c6e 616 kmem_free(kernel_map, (vm_offset_t)kdbip, sizeof(struct kd_bufinfo) * kd_ctrl_page.kdebug_cpus);
6d2010ae
A		 617
618 kdbip = NULL;
619 }
39236c6e
A		 620 kd_ctrl_page.kdebug_iops = NULL;
621 kd_ctrl_page.kdebug_cpus = 0;
6d2010ae 622 kd_ctrl_page.kdebug_flags &= ~KDBG_BUFINIT;
0c530ab8
A		 623}
624
6d2010ae
A		 625void
626release_storage_unit(int cpu, uint32_t kdsp_raw)
0c530ab8 627{
b0d623f7 628 int s = 0;
6d2010ae
A		 629 struct kd_storage *kdsp_actual;
630 struct kd_bufinfo *kdbp;
631 union kds_ptr kdsp;
632
633 kdsp.raw = kdsp_raw;
634
b0d623f7
A		 635 s = ml_set_interrupts_enabled(FALSE);
636 lck_spin_lock(kds_spin_lock);
637
6d2010ae
A		 638 kdbp = &kdbip[cpu];
639
640 if (kdsp.raw == kdbp->kd_list_head.raw) {
b0d623f7 641 /*
6d2010ae 642 * it's possible for the storage unit pointed to
b0d623f7 643 * by kdsp to have already been stolen... so
6d2010ae 644 * check to see if it's still the head of the list
b0d623f7
A		 645 * now that we're behind the lock that protects
646 * adding and removing from the queue...
647 * since we only ever release and steal units from
6d2010ae 648 * that position, if it's no longer the head
b0d623f7
A		 649 * we having nothing to do in this context
650 */
6d2010ae
A		 651 kdsp_actual = POINTER_FROM_KDS_PTR(kdsp);
652 kdbp->kd_list_head = kdsp_actual->kds_next;
39236c6e 653
6d2010ae
A		 654 kdsp_actual->kds_next = kd_ctrl_page.kds_free_list;
655 kd_ctrl_page.kds_free_list = kdsp;
656
657 kd_ctrl_page.kds_inuse_count--;
b0d623f7
A		 658 }
659 lck_spin_unlock(kds_spin_lock);
660 ml_set_interrupts_enabled(s);
661}
662
663
6d2010ae
A		 664boolean_t
665allocate_storage_unit(int cpu)
b0d623f7 666{
6d2010ae 667 union kds_ptr kdsp;
316670eb 668 struct kd_storage *kdsp_actual, *kdsp_next_actual;
6d2010ae 669 struct kd_bufinfo *kdbp, *kdbp_vict, *kdbp_try;
b0d623f7 670 uint64_t oldest_ts, ts;
6d2010ae
A		 671 boolean_t retval = TRUE;
672 int s = 0;
b0d623f7 673
6d2010ae 674 s = ml_set_interrupts_enabled(FALSE);
b0d623f7
A		 675 lck_spin_lock(kds_spin_lock);
676
6d2010ae
A		 677 kdbp = &kdbip[cpu];
678
679 /* If someone beat us to the allocate, return success */
680 if (kdbp->kd_list_tail.raw != KDS_PTR_NULL) {
681 kdsp_actual = POINTER_FROM_KDS_PTR(kdbp->kd_list_tail);
682
683 if (kdsp_actual->kds_bufindx < EVENTS_PER_STORAGE_UNIT)
684 goto out;
685 }
686
687 if ((kdsp = kd_ctrl_page.kds_free_list).raw != KDS_PTR_NULL) {
688 kdsp_actual = POINTER_FROM_KDS_PTR(kdsp);
689 kd_ctrl_page.kds_free_list = kdsp_actual->kds_next;
690
691 kd_ctrl_page.kds_inuse_count++;
692 } else {
693 if (kd_ctrl_page.kdebug_flags & KDBG_NOWRAP) {
694 kd_ctrl_page.kdebug_slowcheck |= SLOW_NOLOG;
695 kdbp->kd_lostevents = TRUE;
696 retval = FALSE;
b0d623f7
A		 697 goto out;
698 }
699 kdbp_vict = NULL;
700 oldest_ts = (uint64_t)-1;
701
39236c6e 702 for (kdbp_try = &kdbip[0]; kdbp_try < &kdbip[kd_ctrl_page.kdebug_cpus]; kdbp_try++) {
b0d623f7 703
6d2010ae 704 if (kdbp_try->kd_list_head.raw == KDS_PTR_NULL) {
b0d623f7
A		 705 /*
706 * no storage unit to steal
707 */
708 continue;
709 }
6d2010ae
A		 710
711 kdsp_actual = POINTER_FROM_KDS_PTR(kdbp_try->kd_list_head);
712
713 if (kdsp_actual->kds_bufcnt < EVENTS_PER_STORAGE_UNIT) {
b0d623f7
A		 714 /*
715 * make sure we don't steal the storage unit
6d2010ae
A		 716 * being actively recorded to... need to
717 * move on because we don't want an out-of-order
718 * set of events showing up later
b0d623f7
A		 719 */
720 continue;
721 }
6d2010ae 722 ts = kdbg_get_timestamp(&kdsp_actual->kds_records[0]);
b0d623f7
A		 723
724 if (ts < oldest_ts) {
725 /*
726 * when 'wrapping', we want to steal the
727 * storage unit that has the 'earliest' time
728 * associated with it (first event time)
729 */
730 oldest_ts = ts;
731 kdbp_vict = kdbp_try;
732 }
733 }
b0d623f7
A		 734 if (kdbp_vict == NULL) {
735 kdebug_enable = 0;
6d2010ae
A		 736 kd_ctrl_page.enabled = 0;
737 retval = FALSE;
738 goto out;
b0d623f7 739 }
b0d623f7 740 kdsp = kdbp_vict->kd_list_head;
6d2010ae 741 kdsp_actual = POINTER_FROM_KDS_PTR(kdsp);
6d2010ae 742 kdbp_vict->kd_list_head = kdsp_actual->kds_next;
b0d623f7 743
316670eb
A		 744 if (kdbp_vict->kd_list_head.raw != KDS_PTR_NULL) {
745 kdsp_next_actual = POINTER_FROM_KDS_PTR(kdbp_vict->kd_list_head);
746 kdsp_next_actual->kds_lostevents = TRUE;
747 } else
748 kdbp_vict->kd_lostevents = TRUE;
749
6d2010ae 750 kd_ctrl_page.kdebug_flags |= KDBG_WRAPPED;
b0d623f7 751 }
6d2010ae
A		 752 kdsp_actual->kds_timestamp = mach_absolute_time();
753 kdsp_actual->kds_next.raw = KDS_PTR_NULL;
754 kdsp_actual->kds_bufcnt = 0;
755 kdsp_actual->kds_readlast = 0;
756
757 kdsp_actual->kds_lostevents = kdbp->kd_lostevents;
758 kdbp->kd_lostevents = FALSE;
759 kdsp_actual->kds_bufindx = 0;
b0d623f7 760
6d2010ae 761 if (kdbp->kd_list_head.raw == KDS_PTR_NULL)
b0d623f7
A		 762 kdbp->kd_list_head = kdsp;
763 else
6d2010ae 764 POINTER_FROM_KDS_PTR(kdbp->kd_list_tail)->kds_next = kdsp;
b0d623f7
A		 765 kdbp->kd_list_tail = kdsp;
766out:
767 lck_spin_unlock(kds_spin_lock);
6d2010ae 768 ml_set_interrupts_enabled(s);
b0d623f7 769
6d2010ae 770 return (retval);
b0d623f7 771}
39236c6e
A		 772
773int
774kernel_debug_register_callback(kd_callback_t callback)
775{
776 kd_iop_t* iop;
777 if (kmem_alloc(kernel_map, (vm_offset_t *)&iop, sizeof(kd_iop_t)) == KERN_SUCCESS) {
778 memcpy(&iop->callback, &callback, sizeof(kd_callback_t));
779
780 /*
781 * <rdar://problem/13351477> Some IOP clients are not providing a name.
782 *
783 * Remove when fixed.
784 */
785 {
786 boolean_t is_valid_name = FALSE;
787 for (uint32_t length=0; length<sizeof(callback.iop_name); ++length) {
788 /* This is roughly isprintable(c) */
789 if (callback.iop_name[length] > 0x20 && callback.iop_name[length] < 0x7F)
790 continue;
791 if (callback.iop_name[length] == 0) {
792 if (length)
793 is_valid_name = TRUE;
794 break;
795 }
796 }
797
798 if (!is_valid_name) {
799 strlcpy(iop->callback.iop_name, "IOP-???", sizeof(iop->callback.iop_name));
800 }
801 }
802
803 iop->last_timestamp = 0;
804
805 do {
806 /*
807 * We use two pieces of state, the old list head
808 * pointer, and the value of old_list_head->cpu_id.
809 * If we read kd_iops more than once, it can change
810 * between reads.
811 *
812 * TLDR; Must not read kd_iops more than once per loop.
813 */
814 iop->next = kd_iops;
815 iop->cpu_id = iop->next ? (iop->next->cpu_id+1) : kdbg_cpu_count(FALSE);
816
817 /*
818 * Header says OSCompareAndSwapPtr has a memory barrier
819 */
820 } while (!OSCompareAndSwapPtr(iop->next, iop, (void* volatile*)&kd_iops));
821
822 return iop->cpu_id;
823 }
824
825 return 0;
826}
827
828void
829kernel_debug_enter(
830 uint32_t coreid,
831 uint32_t debugid,
832 uint64_t timestamp,
833 uintptr_t arg1,
834 uintptr_t arg2,
835 uintptr_t arg3,
836 uintptr_t arg4,
837 uintptr_t threadid
838 )
839{
840 uint32_t bindx;
841 kd_buf *kd;
842 struct kd_bufinfo *kdbp;
843 struct kd_storage *kdsp_actual;
844 union kds_ptr kds_raw;
845
846 if (kd_ctrl_page.kdebug_slowcheck) {
847
848 if ( (kd_ctrl_page.kdebug_slowcheck & SLOW_NOLOG) || !(kdebug_enable & (KDEBUG_ENABLE_TRACE|KDEBUG_ENABLE_PPT)))
849 goto out1;
850
851 if (kd_ctrl_page.kdebug_flags & KDBG_TYPEFILTER_CHECK) {
852 if (isset(type_filter_bitmap, EXTRACT_CSC(debugid)))
853 goto record_event;
854 goto out1;
855 }
856 else if (kd_ctrl_page.kdebug_flags & KDBG_RANGECHECK) {
857 if (debugid >= kdlog_beg && debugid <= kdlog_end)
858 goto record_event;
859 goto out1;
860 }
861 else if (kd_ctrl_page.kdebug_flags & KDBG_VALCHECK) {
862 if ((debugid & DBG_FUNC_MASK) != kdlog_value1 &&
863 (debugid & DBG_FUNC_MASK) != kdlog_value2 &&
864 (debugid & DBG_FUNC_MASK) != kdlog_value3 &&
865 (debugid & DBG_FUNC_MASK) != kdlog_value4)
866 goto out1;
867 }
868 }
869
870record_event:
39236c6e
A		 871
872 disable_preemption();
873
874 if (kd_ctrl_page.enabled == 0)
875 goto out;
876
877 kdbp = &kdbip[coreid];
878 timestamp &= KDBG_TIMESTAMP_MASK;
879
880retry_q:
881 kds_raw = kdbp->kd_list_tail;
882
883 if (kds_raw.raw != KDS_PTR_NULL) {
884 kdsp_actual = POINTER_FROM_KDS_PTR(kds_raw);
885 bindx = kdsp_actual->kds_bufindx;
886 } else
887 kdsp_actual = NULL;
888
889 if (kdsp_actual == NULL || bindx >= EVENTS_PER_STORAGE_UNIT) {
890 if (allocate_storage_unit(coreid) == FALSE) {
891 /*
892 * this can only happen if wrapping
893 * has been disabled
894 */
895 goto out;
896 }
897 goto retry_q;
898 }
899 if ( !OSCompareAndSwap(bindx, bindx + 1, &kdsp_actual->kds_bufindx))
900 goto retry_q;
901
902 // IOP entries can be allocated before xnu allocates and inits the buffer
903 if (timestamp < kdsp_actual->kds_timestamp)
904 kdsp_actual->kds_timestamp = timestamp;
905
906 kd = &kdsp_actual->kds_records[bindx];
907
908 kd->debugid = debugid;
909 kd->arg1 = arg1;
910 kd->arg2 = arg2;
911 kd->arg3 = arg3;
912 kd->arg4 = arg4;
913 kd->arg5 = threadid;
914
915 kdbg_set_timestamp_and_cpu(kd, timestamp, coreid);
916
917 OSAddAtomic(1, &kdsp_actual->kds_bufcnt);
918out:
919 enable_preemption();
920out1:
921 if ((kds_waiter && kd_ctrl_page.kds_inuse_count >= n_storage_threshold)) {
922 boolean_t need_kds_wakeup = FALSE;
923 int s;
924
925 /*
926 * try to take the lock here to synchronize with the
927 * waiter entering the blocked state... use the try
928 * mode to prevent deadlocks caused by re-entering this
929 * routine due to various trace points triggered in the
930 * lck_spin_sleep_xxxx routines used to actually enter
931 * our wait condition... no problem if we fail,
932 * there will be lots of additional events coming in that
933 * will eventually succeed in grabbing this lock
934 */
935 s = ml_set_interrupts_enabled(FALSE);
936
937 if (lck_spin_try_lock(kdw_spin_lock)) {
938
939 if (kds_waiter && kd_ctrl_page.kds_inuse_count >= n_storage_threshold) {
940 kds_waiter = 0;
941 need_kds_wakeup = TRUE;
942 }
943 lck_spin_unlock(kdw_spin_lock);
944
945 ml_set_interrupts_enabled(s);
946
947 if (need_kds_wakeup == TRUE)
948 wakeup(&kds_waiter);
949 }
950 }
951}
952
953
b0d623f7 954
6d2010ae
A		 955void
956kernel_debug_internal(
957 uint32_t debugid,
958 uintptr_t arg1,
959 uintptr_t arg2,
960 uintptr_t arg3,
961 uintptr_t arg4,
fe8ab488 962 uintptr_t arg5);
b0d623f7 963
6d2010ae 964__attribute__((always_inline)) void
b0d623f7
A		 965kernel_debug_internal(
966 uint32_t debugid,
967 uintptr_t arg1,
968 uintptr_t arg2,
969 uintptr_t arg3,
970 uintptr_t arg4,
fe8ab488 971 uintptr_t arg5)
b0d623f7
A		 972{
973 struct proc *curproc;
974 uint64_t now;
6d2010ae
A		 975 uint32_t bindx;
976 boolean_t s;
b0d623f7
A		 977 kd_buf *kd;
978 int cpu;
979 struct kd_bufinfo *kdbp;
6d2010ae 980 struct kd_storage *kdsp_actual;
316670eb 981 union kds_ptr kds_raw;
b0d623f7 982
316670eb 983
91447636 984
6d2010ae 985 if (kd_ctrl_page.kdebug_slowcheck) {
9bccf70c 986
6d2010ae
A		 987 if (kdebug_enable & KDEBUG_ENABLE_CHUD) {
988 kd_chudhook_fn chudhook;
b0d623f7 989 /*
6d2010ae
A		 990 * Mask interrupts to minimize the interval across
991 * which the driver providing the hook could be
992 * unloaded.
b0d623f7 993 */
6d2010ae
A		 994 s = ml_set_interrupts_enabled(FALSE);
995 chudhook = kdebug_chudhook;
996 if (chudhook)
997 chudhook(debugid, arg1, arg2, arg3, arg4, arg5);
998 ml_set_interrupts_enabled(s);
b0d623f7 999 }
316670eb 1000 if ( (kd_ctrl_page.kdebug_slowcheck & SLOW_NOLOG) || !(kdebug_enable & (KDEBUG_ENABLE_TRACE|KDEBUG_ENABLE_PPT)))
6d2010ae 1001 goto out1;
b0d623f7 1002
6d2010ae
A		 1003 if ( !ml_at_interrupt_context()) {
1004 if (kd_ctrl_page.kdebug_flags & KDBG_PIDCHECK) {
1005 /*
1006 * If kdebug flag is not set for current proc, return
1007 */
1008 curproc = current_proc();
1c79356b 1009
6d2010ae 1010 if ((curproc && !(curproc->p_kdebug)) &&
316670eb
A		 1011 ((debugid & 0xffff0000) != (MACHDBG_CODE(DBG_MACH_SCHED, 0) | DBG_FUNC_NONE)) &&
1012 (debugid >> 24 != DBG_TRACE))
6d2010ae
A		 1013 goto out1;
1014 }
1015 else if (kd_ctrl_page.kdebug_flags & KDBG_PIDEXCLUDE) {
1016 /*
1017 * If kdebug flag is set for current proc, return
1018 */
1019 curproc = current_proc();
b0d623f7 1020
6d2010ae 1021 if ((curproc && curproc->p_kdebug) &&
316670eb
A		 1022 ((debugid & 0xffff0000) != (MACHDBG_CODE(DBG_MACH_SCHED, 0) | DBG_FUNC_NONE)) &&
1023 (debugid >> 24 != DBG_TRACE))
6d2010ae
A		 1024 goto out1;
1025 }
1026 }
316670eb
A		 1027
1028 if (kd_ctrl_page.kdebug_flags & KDBG_TYPEFILTER_CHECK) {
1029 /* Always record trace system info */
1030 if (EXTRACT_CLASS(debugid) == DBG_TRACE)
1031 goto record_event;
1032
1033 if (isset(type_filter_bitmap, EXTRACT_CSC(debugid)))
1034 goto record_event;
1035 goto out1;
1036 }
1037 else if (kd_ctrl_page.kdebug_flags & KDBG_RANGECHECK) {
39236c6e
A		 1038 /* Always record trace system info */
1039 if (EXTRACT_CLASS(debugid) == DBG_TRACE)
316670eb 1040 goto record_event;
39236c6e
A		 1041
1042 if (debugid < kdlog_beg || debugid > kdlog_end)
1043 goto out1;
6d2010ae
A		 1044 }
1045 else if (kd_ctrl_page.kdebug_flags & KDBG_VALCHECK) {
39236c6e
A		 1046 /* Always record trace system info */
1047 if (EXTRACT_CLASS(debugid) == DBG_TRACE)
1048 goto record_event;
1049
6d2010ae 1050 if ((debugid & DBG_FUNC_MASK) != kdlog_value1 &&
316670eb
A		 1051 (debugid & DBG_FUNC_MASK) != kdlog_value2 &&
1052 (debugid & DBG_FUNC_MASK) != kdlog_value3 &&
39236c6e 1053 (debugid & DBG_FUNC_MASK) != kdlog_value4)
6d2010ae
A		 1054 goto out1;
1055 }
b0d623f7 1056 }
316670eb 1057record_event:
6d2010ae 1058 disable_preemption();
39236c6e
A		 1059
1060 if (kd_ctrl_page.enabled == 0)
1061 goto out;
1062
6d2010ae 1063 cpu = cpu_number();
b0d623f7 1064 kdbp = &kdbip[cpu];
6d2010ae 1065retry_q:
316670eb
A		 1066 kds_raw = kdbp->kd_list_tail;
1067
1068 if (kds_raw.raw != KDS_PTR_NULL) {
1069 kdsp_actual = POINTER_FROM_KDS_PTR(kds_raw);
6d2010ae
A		 1070 bindx = kdsp_actual->kds_bufindx;
1071 } else
1072 kdsp_actual = NULL;
1073
1074 if (kdsp_actual == NULL || bindx >= EVENTS_PER_STORAGE_UNIT) {
1075 if (allocate_storage_unit(cpu) == FALSE) {
b0d623f7
A		 1076 /*
1077 * this can only happen if wrapping
1078 * has been disabled
1079 */
1080 goto out;
1081 }
6d2010ae 1082 goto retry_q;
b0d623f7 1083 }
6d2010ae
A		 1084 now = mach_absolute_time() & KDBG_TIMESTAMP_MASK;
1085
1086 if ( !OSCompareAndSwap(bindx, bindx + 1, &kdsp_actual->kds_bufindx))
1087 goto retry_q;
1088
1089 kd = &kdsp_actual->kds_records[bindx];
b0d623f7 1090
1c79356b
A		 1091 kd->debugid = debugid;
1092 kd->arg1 = arg1;
1093 kd->arg2 = arg2;
1094 kd->arg3 = arg3;
1095 kd->arg4 = arg4;
0c530ab8 1096 kd->arg5 = arg5;
1c79356b 1097
b0d623f7 1098 kdbg_set_timestamp_and_cpu(kd, now, cpu);
1c79356b 1099
6d2010ae 1100 OSAddAtomic(1, &kdsp_actual->kds_bufcnt);
0c530ab8 1101out:
6d2010ae
A		 1102 enable_preemption();
1103out1:
fe8ab488 1104 if (kds_waiter && kd_ctrl_page.kds_inuse_count >= n_storage_threshold) {
6d2010ae
A		 1105 uint32_t etype;
1106 uint32_t stype;
1107
1108 etype = debugid & DBG_FUNC_MASK;
1109 stype = debugid & DBG_SCALL_MASK;
1110
1111 if (etype == INTERRUPT || etype == MACH_vmfault ||
1112 stype == BSC_SysCall || stype == MACH_SysCall) {
1113
1114 boolean_t need_kds_wakeup = FALSE;
6d2010ae
A		 1115
1116 /*
1117 * try to take the lock here to synchronize with the
1118 * waiter entering the blocked state... use the try
1119 * mode to prevent deadlocks caused by re-entering this
1120 * routine due to various trace points triggered in the
1121 * lck_spin_sleep_xxxx routines used to actually enter
1122 * one of our 2 wait conditions... no problem if we fail,
1123 * there will be lots of additional events coming in that
1124 * will eventually succeed in grabbing this lock
1125 */
1126 s = ml_set_interrupts_enabled(FALSE);
1127
1128 if (lck_spin_try_lock(kdw_spin_lock)) {
1129
1130 if (kds_waiter && kd_ctrl_page.kds_inuse_count >= n_storage_threshold) {
1131 kds_waiter = 0;
1132 need_kds_wakeup = TRUE;
1133 }
6d2010ae
A		 1134 lck_spin_unlock(kdw_spin_lock);
1135 }
1136 ml_set_interrupts_enabled(s);
1137
1138 if (need_kds_wakeup == TRUE)
1139 wakeup(&kds_waiter);
6d2010ae
A		 1140 }
1141 }
1c79356b
A		 1142}
1143
1144void
b0d623f7
A		 1145kernel_debug(
1146 uint32_t debugid,
1147 uintptr_t arg1,
1148 uintptr_t arg2,
1149 uintptr_t arg3,
1150 uintptr_t arg4,
1151 __unused uintptr_t arg5)
1c79356b 1152{
fe8ab488 1153 kernel_debug_internal(debugid, arg1, arg2, arg3, arg4, (uintptr_t)thread_tid(current_thread()));
0c530ab8 1154}
21362eb3 1155
0c530ab8 1156void
b0d623f7
A		 1157kernel_debug1(
1158 uint32_t debugid,
1159 uintptr_t arg1,
1160 uintptr_t arg2,
1161 uintptr_t arg3,
1162 uintptr_t arg4,
1163 uintptr_t arg5)
0c530ab8 1164{
fe8ab488
A		 1165 kernel_debug_internal(debugid, arg1, arg2, arg3, arg4, arg5);
1166}
1167
1168void
1169kernel_debug_string(const char *message)
1170{
1171 uintptr_t arg[4] = {0, 0, 0, 0};
1172
1173 /* Stuff the message string in the args and log it. */
1174 strncpy((char *)arg, message, MIN(sizeof(arg), strlen(message)));
1175 KERNEL_DEBUG_EARLY(
1176 (TRACEDBG_CODE(DBG_TRACE_INFO, 4)) | DBG_FUNC_NONE,
1177 arg[0], arg[1], arg[2], arg[3]);
1178}
1179
1180extern int master_cpu; /* MACH_KERNEL_PRIVATE */
1181/*
1182 * Used prior to start_kern_tracing() being called.
1183 * Log temporarily into a static buffer.
1184 */
1185void
1186kernel_debug_early(
1187 uint32_t debugid,
1188 uintptr_t arg1,
1189 uintptr_t arg2,
1190 uintptr_t arg3,
1191 uintptr_t arg4)
1192{
1193 /* If tracing is already initialized, use it */
1194 if (nkdbufs)
1195 KERNEL_DEBUG_CONSTANT(debugid, arg1, arg2, arg3, arg4, 0);
1196
1197 /* Do nothing if the buffer is full or we're not on the boot cpu */
1198 kd_early_overflow = kd_early_index >= KD_EARLY_BUFFER_MAX;
1199 if (kd_early_overflow ||
1200 cpu_number() != master_cpu)
1201 return;
1202
1203 kd_early_buffer[kd_early_index].debugid = debugid;
1204 kd_early_buffer[kd_early_index].timestamp = mach_absolute_time();
1205 kd_early_buffer[kd_early_index].arg1 = arg1;
1206 kd_early_buffer[kd_early_index].arg2 = arg2;
1207 kd_early_buffer[kd_early_index].arg3 = arg3;
1208 kd_early_buffer[kd_early_index].arg4 = arg4;
1209 kd_early_buffer[kd_early_index].arg5 = 0;
1210 kd_early_index++;
1211}
1212
1213/*
1214 * Transfer the contents of the temporary buffer into the trace buffers.
1215 * Precede that by logging the rebase time (offset) - the TSC-based time (in ns)
1216 * when mach_absolute_time is set to 0.
1217 */
1218static void
1219kernel_debug_early_end(void)
1220{
1221 int i;
1222
1223 if (cpu_number() != master_cpu)
1224 panic("kernel_debug_early_end() not call on boot processor");
1225
1226 /* Fake sentinel marking the start of kernel time relative to TSC */
1227 kernel_debug_enter(
1228 0,
1229 (TRACEDBG_CODE(DBG_TRACE_INFO, 1)) | DBG_FUNC_NONE,
1230 0,
1231 (uint32_t)(tsc_rebase_abs_time >> 32),
1232 (uint32_t)tsc_rebase_abs_time,
1233 0,
1234 0,
1235 0);
1236 for (i = 0; i < kd_early_index; i++) {
1237 kernel_debug_enter(
1238 0,
1239 kd_early_buffer[i].debugid,
1240 kd_early_buffer[i].timestamp,
1241 kd_early_buffer[i].arg1,
1242 kd_early_buffer[i].arg2,
1243 kd_early_buffer[i].arg3,
1244 kd_early_buffer[i].arg4,
1245 0);
1246 }
1247
1248 /* Cut events-lost event on overflow */
1249 if (kd_early_overflow)
1250 KERNEL_DEBUG_CONSTANT(
1251 TRACEDBG_CODE(DBG_TRACE_INFO, 2), 0, 0, 0, 0, 0);
1252
1253 /* This trace marks the start of kernel tracing */
1254 kernel_debug_string("early trace done");
0c530ab8 1255}
6601e61a 1256
6d2010ae
A		 1257/*
1258 * Support syscall SYS_kdebug_trace
1259 */
1260int
1261kdebug_trace(__unused struct proc *p, struct kdebug_trace_args *uap, __unused int32_t *retval)
0c530ab8 1262{
6d2010ae 1263 if ( __probable(kdebug_enable == 0) )
fe8ab488
A		 1264 return(0);
1265
1266 kernel_debug_internal(uap->code, uap->arg1, uap->arg2, uap->arg3, uap->arg4, (uintptr_t)thread_tid(current_thread()));
91447636 1267
6d2010ae
A		 1268 return(0);
1269}
1c79356b 1270
1c79356b 1271
6d2010ae
A		 1272static void
1273kdbg_lock_init(void)
1274{
1275 if (kd_ctrl_page.kdebug_flags & KDBG_LOCKINIT)
1276 return;
6d2010ae 1277
0c530ab8 1278 /*
91447636
A		 1279 * allocate lock group attribute and group
1280 */
0c530ab8
A		 1281 kd_trace_mtx_sysctl_grp_attr = lck_grp_attr_alloc_init();
1282 kd_trace_mtx_sysctl_grp = lck_grp_alloc_init("kdebug", kd_trace_mtx_sysctl_grp_attr);
91447636
A		 1283
1284 /*
1285 * allocate the lock attribute
1286 */
0c530ab8 1287 kd_trace_mtx_sysctl_attr = lck_attr_alloc_init();
91447636
A		 1288
1289
1290 /*
6d2010ae 1291 * allocate and initialize mutex's
91447636 1292 */
0c530ab8 1293 kd_trace_mtx_sysctl = lck_mtx_alloc_init(kd_trace_mtx_sysctl_grp, kd_trace_mtx_sysctl_attr);
b0d623f7 1294 kds_spin_lock = lck_spin_alloc_init(kd_trace_mtx_sysctl_grp, kd_trace_mtx_sysctl_attr);
6d2010ae 1295 kdw_spin_lock = lck_spin_alloc_init(kd_trace_mtx_sysctl_grp, kd_trace_mtx_sysctl_attr);
91447636 1296
6d2010ae 1297 kd_ctrl_page.kdebug_flags |= KDBG_LOCKINIT;
91447636
A		 1298}
1299
1300
1301int
6d2010ae 1302kdbg_bootstrap(boolean_t early_trace)
1c79356b 1303{
6d2010ae 1304 kd_ctrl_page.kdebug_flags &= ~KDBG_WRAPPED;
91447636 1305
6d2010ae 1306 return (create_buffers(early_trace));
1c79356b
A		 1307}
1308
0c530ab8 1309int
6d2010ae 1310kdbg_reinit(boolean_t early_trace)
1c79356b 1311{
b0d623f7 1312 int ret = 0;
91447636 1313
b0d623f7
A		 1314 /*
1315 * Disable trace collecting
1316 * First make sure we're not in
1317 * the middle of cutting a trace
1318 */
316670eb 1319 kdbg_set_tracing_enabled(FALSE, KDEBUG_ENABLE_TRACE);
1c79356b 1320
b0d623f7
A		 1321 /*
1322 * make sure the SLOW_NOLOG is seen
1323 * by everyone that might be trying
1324 * to cut a trace..
1325 */
1326 IOSleep(100);
1c79356b 1327
b0d623f7 1328 delete_buffers();
1c79356b 1329
6d2010ae 1330 if ((kd_ctrl_page.kdebug_flags & KDBG_MAPINIT) && kd_mapsize && kd_mapptr) {
b0d623f7 1331 kmem_free(kernel_map, (vm_offset_t)kd_mapptr, kd_mapsize);
6d2010ae 1332 kd_ctrl_page.kdebug_flags &= ~KDBG_MAPINIT;
b0d623f7
A		 1333 kd_mapsize = 0;
1334 kd_mapptr = (kd_threadmap *) 0;
1335 kd_mapcount = 0;
1336 }
6d2010ae
A		 1337 ret = kdbg_bootstrap(early_trace);
1338
1339 RAW_file_offset = 0;
1340 RAW_file_written = 0;
1c79356b 1341
b0d623f7 1342 return(ret);
1c79356b
A		 1343}
1344
0c530ab8
A		 1345void
1346kdbg_trace_data(struct proc *proc, long *arg_pid)
55e303ae 1347{
b0d623f7
A		 1348 if (!proc)
1349 *arg_pid = 0;
1350 else
1351 *arg_pid = proc->p_pid;
55e303ae
A		 1352}
1353
1354
0c530ab8
A		 1355void
1356kdbg_trace_string(struct proc *proc, long *arg1, long *arg2, long *arg3, long *arg4)
1c79356b 1357{
b0d623f7
A		 1358 char *dbg_nameptr;
1359 int dbg_namelen;
1360 long dbg_parms[4];
1361
1362 if (!proc) {
1363 *arg1 = 0;
1364 *arg2 = 0;
1365 *arg3 = 0;
1366 *arg4 = 0;
1367 return;
1368 }
1369 /*
1370 * Collect the pathname for tracing
1371 */
1372 dbg_nameptr = proc->p_comm;
1373 dbg_namelen = (int)strlen(proc->p_comm);
1374 dbg_parms[0]=0L;
1375 dbg_parms[1]=0L;
1376 dbg_parms[2]=0L;
1377 dbg_parms[3]=0L;
1c79356b 1378
b0d623f7
A		 1379 if(dbg_namelen > (int)sizeof(dbg_parms))
1380 dbg_namelen = (int)sizeof(dbg_parms);
1c79356b 1381
b0d623f7 1382 strncpy((char *)dbg_parms, dbg_nameptr, dbg_namelen);
1c79356b 1383
b0d623f7
A		 1384 *arg1=dbg_parms[0];
1385 *arg2=dbg_parms[1];
1386 *arg3=dbg_parms[2];
1387 *arg4=dbg_parms[3];
1c79356b
A		 1388}
1389
91447636 1390static void
0c530ab8 1391kdbg_resolve_map(thread_t th_act, void *opaque)
1c79356b 1392{
b0d623f7
A		 1393 kd_threadmap *mapptr;
1394 krt_t *t = (krt_t *)opaque;
1395
1396 if (t->count < t->maxcount) {
1397 mapptr = &t->map[t->count];
1398 mapptr->thread = (uintptr_t)thread_tid(th_act);
1399
1400 (void) strlcpy (mapptr->command, t->atts->task_comm,
1401 sizeof(t->atts->task_comm));
1402 /*
1403 * Some kernel threads have no associated pid.
1404 * We still need to mark the entry as valid.
1405 */
1406 if (t->atts->pid)
1407 mapptr->valid = t->atts->pid;
1408 else
1409 mapptr->valid = 1;
1410
1411 t->count++;
1412 }
1c79356b
A		 1413}
1414
39236c6e
A		 1415/*
1416 *
1417 * Writes a cpumap for the given iops_list/cpu_count to the provided buffer.
1418 *
1419 * You may provide a buffer and size, or if you set the buffer to NULL, a
1420 * buffer of sufficient size will be allocated.
1421 *
1422 * If you provide a buffer and it is too small, sets cpumap_size to the number
1423 * of bytes required and returns EINVAL.
1424 *
1425 * On success, if you provided a buffer, cpumap_size is set to the number of
1426 * bytes written. If you did not provide a buffer, cpumap is set to the newly
1427 * allocated buffer and cpumap_size is set to the number of bytes allocated.
1428 *
1429 * NOTE: It may seem redundant to pass both iops and a cpu_count.
1430 *
1431 * We may be reporting data from "now", or from the "past".
1432 *
1433 * The "now" data would be for something like kdbg_readcurcpumap().
1434 * The "past" data would be for kdbg_readcpumap().
1435 *
1436 * If we do not pass both iops and cpu_count, and iops is NULL, this function
1437 * will need to read "now" state to get the number of cpus, which would be in
1438 * error if we were reporting "past" state.
1439 */
1440
1441int
1442kdbg_cpumap_init_internal(kd_iop_t* iops, uint32_t cpu_count, uint8_t** cpumap, uint32_t* cpumap_size)
1443{
1444 assert(cpumap);
1445 assert(cpumap_size);
1446 assert(cpu_count);
1447 assert(!iops || iops->cpu_id + 1 == cpu_count);
1448
1449 uint32_t bytes_needed = sizeof(kd_cpumap_header) + cpu_count * sizeof(kd_cpumap);
1450 uint32_t bytes_available = *cpumap_size;
1451 *cpumap_size = bytes_needed;
1452
1453 if (*cpumap == NULL) {
1454 if (kmem_alloc(kernel_map, (vm_offset_t*)cpumap, (vm_size_t)*cpumap_size) != KERN_SUCCESS) {
1455 return ENOMEM;
1456 }
1457 } else if (bytes_available < bytes_needed) {
1458 return EINVAL;
1459 }
1460
1461 kd_cpumap_header* header = (kd_cpumap_header*)(uintptr_t)*cpumap;
1462
1463 header->version_no = RAW_VERSION1;
1464 header->cpu_count = cpu_count;
1465
1466 kd_cpumap* cpus = (kd_cpumap*)&header[1];
1467
1468 int32_t index = cpu_count - 1;
1469 while (iops) {
1470 cpus[index].cpu_id = iops->cpu_id;
1471 cpus[index].flags = KDBG_CPUMAP_IS_IOP;
1472 bzero(cpus[index].name, sizeof(cpus->name));
1473 strlcpy(cpus[index].name, iops->callback.iop_name, sizeof(cpus->name));
1474
1475 iops = iops->next;
1476 index--;
1477 }
1478
1479 while (index >= 0) {
1480 cpus[index].cpu_id = index;
1481 cpus[index].flags = 0;
1482 bzero(cpus[index].name, sizeof(cpus->name));
1483 strlcpy(cpus[index].name, "AP", sizeof(cpus->name));
1484
1485 index--;
1486 }
1487
1488 return KERN_SUCCESS;
1489}
1490
0c530ab8 1491void
39236c6e 1492kdbg_thrmap_init(void)
1c79356b 1493{
39236c6e
A		 1494 if (kd_ctrl_page.kdebug_flags & KDBG_MAPINIT)
1495 return;
1496
1497 kd_mapptr = kdbg_thrmap_init_internal(0, &kd_mapsize, &kd_mapcount);
1498
1499 if (kd_mapptr)
1500 kd_ctrl_page.kdebug_flags |= KDBG_MAPINIT;
1501}
1502
1503
1504kd_threadmap* kdbg_thrmap_init_internal(unsigned int count, unsigned int *mapsize, unsigned int *mapcount)
1505{
1506 kd_threadmap *mapptr;
b0d623f7
A		 1507 struct proc *p;
1508 struct krt akrt;
1509 int tts_count; /* number of task-to-string structures */
1510 struct tts *tts_mapptr;
1511 unsigned int tts_mapsize = 0;
b0d623f7 1512 int i;
39236c6e 1513 vm_offset_t kaddr;
1c79356b 1514
b0d623f7
A		 1515 /*
1516 * need to use PROC_SCANPROCLIST with proc_iterate
1517 */
2d21ac55
A		 1518 proc_list_lock();
1519
b0d623f7
A		 1520 /*
1521 * Calculate the sizes of map buffers
1522 */
39236c6e
A		 1523 for (p = allproc.lh_first, *mapcount=0, tts_count=0; p; p = p->p_list.le_next) {
1524 *mapcount += get_task_numacts((task_t)p->task);
b0d623f7
A		 1525 tts_count++;
1526 }
2d21ac55
A		 1527 proc_list_unlock();
1528
9bccf70c
A		 1529 /*
1530 * The proc count could change during buffer allocation,
1531 * so introduce a small fudge factor to bump up the
1532 * buffer sizes. This gives new tasks some chance of
39236c6e 1533 * making into the tables. Bump up by 25%.
9bccf70c 1534 */
39236c6e
A		 1535 *mapcount += *mapcount/4;
1536 tts_count += tts_count/4;
1537
1538 *mapsize = *mapcount * sizeof(kd_threadmap);
9bccf70c 1539
39236c6e
A		 1540 if (count && count < *mapcount)
1541 return (0);
b0d623f7 1542
39236c6e
A		 1543 if ((kmem_alloc(kernel_map, &kaddr, (vm_size_t)*mapsize) == KERN_SUCCESS)) {
1544 bzero((void *)kaddr, *mapsize);
1545 mapptr = (kd_threadmap *)kaddr;
b0d623f7 1546 } else
39236c6e 1547 return (0);
1c79356b 1548
9bccf70c 1549 tts_mapsize = tts_count * sizeof(struct tts);
9bccf70c 1550
39236c6e
A		 1551 if ((kmem_alloc(kernel_map, &kaddr, (vm_size_t)tts_mapsize) == KERN_SUCCESS)) {
1552 bzero((void *)kaddr, tts_mapsize);
1553 tts_mapptr = (struct tts *)kaddr;
1554 } else {
1555 kmem_free(kernel_map, (vm_offset_t)mapptr, *mapsize);
9bccf70c 1556
39236c6e
A		 1557 return (0);
1558 }
9bccf70c
A		 1559 /*
1560 * We need to save the procs command string
1561 * and take a reference for each task associated
1562 * with a valid process
1563 */
2d21ac55 1564
39236c6e
A		 1565 proc_list_lock();
1566
1567 /*
1568 * should use proc_iterate
1569 */
1570 for (p = allproc.lh_first, i=0; p && i < tts_count; p = p->p_list.le_next) {
1571 if (p->p_lflag & P_LEXIT)
1572 continue;
1573
1574 if (p->task) {
1575 task_reference(p->task);
1576 tts_mapptr[i].task = p->task;
1577 tts_mapptr[i].pid = p->p_pid;
1578 (void)strlcpy(tts_mapptr[i].task_comm, p->p_comm, sizeof(tts_mapptr[i].task_comm));
1579 i++;
1580 }
9bccf70c 1581 }
39236c6e 1582 tts_count = i;
9bccf70c 1583
39236c6e 1584 proc_list_unlock();
9bccf70c 1585
39236c6e
A		 1586 /*
1587 * Initialize thread map data
1588 */
1589 akrt.map = mapptr;
1590 akrt.count = 0;
1591 akrt.maxcount = *mapcount;
1c79356b 1592
39236c6e
A		 1593 for (i = 0; i < tts_count; i++) {
1594 akrt.atts = &tts_mapptr[i];
1595 task_act_iterate_wth_args(tts_mapptr[i].task, kdbg_resolve_map, &akrt);
1596 task_deallocate((task_t) tts_mapptr[i].task);
b0d623f7 1597 }
39236c6e
A		 1598 kmem_free(kernel_map, (vm_offset_t)tts_mapptr, tts_mapsize);
1599
1600 *mapcount = akrt.count;
1601
1602 return (mapptr);
1c79356b
A		 1603}
1604
91447636
A		 1605static void
1606kdbg_clear(void)
1c79356b 1607{
91447636
A		 1608 /*
1609 * Clean up the trace buffer
1610 * First make sure we're not in
1611 * the middle of cutting a trace
1612 */
316670eb 1613 kdbg_set_tracing_enabled(FALSE, KDEBUG_ENABLE_TRACE);
91447636 1614
0c530ab8
A		 1615 /*
1616 * make sure the SLOW_NOLOG is seen
1617 * by everyone that might be trying
1618 * to cut a trace..
1619 */
1620 IOSleep(100);
1621
91447636 1622 global_state_pid = -1;
6d2010ae
A		 1623 kd_ctrl_page.kdebug_flags &= (unsigned int)~KDBG_CKTYPES;
1624 kd_ctrl_page.kdebug_flags &= ~(KDBG_NOWRAP | KDBG_RANGECHECK | KDBG_VALCHECK);
1625 kd_ctrl_page.kdebug_flags &= ~(KDBG_PIDCHECK | KDBG_PIDEXCLUDE);
316670eb
A		 1626
1627 kdbg_disable_typefilter();
0c530ab8
A		 1628
1629 delete_buffers();
316670eb 1630 nkdbufs = 0;
1c79356b
A		 1631
1632 /* Clean up the thread map buffer */
6d2010ae 1633 kd_ctrl_page.kdebug_flags &= ~KDBG_MAPINIT;
b0d623f7
A		 1634 if (kd_mapptr) {
1635 kmem_free(kernel_map, (vm_offset_t)kd_mapptr, kd_mapsize);
1636 kd_mapptr = (kd_threadmap *) 0;
1637 }
1c79356b
A		 1638 kd_mapsize = 0;
1639 kd_mapcount = 0;
6d2010ae
A		 1640
1641 RAW_file_offset = 0;
1642 RAW_file_written = 0;
1c79356b
A		 1643}
1644
0c530ab8 1645int
1c79356b
A		 1646kdbg_setpid(kd_regtype *kdr)
1647{
b0d623f7
A		 1648 pid_t pid;
1649 int flag, ret=0;
1650 struct proc *p;
1651
1652 pid = (pid_t)kdr->value1;
1653 flag = (int)kdr->value2;
1654
1655 if (pid > 0) {
1656 if ((p = proc_find(pid)) == NULL)
1657 ret = ESRCH;
1658 else {
1659 if (flag == 1) {
1660 /*
1661 * turn on pid check for this and all pids
1662 */
6d2010ae
A		 1663 kd_ctrl_page.kdebug_flags |= KDBG_PIDCHECK;
1664 kd_ctrl_page.kdebug_flags &= ~KDBG_PIDEXCLUDE;
1665 kdbg_set_flags(SLOW_CHECKS, 0, TRUE);
1666
b0d623f7
A		 1667 p->p_kdebug = 1;
1668 } else {
1669 /*
1670 * turn off pid check for this pid value
1671 * Don't turn off all pid checking though
1672 *
6d2010ae 1673 * kd_ctrl_page.kdebug_flags &= ~KDBG_PIDCHECK;
b0d623f7
A		 1674 */
1675 p->p_kdebug = 0;
1676 }
1677 proc_rele(p);
1678 }
1c79356b 1679 }
b0d623f7
A		 1680 else
1681 ret = EINVAL;
1682
1683 return(ret);
1c79356b
A		 1684}
1685
1686/* This is for pid exclusion in the trace buffer */
0c530ab8 1687int
1c79356b
A		 1688kdbg_setpidex(kd_regtype *kdr)
1689{
b0d623f7
A		 1690 pid_t pid;
1691 int flag, ret=0;
1692 struct proc *p;
1693
1694 pid = (pid_t)kdr->value1;
1695 flag = (int)kdr->value2;
1696
1697 if (pid > 0) {
1698 if ((p = proc_find(pid)) == NULL)
1699 ret = ESRCH;
1700 else {
1701 if (flag == 1) {
1702 /*
1703 * turn on pid exclusion
1704 */
6d2010ae
A		 1705 kd_ctrl_page.kdebug_flags |= KDBG_PIDEXCLUDE;
1706 kd_ctrl_page.kdebug_flags &= ~KDBG_PIDCHECK;
1707 kdbg_set_flags(SLOW_CHECKS, 0, TRUE);
b0d623f7
A		 1708
1709 p->p_kdebug = 1;
1710 }
1711 else {
1712 /*
1713 * turn off pid exclusion for this pid value
1714 * Don't turn off all pid exclusion though
1715 *
6d2010ae 1716 * kd_ctrl_page.kdebug_flags &= ~KDBG_PIDEXCLUDE;
b0d623f7
A		 1717 */
1718 p->p_kdebug = 0;
1719 }
1720 proc_rele(p);
1721 }
1722 } else
1723 ret = EINVAL;
1724
1725 return(ret);
1c79356b
A		 1726}
1727
b0d623f7
A		 1728
1729/*
1730 * This is for setting a maximum decrementer value
1731 */
0c530ab8 1732int
1c79356b
A		 1733kdbg_setrtcdec(kd_regtype *kdr)
1734{
b0d623f7
A		 1735 int ret = 0;
1736 natural_t decval;
1c79356b 1737
b0d623f7 1738 decval = (natural_t)kdr->value1;
1c79356b 1739
b0d623f7
A		 1740 if (decval && decval < KDBG_MINRTCDEC)
1741 ret = EINVAL;
3a60a9f5
A		 1742 else
1743 ret = ENOTSUP;
1c79356b 1744
b0d623f7 1745 return(ret);
1c79356b
A		 1746}
1747
316670eb
A		 1748int
1749kdbg_enable_typefilter(void)
1750{
1751 if (kd_ctrl_page.kdebug_flags & KDBG_TYPEFILTER_CHECK) {
1752 /* free the old filter */
1753 kdbg_disable_typefilter();
1754 }
1755
1756 if (kmem_alloc(kernel_map, (vm_offset_t *)&type_filter_bitmap, KDBG_TYPEFILTER_BITMAP_SIZE) != KERN_SUCCESS) {
1757 return ENOSPC;
1758 }
1759
1760 bzero(type_filter_bitmap, KDBG_TYPEFILTER_BITMAP_SIZE);
1761
1762 /* Turn off range and value checks */
1763 kd_ctrl_page.kdebug_flags &= ~(KDBG_RANGECHECK | KDBG_VALCHECK);
1764
1765 /* Enable filter checking */
1766 kd_ctrl_page.kdebug_flags |= KDBG_TYPEFILTER_CHECK;
1767 kdbg_set_flags(SLOW_CHECKS, 0, TRUE);
1768 return 0;
1769}
1770
1771int
1772kdbg_disable_typefilter(void)
1773{
1774 /* Disable filter checking */
1775 kd_ctrl_page.kdebug_flags &= ~KDBG_TYPEFILTER_CHECK;
1776
1777 /* Turn off slow checks unless pid checks are using them */
1778 if ( (kd_ctrl_page.kdebug_flags & (KDBG_PIDCHECK | KDBG_PIDEXCLUDE)) )
1779 kdbg_set_flags(SLOW_CHECKS, 0, TRUE);
1780 else
1781 kdbg_set_flags(SLOW_CHECKS, 0, FALSE);
1782
1783 if(type_filter_bitmap == NULL)
1784 return 0;
1785
1786 vm_offset_t old_bitmap = (vm_offset_t)type_filter_bitmap;
1787 type_filter_bitmap = NULL;
1788
1789 kmem_free(kernel_map, old_bitmap, KDBG_TYPEFILTER_BITMAP_SIZE);
1790 return 0;
1791}
1792
0c530ab8 1793int
1c79356b
A		 1794kdbg_setreg(kd_regtype * kdr)
1795{
0c530ab8 1796 int ret=0;
1c79356b
A		 1797 unsigned int val_1, val_2, val;
1798 switch (kdr->type) {
1799
1800 case KDBG_CLASSTYPE :
1801 val_1 = (kdr->value1 & 0xff);
1802 val_2 = (kdr->value2 & 0xff);
1803 kdlog_beg = (val_1<<24);
1804 kdlog_end = (val_2<<24);
6d2010ae
A		 1805 kd_ctrl_page.kdebug_flags &= (unsigned int)~KDBG_CKTYPES;
1806 kd_ctrl_page.kdebug_flags &= ~KDBG_VALCHECK; /* Turn off specific value check */
1807 kd_ctrl_page.kdebug_flags |= (KDBG_RANGECHECK | KDBG_CLASSTYPE);
1808 kdbg_set_flags(SLOW_CHECKS, 0, TRUE);
1c79356b
A		 1809 break;
1810 case KDBG_SUBCLSTYPE :
1811 val_1 = (kdr->value1 & 0xff);
1812 val_2 = (kdr->value2 & 0xff);
1813 val = val_2 + 1;
1814 kdlog_beg = ((val_1<<24) | (val_2 << 16));
1815 kdlog_end = ((val_1<<24) | (val << 16));
6d2010ae
A		 1816 kd_ctrl_page.kdebug_flags &= (unsigned int)~KDBG_CKTYPES;
1817 kd_ctrl_page.kdebug_flags &= ~KDBG_VALCHECK; /* Turn off specific value check */
1818 kd_ctrl_page.kdebug_flags |= (KDBG_RANGECHECK | KDBG_SUBCLSTYPE);
1819 kdbg_set_flags(SLOW_CHECKS, 0, TRUE);
1c79356b
A		 1820 break;
1821 case KDBG_RANGETYPE :
1822 kdlog_beg = (kdr->value1);
1823 kdlog_end = (kdr->value2);
6d2010ae
A		 1824 kd_ctrl_page.kdebug_flags &= (unsigned int)~KDBG_CKTYPES;
1825 kd_ctrl_page.kdebug_flags &= ~KDBG_VALCHECK; /* Turn off specific value check */
1826 kd_ctrl_page.kdebug_flags |= (KDBG_RANGECHECK | KDBG_RANGETYPE);
1827 kdbg_set_flags(SLOW_CHECKS, 0, TRUE);
1c79356b
A		 1828 break;
1829 case KDBG_VALCHECK:
1830 kdlog_value1 = (kdr->value1);
1831 kdlog_value2 = (kdr->value2);
1832 kdlog_value3 = (kdr->value3);
1833 kdlog_value4 = (kdr->value4);
6d2010ae
A		 1834 kd_ctrl_page.kdebug_flags &= (unsigned int)~KDBG_CKTYPES;
1835 kd_ctrl_page.kdebug_flags &= ~KDBG_RANGECHECK; /* Turn off range check */
1836 kd_ctrl_page.kdebug_flags |= KDBG_VALCHECK; /* Turn on specific value check */
1837 kdbg_set_flags(SLOW_CHECKS, 0, TRUE);
1c79356b
A		 1838 break;
1839 case KDBG_TYPENONE :
6d2010ae 1840 kd_ctrl_page.kdebug_flags &= (unsigned int)~KDBG_CKTYPES;
91447636 1841
316670eb
A		 1842 if ( (kd_ctrl_page.kdebug_flags & (KDBG_RANGECHECK | KDBG_VALCHECK |
1843 KDBG_PIDCHECK | KDBG_PIDEXCLUDE |
1844 KDBG_TYPEFILTER_CHECK)) )
6d2010ae 1845 kdbg_set_flags(SLOW_CHECKS, 0, TRUE);
91447636 1846 else
6d2010ae 1847 kdbg_set_flags(SLOW_CHECKS, 0, FALSE);
91447636 1848
1c79356b
A		 1849 kdlog_beg = 0;
1850 kdlog_end = 0;
1851 break;
1852 default :
1853 ret = EINVAL;
1854 break;
1855 }
1856 return(ret);
1857}
1858
0c530ab8
A		 1859int
1860kdbg_getreg(__unused kd_regtype * kdr)
1c79356b 1861{
0c530ab8 1862#if 0
1c79356b
A		 1863 int i,j, ret=0;
1864 unsigned int val_1, val_2, val;
0c530ab8 1865
1c79356b
A		 1866 switch (kdr->type) {
1867 case KDBG_CLASSTYPE :
1868 val_1 = (kdr->value1 & 0xff);
1869 val_2 = val_1 + 1;
1870 kdlog_beg = (val_1<<24);
1871 kdlog_end = (val_2<<24);
6d2010ae
A		 1872 kd_ctrl_page.kdebug_flags &= (unsigned int)~KDBG_CKTYPES;
1873 kd_ctrl_page.kdebug_flags |= (KDBG_RANGECHECK | KDBG_CLASSTYPE);
1c79356b
A		 1874 break;
1875 case KDBG_SUBCLSTYPE :
1876 val_1 = (kdr->value1 & 0xff);
1877 val_2 = (kdr->value2 & 0xff);
1878 val = val_2 + 1;
1879 kdlog_beg = ((val_1<<24) | (val_2 << 16));
1880 kdlog_end = ((val_1<<24) | (val << 16));
6d2010ae
A		 1881 kd_ctrl_page.kdebug_flags &= (unsigned int)~KDBG_CKTYPES;
1882 kd_ctrl_page.kdebug_flags |= (KDBG_RANGECHECK | KDBG_SUBCLSTYPE);
1c79356b
A		 1883 break;
1884 case KDBG_RANGETYPE :
1885 kdlog_beg = (kdr->value1);
1886 kdlog_end = (kdr->value2);
6d2010ae
A		 1887 kd_ctrl_page.kdebug_flags &= (unsigned int)~KDBG_CKTYPES;
1888 kd_ctrl_page.kdebug_flags |= (KDBG_RANGECHECK | KDBG_RANGETYPE);
1c79356b
A		 1889 break;
1890 case KDBG_TYPENONE :
6d2010ae 1891 kd_ctrl_page.kdebug_flags &= (unsigned int)~KDBG_CKTYPES;
1c79356b
A		 1892 kdlog_beg = 0;
1893 kdlog_end = 0;
1894 break;
1895 default :
1896 ret = EINVAL;
1897 break;
1898 }
1899#endif /* 0 */
1900 return(EINVAL);
1901}
1902
39236c6e
A		 1903int
1904kdbg_readcpumap(user_addr_t user_cpumap, size_t *user_cpumap_size)
1905{
1906 uint8_t* cpumap = NULL;
1907 uint32_t cpumap_size = 0;
1908 int ret = KERN_SUCCESS;
1909
1910 if (kd_ctrl_page.kdebug_flags & KDBG_BUFINIT) {
1911 if (kdbg_cpumap_init_internal(kd_ctrl_page.kdebug_iops, kd_ctrl_page.kdebug_cpus, &cpumap, &cpumap_size) == KERN_SUCCESS) {
1912 if (user_cpumap) {
1913 size_t bytes_to_copy = (*user_cpumap_size >= cpumap_size) ? cpumap_size : *user_cpumap_size;
1914 if (copyout(cpumap, user_cpumap, (size_t)bytes_to_copy)) {
1915 ret = EFAULT;
1916 }
1917 }
1918 *user_cpumap_size = cpumap_size;
1919 kmem_free(kernel_map, (vm_offset_t)cpumap, cpumap_size);
1920 } else
1921 ret = EINVAL;
1922 } else
1923 ret = EINVAL;
1924
1925 return (ret);
1926}
1927
1928int
1929kdbg_readcurthrmap(user_addr_t buffer, size_t *bufsize)
1930{
1931 kd_threadmap *mapptr;
1932 unsigned int mapsize;
1933 unsigned int mapcount;
1934 unsigned int count = 0;
1935 int ret = 0;
1936
1937 count = *bufsize/sizeof(kd_threadmap);
1938 *bufsize = 0;
1939
1940 if ( (mapptr = kdbg_thrmap_init_internal(count, &mapsize, &mapcount)) ) {
1941 if (copyout(mapptr, buffer, mapcount * sizeof(kd_threadmap)))
1942 ret = EFAULT;
1943 else
1944 *bufsize = (mapcount * sizeof(kd_threadmap));
1945
1946 kmem_free(kernel_map, (vm_offset_t)mapptr, mapsize);
1947 } else
1948 ret = EINVAL;
1949
1950 return (ret);
1951}
1c79356b 1952
91447636 1953int
39236c6e 1954kdbg_readthrmap(user_addr_t buffer, size_t *number, vnode_t vp, vfs_context_t ctx)
1c79356b 1955{
b0d623f7
A		 1956 int avail = *number;
1957 int ret = 0;
1958 uint32_t count = 0;
39236c6e 1959 unsigned int mapsize;
1c79356b 1960
b0d623f7 1961 count = avail/sizeof (kd_threadmap);
1c79356b 1962
39236c6e
A		 1963 mapsize = kd_mapcount * sizeof(kd_threadmap);
1964
b0d623f7 1965 if (count && (count <= kd_mapcount))
1c79356b 1966 {
6d2010ae 1967 if ((kd_ctrl_page.kdebug_flags & KDBG_MAPINIT) && kd_mapsize && kd_mapptr)
b0d623f7 1968 {
39236c6e 1969 if (*number < mapsize)
b0d623f7
A		 1970 ret = EINVAL;
1971 else
1972 {
6d2010ae
A		 1973 if (vp)
1974 {
1975 RAW_header header;
1976 clock_sec_t secs;
1977 clock_usec_t usecs;
1978 char *pad_buf;
39236c6e
A		 1979 uint32_t pad_size;
1980 uint32_t extra_thread_count = 0;
1981 uint32_t cpumap_size;
1982
1983 /*
1984 * To write a RAW_VERSION1+ file, we
1985 * must embed a cpumap in the "padding"
1986 * used to page align the events folloing
1987 * the threadmap. If the threadmap happens
1988 * to not require enough padding, we
1989 * artificially increase its footprint
1990 * until it needs enough padding.
1991 */
1992
1993 pad_size = PAGE_SIZE - ((sizeof(RAW_header) + (count * sizeof(kd_threadmap))) & PAGE_MASK_64);
1994 cpumap_size = sizeof(kd_cpumap_header) + kd_ctrl_page.kdebug_cpus * sizeof(kd_cpumap);
1995
1996 if (cpumap_size > pad_size) {
1997 /* Force an overflow onto the next page, we get a full page of padding */
1998 extra_thread_count = (pad_size / sizeof(kd_threadmap)) + 1;
1999 }
6d2010ae
A		 2000
2001 header.version_no = RAW_VERSION1;
39236c6e 2002 header.thread_count = count + extra_thread_count;
6d2010ae
A		 2003
2004 clock_get_calendar_microtime(&secs, &usecs);
2005 header.TOD_secs = secs;
2006 header.TOD_usecs = usecs;
39236c6e 2007
6d2010ae
A		 2008 ret = vn_rdwr(UIO_WRITE, vp, (caddr_t)&header, sizeof(RAW_header), RAW_file_offset,
2009 UIO_SYSSPACE, IO_NODELOCKED|IO_UNIT, vfs_context_ucred(ctx), (int *) 0, vfs_context_proc(ctx));
2010 if (ret)
2011 goto write_error;
2012 RAW_file_offset += sizeof(RAW_header);
2013
39236c6e 2014 ret = vn_rdwr(UIO_WRITE, vp, (caddr_t)kd_mapptr, mapsize, RAW_file_offset,
6d2010ae
A		 2015 UIO_SYSSPACE, IO_NODELOCKED|IO_UNIT, vfs_context_ucred(ctx), (int *) 0, vfs_context_proc(ctx));
2016 if (ret)
2017 goto write_error;
39236c6e 2018 RAW_file_offset += mapsize;
b0d623f7 2019
39236c6e
A		 2020 if (extra_thread_count) {
2021 pad_size = extra_thread_count * sizeof(kd_threadmap);
2022 pad_buf = (char *)kalloc(pad_size);
2023 memset(pad_buf, 0, pad_size);
2024
2025 ret = vn_rdwr(UIO_WRITE, vp, (caddr_t)pad_buf, pad_size, RAW_file_offset,
2026 UIO_SYSSPACE, IO_NODELOCKED|IO_UNIT, vfs_context_ucred(ctx), (int *) 0, vfs_context_proc(ctx));
2027 kfree(pad_buf, pad_size);
2028
2029 if (ret)
2030 goto write_error;
2031 RAW_file_offset += pad_size;
6d2010ae 2032
39236c6e
A		 2033 }
2034
2035 pad_size = PAGE_SIZE - (RAW_file_offset & PAGE_MASK_64);
2036 if (pad_size) {
6d2010ae
A		 2037 pad_buf = (char *)kalloc(pad_size);
2038 memset(pad_buf, 0, pad_size);
2039
39236c6e
A		 2040 /*
2041 * embed a cpumap in the padding bytes.
2042 * older code will skip this.
2043 * newer code will know how to read it.
2044 */
2045 uint32_t temp = pad_size;
2046 if (kdbg_cpumap_init_internal(kd_ctrl_page.kdebug_iops, kd_ctrl_page.kdebug_cpus, (uint8_t**)&pad_buf, &temp) != KERN_SUCCESS) {
2047 memset(pad_buf, 0, pad_size);
2048 }
2049
6d2010ae 2050 ret = vn_rdwr(UIO_WRITE, vp, (caddr_t)pad_buf, pad_size, RAW_file_offset,
39236c6e 2051 UIO_SYSSPACE, IO_NODELOCKED|IO_UNIT, vfs_context_ucred(ctx), (int *) 0, vfs_context_proc(ctx));
6d2010ae 2052 kfree(pad_buf, pad_size);
39236c6e 2053
6d2010ae
A		 2054 if (ret)
2055 goto write_error;
2056 RAW_file_offset += pad_size;
2057 }
39236c6e 2058 RAW_file_written += sizeof(RAW_header) + mapsize + pad_size;
6d2010ae 2059
b0d623f7 2060 } else {
39236c6e 2061 if (copyout(kd_mapptr, buffer, mapsize))
b0d623f7
A		 2062 ret = EINVAL;
2063 }
2064 }
2065 }
2066 else
2067 ret = EINVAL;
2068 }
2069 else
2070 ret = EINVAL;
2071
6d2010ae
A		 2072 if (ret && vp)
2073 {
b0d623f7
A		 2074 count = 0;
2075
2076 vn_rdwr(UIO_WRITE, vp, (caddr_t)&count, sizeof(uint32_t), RAW_file_offset,
2077 UIO_SYSSPACE, IO_NODELOCKED|IO_UNIT, vfs_context_ucred(ctx), (int *) 0, vfs_context_proc(ctx));
2078 RAW_file_offset += sizeof(uint32_t);
6d2010ae 2079 RAW_file_written += sizeof(uint32_t);
1c79356b 2080 }
6d2010ae
A		 2081write_error:
2082 if ((kd_ctrl_page.kdebug_flags & KDBG_MAPINIT) && kd_mapsize && kd_mapptr)
b0d623f7
A		 2083 {
2084 kmem_free(kernel_map, (vm_offset_t)kd_mapptr, kd_mapsize);
6d2010ae 2085 kd_ctrl_page.kdebug_flags &= ~KDBG_MAPINIT;
b0d623f7
A		 2086 kd_mapsize = 0;
2087 kd_mapptr = (kd_threadmap *) 0;
2088 kd_mapcount = 0;
2089 }
b0d623f7 2090 return(ret);
1c79356b
A		 2091}
2092
9bccf70c 2093
316670eb 2094static int
2d21ac55
A		 2095kdbg_set_nkdbufs(unsigned int value)
2096{
2097 /*
b0d623f7 2098 * We allow a maximum buffer size of 50% of either ram or max mapped address, whichever is smaller
2d21ac55
A		 2099 * 'value' is the desired number of trace entries
2100 */
b0d623f7 2101 unsigned int max_entries = (sane_size/2) / sizeof(kd_buf);
2d21ac55
A		 2102
2103 if (value <= max_entries)
316670eb 2104 return (value);
2d21ac55 2105 else
316670eb 2106 return (max_entries);
2d21ac55
A		 2107}
2108
2109
39236c6e 2110static int
316670eb
A		 2111kdbg_enable_bg_trace(void)
2112{
39236c6e
A		 2113 int ret = 0;
2114
316670eb
A		 2115 if (kdlog_bg_trace == TRUE && kdlog_bg_trace_running == FALSE && n_storage_buffers == 0) {
2116 nkdbufs = bg_nkdbufs;
39236c6e
A		 2117 ret = kdbg_reinit(FALSE);
2118 if (0 == ret) {
2119 kdbg_set_tracing_enabled(TRUE, KDEBUG_ENABLE_TRACE);
2120 kdlog_bg_trace_running = TRUE;
2121 }
316670eb 2122 }
39236c6e 2123 return ret;
316670eb
A		 2124}
2125
2126static void
2127kdbg_disable_bg_trace(void)
2128{
2129 if (kdlog_bg_trace_running == TRUE) {
2130 kdlog_bg_trace_running = FALSE;
2131 kdbg_clear();
2132 }
2133}
2134
2135
2136
9bccf70c
A		 2137/*
2138 * This function is provided for the CHUD toolkit only.
2139 * int val:
2140 * zero disables kdebug_chudhook function call
2141 * non-zero enables kdebug_chudhook function call
2142 * char *fn:
2143 * address of the enabled kdebug_chudhook function
2144*/
2145
0c530ab8
A		 2146void
2147kdbg_control_chud(int val, void *fn)
9bccf70c 2148{
6d2010ae
A		 2149 kdbg_lock_init();
2150
2151 if (val) {
2152 /* enable chudhook */
9bccf70c 2153 kdebug_chudhook = fn;
6d2010ae 2154 kdbg_set_flags(SLOW_CHUD, KDEBUG_ENABLE_CHUD, TRUE);
9bccf70c
A		 2155 }
2156 else {
6d2010ae
A		 2157 /* disable chudhook */
2158 kdbg_set_flags(SLOW_CHUD, KDEBUG_ENABLE_CHUD, FALSE);
9bccf70c
A		 2159 kdebug_chudhook = 0;
2160 }
2161}
1c79356b 2162
9bccf70c 2163
0c530ab8 2164int
c910b4d9 2165kdbg_control(int *name, u_int namelen, user_addr_t where, size_t *sizep)
1c79356b 2166{
b0d623f7
A		 2167 int ret = 0;
2168 size_t size = *sizep;
c910b4d9 2169 unsigned int value = 0;
91447636
A		 2170 kd_regtype kd_Reg;
2171 kbufinfo_t kd_bufinfo;
2172 pid_t curpid;
6d2010ae 2173 proc_t p, curproc;
91447636 2174
c910b4d9 2175 if (name[0] == KERN_KDGETENTROPY ||
39236c6e
A		 2176 name[0] == KERN_KDWRITETR ||
2177 name[0] == KERN_KDWRITEMAP ||
c910b4d9
A		 2178 name[0] == KERN_KDEFLAGS ||
2179 name[0] == KERN_KDDFLAGS ||
2180 name[0] == KERN_KDENABLE ||
39236c6e 2181 name[0] == KERN_KDENABLE_BG_TRACE ||
c910b4d9
A		 2182 name[0] == KERN_KDSETBUF) {
2183
2184 if ( namelen < 2 )
6d2010ae 2185 return(EINVAL);
c910b4d9
A		 2186 value = name[1];
2187 }
2188
91447636 2189 kdbg_lock_init();
0c530ab8 2190
6d2010ae 2191 if ( !(kd_ctrl_page.kdebug_flags & KDBG_LOCKINIT))
39236c6e 2192 return(ENOSPC);
0c530ab8
A		 2193
2194 lck_mtx_lock(kd_trace_mtx_sysctl);
91447636 2195
316670eb 2196 switch(name[0]) {
39236c6e 2197 case KERN_KDGETBUF:
b0d623f7 2198 /*
39236c6e
A		 2199 * Does not alter the global_state_pid
2200 * This is a passive request.
b0d623f7 2201 */
39236c6e
A		 2202 if (size < sizeof(kd_bufinfo.nkdbufs)) {
2203 /*
2204 * There is not enough room to return even
2205 * the first element of the info structure.
2206 */
b0d623f7 2207 ret = EINVAL;
39236c6e
A		 2208 goto out;
2209 }
2210 kd_bufinfo.nkdbufs = nkdbufs;
2211 kd_bufinfo.nkdthreads = kd_mapcount;
2212
2213 if ( (kd_ctrl_page.kdebug_slowcheck & SLOW_NOLOG) )
2214 kd_bufinfo.nolog = 1;
2215 else
2216 kd_bufinfo.nolog = 0;
2217
2218 kd_bufinfo.flags = kd_ctrl_page.kdebug_flags;
2219#if defined(__LP64__)
2220 kd_bufinfo.flags |= KDBG_LP64;
2221#endif
2222 kd_bufinfo.bufid = global_state_pid;
2223
2224 if (size >= sizeof(kd_bufinfo)) {
2225 /*
2226 * Provide all the info we have
2227 */
2228 if (copyout(&kd_bufinfo, where, sizeof(kd_bufinfo)))
2229 ret = EINVAL;
2230 } else {
2231 /*
2232 * For backwards compatibility, only provide
2233 * as much info as there is room for.
2234 */
2235 if (copyout(&kd_bufinfo, where, size))
2236 ret = EINVAL;
2237 }
2238 goto out;
39236c6e 2239
fe8ab488
A		 2240 case KERN_KDGETENTROPY: {
2241 /* Obsolescent - just fake with a random buffer */
2242 char *buffer = (char *) kalloc(size);
2243 read_frandom((void *) buffer, size);
2244 ret = copyout(buffer, where, size);
2245 kfree(buffer, size);
39236c6e 2246 goto out;
fe8ab488 2247 }
39236c6e
A		 2248
2249 case KERN_KDENABLE_BG_TRACE:
2250 bg_nkdbufs = kdbg_set_nkdbufs(value);
2251 kdlog_bg_trace = TRUE;
2252 ret = kdbg_enable_bg_trace();
2253 goto out;
39236c6e
A		 2254
2255 case KERN_KDDISABLE_BG_TRACE:
2256 kdlog_bg_trace = FALSE;
2257 kdbg_disable_bg_trace();
2258 goto out;
91447636
A		 2259 }
2260
0c530ab8 2261 if ((curproc = current_proc()) != NULL)
b0d623f7 2262 curpid = curproc->p_pid;
91447636 2263 else {
b0d623f7
A		 2264 ret = ESRCH;
2265 goto out;
91447636 2266 }
39236c6e 2267 if (global_state_pid == -1)
b0d623f7 2268 global_state_pid = curpid;
91447636 2269 else if (global_state_pid != curpid) {
b0d623f7
A		 2270 if ((p = proc_find(global_state_pid)) == NULL) {
2271 /*
2272 * The global pid no longer exists
2273 */
2274 global_state_pid = curpid;
2275 } else {
2276 /*
2277 * The global pid exists, deny this request
2278 */
2279 proc_rele(p);
91447636 2280
b0d623f7
A		 2281 ret = EBUSY;
2282 goto out;
2283 }
91447636 2284 }
1c79356b
A		 2285
2286 switch(name[0]) {
2287 case KERN_KDEFLAGS:
316670eb
A		 2288 kdbg_disable_bg_trace();
2289
1c79356b 2290 value &= KDBG_USERFLAGS;
6d2010ae 2291 kd_ctrl_page.kdebug_flags |= value;
1c79356b
A		 2292 break;
2293 case KERN_KDDFLAGS:
316670eb
A		 2294 kdbg_disable_bg_trace();
2295
1c79356b 2296 value &= KDBG_USERFLAGS;
6d2010ae 2297 kd_ctrl_page.kdebug_flags &= ~value;
1c79356b 2298 break;
b0d623f7
A		 2299 case KERN_KDENABLE:
2300 /*
316670eb
A		 2301 * Enable tracing mechanism. Two types:
2302 * KDEBUG_TRACE is the standard one,
2303 * and KDEBUG_PPT which is a carefully
2304 * chosen subset to avoid performance impact.
b0d623f7
A		 2305 */
2306 if (value) {
2307 /*
2308 * enable only if buffer is initialized
2309 */
316670eb
A		 2310 if (!(kd_ctrl_page.kdebug_flags & KDBG_BUFINIT) ||
2311 !(value == KDEBUG_ENABLE_TRACE || value == KDEBUG_ENABLE_PPT)) {
b0d623f7
A		 2312 ret = EINVAL;
2313 break;
2314 }
39236c6e 2315 kdbg_thrmap_init();
b0d623f7 2316
316670eb 2317 kdbg_set_tracing_enabled(TRUE, value);
1c79356b 2318 }
6d2010ae 2319 else
316670eb
A		 2320 {
2321 kdbg_set_tracing_enabled(FALSE, 0);
2322 }
b0d623f7 2323 break;
1c79356b 2324 case KERN_KDSETBUF:
316670eb
A		 2325 kdbg_disable_bg_trace();
2326
2327 nkdbufs = kdbg_set_nkdbufs(value);
1c79356b 2328 break;
1c79356b 2329 case KERN_KDSETUP:
316670eb
A		 2330 kdbg_disable_bg_trace();
2331
6d2010ae 2332 ret = kdbg_reinit(FALSE);
1c79356b
A		 2333 break;
2334 case KERN_KDREMOVE:
2335 kdbg_clear();
39236c6e 2336 ret = kdbg_enable_bg_trace();
1c79356b
A		 2337 break;
2338 case KERN_KDSETREG:
2339 if(size < sizeof(kd_regtype)) {
b0d623f7 2340 ret = EINVAL;
1c79356b
A		 2341 break;
2342 }
2343 if (copyin(where, &kd_Reg, sizeof(kd_regtype))) {
b0d623f7 2344 ret = EINVAL;
1c79356b
A		 2345 break;
2346 }
316670eb
A		 2347 kdbg_disable_bg_trace();
2348
1c79356b
A		 2349 ret = kdbg_setreg(&kd_Reg);
2350 break;
2351 case KERN_KDGETREG:
b0d623f7 2352 if (size < sizeof(kd_regtype)) {
1c79356b
A		 2353 ret = EINVAL;
2354 break;
2355 }
2356 ret = kdbg_getreg(&kd_Reg);
39236c6e 2357 if (copyout(&kd_Reg, where, sizeof(kd_regtype))) {
b0d623f7 2358 ret = EINVAL;
1c79356b 2359 }
316670eb
A		 2360 kdbg_disable_bg_trace();
2361
1c79356b
A		 2362 break;
2363 case KERN_KDREADTR:
b0d623f7 2364 ret = kdbg_read(where, sizep, NULL, NULL);
1c79356b 2365 break;
39236c6e
A		 2366 case KERN_KDWRITETR:
2367 case KERN_KDWRITEMAP:
6d2010ae
A		 2368 {
2369 struct vfs_context context;
2370 struct fileproc *fp;
2371 size_t number;
2372 vnode_t vp;
2373 int fd;
2374
316670eb
A		 2375 kdbg_disable_bg_trace();
2376
6d2010ae
A		 2377 if (name[0] == KERN_KDWRITETR) {
2378 int s;
2379 int wait_result = THREAD_AWAKENED;
2380 u_int64_t abstime;
2381 u_int64_t ns;
2382
2383 if (*sizep) {
2384 ns = ((u_int64_t)*sizep) * (u_int64_t)(1000 * 1000);
2385 nanoseconds_to_absolutetime(ns, &abstime );
2386 clock_absolutetime_interval_to_deadline( abstime, &abstime );
2387 } else
2388 abstime = 0;
2389
2390 s = ml_set_interrupts_enabled(FALSE);
2391 lck_spin_lock(kdw_spin_lock);
2392
2393 while (wait_result == THREAD_AWAKENED && kd_ctrl_page.kds_inuse_count < n_storage_threshold) {
2394
2395 kds_waiter = 1;
2396
2397 if (abstime)
2398 wait_result = lck_spin_sleep_deadline(kdw_spin_lock, 0, &kds_waiter, THREAD_ABORTSAFE, abstime);
2399 else
2400 wait_result = lck_spin_sleep(kdw_spin_lock, 0, &kds_waiter, THREAD_ABORTSAFE);
2401
2402 kds_waiter = 0;
2403 }
2404 lck_spin_unlock(kdw_spin_lock);
2405 ml_set_interrupts_enabled(s);
2406 }
2407 p = current_proc();
2408 fd = value;
2409
2410 proc_fdlock(p);
2411 if ( (ret = fp_lookup(p, fd, &fp, 1)) ) {
2412 proc_fdunlock(p);
2413 break;
2414 }
2415 context.vc_thread = current_thread();
2416 context.vc_ucred = fp->f_fglob->fg_cred;
2417
39236c6e 2418 if (FILEGLOB_DTYPE(fp->f_fglob) != DTYPE_VNODE) {
6d2010ae
A		 2419 fp_drop(p, fd, fp, 1);
2420 proc_fdunlock(p);
2421
2422 ret = EBADF;
2423 break;
2424 }
2425 vp = (struct vnode *)fp->f_fglob->fg_data;
2426 proc_fdunlock(p);
2427
2428 if ((ret = vnode_getwithref(vp)) == 0) {
fe8ab488 2429 RAW_file_offset = fp->f_fglob->fg_offset;
6d2010ae
A		 2430 if (name[0] == KERN_KDWRITETR) {
2431 number = nkdbufs * sizeof(kd_buf);
2432
2433 KERNEL_DEBUG_CONSTANT((TRACEDBG_CODE(DBG_TRACE_INFO, 3)) | DBG_FUNC_START, 0, 0, 0, 0, 0);
2434 ret = kdbg_read(0, &number, vp, &context);
2435 KERNEL_DEBUG_CONSTANT((TRACEDBG_CODE(DBG_TRACE_INFO, 3)) | DBG_FUNC_END, number, 0, 0, 0, 0);
2436
2437 *sizep = number;
2438 } else {
39236c6e
A		 2439 number = kd_mapcount * sizeof(kd_threadmap);
2440 kdbg_readthrmap(0, &number, vp, &context);
6d2010ae 2441 }
fe8ab488 2442 fp->f_fglob->fg_offset = RAW_file_offset;
6d2010ae
A		 2443 vnode_put(vp);
2444 }
2445 fp_drop(p, fd, fp, 0);
2446
2447 break;
2448 }
39236c6e
A		 2449 case KERN_KDBUFWAIT:
2450 {
2451 /* WRITETR lite -- just block until there's data */
2452 int s;
2453 int wait_result = THREAD_AWAKENED;
2454 u_int64_t abstime;
2455 u_int64_t ns;
2456 size_t number = 0;
2457
2458 kdbg_disable_bg_trace();
2459
2460
2461 if (*sizep) {
2462 ns = ((u_int64_t)*sizep) * (u_int64_t)(1000 * 1000);
2463 nanoseconds_to_absolutetime(ns, &abstime );
2464 clock_absolutetime_interval_to_deadline( abstime, &abstime );
2465 } else
2466 abstime = 0;
2467
2468 s = ml_set_interrupts_enabled(FALSE);
2469 if( !s )
2470 panic("trying to wait with interrupts off");
2471 lck_spin_lock(kdw_spin_lock);
2472
2473 /* drop the mutex so don't exclude others from
2474 * accessing trace
2475 */
2476 lck_mtx_unlock(kd_trace_mtx_sysctl);
2477
2478 while (wait_result == THREAD_AWAKENED &&
2479 kd_ctrl_page.kds_inuse_count < n_storage_threshold) {
2480
2481 kds_waiter = 1;
2482
2483 if (abstime)
2484 wait_result = lck_spin_sleep_deadline(kdw_spin_lock, 0, &kds_waiter, THREAD_ABORTSAFE, abstime);
2485 else
2486 wait_result = lck_spin_sleep(kdw_spin_lock, 0, &kds_waiter, THREAD_ABORTSAFE);
2487
2488 kds_waiter = 0;
2489 }
2490
2491 /* check the count under the spinlock */
2492 number = (kd_ctrl_page.kds_inuse_count >= n_storage_threshold);
2493
2494 lck_spin_unlock(kdw_spin_lock);
2495 ml_set_interrupts_enabled(s);
2496
2497 /* pick the mutex back up again */
2498 lck_mtx_lock(kd_trace_mtx_sysctl);
2499
2500 /* write out whether we've exceeded the threshold */
2501 *sizep = number;
2502 break;
2503 }
1c79356b
A		 2504 case KERN_KDPIDTR:
2505 if (size < sizeof(kd_regtype)) {
2506 ret = EINVAL;
2507 break;
2508 }
2509 if (copyin(where, &kd_Reg, sizeof(kd_regtype))) {
b0d623f7 2510 ret = EINVAL;
1c79356b
A		 2511 break;
2512 }
316670eb
A		 2513 kdbg_disable_bg_trace();
2514
1c79356b
A		 2515 ret = kdbg_setpid(&kd_Reg);
2516 break;
2517 case KERN_KDPIDEX:
2518 if (size < sizeof(kd_regtype)) {
2519 ret = EINVAL;
2520 break;
2521 }
2522 if (copyin(where, &kd_Reg, sizeof(kd_regtype))) {
b0d623f7 2523 ret = EINVAL;
1c79356b
A		 2524 break;
2525 }
316670eb
A		 2526 kdbg_disable_bg_trace();
2527
1c79356b
A		 2528 ret = kdbg_setpidex(&kd_Reg);
2529 break;
39236c6e
A		 2530 case KERN_KDCPUMAP:
2531 ret = kdbg_readcpumap(where, sizep);
2532 break;
2533 case KERN_KDTHRMAP:
2534 ret = kdbg_readthrmap(where, sizep, NULL, NULL);
2535 break;
2536 case KERN_KDREADCURTHRMAP:
2537 ret = kdbg_readcurthrmap(where, sizep);
2538 break;
2539 case KERN_KDSETRTCDEC:
1c79356b
A		 2540 if (size < sizeof(kd_regtype)) {
2541 ret = EINVAL;
2542 break;
2543 }
2544 if (copyin(where, &kd_Reg, sizeof(kd_regtype))) {
b0d623f7 2545 ret = EINVAL;
1c79356b
A		 2546 break;
2547 }
316670eb
A		 2548 kdbg_disable_bg_trace();
2549
1c79356b
A		 2550 ret = kdbg_setrtcdec(&kd_Reg);
2551 break;
316670eb
A		 2552 case KERN_KDSET_TYPEFILTER:
2553 kdbg_disable_bg_trace();
2554
2555 if ((kd_ctrl_page.kdebug_flags & KDBG_TYPEFILTER_CHECK) == 0){
2556 if ((ret = kdbg_enable_typefilter()))
2557 break;
2558 }
2559
2560 if (size != KDBG_TYPEFILTER_BITMAP_SIZE) {
2561 ret = EINVAL;
2562 break;
2563 }
2564
2565 if (copyin(where, type_filter_bitmap, KDBG_TYPEFILTER_BITMAP_SIZE)) {
2566 ret = EINVAL;
2567 break;
2568 }
39236c6e 2569 kdbg_iop_list_callback(kd_ctrl_page.kdebug_iops, KD_CALLBACK_TYPEFILTER_CHANGED, type_filter_bitmap);
316670eb 2570 break;
1c79356b 2571 default:
b0d623f7 2572 ret = EINVAL;
1c79356b 2573 }
b0d623f7 2574out:
0c530ab8 2575 lck_mtx_unlock(kd_trace_mtx_sysctl);
91447636 2576
1c79356b
A		 2577 return(ret);
2578}
2579
0c530ab8
A		 2580
2581/*
b0d623f7
A		 2582 * This code can run for the most part concurrently with kernel_debug_internal()...
2583 * 'release_storage_unit' will take the kds_spin_lock which may cause us to briefly
2584 * synchronize with the recording side of this puzzle... otherwise, we are able to
2585 * move through the lists w/o use of any locks
0c530ab8
A		 2586 */
2587int
b0d623f7 2588kdbg_read(user_addr_t buffer, size_t *number, vnode_t vp, vfs_context_t ctx)
1c79356b 2589{
0c530ab8 2590 unsigned int count;
6d2010ae 2591 unsigned int cpu, min_cpu;
39236c6e 2592 uint64_t mintime, t, barrier = 0;
6d2010ae 2593 int error = 0;
0c530ab8 2594 kd_buf *tempbuf;
6d2010ae
A		 2595 uint32_t rcursor;
2596 kd_buf lostevent;
2597 union kds_ptr kdsp;
2598 struct kd_storage *kdsp_actual;
b0d623f7 2599 struct kd_bufinfo *kdbp;
6d2010ae 2600 struct kd_bufinfo *min_kdbp;
0c530ab8
A		 2601 uint32_t tempbuf_count;
2602 uint32_t tempbuf_number;
b0d623f7
A		 2603 uint32_t old_kdebug_flags;
2604 uint32_t old_kdebug_slowcheck;
6d2010ae
A		 2605 boolean_t lostevents = FALSE;
2606 boolean_t out_of_events = FALSE;
2d21ac55 2607
0c530ab8
A		 2608 count = *number/sizeof(kd_buf);
2609 *number = 0;
2610
6d2010ae 2611 if (count == 0 || !(kd_ctrl_page.kdebug_flags & KDBG_BUFINIT) || kdcopybuf == 0)
0c530ab8 2612 return EINVAL;
1c79356b 2613
6d2010ae
A		 2614 memset(&lostevent, 0, sizeof(lostevent));
2615 lostevent.debugid = TRACEDBG_CODE(DBG_TRACE_INFO, 2);
2616
39236c6e
A		 2617 /* Capture timestamp. Only sort events that have occured before the timestamp.
2618 * Since the iop is being flushed here, its possible that events occur on the AP
2619 * while running live tracing. If we are disabled, no new events should
2620 * occur on the AP.
2621 */
2622
2623 if (kd_ctrl_page.enabled)
2624 {
2625 // timestamp is non-zero value
2626 barrier = mach_absolute_time() & KDBG_TIMESTAMP_MASK;
2627 }
2628
2629 // Request each IOP to provide us with up to date entries before merging buffers together.
2630 kdbg_iop_list_callback(kd_ctrl_page.kdebug_iops, KD_CALLBACK_SYNC_FLUSH, NULL);
2631
0c530ab8
A		 2632 /*
2633 * because we hold kd_trace_mtx_sysctl, no other control threads can
2634 * be playing with kdebug_flags... the code that cuts new events could
b0d623f7
A		 2635 * be running, but it grabs kds_spin_lock if it needs to acquire a new
2636 * storage chunk which is where it examines kdebug_flags... it its adding
2637 * to the same chunk we're reading from, no problem...
0c530ab8 2638 */
0c530ab8 2639
6d2010ae 2640 disable_wrap(&old_kdebug_slowcheck, &old_kdebug_flags);
4452a7af 2641
0c530ab8
A		 2642 if (count > nkdbufs)
2643 count = nkdbufs;
4452a7af 2644
0c530ab8
A		 2645 if ((tempbuf_count = count) > KDCOPYBUF_COUNT)
2646 tempbuf_count = KDCOPYBUF_COUNT;
4452a7af 2647
0c530ab8 2648 while (count) {
39236c6e 2649 tempbuf = kdcopybuf;
0c530ab8
A		 2650 tempbuf_number = 0;
2651
316670eb 2652 // While space
39236c6e 2653 while (tempbuf_count) {
6d2010ae
A		 2654 mintime = 0xffffffffffffffffULL;
2655 min_kdbp = NULL;
2656 min_cpu = 0;
0c530ab8 2657
316670eb 2658 // Check all CPUs
39236c6e 2659 for (cpu = 0, kdbp = &kdbip[0]; cpu < kd_ctrl_page.kdebug_cpus; cpu++, kdbp++) {
b0d623f7 2660
316670eb 2661 // Find one with raw data
6d2010ae 2662 if ((kdsp = kdbp->kd_list_head).raw == KDS_PTR_NULL)
0c530ab8 2663 continue;
39236c6e
A		 2664 /* Debugging aid: maintain a copy of the "kdsp"
2665 * index.
2666 */
2667 volatile union kds_ptr kdsp_shadow;
2668
2669 kdsp_shadow = kdsp;
316670eb
A		 2670
2671 // Get from cpu data to buffer header to buffer
6d2010ae
A		 2672 kdsp_actual = POINTER_FROM_KDS_PTR(kdsp);
2673
39236c6e
A		 2674 volatile struct kd_storage *kdsp_actual_shadow;
2675
2676 kdsp_actual_shadow = kdsp_actual;
2677
316670eb 2678 // See if there are actual data left in this buffer
6d2010ae 2679 rcursor = kdsp_actual->kds_readlast;
b0d623f7 2680
6d2010ae 2681 if (rcursor == kdsp_actual->kds_bufindx)
b0d623f7 2682 continue;
0c530ab8 2683
6d2010ae
A		 2684 t = kdbg_get_timestamp(&kdsp_actual->kds_records[rcursor]);
2685
39236c6e
A		 2686 if ((t > barrier) && (barrier > 0)) {
2687 /*
2688 * Need to wait to flush iop again before we
2689 * sort any more data from the buffers
2690 */
2691 out_of_events = TRUE;
2692 break;
2693 }
6d2010ae
A		 2694 if (t < kdsp_actual->kds_timestamp) {
2695 /*
2696 * indicates we've not yet completed filling
2697 * in this event...
2698 * this should only occur when we're looking
2699 * at the buf that the record head is utilizing
2700 * we'll pick these events up on the next
2701 * call to kdbg_read
2702 * we bail at this point so that we don't
2703 * get an out-of-order timestream by continuing
2704 * to read events from the other CPUs' timestream(s)
2705 */
2706 out_of_events = TRUE;
2707 break;
2708 }
0c530ab8 2709 if (t < mintime) {
b0d623f7 2710 mintime = t;
6d2010ae
A		 2711 min_kdbp = kdbp;
2712 min_cpu = cpu;
91447636
A		 2713 }
2714 }
6d2010ae
A		 2715 if (min_kdbp == NULL || out_of_events == TRUE) {
2716 /*
b0d623f7 2717 * all buffers ran empty
91447636 2718 */
6d2010ae
A		 2719 out_of_events = TRUE;
2720 break;
2721 }
316670eb
A		 2722
2723 // Get data
6d2010ae
A		 2724 kdsp = min_kdbp->kd_list_head;
2725 kdsp_actual = POINTER_FROM_KDS_PTR(kdsp);
0c530ab8 2726
6d2010ae 2727 if (kdsp_actual->kds_lostevents == TRUE) {
316670eb 2728 kdbg_set_timestamp_and_cpu(&lostevent, kdsp_actual->kds_records[kdsp_actual->kds_readlast].timestamp, min_cpu);
6d2010ae
A		 2729 *tempbuf = lostevent;
2730
2731 kdsp_actual->kds_lostevents = FALSE;
2732 lostevents = TRUE;
2d21ac55 2733
6d2010ae 2734 goto nextevent;
2d21ac55 2735 }
316670eb
A		 2736
2737 // Copy into buffer
6d2010ae
A		 2738 *tempbuf = kdsp_actual->kds_records[kdsp_actual->kds_readlast++];
2739
2740 if (kdsp_actual->kds_readlast == EVENTS_PER_STORAGE_UNIT)
2741 release_storage_unit(min_cpu, kdsp.raw);
2742
b0d623f7
A		 2743 /*
2744 * Watch for out of order timestamps
2745 */
6d2010ae 2746 if (mintime < min_kdbp->kd_prev_timebase) {
b0d623f7
A		 2747 /*
2748 * if so, use the previous timestamp + 1 cycle
2749 */
6d2010ae
A		 2750 min_kdbp->kd_prev_timebase++;
2751 kdbg_set_timestamp_and_cpu(tempbuf, min_kdbp->kd_prev_timebase, kdbg_get_cpu(tempbuf));
b0d623f7 2752 } else
6d2010ae
A		 2753 min_kdbp->kd_prev_timebase = mintime;
2754nextevent:
0c530ab8
A		 2755 tempbuf_count--;
2756 tempbuf_number++;
b0d623f7 2757 tempbuf++;
6d2010ae
A		 2758
2759 if ((RAW_file_written += sizeof(kd_buf)) >= RAW_FLUSH_SIZE)
2760 break;
0c530ab8
A		 2761 }
2762 if (tempbuf_number) {
b0d623f7
A		 2763
2764 if (vp) {
2765 error = vn_rdwr(UIO_WRITE, vp, (caddr_t)kdcopybuf, tempbuf_number * sizeof(kd_buf), RAW_file_offset,
2766 UIO_SYSSPACE, IO_NODELOCKED|IO_UNIT, vfs_context_ucred(ctx), (int *) 0, vfs_context_proc(ctx));
2767
2768 RAW_file_offset += (tempbuf_number * sizeof(kd_buf));
6d2010ae
A		 2769
2770 if (RAW_file_written >= RAW_FLUSH_SIZE) {
2771 cluster_push(vp, 0);
2772
2773 RAW_file_written = 0;
2774 }
b0d623f7
A		 2775 } else {
2776 error = copyout(kdcopybuf, buffer, tempbuf_number * sizeof(kd_buf));
2777 buffer += (tempbuf_number * sizeof(kd_buf));
2778 }
2779 if (error) {
2780 *number = 0;
0c530ab8
A		 2781 error = EINVAL;
2782 break;
6601e61a 2783 }
0c530ab8
A		 2784 count -= tempbuf_number;
2785 *number += tempbuf_number;
0c530ab8 2786 }
6d2010ae 2787 if (out_of_events == TRUE)
0c530ab8
A		 2788 /*
2789 * all trace buffers are empty
2790 */
2791 break;
89b3af67 2792
0c530ab8
A		 2793 if ((tempbuf_count = count) > KDCOPYBUF_COUNT)
2794 tempbuf_count = KDCOPYBUF_COUNT;
2795 }
2796 if ( !(old_kdebug_flags & KDBG_NOWRAP)) {
6d2010ae 2797 enable_wrap(old_kdebug_slowcheck, lostevents);
0c530ab8
A		 2798 }
2799 return (error);
6601e61a 2800}
4452a7af 2801
0c530ab8 2802
55e303ae
A		 2803unsigned char *getProcName(struct proc *proc);
2804unsigned char *getProcName(struct proc *proc) {
2805
2806 return (unsigned char *) &proc->p_comm; /* Return pointer to the proc name */
2807
2808}
0c530ab8
A		 2809
2810#define STACKSHOT_SUBSYS_LOCK() lck_mtx_lock(&stackshot_subsys_mutex)
2811#define STACKSHOT_SUBSYS_UNLOCK() lck_mtx_unlock(&stackshot_subsys_mutex)
b0d623f7 2812#if defined(__i386__) || defined (__x86_64__)
0c530ab8 2813#define TRAP_DEBUGGER __asm__ volatile("int3");
39236c6e
A		 2814#else
2815#error No TRAP_DEBUGGER definition for this architecture
0c530ab8 2816#endif
0c530ab8 2817
d41d1dae 2818#define SANE_TRACEBUF_SIZE (8 * 1024 * 1024)
39236c6e 2819#define SANE_BOOTPROFILE_TRACEBUF_SIZE (64 * 1024 * 1024)
0c530ab8
A		 2820
2821/* Initialize the mutex governing access to the stack snapshot subsystem */
2822__private_extern__ void
2823stackshot_lock_init( void )
2824{
2825 stackshot_subsys_lck_grp_attr = lck_grp_attr_alloc_init();
2826
2827 stackshot_subsys_lck_grp = lck_grp_alloc_init("stackshot_subsys_lock", stackshot_subsys_lck_grp_attr);
2828
2829 stackshot_subsys_lck_attr = lck_attr_alloc_init();
2830
2831 lck_mtx_init(&stackshot_subsys_mutex, stackshot_subsys_lck_grp, stackshot_subsys_lck_attr);
2832}
2833
2834/*
2835 * stack_snapshot: Obtains a coherent set of stack traces for all threads
2836 * on the system, tracing both kernel and user stacks
2837 * where available. Uses machine specific trace routines
2838 * for ppc, ppc64 and x86.
2839 * Inputs: uap->pid - process id of process to be traced, or -1
2840 * for the entire system
2841 * uap->tracebuf - address of the user space destination
2842 * buffer
2843 * uap->tracebuf_size - size of the user space trace buffer
2844 * uap->options - various options, including the maximum
2845 * number of frames to trace.
2846 * Outputs: EPERM if the caller is not privileged
2847 * EINVAL if the supplied trace buffer isn't sanely sized
2848 * ENOMEM if we don't have enough memory to satisfy the
2849 * request
2850 * ENOENT if the target pid isn't found
2851 * ENOSPC if the supplied buffer is insufficient
2852 * *retval contains the number of bytes traced, if successful
2853 * and -1 otherwise. If the request failed due to
2854 * tracebuffer exhaustion, we copyout as much as possible.
2855 */
2856int
b0d623f7 2857stack_snapshot(struct proc *p, register struct stack_snapshot_args *uap, int32_t *retval) {
0c530ab8
A		 2858 int error = 0;
2859
2860 if ((error = suser(kauth_cred_get(), &p->p_acflag)))
2861 return(error);
2862
2863 return stack_snapshot2(uap->pid, uap->tracebuf, uap->tracebuf_size,
b7266188 2864 uap->flags, uap->dispatch_offset, retval);
0c530ab8
A		 2865}
2866
39236c6e
A		 2867int
2868stack_snapshot_from_kernel(pid_t pid, void *buf, uint32_t size, uint32_t flags, unsigned *bytesTraced)
2869{
2870 int error = 0;
2871 boolean_t istate;
2872
2873 if ((buf == NULL) || (size <= 0) || (bytesTraced == NULL)) {
2874 return -1;
2875 }
2876
2877 /* cap in individual stackshot to SANE_TRACEBUF_SIZE */
2878 if (size > SANE_TRACEBUF_SIZE) {
2879 size = SANE_TRACEBUF_SIZE;
2880 }
2881
2882/* Serialize tracing */
2883 STACKSHOT_SUBSYS_LOCK();
2884 istate = ml_set_interrupts_enabled(FALSE);
2885
2886
2887/* Preload trace parameters*/
2888 kdp_snapshot_preflight(pid, buf, size, flags, 0);
2889
2890/* Trap to the debugger to obtain a coherent stack snapshot; this populates
2891 * the trace buffer
2892 */
2893 TRAP_DEBUGGER;
2894
2895 ml_set_interrupts_enabled(istate);
2896
2897 *bytesTraced = kdp_stack_snapshot_bytes_traced();
2898
2899 error = kdp_stack_snapshot_geterror();
2900
2901 STACKSHOT_SUBSYS_UNLOCK();
2902
2903 return error;
2904
2905}
2906
0c530ab8 2907int
b7266188 2908stack_snapshot2(pid_t pid, user_addr_t tracebuf, uint32_t tracebuf_size, uint32_t flags, uint32_t dispatch_offset, int32_t *retval)
0c530ab8 2909{
39236c6e 2910 boolean_t istate;
0c530ab8
A		 2911 int error = 0;
2912 unsigned bytesTraced = 0;
39236c6e
A		 2913
2914#if CONFIG_TELEMETRY
2915 if (flags & STACKSHOT_GLOBAL_MICROSTACKSHOT_ENABLE) {
2916 telemetry_global_ctl(1);
2917 *retval = 0;
2918 return (0);
2919 } else if (flags & STACKSHOT_GLOBAL_MICROSTACKSHOT_DISABLE) {
2920 telemetry_global_ctl(0);
2921 *retval = 0;
2922 return (0);
2923 }
fe8ab488
A		 2924
2925 if (flags & STACKSHOT_WINDOWED_MICROSTACKSHOTS_ENABLE) {
2926 error = telemetry_enable_window();
2927
2928 if (error != KERN_SUCCESS) {
2929 /* We are probably out of memory */
2930 *retval = -1;
2931 return ENOMEM;
2932 }
2933
2934 *retval = 0;
2935 return (0);
2936 } else if (flags & STACKSHOT_WINDOWED_MICROSTACKSHOTS_DISABLE) {
2937 telemetry_disable_window();
2938 *retval = 0;
2939 return (0);
2940 }
39236c6e 2941#endif
0c530ab8
A		 2942
2943 *retval = -1;
2944/* Serialize tracing */
2945 STACKSHOT_SUBSYS_LOCK();
2946
39236c6e
A		 2947 if (tracebuf_size <= 0) {
2948 error = EINVAL;
2949 goto error_exit;
2950 }
2951
2952#if CONFIG_TELEMETRY
2953 if (flags & STACKSHOT_GET_MICROSTACKSHOT) {
2954
2955 if (tracebuf_size > SANE_TRACEBUF_SIZE) {
2956 error = EINVAL;
2957 goto error_exit;
2958 }
2959
2960 bytesTraced = tracebuf_size;
2961 error = telemetry_gather(tracebuf, &bytesTraced,
2962 (flags & STACKSHOT_SET_MICROSTACKSHOT_MARK) ? TRUE : FALSE);
2963 if (error == KERN_NO_SPACE) {
2964 error = ENOSPC;
2965 }
2966
2967 *retval = (int)bytesTraced;
2968 goto error_exit;
2969 }
2970
fe8ab488
A		 2971 if (flags & STACKSHOT_GET_WINDOWED_MICROSTACKSHOTS) {
2972
2973 if (tracebuf_size > SANE_TRACEBUF_SIZE) {
2974 error = EINVAL;
2975 goto error_exit;
2976 }
2977
2978 bytesTraced = tracebuf_size;
2979 error = telemetry_gather_windowed(tracebuf, &bytesTraced);
2980 if (error == KERN_NO_SPACE) {
2981 error = ENOSPC;
2982 }
2983
2984 *retval = (int)bytesTraced;
2985 goto error_exit;
2986 }
2987
39236c6e
A		 2988 if (flags & STACKSHOT_GET_BOOT_PROFILE) {
2989
2990 if (tracebuf_size > SANE_BOOTPROFILE_TRACEBUF_SIZE) {
2991 error = EINVAL;
2992 goto error_exit;
2993 }
2994
2995 bytesTraced = tracebuf_size;
2996 error = bootprofile_gather(tracebuf, &bytesTraced);
2997 if (error == KERN_NO_SPACE) {
2998 error = ENOSPC;
2999 }
3000
3001 *retval = (int)bytesTraced;
3002 goto error_exit;
3003 }
3004#endif
3005
3006 if (tracebuf_size > SANE_TRACEBUF_SIZE) {
0c530ab8
A		 3007 error = EINVAL;
3008 goto error_exit;
3009 }
3010
d41d1dae
A		 3011 assert(stackshot_snapbuf == NULL);
3012 if (kmem_alloc_kobject(kernel_map, (vm_offset_t *)&stackshot_snapbuf, tracebuf_size) != KERN_SUCCESS) {
3013 error = ENOMEM;
3014 goto error_exit;
3015 }
0c530ab8 3016
d41d1dae 3017 if (panic_active()) {
0c530ab8
A		 3018 error = ENOMEM;
3019 goto error_exit;
3020 }
d41d1dae
A		 3021
3022 istate = ml_set_interrupts_enabled(FALSE);
0c530ab8 3023/* Preload trace parameters*/
b7266188 3024 kdp_snapshot_preflight(pid, stackshot_snapbuf, tracebuf_size, flags, dispatch_offset);
0c530ab8
A		 3025
3026/* Trap to the debugger to obtain a coherent stack snapshot; this populates
3027 * the trace buffer
3028 */
2d21ac55 3029
0c530ab8
A		 3030 TRAP_DEBUGGER;
3031
d41d1dae
A		 3032 ml_set_interrupts_enabled(istate);
3033
0c530ab8
A		 3034 bytesTraced = kdp_stack_snapshot_bytes_traced();
3035
3036 if (bytesTraced > 0) {
3037 if ((error = copyout(stackshot_snapbuf, tracebuf,
3038 ((bytesTraced < tracebuf_size) ?
3039 bytesTraced : tracebuf_size))))
3040 goto error_exit;
3041 *retval = bytesTraced;
3042 }
3043 else {
3044 error = ENOENT;
3045 goto error_exit;
3046 }
3047
3048 error = kdp_stack_snapshot_geterror();
3049 if (error == -1) {
3050 error = ENOSPC;
3051 *retval = -1;
3052 goto error_exit;
3053 }
3054
3055error_exit:
3056 if (stackshot_snapbuf != NULL)
d41d1dae 3057 kmem_free(kernel_map, (vm_offset_t) stackshot_snapbuf, tracebuf_size);
0c530ab8
A		 3058 stackshot_snapbuf = NULL;
3059 STACKSHOT_SUBSYS_UNLOCK();
3060 return error;
3061}
2d21ac55
A		 3062
3063void
fe8ab488
A		 3064start_kern_tracing(unsigned int new_nkdbufs, boolean_t need_map)
3065{
6d2010ae 3066
2d21ac55
A		 3067 if (!new_nkdbufs)
3068 return;
316670eb 3069 nkdbufs = kdbg_set_nkdbufs(new_nkdbufs);
2d21ac55 3070 kdbg_lock_init();
39236c6e 3071
fe8ab488
A		 3072 kernel_debug_string("start_kern_tracing");
3073
39236c6e
A		 3074 if (0 == kdbg_reinit(TRUE)) {
3075
3076 if (need_map == TRUE) {
3077 uint32_t old1, old2;
3078
3079 kdbg_thrmap_init();
3080
3081 disable_wrap(&old1, &old2);
3082 }
fe8ab488
A		 3083
3084 /* Hold off interrupts until the early traces are cut */
3085 boolean_t s = ml_set_interrupts_enabled(FALSE);
3086
39236c6e 3087 kdbg_set_tracing_enabled(TRUE, KDEBUG_ENABLE_TRACE);
b0d623f7 3088
fe8ab488
A		 3089 /*
3090 * Transfer all very early events from the static buffer
3091 * into the real buffers.
3092 */
3093 kernel_debug_early_end();
3094
3095 ml_set_interrupts_enabled(s);
b0d623f7 3096
39236c6e
A		 3097 printf("kernel tracing started\n");
3098 } else {
3099 printf("error from kdbg_reinit,kernel tracing not started\n");
3100 }
2d21ac55 3101}
b0d623f7 3102
fe8ab488
A		 3103void
3104start_kern_tracing_with_typefilter(unsigned int new_nkdbufs,
3105 boolean_t need_map,
3106 unsigned int typefilter)
3107{
3108 /* startup tracing */
3109 start_kern_tracing(new_nkdbufs, need_map);
3110
3111 /* check that tracing was actually enabled */
3112 if (!(kdebug_enable & KDEBUG_ENABLE_TRACE))
3113 return;
3114
3115 /* setup the typefiltering */
3116 if (0 == kdbg_enable_typefilter())
3117 setbit(type_filter_bitmap, typefilter & (CSC_MASK >> CSC_OFFSET));
3118}
3119
b0d623f7
A		 3120void
3121kdbg_dump_trace_to_file(const char *filename)
3122{
3123 vfs_context_t ctx;
3124 vnode_t vp;
3125 int error;
3126 size_t number;
3127
3128
6d2010ae 3129 if ( !(kdebug_enable & KDEBUG_ENABLE_TRACE))
b0d623f7
A		 3130 return;
3131
3132 if (global_state_pid != -1) {
3133 if ((proc_find(global_state_pid)) != NULL) {
3134 /*
3135 * The global pid exists, we're running
3136 * due to fs_usage, latency, etc...
3137 * don't cut the panic/shutdown trace file
39236c6e
A		 3138 * Disable tracing from this point to avoid
3139 * perturbing state.
b0d623f7 3140 */
39236c6e
A		 3141 kdebug_enable = 0;
3142 kd_ctrl_page.enabled = 0;
b0d623f7
A		 3143 return;
3144 }
3145 }
3146 KERNEL_DEBUG_CONSTANT((TRACEDBG_CODE(DBG_TRACE_INFO, 0)) | DBG_FUNC_NONE, 0, 0, 0, 0, 0);
3147
3148 kdebug_enable = 0;
6d2010ae 3149 kd_ctrl_page.enabled = 0;
b0d623f7
A		 3150
3151 ctx = vfs_context_kernel();
3152
3153 if ((error = vnode_open(filename, (O_CREAT | FWRITE | O_NOFOLLOW), 0600, 0, &vp, ctx)))
3154 return;
3155
39236c6e
A		 3156 number = kd_mapcount * sizeof(kd_threadmap);
3157 kdbg_readthrmap(0, &number, vp, ctx);
b0d623f7
A		 3158
3159 number = nkdbufs*sizeof(kd_buf);
3160 kdbg_read(0, &number, vp, ctx);
3161
3162 vnode_close(vp, FWRITE, ctx);
3163
3164 sync(current_proc(), (void *)NULL, (int *)NULL);
3165}
6d2010ae
A		 3166
3167/* Helper function for filling in the BSD name for an address space
3168 * Defined here because the machine bindings know only Mach threads
3169 * and nothing about BSD processes.
3170 *
3171 * FIXME: need to grab a lock during this?
3172 */
3173void kdbg_get_task_name(char* name_buf, int len, task_t task)
3174{
3175 proc_t proc;
3176
3177 /* Note: we can't use thread->task (and functions that rely on it) here
3178 * because it hasn't been initialized yet when this function is called.
3179 * We use the explicitly-passed task parameter instead.
3180 */
3181 proc = get_bsdtask_info(task);
3182 if (proc != PROC_NULL)
3183 snprintf(name_buf, len, "%s/%d", proc->p_comm, proc->p_pid);
3184 else
3185 snprintf(name_buf, len, "%p [!bsd]", task);
3186}
mirror of XNU