[apple/xnu.git] / san / ksancov.c

/*
 * Copyright (c) 2019 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 *
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */

#include <string.h>
#include <stdbool.h>
#include <stdatomic.h>

#include <kern/assert.h>
#include <kern/cpu_data.h>
#include <kern/locks.h>
#include <kern/debug.h>
#include <kern/kalloc.h>
#include <kern/zalloc.h>
#include <kern/task.h>
#include <kern/thread.h>

#include <vm/vm_kern.h>
#include <vm/vm_protos.h>

#include <mach/mach_vm.h>
#include <mach/mach_types.h>
#include <mach/mach_port.h>
#include <mach/vm_map.h>
#include <mach/vm_param.h>
#include <mach/machine/vm_param.h>
#include <machine/atomic.h>

#include <sys/stat.h> /* dev_t */
#include <miscfs/devfs/devfs.h> /* must come after sys/stat.h */
#include <sys/conf.h> /* must come after sys/stat.h */

#include <libkern/libkern.h>
#include <libkern/OSAtomic.h>
#include <os/overflow.h>

#include <san/ksancov.h>

/* header mess... */
struct uthread;
typedef struct uthread * uthread_t;

#include <sys/sysproto.h>
#include <sys/queue.h>
#include <sys/sysctl.h>

#define USE_PC_TABLE 0
#define KSANCOV_MAX_DEV 64

extern boolean_t ml_at_interrupt_context(void);
extern boolean_t ml_get_interrupts_enabled(void);

static int ksancov_detach(dev_t dev);

static int dev_major;
static size_t nedges = 0;
static uint32_t __unused npcs = 0;

static _Atomic unsigned active_devs;

enum {
	KS_MODE_NONE,
	KS_MODE_TRACE,
	KS_MODE_COUNTERS,
	KS_MODE_MAX
};

struct ksancov_dev {
	unsigned mode;

	union {
		struct ksancov_trace *trace;
		struct ksancov_counters *counters;
	};
	size_t sz;     /* size of allocated trace/counters buffer */

	size_t maxpcs;

	thread_t thread;
	dev_t dev;
};

/* array of devices indexed by devnode minor */
static struct ksancov_dev *ksancov_devs[KSANCOV_MAX_DEV];

static struct ksancov_edgemap *ksancov_edgemap;

static inline struct ksancov_dev *
get_dev(dev_t dev)
{
	int mn = minor(dev);
	return ksancov_devs[mn];
}

void
__sanitizer_cov_trace_pc_indirect(void * __unused callee)
{
	return;
}

#define GUARD_SEEN     (uint32_t)0x80000000
#define GUARD_IDX_MASK (uint32_t)0x0fffffff

static inline void __attribute__((always_inline))
trace_pc_guard(uint32_t *guardp, void *caller)
{
	/* record the pc for this guard */
	if (guardp) {
		uint32_t gd = *guardp;
		if (__improbable(gd && !(gd & GUARD_SEEN) && ksancov_edgemap)) {
			size_t idx = gd & GUARD_IDX_MASK;
			if (idx < ksancov_edgemap->nedges) {
				ksancov_edgemap->addrs[idx] = (uint32_t)(VM_KERNEL_UNSLIDE(caller) - VM_MIN_KERNEL_ADDRESS - 1);
				*guardp |= GUARD_SEEN;
			}
		}
	}

	if (__probable(os_atomic_load(&active_devs, relaxed) == 0)) {
		/* early exit when nothing is active */
		return;
	}

	if (ml_at_interrupt_context()) {
		return;
	}

	uint32_t pc = (uint32_t)(VM_KERNEL_UNSLIDE(caller) - VM_MIN_KERNEL_ADDRESS - 1);

	thread_t th = current_thread();
	if (__improbable(th == THREAD_NULL)) {
		return;
	}

	struct ksancov_dev *dev = *(struct ksancov_dev **)__sanitizer_get_thread_data(th);
	if (__probable(dev == NULL)) {
		return;
	}

	if (dev->mode == KS_MODE_TRACE) {
		struct ksancov_trace *trace = dev->trace;
		if (os_atomic_load(&trace->enabled, relaxed) == 0) {
			return;
		}

		if (os_atomic_load(&trace->head, relaxed) >= dev->maxpcs) {
			return; /* overflow */
		}

		uint32_t idx = os_atomic_inc_orig(&trace->head, relaxed);
		if (__improbable(idx >= dev->maxpcs)) {
			return;
		}

		trace->pcs[idx] = pc;
	} else {
		size_t idx = *guardp & GUARD_IDX_MASK;

		struct ksancov_counters *counters = dev->counters;
		if (os_atomic_load(&counters->enabled, relaxed) == 0) {
			return;
		}

		/* saturating 8bit add */
		if (counters->hits[idx] < KSANCOV_MAX_HITS) {
			counters->hits[idx]++;
		}
	}
}

void __attribute__((noinline))
__sanitizer_cov_trace_pc(void)
{
	trace_pc_guard(NULL, __builtin_return_address(0));
}

void __attribute__((noinline))
__sanitizer_cov_trace_pc_guard(uint32_t *guardp)
{
	trace_pc_guard(guardp, __builtin_return_address(0));
}

void
__sanitizer_cov_trace_pc_guard_init(uint32_t *start, uint32_t *stop)
{
	/* assign a unique number to each guard */
	for (; start != stop; start++) {
		if (*start == 0) {
			if (nedges < KSANCOV_MAX_EDGES) {
				*start = ++nedges;
			}
		}
	}
}

void
__sanitizer_cov_pcs_init(uintptr_t *start, uintptr_t *stop)
{
#if USE_PC_TABLE
	static const uintptr_t pc_table_seen_flag = 0x100;

	for (; start < stop; start += 2) {
		uintptr_t pc = start[0];
		uintptr_t flags = start[1];

		/*
		 * This function gets called multiple times on the same range, so mark the
		 * ones we've seen using unused bits in the flags field.
		 */
		if (flags & pc_table_seen_flag) {
			continue;
		}

		start[1] |= pc_table_seen_flag;
		assert(npcs < KSANCOV_MAX_EDGES - 1);
		edge_addrs[++npcs] = pc;
	}
#else
	(void)start;
	(void)stop;
#endif
}

static void *
ksancov_do_map(uintptr_t base, size_t sz, vm_prot_t prot)
{
	kern_return_t kr;
	mach_port_t mem_entry = MACH_PORT_NULL;
	mach_vm_address_t user_addr = 0;
	memory_object_size_t size = sz;

	kr = mach_make_memory_entry_64(kernel_map,
	    &size,
	    (mach_vm_offset_t)base,
	    MAP_MEM_VM_SHARE | prot,
	    &mem_entry,
	    MACH_PORT_NULL);
	if (kr != KERN_SUCCESS) {
		return NULL;
	}

	kr = mach_vm_map_kernel(get_task_map(current_task()),
	    &user_addr,
	    size,
	    0,
	    VM_FLAGS_ANYWHERE,
	    VM_MAP_KERNEL_FLAGS_NONE,
	    VM_KERN_MEMORY_NONE,
	    mem_entry,
	    0,
	    FALSE,
	    prot,
	    prot,
	    VM_INHERIT_SHARE);

	/*
	 * At this point, either vm_map() has taken a reference on the memory entry
	 * and we can release our local reference, or the map failed and the entry
	 * needs to be freed.
	 */
	mach_memory_entry_port_release(mem_entry);

	if (kr != KERN_SUCCESS) {
		return NULL;
	}

	return (void *)user_addr;
}

/*
 * map the sancov buffer into the current process
 */
static int
ksancov_map(dev_t dev, void **bufp, size_t *sizep)
{
	struct ksancov_dev *d = get_dev(dev);
	if (!d) {
		return EINVAL;
	}

	uintptr_t addr;
	size_t size = d->sz;

	if (d->mode == KS_MODE_TRACE) {
		if (!d->trace) {
			return EINVAL;
		}
		addr = (uintptr_t)d->trace;
	} else if (d->mode == KS_MODE_COUNTERS) {
		if (!d->counters) {
			return EINVAL;
		}
		addr = (uintptr_t)d->counters;
	} else {
		return EINVAL; /* not configured */
	}

	void *buf = ksancov_do_map(addr, size, VM_PROT_READ | VM_PROT_WRITE);
	if (buf == NULL) {
		return ENOMEM;
	}

	*bufp = buf;
	*sizep = size;
	return 0;
}

/*
 * map the edge -> pc mapping as read-only
 */
static int
ksancov_map_edgemap(dev_t dev, void **bufp, size_t *sizep)
{
	struct ksancov_dev *d = get_dev(dev);
	if (!d) {
		return EINVAL;
	}

	uintptr_t addr = (uintptr_t)ksancov_edgemap;
	size_t size = sizeof(struct ksancov_edgemap) + ksancov_edgemap->nedges * sizeof(uint32_t);

	void *buf = ksancov_do_map(addr, size, VM_PROT_READ);
	if (buf == NULL) {
		return ENOMEM;
	}

	*bufp = buf;
	*sizep = size;
	return 0;
}


/*
 * Device node management
 */

static int
ksancov_open(dev_t dev, int flags, int devtype, proc_t p)
{
#pragma unused(flags,devtype,p)
	if (minor(dev) >= KSANCOV_MAX_DEV) {
		return EBUSY;
	}

	/* allocate a device entry */
	struct ksancov_dev *d = kalloc_tag(sizeof(struct ksancov_dev), VM_KERN_MEMORY_DIAG);
	if (!d) {
		return ENOMEM;
	}

	d->mode = KS_MODE_NONE;
	d->trace = NULL;
	d->maxpcs = 1024U * 64; /* default to 256k buffer => 64k pcs */
	d->dev = dev;
	d->thread = THREAD_NULL;

	ksancov_devs[minor(dev)] = d;

	return 0;
}

static int
ksancov_trace_alloc(dev_t dev, size_t maxpcs)
{
	struct ksancov_dev *d = get_dev(dev);
	if (!d) {
		return EINVAL;
	}

	if (d->mode != KS_MODE_NONE) {
		return EBUSY; /* trace/counters already created */
	}
	assert(d->trace == NULL);

	uintptr_t buf;
	size_t sz;
	if (os_mul_and_add_overflow(maxpcs, sizeof(uint32_t), sizeof(struct ksancov_trace), &sz)) {
		return EINVAL;
	}

	/* allocate the shared memory buffer */
	kern_return_t kr = kmem_alloc_flags(kernel_map, &buf, sz, VM_KERN_MEMORY_DIAG, KMA_ZERO);
	if (kr != KERN_SUCCESS) {
		return ENOMEM;
	}

	struct ksancov_trace *trace = (struct ksancov_trace *)buf;
	trace->magic = KSANCOV_TRACE_MAGIC;
	trace->offset = VM_MIN_KERNEL_ADDRESS;
	trace->head = 0;
	trace->enabled = 0;
	trace->maxpcs = maxpcs;

	d->trace = trace;
	d->sz = sz;
	d->maxpcs = maxpcs;
	d->mode = KS_MODE_TRACE;

	return 0;
}

static int
ksancov_counters_alloc(dev_t dev)
{
	struct ksancov_dev *d = get_dev(dev);
	if (!d) {
		return EINVAL;
	}

	if (d->mode != KS_MODE_NONE) {
		return EBUSY; /* trace/counters already created */
	}
	assert(d->counters == NULL);

	uintptr_t buf;
	size_t sz = sizeof(struct ksancov_counters) + ksancov_edgemap->nedges * sizeof(uint8_t);

	/* allocate the shared memory buffer */
	kern_return_t kr = kmem_alloc_flags(kernel_map, &buf, sz, VM_KERN_MEMORY_DIAG, KMA_ZERO);
	if (kr != KERN_SUCCESS) {
		return ENOMEM;
	}

	struct ksancov_counters *counters = (struct ksancov_counters *)buf;
	counters->magic = KSANCOV_COUNTERS_MAGIC;
	counters->nedges = ksancov_edgemap->nedges;
	counters->enabled = 0;

	d->counters = counters;
	d->sz = sz;
	d->mode = KS_MODE_COUNTERS;

	return 0;
}

/*
 * attach a thread to a ksancov dev instance
 */
static int
ksancov_attach(dev_t dev, thread_t th)
{
	struct ksancov_dev *d = get_dev(dev);
	if (!d) {
		return EINVAL;
	}

	if (d->thread != THREAD_NULL) {
		int ret = ksancov_detach(dev);
		if (ret) {
			return ret;
		}
	}

	if (th != current_thread()) {
		/* can only attach to self presently */
		return EINVAL;
	}

	struct ksancov_dev **devp = (void *)__sanitizer_get_thread_data(th);
	if (*devp) {
		return EBUSY; /* one dev per thread */
	}

	d->thread = th;
	thread_reference(d->thread);

	os_atomic_store(devp, d, relaxed);
	os_atomic_add(&active_devs, 1, relaxed);

	return 0;
}

extern void
thread_wait(
	thread_t        thread,
	boolean_t       until_not_runnable);


/*
 * disconnect thread from ksancov dev
 */
static int
ksancov_detach(dev_t dev)
{
	struct ksancov_dev *d = get_dev(dev);
	if (!d) {
		return EINVAL;
	}

	if (d->thread == THREAD_NULL) {
		/* no thread attached */
		return 0;
	}

	/* disconnect dev from thread */
	struct ksancov_dev **devp = (void *)__sanitizer_get_thread_data(d->thread);
	if (*devp != NULL) {
		assert(*devp == d);
		os_atomic_store(devp, NULL, relaxed);
	}

	if (d->thread != current_thread()) {
		/* wait until it's safe to yank */
		thread_wait(d->thread, TRUE);
	}

	/* drop our thread reference */
	thread_deallocate(d->thread);
	d->thread = THREAD_NULL;

	return 0;
}

static int
ksancov_close(dev_t dev, int flags, int devtype, proc_t p)
{
#pragma unused(flags,devtype,p)
	struct ksancov_dev *d = get_dev(dev);
	if (!d) {
		return EINVAL;
	}

	if (d->mode == KS_MODE_TRACE) {
		struct ksancov_trace *trace = d->trace;
		if (trace) {
			/* trace allocated - delete it */

			os_atomic_sub(&active_devs, 1, relaxed);
			os_atomic_store(&trace->enabled, 0, relaxed); /* stop tracing */

			ksancov_detach(dev);

			/* free trace */
			kmem_free(kernel_map, (uintptr_t)d->trace, d->sz);
			d->trace = NULL;
			d->sz = 0;
		}
	} else if (d->mode == KS_MODE_COUNTERS) {
		struct ksancov_counters *counters = d->counters;
		if (counters) {
			os_atomic_sub(&active_devs, 1, relaxed);
			os_atomic_store(&counters->enabled, 0, relaxed); /* stop tracing */

			ksancov_detach(dev);

			/* free counters */
			kmem_free(kernel_map, (uintptr_t)d->counters, d->sz);
			d->counters = NULL;
			d->sz = 0;
		}
	}

	ksancov_devs[minor(dev)] = NULL; /* dev no longer discoverable */

	/* free the ksancov device instance */
	kfree(d, sizeof(struct ksancov_dev));

	return 0;
}

static void
ksancov_testpanic(volatile uint64_t guess)
{
	const uint64_t tgt = 0xf85de3b12891c817UL;

#define X(n) ((tgt & (0xfUL << (4*n))) == (guess & (0xfUL << (4*n))))

	if (X(0)) {
		if (X(1)) {
			if (X(2)) {
				if (X(3)) {
					if (X(4)) {
						if (X(5)) {
							if (X(6)) {
								if (X(7)) {
									if (X(8)) {
										if (X(9)) {
											if (X(10)) {
												if (X(11)) {
													if (X(12)) {
														if (X(13)) {
															if (X(14)) {
																if (X(15)) {
																	panic("ksancov: found test value\n");
																}
															}
														}
													}
												}
											}
										}
									}
								}
							}
						}
					}
				}
			}
		}
	}
}

static int
ksancov_ioctl(dev_t dev, unsigned long cmd, caddr_t _data, int fflag, proc_t p)
{
#pragma unused(fflag,p)
	int ret = 0;
	void *data = (void *)_data;

	struct ksancov_dev *d = get_dev(dev);
	if (!d) {
		return EINVAL; /* dev not open */
	}

	if (cmd == KSANCOV_IOC_TRACE) {
		size_t maxpcs = *(size_t *)data;
		ret = ksancov_trace_alloc(dev, maxpcs);
		if (ret) {
			return ret;
		}
	} else if (cmd == KSANCOV_IOC_COUNTERS) {
		ret = ksancov_counters_alloc(dev);
		if (ret) {
			return ret;
		}
	} else if (cmd == KSANCOV_IOC_MAP) {
		struct ksancov_buf_desc *mcmd = (struct ksancov_buf_desc *)data;

		if (d->mode == KS_MODE_NONE) {
			return EINVAL; /* mode not configured */
		}

		/* map buffer into the userspace VA space */
		void *buf;
		size_t size;
		ret = ksancov_map(dev, &buf, &size);
		if (ret) {
			return ret;
		}

		mcmd->ptr = (uintptr_t)buf;
		mcmd->sz = size;
	} else if (cmd == KSANCOV_IOC_MAP_EDGEMAP) {
		struct ksancov_buf_desc *mcmd = (struct ksancov_buf_desc *)data;

		/* map buffer into the userspace VA space */
		void *buf;
		size_t size;
		ret = ksancov_map_edgemap(dev, &buf, &size);
		if (ret) {
			return ret;
		}

		mcmd->ptr = (uintptr_t)buf;
		mcmd->sz = size;
	} else if (cmd == KSANCOV_IOC_START) {
		if (d->mode == KS_MODE_NONE) {
			return EINVAL; /* not configured */
		}

		ret = ksancov_attach(dev, current_thread());
		if (ret) {
			return ret;
		}
	} else if (cmd == KSANCOV_IOC_NEDGES) {
		size_t *nptr = (size_t *)data;
		*nptr = nedges;
	} else if (cmd == KSANCOV_IOC_TESTPANIC) {
		uint64_t guess = *(uint64_t *)data;
		ksancov_testpanic(guess);
	} else {
		/* unknown ioctl */
		return ENODEV;
	}

	return ret;
}

static int
ksancov_dev_clone(dev_t dev, int action)
{
#pragma unused(dev)
	if (action == DEVFS_CLONE_ALLOC) {
		for (size_t i = 0; i < KSANCOV_MAX_DEV; i++) {
			if (ksancov_devs[i] == NULL) {
				return i;
			}
		}
	} else if (action == DEVFS_CLONE_FREE) {
		return 0;
	}

	return -1;
}

static struct cdevsw
    ksancov_cdev = {
	.d_open =  ksancov_open,
	.d_close = ksancov_close,
	.d_ioctl = ksancov_ioctl,

	.d_read = eno_rdwrt,
	.d_write = eno_rdwrt,
	.d_stop = eno_stop,
	.d_reset = eno_reset,
	.d_select = eno_select,
	.d_mmap = eno_mmap,
	.d_strategy = eno_strat,
	.d_type = 0
};

int
ksancov_init_dev(void)
{
	dev_major = cdevsw_add(-1, &ksancov_cdev);
	if (dev_major < 0) {
		printf("ksancov: failed to allocate major device node\n");
		return -1;
	}

	dev_t dev = makedev(dev_major, 0);
	void *node = devfs_make_node_clone(dev, DEVFS_CHAR, UID_ROOT, GID_WHEEL, 0666,
	    ksancov_dev_clone, KSANCOV_DEVNODE);
	if (!node) {
		printf("ksancov: failed to create device node\n");
		return -1;
	}

	/* This could be moved to the first use of /dev/ksancov to save memory */
	uintptr_t buf;
	size_t sz = sizeof(struct ksancov_edgemap) + KSANCOV_MAX_EDGES * sizeof(uint32_t);

	kern_return_t kr = kmem_alloc_flags(kernel_map, &buf, sz, VM_KERN_MEMORY_DIAG, KMA_ZERO);
	if (kr) {
		printf("ksancov: failed to allocate edge addr map\n");
		return -1;
	}

	ksancov_edgemap = (void *)buf;
	ksancov_edgemap->magic = KSANCOV_EDGEMAP_MAGIC;
	ksancov_edgemap->nedges = nedges;
	ksancov_edgemap->offset = VM_MIN_KERNEL_ADDRESS;

	return 0;
}
Commit	Line	Data
cb323159 A	1	/*
	2	* Copyright (c) 2019 Apple Inc. All rights reserved.
	3	*
	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
	5	*
	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
	14	*
	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
	25	*
	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
	27	*/
	28
	29	#include <string.h>
	30	#include <stdbool.h>
	31	#include <stdatomic.h>
	32
	33	#include <kern/assert.h>
	34	#include <kern/cpu_data.h>
	35	#include <kern/locks.h>
	36	#include <kern/debug.h>
	37	#include <kern/kalloc.h>
	38	#include <kern/zalloc.h>
	39	#include <kern/task.h>
	40	#include <kern/thread.h>
	41
	42	#include <vm/vm_kern.h>
	43	#include <vm/vm_protos.h>
	44
	45	#include <mach/mach_vm.h>
	46	#include <mach/mach_types.h>
	47	#include <mach/mach_port.h>
	48	#include <mach/vm_map.h>
	49	#include <mach/vm_param.h>
	50	#include <mach/machine/vm_param.h>
	51	#include <machine/atomic.h>
	52
	53	#include <sys/stat.h> /* dev_t */
	54	#include <miscfs/devfs/devfs.h> /* must come after sys/stat.h */
	55	#include <sys/conf.h> /* must come after sys/stat.h */
	56
	57	#include <libkern/libkern.h>
	58	#include <libkern/OSAtomic.h>
	59	#include <os/overflow.h>
	60
	61	#include <san/ksancov.h>
	62
	63	/* header mess... */
	64	struct uthread;
65	typedef struct uthread * uthread_t;
66
67	#include <sys/sysproto.h>
68	#include <sys/queue.h>
69	#include <sys/sysctl.h>
70
71	#define USE_PC_TABLE 0
72	#define KSANCOV_MAX_DEV 64
73
74	extern boolean_t ml_at_interrupt_context(void);
75	extern boolean_t ml_get_interrupts_enabled(void);
76
77	static int ksancov_detach(dev_t dev);
78
79	static int dev_major;
80	static size_t nedges = 0;
81	static uint32_t __unused npcs = 0;
82
83	static _Atomic unsigned active_devs;
84
85	enum {
86	KS_MODE_NONE,
87	KS_MODE_TRACE,
88	KS_MODE_COUNTERS,
89	KS_MODE_MAX
90	};
91
92	struct ksancov_dev {
93	unsigned mode;
94
95	union {
96	struct ksancov_trace *trace;
97	struct ksancov_counters *counters;
98	};
99	size_t sz; /* size of allocated trace/counters buffer */
100
101	size_t maxpcs;
102
103	thread_t thread;
104	dev_t dev;
105	};
106
107	/* array of devices indexed by devnode minor */
108	static struct ksancov_dev *ksancov_devs[KSANCOV_MAX_DEV];
109
110	static struct ksancov_edgemap *ksancov_edgemap;
111
112	static inline struct ksancov_dev *
113	get_dev(dev_t dev)
114	{
115	int mn = minor(dev);
116	return ksancov_devs[mn];
117	}
118
119	void
120	__sanitizer_cov_trace_pc_indirect(void * __unused callee)
121	{
122	return;
123	}
124
125	#define GUARD_SEEN (uint32_t)0x80000000
126	#define GUARD_IDX_MASK (uint32_t)0x0fffffff
127
128	static inline void __attribute__((always_inline))
129	trace_pc_guard(uint32_t guardp, void caller)
130	{
131	/* record the pc for this guard */
132	if (guardp) {
133	uint32_t gd = *guardp;
134	if (__improbable(gd && !(gd & GUARD_SEEN) && ksancov_edgemap)) {
135	size_t idx = gd & GUARD_IDX_MASK;
136	if (idx < ksancov_edgemap->nedges) {
137	ksancov_edgemap->addrs[idx] = (uint32_t)(VM_KERNEL_UNSLIDE(caller) - VM_MIN_KERNEL_ADDRESS - 1);
138	*guardp \|= GUARD_SEEN;
139	}
140	}
141	}
142
143	if (__probable(os_atomic_load(&active_devs, relaxed) == 0)) {
144	/* early exit when nothing is active */
145	return;
146	}
147
148	if (ml_at_interrupt_context()) {
149	return;
150	}
151
152	uint32_t pc = (uint32_t)(VM_KERNEL_UNSLIDE(caller) - VM_MIN_KERNEL_ADDRESS - 1);
153
154	thread_t th = current_thread();
155	if (__improbable(th == THREAD_NULL)) {
156	return;
157	}
158
159	struct ksancov_dev dev = (struct ksancov_dev **)__sanitizer_get_thread_data(th);
160	if (__probable(dev == NULL)) {
161	return;
162	}
163
164	if (dev->mode == KS_MODE_TRACE) {
165	struct ksancov_trace *trace = dev->trace;
166	if (os_atomic_load(&trace->enabled, relaxed) == 0) {
167	return;
168	}
169
170	if (os_atomic_load(&trace->head, relaxed) >= dev->maxpcs) {
171	return; /* overflow */
172	}
173
174	uint32_t idx = os_atomic_inc_orig(&trace->head, relaxed);
175	if (__improbable(idx >= dev->maxpcs)) {
176	return;
177	}
178
179	trace->pcs[idx] = pc;
180	} else {
181	size_t idx = *guardp & GUARD_IDX_MASK;
182
183	struct ksancov_counters *counters = dev->counters;
184	if (os_atomic_load(&counters->enabled, relaxed) == 0) {
185	return;
186	}
187
188	/* saturating 8bit add */
189	if (counters->hits[idx] < KSANCOV_MAX_HITS) {
190	counters->hits[idx]++;
191	}
192	}
193	}
194
195	void __attribute__((noinline))
196	__sanitizer_cov_trace_pc(void)
197	{
198	trace_pc_guard(NULL, __builtin_return_address(0));
199	}
200
201	void __attribute__((noinline))
202	__sanitizer_cov_trace_pc_guard(uint32_t *guardp)
203	{
204	trace_pc_guard(guardp, __builtin_return_address(0));
205	}
206
207	void
208	__sanitizer_cov_trace_pc_guard_init(uint32_t start, uint32_t stop)
209	{
210	/* assign a unique number to each guard */
211	for (; start != stop; start++) {
212	if (*start == 0) {
213	if (nedges < KSANCOV_MAX_EDGES) {
214	*start = ++nedges;
215	}
216	}
217	}
218	}
219
220	void
221	__sanitizer_cov_pcs_init(uintptr_t start, uintptr_t stop)
222	{
223	#if USE_PC_TABLE
224	static const uintptr_t pc_table_seen_flag = 0x100;
225
226	for (; start < stop; start += 2) {
227	uintptr_t pc = start[0];
228	uintptr_t flags = start[1];
229
230	/*
231	* This function gets called multiple times on the same range, so mark the
232	* ones we've seen using unused bits in the flags field.
233	*/
234	if (flags & pc_table_seen_flag) {
235	continue;
236	}
237
238	start[1] \|= pc_table_seen_flag;
239	assert(npcs < KSANCOV_MAX_EDGES - 1);
240	edge_addrs[++npcs] = pc;
241	}
242	#else
243	(void)start;
244	(void)stop;
245	#endif
246	}
247
248	static void *
249	ksancov_do_map(uintptr_t base, size_t sz, vm_prot_t prot)
250	{
251	kern_return_t kr;
252	mach_port_t mem_entry = MACH_PORT_NULL;
253	mach_vm_address_t user_addr = 0;
254	memory_object_size_t size = sz;
255
256	kr = mach_make_memory_entry_64(kernel_map,
257	&size,
258	(mach_vm_offset_t)base,
259	MAP_MEM_VM_SHARE \| prot,
260	&mem_entry,
261	MACH_PORT_NULL);
262	if (kr != KERN_SUCCESS) {
263	return NULL;
264	}
265
266	kr = mach_vm_map_kernel(get_task_map(current_task()),
267	&user_addr,
268	size,
269	0,
270	VM_FLAGS_ANYWHERE,
271	VM_MAP_KERNEL_FLAGS_NONE,
272	VM_KERN_MEMORY_NONE,
273	mem_entry,
274	0,
275	FALSE,
276	prot,
277	prot,
278	VM_INHERIT_SHARE);
279
280	/*
281	* At this point, either vm_map() has taken a reference on the memory entry
282	* and we can release our local reference, or the map failed and the entry
283	* needs to be freed.
284	*/
285	mach_memory_entry_port_release(mem_entry);
286
287	if (kr != KERN_SUCCESS) {
288	return NULL;
289	}
290
291	return (void *)user_addr;
292	}
293
294	/*
295	* map the sancov buffer into the current process
296	*/
297	static int
298	ksancov_map(dev_t dev, void *bufp, size_t sizep)
299	{
300	struct ksancov_dev *d = get_dev(dev);
301	if (!d) {
302	return EINVAL;
303	}
304
305	uintptr_t addr;
306	size_t size = d->sz;
307
308	if (d->mode == KS_MODE_TRACE) {
309	if (!d->trace) {
310	return EINVAL;
311	}
312	addr = (uintptr_t)d->trace;
313	} else if (d->mode == KS_MODE_COUNTERS) {
314	if (!d->counters) {
315	return EINVAL;
316	}
317	addr = (uintptr_t)d->counters;
318	} else {
319	return EINVAL; /* not configured */
320	}
321
322	void *buf = ksancov_do_map(addr, size, VM_PROT_READ \| VM_PROT_WRITE);
323	if (buf == NULL) {
324	return ENOMEM;
325	}
326
327	*bufp = buf;
328	*sizep = size;
329	return 0;
330	}
331
332	/*
333	* map the edge -> pc mapping as read-only
334	*/
335	static int
336	ksancov_map_edgemap(dev_t dev, void *bufp, size_t sizep)
337	{
338	struct ksancov_dev *d = get_dev(dev);
339	if (!d) {
340	return EINVAL;
341	}
342
343	uintptr_t addr = (uintptr_t)ksancov_edgemap;
344	size_t size = sizeof(struct ksancov_edgemap) + ksancov_edgemap->nedges * sizeof(uint32_t);
345
346	void *buf = ksancov_do_map(addr, size, VM_PROT_READ);
347	if (buf == NULL) {
348	return ENOMEM;
349	}
350
351	*bufp = buf;
352	*sizep = size;
353	return 0;
354	}
355
356
357	/*
358	* Device node management
359	*/
360
361	static int
362	ksancov_open(dev_t dev, int flags, int devtype, proc_t p)
363	{
364	#pragma unused(flags,devtype,p)
365	if (minor(dev) >= KSANCOV_MAX_DEV) {
366	return EBUSY;
367	}
368
369	/* allocate a device entry */
370	struct ksancov_dev *d = kalloc_tag(sizeof(struct ksancov_dev), VM_KERN_MEMORY_DIAG);
371	if (!d) {
372	return ENOMEM;
373	}
374
375	d->mode = KS_MODE_NONE;
376	d->trace = NULL;
377	d->maxpcs = 1024U * 64; /* default to 256k buffer => 64k pcs */
378	d->dev = dev;
379	d->thread = THREAD_NULL;
380
381	ksancov_devs[minor(dev)] = d;
382
383	return 0;
384	}
385
386	static int
387	ksancov_trace_alloc(dev_t dev, size_t maxpcs)
388	{
389	struct ksancov_dev *d = get_dev(dev);
390	if (!d) {
391	return EINVAL;
392	}
393
394	if (d->mode != KS_MODE_NONE) {
395	return EBUSY; /* trace/counters already created */
396	}
397	assert(d->trace == NULL);
398
399	uintptr_t buf;
400	size_t sz;
401	if (os_mul_and_add_overflow(maxpcs, sizeof(uint32_t), sizeof(struct ksancov_trace), &sz)) {
402	return EINVAL;
403	}
404
405	/* allocate the shared memory buffer */
406	kern_return_t kr = kmem_alloc_flags(kernel_map, &buf, sz, VM_KERN_MEMORY_DIAG, KMA_ZERO);
407	if (kr != KERN_SUCCESS) {
408	return ENOMEM;
409	}
410
411	struct ksancov_trace trace = (struct ksancov_trace )buf;
412	trace->magic = KSANCOV_TRACE_MAGIC;
413	trace->offset = VM_MIN_KERNEL_ADDRESS;
414	trace->head = 0;
415	trace->enabled = 0;
416	trace->maxpcs = maxpcs;
417
418	d->trace = trace;
419	d->sz = sz;
420	d->maxpcs = maxpcs;
421	d->mode = KS_MODE_TRACE;
422
423	return 0;
424	}
425
426	static int
427	ksancov_counters_alloc(dev_t dev)
428	{
429	struct ksancov_dev *d = get_dev(dev);
430	if (!d) {
431	return EINVAL;
432	}
433
434	if (d->mode != KS_MODE_NONE) {
435	return EBUSY; /* trace/counters already created */
436	}
437	assert(d->counters == NULL);
438
439	uintptr_t buf;
440	size_t sz = sizeof(struct ksancov_counters) + ksancov_edgemap->nedges * sizeof(uint8_t);
441
442	/* allocate the shared memory buffer */
443	kern_return_t kr = kmem_alloc_flags(kernel_map, &buf, sz, VM_KERN_MEMORY_DIAG, KMA_ZERO);
444	if (kr != KERN_SUCCESS) {
445	return ENOMEM;
446	}
447
448	struct ksancov_counters counters = (struct ksancov_counters )buf;
449	counters->magic = KSANCOV_COUNTERS_MAGIC;
450	counters->nedges = ksancov_edgemap->nedges;
451	counters->enabled = 0;
452
453	d->counters = counters;
454	d->sz = sz;
455	d->mode = KS_MODE_COUNTERS;
456
457	return 0;
458	}
459
460	/*
461	* attach a thread to a ksancov dev instance
462	*/
463	static int
464	ksancov_attach(dev_t dev, thread_t th)
465	{
466	struct ksancov_dev *d = get_dev(dev);
467	if (!d) {
468	return EINVAL;
469	}
470
471	if (d->thread != THREAD_NULL) {
472	int ret = ksancov_detach(dev);
473	if (ret) {
474	return ret;
475	}
476	}
477
478	if (th != current_thread()) {
479	/* can only attach to self presently */
480	return EINVAL;
481	}
482
483	struct ksancov_dev *devp = (void )__sanitizer_get_thread_data(th);
484	if (*devp) {
485	return EBUSY; /* one dev per thread */
486	}
487
488	d->thread = th;
489	thread_reference(d->thread);
490
491	os_atomic_store(devp, d, relaxed);
492	os_atomic_add(&active_devs, 1, relaxed);
493
494	return 0;
495	}
496
497	extern void
498	thread_wait(
499	thread_t thread,
500	boolean_t until_not_runnable);
501
502
503	/*
504	* disconnect thread from ksancov dev
505	*/
506	static int
507	ksancov_detach(dev_t dev)
508	{
509	struct ksancov_dev *d = get_dev(dev);
510	if (!d) {
511	return EINVAL;
512	}
513
514	if (d->thread == THREAD_NULL) {
515	/* no thread attached */
516	return 0;
517	}
518
519	/* disconnect dev from thread */
520	struct ksancov_dev *devp = (void )__sanitizer_get_thread_data(d->thread);
521	if (*devp != NULL) {
522	assert(*devp == d);
523	os_atomic_store(devp, NULL, relaxed);
524	}
525
526	if (d->thread != current_thread()) {
527	/* wait until it's safe to yank */
528	thread_wait(d->thread, TRUE);
529	}
530
531	/* drop our thread reference */
532	thread_deallocate(d->thread);
533	d->thread = THREAD_NULL;
534
535	return 0;
536	}
537
538	static int
539	ksancov_close(dev_t dev, int flags, int devtype, proc_t p)
540	{
541	#pragma unused(flags,devtype,p)
542	struct ksancov_dev *d = get_dev(dev);
543	if (!d) {
544	return EINVAL;
545	}
546
547	if (d->mode == KS_MODE_TRACE) {
548	struct ksancov_trace *trace = d->trace;
549	if (trace) {
550	/* trace allocated - delete it */
551
552	os_atomic_sub(&active_devs, 1, relaxed);
553	os_atomic_store(&trace->enabled, 0, relaxed); /* stop tracing */
554
555	ksancov_detach(dev);
556
557	/* free trace */
558	kmem_free(kernel_map, (uintptr_t)d->trace, d->sz);
559	d->trace = NULL;
560	d->sz = 0;
561	}
562	} else if (d->mode == KS_MODE_COUNTERS) {
563	struct ksancov_counters *counters = d->counters;
564	if (counters) {
565	os_atomic_sub(&active_devs, 1, relaxed);
566	os_atomic_store(&counters->enabled, 0, relaxed); /* stop tracing */
567
568	ksancov_detach(dev);
569
570	/* free counters */
571	kmem_free(kernel_map, (uintptr_t)d->counters, d->sz);
572	d->counters = NULL;
573	d->sz = 0;
574	}
575	}
576
577	ksancov_devs[minor(dev)] = NULL; /* dev no longer discoverable */
578
579	/* free the ksancov device instance */
580	kfree(d, sizeof(struct ksancov_dev));
581
582	return 0;
583	}
584
585	static void
586	ksancov_testpanic(volatile uint64_t guess)
587	{
588	const uint64_t tgt = 0xf85de3b12891c817UL;
589
590	#define X(n) ((tgt & (0xfUL << (4n))) == (guess & (0xfUL << (4n))))
591
592	if (X(0)) {
593	if (X(1)) {
594	if (X(2)) {
595	if (X(3)) {
596	if (X(4)) {
597	if (X(5)) {
598	if (X(6)) {
599	if (X(7)) {
600	if (X(8)) {
601	if (X(9)) {
602	if (X(10)) {
603	if (X(11)) {
604	if (X(12)) {
605	if (X(13)) {
606	if (X(14)) {
607	if (X(15)) {
608	panic("ksancov: found test value\n");
609	}
610	}
611	}
612	}
613	}
614	}
615	}
616	}
617	}
618	}
619	}
620	}
621	}
622	}
623	}
624	}
625	}
626
627	static int
628	ksancov_ioctl(dev_t dev, unsigned long cmd, caddr_t _data, int fflag, proc_t p)
629	{
630	#pragma unused(fflag,p)
631	int ret = 0;
632	void data = (void )_data;
633
634	struct ksancov_dev *d = get_dev(dev);
635	if (!d) {
636	return EINVAL; /* dev not open */
637	}
638
639	if (cmd == KSANCOV_IOC_TRACE) {
640	size_t maxpcs = (size_t )data;
641	ret = ksancov_trace_alloc(dev, maxpcs);
642	if (ret) {
643	return ret;
644	}
645	} else if (cmd == KSANCOV_IOC_COUNTERS) {
646	ret = ksancov_counters_alloc(dev);
647	if (ret) {
648	return ret;
649	}
650	} else if (cmd == KSANCOV_IOC_MAP) {
651	struct ksancov_buf_desc mcmd = (struct ksancov_buf_desc )data;
652
653	if (d->mode == KS_MODE_NONE) {
654	return EINVAL; /* mode not configured */
655	}
656
657	/* map buffer into the userspace VA space */
658	void *buf;
659	size_t size;
660	ret = ksancov_map(dev, &buf, &size);
661	if (ret) {
662	return ret;
663	}
664
665	mcmd->ptr = (uintptr_t)buf;
666	mcmd->sz = size;
667	} else if (cmd == KSANCOV_IOC_MAP_EDGEMAP) {
668	struct ksancov_buf_desc mcmd = (struct ksancov_buf_desc )data;
669
670	/* map buffer into the userspace VA space */
671	void *buf;
672	size_t size;
673	ret = ksancov_map_edgemap(dev, &buf, &size);
674	if (ret) {
675	return ret;
676	}
677
678	mcmd->ptr = (uintptr_t)buf;
679	mcmd->sz = size;
680	} else if (cmd == KSANCOV_IOC_START) {
681	if (d->mode == KS_MODE_NONE) {
682	return EINVAL; /* not configured */
683	}
684
685	ret = ksancov_attach(dev, current_thread());
686	if (ret) {
687	return ret;
688	}
689	} else if (cmd == KSANCOV_IOC_NEDGES) {
690	size_t nptr = (size_t )data;
691	*nptr = nedges;
692	} else if (cmd == KSANCOV_IOC_TESTPANIC) {
693	uint64_t guess = (uint64_t )data;
694	ksancov_testpanic(guess);
695	} else {
696	/* unknown ioctl */
697	return ENODEV;
698	}
699
700	return ret;
701	}
702
703	static int
704	ksancov_dev_clone(dev_t dev, int action)
705	{
706	#pragma unused(dev)
707	if (action == DEVFS_CLONE_ALLOC) {
708	for (size_t i = 0; i < KSANCOV_MAX_DEV; i++) {
709	if (ksancov_devs[i] == NULL) {
710	return i;
711	}
712	}
713	} else if (action == DEVFS_CLONE_FREE) {
714	return 0;
715	}
716
717	return -1;
718	}
719
720	static struct cdevsw
721	ksancov_cdev = {
722	.d_open = ksancov_open,
723	.d_close = ksancov_close,
724	.d_ioctl = ksancov_ioctl,
725
726	.d_read = eno_rdwrt,
727	.d_write = eno_rdwrt,
728	.d_stop = eno_stop,
729	.d_reset = eno_reset,
730	.d_select = eno_select,
731	.d_mmap = eno_mmap,
732	.d_strategy = eno_strat,
733	.d_type = 0
734	};
735
736	int
737	ksancov_init_dev(void)
738	{
739	dev_major = cdevsw_add(-1, &ksancov_cdev);
740	if (dev_major < 0) {
741	printf("ksancov: failed to allocate major device node\n");
742	return -1;
743	}
744
745	dev_t dev = makedev(dev_major, 0);
746	void *node = devfs_make_node_clone(dev, DEVFS_CHAR, UID_ROOT, GID_WHEEL, 0666,
747	ksancov_dev_clone, KSANCOV_DEVNODE);
748	if (!node) {
749	printf("ksancov: failed to create device node\n");
750	return -1;
751	}
752
753	/* This could be moved to the first use of /dev/ksancov to save memory */
754	uintptr_t buf;
755	size_t sz = sizeof(struct ksancov_edgemap) + KSANCOV_MAX_EDGES * sizeof(uint32_t);
756
757	kern_return_t kr = kmem_alloc_flags(kernel_map, &buf, sz, VM_KERN_MEMORY_DIAG, KMA_ZERO);
758	if (kr) {
759	printf("ksancov: failed to allocate edge addr map\n");
760	return -1;
761	}
762
763	ksancov_edgemap = (void *)buf;
764	ksancov_edgemap->magic = KSANCOV_EDGEMAP_MAGIC;
765	ksancov_edgemap->nedges = nedges;
766	ksancov_edgemap->offset = VM_MIN_KERNEL_ADDRESS;
767
768	return 0;
769	}