[apple/xnu.git] / san / ksancov.c

/*
 * Copyright (c) 2019 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 *
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */

#include <string.h>
#include <stdbool.h>

#include <kern/assert.h>
#include <kern/cpu_data.h>
#include <kern/locks.h>
#include <kern/debug.h>
#include <kern/kalloc.h>
#include <kern/zalloc.h>
#include <kern/task.h>
#include <kern/thread.h>

#include <vm/vm_kern.h>
#include <vm/vm_protos.h>

#include <mach/mach_vm.h>
#include <mach/mach_types.h>
#include <mach/mach_port.h>
#include <mach/vm_map.h>
#include <mach/vm_param.h>
#include <mach/machine/vm_param.h>

#include <sys/stat.h> /* dev_t */
#include <miscfs/devfs/devfs.h> /* must come after sys/stat.h */
#include <sys/conf.h> /* must come after sys/stat.h */

#include <libkern/libkern.h>
#include <os/atomic_private.h>
#include <os/overflow.h>

#include <san/ksancov.h>

/* header mess... */
struct uthread;
typedef struct uthread * uthread_t;

#include <sys/sysproto.h>
#include <sys/queue.h>
#include <sys/sysctl.h>

#define USE_PC_TABLE 0
#define KSANCOV_MAX_DEV 64
#define KSANCOV_MAX_PCS (1024U * 64)  /* default to 256k buffer => 64k pcs */

enum {
	KS_MODE_NONE,
	KS_MODE_TRACE,
	KS_MODE_COUNTERS,
	KS_MODE_MAX
};

struct ksancov_dev {
	unsigned mode;

	union {
		struct ksancov_trace *trace;
		struct ksancov_counters *counters;
	};
	size_t sz;     /* size of allocated trace/counters buffer */

	size_t maxpcs;

	thread_t thread;
	dev_t dev;
	lck_mtx_t lock;
};
typedef struct ksancov_dev * ksancov_dev_t;

extern boolean_t ml_at_interrupt_context(void);
extern boolean_t ml_get_interrupts_enabled(void);

static void ksancov_detach(ksancov_dev_t);

static int dev_major;
static size_t nedges = 0;
static uint32_t __unused npcs = 0;

static _Atomic unsigned active_devs;

static LCK_GRP_DECLARE(ksancov_lck_grp, "ksancov_lck_grp");
static lck_rw_t *ksancov_devs_lck;

/* array of devices indexed by devnode minor */
static ksancov_dev_t ksancov_devs[KSANCOV_MAX_DEV];
static struct ksancov_edgemap *ksancov_edgemap;


static ksancov_dev_t
create_dev(dev_t dev)
{
	ksancov_dev_t d = kalloc_tag(sizeof(struct ksancov_dev), VM_KERN_MEMORY_DIAG);
	if (!d) {
		return NULL;
	}

	d->mode = KS_MODE_NONE;
	d->trace = NULL;
	d->sz = 0;
	d->maxpcs = KSANCOV_MAX_PCS;
	d->dev = dev;
	d->thread = THREAD_NULL;
	lck_mtx_init(&d->lock, &ksancov_lck_grp, LCK_ATTR_NULL);

	return d;
}

static void
free_dev(ksancov_dev_t d)
{
	if (d->mode == KS_MODE_TRACE && d->trace) {
		kmem_free(kernel_map, (uintptr_t)d->trace, d->sz);
	} else if (d->mode == KS_MODE_COUNTERS && d->counters) {
		kmem_free(kernel_map, (uintptr_t)d->counters, d->sz);
	}
	lck_mtx_destroy(&d->lock, &ksancov_lck_grp);
	kfree(d, sizeof(struct ksancov_dev));
}

void
__sanitizer_cov_trace_pc_indirect(void * __unused callee)
{
	return;
}

#define GUARD_SEEN     (uint32_t)0x80000000
#define GUARD_IDX_MASK (uint32_t)0x0fffffff

static inline void __attribute__((always_inline))
trace_pc_guard(uint32_t *guardp, void *caller)
{
	/* record the pc for this guard */
	if (guardp) {
		uint32_t gd = *guardp;
		if (__improbable(gd && !(gd & GUARD_SEEN) && ksancov_edgemap)) {
			size_t idx = gd & GUARD_IDX_MASK;
			if (idx < ksancov_edgemap->nedges) {
				ksancov_edgemap->addrs[idx] = (uint32_t)(VM_KERNEL_UNSLIDE(caller) - KSANCOV_PC_OFFSET - 1);
				*guardp |= GUARD_SEEN;
			}
		}
	}

	if (__probable(os_atomic_load(&active_devs, relaxed) == 0)) {
		/* early exit when nothing is active */
		return;
	}

	if (ml_at_interrupt_context()) {
		return;
	}

	uint32_t pc = (uint32_t)(VM_KERNEL_UNSLIDE(caller) - KSANCOV_PC_OFFSET - 1);

	thread_t th = current_thread();
	if (__improbable(th == THREAD_NULL)) {
		return;
	}

	ksancov_dev_t dev = *(ksancov_dev_t *)__sanitizer_get_thread_data(th);
	if (__probable(dev == NULL)) {
		return;
	}

	if (dev->mode == KS_MODE_TRACE) {
		struct ksancov_trace *trace = dev->trace;
		if (os_atomic_load(&trace->enabled, relaxed) == 0) {
			return;
		}

		if (os_atomic_load(&trace->head, relaxed) >= dev->maxpcs) {
			return; /* overflow */
		}

		uint32_t idx = os_atomic_inc_orig(&trace->head, relaxed);
		if (__improbable(idx >= dev->maxpcs)) {
			return;
		}

		trace->pcs[idx] = pc;
	} else {
		size_t idx = *guardp & GUARD_IDX_MASK;

		struct ksancov_counters *counters = dev->counters;
		if (os_atomic_load(&counters->enabled, relaxed) == 0) {
			return;
		}

		/* saturating 8bit add */
		if (counters->hits[idx] < KSANCOV_MAX_HITS) {
			counters->hits[idx]++;
		}
	}
}

void __attribute__((noinline))
__sanitizer_cov_trace_pc(void)
{
	trace_pc_guard(NULL, __builtin_return_address(0));
}

void __attribute__((noinline))
__sanitizer_cov_trace_pc_guard(uint32_t *guardp)
{
	trace_pc_guard(guardp, __builtin_return_address(0));
}

void
__sanitizer_cov_trace_pc_guard_init(uint32_t *start, uint32_t *stop)
{
	/* assign a unique number to each guard */
	for (; start != stop; start++) {
		if (*start == 0) {
			if (nedges < KSANCOV_MAX_EDGES) {
				*start = (uint32_t)++nedges;
			}
		}
	}
}

void
__sanitizer_cov_pcs_init(uintptr_t *start, uintptr_t *stop)
{
#if USE_PC_TABLE
	static const uintptr_t pc_table_seen_flag = 0x100;

	for (; start < stop; start += 2) {
		uintptr_t pc = start[0];
		uintptr_t flags = start[1];

		/*
		 * This function gets called multiple times on the same range, so mark the
		 * ones we've seen using unused bits in the flags field.
		 */
		if (flags & pc_table_seen_flag) {
			continue;
		}

		start[1] |= pc_table_seen_flag;
		assert(npcs < KSANCOV_MAX_EDGES - 1);
		edge_addrs[++npcs] = pc;
	}
#else
	(void)start;
	(void)stop;
#endif
}

static void *
ksancov_do_map(uintptr_t base, size_t sz, vm_prot_t prot)
{
	kern_return_t kr;
	mach_port_t mem_entry = MACH_PORT_NULL;
	mach_vm_address_t user_addr = 0;
	memory_object_size_t size = sz;

	kr = mach_make_memory_entry_64(kernel_map,
	    &size,
	    (mach_vm_offset_t)base,
	    MAP_MEM_VM_SHARE | prot,
	    &mem_entry,
	    MACH_PORT_NULL);
	if (kr != KERN_SUCCESS) {
		return NULL;
	}

	kr = mach_vm_map_kernel(get_task_map(current_task()),
	    &user_addr,
	    size,
	    0,
	    VM_FLAGS_ANYWHERE,
	    VM_MAP_KERNEL_FLAGS_NONE,
	    VM_KERN_MEMORY_NONE,
	    mem_entry,
	    0,
	    FALSE,
	    prot,
	    prot,
	    VM_INHERIT_SHARE);

	/*
	 * At this point, either vm_map() has taken a reference on the memory entry
	 * and we can release our local reference, or the map failed and the entry
	 * needs to be freed.
	 */
	mach_memory_entry_port_release(mem_entry);

	if (kr != KERN_SUCCESS) {
		return NULL;
	}

	return (void *)user_addr;
}

/*
 * map the sancov buffer into the current process
 */
static int
ksancov_map(ksancov_dev_t d, uintptr_t *bufp, size_t *sizep)
{
	uintptr_t addr;
	size_t size = d->sz;

	if (d->mode == KS_MODE_TRACE) {
		if (!d->trace) {
			return EINVAL;
		}
		addr = (uintptr_t)d->trace;
	} else if (d->mode == KS_MODE_COUNTERS) {
		if (!d->counters) {
			return EINVAL;
		}
		addr = (uintptr_t)d->counters;
	} else {
		return EINVAL; /* not configured */
	}

	void *buf = ksancov_do_map(addr, size, VM_PROT_READ | VM_PROT_WRITE);
	if (buf == NULL) {
		return ENOMEM;
	}

	*bufp = (uintptr_t)buf;
	*sizep = size;

	return 0;
}

/*
 * map the edge -> pc mapping as read-only
 */
static int
ksancov_map_edgemap(uintptr_t *bufp, size_t *sizep)
{
	uintptr_t addr = (uintptr_t)ksancov_edgemap;
	size_t size = sizeof(struct ksancov_edgemap) + ksancov_edgemap->nedges * sizeof(uint32_t);

	void *buf = ksancov_do_map(addr, size, VM_PROT_READ);
	if (buf == NULL) {
		return ENOMEM;
	}

	*bufp = (uintptr_t)buf;
	*sizep = size;
	return 0;
}

/*
 * Device node management
 */

static int
ksancov_open(dev_t dev, int flags, int devtype, proc_t p)
{
#pragma unused(flags,devtype,p)
	const int minor_num = minor(dev);

	if (minor_num >= KSANCOV_MAX_DEV) {
		return EBUSY;
	}

	lck_rw_lock_exclusive(ksancov_devs_lck);

	if (ksancov_devs[minor_num]) {
		lck_rw_unlock_exclusive(ksancov_devs_lck);
		return EBUSY;
	}

	ksancov_dev_t d = create_dev(dev);
	if (!d) {
		lck_rw_unlock_exclusive(ksancov_devs_lck);
		return ENOMEM;
	}
	ksancov_devs[minor_num] = d;

	lck_rw_unlock_exclusive(ksancov_devs_lck);

	return 0;
}

static int
ksancov_trace_alloc(ksancov_dev_t d, size_t maxpcs)
{
	if (d->mode != KS_MODE_NONE) {
		return EBUSY; /* trace/counters already created */
	}
	assert(d->trace == NULL);

	uintptr_t buf;
	size_t sz;
	if (os_mul_and_add_overflow(maxpcs, sizeof(uint32_t), sizeof(struct ksancov_trace), &sz)) {
		return EINVAL;
	}

	/* allocate the shared memory buffer */
	kern_return_t kr = kmem_alloc_flags(kernel_map, &buf, sz, VM_KERN_MEMORY_DIAG, KMA_ZERO);
	if (kr != KERN_SUCCESS) {
		return ENOMEM;
	}

	struct ksancov_trace *trace = (struct ksancov_trace *)buf;
	trace->magic = KSANCOV_TRACE_MAGIC;
	trace->offset = KSANCOV_PC_OFFSET;
	os_atomic_init(&trace->head, 0);
	os_atomic_init(&trace->enabled, 0);
	trace->maxpcs = (uint32_t)maxpcs;

	d->trace = trace;
	d->sz = sz;
	d->maxpcs = maxpcs;
	d->mode = KS_MODE_TRACE;

	return 0;
}

static int
ksancov_counters_alloc(ksancov_dev_t d)
{
	if (d->mode != KS_MODE_NONE) {
		return EBUSY; /* trace/counters already created */
	}
	assert(d->counters == NULL);

	uintptr_t buf;
	size_t sz = sizeof(struct ksancov_counters) + ksancov_edgemap->nedges * sizeof(uint8_t);

	/* allocate the shared memory buffer */
	kern_return_t kr = kmem_alloc_flags(kernel_map, &buf, sz, VM_KERN_MEMORY_DIAG, KMA_ZERO);
	if (kr != KERN_SUCCESS) {
		return ENOMEM;
	}

	struct ksancov_counters *counters = (struct ksancov_counters *)buf;
	counters->magic = KSANCOV_COUNTERS_MAGIC;
	counters->nedges = ksancov_edgemap->nedges;
	os_atomic_init(&counters->enabled, 0);

	d->counters = counters;
	d->sz = sz;
	d->mode = KS_MODE_COUNTERS;

	return 0;
}

/*
 * attach a thread to a ksancov dev instance
 */
static int
ksancov_attach(ksancov_dev_t d, thread_t th)
{
	if (d->mode == KS_MODE_NONE) {
		return EINVAL; /* not configured */
	}

	if (th != current_thread()) {
		/* can only attach to self presently */
		return EINVAL;
	}

	ksancov_dev_t *devp = (void *)__sanitizer_get_thread_data(th);
	if (*devp) {
		return EBUSY; /* one dev per thread */
	}

	if (d->thread != THREAD_NULL) {
		ksancov_detach(d);
	}

	d->thread = th;
	thread_reference(d->thread);

	os_atomic_store(devp, d, relaxed);
	os_atomic_add(&active_devs, 1, relaxed);

	return 0;
}

extern void
thread_wait(
	thread_t        thread,
	boolean_t       until_not_runnable);


/*
 * disconnect thread from ksancov dev
 */
static void
ksancov_detach(ksancov_dev_t d)
{
	if (d->thread == THREAD_NULL) {
		/* no thread attached */
		return;
	}

	/* disconnect dev from thread */
	ksancov_dev_t *devp = (void *)__sanitizer_get_thread_data(d->thread);
	if (*devp != NULL) {
		assert(*devp == d);
		os_atomic_store(devp, NULL, relaxed);
	}

	if (d->thread != current_thread()) {
		/* wait until it's safe to yank */
		thread_wait(d->thread, TRUE);
	}

	/* drop our thread reference */
	thread_deallocate(d->thread);
	d->thread = THREAD_NULL;
}

static int
ksancov_close(dev_t dev, int flags, int devtype, proc_t p)
{
#pragma unused(flags,devtype,p)
	const int minor_num = minor(dev);

	lck_rw_lock_exclusive(ksancov_devs_lck);
	ksancov_dev_t d = ksancov_devs[minor_num];
	ksancov_devs[minor_num] = NULL; /* dev no longer discoverable */
	lck_rw_unlock_exclusive(ksancov_devs_lck);

	/*
	 * No need to lock d here as there is and will be no one having its
	 * reference except for this thread and the one which is going to
	 * be detached below.
	 */

	if (!d) {
		return ENXIO;
	}

	if (d->mode == KS_MODE_TRACE && d->trace) {
		os_atomic_sub(&active_devs, 1, relaxed);
		os_atomic_store(&d->trace->enabled, 0, relaxed); /* stop tracing */
	} else if (d->mode == KS_MODE_COUNTERS && d->counters) {
		os_atomic_sub(&active_devs, 1, relaxed);
		os_atomic_store(&d->counters->enabled, 0, relaxed);         /* stop tracing */
	}

	ksancov_detach(d);
	free_dev(d);

	return 0;
}

static void
ksancov_testpanic(volatile uint64_t guess)
{
	const uint64_t tgt = 0xf85de3b12891c817UL;

#define X(n) ((tgt & (0xfUL << (4*n))) == (guess & (0xfUL << (4*n))))

	if (X(0)) {
		if (X(1)) {
			if (X(2)) {
				if (X(3)) {
					if (X(4)) {
						if (X(5)) {
							if (X(6)) {
								if (X(7)) {
									if (X(8)) {
										if (X(9)) {
											if (X(10)) {
												if (X(11)) {
													if (X(12)) {
														if (X(13)) {
															if (X(14)) {
																if (X(15)) {
																	panic("ksancov: found test value\n");
																}
															}
														}
													}
												}
											}
										}
									}
								}
							}
						}
					}
				}
			}
		}
	}
}

static int
ksancov_ioctl(dev_t dev, unsigned long cmd, caddr_t _data, int fflag, proc_t p)
{
#pragma unused(fflag,p)
	struct ksancov_buf_desc *mcmd;
	void *data = (void *)_data;

	lck_rw_lock_shared(ksancov_devs_lck);
	ksancov_dev_t d = ksancov_devs[minor(dev)];
	if (!d) {
		lck_rw_unlock_shared(ksancov_devs_lck);
		return EINVAL;         /* dev not open */
	}

	int ret = 0;

	switch (cmd) {
	case KSANCOV_IOC_TRACE:
		lck_mtx_lock(&d->lock);
		ret = ksancov_trace_alloc(d, *(size_t *)data);
		lck_mtx_unlock(&d->lock);
		break;
	case KSANCOV_IOC_COUNTERS:
		lck_mtx_lock(&d->lock);
		ret = ksancov_counters_alloc(d);
		lck_mtx_unlock(&d->lock);
		break;
	case KSANCOV_IOC_MAP:
		mcmd = (struct ksancov_buf_desc *)data;
		lck_mtx_lock(&d->lock);
		ret = ksancov_map(d, &mcmd->ptr, &mcmd->sz);
		lck_mtx_unlock(&d->lock);
		break;
	case KSANCOV_IOC_MAP_EDGEMAP:
		mcmd = (struct ksancov_buf_desc *)data;
		ret = ksancov_map_edgemap(&mcmd->ptr, &mcmd->sz);
		break;
	case KSANCOV_IOC_START:
		lck_mtx_lock(&d->lock);
		ret = ksancov_attach(d, current_thread());
		lck_mtx_unlock(&d->lock);
		break;
	case KSANCOV_IOC_NEDGES:
		*(size_t *)data = nedges;
		break;
	case KSANCOV_IOC_TESTPANIC:
		ksancov_testpanic(*(uint64_t *)data);
		break;
	default:
		ret = EINVAL;
		break;
	}

	lck_rw_unlock_shared(ksancov_devs_lck);

	return ret;
}

static int
ksancov_dev_clone(dev_t dev, int action)
{
#pragma unused(dev)
	if (action == DEVFS_CLONE_ALLOC) {
		for (int i = 0; i < KSANCOV_MAX_DEV; i++) {
			if (ksancov_devs[i] == NULL) {
				return i;
			}
		}
	} else if (action == DEVFS_CLONE_FREE) {
		return 0;
	}

	return -1;
}

static const struct cdevsw
    ksancov_cdev = {
	.d_open =  ksancov_open,
	.d_close = ksancov_close,
	.d_ioctl = ksancov_ioctl,

	.d_read = eno_rdwrt,
	.d_write = eno_rdwrt,
	.d_stop = eno_stop,
	.d_reset = eno_reset,
	.d_select = eno_select,
	.d_mmap = eno_mmap,
	.d_strategy = eno_strat,
	.d_type = 0
};

int
ksancov_init_dev(void)
{
	dev_major = cdevsw_add(-1, &ksancov_cdev);
	if (dev_major < 0) {
		printf("ksancov: failed to allocate major device node\n");
		return -1;
	}

	dev_t dev = makedev(dev_major, 0);
	void *node = devfs_make_node_clone(dev, DEVFS_CHAR, UID_ROOT, GID_WHEEL, 0666,
	    ksancov_dev_clone, KSANCOV_DEVNODE);
	if (!node) {
		printf("ksancov: failed to create device node\n");
		return -1;
	}

	/* This could be moved to the first use of /dev/ksancov to save memory */
	uintptr_t buf;
	size_t sz = sizeof(struct ksancov_edgemap) + KSANCOV_MAX_EDGES * sizeof(uint32_t);

	kern_return_t kr = kmem_alloc_flags(kernel_map, &buf, sz, VM_KERN_MEMORY_DIAG, KMA_ZERO);
	if (kr) {
		printf("ksancov: failed to allocate edge addr map\n");
		return -1;
	}

	ksancov_edgemap = (void *)buf;
	ksancov_edgemap->magic = KSANCOV_EDGEMAP_MAGIC;
	ksancov_edgemap->nedges = (uint32_t)nedges;
	ksancov_edgemap->offset = KSANCOV_PC_OFFSET;

	ksancov_devs_lck = lck_rw_alloc_init(&ksancov_lck_grp, LCK_ATTR_NULL);

	return 0;
}
Commit	Line	Data
cb323159 A	1	/*
	2	* Copyright (c) 2019 Apple Inc. All rights reserved.
	3	*
	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
	5	*
	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
	14	*
	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
	25	*
	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
	27	*/
	28
	29	#include <string.h>
	30	#include <stdbool.h>
cb323159 A	31
	32	#include <kern/assert.h>
	33	#include <kern/cpu_data.h>
	34	#include <kern/locks.h>
	35	#include <kern/debug.h>
	36	#include <kern/kalloc.h>
	37	#include <kern/zalloc.h>
	38	#include <kern/task.h>
	39	#include <kern/thread.h>
	40
	41	#include <vm/vm_kern.h>
	42	#include <vm/vm_protos.h>
	43
	44	#include <mach/mach_vm.h>
	45	#include <mach/mach_types.h>
	46	#include <mach/mach_port.h>
	47	#include <mach/vm_map.h>
	48	#include <mach/vm_param.h>
	49	#include <mach/machine/vm_param.h>
cb323159 A	50
	51	#include <sys/stat.h> /* dev_t */
	52	#include <miscfs/devfs/devfs.h> /* must come after sys/stat.h */
	53	#include <sys/conf.h> /* must come after sys/stat.h */
	54
	55	#include <libkern/libkern.h>
f427ee49	56	#include <os/atomic_private.h>
cb323159 A	57	#include <os/overflow.h>
	58
	59	#include <san/ksancov.h>
	60
	61	/* header mess... */
	62	struct uthread;
	63	typedef struct uthread * uthread_t;
	64
	65	#include <sys/sysproto.h>
	66	#include <sys/queue.h>
	67	#include <sys/sysctl.h>
	68
	69	#define USE_PC_TABLE 0
	70	#define KSANCOV_MAX_DEV 64
f427ee49	71	#define KSANCOV_MAX_PCS (1024U * 64) /* default to 256k buffer => 64k pcs */
cb323159 A	72
	73	enum {
	74	KS_MODE_NONE,
	75	KS_MODE_TRACE,
	76	KS_MODE_COUNTERS,
	77	KS_MODE_MAX
	78	};
	79
	80	struct ksancov_dev {
	81	unsigned mode;
	82
	83	union {
	84	struct ksancov_trace *trace;
	85	struct ksancov_counters *counters;
	86	};
	87	size_t sz; /* size of allocated trace/counters buffer */
	88
	89	size_t maxpcs;
	90
	91	thread_t thread;
	92	dev_t dev;
f427ee49	93	lck_mtx_t lock;
cb323159	94	};
f427ee49	95	typedef struct ksancov_dev * ksancov_dev_t;
cb323159	96
f427ee49 A	97	extern boolean_t ml_at_interrupt_context(void);
	98	extern boolean_t ml_get_interrupts_enabled(void);
	99
	100	static void ksancov_detach(ksancov_dev_t);
	101
	102	static int dev_major;
	103	static size_t nedges = 0;
	104	static uint32_t __unused npcs = 0;
	105
	106	static _Atomic unsigned active_devs;
	107
	108	static LCK_GRP_DECLARE(ksancov_lck_grp, "ksancov_lck_grp");
	109	static lck_rw_t *ksancov_devs_lck;
cb323159	110
f427ee49 A	111	/* array of devices indexed by devnode minor */
f427ee49 A	112	static ksancov_dev_t ksancov_devs[KSANCOV_MAX_DEV];
cb323159 A	113	static struct ksancov_edgemap *ksancov_edgemap;
cb323159 A	114
f427ee49 A	115
	116	static ksancov_dev_t
	117	create_dev(dev_t dev)
cb323159	118	{
f427ee49 A	119	ksancov_dev_t d = kalloc_tag(sizeof(struct ksancov_dev), VM_KERN_MEMORY_DIAG);
	120	if (!d) {
	121	return NULL;
	122	}
	123
	124	d->mode = KS_MODE_NONE;
	125	d->trace = NULL;
	126	d->sz = 0;
	127	d->maxpcs = KSANCOV_MAX_PCS;
	128	d->dev = dev;
	129	d->thread = THREAD_NULL;
	130	lck_mtx_init(&d->lock, &ksancov_lck_grp, LCK_ATTR_NULL);
	131
	132	return d;
	133	}
	134
	135	static void
	136	free_dev(ksancov_dev_t d)
	137	{
	138	if (d->mode == KS_MODE_TRACE && d->trace) {
	139	kmem_free(kernel_map, (uintptr_t)d->trace, d->sz);
	140	} else if (d->mode == KS_MODE_COUNTERS && d->counters) {
	141	kmem_free(kernel_map, (uintptr_t)d->counters, d->sz);
	142	}
	143	lck_mtx_destroy(&d->lock, &ksancov_lck_grp);
	144	kfree(d, sizeof(struct ksancov_dev));
cb323159 A	145	}
	146
	147	void
	148	__sanitizer_cov_trace_pc_indirect(void * __unused callee)
	149	{
	150	return;
	151	}
	152
	153	#define GUARD_SEEN (uint32_t)0x80000000
	154	#define GUARD_IDX_MASK (uint32_t)0x0fffffff
	155
	156	static inline void __attribute__((always_inline))
	157	trace_pc_guard(uint32_t guardp, void caller)
	158	{
	159	/* record the pc for this guard */
	160	if (guardp) {
	161	uint32_t gd = *guardp;
	162	if (__improbable(gd && !(gd & GUARD_SEEN) && ksancov_edgemap)) {
	163	size_t idx = gd & GUARD_IDX_MASK;
	164	if (idx < ksancov_edgemap->nedges) {
f427ee49	165	ksancov_edgemap->addrs[idx] = (uint32_t)(VM_KERNEL_UNSLIDE(caller) - KSANCOV_PC_OFFSET - 1);
cb323159 A	166	*guardp \|= GUARD_SEEN;
	167	}
	168	}
	169	}
	170
	171	if (__probable(os_atomic_load(&active_devs, relaxed) == 0)) {
	172	/* early exit when nothing is active */
	173	return;
	174	}
	175
	176	if (ml_at_interrupt_context()) {
	177	return;
	178	}
	179
f427ee49	180	uint32_t pc = (uint32_t)(VM_KERNEL_UNSLIDE(caller) - KSANCOV_PC_OFFSET - 1);
cb323159 A	181
	182	thread_t th = current_thread();
	183	if (__improbable(th == THREAD_NULL)) {
	184	return;
	185	}
	186
f427ee49	187	ksancov_dev_t dev = (ksancov_dev_t )__sanitizer_get_thread_data(th);
cb323159 A	188	if (__probable(dev == NULL)) {
	189	return;
	190	}
	191
	192	if (dev->mode == KS_MODE_TRACE) {
	193	struct ksancov_trace *trace = dev->trace;
	194	if (os_atomic_load(&trace->enabled, relaxed) == 0) {
	195	return;
	196	}
	197
	198	if (os_atomic_load(&trace->head, relaxed) >= dev->maxpcs) {
	199	return; /* overflow */
	200	}
	201
	202	uint32_t idx = os_atomic_inc_orig(&trace->head, relaxed);
	203	if (__improbable(idx >= dev->maxpcs)) {
	204	return;
	205	}
	206
	207	trace->pcs[idx] = pc;
	208	} else {
	209	size_t idx = *guardp & GUARD_IDX_MASK;
	210
	211	struct ksancov_counters *counters = dev->counters;
	212	if (os_atomic_load(&counters->enabled, relaxed) == 0) {
	213	return;
	214	}
	215
	216	/* saturating 8bit add */
	217	if (counters->hits[idx] < KSANCOV_MAX_HITS) {
	218	counters->hits[idx]++;
	219	}
	220	}
	221	}
	222
	223	void __attribute__((noinline))
	224	__sanitizer_cov_trace_pc(void)
	225	{
	226	trace_pc_guard(NULL, __builtin_return_address(0));
	227	}
	228
	229	void __attribute__((noinline))
	230	__sanitizer_cov_trace_pc_guard(uint32_t *guardp)
	231	{
	232	trace_pc_guard(guardp, __builtin_return_address(0));
	233	}
	234
	235	void
	236	__sanitizer_cov_trace_pc_guard_init(uint32_t start, uint32_t stop)
	237	{
	238	/* assign a unique number to each guard */
	239	for (; start != stop; start++) {
	240	if (*start == 0) {
	241	if (nedges < KSANCOV_MAX_EDGES) {
f427ee49	242	*start = (uint32_t)++nedges;
cb323159 A	243	}
	244	}
	245	}
	246	}
	247
	248	void
	249	__sanitizer_cov_pcs_init(uintptr_t start, uintptr_t stop)
	250	{
	251	#if USE_PC_TABLE
	252	static const uintptr_t pc_table_seen_flag = 0x100;
	253
	254	for (; start < stop; start += 2) {
	255	uintptr_t pc = start[0];
	256	uintptr_t flags = start[1];
	257
	258	/*
	259	* This function gets called multiple times on the same range, so mark the
	260	* ones we've seen using unused bits in the flags field.
	261	*/
	262	if (flags & pc_table_seen_flag) {
	263	continue;
	264	}
	265
	266	start[1] \|= pc_table_seen_flag;
	267	assert(npcs < KSANCOV_MAX_EDGES - 1);
	268	edge_addrs[++npcs] = pc;
	269	}
	270	#else
	271	(void)start;
	272	(void)stop;
	273	#endif
	274	}
	275
	276	static void *
	277	ksancov_do_map(uintptr_t base, size_t sz, vm_prot_t prot)
	278	{
	279	kern_return_t kr;
	280	mach_port_t mem_entry = MACH_PORT_NULL;
	281	mach_vm_address_t user_addr = 0;
	282	memory_object_size_t size = sz;
	283
	284	kr = mach_make_memory_entry_64(kernel_map,
	285	&size,
	286	(mach_vm_offset_t)base,
	287	MAP_MEM_VM_SHARE \| prot,
	288	&mem_entry,
	289	MACH_PORT_NULL);
	290	if (kr != KERN_SUCCESS) {
	291	return NULL;
	292	}
	293
	294	kr = mach_vm_map_kernel(get_task_map(current_task()),
	295	&user_addr,
	296	size,
	297	0,
	298	VM_FLAGS_ANYWHERE,
	299	VM_MAP_KERNEL_FLAGS_NONE,
	300	VM_KERN_MEMORY_NONE,
	301	mem_entry,
	302	0,
	303	FALSE,
	304	prot,
	305	prot,
	306	VM_INHERIT_SHARE);
307
308	/*
309	* At this point, either vm_map() has taken a reference on the memory entry
310	* and we can release our local reference, or the map failed and the entry
311	* needs to be freed.
312	*/
313	mach_memory_entry_port_release(mem_entry);
314
315	if (kr != KERN_SUCCESS) {
316	return NULL;
317	}
318
319	return (void *)user_addr;
320	}
321
322	/*
323	* map the sancov buffer into the current process
324	*/
325	static int
f427ee49	326	ksancov_map(ksancov_dev_t d, uintptr_t bufp, size_t sizep)
cb323159	327	{
cb323159 A	328	uintptr_t addr;
	329	size_t size = d->sz;
	330
	331	if (d->mode == KS_MODE_TRACE) {
	332	if (!d->trace) {
	333	return EINVAL;
	334	}
	335	addr = (uintptr_t)d->trace;
	336	} else if (d->mode == KS_MODE_COUNTERS) {
	337	if (!d->counters) {
	338	return EINVAL;
	339	}
	340	addr = (uintptr_t)d->counters;
	341	} else {
	342	return EINVAL; /* not configured */
	343	}
	344
	345	void *buf = ksancov_do_map(addr, size, VM_PROT_READ \| VM_PROT_WRITE);
	346	if (buf == NULL) {
	347	return ENOMEM;
	348	}
	349
f427ee49	350	*bufp = (uintptr_t)buf;
cb323159	351	*sizep = size;
f427ee49	352
cb323159 A	353	return 0;
	354	}
	355
	356	/*
	357	* map the edge -> pc mapping as read-only
	358	*/
	359	static int
f427ee49	360	ksancov_map_edgemap(uintptr_t bufp, size_t sizep)
cb323159	361	{
cb323159 A	362	uintptr_t addr = (uintptr_t)ksancov_edgemap;
	363	size_t size = sizeof(struct ksancov_edgemap) + ksancov_edgemap->nedges * sizeof(uint32_t);
	364
	365	void *buf = ksancov_do_map(addr, size, VM_PROT_READ);
	366	if (buf == NULL) {
	367	return ENOMEM;
	368	}
	369
f427ee49	370	*bufp = (uintptr_t)buf;
cb323159 A	371	*sizep = size;
	372	return 0;
	373	}
	374
cb323159 A	375	/*
	376	* Device node management
	377	*/
	378
	379	static int
	380	ksancov_open(dev_t dev, int flags, int devtype, proc_t p)
	381	{
	382	#pragma unused(flags,devtype,p)
f427ee49 A	383	const int minor_num = minor(dev);
	384
	385	if (minor_num >= KSANCOV_MAX_DEV) {
	386	return EBUSY;
	387	}
	388
	389	lck_rw_lock_exclusive(ksancov_devs_lck);
	390
	391	if (ksancov_devs[minor_num]) {
	392	lck_rw_unlock_exclusive(ksancov_devs_lck);
cb323159 A	393	return EBUSY;
	394	}
	395
f427ee49	396	ksancov_dev_t d = create_dev(dev);
cb323159	397	if (!d) {
f427ee49	398	lck_rw_unlock_exclusive(ksancov_devs_lck);
cb323159 A	399	return ENOMEM;
cb323159 A	400	}
f427ee49	401	ksancov_devs[minor_num] = d;
cb323159	402
f427ee49	403	lck_rw_unlock_exclusive(ksancov_devs_lck);
cb323159 A	404
	405	return 0;
	406	}
	407
	408	static int
f427ee49	409	ksancov_trace_alloc(ksancov_dev_t d, size_t maxpcs)
cb323159	410	{
cb323159 A	411	if (d->mode != KS_MODE_NONE) {
	412	return EBUSY; /* trace/counters already created */
	413	}
	414	assert(d->trace == NULL);
	415
	416	uintptr_t buf;
	417	size_t sz;
	418	if (os_mul_and_add_overflow(maxpcs, sizeof(uint32_t), sizeof(struct ksancov_trace), &sz)) {
	419	return EINVAL;
	420	}
	421
	422	/* allocate the shared memory buffer */
	423	kern_return_t kr = kmem_alloc_flags(kernel_map, &buf, sz, VM_KERN_MEMORY_DIAG, KMA_ZERO);
	424	if (kr != KERN_SUCCESS) {
	425	return ENOMEM;
	426	}
	427
	428	struct ksancov_trace trace = (struct ksancov_trace )buf;
	429	trace->magic = KSANCOV_TRACE_MAGIC;
f427ee49 A	430	trace->offset = KSANCOV_PC_OFFSET;
	431	os_atomic_init(&trace->head, 0);
	432	os_atomic_init(&trace->enabled, 0);
	433	trace->maxpcs = (uint32_t)maxpcs;
cb323159 A	434
	435	d->trace = trace;
	436	d->sz = sz;
	437	d->maxpcs = maxpcs;
	438	d->mode = KS_MODE_TRACE;
	439
	440	return 0;
	441	}
	442
	443	static int
f427ee49	444	ksancov_counters_alloc(ksancov_dev_t d)
cb323159	445	{
cb323159 A	446	if (d->mode != KS_MODE_NONE) {
	447	return EBUSY; /* trace/counters already created */
	448	}
	449	assert(d->counters == NULL);
	450
	451	uintptr_t buf;
	452	size_t sz = sizeof(struct ksancov_counters) + ksancov_edgemap->nedges * sizeof(uint8_t);
	453
	454	/* allocate the shared memory buffer */
	455	kern_return_t kr = kmem_alloc_flags(kernel_map, &buf, sz, VM_KERN_MEMORY_DIAG, KMA_ZERO);
	456	if (kr != KERN_SUCCESS) {
	457	return ENOMEM;
	458	}
	459
	460	struct ksancov_counters counters = (struct ksancov_counters )buf;
	461	counters->magic = KSANCOV_COUNTERS_MAGIC;
	462	counters->nedges = ksancov_edgemap->nedges;
f427ee49	463	os_atomic_init(&counters->enabled, 0);
cb323159 A	464
	465	d->counters = counters;
	466	d->sz = sz;
	467	d->mode = KS_MODE_COUNTERS;
	468
	469	return 0;
	470	}
	471
	472	/*
	473	* attach a thread to a ksancov dev instance
	474	*/
	475	static int
f427ee49	476	ksancov_attach(ksancov_dev_t d, thread_t th)
cb323159	477	{
f427ee49 A	478	if (d->mode == KS_MODE_NONE) {
f427ee49 A	479	return EINVAL; /* not configured */
cb323159 A	480	}
	481
	482	if (th != current_thread()) {
	483	/* can only attach to self presently */
	484	return EINVAL;
	485	}
	486
f427ee49	487	ksancov_dev_t devp = (void )__sanitizer_get_thread_data(th);
cb323159 A	488	if (*devp) {
	489	return EBUSY; /* one dev per thread */
	490	}
	491
f427ee49 A	492	if (d->thread != THREAD_NULL) {
	493	ksancov_detach(d);
	494	}
	495
cb323159 A	496	d->thread = th;
	497	thread_reference(d->thread);
	498
	499	os_atomic_store(devp, d, relaxed);
	500	os_atomic_add(&active_devs, 1, relaxed);
	501
	502	return 0;
	503	}
	504
	505	extern void
	506	thread_wait(
	507	thread_t thread,
	508	boolean_t until_not_runnable);
	509
	510
	511	/*
	512	* disconnect thread from ksancov dev
	513	*/
f427ee49 A	514	static void
f427ee49 A	515	ksancov_detach(ksancov_dev_t d)
cb323159	516	{
cb323159 A	517	if (d->thread == THREAD_NULL) {
cb323159 A	518	/* no thread attached */
f427ee49	519	return;
cb323159 A	520	}
	521
	522	/* disconnect dev from thread */
f427ee49	523	ksancov_dev_t devp = (void )__sanitizer_get_thread_data(d->thread);
cb323159 A	524	if (*devp != NULL) {
	525	assert(*devp == d);
	526	os_atomic_store(devp, NULL, relaxed);
	527	}
	528
	529	if (d->thread != current_thread()) {
	530	/* wait until it's safe to yank */
	531	thread_wait(d->thread, TRUE);
	532	}
	533
	534	/* drop our thread reference */
	535	thread_deallocate(d->thread);
	536	d->thread = THREAD_NULL;
cb323159 A	537	}
	538
	539	static int
	540	ksancov_close(dev_t dev, int flags, int devtype, proc_t p)
	541	{
	542	#pragma unused(flags,devtype,p)
f427ee49	543	const int minor_num = minor(dev);
cb323159	544
f427ee49 A	545	lck_rw_lock_exclusive(ksancov_devs_lck);
	546	ksancov_dev_t d = ksancov_devs[minor_num];
	547	ksancov_devs[minor_num] = NULL; /* dev no longer discoverable */
	548	lck_rw_unlock_exclusive(ksancov_devs_lck);
cb323159	549
f427ee49 A	550	/*
	551	* No need to lock d here as there is and will be no one having its
	552	* reference except for this thread and the one which is going to
	553	* be detached below.
	554	*/
cb323159	555
f427ee49 A	556	if (!d) {
f427ee49 A	557	return ENXIO;
cb323159 A	558	}
cb323159 A	559
f427ee49 A	560	if (d->mode == KS_MODE_TRACE && d->trace) {
	561	os_atomic_sub(&active_devs, 1, relaxed);
	562	os_atomic_store(&d->trace->enabled, 0, relaxed); /* stop tracing */
	563	} else if (d->mode == KS_MODE_COUNTERS && d->counters) {
	564	os_atomic_sub(&active_devs, 1, relaxed);
	565	os_atomic_store(&d->counters->enabled, 0, relaxed); /* stop tracing */
	566	}
cb323159	567
f427ee49 A	568	ksancov_detach(d);
f427ee49 A	569	free_dev(d);
cb323159 A	570
	571	return 0;
	572	}
	573
	574	static void
	575	ksancov_testpanic(volatile uint64_t guess)
	576	{
	577	const uint64_t tgt = 0xf85de3b12891c817UL;
	578
	579	#define X(n) ((tgt & (0xfUL << (4n))) == (guess & (0xfUL << (4n))))
	580
	581	if (X(0)) {
	582	if (X(1)) {
	583	if (X(2)) {
	584	if (X(3)) {
	585	if (X(4)) {
	586	if (X(5)) {
	587	if (X(6)) {
	588	if (X(7)) {
	589	if (X(8)) {
	590	if (X(9)) {
	591	if (X(10)) {
	592	if (X(11)) {
	593	if (X(12)) {
	594	if (X(13)) {
	595	if (X(14)) {
	596	if (X(15)) {
	597	panic("ksancov: found test value\n");
	598	}
	599	}
	600	}
	601	}
	602	}
	603	}
	604	}
	605	}
	606	}
	607	}
	608	}
	609	}
	610	}
	611	}
	612	}
	613	}
	614	}
	615
	616	static int
	617	ksancov_ioctl(dev_t dev, unsigned long cmd, caddr_t _data, int fflag, proc_t p)
	618	{
	619	#pragma unused(fflag,p)
f427ee49	620	struct ksancov_buf_desc *mcmd;
cb323159 A	621	void data = (void )_data;
cb323159 A	622
f427ee49 A	623	lck_rw_lock_shared(ksancov_devs_lck);
f427ee49 A	624	ksancov_dev_t d = ksancov_devs[minor(dev)];
cb323159	625	if (!d) {
f427ee49 A	626	lck_rw_unlock_shared(ksancov_devs_lck);
f427ee49 A	627	return EINVAL; /* dev not open */
cb323159 A	628	}
cb323159 A	629
f427ee49	630	int ret = 0;
cb323159	631
f427ee49 A	632	switch (cmd) {
	633	case KSANCOV_IOC_TRACE:
	634	lck_mtx_lock(&d->lock);
	635	ret = ksancov_trace_alloc(d, (size_t )data);
	636	lck_mtx_unlock(&d->lock);
	637	break;
	638	case KSANCOV_IOC_COUNTERS:
	639	lck_mtx_lock(&d->lock);
	640	ret = ksancov_counters_alloc(d);
	641	lck_mtx_unlock(&d->lock);
	642	break;
	643	case KSANCOV_IOC_MAP:
	644	mcmd = (struct ksancov_buf_desc *)data;
	645	lck_mtx_lock(&d->lock);
	646	ret = ksancov_map(d, &mcmd->ptr, &mcmd->sz);
	647	lck_mtx_unlock(&d->lock);
	648	break;
	649	case KSANCOV_IOC_MAP_EDGEMAP:
	650	mcmd = (struct ksancov_buf_desc *)data;
	651	ret = ksancov_map_edgemap(&mcmd->ptr, &mcmd->sz);
	652	break;
	653	case KSANCOV_IOC_START:
	654	lck_mtx_lock(&d->lock);
	655	ret = ksancov_attach(d, current_thread());
	656	lck_mtx_unlock(&d->lock);
	657	break;
	658	case KSANCOV_IOC_NEDGES:
	659	(size_t )data = nedges;
	660	break;
	661	case KSANCOV_IOC_TESTPANIC:
	662	ksancov_testpanic((uint64_t )data);
	663	break;
	664	default:
	665	ret = EINVAL;
	666	break;
	667	}
	668
	669	lck_rw_unlock_shared(ksancov_devs_lck);
cb323159 A	670
	671	return ret;
	672	}
	673
	674	static int
	675	ksancov_dev_clone(dev_t dev, int action)
	676	{
	677	#pragma unused(dev)
	678	if (action == DEVFS_CLONE_ALLOC) {
f427ee49	679	for (int i = 0; i < KSANCOV_MAX_DEV; i++) {
cb323159 A	680	if (ksancov_devs[i] == NULL) {
	681	return i;
	682	}
	683	}
	684	} else if (action == DEVFS_CLONE_FREE) {
	685	return 0;
	686	}
	687
	688	return -1;
	689	}
	690
f427ee49	691	static const struct cdevsw
cb323159 A	692	ksancov_cdev = {
	693	.d_open = ksancov_open,
	694	.d_close = ksancov_close,
	695	.d_ioctl = ksancov_ioctl,
	696
	697	.d_read = eno_rdwrt,
	698	.d_write = eno_rdwrt,
	699	.d_stop = eno_stop,
	700	.d_reset = eno_reset,
	701	.d_select = eno_select,
	702	.d_mmap = eno_mmap,
	703	.d_strategy = eno_strat,
	704	.d_type = 0
	705	};
	706
	707	int
	708	ksancov_init_dev(void)
	709	{
	710	dev_major = cdevsw_add(-1, &ksancov_cdev);
	711	if (dev_major < 0) {
	712	printf("ksancov: failed to allocate major device node\n");
	713	return -1;
	714	}
	715
	716	dev_t dev = makedev(dev_major, 0);
	717	void *node = devfs_make_node_clone(dev, DEVFS_CHAR, UID_ROOT, GID_WHEEL, 0666,
	718	ksancov_dev_clone, KSANCOV_DEVNODE);
	719	if (!node) {
	720	printf("ksancov: failed to create device node\n");
	721	return -1;
	722	}
	723
	724	/* This could be moved to the first use of /dev/ksancov to save memory */
	725	uintptr_t buf;
	726	size_t sz = sizeof(struct ksancov_edgemap) + KSANCOV_MAX_EDGES * sizeof(uint32_t);
	727
	728	kern_return_t kr = kmem_alloc_flags(kernel_map, &buf, sz, VM_KERN_MEMORY_DIAG, KMA_ZERO);
	729	if (kr) {
	730	printf("ksancov: failed to allocate edge addr map\n");
	731	return -1;
	732	}
	733
	734	ksancov_edgemap = (void *)buf;
	735	ksancov_edgemap->magic = KSANCOV_EDGEMAP_MAGIC;
f427ee49 A	736	ksancov_edgemap->nedges = (uint32_t)nedges;
	737	ksancov_edgemap->offset = KSANCOV_PC_OFFSET;
	738
	739	ksancov_devs_lck = lck_rw_alloc_init(&ksancov_lck_grp, LCK_ATTR_NULL);
cb323159 A	740
	741	return 0;
	742	}