[apple/xnu.git] / osfmk / arm64 / copyio.c

/*
 * Copyright (c) 2012-2013 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 *
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */

#include <arm/cpu_data_internal.h>
#include <arm/misc_protos.h>
#include <kern/thread.h>
#include <sys/errno.h>
#include <vm/pmap.h>
#include <vm/vm_map.h>
#include <san/kasan.h>

#undef copyin
#undef copyout

extern int _bcopyin(const char *src, char *dst, vm_size_t len);
extern int _bcopyinstr(const char *src, char *dst, vm_size_t max, vm_size_t *actual);
extern int _bcopyout(const char *src, char *dst, vm_size_t len);
extern int _copyin_word(const char *src, uint64_t *dst, vm_size_t len);

extern pmap_t kernel_pmap;

/* On by default, optionally disabled by boot-arg */
extern boolean_t copyio_zalloc_check;

typedef enum copyio_type {
	COPYIO_IN,
	COPYIO_IN_WORD,
	COPYIO_INSTR,
	COPYIO_OUT,
} copyio_type_t;

static inline void
user_access_enable(void)
{
#if __ARM_PAN_AVAILABLE__
	__builtin_arm_wsr("pan", 0);
#endif  /* __ARM_PAN_AVAILABLE__ */
}

static inline void
user_access_disable(void)
{
#if __ARM_PAN_AVAILABLE__
	__builtin_arm_wsr("pan", 1);
#endif  /* __ARM_PAN_AVAILABLE__ */
}

static int
copyio(copyio_type_t copytype, const char *src, char *dst,
	   vm_size_t nbytes, vm_size_t *lencopied)
{
	int result = 0;
	vm_size_t bytes_copied = 0;
	vm_size_t kernel_buf_size = 0;
	void * kernel_addr = NULL;

	/* Reject TBI addresses */
	if (copytype == COPYIO_OUT) {
		if ((uintptr_t)dst & TBI_MASK)
			return EINVAL;
	} else {
		if ((uintptr_t)src & TBI_MASK)
			return EINVAL;
	}

	if (__probable(copyio_zalloc_check)) {
		if (copytype == COPYIO_IN || copytype == COPYIO_INSTR || copytype == COPYIO_IN_WORD) {
			kernel_addr = (void*)dst;
		} else if (copytype == COPYIO_OUT) {
			kernel_addr = (void*)(uintptr_t)src;
		}
		if (kernel_addr)
			kernel_buf_size = zone_element_size(kernel_addr, NULL);
		if (__improbable(kernel_buf_size && kernel_buf_size < nbytes))
			panic("copyio: kernel buffer %p has size %lu < nbytes %lu", kernel_addr, kernel_buf_size, nbytes);
	}

#if KASAN
	/* For user copies, asan-check the kernel-side buffer */
	if (copytype == COPYIO_IN || copytype == COPYIO_INSTR || copytype == COPYIO_IN_WORD) {
		__asan_storeN((uintptr_t)dst, nbytes);
	} else if (copytype == COPYIO_OUT) {
		__asan_loadN((uintptr_t)src, nbytes);
	}
#endif

	user_access_enable();

	/* Select copy routines based on direction:
	 *   COPYIO_IN - Use unprivileged loads to read from user address
	 *   COPYIO_OUT - Use unprivleged stores to write to user address
	 */

	switch (copytype) {
	case COPYIO_IN:
		result = _bcopyin(src, dst, nbytes);
		break;
	case COPYIO_INSTR:
		result = _bcopyinstr(src, dst, nbytes, &bytes_copied);
		if (result != EFAULT) {
			*lencopied = bytes_copied;
		}
		break;
	case COPYIO_IN_WORD:
		result = _copyin_word(src, (uint64_t *)(uintptr_t)dst, nbytes);
		break;
	case COPYIO_OUT:
		result = _bcopyout(src, dst, nbytes);
		break;
	default:
		result = EINVAL;
	}

	user_access_disable();
	return result;
}

int
copyin_kern(const user_addr_t user_addr, char *kernel_addr, vm_size_t nbytes)
{
	bcopy((const char*)(uintptr_t)user_addr, kernel_addr, nbytes);

	return 0;
}

int
copyout_kern(const char *kernel_addr, user_addr_t user_addr, vm_size_t nbytes)
{
	bcopy(kernel_addr, (char *)(uintptr_t)user_addr, nbytes);

	return 0;
}

int
copyin(const user_addr_t user_addr, void *kernel_addr, vm_size_t nbytes)
{
	int result;

	if (nbytes == 0)
		return 0;

	result = copyin_validate(user_addr, (uintptr_t)kernel_addr, nbytes);
	if (result) return result;

	if (current_thread()->map->pmap == kernel_pmap)
		return copyin_kern(user_addr, kernel_addr, nbytes);
	else
		return copyio(COPYIO_IN, (const char *)(uintptr_t)user_addr, kernel_addr, nbytes, NULL);
}

/*
 * copyin_word
 * Read an aligned value from userspace as a single memory transaction.
 * This function supports userspace synchronization features
 */
int
copyin_word(const user_addr_t user_addr, uint64_t *kernel_addr, vm_size_t nbytes)
{
	int			result;

	/* Verify sizes */
	if ((nbytes != 4) && (nbytes != 8))
		return EINVAL;

	/* Test alignment */
	if (user_addr & (nbytes - 1))
		return EINVAL;

	result = copyin_validate(user_addr, (uintptr_t)kernel_addr, nbytes);
	if (result)
		return result;

	return copyio(COPYIO_IN_WORD, (const char *)user_addr, (char *)(uintptr_t)kernel_addr, nbytes, NULL);
}

int
copyinstr(const user_addr_t user_addr, char *kernel_addr, vm_size_t nbytes, vm_size_t *lencopied)
{
	int result;

	*lencopied = 0;
	if (nbytes == 0)
		return ENAMETOOLONG;

	result = copyin_validate(user_addr, (uintptr_t)kernel_addr, nbytes);

	if (result) return result;

	return copyio(COPYIO_INSTR, (const char *)(uintptr_t)user_addr, kernel_addr, nbytes, lencopied);
}

int
copyout(const void *kernel_addr, user_addr_t user_addr, vm_size_t nbytes)
{
	int result;

	if (nbytes == 0)
		return 0;

	result = copyout_validate((uintptr_t)kernel_addr, user_addr, nbytes);
	if (result) return result;

	if (current_thread()->map->pmap == kernel_pmap)
		return copyout_kern(kernel_addr, user_addr, nbytes);
	else
		return copyio(COPYIO_OUT, kernel_addr, (char *)(uintptr_t)user_addr, nbytes, NULL);
}


/*
 * Copy sizes bigger than this value will cause a kernel panic.
 *
 * Yes, this is an arbitrary fixed limit, but it's almost certainly
 * a programming error to be copying more than this amount between
 * user and wired kernel memory in a single invocation on this
 * platform.
 */
const int copysize_limit_panic = (64 * 1024 * 1024);

/*
 * Validate the arguments to copy{in,out} on this platform.
 */
static int
copy_validate(const user_addr_t user_addr,
	uintptr_t kernel_addr, vm_size_t nbytes)
{
	uintptr_t kernel_addr_last = kernel_addr + nbytes;

	if (__improbable(kernel_addr < VM_MIN_KERNEL_ADDRESS ||
	    kernel_addr > VM_MAX_KERNEL_ADDRESS ||
	    kernel_addr_last < kernel_addr ||
	    kernel_addr_last > VM_MAX_KERNEL_ADDRESS))
		panic("%s(%p, %p, %lu) - kaddr not in kernel", __func__,
		       (void *)user_addr, (void *)kernel_addr, nbytes);

	user_addr_t user_addr_last = user_addr + nbytes;

	if (__improbable((user_addr_last < user_addr) || ((user_addr + nbytes) > vm_map_max(current_thread()->map)) ||
	    (user_addr < vm_map_min(current_thread()->map))))
		return (EFAULT);

	if (__improbable(nbytes > copysize_limit_panic))
		panic("%s(%p, %p, %lu) - transfer too large", __func__,
		       (void *)user_addr, (void *)kernel_addr, nbytes);

	return (0);
}

int
copyin_validate(const user_addr_t ua, uintptr_t ka, vm_size_t nbytes)
{
	return (copy_validate(ua, ka, nbytes));
}

int
copyout_validate(uintptr_t ka, const user_addr_t ua, vm_size_t nbytes)
{
	return (copy_validate(ua, ka, nbytes));
}
Commit	Line	Data
5ba3f43e A	1	/*
	2	* Copyright (c) 2012-2013 Apple Inc. All rights reserved.
	3	*
	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
	5	*
	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
	14	*
	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
	25	*
	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
	27	*/
	28
	29	#include <arm/cpu_data_internal.h>
	30	#include <arm/misc_protos.h>
	31	#include <kern/thread.h>
	32	#include <sys/errno.h>
	33	#include <vm/pmap.h>
	34	#include <vm/vm_map.h>
	35	#include <san/kasan.h>
	36
d9a64523 A	37	#undef copyin
	38	#undef copyout
	39
5ba3f43e A	40	extern int _bcopyin(const char src, char dst, vm_size_t len);
	41	extern int _bcopyinstr(const char src, char dst, vm_size_t max, vm_size_t *actual);
	42	extern int _bcopyout(const char src, char dst, vm_size_t len);
	43	extern int _copyin_word(const char src, uint64_t dst, vm_size_t len);
	44
	45	extern pmap_t kernel_pmap;
5ba3f43e	46
d9a64523 A	47	/* On by default, optionally disabled by boot-arg */
	48	extern boolean_t copyio_zalloc_check;
	49
5ba3f43e A	50	typedef enum copyio_type {
	51	COPYIO_IN,
	52	COPYIO_IN_WORD,
	53	COPYIO_INSTR,
	54	COPYIO_OUT,
	55	} copyio_type_t;
	56
5ba3f43e A	57	static inline void
	58	user_access_enable(void)
	59	{
	60	#if __ARM_PAN_AVAILABLE__
cc8bc92a	61	__builtin_arm_wsr("pan", 0);
5ba3f43e A	62	#endif /* __ARM_PAN_AVAILABLE__ */
	63	}
	64
	65	static inline void
	66	user_access_disable(void)
	67	{
	68	#if __ARM_PAN_AVAILABLE__
cc8bc92a	69	__builtin_arm_wsr("pan", 1);
5ba3f43e A	70	#endif /* __ARM_PAN_AVAILABLE__ */
	71	}
	72
	73	static int
	74	copyio(copyio_type_t copytype, const char src, char dst,
	75	vm_size_t nbytes, vm_size_t *lencopied)
	76	{
	77	int result = 0;
	78	vm_size_t bytes_copied = 0;
d9a64523 A	79	vm_size_t kernel_buf_size = 0;
d9a64523 A	80	void * kernel_addr = NULL;
5ba3f43e A	81
	82	/* Reject TBI addresses */
	83	if (copytype == COPYIO_OUT) {
	84	if ((uintptr_t)dst & TBI_MASK)
	85	return EINVAL;
	86	} else {
	87	if ((uintptr_t)src & TBI_MASK)
	88	return EINVAL;
	89	}
	90
d9a64523 A	91	if (__probable(copyio_zalloc_check)) {
	92	if (copytype == COPYIO_IN \|\| copytype == COPYIO_INSTR \|\| copytype == COPYIO_IN_WORD) {
	93	kernel_addr = (void*)dst;
	94	} else if (copytype == COPYIO_OUT) {
	95	kernel_addr = (void*)(uintptr_t)src;
	96	}
	97	if (kernel_addr)
	98	kernel_buf_size = zone_element_size(kernel_addr, NULL);
	99	if (__improbable(kernel_buf_size && kernel_buf_size < nbytes))
	100	panic("copyio: kernel buffer %p has size %lu < nbytes %lu", kernel_addr, kernel_buf_size, nbytes);
5ba3f43e A	101	}
	102
	103	#if KASAN
	104	/* For user copies, asan-check the kernel-side buffer */
	105	if (copytype == COPYIO_IN \|\| copytype == COPYIO_INSTR \|\| copytype == COPYIO_IN_WORD) {
	106	__asan_storeN((uintptr_t)dst, nbytes);
	107	} else if (copytype == COPYIO_OUT) {
	108	__asan_loadN((uintptr_t)src, nbytes);
	109	}
	110	#endif
	111
cc8bc92a	112	user_access_enable();
5ba3f43e A	113
	114	/* Select copy routines based on direction:
	115	* COPYIO_IN - Use unprivileged loads to read from user address
	116	* COPYIO_OUT - Use unprivleged stores to write to user address
	117	*/
	118
	119	switch (copytype) {
	120	case COPYIO_IN:
	121	result = _bcopyin(src, dst, nbytes);
	122	break;
	123	case COPYIO_INSTR:
	124	result = _bcopyinstr(src, dst, nbytes, &bytes_copied);
	125	if (result != EFAULT) {
	126	*lencopied = bytes_copied;
	127	}
	128	break;
	129	case COPYIO_IN_WORD:
	130	result = _copyin_word(src, (uint64_t *)(uintptr_t)dst, nbytes);
	131	break;
	132	case COPYIO_OUT:
	133	result = _bcopyout(src, dst, nbytes);
	134	break;
	135	default:
	136	result = EINVAL;
	137	}
	138
cc8bc92a	139	user_access_disable();
5ba3f43e A	140	return result;
	141	}
	142
	143	int
	144	copyin_kern(const user_addr_t user_addr, char *kernel_addr, vm_size_t nbytes)
	145	{
	146	bcopy((const char*)(uintptr_t)user_addr, kernel_addr, nbytes);
	147
	148	return 0;
	149	}
	150
	151	int
	152	copyout_kern(const char *kernel_addr, user_addr_t user_addr, vm_size_t nbytes)
	153	{
	154	bcopy(kernel_addr, (char *)(uintptr_t)user_addr, nbytes);
	155
	156	return 0;
	157	}
	158
	159	int
d9a64523	160	copyin(const user_addr_t user_addr, void *kernel_addr, vm_size_t nbytes)
5ba3f43e A	161	{
	162	int result;
	163
d9a64523 A	164	if (nbytes == 0)
d9a64523 A	165	return 0;
5ba3f43e	166
d9a64523	167	result = copyin_validate(user_addr, (uintptr_t)kernel_addr, nbytes);
5ba3f43e A	168	if (result) return result;
5ba3f43e A	169
d9a64523 A	170	if (current_thread()->map->pmap == kernel_pmap)
	171	return copyin_kern(user_addr, kernel_addr, nbytes);
	172	else
	173	return copyio(COPYIO_IN, (const char *)(uintptr_t)user_addr, kernel_addr, nbytes, NULL);
5ba3f43e A	174	}
	175
	176	/*
	177	* copyin_word
	178	* Read an aligned value from userspace as a single memory transaction.
	179	* This function supports userspace synchronization features
	180	*/
	181	int
	182	copyin_word(const user_addr_t user_addr, uint64_t *kernel_addr, vm_size_t nbytes)
	183	{
	184	int result;
	185
	186	/* Verify sizes */
	187	if ((nbytes != 4) && (nbytes != 8))
	188	return EINVAL;
	189
	190	/* Test alignment */
	191	if (user_addr & (nbytes - 1))
	192	return EINVAL;
	193
d9a64523	194	result = copyin_validate(user_addr, (uintptr_t)kernel_addr, nbytes);
5ba3f43e A	195	if (result)
	196	return result;
	197
	198	return copyio(COPYIO_IN_WORD, (const char )user_addr, (char )(uintptr_t)kernel_addr, nbytes, NULL);
	199	}
	200
	201	int
	202	copyinstr(const user_addr_t user_addr, char kernel_addr, vm_size_t nbytes, vm_size_t lencopied)
	203	{
	204	int result;
	205
d9a64523 A	206	*lencopied = 0;
	207	if (nbytes == 0)
	208	return ENAMETOOLONG;
5ba3f43e	209
d9a64523	210	result = copyin_validate(user_addr, (uintptr_t)kernel_addr, nbytes);
5ba3f43e A	211
	212	if (result) return result;
	213
5ba3f43e A	214	return copyio(COPYIO_INSTR, (const char *)(uintptr_t)user_addr, kernel_addr, nbytes, lencopied);
	215	}
	216
	217	int
	218	copyout(const void *kernel_addr, user_addr_t user_addr, vm_size_t nbytes)
	219	{
	220	int result;
	221
d9a64523 A	222	if (nbytes == 0)
d9a64523 A	223	return 0;
5ba3f43e	224
d9a64523	225	result = copyout_validate((uintptr_t)kernel_addr, user_addr, nbytes);
5ba3f43e A	226	if (result) return result;
5ba3f43e A	227
d9a64523 A	228	if (current_thread()->map->pmap == kernel_pmap)
	229	return copyout_kern(kernel_addr, user_addr, nbytes);
	230	else
	231	return copyio(COPYIO_OUT, kernel_addr, (char *)(uintptr_t)user_addr, nbytes, NULL);
5ba3f43e A	232	}
	233
	234
	235	/*
	236	* Copy sizes bigger than this value will cause a kernel panic.
	237	*
	238	* Yes, this is an arbitrary fixed limit, but it's almost certainly
	239	* a programming error to be copying more than this amount between
	240	* user and wired kernel memory in a single invocation on this
	241	* platform.
	242	*/
	243	const int copysize_limit_panic = (64 * 1024 * 1024);
	244
	245	/*
	246	* Validate the arguments to copy{in,out} on this platform.
5ba3f43e A	247	*/
	248	static int
	249	copy_validate(const user_addr_t user_addr,
	250	uintptr_t kernel_addr, vm_size_t nbytes)
	251	{
	252	uintptr_t kernel_addr_last = kernel_addr + nbytes;
	253
d9a64523	254	if (__improbable(kernel_addr < VM_MIN_KERNEL_ADDRESS \|\|
5ba3f43e A	255	kernel_addr > VM_MAX_KERNEL_ADDRESS \|\|
5ba3f43e A	256	kernel_addr_last < kernel_addr \|\|
d9a64523	257	kernel_addr_last > VM_MAX_KERNEL_ADDRESS))
5ba3f43e A	258	panic("%s(%p, %p, %lu) - kaddr not in kernel", __func__,
	259	(void )user_addr, (void )kernel_addr, nbytes);
	260
	261	user_addr_t user_addr_last = user_addr + nbytes;
	262
d9a64523 A	263	if (__improbable((user_addr_last < user_addr) \|\| ((user_addr + nbytes) > vm_map_max(current_thread()->map)) \|\|
d9a64523 A	264	(user_addr < vm_map_min(current_thread()->map))))
5ba3f43e A	265	return (EFAULT);
	266
	267	if (__improbable(nbytes > copysize_limit_panic))
	268	panic("%s(%p, %p, %lu) - transfer too large", __func__,
	269	(void )user_addr, (void )kernel_addr, nbytes);
	270
	271	return (0);
	272	}
	273
	274	int
	275	copyin_validate(const user_addr_t ua, uintptr_t ka, vm_size_t nbytes)
	276	{
	277	return (copy_validate(ua, ka, nbytes));
	278	}
	279
	280	int
	281	copyout_validate(uintptr_t ka, const user_addr_t ua, vm_size_t nbytes)
	282	{
	283	return (copy_validate(ua, ka, nbytes));
	284	}
	285