[apple/xnu.git] / osfmk / arm / loose_ends.c

/*
 * Copyright (c) 2007-2016 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 *
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */

#include <mach_assert.h>
#include <mach/vm_types.h>
#include <mach/mach_time.h>
#include <kern/timer.h>
#include <kern/clock.h>
#include <kern/machine.h>
#include <mach/machine.h>
#include <mach/machine/vm_param.h>
#include <mach_kdp.h>
#include <kdp/kdp_udp.h>
#if !MACH_KDP
#include <kdp/kdp_callout.h>
#endif /* !MACH_KDP */
#include <arm/cpu_data.h>
#include <arm/cpu_data_internal.h>
#include <arm/caches_internal.h>

#include <vm/vm_kern.h>
#include <vm/vm_map.h>
#include <vm/pmap.h>

#include <arm/misc_protos.h>

#include <sys/errno.h>

#define INT_SIZE        (BYTE_SIZE * sizeof (int))


void
bcopy_phys(addr64_t src, addr64_t dst, vm_size_t bytes)
{
	unsigned int    src_index;
	unsigned int    dst_index;
	vm_offset_t     src_offset;
	vm_offset_t     dst_offset;
	unsigned int    cpu_num;
	unsigned int    wimg_bits_src, wimg_bits_dst;
	ppnum_t         pn_src = (src >> PAGE_SHIFT);
	ppnum_t         pn_dst = (dst >> PAGE_SHIFT);

	wimg_bits_src = pmap_cache_attributes(pn_src);
	wimg_bits_dst = pmap_cache_attributes(pn_dst);

	if (mmu_kvtop_wpreflight(phystokv((pmap_paddr_t) dst)) &&
	    ((wimg_bits_src & VM_WIMG_MASK) == VM_WIMG_DEFAULT) &&
	    ((wimg_bits_dst & VM_WIMG_MASK) == VM_WIMG_DEFAULT)) {
		/* Fast path - dst is writable and both source and destination have default attributes */
		bcopy((char *)phystokv((pmap_paddr_t) src), (char *)phystokv((pmap_paddr_t) dst), bytes);
		return;
	}

	src_offset = src & PAGE_MASK;
	dst_offset = dst & PAGE_MASK;

	if ((src_offset + bytes) > PAGE_SIZE || (dst_offset + bytes) > PAGE_SIZE) {
		panic("bcopy extends beyond copy windows");
	}

	mp_disable_preemption();
	cpu_num = cpu_number();
	src_index = pmap_map_cpu_windows_copy(pn_src, VM_PROT_READ, wimg_bits_src);
	dst_index = pmap_map_cpu_windows_copy(pn_dst, VM_PROT_READ | VM_PROT_WRITE, wimg_bits_dst);

	bcopy((char *)(pmap_cpu_windows_copy_addr(cpu_num, src_index) + src_offset),
	    (char *)(pmap_cpu_windows_copy_addr(cpu_num, dst_index) + dst_offset),
	    bytes);

	pmap_unmap_cpu_windows_copy(src_index);
	pmap_unmap_cpu_windows_copy(dst_index);
	mp_enable_preemption();
}

void
bzero_phys_nc(addr64_t src64, vm_size_t bytes)
{
	bzero_phys(src64, bytes);
}

/* Zero bytes starting at a physical address */
void
bzero_phys(addr64_t src, vm_size_t bytes)
{
	unsigned int    wimg_bits;
	ppnum_t         pn = (src >> PAGE_SHIFT);

	wimg_bits = pmap_cache_attributes(pn);
	if ((wimg_bits & VM_WIMG_MASK) == VM_WIMG_DEFAULT) {
		/* Fast path - default attributes */
		bzero((char *)phystokv((pmap_paddr_t) src), bytes);
	} else {
		mp_disable_preemption();

		unsigned int cpu_num = cpu_number();

		while (bytes > 0) {
			vm_offset_t offset = src & PAGE_MASK;
			uint32_t count = PAGE_SIZE - offset;

			if (count > bytes) {
				count = bytes;
			}

			unsigned int index = pmap_map_cpu_windows_copy(src >> PAGE_SHIFT, VM_PROT_READ | VM_PROT_WRITE, wimg_bits);

			bzero((char *)(pmap_cpu_windows_copy_addr(cpu_num, index) + offset), count);

			pmap_unmap_cpu_windows_copy(index);

			src += count;
			bytes -= count;
		}

		mp_enable_preemption();
	}
}

/*
 *  Read data from a physical address.
 */


static unsigned int
ml_phys_read_data(pmap_paddr_t paddr, int size)
{
	unsigned int    index;
	unsigned int    result;
	unsigned int    wimg_bits;
	ppnum_t         pn = (paddr >> PAGE_SHIFT);
	unsigned char   s1;
	unsigned short  s2;
	vm_offset_t     copywindow_vaddr = 0;

	mp_disable_preemption();
	wimg_bits = pmap_cache_attributes(pn);
	index = pmap_map_cpu_windows_copy(pn, VM_PROT_READ, wimg_bits);
	copywindow_vaddr = pmap_cpu_windows_copy_addr(cpu_number(), index) | ((uint32_t)paddr & PAGE_MASK);;

	switch (size) {
	case 1:
		s1 = *(volatile unsigned char *)(copywindow_vaddr);
		result = s1;
		break;
	case 2:
		s2 = *(volatile unsigned short *)(copywindow_vaddr);
		result = s2;
		break;
	case 4:
	default:
		result = *(volatile unsigned int *)(copywindow_vaddr);
		break;
	}

	pmap_unmap_cpu_windows_copy(index);
	mp_enable_preemption();

	return result;
}

static unsigned long long
ml_phys_read_long_long(pmap_paddr_t paddr)
{
	unsigned int    index;
	unsigned int    result;
	unsigned int    wimg_bits;
	ppnum_t         pn = (paddr >> PAGE_SHIFT);

	mp_disable_preemption();
	wimg_bits = pmap_cache_attributes(pn);
	index = pmap_map_cpu_windows_copy(pn, VM_PROT_READ, wimg_bits);

	result = *(volatile unsigned long long *)(pmap_cpu_windows_copy_addr(cpu_number(), index)
	    | ((uint32_t)paddr & PAGE_MASK));

	pmap_unmap_cpu_windows_copy(index);
	mp_enable_preemption();

	return result;
}

unsigned int
ml_phys_read( vm_offset_t paddr)
{
	return ml_phys_read_data((pmap_paddr_t)paddr, 4);
}

unsigned int
ml_phys_read_word(vm_offset_t paddr)
{
	return ml_phys_read_data((pmap_paddr_t)paddr, 4);
}

unsigned int
ml_phys_read_64(addr64_t paddr64)
{
	return ml_phys_read_data((pmap_paddr_t)paddr64, 4);
}

unsigned int
ml_phys_read_word_64(addr64_t paddr64)
{
	return ml_phys_read_data((pmap_paddr_t)paddr64, 4);
}

unsigned int
ml_phys_read_half(vm_offset_t paddr)
{
	return ml_phys_read_data((pmap_paddr_t)paddr, 2);
}

unsigned int
ml_phys_read_half_64(addr64_t paddr64)
{
	return ml_phys_read_data((pmap_paddr_t)paddr64, 2);
}

unsigned int
ml_phys_read_byte(vm_offset_t paddr)
{
	return ml_phys_read_data((pmap_paddr_t)paddr, 1);
}

unsigned int
ml_phys_read_byte_64(addr64_t paddr64)
{
	return ml_phys_read_data((pmap_paddr_t)paddr64, 1);
}

unsigned long long
ml_phys_read_double(vm_offset_t paddr)
{
	return ml_phys_read_long_long((pmap_paddr_t)paddr);
}

unsigned long long
ml_phys_read_double_64(addr64_t paddr64)
{
	return ml_phys_read_long_long((pmap_paddr_t)paddr64);
}


/*
 *  Write data to a physical address.
 */

static void
ml_phys_write_data(pmap_paddr_t paddr, unsigned long data, int size)
{
	unsigned int    index;
	unsigned int    wimg_bits;
	ppnum_t         pn = (paddr >> PAGE_SHIFT);
	vm_offset_t     copywindow_vaddr = 0;

	mp_disable_preemption();
	wimg_bits = pmap_cache_attributes(pn);
	index = pmap_map_cpu_windows_copy(pn, VM_PROT_READ | VM_PROT_WRITE, wimg_bits);
	copywindow_vaddr = pmap_cpu_windows_copy_addr(cpu_number(), index) | ((uint32_t) paddr & PAGE_MASK);

	switch (size) {
	case 1:
		*(volatile unsigned char *)(copywindow_vaddr) = (unsigned char)data;
		break;
	case 2:
		*(volatile unsigned short *)(copywindow_vaddr) = (unsigned short)data;
		break;
	case 4:
	default:
		*(volatile unsigned int *)(copywindow_vaddr) = (uint32_t)data;
		break;
	}

	pmap_unmap_cpu_windows_copy(index);
	mp_enable_preemption();
}

static void
ml_phys_write_long_long(pmap_paddr_t paddr, unsigned long long data)
{
	unsigned int    index;
	unsigned int    wimg_bits;
	ppnum_t         pn = (paddr >> PAGE_SHIFT);

	mp_disable_preemption();
	wimg_bits = pmap_cache_attributes(pn);
	index = pmap_map_cpu_windows_copy(pn, VM_PROT_READ | VM_PROT_WRITE, wimg_bits);

	*(volatile unsigned long long *)(pmap_cpu_windows_copy_addr(cpu_number(), index)
	| ((uint32_t)paddr & PAGE_MASK)) = data;

	pmap_unmap_cpu_windows_copy(index);
	mp_enable_preemption();
}


void
ml_phys_write_byte(vm_offset_t paddr, unsigned int data)
{
	ml_phys_write_data((pmap_paddr_t)paddr, data, 1);
}

void
ml_phys_write_byte_64(addr64_t paddr64, unsigned int data)
{
	ml_phys_write_data((pmap_paddr_t)paddr64, data, 1);
}

void
ml_phys_write_half(vm_offset_t paddr, unsigned int data)
{
	ml_phys_write_data((pmap_paddr_t)paddr, data, 2);
}

void
ml_phys_write_half_64(addr64_t paddr64, unsigned int data)
{
	ml_phys_write_data((pmap_paddr_t)paddr64, data, 2);
}

void
ml_phys_write(vm_offset_t paddr, unsigned int data)
{
	ml_phys_write_data((pmap_paddr_t)paddr, data, 4);
}

void
ml_phys_write_64(addr64_t paddr64, unsigned int data)
{
	ml_phys_write_data((pmap_paddr_t)paddr64, data, 4);
}

void
ml_phys_write_word(vm_offset_t paddr, unsigned int data)
{
	ml_phys_write_data((pmap_paddr_t)paddr, data, 4);
}

void
ml_phys_write_word_64(addr64_t paddr64, unsigned int data)
{
	ml_phys_write_data((pmap_paddr_t)paddr64, data, 4);
}

void
ml_phys_write_double(vm_offset_t paddr, unsigned long long data)
{
	ml_phys_write_long_long((pmap_paddr_t)paddr, data);
}

void
ml_phys_write_double_64(addr64_t paddr64, unsigned long long data)
{
	ml_phys_write_long_long((pmap_paddr_t)paddr64, data);
}


/*
 * Set indicated bit in bit string.
 */
void
setbit(int bitno, int *s)
{
	s[bitno / INT_SIZE] |= 1 << (bitno % INT_SIZE);
}

/*
 * Clear indicated bit in bit string.
 */
void
clrbit(int bitno, int *s)
{
	s[bitno / INT_SIZE] &= ~(1 << (bitno % INT_SIZE));
}

/*
 * Test if indicated bit is set in bit string.
 */
int
testbit(int bitno, int *s)
{
	return s[bitno / INT_SIZE] & (1 << (bitno % INT_SIZE));
}

/*
 * Find first bit set in bit string.
 */
int
ffsbit(int *s)
{
	int             offset;

	for (offset = 0; !*s; offset += INT_SIZE, ++s) {
		;
	}
	return offset + __builtin_ctz(*s);
}

int
ffs(unsigned int mask)
{
	if (mask == 0) {
		return 0;
	}

	/*
	 * NOTE: cannot use __builtin_ffs because it generates a call to
	 * 'ffs'
	 */
	return 1 + __builtin_ctz(mask);
}

int
ffsll(unsigned long long mask)
{
	if (mask == 0) {
		return 0;
	}

	/*
	 * NOTE: cannot use __builtin_ffsll because it generates a call to
	 * 'ffsll'
	 */
	return 1 + __builtin_ctzll(mask);
}

/*
 * Find last bit set in bit string.
 */
int
fls(unsigned int mask)
{
	if (mask == 0) {
		return 0;
	}

	return (sizeof(mask) << 3) - __builtin_clz(mask);
}

int
flsll(unsigned long long mask)
{
	if (mask == 0) {
		return 0;
	}

	return (sizeof(mask) << 3) - __builtin_clzll(mask);
}

int
bcmp(
	const void *pa,
	const void *pb,
	size_t len)
{
	const char     *a = (const char *) pa;
	const char     *b = (const char *) pb;

	if (len == 0) {
		return 0;
	}

	do{
		if (*a++ != *b++) {
			break;
		}
	} while (--len);

	return len;
}

int
memcmp(const void *s1, const void *s2, size_t n)
{
	if (n != 0) {
		const unsigned char *p1 = s1, *p2 = s2;

		do {
			if (*p1++ != *p2++) {
				return *--p1 - *--p2;
			}
		} while (--n != 0);
	}
	return 0;
}

kern_return_t
copypv(addr64_t source, addr64_t sink, unsigned int size, int which)
{
	kern_return_t   retval = KERN_SUCCESS;
	void            *from, *to;
	unsigned int    from_wimg_bits, to_wimg_bits;

	from = CAST_DOWN(void *, source);
	to = CAST_DOWN(void *, sink);

	if ((which & (cppvPsrc | cppvPsnk)) == 0) {     /* Make sure that only
		                                         * one is virtual */
		panic("copypv: no more than 1 parameter may be virtual\n");     /* Not allowed */
	}
	if (which & cppvPsrc) {
		from = (void *)phystokv((pmap_paddr_t)from);
	}
	if (which & cppvPsnk) {
		to = (void *)phystokv((pmap_paddr_t)to);
	}

	if ((which & (cppvPsrc | cppvKmap)) == 0) {     /* Source is virtual in
		                                         * current map */
		retval = copyin((user_addr_t) from, to, size);
	} else if ((which & (cppvPsnk | cppvKmap)) == 0) { /* Sink is virtual in
		                                            * current map */
		retval = copyout(from, (user_addr_t) to, size);
	} else {                /* both addresses are physical or kernel map */
		bcopy(from, to, size);
	}

	if (which & cppvFsrc) {
		flush_dcache64(source, size, ((which & cppvPsrc) == cppvPsrc));
	} else if (which & cppvPsrc) {
		from_wimg_bits = pmap_cache_attributes(source >> PAGE_SHIFT);
		if ((from_wimg_bits != VM_WIMG_COPYBACK) && (from_wimg_bits != VM_WIMG_WTHRU)) {
			flush_dcache64(source, size, TRUE);
		}
	}

	if (which & cppvFsnk) {
		flush_dcache64(sink, size, ((which & cppvPsnk) == cppvPsnk));
	} else if (which & cppvPsnk) {
		to_wimg_bits = pmap_cache_attributes(sink >> PAGE_SHIFT);
		if (to_wimg_bits != VM_WIMG_COPYBACK) {
			flush_dcache64(sink, size, TRUE);
		}
	}
	return retval;
}

/*
 * Copy sizes bigger than this value will cause a kernel panic.
 *
 * Yes, this is an arbitrary fixed limit, but it's almost certainly
 * a programming error to be copying more than this amount between
 * user and wired kernel memory in a single invocation on this
 * platform.
 */
const int copysize_limit_panic = (64 * 1024 * 1024);

/*
 * Validate the arguments to copy{in,out} on this platform.
 *
 * Called when nbytes is "large" e.g. more than a page.  Such sizes are
 * infrequent, and very large sizes are likely indications of attempts
 * to exploit kernel programming errors (bugs).
 */
static int
copy_validate(const user_addr_t user_addr,
    uintptr_t kernel_addr, vm_size_t nbytes)
{
	uintptr_t kernel_addr_last = kernel_addr + nbytes;

	if (__improbable(kernel_addr < VM_MIN_KERNEL_ADDRESS ||
	    kernel_addr > VM_MAX_KERNEL_ADDRESS ||
	    kernel_addr_last < kernel_addr ||
	    kernel_addr_last > VM_MAX_KERNEL_ADDRESS)) {
		panic("%s(%p, %p, %u) - kaddr not in kernel", __func__,
		    (void *)user_addr, (void *)kernel_addr, nbytes);
	}

	user_addr_t user_addr_last = user_addr + nbytes;

	if (__improbable((user_addr_last < user_addr) || ((user_addr + nbytes) > vm_map_max(current_thread()->map)) ||
	    (user_addr < vm_map_min(current_thread()->map)))) {
		return EFAULT;
	}

	if (__improbable(nbytes > copysize_limit_panic)) {
		panic("%s(%p, %p, %u) - transfer too large", __func__,
		    (void *)user_addr, (void *)kernel_addr, nbytes);
	}

	return 0;
}

int
copyin_validate(const user_addr_t ua, uintptr_t ka, vm_size_t nbytes)
{
	return copy_validate(ua, ka, nbytes);
}

int
copyout_validate(uintptr_t ka, const user_addr_t ua, vm_size_t nbytes)
{
	return copy_validate(ua, ka, nbytes);
}

#if     MACH_ASSERT

extern int copyinframe(vm_address_t fp, char *frame);

/*
 * Machine-dependent routine to fill in an array with up to callstack_max
 * levels of return pc information.
 */
void
machine_callstack(
	uintptr_t * buf,
	vm_size_t callstack_max)
{
	/* Captures the USER call stack */
	uint32_t i = 0;
	uint32_t frame[2];

	struct arm_saved_state* state = find_user_regs(current_thread());

	if (!state) {
		while (i < callstack_max) {
			buf[i++] = 0;
		}
	} else {
		buf[i++] = (uintptr_t)state->pc;
		frame[0] = state->r[7];

		while (i < callstack_max && frame[0] != 0) {
			if (copyinframe(frame[0], (void*) frame)) {
				break;
			}
			buf[i++] = (uintptr_t)frame[1];
		}

		while (i < callstack_max) {
			buf[i++] = 0;
		}
	}
}

#endif                          /* MACH_ASSERT */

int
clr_be_bit(void)
{
	panic("clr_be_bit");
	return 0;
}

boolean_t
ml_probe_read(
	__unused vm_offset_t paddr,
	__unused unsigned int *val)
{
	panic("ml_probe_read() unimplemented");
	return 1;
}

boolean_t
ml_probe_read_64(
	__unused addr64_t paddr,
	__unused unsigned int *val)
{
	panic("ml_probe_read_64() unimplemented");
	return 1;
}


void
ml_thread_policy(
	__unused thread_t thread,
	__unused unsigned policy_id,
	__unused unsigned policy_info)
{
	//    <rdar://problem/7141284>: Reduce print noise
	//	kprintf("ml_thread_policy() unimplemented\n");
}

#if !MACH_KDP
void
kdp_register_callout(kdp_callout_fn_t fn, void *arg)
{
#pragma unused(fn,arg)
}
#endif
Commit	Line	Data
5ba3f43e A	1	/*
	2	* Copyright (c) 2007-2016 Apple Inc. All rights reserved.
	3	*
	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
	5	*
	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
	14	*
	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
	25	*
	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
	27	*/
	28
	29	#include <mach_assert.h>
	30	#include <mach/vm_types.h>
	31	#include <mach/mach_time.h>
	32	#include <kern/timer.h>
	33	#include <kern/clock.h>
	34	#include <kern/machine.h>
	35	#include <mach/machine.h>
	36	#include <mach/machine/vm_param.h>
	37	#include <mach_kdp.h>
	38	#include <kdp/kdp_udp.h>
	39	#if !MACH_KDP
	40	#include <kdp/kdp_callout.h>
	41	#endif /* !MACH_KDP */
	42	#include <arm/cpu_data.h>
	43	#include <arm/cpu_data_internal.h>
	44	#include <arm/caches_internal.h>
	45
	46	#include <vm/vm_kern.h>
	47	#include <vm/vm_map.h>
	48	#include <vm/pmap.h>
	49
	50	#include <arm/misc_protos.h>
	51
	52	#include <sys/errno.h>
	53
	54	#define INT_SIZE (BYTE_SIZE * sizeof (int))
	55
	56
	57	void
	58	bcopy_phys(addr64_t src, addr64_t dst, vm_size_t bytes)
	59	{
	60	unsigned int src_index;
	61	unsigned int dst_index;
	62	vm_offset_t src_offset;
	63	vm_offset_t dst_offset;
	64	unsigned int cpu_num;
65	unsigned int wimg_bits_src, wimg_bits_dst;
66	ppnum_t pn_src = (src >> PAGE_SHIFT);
67	ppnum_t pn_dst = (dst >> PAGE_SHIFT);
68
69	wimg_bits_src = pmap_cache_attributes(pn_src);
70	wimg_bits_dst = pmap_cache_attributes(pn_dst);
71
72	if (mmu_kvtop_wpreflight(phystokv((pmap_paddr_t) dst)) &&
0a7de745 A	73	((wimg_bits_src & VM_WIMG_MASK) == VM_WIMG_DEFAULT) &&
0a7de745 A	74	((wimg_bits_dst & VM_WIMG_MASK) == VM_WIMG_DEFAULT)) {
5ba3f43e A	75	/* Fast path - dst is writable and both source and destination have default attributes */
	76	bcopy((char )phystokv((pmap_paddr_t) src), (char )phystokv((pmap_paddr_t) dst), bytes);
	77	return;
	78	}
	79
	80	src_offset = src & PAGE_MASK;
	81	dst_offset = dst & PAGE_MASK;
	82
0a7de745	83	if ((src_offset + bytes) > PAGE_SIZE \|\| (dst_offset + bytes) > PAGE_SIZE) {
5ba3f43e	84	panic("bcopy extends beyond copy windows");
0a7de745	85	}
5ba3f43e A	86
	87	mp_disable_preemption();
	88	cpu_num = cpu_number();
	89	src_index = pmap_map_cpu_windows_copy(pn_src, VM_PROT_READ, wimg_bits_src);
0a7de745	90	dst_index = pmap_map_cpu_windows_copy(pn_dst, VM_PROT_READ \| VM_PROT_WRITE, wimg_bits_dst);
5ba3f43e	91
0a7de745 A	92	bcopy((char *)(pmap_cpu_windows_copy_addr(cpu_num, src_index) + src_offset),
	93	(char *)(pmap_cpu_windows_copy_addr(cpu_num, dst_index) + dst_offset),
	94	bytes);
5ba3f43e A	95
	96	pmap_unmap_cpu_windows_copy(src_index);
	97	pmap_unmap_cpu_windows_copy(dst_index);
	98	mp_enable_preemption();
	99	}
	100
	101	void
	102	bzero_phys_nc(addr64_t src64, vm_size_t bytes)
	103	{
	104	bzero_phys(src64, bytes);
	105	}
	106
	107	/* Zero bytes starting at a physical address */
	108	void
	109	bzero_phys(addr64_t src, vm_size_t bytes)
	110	{
	111	unsigned int wimg_bits;
	112	ppnum_t pn = (src >> PAGE_SHIFT);
	113
	114	wimg_bits = pmap_cache_attributes(pn);
	115	if ((wimg_bits & VM_WIMG_MASK) == VM_WIMG_DEFAULT) {
	116	/* Fast path - default attributes */
	117	bzero((char *)phystokv((pmap_paddr_t) src), bytes);
	118	} else {
	119	mp_disable_preemption();
	120
	121	unsigned int cpu_num = cpu_number();
	122
	123	while (bytes > 0) {
	124	vm_offset_t offset = src & PAGE_MASK;
	125	uint32_t count = PAGE_SIZE - offset;
	126
0a7de745	127	if (count > bytes) {
5ba3f43e	128	count = bytes;
0a7de745	129	}
5ba3f43e A	130
	131	unsigned int index = pmap_map_cpu_windows_copy(src >> PAGE_SHIFT, VM_PROT_READ \| VM_PROT_WRITE, wimg_bits);
	132
	133	bzero((char *)(pmap_cpu_windows_copy_addr(cpu_num, index) + offset), count);
	134
	135	pmap_unmap_cpu_windows_copy(index);
	136
	137	src += count;
	138	bytes -= count;
	139	}
	140
	141	mp_enable_preemption();
	142	}
	143	}
	144
	145	/*
	146	* Read data from a physical address.
	147	*/
	148
	149
	150	static unsigned int
	151	ml_phys_read_data(pmap_paddr_t paddr, int size)
	152	{
	153	unsigned int index;
	154	unsigned int result;
	155	unsigned int wimg_bits;
	156	ppnum_t pn = (paddr >> PAGE_SHIFT);
	157	unsigned char s1;
	158	unsigned short s2;
	159	vm_offset_t copywindow_vaddr = 0;
	160
	161	mp_disable_preemption();
	162	wimg_bits = pmap_cache_attributes(pn);
	163	index = pmap_map_cpu_windows_copy(pn, VM_PROT_READ, wimg_bits);
	164	copywindow_vaddr = pmap_cpu_windows_copy_addr(cpu_number(), index) \| ((uint32_t)paddr & PAGE_MASK);;
	165
	166	switch (size) {
0a7de745 A	167	case 1:
	168	s1 = (volatile unsigned char )(copywindow_vaddr);
	169	result = s1;
	170	break;
	171	case 2:
	172	s2 = (volatile unsigned short )(copywindow_vaddr);
	173	result = s2;
	174	break;
	175	case 4:
	176	default:
	177	result = (volatile unsigned int )(copywindow_vaddr);
	178	break;
5ba3f43e A	179	}
	180
	181	pmap_unmap_cpu_windows_copy(index);
	182	mp_enable_preemption();
	183
	184	return result;
	185	}
	186
	187	static unsigned long long
	188	ml_phys_read_long_long(pmap_paddr_t paddr)
	189	{
	190	unsigned int index;
	191	unsigned int result;
	192	unsigned int wimg_bits;
	193	ppnum_t pn = (paddr >> PAGE_SHIFT);
	194
	195	mp_disable_preemption();
	196	wimg_bits = pmap_cache_attributes(pn);
	197	index = pmap_map_cpu_windows_copy(pn, VM_PROT_READ, wimg_bits);
	198
	199	result = (volatile unsigned long long )(pmap_cpu_windows_copy_addr(cpu_number(), index)
0a7de745	200	\| ((uint32_t)paddr & PAGE_MASK));
5ba3f43e A	201
	202	pmap_unmap_cpu_windows_copy(index);
	203	mp_enable_preemption();
	204
	205	return result;
	206	}
	207
0a7de745 A	208	unsigned int
0a7de745 A	209	ml_phys_read( vm_offset_t paddr)
5ba3f43e	210	{
0a7de745	211	return ml_phys_read_data((pmap_paddr_t)paddr, 4);
5ba3f43e A	212	}
5ba3f43e A	213
0a7de745 A	214	unsigned int
	215	ml_phys_read_word(vm_offset_t paddr)
	216	{
	217	return ml_phys_read_data((pmap_paddr_t)paddr, 4);
5ba3f43e A	218	}
5ba3f43e A	219
0a7de745 A	220	unsigned int
0a7de745 A	221	ml_phys_read_64(addr64_t paddr64)
5ba3f43e	222	{
0a7de745	223	return ml_phys_read_data((pmap_paddr_t)paddr64, 4);
5ba3f43e A	224	}
5ba3f43e A	225
0a7de745 A	226	unsigned int
0a7de745 A	227	ml_phys_read_word_64(addr64_t paddr64)
5ba3f43e	228	{
0a7de745	229	return ml_phys_read_data((pmap_paddr_t)paddr64, 4);
5ba3f43e A	230	}
5ba3f43e A	231
0a7de745 A	232	unsigned int
0a7de745 A	233	ml_phys_read_half(vm_offset_t paddr)
5ba3f43e	234	{
0a7de745	235	return ml_phys_read_data((pmap_paddr_t)paddr, 2);
5ba3f43e A	236	}
5ba3f43e A	237
0a7de745 A	238	unsigned int
0a7de745 A	239	ml_phys_read_half_64(addr64_t paddr64)
5ba3f43e	240	{
0a7de745	241	return ml_phys_read_data((pmap_paddr_t)paddr64, 2);
5ba3f43e A	242	}
5ba3f43e A	243
0a7de745 A	244	unsigned int
0a7de745 A	245	ml_phys_read_byte(vm_offset_t paddr)
5ba3f43e	246	{
0a7de745	247	return ml_phys_read_data((pmap_paddr_t)paddr, 1);
5ba3f43e A	248	}
5ba3f43e A	249
0a7de745 A	250	unsigned int
0a7de745 A	251	ml_phys_read_byte_64(addr64_t paddr64)
5ba3f43e	252	{
0a7de745	253	return ml_phys_read_data((pmap_paddr_t)paddr64, 1);
5ba3f43e A	254	}
5ba3f43e A	255
0a7de745 A	256	unsigned long long
0a7de745 A	257	ml_phys_read_double(vm_offset_t paddr)
5ba3f43e	258	{
0a7de745	259	return ml_phys_read_long_long((pmap_paddr_t)paddr);
5ba3f43e A	260	}
5ba3f43e A	261
0a7de745 A	262	unsigned long long
0a7de745 A	263	ml_phys_read_double_64(addr64_t paddr64)
5ba3f43e	264	{
0a7de745	265	return ml_phys_read_long_long((pmap_paddr_t)paddr64);
5ba3f43e A	266	}
	267
	268
	269
	270	/*
	271	* Write data to a physical address.
	272	*/
	273
	274	static void
	275	ml_phys_write_data(pmap_paddr_t paddr, unsigned long data, int size)
	276	{
	277	unsigned int index;
	278	unsigned int wimg_bits;
	279	ppnum_t pn = (paddr >> PAGE_SHIFT);
	280	vm_offset_t copywindow_vaddr = 0;
	281
	282	mp_disable_preemption();
	283	wimg_bits = pmap_cache_attributes(pn);
0a7de745	284	index = pmap_map_cpu_windows_copy(pn, VM_PROT_READ \| VM_PROT_WRITE, wimg_bits);
5ba3f43e A	285	copywindow_vaddr = pmap_cpu_windows_copy_addr(cpu_number(), index) \| ((uint32_t) paddr & PAGE_MASK);
	286
	287	switch (size) {
0a7de745 A	288	case 1:
	289	(volatile unsigned char )(copywindow_vaddr) = (unsigned char)data;
	290	break;
	291	case 2:
	292	(volatile unsigned short )(copywindow_vaddr) = (unsigned short)data;
	293	break;
	294	case 4:
	295	default:
	296	(volatile unsigned int )(copywindow_vaddr) = (uint32_t)data;
	297	break;
5ba3f43e A	298	}
	299
	300	pmap_unmap_cpu_windows_copy(index);
	301	mp_enable_preemption();
	302	}
	303
	304	static void
	305	ml_phys_write_long_long(pmap_paddr_t paddr, unsigned long long data)
	306	{
	307	unsigned int index;
	308	unsigned int wimg_bits;
	309	ppnum_t pn = (paddr >> PAGE_SHIFT);
	310
	311	mp_disable_preemption();
	312	wimg_bits = pmap_cache_attributes(pn);
0a7de745	313	index = pmap_map_cpu_windows_copy(pn, VM_PROT_READ \| VM_PROT_WRITE, wimg_bits);
5ba3f43e A	314
5ba3f43e A	315	(volatile unsigned long long )(pmap_cpu_windows_copy_addr(cpu_number(), index)
0a7de745	316	\| ((uint32_t)paddr & PAGE_MASK)) = data;
5ba3f43e A	317
	318	pmap_unmap_cpu_windows_copy(index);
	319	mp_enable_preemption();
	320	}
	321
	322
	323
0a7de745 A	324	void
0a7de745 A	325	ml_phys_write_byte(vm_offset_t paddr, unsigned int data)
5ba3f43e	326	{
0a7de745	327	ml_phys_write_data((pmap_paddr_t)paddr, data, 1);
5ba3f43e A	328	}
5ba3f43e A	329
0a7de745 A	330	void
0a7de745 A	331	ml_phys_write_byte_64(addr64_t paddr64, unsigned int data)
5ba3f43e	332	{
0a7de745	333	ml_phys_write_data((pmap_paddr_t)paddr64, data, 1);
5ba3f43e A	334	}
5ba3f43e A	335
0a7de745 A	336	void
0a7de745 A	337	ml_phys_write_half(vm_offset_t paddr, unsigned int data)
5ba3f43e	338	{
0a7de745	339	ml_phys_write_data((pmap_paddr_t)paddr, data, 2);
5ba3f43e A	340	}
5ba3f43e A	341
0a7de745 A	342	void
0a7de745 A	343	ml_phys_write_half_64(addr64_t paddr64, unsigned int data)
5ba3f43e	344	{
0a7de745	345	ml_phys_write_data((pmap_paddr_t)paddr64, data, 2);
5ba3f43e A	346	}
5ba3f43e A	347
0a7de745 A	348	void
0a7de745 A	349	ml_phys_write(vm_offset_t paddr, unsigned int data)
5ba3f43e	350	{
0a7de745	351	ml_phys_write_data((pmap_paddr_t)paddr, data, 4);
5ba3f43e A	352	}
5ba3f43e A	353
0a7de745 A	354	void
0a7de745 A	355	ml_phys_write_64(addr64_t paddr64, unsigned int data)
5ba3f43e	356	{
0a7de745	357	ml_phys_write_data((pmap_paddr_t)paddr64, data, 4);
5ba3f43e A	358	}
5ba3f43e A	359
0a7de745 A	360	void
0a7de745 A	361	ml_phys_write_word(vm_offset_t paddr, unsigned int data)
5ba3f43e	362	{
0a7de745	363	ml_phys_write_data((pmap_paddr_t)paddr, data, 4);
5ba3f43e A	364	}
5ba3f43e A	365
0a7de745 A	366	void
0a7de745 A	367	ml_phys_write_word_64(addr64_t paddr64, unsigned int data)
5ba3f43e	368	{
0a7de745	369	ml_phys_write_data((pmap_paddr_t)paddr64, data, 4);
5ba3f43e A	370	}
5ba3f43e A	371
0a7de745 A	372	void
0a7de745 A	373	ml_phys_write_double(vm_offset_t paddr, unsigned long long data)
5ba3f43e	374	{
0a7de745	375	ml_phys_write_long_long((pmap_paddr_t)paddr, data);
5ba3f43e A	376	}
5ba3f43e A	377
0a7de745 A	378	void
0a7de745 A	379	ml_phys_write_double_64(addr64_t paddr64, unsigned long long data)
5ba3f43e	380	{
0a7de745	381	ml_phys_write_long_long((pmap_paddr_t)paddr64, data);
5ba3f43e A	382	}
	383
	384
	385	/*
	386	* Set indicated bit in bit string.
	387	*/
	388	void
	389	setbit(int bitno, int *s)
	390	{
	391	s[bitno / INT_SIZE] \|= 1 << (bitno % INT_SIZE);
	392	}
	393
	394	/*
	395	* Clear indicated bit in bit string.
	396	*/
	397	void
	398	clrbit(int bitno, int *s)
	399	{
	400	s[bitno / INT_SIZE] &= ~(1 << (bitno % INT_SIZE));
	401	}
	402
	403	/*
	404	* Test if indicated bit is set in bit string.
	405	*/
	406	int
	407	testbit(int bitno, int *s)
	408	{
	409	return s[bitno / INT_SIZE] & (1 << (bitno % INT_SIZE));
	410	}
	411
	412	/*
	413	* Find first bit set in bit string.
	414	*/
	415	int
	416	ffsbit(int *s)
	417	{
	418	int offset;
	419
0a7de745 A	420	for (offset = 0; !*s; offset += INT_SIZE, ++s) {
	421	;
	422	}
5ba3f43e A	423	return offset + __builtin_ctz(*s);
	424	}
	425
	426	int
	427	ffs(unsigned int mask)
	428	{
0a7de745	429	if (mask == 0) {
5ba3f43e	430	return 0;
0a7de745	431	}
5ba3f43e A	432
	433	/*
	434	* NOTE: cannot use __builtin_ffs because it generates a call to
	435	* 'ffs'
	436	*/
	437	return 1 + __builtin_ctz(mask);
	438	}
	439
	440	int
	441	ffsll(unsigned long long mask)
	442	{
0a7de745	443	if (mask == 0) {
5ba3f43e	444	return 0;
0a7de745	445	}
5ba3f43e A	446
	447	/*
	448	* NOTE: cannot use __builtin_ffsll because it generates a call to
	449	* 'ffsll'
	450	*/
	451	return 1 + __builtin_ctzll(mask);
	452	}
	453
	454	/*
	455	* Find last bit set in bit string.
	456	*/
	457	int
	458	fls(unsigned int mask)
	459	{
0a7de745	460	if (mask == 0) {
5ba3f43e	461	return 0;
0a7de745	462	}
5ba3f43e	463
0a7de745	464	return (sizeof(mask) << 3) - __builtin_clz(mask);
5ba3f43e A	465	}
	466
	467	int
	468	flsll(unsigned long long mask)
	469	{
0a7de745	470	if (mask == 0) {
5ba3f43e	471	return 0;
0a7de745	472	}
5ba3f43e	473
0a7de745	474	return (sizeof(mask) << 3) - __builtin_clzll(mask);
5ba3f43e A	475	}
5ba3f43e A	476
0a7de745	477	int
5ba3f43e	478	bcmp(
0a7de745 A	479	const void *pa,
	480	const void *pb,
	481	size_t len)
5ba3f43e A	482	{
	483	const char a = (const char ) pa;
	484	const char b = (const char ) pb;
	485
0a7de745	486	if (len == 0) {
5ba3f43e	487	return 0;
0a7de745	488	}
5ba3f43e	489
0a7de745 A	490	do{
0a7de745 A	491	if (a++ != b++) {
5ba3f43e	492	break;
0a7de745 A	493	}
0a7de745 A	494	} while (--len);
5ba3f43e A	495
	496	return len;
	497	}
	498
	499	int
	500	memcmp(const void s1, const void s2, size_t n)
	501	{
	502	if (n != 0) {
	503	const unsigned char p1 = s1, p2 = s2;
	504
	505	do {
0a7de745 A	506	if (p1++ != p2++) {
	507	return --p1 - --p2;
	508	}
5ba3f43e A	509	} while (--n != 0);
5ba3f43e A	510	}
0a7de745	511	return 0;
5ba3f43e A	512	}
	513
	514	kern_return_t
	515	copypv(addr64_t source, addr64_t sink, unsigned int size, int which)
	516	{
	517	kern_return_t retval = KERN_SUCCESS;
0a7de745 A	518	void from, to;
0a7de745 A	519	unsigned int from_wimg_bits, to_wimg_bits;
5ba3f43e A	520
	521	from = CAST_DOWN(void *, source);
	522	to = CAST_DOWN(void *, sink);
	523
0a7de745 A	524	if ((which & (cppvPsrc \| cppvPsnk)) == 0) { /* Make sure that only
	525	* one is virtual */
	526	panic("copypv: no more than 1 parameter may be virtual\n"); /* Not allowed */
	527	}
	528	if (which & cppvPsrc) {
d9a64523	529	from = (void *)phystokv((pmap_paddr_t)from);
0a7de745 A	530	}
0a7de745 A	531	if (which & cppvPsnk) {
d9a64523	532	to = (void *)phystokv((pmap_paddr_t)to);
0a7de745	533	}
5ba3f43e	534
0a7de745 A	535	if ((which & (cppvPsrc \| cppvKmap)) == 0) { /* Source is virtual in
0a7de745 A	536	* current map */
5ba3f43e	537	retval = copyin((user_addr_t) from, to, size);
0a7de745 A	538	} else if ((which & (cppvPsnk \| cppvKmap)) == 0) { /* Sink is virtual in
0a7de745 A	539	* current map */
5ba3f43e	540	retval = copyout(from, (user_addr_t) to, size);
0a7de745	541	} else { /* both addresses are physical or kernel map */
5ba3f43e	542	bcopy(from, to, size);
0a7de745	543	}
5ba3f43e A	544
	545	if (which & cppvFsrc) {
	546	flush_dcache64(source, size, ((which & cppvPsrc) == cppvPsrc));
	547	} else if (which & cppvPsrc) {
	548	from_wimg_bits = pmap_cache_attributes(source >> PAGE_SHIFT);
0a7de745	549	if ((from_wimg_bits != VM_WIMG_COPYBACK) && (from_wimg_bits != VM_WIMG_WTHRU)) {
5ba3f43e	550	flush_dcache64(source, size, TRUE);
0a7de745	551	}
5ba3f43e A	552	}
	553
	554	if (which & cppvFsnk) {
	555	flush_dcache64(sink, size, ((which & cppvPsnk) == cppvPsnk));
0a7de745	556	} else if (which & cppvPsnk) {
5ba3f43e	557	to_wimg_bits = pmap_cache_attributes(sink >> PAGE_SHIFT);
0a7de745	558	if (to_wimg_bits != VM_WIMG_COPYBACK) {
5ba3f43e	559	flush_dcache64(sink, size, TRUE);
0a7de745	560	}
5ba3f43e A	561	}
	562	return retval;
	563	}
	564
	565	/*
	566	* Copy sizes bigger than this value will cause a kernel panic.
	567	*
	568	* Yes, this is an arbitrary fixed limit, but it's almost certainly
	569	* a programming error to be copying more than this amount between
	570	* user and wired kernel memory in a single invocation on this
	571	* platform.
	572	*/
	573	const int copysize_limit_panic = (64 * 1024 * 1024);
	574
	575	/*
	576	* Validate the arguments to copy{in,out} on this platform.
	577	*
	578	* Called when nbytes is "large" e.g. more than a page. Such sizes are
	579	* infrequent, and very large sizes are likely indications of attempts
	580	* to exploit kernel programming errors (bugs).
	581	*/
	582	static int
	583	copy_validate(const user_addr_t user_addr,
	584	uintptr_t kernel_addr, vm_size_t nbytes)
	585	{
	586	uintptr_t kernel_addr_last = kernel_addr + nbytes;
	587
d9a64523	588	if (__improbable(kernel_addr < VM_MIN_KERNEL_ADDRESS \|\|
5ba3f43e A	589	kernel_addr > VM_MAX_KERNEL_ADDRESS \|\|
5ba3f43e A	590	kernel_addr_last < kernel_addr \|\|
0a7de745	591	kernel_addr_last > VM_MAX_KERNEL_ADDRESS)) {
5ba3f43e A	592	panic("%s(%p, %p, %u) - kaddr not in kernel", __func__,
5ba3f43e A	593	(void )user_addr, (void )kernel_addr, nbytes);
0a7de745	594	}
5ba3f43e A	595
	596	user_addr_t user_addr_last = user_addr + nbytes;
	597
d9a64523	598	if (__improbable((user_addr_last < user_addr) \|\| ((user_addr + nbytes) > vm_map_max(current_thread()->map)) \|\|
0a7de745 A	599	(user_addr < vm_map_min(current_thread()->map)))) {
	600	return EFAULT;
	601	}
5ba3f43e	602
0a7de745	603	if (__improbable(nbytes > copysize_limit_panic)) {
5ba3f43e A	604	panic("%s(%p, %p, %u) - transfer too large", __func__,
5ba3f43e A	605	(void )user_addr, (void )kernel_addr, nbytes);
0a7de745	606	}
5ba3f43e	607
0a7de745	608	return 0;
5ba3f43e A	609	}
	610
	611	int
	612	copyin_validate(const user_addr_t ua, uintptr_t ka, vm_size_t nbytes)
	613	{
0a7de745	614	return copy_validate(ua, ka, nbytes);
5ba3f43e A	615	}
	616
	617	int
	618	copyout_validate(uintptr_t ka, const user_addr_t ua, vm_size_t nbytes)
	619	{
0a7de745	620	return copy_validate(ua, ka, nbytes);
5ba3f43e A	621	}
	622
	623	#if MACH_ASSERT
	624
0a7de745	625	extern int copyinframe(vm_address_t fp, char *frame);
5ba3f43e A	626
	627	/*
	628	* Machine-dependent routine to fill in an array with up to callstack_max
	629	* levels of return pc information.
	630	*/
	631	void
	632	machine_callstack(
0a7de745 A	633	uintptr_t * buf,
0a7de745 A	634	vm_size_t callstack_max)
5ba3f43e A	635	{
5ba3f43e A	636	/* Captures the USER call stack */
0a7de745	637	uint32_t i = 0;
5ba3f43e A	638	uint32_t frame[2];
	639
	640	struct arm_saved_state* state = find_user_regs(current_thread());
	641
	642	if (!state) {
0a7de745	643	while (i < callstack_max) {
5ba3f43e	644	buf[i++] = 0;
0a7de745	645	}
5ba3f43e A	646	} else {
	647	buf[i++] = (uintptr_t)state->pc;
	648	frame[0] = state->r[7];
	649
0a7de745 A	650	while (i < callstack_max && frame[0] != 0) {
0a7de745 A	651	if (copyinframe(frame[0], (void*) frame)) {
5ba3f43e	652	break;
0a7de745	653	}
5ba3f43e A	654	buf[i++] = (uintptr_t)frame[1];
	655	}
	656
0a7de745	657	while (i < callstack_max) {
5ba3f43e	658	buf[i++] = 0;
0a7de745	659	}
5ba3f43e A	660	}
	661	}
	662
0a7de745	663	#endif /* MACH_ASSERT */
5ba3f43e A	664
	665	int
	666	clr_be_bit(void)
	667	{
	668	panic("clr_be_bit");
	669	return 0;
	670	}
	671
	672	boolean_t
	673	ml_probe_read(
0a7de745 A	674	__unused vm_offset_t paddr,
0a7de745 A	675	__unused unsigned int *val)
5ba3f43e A	676	{
	677	panic("ml_probe_read() unimplemented");
	678	return 1;
	679	}
	680
	681	boolean_t
	682	ml_probe_read_64(
0a7de745 A	683	__unused addr64_t paddr,
0a7de745 A	684	__unused unsigned int *val)
5ba3f43e A	685	{
	686	panic("ml_probe_read_64() unimplemented");
	687	return 1;
	688	}
	689
	690
	691	void
	692	ml_thread_policy(
0a7de745 A	693	__unused thread_t thread,
	694	__unused unsigned policy_id,
	695	__unused unsigned policy_info)
5ba3f43e	696	{
0a7de745 A	697	// <rdar://problem/7141284>: Reduce print noise
0a7de745 A	698	// kprintf("ml_thread_policy() unimplemented\n");
5ba3f43e A	699	}
	700
	701	#if !MACH_KDP
	702	void
	703	kdp_register_callout(kdp_callout_fn_t fn, void *arg)
	704	{
	705	#pragma unused(fn,arg)
	706	}
	707	#endif