[apple/xnu.git] / osfmk / arm64 / loose_ends.c

/*
 * Copyright (c) 2007-2016 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 *
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */

#include <mach_assert.h>
#include <mach/vm_types.h>
#include <mach/mach_time.h>
#include <kern/timer.h>
#include <kern/clock.h>
#include <kern/machine.h>
#include <mach/machine.h>
#include <mach/machine/vm_param.h>
#include <mach_kdp.h>
#include <kdp/kdp_udp.h>
#if !MACH_KDP
#include <kdp/kdp_callout.h>
#endif /* !MACH_KDP */
#include <arm/cpu_data.h>
#include <arm/cpu_data_internal.h>
#include <arm/caches_internal.h>

#include <vm/vm_kern.h>
#include <vm/vm_map.h>
#include <vm/pmap.h>

#include <arm/misc_protos.h>

#include <sys/errno.h>

#define INT_SIZE        (BYTE_SIZE * sizeof (int))

#define BCOPY_PHYS_SRC_IS_PHYS(flags) (((flags) & cppvPsrc) != 0)
#define BCOPY_PHYS_DST_IS_PHYS(flags) (((flags) & cppvPsnk) != 0)
#define BCOPY_PHYS_SRC_IS_USER(flags) (((flags) & (cppvPsrc | cppvKmap)) == 0)
#define BCOPY_PHYS_DST_IS_USER(flags) (((flags) & (cppvPsnk | cppvKmap)) == 0)

static kern_return_t
bcopy_phys_internal(addr64_t src, addr64_t dst, vm_size_t bytes, int flags)
{
	unsigned int    src_index;
	unsigned int    dst_index;
	vm_offset_t     src_offset;
	vm_offset_t     dst_offset;
	unsigned int    wimg_bits_src, wimg_bits_dst;
	unsigned int    cpu_num = 0;
	ppnum_t         pn_src;
	ppnum_t         pn_dst;
	addr64_t        end __assert_only;
	kern_return_t   res = KERN_SUCCESS;

	assert(!__improbable(os_add_overflow(src, bytes, &end)));
	assert(!__improbable(os_add_overflow(dst, bytes, &end)));

	while ((bytes > 0) && (res == KERN_SUCCESS)) {
		src_offset = src & PAGE_MASK;
		dst_offset = dst & PAGE_MASK;
		boolean_t use_copy_window_src = FALSE;
		boolean_t use_copy_window_dst = FALSE;
		vm_size_t count = bytes;
		vm_size_t count2 = bytes;
		if (BCOPY_PHYS_SRC_IS_PHYS(flags)) {
			use_copy_window_src = !pmap_valid_address(src);
			pn_src = (ppnum_t)(src >> PAGE_SHIFT);
#if !defined(__ARM_COHERENT_IO__) && !__ARM_PTE_PHYSMAP__
			count = PAGE_SIZE - src_offset;
			wimg_bits_src = pmap_cache_attributes(pn_src);
			if ((wimg_bits_src & VM_WIMG_MASK) != VM_WIMG_DEFAULT)
				use_copy_window_src = TRUE;
#else
			if (use_copy_window_src) {
				wimg_bits_src = pmap_cache_attributes(pn_src);
				count = PAGE_SIZE - src_offset;
			}
#endif
		}
		if (BCOPY_PHYS_DST_IS_PHYS(flags)) {
			// write preflighting needed for things like dtrace which may write static read-only mappings
			use_copy_window_dst = (!pmap_valid_address(dst) || !mmu_kvtop_wpreflight(phystokv((pmap_paddr_t)dst)));
			pn_dst = (ppnum_t)(dst >> PAGE_SHIFT);
#if !defined(__ARM_COHERENT_IO__) && !__ARM_PTE_PHYSMAP__
			count2 = PAGE_SIZE - dst_offset;
			wimg_bits_dst = pmap_cache_attributes(pn_dst);
			if ((wimg_bits_dst & VM_WIMG_MASK) != VM_WIMG_DEFAULT)
				use_copy_window_dst = TRUE;
#else
			if (use_copy_window_dst) {
				wimg_bits_dst = pmap_cache_attributes(pn_dst);
				count2 = PAGE_SIZE - dst_offset;
			}
#endif
		}

		char *tmp_src;
		char *tmp_dst;

		if (use_copy_window_src || use_copy_window_dst) {
			mp_disable_preemption();
			cpu_num = cpu_number();
		}

		if (use_copy_window_src) {
			src_index = pmap_map_cpu_windows_copy(pn_src, VM_PROT_READ, wimg_bits_src);
			tmp_src = (char*)(pmap_cpu_windows_copy_addr(cpu_num, src_index) + src_offset);
		} else if (BCOPY_PHYS_SRC_IS_PHYS(flags)) {
			tmp_src = (char*)phystokv_range((pmap_paddr_t)src, &count);
		} else {
			tmp_src = (char*)src;
		}
		if (use_copy_window_dst) {
			dst_index = pmap_map_cpu_windows_copy(pn_dst, VM_PROT_READ | VM_PROT_WRITE, wimg_bits_dst);
			tmp_dst = (char*)(pmap_cpu_windows_copy_addr(cpu_num, dst_index) + dst_offset);
		} else if (BCOPY_PHYS_DST_IS_PHYS(flags)) {
			tmp_dst = (char*)phystokv_range((pmap_paddr_t)dst, &count2);
		} else {
			tmp_dst = (char*)dst;
		}

		if (count > count2)
			count = count2;
		if (count > bytes)
			count = bytes;

		if (BCOPY_PHYS_SRC_IS_USER(flags))
			res = copyin((user_addr_t)src, tmp_dst, count);
		else if (BCOPY_PHYS_DST_IS_USER(flags))
			res = copyout(tmp_src, (user_addr_t)dst, count);
		else
			bcopy(tmp_src, tmp_dst, count);

		if (use_copy_window_src)
			pmap_unmap_cpu_windows_copy(src_index);
		if (use_copy_window_dst)
			pmap_unmap_cpu_windows_copy(dst_index);
		if (use_copy_window_src || use_copy_window_dst)
			mp_enable_preemption();

		src += count;
		dst += count;
		bytes -= count;
	}
	return res;
}

void
bcopy_phys(addr64_t src, addr64_t dst, vm_size_t bytes)
{
	bcopy_phys_internal(src, dst, bytes, cppvPsrc | cppvPsnk);
}

void
bzero_phys_nc(addr64_t src64, vm_size_t bytes)
{
	bzero_phys(src64, bytes);
}

/* Zero bytes starting at a physical address */
void
bzero_phys(addr64_t src, vm_size_t bytes)
{
	unsigned int    wimg_bits;
	unsigned int    cpu_num = cpu_number();
	ppnum_t         pn;
	addr64_t        end __assert_only;

	assert(!__improbable(os_add_overflow(src, bytes, &end)));

	vm_offset_t offset = src & PAGE_MASK;
	while (bytes > 0) {
		vm_size_t count = bytes;

		boolean_t use_copy_window = !pmap_valid_address(src);
		pn = (ppnum_t)(src >> PAGE_SHIFT);
#if !defined(__ARM_COHERENT_IO__) && !__ARM_PTE_PHYSMAP__
		count = PAGE_SIZE - offset;
		wimg_bits = pmap_cache_attributes(pn);
		if ((wimg_bits & VM_WIMG_MASK) != VM_WIMG_DEFAULT)
			use_copy_window = TRUE;
#else
		if (use_copy_window) {
			wimg_bits = pmap_cache_attributes(pn);
			count = PAGE_SIZE - offset;
		}
#endif
		char *buf;
		unsigned int index;
		if (use_copy_window) {
			mp_disable_preemption();
			cpu_num = cpu_number();
			index = pmap_map_cpu_windows_copy(pn, VM_PROT_READ | VM_PROT_WRITE, wimg_bits);
			buf = (char *)(pmap_cpu_windows_copy_addr(cpu_num, index) + offset);
		} else {
			buf = (char *)phystokv_range((pmap_paddr_t)src, &count);
		}

		if (count > bytes)
			count = bytes;

		bzero(buf, count);

		if (use_copy_window) {
			pmap_unmap_cpu_windows_copy(index);
			mp_enable_preemption();
		}

		src += count;
		bytes -= count;
		offset = 0;
	}
}

/*
 *  Read data from a physical address.
 */


static unsigned long long
ml_phys_read_data(pmap_paddr_t paddr, int size)
{
	unsigned int   index;
	unsigned int   wimg_bits;
	ppnum_t        pn = (ppnum_t)(paddr >> PAGE_SHIFT);
	ppnum_t        pn_end = (ppnum_t)((paddr + size - 1) >> PAGE_SHIFT); 
	unsigned long  long result = 0;
	vm_offset_t    copywindow_vaddr = 0;
	unsigned char  s1;
	unsigned short s2;
	unsigned int   s4;

	if (__improbable(pn_end != pn))
		panic("%s: paddr 0x%llx spans a page boundary", __func__, (uint64_t)paddr);

#if defined(__ARM_COHERENT_IO__) || __ARM_PTE_PHYSMAP__
	if (pmap_valid_address(paddr)) {
		switch (size) {
		case 1:
			s1 = *(volatile unsigned char *)phystokv(paddr);
			result = s1;
			break;
		case 2:
			s2 = *(volatile unsigned short *)phystokv(paddr);
			result = s2;
			break;
		case 4:
			s4 = *(volatile unsigned int *)phystokv(paddr);
			result = s4;
			break;
		case 8:
			result = *(volatile unsigned long long *)phystokv(paddr);
			break;
		default:
			panic("Invalid size %d for ml_phys_read_data\n", size);
                	break;
		}
		return result;
	}
#endif

	mp_disable_preemption();
	wimg_bits = pmap_cache_attributes(pn);
	index = pmap_map_cpu_windows_copy(pn, VM_PROT_READ, wimg_bits);
	copywindow_vaddr = pmap_cpu_windows_copy_addr(cpu_number(), index) | ((uint32_t)paddr & PAGE_MASK);

	switch (size) {
		case 1:
			s1 = *(volatile unsigned char *)copywindow_vaddr;
			result = s1;
			break;
		case 2:
			s2 = *(volatile unsigned short *)copywindow_vaddr;
			result = s2;
			break;
		case 4:
			s4 = *(volatile unsigned int *)copywindow_vaddr;
			result = s4;
			break;
		case 8:
			result = *(volatile unsigned long long*)copywindow_vaddr;
			break;
		default:
			panic("Invalid size %d for ml_phys_read_data\n", size);
                	break;

	}

	pmap_unmap_cpu_windows_copy(index);
	mp_enable_preemption();

	return result;
}

unsigned int ml_phys_read( vm_offset_t paddr)
{
        return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr, 4);
}

unsigned int ml_phys_read_word(vm_offset_t paddr) {

        return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr, 4);
}

unsigned int ml_phys_read_64(addr64_t paddr64)
{
        return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr64, 4);
}

unsigned int ml_phys_read_word_64(addr64_t paddr64)
{
        return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr64, 4);
}

unsigned int ml_phys_read_half(vm_offset_t paddr)
{
        return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr, 2);
}

unsigned int ml_phys_read_half_64(addr64_t paddr64)
{
        return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr64, 2);
}

unsigned int ml_phys_read_byte(vm_offset_t paddr)
{
        return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr, 1);
}

unsigned int ml_phys_read_byte_64(addr64_t paddr64)
{
        return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr64, 1);
}

unsigned long long ml_phys_read_double(vm_offset_t paddr)
{
        return ml_phys_read_data((pmap_paddr_t)paddr, 8);
}

unsigned long long ml_phys_read_double_64(addr64_t paddr64)
{
        return ml_phys_read_data((pmap_paddr_t)paddr64, 8);
}


/*
 *  Write data to a physical address.
 */

static void
ml_phys_write_data(pmap_paddr_t paddr, unsigned long long data, int size)
{
	unsigned int    index;
	unsigned int    wimg_bits;
	ppnum_t         pn = (ppnum_t)(paddr >> PAGE_SHIFT);
	ppnum_t         pn_end = (ppnum_t)((paddr + size - 1) >> PAGE_SHIFT); 
	vm_offset_t     copywindow_vaddr = 0;

	if (__improbable(pn_end != pn))
		panic("%s: paddr 0x%llx spans a page boundary", __func__, (uint64_t)paddr);

#if defined(__ARM_COHERENT_IO__) || __ARM_PTE_PHYSMAP__
	if (pmap_valid_address(paddr)) {
		switch (size) {
		case 1:
			*(volatile unsigned char *)phystokv(paddr) = (unsigned char)data;
			return;
		case 2:
			*(volatile unsigned short *)phystokv(paddr) = (unsigned short)data;
			return;
		case 4:
			*(volatile unsigned int *)phystokv(paddr) = (unsigned int)data;
			return;
		case 8:
			*(volatile unsigned long long *)phystokv(paddr) = data;
			return;
		default:
			panic("Invalid size %d for ml_phys_write_data\n", size);
		}
	}
#endif

	mp_disable_preemption();
	wimg_bits = pmap_cache_attributes(pn);
	index = pmap_map_cpu_windows_copy(pn, VM_PROT_READ|VM_PROT_WRITE, wimg_bits);
	copywindow_vaddr = pmap_cpu_windows_copy_addr(cpu_number(), index) | ((uint32_t)paddr & PAGE_MASK);

	switch (size) {
		case 1:
			*(volatile unsigned char *)(copywindow_vaddr) =
			                        (unsigned char)data;
			break;
		case 2:
			*(volatile unsigned short *)(copywindow_vaddr) =
			                         (unsigned short)data;
			break;
		case 4:
			*(volatile unsigned int *)(copywindow_vaddr) =
			                           (uint32_t)data;
			break;
		case 8:
			*(volatile unsigned long long *)(copywindow_vaddr) =
			                         (unsigned long long)data;
			break;
		default:
			panic("Invalid size %d for ml_phys_write_data\n", size);
			break;
	}

	pmap_unmap_cpu_windows_copy(index);
	mp_enable_preemption();
}

void ml_phys_write_byte(vm_offset_t paddr, unsigned int data)
{
        ml_phys_write_data((pmap_paddr_t)paddr, data, 1);
}

void ml_phys_write_byte_64(addr64_t paddr64, unsigned int data)
{
        ml_phys_write_data((pmap_paddr_t)paddr64, data, 1);
}

void ml_phys_write_half(vm_offset_t paddr, unsigned int data)
{
        ml_phys_write_data((pmap_paddr_t)paddr, data, 2);
}

void ml_phys_write_half_64(addr64_t paddr64, unsigned int data)
{
        ml_phys_write_data((pmap_paddr_t)paddr64, data, 2);
}

void ml_phys_write(vm_offset_t paddr, unsigned int data)
{
        ml_phys_write_data((pmap_paddr_t)paddr, data, 4);
}

void ml_phys_write_64(addr64_t paddr64, unsigned int data)
{
        ml_phys_write_data((pmap_paddr_t)paddr64, data, 4);
}

void ml_phys_write_word(vm_offset_t paddr, unsigned int data)
{
        ml_phys_write_data((pmap_paddr_t)paddr, data, 4);
}

void ml_phys_write_word_64(addr64_t paddr64, unsigned int data)
{
        ml_phys_write_data((pmap_paddr_t)paddr64, data, 4);
}

void ml_phys_write_double(vm_offset_t paddr, unsigned long long data)
{
        ml_phys_write_data((pmap_paddr_t)paddr, data, 8);
}

void ml_phys_write_double_64(addr64_t paddr64, unsigned long long data)
{
        ml_phys_write_data((pmap_paddr_t)paddr64, data, 8);
}


/*
 * Set indicated bit in bit string.
 */
void
setbit(int bitno, int *s)
{
	s[bitno / INT_SIZE] |= 1 << (bitno % INT_SIZE);
}

/*
 * Clear indicated bit in bit string.
 */
void
clrbit(int bitno, int *s)
{
	s[bitno / INT_SIZE] &= ~(1 << (bitno % INT_SIZE));
}

/*
 * Test if indicated bit is set in bit string.
 */
int
testbit(int bitno, int *s)
{
	return s[bitno / INT_SIZE] & (1 << (bitno % INT_SIZE));
}

/*
 * Find first bit set in bit string.
 */
int
ffsbit(int *s)
{
	int             offset;

	for (offset = 0; !*s; offset += INT_SIZE, ++s);
	return offset + __builtin_ctz(*s);
}

int
ffs(unsigned int mask)
{
	if (mask == 0)
		return 0;

	/*
	 * NOTE: cannot use __builtin_ffs because it generates a call to
	 * 'ffs'
	 */
	return 1 + __builtin_ctz(mask);
}

int
ffsll(unsigned long long mask)
{
	if (mask == 0)
		return 0;

	/*
	 * NOTE: cannot use __builtin_ffsll because it generates a call to
	 * 'ffsll'
	 */
	return 1 + __builtin_ctzll(mask);
}

/*
 * Find last bit set in bit string.
 */
int
fls(unsigned int mask)
{
	if (mask == 0)
		return 0;

	return (sizeof (mask) << 3) - __builtin_clz(mask);
}

int
flsll(unsigned long long mask)
{
	if (mask == 0)
		return 0;

	return (sizeof (mask) << 3) - __builtin_clzll(mask);
}

#undef bcmp
int 
bcmp(
     const void *pa,
     const void *pb,
     size_t len)
{
	const char     *a = (const char *) pa;
	const char     *b = (const char *) pb;

	if (len == 0)
		return 0;

	do
		if (*a++ != *b++)
			break;
	while (--len);

	/*
	 * Check for the overflow case but continue to handle the non-overflow
	 * case the same way just in case someone is using the return value
	 * as more than zero/non-zero
	 */
	if ((len & 0xFFFFFFFF00000000ULL) && !(len & 0x00000000FFFFFFFFULL))
		return 0xFFFFFFFFL;
	else
		return (int)len;
}

#undef memcmp
int
memcmp(const void *s1, const void *s2, size_t n)
{
	if (n != 0) {
		const unsigned char *p1 = s1, *p2 = s2;

		do {
			if (*p1++ != *p2++)
				return (*--p1 - *--p2);
		} while (--n != 0);
	}
	return (0);
}

kern_return_t
copypv(addr64_t source, addr64_t sink, unsigned int size, int which)
{
	if ((which & (cppvPsrc | cppvPsnk)) == 0)	/* Make sure that only one is virtual */
		panic("%s: no more than 1 parameter may be virtual", __func__);

	kern_return_t res = bcopy_phys_internal(source, sink, size, which);

#ifndef __ARM_COHERENT_IO__
        if (which & cppvFsrc)
                flush_dcache64(source, size, ((which & cppvPsrc) == cppvPsrc));

        if (which & cppvFsnk)
                flush_dcache64(sink, size, ((which & cppvPsnk) == cppvPsnk));
#endif

	return res;
}

#if     MACH_ASSERT

extern int copyinframe(vm_address_t fp, char *frame, boolean_t is64bit);

/*
 * Machine-dependent routine to fill in an array with up to callstack_max
 * levels of return pc information.
 */
void
machine_callstack(
		  uintptr_t * buf,
		  vm_size_t callstack_max)
{
	/* Captures the USER call stack */
	uint32_t i=0;

	struct arm_saved_state *state = find_user_regs(current_thread());

	if (!state) {
		while (i<callstack_max)
			buf[i++] = 0;
	} else {
		if (is_saved_state64(state)) {
			uint64_t frame[2];
			buf[i++] = (uintptr_t)get_saved_state_pc(state);
			frame[0] = get_saved_state_fp(state);
			while (i<callstack_max && frame[0] != 0) {
				if (copyinframe(frame[0], (void*) frame, TRUE))
					break;
				buf[i++] = (uintptr_t)frame[1];
			}
		}
		else {
			uint32_t frame[2];
			buf[i++] = (uintptr_t)get_saved_state_pc(state);
			frame[0] = (uint32_t)get_saved_state_fp(state);
			while (i<callstack_max && frame[0] != 0) {
				if (copyinframe(frame[0], (void*) frame, FALSE))
					break;
				buf[i++] = (uintptr_t)frame[1];
			}
		}

		while (i<callstack_max)
			buf[i++] = 0;
	}
}

#endif				/* MACH_ASSERT */

int
clr_be_bit(void)
{
	panic("clr_be_bit");
	return 0;
}

boolean_t
ml_probe_read(
	      __unused vm_offset_t paddr,
	      __unused unsigned int *val)
{
	panic("ml_probe_read() unimplemented");
	return 1;
}

boolean_t
ml_probe_read_64(
		 __unused addr64_t paddr,
		 __unused unsigned int *val)
{
	panic("ml_probe_read_64() unimplemented");
	return 1;
}


void
ml_thread_policy(
		 __unused thread_t thread,
		 __unused unsigned policy_id,
		 __unused unsigned policy_info)
{
  //    <rdar://problem/7141284>: Reduce print noise
  //	kprintf("ml_thread_policy() unimplemented\n");
}

void
panic_unimplemented() 
{
	panic("Not yet implemented.");
}

/* ARM64_TODO <rdar://problem/9198953> */
void abort(void);

void
abort() 
{
	panic("Abort.");
}


#if !MACH_KDP
void
kdp_register_callout(kdp_callout_fn_t fn, void *arg)
{
#pragma unused(fn,arg)
}
#endif
Commit	Line	Data
5ba3f43e A	1	/*
	2	* Copyright (c) 2007-2016 Apple Inc. All rights reserved.
	3	*
	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
	5	*
	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
	14	*
	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
	25	*
	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
	27	*/
	28
	29	#include <mach_assert.h>
	30	#include <mach/vm_types.h>
	31	#include <mach/mach_time.h>
	32	#include <kern/timer.h>
	33	#include <kern/clock.h>
	34	#include <kern/machine.h>
	35	#include <mach/machine.h>
	36	#include <mach/machine/vm_param.h>
	37	#include <mach_kdp.h>
	38	#include <kdp/kdp_udp.h>
	39	#if !MACH_KDP
	40	#include <kdp/kdp_callout.h>
	41	#endif /* !MACH_KDP */
	42	#include <arm/cpu_data.h>
	43	#include <arm/cpu_data_internal.h>
	44	#include <arm/caches_internal.h>
	45
	46	#include <vm/vm_kern.h>
	47	#include <vm/vm_map.h>
	48	#include <vm/pmap.h>
	49
	50	#include <arm/misc_protos.h>
	51
	52	#include <sys/errno.h>
	53
	54	#define INT_SIZE (BYTE_SIZE * sizeof (int))
	55
d9a64523 A	56	#define BCOPY_PHYS_SRC_IS_PHYS(flags) (((flags) & cppvPsrc) != 0)
	57	#define BCOPY_PHYS_DST_IS_PHYS(flags) (((flags) & cppvPsnk) != 0)
	58	#define BCOPY_PHYS_SRC_IS_USER(flags) (((flags) & (cppvPsrc \| cppvKmap)) == 0)
	59	#define BCOPY_PHYS_DST_IS_USER(flags) (((flags) & (cppvPsnk \| cppvKmap)) == 0)
	60
	61	static kern_return_t
	62	bcopy_phys_internal(addr64_t src, addr64_t dst, vm_size_t bytes, int flags)
5ba3f43e A	63	{
	64	unsigned int src_index;
	65	unsigned int dst_index;
	66	vm_offset_t src_offset;
	67	vm_offset_t dst_offset;
	68	unsigned int wimg_bits_src, wimg_bits_dst;
	69	unsigned int cpu_num = 0;
d9a64523 A	70	ppnum_t pn_src;
	71	ppnum_t pn_dst;
	72	addr64_t end __assert_only;
	73	kern_return_t res = KERN_SUCCESS;
	74
	75	assert(!__improbable(os_add_overflow(src, bytes, &end)));
	76	assert(!__improbable(os_add_overflow(dst, bytes, &end)));
	77
	78	while ((bytes > 0) && (res == KERN_SUCCESS)) {
	79	src_offset = src & PAGE_MASK;
	80	dst_offset = dst & PAGE_MASK;
	81	boolean_t use_copy_window_src = FALSE;
	82	boolean_t use_copy_window_dst = FALSE;
	83	vm_size_t count = bytes;
	84	vm_size_t count2 = bytes;
	85	if (BCOPY_PHYS_SRC_IS_PHYS(flags)) {
	86	use_copy_window_src = !pmap_valid_address(src);
	87	pn_src = (ppnum_t)(src >> PAGE_SHIFT);
	88	#if !defined(__ARM_COHERENT_IO__) && !__ARM_PTE_PHYSMAP__
	89	count = PAGE_SIZE - src_offset;
	90	wimg_bits_src = pmap_cache_attributes(pn_src);
	91	if ((wimg_bits_src & VM_WIMG_MASK) != VM_WIMG_DEFAULT)
	92	use_copy_window_src = TRUE;
	93	#else
	94	if (use_copy_window_src) {
	95	wimg_bits_src = pmap_cache_attributes(pn_src);
	96	count = PAGE_SIZE - src_offset;
	97	}
5ba3f43e	98	#endif
d9a64523 A	99	}
	100	if (BCOPY_PHYS_DST_IS_PHYS(flags)) {
	101	// write preflighting needed for things like dtrace which may write static read-only mappings
	102	use_copy_window_dst = (!pmap_valid_address(dst) \|\| !mmu_kvtop_wpreflight(phystokv((pmap_paddr_t)dst)));
	103	pn_dst = (ppnum_t)(dst >> PAGE_SHIFT);
	104	#if !defined(__ARM_COHERENT_IO__) && !__ARM_PTE_PHYSMAP__
	105	count2 = PAGE_SIZE - dst_offset;
	106	wimg_bits_dst = pmap_cache_attributes(pn_dst);
	107	if ((wimg_bits_dst & VM_WIMG_MASK) != VM_WIMG_DEFAULT)
	108	use_copy_window_dst = TRUE;
	109	#else
	110	if (use_copy_window_dst) {
	111	wimg_bits_dst = pmap_cache_attributes(pn_dst);
	112	count2 = PAGE_SIZE - dst_offset;
	113	}
	114	#endif
	115	}
5ba3f43e	116
d9a64523 A	117	char *tmp_src;
d9a64523 A	118	char *tmp_dst;
5ba3f43e	119
d9a64523 A	120	if (use_copy_window_src \|\| use_copy_window_dst) {
	121	mp_disable_preemption();
	122	cpu_num = cpu_number();
	123	}
	124
	125	if (use_copy_window_src) {
	126	src_index = pmap_map_cpu_windows_copy(pn_src, VM_PROT_READ, wimg_bits_src);
	127	tmp_src = (char*)(pmap_cpu_windows_copy_addr(cpu_num, src_index) + src_offset);
	128	} else if (BCOPY_PHYS_SRC_IS_PHYS(flags)) {
	129	tmp_src = (char*)phystokv_range((pmap_paddr_t)src, &count);
	130	} else {
	131	tmp_src = (char*)src;
	132	}
	133	if (use_copy_window_dst) {
	134	dst_index = pmap_map_cpu_windows_copy(pn_dst, VM_PROT_READ \| VM_PROT_WRITE, wimg_bits_dst);
	135	tmp_dst = (char*)(pmap_cpu_windows_copy_addr(cpu_num, dst_index) + dst_offset);
	136	} else if (BCOPY_PHYS_DST_IS_PHYS(flags)) {
	137	tmp_dst = (char*)phystokv_range((pmap_paddr_t)dst, &count2);
	138	} else {
	139	tmp_dst = (char*)dst;
	140	}
5ba3f43e	141
d9a64523 A	142	if (count > count2)
	143	count = count2;
	144	if (count > bytes)
	145	count = bytes;
5ba3f43e	146
d9a64523 A	147	if (BCOPY_PHYS_SRC_IS_USER(flags))
	148	res = copyin((user_addr_t)src, tmp_dst, count);
	149	else if (BCOPY_PHYS_DST_IS_USER(flags))
	150	res = copyout(tmp_src, (user_addr_t)dst, count);
	151	else
	152	bcopy(tmp_src, tmp_dst, count);
5ba3f43e	153
d9a64523 A	154	if (use_copy_window_src)
	155	pmap_unmap_cpu_windows_copy(src_index);
	156	if (use_copy_window_dst)
	157	pmap_unmap_cpu_windows_copy(dst_index);
	158	if (use_copy_window_src \|\| use_copy_window_dst)
	159	mp_enable_preemption();
5ba3f43e	160
d9a64523 A	161	src += count;
	162	dst += count;
	163	bytes -= count;
	164	}
	165	return res;
	166	}
5ba3f43e	167
d9a64523 A	168	void
	169	bcopy_phys(addr64_t src, addr64_t dst, vm_size_t bytes)
	170	{
	171	bcopy_phys_internal(src, dst, bytes, cppvPsrc \| cppvPsnk);
5ba3f43e A	172	}
	173
	174	void
	175	bzero_phys_nc(addr64_t src64, vm_size_t bytes)
	176	{
	177	bzero_phys(src64, bytes);
	178	}
	179
	180	/* Zero bytes starting at a physical address */
	181	void
	182	bzero_phys(addr64_t src, vm_size_t bytes)
	183	{
	184	unsigned int wimg_bits;
	185	unsigned int cpu_num = cpu_number();
d9a64523 A	186	ppnum_t pn;
d9a64523 A	187	addr64_t end __assert_only;
5ba3f43e	188
d9a64523	189	assert(!__improbable(os_add_overflow(src, bytes, &end)));
5ba3f43e	190
d9a64523 A	191	vm_offset_t offset = src & PAGE_MASK;
	192	while (bytes > 0) {
	193	vm_size_t count = bytes;
5ba3f43e	194
d9a64523 A	195	boolean_t use_copy_window = !pmap_valid_address(src);
	196	pn = (ppnum_t)(src >> PAGE_SHIFT);
	197	#if !defined(__ARM_COHERENT_IO__) && !__ARM_PTE_PHYSMAP__
	198	count = PAGE_SIZE - offset;
	199	wimg_bits = pmap_cache_attributes(pn);
	200	if ((wimg_bits & VM_WIMG_MASK) != VM_WIMG_DEFAULT)
	201	use_copy_window = TRUE;
	202	#else
	203	if (use_copy_window) {
	204	wimg_bits = pmap_cache_attributes(pn);
	205	count = PAGE_SIZE - offset;
	206	}
5ba3f43e	207	#endif
d9a64523 A	208	char *buf;
	209	unsigned int index;
	210	if (use_copy_window) {
	211	mp_disable_preemption();
	212	cpu_num = cpu_number();
	213	index = pmap_map_cpu_windows_copy(pn, VM_PROT_READ \| VM_PROT_WRITE, wimg_bits);
	214	buf = (char *)(pmap_cpu_windows_copy_addr(cpu_num, index) + offset);
	215	} else {
	216	buf = (char *)phystokv_range((pmap_paddr_t)src, &count);
	217	}
5ba3f43e A	218
	219	if (count > bytes)
	220	count = bytes;
	221
d9a64523	222	bzero(buf, count);
5ba3f43e	223
d9a64523 A	224	if (use_copy_window) {
	225	pmap_unmap_cpu_windows_copy(index);
	226	mp_enable_preemption();
	227	}
5ba3f43e A	228
	229	src += count;
	230	bytes -= count;
d9a64523	231	offset = 0;
5ba3f43e	232	}
5ba3f43e A	233	}
	234
	235	/*
	236	* Read data from a physical address.
	237	*/
	238
	239
	240	static unsigned long long
	241	ml_phys_read_data(pmap_paddr_t paddr, int size)
	242	{
	243	unsigned int index;
	244	unsigned int wimg_bits;
	245	ppnum_t pn = (ppnum_t)(paddr >> PAGE_SHIFT);
d9a64523	246	ppnum_t pn_end = (ppnum_t)((paddr + size - 1) >> PAGE_SHIFT);
5ba3f43e A	247	unsigned long long result = 0;
	248	vm_offset_t copywindow_vaddr = 0;
	249	unsigned char s1;
	250	unsigned short s2;
	251	unsigned int s4;
	252
d9a64523 A	253	if (__improbable(pn_end != pn))
	254	panic("%s: paddr 0x%llx spans a page boundary", __func__, (uint64_t)paddr);
	255
	256	#if defined(__ARM_COHERENT_IO__) \|\| __ARM_PTE_PHYSMAP__
5ba3f43e A	257	if (pmap_valid_address(paddr)) {
	258	switch (size) {
	259	case 1:
	260	s1 = (volatile unsigned char )phystokv(paddr);
	261	result = s1;
	262	break;
	263	case 2:
	264	s2 = (volatile unsigned short )phystokv(paddr);
	265	result = s2;
	266	break;
	267	case 4:
	268	s4 = (volatile unsigned int )phystokv(paddr);
	269	result = s4;
	270	break;
	271	case 8:
	272	result = (volatile unsigned long long )phystokv(paddr);
	273	break;
	274	default:
	275	panic("Invalid size %d for ml_phys_read_data\n", size);
	276	break;
	277	}
	278	return result;
	279	}
	280	#endif
	281
	282	mp_disable_preemption();
	283	wimg_bits = pmap_cache_attributes(pn);
	284	index = pmap_map_cpu_windows_copy(pn, VM_PROT_READ, wimg_bits);
	285	copywindow_vaddr = pmap_cpu_windows_copy_addr(cpu_number(), index) \| ((uint32_t)paddr & PAGE_MASK);
	286
	287	switch (size) {
	288	case 1:
	289	s1 = (volatile unsigned char )copywindow_vaddr;
	290	result = s1;
	291	break;
	292	case 2:
	293	s2 = (volatile unsigned short )copywindow_vaddr;
	294	result = s2;
	295	break;
	296	case 4:
	297	s4 = (volatile unsigned int )copywindow_vaddr;
	298	result = s4;
	299	break;
	300	case 8:
	301	result = (volatile unsigned long long)copywindow_vaddr;
	302	break;
	303	default:
	304	panic("Invalid size %d for ml_phys_read_data\n", size);
	305	break;
	306
	307	}
	308
	309	pmap_unmap_cpu_windows_copy(index);
	310	mp_enable_preemption();
	311
	312	return result;
	313	}
	314
	315	unsigned int ml_phys_read( vm_offset_t paddr)
	316	{
	317	return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr, 4);
	318	}
	319
	320	unsigned int ml_phys_read_word(vm_offset_t paddr) {
321
322	return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr, 4);
323	}
324
325	unsigned int ml_phys_read_64(addr64_t paddr64)
326	{
327	return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr64, 4);
328	}
329
330	unsigned int ml_phys_read_word_64(addr64_t paddr64)
331	{
332	return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr64, 4);
333	}
334
335	unsigned int ml_phys_read_half(vm_offset_t paddr)
336	{
337	return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr, 2);
338	}
339
340	unsigned int ml_phys_read_half_64(addr64_t paddr64)
341	{
342	return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr64, 2);
343	}
344
345	unsigned int ml_phys_read_byte(vm_offset_t paddr)
346	{
347	return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr, 1);
348	}
349
350	unsigned int ml_phys_read_byte_64(addr64_t paddr64)
351	{
352	return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr64, 1);
353	}
354
355	unsigned long long ml_phys_read_double(vm_offset_t paddr)
356	{
357	return ml_phys_read_data((pmap_paddr_t)paddr, 8);
358	}
359
360	unsigned long long ml_phys_read_double_64(addr64_t paddr64)
361	{
362	return ml_phys_read_data((pmap_paddr_t)paddr64, 8);
363	}
364
365
366
367	/*
368	* Write data to a physical address.
369	*/
370
371	static void
372	ml_phys_write_data(pmap_paddr_t paddr, unsigned long long data, int size)
373	{
374	unsigned int index;
375	unsigned int wimg_bits;
376	ppnum_t pn = (ppnum_t)(paddr >> PAGE_SHIFT);
d9a64523	377	ppnum_t pn_end = (ppnum_t)((paddr + size - 1) >> PAGE_SHIFT);
5ba3f43e A	378	vm_offset_t copywindow_vaddr = 0;
5ba3f43e A	379
d9a64523 A	380	if (__improbable(pn_end != pn))
	381	panic("%s: paddr 0x%llx spans a page boundary", __func__, (uint64_t)paddr);
	382
	383	#if defined(__ARM_COHERENT_IO__) \|\| __ARM_PTE_PHYSMAP__
5ba3f43e A	384	if (pmap_valid_address(paddr)) {
	385	switch (size) {
	386	case 1:
	387	(volatile unsigned char )phystokv(paddr) = (unsigned char)data;
	388	return;
	389	case 2:
	390	(volatile unsigned short )phystokv(paddr) = (unsigned short)data;
	391	return;
	392	case 4:
	393	(volatile unsigned int )phystokv(paddr) = (unsigned int)data;
	394	return;
	395	case 8:
	396	(volatile unsigned long long )phystokv(paddr) = data;
	397	return;
	398	default:
	399	panic("Invalid size %d for ml_phys_write_data\n", size);
	400	}
	401	}
	402	#endif
	403
	404	mp_disable_preemption();
	405	wimg_bits = pmap_cache_attributes(pn);
	406	index = pmap_map_cpu_windows_copy(pn, VM_PROT_READ\|VM_PROT_WRITE, wimg_bits);
	407	copywindow_vaddr = pmap_cpu_windows_copy_addr(cpu_number(), index) \| ((uint32_t)paddr & PAGE_MASK);
	408
	409	switch (size) {
	410	case 1:
	411	(volatile unsigned char )(copywindow_vaddr) =
	412	(unsigned char)data;
	413	break;
	414	case 2:
	415	(volatile unsigned short )(copywindow_vaddr) =
	416	(unsigned short)data;
	417	break;
	418	case 4:
	419	(volatile unsigned int )(copywindow_vaddr) =
	420	(uint32_t)data;
	421	break;
	422	case 8:
	423	(volatile unsigned long long )(copywindow_vaddr) =
	424	(unsigned long long)data;
	425	break;
	426	default:
	427	panic("Invalid size %d for ml_phys_write_data\n", size);
	428	break;
	429	}
	430
	431	pmap_unmap_cpu_windows_copy(index);
	432	mp_enable_preemption();
	433	}
	434
	435	void ml_phys_write_byte(vm_offset_t paddr, unsigned int data)
	436	{
	437	ml_phys_write_data((pmap_paddr_t)paddr, data, 1);
	438	}
	439
	440	void ml_phys_write_byte_64(addr64_t paddr64, unsigned int data)
	441	{
	442	ml_phys_write_data((pmap_paddr_t)paddr64, data, 1);
	443	}
	444
	445	void ml_phys_write_half(vm_offset_t paddr, unsigned int data)
	446	{
	447	ml_phys_write_data((pmap_paddr_t)paddr, data, 2);
448	}
449
450	void ml_phys_write_half_64(addr64_t paddr64, unsigned int data)
451	{
452	ml_phys_write_data((pmap_paddr_t)paddr64, data, 2);
453	}
454
455	void ml_phys_write(vm_offset_t paddr, unsigned int data)
456	{
457	ml_phys_write_data((pmap_paddr_t)paddr, data, 4);
458	}
459
460	void ml_phys_write_64(addr64_t paddr64, unsigned int data)
461	{
462	ml_phys_write_data((pmap_paddr_t)paddr64, data, 4);
463	}
464
465	void ml_phys_write_word(vm_offset_t paddr, unsigned int data)
466	{
467	ml_phys_write_data((pmap_paddr_t)paddr, data, 4);
468	}
469
470	void ml_phys_write_word_64(addr64_t paddr64, unsigned int data)
471	{
472	ml_phys_write_data((pmap_paddr_t)paddr64, data, 4);
473	}
474
475	void ml_phys_write_double(vm_offset_t paddr, unsigned long long data)
476	{
477	ml_phys_write_data((pmap_paddr_t)paddr, data, 8);
478	}
479
480	void ml_phys_write_double_64(addr64_t paddr64, unsigned long long data)
481	{
482	ml_phys_write_data((pmap_paddr_t)paddr64, data, 8);
483	}
484
485
486	/*
487	* Set indicated bit in bit string.
488	*/
489	void
490	setbit(int bitno, int *s)
491	{
492	s[bitno / INT_SIZE] \|= 1 << (bitno % INT_SIZE);
493	}
494
495	/*
496	* Clear indicated bit in bit string.
497	*/
498	void
499	clrbit(int bitno, int *s)
500	{
501	s[bitno / INT_SIZE] &= ~(1 << (bitno % INT_SIZE));
502	}
503
504	/*
505	* Test if indicated bit is set in bit string.
506	*/
507	int
508	testbit(int bitno, int *s)
509	{
510	return s[bitno / INT_SIZE] & (1 << (bitno % INT_SIZE));
511	}
512
513	/*
514	* Find first bit set in bit string.
515	*/
516	int
517	ffsbit(int *s)
518	{
519	int offset;
520
521	for (offset = 0; !*s; offset += INT_SIZE, ++s);
522	return offset + __builtin_ctz(*s);
523	}
524
525	int
526	ffs(unsigned int mask)
527	{
528	if (mask == 0)
529	return 0;
530
531	/*
532	* NOTE: cannot use __builtin_ffs because it generates a call to
533	* 'ffs'
534	*/
535	return 1 + __builtin_ctz(mask);
536	}
537
538	int
539	ffsll(unsigned long long mask)
540	{
541	if (mask == 0)
542	return 0;
543
544	/*
545	* NOTE: cannot use __builtin_ffsll because it generates a call to
546	* 'ffsll'
547	*/
548	return 1 + __builtin_ctzll(mask);
549	}
550
551	/*
552	* Find last bit set in bit string.
553	*/
554	int
555	fls(unsigned int mask)
556	{
557	if (mask == 0)
558	return 0;
559
560	return (sizeof (mask) << 3) - __builtin_clz(mask);
561	}
562
563	int
564	flsll(unsigned long long mask)
565	{
566	if (mask == 0)
567	return 0;
568
569	return (sizeof (mask) << 3) - __builtin_clzll(mask);
570	}
571
572	#undef bcmp
573	int
574	bcmp(
575	const void *pa,
576	const void *pb,
577	size_t len)
578	{
579	const char a = (const char ) pa;
580	const char b = (const char ) pb;
581
582	if (len == 0)
583	return 0;
584
585	do
586	if (a++ != b++)
587	break;
588	while (--len);
589
590	/*
591	* Check for the overflow case but continue to handle the non-overflow
592	* case the same way just in case someone is using the return value
593	* as more than zero/non-zero
594	*/
595	if ((len & 0xFFFFFFFF00000000ULL) && !(len & 0x00000000FFFFFFFFULL))
596	return 0xFFFFFFFFL;
597	else
598	return (int)len;
599	}
600
601	#undef memcmp
602	int
603	memcmp(const void s1, const void s2, size_t n)
604	{
605	if (n != 0) {
606	const unsigned char p1 = s1, p2 = s2;
607
608	do {
609	if (p1++ != p2++)
610	return (--p1 - --p2);
611	} while (--n != 0);
612	}
613	return (0);
614	}
615
616	kern_return_t
617	copypv(addr64_t source, addr64_t sink, unsigned int size, int which)
618	{
d9a64523 A	619	if ((which & (cppvPsrc \| cppvPsnk)) == 0) /* Make sure that only one is virtual */
d9a64523 A	620	panic("%s: no more than 1 parameter may be virtual", __func__);
5ba3f43e	621
d9a64523	622	kern_return_t res = bcopy_phys_internal(source, sink, size, which);
5ba3f43e	623
d9a64523 A	624	#ifndef __ARM_COHERENT_IO__
	625	if (which & cppvFsrc)
	626	flush_dcache64(source, size, ((which & cppvPsrc) == cppvPsrc));
	627
	628	if (which & cppvFsnk)
	629	flush_dcache64(sink, size, ((which & cppvPsnk) == cppvPsnk));
5ba3f43e	630	#endif
5ba3f43e	631
d9a64523 A	632	return res;
d9a64523 A	633	}
5ba3f43e A	634
	635	#if MACH_ASSERT
	636
	637	extern int copyinframe(vm_address_t fp, char *frame, boolean_t is64bit);
	638
	639	/*
	640	* Machine-dependent routine to fill in an array with up to callstack_max
	641	* levels of return pc information.
	642	*/
	643	void
	644	machine_callstack(
	645	uintptr_t * buf,
	646	vm_size_t callstack_max)
	647	{
	648	/* Captures the USER call stack */
	649	uint32_t i=0;
	650
	651	struct arm_saved_state *state = find_user_regs(current_thread());
	652
	653	if (!state) {
	654	while (i<callstack_max)
	655	buf[i++] = 0;
	656	} else {
	657	if (is_saved_state64(state)) {
	658	uint64_t frame[2];
	659	buf[i++] = (uintptr_t)get_saved_state_pc(state);
	660	frame[0] = get_saved_state_fp(state);
	661	while (i<callstack_max && frame[0] != 0) {
	662	if (copyinframe(frame[0], (void*) frame, TRUE))
	663	break;
	664	buf[i++] = (uintptr_t)frame[1];
	665	}
	666	}
	667	else {
	668	uint32_t frame[2];
	669	buf[i++] = (uintptr_t)get_saved_state_pc(state);
	670	frame[0] = (uint32_t)get_saved_state_fp(state);
	671	while (i<callstack_max && frame[0] != 0) {
	672	if (copyinframe(frame[0], (void*) frame, FALSE))
	673	break;
	674	buf[i++] = (uintptr_t)frame[1];
	675	}
	676	}
	677
	678	while (i<callstack_max)
	679	buf[i++] = 0;
	680	}
	681	}
	682
	683	#endif /* MACH_ASSERT */
	684
	685	int
	686	clr_be_bit(void)
	687	{
	688	panic("clr_be_bit");
	689	return 0;
	690	}
	691
	692	boolean_t
	693	ml_probe_read(
	694	__unused vm_offset_t paddr,
	695	__unused unsigned int *val)
	696	{
	697	panic("ml_probe_read() unimplemented");
698	return 1;
699	}
700
701	boolean_t
702	ml_probe_read_64(
703	__unused addr64_t paddr,
704	__unused unsigned int *val)
705	{
706	panic("ml_probe_read_64() unimplemented");
707	return 1;
708	}
709
710
711	void
712	ml_thread_policy(
713	__unused thread_t thread,
714	__unused unsigned policy_id,
715	__unused unsigned policy_info)
716	{
717	// <rdar://problem/7141284>: Reduce print noise
718	// kprintf("ml_thread_policy() unimplemented\n");
719	}
720
721	void
722	panic_unimplemented()
723	{
724	panic("Not yet implemented.");
725	}
726
727	/* ARM64_TODO <rdar://problem/9198953> */
728	void abort(void);
729
730	void
731	abort()
732	{
733	panic("Abort.");
734	}
735
736
737	#if !MACH_KDP
738	void
739	kdp_register_callout(kdp_callout_fn_t fn, void *arg)
740	{
741	#pragma unused(fn,arg)
742	}
743	#endif
744