X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/5ba3f43ea354af8ad55bea84372a2bc834d8757c..HEAD:/osfmk/arm64/loose_ends.c diff --git a/osfmk/arm64/loose_ends.c b/osfmk/arm64/loose_ends.c index 1eec53104..8479f24bf 100644 --- a/osfmk/arm64/loose_ends.c +++ b/osfmk/arm64/loose_ends.c @@ -50,11 +50,17 @@ #include #include +#include #define INT_SIZE (BYTE_SIZE * sizeof (int)) -void -bcopy_phys(addr64_t src, addr64_t dst, vm_size_t bytes) +#define BCOPY_PHYS_SRC_IS_PHYS(flags) (((flags) & cppvPsrc) != 0) +#define BCOPY_PHYS_DST_IS_PHYS(flags) (((flags) & cppvPsnk) != 0) +#define BCOPY_PHYS_SRC_IS_USER(flags) (((flags) & (cppvPsrc | cppvKmap)) == 0) +#define BCOPY_PHYS_DST_IS_USER(flags) (((flags) & (cppvPsnk | cppvKmap)) == 0) + +static kern_return_t +bcopy_phys_internal(addr64_t src, addr64_t dst, vm_size_t bytes, int flags) { unsigned int src_index; unsigned int dst_index; @@ -62,49 +68,120 @@ bcopy_phys(addr64_t src, addr64_t dst, vm_size_t bytes) vm_offset_t dst_offset; unsigned int wimg_bits_src, wimg_bits_dst; unsigned int cpu_num = 0; - ppnum_t pn_src = (ppnum_t)(src >> PAGE_SHIFT); - ppnum_t pn_dst = (ppnum_t)(dst >> PAGE_SHIFT); - -#ifdef __ARM_COHERENT_IO__ - if (pmap_valid_address(src) && - pmap_valid_address(dst) && - (mmu_kvtop_wpreflight(phystokv((pmap_paddr_t) dst)))) { - bcopy((char *)phystokv((pmap_paddr_t) src), (char *)phystokv((pmap_paddr_t) dst), bytes); - return; + ppnum_t pn_src; + ppnum_t pn_dst; + addr64_t end __assert_only; + kern_return_t res = KERN_SUCCESS; + + if (!BCOPY_PHYS_SRC_IS_USER(flags)) { + assert(!__improbable(os_add_overflow(src, bytes, &end))); + } + if (!BCOPY_PHYS_DST_IS_USER(flags)) { + assert(!__improbable(os_add_overflow(dst, bytes, &end))); } + + while ((bytes > 0) && (res == KERN_SUCCESS)) { + src_offset = src & PAGE_MASK; + dst_offset = dst & PAGE_MASK; + boolean_t use_copy_window_src = FALSE; + boolean_t use_copy_window_dst = FALSE; + vm_size_t count = bytes; + vm_size_t count2 = bytes; + if (BCOPY_PHYS_SRC_IS_PHYS(flags)) { + use_copy_window_src = !pmap_valid_address(src); + pn_src = (ppnum_t)(src >> PAGE_SHIFT); +#if !defined(__ARM_COHERENT_IO__) && !__ARM_PTE_PHYSMAP__ + count = PAGE_SIZE - src_offset; + wimg_bits_src = pmap_cache_attributes(pn_src); + if ((wimg_bits_src & VM_WIMG_MASK) != VM_WIMG_DEFAULT) { + use_copy_window_src = TRUE; + } +#else + if (use_copy_window_src) { + wimg_bits_src = pmap_cache_attributes(pn_src); + count = PAGE_SIZE - src_offset; + } #endif + } + if (BCOPY_PHYS_DST_IS_PHYS(flags)) { + // write preflighting needed for things like dtrace which may write static read-only mappings + use_copy_window_dst = (!pmap_valid_address(dst) || !mmu_kvtop_wpreflight(phystokv((pmap_paddr_t)dst))); + pn_dst = (ppnum_t)(dst >> PAGE_SHIFT); +#if !defined(__ARM_COHERENT_IO__) && !__ARM_PTE_PHYSMAP__ + count2 = PAGE_SIZE - dst_offset; + wimg_bits_dst = pmap_cache_attributes(pn_dst); + if ((wimg_bits_dst & VM_WIMG_MASK) != VM_WIMG_DEFAULT) { + use_copy_window_dst = TRUE; + } +#else + if (use_copy_window_dst) { + wimg_bits_dst = pmap_cache_attributes(pn_dst); + count2 = PAGE_SIZE - dst_offset; + } +#endif + } - wimg_bits_src = pmap_cache_attributes(pn_src); - wimg_bits_dst = pmap_cache_attributes(pn_dst); + char *tmp_src; + char *tmp_dst; -#ifndef __ARM_COHERENT_IO__ - if (((wimg_bits_src & VM_WIMG_MASK) == VM_WIMG_DEFAULT) && - ((wimg_bits_dst & VM_WIMG_MASK) == VM_WIMG_DEFAULT) && - (mmu_kvtop_wpreflight(phystokv((pmap_paddr_t) dst)))) { - /* Fast path - dst is writable and both source and destination have default attributes */ - bcopy((char *)phystokv((pmap_paddr_t) src), (char *)phystokv((pmap_paddr_t) dst), bytes); - return; - } -#endif + if (use_copy_window_src || use_copy_window_dst) { + mp_disable_preemption(); + cpu_num = cpu_number(); + } - src_offset = src & PAGE_MASK; - dst_offset = dst & PAGE_MASK; + if (use_copy_window_src) { + src_index = pmap_map_cpu_windows_copy(pn_src, VM_PROT_READ, wimg_bits_src); + tmp_src = (char*)(pmap_cpu_windows_copy_addr(cpu_num, src_index) + src_offset); + } else if (BCOPY_PHYS_SRC_IS_PHYS(flags)) { + tmp_src = (char*)phystokv_range((pmap_paddr_t)src, &count); + } else { + tmp_src = (char*)src; + } + if (use_copy_window_dst) { + dst_index = pmap_map_cpu_windows_copy(pn_dst, VM_PROT_READ | VM_PROT_WRITE, wimg_bits_dst); + tmp_dst = (char*)(pmap_cpu_windows_copy_addr(cpu_num, dst_index) + dst_offset); + } else if (BCOPY_PHYS_DST_IS_PHYS(flags)) { + tmp_dst = (char*)phystokv_range((pmap_paddr_t)dst, &count2); + } else { + tmp_dst = (char*)dst; + } - if ((src_offset + bytes) > PAGE_SIZE || (dst_offset + bytes) > PAGE_SIZE) - panic("bcopy extends beyond copy windows"); + if (count > count2) { + count = count2; + } + if (count > bytes) { + count = bytes; + } - mp_disable_preemption(); - cpu_num = cpu_number(); - src_index = pmap_map_cpu_windows_copy(pn_src, VM_PROT_READ, wimg_bits_src); - dst_index = pmap_map_cpu_windows_copy(pn_dst, VM_PROT_READ|VM_PROT_WRITE, wimg_bits_dst); + if (BCOPY_PHYS_SRC_IS_USER(flags)) { + res = copyin((user_addr_t)src, tmp_dst, count); + } else if (BCOPY_PHYS_DST_IS_USER(flags)) { + res = copyout(tmp_src, (user_addr_t)dst, count); + } else { + bcopy(tmp_src, tmp_dst, count); + } - bcopy((char *)(pmap_cpu_windows_copy_addr(cpu_num, src_index) + src_offset), - (char *)(pmap_cpu_windows_copy_addr(cpu_num, dst_index) + dst_offset), - bytes); + if (use_copy_window_src) { + pmap_unmap_cpu_windows_copy(src_index); + } + if (use_copy_window_dst) { + pmap_unmap_cpu_windows_copy(dst_index); + } + if (use_copy_window_src || use_copy_window_dst) { + mp_enable_preemption(); + } - pmap_unmap_cpu_windows_copy(src_index); - pmap_unmap_cpu_windows_copy(dst_index); - mp_enable_preemption(); + src += count; + dst += count; + bytes -= count; + } + return res; +} + +void +bcopy_phys(addr64_t src, addr64_t dst, vm_size_t bytes) +{ + bcopy_phys_internal(src, dst, bytes, cppvPsrc | cppvPsnk); } void @@ -113,54 +190,72 @@ bzero_phys_nc(addr64_t src64, vm_size_t bytes) bzero_phys(src64, bytes); } +extern void *secure_memset(void *, int, size_t); + /* Zero bytes starting at a physical address */ void bzero_phys(addr64_t src, vm_size_t bytes) { unsigned int wimg_bits; unsigned int cpu_num = cpu_number(); - ppnum_t pn = (ppnum_t)(src >> PAGE_SHIFT); - -#ifdef __ARM_COHERENT_IO__ - if (pmap_valid_address(src)) { - bzero((char *)phystokv((pmap_paddr_t) src), bytes); - return; - } -#endif + ppnum_t pn; + addr64_t end __assert_only; - wimg_bits = pmap_cache_attributes(pn); - -#ifndef __ARM_COHERENT_IO__ - if ((wimg_bits & VM_WIMG_MASK) == VM_WIMG_DEFAULT) { - /* Fast path - default attributes */ - bzero((char *)phystokv((pmap_paddr_t) src), bytes); - return; - } -#endif - - mp_disable_preemption(); - cpu_num = cpu_number(); + assert(!__improbable(os_add_overflow(src, bytes, &end))); + vm_offset_t offset = src & PAGE_MASK; while (bytes > 0) { - vm_offset_t offset = src & PAGE_MASK; - uint64_t count = PAGE_SIZE - offset; - - if (count > bytes) - count = bytes; + vm_size_t count = bytes; + boolean_t use_copy_window = !pmap_valid_address(src); pn = (ppnum_t)(src >> PAGE_SHIFT); + wimg_bits = pmap_cache_attributes(pn); +#if !defined(__ARM_COHERENT_IO__) && !__ARM_PTE_PHYSMAP__ + count = PAGE_SIZE - offset; + if ((wimg_bits & VM_WIMG_MASK) != VM_WIMG_DEFAULT) { + use_copy_window = TRUE; + } +#else + if (use_copy_window) { + count = PAGE_SIZE - offset; + } +#endif + char *buf; + unsigned int index; + if (use_copy_window) { + mp_disable_preemption(); + cpu_num = cpu_number(); + index = pmap_map_cpu_windows_copy(pn, VM_PROT_READ | VM_PROT_WRITE, wimg_bits); + buf = (char *)(pmap_cpu_windows_copy_addr(cpu_num, index) + offset); + } else { + buf = (char *)phystokv_range((pmap_paddr_t)src, &count); + } - unsigned int index = pmap_map_cpu_windows_copy(pn, VM_PROT_READ | VM_PROT_WRITE, wimg_bits); + if (count > bytes) { + count = bytes; + } - bzero((char *)(pmap_cpu_windows_copy_addr(cpu_num, index) + offset), count); + switch (wimg_bits & VM_WIMG_MASK) { + case VM_WIMG_DEFAULT: + case VM_WIMG_WCOMB: + case VM_WIMG_INNERWBACK: + case VM_WIMG_WTHRU: + bzero(buf, count); + break; + default: + /* 'dc zva' performed by bzero is not safe for device memory */ + secure_memset((void*)buf, 0, count); + } - pmap_unmap_cpu_windows_copy(index); + if (use_copy_window) { + pmap_unmap_cpu_windows_copy(index); + mp_enable_preemption(); + } src += count; bytes -= count; + offset = 0; } - - mp_enable_preemption(); } /* @@ -174,13 +269,18 @@ ml_phys_read_data(pmap_paddr_t paddr, int size) unsigned int index; unsigned int wimg_bits; ppnum_t pn = (ppnum_t)(paddr >> PAGE_SHIFT); + ppnum_t pn_end = (ppnum_t)((paddr + size - 1) >> PAGE_SHIFT); unsigned long long result = 0; vm_offset_t copywindow_vaddr = 0; unsigned char s1; unsigned short s2; unsigned int s4; -#ifdef __ARM_COHERENT_IO__ + if (__improbable(pn_end != pn)) { + panic("%s: paddr 0x%llx spans a page boundary", __func__, (uint64_t)paddr); + } + +#if defined(__ARM_COHERENT_IO__) || __ARM_PTE_PHYSMAP__ if (pmap_valid_address(paddr)) { switch (size) { case 1: @@ -200,7 +300,7 @@ ml_phys_read_data(pmap_paddr_t paddr, int size) break; default: panic("Invalid size %d for ml_phys_read_data\n", size); - break; + break; } return result; } @@ -212,25 +312,24 @@ ml_phys_read_data(pmap_paddr_t paddr, int size) copywindow_vaddr = pmap_cpu_windows_copy_addr(cpu_number(), index) | ((uint32_t)paddr & PAGE_MASK); switch (size) { - case 1: - s1 = *(volatile unsigned char *)copywindow_vaddr; - result = s1; - break; - case 2: - s2 = *(volatile unsigned short *)copywindow_vaddr; - result = s2; - break; - case 4: - s4 = *(volatile unsigned int *)copywindow_vaddr; - result = s4; - break; - case 8: - result = *(volatile unsigned long long*)copywindow_vaddr; - break; - default: - panic("Invalid size %d for ml_phys_read_data\n", size); - break; - + case 1: + s1 = *(volatile unsigned char *)copywindow_vaddr; + result = s1; + break; + case 2: + s2 = *(volatile unsigned short *)copywindow_vaddr; + result = s2; + break; + case 4: + s4 = *(volatile unsigned int *)copywindow_vaddr; + result = s4; + break; + case 8: + result = *(volatile unsigned long long*)copywindow_vaddr; + break; + default: + panic("Invalid size %d for ml_phys_read_data\n", size); + break; } pmap_unmap_cpu_windows_copy(index); @@ -239,54 +338,64 @@ ml_phys_read_data(pmap_paddr_t paddr, int size) return result; } -unsigned int ml_phys_read( vm_offset_t paddr) +unsigned int +ml_phys_read( vm_offset_t paddr) { - return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr, 4); + return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr, 4); } -unsigned int ml_phys_read_word(vm_offset_t paddr) { - - return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr, 4); +unsigned int +ml_phys_read_word(vm_offset_t paddr) +{ + return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr, 4); } -unsigned int ml_phys_read_64(addr64_t paddr64) +unsigned int +ml_phys_read_64(addr64_t paddr64) { - return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr64, 4); + return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr64, 4); } -unsigned int ml_phys_read_word_64(addr64_t paddr64) +unsigned int +ml_phys_read_word_64(addr64_t paddr64) { - return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr64, 4); + return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr64, 4); } -unsigned int ml_phys_read_half(vm_offset_t paddr) +unsigned int +ml_phys_read_half(vm_offset_t paddr) { - return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr, 2); + return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr, 2); } -unsigned int ml_phys_read_half_64(addr64_t paddr64) +unsigned int +ml_phys_read_half_64(addr64_t paddr64) { - return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr64, 2); + return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr64, 2); } -unsigned int ml_phys_read_byte(vm_offset_t paddr) +unsigned int +ml_phys_read_byte(vm_offset_t paddr) { - return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr, 1); + return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr, 1); } -unsigned int ml_phys_read_byte_64(addr64_t paddr64) +unsigned int +ml_phys_read_byte_64(addr64_t paddr64) { - return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr64, 1); + return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr64, 1); } -unsigned long long ml_phys_read_double(vm_offset_t paddr) +unsigned long long +ml_phys_read_double(vm_offset_t paddr) { - return ml_phys_read_data((pmap_paddr_t)paddr, 8); + return ml_phys_read_data((pmap_paddr_t)paddr, 8); } -unsigned long long ml_phys_read_double_64(addr64_t paddr64) +unsigned long long +ml_phys_read_double_64(addr64_t paddr64) { - return ml_phys_read_data((pmap_paddr_t)paddr64, 8); + return ml_phys_read_data((pmap_paddr_t)paddr64, 8); } @@ -301,9 +410,14 @@ ml_phys_write_data(pmap_paddr_t paddr, unsigned long long data, int size) unsigned int index; unsigned int wimg_bits; ppnum_t pn = (ppnum_t)(paddr >> PAGE_SHIFT); + ppnum_t pn_end = (ppnum_t)((paddr + size - 1) >> PAGE_SHIFT); vm_offset_t copywindow_vaddr = 0; -#ifdef __ARM_COHERENT_IO__ + if (__improbable(pn_end != pn)) { + panic("%s: paddr 0x%llx spans a page boundary", __func__, (uint64_t)paddr); + } + +#if defined(__ARM_COHERENT_IO__) || __ARM_PTE_PHYSMAP__ if (pmap_valid_address(paddr)) { switch (size) { case 1: @@ -326,83 +440,93 @@ ml_phys_write_data(pmap_paddr_t paddr, unsigned long long data, int size) mp_disable_preemption(); wimg_bits = pmap_cache_attributes(pn); - index = pmap_map_cpu_windows_copy(pn, VM_PROT_READ|VM_PROT_WRITE, wimg_bits); + index = pmap_map_cpu_windows_copy(pn, VM_PROT_READ | VM_PROT_WRITE, wimg_bits); copywindow_vaddr = pmap_cpu_windows_copy_addr(cpu_number(), index) | ((uint32_t)paddr & PAGE_MASK); switch (size) { - case 1: - *(volatile unsigned char *)(copywindow_vaddr) = - (unsigned char)data; - break; - case 2: - *(volatile unsigned short *)(copywindow_vaddr) = - (unsigned short)data; - break; - case 4: - *(volatile unsigned int *)(copywindow_vaddr) = - (uint32_t)data; - break; - case 8: - *(volatile unsigned long long *)(copywindow_vaddr) = - (unsigned long long)data; - break; - default: - panic("Invalid size %d for ml_phys_write_data\n", size); - break; + case 1: + *(volatile unsigned char *)(copywindow_vaddr) = + (unsigned char)data; + break; + case 2: + *(volatile unsigned short *)(copywindow_vaddr) = + (unsigned short)data; + break; + case 4: + *(volatile unsigned int *)(copywindow_vaddr) = + (uint32_t)data; + break; + case 8: + *(volatile unsigned long long *)(copywindow_vaddr) = + (unsigned long long)data; + break; + default: + panic("Invalid size %d for ml_phys_write_data\n", size); + break; } pmap_unmap_cpu_windows_copy(index); mp_enable_preemption(); } -void ml_phys_write_byte(vm_offset_t paddr, unsigned int data) +void +ml_phys_write_byte(vm_offset_t paddr, unsigned int data) { - ml_phys_write_data((pmap_paddr_t)paddr, data, 1); + ml_phys_write_data((pmap_paddr_t)paddr, data, 1); } -void ml_phys_write_byte_64(addr64_t paddr64, unsigned int data) +void +ml_phys_write_byte_64(addr64_t paddr64, unsigned int data) { - ml_phys_write_data((pmap_paddr_t)paddr64, data, 1); + ml_phys_write_data((pmap_paddr_t)paddr64, data, 1); } -void ml_phys_write_half(vm_offset_t paddr, unsigned int data) +void +ml_phys_write_half(vm_offset_t paddr, unsigned int data) { - ml_phys_write_data((pmap_paddr_t)paddr, data, 2); + ml_phys_write_data((pmap_paddr_t)paddr, data, 2); } -void ml_phys_write_half_64(addr64_t paddr64, unsigned int data) +void +ml_phys_write_half_64(addr64_t paddr64, unsigned int data) { - ml_phys_write_data((pmap_paddr_t)paddr64, data, 2); + ml_phys_write_data((pmap_paddr_t)paddr64, data, 2); } -void ml_phys_write(vm_offset_t paddr, unsigned int data) +void +ml_phys_write(vm_offset_t paddr, unsigned int data) { - ml_phys_write_data((pmap_paddr_t)paddr, data, 4); + ml_phys_write_data((pmap_paddr_t)paddr, data, 4); } -void ml_phys_write_64(addr64_t paddr64, unsigned int data) +void +ml_phys_write_64(addr64_t paddr64, unsigned int data) { - ml_phys_write_data((pmap_paddr_t)paddr64, data, 4); + ml_phys_write_data((pmap_paddr_t)paddr64, data, 4); } -void ml_phys_write_word(vm_offset_t paddr, unsigned int data) +void +ml_phys_write_word(vm_offset_t paddr, unsigned int data) { - ml_phys_write_data((pmap_paddr_t)paddr, data, 4); + ml_phys_write_data((pmap_paddr_t)paddr, data, 4); } -void ml_phys_write_word_64(addr64_t paddr64, unsigned int data) +void +ml_phys_write_word_64(addr64_t paddr64, unsigned int data) { - ml_phys_write_data((pmap_paddr_t)paddr64, data, 4); + ml_phys_write_data((pmap_paddr_t)paddr64, data, 4); } -void ml_phys_write_double(vm_offset_t paddr, unsigned long long data) +void +ml_phys_write_double(vm_offset_t paddr, unsigned long long data) { - ml_phys_write_data((pmap_paddr_t)paddr, data, 8); + ml_phys_write_data((pmap_paddr_t)paddr, data, 8); } -void ml_phys_write_double_64(addr64_t paddr64, unsigned long long data) +void +ml_phys_write_double_64(addr64_t paddr64, unsigned long long data) { - ml_phys_write_data((pmap_paddr_t)paddr64, data, 8); + ml_phys_write_data((pmap_paddr_t)paddr64, data, 8); } @@ -412,7 +536,7 @@ void ml_phys_write_double_64(addr64_t paddr64, unsigned long long data) void setbit(int bitno, int *s) { - s[bitno / INT_SIZE] |= 1 << (bitno % INT_SIZE); + s[bitno / INT_SIZE] |= 1U << (bitno % INT_SIZE); } /* @@ -421,7 +545,7 @@ setbit(int bitno, int *s) void clrbit(int bitno, int *s) { - s[bitno / INT_SIZE] &= ~(1 << (bitno % INT_SIZE)); + s[bitno / INT_SIZE] &= ~(1U << (bitno % INT_SIZE)); } /* @@ -430,7 +554,7 @@ clrbit(int bitno, int *s) int testbit(int bitno, int *s) { - return s[bitno / INT_SIZE] & (1 << (bitno % INT_SIZE)); + return s[bitno / INT_SIZE] & (1U << (bitno % INT_SIZE)); } /* @@ -441,15 +565,18 @@ ffsbit(int *s) { int offset; - for (offset = 0; !*s; offset += INT_SIZE, ++s); + for (offset = 0; !*s; offset += INT_SIZE, ++s) { + ; + } return offset + __builtin_ctz(*s); } int ffs(unsigned int mask) { - if (mask == 0) + if (mask == 0) { return 0; + } /* * NOTE: cannot use __builtin_ffs because it generates a call to @@ -461,8 +588,9 @@ ffs(unsigned int mask) int ffsll(unsigned long long mask) { - if (mask == 0) + if (mask == 0) { return 0; + } /* * NOTE: cannot use __builtin_ffsll because it generates a call to @@ -477,51 +605,57 @@ ffsll(unsigned long long mask) int fls(unsigned int mask) { - if (mask == 0) + if (mask == 0) { return 0; + } - return (sizeof (mask) << 3) - __builtin_clz(mask); + return (sizeof(mask) << 3) - __builtin_clz(mask); } int flsll(unsigned long long mask) { - if (mask == 0) + if (mask == 0) { return 0; + } - return (sizeof (mask) << 3) - __builtin_clzll(mask); + return (sizeof(mask) << 3) - __builtin_clzll(mask); } #undef bcmp -int +int bcmp( - const void *pa, - const void *pb, - size_t len) + const void *pa, + const void *pb, + size_t len) { const char *a = (const char *) pa; const char *b = (const char *) pb; - if (len == 0) + if (len == 0) { return 0; + } - do - if (*a++ != *b++) + do{ + if (*a++ != *b++) { break; - while (--len); + } + } while (--len); /* * Check for the overflow case but continue to handle the non-overflow * case the same way just in case someone is using the return value * as more than zero/non-zero */ - if ((len & 0xFFFFFFFF00000000ULL) && !(len & 0x00000000FFFFFFFFULL)) + if ((len & 0xFFFFFFFF00000000ULL) && !(len & 0x00000000FFFFFFFFULL)) { return 0xFFFFFFFFL; - else + } else { return (int)len; + } } #undef memcmp +MARK_AS_HIBERNATE_TEXT int memcmp(const void *s1, const void *s2, size_t n) { @@ -529,63 +663,35 @@ memcmp(const void *s1, const void *s2, size_t n) const unsigned char *p1 = s1, *p2 = s2; do { - if (*p1++ != *p2++) - return (*--p1 - *--p2); + if (*p1++ != *p2++) { + return *--p1 - *--p2; + } } while (--n != 0); } - return (0); + return 0; } kern_return_t copypv(addr64_t source, addr64_t sink, unsigned int size, int which) { - kern_return_t retval = KERN_SUCCESS; - void *from, *to; -#ifndef __ARM_COHERENT_IO__ - unsigned int from_wimg_bits, to_wimg_bits; -#endif - - from = CAST_DOWN(void *, source); - to = CAST_DOWN(void *, sink); - - if ((which & (cppvPsrc | cppvPsnk)) == 0) /* Make sure that only - * one is virtual */ - panic("copypv: no more than 1 parameter may be virtual\n"); /* Not allowed */ - - if (which & cppvPsrc) - from = (void *)phystokv(from); - if (which & cppvPsnk) - to = (void *)phystokv(to); + if ((which & (cppvPsrc | cppvPsnk)) == 0) { /* Make sure that only one is virtual */ + panic("%s: no more than 1 parameter may be virtual", __func__); + } - if ((which & (cppvPsrc | cppvKmap)) == 0) /* Source is virtual in - * current map */ - retval = copyin((user_addr_t) from, to, size); - else if ((which & (cppvPsnk | cppvKmap)) == 0) /* Sink is virtual in - * current map */ - retval = copyout(from, (user_addr_t) to, size); - else /* both addresses are physical or kernel map */ - bcopy(from, to, size); + kern_return_t res = bcopy_phys_internal(source, sink, size, which); -#ifndef __ARM_COHERENT_IO__ +#ifndef __ARM_COHERENT_IO__ if (which & cppvFsrc) { flush_dcache64(source, size, ((which & cppvPsrc) == cppvPsrc)); - } else if (which & cppvPsrc) { - from_wimg_bits = pmap_cache_attributes(source >> PAGE_SHIFT); - if ((from_wimg_bits != VM_WIMG_COPYBACK) && (from_wimg_bits != VM_WIMG_WTHRU)) - flush_dcache64(source, size, TRUE); } if (which & cppvFsnk) { flush_dcache64(sink, size, ((which & cppvPsnk) == cppvPsnk)); - } else if (which & cppvPsnk) { - to_wimg_bits = pmap_cache_attributes(sink >> PAGE_SHIFT); - if (to_wimg_bits != VM_WIMG_COPYBACK) - flush_dcache64(sink, size, TRUE); } #endif - return retval; -} + return res; +} #if MACH_ASSERT @@ -597,45 +703,48 @@ extern int copyinframe(vm_address_t fp, char *frame, boolean_t is64bit); */ void machine_callstack( - uintptr_t * buf, - vm_size_t callstack_max) + uintptr_t * buf, + vm_size_t callstack_max) { /* Captures the USER call stack */ - uint32_t i=0; + uint32_t i = 0; struct arm_saved_state *state = find_user_regs(current_thread()); if (!state) { - while (i: Reduce print noise - // kprintf("ml_thread_policy() unimplemented\n"); + // : Reduce print noise + // kprintf("ml_thread_policy() unimplemented\n"); } +__dead2 void -panic_unimplemented() +panic_unimplemented(void) { panic("Not yet implemented."); } /* ARM64_TODO */ -void abort(void); +void abort(void) __dead2; void -abort() +abort(void) { panic("Abort."); } @@ -697,3 +807,35 @@ kdp_register_callout(kdp_callout_fn_t fn, void *arg) } #endif +/* + * Get a quick virtual mapping of a physical page and run a callback on that + * page's virtual address. + * + * @param dst64 Physical address to access (doesn't need to be page-aligned). + * @param bytes Number of bytes to be accessed. This cannot cross page boundaries. + * @param func Callback function to call with the page's virtual address. + * @param arg Argument passed directly to `func`. + * + * @return The return value from `func`. + */ +int +apply_func_phys( + addr64_t dst64, + vm_size_t bytes, + int (*func)(void * buffer, vm_size_t bytes, void * arg), + void * arg) +{ + /* The physical aperture is only guaranteed to work with kernel-managed addresses. */ + if (!pmap_valid_address(dst64)) { + panic("%s address error: passed in address (%#llx) not a kernel managed address", + __FUNCTION__, dst64); + } + + /* Ensure we stay within a single page */ + if (((((uint32_t)dst64 & (ARM_PGBYTES - 1)) + bytes) > ARM_PGBYTES)) { + panic("%s alignment error: tried accessing addresses spanning more than one page %#llx %#lx", + __FUNCTION__, dst64, bytes); + } + + return func((void*)phystokv(dst64), bytes, arg); +}