X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/5ba3f43ea354af8ad55bea84372a2bc834d8757c..refs/heads/master:/osfmk/arm/loose_ends.c diff --git a/osfmk/arm/loose_ends.c b/osfmk/arm/loose_ends.c index 46aeec6da..a78f820ee 100644 --- a/osfmk/arm/loose_ends.c +++ b/osfmk/arm/loose_ends.c @@ -53,6 +53,12 @@ #define INT_SIZE (BYTE_SIZE * sizeof (int)) +/* machine_routines_asm.s calls these */ +extern int copyin_validate(const user_addr_t, uintptr_t, vm_size_t); +extern int copyin_user_validate(const user_addr_t, uintptr_t, vm_size_t); +extern int copyout_validate(uintptr_t, const user_addr_t, vm_size_t); +extern int copyio_user_validate(int, int, user_addr_t, vm_size_t); +extern int copyoutstr_prevalidate(const void *, user_addr_t, size_t); void bcopy_phys(addr64_t src, addr64_t dst, vm_size_t bytes) @@ -70,8 +76,8 @@ bcopy_phys(addr64_t src, addr64_t dst, vm_size_t bytes) wimg_bits_dst = pmap_cache_attributes(pn_dst); if (mmu_kvtop_wpreflight(phystokv((pmap_paddr_t) dst)) && - ((wimg_bits_src & VM_WIMG_MASK) == VM_WIMG_DEFAULT) && - ((wimg_bits_dst & VM_WIMG_MASK) == VM_WIMG_DEFAULT)) { + ((wimg_bits_src & VM_WIMG_MASK) == VM_WIMG_DEFAULT) && + ((wimg_bits_dst & VM_WIMG_MASK) == VM_WIMG_DEFAULT)) { /* Fast path - dst is writable and both source and destination have default attributes */ bcopy((char *)phystokv((pmap_paddr_t) src), (char *)phystokv((pmap_paddr_t) dst), bytes); return; @@ -80,17 +86,18 @@ bcopy_phys(addr64_t src, addr64_t dst, vm_size_t bytes) src_offset = src & PAGE_MASK; dst_offset = dst & PAGE_MASK; - if ((src_offset + bytes) > PAGE_SIZE || (dst_offset + bytes) > PAGE_SIZE) + if ((src_offset + bytes) > PAGE_SIZE || (dst_offset + bytes) > PAGE_SIZE) { panic("bcopy extends beyond copy windows"); + } mp_disable_preemption(); cpu_num = cpu_number(); src_index = pmap_map_cpu_windows_copy(pn_src, VM_PROT_READ, wimg_bits_src); - dst_index = pmap_map_cpu_windows_copy(pn_dst, VM_PROT_READ|VM_PROT_WRITE, wimg_bits_dst); + dst_index = pmap_map_cpu_windows_copy(pn_dst, VM_PROT_READ | VM_PROT_WRITE, wimg_bits_dst); - bcopy((char *)(pmap_cpu_windows_copy_addr(cpu_num, src_index)+src_offset), - (char *)(pmap_cpu_windows_copy_addr(cpu_num, dst_index)+dst_offset), - bytes); + bcopy((char *)(pmap_cpu_windows_copy_addr(cpu_num, src_index) + src_offset), + (char *)(pmap_cpu_windows_copy_addr(cpu_num, dst_index) + dst_offset), + bytes); pmap_unmap_cpu_windows_copy(src_index); pmap_unmap_cpu_windows_copy(dst_index); @@ -111,7 +118,7 @@ bzero_phys(addr64_t src, vm_size_t bytes) ppnum_t pn = (src >> PAGE_SHIFT); wimg_bits = pmap_cache_attributes(pn); - if ((wimg_bits & VM_WIMG_MASK) == VM_WIMG_DEFAULT) { + if (__probable((wimg_bits & VM_WIMG_MASK) == VM_WIMG_DEFAULT)) { /* Fast path - default attributes */ bzero((char *)phystokv((pmap_paddr_t) src), bytes); } else { @@ -123,8 +130,9 @@ bzero_phys(addr64_t src, vm_size_t bytes) vm_offset_t offset = src & PAGE_MASK; uint32_t count = PAGE_SIZE - offset; - if (count > bytes) + if (count > bytes) { count = bytes; + } unsigned int index = pmap_map_cpu_windows_copy(src >> PAGE_SHIFT, VM_PROT_READ | VM_PROT_WRITE, wimg_bits); @@ -162,18 +170,18 @@ ml_phys_read_data(pmap_paddr_t paddr, int size) copywindow_vaddr = pmap_cpu_windows_copy_addr(cpu_number(), index) | ((uint32_t)paddr & PAGE_MASK);; switch (size) { - case 1: - s1 = *(volatile unsigned char *)(copywindow_vaddr); - result = s1; - break; - case 2: - s2 = *(volatile unsigned short *)(copywindow_vaddr); - result = s2; - break; - case 4: - default: - result = *(volatile unsigned int *)(copywindow_vaddr); - break; + case 1: + s1 = *(volatile unsigned char *)(copywindow_vaddr); + result = s1; + break; + case 2: + s2 = *(volatile unsigned short *)(copywindow_vaddr); + result = s2; + break; + case 4: + default: + result = *(volatile unsigned int *)(copywindow_vaddr); + break; } pmap_unmap_cpu_windows_copy(index); @@ -195,7 +203,7 @@ ml_phys_read_long_long(pmap_paddr_t paddr) index = pmap_map_cpu_windows_copy(pn, VM_PROT_READ, wimg_bits); result = *(volatile unsigned long long *)(pmap_cpu_windows_copy_addr(cpu_number(), index) - | ((uint32_t)paddr & PAGE_MASK)); + | ((uint32_t)paddr & PAGE_MASK)); pmap_unmap_cpu_windows_copy(index); mp_enable_preemption(); @@ -203,54 +211,64 @@ ml_phys_read_long_long(pmap_paddr_t paddr) return result; } -unsigned int ml_phys_read( vm_offset_t paddr) +unsigned int +ml_phys_read( vm_offset_t paddr) { - return ml_phys_read_data((pmap_paddr_t)paddr, 4); + return ml_phys_read_data((pmap_paddr_t)paddr, 4); } -unsigned int ml_phys_read_word(vm_offset_t paddr) { - - return ml_phys_read_data((pmap_paddr_t)paddr, 4); +unsigned int +ml_phys_read_word(vm_offset_t paddr) +{ + return ml_phys_read_data((pmap_paddr_t)paddr, 4); } -unsigned int ml_phys_read_64(addr64_t paddr64) +unsigned int +ml_phys_read_64(addr64_t paddr64) { - return ml_phys_read_data((pmap_paddr_t)paddr64, 4); + return ml_phys_read_data((pmap_paddr_t)paddr64, 4); } -unsigned int ml_phys_read_word_64(addr64_t paddr64) +unsigned int +ml_phys_read_word_64(addr64_t paddr64) { - return ml_phys_read_data((pmap_paddr_t)paddr64, 4); + return ml_phys_read_data((pmap_paddr_t)paddr64, 4); } -unsigned int ml_phys_read_half(vm_offset_t paddr) +unsigned int +ml_phys_read_half(vm_offset_t paddr) { - return ml_phys_read_data((pmap_paddr_t)paddr, 2); + return ml_phys_read_data((pmap_paddr_t)paddr, 2); } -unsigned int ml_phys_read_half_64(addr64_t paddr64) +unsigned int +ml_phys_read_half_64(addr64_t paddr64) { - return ml_phys_read_data((pmap_paddr_t)paddr64, 2); + return ml_phys_read_data((pmap_paddr_t)paddr64, 2); } -unsigned int ml_phys_read_byte(vm_offset_t paddr) +unsigned int +ml_phys_read_byte(vm_offset_t paddr) { - return ml_phys_read_data((pmap_paddr_t)paddr, 1); + return ml_phys_read_data((pmap_paddr_t)paddr, 1); } -unsigned int ml_phys_read_byte_64(addr64_t paddr64) +unsigned int +ml_phys_read_byte_64(addr64_t paddr64) { - return ml_phys_read_data((pmap_paddr_t)paddr64, 1); + return ml_phys_read_data((pmap_paddr_t)paddr64, 1); } -unsigned long long ml_phys_read_double(vm_offset_t paddr) +unsigned long long +ml_phys_read_double(vm_offset_t paddr) { - return ml_phys_read_long_long((pmap_paddr_t)paddr); + return ml_phys_read_long_long((pmap_paddr_t)paddr); } -unsigned long long ml_phys_read_double_64(addr64_t paddr64) +unsigned long long +ml_phys_read_double_64(addr64_t paddr64) { - return ml_phys_read_long_long((pmap_paddr_t)paddr64); + return ml_phys_read_long_long((pmap_paddr_t)paddr64); } @@ -269,20 +287,20 @@ ml_phys_write_data(pmap_paddr_t paddr, unsigned long data, int size) mp_disable_preemption(); wimg_bits = pmap_cache_attributes(pn); - index = pmap_map_cpu_windows_copy(pn, VM_PROT_READ|VM_PROT_WRITE, wimg_bits); + index = pmap_map_cpu_windows_copy(pn, VM_PROT_READ | VM_PROT_WRITE, wimg_bits); copywindow_vaddr = pmap_cpu_windows_copy_addr(cpu_number(), index) | ((uint32_t) paddr & PAGE_MASK); switch (size) { - case 1: - *(volatile unsigned char *)(copywindow_vaddr) = (unsigned char)data; - break; - case 2: - *(volatile unsigned short *)(copywindow_vaddr) = (unsigned short)data; - break; - case 4: - default: - *(volatile unsigned int *)(copywindow_vaddr) = (uint32_t)data; - break; + case 1: + *(volatile unsigned char *)(copywindow_vaddr) = (unsigned char)data; + break; + case 2: + *(volatile unsigned short *)(copywindow_vaddr) = (unsigned short)data; + break; + case 4: + default: + *(volatile unsigned int *)(copywindow_vaddr) = (uint32_t)data; + break; } pmap_unmap_cpu_windows_copy(index); @@ -298,10 +316,10 @@ ml_phys_write_long_long(pmap_paddr_t paddr, unsigned long long data) mp_disable_preemption(); wimg_bits = pmap_cache_attributes(pn); - index = pmap_map_cpu_windows_copy(pn, VM_PROT_READ|VM_PROT_WRITE, wimg_bits); + index = pmap_map_cpu_windows_copy(pn, VM_PROT_READ | VM_PROT_WRITE, wimg_bits); *(volatile unsigned long long *)(pmap_cpu_windows_copy_addr(cpu_number(), index) - | ((uint32_t)paddr & PAGE_MASK)) = data; + | ((uint32_t)paddr & PAGE_MASK)) = data; pmap_unmap_cpu_windows_copy(index); mp_enable_preemption(); @@ -309,54 +327,64 @@ ml_phys_write_long_long(pmap_paddr_t paddr, unsigned long long data) -void ml_phys_write_byte(vm_offset_t paddr, unsigned int data) +void +ml_phys_write_byte(vm_offset_t paddr, unsigned int data) { - ml_phys_write_data((pmap_paddr_t)paddr, data, 1); + ml_phys_write_data((pmap_paddr_t)paddr, data, 1); } -void ml_phys_write_byte_64(addr64_t paddr64, unsigned int data) +void +ml_phys_write_byte_64(addr64_t paddr64, unsigned int data) { - ml_phys_write_data((pmap_paddr_t)paddr64, data, 1); + ml_phys_write_data((pmap_paddr_t)paddr64, data, 1); } -void ml_phys_write_half(vm_offset_t paddr, unsigned int data) +void +ml_phys_write_half(vm_offset_t paddr, unsigned int data) { - ml_phys_write_data((pmap_paddr_t)paddr, data, 2); + ml_phys_write_data((pmap_paddr_t)paddr, data, 2); } -void ml_phys_write_half_64(addr64_t paddr64, unsigned int data) +void +ml_phys_write_half_64(addr64_t paddr64, unsigned int data) { - ml_phys_write_data((pmap_paddr_t)paddr64, data, 2); + ml_phys_write_data((pmap_paddr_t)paddr64, data, 2); } -void ml_phys_write(vm_offset_t paddr, unsigned int data) +void +ml_phys_write(vm_offset_t paddr, unsigned int data) { - ml_phys_write_data((pmap_paddr_t)paddr, data, 4); + ml_phys_write_data((pmap_paddr_t)paddr, data, 4); } -void ml_phys_write_64(addr64_t paddr64, unsigned int data) +void +ml_phys_write_64(addr64_t paddr64, unsigned int data) { - ml_phys_write_data((pmap_paddr_t)paddr64, data, 4); + ml_phys_write_data((pmap_paddr_t)paddr64, data, 4); } -void ml_phys_write_word(vm_offset_t paddr, unsigned int data) +void +ml_phys_write_word(vm_offset_t paddr, unsigned int data) { - ml_phys_write_data((pmap_paddr_t)paddr, data, 4); + ml_phys_write_data((pmap_paddr_t)paddr, data, 4); } -void ml_phys_write_word_64(addr64_t paddr64, unsigned int data) +void +ml_phys_write_word_64(addr64_t paddr64, unsigned int data) { - ml_phys_write_data((pmap_paddr_t)paddr64, data, 4); + ml_phys_write_data((pmap_paddr_t)paddr64, data, 4); } -void ml_phys_write_double(vm_offset_t paddr, unsigned long long data) +void +ml_phys_write_double(vm_offset_t paddr, unsigned long long data) { - ml_phys_write_long_long((pmap_paddr_t)paddr, data); + ml_phys_write_long_long((pmap_paddr_t)paddr, data); } -void ml_phys_write_double_64(addr64_t paddr64, unsigned long long data) +void +ml_phys_write_double_64(addr64_t paddr64, unsigned long long data) { - ml_phys_write_long_long((pmap_paddr_t)paddr64, data); + ml_phys_write_long_long((pmap_paddr_t)paddr64, data); } @@ -395,15 +423,18 @@ ffsbit(int *s) { int offset; - for (offset = 0; !*s; offset += INT_SIZE, ++s); + for (offset = 0; !*s; offset += INT_SIZE, ++s) { + ; + } return offset + __builtin_ctz(*s); } int ffs(unsigned int mask) { - if (mask == 0) + if (mask == 0) { return 0; + } /* * NOTE: cannot use __builtin_ffs because it generates a call to @@ -415,8 +446,9 @@ ffs(unsigned int mask) int ffsll(unsigned long long mask) { - if (mask == 0) + if (mask == 0) { return 0; + } /* * NOTE: cannot use __builtin_ffsll because it generates a call to @@ -431,37 +463,41 @@ ffsll(unsigned long long mask) int fls(unsigned int mask) { - if (mask == 0) + if (mask == 0) { return 0; + } - return (sizeof (mask) << 3) - __builtin_clz(mask); + return (sizeof(mask) << 3) - __builtin_clz(mask); } int flsll(unsigned long long mask) { - if (mask == 0) + if (mask == 0) { return 0; + } - return (sizeof (mask) << 3) - __builtin_clzll(mask); + return (sizeof(mask) << 3) - __builtin_clzll(mask); } -int +int bcmp( - const void *pa, - const void *pb, - size_t len) + const void *pa, + const void *pb, + size_t len) { const char *a = (const char *) pa; const char *b = (const char *) pb; - if (len == 0) + if (len == 0) { return 0; + } - do - if (*a++ != *b++) + do{ + if (*a++ != *b++) { break; - while (--len); + } + } while (--len); return len; } @@ -473,55 +509,91 @@ memcmp(const void *s1, const void *s2, size_t n) const unsigned char *p1 = s1, *p2 = s2; do { - if (*p1++ != *p2++) - return (*--p1 - *--p2); + if (*p1++ != *p2++) { + return *--p1 - *--p2; + } } while (--n != 0); } - return (0); + return 0; +} + +unsigned long +memcmp_zero_ptr_aligned(const void *s, size_t n) +{ + uintptr_t p = (uintptr_t)s; + uintptr_t end = (uintptr_t)s + n; + uint32_t a, b; + + static_assert(sizeof(unsigned long) == sizeof(uint32_t)); + + a = *(const uint32_t *)p; + b = *(const uint32_t *)(end - sizeof(uint32_t)); + + /* + * align p to the next 64bit boundary + * align end to the previous 64bit boundary + * + * and do a nice ldrd loop. + */ + p = (p + sizeof(uint64_t) - 1) & -sizeof(uint64_t); + end &= -sizeof(uint64_t); + + for (; p < end; p += sizeof(uint64_t)) { + uint64_t v = *(const uint64_t *)p; + a |= (uint32_t)v; + b |= (uint32_t)(v >> 32); + } + + return a | b; } kern_return_t copypv(addr64_t source, addr64_t sink, unsigned int size, int which) { kern_return_t retval = KERN_SUCCESS; - void *from, *to; - unsigned int from_wimg_bits, to_wimg_bits; + void *from, *to; + unsigned int from_wimg_bits, to_wimg_bits; from = CAST_DOWN(void *, source); to = CAST_DOWN(void *, sink); - if ((which & (cppvPsrc | cppvPsnk)) == 0) /* Make sure that only - * one is virtual */ - panic("copypv: no more than 1 parameter may be virtual\n"); /* Not allowed */ - - if (which & cppvPsrc) - from = (void *)phystokv(from); - if (which & cppvPsnk) - to = (void *)phystokv(to); + if ((which & (cppvPsrc | cppvPsnk)) == 0) { /* Make sure that only + * one is virtual */ + panic("copypv: no more than 1 parameter may be virtual\n"); /* Not allowed */ + } + if (which & cppvPsrc) { + from = (void *)phystokv((pmap_paddr_t)from); + } + if (which & cppvPsnk) { + to = (void *)phystokv((pmap_paddr_t)to); + } - if ((which & (cppvPsrc | cppvKmap)) == 0) /* Source is virtual in - * current map */ + if ((which & (cppvPsrc | cppvKmap)) == 0) { /* Source is virtual in + * current map */ retval = copyin((user_addr_t) from, to, size); - else if ((which & (cppvPsnk | cppvKmap)) == 0) /* Sink is virtual in - * current map */ + } else if ((which & (cppvPsnk | cppvKmap)) == 0) { /* Sink is virtual in + * current map */ retval = copyout(from, (user_addr_t) to, size); - else /* both addresses are physical or kernel map */ + } else { /* both addresses are physical or kernel map */ bcopy(from, to, size); + } if (which & cppvFsrc) { flush_dcache64(source, size, ((which & cppvPsrc) == cppvPsrc)); } else if (which & cppvPsrc) { from_wimg_bits = pmap_cache_attributes(source >> PAGE_SHIFT); - if ((from_wimg_bits != VM_WIMG_COPYBACK) && (from_wimg_bits != VM_WIMG_WTHRU)) + if ((from_wimg_bits != VM_WIMG_COPYBACK) && (from_wimg_bits != VM_WIMG_WTHRU)) { flush_dcache64(source, size, TRUE); + } } if (which & cppvFsnk) { flush_dcache64(sink, size, ((which & cppvPsnk) == cppvPsnk)); - } else if (which & cppvPsnk) { + } else if (which & cppvPsnk) { to_wimg_bits = pmap_cache_attributes(sink >> PAGE_SHIFT); - if (to_wimg_bits != VM_WIMG_COPYBACK) + if (to_wimg_bits != VM_WIMG_COPYBACK) { flush_dcache64(sink, size, TRUE); + } } return retval; } @@ -536,6 +608,36 @@ copypv(addr64_t source, addr64_t sink, unsigned int size, int which) */ const int copysize_limit_panic = (64 * 1024 * 1024); +static inline bool +is_kernel_to_kernel_copy() +{ + return current_thread()->map->pmap == kernel_pmap; +} + +static int +copy_validate_user(const user_addr_t user_addr, vm_size_t nbytes, bool kern_to_kern_allowed) +{ + user_addr_t user_addr_last = user_addr + nbytes; + thread_t self = current_thread(); + + if (__improbable(!kern_to_kern_allowed && is_kernel_to_kernel_copy())) { + return EFAULT; + } + + if (__improbable((user_addr_last < user_addr) || + ((user_addr + nbytes) > vm_map_max(self->map)) || + (user_addr < vm_map_min(self->map)))) { + return EFAULT; + } + + if (__improbable(nbytes > copysize_limit_panic)) { + panic("%s(%p, ..., %u) - transfer too large", __func__, + (void *)user_addr, nbytes); + } + + return 0; +} + /* * Validate the arguments to copy{in,out} on this platform. * @@ -545,45 +647,59 @@ const int copysize_limit_panic = (64 * 1024 * 1024); */ static int copy_validate(const user_addr_t user_addr, - uintptr_t kernel_addr, vm_size_t nbytes) + uintptr_t kernel_addr, vm_size_t nbytes, bool kern_to_kern_allowed) { uintptr_t kernel_addr_last = kernel_addr + nbytes; - if (kernel_addr < VM_MIN_KERNEL_ADDRESS || + if (__improbable(kernel_addr < VM_MIN_KERNEL_ADDRESS || kernel_addr > VM_MAX_KERNEL_ADDRESS || kernel_addr_last < kernel_addr || - kernel_addr_last > VM_MAX_KERNEL_ADDRESS) + kernel_addr_last > VM_MAX_KERNEL_ADDRESS)) { panic("%s(%p, %p, %u) - kaddr not in kernel", __func__, (void *)user_addr, (void *)kernel_addr, nbytes); + } - user_addr_t user_addr_last = user_addr + nbytes; + return copy_validate_user(user_addr, nbytes, kern_to_kern_allowed); +} - if (user_addr_last < user_addr || - user_addr_last > VM_MIN_KERNEL_ADDRESS) - return (EFAULT); +int +copyin_validate(const user_addr_t ua, uintptr_t ka, vm_size_t nbytes) +{ + return copy_validate(ua, ka, nbytes, true); +} - if (__improbable(nbytes > copysize_limit_panic)) - panic("%s(%p, %p, %u) - transfer too large", __func__, - (void *)user_addr, (void *)kernel_addr, nbytes); +int +copyin_user_validate(const user_addr_t ua, uintptr_t ka, vm_size_t nbytes) +{ + return copy_validate(ua, ka, nbytes, false); +} - return (0); +int +copyout_validate(uintptr_t ka, const user_addr_t ua, vm_size_t nbytes) +{ + return copy_validate(ua, ka, nbytes, true); } int -copyin_validate(const user_addr_t ua, uintptr_t ka, vm_size_t nbytes) +copyio_user_validate(int a __unused, int b __unused, + user_addr_t ua, vm_size_t nbytes) { - return (copy_validate(ua, ka, nbytes)); + return copy_validate_user(ua, nbytes, false); } int -copyout_validate(uintptr_t ka, const user_addr_t ua, vm_size_t nbytes) +copyoutstr_prevalidate(const void *__unused kaddr, user_addr_t __unused uaddr, size_t __unused len) { - return (copy_validate(ua, ka, nbytes)); + if (__improbable(is_kernel_to_kernel_copy())) { + return EFAULT; + } + + return 0; } #if MACH_ASSERT -extern int copyinframe(vm_address_t fp, char *frame); +extern int copyinframe(vm_address_t fp, char *frame); /* * Machine-dependent routine to fill in an array with up to callstack_max @@ -591,34 +707,37 @@ extern int copyinframe(vm_address_t fp, char *frame); */ void machine_callstack( - uintptr_t * buf, - vm_size_t callstack_max) + uintptr_t * buf, + vm_size_t callstack_max) { /* Captures the USER call stack */ - uint32_t i=0; + uint32_t i = 0; uint32_t frame[2]; struct arm_saved_state* state = find_user_regs(current_thread()); if (!state) { - while (ipc; frame[0] = state->r[7]; - while (i: Reduce print noise - // kprintf("ml_thread_policy() unimplemented\n"); + // : Reduce print noise + // kprintf("ml_thread_policy() unimplemented\n"); } #if !MACH_KDP