xnu-7195.101.1.tar.gz

[apple/xnu.git] / osfmk / arm / loose_ends.c

diff --git a/osfmk/arm/loose_ends.c b/osfmk/arm/loose_ends.c
index 46aeec6daa19e519a597dc8e54d834eb083c8b91..a78f820ee43d9ec9801cfd10d96858e616c3e484 100644 (file)
--- a/osfmk/arm/loose_ends.c
+++ b/osfmk/arm/loose_ends.c
@@ -53,6 +53,12 @@
 
 #define INT_SIZE        (BYTE_SIZE * sizeof (int))
 
 
 #define INT_SIZE        (BYTE_SIZE * sizeof (int))
 

+/* machine_routines_asm.s calls these */
+extern int copyin_validate(const user_addr_t, uintptr_t, vm_size_t);
+extern int copyin_user_validate(const user_addr_t, uintptr_t, vm_size_t);
+extern int copyout_validate(uintptr_t, const user_addr_t, vm_size_t);
+extern int copyio_user_validate(int, int, user_addr_t, vm_size_t);
+extern int copyoutstr_prevalidate(const void *, user_addr_t, size_t);

 
 void
 bcopy_phys(addr64_t src, addr64_t dst, vm_size_t bytes)
 
 void
 bcopy_phys(addr64_t src, addr64_t dst, vm_size_t bytes)


@@ -70,8 +76,8 @@ bcopy_phys(addr64_t src, addr64_t dst, vm_size_t bytes)
        wimg_bits_dst = pmap_cache_attributes(pn_dst);
 
        if (mmu_kvtop_wpreflight(phystokv((pmap_paddr_t) dst)) &&
        wimg_bits_dst = pmap_cache_attributes(pn_dst);
 
        if (mmu_kvtop_wpreflight(phystokv((pmap_paddr_t) dst)) &&

-                       ((wimg_bits_src & VM_WIMG_MASK) == VM_WIMG_DEFAULT) &&
-                       ((wimg_bits_dst & VM_WIMG_MASK) == VM_WIMG_DEFAULT)) {
+           ((wimg_bits_src & VM_WIMG_MASK) == VM_WIMG_DEFAULT) &&
+           ((wimg_bits_dst & VM_WIMG_MASK) == VM_WIMG_DEFAULT)) {

                /* Fast path - dst is writable and both source and destination have default attributes */
                bcopy((char *)phystokv((pmap_paddr_t) src), (char *)phystokv((pmap_paddr_t) dst), bytes);
                return;
                /* Fast path - dst is writable and both source and destination have default attributes */
                bcopy((char *)phystokv((pmap_paddr_t) src), (char *)phystokv((pmap_paddr_t) dst), bytes);
                return;


@@ -80,17 +86,18 @@ bcopy_phys(addr64_t src, addr64_t dst, vm_size_t bytes)
        src_offset = src & PAGE_MASK;
        dst_offset = dst & PAGE_MASK;
 
        src_offset = src & PAGE_MASK;
        dst_offset = dst & PAGE_MASK;
 

-       if ((src_offset + bytes) > PAGE_SIZE || (dst_offset + bytes) > PAGE_SIZE)
+       if ((src_offset + bytes) > PAGE_SIZE || (dst_offset + bytes) > PAGE_SIZE) {

                panic("bcopy extends beyond copy windows");
                panic("bcopy extends beyond copy windows");

+       }

 
        mp_disable_preemption();
        cpu_num = cpu_number();
        src_index = pmap_map_cpu_windows_copy(pn_src, VM_PROT_READ, wimg_bits_src);
 
        mp_disable_preemption();
        cpu_num = cpu_number();
        src_index = pmap_map_cpu_windows_copy(pn_src, VM_PROT_READ, wimg_bits_src);

-       dst_index = pmap_map_cpu_windows_copy(pn_dst, VM_PROT_READ|VM_PROT_WRITE, wimg_bits_dst);
+       dst_index = pmap_map_cpu_windows_copy(pn_dst, VM_PROT_READ | VM_PROT_WRITE, wimg_bits_dst);

 
 

-       bcopy((char *)(pmap_cpu_windows_copy_addr(cpu_num, src_index)+src_offset),
-             (char *)(pmap_cpu_windows_copy_addr(cpu_num, dst_index)+dst_offset),
-             bytes);
+       bcopy((char *)(pmap_cpu_windows_copy_addr(cpu_num, src_index) + src_offset),
+           (char *)(pmap_cpu_windows_copy_addr(cpu_num, dst_index) + dst_offset),
+           bytes);

 
        pmap_unmap_cpu_windows_copy(src_index);
        pmap_unmap_cpu_windows_copy(dst_index);
 
        pmap_unmap_cpu_windows_copy(src_index);
        pmap_unmap_cpu_windows_copy(dst_index);


@@ -111,7 +118,7 @@ bzero_phys(addr64_t src, vm_size_t bytes)
        ppnum_t         pn = (src >> PAGE_SHIFT);
 
        wimg_bits = pmap_cache_attributes(pn);
        ppnum_t         pn = (src >> PAGE_SHIFT);
 
        wimg_bits = pmap_cache_attributes(pn);

-       if ((wimg_bits & VM_WIMG_MASK) == VM_WIMG_DEFAULT) {
+       if (__probable((wimg_bits & VM_WIMG_MASK) == VM_WIMG_DEFAULT)) {

                /* Fast path - default attributes */
                bzero((char *)phystokv((pmap_paddr_t) src), bytes);
        } else {
                /* Fast path - default attributes */
                bzero((char *)phystokv((pmap_paddr_t) src), bytes);
        } else {


@@ -123,8 +130,9 @@ bzero_phys(addr64_t src, vm_size_t bytes)
                        vm_offset_t offset = src & PAGE_MASK;
                        uint32_t count = PAGE_SIZE - offset;
 
                        vm_offset_t offset = src & PAGE_MASK;
                        uint32_t count = PAGE_SIZE - offset;
 

-                       if (count > bytes)
+                       if (count > bytes) {

                                count = bytes;
                                count = bytes;

+                       }

 
                        unsigned int index = pmap_map_cpu_windows_copy(src >> PAGE_SHIFT, VM_PROT_READ | VM_PROT_WRITE, wimg_bits);
 
 
                        unsigned int index = pmap_map_cpu_windows_copy(src >> PAGE_SHIFT, VM_PROT_READ | VM_PROT_WRITE, wimg_bits);
 


@@ -162,18 +170,18 @@ ml_phys_read_data(pmap_paddr_t paddr, int size)
        copywindow_vaddr = pmap_cpu_windows_copy_addr(cpu_number(), index) | ((uint32_t)paddr & PAGE_MASK);;
 
        switch (size) {
        copywindow_vaddr = pmap_cpu_windows_copy_addr(cpu_number(), index) | ((uint32_t)paddr & PAGE_MASK);;
 
        switch (size) {

-               case 1:
-                       s1 = *(volatile unsigned char *)(copywindow_vaddr);
-                       result = s1;
-                       break;
-               case 2:
-                       s2 = *(volatile unsigned short *)(copywindow_vaddr);
-                       result = s2;
-                       break;
-               case 4:
-               default:
-                       result = *(volatile unsigned int *)(copywindow_vaddr);
-                       break;
+       case 1:
+               s1 = *(volatile unsigned char *)(copywindow_vaddr);
+               result = s1;
+               break;
+       case 2:
+               s2 = *(volatile unsigned short *)(copywindow_vaddr);
+               result = s2;
+               break;
+       case 4:
+       default:
+               result = *(volatile unsigned int *)(copywindow_vaddr);
+               break;

        }
 
        pmap_unmap_cpu_windows_copy(index);
        }
 
        pmap_unmap_cpu_windows_copy(index);


@@ -195,7 +203,7 @@ ml_phys_read_long_long(pmap_paddr_t paddr)
        index = pmap_map_cpu_windows_copy(pn, VM_PROT_READ, wimg_bits);
 
        result = *(volatile unsigned long long *)(pmap_cpu_windows_copy_addr(cpu_number(), index)
        index = pmap_map_cpu_windows_copy(pn, VM_PROT_READ, wimg_bits);
 
        result = *(volatile unsigned long long *)(pmap_cpu_windows_copy_addr(cpu_number(), index)

-                                        | ((uint32_t)paddr & PAGE_MASK));
+           | ((uint32_t)paddr & PAGE_MASK));

 
        pmap_unmap_cpu_windows_copy(index);
        mp_enable_preemption();
 
        pmap_unmap_cpu_windows_copy(index);
        mp_enable_preemption();


@@ -203,54 +211,64 @@ ml_phys_read_long_long(pmap_paddr_t paddr)
        return result;
 }
 
        return result;
 }
 

-unsigned int ml_phys_read( vm_offset_t paddr)
+unsigned int
+ml_phys_read( vm_offset_t paddr)

 {
 {

-        return ml_phys_read_data((pmap_paddr_t)paddr, 4);
+       return ml_phys_read_data((pmap_paddr_t)paddr, 4);

 }
 
 }
 

-unsigned int ml_phys_read_word(vm_offset_t paddr) {
-
-        return ml_phys_read_data((pmap_paddr_t)paddr, 4);
+unsigned int
+ml_phys_read_word(vm_offset_t paddr)
+{
+       return ml_phys_read_data((pmap_paddr_t)paddr, 4);

 }
 
 }
 

-unsigned int ml_phys_read_64(addr64_t paddr64)
+unsigned int
+ml_phys_read_64(addr64_t paddr64)

 {
 {

-        return ml_phys_read_data((pmap_paddr_t)paddr64, 4);
+       return ml_phys_read_data((pmap_paddr_t)paddr64, 4);

 }
 
 }
 

-unsigned int ml_phys_read_word_64(addr64_t paddr64)
+unsigned int
+ml_phys_read_word_64(addr64_t paddr64)

 {
 {

-        return ml_phys_read_data((pmap_paddr_t)paddr64, 4);
+       return ml_phys_read_data((pmap_paddr_t)paddr64, 4);

 }
 
 }
 

-unsigned int ml_phys_read_half(vm_offset_t paddr)
+unsigned int
+ml_phys_read_half(vm_offset_t paddr)

 {
 {

-        return ml_phys_read_data((pmap_paddr_t)paddr, 2);
+       return ml_phys_read_data((pmap_paddr_t)paddr, 2);

 }
 
 }
 

-unsigned int ml_phys_read_half_64(addr64_t paddr64)
+unsigned int
+ml_phys_read_half_64(addr64_t paddr64)

 {
 {

-        return ml_phys_read_data((pmap_paddr_t)paddr64, 2);
+       return ml_phys_read_data((pmap_paddr_t)paddr64, 2);

 }
 
 }
 

-unsigned int ml_phys_read_byte(vm_offset_t paddr)
+unsigned int
+ml_phys_read_byte(vm_offset_t paddr)

 {
 {

-        return ml_phys_read_data((pmap_paddr_t)paddr, 1);
+       return ml_phys_read_data((pmap_paddr_t)paddr, 1);

 }
 
 }
 

-unsigned int ml_phys_read_byte_64(addr64_t paddr64)
+unsigned int
+ml_phys_read_byte_64(addr64_t paddr64)

 {
 {

-        return ml_phys_read_data((pmap_paddr_t)paddr64, 1);
+       return ml_phys_read_data((pmap_paddr_t)paddr64, 1);

 }
 
 }
 

-unsigned long long ml_phys_read_double(vm_offset_t paddr)
+unsigned long long
+ml_phys_read_double(vm_offset_t paddr)

 {
 {

-        return ml_phys_read_long_long((pmap_paddr_t)paddr);
+       return ml_phys_read_long_long((pmap_paddr_t)paddr);

 }
 
 }
 

-unsigned long long ml_phys_read_double_64(addr64_t paddr64)
+unsigned long long
+ml_phys_read_double_64(addr64_t paddr64)

 {
 {

-        return ml_phys_read_long_long((pmap_paddr_t)paddr64);
+       return ml_phys_read_long_long((pmap_paddr_t)paddr64);

 }
 
 
 }
 
 


@@ -269,20 +287,20 @@ ml_phys_write_data(pmap_paddr_t paddr, unsigned long data, int size)
 
        mp_disable_preemption();
        wimg_bits = pmap_cache_attributes(pn);
 
        mp_disable_preemption();
        wimg_bits = pmap_cache_attributes(pn);

-       index = pmap_map_cpu_windows_copy(pn, VM_PROT_READ|VM_PROT_WRITE, wimg_bits);
+       index = pmap_map_cpu_windows_copy(pn, VM_PROT_READ | VM_PROT_WRITE, wimg_bits);

        copywindow_vaddr = pmap_cpu_windows_copy_addr(cpu_number(), index) | ((uint32_t) paddr & PAGE_MASK);
 
        switch (size) {
        copywindow_vaddr = pmap_cpu_windows_copy_addr(cpu_number(), index) | ((uint32_t) paddr & PAGE_MASK);
 
        switch (size) {

-               case 1:
-                       *(volatile unsigned char *)(copywindow_vaddr) = (unsigned char)data;
-                       break;
-               case 2:
-                       *(volatile unsigned short *)(copywindow_vaddr) = (unsigned short)data;
-                       break;
-               case 4:
-               default:
-                        *(volatile unsigned int *)(copywindow_vaddr) = (uint32_t)data;
-                       break;
+       case 1:
+               *(volatile unsigned char *)(copywindow_vaddr) = (unsigned char)data;
+               break;
+       case 2:
+               *(volatile unsigned short *)(copywindow_vaddr) = (unsigned short)data;
+               break;
+       case 4:
+       default:
+               *(volatile unsigned int *)(copywindow_vaddr) = (uint32_t)data;
+               break;

        }
 
        pmap_unmap_cpu_windows_copy(index);
        }
 
        pmap_unmap_cpu_windows_copy(index);


@@ -298,10 +316,10 @@ ml_phys_write_long_long(pmap_paddr_t paddr, unsigned long long data)
 
        mp_disable_preemption();
        wimg_bits = pmap_cache_attributes(pn);
 
        mp_disable_preemption();
        wimg_bits = pmap_cache_attributes(pn);

-       index = pmap_map_cpu_windows_copy(pn, VM_PROT_READ|VM_PROT_WRITE, wimg_bits);
+       index = pmap_map_cpu_windows_copy(pn, VM_PROT_READ | VM_PROT_WRITE, wimg_bits);

 
        *(volatile unsigned long long *)(pmap_cpu_windows_copy_addr(cpu_number(), index)
 
        *(volatile unsigned long long *)(pmap_cpu_windows_copy_addr(cpu_number(), index)

-                               | ((uint32_t)paddr & PAGE_MASK)) = data;
+       | ((uint32_t)paddr & PAGE_MASK)) = data;

 
        pmap_unmap_cpu_windows_copy(index);
        mp_enable_preemption();
 
        pmap_unmap_cpu_windows_copy(index);
        mp_enable_preemption();


@@ -309,54 +327,64 @@ ml_phys_write_long_long(pmap_paddr_t paddr, unsigned long long data)
 
 
 
 
 
 

-void ml_phys_write_byte(vm_offset_t paddr, unsigned int data)
+void
+ml_phys_write_byte(vm_offset_t paddr, unsigned int data)

 {
 {

-        ml_phys_write_data((pmap_paddr_t)paddr, data, 1);
+       ml_phys_write_data((pmap_paddr_t)paddr, data, 1);

 }
 
 }
 

-void ml_phys_write_byte_64(addr64_t paddr64, unsigned int data)
+void
+ml_phys_write_byte_64(addr64_t paddr64, unsigned int data)

 {
 {

-        ml_phys_write_data((pmap_paddr_t)paddr64, data, 1);
+       ml_phys_write_data((pmap_paddr_t)paddr64, data, 1);

 }
 
 }
 

-void ml_phys_write_half(vm_offset_t paddr, unsigned int data)
+void
+ml_phys_write_half(vm_offset_t paddr, unsigned int data)

 {
 {

-        ml_phys_write_data((pmap_paddr_t)paddr, data, 2);
+       ml_phys_write_data((pmap_paddr_t)paddr, data, 2);

 }
 
 }
 

-void ml_phys_write_half_64(addr64_t paddr64, unsigned int data)
+void
+ml_phys_write_half_64(addr64_t paddr64, unsigned int data)

 {
 {

-        ml_phys_write_data((pmap_paddr_t)paddr64, data, 2);
+       ml_phys_write_data((pmap_paddr_t)paddr64, data, 2);

 }
 
 }
 

-void ml_phys_write(vm_offset_t paddr, unsigned int data)
+void
+ml_phys_write(vm_offset_t paddr, unsigned int data)

 {
 {

-        ml_phys_write_data((pmap_paddr_t)paddr, data, 4);
+       ml_phys_write_data((pmap_paddr_t)paddr, data, 4);

 }
 
 }
 

-void ml_phys_write_64(addr64_t paddr64, unsigned int data)
+void
+ml_phys_write_64(addr64_t paddr64, unsigned int data)

 {
 {

-        ml_phys_write_data((pmap_paddr_t)paddr64, data, 4);
+       ml_phys_write_data((pmap_paddr_t)paddr64, data, 4);

 }
 
 }
 

-void ml_phys_write_word(vm_offset_t paddr, unsigned int data)
+void
+ml_phys_write_word(vm_offset_t paddr, unsigned int data)

 {
 {

-        ml_phys_write_data((pmap_paddr_t)paddr, data, 4);
+       ml_phys_write_data((pmap_paddr_t)paddr, data, 4);

 }
 
 }
 

-void ml_phys_write_word_64(addr64_t paddr64, unsigned int data)
+void
+ml_phys_write_word_64(addr64_t paddr64, unsigned int data)

 {
 {

-        ml_phys_write_data((pmap_paddr_t)paddr64, data, 4);
+       ml_phys_write_data((pmap_paddr_t)paddr64, data, 4);

 }
 
 }
 

-void ml_phys_write_double(vm_offset_t paddr, unsigned long long data)
+void
+ml_phys_write_double(vm_offset_t paddr, unsigned long long data)

 {
 {

-        ml_phys_write_long_long((pmap_paddr_t)paddr, data);
+       ml_phys_write_long_long((pmap_paddr_t)paddr, data);

 }
 
 }
 

-void ml_phys_write_double_64(addr64_t paddr64, unsigned long long data)
+void
+ml_phys_write_double_64(addr64_t paddr64, unsigned long long data)

 {
 {

-        ml_phys_write_long_long((pmap_paddr_t)paddr64, data);
+       ml_phys_write_long_long((pmap_paddr_t)paddr64, data);

 }
 
 
 }
 
 


@@ -395,15 +423,18 @@ ffsbit(int *s)
 {
        int             offset;
 
 {
        int             offset;
 

-       for (offset = 0; !*s; offset += INT_SIZE, ++s);
+       for (offset = 0; !*s; offset += INT_SIZE, ++s) {
+               ;
+       }

        return offset + __builtin_ctz(*s);
 }
 
 int
 ffs(unsigned int mask)
 {
        return offset + __builtin_ctz(*s);
 }
 
 int
 ffs(unsigned int mask)
 {

-       if (mask == 0)
+       if (mask == 0) {

                return 0;
                return 0;

+       }

 
        /*
         * NOTE: cannot use __builtin_ffs because it generates a call to
 
        /*
         * NOTE: cannot use __builtin_ffs because it generates a call to


@@ -415,8 +446,9 @@ ffs(unsigned int mask)
 int
 ffsll(unsigned long long mask)
 {
 int
 ffsll(unsigned long long mask)
 {

-       if (mask == 0)
+       if (mask == 0) {

                return 0;
                return 0;

+       }

 
        /*
         * NOTE: cannot use __builtin_ffsll because it generates a call to
 
        /*
         * NOTE: cannot use __builtin_ffsll because it generates a call to


@@ -431,37 +463,41 @@ ffsll(unsigned long long mask)
 int
 fls(unsigned int mask)
 {
 int
 fls(unsigned int mask)
 {

-       if (mask == 0)
+       if (mask == 0) {

                return 0;
                return 0;

+       }

 
 

-       return (sizeof (mask) << 3) - __builtin_clz(mask);
+       return (sizeof(mask) << 3) - __builtin_clz(mask);

 }
 
 int
 flsll(unsigned long long mask)
 {
 }
 
 int
 flsll(unsigned long long mask)
 {

-       if (mask == 0)
+       if (mask == 0) {

                return 0;
                return 0;

+       }

 
 

-       return (sizeof (mask) << 3) - __builtin_clzll(mask);
+       return (sizeof(mask) << 3) - __builtin_clzll(mask);

 }
 
 }
 

-int 
+int

 bcmp(
 bcmp(

-     const void *pa,
-     const void *pb,
-     size_t len)
+       const void *pa,
+       const void *pb,
+       size_t len)

 {
        const char     *a = (const char *) pa;
        const char     *b = (const char *) pb;
 
 {
        const char     *a = (const char *) pa;
        const char     *b = (const char *) pb;
 

-       if (len == 0)
+       if (len == 0) {

                return 0;
                return 0;

+       }

 
 

-       do
-               if (*a++ != *b++)
+       do{
+               if (*a++ != *b++) {

                        break;
                        break;

-       while (--len);
+               }
+       } while (--len);

 
        return len;
 }
 
        return len;
 }


@@ -473,55 +509,91 @@ memcmp(const void *s1, const void *s2, size_t n)
                const unsigned char *p1 = s1, *p2 = s2;
 
                do {
                const unsigned char *p1 = s1, *p2 = s2;
 
                do {

-                       if (*p1++ != *p2++)
-                               return (*--p1 - *--p2);
+                       if (*p1++ != *p2++) {
+                               return *--p1 - *--p2;
+                       }

                } while (--n != 0);
        }
                } while (--n != 0);
        }

-       return (0);
+       return 0;
+}
+
+unsigned long
+memcmp_zero_ptr_aligned(const void *s, size_t n)
+{
+       uintptr_t p = (uintptr_t)s;
+       uintptr_t end = (uintptr_t)s + n;
+       uint32_t a, b;
+
+       static_assert(sizeof(unsigned long) == sizeof(uint32_t));
+
+       a = *(const uint32_t *)p;
+       b = *(const uint32_t *)(end - sizeof(uint32_t));
+
+       /*
+        * align p to the next 64bit boundary
+        * align end to the previous 64bit boundary
+        *
+        * and do a nice ldrd loop.
+        */
+       p = (p + sizeof(uint64_t) - 1) & -sizeof(uint64_t);
+       end &= -sizeof(uint64_t);
+
+       for (; p < end; p += sizeof(uint64_t)) {
+               uint64_t v = *(const uint64_t *)p;
+               a |= (uint32_t)v;
+               b |= (uint32_t)(v >> 32);
+       }
+
+       return a | b;

 }
 
 kern_return_t
 copypv(addr64_t source, addr64_t sink, unsigned int size, int which)
 {
        kern_return_t   retval = KERN_SUCCESS;
 }
 
 kern_return_t
 copypv(addr64_t source, addr64_t sink, unsigned int size, int which)
 {
        kern_return_t   retval = KERN_SUCCESS;

-       void            *from, *to;
-       unsigned int    from_wimg_bits, to_wimg_bits;
+       void            *from, *to;
+       unsigned int    from_wimg_bits, to_wimg_bits;

 
        from = CAST_DOWN(void *, source);
        to = CAST_DOWN(void *, sink);
 
 
        from = CAST_DOWN(void *, source);
        to = CAST_DOWN(void *, sink);
 

-       if ((which & (cppvPsrc | cppvPsnk)) == 0)       /* Make sure that only
-                                                        * one is virtual */
-               panic("copypv: no more than 1 parameter may be virtual\n");     /* Not allowed */
-
-       if (which & cppvPsrc)
-               from = (void *)phystokv(from);
-       if (which & cppvPsnk)
-               to = (void *)phystokv(to);
+       if ((which & (cppvPsrc | cppvPsnk)) == 0) {     /* Make sure that only
+                                                        * one is virtual */
+               panic("copypv: no more than 1 parameter may be virtual\n");     /* Not allowed */
+       }
+       if (which & cppvPsrc) {
+               from = (void *)phystokv((pmap_paddr_t)from);
+       }
+       if (which & cppvPsnk) {
+               to = (void *)phystokv((pmap_paddr_t)to);
+       }

 
 

-       if ((which & (cppvPsrc | cppvKmap)) == 0)       /* Source is virtual in
-                                                        * current map */
+       if ((which & (cppvPsrc | cppvKmap)) == 0) {     /* Source is virtual in
+                                                        * current map */

                retval = copyin((user_addr_t) from, to, size);
                retval = copyin((user_addr_t) from, to, size);

-       else if ((which & (cppvPsnk | cppvKmap)) == 0)  /* Sink is virtual in
-                                                        * current map */
+       } else if ((which & (cppvPsnk | cppvKmap)) == 0) { /* Sink is virtual in
+                                                           * current map */

                retval = copyout(from, (user_addr_t) to, size);
                retval = copyout(from, (user_addr_t) to, size);

-       else                    /* both addresses are physical or kernel map */
+       } else {                /* both addresses are physical or kernel map */

                bcopy(from, to, size);
                bcopy(from, to, size);

+       }

 
        if (which & cppvFsrc) {
                flush_dcache64(source, size, ((which & cppvPsrc) == cppvPsrc));
        } else if (which & cppvPsrc) {
                from_wimg_bits = pmap_cache_attributes(source >> PAGE_SHIFT);
 
        if (which & cppvFsrc) {
                flush_dcache64(source, size, ((which & cppvPsrc) == cppvPsrc));
        } else if (which & cppvPsrc) {
                from_wimg_bits = pmap_cache_attributes(source >> PAGE_SHIFT);

-               if ((from_wimg_bits != VM_WIMG_COPYBACK) && (from_wimg_bits != VM_WIMG_WTHRU))
+               if ((from_wimg_bits != VM_WIMG_COPYBACK) && (from_wimg_bits != VM_WIMG_WTHRU)) {

                        flush_dcache64(source, size, TRUE);
                        flush_dcache64(source, size, TRUE);

+               }

        }
 
        if (which & cppvFsnk) {
                flush_dcache64(sink, size, ((which & cppvPsnk) == cppvPsnk));
        }
 
        if (which & cppvFsnk) {
                flush_dcache64(sink, size, ((which & cppvPsnk) == cppvPsnk));

-       } else if (which & cppvPsnk) { 
+       } else if (which & cppvPsnk) {

                to_wimg_bits = pmap_cache_attributes(sink >> PAGE_SHIFT);
                to_wimg_bits = pmap_cache_attributes(sink >> PAGE_SHIFT);

-               if (to_wimg_bits != VM_WIMG_COPYBACK)
+               if (to_wimg_bits != VM_WIMG_COPYBACK) {

                        flush_dcache64(sink, size, TRUE);
                        flush_dcache64(sink, size, TRUE);

+               }

        }
        return retval;
 }
        }
        return retval;
 }


@@ -536,6 +608,36 @@ copypv(addr64_t source, addr64_t sink, unsigned int size, int which)
  */
 const int copysize_limit_panic = (64 * 1024 * 1024);
 
  */
 const int copysize_limit_panic = (64 * 1024 * 1024);
 

+static inline bool
+is_kernel_to_kernel_copy()
+{
+       return current_thread()->map->pmap == kernel_pmap;
+}
+
+static int
+copy_validate_user(const user_addr_t user_addr, vm_size_t nbytes, bool kern_to_kern_allowed)
+{
+       user_addr_t user_addr_last = user_addr + nbytes;
+       thread_t self = current_thread();
+
+       if (__improbable(!kern_to_kern_allowed && is_kernel_to_kernel_copy())) {
+               return EFAULT;
+       }
+
+       if (__improbable((user_addr_last < user_addr) ||
+           ((user_addr + nbytes) > vm_map_max(self->map)) ||
+           (user_addr < vm_map_min(self->map)))) {
+               return EFAULT;
+       }
+
+       if (__improbable(nbytes > copysize_limit_panic)) {
+               panic("%s(%p, ..., %u) - transfer too large", __func__,
+                   (void *)user_addr, nbytes);
+       }
+
+       return 0;
+}
+

 /*
  * Validate the arguments to copy{in,out} on this platform.
  *
 /*
  * Validate the arguments to copy{in,out} on this platform.
  *


@@ -545,45 +647,59 @@ const int copysize_limit_panic = (64 * 1024 * 1024);
  */
 static int
 copy_validate(const user_addr_t user_addr,
  */
 static int
 copy_validate(const user_addr_t user_addr,

-    uintptr_t kernel_addr, vm_size_t nbytes)
+    uintptr_t kernel_addr, vm_size_t nbytes, bool kern_to_kern_allowed)

 {
        uintptr_t kernel_addr_last = kernel_addr + nbytes;
 
 {
        uintptr_t kernel_addr_last = kernel_addr + nbytes;
 

-       if (kernel_addr < VM_MIN_KERNEL_ADDRESS ||
+       if (__improbable(kernel_addr < VM_MIN_KERNEL_ADDRESS ||

            kernel_addr > VM_MAX_KERNEL_ADDRESS ||
            kernel_addr_last < kernel_addr ||
            kernel_addr > VM_MAX_KERNEL_ADDRESS ||
            kernel_addr_last < kernel_addr ||

-           kernel_addr_last > VM_MAX_KERNEL_ADDRESS)
+           kernel_addr_last > VM_MAX_KERNEL_ADDRESS)) {

                panic("%s(%p, %p, %u) - kaddr not in kernel", __func__,
                    (void *)user_addr, (void *)kernel_addr, nbytes);
                panic("%s(%p, %p, %u) - kaddr not in kernel", __func__,
                    (void *)user_addr, (void *)kernel_addr, nbytes);

+       }

 
 

-       user_addr_t user_addr_last = user_addr + nbytes;
+       return copy_validate_user(user_addr, nbytes, kern_to_kern_allowed);
+}

 
 

-       if (user_addr_last < user_addr ||
-           user_addr_last > VM_MIN_KERNEL_ADDRESS)
-               return (EFAULT);
+int
+copyin_validate(const user_addr_t ua, uintptr_t ka, vm_size_t nbytes)
+{
+       return copy_validate(ua, ka, nbytes, true);
+}

 
 

-       if (__improbable(nbytes > copysize_limit_panic))
-               panic("%s(%p, %p, %u) - transfer too large", __func__,
-                   (void *)user_addr, (void *)kernel_addr, nbytes);
+int
+copyin_user_validate(const user_addr_t ua, uintptr_t ka, vm_size_t nbytes)
+{
+       return copy_validate(ua, ka, nbytes, false);
+}

 
 

-       return (0);
+int
+copyout_validate(uintptr_t ka, const user_addr_t ua, vm_size_t nbytes)
+{
+       return copy_validate(ua, ka, nbytes, true);

 }
 
 int
 }
 
 int

-copyin_validate(const user_addr_t ua, uintptr_t ka, vm_size_t nbytes)
+copyio_user_validate(int a __unused, int b __unused,
+    user_addr_t ua, vm_size_t nbytes)

 {
 {

-       return (copy_validate(ua, ka, nbytes));
+       return copy_validate_user(ua, nbytes, false);

 }
 
 int
 }
 
 int

-copyout_validate(uintptr_t ka, const user_addr_t ua, vm_size_t nbytes)
+copyoutstr_prevalidate(const void *__unused kaddr, user_addr_t __unused uaddr, size_t __unused len)

 {
 {

-       return (copy_validate(ua, ka, nbytes));
+       if (__improbable(is_kernel_to_kernel_copy())) {
+               return EFAULT;
+       }
+
+       return 0;

 }
 
 #if     MACH_ASSERT
 
 }
 
 #if     MACH_ASSERT
 

-extern int copyinframe(vm_address_t fp, char *frame);  
+extern int copyinframe(vm_address_t fp, char *frame);

 
 /*
  * Machine-dependent routine to fill in an array with up to callstack_max
 
 /*
  * Machine-dependent routine to fill in an array with up to callstack_max


@@ -591,34 +707,37 @@ extern int copyinframe(vm_address_t fp, char *frame);
  */
 void
 machine_callstack(
  */
 void
 machine_callstack(

-                 uintptr_t * buf,
-                 vm_size_t callstack_max)
+       uintptr_t * buf,
+       vm_size_t callstack_max)

 {
        /* Captures the USER call stack */
 {
        /* Captures the USER call stack */

-       uint32_t i=0;
+       uint32_t i = 0;

        uint32_t frame[2];
 
        struct arm_saved_state* state = find_user_regs(current_thread());
 
        if (!state) {
        uint32_t frame[2];
 
        struct arm_saved_state* state = find_user_regs(current_thread());
 
        if (!state) {

-               while (i<callstack_max)
+               while (i < callstack_max) {

                        buf[i++] = 0;
                        buf[i++] = 0;

+               }

        } else {
                buf[i++] = (uintptr_t)state->pc;
                frame[0] = state->r[7];
 
        } else {
                buf[i++] = (uintptr_t)state->pc;
                frame[0] = state->r[7];
 

-               while (i<callstack_max && frame[0] != 0) {
-                       if (copyinframe(frame[0], (void*) frame))
+               while (i < callstack_max && frame[0] != 0) {
+                       if (copyinframe(frame[0], (void*) frame)) {

                                break;
                                break;

+                       }

                        buf[i++] = (uintptr_t)frame[1];
                }
 
                        buf[i++] = (uintptr_t)frame[1];
                }
 

-               while (i<callstack_max)
+               while (i < callstack_max) {

                        buf[i++] = 0;
                        buf[i++] = 0;

+               }

        }
 }
 
        }
 }
 

-#endif                         /* MACH_ASSERT */
+#endif                          /* MACH_ASSERT */

 
 int
 clr_be_bit(void)
 
 int
 clr_be_bit(void)


@@ -629,8 +748,8 @@ clr_be_bit(void)
 
 boolean_t
 ml_probe_read(
 
 boolean_t
 ml_probe_read(

-             __unused vm_offset_t paddr,
-             __unused unsigned int *val)
+       __unused vm_offset_t paddr,
+       __unused unsigned int *val)

 {
        panic("ml_probe_read() unimplemented");
        return 1;
 {
        panic("ml_probe_read() unimplemented");
        return 1;


@@ -638,8 +757,8 @@ ml_probe_read(
 
 boolean_t
 ml_probe_read_64(
 
 boolean_t
 ml_probe_read_64(

-                __unused addr64_t paddr,
-                __unused unsigned int *val)
+       __unused addr64_t paddr,
+       __unused unsigned int *val)

 {
        panic("ml_probe_read_64() unimplemented");
        return 1;
 {
        panic("ml_probe_read_64() unimplemented");
        return 1;


@@ -648,12 +767,12 @@ ml_probe_read_64(
 
 void
 ml_thread_policy(
 
 void
 ml_thread_policy(

-                __unused thread_t thread,
-                __unused unsigned policy_id,
-                __unused unsigned policy_info)
+       __unused thread_t thread,
+       __unused unsigned policy_id,
+       __unused unsigned policy_info)

 {
 {

-  //    <rdar://problem/7141284>: Reduce print noise
-  //   kprintf("ml_thread_policy() unimplemented\n");
+       //    <rdar://problem/7141284>: Reduce print noise
+       //      kprintf("ml_thread_policy() unimplemented\n");

 }
 
 #if !MACH_KDP
 }
 
 #if !MACH_KDP