osfmk/arm64/copyio.c

   1 /*
   2  * Copyright (c) 2012-2013 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <arm/cpu_data_internal.h>
  30 #include <arm/misc_protos.h>
  31 #include <kern/thread.h>
  32 #include <sys/errno.h>
  33 #include <vm/pmap.h>
  34 #include <vm/vm_map.h>
  35 #include <san/kasan.h>
  36
  37 #undef copyin
  38 #undef copyout
  39
  40 extern int _bcopyin(const char *src, char *dst, vm_size_t len);
  41 extern int _bcopyinstr(const char *src, char *dst, vm_size_t max, vm_size_t *actual);
  42 extern int _bcopyout(const char *src, char *dst, vm_size_t len);
  43 extern int _copyin_word(const char *src, uint64_t *dst, vm_size_t len);
  44
  45 extern pmap_t kernel_pmap;
  46
  47 /* On by default, optionally disabled by boot-arg */
  48 extern boolean_t copyio_zalloc_check;
  49
  50 typedef enum copyio_type {
  51         COPYIO_IN,
  52         COPYIO_IN_WORD,
  53         COPYIO_INSTR,
  54         COPYIO_OUT,
  55 } copyio_type_t;
  56
  57 static inline void
  58 user_access_enable(void)
  59 {
  60 #if __ARM_PAN_AVAILABLE__
  61         __builtin_arm_wsr("pan", 0);
  62 #endif  /* __ARM_PAN_AVAILABLE__ */
  63 }
  64
  65 static inline void
  66 user_access_disable(void)
  67 {
  68 #if __ARM_PAN_AVAILABLE__
  69         __builtin_arm_wsr("pan", 1);
  70 #endif  /* __ARM_PAN_AVAILABLE__ */
  71 }
  72
  73 static int
  74 copyio(copyio_type_t copytype, const char *src, char *dst,
  75            vm_size_t nbytes, vm_size_t *lencopied)
  76 {
  77         int result = 0;
  78         vm_size_t bytes_copied = 0;
  79         vm_size_t kernel_buf_size = 0;
  80         void * kernel_addr = NULL;
  81
  82         /* Reject TBI addresses */
  83         if (copytype == COPYIO_OUT) {
  84                 if ((uintptr_t)dst & TBI_MASK)
  85                         return EINVAL;
  86         } else {
  87                 if ((uintptr_t)src & TBI_MASK)
  88                         return EINVAL;
  89         }
  90
  91         if (__probable(copyio_zalloc_check)) {
  92                 if (copytype == COPYIO_IN || copytype == COPYIO_INSTR || copytype == COPYIO_IN_WORD) {
  93                         kernel_addr = (void*)dst;
  94                 } else if (copytype == COPYIO_OUT) {
  95                         kernel_addr = (void*)(uintptr_t)src;
  96                 }
  97                 if (kernel_addr)
  98                         kernel_buf_size = zone_element_size(kernel_addr, NULL);
  99                 if (__improbable(kernel_buf_size && kernel_buf_size < nbytes))
 100                         panic("copyio: kernel buffer %p has size %lu < nbytes %lu", kernel_addr, kernel_buf_size, nbytes);
 101         }
 102
 103 #if KASAN
 104         /* For user copies, asan-check the kernel-side buffer */
 105         if (copytype == COPYIO_IN || copytype == COPYIO_INSTR || copytype == COPYIO_IN_WORD) {
 106                 __asan_storeN((uintptr_t)dst, nbytes);
 107         } else if (copytype == COPYIO_OUT) {
 108                 __asan_loadN((uintptr_t)src, nbytes);
 109         }
 110 #endif
 111
 112         user_access_enable();
 113
 114         /* Select copy routines based on direction:
 115          *   COPYIO_IN - Use unprivileged loads to read from user address
 116          *   COPYIO_OUT - Use unprivleged stores to write to user address
 117          */
 118
 119         switch (copytype) {
 120         case COPYIO_IN:
 121                 result = _bcopyin(src, dst, nbytes);
 122                 break;
 123         case COPYIO_INSTR:
 124                 result = _bcopyinstr(src, dst, nbytes, &bytes_copied);
 125                 if (result != EFAULT) {
 126                         *lencopied = bytes_copied;
 127                 }
 128                 break;
 129         case COPYIO_IN_WORD:
 130                 result = _copyin_word(src, (uint64_t *)(uintptr_t)dst, nbytes);
 131                 break;
 132         case COPYIO_OUT:
 133                 result = _bcopyout(src, dst, nbytes);
 134                 break;
 135         default:
 136                 result = EINVAL;
 137         }
 138
 139         user_access_disable();
 140         return result;
 141 }
 142
 143 int
 144 copyin_kern(const user_addr_t user_addr, char *kernel_addr, vm_size_t nbytes)
 145 {
 146         bcopy((const char*)(uintptr_t)user_addr, kernel_addr, nbytes);
 147
 148         return 0;
 149 }
 150
 151 int
 152 copyout_kern(const char *kernel_addr, user_addr_t user_addr, vm_size_t nbytes)
 153 {
 154         bcopy(kernel_addr, (char *)(uintptr_t)user_addr, nbytes);
 155
 156         return 0;
 157 }
 158
 159 int
 160 copyin(const user_addr_t user_addr, void *kernel_addr, vm_size_t nbytes)
 161 {
 162         int result;
 163
 164         if (nbytes == 0)
 165                 return 0;
 166
 167         result = copyin_validate(user_addr, (uintptr_t)kernel_addr, nbytes);
 168         if (result) return result;
 169
 170         if (current_thread()->map->pmap == kernel_pmap)
 171                 return copyin_kern(user_addr, kernel_addr, nbytes);
 172         else
 173                 return copyio(COPYIO_IN, (const char *)(uintptr_t)user_addr, kernel_addr, nbytes, NULL);
 174 }
 175
 176 /*
 177  * copyin_word
 178  * Read an aligned value from userspace as a single memory transaction.
 179  * This function supports userspace synchronization features
 180  */
 181 int
 182 copyin_word(const user_addr_t user_addr, uint64_t *kernel_addr, vm_size_t nbytes)
 183 {
 184         int                     result;
 185
 186         /* Verify sizes */
 187         if ((nbytes != 4) && (nbytes != 8))
 188                 return EINVAL;
 189
 190         /* Test alignment */
 191         if (user_addr & (nbytes - 1))
 192                 return EINVAL;
 193
 194         result = copyin_validate(user_addr, (uintptr_t)kernel_addr, nbytes);
 195         if (result)
 196                 return result;
 197
 198         return copyio(COPYIO_IN_WORD, (const char *)user_addr, (char *)(uintptr_t)kernel_addr, nbytes, NULL);
 199 }
 200
 201 int
 202 copyinstr(const user_addr_t user_addr, char *kernel_addr, vm_size_t nbytes, vm_size_t *lencopied)
 203 {
 204         int result;
 205
 206         *lencopied = 0;
 207         if (nbytes == 0)
 208                 return ENAMETOOLONG;
 209
 210         result = copyin_validate(user_addr, (uintptr_t)kernel_addr, nbytes);
 211
 212         if (result) return result;
 213
 214         return copyio(COPYIO_INSTR, (const char *)(uintptr_t)user_addr, kernel_addr, nbytes, lencopied);
 215 }
 216
 217 int
 218 copyout(const void *kernel_addr, user_addr_t user_addr, vm_size_t nbytes)
 219 {
 220         int result;
 221
 222         if (nbytes == 0)
 223                 return 0;
 224
 225         result = copyout_validate((uintptr_t)kernel_addr, user_addr, nbytes);
 226         if (result) return result;
 227
 228         if (current_thread()->map->pmap == kernel_pmap)
 229                 return copyout_kern(kernel_addr, user_addr, nbytes);
 230         else
 231                 return copyio(COPYIO_OUT, kernel_addr, (char *)(uintptr_t)user_addr, nbytes, NULL);
 232 }
 233
 234
 235 /*
 236  * Copy sizes bigger than this value will cause a kernel panic.
 237  *
 238  * Yes, this is an arbitrary fixed limit, but it's almost certainly
 239  * a programming error to be copying more than this amount between
 240  * user and wired kernel memory in a single invocation on this
 241  * platform.
 242  */
 243 const int copysize_limit_panic = (64 * 1024 * 1024);
 244
 245 /*
 246  * Validate the arguments to copy{in,out} on this platform.
 247  */
 248 static int
 249 copy_validate(const user_addr_t user_addr,
 250         uintptr_t kernel_addr, vm_size_t nbytes)
 251 {
 252         uintptr_t kernel_addr_last = kernel_addr + nbytes;
 253
 254         if (__improbable(kernel_addr < VM_MIN_KERNEL_ADDRESS ||
 255             kernel_addr > VM_MAX_KERNEL_ADDRESS ||
 256             kernel_addr_last < kernel_addr ||
 257             kernel_addr_last > VM_MAX_KERNEL_ADDRESS))
 258                 panic("%s(%p, %p, %lu) - kaddr not in kernel", __func__,
 259                        (void *)user_addr, (void *)kernel_addr, nbytes);
 260
 261         user_addr_t user_addr_last = user_addr + nbytes;
 262
 263         if (__improbable((user_addr_last < user_addr) || ((user_addr + nbytes) > vm_map_max(current_thread()->map)) ||
 264             (user_addr < vm_map_min(current_thread()->map))))
 265                 return (EFAULT);
 266
 267         if (__improbable(nbytes > copysize_limit_panic))
 268                 panic("%s(%p, %p, %lu) - transfer too large", __func__,
 269                        (void *)user_addr, (void *)kernel_addr, nbytes);
 270
 271         return (0);
 272 }
 273
 274 int
 275 copyin_validate(const user_addr_t ua, uintptr_t ka, vm_size_t nbytes)
 276 {
 277         return (copy_validate(ua, ka, nbytes));
 278 }
 279
 280 int
 281 copyout_validate(uintptr_t ka, const user_addr_t ua, vm_size_t nbytes)
 282 {
 283         return (copy_validate(ua, ka, nbytes));
 284 }
 285