osfmk/x86_64/copyio.c

   1 /*
   2  * Copyright (c) 2009 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 #include <mach_assert.h>
  29
  30 #include <sys/errno.h>
  31 #include <i386/param.h>
  32 #include <i386/misc_protos.h>
  33 #include <i386/cpu_data.h>
  34 #include <i386/machine_routines.h>
  35 #include <i386/cpuid.h>
  36 #include <i386/vmx.h>
  37 #include <vm/pmap.h>
  38 #include <vm/vm_map.h>
  39 #include <vm/vm_kern.h>
  40 #include <vm/vm_fault.h>
  41
  42 #include <sys/kdebug.h>
  43
  44 static int copyio(int, user_addr_t, char *, vm_size_t, vm_size_t *, int);
  45 static int copyio_phys(addr64_t, addr64_t, vm_size_t, int);
  46
  47 /*
  48  * The copy engine has the following characteristics
  49  *   - copyio() handles copies to/from user or kernel space
  50  *   - copypv() deals with physical or virtual addresses
  51  *
  52  * Readers familiar with the 32-bit kernel will expect Joe's thesis at this
  53  * point describing the full glory of the copy window implementation. In K64,
  54  * however, there is no need for windowing. Thanks to the vast shared address
  55  * space, the kernel has direct access to userspace and to physical memory.
  56  *
  57  * User virtual addresses are accessible provided the user's cr3 is loaded.
  58  * Physical addresses are accessible via the direct map and the PHYSMAP_PTOV()
  59  * translation.
  60  *
  61  * Copyin/out variants all boil done to just these 2 routines in locore.s which
  62  * provide fault-recoverable copying:
  63  */
  64 extern int _bcopy(const void *, void *, vm_size_t);
  65 extern int _bcopystr(const void *, void *, vm_size_t, vm_size_t *);
  66
  67
  68 /*
  69  * Types of copies:
  70  */
  71 #define COPYIN          0       /* from user virtual to kernel virtual */
  72 #define COPYOUT         1       /* from kernel virtual to user virtual */
  73 #define COPYINSTR       2       /* string variant of copyout */
  74 #define COPYINPHYS      3       /* from user virtual to kernel physical */
  75 #define COPYOUTPHYS     4       /* from kernel physical to user virtual */
  76
  77 static int
  78 copyio(int copy_type, user_addr_t user_addr, char *kernel_addr,
  79        vm_size_t nbytes, vm_size_t *lencopied, int use_kernel_map)
  80 {
  81         thread_t        thread;
  82         pmap_t          pmap;
  83         vm_size_t       bytes_copied;
  84         int             error = 0;
  85         boolean_t       istate = FALSE;
  86         boolean_t       recursive_CopyIOActive;
  87 #if KDEBUG
  88         int             debug_type = 0xeff70010;
  89         debug_type += (copy_type << 2);
  90 #endif
  91
  92         thread = current_thread();
  93
  94         KERNEL_DEBUG(debug_type | DBG_FUNC_START,
  95                      (unsigned)(user_addr >> 32), (unsigned)user_addr,
  96                      nbytes, thread->machine.copyio_state, 0);
  97
  98         if (nbytes == 0)
  99                 goto out;
 100
 101         pmap = thread->map->pmap;
 102
 103         if ((copy_type != COPYINPHYS) && (copy_type != COPYOUTPHYS) && ((vm_offset_t)kernel_addr < VM_MIN_KERNEL_AND_KEXT_ADDRESS)) {
 104                 panic("Invalid copy parameter, copy type: %d, kernel address: %p", copy_type, kernel_addr);
 105         }
 106
 107         /* Sanity and security check for addresses to/from a user */
 108
 109         if (((pmap != kernel_pmap) && (use_kernel_map == 0)) &&
 110             ((nbytes && (user_addr+nbytes <= user_addr)) || ((user_addr + nbytes) > vm_map_max(thread->map)))) {
 111                 error = EFAULT;
 112                 goto out;
 113         }
 114
 115         /*
 116          * If the no_shared_cr3 boot-arg is set (true), the kernel runs on
 117          * its own pmap and cr3 rather than the user's -- so that wild accesses
 118          * from kernel or kexts can be trapped. So, during copyin and copyout,
 119          * we need to switch back to the user's map/cr3. The thread is flagged
 120          * "CopyIOActive" at this time so that if the thread is pre-empted,
 121          * we will later restore the correct cr3.
 122          */
 123         recursive_CopyIOActive = thread->machine.specFlags & CopyIOActive;
 124         thread->machine.specFlags |= CopyIOActive;
 125         if (no_shared_cr3) {
 126                 istate = ml_set_interrupts_enabled(FALSE);
 127                 if (get_cr3_base() != pmap->pm_cr3)
 128                         set_cr3_raw(pmap->pm_cr3);
 129         }
 130
 131         /*
 132          * Ensure that we're running on the target thread's cr3.
 133          */
 134         if ((pmap != kernel_pmap) && !use_kernel_map &&
 135             (get_cr3_base() != pmap->pm_cr3)) {
 136                 panic("copyio(%d,%p,%p,%ld,%p,%d) cr3 is %p expects %p",
 137                         copy_type, (void *)user_addr, kernel_addr, nbytes, lencopied, use_kernel_map,
 138                         (void *) get_cr3_raw(), (void *) pmap->pm_cr3);
 139         }
 140         if (no_shared_cr3)
 141                 (void) ml_set_interrupts_enabled(istate);
 142
 143         KERNEL_DEBUG(0xeff70044 | DBG_FUNC_NONE, (unsigned)user_addr,
 144                      (unsigned)kernel_addr, nbytes, 0, 0);
 145
 146         switch (copy_type) {
 147
 148         case COPYIN:
 149                 error = _bcopy((const void *) user_addr,
 150                                 kernel_addr,
 151                                 nbytes);
 152                 break;
 153
 154         case COPYOUT:
 155                 error = _bcopy(kernel_addr,
 156                                 (void *) user_addr,
 157                                 nbytes);
 158                 break;
 159
 160         case COPYINPHYS:
 161                 error = _bcopy((const void *) user_addr,
 162                                 PHYSMAP_PTOV(kernel_addr),
 163                                 nbytes);
 164                 break;
 165
 166         case COPYOUTPHYS:
 167                 error = _bcopy((const void *) PHYSMAP_PTOV(kernel_addr),
 168                                 (void *) user_addr,
 169                                 nbytes);
 170                 break;
 171
 172         case COPYINSTR:
 173                 error = _bcopystr((const void *) user_addr,
 174                                 kernel_addr,
 175                                 (int) nbytes,
 176                                 &bytes_copied);
 177
 178                 /*
 179                  * lencopied should be updated on success
 180                  * or ENAMETOOLONG...  but not EFAULT
 181                  */
 182                 if (error != EFAULT)
 183                         *lencopied = bytes_copied;
 184
 185                 if (error) {
 186 #if KDEBUG
 187                         nbytes = *lencopied;
 188 #endif
 189                         break;
 190                 }
 191                 if (*(kernel_addr + bytes_copied - 1) == 0) {
 192                         /*
 193                          * we found a NULL terminator... we're done
 194                          */
 195 #if KDEBUG
 196                         nbytes = *lencopied;
 197 #endif
 198                         break;
 199                 } else {
 200                         /*
 201                          * no more room in the buffer and we haven't
 202                          * yet come across a NULL terminator
 203                          */
 204 #if KDEBUG
 205                         nbytes = *lencopied;
 206 #endif
 207                         error = ENAMETOOLONG;
 208                         break;
 209                 }
 210                 break;
 211         }
 212
 213         if (!recursive_CopyIOActive) {
 214                 thread->machine.specFlags &= ~CopyIOActive;
 215         }
 216         if (no_shared_cr3) {
 217                 istate = ml_set_interrupts_enabled(FALSE);
 218                 if  (get_cr3_raw() != kernel_pmap->pm_cr3)
 219                         set_cr3_raw(kernel_pmap->pm_cr3);
 220                 (void) ml_set_interrupts_enabled(istate);
 221         }
 222
 223 out:
 224         KERNEL_DEBUG(debug_type | DBG_FUNC_END, (unsigned)user_addr,
 225                      (unsigned)kernel_addr, (unsigned)nbytes, error, 0);
 226
 227         return (error);
 228 }
 229
 230
 231 static int
 232 copyio_phys(addr64_t source, addr64_t sink, vm_size_t csize, int which)
 233 {
 234         char        *paddr;
 235         user_addr_t vaddr;
 236         int         ctype;
 237
 238         if (which & cppvPsnk) {
 239                 paddr  = (char *)sink;
 240                 vaddr  = (user_addr_t)source;
 241                 ctype  = COPYINPHYS;
 242         } else {
 243                 paddr  = (char *)source;
 244                 vaddr  = (user_addr_t)sink;
 245                 ctype  = COPYOUTPHYS;
 246         }
 247         return copyio(ctype, vaddr, paddr, csize, NULL, which & cppvKmap);
 248 }
 249
 250 int
 251 copyinmsg(const user_addr_t user_addr, char *kernel_addr, mach_msg_size_t nbytes)
 252 {
 253     return copyio(COPYIN, user_addr, kernel_addr, nbytes, NULL, 0);
 254 }
 255
 256 int
 257 copyin(const user_addr_t user_addr, char *kernel_addr, vm_size_t nbytes)
 258 {
 259     return copyio(COPYIN, user_addr, kernel_addr, nbytes, NULL, 0);
 260 }
 261
 262 int
 263 copyinstr(const user_addr_t user_addr,  char *kernel_addr, vm_size_t nbytes, vm_size_t *lencopied)
 264 {
 265     *lencopied = 0;
 266
 267     return copyio(COPYINSTR, user_addr, kernel_addr, nbytes, lencopied, 0);
 268 }
 269
 270 int
 271 copyoutmsg(const char *kernel_addr, user_addr_t user_addr, mach_msg_size_t nbytes)
 272 {
 273     return copyio(COPYOUT, user_addr, (char *)(uintptr_t)kernel_addr, nbytes, NULL, 0);
 274 }
 275
 276 int
 277 copyout(const void *kernel_addr, user_addr_t user_addr, vm_size_t nbytes)
 278 {
 279     return copyio(COPYOUT, user_addr, (char *)(uintptr_t)kernel_addr, nbytes, NULL, 0);
 280 }
 281
 282
 283 kern_return_t
 284 copypv(addr64_t src64, addr64_t snk64, unsigned int size, int which)
 285 {
 286         unsigned int lop, csize;
 287         int bothphys = 0;
 288
 289         KERNEL_DEBUG(0xeff7004c | DBG_FUNC_START, (unsigned)src64,
 290                      (unsigned)snk64, size, which, 0);
 291
 292         if ((which & (cppvPsrc | cppvPsnk)) == 0 )                              /* Make sure that only one is virtual */
 293                 panic("copypv: no more than 1 parameter may be virtual\n");     /* Not allowed */
 294
 295         if ((which & (cppvPsrc | cppvPsnk)) == (cppvPsrc | cppvPsnk))
 296                 bothphys = 1;                                                   /* both are physical */
 297
 298         while (size) {
 299
 300                 if (bothphys) {
 301                         lop = (unsigned int)(PAGE_SIZE - (snk64 & (PAGE_SIZE - 1)));            /* Assume sink smallest */
 302
 303                         if (lop > (unsigned int)(PAGE_SIZE - (src64 & (PAGE_SIZE - 1))))
 304                                 lop = (unsigned int)(PAGE_SIZE - (src64 & (PAGE_SIZE - 1)));    /* No, source is smaller */
 305                 } else {
 306                         /*
 307                          * only need to compute the resid for the physical page
 308                          * address... we don't care about where we start/finish in
 309                          * the virtual since we just call the normal copyin/copyout
 310                          */
 311                         if (which & cppvPsrc)
 312                                 lop = (unsigned int)(PAGE_SIZE - (src64 & (PAGE_SIZE - 1)));
 313                         else
 314                                 lop = (unsigned int)(PAGE_SIZE - (snk64 & (PAGE_SIZE - 1)));
 315                 }
 316                 csize = size;                                           /* Assume we can copy it all */
 317                 if (lop < size)
 318                         csize = lop;                                    /* Nope, we can't do it all */
 319 #if 0
 320                 /*
 321                  * flush_dcache64 is currently a nop on the i386...
 322                  * it's used when copying to non-system memory such
 323                  * as video capture cards... on PPC there was a need
 324                  * to flush due to how we mapped this memory... not
 325                  * sure if it's needed on i386.
 326                  */
 327                 if (which & cppvFsrc)
 328                         flush_dcache64(src64, csize, 1);                /* If requested, flush source before move */
 329                 if (which & cppvFsnk)
 330                         flush_dcache64(snk64, csize, 1);                /* If requested, flush sink before move */
 331 #endif
 332                 if (bothphys)
 333                         bcopy_phys(src64, snk64, csize);                /* Do a physical copy, virtually */
 334                 else {
 335                         if (copyio_phys(src64, snk64, csize, which))
 336                                 return (KERN_FAILURE);
 337                 }
 338 #if 0
 339                 if (which & cppvFsrc)
 340                         flush_dcache64(src64, csize, 1);        /* If requested, flush source after move */
 341                 if (which & cppvFsnk)
 342                         flush_dcache64(snk64, csize, 1);        /* If requested, flush sink after move */
 343 #endif
 344                 size   -= csize;                                        /* Calculate what is left */
 345                 snk64 += csize;                                 /* Bump sink to next physical address */
 346                 src64 += csize;                                 /* Bump source to next physical address */
 347         }
 348         KERNEL_DEBUG(0xeff7004c | DBG_FUNC_END, (unsigned)src64,
 349                      (unsigned)snk64, size, which, 0);
 350
 351         return KERN_SUCCESS;
 352 }