osfmk/x86_64/copyio.c

   1 /*
   2  * Copyright (c) 2009-2016 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 #include <mach_assert.h>
  29
  30 #include <sys/errno.h>
  31 #include <i386/param.h>
  32 #include <i386/misc_protos.h>
  33 #include <i386/cpu_data.h>
  34 #include <i386/machine_cpu.h>
  35 #include <i386/machine_routines.h>
  36 #include <i386/cpuid.h>
  37 #include <i386/vmx.h>
  38 #include <vm/pmap.h>
  39 #include <vm/vm_map.h>
  40 #include <vm/vm_kern.h>
  41 #include <vm/vm_fault.h>
  42 #include <san/kasan.h>
  43
  44 #include <sys/kdebug.h>
  45
  46 #include <kern/copyout_shim.h>
  47
  48 #undef copyin
  49 #undef copyout
  50
  51 static int copyio(int, user_addr_t, char *, vm_size_t, vm_size_t *, int);
  52 static int copyio_phys(addr64_t, addr64_t, vm_size_t, int);
  53
  54 /*
  55  * Copy sizes bigger than this value will cause a kernel panic.
  56  *
  57  * Yes, this is an arbitrary fixed limit, but it's almost certainly
  58  * a programming error to be copying more than this amount between
  59  * user and wired kernel memory in a single invocation on this
  60  * platform.
  61  */
  62 const int copysize_limit_panic = (64 * MB);
  63
  64 /*
  65  * The copy engine has the following characteristics
  66  *   - copyio() handles copies to/from user or kernel space
  67  *   - copypv() deals with physical or virtual addresses
  68  *
  69  * Readers familiar with the 32-bit kernel will expect Joe's thesis at this
  70  * point describing the full glory of the copy window implementation. In K64,
  71  * however, there is no need for windowing. Thanks to the vast shared address
  72  * space, the kernel has direct access to userspace and to physical memory.
  73  *
  74  * User virtual addresses are accessible provided the user's cr3 is loaded.
  75  * Physical addresses are accessible via the direct map and the PHYSMAP_PTOV()
  76  * translation.
  77  *
  78  * Copyin/out variants all boil done to just these 2 routines in locore.s which
  79  * provide fault-recoverable copying:
  80  */
  81 extern int _bcopy(const void *, void *, vm_size_t);
  82 extern int _bcopystr(const void *, void *, vm_size_t, vm_size_t *);
  83 extern int _copyin_atomic32(const char *src, uint32_t *dst);
  84 extern int _copyin_atomic64(const char *src, uint64_t *dst);
  85 extern int _copyout_atomic32(const uint32_t *u32, char *src);
  86 extern int _copyout_atomic64(const uint64_t *u64, char *src);
  87
  88 /* On by default, optionally disabled by boot-arg */
  89 extern boolean_t copyio_zalloc_check;
  90
  91 /*
  92  * Types of copies:
  93  */
  94 #define COPYIN          0       /* from user virtual to kernel virtual */
  95 #define COPYOUT         1       /* from kernel virtual to user virtual */
  96 #define COPYINSTR       2       /* string variant of copyout */
  97 #define COPYINPHYS      3       /* from user virtual to kernel physical */
  98 #define COPYOUTPHYS     4       /* from kernel physical to user virtual */
  99 #define COPYINATOMIC32  5       /* from user virtual to kernel virtual */
 100 #define COPYINATOMIC64  6       /* from user virtual to kernel virtual */
 101 #define COPYOUTATOMIC32 7       /* from user virtual to kernel virtual */
 102 #define COPYOUTATOMIC64 8       /* from user virtual to kernel virtual */
 103
 104 #if ENABLE_SMAPLOG
 105 typedef struct {
 106         uint64_t        timestamp;
 107         thread_t        thread;
 108         uintptr_t       cr4;
 109         uint8_t         cpuid;
 110         uint8_t         smap_state;
 111         uint8_t         copyio_active;
 112 } smaplog_entry_t;
 113
 114 #define SMAPLOG_BUFFER_SIZE (50)
 115 static smaplog_entry_t  smaplog_cbuf[SMAPLOG_BUFFER_SIZE];
 116 static uint32_t         smaplog_head = 0;
 117
 118 static void
 119 smaplog_add_entry(boolean_t enabling)
 120 {
 121         uint32_t index = 0;
 122         thread_t thread = current_thread();
 123
 124         do {
 125                 index = smaplog_head;
 126         } while (!OSCompareAndSwap(index, (index + 1) % SMAPLOG_BUFFER_SIZE, &smaplog_head));
 127
 128         assert(index < SMAPLOG_BUFFER_SIZE);
 129         assert(smaplog_head < SMAPLOG_BUFFER_SIZE);
 130         assert(thread);
 131
 132         smaplog_cbuf[index].timestamp = mach_absolute_time();
 133         smaplog_cbuf[index].thread = thread;
 134         smaplog_cbuf[index].cpuid = cpu_number();
 135         smaplog_cbuf[index].cr4 = get_cr4();
 136         smaplog_cbuf[index].smap_state = enabling;
 137         smaplog_cbuf[index].copyio_active = (thread->machine.specFlags & CopyIOActive) ? 1 : 0;
 138 }
 139 #endif /* ENABLE_SMAPLOG */
 140
 141 extern boolean_t pmap_smap_enabled;
 142 static inline void
 143 user_access_enable(void)
 144 {
 145         if (pmap_smap_enabled) {
 146                 stac();
 147 #if ENABLE_SMAPLOG
 148                 smaplog_add_entry(TRUE);
 149 #endif
 150         }
 151 }
 152 static inline void
 153 user_access_disable(void)
 154 {
 155         if (pmap_smap_enabled) {
 156                 clac();
 157 #if ENABLE_SMAPLOG
 158                 smaplog_add_entry(FALSE);
 159 #endif
 160         }
 161 }
 162
 163 #if COPYIO_TRACE_ENABLED
 164 #define COPYIO_TRACE(x, a, b, c, d, e) KERNEL_DEBUG_CONSTANT(x, a, b, c, d, e)
 165 #else
 166 #define COPYIO_TRACE(x, a, b, c, d, e) do { } while(0)
 167 #endif
 168
 169 static int
 170 copyio(int copy_type, user_addr_t user_addr, char *kernel_addr,
 171     vm_size_t nbytes, vm_size_t *lencopied, int use_kernel_map)
 172 {
 173         thread_t        thread = current_thread();
 174         pmap_t          pmap;
 175         vm_size_t       bytes_copied;
 176         int             error = 0;
 177         boolean_t       istate = FALSE;
 178         boolean_t       recursive_CopyIOActive;
 179 #if     COPYIO_TRACE_ENABLED
 180         int             debug_type = 0xeff70010;
 181         debug_type += (copy_type << 2);
 182 #endif
 183         vm_size_t kernel_buf_size = 0;
 184
 185         if (__improbable(nbytes > copysize_limit_panic)) {
 186                 panic("%s(%p, %p, %lu) - transfer too large", __func__,
 187                     (void *)user_addr, (void *)kernel_addr, nbytes);
 188         }
 189
 190         COPYIO_TRACE(debug_type | DBG_FUNC_START,
 191             user_addr, kernel_addr, nbytes, use_kernel_map, 0);
 192
 193         if (__improbable(nbytes == 0)) {
 194                 goto out;
 195         }
 196
 197         pmap = thread->map->pmap;
 198         boolean_t nopagezero = thread->map->pmap->pagezero_accessible;
 199
 200         if ((copy_type != COPYINPHYS) && (copy_type != COPYOUTPHYS)) {
 201                 if (__improbable((vm_offset_t)kernel_addr < VM_MIN_KERNEL_AND_KEXT_ADDRESS)) {
 202                         panic("Invalid copy parameter, copy type: %d, kernel address: %p", copy_type, kernel_addr);
 203                 }
 204                 if (__probable(copyio_zalloc_check)) {
 205                         kernel_buf_size = zone_element_size(kernel_addr, NULL);
 206                         if (__improbable(kernel_buf_size && kernel_buf_size < nbytes)) {
 207                                 panic("copyio: kernel buffer %p has size %lu < nbytes %lu", kernel_addr, kernel_buf_size, nbytes);
 208                         }
 209                 }
 210         }
 211
 212         /* Sanity and security check for addresses to/from a user */
 213
 214         if (__improbable(((pmap != kernel_pmap) && (use_kernel_map == 0)) &&
 215             ((nbytes && (user_addr + nbytes <= user_addr)) || ((user_addr + nbytes) > vm_map_max(thread->map))))) {
 216                 error = EFAULT;
 217                 goto out;
 218         }
 219
 220         if (copy_type >= COPYINATOMIC32 && copy_type <= COPYOUTATOMIC64) {
 221                 if (__improbable(pmap == kernel_pmap)) {
 222                         error = EFAULT;
 223                         goto out;
 224                 }
 225         }
 226
 227 #if KASAN
 228         switch (copy_type) {
 229         case COPYIN:
 230         case COPYINSTR:
 231         case COPYINATOMIC32:
 232         case COPYINATOMIC64:
 233                 __asan_storeN((uptr)kernel_addr, nbytes);
 234                 break;
 235         case COPYOUT:
 236         case COPYOUTATOMIC32:
 237         case COPYOUTATOMIC64:
 238                 __asan_loadN((uptr)kernel_addr, nbytes);
 239                 kasan_check_uninitialized((vm_address_t)kernel_addr, nbytes);
 240                 break;
 241         }
 242 #endif
 243
 244         /*
 245          * If the no_shared_cr3 boot-arg is set (true), the kernel runs on
 246          * its own pmap and cr3 rather than the user's -- so that wild accesses
 247          * from kernel or kexts can be trapped. So, during copyin and copyout,
 248          * we need to switch back to the user's map/cr3. The thread is flagged
 249          * "CopyIOActive" at this time so that if the thread is pre-empted,
 250          * we will later restore the correct cr3.
 251          */
 252         recursive_CopyIOActive = thread->machine.specFlags & CopyIOActive;
 253
 254         boolean_t pdswitch = no_shared_cr3 || nopagezero;
 255
 256         if (__improbable(pdswitch)) {
 257                 istate = ml_set_interrupts_enabled(FALSE);
 258                 if (nopagezero && pmap_pcid_ncpus) {
 259                         pmap_pcid_activate(pmap, cpu_number(), TRUE, TRUE);
 260                 } else if (get_cr3_base() != pmap->pm_cr3) {
 261                         set_cr3_raw(pmap->pm_cr3);
 262                 }
 263                 thread->machine.specFlags |= CopyIOActive;
 264         } else {
 265                 thread->machine.specFlags |= CopyIOActive;
 266         }
 267
 268         user_access_enable();
 269
 270 #if DEVELOPMENT || DEBUG
 271         /*
 272          * Ensure that we're running on the target thread's cr3.
 273          */
 274         if ((pmap != kernel_pmap) && !use_kernel_map &&
 275             (get_cr3_base() != pmap->pm_cr3)) {
 276                 panic("copyio(%d,%p,%p,%ld,%p,%d) cr3 is %p expects %p",
 277                     copy_type, (void *)user_addr, kernel_addr, nbytes, lencopied, use_kernel_map,
 278                     (void *) get_cr3_raw(), (void *) pmap->pm_cr3);
 279         }
 280 #endif
 281
 282         if (__improbable(pdswitch)) {
 283                 (void) ml_set_interrupts_enabled(istate);
 284         }
 285
 286         COPYIO_TRACE(0xeff70044 | DBG_FUNC_NONE, user_addr,
 287             kernel_addr, nbytes, 0, 0);
 288
 289         switch (copy_type) {
 290         case COPYIN:
 291                 error = _bcopy((const void *) user_addr,
 292                     kernel_addr,
 293                     nbytes);
 294                 break;
 295
 296         case COPYOUT:
 297                 error = _bcopy(kernel_addr,
 298                     (void *) user_addr,
 299                     nbytes);
 300                 break;
 301
 302         case COPYINPHYS:
 303                 error = _bcopy((const void *) user_addr,
 304                     PHYSMAP_PTOV(kernel_addr),
 305                     nbytes);
 306                 break;
 307
 308         case COPYOUTPHYS:
 309                 error = _bcopy((const void *) PHYSMAP_PTOV(kernel_addr),
 310                     (void *) user_addr,
 311                     nbytes);
 312                 break;
 313
 314         case COPYINATOMIC32:
 315                 error = _copyin_atomic32((const void *) user_addr,
 316                     (void *) kernel_addr);
 317                 break;
 318
 319         case COPYINATOMIC64:
 320                 error = _copyin_atomic64((const void *) user_addr,
 321                     (void *) kernel_addr);
 322                 break;
 323
 324         case COPYOUTATOMIC32:
 325                 error = _copyout_atomic32((const void *) kernel_addr,
 326                     (void *) user_addr);
 327                 break;
 328
 329         case COPYOUTATOMIC64:
 330                 error = _copyout_atomic64((const void *) kernel_addr,
 331                     (void *) user_addr);
 332                 break;
 333
 334         case COPYINSTR:
 335                 error = _bcopystr((const void *) user_addr,
 336                     kernel_addr,
 337                     (int) nbytes,
 338                     &bytes_copied);
 339
 340                 /*
 341                  * lencopied should be updated on success
 342                  * or ENAMETOOLONG...  but not EFAULT
 343                  */
 344                 if (error != EFAULT) {
 345                         *lencopied = bytes_copied;
 346                 }
 347
 348                 if (error) {
 349 #if KDEBUG
 350                         nbytes = *lencopied;
 351 #endif
 352                         break;
 353                 }
 354                 if (*(kernel_addr + bytes_copied - 1) == 0) {
 355                         /*
 356                          * we found a NULL terminator... we're done
 357                          */
 358 #if KDEBUG
 359                         nbytes = *lencopied;
 360 #endif
 361                         break;
 362                 } else {
 363                         /*
 364                          * no more room in the buffer and we haven't
 365                          * yet come across a NULL terminator
 366                          */
 367 #if KDEBUG
 368                         nbytes = *lencopied;
 369 #endif
 370                         error = ENAMETOOLONG;
 371                         break;
 372                 }
 373         }
 374
 375         user_access_disable();
 376
 377         if (__improbable(pdswitch)) {
 378                 istate = ml_set_interrupts_enabled(FALSE);
 379                 if (!recursive_CopyIOActive && (get_cr3_raw() != kernel_pmap->pm_cr3)) {
 380                         if (nopagezero && pmap_pcid_ncpus) {
 381                                 pmap_pcid_activate(pmap, cpu_number(), TRUE, FALSE);
 382                         } else {
 383                                 set_cr3_raw(kernel_pmap->pm_cr3);
 384                         }
 385                 }
 386
 387                 if (!recursive_CopyIOActive) {
 388                         thread->machine.specFlags &= ~CopyIOActive;
 389                 }
 390                 (void) ml_set_interrupts_enabled(istate);
 391         } else if (!recursive_CopyIOActive) {
 392                 thread->machine.specFlags &= ~CopyIOActive;
 393         }
 394
 395 out:
 396         COPYIO_TRACE(debug_type | DBG_FUNC_END, user_addr, kernel_addr, nbytes, error, 0);
 397
 398         return error;
 399 }
 400
 401
 402 static int
 403 copyio_phys(addr64_t source, addr64_t sink, vm_size_t csize, int which)
 404 {
 405         char        *paddr;
 406         user_addr_t vaddr;
 407         int         ctype;
 408
 409         if (which & cppvPsnk) {
 410                 paddr  = (char *)sink;
 411                 vaddr  = (user_addr_t)source;
 412                 ctype  = COPYINPHYS;
 413         } else {
 414                 paddr  = (char *)source;
 415                 vaddr  = (user_addr_t)sink;
 416                 ctype  = COPYOUTPHYS;
 417                 CALL_COPYOUT_SHIM_PHYS((void *)PHYSMAP_PTOV(source), sink, csize)
 418         }
 419         return copyio(ctype, vaddr, paddr, csize, NULL, which & cppvKmap);
 420 }
 421
 422 int
 423 copyinmsg(const user_addr_t user_addr, char *kernel_addr, mach_msg_size_t nbytes)
 424 {
 425         return copyio(COPYIN, user_addr, kernel_addr, nbytes, NULL, 0);
 426 }
 427
 428 int
 429 copyin(const user_addr_t user_addr, void *kernel_addr, vm_size_t nbytes)
 430 {
 431         return copyio(COPYIN, user_addr, kernel_addr, nbytes, NULL, 0);
 432 }
 433
 434 /*
 435  * copy{in,out}_atomic{32,64}
 436  * Read or store an aligned value from userspace as a single memory transaction.
 437  * These functions support userspace synchronization features
 438  */
 439 int
 440 copyin_atomic32(const user_addr_t user_addr, uint32_t *kernel_addr)
 441 {
 442         /* Test alignment */
 443         if (user_addr & 3) {
 444                 return EINVAL;
 445         }
 446         return copyio(COPYINATOMIC32, user_addr, (char *)(uintptr_t)kernel_addr, 4, NULL, 0);
 447 }
 448
 449 int
 450 copyin_atomic32_wait_if_equals(const user_addr_t user_addr, uint32_t value)
 451 {
 452         uint32_t u32;
 453         int result = copyin_atomic32(user_addr, &u32);
 454         if (__improbable(result)) {
 455                 return result;
 456         }
 457         if (u32 != value) {
 458                 return ESTALE;
 459         }
 460         cpu_pause();
 461         return 0;
 462 }
 463
 464 int
 465 copyin_atomic64(const user_addr_t user_addr, uint64_t *kernel_addr)
 466 {
 467         /* Test alignment */
 468         if (user_addr & 7) {
 469                 return EINVAL;
 470         }
 471         return copyio(COPYINATOMIC64, user_addr, (char *)(uintptr_t)kernel_addr, 8, NULL, 0);
 472 }
 473
 474 int
 475 copyout_atomic32(uint32_t value, user_addr_t user_addr)
 476 {
 477         /* Test alignment */
 478         if (user_addr & 3) {
 479                 return EINVAL;
 480         }
 481         return copyio(COPYOUTATOMIC32, user_addr, (char *)&value, 4, NULL, 0);
 482 }
 483
 484 int
 485 copyout_atomic64(uint64_t value, user_addr_t user_addr)
 486 {
 487         /* Test alignment */
 488         if (user_addr & 7) {
 489                 return EINVAL;
 490         }
 491         return copyio(COPYOUTATOMIC64, user_addr, (char *)&value, 8, NULL, 0);
 492 }
 493
 494 int
 495 copyinstr(const user_addr_t user_addr, char *kernel_addr, vm_size_t nbytes, vm_size_t *lencopied)
 496 {
 497         *lencopied = 0;
 498
 499         return copyio(COPYINSTR, user_addr, kernel_addr, nbytes, lencopied, 0);
 500 }
 501
 502 int
 503 copyoutmsg(const char *kernel_addr, user_addr_t user_addr, mach_msg_size_t nbytes)
 504 {
 505         CALL_COPYOUT_SHIM_MSG(kernel_addr, user_addr, (vm_size_t)nbytes)
 506         return copyio(COPYOUT, user_addr, (char *)(uintptr_t)kernel_addr, nbytes, NULL, 0);
 507 }
 508
 509 int
 510 copyout(const void *kernel_addr, user_addr_t user_addr, vm_size_t nbytes)
 511 {
 512         CALL_COPYOUT_SHIM_NRML(kernel_addr, user_addr, nbytes)
 513         return copyio(COPYOUT, user_addr, (char *)(uintptr_t)kernel_addr, nbytes, NULL, 0);
 514 }
 515
 516
 517 kern_return_t
 518 copypv(addr64_t src64, addr64_t snk64, unsigned int size, int which)
 519 {
 520         unsigned int lop, csize;
 521         int bothphys = 0;
 522
 523         KERNEL_DEBUG(0xeff7004c | DBG_FUNC_START, (unsigned)src64,
 524             (unsigned)snk64, size, which, 0);
 525
 526         if ((which & (cppvPsrc | cppvPsnk)) == 0) {                             /* Make sure that only one is virtual */
 527                 panic("copypv: no more than 1 parameter may be virtual\n");     /* Not allowed */
 528         }
 529         if ((which & (cppvPsrc | cppvPsnk)) == (cppvPsrc | cppvPsnk)) {
 530                 bothphys = 1;                                                   /* both are physical */
 531         }
 532         while (size) {
 533                 if (bothphys) {
 534                         lop = (unsigned int)(PAGE_SIZE - (snk64 & (PAGE_SIZE - 1)));            /* Assume sink smallest */
 535
 536                         if (lop > (unsigned int)(PAGE_SIZE - (src64 & (PAGE_SIZE - 1)))) {
 537                                 lop = (unsigned int)(PAGE_SIZE - (src64 & (PAGE_SIZE - 1)));    /* No, source is smaller */
 538                         }
 539                 } else {
 540                         /*
 541                          * only need to compute the resid for the physical page
 542                          * address... we don't care about where we start/finish in
 543                          * the virtual since we just call the normal copyin/copyout
 544                          */
 545                         if (which & cppvPsrc) {
 546                                 lop = (unsigned int)(PAGE_SIZE - (src64 & (PAGE_SIZE - 1)));
 547                         } else {
 548                                 lop = (unsigned int)(PAGE_SIZE - (snk64 & (PAGE_SIZE - 1)));
 549                         }
 550                 }
 551                 csize = size;                                           /* Assume we can copy it all */
 552                 if (lop < size) {
 553                         csize = lop;                                    /* Nope, we can't do it all */
 554                 }
 555 #if 0
 556                 /*
 557                  * flush_dcache64 is currently a nop on the i386...
 558                  * it's used when copying to non-system memory such
 559                  * as video capture cards... on PPC there was a need
 560                  * to flush due to how we mapped this memory... not
 561                  * sure if it's needed on i386.
 562                  */
 563                 if (which & cppvFsrc) {
 564                         flush_dcache64(src64, csize, 1);                /* If requested, flush source before move */
 565                 }
 566                 if (which & cppvFsnk) {
 567                         flush_dcache64(snk64, csize, 1);                /* If requested, flush sink before move */
 568                 }
 569 #endif
 570                 if (bothphys) {
 571                         bcopy_phys(src64, snk64, csize);                /* Do a physical copy, virtually */
 572                 } else {
 573                         if (copyio_phys(src64, snk64, csize, which)) {
 574                                 return KERN_FAILURE;
 575                         }
 576                 }
 577 #if 0
 578                 if (which & cppvFsrc) {
 579                         flush_dcache64(src64, csize, 1);        /* If requested, flush source after move */
 580                 }
 581                 if (which & cppvFsnk) {
 582                         flush_dcache64(snk64, csize, 1);        /* If requested, flush sink after move */
 583                 }
 584 #endif
 585                 size   -= csize;                                        /* Calculate what is left */
 586                 snk64 += csize;                                 /* Bump sink to next physical address */
 587                 src64 += csize;                                 /* Bump source to next physical address */
 588         }
 589         KERNEL_DEBUG(0xeff7004c | DBG_FUNC_END, (unsigned)src64,
 590             (unsigned)snk64, size, which, 0);
 591
 592         return KERN_SUCCESS;
 593 }