osfmk/x86_64/copyio.c

   1 /*
   2  * Copyright (c) 2009-2020 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 #include <mach_assert.h>
  29
  30 #include <sys/errno.h>
  31 #include <i386/param.h>
  32 #include <i386/misc_protos.h>
  33 #include <i386/cpu_data.h>
  34 #include <i386/machine_cpu.h>
  35 #include <i386/machine_routines.h>
  36 #include <i386/cpuid.h>
  37 #include <i386/vmx.h>
  38 #include <vm/pmap.h>
  39 #include <vm/vm_map.h>
  40 #include <vm/vm_kern.h>
  41 #include <vm/vm_fault.h>
  42 #include <san/kasan.h>
  43
  44 #include <sys/kdebug.h>
  45
  46 #include <kern/copyout_shim.h>
  47 #include <kern/zalloc_internal.h>
  48
  49 #undef copyin
  50 #undef copyout
  51
  52 static int copyio(int, user_addr_t, char *, vm_size_t, vm_size_t *, int);
  53 static int copyio_phys(addr64_t, addr64_t, vm_size_t, int);
  54
  55 /*
  56  * Copy sizes bigger than this value will cause a kernel panic.
  57  *
  58  * Yes, this is an arbitrary fixed limit, but it's almost certainly
  59  * a programming error to be copying more than this amount between
  60  * user and wired kernel memory in a single invocation on this
  61  * platform.
  62  */
  63 const int copysize_limit_panic = (64 * MB);
  64
  65 /*
  66  * The copy engine has the following characteristics
  67  *   - copyio() handles copies to/from user or kernel space
  68  *   - copypv() deals with physical or virtual addresses
  69  *
  70  * Readers familiar with the 32-bit kernel will expect Joe's thesis at this
  71  * point describing the full glory of the copy window implementation. In K64,
  72  * however, there is no need for windowing. Thanks to the vast shared address
  73  * space, the kernel has direct access to userspace and to physical memory.
  74  *
  75  * User virtual addresses are accessible provided the user's cr3 is loaded.
  76  * Physical addresses are accessible via the direct map and the PHYSMAP_PTOV()
  77  * translation.
  78  *
  79  * Copyin/out variants all boil done to just these 2 routines in locore.s which
  80  * provide fault-recoverable copying:
  81  */
  82 extern int _bcopy(const void *, void *, vm_size_t);
  83 extern int _bcopystr(const void *, void *, vm_size_t, vm_size_t *);
  84 extern int _copyin_atomic32(const char *src, uint32_t *dst);
  85 extern int _copyin_atomic64(const char *src, uint64_t *dst);
  86 extern int _copyout_atomic32(const uint32_t *u32, char *src);
  87 extern int _copyout_atomic64(const uint64_t *u64, char *src);
  88
  89 /*
  90  * Types of copies:
  91  */
  92 #define COPYIN          0       /* from user virtual to kernel virtual */
  93 #define COPYOUT         1       /* from kernel virtual to user virtual */
  94 #define COPYINSTR       2       /* string variant of copyout */
  95 #define COPYINPHYS      3       /* from user virtual to kernel physical */
  96 #define COPYOUTPHYS     4       /* from kernel physical to user virtual */
  97 #define COPYINATOMIC32  5       /* from user virtual to kernel virtual */
  98 #define COPYINATOMIC64  6       /* from user virtual to kernel virtual */
  99 #define COPYOUTATOMIC32 7       /* from user virtual to kernel virtual */
 100 #define COPYOUTATOMIC64 8       /* from user virtual to kernel virtual */
 101
 102 #if ENABLE_SMAPLOG
 103 typedef struct {
 104         uint64_t        timestamp;
 105         thread_t        thread;
 106         uintptr_t       cr4;
 107         uint8_t         cpuid;
 108         uint8_t         smap_state;
 109         uint8_t         copyio_active;
 110 } smaplog_entry_t;
 111
 112 #define SMAPLOG_BUFFER_SIZE (50)
 113 static smaplog_entry_t  smaplog_cbuf[SMAPLOG_BUFFER_SIZE];
 114 static uint32_t         smaplog_head = 0;
 115
 116 static void
 117 smaplog_add_entry(boolean_t enabling)
 118 {
 119         uint32_t index = 0;
 120         thread_t thread = current_thread();
 121
 122         do {
 123                 index = smaplog_head;
 124         } while (!OSCompareAndSwap(index, (index + 1) % SMAPLOG_BUFFER_SIZE, &smaplog_head));
 125
 126         assert(index < SMAPLOG_BUFFER_SIZE);
 127         assert(smaplog_head < SMAPLOG_BUFFER_SIZE);
 128         assert(thread);
 129
 130         smaplog_cbuf[index].timestamp = mach_absolute_time();
 131         smaplog_cbuf[index].thread = thread;
 132         smaplog_cbuf[index].cpuid = cpu_number();
 133         smaplog_cbuf[index].cr4 = get_cr4();
 134         smaplog_cbuf[index].smap_state = enabling;
 135         smaplog_cbuf[index].copyio_active = (thread->machine.specFlags & CopyIOActive) ? 1 : 0;
 136 }
 137 #endif /* ENABLE_SMAPLOG */
 138
 139 extern boolean_t pmap_smap_enabled;
 140 static inline void
 141 user_access_enable(void)
 142 {
 143         if (pmap_smap_enabled) {
 144                 stac();
 145 #if ENABLE_SMAPLOG
 146                 smaplog_add_entry(TRUE);
 147 #endif
 148         }
 149 }
 150 static inline void
 151 user_access_disable(void)
 152 {
 153         if (pmap_smap_enabled) {
 154                 clac();
 155 #if ENABLE_SMAPLOG
 156                 smaplog_add_entry(FALSE);
 157 #endif
 158         }
 159 }
 160
 161 #if COPYIO_TRACE_ENABLED
 162 #define COPYIO_TRACE(x, a, b, c, d, e) KERNEL_DEBUG_CONSTANT(x, a, b, c, d, e)
 163 #else
 164 #define COPYIO_TRACE(x, a, b, c, d, e) do { } while(0)
 165 #endif
 166
 167 static int
 168 copyio(int copy_type, user_addr_t user_addr, char *kernel_addr,
 169     vm_size_t nbytes, vm_size_t *lencopied, int use_kernel_map)
 170 {
 171         thread_t        thread = current_thread();
 172         pmap_t          pmap;
 173         vm_size_t       bytes_copied;
 174         int             error = 0;
 175         boolean_t       istate = FALSE;
 176         boolean_t       recursive_CopyIOActive;
 177 #if     COPYIO_TRACE_ENABLED
 178         int             debug_type = 0xeff70010;
 179         debug_type += (copy_type << 2);
 180 #endif
 181         vm_size_t kernel_buf_size = 0;
 182
 183         if (__improbable(nbytes > copysize_limit_panic)) {
 184                 panic("%s(%p, %p, %lu) - transfer too large", __func__,
 185                     (void *)user_addr, (void *)kernel_addr, nbytes);
 186         }
 187
 188         COPYIO_TRACE(debug_type | DBG_FUNC_START,
 189             user_addr, kernel_addr, nbytes, use_kernel_map, 0);
 190
 191         if (__improbable(nbytes == 0)) {
 192                 goto out;
 193         }
 194
 195         pmap = thread->map->pmap;
 196         boolean_t nopagezero = thread->map->pmap->pagezero_accessible;
 197
 198         if ((copy_type != COPYINPHYS) && (copy_type != COPYOUTPHYS)) {
 199                 if (__improbable((vm_offset_t)kernel_addr < VM_MIN_KERNEL_AND_KEXT_ADDRESS)) {
 200                         panic("Invalid copy parameter, copy type: %d, kernel address: %p", copy_type, kernel_addr);
 201                 }
 202                 if (__probable(!zalloc_disable_copyio_check)) {
 203                         zone_t src_zone = NULL;
 204                         kernel_buf_size = zone_element_size(kernel_addr, &src_zone);
 205                         /*
 206                          * Size of elements in the permanent zone is not saved as a part of the
 207                          * zone's info
 208                          */
 209                         if (__improbable(src_zone && !src_zone->permanent &&
 210                             kernel_buf_size < nbytes)) {
 211                                 panic("copyio: kernel buffer %p has size %lu < nbytes %lu", kernel_addr, kernel_buf_size, nbytes);
 212                         }
 213                 }
 214         }
 215
 216         /* Sanity and security check for addresses to/from a user */
 217
 218         if (__improbable(((pmap != kernel_pmap) && (use_kernel_map == 0)) &&
 219             ((nbytes && (user_addr + nbytes <= user_addr)) || ((user_addr + nbytes) > vm_map_max(thread->map))))) {
 220                 error = EFAULT;
 221                 goto out;
 222         }
 223
 224         if (copy_type >= COPYINATOMIC32 && copy_type <= COPYOUTATOMIC64) {
 225                 if (__improbable(pmap == kernel_pmap)) {
 226                         error = EFAULT;
 227                         goto out;
 228                 }
 229         }
 230
 231 #if KASAN
 232         switch (copy_type) {
 233         case COPYIN:
 234         case COPYINSTR:
 235         case COPYINATOMIC32:
 236         case COPYINATOMIC64:
 237                 __asan_storeN((uptr)kernel_addr, nbytes);
 238                 break;
 239         case COPYOUT:
 240         case COPYOUTATOMIC32:
 241         case COPYOUTATOMIC64:
 242                 __asan_loadN((uptr)kernel_addr, nbytes);
 243                 kasan_check_uninitialized((vm_address_t)kernel_addr, nbytes);
 244                 break;
 245         }
 246 #endif
 247
 248         /*
 249          * If the no_shared_cr3 boot-arg is set (true), the kernel runs on
 250          * its own pmap and cr3 rather than the user's -- so that wild accesses
 251          * from kernel or kexts can be trapped. So, during copyin and copyout,
 252          * we need to switch back to the user's map/cr3. The thread is flagged
 253          * "CopyIOActive" at this time so that if the thread is pre-empted,
 254          * we will later restore the correct cr3.
 255          */
 256         recursive_CopyIOActive = thread->machine.specFlags & CopyIOActive;
 257
 258         boolean_t pdswitch = no_shared_cr3 || nopagezero;
 259
 260         if (__improbable(pdswitch)) {
 261                 istate = ml_set_interrupts_enabled(FALSE);
 262                 if (nopagezero && pmap_pcid_ncpus) {
 263                         pmap_pcid_activate(pmap, cpu_number(), TRUE, TRUE);
 264                 } else if (get_cr3_base() != pmap->pm_cr3) {
 265                         set_cr3_raw(pmap->pm_cr3);
 266                 }
 267                 thread->machine.specFlags |= CopyIOActive;
 268         } else {
 269                 thread->machine.specFlags |= CopyIOActive;
 270         }
 271
 272         user_access_enable();
 273
 274 #if DEVELOPMENT || DEBUG
 275         /*
 276          * Ensure that we're running on the target thread's cr3.
 277          */
 278         if ((pmap != kernel_pmap) && !use_kernel_map &&
 279             (get_cr3_base() != pmap->pm_cr3)) {
 280                 panic("copyio(%d,%p,%p,%ld,%p,%d) cr3 is %p expects %p",
 281                     copy_type, (void *)user_addr, kernel_addr, nbytes, lencopied, use_kernel_map,
 282                     (void *) get_cr3_raw(), (void *) pmap->pm_cr3);
 283         }
 284 #endif
 285
 286         if (__improbable(pdswitch)) {
 287                 (void) ml_set_interrupts_enabled(istate);
 288         }
 289
 290         COPYIO_TRACE(0xeff70044 | DBG_FUNC_NONE, user_addr,
 291             kernel_addr, nbytes, 0, 0);
 292
 293         switch (copy_type) {
 294         case COPYIN:
 295                 error = _bcopy((const void *) user_addr,
 296                     kernel_addr,
 297                     nbytes);
 298                 break;
 299
 300         case COPYOUT:
 301                 error = _bcopy(kernel_addr,
 302                     (void *) user_addr,
 303                     nbytes);
 304                 break;
 305
 306         case COPYINPHYS:
 307                 error = _bcopy((const void *) user_addr,
 308                     PHYSMAP_PTOV(kernel_addr),
 309                     nbytes);
 310                 break;
 311
 312         case COPYOUTPHYS:
 313                 error = _bcopy((const void *) PHYSMAP_PTOV(kernel_addr),
 314                     (void *) user_addr,
 315                     nbytes);
 316                 break;
 317
 318         case COPYINATOMIC32:
 319                 error = _copyin_atomic32((const void *) user_addr,
 320                     (void *) kernel_addr);
 321                 break;
 322
 323         case COPYINATOMIC64:
 324                 error = _copyin_atomic64((const void *) user_addr,
 325                     (void *) kernel_addr);
 326                 break;
 327
 328         case COPYOUTATOMIC32:
 329                 error = _copyout_atomic32((const void *) kernel_addr,
 330                     (void *) user_addr);
 331                 break;
 332
 333         case COPYOUTATOMIC64:
 334                 error = _copyout_atomic64((const void *) kernel_addr,
 335                     (void *) user_addr);
 336                 break;
 337
 338         case COPYINSTR:
 339                 error = _bcopystr((const void *) user_addr,
 340                     kernel_addr,
 341                     (int) nbytes,
 342                     &bytes_copied);
 343
 344                 /*
 345                  * lencopied should be updated on success
 346                  * or ENAMETOOLONG...  but not EFAULT
 347                  */
 348                 if (error != EFAULT) {
 349                         *lencopied = bytes_copied;
 350                 }
 351
 352                 if (error) {
 353 #if KDEBUG
 354                         nbytes = *lencopied;
 355 #endif
 356                         break;
 357                 }
 358                 if (*(kernel_addr + bytes_copied - 1) == 0) {
 359                         /*
 360                          * we found a NULL terminator... we're done
 361                          */
 362 #if KDEBUG
 363                         nbytes = *lencopied;
 364 #endif
 365                         break;
 366                 } else {
 367                         /*
 368                          * no more room in the buffer and we haven't
 369                          * yet come across a NULL terminator
 370                          */
 371 #if KDEBUG
 372                         nbytes = *lencopied;
 373 #endif
 374                         error = ENAMETOOLONG;
 375                         break;
 376                 }
 377         }
 378
 379         user_access_disable();
 380
 381         if (__improbable(pdswitch)) {
 382                 istate = ml_set_interrupts_enabled(FALSE);
 383                 if (!recursive_CopyIOActive && (get_cr3_raw() != kernel_pmap->pm_cr3)) {
 384                         if (nopagezero && pmap_pcid_ncpus) {
 385                                 pmap_pcid_activate(pmap, cpu_number(), TRUE, FALSE);
 386                         } else {
 387                                 set_cr3_raw(kernel_pmap->pm_cr3);
 388                         }
 389                 }
 390
 391                 if (!recursive_CopyIOActive) {
 392                         thread->machine.specFlags &= ~CopyIOActive;
 393                 }
 394                 (void) ml_set_interrupts_enabled(istate);
 395         } else if (!recursive_CopyIOActive) {
 396                 thread->machine.specFlags &= ~CopyIOActive;
 397         }
 398
 399 out:
 400         COPYIO_TRACE(debug_type | DBG_FUNC_END, user_addr, kernel_addr, nbytes, error, 0);
 401
 402         return error;
 403 }
 404
 405
 406 static int
 407 copyio_phys(addr64_t source, addr64_t sink, vm_size_t csize, int which)
 408 {
 409         char        *paddr;
 410         user_addr_t vaddr;
 411         int         ctype;
 412
 413         if (which & cppvPsnk) {
 414                 paddr  = (char *)sink;
 415                 vaddr  = (user_addr_t)source;
 416                 ctype  = COPYINPHYS;
 417         } else {
 418                 paddr  = (char *)source;
 419                 vaddr  = (user_addr_t)sink;
 420                 ctype  = COPYOUTPHYS;
 421                 CALL_COPYOUT_SHIM_PHYS((void *)PHYSMAP_PTOV(source), sink, csize)
 422         }
 423         return copyio(ctype, vaddr, paddr, csize, NULL, which & cppvKmap);
 424 }
 425
 426 int
 427 copyinmsg(const user_addr_t user_addr, char *kernel_addr, mach_msg_size_t nbytes)
 428 {
 429         return copyio(COPYIN, user_addr, kernel_addr, nbytes, NULL, 0);
 430 }
 431
 432 int
 433 copyin(const user_addr_t user_addr, void *kernel_addr, vm_size_t nbytes)
 434 {
 435         return copyio(COPYIN, user_addr, kernel_addr, nbytes, NULL, 0);
 436 }
 437
 438 /*
 439  * copy{in,out}_atomic{32,64}
 440  * Read or store an aligned value from userspace as a single memory transaction.
 441  * These functions support userspace synchronization features
 442  */
 443 int
 444 copyin_atomic32(const user_addr_t user_addr, uint32_t *kernel_addr)
 445 {
 446         /* Test alignment */
 447         if (user_addr & 3) {
 448                 return EINVAL;
 449         }
 450         return copyio(COPYINATOMIC32, user_addr, (char *)(uintptr_t)kernel_addr, 4, NULL, 0);
 451 }
 452
 453 int
 454 copyin_atomic32_wait_if_equals(const user_addr_t user_addr, uint32_t value)
 455 {
 456         uint32_t u32;
 457         int result = copyin_atomic32(user_addr, &u32);
 458         if (__improbable(result)) {
 459                 return result;
 460         }
 461         if (u32 != value) {
 462                 return ESTALE;
 463         }
 464         cpu_pause();
 465         return 0;
 466 }
 467
 468 int
 469 copyin_atomic64(const user_addr_t user_addr, uint64_t *kernel_addr)
 470 {
 471         /* Test alignment */
 472         if (user_addr & 7) {
 473                 return EINVAL;
 474         }
 475         return copyio(COPYINATOMIC64, user_addr, (char *)(uintptr_t)kernel_addr, 8, NULL, 0);
 476 }
 477
 478 int
 479 copyout_atomic32(uint32_t value, user_addr_t user_addr)
 480 {
 481         /* Test alignment */
 482         if (user_addr & 3) {
 483                 return EINVAL;
 484         }
 485         return copyio(COPYOUTATOMIC32, user_addr, (char *)&value, 4, NULL, 0);
 486 }
 487
 488 int
 489 copyout_atomic64(uint64_t value, user_addr_t user_addr)
 490 {
 491         /* Test alignment */
 492         if (user_addr & 7) {
 493                 return EINVAL;
 494         }
 495         return copyio(COPYOUTATOMIC64, user_addr, (char *)&value, 8, NULL, 0);
 496 }
 497
 498 int
 499 copyinstr(const user_addr_t user_addr, char *kernel_addr, vm_size_t nbytes, vm_size_t *lencopied)
 500 {
 501         *lencopied = 0;
 502
 503         return copyio(COPYINSTR, user_addr, kernel_addr, nbytes, lencopied, 0);
 504 }
 505
 506 int
 507 copyoutmsg(const char *kernel_addr, user_addr_t user_addr, mach_msg_size_t nbytes)
 508 {
 509         CALL_COPYOUT_SHIM_MSG(kernel_addr, user_addr, (vm_size_t)nbytes)
 510         return copyio(COPYOUT, user_addr, (char *)(uintptr_t)kernel_addr, nbytes, NULL, 0);
 511 }
 512
 513 int
 514 copyout(const void *kernel_addr, user_addr_t user_addr, vm_size_t nbytes)
 515 {
 516         CALL_COPYOUT_SHIM_NRML(kernel_addr, user_addr, nbytes)
 517         return copyio(COPYOUT, user_addr, (char *)(uintptr_t)kernel_addr, nbytes, NULL, 0);
 518 }
 519
 520
 521 kern_return_t
 522 copypv(addr64_t src64, addr64_t snk64, unsigned int size, int which)
 523 {
 524         unsigned int lop, csize;
 525         int bothphys = 0;
 526
 527         KERNEL_DEBUG(0xeff7004c | DBG_FUNC_START, (unsigned)src64,
 528             (unsigned)snk64, size, which, 0);
 529
 530         if ((which & (cppvPsrc | cppvPsnk)) == 0) {                             /* Make sure that only one is virtual */
 531                 panic("copypv: no more than 1 parameter may be virtual\n");     /* Not allowed */
 532         }
 533         if ((which & (cppvPsrc | cppvPsnk)) == (cppvPsrc | cppvPsnk)) {
 534                 bothphys = 1;                                                   /* both are physical */
 535         }
 536         while (size) {
 537                 if (bothphys) {
 538                         lop = (unsigned int)(PAGE_SIZE - (snk64 & (PAGE_SIZE - 1)));            /* Assume sink smallest */
 539
 540                         if (lop > (unsigned int)(PAGE_SIZE - (src64 & (PAGE_SIZE - 1)))) {
 541                                 lop = (unsigned int)(PAGE_SIZE - (src64 & (PAGE_SIZE - 1)));    /* No, source is smaller */
 542                         }
 543                 } else {
 544                         /*
 545                          * only need to compute the resid for the physical page
 546                          * address... we don't care about where we start/finish in
 547                          * the virtual since we just call the normal copyin/copyout
 548                          */
 549                         if (which & cppvPsrc) {
 550                                 lop = (unsigned int)(PAGE_SIZE - (src64 & (PAGE_SIZE - 1)));
 551                         } else {
 552                                 lop = (unsigned int)(PAGE_SIZE - (snk64 & (PAGE_SIZE - 1)));
 553                         }
 554                 }
 555                 csize = size;                                           /* Assume we can copy it all */
 556                 if (lop < size) {
 557                         csize = lop;                                    /* Nope, we can't do it all */
 558                 }
 559 #if 0
 560                 /*
 561                  * flush_dcache64 is currently a nop on the i386...
 562                  * it's used when copying to non-system memory such
 563                  * as video capture cards... on PPC there was a need
 564                  * to flush due to how we mapped this memory... not
 565                  * sure if it's needed on i386.
 566                  */
 567                 if (which & cppvFsrc) {
 568                         flush_dcache64(src64, csize, 1);                /* If requested, flush source before move */
 569                 }
 570                 if (which & cppvFsnk) {
 571                         flush_dcache64(snk64, csize, 1);                /* If requested, flush sink before move */
 572                 }
 573 #endif
 574                 if (bothphys) {
 575                         bcopy_phys(src64, snk64, csize);                /* Do a physical copy, virtually */
 576                 } else {
 577                         if (copyio_phys(src64, snk64, csize, which)) {
 578                                 return KERN_FAILURE;
 579                         }
 580                 }
 581 #if 0
 582                 if (which & cppvFsrc) {
 583                         flush_dcache64(src64, csize, 1);        /* If requested, flush source after move */
 584                 }
 585                 if (which & cppvFsnk) {
 586                         flush_dcache64(snk64, csize, 1);        /* If requested, flush sink after move */
 587                 }
 588 #endif
 589                 size   -= csize;                                        /* Calculate what is left */
 590                 snk64 += csize;                                 /* Bump sink to next physical address */
 591                 src64 += csize;                                 /* Bump source to next physical address */
 592         }
 593         KERNEL_DEBUG(0xeff7004c | DBG_FUNC_END, (unsigned)src64,
 594             (unsigned)snk64, size, which, 0);
 595
 596         return KERN_SUCCESS;
 597 }