bsd/kern/kern_symfile.c

   1 /*
   2  * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 /* Copyright (c) 1998 Apple Computer, Inc.  All rights reserved.
  29  *
  30  *      File:   bsd/kern/kern_symfile.c
  31  *
  32  * HISTORY
  33  */
  34
  35 #include <mach/vm_param.h>
  36
  37 #include <sys/param.h>
  38 #include <sys/systm.h>
  39 #include <sys/signalvar.h>
  40 #include <sys/resourcevar.h>
  41 #include <sys/namei.h>
  42 #include <sys/vnode_internal.h>
  43 #include <sys/proc_internal.h>
  44 #include <sys/kauth.h>
  45 #include <sys/timeb.h>
  46 #include <sys/times.h>
  47 #include <sys/acct.h>
  48 #include <sys/file_internal.h>
  49 #include <sys/uio.h>
  50 #include <sys/kernel.h>
  51 #include <sys/stat.h>
  52 #include <sys/disk.h>
  53 #include <sys/conf.h>
  54 #include <sys/content_protection.h>
  55 #include <sys/fsctl.h>
  56
  57 #include <mach-o/loader.h>
  58 #include <mach-o/nlist.h>
  59
  60 #include <kern/kalloc.h>
  61 #include <vm/vm_kern.h>
  62 #include <pexpert/pexpert.h>
  63 #include <IOKit/IOPolledInterface.h>
  64
  65 #define HIBERNATE_MIN_PHYSICAL_LBA_512    (34)
  66 #define HIBERNATE_MIN_PHYSICAL_LBA_4096   (6)
  67 #define HIBERNATE_MIN_FILE_SIZE           (1024*1024)
  68
  69 /* This function is called from kern_sysctl in the current process context;
  70  * it is exported with the System6.0.exports, but this appears to be a legacy
  71  * export, as there are no internal consumers.
  72  */
  73 int
  74 get_kernel_symfile(__unused proc_t p, __unused char const **symfile);
  75 int
  76 get_kernel_symfile(__unused proc_t p, __unused char const **symfile)
  77 {
  78         return KERN_FAILURE;
  79 }
  80
  81 struct kern_direct_file_io_ref_t {
  82         vfs_context_t      ctx;
  83         struct vnode      * vp;
  84         dev_t               device;
  85         uint32_t            blksize;
  86         off_t               filelength;
  87         char                cf;
  88         char                pinned;
  89         char                frozen;
  90         char                wbcranged;
  91 };
  92
  93
  94 static int
  95 file_ioctl(void * p1, void * p2, u_long theIoctl, caddr_t result)
  96 {
  97         dev_t device = *(dev_t*) p1;
  98
  99         return (*bdevsw[major(device)].d_ioctl)
 100                (device, theIoctl, result, S_IFBLK, p2);
 101 }
 102
 103 static int
 104 device_ioctl(void * p1, __unused void * p2, u_long theIoctl, caddr_t result)
 105 {
 106         return VNOP_IOCTL(p1, theIoctl, result, 0, p2);
 107 }
 108
 109 static int
 110 kern_ioctl_file_extents(struct kern_direct_file_io_ref_t * ref, u_long theIoctl, off_t offset, off_t end)
 111 {
 112         int error = 0;
 113         int (*do_ioctl)(void * p1, void * p2, u_long theIoctl, caddr_t result);
 114         void * p1;
 115         void * p2;
 116         uint64_t    fileblk;
 117         size_t      filechunk;
 118         dk_extent_t  extent;
 119         dk_unmap_t   unmap;
 120         _dk_cs_pin_t pin;
 121
 122         bzero(&extent, sizeof(dk_extent_t));
 123         bzero(&unmap, sizeof(dk_unmap_t));
 124         bzero(&pin, sizeof(pin));
 125         if (ref->vp->v_type == VREG) {
 126                 p1 = &ref->device;
 127                 p2 = kernproc;
 128                 do_ioctl = &file_ioctl;
 129         } else {
 130                 /* Partition. */
 131                 p1 = ref->vp;
 132                 p2 = ref->ctx;
 133                 do_ioctl = &device_ioctl;
 134         }
 135
 136         if (_DKIOCCSPINEXTENT == theIoctl) {
 137                 /* Tell CS the image size, so it knows whether to place the subsequent pins SSD/HDD */
 138                 pin.cp_extent.length = end;
 139                 pin.cp_flags = _DKIOCCSHIBERNATEIMGSIZE;
 140                 (void) do_ioctl(p1, p2, _DKIOCCSPINEXTENT, (caddr_t)&pin);
 141         } else if (_DKIOCCSUNPINEXTENT == theIoctl) {
 142                 /* Tell CS hibernation is done, so it can stop blocking overlapping writes */
 143                 pin.cp_flags = _DKIOCCSPINDISCARDBLACKLIST;
 144                 (void) do_ioctl(p1, p2, _DKIOCCSUNPINEXTENT, (caddr_t)&pin);
 145         }
 146
 147         for (; offset < end; offset += filechunk) {
 148                 if (ref->vp->v_type == VREG) {
 149                         daddr64_t blkno;
 150                         filechunk = 1 * 1024 * 1024 * 1024;
 151                         if (filechunk > (size_t)(end - offset)) {
 152                                 filechunk = (size_t)(end - offset);
 153                         }
 154                         error = VNOP_BLOCKMAP(ref->vp, offset, filechunk, &blkno,
 155                             &filechunk, NULL, VNODE_WRITE | VNODE_BLOCKMAP_NO_TRACK, NULL);
 156                         if (error) {
 157                                 break;
 158                         }
 159                         if (-1LL == blkno) {
 160                                 continue;
 161                         }
 162                         fileblk = blkno * ref->blksize;
 163                 } else if ((ref->vp->v_type == VBLK) || (ref->vp->v_type == VCHR)) {
 164                         fileblk = offset;
 165                         filechunk = (unsigned long)((ref->filelength > ULONG_MAX) ? ULONG_MAX: ref->filelength);
 166                 }
 167
 168                 if (DKIOCUNMAP == theIoctl) {
 169                         extent.offset = fileblk;
 170                         extent.length = filechunk;
 171                         unmap.extents = &extent;
 172                         unmap.extentsCount = 1;
 173                         error = do_ioctl(p1, p2, theIoctl, (caddr_t)&unmap);
 174 //          printf("DKIOCUNMAP(%d) 0x%qx, 0x%qx\n", error, extent.offset, extent.length);
 175                 } else if (_DKIOCCSPINEXTENT == theIoctl) {
 176                         pin.cp_extent.offset = fileblk;
 177                         pin.cp_extent.length = filechunk;
 178                         pin.cp_flags = _DKIOCCSPINFORHIBERNATION;
 179                         error = do_ioctl(p1, p2, theIoctl, (caddr_t)&pin);
 180                         if (error && (ENOTTY != error)) {
 181                                 printf("_DKIOCCSPINEXTENT(%d) 0x%qx, 0x%qx\n", error, pin.cp_extent.offset, pin.cp_extent.length);
 182                         }
 183                 } else if (_DKIOCCSUNPINEXTENT == theIoctl) {
 184                         pin.cp_extent.offset = fileblk;
 185                         pin.cp_extent.length = filechunk;
 186                         pin.cp_flags = _DKIOCCSPINFORHIBERNATION;
 187                         error = do_ioctl(p1, p2, theIoctl, (caddr_t)&pin);
 188                         if (error && (ENOTTY != error)) {
 189                                 printf("_DKIOCCSUNPINEXTENT(%d) 0x%qx, 0x%qx\n", error, pin.cp_extent.offset, pin.cp_extent.length);
 190                         }
 191                 } else {
 192                         error = EINVAL;
 193                 }
 194
 195                 if (error) {
 196                         break;
 197                 }
 198         }
 199         return error;
 200 }
 201
 202 extern uint32_t freespace_mb(vnode_t vp);
 203
 204 struct kern_direct_file_io_ref_t *
 205 kern_open_file_for_direct_io(const char * name,
 206     uint32_t iflags,
 207     kern_get_file_extents_callback_t callback,
 208     void * callback_ref,
 209     off_t set_file_size,
 210     off_t fs_free_size,
 211     off_t write_file_offset,
 212     void * write_file_addr,
 213     size_t write_file_len,
 214     dev_t * partition_device_result,
 215     dev_t * image_device_result,
 216     uint64_t * partitionbase_result,
 217     uint64_t * maxiocount_result,
 218     uint32_t * oflags)
 219 {
 220         struct kern_direct_file_io_ref_t * ref;
 221
 222         proc_t            p;
 223         struct vnode_attr va;
 224         dk_apfs_wbc_range_t wbc_range;
 225         int               error;
 226         off_t             f_offset;
 227         uint64_t          fileblk;
 228         size_t            filechunk;
 229         uint64_t          physoffset, minoffset;
 230         dev_t             device;
 231         dev_t             target = 0;
 232         int               isssd = 0;
 233         uint32_t          flags = 0;
 234         uint32_t          blksize;
 235         off_t             maxiocount, count, segcount, wbctotal;
 236         boolean_t         locked = FALSE;
 237         int               fmode;
 238         mode_t                    cmode;
 239         struct            nameidata nd;
 240         u_int32_t         ndflags;
 241         off_t             mpFree;
 242
 243         int (*do_ioctl)(void * p1, void * p2, u_long theIoctl, caddr_t result);
 244         void * p1 = NULL;
 245         void * p2 = NULL;
 246
 247         error = EFAULT;
 248
 249         ref = (struct kern_direct_file_io_ref_t *) kalloc(sizeof(struct kern_direct_file_io_ref_t));
 250         if (!ref) {
 251                 error = EFAULT;
 252                 goto out;
 253         }
 254
 255         bzero(ref, sizeof(*ref));
 256         p = kernproc;
 257         ref->ctx = vfs_context_kernel();
 258
 259         fmode  = (kIOPolledFileCreate & iflags) ? (O_CREAT | FWRITE) : FWRITE;
 260         cmode =  S_IRUSR | S_IWUSR;
 261         ndflags = NOFOLLOW;
 262         NDINIT(&nd, LOOKUP, OP_OPEN, ndflags, UIO_SYSSPACE, CAST_USER_ADDR_T(name), ref->ctx);
 263         VATTR_INIT(&va);
 264         VATTR_SET(&va, va_mode, cmode);
 265         VATTR_SET(&va, va_dataprotect_flags, VA_DP_RAWENCRYPTED);
 266         VATTR_SET(&va, va_dataprotect_class, PROTECTION_CLASS_D);
 267         if ((error = vn_open_auth(&nd, &fmode, &va))) {
 268                 kprintf("vn_open_auth(fmode: %d, cmode: %d) failed with error: %d\n", fmode, cmode, error);
 269                 goto out;
 270         }
 271
 272         ref->vp = nd.ni_vp;
 273         if (ref->vp->v_type == VREG) {
 274                 vnode_lock_spin(ref->vp);
 275                 SET(ref->vp->v_flag, VSWAP);
 276                 vnode_unlock(ref->vp);
 277         }
 278
 279         if (write_file_addr && write_file_len) {
 280                 if ((error = kern_write_file(ref, write_file_offset, write_file_addr, write_file_len, IO_SKIP_ENCRYPTION))) {
 281                         kprintf("kern_write_file() failed with error: %d\n", error);
 282                         goto out;
 283                 }
 284         }
 285
 286         VATTR_INIT(&va);
 287         VATTR_WANTED(&va, va_rdev);
 288         VATTR_WANTED(&va, va_fsid);
 289         VATTR_WANTED(&va, va_devid);
 290         VATTR_WANTED(&va, va_data_size);
 291         VATTR_WANTED(&va, va_data_alloc);
 292         VATTR_WANTED(&va, va_nlink);
 293         error = EFAULT;
 294         if (vnode_getattr(ref->vp, &va, ref->ctx)) {
 295                 goto out;
 296         }
 297
 298         wbctotal = 0;
 299         mpFree = freespace_mb(ref->vp);
 300         mpFree <<= 20;
 301         kprintf("kern_direct_file(%s): vp size %qd, alloc %qd, mp free %qd, keep free %qd\n",
 302             name, va.va_data_size, va.va_data_alloc, mpFree, fs_free_size);
 303
 304         if (ref->vp->v_type == VREG) {
 305                 /* Don't dump files with links. */
 306                 if (va.va_nlink != 1) {
 307                         goto out;
 308                 }
 309
 310                 device = (VATTR_IS_SUPPORTED(&va, va_devid)) ? va.va_devid : va.va_fsid;
 311                 ref->filelength = va.va_data_size;
 312
 313                 p1 = &device;
 314                 p2 = p;
 315                 do_ioctl = &file_ioctl;
 316
 317                 if (kIOPolledFileHibernate & iflags) {
 318                         error = do_ioctl(p1, p2, DKIOCAPFSGETWBCRANGE, (caddr_t) &wbc_range);
 319                         ref->wbcranged = (error == 0);
 320                 }
 321                 if (ref->wbcranged) {
 322                         uint32_t idx;
 323                         assert(wbc_range.count <= (sizeof(wbc_range.extents) / sizeof(wbc_range.extents[0])));
 324                         for (idx = 0; idx < wbc_range.count; idx++) {
 325                                 wbctotal += wbc_range.extents[idx].length;
 326                         }
 327                         kprintf("kern_direct_file(%s): wbc %qd\n", name, wbctotal);
 328                         if (wbctotal) {
 329                                 target = wbc_range.dev;
 330                         }
 331                 }
 332
 333                 if (set_file_size) {
 334                         if (wbctotal) {
 335                                 if (wbctotal >= set_file_size) {
 336                                         set_file_size = HIBERNATE_MIN_FILE_SIZE;
 337                                 } else {
 338                                         set_file_size -= wbctotal;
 339                                         if (set_file_size < HIBERNATE_MIN_FILE_SIZE) {
 340                                                 set_file_size = HIBERNATE_MIN_FILE_SIZE;
 341                                         }
 342                                 }
 343                         }
 344                         if (fs_free_size) {
 345                                 mpFree += va.va_data_alloc;
 346                                 if ((mpFree < set_file_size) || ((mpFree - set_file_size) < fs_free_size)) {
 347                                         error = ENOSPC;
 348                                         goto out;
 349                                 }
 350                         }
 351                         error = vnode_setsize(ref->vp, set_file_size, IO_NOZEROFILL | IO_NOAUTH, ref->ctx);
 352                         if (error) {
 353                                 goto out;
 354                         }
 355                         ref->filelength = set_file_size;
 356                 }
 357         } else if ((ref->vp->v_type == VBLK) || (ref->vp->v_type == VCHR)) {
 358                 /* Partition. */
 359                 device = va.va_rdev;
 360
 361                 p1 = ref->vp;
 362                 p2 = ref->ctx;
 363                 do_ioctl = &device_ioctl;
 364         } else {
 365                 /* Don't dump to non-regular files. */
 366                 error = EFAULT;
 367                 goto out;
 368         }
 369         ref->device = device;
 370
 371         // probe for CF
 372         dk_corestorage_info_t cs_info;
 373         memset(&cs_info, 0, sizeof(dk_corestorage_info_t));
 374         error = do_ioctl(p1, p2, DKIOCCORESTORAGE, (caddr_t)&cs_info);
 375         ref->cf = (error == 0) && (cs_info.flags & DK_CORESTORAGE_ENABLE_HOTFILES);
 376
 377         // get block size
 378
 379         error = do_ioctl(p1, p2, DKIOCGETBLOCKSIZE, (caddr_t) &ref->blksize);
 380         if (error) {
 381                 goto out;
 382         }
 383
 384         if (ref->blksize == 4096) {
 385                 minoffset = HIBERNATE_MIN_PHYSICAL_LBA_4096 * ref->blksize;
 386         } else {
 387                 minoffset = HIBERNATE_MIN_PHYSICAL_LBA_512 * ref->blksize;
 388         }
 389
 390         if (ref->vp->v_type != VREG) {
 391                 error = do_ioctl(p1, p2, DKIOCGETBLOCKCOUNT, (caddr_t) &fileblk);
 392                 if (error) {
 393                         goto out;
 394                 }
 395                 ref->filelength = fileblk * ref->blksize;
 396         }
 397
 398         // pin logical extents, CS version
 399
 400         error = kern_ioctl_file_extents(ref, _DKIOCCSPINEXTENT, 0, ref->filelength);
 401         if (error && (ENOTTY != error)) {
 402                 goto out;
 403         }
 404         ref->pinned = (error == 0);
 405
 406         // pin logical extents, apfs version
 407
 408         error = VNOP_IOCTL(ref->vp, FSCTL_FREEZE_EXTENTS, NULL, 0, ref->ctx);
 409         if (error && (ENOTTY != error)) {
 410                 goto out;
 411         }
 412         ref->frozen = (error == 0);
 413
 414         // generate the block list
 415
 416         error = do_ioctl(p1, p2, DKIOCLOCKPHYSICALEXTENTS, NULL);
 417         if (error) {
 418                 goto out;
 419         }
 420         locked = TRUE;
 421
 422         f_offset = 0;
 423         for (; f_offset < ref->filelength; f_offset += filechunk) {
 424                 if (ref->vp->v_type == VREG) {
 425                         filechunk = 1 * 1024 * 1024 * 1024;
 426                         daddr64_t blkno;
 427
 428                         error = VNOP_BLOCKMAP(ref->vp, f_offset, filechunk, &blkno,
 429                             &filechunk, NULL, VNODE_WRITE | VNODE_BLOCKMAP_NO_TRACK, NULL);
 430                         if (error) {
 431                                 goto out;
 432                         }
 433                         if (-1LL == blkno) {
 434                                 continue;
 435                         }
 436                         fileblk = blkno * ref->blksize;
 437                 } else if ((ref->vp->v_type == VBLK) || (ref->vp->v_type == VCHR)) {
 438                         fileblk = f_offset;
 439                         filechunk = f_offset ? 0 : (unsigned long)ref->filelength;
 440                 }
 441
 442                 physoffset = 0;
 443                 while (physoffset < filechunk) {
 444                         dk_physical_extent_t getphysreq;
 445                         bzero(&getphysreq, sizeof(getphysreq));
 446
 447                         getphysreq.offset = fileblk + physoffset;
 448                         getphysreq.length = (filechunk - physoffset);
 449                         error = do_ioctl(p1, p2, DKIOCGETPHYSICALEXTENT, (caddr_t) &getphysreq);
 450                         if (error) {
 451                                 goto out;
 452                         }
 453                         if (!target) {
 454                                 target = getphysreq.dev;
 455                         } else if (target != getphysreq.dev) {
 456                                 error = ENOTSUP;
 457                                 goto out;
 458                         }
 459
 460                         assert(getphysreq.offset >= minoffset);
 461
 462 #if HIBFRAGMENT
 463                         uint64_t rev;
 464                         for (rev = 4096; rev <= getphysreq.length; rev += 4096) {
 465                                 callback(callback_ref, getphysreq.offset + getphysreq.length - rev, 4096);
 466                         }
 467 #else
 468                         callback(callback_ref, getphysreq.offset, getphysreq.length);
 469 #endif
 470                         physoffset += getphysreq.length;
 471                 }
 472         }
 473         if (ref->wbcranged) {
 474                 uint32_t idx;
 475                 for (idx = 0; idx < wbc_range.count; idx++) {
 476                         assert(wbc_range.extents[idx].offset >= minoffset);
 477                         callback(callback_ref, wbc_range.extents[idx].offset, wbc_range.extents[idx].length);
 478                 }
 479         }
 480         callback(callback_ref, 0ULL, 0ULL);
 481
 482         if (ref->vp->v_type == VREG) {
 483                 p1 = &target;
 484         } else {
 485                 p1 = &target;
 486                 p2 = p;
 487                 do_ioctl = &file_ioctl;
 488         }
 489
 490         // get partition base
 491
 492         if (partitionbase_result) {
 493                 error = do_ioctl(p1, p2, DKIOCGETBASE, (caddr_t) partitionbase_result);
 494                 if (error) {
 495                         goto out;
 496                 }
 497         }
 498
 499         // get block size & constraints
 500
 501         error = do_ioctl(p1, p2, DKIOCGETBLOCKSIZE, (caddr_t) &blksize);
 502         if (error) {
 503                 goto out;
 504         }
 505
 506         maxiocount = 1 * 1024 * 1024 * 1024;
 507
 508         error = do_ioctl(p1, p2, DKIOCGETMAXBLOCKCOUNTREAD, (caddr_t) &count);
 509         if (error) {
 510                 count = 0;
 511         }
 512         count *= blksize;
 513         if (count && (count < maxiocount)) {
 514                 maxiocount = count;
 515         }
 516
 517         error = do_ioctl(p1, p2, DKIOCGETMAXBLOCKCOUNTWRITE, (caddr_t) &count);
 518         if (error) {
 519                 count = 0;
 520         }
 521         count *= blksize;
 522         if (count && (count < maxiocount)) {
 523                 maxiocount = count;
 524         }
 525
 526         error = do_ioctl(p1, p2, DKIOCGETMAXBYTECOUNTREAD, (caddr_t) &count);
 527         if (error) {
 528                 count = 0;
 529         }
 530         if (count && (count < maxiocount)) {
 531                 maxiocount = count;
 532         }
 533
 534         error = do_ioctl(p1, p2, DKIOCGETMAXBYTECOUNTWRITE, (caddr_t) &count);
 535         if (error) {
 536                 count = 0;
 537         }
 538         if (count && (count < maxiocount)) {
 539                 maxiocount = count;
 540         }
 541
 542         error = do_ioctl(p1, p2, DKIOCGETMAXSEGMENTBYTECOUNTREAD, (caddr_t) &count);
 543         if (!error) {
 544                 error = do_ioctl(p1, p2, DKIOCGETMAXSEGMENTCOUNTREAD, (caddr_t) &segcount);
 545         }
 546         if (error) {
 547                 count = segcount = 0;
 548         }
 549         count *= segcount;
 550         if (count && (count < maxiocount)) {
 551                 maxiocount = count;
 552         }
 553
 554         error = do_ioctl(p1, p2, DKIOCGETMAXSEGMENTBYTECOUNTWRITE, (caddr_t) &count);
 555         if (!error) {
 556                 error = do_ioctl(p1, p2, DKIOCGETMAXSEGMENTCOUNTWRITE, (caddr_t) &segcount);
 557         }
 558         if (error) {
 559                 count = segcount = 0;
 560         }
 561         count *= segcount;
 562         if (count && (count < maxiocount)) {
 563                 maxiocount = count;
 564         }
 565
 566         kprintf("max io 0x%qx bytes\n", maxiocount);
 567         if (maxiocount_result) {
 568                 *maxiocount_result = maxiocount;
 569         }
 570
 571         error = do_ioctl(p1, p2, DKIOCISSOLIDSTATE, (caddr_t)&isssd);
 572         if (!error && isssd) {
 573                 flags |= kIOPolledFileSSD;
 574         }
 575
 576         if (partition_device_result) {
 577                 *partition_device_result = device;
 578         }
 579         if (image_device_result) {
 580                 *image_device_result = target;
 581         }
 582         if (oflags) {
 583                 *oflags = flags;
 584         }
 585
 586         if ((ref->vp->v_type == VBLK) || (ref->vp->v_type == VCHR)) {
 587                 vnode_close(ref->vp, FWRITE, ref->ctx);
 588                 ref->vp = NULLVP;
 589                 ref->ctx = NULL;
 590         }
 591
 592 out:
 593         printf("kern_open_file_for_direct_io(%p, %d)\n", ref, error);
 594
 595
 596         if (error && locked) {
 597                 p1 = &device;
 598                 (void) do_ioctl(p1, p2, DKIOCUNLOCKPHYSICALEXTENTS, NULL);
 599         }
 600
 601         if (error && ref) {
 602                 if (ref->vp) {
 603                         (void) kern_ioctl_file_extents(ref, _DKIOCCSUNPINEXTENT, 0, (ref->pinned && ref->cf) ? ref->filelength : 0);
 604
 605                         if (ref->frozen) {
 606                                 (void) VNOP_IOCTL(ref->vp, FSCTL_THAW_EXTENTS, NULL, 0, ref->ctx);
 607                         }
 608                         if (ref->wbcranged) {
 609                                 (void) do_ioctl(p1, p2, DKIOCAPFSRELEASEWBCRANGE, (caddr_t) NULL);
 610                         }
 611                         vnode_close(ref->vp, FWRITE, ref->ctx);
 612                         ref->vp = NULLVP;
 613                 }
 614                 ref->ctx = NULL;
 615                 kfree(ref, sizeof(struct kern_direct_file_io_ref_t));
 616                 ref = NULL;
 617         }
 618
 619         return ref;
 620 }
 621
 622 int
 623 kern_write_file(struct kern_direct_file_io_ref_t * ref, off_t offset, void * addr, size_t len, int ioflag)
 624 {
 625         assert(len <= INT32_MAX);
 626         return vn_rdwr(UIO_WRITE, ref->vp,
 627                    addr, (int)len, offset,
 628                    UIO_SYSSPACE, ioflag | IO_SYNC | IO_NODELOCKED | IO_UNIT,
 629                    vfs_context_ucred(ref->ctx), (int *) 0,
 630                    vfs_context_proc(ref->ctx));
 631 }
 632
 633 int
 634 kern_read_file(struct kern_direct_file_io_ref_t * ref, off_t offset, void * addr, size_t len, int ioflag)
 635 {
 636         assert(len <= INT32_MAX);
 637         return vn_rdwr(UIO_READ, ref->vp,
 638                    addr, (int)len, offset,
 639                    UIO_SYSSPACE, ioflag | IO_SYNC | IO_NODELOCKED | IO_UNIT,
 640                    vfs_context_ucred(ref->ctx), (int *) 0,
 641                    vfs_context_proc(ref->ctx));
 642 }
 643
 644
 645 struct mount *
 646 kern_file_mount(struct kern_direct_file_io_ref_t * ref)
 647 {
 648         return ref->vp->v_mount;
 649 }
 650
 651 void
 652 kern_close_file_for_direct_io(struct kern_direct_file_io_ref_t * ref,
 653     off_t write_offset, void * addr, size_t write_length,
 654     off_t discard_offset, off_t discard_end)
 655 {
 656         int error;
 657         printf("kern_close_file_for_direct_io(%p)\n", ref);
 658
 659         if (!ref) {
 660                 return;
 661         }
 662
 663         if (ref->vp) {
 664                 int (*do_ioctl)(void * p1, void * p2, u_long theIoctl, caddr_t result);
 665                 void * p1;
 666                 void * p2;
 667
 668                 discard_offset = ((discard_offset + ref->blksize - 1) & ~(((off_t) ref->blksize) - 1));
 669                 discard_end    = ((discard_end)                       & ~(((off_t) ref->blksize) - 1));
 670
 671                 if (ref->vp->v_type == VREG) {
 672                         p1 = &ref->device;
 673                         p2 = kernproc;
 674                         do_ioctl = &file_ioctl;
 675                 } else {
 676                         /* Partition. */
 677                         p1 = ref->vp;
 678                         p2 = ref->ctx;
 679                         do_ioctl = &device_ioctl;
 680                 }
 681                 (void) do_ioctl(p1, p2, DKIOCUNLOCKPHYSICALEXTENTS, NULL);
 682
 683                 //XXX If unmapping extents then don't also need to unpin; except ...
 684                 //XXX if file unaligned (HFS 4k / Fusion 128k) then pin is superset and
 685                 //XXX unmap is subset, so save extra walk over file extents (and the risk
 686                 //XXX that CF drain starts) vs leaving partial units pinned to SSD
 687                 //XXX (until whatever was sharing also unmaps).  Err on cleaning up fully.
 688                 boolean_t will_unmap = (!ref->pinned || ref->cf) && (discard_end > discard_offset);
 689                 boolean_t will_unpin = (ref->pinned && ref->cf /* && !will_unmap */);
 690
 691                 (void) kern_ioctl_file_extents(ref, _DKIOCCSUNPINEXTENT, 0, (will_unpin) ? ref->filelength : 0);
 692
 693                 if (will_unmap) {
 694                         (void) kern_ioctl_file_extents(ref, DKIOCUNMAP, discard_offset, (ref->cf) ? ref->filelength : discard_end);
 695                 }
 696
 697                 if (ref->frozen) {
 698                         (void) VNOP_IOCTL(ref->vp, FSCTL_THAW_EXTENTS, NULL, 0, ref->ctx);
 699                 }
 700                 if (ref->wbcranged) {
 701                         (void) do_ioctl(p1, p2, DKIOCAPFSRELEASEWBCRANGE, (caddr_t) NULL);
 702                 }
 703
 704                 if (addr && write_length) {
 705                         (void) kern_write_file(ref, write_offset, addr, write_length, IO_SKIP_ENCRYPTION);
 706                 }
 707
 708                 error = vnode_close(ref->vp, FWRITE, ref->ctx);
 709
 710                 ref->vp = NULLVP;
 711                 kprintf("vnode_close(%d)\n", error);
 712
 713         }
 714
 715         ref->ctx = NULL;
 716
 717         kfree(ref, sizeof(struct kern_direct_file_io_ref_t));
 718 }