[apple/xnu.git] / bsd / kern / kern_symfile.c

/*
 * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 * 
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */
/* Copyright (c) 1998 Apple Computer, Inc.  All rights reserved.
 *
 *	File:	bsd/kern/kern_symfile.c
 *
 * HISTORY
 */

#include <mach/vm_param.h>

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/signalvar.h>
#include <sys/resourcevar.h>
#include <sys/namei.h>
#include <sys/vnode_internal.h>
#include <sys/proc_internal.h>
#include <sys/kauth.h>
#include <sys/timeb.h>
#include <sys/times.h>
#include <sys/acct.h>
#include <sys/file_internal.h>
#include <sys/uio.h>
#include <sys/kernel.h>
#include <sys/stat.h>
#include <sys/disk.h>
#include <sys/conf.h>
#include <sys/content_protection.h>

#include <mach-o/loader.h>
#include <mach-o/nlist.h>

#include <kern/kalloc.h>
#include <vm/vm_kern.h>
#include <pexpert/pexpert.h>
#include <IOKit/IOPolledInterface.h>

/* This function is called from kern_sysctl in the current process context;
 * it is exported with the System6.0.exports, but this appears to be a legacy
 * export, as there are no internal consumers.
 */
int
get_kernel_symfile(__unused proc_t p, __unused char const **symfile);
int
get_kernel_symfile(__unused proc_t p, __unused char const **symfile)
{
    return KERN_FAILURE;
}

struct kern_direct_file_io_ref_t
{
    vfs_context_t  ctx;
    struct vnode * vp;
    dev_t          device;
    uint32_t	   blksize;
    off_t          filelength;
    char           cf;
    char           pinned;
};


static int file_ioctl(void * p1, void * p2, u_long theIoctl, caddr_t result)
{
    dev_t device = *(dev_t*) p1;

    return ((*bdevsw[major(device)].d_ioctl)
		    (device, theIoctl, result, S_IFBLK, p2));
}

static int device_ioctl(void * p1, __unused void * p2, u_long theIoctl, caddr_t result)
{
    return (VNOP_IOCTL(p1, theIoctl, result, 0, p2));
}

static int
kern_ioctl_file_extents(struct kern_direct_file_io_ref_t * ref, u_long theIoctl, off_t offset, off_t end)
{
    int error = 0;
    int (*do_ioctl)(void * p1, void * p2, u_long theIoctl, caddr_t result);
    void * p1;
    void * p2;
    uint64_t    fileblk;
    size_t      filechunk;
    dk_extent_t  extent;
    dk_unmap_t   unmap;
    _dk_cs_pin_t pin;

    bzero(&extent, sizeof(dk_extent_t));
    bzero(&unmap, sizeof(dk_unmap_t));
    bzero(&pin, sizeof(pin));
    if (ref->vp->v_type == VREG)
    {
	 p1 = &ref->device;
	 p2 = kernproc;
	 do_ioctl = &file_ioctl;
    }
    else
    {
	/* Partition. */
	p1 = ref->vp;
	p2 = ref->ctx;
	do_ioctl = &device_ioctl;
    }

    if (_DKIOCCSPINEXTENT == theIoctl) {
	    /* Tell CS the image size, so it knows whether to place the subsequent pins SSD/HDD */
	    pin.cp_extent.length = end;
	    pin.cp_flags = _DKIOCCSHIBERNATEIMGSIZE;
	    (void) do_ioctl(p1, p2, _DKIOCCSPINEXTENT, (caddr_t)&pin);
    } else if (_DKIOCCSUNPINEXTENT == theIoctl) {
	    /* Tell CS hibernation is done, so it can stop blocking overlapping writes */
	    pin.cp_flags = _DKIOCCSPINDISCARDBLACKLIST;
	    (void) do_ioctl(p1, p2, _DKIOCCSUNPINEXTENT, (caddr_t)&pin);
    }

    for (; offset < end; offset += filechunk)
    {
        if (ref->vp->v_type == VREG)
        {
            daddr64_t blkno;
	    filechunk = 1*1024*1024*1024;
	    if (filechunk > (size_t)(end - offset))
	    filechunk = (size_t)(end - offset);
            error = VNOP_BLOCKMAP(ref->vp, offset, filechunk, &blkno,
								  &filechunk, NULL, VNODE_WRITE | VNODE_BLOCKMAP_NO_TRACK, NULL);
            if (error) break;
            if (-1LL == blkno) continue;
            fileblk = blkno * ref->blksize;
        }
        else if ((ref->vp->v_type == VBLK) || (ref->vp->v_type == VCHR))
        {
            fileblk = offset;
            filechunk = ref->filelength;
        }

	if (DKIOCUNMAP == theIoctl)
	{
	    extent.offset = fileblk;
	    extent.length = filechunk;
	    unmap.extents = &extent;
	    unmap.extentsCount = 1;
	    error = do_ioctl(p1, p2, theIoctl, (caddr_t)&unmap);
// 	    printf("DKIOCUNMAP(%d) 0x%qx, 0x%qx\n", error, extent.offset, extent.length);
	}
	else if (_DKIOCCSPINEXTENT == theIoctl)
	{
	    pin.cp_extent.offset = fileblk;
	    pin.cp_extent.length = filechunk;
	    pin.cp_flags = _DKIOCCSPINFORHIBERNATION;
	    error = do_ioctl(p1, p2, theIoctl, (caddr_t)&pin);
	    if (error && (ENOTTY != error))
	    {
		printf("_DKIOCCSPINEXTENT(%d) 0x%qx, 0x%qx\n", error, pin.cp_extent.offset, pin.cp_extent.length);
	    }
	}
	else if (_DKIOCCSUNPINEXTENT == theIoctl)
	{
	    pin.cp_extent.offset = fileblk;
	    pin.cp_extent.length = filechunk;
	    pin.cp_flags = _DKIOCCSPINFORHIBERNATION;
	    error = do_ioctl(p1, p2, theIoctl, (caddr_t)&pin);
	    if (error && (ENOTTY != error))
	    {
		printf("_DKIOCCSUNPINEXTENT(%d) 0x%qx, 0x%qx\n", error, pin.cp_extent.offset, pin.cp_extent.length);
	    }
	}
	else error = EINVAL;

        if (error) break;
    }
    return (error);
}

extern uint32_t freespace_mb(vnode_t vp);

struct kern_direct_file_io_ref_t *
kern_open_file_for_direct_io(const char * name, 
			     boolean_t create_file,
			     kern_get_file_extents_callback_t callback, 
			     void * callback_ref,
                             off_t set_file_size,
                             off_t fs_free_size,
                             off_t write_file_offset,
                             void * write_file_addr,
                             size_t write_file_len,
			     dev_t * partition_device_result,
			     dev_t * image_device_result,
                             uint64_t * partitionbase_result,
                             uint64_t * maxiocount_result,
                             uint32_t * oflags)
{
    struct kern_direct_file_io_ref_t * ref;

    proc_t            p;
    struct vnode_attr va;
    int               error;
    off_t             f_offset;
    uint64_t          fileblk;
    size_t            filechunk;
    uint64_t          physoffset;
    dev_t             device;
    dev_t             target = 0;
    int               isssd = 0;
    uint32_t          flags = 0;
    uint32_t          blksize;
    off_t             maxiocount, count, segcount;
    boolean_t         locked = FALSE;
    int               fmode, cmode;
    struct            nameidata nd;
    u_int32_t         ndflags;
    off_t             mpFree;

    int (*do_ioctl)(void * p1, void * p2, u_long theIoctl, caddr_t result);
    void * p1 = NULL;
    void * p2 = NULL;

    error = EFAULT;

    ref = (struct kern_direct_file_io_ref_t *) kalloc(sizeof(struct kern_direct_file_io_ref_t));
    if (!ref)
    {
	error = EFAULT;
    	goto out;
    }

    bzero(ref, sizeof(*ref));
    p = kernproc;
    ref->ctx = vfs_context_kernel();

    fmode  = (create_file) ? (O_CREAT | FWRITE) : FWRITE;
    cmode =  S_IRUSR | S_IWUSR;
    ndflags = NOFOLLOW;
    NDINIT(&nd, LOOKUP, OP_OPEN, ndflags, UIO_SYSSPACE, CAST_USER_ADDR_T(name), ref->ctx);
    VATTR_INIT(&va);
    VATTR_SET(&va, va_mode, cmode);
    VATTR_SET(&va, va_dataprotect_flags, VA_DP_RAWENCRYPTED);
    VATTR_SET(&va, va_dataprotect_class, PROTECTION_CLASS_D);
    if ((error = vn_open_auth(&nd, &fmode, &va))) {
	kprintf("vn_open_auth(fmode: %d, cmode: %d) failed with error: %d\n", fmode, cmode, error);
	goto out;
    }

    ref->vp = nd.ni_vp;
    if (ref->vp->v_type == VREG)
    {
        vnode_lock_spin(ref->vp);
        SET(ref->vp->v_flag, VSWAP);
        vnode_unlock(ref->vp);
    }

    if (write_file_addr && write_file_len)
    {
	if ((error = kern_write_file(ref, write_file_offset, write_file_addr, write_file_len, IO_SKIP_ENCRYPTION))) {
		kprintf("kern_write_file() failed with error: %d\n", error);
		goto out;
	}
    }

    VATTR_INIT(&va);
    VATTR_WANTED(&va, va_rdev);
    VATTR_WANTED(&va, va_fsid);
    VATTR_WANTED(&va, va_devid);
    VATTR_WANTED(&va, va_data_size);
    VATTR_WANTED(&va, va_data_alloc);
    VATTR_WANTED(&va, va_nlink);
    error = EFAULT;
    if (vnode_getattr(ref->vp, &va, ref->ctx)) goto out;

    mpFree = freespace_mb(ref->vp);
    mpFree <<= 20;
    kprintf("kern_direct_file(%s): vp size %qd, alloc %qd, mp free %qd, keep free %qd\n", 
    		name, va.va_data_size, va.va_data_alloc, mpFree, fs_free_size);

    if (ref->vp->v_type == VREG)
    {
        /* Don't dump files with links. */
        if (va.va_nlink != 1) goto out;

        device = (VATTR_IS_SUPPORTED(&va, va_devid)) ? va.va_devid : va.va_fsid;
        ref->filelength = va.va_data_size;

        p1 = &device;
        p2 = p;
        do_ioctl = &file_ioctl;

        if (set_file_size)
        {
            if (fs_free_size)
            {
		mpFree += va.va_data_alloc;
		if ((mpFree < set_file_size) || ((mpFree - set_file_size) < fs_free_size))
		{
		    error = ENOSPC;
		    goto out;
		}
	    }
	    error = vnode_setsize(ref->vp, set_file_size, IO_NOZEROFILL | IO_NOAUTH, ref->ctx);
	    if (error) goto out;
	    ref->filelength = set_file_size;
        }
    }
    else if ((ref->vp->v_type == VBLK) || (ref->vp->v_type == VCHR))
    {
	/* Partition. */
        device = va.va_rdev;

        p1 = ref->vp;
        p2 = ref->ctx;
        do_ioctl = &device_ioctl;
    }
    else
    {
	/* Don't dump to non-regular files. */
        error = EFAULT;
        goto out;
    }
    ref->device = device;

    // probe for CF
    dk_corestorage_info_t cs_info;
    memset(&cs_info, 0, sizeof(dk_corestorage_info_t));
    error = do_ioctl(p1, p2, DKIOCCORESTORAGE, (caddr_t)&cs_info);
    ref->cf = (error == 0) && (cs_info.flags & DK_CORESTORAGE_ENABLE_HOTFILES);

    // get block size

    error = do_ioctl(p1, p2, DKIOCGETBLOCKSIZE, (caddr_t) &ref->blksize);
    if (error)
        goto out;

    if (ref->vp->v_type != VREG)
    {
        error = do_ioctl(p1, p2, DKIOCGETBLOCKCOUNT, (caddr_t) &fileblk);
        if (error) goto out;
	ref->filelength = fileblk * ref->blksize;    
    }

    // pin logical extents

    error = kern_ioctl_file_extents(ref, _DKIOCCSPINEXTENT, 0, ref->filelength);
    if (error && (ENOTTY != error)) goto out;
    ref->pinned = (error == 0);

    // generate the block list

    error = do_ioctl(p1, p2, DKIOCLOCKPHYSICALEXTENTS, NULL);
    if (error) goto out;
    locked = TRUE;

    f_offset = 0;
    for (; f_offset < ref->filelength; f_offset += filechunk)
    {
        if (ref->vp->v_type == VREG)
        {
            filechunk = 1*1024*1024*1024;
            daddr64_t blkno;

            error = VNOP_BLOCKMAP(ref->vp, f_offset, filechunk, &blkno,
								  &filechunk, NULL, VNODE_WRITE | VNODE_BLOCKMAP_NO_TRACK, NULL);
            if (error) goto out;
            if (-1LL == blkno) continue;
            fileblk = blkno * ref->blksize;
        }
        else if ((ref->vp->v_type == VBLK) || (ref->vp->v_type == VCHR))
        {
            fileblk = f_offset;
            filechunk = f_offset ? 0 : ref->filelength;
        }

        physoffset = 0;
        while (physoffset < filechunk)
        {
            dk_physical_extent_t getphysreq;
            bzero(&getphysreq, sizeof(getphysreq));

            getphysreq.offset = fileblk + physoffset;
            getphysreq.length = (filechunk - physoffset);
            error = do_ioctl(p1, p2, DKIOCGETPHYSICALEXTENT, (caddr_t) &getphysreq);
            if (error) goto out;
            if (!target)
            {
                target = getphysreq.dev;
            }
            else if (target != getphysreq.dev)
            {
                error = ENOTSUP;
                goto out;
            }
#if HIBFRAGMENT
	    uint64_t rev;
	    for (rev = 4096; rev <= getphysreq.length; rev += 4096)
	    {
		callback(callback_ref, getphysreq.offset + getphysreq.length - rev, 4096);
	    }
#else
            callback(callback_ref, getphysreq.offset, getphysreq.length);
#endif
            physoffset += getphysreq.length;
        }
    }
    callback(callback_ref, 0ULL, 0ULL);

    if (ref->vp->v_type == VREG) p1 = &target;
    else
    {
	p1 = &target;
	p2 = p;
	do_ioctl = &file_ioctl;
    }

    // get partition base

    if (partitionbase_result) 
    {
        error = do_ioctl(p1, p2, DKIOCGETBASE, (caddr_t) partitionbase_result);
        if (error)
            goto out;
    }

    // get block size & constraints

    error = do_ioctl(p1, p2, DKIOCGETBLOCKSIZE, (caddr_t) &blksize);
    if (error)
        goto out;

    maxiocount = 1*1024*1024*1024;

    error = do_ioctl(p1, p2, DKIOCGETMAXBLOCKCOUNTREAD, (caddr_t) &count);
    if (error)
        count = 0;
    count *= blksize;
    if (count && (count < maxiocount))
        maxiocount = count;

    error = do_ioctl(p1, p2, DKIOCGETMAXBLOCKCOUNTWRITE, (caddr_t) &count);
    if (error)
        count = 0;
    count *= blksize;
    if (count && (count < maxiocount))
        maxiocount = count;

    error = do_ioctl(p1, p2, DKIOCGETMAXBYTECOUNTREAD, (caddr_t) &count);
    if (error)
        count = 0;
    if (count && (count < maxiocount))
        maxiocount = count;

    error = do_ioctl(p1, p2, DKIOCGETMAXBYTECOUNTWRITE, (caddr_t) &count);
    if (error)
        count = 0;
    if (count && (count < maxiocount))
        maxiocount = count;

    error = do_ioctl(p1, p2, DKIOCGETMAXSEGMENTBYTECOUNTREAD, (caddr_t) &count);
    if (!error)
	error = do_ioctl(p1, p2, DKIOCGETMAXSEGMENTCOUNTREAD, (caddr_t) &segcount);
    if (error)
        count = segcount = 0;
    count *= segcount;
    if (count && (count < maxiocount))
        maxiocount = count;

    error = do_ioctl(p1, p2, DKIOCGETMAXSEGMENTBYTECOUNTWRITE, (caddr_t) &count);
    if (!error)
	error = do_ioctl(p1, p2, DKIOCGETMAXSEGMENTCOUNTWRITE, (caddr_t) &segcount);
    if (error)
        count = segcount = 0;
    count *= segcount;
    if (count && (count < maxiocount))
        maxiocount = count;

    kprintf("max io 0x%qx bytes\n", maxiocount);
    if (maxiocount_result)
        *maxiocount_result = maxiocount;

    error = do_ioctl(p1, p2, DKIOCISSOLIDSTATE, (caddr_t)&isssd);
    if (!error && isssd)
        flags |= kIOPolledFileSSD;

    if (partition_device_result)
        *partition_device_result = device;
    if (image_device_result)
        *image_device_result = target;
    if (oflags)
        *oflags = flags;

    if ((ref->vp->v_type == VBLK) || (ref->vp->v_type == VCHR))
    {
        vnode_close(ref->vp, FWRITE, ref->ctx);
        ref->vp = NULLVP;
	ref->ctx = NULL;
    }

out:
    printf("kern_open_file_for_direct_io(%p, %d)\n", ref, error);


    if (error && locked)
    {
        p1 = &device;
        (void) do_ioctl(p1, p2, DKIOCUNLOCKPHYSICALEXTENTS, NULL);
    }

    if (error && ref)
    {
	if (ref->vp)
	{
	    (void) kern_ioctl_file_extents(ref, _DKIOCCSUNPINEXTENT, 0, (ref->pinned && ref->cf) ? ref->filelength : 0);
	    vnode_close(ref->vp, FWRITE, ref->ctx);
	    ref->vp = NULLVP;
	}
	ref->ctx = NULL;
	kfree(ref, sizeof(struct kern_direct_file_io_ref_t));
	ref = NULL;
    }

    return(ref);
}

int
kern_write_file(struct kern_direct_file_io_ref_t * ref, off_t offset, void * addr, size_t len, int ioflag)
{
    return (vn_rdwr(UIO_WRITE, ref->vp,
			addr, len, offset,
			UIO_SYSSPACE, ioflag|IO_SYNC|IO_NODELOCKED|IO_UNIT, 
                        vfs_context_ucred(ref->ctx), (int *) 0,
			vfs_context_proc(ref->ctx)));
}

int
kern_read_file(struct kern_direct_file_io_ref_t * ref, off_t offset, void * addr, size_t len, int ioflag)
{
    return (vn_rdwr(UIO_READ, ref->vp,
			addr, len, offset,
			UIO_SYSSPACE, ioflag|IO_SYNC|IO_NODELOCKED|IO_UNIT, 
                        vfs_context_ucred(ref->ctx), (int *) 0,
			vfs_context_proc(ref->ctx)));
}


struct mount *
kern_file_mount(struct kern_direct_file_io_ref_t * ref)
{
    return (ref->vp->v_mount);
}

void
kern_close_file_for_direct_io(struct kern_direct_file_io_ref_t * ref,
			      off_t write_offset, void * addr, size_t write_length,
			      off_t discard_offset, off_t discard_end)
{
    int error;
    printf("kern_close_file_for_direct_io(%p)\n", ref);

    if (!ref) return;

    if (ref->vp)
    {
        int (*do_ioctl)(void * p1, void * p2, u_long theIoctl, caddr_t result);
        void * p1;
        void * p2;

        if (ref->vp->v_type == VREG)
        {
            p1 = &ref->device;
            p2 = kernproc;
            do_ioctl = &file_ioctl;
        }
        else
        {
            /* Partition. */
            p1 = ref->vp;
            p2 = ref->ctx;
            do_ioctl = &device_ioctl;
        }
        (void) do_ioctl(p1, p2, DKIOCUNLOCKPHYSICALEXTENTS, NULL);

		//XXX If unmapping extents then don't also need to unpin; except ...
		//XXX if file unaligned (HFS 4k / Fusion 128k) then pin is superset and
		//XXX unmap is subset, so save extra walk over file extents (and the risk
		//XXX that CF drain starts) vs leaving partial units pinned to SSD
		//XXX (until whatever was sharing also unmaps).  Err on cleaning up fully.
		boolean_t will_unmap = (!ref->pinned || ref->cf) && (discard_end > discard_offset);
		boolean_t will_unpin = (ref->pinned && ref->cf /* && !will_unmap */);

		(void) kern_ioctl_file_extents(ref, _DKIOCCSUNPINEXTENT, 0, (will_unpin) ? ref->filelength : 0);

        if (will_unmap)
        {
            (void) kern_ioctl_file_extents(ref, DKIOCUNMAP, discard_offset, (ref->cf) ? ref->filelength : discard_end);
        }

        if (addr && write_length)
        {
            (void) kern_write_file(ref, write_offset, addr, write_length, IO_SKIP_ENCRYPTION);
        }

        error = vnode_close(ref->vp, FWRITE, ref->ctx);

        ref->vp = NULLVP;
        kprintf("vnode_close(%d)\n", error);

    }

    ref->ctx = NULL;

    kfree(ref, sizeof(struct kern_direct_file_io_ref_t));
}
Commit	Line	Data
1c79356b	1	/*
2d21ac55	2	* Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
5d5c5d0d	3	*
2d21ac55	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
1c79356b	5	*
2d21ac55 A	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
8f6c56a5	14	*
2d21ac55 A	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
8f6c56a5 A	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
8f6c56a5 A	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
2d21ac55 A	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
8f6c56a5	25	*
2d21ac55	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
1c79356b A	27	*/
	28	/* Copyright (c) 1998 Apple Computer, Inc. All rights reserved.
	29	*
	30	* File: bsd/kern/kern_symfile.c
	31	*
1c79356b	32	* HISTORY
1c79356b A	33	*/
	34
	35	#include <mach/vm_param.h>
	36
	37	#include <sys/param.h>
	38	#include <sys/systm.h>
	39	#include <sys/signalvar.h>
	40	#include <sys/resourcevar.h>
	41	#include <sys/namei.h>
91447636 A	42	#include <sys/vnode_internal.h>
	43	#include <sys/proc_internal.h>
	44	#include <sys/kauth.h>
1c79356b A	45	#include <sys/timeb.h>
1c79356b A	46	#include <sys/times.h>
1c79356b	47	#include <sys/acct.h>
91447636	48	#include <sys/file_internal.h>
1c79356b A	49	#include <sys/uio.h>
	50	#include <sys/kernel.h>
	51	#include <sys/stat.h>
91447636 A	52	#include <sys/disk.h>
91447636 A	53	#include <sys/conf.h>
3e170ce0	54	#include <sys/content_protection.h>
1c79356b A	55
	56	#include <mach-o/loader.h>
	57	#include <mach-o/nlist.h>
	58
91447636	59	#include <kern/kalloc.h>
1c79356b	60	#include <vm/vm_kern.h>
91447636	61	#include <pexpert/pexpert.h>
3e170ce0	62	#include <IOKit/IOPolledInterface.h>
1c79356b	63
2d21ac55 A	64	/* This function is called from kern_sysctl in the current process context;
	65	* it is exported with the System6.0.exports, but this appears to be a legacy
	66	* export, as there are no internal consumers.
1c79356b	67	*/
2d21ac55	68	int
7ddcb079 A	69	get_kernel_symfile(__unused proc_t p, __unused char const **symfile);
7ddcb079 A	70	int
2d21ac55	71	get_kernel_symfile(__unused proc_t p, __unused char const **symfile)
1c79356b	72	{
2d21ac55	73	return KERN_FAILURE;
0b4e3aa0	74	}
91447636	75
3a60a9f5 A	76	struct kern_direct_file_io_ref_t
3a60a9f5 A	77	{
6d2010ae A	78	vfs_context_t ctx;
	79	struct vnode * vp;
	80	dev_t device;
7ddcb079	81	uint32_t blksize;
316670eb	82	off_t filelength;
3e170ce0	83	char cf;
316670eb	84	char pinned;
3a60a9f5 A	85	};
	86
	87
6d2010ae	88	static int file_ioctl(void * p1, void * p2, u_long theIoctl, caddr_t result)
3a60a9f5	89	{
b0d623f7	90	dev_t device = (dev_t) p1;
3a60a9f5 A	91
	92	return ((*bdevsw[major(device)].d_ioctl)
	93	(device, theIoctl, result, S_IFBLK, p2));
	94	}
	95
6d2010ae	96	static int device_ioctl(void * p1, __unused void * p2, u_long theIoctl, caddr_t result)
3a60a9f5 A	97	{
	98	return (VNOP_IOCTL(p1, theIoctl, result, 0, p2));
	99	}
	100
316670eb A	101	static int
	102	kern_ioctl_file_extents(struct kern_direct_file_io_ref_t * ref, u_long theIoctl, off_t offset, off_t end)
	103	{
3e170ce0	104	int error = 0;
316670eb A	105	int (do_ioctl)(void p1, void * p2, u_long theIoctl, caddr_t result);
	106	void * p1;
	107	void * p2;
	108	uint64_t fileblk;
	109	size_t filechunk;
	110	dk_extent_t extent;
	111	dk_unmap_t unmap;
	112	_dk_cs_pin_t pin;
	113
	114	bzero(&extent, sizeof(dk_extent_t));
	115	bzero(&unmap, sizeof(dk_unmap_t));
	116	bzero(&pin, sizeof(pin));
	117	if (ref->vp->v_type == VREG)
	118	{
	119	p1 = &ref->device;
	120	p2 = kernproc;
	121	do_ioctl = &file_ioctl;
	122	}
	123	else
	124	{
	125	/* Partition. */
	126	p1 = ref->vp;
	127	p2 = ref->ctx;
	128	do_ioctl = &device_ioctl;
	129	}
3e170ce0 A	130
	131	if (_DKIOCCSPINEXTENT == theIoctl) {
	132	/* Tell CS the image size, so it knows whether to place the subsequent pins SSD/HDD */
	133	pin.cp_extent.length = end;
	134	pin.cp_flags = _DKIOCCSHIBERNATEIMGSIZE;
	135	(void) do_ioctl(p1, p2, _DKIOCCSPINEXTENT, (caddr_t)&pin);
	136	} else if (_DKIOCCSUNPINEXTENT == theIoctl) {
	137	/* Tell CS hibernation is done, so it can stop blocking overlapping writes */
	138	pin.cp_flags = _DKIOCCSPINDISCARDBLACKLIST;
	139	(void) do_ioctl(p1, p2, _DKIOCCSUNPINEXTENT, (caddr_t)&pin);
	140	}
	141
39037602	142	for (; offset < end; offset += filechunk)
316670eb A	143	{
	144	if (ref->vp->v_type == VREG)
	145	{
	146	daddr64_t blkno;
	147	filechunk = 110241024*1024;
	148	if (filechunk > (size_t)(end - offset))
	149	filechunk = (size_t)(end - offset);
fe8ab488	150	error = VNOP_BLOCKMAP(ref->vp, offset, filechunk, &blkno,
5ba3f43e	151	&filechunk, NULL, VNODE_WRITE \| VNODE_BLOCKMAP_NO_TRACK, NULL);
39037602 A	152	if (error) break;
39037602 A	153	if (-1LL == blkno) continue;
316670eb A	154	fileblk = blkno * ref->blksize;
	155	}
	156	else if ((ref->vp->v_type == VBLK) \|\| (ref->vp->v_type == VCHR))
	157	{
	158	fileblk = offset;
	159	filechunk = ref->filelength;
	160	}
	161
	162	if (DKIOCUNMAP == theIoctl)
	163	{
	164	extent.offset = fileblk;
	165	extent.length = filechunk;
	166	unmap.extents = &extent;
	167	unmap.extentsCount = 1;
	168	error = do_ioctl(p1, p2, theIoctl, (caddr_t)&unmap);
	169	// printf("DKIOCUNMAP(%d) 0x%qx, 0x%qx\n", error, extent.offset, extent.length);
	170	}
	171	else if (_DKIOCCSPINEXTENT == theIoctl)
	172	{
	173	pin.cp_extent.offset = fileblk;
	174	pin.cp_extent.length = filechunk;
fe8ab488	175	pin.cp_flags = _DKIOCCSPINFORHIBERNATION;
316670eb A	176	error = do_ioctl(p1, p2, theIoctl, (caddr_t)&pin);
	177	if (error && (ENOTTY != error))
	178	{
3e170ce0 A	179	printf("_DKIOCCSPINEXTENT(%d) 0x%qx, 0x%qx\n", error, pin.cp_extent.offset, pin.cp_extent.length);
	180	}
	181	}
	182	else if (_DKIOCCSUNPINEXTENT == theIoctl)
	183	{
	184	pin.cp_extent.offset = fileblk;
	185	pin.cp_extent.length = filechunk;
	186	pin.cp_flags = _DKIOCCSPINFORHIBERNATION;
	187	error = do_ioctl(p1, p2, theIoctl, (caddr_t)&pin);
	188	if (error && (ENOTTY != error))
	189	{
	190	printf("_DKIOCCSUNPINEXTENT(%d) 0x%qx, 0x%qx\n", error, pin.cp_extent.offset, pin.cp_extent.length);
316670eb A	191	}
	192	}
	193	else error = EINVAL;
	194
3e170ce0	195	if (error) break;
316670eb A	196	}
	197	return (error);
	198	}
	199
3e170ce0	200	extern uint32_t freespace_mb(vnode_t vp);
7ddcb079	201
3a60a9f5 A	202	struct kern_direct_file_io_ref_t *
3a60a9f5 A	203	kern_open_file_for_direct_io(const char * name,
3e170ce0	204	boolean_t create_file,
3a60a9f5 A	205	kern_get_file_extents_callback_t callback,
3a60a9f5 A	206	void * callback_ref,
db609669	207	off_t set_file_size,
3e170ce0	208	off_t fs_free_size,
db609669	209	off_t write_file_offset,
3e170ce0 A	210	void * write_file_addr,
3e170ce0 A	211	size_t write_file_len,
6d2010ae A	212	dev_t * partition_device_result,
6d2010ae A	213	dev_t * image_device_result,
3a60a9f5	214	uint64_t * partitionbase_result,
0b4c1975	215	uint64_t * maxiocount_result,
db609669	216	uint32_t * oflags)
3a60a9f5 A	217	{
	218	struct kern_direct_file_io_ref_t * ref;
	219
3e170ce0 A	220	proc_t p;
	221	struct vnode_attr va;
	222	int error;
	223	off_t f_offset;
	224	uint64_t fileblk;
	225	size_t filechunk;
	226	uint64_t physoffset;
	227	dev_t device;
	228	dev_t target = 0;
	229	int isssd = 0;
	230	uint32_t flags = 0;
	231	uint32_t blksize;
	232	off_t maxiocount, count, segcount;
	233	boolean_t locked = FALSE;
	234	int fmode, cmode;
	235	struct nameidata nd;
	236	u_int32_t ndflags;
	237	off_t mpFree;
3a60a9f5	238
6d2010ae A	239	int (do_ioctl)(void p1, void * p2, u_long theIoctl, caddr_t result);
	240	void * p1 = NULL;
	241	void * p2 = NULL;
3a60a9f5 A	242
	243	error = EFAULT;
	244
	245	ref = (struct kern_direct_file_io_ref_t *) kalloc(sizeof(struct kern_direct_file_io_ref_t));
	246	if (!ref)
	247	{
	248	error = EFAULT;
	249	goto out;
	250	}
	251
316670eb	252	bzero(ref, sizeof(*ref));
6d2010ae	253	p = kernproc;
00867663	254	ref->ctx = vfs_context_kernel();
3a60a9f5	255
3e170ce0 A	256	fmode = (create_file) ? (O_CREAT \| FWRITE) : FWRITE;
	257	cmode = S_IRUSR \| S_IWUSR;
	258	ndflags = NOFOLLOW;
	259	NDINIT(&nd, LOOKUP, OP_OPEN, ndflags, UIO_SYSSPACE, CAST_USER_ADDR_T(name), ref->ctx);
	260	VATTR_INIT(&va);
	261	VATTR_SET(&va, va_mode, cmode);
	262	VATTR_SET(&va, va_dataprotect_flags, VA_DP_RAWENCRYPTED);
	263	VATTR_SET(&va, va_dataprotect_class, PROTECTION_CLASS_D);
00867663 A	264	if ((error = vn_open_auth(&nd, &fmode, &va))) {
	265	kprintf("vn_open_auth(fmode: %d, cmode: %d) failed with error: %d\n", fmode, cmode, error);
	266	goto out;
	267	}
3a60a9f5	268
3e170ce0	269	ref->vp = nd.ni_vp;
39236c6e A	270	if (ref->vp->v_type == VREG)
	271	{
	272	vnode_lock_spin(ref->vp);
	273	SET(ref->vp->v_flag, VSWAP);
	274	vnode_unlock(ref->vp);
	275	}
	276
db609669	277	if (write_file_addr && write_file_len)
6d2010ae	278	{
00867663 A	279	if ((error = kern_write_file(ref, write_file_offset, write_file_addr, write_file_len, IO_SKIP_ENCRYPTION))) {
	280	kprintf("kern_write_file() failed with error: %d\n", error);
	281	goto out;
	282	}
6d2010ae A	283	}
6d2010ae A	284
3a60a9f5 A	285	VATTR_INIT(&va);
	286	VATTR_WANTED(&va, va_rdev);
	287	VATTR_WANTED(&va, va_fsid);
39037602	288	VATTR_WANTED(&va, va_devid);
3a60a9f5	289	VATTR_WANTED(&va, va_data_size);
39236c6e	290	VATTR_WANTED(&va, va_data_alloc);
3a60a9f5 A	291	VATTR_WANTED(&va, va_nlink);
3a60a9f5 A	292	error = EFAULT;
3e170ce0	293	if (vnode_getattr(ref->vp, &va, ref->ctx)) goto out;
3a60a9f5	294
3e170ce0 A	295	mpFree = freespace_mb(ref->vp);
	296	mpFree <<= 20;
	297	kprintf("kern_direct_file(%s): vp size %qd, alloc %qd, mp free %qd, keep free %qd\n",
	298	name, va.va_data_size, va.va_data_alloc, mpFree, fs_free_size);
3a60a9f5 A	299
	300	if (ref->vp->v_type == VREG)
	301	{
3e170ce0 A	302	/* Don't dump files with links. */
3e170ce0 A	303	if (va.va_nlink != 1) goto out;
3a60a9f5	304
39037602	305	device = (VATTR_IS_SUPPORTED(&va, va_devid)) ? va.va_devid : va.va_fsid;
db609669 A	306	ref->filelength = va.va_data_size;
db609669 A	307
b0d623f7	308	p1 = &device;
3a60a9f5 A	309	p2 = p;
3a60a9f5 A	310	do_ioctl = &file_ioctl;
db609669	311
3e170ce0 A	312	if (set_file_size)
	313	{
	314	if (fs_free_size)
	315	{
	316	mpFree += va.va_data_alloc;
	317	if ((mpFree < set_file_size) \|\| ((mpFree - set_file_size) < fs_free_size))
	318	{
	319	error = ENOSPC;
	320	goto out;
fe8ab488	321	}
3e170ce0 A	322	}
	323	error = vnode_setsize(ref->vp, set_file_size, IO_NOZEROFILL \| IO_NOAUTH, ref->ctx);
	324	if (error) goto out;
	325	ref->filelength = set_file_size;
	326	}
3a60a9f5 A	327	}
	328	else if ((ref->vp->v_type == VBLK) \|\| (ref->vp->v_type == VCHR))
	329	{
	330	/* Partition. */
	331	device = va.va_rdev;
	332
	333	p1 = ref->vp;
2d21ac55	334	p2 = ref->ctx;
3a60a9f5 A	335	do_ioctl = &device_ioctl;
	336	}
	337	else
	338	{
	339	/* Don't dump to non-regular files. */
3e170ce0	340	error = EFAULT;
3a60a9f5 A	341	goto out;
3a60a9f5 A	342	}
6d2010ae A	343	ref->device = device;
6d2010ae A	344
3e170ce0 A	345	// probe for CF
	346	dk_corestorage_info_t cs_info;
	347	memset(&cs_info, 0, sizeof(dk_corestorage_info_t));
	348	error = do_ioctl(p1, p2, DKIOCCORESTORAGE, (caddr_t)&cs_info);
	349	ref->cf = (error == 0) && (cs_info.flags & DK_CORESTORAGE_ENABLE_HOTFILES);
	350
6d2010ae A	351	// get block size
6d2010ae A	352
7ddcb079	353	error = do_ioctl(p1, p2, DKIOCGETBLOCKSIZE, (caddr_t) &ref->blksize);
6d2010ae A	354	if (error)
	355	goto out;
	356
db609669	357	if (ref->vp->v_type != VREG)
6d2010ae A	358	{
6d2010ae A	359	error = do_ioctl(p1, p2, DKIOCGETBLOCKCOUNT, (caddr_t) &fileblk);
3e170ce0	360	if (error) goto out;
7ddcb079	361	ref->filelength = fileblk * ref->blksize;
6d2010ae A	362	}
6d2010ae A	363
316670eb A	364	// pin logical extents
	365
	366	error = kern_ioctl_file_extents(ref, _DKIOCCSPINEXTENT, 0, ref->filelength);
	367	if (error && (ENOTTY != error)) goto out;
	368	ref->pinned = (error == 0);
	369
	370	// generate the block list
	371
	372	error = do_ioctl(p1, p2, DKIOCLOCKPHYSICALEXTENTS, NULL);
3e170ce0	373	if (error) goto out;
316670eb A	374	locked = TRUE;
316670eb A	375
6d2010ae	376	f_offset = 0;
39037602	377	for (; f_offset < ref->filelength; f_offset += filechunk)
6d2010ae A	378	{
	379	if (ref->vp->v_type == VREG)
	380	{
	381	filechunk = 110241024*1024;
	382	daddr64_t blkno;
	383
fe8ab488	384	error = VNOP_BLOCKMAP(ref->vp, f_offset, filechunk, &blkno,
5ba3f43e	385	&filechunk, NULL, VNODE_WRITE \| VNODE_BLOCKMAP_NO_TRACK, NULL);
3e170ce0	386	if (error) goto out;
39037602	387	if (-1LL == blkno) continue;
7ddcb079	388	fileblk = blkno * ref->blksize;
6d2010ae A	389	}
	390	else if ((ref->vp->v_type == VBLK) \|\| (ref->vp->v_type == VCHR))
	391	{
	392	fileblk = f_offset;
7ddcb079	393	filechunk = f_offset ? 0 : ref->filelength;
6d2010ae A	394	}
	395
	396	physoffset = 0;
	397	while (physoffset < filechunk)
	398	{
	399	dk_physical_extent_t getphysreq;
	400	bzero(&getphysreq, sizeof(getphysreq));
	401
	402	getphysreq.offset = fileblk + physoffset;
	403	getphysreq.length = (filechunk - physoffset);
	404	error = do_ioctl(p1, p2, DKIOCGETPHYSICALEXTENT, (caddr_t) &getphysreq);
3e170ce0	405	if (error) goto out;
6d2010ae A	406	if (!target)
	407	{
	408	target = getphysreq.dev;
	409	}
	410	else if (target != getphysreq.dev)
	411	{
	412	error = ENOTSUP;
	413	goto out;
	414	}
39236c6e A	415	#if HIBFRAGMENT
	416	uint64_t rev;
	417	for (rev = 4096; rev <= getphysreq.length; rev += 4096)
	418	{
	419	callback(callback_ref, getphysreq.offset + getphysreq.length - rev, 4096);
	420	}
	421	#else
6d2010ae	422	callback(callback_ref, getphysreq.offset, getphysreq.length);
39236c6e	423	#endif
6d2010ae A	424	physoffset += getphysreq.length;
6d2010ae A	425	}
6d2010ae A	426	}
	427	callback(callback_ref, 0ULL, 0ULL);
	428
3e170ce0 A	429	if (ref->vp->v_type == VREG) p1 = &target;
	430	else
	431	{
	432	p1 = &target;
	433	p2 = p;
	434	do_ioctl = &file_ioctl;
	435	}
3a60a9f5 A	436
	437	// get partition base
	438
fe8ab488 A	439	if (partitionbase_result)
	440	{
	441	error = do_ioctl(p1, p2, DKIOCGETBASE, (caddr_t) partitionbase_result);
	442	if (error)
	443	goto out;
	444	}
3a60a9f5 A	445
	446	// get block size & constraints
	447
	448	error = do_ioctl(p1, p2, DKIOCGETBLOCKSIZE, (caddr_t) &blksize);
	449	if (error)
	450	goto out;
	451
	452	maxiocount = 110241024*1024;
	453
	454	error = do_ioctl(p1, p2, DKIOCGETMAXBLOCKCOUNTREAD, (caddr_t) &count);
	455	if (error)
	456	count = 0;
	457	count *= blksize;
	458	if (count && (count < maxiocount))
	459	maxiocount = count;
	460
	461	error = do_ioctl(p1, p2, DKIOCGETMAXBLOCKCOUNTWRITE, (caddr_t) &count);
	462	if (error)
	463	count = 0;
	464	count *= blksize;
	465	if (count && (count < maxiocount))
	466	maxiocount = count;
	467
	468	error = do_ioctl(p1, p2, DKIOCGETMAXBYTECOUNTREAD, (caddr_t) &count);
	469	if (error)
	470	count = 0;
	471	if (count && (count < maxiocount))
	472	maxiocount = count;
	473
	474	error = do_ioctl(p1, p2, DKIOCGETMAXBYTECOUNTWRITE, (caddr_t) &count);
	475	if (error)
	476	count = 0;
	477	if (count && (count < maxiocount))
	478	maxiocount = count;
	479
	480	error = do_ioctl(p1, p2, DKIOCGETMAXSEGMENTBYTECOUNTREAD, (caddr_t) &count);
bd504ef0 A	481	if (!error)
bd504ef0 A	482	error = do_ioctl(p1, p2, DKIOCGETMAXSEGMENTCOUNTREAD, (caddr_t) &segcount);
3a60a9f5	483	if (error)
bd504ef0 A	484	count = segcount = 0;
bd504ef0 A	485	count *= segcount;
3a60a9f5 A	486	if (count && (count < maxiocount))
	487	maxiocount = count;
	488
	489	error = do_ioctl(p1, p2, DKIOCGETMAXSEGMENTBYTECOUNTWRITE, (caddr_t) &count);
bd504ef0 A	490	if (!error)
bd504ef0 A	491	error = do_ioctl(p1, p2, DKIOCGETMAXSEGMENTCOUNTWRITE, (caddr_t) &segcount);
3a60a9f5	492	if (error)
bd504ef0 A	493	count = segcount = 0;
bd504ef0 A	494	count *= segcount;
3a60a9f5 A	495	if (count && (count < maxiocount))
	496	maxiocount = count;
	497
	498	kprintf("max io 0x%qx bytes\n", maxiocount);
	499	if (maxiocount_result)
	500	*maxiocount_result = maxiocount;
	501
6d2010ae A	502	error = do_ioctl(p1, p2, DKIOCISSOLIDSTATE, (caddr_t)&isssd);
6d2010ae A	503	if (!error && isssd)
3e170ce0	504	flags \|= kIOPolledFileSSD;
3a60a9f5	505
6d2010ae A	506	if (partition_device_result)
	507	*partition_device_result = device;
	508	if (image_device_result)
	509	*image_device_result = target;
fe8ab488	510	if (oflags)
6d2010ae	511	*oflags = flags;
3a60a9f5	512
3e170ce0 A	513	if ((ref->vp->v_type == VBLK) \|\| (ref->vp->v_type == VCHR))
	514	{
	515	vnode_close(ref->vp, FWRITE, ref->ctx);
	516	ref->vp = NULLVP;
3e170ce0 A	517	ref->ctx = NULL;
	518	}
	519
3a60a9f5	520	out:
813fb2f6	521	printf("kern_open_file_for_direct_io(%p, %d)\n", ref, error);
3a60a9f5	522
00867663	523
6d2010ae A	524	if (error && locked)
	525	{
	526	p1 = &device;
	527	(void) do_ioctl(p1, p2, DKIOCUNLOCKPHYSICALEXTENTS, NULL);
	528	}
	529
	530	if (error && ref)
	531	{
	532	if (ref->vp)
	533	{
3e170ce0	534	(void) kern_ioctl_file_extents(ref, _DKIOCCSUNPINEXTENT, 0, (ref->pinned && ref->cf) ? ref->filelength : 0);
2d21ac55 A	535	vnode_close(ref->vp, FWRITE, ref->ctx);
	536	ref->vp = NULLVP;
	537	}
00867663	538	ref->ctx = NULL;
2d21ac55 A	539	kfree(ref, sizeof(struct kern_direct_file_io_ref_t));
2d21ac55 A	540	ref = NULL;
3a60a9f5	541	}
db609669	542
3a60a9f5 A	543	return(ref);
	544	}
	545
	546	int
3e170ce0	547	kern_write_file(struct kern_direct_file_io_ref_t * ref, off_t offset, void * addr, size_t len, int ioflag)
3a60a9f5 A	548	{
	549	return (vn_rdwr(UIO_WRITE, ref->vp,
	550	addr, len, offset,
fe8ab488	551	UIO_SYSSPACE, ioflag\|IO_SYNC\|IO_NODELOCKED\|IO_UNIT,
2d21ac55 A	552	vfs_context_ucred(ref->ctx), (int *) 0,
2d21ac55 A	553	vfs_context_proc(ref->ctx)));
3a60a9f5 A	554	}
3a60a9f5 A	555
3e170ce0 A	556	int
	557	kern_read_file(struct kern_direct_file_io_ref_t * ref, off_t offset, void * addr, size_t len, int ioflag)
	558	{
	559	return (vn_rdwr(UIO_READ, ref->vp,
	560	addr, len, offset,
	561	UIO_SYSSPACE, ioflag\|IO_SYNC\|IO_NODELOCKED\|IO_UNIT,
	562	vfs_context_ucred(ref->ctx), (int *) 0,
	563	vfs_context_proc(ref->ctx)));
	564	}
	565
	566
	567	struct mount *
	568	kern_file_mount(struct kern_direct_file_io_ref_t * ref)
	569	{
	570	return (ref->vp->v_mount);
	571	}
7ddcb079	572
3a60a9f5	573	void
6d2010ae	574	kern_close_file_for_direct_io(struct kern_direct_file_io_ref_t * ref,
3e170ce0	575	off_t write_offset, void * addr, size_t write_length,
7ddcb079	576	off_t discard_offset, off_t discard_end)
3a60a9f5	577	{
6d2010ae	578	int error;
813fb2f6	579	printf("kern_close_file_for_direct_io(%p)\n", ref);
3a60a9f5	580
6d2010ae	581	if (!ref) return;
3a60a9f5	582
6d2010ae A	583	if (ref->vp)
	584	{
	585	int (do_ioctl)(void p1, void * p2, u_long theIoctl, caddr_t result);
	586	void * p1;
	587	void * p2;
	588
	589	if (ref->vp->v_type == VREG)
	590	{
	591	p1 = &ref->device;
	592	p2 = kernproc;
	593	do_ioctl = &file_ioctl;
	594	}
	595	else
	596	{
	597	/* Partition. */
	598	p1 = ref->vp;
	599	p2 = ref->ctx;
	600	do_ioctl = &device_ioctl;
	601	}
	602	(void) do_ioctl(p1, p2, DKIOCUNLOCKPHYSICALEXTENTS, NULL);
fe8ab488	603
3e170ce0 A	604	//XXX If unmapping extents then don't also need to unpin; except ...
	605	//XXX if file unaligned (HFS 4k / Fusion 128k) then pin is superset and
	606	//XXX unmap is subset, so save extra walk over file extents (and the risk
	607	//XXX that CF drain starts) vs leaving partial units pinned to SSD
	608	//XXX (until whatever was sharing also unmaps). Err on cleaning up fully.
	609	boolean_t will_unmap = (!ref->pinned \|\| ref->cf) && (discard_end > discard_offset);
	610	boolean_t will_unpin = (ref->pinned && ref->cf /* && !will_unmap */);
fe8ab488	611
3e170ce0 A	612	(void) kern_ioctl_file_extents(ref, _DKIOCCSUNPINEXTENT, 0, (will_unpin) ? ref->filelength : 0);
	613
	614	if (will_unmap)
6d2010ae	615	{
3e170ce0	616	(void) kern_ioctl_file_extents(ref, DKIOCUNMAP, discard_offset, (ref->cf) ? ref->filelength : discard_end);
6d2010ae	617	}
3e170ce0	618
39236c6e A	619	if (addr && write_length)
39236c6e A	620	{
00867663	621	(void) kern_write_file(ref, write_offset, addr, write_length, IO_SKIP_ENCRYPTION);
39236c6e	622	}
6d2010ae A	623
	624	error = vnode_close(ref->vp, FWRITE, ref->ctx);
	625
	626	ref->vp = NULLVP;
	627	kprintf("vnode_close(%d)\n", error);
00867663	628
3a60a9f5	629	}
00867663 A	630
	631	ref->ctx = NULL;
	632
6d2010ae	633	kfree(ref, sizeof(struct kern_direct_file_io_ref_t));
3a60a9f5	634	}