X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/813fb2f63a553c957e917ede5f119b021d6ce391..a991bd8d3e7fe02dbca0644054bab73c5b75324a:/bsd/kern/kern_symfile.c?ds=sidebyside diff --git a/bsd/kern/kern_symfile.c b/bsd/kern/kern_symfile.c index a612b23fc..720ebf9bf 100644 --- a/bsd/kern/kern_symfile.c +++ b/bsd/kern/kern_symfile.c @@ -2,7 +2,7 @@ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * + * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in @@ -11,10 +11,10 @@ * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. - * + * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. - * + * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, @@ -22,7 +22,7 @@ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. - * + * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ /* Copyright (c) 1998 Apple Computer, Inc. All rights reserved. @@ -52,6 +52,7 @@ #include #include #include +#include #include #include @@ -61,6 +62,10 @@ #include #include +#define HIBERNATE_MIN_PHYSICAL_LBA_512 (34) +#define HIBERNATE_MIN_PHYSICAL_LBA_4096 (6) +#define HIBERNATE_MIN_FILE_SIZE (1024*1024) + /* This function is called from kern_sysctl in the current process context; * it is exported with the System6.0.exports, but this appears to be a legacy * export, as there are no internal consumers. @@ -70,530 +75,610 @@ get_kernel_symfile(__unused proc_t p, __unused char const **symfile); int get_kernel_symfile(__unused proc_t p, __unused char const **symfile) { - return KERN_FAILURE; + return KERN_FAILURE; } -struct kern_direct_file_io_ref_t -{ - vfs_context_t ctx; - struct vnode * vp; - dev_t device; - uint32_t blksize; - off_t filelength; - char cf; - char pinned; +struct kern_direct_file_io_ref_t { + vfs_context_t ctx; + struct vnode * vp; + dev_t device; + uint32_t blksize; + off_t filelength; + char cf; + char pinned; + char frozen; + char wbcranged; }; -static int file_ioctl(void * p1, void * p2, u_long theIoctl, caddr_t result) +static int +file_ioctl(void * p1, void * p2, u_long theIoctl, caddr_t result) { - dev_t device = *(dev_t*) p1; + dev_t device = *(dev_t*) p1; - return ((*bdevsw[major(device)].d_ioctl) - (device, theIoctl, result, S_IFBLK, p2)); + return (*bdevsw[major(device)].d_ioctl) + (device, theIoctl, result, S_IFBLK, p2); } -static int device_ioctl(void * p1, __unused void * p2, u_long theIoctl, caddr_t result) +static int +device_ioctl(void * p1, __unused void * p2, u_long theIoctl, caddr_t result) { - return (VNOP_IOCTL(p1, theIoctl, result, 0, p2)); + return VNOP_IOCTL(p1, theIoctl, result, 0, p2); } static int kern_ioctl_file_extents(struct kern_direct_file_io_ref_t * ref, u_long theIoctl, off_t offset, off_t end) { - int error = 0; - int (*do_ioctl)(void * p1, void * p2, u_long theIoctl, caddr_t result); - void * p1; - void * p2; - uint64_t fileblk; - size_t filechunk; - dk_extent_t extent; - dk_unmap_t unmap; - _dk_cs_pin_t pin; - - bzero(&extent, sizeof(dk_extent_t)); - bzero(&unmap, sizeof(dk_unmap_t)); - bzero(&pin, sizeof(pin)); - if (ref->vp->v_type == VREG) - { - p1 = &ref->device; - p2 = kernproc; - do_ioctl = &file_ioctl; - } - else - { - /* Partition. */ - p1 = ref->vp; - p2 = ref->ctx; - do_ioctl = &device_ioctl; - } - - if (_DKIOCCSPINEXTENT == theIoctl) { - /* Tell CS the image size, so it knows whether to place the subsequent pins SSD/HDD */ - pin.cp_extent.length = end; - pin.cp_flags = _DKIOCCSHIBERNATEIMGSIZE; - (void) do_ioctl(p1, p2, _DKIOCCSPINEXTENT, (caddr_t)&pin); - } else if (_DKIOCCSUNPINEXTENT == theIoctl) { - /* Tell CS hibernation is done, so it can stop blocking overlapping writes */ - pin.cp_flags = _DKIOCCSPINDISCARDBLACKLIST; - (void) do_ioctl(p1, p2, _DKIOCCSUNPINEXTENT, (caddr_t)&pin); - } - - for (; offset < end; offset += filechunk) - { - if (ref->vp->v_type == VREG) - { - daddr64_t blkno; - filechunk = 1*1024*1024*1024; - if (filechunk > (size_t)(end - offset)) - filechunk = (size_t)(end - offset); - error = VNOP_BLOCKMAP(ref->vp, offset, filechunk, &blkno, - &filechunk, NULL, VNODE_WRITE, NULL); - if (error) break; - if (-1LL == blkno) continue; - fileblk = blkno * ref->blksize; - } - else if ((ref->vp->v_type == VBLK) || (ref->vp->v_type == VCHR)) - { - fileblk = offset; - filechunk = ref->filelength; - } - - if (DKIOCUNMAP == theIoctl) - { - extent.offset = fileblk; - extent.length = filechunk; - unmap.extents = &extent; - unmap.extentsCount = 1; - error = do_ioctl(p1, p2, theIoctl, (caddr_t)&unmap); -// printf("DKIOCUNMAP(%d) 0x%qx, 0x%qx\n", error, extent.offset, extent.length); - } - else if (_DKIOCCSPINEXTENT == theIoctl) - { - pin.cp_extent.offset = fileblk; - pin.cp_extent.length = filechunk; - pin.cp_flags = _DKIOCCSPINFORHIBERNATION; - error = do_ioctl(p1, p2, theIoctl, (caddr_t)&pin); - if (error && (ENOTTY != error)) - { - printf("_DKIOCCSPINEXTENT(%d) 0x%qx, 0x%qx\n", error, pin.cp_extent.offset, pin.cp_extent.length); - } - } - else if (_DKIOCCSUNPINEXTENT == theIoctl) - { - pin.cp_extent.offset = fileblk; - pin.cp_extent.length = filechunk; - pin.cp_flags = _DKIOCCSPINFORHIBERNATION; - error = do_ioctl(p1, p2, theIoctl, (caddr_t)&pin); - if (error && (ENOTTY != error)) - { - printf("_DKIOCCSUNPINEXTENT(%d) 0x%qx, 0x%qx\n", error, pin.cp_extent.offset, pin.cp_extent.length); - } - } - else error = EINVAL; - - if (error) break; - } - return (error); + int error = 0; + int (*do_ioctl)(void * p1, void * p2, u_long theIoctl, caddr_t result); + void * p1; + void * p2; + uint64_t fileblk; + size_t filechunk; + dk_extent_t extent; + dk_unmap_t unmap; + _dk_cs_pin_t pin; + + bzero(&extent, sizeof(dk_extent_t)); + bzero(&unmap, sizeof(dk_unmap_t)); + bzero(&pin, sizeof(pin)); + if (ref->vp->v_type == VREG) { + p1 = &ref->device; + p2 = kernproc; + do_ioctl = &file_ioctl; + } else { + /* Partition. */ + p1 = ref->vp; + p2 = ref->ctx; + do_ioctl = &device_ioctl; + } + + if (_DKIOCCSPINEXTENT == theIoctl) { + /* Tell CS the image size, so it knows whether to place the subsequent pins SSD/HDD */ + pin.cp_extent.length = end; + pin.cp_flags = _DKIOCCSHIBERNATEIMGSIZE; + (void) do_ioctl(p1, p2, _DKIOCCSPINEXTENT, (caddr_t)&pin); + } else if (_DKIOCCSUNPINEXTENT == theIoctl) { + /* Tell CS hibernation is done, so it can stop blocking overlapping writes */ + pin.cp_flags = _DKIOCCSPINDISCARDBLACKLIST; + (void) do_ioctl(p1, p2, _DKIOCCSUNPINEXTENT, (caddr_t)&pin); + } + + for (; offset < end; offset += filechunk) { + if (ref->vp->v_type == VREG) { + daddr64_t blkno; + filechunk = 1 * 1024 * 1024 * 1024; + if (filechunk > (size_t)(end - offset)) { + filechunk = (size_t)(end - offset); + } + error = VNOP_BLOCKMAP(ref->vp, offset, filechunk, &blkno, + &filechunk, NULL, VNODE_WRITE | VNODE_BLOCKMAP_NO_TRACK, NULL); + if (error) { + break; + } + if (-1LL == blkno) { + continue; + } + fileblk = blkno * ref->blksize; + } else if ((ref->vp->v_type == VBLK) || (ref->vp->v_type == VCHR)) { + fileblk = offset; + filechunk = (unsigned long)((ref->filelength > ULONG_MAX) ? ULONG_MAX: ref->filelength); + } + + if (DKIOCUNMAP == theIoctl) { + extent.offset = fileblk; + extent.length = filechunk; + unmap.extents = &extent; + unmap.extentsCount = 1; + error = do_ioctl(p1, p2, theIoctl, (caddr_t)&unmap); +// printf("DKIOCUNMAP(%d) 0x%qx, 0x%qx\n", error, extent.offset, extent.length); + } else if (_DKIOCCSPINEXTENT == theIoctl) { + pin.cp_extent.offset = fileblk; + pin.cp_extent.length = filechunk; + pin.cp_flags = _DKIOCCSPINFORHIBERNATION; + error = do_ioctl(p1, p2, theIoctl, (caddr_t)&pin); + if (error && (ENOTTY != error)) { + printf("_DKIOCCSPINEXTENT(%d) 0x%qx, 0x%qx\n", error, pin.cp_extent.offset, pin.cp_extent.length); + } + } else if (_DKIOCCSUNPINEXTENT == theIoctl) { + pin.cp_extent.offset = fileblk; + pin.cp_extent.length = filechunk; + pin.cp_flags = _DKIOCCSPINFORHIBERNATION; + error = do_ioctl(p1, p2, theIoctl, (caddr_t)&pin); + if (error && (ENOTTY != error)) { + printf("_DKIOCCSUNPINEXTENT(%d) 0x%qx, 0x%qx\n", error, pin.cp_extent.offset, pin.cp_extent.length); + } + } else { + error = EINVAL; + } + + if (error) { + break; + } + } + return error; } extern uint32_t freespace_mb(vnode_t vp); struct kern_direct_file_io_ref_t * -kern_open_file_for_direct_io(const char * name, - boolean_t create_file, - kern_get_file_extents_callback_t callback, - void * callback_ref, - off_t set_file_size, - off_t fs_free_size, - off_t write_file_offset, - void * write_file_addr, - size_t write_file_len, - dev_t * partition_device_result, - dev_t * image_device_result, - uint64_t * partitionbase_result, - uint64_t * maxiocount_result, - uint32_t * oflags) +kern_open_file_for_direct_io(const char * name, + uint32_t iflags, + kern_get_file_extents_callback_t callback, + void * callback_ref, + off_t set_file_size, + off_t fs_free_size, + off_t write_file_offset, + void * write_file_addr, + size_t write_file_len, + dev_t * partition_device_result, + dev_t * image_device_result, + uint64_t * partitionbase_result, + uint64_t * maxiocount_result, + uint32_t * oflags) { - struct kern_direct_file_io_ref_t * ref; - - proc_t p; - struct vnode_attr va; - int error; - off_t f_offset; - uint64_t fileblk; - size_t filechunk; - uint64_t physoffset; - dev_t device; - dev_t target = 0; - int isssd = 0; - uint32_t flags = 0; - uint32_t blksize; - off_t maxiocount, count, segcount; - boolean_t locked = FALSE; - int fmode, cmode; - struct nameidata nd; - u_int32_t ndflags; - off_t mpFree; - - int (*do_ioctl)(void * p1, void * p2, u_long theIoctl, caddr_t result); - void * p1 = NULL; - void * p2 = NULL; - - error = EFAULT; - - ref = (struct kern_direct_file_io_ref_t *) kalloc(sizeof(struct kern_direct_file_io_ref_t)); - if (!ref) - { + struct kern_direct_file_io_ref_t * ref; + + proc_t p; + struct vnode_attr va; + dk_apfs_wbc_range_t wbc_range; + int error; + off_t f_offset; + uint64_t fileblk; + size_t filechunk; + uint64_t physoffset, minoffset; + dev_t device; + dev_t target = 0; + int isssd = 0; + uint32_t flags = 0; + uint32_t blksize; + off_t maxiocount, count, segcount, wbctotal; + boolean_t locked = FALSE; + int fmode; + mode_t cmode; + struct nameidata nd; + u_int32_t ndflags; + off_t mpFree; + + int (*do_ioctl)(void * p1, void * p2, u_long theIoctl, caddr_t result); + void * p1 = NULL; + void * p2 = NULL; + + error = EFAULT; + + ref = (struct kern_direct_file_io_ref_t *) kalloc(sizeof(struct kern_direct_file_io_ref_t)); + if (!ref) { + error = EFAULT; + goto out; + } + + bzero(ref, sizeof(*ref)); + p = kernproc; + ref->ctx = vfs_context_kernel(); + + fmode = (kIOPolledFileCreate & iflags) ? (O_CREAT | FWRITE) : FWRITE; + cmode = S_IRUSR | S_IWUSR; + ndflags = NOFOLLOW; + NDINIT(&nd, LOOKUP, OP_OPEN, ndflags, UIO_SYSSPACE, CAST_USER_ADDR_T(name), ref->ctx); + VATTR_INIT(&va); + VATTR_SET(&va, va_mode, cmode); + VATTR_SET(&va, va_dataprotect_flags, VA_DP_RAWENCRYPTED); + VATTR_SET(&va, va_dataprotect_class, PROTECTION_CLASS_D); + if ((error = vn_open_auth(&nd, &fmode, &va))) { + kprintf("vn_open_auth(fmode: %d, cmode: %d) failed with error: %d\n", fmode, cmode, error); + goto out; + } + + ref->vp = nd.ni_vp; + if (ref->vp->v_type == VREG) { + vnode_lock_spin(ref->vp); + SET(ref->vp->v_flag, VSWAP); + vnode_unlock(ref->vp); + } + + if (write_file_addr && write_file_len) { + if ((error = kern_write_file(ref, write_file_offset, write_file_addr, write_file_len, IO_SKIP_ENCRYPTION))) { + kprintf("kern_write_file() failed with error: %d\n", error); + goto out; + } + } + + VATTR_INIT(&va); + VATTR_WANTED(&va, va_rdev); + VATTR_WANTED(&va, va_fsid); + VATTR_WANTED(&va, va_devid); + VATTR_WANTED(&va, va_data_size); + VATTR_WANTED(&va, va_data_alloc); + VATTR_WANTED(&va, va_nlink); error = EFAULT; - goto out; - } - - bzero(ref, sizeof(*ref)); - p = kernproc; - ref->ctx = vfs_context_create(vfs_context_kernel()); - - fmode = (create_file) ? (O_CREAT | FWRITE) : FWRITE; - cmode = S_IRUSR | S_IWUSR; - ndflags = NOFOLLOW; - NDINIT(&nd, LOOKUP, OP_OPEN, ndflags, UIO_SYSSPACE, CAST_USER_ADDR_T(name), ref->ctx); - VATTR_INIT(&va); - VATTR_SET(&va, va_mode, cmode); - VATTR_SET(&va, va_dataprotect_flags, VA_DP_RAWENCRYPTED); - VATTR_SET(&va, va_dataprotect_class, PROTECTION_CLASS_D); - if ((error = vn_open_auth(&nd, &fmode, &va))) goto out; - - ref->vp = nd.ni_vp; - if (ref->vp->v_type == VREG) - { - vnode_lock_spin(ref->vp); - SET(ref->vp->v_flag, VSWAP); - vnode_unlock(ref->vp); - } - - if (write_file_addr && write_file_len) - { - if ((error = kern_write_file(ref, write_file_offset, write_file_addr, write_file_len, 0))) goto out; - } - - VATTR_INIT(&va); - VATTR_WANTED(&va, va_rdev); - VATTR_WANTED(&va, va_fsid); - VATTR_WANTED(&va, va_devid); - VATTR_WANTED(&va, va_data_size); - VATTR_WANTED(&va, va_data_alloc); - VATTR_WANTED(&va, va_nlink); - error = EFAULT; - if (vnode_getattr(ref->vp, &va, ref->ctx)) goto out; - - mpFree = freespace_mb(ref->vp); - mpFree <<= 20; - kprintf("kern_direct_file(%s): vp size %qd, alloc %qd, mp free %qd, keep free %qd\n", - name, va.va_data_size, va.va_data_alloc, mpFree, fs_free_size); - - if (ref->vp->v_type == VREG) - { - /* Don't dump files with links. */ - if (va.va_nlink != 1) goto out; - - device = (VATTR_IS_SUPPORTED(&va, va_devid)) ? va.va_devid : va.va_fsid; - ref->filelength = va.va_data_size; - - p1 = &device; - p2 = p; - do_ioctl = &file_ioctl; - - if (set_file_size) - { - if (fs_free_size) - { - mpFree += va.va_data_alloc; - if ((mpFree < set_file_size) || ((mpFree - set_file_size) < fs_free_size)) - { - error = ENOSPC; - goto out; + if (vnode_getattr(ref->vp, &va, ref->ctx)) { + goto out; + } + + wbctotal = 0; + mpFree = freespace_mb(ref->vp); + mpFree <<= 20; + kprintf("kern_direct_file(%s): vp size %qd, alloc %qd, mp free %qd, keep free %qd\n", + name, va.va_data_size, va.va_data_alloc, mpFree, fs_free_size); + + if (ref->vp->v_type == VREG) { + /* Don't dump files with links. */ + if (va.va_nlink != 1) { + goto out; + } + + device = (VATTR_IS_SUPPORTED(&va, va_devid)) ? va.va_devid : va.va_fsid; + ref->filelength = va.va_data_size; + + p1 = &device; + p2 = p; + do_ioctl = &file_ioctl; + + if (kIOPolledFileHibernate & iflags) { + error = do_ioctl(p1, p2, DKIOCAPFSGETWBCRANGE, (caddr_t) &wbc_range); + ref->wbcranged = (error == 0); + } + if (ref->wbcranged) { + uint32_t idx; + assert(wbc_range.count <= (sizeof(wbc_range.extents) / sizeof(wbc_range.extents[0]))); + for (idx = 0; idx < wbc_range.count; idx++) { + wbctotal += wbc_range.extents[idx].length; + } + kprintf("kern_direct_file(%s): wbc %qd\n", name, wbctotal); + if (wbctotal) { + target = wbc_range.dev; + } + } + + if (set_file_size) { + if (wbctotal) { + if (wbctotal >= set_file_size) { + set_file_size = HIBERNATE_MIN_FILE_SIZE; + } else { + set_file_size -= wbctotal; + if (set_file_size < HIBERNATE_MIN_FILE_SIZE) { + set_file_size = HIBERNATE_MIN_FILE_SIZE; + } + } + } + if (fs_free_size) { + mpFree += va.va_data_alloc; + if ((mpFree < set_file_size) || ((mpFree - set_file_size) < fs_free_size)) { + error = ENOSPC; + goto out; + } + } + error = vnode_setsize(ref->vp, set_file_size, IO_NOZEROFILL | IO_NOAUTH, ref->ctx); + if (error) { + goto out; + } + ref->filelength = set_file_size; + } + } else if ((ref->vp->v_type == VBLK) || (ref->vp->v_type == VCHR)) { + /* Partition. */ + device = va.va_rdev; + + p1 = ref->vp; + p2 = ref->ctx; + do_ioctl = &device_ioctl; + } else { + /* Don't dump to non-regular files. */ + error = EFAULT; + goto out; + } + ref->device = device; + + // probe for CF + dk_corestorage_info_t cs_info; + memset(&cs_info, 0, sizeof(dk_corestorage_info_t)); + error = do_ioctl(p1, p2, DKIOCCORESTORAGE, (caddr_t)&cs_info); + ref->cf = (error == 0) && (cs_info.flags & DK_CORESTORAGE_ENABLE_HOTFILES); + + // get block size + + error = do_ioctl(p1, p2, DKIOCGETBLOCKSIZE, (caddr_t) &ref->blksize); + if (error) { + goto out; + } + + if (ref->blksize == 4096) { + minoffset = HIBERNATE_MIN_PHYSICAL_LBA_4096 * ref->blksize; + } else { + minoffset = HIBERNATE_MIN_PHYSICAL_LBA_512 * ref->blksize; + } + + if (ref->vp->v_type != VREG) { + error = do_ioctl(p1, p2, DKIOCGETBLOCKCOUNT, (caddr_t) &fileblk); + if (error) { + goto out; + } + ref->filelength = fileblk * ref->blksize; + } + + // pin logical extents, CS version + + error = kern_ioctl_file_extents(ref, _DKIOCCSPINEXTENT, 0, ref->filelength); + if (error && (ENOTTY != error)) { + goto out; + } + ref->pinned = (error == 0); + + // pin logical extents, apfs version + + error = VNOP_IOCTL(ref->vp, FSCTL_FREEZE_EXTENTS, NULL, 0, ref->ctx); + if (error && (ENOTTY != error)) { + goto out; + } + ref->frozen = (error == 0); + + // generate the block list + + error = do_ioctl(p1, p2, DKIOCLOCKPHYSICALEXTENTS, NULL); + if (error) { + goto out; + } + locked = TRUE; + + f_offset = 0; + for (; f_offset < ref->filelength; f_offset += filechunk) { + if (ref->vp->v_type == VREG) { + filechunk = 1 * 1024 * 1024 * 1024; + daddr64_t blkno; + + error = VNOP_BLOCKMAP(ref->vp, f_offset, filechunk, &blkno, + &filechunk, NULL, VNODE_WRITE | VNODE_BLOCKMAP_NO_TRACK, NULL); + if (error) { + goto out; + } + if (-1LL == blkno) { + continue; + } + fileblk = blkno * ref->blksize; + } else if ((ref->vp->v_type == VBLK) || (ref->vp->v_type == VCHR)) { + fileblk = f_offset; + filechunk = f_offset ? 0 : (unsigned long)ref->filelength; } - } - error = vnode_setsize(ref->vp, set_file_size, IO_NOZEROFILL | IO_NOAUTH, ref->ctx); - if (error) goto out; - ref->filelength = set_file_size; - } - } - else if ((ref->vp->v_type == VBLK) || (ref->vp->v_type == VCHR)) - { - /* Partition. */ - device = va.va_rdev; - - p1 = ref->vp; - p2 = ref->ctx; - do_ioctl = &device_ioctl; - } - else - { - /* Don't dump to non-regular files. */ - error = EFAULT; - goto out; - } - ref->device = device; - - // probe for CF - dk_corestorage_info_t cs_info; - memset(&cs_info, 0, sizeof(dk_corestorage_info_t)); - error = do_ioctl(p1, p2, DKIOCCORESTORAGE, (caddr_t)&cs_info); - ref->cf = (error == 0) && (cs_info.flags & DK_CORESTORAGE_ENABLE_HOTFILES); - - // get block size - - error = do_ioctl(p1, p2, DKIOCGETBLOCKSIZE, (caddr_t) &ref->blksize); - if (error) - goto out; - - if (ref->vp->v_type != VREG) - { - error = do_ioctl(p1, p2, DKIOCGETBLOCKCOUNT, (caddr_t) &fileblk); - if (error) goto out; - ref->filelength = fileblk * ref->blksize; - } - - // pin logical extents - - error = kern_ioctl_file_extents(ref, _DKIOCCSPINEXTENT, 0, ref->filelength); - if (error && (ENOTTY != error)) goto out; - ref->pinned = (error == 0); - - // generate the block list - - error = do_ioctl(p1, p2, DKIOCLOCKPHYSICALEXTENTS, NULL); - if (error) goto out; - locked = TRUE; - - f_offset = 0; - for (; f_offset < ref->filelength; f_offset += filechunk) - { - if (ref->vp->v_type == VREG) - { - filechunk = 1*1024*1024*1024; - daddr64_t blkno; - - error = VNOP_BLOCKMAP(ref->vp, f_offset, filechunk, &blkno, - &filechunk, NULL, VNODE_WRITE, NULL); - if (error) goto out; - if (-1LL == blkno) continue; - fileblk = blkno * ref->blksize; - } - else if ((ref->vp->v_type == VBLK) || (ref->vp->v_type == VCHR)) - { - fileblk = f_offset; - filechunk = f_offset ? 0 : ref->filelength; - } - - physoffset = 0; - while (physoffset < filechunk) - { - dk_physical_extent_t getphysreq; - bzero(&getphysreq, sizeof(getphysreq)); - - getphysreq.offset = fileblk + physoffset; - getphysreq.length = (filechunk - physoffset); - error = do_ioctl(p1, p2, DKIOCGETPHYSICALEXTENT, (caddr_t) &getphysreq); - if (error) goto out; - if (!target) - { - target = getphysreq.dev; - } - else if (target != getphysreq.dev) - { - error = ENOTSUP; - goto out; - } + + physoffset = 0; + while (physoffset < filechunk) { + dk_physical_extent_t getphysreq; + bzero(&getphysreq, sizeof(getphysreq)); + + getphysreq.offset = fileblk + physoffset; + getphysreq.length = (filechunk - physoffset); + error = do_ioctl(p1, p2, DKIOCGETPHYSICALEXTENT, (caddr_t) &getphysreq); + if (error) { + goto out; + } + if (!target) { + target = getphysreq.dev; + } else if (target != getphysreq.dev) { + error = ENOTSUP; + goto out; + } + + assert(getphysreq.offset >= minoffset); + #if HIBFRAGMENT - uint64_t rev; - for (rev = 4096; rev <= getphysreq.length; rev += 4096) - { - callback(callback_ref, getphysreq.offset + getphysreq.length - rev, 4096); - } + uint64_t rev; + for (rev = 4096; rev <= getphysreq.length; rev += 4096) { + callback(callback_ref, getphysreq.offset + getphysreq.length - rev, 4096); + } #else - callback(callback_ref, getphysreq.offset, getphysreq.length); + callback(callback_ref, getphysreq.offset, getphysreq.length); #endif - physoffset += getphysreq.length; - } - } - callback(callback_ref, 0ULL, 0ULL); - - if (ref->vp->v_type == VREG) p1 = ⌖ - else - { - p1 = ⌖ - p2 = p; - do_ioctl = &file_ioctl; - } - - // get partition base - - if (partitionbase_result) - { - error = do_ioctl(p1, p2, DKIOCGETBASE, (caddr_t) partitionbase_result); - if (error) - goto out; - } - - // get block size & constraints - - error = do_ioctl(p1, p2, DKIOCGETBLOCKSIZE, (caddr_t) &blksize); - if (error) - goto out; - - maxiocount = 1*1024*1024*1024; - - error = do_ioctl(p1, p2, DKIOCGETMAXBLOCKCOUNTREAD, (caddr_t) &count); - if (error) - count = 0; - count *= blksize; - if (count && (count < maxiocount)) - maxiocount = count; - - error = do_ioctl(p1, p2, DKIOCGETMAXBLOCKCOUNTWRITE, (caddr_t) &count); - if (error) - count = 0; - count *= blksize; - if (count && (count < maxiocount)) - maxiocount = count; - - error = do_ioctl(p1, p2, DKIOCGETMAXBYTECOUNTREAD, (caddr_t) &count); - if (error) - count = 0; - if (count && (count < maxiocount)) - maxiocount = count; - - error = do_ioctl(p1, p2, DKIOCGETMAXBYTECOUNTWRITE, (caddr_t) &count); - if (error) - count = 0; - if (count && (count < maxiocount)) - maxiocount = count; - - error = do_ioctl(p1, p2, DKIOCGETMAXSEGMENTBYTECOUNTREAD, (caddr_t) &count); - if (!error) - error = do_ioctl(p1, p2, DKIOCGETMAXSEGMENTCOUNTREAD, (caddr_t) &segcount); - if (error) - count = segcount = 0; - count *= segcount; - if (count && (count < maxiocount)) - maxiocount = count; - - error = do_ioctl(p1, p2, DKIOCGETMAXSEGMENTBYTECOUNTWRITE, (caddr_t) &count); - if (!error) - error = do_ioctl(p1, p2, DKIOCGETMAXSEGMENTCOUNTWRITE, (caddr_t) &segcount); - if (error) - count = segcount = 0; - count *= segcount; - if (count && (count < maxiocount)) - maxiocount = count; - - kprintf("max io 0x%qx bytes\n", maxiocount); - if (maxiocount_result) - *maxiocount_result = maxiocount; - - error = do_ioctl(p1, p2, DKIOCISSOLIDSTATE, (caddr_t)&isssd); - if (!error && isssd) - flags |= kIOPolledFileSSD; - - if (partition_device_result) - *partition_device_result = device; - if (image_device_result) - *image_device_result = target; - if (oflags) - *oflags = flags; - - if ((ref->vp->v_type == VBLK) || (ref->vp->v_type == VCHR)) - { - vnode_close(ref->vp, FWRITE, ref->ctx); - ref->vp = NULLVP; - vfs_context_rele(ref->ctx); - ref->ctx = NULL; - } + physoffset += getphysreq.length; + } + } + if (ref->wbcranged) { + uint32_t idx; + for (idx = 0; idx < wbc_range.count; idx++) { + assert(wbc_range.extents[idx].offset >= minoffset); + callback(callback_ref, wbc_range.extents[idx].offset, wbc_range.extents[idx].length); + } + } + callback(callback_ref, 0ULL, 0ULL); + + if (ref->vp->v_type == VREG) { + p1 = ⌖ + } else { + p1 = ⌖ + p2 = p; + do_ioctl = &file_ioctl; + } + + // get partition base + + if (partitionbase_result) { + error = do_ioctl(p1, p2, DKIOCGETBASE, (caddr_t) partitionbase_result); + if (error) { + goto out; + } + } + + // get block size & constraints + + error = do_ioctl(p1, p2, DKIOCGETBLOCKSIZE, (caddr_t) &blksize); + if (error) { + goto out; + } + + maxiocount = 1 * 1024 * 1024 * 1024; + + error = do_ioctl(p1, p2, DKIOCGETMAXBLOCKCOUNTREAD, (caddr_t) &count); + if (error) { + count = 0; + } + count *= blksize; + if (count && (count < maxiocount)) { + maxiocount = count; + } + + error = do_ioctl(p1, p2, DKIOCGETMAXBLOCKCOUNTWRITE, (caddr_t) &count); + if (error) { + count = 0; + } + count *= blksize; + if (count && (count < maxiocount)) { + maxiocount = count; + } + + error = do_ioctl(p1, p2, DKIOCGETMAXBYTECOUNTREAD, (caddr_t) &count); + if (error) { + count = 0; + } + if (count && (count < maxiocount)) { + maxiocount = count; + } + + error = do_ioctl(p1, p2, DKIOCGETMAXBYTECOUNTWRITE, (caddr_t) &count); + if (error) { + count = 0; + } + if (count && (count < maxiocount)) { + maxiocount = count; + } + + error = do_ioctl(p1, p2, DKIOCGETMAXSEGMENTBYTECOUNTREAD, (caddr_t) &count); + if (!error) { + error = do_ioctl(p1, p2, DKIOCGETMAXSEGMENTCOUNTREAD, (caddr_t) &segcount); + } + if (error) { + count = segcount = 0; + } + count *= segcount; + if (count && (count < maxiocount)) { + maxiocount = count; + } + + error = do_ioctl(p1, p2, DKIOCGETMAXSEGMENTBYTECOUNTWRITE, (caddr_t) &count); + if (!error) { + error = do_ioctl(p1, p2, DKIOCGETMAXSEGMENTCOUNTWRITE, (caddr_t) &segcount); + } + if (error) { + count = segcount = 0; + } + count *= segcount; + if (count && (count < maxiocount)) { + maxiocount = count; + } + + kprintf("max io 0x%qx bytes\n", maxiocount); + if (maxiocount_result) { + *maxiocount_result = maxiocount; + } + + error = do_ioctl(p1, p2, DKIOCISSOLIDSTATE, (caddr_t)&isssd); + if (!error && isssd) { + flags |= kIOPolledFileSSD; + } + + if (partition_device_result) { + *partition_device_result = device; + } + if (image_device_result) { + *image_device_result = target; + } + if (oflags) { + *oflags = flags; + } + + if ((ref->vp->v_type == VBLK) || (ref->vp->v_type == VCHR)) { + vnode_close(ref->vp, FWRITE, ref->ctx); + ref->vp = NULLVP; + ref->ctx = NULL; + } out: - printf("kern_open_file_for_direct_io(%p, %d)\n", ref, error); - - if (error && locked) - { - p1 = &device; - (void) do_ioctl(p1, p2, DKIOCUNLOCKPHYSICALEXTENTS, NULL); - } - - if (error && ref) - { - if (ref->vp) - { - (void) kern_ioctl_file_extents(ref, _DKIOCCSUNPINEXTENT, 0, (ref->pinned && ref->cf) ? ref->filelength : 0); - vnode_close(ref->vp, FWRITE, ref->ctx); - ref->vp = NULLVP; - } - vfs_context_rele(ref->ctx); - kfree(ref, sizeof(struct kern_direct_file_io_ref_t)); - ref = NULL; - } + printf("kern_open_file_for_direct_io(%p, %d)\n", ref, error); + - return(ref); + if (error && locked) { + p1 = &device; + (void) do_ioctl(p1, p2, DKIOCUNLOCKPHYSICALEXTENTS, NULL); + } + + if (error && ref) { + if (ref->vp) { + (void) kern_ioctl_file_extents(ref, _DKIOCCSUNPINEXTENT, 0, (ref->pinned && ref->cf) ? ref->filelength : 0); + + if (ref->frozen) { + (void) VNOP_IOCTL(ref->vp, FSCTL_THAW_EXTENTS, NULL, 0, ref->ctx); + } + if (ref->wbcranged) { + (void) do_ioctl(p1, p2, DKIOCAPFSRELEASEWBCRANGE, (caddr_t) NULL); + } + vnode_close(ref->vp, FWRITE, ref->ctx); + ref->vp = NULLVP; + } + ref->ctx = NULL; + kfree(ref, sizeof(struct kern_direct_file_io_ref_t)); + ref = NULL; + } + + return ref; } int kern_write_file(struct kern_direct_file_io_ref_t * ref, off_t offset, void * addr, size_t len, int ioflag) { - return (vn_rdwr(UIO_WRITE, ref->vp, - addr, len, offset, - UIO_SYSSPACE, ioflag|IO_SYNC|IO_NODELOCKED|IO_UNIT, - vfs_context_ucred(ref->ctx), (int *) 0, - vfs_context_proc(ref->ctx))); + assert(len <= INT32_MAX); + return vn_rdwr(UIO_WRITE, ref->vp, + addr, (int)len, offset, + UIO_SYSSPACE, ioflag | IO_SYNC | IO_NODELOCKED | IO_UNIT, + vfs_context_ucred(ref->ctx), (int *) 0, + vfs_context_proc(ref->ctx)); } int kern_read_file(struct kern_direct_file_io_ref_t * ref, off_t offset, void * addr, size_t len, int ioflag) { - return (vn_rdwr(UIO_READ, ref->vp, - addr, len, offset, - UIO_SYSSPACE, ioflag|IO_SYNC|IO_NODELOCKED|IO_UNIT, - vfs_context_ucred(ref->ctx), (int *) 0, - vfs_context_proc(ref->ctx))); + assert(len <= INT32_MAX); + return vn_rdwr(UIO_READ, ref->vp, + addr, (int)len, offset, + UIO_SYSSPACE, ioflag | IO_SYNC | IO_NODELOCKED | IO_UNIT, + vfs_context_ucred(ref->ctx), (int *) 0, + vfs_context_proc(ref->ctx)); } struct mount * kern_file_mount(struct kern_direct_file_io_ref_t * ref) { - return (ref->vp->v_mount); + return ref->vp->v_mount; } void kern_close_file_for_direct_io(struct kern_direct_file_io_ref_t * ref, - off_t write_offset, void * addr, size_t write_length, - off_t discard_offset, off_t discard_end) + off_t write_offset, void * addr, size_t write_length, + off_t discard_offset, off_t discard_end) { - int error; - printf("kern_close_file_for_direct_io(%p)\n", ref); - - if (!ref) return; - - if (ref->vp) - { - int (*do_ioctl)(void * p1, void * p2, u_long theIoctl, caddr_t result); - void * p1; - void * p2; - - if (ref->vp->v_type == VREG) - { - p1 = &ref->device; - p2 = kernproc; - do_ioctl = &file_ioctl; - } - else - { - /* Partition. */ - p1 = ref->vp; - p2 = ref->ctx; - do_ioctl = &device_ioctl; - } - (void) do_ioctl(p1, p2, DKIOCUNLOCKPHYSICALEXTENTS, NULL); + int error; + printf("kern_close_file_for_direct_io(%p)\n", ref); + + if (!ref) { + return; + } + + if (ref->vp) { + int (*do_ioctl)(void * p1, void * p2, u_long theIoctl, caddr_t result); + void * p1; + void * p2; + + discard_offset = ((discard_offset + ref->blksize - 1) & ~(((off_t) ref->blksize) - 1)); + discard_end = ((discard_end) & ~(((off_t) ref->blksize) - 1)); + + if (ref->vp->v_type == VREG) { + p1 = &ref->device; + p2 = kernproc; + do_ioctl = &file_ioctl; + } else { + /* Partition. */ + p1 = ref->vp; + p2 = ref->ctx; + do_ioctl = &device_ioctl; + } + (void) do_ioctl(p1, p2, DKIOCUNLOCKPHYSICALEXTENTS, NULL); //XXX If unmapping extents then don't also need to unpin; except ... //XXX if file unaligned (HFS 4k / Fusion 128k) then pin is superset and @@ -605,25 +690,29 @@ kern_close_file_for_direct_io(struct kern_direct_file_io_ref_t * ref, (void) kern_ioctl_file_extents(ref, _DKIOCCSUNPINEXTENT, 0, (will_unpin) ? ref->filelength : 0); - if (will_unmap) - { - (void) kern_ioctl_file_extents(ref, DKIOCUNMAP, discard_offset, (ref->cf) ? ref->filelength : discard_end); - } + if (will_unmap) { + (void) kern_ioctl_file_extents(ref, DKIOCUNMAP, discard_offset, (ref->cf) ? ref->filelength : discard_end); + } - if (addr && write_length) - { - (void) kern_write_file(ref, write_offset, addr, write_length, 0); - } + if (ref->frozen) { + (void) VNOP_IOCTL(ref->vp, FSCTL_THAW_EXTENTS, NULL, 0, ref->ctx); + } + if (ref->wbcranged) { + (void) do_ioctl(p1, p2, DKIOCAPFSRELEASEWBCRANGE, (caddr_t) NULL); + } - error = vnode_close(ref->vp, FWRITE, ref->ctx); + if (addr && write_length) { + (void) kern_write_file(ref, write_offset, addr, write_length, IO_SKIP_ENCRYPTION); + } + + error = vnode_close(ref->vp, FWRITE, ref->ctx); + + ref->vp = NULLVP; + kprintf("vnode_close(%d)\n", error); + + } - ref->vp = NULLVP; - kprintf("vnode_close(%d)\n", error); - } - if (ref->ctx) - { - vfs_context_rele(ref->ctx); ref->ctx = NULL; - } - kfree(ref, sizeof(struct kern_direct_file_io_ref_t)); + + kfree(ref, sizeof(struct kern_direct_file_io_ref_t)); }