/*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
- *
+ *
* This file contains Original Code and/or Modifications of Original Code
* as defined in and that are subject to the Apple Public Source License
* Version 2.0 (the 'License'). You may not use this file except in
* unlawful or unlicensed copies of an Apple operating system, or to
* circumvent, violate, or enable the circumvention or violation of, any
* terms of an Apple operating system software license agreement.
- *
+ *
* Please obtain a copy of the License at
* http://www.opensource.apple.com/apsl/ and read it before using this file.
- *
+ *
* The Original Code and all software distributed under the License are
* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
* Please see the License for the specific language governing rights and
* limitations under the License.
- *
+ *
* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
*/
/* Copyright (c) 1998 Apple Computer, Inc. All rights reserved.
*
* File: bsd/kern/kern_symfile.c
*
- * This file contains creates a dummy symbol file for mach_kernel
- * based on the symbol table information passed by the
- * SecondaryLoader/PlatformExpert. This allows us to correctly
- * link other executables (drivers, etc) against the the kernel in
- * cases where the kernel image on the root device does not match
- * the live kernel. This can occur during net-booting where the
- * actual kernel image is obtained from the network via tftp rather
- * than the root device.
- *
- * If a symbol table is available, then the file /mach.sym will be
- * created containing a Mach Header and a LC_SYMTAB load command
- * followed by the the symbol table data for mach_kernel.
- *
- * NOTE: This file supports only 32 bit kernels at the present time;
- * adding support for 64 bit kernels is possible, but is not
- * necessary at the present time.
- *
* HISTORY
- *
- * .
*/
#include <mach/vm_param.h>
#include <sys/stat.h>
#include <sys/disk.h>
#include <sys/conf.h>
+#include <sys/content_protection.h>
+#include <sys/fsctl.h>
#include <mach-o/loader.h>
#include <mach-o/nlist.h>
#include <kern/kalloc.h>
#include <vm/vm_kern.h>
#include <pexpert/pexpert.h>
-#include <IOKit/IOHibernatePrivate.h>
+#include <IOKit/IOPolledInterface.h>
+
+#define HIBERNATE_MIN_PHYSICAL_LBA_512 (34)
+#define HIBERNATE_MIN_PHYSICAL_LBA_4096 (6)
+#define HIBERNATE_MIN_FILE_SIZE (1024*1024)
-extern unsigned char rootdevice[];
-extern struct mach_header _mh_execute_header;
+/* This function is called from kern_sysctl in the current process context;
+ * it is exported with the System6.0.exports, but this appears to be a legacy
+ * export, as there are no internal consumers.
+ */
+int
+get_kernel_symfile(__unused proc_t p, __unused char const **symfile);
+int
+get_kernel_symfile(__unused proc_t p, __unused char const **symfile)
+{
+ return KERN_FAILURE;
+}
-static int kernel_symfile_opened = 0;
-static int error_code = 0;
+struct kern_direct_file_io_ref_t {
+ vfs_context_t ctx;
+ struct vnode * vp;
+ dev_t device;
+ uint32_t blksize;
+ off_t filelength;
+ char cf;
+ char pinned;
+ char frozen;
+ char wbcranged;
+};
-extern int IODTGetLoaderInfo(char *key, void **infoAddr, int *infoSize);
-extern void IODTFreeLoaderInfo(char *key, void *infoAddr, int infoSize);
-/*
- * Can only operate against currently running 32 bit mach_kernel
- */
static int
-output_kernel_symbols(struct proc *p)
+file_ioctl(void * p1, void * p2, u_long theIoctl, caddr_t result)
{
- struct vnode *vp;
- kauth_cred_t cred = p->p_ucred; /* XXX unsafe */
- struct vnode_attr va;
- struct vfs_context context;
- struct load_command *cmd;
- struct mach_header *orig_mh, *mh;
- struct segment_command *orig_ds, *orig_ts, *orig_le, *sg;
- struct section *se, *const_text;
- struct symtab_command *st, *orig_st;
- struct nlist *sym;
- vm_size_t orig_mhsize, orig_st_size;
- vm_offset_t header;
- vm_size_t header_size = 0; /* out: protected by header */
- int error, error1;
- unsigned int i, j;
- caddr_t addr;
- vm_offset_t offset;
- int rc_mh, rc_sc;
-
- error = EFAULT;
-
- vp = NULL;
- header = NULL;
- orig_mh = NULL;
- orig_st = NULL;
-
- context.vc_proc = p;
- context.vc_ucred = kauth_cred_proc_ref(p);
-
- // Dispose of unnecessary gumf, the booter doesn't need to load these
- rc_mh = IODTGetLoaderInfo("Kernel-__HEADER",
- (void **)&orig_mh, &orig_mhsize);
- if (rc_mh == 0 && orig_mh)
- IODTFreeLoaderInfo("Kernel-__HEADER",
- (void *)orig_mh, round_page_32(orig_mhsize));
-
- rc_sc = IODTGetLoaderInfo("Kernel-__SYMTAB",
- (void **) &orig_st, &orig_st_size);
- if (rc_sc == 0 && orig_st)
- IODTFreeLoaderInfo("Kernel-__SYMTAB",
- (void *)orig_st, round_page_32(orig_st_size));
-
- if (cred->cr_svuid != cred->cr_ruid || cred->cr_svgid != cred->cr_rgid)
- goto out;
-
- // Check to see if the root is 'e' or 'n', is this a test for network?
- if (rootdevice[0] == 'e' && rootdevice[1] == 'n')
- goto out;
-
- if ((error = vnode_open("mach.sym", (O_CREAT | FWRITE), (S_IRUSR | S_IRGRP | S_IROTH), 0, &vp, &context)))
- goto out;
-
- /* Don't dump to non-regular files or files with links. */
- error = EFAULT;
- VATTR_INIT(&va);
- VATTR_WANTED(&va, va_nlink);
- if ((vp->v_type != VREG) || vnode_getattr(vp, &va, &context) || (va.va_nlink != 1))
- goto out;
-
- VATTR_INIT(&va); /* better to do it here than waste more stack in vnode_getsize */
- VATTR_SET(&va, va_data_size, 0);
- vnode_setattr(vp, &va, &context);
- p->p_acflag |= ACORE;
-
- // If the file type is MH_EXECUTE then this must be a kernel
- // as all Kernel extensions must be of type MH_OBJECT
- orig_ds = orig_ts = orig_le = NULL;
- orig_st = NULL;
- orig_mh = &_mh_execute_header;
- cmd = (struct load_command *) &orig_mh[1];
- for (i = 0; i < orig_mh->ncmds; i++) {
- if (cmd->cmd == LC_SEGMENT) {
- struct segment_command *orig_sg = (struct segment_command *) cmd;
-
- if (!strcmp(SEG_TEXT, orig_sg->segname))
- orig_ts = orig_sg;
- else if (!strcmp(SEG_DATA, orig_sg->segname))
- orig_ds = orig_sg;
- else if (!strcmp(SEG_LINKEDIT, orig_sg->segname))
- orig_le = orig_sg;
- }
- else if (cmd->cmd == LC_SYMTAB)
- orig_st = (struct symtab_command *) cmd;
-
- cmd = (struct load_command *) ((caddr_t) cmd + cmd->cmdsize);
- }
-
- if (!orig_ts || !orig_ds || !orig_le || !orig_st)
- goto out;
-
- const_text = NULL;
- se = (struct section *) &orig_ts[1];
- for (i = 0; i < orig_ts->nsects; i++, se++) {
- if (!strcmp("__const", se->sectname)) {
- const_text = se;
- break;
- }
- }
- if (!const_text)
- goto out;
-
- header_size = sizeof(struct mach_header)
- + orig_ts->cmdsize
- + orig_ds->cmdsize
- + sizeof(struct symtab_command);
-
- (void) kmem_alloc(kernel_map,
- (vm_offset_t *) &header,
- (vm_size_t) header_size);
- if (header)
- bzero((void *) header, header_size);
- else
- goto out;
-
- /*
- * Set up Mach-O header.
- */
- mh = (struct mach_header *) header;
- mh->magic = orig_mh->magic;
- mh->cputype = orig_mh->cputype;
- mh->cpusubtype = orig_mh->cpusubtype;
- mh->filetype = orig_mh->filetype;
- mh->ncmds = 3;
- mh->sizeofcmds = header_size - sizeof(struct mach_header);
- mh->flags = orig_mh->flags;
-
- // Initialise the current file offset and addr
- offset = round_page(header_size);
- addr = (caddr_t) const_text->addr; // Load address of __TEXT,__const
-
- /*
- * Construct a TEXT segment load command
- * the only part of the TEXT segment we keep is the __TEXT,__const
- * which contains the kernel vtables.
- */
- sg = (struct segment_command *) &mh[1];
- bcopy(orig_ts, sg, orig_ts->cmdsize);
- sg->vmaddr = (unsigned long) addr;
- sg->vmsize = const_text->size;
- sg->fileoff = 0;
- sg->filesize = const_text->size + round_page(header_size);
- sg->maxprot = 0;
- sg->initprot = 0;
- sg->flags = 0;
- se = (struct section *)(sg+1);
- for ( j = 0; j < sg->nsects; j++, se++ ) {
- se->addr = (unsigned long) addr;
- se->size = 0;
- se->offset = offset;
- se->nreloc = 0;
- if (!strcmp("__const", se->sectname)) {
- se->size = const_text->size;
- addr += const_text->size;
- offset += const_text->size;
- const_text = se;
- }
- }
- offset = round_page(offset);
-
- // Now copy of the __DATA segment load command, the image need
- // not be stored to disk nobody needs it, yet!
- sg = (struct segment_command *)((int)sg + sg->cmdsize);
- bcopy(orig_ds, sg, orig_ds->cmdsize);
-
- sg->vmaddr = (unsigned long) addr;
- sg->vmsize = 0x1000; // One page for some reason?
- sg->fileoff = offset;
- sg->filesize = 0;
- sg->maxprot = 0;
- sg->initprot = 0;
- sg->flags = 0;
- se = (struct section *)(sg+1);
- for ( j = 0; j < sg->nsects; j++, se++ ) {
- se->addr = (unsigned long) addr;
- se->size = 0;
- se->offset = offset;
- se->nreloc = 0;
- }
- offset = round_page(offset);
-
-
- /*
- * Set up LC_SYMTAB command
- */
- st = (struct symtab_command *)((int)sg + sg->cmdsize);
- st->cmd = LC_SYMTAB;
- st->cmdsize = sizeof(struct symtab_command);
- st->symoff = offset;
- st->nsyms = orig_st->nsyms;
- st->strsize = orig_st->strsize;
- st->stroff = offset + st->nsyms * sizeof(struct nlist);
-
- /*
- * Convert the symbol table in place from section references
- * to absolute references.
- */
- sym = (struct nlist *) orig_le->vmaddr;
- for (i = 0; i < st->nsyms; i++, sym++ ) {
- if ( (sym->n_type & N_TYPE) == N_SECT) {
- sym->n_sect = NO_SECT;
- sym->n_type = (sym->n_type & ~N_TYPE) | N_ABS;
- }
- }
-
- /*
- * Write out the load commands at the beginning of the file.
- */
- error = vn_rdwr(UIO_WRITE, vp, (caddr_t) mh, header_size, (off_t) 0,
- UIO_SYSSPACE32, IO_NODELOCKED|IO_UNIT, cred, (int *) 0, p);
- if (error)
- goto out;
-
- /*
- * Write out the __TEXT,__const data segment.
- */
- error = vn_rdwr(UIO_WRITE, vp, (caddr_t) const_text->addr,
- const_text->size, const_text->offset,
- UIO_SYSSPACE32, IO_NODELOCKED|IO_UNIT, cred, (int *) 0, p);
- if (error)
- goto out;
-
- /*
- * Write out kernel symbols
- */
- offset = st->nsyms * sizeof(struct nlist) + st->strsize; // symtab size
- error = vn_rdwr(UIO_WRITE, vp,
- (caddr_t) orig_le->vmaddr, offset, st->symoff,
- UIO_SYSSPACE32, IO_NODELOCKED|IO_UNIT, cred, (int *) 0, p);
-out:
- if (header)
- kmem_free(kernel_map, header, header_size);
-
- if (vp) {
- error1 = vnode_close(vp, FWRITE, &context);
- if (!error) error = error1;
- }
+ dev_t device = *(dev_t*) p1;
- kauth_cred_unref(&context.vc_ucred);
- return(error);
+ return (*bdevsw[major(device)].d_ioctl)
+ (device, theIoctl, result, S_IFBLK, p2);
}
-/*
- *
- */
-int get_kernel_symfile(struct proc *p, char **symfile)
+
+static int
+device_ioctl(void * p1, __unused void * p2, u_long theIoctl, caddr_t result)
{
- if (!kernel_symfile_opened) {
- kernel_symfile_opened = 1;
- error_code = output_kernel_symbols(p);
- }
- if (!error_code)
- *symfile = "\\mach.sym";
-
- return error_code;
+ return VNOP_IOCTL(p1, theIoctl, result, 0, p2);
}
-struct kern_direct_file_io_ref_t
+static int
+kern_ioctl_file_extents(struct kern_direct_file_io_ref_t * ref, u_long theIoctl, off_t offset, off_t end)
{
- struct vfs_context context;
- struct vnode *vp;
-};
+ int error = 0;
+ int (*do_ioctl)(void * p1, void * p2, u_long theIoctl, caddr_t result);
+ void * p1;
+ void * p2;
+ uint64_t fileblk;
+ size_t filechunk;
+ dk_extent_t extent;
+ dk_unmap_t unmap;
+ _dk_cs_pin_t pin;
+
+ bzero(&extent, sizeof(dk_extent_t));
+ bzero(&unmap, sizeof(dk_unmap_t));
+ bzero(&pin, sizeof(pin));
+ if (ref->vp->v_type == VREG) {
+ p1 = &ref->device;
+ p2 = kernproc;
+ do_ioctl = &file_ioctl;
+ } else {
+ /* Partition. */
+ p1 = ref->vp;
+ p2 = ref->ctx;
+ do_ioctl = &device_ioctl;
+ }
+ if (_DKIOCCSPINEXTENT == theIoctl) {
+ /* Tell CS the image size, so it knows whether to place the subsequent pins SSD/HDD */
+ pin.cp_extent.length = end;
+ pin.cp_flags = _DKIOCCSHIBERNATEIMGSIZE;
+ (void) do_ioctl(p1, p2, _DKIOCCSPINEXTENT, (caddr_t)&pin);
+ } else if (_DKIOCCSUNPINEXTENT == theIoctl) {
+ /* Tell CS hibernation is done, so it can stop blocking overlapping writes */
+ pin.cp_flags = _DKIOCCSPINDISCARDBLACKLIST;
+ (void) do_ioctl(p1, p2, _DKIOCCSUNPINEXTENT, (caddr_t)&pin);
+ }
-static int file_ioctl(void * p1, void * p2, int theIoctl, caddr_t result)
-{
- dev_t device = (dev_t) p1;
+ for (; offset < end; offset += filechunk) {
+ if (ref->vp->v_type == VREG) {
+ daddr64_t blkno;
+ filechunk = 1 * 1024 * 1024 * 1024;
+ if (filechunk > (size_t)(end - offset)) {
+ filechunk = (size_t)(end - offset);
+ }
+ error = VNOP_BLOCKMAP(ref->vp, offset, filechunk, &blkno,
+ &filechunk, NULL, VNODE_WRITE | VNODE_BLOCKMAP_NO_TRACK, NULL);
+ if (error) {
+ break;
+ }
+ if (-1LL == blkno) {
+ continue;
+ }
+ fileblk = blkno * ref->blksize;
+ } else if ((ref->vp->v_type == VBLK) || (ref->vp->v_type == VCHR)) {
+ fileblk = offset;
+ filechunk = (unsigned long)((ref->filelength > ULONG_MAX) ? ULONG_MAX: ref->filelength);
+ }
- return ((*bdevsw[major(device)].d_ioctl)
- (device, theIoctl, result, S_IFBLK, p2));
-}
+ if (DKIOCUNMAP == theIoctl) {
+ extent.offset = fileblk;
+ extent.length = filechunk;
+ unmap.extents = &extent;
+ unmap.extentsCount = 1;
+ error = do_ioctl(p1, p2, theIoctl, (caddr_t)&unmap);
+// printf("DKIOCUNMAP(%d) 0x%qx, 0x%qx\n", error, extent.offset, extent.length);
+ } else if (_DKIOCCSPINEXTENT == theIoctl) {
+ pin.cp_extent.offset = fileblk;
+ pin.cp_extent.length = filechunk;
+ pin.cp_flags = _DKIOCCSPINFORHIBERNATION;
+ error = do_ioctl(p1, p2, theIoctl, (caddr_t)&pin);
+ if (error && (ENOTTY != error)) {
+ printf("_DKIOCCSPINEXTENT(%d) 0x%qx, 0x%qx\n", error, pin.cp_extent.offset, pin.cp_extent.length);
+ }
+ } else if (_DKIOCCSUNPINEXTENT == theIoctl) {
+ pin.cp_extent.offset = fileblk;
+ pin.cp_extent.length = filechunk;
+ pin.cp_flags = _DKIOCCSPINFORHIBERNATION;
+ error = do_ioctl(p1, p2, theIoctl, (caddr_t)&pin);
+ if (error && (ENOTTY != error)) {
+ printf("_DKIOCCSUNPINEXTENT(%d) 0x%qx, 0x%qx\n", error, pin.cp_extent.offset, pin.cp_extent.length);
+ }
+ } else {
+ error = EINVAL;
+ }
-static int device_ioctl(void * p1, __unused void * p2, int theIoctl, caddr_t result)
-{
- return (VNOP_IOCTL(p1, theIoctl, result, 0, p2));
+ if (error) {
+ break;
+ }
+ }
+ return error;
}
+extern uint32_t freespace_mb(vnode_t vp);
+
struct kern_direct_file_io_ref_t *
-kern_open_file_for_direct_io(const char * name,
- kern_get_file_extents_callback_t callback,
- void * callback_ref,
- dev_t * device_result,
- uint64_t * partitionbase_result,
- uint64_t * maxiocount_result)
+kern_open_file_for_direct_io(const char * name,
+ uint32_t iflags,
+ kern_get_file_extents_callback_t callback,
+ void * callback_ref,
+ off_t set_file_size,
+ off_t fs_free_size,
+ off_t write_file_offset,
+ void * write_file_addr,
+ size_t write_file_len,
+ dev_t * partition_device_result,
+ dev_t * image_device_result,
+ uint64_t * partitionbase_result,
+ uint64_t * maxiocount_result,
+ uint32_t * oflags)
{
- struct kern_direct_file_io_ref_t * ref;
-
- struct proc *p;
- struct vnode_attr va;
- int error;
- off_t f_offset;
- uint32_t blksize;
- uint64_t size;
- dev_t device;
- off_t maxiocount, count;
-
- int (*do_ioctl)(void * p1, void * p2, int theIoctl, caddr_t result);
- void * p1;
- void * p2;
-
- error = EFAULT;
-
- ref = (struct kern_direct_file_io_ref_t *) kalloc(sizeof(struct kern_direct_file_io_ref_t));
- if (!ref)
- {
+ struct kern_direct_file_io_ref_t * ref;
+
+ proc_t p;
+ struct vnode_attr va;
+ dk_apfs_wbc_range_t wbc_range;
+ int error;
+ off_t f_offset;
+ uint64_t fileblk;
+ size_t filechunk;
+ uint64_t physoffset, minoffset;
+ dev_t device;
+ dev_t target = 0;
+ int isssd = 0;
+ uint32_t flags = 0;
+ uint32_t blksize;
+ off_t maxiocount, count, segcount, wbctotal;
+ boolean_t locked = FALSE;
+ int fmode;
+ mode_t cmode;
+ struct nameidata nd;
+ u_int32_t ndflags;
+ off_t mpFree;
+
+ int (*do_ioctl)(void * p1, void * p2, u_long theIoctl, caddr_t result);
+ void * p1 = NULL;
+ void * p2 = NULL;
+
error = EFAULT;
- goto out;
- }
-
- ref->vp = NULL;
- p = current_proc(); // kernproc;
- ref->context.vc_proc = p;
- ref->context.vc_ucred = kauth_cred_proc_ref(p);
-
- if ((error = vnode_open(name, (O_CREAT | FWRITE), (0), 0, &ref->vp, &ref->context)))
- goto out;
-
- VATTR_INIT(&va);
- VATTR_WANTED(&va, va_rdev);
- VATTR_WANTED(&va, va_fsid);
- VATTR_WANTED(&va, va_data_size);
- VATTR_WANTED(&va, va_nlink);
- error = EFAULT;
- if (vnode_getattr(ref->vp, &va, &ref->context))
- goto out;
-
- kprintf("vp va_rdev major %d minor %d\n", major(va.va_rdev), minor(va.va_rdev));
- kprintf("vp va_fsid major %d minor %d\n", major(va.va_fsid), minor(va.va_fsid));
- kprintf("vp size %qd\n", va.va_data_size);
-
- if (ref->vp->v_type == VREG)
- {
- /* Don't dump files with links. */
- if (va.va_nlink != 1)
- goto out;
-
- device = va.va_fsid;
- p1 = (void *) device;
- p2 = p;
- do_ioctl = &file_ioctl;
- }
- else if ((ref->vp->v_type == VBLK) || (ref->vp->v_type == VCHR))
- {
- /* Partition. */
- device = va.va_rdev;
-
- p1 = ref->vp;
- p2 = &ref->context;
- do_ioctl = &device_ioctl;
- }
- else
- {
- /* Don't dump to non-regular files. */
+
+ ref = (struct kern_direct_file_io_ref_t *) kalloc(sizeof(struct kern_direct_file_io_ref_t));
+ if (!ref) {
+ error = EFAULT;
+ goto out;
+ }
+
+ bzero(ref, sizeof(*ref));
+ p = kernproc;
+ ref->ctx = vfs_context_kernel();
+
+ fmode = (kIOPolledFileCreate & iflags) ? (O_CREAT | FWRITE) : FWRITE;
+ cmode = S_IRUSR | S_IWUSR;
+ ndflags = NOFOLLOW;
+ NDINIT(&nd, LOOKUP, OP_OPEN, ndflags, UIO_SYSSPACE, CAST_USER_ADDR_T(name), ref->ctx);
+ VATTR_INIT(&va);
+ VATTR_SET(&va, va_mode, cmode);
+ VATTR_SET(&va, va_dataprotect_flags, VA_DP_RAWENCRYPTED);
+ VATTR_SET(&va, va_dataprotect_class, PROTECTION_CLASS_D);
+ if ((error = vn_open_auth(&nd, &fmode, &va))) {
+ kprintf("vn_open_auth(fmode: %d, cmode: %d) failed with error: %d\n", fmode, cmode, error);
+ goto out;
+ }
+
+ ref->vp = nd.ni_vp;
+ if (ref->vp->v_type == VREG) {
+ vnode_lock_spin(ref->vp);
+ SET(ref->vp->v_flag, VSWAP);
+ vnode_unlock(ref->vp);
+ }
+
+ if (write_file_addr && write_file_len) {
+ if ((error = kern_write_file(ref, write_file_offset, write_file_addr, write_file_len, IO_SKIP_ENCRYPTION))) {
+ kprintf("kern_write_file() failed with error: %d\n", error);
+ goto out;
+ }
+ }
+
+ VATTR_INIT(&va);
+ VATTR_WANTED(&va, va_rdev);
+ VATTR_WANTED(&va, va_fsid);
+ VATTR_WANTED(&va, va_devid);
+ VATTR_WANTED(&va, va_data_size);
+ VATTR_WANTED(&va, va_data_alloc);
+ VATTR_WANTED(&va, va_nlink);
error = EFAULT;
- goto out;
- }
-
- // get partition base
-
- error = do_ioctl(p1, p2, DKIOCGETBASE, (caddr_t) partitionbase_result);
- if (error)
- goto out;
-
- // get block size & constraints
-
- error = do_ioctl(p1, p2, DKIOCGETBLOCKSIZE, (caddr_t) &blksize);
- if (error)
- goto out;
-
- maxiocount = 1*1024*1024*1024;
-
- error = do_ioctl(p1, p2, DKIOCGETMAXBLOCKCOUNTREAD, (caddr_t) &count);
- if (error)
- count = 0;
- count *= blksize;
- if (count && (count < maxiocount))
- maxiocount = count;
-
- error = do_ioctl(p1, p2, DKIOCGETMAXBLOCKCOUNTWRITE, (caddr_t) &count);
- if (error)
- count = 0;
- count *= blksize;
- if (count && (count < maxiocount))
- maxiocount = count;
-
- error = do_ioctl(p1, p2, DKIOCGETMAXBYTECOUNTREAD, (caddr_t) &count);
- if (error)
- count = 0;
- if (count && (count < maxiocount))
- maxiocount = count;
-
- error = do_ioctl(p1, p2, DKIOCGETMAXBYTECOUNTWRITE, (caddr_t) &count);
- if (error)
- count = 0;
- if (count && (count < maxiocount))
- maxiocount = count;
-
- error = do_ioctl(p1, p2, DKIOCGETMAXSEGMENTBYTECOUNTREAD, (caddr_t) &count);
- if (error)
- count = 0;
- if (count && (count < maxiocount))
- maxiocount = count;
-
- error = do_ioctl(p1, p2, DKIOCGETMAXSEGMENTBYTECOUNTWRITE, (caddr_t) &count);
- if (error)
- count = 0;
- if (count && (count < maxiocount))
- maxiocount = count;
-
- kprintf("max io 0x%qx bytes\n", maxiocount);
- if (maxiocount_result)
- *maxiocount_result = maxiocount;
-
- // generate the block list
-
- error = 0;
- if (ref->vp->v_type == VREG)
- {
- f_offset = 0;
- while(f_offset < (off_t) va.va_data_size)
- {
- size_t io_size = 1*1024*1024*1024;
- daddr64_t blkno;
+ if (vnode_getattr(ref->vp, &va, ref->ctx)) {
+ goto out;
+ }
+
+ wbctotal = 0;
+ mpFree = freespace_mb(ref->vp);
+ mpFree <<= 20;
+ kprintf("kern_direct_file(%s): vp size %qd, alloc %qd, mp free %qd, keep free %qd\n",
+ name, va.va_data_size, va.va_data_alloc, mpFree, fs_free_size);
+
+ if (ref->vp->v_type == VREG) {
+ /* Don't dump files with links. */
+ if (va.va_nlink != 1) {
+ goto out;
+ }
+
+ device = (VATTR_IS_SUPPORTED(&va, va_devid)) ? va.va_devid : va.va_fsid;
+ ref->filelength = va.va_data_size;
+
+ p1 = &device;
+ p2 = p;
+ do_ioctl = &file_ioctl;
+
+ if (kIOPolledFileHibernate & iflags) {
+ error = do_ioctl(p1, p2, DKIOCAPFSGETWBCRANGE, (caddr_t) &wbc_range);
+ ref->wbcranged = (error == 0);
+ }
+ if (ref->wbcranged) {
+ uint32_t idx;
+ assert(wbc_range.count <= (sizeof(wbc_range.extents) / sizeof(wbc_range.extents[0])));
+ for (idx = 0; idx < wbc_range.count; idx++) {
+ wbctotal += wbc_range.extents[idx].length;
+ }
+ kprintf("kern_direct_file(%s): wbc %qd\n", name, wbctotal);
+ if (wbctotal) {
+ target = wbc_range.dev;
+ }
+ }
+
+ if (set_file_size) {
+ if (wbctotal) {
+ if (wbctotal >= set_file_size) {
+ set_file_size = HIBERNATE_MIN_FILE_SIZE;
+ } else {
+ set_file_size -= wbctotal;
+ if (set_file_size < HIBERNATE_MIN_FILE_SIZE) {
+ set_file_size = HIBERNATE_MIN_FILE_SIZE;
+ }
+ }
+ }
+ if (fs_free_size) {
+ mpFree += va.va_data_alloc;
+ if ((mpFree < set_file_size) || ((mpFree - set_file_size) < fs_free_size)) {
+ error = ENOSPC;
+ goto out;
+ }
+ }
+ error = vnode_setsize(ref->vp, set_file_size, IO_NOZEROFILL | IO_NOAUTH, ref->ctx);
+ if (error) {
+ goto out;
+ }
+ ref->filelength = set_file_size;
+ }
+ } else if ((ref->vp->v_type == VBLK) || (ref->vp->v_type == VCHR)) {
+ /* Partition. */
+ device = va.va_rdev;
+
+ p1 = ref->vp;
+ p2 = ref->ctx;
+ do_ioctl = &device_ioctl;
+ } else {
+ /* Don't dump to non-regular files. */
+ error = EFAULT;
+ goto out;
+ }
+ ref->device = device;
+
+ // probe for CF
+ dk_corestorage_info_t cs_info;
+ memset(&cs_info, 0, sizeof(dk_corestorage_info_t));
+ error = do_ioctl(p1, p2, DKIOCCORESTORAGE, (caddr_t)&cs_info);
+ ref->cf = (error == 0) && (cs_info.flags & DK_CORESTORAGE_ENABLE_HOTFILES);
+
+ // get block size
+
+ error = do_ioctl(p1, p2, DKIOCGETBLOCKSIZE, (caddr_t) &ref->blksize);
+ if (error) {
+ goto out;
+ }
+
+ if (ref->blksize == 4096) {
+ minoffset = HIBERNATE_MIN_PHYSICAL_LBA_4096 * ref->blksize;
+ } else {
+ minoffset = HIBERNATE_MIN_PHYSICAL_LBA_512 * ref->blksize;
+ }
+
+ if (ref->vp->v_type != VREG) {
+ error = do_ioctl(p1, p2, DKIOCGETBLOCKCOUNT, (caddr_t) &fileblk);
+ if (error) {
+ goto out;
+ }
+ ref->filelength = fileblk * ref->blksize;
+ }
+
+ // pin logical extents, CS version
+
+ error = kern_ioctl_file_extents(ref, _DKIOCCSPINEXTENT, 0, ref->filelength);
+ if (error && (ENOTTY != error)) {
+ goto out;
+ }
+ ref->pinned = (error == 0);
+
+ // pin logical extents, apfs version
+
+ error = VNOP_IOCTL(ref->vp, FSCTL_FREEZE_EXTENTS, NULL, 0, ref->ctx);
+ if (error && (ENOTTY != error)) {
+ goto out;
+ }
+ ref->frozen = (error == 0);
- error = VNOP_BLOCKMAP(ref->vp, f_offset, io_size, &blkno, (size_t *)&io_size, NULL, 0, NULL);
- if (error)
+ // generate the block list
+
+ error = do_ioctl(p1, p2, DKIOCLOCKPHYSICALEXTENTS, NULL);
+ if (error) {
goto out;
- callback(callback_ref, ((uint64_t) blkno) * blksize, (uint64_t) io_size);
- f_offset += io_size;
+ }
+ locked = TRUE;
+
+ f_offset = 0;
+ for (; f_offset < ref->filelength; f_offset += filechunk) {
+ if (ref->vp->v_type == VREG) {
+ filechunk = 1 * 1024 * 1024 * 1024;
+ daddr64_t blkno;
+
+ error = VNOP_BLOCKMAP(ref->vp, f_offset, filechunk, &blkno,
+ &filechunk, NULL, VNODE_WRITE | VNODE_BLOCKMAP_NO_TRACK, NULL);
+ if (error) {
+ goto out;
+ }
+ if (-1LL == blkno) {
+ continue;
+ }
+ fileblk = blkno * ref->blksize;
+ } else if ((ref->vp->v_type == VBLK) || (ref->vp->v_type == VCHR)) {
+ fileblk = f_offset;
+ filechunk = f_offset ? 0 : (unsigned long)ref->filelength;
+ }
+
+ physoffset = 0;
+ while (physoffset < filechunk) {
+ dk_physical_extent_t getphysreq;
+ bzero(&getphysreq, sizeof(getphysreq));
+
+ getphysreq.offset = fileblk + physoffset;
+ getphysreq.length = (filechunk - physoffset);
+ error = do_ioctl(p1, p2, DKIOCGETPHYSICALEXTENT, (caddr_t) &getphysreq);
+ if (error) {
+ goto out;
+ }
+ if (!target) {
+ target = getphysreq.dev;
+ } else if (target != getphysreq.dev) {
+ error = ENOTSUP;
+ goto out;
+ }
+
+ assert(getphysreq.offset >= minoffset);
+
+#if HIBFRAGMENT
+ uint64_t rev;
+ for (rev = 4096; rev <= getphysreq.length; rev += 4096) {
+ callback(callback_ref, getphysreq.offset + getphysreq.length - rev, 4096);
+ }
+#else
+ callback(callback_ref, getphysreq.offset, getphysreq.length);
+#endif
+ physoffset += getphysreq.length;
+ }
+ }
+ if (ref->wbcranged) {
+ uint32_t idx;
+ for (idx = 0; idx < wbc_range.count; idx++) {
+ assert(wbc_range.extents[idx].offset >= minoffset);
+ callback(callback_ref, wbc_range.extents[idx].offset, wbc_range.extents[idx].length);
+ }
}
callback(callback_ref, 0ULL, 0ULL);
- }
- else if ((ref->vp->v_type == VBLK) || (ref->vp->v_type == VCHR))
- {
- error = do_ioctl(p1, p2, DKIOCGETBLOCKCOUNT, (caddr_t) &size);
- if (error)
- goto out;
- size *= blksize;
- callback(callback_ref, 0ULL, size);
- callback(callback_ref, size, 0ULL);
- }
-
- if (device_result)
- *device_result = device;
+
+ if (ref->vp->v_type == VREG) {
+ p1 = ⌖
+ } else {
+ p1 = ⌖
+ p2 = p;
+ do_ioctl = &file_ioctl;
+ }
+
+ // get partition base
+
+ if (partitionbase_result) {
+ error = do_ioctl(p1, p2, DKIOCGETBASE, (caddr_t) partitionbase_result);
+ if (error) {
+ goto out;
+ }
+ }
+
+ // get block size & constraints
+
+ error = do_ioctl(p1, p2, DKIOCGETBLOCKSIZE, (caddr_t) &blksize);
+ if (error) {
+ goto out;
+ }
+
+ maxiocount = 1 * 1024 * 1024 * 1024;
+
+ error = do_ioctl(p1, p2, DKIOCGETMAXBLOCKCOUNTREAD, (caddr_t) &count);
+ if (error) {
+ count = 0;
+ }
+ count *= blksize;
+ if (count && (count < maxiocount)) {
+ maxiocount = count;
+ }
+
+ error = do_ioctl(p1, p2, DKIOCGETMAXBLOCKCOUNTWRITE, (caddr_t) &count);
+ if (error) {
+ count = 0;
+ }
+ count *= blksize;
+ if (count && (count < maxiocount)) {
+ maxiocount = count;
+ }
+
+ error = do_ioctl(p1, p2, DKIOCGETMAXBYTECOUNTREAD, (caddr_t) &count);
+ if (error) {
+ count = 0;
+ }
+ if (count && (count < maxiocount)) {
+ maxiocount = count;
+ }
+
+ error = do_ioctl(p1, p2, DKIOCGETMAXBYTECOUNTWRITE, (caddr_t) &count);
+ if (error) {
+ count = 0;
+ }
+ if (count && (count < maxiocount)) {
+ maxiocount = count;
+ }
+
+ error = do_ioctl(p1, p2, DKIOCGETMAXSEGMENTBYTECOUNTREAD, (caddr_t) &count);
+ if (!error) {
+ error = do_ioctl(p1, p2, DKIOCGETMAXSEGMENTCOUNTREAD, (caddr_t) &segcount);
+ }
+ if (error) {
+ count = segcount = 0;
+ }
+ count *= segcount;
+ if (count && (count < maxiocount)) {
+ maxiocount = count;
+ }
+
+ error = do_ioctl(p1, p2, DKIOCGETMAXSEGMENTBYTECOUNTWRITE, (caddr_t) &count);
+ if (!error) {
+ error = do_ioctl(p1, p2, DKIOCGETMAXSEGMENTCOUNTWRITE, (caddr_t) &segcount);
+ }
+ if (error) {
+ count = segcount = 0;
+ }
+ count *= segcount;
+ if (count && (count < maxiocount)) {
+ maxiocount = count;
+ }
+
+ kprintf("max io 0x%qx bytes\n", maxiocount);
+ if (maxiocount_result) {
+ *maxiocount_result = maxiocount;
+ }
+
+ error = do_ioctl(p1, p2, DKIOCISSOLIDSTATE, (caddr_t)&isssd);
+ if (!error && isssd) {
+ flags |= kIOPolledFileSSD;
+ }
+
+ if (partition_device_result) {
+ *partition_device_result = device;
+ }
+ if (image_device_result) {
+ *image_device_result = target;
+ }
+ if (oflags) {
+ *oflags = flags;
+ }
+
+ if ((ref->vp->v_type == VBLK) || (ref->vp->v_type == VCHR)) {
+ vnode_close(ref->vp, FWRITE, ref->ctx);
+ ref->vp = NULLVP;
+ ref->ctx = NULL;
+ }
out:
- kprintf("kern_open_file_for_direct_io(%d)\n", error);
+ printf("kern_open_file_for_direct_io(%p, %d)\n", ref, error);
+
- if (error && ref) {
+ if (error && locked) {
+ p1 = &device;
+ (void) do_ioctl(p1, p2, DKIOCUNLOCKPHYSICALEXTENTS, NULL);
+ }
+
+ if (error && ref) {
if (ref->vp) {
- vnode_close(ref->vp, FWRITE, &ref->context);
+ (void) kern_ioctl_file_extents(ref, _DKIOCCSUNPINEXTENT, 0, (ref->pinned && ref->cf) ? ref->filelength : 0);
+
+ if (ref->frozen) {
+ (void) VNOP_IOCTL(ref->vp, FSCTL_THAW_EXTENTS, NULL, 0, ref->ctx);
+ }
+ if (ref->wbcranged) {
+ (void) do_ioctl(p1, p2, DKIOCAPFSRELEASEWBCRANGE, (caddr_t) NULL);
+ }
+ vnode_close(ref->vp, FWRITE, ref->ctx);
ref->vp = NULLVP;
}
-
- kauth_cred_unref(&ref->context.vc_ucred);
+ ref->ctx = NULL;
kfree(ref, sizeof(struct kern_direct_file_io_ref_t));
ref = NULL;
- }
+ }
+
+ return ref;
+}
- return(ref);
+int
+kern_write_file(struct kern_direct_file_io_ref_t * ref, off_t offset, void * addr, size_t len, int ioflag)
+{
+ assert(len <= INT32_MAX);
+ return vn_rdwr(UIO_WRITE, ref->vp,
+ addr, (int)len, offset,
+ UIO_SYSSPACE, ioflag | IO_SYNC | IO_NODELOCKED | IO_UNIT,
+ vfs_context_ucred(ref->ctx), (int *) 0,
+ vfs_context_proc(ref->ctx));
}
int
-kern_write_file(struct kern_direct_file_io_ref_t * ref, off_t offset, caddr_t addr, vm_size_t len)
+kern_read_file(struct kern_direct_file_io_ref_t * ref, off_t offset, void * addr, size_t len, int ioflag)
{
- return (vn_rdwr(UIO_WRITE, ref->vp,
- addr, len, offset,
- UIO_SYSSPACE32, IO_SYNC|IO_NODELOCKED|IO_UNIT,
- ref->context.vc_ucred, (int *) 0, ref->context.vc_proc));
+ assert(len <= INT32_MAX);
+ return vn_rdwr(UIO_READ, ref->vp,
+ addr, (int)len, offset,
+ UIO_SYSSPACE, ioflag | IO_SYNC | IO_NODELOCKED | IO_UNIT,
+ vfs_context_ucred(ref->ctx), (int *) 0,
+ vfs_context_proc(ref->ctx));
+}
+
+
+struct mount *
+kern_file_mount(struct kern_direct_file_io_ref_t * ref)
+{
+ return ref->vp->v_mount;
}
void
-kern_close_file_for_direct_io(struct kern_direct_file_io_ref_t * ref)
+kern_close_file_for_direct_io(struct kern_direct_file_io_ref_t * ref,
+ off_t write_offset, void * addr, size_t write_length,
+ off_t discard_offset, off_t discard_end)
{
- kprintf("kern_close_file_for_direct_io\n");
+ int error;
+ printf("kern_close_file_for_direct_io(%p)\n", ref);
- if (ref) {
- int error;
+ if (!ref) {
+ return;
+ }
if (ref->vp) {
- error = vnode_close(ref->vp, FWRITE, &ref->context);
- kprintf("vnode_close(%d)\n", error);
+ int (*do_ioctl)(void * p1, void * p2, u_long theIoctl, caddr_t result);
+ void * p1;
+ void * p2;
+
+ discard_offset = ((discard_offset + ref->blksize - 1) & ~(((off_t) ref->blksize) - 1));
+ discard_end = ((discard_end) & ~(((off_t) ref->blksize) - 1));
+
+ if (ref->vp->v_type == VREG) {
+ p1 = &ref->device;
+ p2 = kernproc;
+ do_ioctl = &file_ioctl;
+ } else {
+ /* Partition. */
+ p1 = ref->vp;
+ p2 = ref->ctx;
+ do_ioctl = &device_ioctl;
+ }
+ (void) do_ioctl(p1, p2, DKIOCUNLOCKPHYSICALEXTENTS, NULL);
+
+ //XXX If unmapping extents then don't also need to unpin; except ...
+ //XXX if file unaligned (HFS 4k / Fusion 128k) then pin is superset and
+ //XXX unmap is subset, so save extra walk over file extents (and the risk
+ //XXX that CF drain starts) vs leaving partial units pinned to SSD
+ //XXX (until whatever was sharing also unmaps). Err on cleaning up fully.
+ boolean_t will_unmap = (!ref->pinned || ref->cf) && (discard_end > discard_offset);
+ boolean_t will_unpin = (ref->pinned && ref->cf /* && !will_unmap */);
+
+ (void) kern_ioctl_file_extents(ref, _DKIOCCSUNPINEXTENT, 0, (will_unpin) ? ref->filelength : 0);
+
+ if (will_unmap) {
+ (void) kern_ioctl_file_extents(ref, DKIOCUNMAP, discard_offset, (ref->cf) ? ref->filelength : discard_end);
+ }
+
+ if (ref->frozen) {
+ (void) VNOP_IOCTL(ref->vp, FSCTL_THAW_EXTENTS, NULL, 0, ref->ctx);
+ }
+ if (ref->wbcranged) {
+ (void) do_ioctl(p1, p2, DKIOCAPFSRELEASEWBCRANGE, (caddr_t) NULL);
+ }
+
+ if (addr && write_length) {
+ (void) kern_write_file(ref, write_offset, addr, write_length, IO_SKIP_ENCRYPTION);
+ }
+
+ error = vnode_close(ref->vp, FWRITE, ref->ctx);
+
ref->vp = NULLVP;
+ kprintf("vnode_close(%d)\n", error);
+
}
- kauth_cred_unref(&ref->context.vc_ucred);
+
+ ref->ctx = NULL;
+
kfree(ref, sizeof(struct kern_direct_file_io_ref_t));
- }
}
projects / apple / xnu.git / blobdiff