#include <sys/mount_internal.h> /* needs internal due to fhandle_t */
#include <sys/ubc_internal.h>
#include <sys/lock.h>
+#include <sys/disk.h> /* For DKIOC calls */
#include <mach/mach_types.h>
#include <mach/memory_object_types.h>
+#include <mach/memory_object_control.h>
+#include <mach/vm_map.h>
+#include <mach/mach_vm.h>
+#include <mach/upl.h>
#include <mach/sdt.h>
#include <vm/vm_map.h>
#include <vm/vm_protos.h>
-unsigned int vp_pagein=0;
-unsigned int vp_pgodirty=0;
-unsigned int vp_pgoclean=0;
-unsigned int dp_pgouts=0; /* Default pager pageouts */
-unsigned int dp_pgins=0; /* Default pager pageins */
+
+void
+vnode_pager_throttle()
+{
+ struct uthread *ut;
+
+ ut = get_bsdthread_info(current_thread());
+
+ if (ut->uu_lowpri_window)
+ throttle_lowpri_io(TRUE);
+}
+
+
+boolean_t
+vnode_pager_isSSD(vnode_t vp)
+{
+ if (vp->v_mount->mnt_kern_flag & MNTK_SSD)
+ return (TRUE);
+ return (FALSE);
+}
+
+
+uint32_t
+vnode_pager_isinuse(struct vnode *vp)
+{
+ if (vp->v_usecount > vp->v_kusecount)
+ return (1);
+ return (0);
+}
+
+uint32_t
+vnode_pager_return_hard_throttle_limit(struct vnode *vp, uint32_t *limit, uint32_t hard_throttle)
+{
+ return(cluster_hard_throttle_limit(vp, limit, hard_throttle));
+}
vm_object_offset_t
vnode_pager_get_filesize(struct vnode *vp)
return KERN_SUCCESS;
}
+/*
+ * vnode_trim:
+ * Used to call the DKIOCUNMAP ioctl on the underlying disk device for the specified vnode.
+ * Trims the region at offset bytes into the file, for length bytes.
+ *
+ * Care must be taken to ensure that the vnode is sufficiently reference counted at the time this
+ * function is called; no iocounts or usecounts are taken on the vnode.
+ * This function is non-idempotent in error cases; We cannot un-discard the blocks if only some of them
+ * are successfully discarded.
+ */
+u_int32_t vnode_trim (
+ struct vnode *vp,
+ off_t offset,
+ size_t length)
+{
+ daddr64_t io_blockno; /* Block number corresponding to the start of the extent */
+ size_t io_bytecount; /* Number of bytes in current extent for the specified range */
+ size_t trimmed = 0;
+ off_t current_offset = offset;
+ size_t remaining_length = length;
+ int error = 0;
+ u_int32_t blocksize = 0;
+ struct vnode *devvp;
+ dk_extent_t extent;
+ dk_unmap_t unmap;
+
+
+ /* Get the underlying device vnode */
+ devvp = vp->v_mount->mnt_devvp;
+
+ /* Figure out the underlying device block size */
+ error = VNOP_IOCTL(devvp, DKIOCGETBLOCKSIZE, (caddr_t)&blocksize, 0, vfs_context_kernel());
+ if (error) {
+ goto trim_exit;
+ }
+
+ /*
+ * We may not get the entire range from offset -> offset+length in a single
+ * extent from the blockmap call. Keep looping/going until we are sure we've hit
+ * the whole range or if we encounter an error.
+ */
+ while (trimmed < length) {
+ /*
+ * VNOP_BLOCKMAP will tell us the logical to physical block number mapping for the
+ * specified offset. It returns blocks in contiguous chunks, so if the logical range is
+ * broken into multiple extents, it must be called multiple times, increasing the offset
+ * in each call to ensure that the entire range is covered.
+ */
+ error = VNOP_BLOCKMAP (vp, current_offset, remaining_length,
+ &io_blockno, &io_bytecount, NULL, VNODE_READ, NULL);
+
+ if (error) {
+ goto trim_exit;
+ }
+ /*
+ * We have a contiguous run. Prepare & issue the ioctl for the device.
+ * the DKIOCUNMAP ioctl takes offset in bytes from the start of the device.
+ */
+ memset (&extent, 0, sizeof(dk_extent_t));
+ memset (&unmap, 0, sizeof(dk_unmap_t));
+ extent.offset = (uint64_t) io_blockno * (u_int64_t) blocksize;
+ extent.length = io_bytecount;
+ unmap.extents = &extent;
+ unmap.extentsCount = 1;
+ error = VNOP_IOCTL(devvp, DKIOCUNMAP, (caddr_t)&unmap, 0, vfs_context_kernel());
+
+ if (error) {
+ goto trim_exit;
+ }
+ remaining_length = remaining_length - io_bytecount;
+ trimmed = trimmed + io_bytecount;
+ current_offset = current_offset + io_bytecount;
+ }
+trim_exit:
+
+ return error;
+
+}
+
pager_return_t
vnode_pageout(struct vnode *vp,
upl_t upl,
- vm_offset_t upl_offset,
+ upl_offset_t upl_offset,
vm_object_offset_t f_offset,
- vm_size_t size,
+ upl_size_t size,
int flags,
int *errorp)
{
int isize;
int pg_index;
int base_index;
- int offset;
+ upl_offset_t offset;
upl_page_info_t *pl;
vfs_context_t ctx = vfs_context_current(); /* pager context */
* just go ahead and call vnop_pageout since
* it has already sorted out the dirty ranges
*/
- dp_pgouts++;
-
- KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START,
- size, 1, 0, 0, 0);
+ KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
+ (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START,
+ size, 1, 0, 0, 0);
if ( (error_ret = VNOP_PAGEOUT(vp, upl, upl_offset, (off_t)f_offset,
(size_t)size, flags, ctx)) )
result = PAGER_ERROR;
- KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END,
- size, 1, 0, 0, 0);
+ KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
+ (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END,
+ size, 1, 0, 0, 0);
goto out;
}
+ if (upl == NULL) {
+ int request_flags;
+
+ if (vp->v_mount->mnt_vtable->vfc_vfsflags & VFC_VFSVNOP_PAGEOUTV2) {
+ /*
+ * filesystem has requested the new form of VNOP_PAGEOUT for file
+ * backed objects... we will not grab the UPL befofe calling VNOP_PAGEOUT...
+ * it is the fileystem's responsibility to grab the range we're denoting
+ * via 'f_offset' and 'size' into a UPL... this allows the filesystem to first
+ * take any locks it needs, before effectively locking the pages into a UPL...
+ */
+ KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
+ (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START,
+ size, (int)f_offset, 0, 0, 0);
+
+ if ( (error_ret = VNOP_PAGEOUT(vp, NULL, upl_offset, (off_t)f_offset,
+ size, flags, ctx)) ) {
+ result = PAGER_ERROR;
+ }
+ KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
+ (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END,
+ size, 0, 0, 0, 0);
+
+ goto out;
+ }
+ if (flags & UPL_MSYNC)
+ request_flags = UPL_UBC_MSYNC | UPL_RET_ONLY_DIRTY;
+ else
+ request_flags = UPL_UBC_PAGEOUT | UPL_RET_ONLY_DIRTY;
+
+ if (ubc_create_upl(vp, f_offset, size, &upl, &pl, request_flags) != KERN_SUCCESS) {
+ result = PAGER_ERROR;
+ error_ret = EINVAL;
+ goto out;
+ }
+ upl_offset = 0;
+ } else
+ pl = ubc_upl_pageinfo(upl);
+
/*
* we come here for pageouts to 'real' files and
* for msyncs... the upl may not contain any
* through it and find the 'runs' of dirty pages
* to call VNOP_PAGEOUT on...
*/
- pl = ubc_upl_pageinfo(upl);
-
if (ubc_getsize(vp) == 0) {
/*
* if the file has been effectively deleted, then
* Note we must not sleep here if the buffer is busy - that is
* a lock inversion which causes deadlock.
*/
- vp_pgoclean++;
-
#if NFSCLIENT
if (vp->v_tag == VT_NFS)
/* check with nfs if page is OK to drop */
continue;
}
- vp_pgodirty++;
-
num_of_pages = 1;
xsize = isize - PAGE_SIZE;
}
xsize = num_of_pages * PAGE_SIZE;
- KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START,
- xsize, (int)f_offset, 0, 0, 0);
+ KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
+ (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START,
+ xsize, (int)f_offset, 0, 0, 0);
- if ( (error = VNOP_PAGEOUT(vp, upl, (vm_offset_t)offset, (off_t)f_offset,
+ if ( (error = VNOP_PAGEOUT(vp, upl, offset, (off_t)f_offset,
xsize, flags, ctx)) ) {
if (error_ret == 0)
error_ret = error;
result = PAGER_ERROR;
}
- KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END,
- xsize, 0, 0, 0, 0);
+ KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
+ (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END,
+ xsize, 0, 0, 0, 0);
f_offset += xsize;
offset += xsize;
vnode_pagein(
struct vnode *vp,
upl_t upl,
- vm_offset_t upl_offset,
+ upl_offset_t upl_offset,
vm_object_offset_t f_offset,
- vm_size_t size,
+ upl_size_t size,
int flags,
int *errorp)
{
goto out;
}
if (upl == (upl_t)NULL) {
- if (size > (MAX_UPL_SIZE * PAGE_SIZE)) {
-
- panic("vnode_pagein: size = %x\n", size);
+ flags &= ~UPL_NOCOMMIT;
+ if (size > (MAX_UPL_SIZE * PAGE_SIZE)) {
result = PAGER_ERROR;
error = PAGER_ERROR;
goto out;
}
- ubc_create_upl(vp, f_offset, size, &upl, &pl, UPL_NOBLOCK | UPL_RET_ONLY_ABSENT | UPL_SET_LITE);
+ if (vp->v_mount->mnt_vtable->vfc_vfsflags & VFC_VFSVNOP_PAGEINV2) {
+ /*
+ * filesystem has requested the new form of VNOP_PAGEIN for file
+ * backed objects... we will not grab the UPL befofe calling VNOP_PAGEIN...
+ * it is the fileystem's responsibility to grab the range we're denoting
+ * via 'f_offset' and 'size' into a UPL... this allows the filesystem to first
+ * take any locks it needs, before effectively locking the pages into a UPL...
+ * so we pass a NULL into the filesystem instead of a UPL pointer... the 'upl_offset'
+ * is used to identify the "must have" page in the extent... the filesystem is free
+ * to clip the extent to better fit the underlying FS blocksize if it desires as
+ * long as it continues to include the "must have" page... 'f_offset' + 'upl_offset'
+ * identifies that page
+ */
+ if ( (error = VNOP_PAGEIN(vp, NULL, upl_offset, (off_t)f_offset,
+ size, flags, vfs_context_current())) ) {
+ result = PAGER_ERROR;
+ error = PAGER_ERROR;
+ }
+ goto out;
+ }
+ ubc_create_upl(vp, f_offset, size, &upl, &pl, UPL_UBC_PAGEIN | UPL_RET_ONLY_ABSENT);
if (upl == (upl_t)NULL) {
-
- panic("vnode_pagein: ubc_create_upl failed\n");
-
result = PAGER_ABSENT;
error = PAGER_ABSENT;
goto out;
}
+ ubc_upl_range_needed(upl, upl_offset / PAGE_SIZE, 1);
+
upl_offset = 0;
first_pg = 0;
* are responsible for commiting/aborting it
* regardless of what the caller has passed in
*/
- flags &= ~UPL_NOCOMMIT;
must_commit = 1;
-
- vp_pagein++;
} else {
pl = ubc_upl_pageinfo(upl);
first_pg = upl_offset / PAGE_SIZE;
-
- dp_pgins++;
}
pages_in_upl = size / PAGE_SIZE;
DTRACE_VM2(pgpgin, int, pages_in_upl, (uint64_t *), NULL);
xsize = (last_pg - start_pg) * PAGE_SIZE;
xoff = start_pg * PAGE_SIZE;
- if ( (error = VNOP_PAGEIN(vp, upl, (vm_offset_t) xoff,
+ if ( (error = VNOP_PAGEIN(vp, upl, (upl_offset_t) xoff,
(off_t)f_offset + xoff,
xsize, flags, vfs_context_current())) ) {
+ /*
+ * Usually this UPL will be aborted/committed by the lower cluster layer.
+ *
+ * a) In the case of decmpfs, however, we may return an error (EAGAIN) to avoid
+ * a deadlock with another thread already inflating the file.
+ *
+ * b) In the case of content protection, EPERM is a valid error and we should respect it.
+ *
+ * In those cases, we must take care of our UPL at this layer itself.
+ */
+ if (must_commit) {
+ if(error == EAGAIN) {
+ ubc_upl_abort_range(upl, (upl_offset_t) xoff, xsize, UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_RESTART);
+ }
+#if CONFIG_PROTECT
+ if(error == EPERM) {
+ ubc_upl_abort_range(upl, (upl_offset_t) xoff, xsize, UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_ERROR);
+ }
+#endif
+ }
result = PAGER_ERROR;
error = PAGER_ERROR;
projects / apple / xnu.git / blobdiff