xnu-7195.101.1.tar.gz

[apple/xnu.git] / bsd / vm / vnode_pager.c

diff --git a/bsd/vm/vnode_pager.c b/bsd/vm/vnode_pager.c
index f86ba0148f42e2d61b22a07672f8adf17919d25f..fc3426fd5976c661098acdda09a13b303fc6f683 100644 (file)
--- a/bsd/vm/vnode_pager.c
+++ b/bsd/vm/vnode_pager.c
@@ -1,8 +1,8 @@
 /*
 /*

- * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2020 Apple Inc. All rights reserved.

  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@

- * 
+ *

  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in
  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in


@@ -11,10 +11,10 @@
  * unlawful or unlicensed copies of an Apple operating system, or to
  * circumvent, violate, or enable the circumvention or violation of, any
  * terms of an Apple operating system software license agreement.
  * unlawful or unlicensed copies of an Apple operating system, or to
  * circumvent, violate, or enable the circumvention or violation of, any
  * terms of an Apple operating system software license agreement.

- * 
+ *

  * Please obtain a copy of the License at
  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  * Please obtain a copy of the License at
  * http://www.opensource.apple.com/apsl/ and read it before using this file.

- * 
+ *

  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,


@@ -22,10 +22,10 @@
  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  * Please see the License for the specific language governing rights and
  * limitations under the License.
  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  * Please see the License for the specific language governing rights and
  * limitations under the License.

- * 
+ *

  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */

-/* 
+/*

  * Mach Operating System
  * Copyright (c) 1987 Carnegie-Mellon University
  * All rights reserved.  The CMU software License Agreement specifies
  * Mach Operating System
  * Copyright (c) 1987 Carnegie-Mellon University
  * All rights reserved.  The CMU software License Agreement specifies


@@ -49,14 +49,13 @@
 #include <sys/uio.h>
 #include <sys/vnode_internal.h>
 #include <sys/namei.h>
 #include <sys/uio.h>
 #include <sys/vnode_internal.h>
 #include <sys/namei.h>

-#include <sys/mount_internal.h>        /* needs internal due to fhandle_t */
+#include <sys/mount_internal.h> /* needs internal due to fhandle_t */

 #include <sys/ubc_internal.h>
 #include <sys/lock.h>
 #include <sys/ubc_internal.h>
 #include <sys/lock.h>

-#include <sys/disk.h>          /* For DKIOC calls */
+#include <sys/disk.h>           /* For DKIOC calls */

 
 #include <mach/mach_types.h>
 #include <mach/memory_object_types.h>
 
 #include <mach/mach_types.h>
 #include <mach/memory_object_types.h>

-#include <mach/memory_object_control.h>

 #include <mach/vm_map.h>
 #include <mach/mach_vm.h>
 #include <mach/upl.h>
 #include <mach/vm_map.h>
 #include <mach/mach_vm.h>
 #include <mach/upl.h>


@@ -65,7 +64,6 @@
 #include <vm/vm_map.h>
 #include <vm/vm_kern.h>
 #include <kern/zalloc.h>
 #include <vm/vm_map.h>
 #include <vm/vm_kern.h>
 #include <kern/zalloc.h>

-#include <kern/kalloc.h>

 #include <libkern/libkern.h>
 
 #include <vm/vnode_pager.h>
 #include <libkern/libkern.h>
 
 #include <vm/vnode_pager.h>


@@ -73,93 +71,145 @@
 
 #include <kern/assert.h>
 #include <sys/kdebug.h>
 
 #include <kern/assert.h>
 #include <sys/kdebug.h>

-#include <machine/spl.h>
-
+#include <nfs/nfs_conf.h>

 #include <nfs/rpcv2.h>
 #include <nfs/nfsproto.h>
 #include <nfs/nfs.h>
 
 #include <vm/vm_protos.h>
 
 #include <nfs/rpcv2.h>
 #include <nfs/nfsproto.h>
 #include <nfs/nfs.h>
 
 #include <vm/vm_protos.h>
 

+#include <vfs/vfs_disk_conditioner.h>

 
 void
 
 void

-vnode_pager_throttle()
+vnode_pager_throttle(void)

 {
        struct uthread *ut;
 
        ut = get_bsdthread_info(current_thread());
 
 {
        struct uthread *ut;
 
        ut = get_bsdthread_info(current_thread());
 

-       if (ut->uu_lowpri_window)
-               throttle_lowpri_io(TRUE);
+       if (ut->uu_lowpri_window) {
+               throttle_lowpri_io(1);
+       }

 }
 
 }
 

-

 boolean_t
 vnode_pager_isSSD(vnode_t vp)
 {
 boolean_t
 vnode_pager_isSSD(vnode_t vp)
 {

-       if (vp->v_mount->mnt_kern_flag & MNTK_SSD)
-               return (TRUE);
-       return (FALSE);
+       return disk_conditioner_mount_is_ssd(vp->v_mount);

 }
 
 }
 

+#if CONFIG_IOSCHED
+void
+vnode_pager_issue_reprioritize_io(struct vnode *devvp, uint64_t blkno, uint32_t len, int priority)
+{
+       u_int32_t blocksize = 0;
+       dk_extent_t extent;
+       dk_set_tier_t set_tier;
+       int error = 0;
+
+       error = VNOP_IOCTL(devvp, DKIOCGETBLOCKSIZE, (caddr_t)&blocksize, 0, vfs_context_kernel());
+       if (error) {
+               return;
+       }
+
+       memset(&extent, 0, sizeof(dk_extent_t));
+       memset(&set_tier, 0, sizeof(dk_set_tier_t));
+
+       extent.offset = blkno * (u_int64_t) blocksize;
+       extent.length = len;
+
+       set_tier.extents = &extent;
+       set_tier.extentsCount = 1;
+       set_tier.tier = (uint8_t)priority;
+
+       error = VNOP_IOCTL(devvp, DKIOCSETTIER, (caddr_t)&set_tier, 0, vfs_context_kernel());
+       return;
+}
+#endif
+
+void
+vnode_pager_was_dirtied(
+       struct vnode            *vp,
+       vm_object_offset_t      s_offset,
+       vm_object_offset_t      e_offset)
+{
+       cluster_update_state(vp, s_offset, e_offset, TRUE);
+}

 
 uint32_t
 vnode_pager_isinuse(struct vnode *vp)
 {
 
 uint32_t
 vnode_pager_isinuse(struct vnode *vp)
 {

-       if (vp->v_usecount > vp->v_kusecount)
-               return (1);
-       return (0);
+       if (vp->v_usecount > vp->v_kusecount) {
+               return 1;
+       }
+       return 0;

 }
 
 uint32_t
 }
 
 uint32_t

-vnode_pager_return_hard_throttle_limit(struct vnode *vp, uint32_t *limit, uint32_t hard_throttle)
+vnode_pager_return_throttle_io_limit(struct vnode *vp, uint32_t *limit)

 {
 {

-       return(cluster_hard_throttle_limit(vp, limit, hard_throttle));
+       return cluster_throttle_io_limit(vp, limit);

 }
 
 vm_object_offset_t
 vnode_pager_get_filesize(struct vnode *vp)
 {
 }
 
 vm_object_offset_t
 vnode_pager_get_filesize(struct vnode *vp)
 {

-

        return (vm_object_offset_t) ubc_getsize(vp);
 }
 
        return (vm_object_offset_t) ubc_getsize(vp);
 }
 

+extern int safe_getpath(struct vnode *dvp, char *leafname, char *path, int _len, int *truncated_path);
+

 kern_return_t
 kern_return_t

-vnode_pager_get_pathname(
-       struct vnode    *vp,
-       char            *pathname,
-       vm_size_t       *length_p)
+vnode_pager_get_name(
+       struct vnode    *vp,
+       char            *pathname,
+       vm_size_t       pathname_len,
+       char            *filename,
+       vm_size_t       filename_len,
+       boolean_t       *truncated_path_p)

 {
 {

-       int     error, len;
-
-       len = (int) *length_p;
-       error = vn_getpath(vp, pathname, &len);
-       if (error != 0) {
-               return KERN_FAILURE;
+       *truncated_path_p = FALSE;
+       if (pathname != NULL) {
+               /* get the path name */
+               safe_getpath(vp, NULL,
+                   pathname, (int) pathname_len,
+                   truncated_path_p);
+       }
+       if ((pathname == NULL || *truncated_path_p) &&
+           filename != NULL) {
+               /* get the file name */
+               const char *name;
+
+               name = vnode_getname_printable(vp);
+               strlcpy(filename, name, (size_t) filename_len);
+               vnode_putname_printable(name);

        }
        }

-       *length_p = (vm_size_t) len;

        return KERN_SUCCESS;
 }
 
 kern_return_t
        return KERN_SUCCESS;
 }
 
 kern_return_t

-vnode_pager_get_filename(
-       struct vnode    *vp,
-       const char      **filename)
+vnode_pager_get_mtime(
+       struct vnode    *vp,
+       struct timespec *current_mtime,
+       struct timespec *cs_mtime)

 {
 {

-       *filename = vp->v_name;
+       vnode_mtime(vp, current_mtime, vfs_context_current());
+       if (cs_mtime != NULL) {
+               ubc_get_cs_mtime(vp, cs_mtime);
+       }

        return KERN_SUCCESS;
 }
 
 kern_return_t
 vnode_pager_get_cs_blobs(
        return KERN_SUCCESS;
 }
 
 kern_return_t
 vnode_pager_get_cs_blobs(

-       struct vnode    *vp,
-       void            **blobs)
+       struct vnode    *vp,
+       void            **blobs)

 {
        *blobs = ubc_get_cs_blobs(vp);
        return KERN_SUCCESS;
 }
 
 {
        *blobs = ubc_get_cs_blobs(vp);
        return KERN_SUCCESS;
 }
 

-/* 
+/*

  * vnode_trim:
  * Used to call the DKIOCUNMAP ioctl on the underlying disk device for the specified vnode.
  * Trims the region at offset bytes into the file, for length bytes.
  * vnode_trim:
  * Used to call the DKIOCUNMAP ioctl on the underlying disk device for the specified vnode.
  * Trims the region at offset bytes into the file, for length bytes.


@@ -169,15 +219,16 @@ vnode_pager_get_cs_blobs(
  * This function is non-idempotent in error cases;  We cannot un-discard the blocks if only some of them
  * are successfully discarded.
  */
  * This function is non-idempotent in error cases;  We cannot un-discard the blocks if only some of them
  * are successfully discarded.
  */

-u_int32_t vnode_trim (
-               struct vnode *vp,
-               off_t offset,
-               size_t length)
+u_int32_t
+vnode_trim(
+       struct vnode *vp,
+       off_t offset,
+       size_t length)

 {
 {

-       daddr64_t io_blockno;    /* Block number corresponding to the start of the extent */
-       size_t io_bytecount;    /* Number of bytes in current extent for the specified range */
+       daddr64_t io_blockno;    /* Block number corresponding to the start of the extent */
+       size_t io_bytecount;    /* Number of bytes in current extent for the specified range */

        size_t trimmed = 0;
        size_t trimmed = 0;

-       off_t current_offset = offset; 
+       off_t current_offset = offset;

        size_t remaining_length = length;
        int error = 0;
        u_int32_t blocksize = 0;
        size_t remaining_length = length;
        int error = 0;
        u_int32_t blocksize = 0;


@@ -195,7 +246,7 @@ u_int32_t vnode_trim (
                goto trim_exit;
        }
 
                goto trim_exit;
        }
 

-       /* 
+       /*

         * We may not get the entire range from offset -> offset+length in a single
         * extent from the blockmap call.  Keep looping/going until we are sure we've hit
         * the whole range or if we encounter an error.
         * We may not get the entire range from offset -> offset+length in a single
         * extent from the blockmap call.  Keep looping/going until we are sure we've hit
         * the whole range or if we encounter an error.


@@ -203,22 +254,22 @@ u_int32_t vnode_trim (
        while (trimmed < length) {
                /*
                 * VNOP_BLOCKMAP will tell us the logical to physical block number mapping for the
        while (trimmed < length) {
                /*
                 * VNOP_BLOCKMAP will tell us the logical to physical block number mapping for the

-                * specified offset.  It returns blocks in contiguous chunks, so if the logical range is 
+                * specified offset.  It returns blocks in contiguous chunks, so if the logical range is

                 * broken into multiple extents, it must be called multiple times, increasing the offset
                 * in each call to ensure that the entire range is covered.
                 */
                 * broken into multiple extents, it must be called multiple times, increasing the offset
                 * in each call to ensure that the entire range is covered.
                 */

-               error = VNOP_BLOCKMAP (vp, current_offset, remaining_length, 
-                               &io_blockno, &io_bytecount, NULL, VNODE_READ, NULL);
+               error = VNOP_BLOCKMAP(vp, current_offset, remaining_length,
+                   &io_blockno, &io_bytecount, NULL, VNODE_READ | VNODE_BLOCKMAP_NO_TRACK, NULL);

 
                if (error) {
                        goto trim_exit;
                }
 
                if (error) {
                        goto trim_exit;
                }

-               /* 
+               /*

                 * We have a contiguous run.  Prepare & issue the ioctl for the device.
                 * the DKIOCUNMAP ioctl takes offset in bytes from the start of the device.
                 */
                 * We have a contiguous run.  Prepare & issue the ioctl for the device.
                 * the DKIOCUNMAP ioctl takes offset in bytes from the start of the device.
                 */

-               memset (&extent, 0, sizeof(dk_extent_t));
-               memset (&unmap, 0, sizeof(dk_unmap_t));
+               memset(&extent, 0, sizeof(dk_extent_t));
+               memset(&unmap, 0, sizeof(dk_unmap_t));

                extent.offset = (uint64_t) io_blockno * (u_int64_t) blocksize;
                extent.length = io_bytecount;
                unmap.extents = &extent;
                extent.offset = (uint64_t) io_blockno * (u_int64_t) blocksize;
                extent.length = io_bytecount;
                unmap.extents = &extent;


@@ -235,33 +286,52 @@ u_int32_t vnode_trim (
 trim_exit:
 
        return error;
 trim_exit:
 
        return error;

-

 }
 
 pager_return_t
 vnode_pageout(struct vnode *vp,
 }
 
 pager_return_t
 vnode_pageout(struct vnode *vp,

-       upl_t                   upl,
-       upl_offset_t            upl_offset,
-       vm_object_offset_t      f_offset,
-       upl_size_t              size,
-       int                     flags,
-       int                     *errorp)
+    upl_t                   upl,
+    upl_offset_t            upl_offset,
+    vm_object_offset_t      f_offset,
+    upl_size_t              size,
+    int                     flags,
+    int                     *errorp)

 {
 {

-       int             result = PAGER_SUCCESS;
-       int             error = 0;
-       int             error_ret = 0;
+       int             result = PAGER_SUCCESS;
+       int             error = 0;
+       int             error_ret = 0;

        daddr64_t blkno;
        int isize;
        int pg_index;
        int base_index;
        upl_offset_t offset;
        upl_page_info_t *pl;
        daddr64_t blkno;
        int isize;
        int pg_index;
        int base_index;
        upl_offset_t offset;
        upl_page_info_t *pl;

-       vfs_context_t ctx = vfs_context_current();      /* pager context */
+       vfs_context_t ctx = vfs_context_current();      /* pager context */

 
        isize = (int)size;
 
 
        isize = (int)size;
 

+       /*
+        * This call is non-blocking and does not ever fail but it can
+        * only be made when there is other explicit synchronization
+        * with reclaiming of the vnode which, in this path, is provided
+        * by the paging in progress counter.
+        *
+        * In addition, this may also be entered via explicit ubc_msync
+        * calls or vm_swapfile_io where the existing iocount provides
+        * the necessary synchronization. Ideally we would not take an
+        * additional iocount here in the cases where an explcit iocount
+        * has already been taken but this call doesn't cause a deadlock
+        * as other forms of vnode_get* might if this thread has already
+        * taken an iocount.
+        */
+       error = vnode_getalways_from_pager(vp);
+       if (error != 0) {
+               /* This can't happen */
+               panic("vnode_getalways returned %d for vp %p", error, vp);
+       }
+

        if (isize <= 0) {
        if (isize <= 0) {

-               result    = PAGER_ERROR;
+               result    = PAGER_ERROR;

                error_ret = EINVAL;
                goto out;
        }
                error_ret = EINVAL;
                goto out;
        }


@@ -270,32 +340,34 @@ vnode_pageout(struct vnode *vp,
                result    = PAGER_ERROR;
                error_ret = EINVAL;
 
                result    = PAGER_ERROR;
                error_ret = EINVAL;
 

-               if (upl && !(flags & UPL_NOCOMMIT))
-                       ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY);
+               if (upl && !(flags & UPL_NOCOMMIT)) {
+                       ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY);
+               }

                goto out;
        }
                goto out;
        }

-       if ( !(flags & UPL_VNODE_PAGER)) {
+       if (!(flags & UPL_VNODE_PAGER)) {

                /*
                 * This is a pageout from the default pager,
                 * just go ahead and call vnop_pageout since
                 * it has already sorted out the dirty ranges
                 */
                KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
                /*
                 * This is a pageout from the default pager,
                 * just go ahead and call vnop_pageout since
                 * it has already sorted out the dirty ranges
                 */
                KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,

-                       (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START, 
-                       size, 1, 0, 0, 0);
+                   (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START,
+                   size, 1, 0, 0, 0);

 
 

-               if ( (error_ret = VNOP_PAGEOUT(vp, upl, upl_offset, (off_t)f_offset,
-                                              (size_t)size, flags, ctx)) )
+               if ((error_ret = VNOP_PAGEOUT(vp, upl, upl_offset, (off_t)f_offset,
+                   (size_t)size, flags, ctx))) {

                        result = PAGER_ERROR;
                        result = PAGER_ERROR;

+               }

 
                KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
 
                KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,

-                       (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END, 
-                       size, 1, 0, 0, 0);
+                   (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END,
+                   size, 1, 0, 0, 0);

 
                goto out;
        }
        if (upl == NULL) {
 
                goto out;
        }
        if (upl == NULL) {

-               int                     request_flags;
+               int                     request_flags;

 
                if (vp->v_mount->mnt_vtable->vfc_vfsflags & VFC_VFSVNOP_PAGEOUTV2) {
                        /*
 
                if (vp->v_mount->mnt_vtable->vfc_vfsflags & VFC_VFSVNOP_PAGEOUTV2) {
                        /*


@@ -305,33 +377,61 @@ vnode_pageout(struct vnode *vp,
                         * via 'f_offset' and 'size' into a UPL... this allows the filesystem to first
                         * take any locks it needs, before effectively locking the pages into a UPL...
                         */
                         * via 'f_offset' and 'size' into a UPL... this allows the filesystem to first
                         * take any locks it needs, before effectively locking the pages into a UPL...
                         */

-                       KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, 
-                               (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START, 
-                               size, (int)f_offset, 0, 0, 0);
+                       KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
+                           (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START,
+                           size, (int)f_offset, 0, 0, 0);

 
 

-                       if ( (error_ret = VNOP_PAGEOUT(vp, NULL, upl_offset, (off_t)f_offset,
-                                                      size, flags, ctx)) ) {
+                       if ((error_ret = VNOP_PAGEOUT(vp, NULL, upl_offset, (off_t)f_offset,
+                           size, flags, ctx))) {

                                result = PAGER_ERROR;
                        }
                        KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
                                result = PAGER_ERROR;
                        }
                        KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,

-                               (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END, 
-                               size, 0, 0, 0, 0);
+                           (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END,
+                           size, 0, 0, 0, 0);

 
                        goto out;
                }
 
                        goto out;
                }

-               if (flags & UPL_MSYNC)
+               if (flags & UPL_MSYNC) {

                        request_flags = UPL_UBC_MSYNC | UPL_RET_ONLY_DIRTY;
                        request_flags = UPL_UBC_MSYNC | UPL_RET_ONLY_DIRTY;

-               else
+               } else {

                        request_flags = UPL_UBC_PAGEOUT | UPL_RET_ONLY_DIRTY;
                        request_flags = UPL_UBC_PAGEOUT | UPL_RET_ONLY_DIRTY;

-               
-               if (ubc_create_upl(vp, f_offset, size, &upl, &pl, request_flags) != KERN_SUCCESS) {
+               }
+
+               if (ubc_create_upl_kernel(vp, f_offset, size, &upl, &pl, request_flags, VM_KERN_MEMORY_FILE) != KERN_SUCCESS) {

                        result    = PAGER_ERROR;
                        error_ret = EINVAL;
                        goto out;
                }
                upl_offset = 0;
                        result    = PAGER_ERROR;
                        error_ret = EINVAL;
                        goto out;
                }
                upl_offset = 0;

-       } else 
+       } else {

                pl = ubc_upl_pageinfo(upl);
                pl = ubc_upl_pageinfo(upl);

+       }
+
+       /*
+        * Ignore any non-present pages at the end of the
+        * UPL so that we aren't looking at a upl that
+        * may already have been freed by the preceeding
+        * aborts/completions.
+        */
+       base_index = upl_offset / PAGE_SIZE;
+
+       for (pg_index = (upl_offset + isize) / PAGE_SIZE; pg_index > base_index;) {
+               if (upl_page_present(pl, --pg_index)) {
+                       break;
+               }
+               if (pg_index == base_index) {
+                       /*
+                        * no pages were returned, so release
+                        * our hold on the upl and leave
+                        */
+                       if (!(flags & UPL_NOCOMMIT)) {
+                               ubc_upl_abort_range(upl, upl_offset, isize, UPL_ABORT_FREE_ON_EMPTY);
+                       }
+
+                       goto out;
+               }
+       }
+       isize = ((pg_index + 1) - base_index) * PAGE_SIZE;

 
        /*
         * we come here for pageouts to 'real' files and
 
        /*
         * we come here for pageouts to 'real' files and


@@ -340,61 +440,40 @@ vnode_pageout(struct vnode *vp,
         * through it and find the 'runs' of dirty pages
         * to call VNOP_PAGEOUT on...
         */
         * through it and find the 'runs' of dirty pages
         * to call VNOP_PAGEOUT on...
         */

+

        if (ubc_getsize(vp) == 0) {
        if (ubc_getsize(vp) == 0) {

-               /*
+               /*

                 * if the file has been effectively deleted, then
                 * we need to go through the UPL and invalidate any
                 * buffer headers we might have that reference any
                 * of it's pages
                 */
                for (offset = upl_offset; isize; isize -= PAGE_SIZE, offset += PAGE_SIZE) {
                 * if the file has been effectively deleted, then
                 * we need to go through the UPL and invalidate any
                 * buffer headers we might have that reference any
                 * of it's pages
                 */
                for (offset = upl_offset; isize; isize -= PAGE_SIZE, offset += PAGE_SIZE) {

-#if NFSCLIENT
-                       if (vp->v_tag == VT_NFS)
+#if CONFIG_NFS_CLIENT
+                       if (vp->v_tag == VT_NFS) {

                                /* check with nfs if page is OK to drop */
                                error = nfs_buf_page_inval(vp, (off_t)f_offset);
                                /* check with nfs if page is OK to drop */
                                error = nfs_buf_page_inval(vp, (off_t)f_offset);

-                       else
-#endif
+                       } else
+#endif /* CONFIG_NFS_CLIENT */

                        {
                        {

-                               blkno = ubc_offtoblk(vp, (off_t)f_offset);
-                               error = buf_invalblkno(vp, blkno, 0);
+                               blkno = ubc_offtoblk(vp, (off_t)f_offset);
+                               error = buf_invalblkno(vp, blkno, 0);

                        }
                        if (error) {
                        }
                        if (error) {

-                               if ( !(flags & UPL_NOCOMMIT))
-                                       ubc_upl_abort_range(upl, offset, PAGE_SIZE, UPL_ABORT_FREE_ON_EMPTY);
-                               if (error_ret == 0)
-                                       error_ret = error;
+                               if (!(flags & UPL_NOCOMMIT)) {
+                                       ubc_upl_abort_range(upl, offset, PAGE_SIZE, UPL_ABORT_FREE_ON_EMPTY);
+                               }
+                               if (error_ret == 0) {
+                                       error_ret = error;
+                               }

                                result = PAGER_ERROR;
                                result = PAGER_ERROR;

-
-                       } else if ( !(flags & UPL_NOCOMMIT)) {
-                               ubc_upl_commit_range(upl, offset, PAGE_SIZE, UPL_COMMIT_FREE_ON_EMPTY);
+                       } else if (!(flags & UPL_NOCOMMIT)) {
+                               ubc_upl_commit_range(upl, offset, PAGE_SIZE, UPL_COMMIT_FREE_ON_EMPTY);

                        }
                        f_offset += PAGE_SIZE;
                }
                goto out;
        }
                        }
                        f_offset += PAGE_SIZE;
                }
                goto out;
        }

-       /*
-        * Ignore any non-present pages at the end of the
-        * UPL so that we aren't looking at a upl that 
-        * may already have been freed by the preceeding
-        * aborts/completions.
-        */
-       base_index = upl_offset / PAGE_SIZE;
-
-       for (pg_index = (upl_offset + isize) / PAGE_SIZE; pg_index > base_index;) {
-               if (upl_page_present(pl, --pg_index))
-                       break;
-               if (pg_index == base_index) {
-                       /*
-                        * no pages were returned, so release
-                        * our hold on the upl and leave
-                        */
-                       if ( !(flags & UPL_NOCOMMIT))
-                               ubc_upl_abort_range(upl, upl_offset, isize, UPL_ABORT_FREE_ON_EMPTY);
-
-                       goto out;
-               }
-       }
-       isize = ((pg_index + 1) - base_index) * PAGE_SIZE;

 
        offset = upl_offset;
        pg_index = base_index;
 
        offset = upl_offset;
        pg_index = base_index;


@@ -403,20 +482,20 @@ vnode_pageout(struct vnode *vp,
                int  xsize;
                int  num_of_pages;
 
                int  xsize;
                int  num_of_pages;
 

-               if ( !upl_page_present(pl, pg_index)) {
-                       /*
+               if (!upl_page_present(pl, pg_index)) {
+                       /*

                         * we asked for RET_ONLY_DIRTY, so it's possible
                         * to get back empty slots in the UPL
                         * just skip over them
                         */
                         * we asked for RET_ONLY_DIRTY, so it's possible
                         * to get back empty slots in the UPL
                         * just skip over them
                         */

-                       f_offset += PAGE_SIZE;
+                       f_offset += PAGE_SIZE;

                        offset   += PAGE_SIZE;
                        isize    -= PAGE_SIZE;
                        pg_index++;
 
                        continue;
                }
                        offset   += PAGE_SIZE;
                        isize    -= PAGE_SIZE;
                        pg_index++;
 
                        continue;
                }

-               if ( !upl_dirty_page(pl, pg_index)) {
+               if (!upl_dirty_page(pl, pg_index)) {

                        /*
                         * if the page is not dirty and reached here it is
                         * marked precious or it is due to invalidation in
                        /*
                         * if the page is not dirty and reached here it is
                         * marked precious or it is due to invalidation in


@@ -427,27 +506,28 @@ vnode_pageout(struct vnode *vp,
                         * Note we must not sleep here if the buffer is busy - that is
                         * a lock inversion which causes deadlock.
                         */
                         * Note we must not sleep here if the buffer is busy - that is
                         * a lock inversion which causes deadlock.
                         */

-#if NFSCLIENT
-                       if (vp->v_tag == VT_NFS)
+#if CONFIG_NFS_CLIENT
+                       if (vp->v_tag == VT_NFS) {

                                /* check with nfs if page is OK to drop */
                                error = nfs_buf_page_inval(vp, (off_t)f_offset);
                                /* check with nfs if page is OK to drop */
                                error = nfs_buf_page_inval(vp, (off_t)f_offset);

-                       else
-#endif
+                       } else
+#endif /* CONFIG_NFS_CLIENT */

                        {
                        {

-                               blkno = ubc_offtoblk(vp, (off_t)f_offset);
-                               error = buf_invalblkno(vp, blkno, 0);
+                               blkno = ubc_offtoblk(vp, (off_t)f_offset);
+                               error = buf_invalblkno(vp, blkno, 0);

                        }
                        if (error) {
                        }
                        if (error) {

-                               if ( !(flags & UPL_NOCOMMIT))
-                                       ubc_upl_abort_range(upl, offset, PAGE_SIZE, UPL_ABORT_FREE_ON_EMPTY);
-                               if (error_ret == 0)
-                                       error_ret = error;
+                               if (!(flags & UPL_NOCOMMIT)) {
+                                       ubc_upl_abort_range(upl, offset, PAGE_SIZE, UPL_ABORT_FREE_ON_EMPTY);
+                               }
+                               if (error_ret == 0) {
+                                       error_ret = error;
+                               }

                                result = PAGER_ERROR;
                                result = PAGER_ERROR;

-
-                       } else if ( !(flags & UPL_NOCOMMIT)) {
-                               ubc_upl_commit_range(upl, offset, PAGE_SIZE, UPL_COMMIT_FREE_ON_EMPTY);
+                       } else if (!(flags & UPL_NOCOMMIT)) {
+                               ubc_upl_commit_range(upl, offset, PAGE_SIZE, UPL_COMMIT_FREE_ON_EMPTY);

                        }
                        }

-                       f_offset += PAGE_SIZE;
+                       f_offset += PAGE_SIZE;

                        offset   += PAGE_SIZE;
                        isize    -= PAGE_SIZE;
                        pg_index++;
                        offset   += PAGE_SIZE;
                        isize    -= PAGE_SIZE;
                        pg_index++;


@@ -458,78 +538,108 @@ vnode_pageout(struct vnode *vp,
                xsize = isize - PAGE_SIZE;
 
                while (xsize) {
                xsize = isize - PAGE_SIZE;
 
                while (xsize) {

-                       if ( !upl_dirty_page(pl, pg_index + num_of_pages))
+                       if (!upl_dirty_page(pl, pg_index + num_of_pages)) {

                                break;
                                break;

+                       }

                        num_of_pages++;
                        xsize -= PAGE_SIZE;
                }
                xsize = num_of_pages * PAGE_SIZE;
 
                KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
                        num_of_pages++;
                        xsize -= PAGE_SIZE;
                }
                xsize = num_of_pages * PAGE_SIZE;
 
                KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,

-                       (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START, 
-                       xsize, (int)f_offset, 0, 0, 0);
+                   (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START,
+                   xsize, (int)f_offset, 0, 0, 0);

 
 

-               if ( (error = VNOP_PAGEOUT(vp, upl, offset, (off_t)f_offset,
-                                          xsize, flags, ctx)) ) {
-                       if (error_ret == 0)
-                               error_ret = error;
+               if ((error = VNOP_PAGEOUT(vp, upl, offset, (off_t)f_offset,
+                   xsize, flags, ctx))) {
+                       if (error_ret == 0) {
+                               error_ret = error;
+                       }

                        result = PAGER_ERROR;
                }
                KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
                        result = PAGER_ERROR;
                }
                KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,

-                       (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END, 
-                       xsize, 0, 0, 0, 0);
+                   (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END,
+                   xsize, 0, 0, 0, 0);

 
 

-               f_offset += xsize;
+               f_offset += xsize;

                offset   += xsize;
                isize    -= xsize;
                pg_index += num_of_pages;
        }
 out:
                offset   += xsize;
                isize    -= xsize;
                pg_index += num_of_pages;
        }
 out:

-       if (errorp)
+       vnode_put_from_pager(vp);
+
+       if (errorp) {

                *errorp = error_ret;
                *errorp = error_ret;

+       }

 
 

-       return (result);
+       return result;

 }
 
 
 pager_return_t
 vnode_pagein(
 }
 
 
 pager_return_t
 vnode_pagein(

-       struct vnode            *vp,
-       upl_t                   upl,
-       upl_offset_t            upl_offset,
-       vm_object_offset_t      f_offset,
-       upl_size_t              size,
-       int                     flags,
-       int                     *errorp)
+       struct vnode            *vp,
+       upl_t                   upl,
+       upl_offset_t            upl_offset,
+       vm_object_offset_t      f_offset,
+       upl_size_t              size,
+       int                     flags,
+       int                     *errorp)

 {
 {

-        struct uthread *ut;
-        upl_page_info_t *pl;
-       int             result = PAGER_SUCCESS;
-       int             error = 0;
-        int             pages_in_upl;
-        int             start_pg;
-        int             last_pg;
+       upl_page_info_t *pl;
+       int             result = PAGER_SUCCESS;
+       int             error = 0;
+       int             pages_in_upl;
+       int             start_pg;
+       int             last_pg;

        int             first_pg;
        int             first_pg;

-        int             xsize;
-       int             must_commit = 1;
+       int             xsize;
+       int             must_commit = 1;
+       int             ignore_valid_page_check = 0;
+
+       if (flags & UPL_NOCOMMIT) {
+               must_commit = 0;
+       }
+
+       if (flags & UPL_IGNORE_VALID_PAGE_CHECK) {
+               ignore_valid_page_check = 1;
+       }

 
 

-       if (flags & UPL_NOCOMMIT)
-               must_commit = 0;
+       /*
+        * This call is non-blocking and does not ever fail but it can
+        * only be made when there is other explicit synchronization
+        * with reclaiming of the vnode which, in this path, is provided
+        * by the paging in progress counter.
+        *
+        * In addition, this may also be entered via vm_swapfile_io
+        * where the existing iocount provides the necessary synchronization.
+        * Ideally we would not take an additional iocount here in the cases
+        * where an explcit iocount has already been taken but this call
+        * doesn't cause a deadlock as other forms of vnode_get* might if
+        * this thread has already taken an iocount.
+        */
+       error = vnode_getalways_from_pager(vp);
+       if (error != 0) {
+               /* This can't happen */
+               panic("vnode_getalways returned %d for vp %p", error, vp);
+       }

 
        if (UBCINFOEXISTS(vp) == 0) {
                result = PAGER_ERROR;
                error  = PAGER_ERROR;
 
 
        if (UBCINFOEXISTS(vp) == 0) {
                result = PAGER_ERROR;
                error  = PAGER_ERROR;
 

-               if (upl && must_commit)
+               if (upl && must_commit) {

                        ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_ERROR);
                        ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_ERROR);

+               }

 
                goto out;
        }
        if (upl == (upl_t)NULL) {
                flags &= ~UPL_NOCOMMIT;
 
 
                goto out;
        }
        if (upl == (upl_t)NULL) {
                flags &= ~UPL_NOCOMMIT;
 

-               if (size > (MAX_UPL_SIZE * PAGE_SIZE)) {
-                       result = PAGER_ERROR;
+               if (size > MAX_UPL_SIZE_BYTES) {
+                       result = PAGER_ERROR;

                        error  = PAGER_ERROR;
                        goto out;
                }
                        error  = PAGER_ERROR;
                        goto out;
                }


@@ -542,21 +652,22 @@ vnode_pagein(
                         * take any locks it needs, before effectively locking the pages into a UPL...
                         * so we pass a NULL into the filesystem instead of a UPL pointer... the 'upl_offset'
                         * is used to identify the "must have" page in the extent... the filesystem is free
                         * take any locks it needs, before effectively locking the pages into a UPL...
                         * so we pass a NULL into the filesystem instead of a UPL pointer... the 'upl_offset'
                         * is used to identify the "must have" page in the extent... the filesystem is free

-                        * to clip the extent to better fit the underlying FS blocksize if it desires as 
+                        * to clip the extent to better fit the underlying FS blocksize if it desires as

                         * long as it continues to include the "must have" page... 'f_offset' + 'upl_offset'
                         * identifies that page
                         */
                         * long as it continues to include the "must have" page... 'f_offset' + 'upl_offset'
                         * identifies that page
                         */

-                       if ( (error = VNOP_PAGEIN(vp, NULL, upl_offset, (off_t)f_offset,
-                                                 size, flags, vfs_context_current())) ) {
+                       if ((error = VNOP_PAGEIN(vp, NULL, upl_offset, (off_t)f_offset,
+                           size, flags, vfs_context_current()))) {
+                               set_thread_pagein_error(current_thread(), error);

                                result = PAGER_ERROR;
                                error  = PAGER_ERROR;
                        }
                        goto out;
                }
                                result = PAGER_ERROR;
                                error  = PAGER_ERROR;
                        }
                        goto out;
                }

-               ubc_create_upl(vp, f_offset, size, &upl, &pl, UPL_UBC_PAGEIN | UPL_RET_ONLY_ABSENT);
+               ubc_create_upl_kernel(vp, f_offset, size, &upl, &pl, UPL_UBC_PAGEIN | UPL_RET_ONLY_ABSENT, VM_KERN_MEMORY_FILE);

 
                if (upl == (upl_t)NULL) {
 
                if (upl == (upl_t)NULL) {

-                       result =  PAGER_ABSENT;
+                       result =  PAGER_ABSENT;

                        error = PAGER_ABSENT;
                        goto out;
                }
                        error = PAGER_ABSENT;
                        goto out;
                }


@@ -564,7 +675,7 @@ vnode_pagein(
 
                upl_offset = 0;
                first_pg = 0;
 
                upl_offset = 0;
                first_pg = 0;

-               
+

                /*
                 * if we get here, we've created the upl and
                 * are responsible for commiting/aborting it
                /*
                 * if we get here, we've created the upl and
                 * are responsible for commiting/aborting it


@@ -572,26 +683,28 @@ vnode_pagein(
                 */
                must_commit = 1;
        } else {
                 */
                must_commit = 1;
        } else {

-               pl = ubc_upl_pageinfo(upl);
+               pl = ubc_upl_pageinfo(upl);

                first_pg = upl_offset / PAGE_SIZE;
        }
        pages_in_upl = size / PAGE_SIZE;
        DTRACE_VM2(pgpgin, int, pages_in_upl, (uint64_t *), NULL);
 
        /*
                first_pg = upl_offset / PAGE_SIZE;
        }
        pages_in_upl = size / PAGE_SIZE;
        DTRACE_VM2(pgpgin, int, pages_in_upl, (uint64_t *), NULL);
 
        /*

-        * before we start marching forward, we must make sure we end on 
+        * before we start marching forward, we must make sure we end on

         * a present page, otherwise we will be working with a freed
         * a present page, otherwise we will be working with a freed

-         * upl
+        * upl

         */
        for (last_pg = pages_in_upl - 1; last_pg >= first_pg; last_pg--) {
         */
        for (last_pg = pages_in_upl - 1; last_pg >= first_pg; last_pg--) {

-               if (upl_page_present(pl, last_pg))
+               if (upl_page_present(pl, last_pg)) {

                        break;
                        break;

+               }

                if (last_pg == first_pg) {
                if (last_pg == first_pg) {

-                       /*
+                       /*

                         * empty UPL, no pages are present
                         */
                         * empty UPL, no pages are present
                         */

-                       if (must_commit)
-                               ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY);
+                       if (must_commit) {
+                               ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY);
+                       }

                        goto out;
                }
        }
                        goto out;
                }
        }


@@ -599,48 +712,58 @@ vnode_pagein(
        last_pg = first_pg;
 
        while (last_pg < pages_in_upl) {
        last_pg = first_pg;
 
        while (last_pg < pages_in_upl) {

-               /*
+               /*

                 * skip over missing pages...
                 */
                 * skip over missing pages...
                 */

-               for ( ; last_pg < pages_in_upl; last_pg++) {
-                       if (upl_page_present(pl, last_pg))
-                               break;
+               for (; last_pg < pages_in_upl; last_pg++) {
+                       if (upl_page_present(pl, last_pg)) {
+                               break;
+                       }

                }
                }

-               /*
-                * skip over 'valid' pages... we don't want to issue I/O for these
-                */
-               for (start_pg = last_pg; last_pg < pages_in_upl; last_pg++) {
-                       if (!upl_valid_page(pl, last_pg))
-                               break;
+
+               if (ignore_valid_page_check == 1) {
+                       start_pg = last_pg;
+               } else {
+                       /*
+                        * skip over 'valid' pages... we don't want to issue I/O for these
+                        */
+                       for (start_pg = last_pg; last_pg < pages_in_upl; last_pg++) {
+                               if (!upl_valid_page(pl, last_pg)) {
+                                       break;
+                               }
+                       }

                }
                }

+

                if (last_pg > start_pg) {
                if (last_pg > start_pg) {

-                       /*
+                       /*

                         * we've found a range of valid pages
                         * if we've got COMMIT responsibility
                         * commit this range of pages back to the
                         * cache unchanged
                         */
                         * we've found a range of valid pages
                         * if we've got COMMIT responsibility
                         * commit this range of pages back to the
                         * cache unchanged
                         */

-                       xsize = (last_pg - start_pg) * PAGE_SIZE;
+                       xsize = (last_pg - start_pg) * PAGE_SIZE;

 
 

-                       if (must_commit)
-                               ubc_upl_abort_range(upl, start_pg * PAGE_SIZE, xsize, UPL_ABORT_FREE_ON_EMPTY);
+                       if (must_commit) {
+                               ubc_upl_abort_range(upl, start_pg * PAGE_SIZE, xsize, UPL_ABORT_FREE_ON_EMPTY);
+                       }

                }
                }

-               if (last_pg == pages_in_upl)
-                       /*
+               if (last_pg == pages_in_upl) {
+                       /*

                         * we're done... all pages that were present
                         * we're done... all pages that were present

-                        * have either had I/O issued on them or 
+                        * have either had I/O issued on them or

                         * were aborted unchanged...
                         */
                         * were aborted unchanged...
                         */

-                       break;
+                       break;
+               }

 
                if (!upl_page_present(pl, last_pg)) {
 
                if (!upl_page_present(pl, last_pg)) {

-                       /*
-                        * we found a range of valid pages 
+                       /*
+                        * we found a range of valid pages

                         * terminated by a missing page...
                         * bump index to the next page and continue on
                         */
                         * terminated by a missing page...
                         * bump index to the next page and continue on
                         */

-                       last_pg++;
-                       continue;
+                       last_pg++;
+                       continue;

                }
                /*
                 * scan from the found invalid page looking for a valid
                }
                /*
                 * scan from the found invalid page looking for a valid


@@ -649,83 +772,54 @@ vnode_pagein(
                 * 'cluster_io'
                 */
                for (start_pg = last_pg; last_pg < pages_in_upl; last_pg++) {
                 * 'cluster_io'
                 */
                for (start_pg = last_pg; last_pg < pages_in_upl; last_pg++) {

-                       if (upl_valid_page(pl, last_pg) || !upl_page_present(pl, last_pg))
-                               break;
+                       if ((!ignore_valid_page_check && upl_valid_page(pl, last_pg)) || !upl_page_present(pl, last_pg)) {
+                               break;
+                       }

                }
                if (last_pg > start_pg) {
                }
                if (last_pg > start_pg) {

-                       int xoff;
-                       xsize = (last_pg - start_pg) * PAGE_SIZE;
+                       int xoff;
+                       xsize = (last_pg - start_pg) * PAGE_SIZE;

                        xoff  = start_pg * PAGE_SIZE;
 
                        xoff  = start_pg * PAGE_SIZE;
 

-                       if ( (error = VNOP_PAGEIN(vp, upl, (upl_offset_t) xoff,
-                                              (off_t)f_offset + xoff,
-                                              xsize, flags, vfs_context_current())) ) {
-                               /*
+                       if ((error = VNOP_PAGEIN(vp, upl, (upl_offset_t) xoff,
+                           (off_t)f_offset + xoff,
+                           xsize, flags, vfs_context_current()))) {
+                               /*

                                 * Usually this UPL will be aborted/committed by the lower cluster layer.
                                 *
                                 * a)   In the case of decmpfs, however, we may return an error (EAGAIN) to avoid
                                 * Usually this UPL will be aborted/committed by the lower cluster layer.
                                 *
                                 * a)   In the case of decmpfs, however, we may return an error (EAGAIN) to avoid

-                                *      a deadlock with another thread already inflating the file. 
+                                *      a deadlock with another thread already inflating the file.

                                 *
                                 * b)   In the case of content protection, EPERM is a valid error and we should respect it.
                                 *
                                 * In those cases, we must take care of our UPL at this layer itself.
                                 */
                                if (must_commit) {
                                 *
                                 * b)   In the case of content protection, EPERM is a valid error and we should respect it.
                                 *
                                 * In those cases, we must take care of our UPL at this layer itself.
                                 */
                                if (must_commit) {

-                                       if(error == EAGAIN) {
-                                               ubc_upl_abort_range(upl, (upl_offset_t) xoff, xsize, UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_RESTART);
+                                       if (error == EAGAIN) {
+                                               ubc_upl_abort_range(upl, (upl_offset_t) xoff, xsize, UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_RESTART);

                                        }
                                        }

-#if CONFIG_PROTECT
-                                       if(error == EPERM) {
-                                               ubc_upl_abort_range(upl, (upl_offset_t) xoff, xsize, UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_ERROR);
+                                       if (error == EPERM) {
+                                               ubc_upl_abort_range(upl, (upl_offset_t) xoff, xsize, UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_ERROR);

                                        }
                                        }

-#endif

                                }
                                }

+                               set_thread_pagein_error(current_thread(), error);

                                result = PAGER_ERROR;
                                error  = PAGER_ERROR;
                                result = PAGER_ERROR;
                                error  = PAGER_ERROR;

-

                        }
                }
                        }
                }

-        }
+       }

 out:
 out:

-       if (errorp)
-               *errorp = result;
+       vnode_put_from_pager(vp);

 
 

-       ut = get_bsdthread_info(current_thread());
-
-       if (ut->uu_lowpri_window) {
-               /*
-                * task is marked as a low priority I/O type
-                * and the I/O we issued while in this page fault
-                * collided with normal I/O operations... we'll
-                * delay in order to mitigate the impact of this
-                * task on the normal operation of the system
-                */
-               throttle_lowpri_io(TRUE);
+       if (errorp) {
+               *errorp = result;

        }
        }

-       return (error);
-}
-
-void
-vnode_pager_shutdown(void)
-{
-       int i;
-       vnode_t vp;
-
-       for(i = 0; i < MAX_BACKING_STORE; i++) {
-               vp = (vnode_t)(bs_port_table[i]).vp;
-               if (vp) {
-                       (bs_port_table[i]).vp = 0;

 
 

-                       /* get rid of macx_swapon() reference */
-                       vnode_rele(vp);
-               }
-       }
+       return error;

 }
 
 }
 

-

 void *
 upl_get_internal_page_list(upl_t upl)
 {
 void *
 upl_get_internal_page_list(upl_t upl)
 {

-  return(UPL_GET_INTERNAL_PAGE_LIST(upl));
-
+       return UPL_GET_INTERNAL_PAGE_LIST(upl);

 }
 }