xnu-3789.70.16.tar.gz

[apple/xnu.git] / bsd / vm / vnode_pager.c

diff --git a/bsd/vm/vnode_pager.c b/bsd/vm/vnode_pager.c
index d12a65652de5d5d05018d5b2482dd018059f2b90..b70717b92f5b005775bce5c1463e0eb1c22829bf 100644 (file)
--- a/bsd/vm/vnode_pager.c
+++ b/bsd/vm/vnode_pager.c
@@ -90,7 +90,7 @@ vnode_pager_throttle()
        ut = get_bsdthread_info(current_thread());
 
        if (ut->uu_lowpri_window)
        ut = get_bsdthread_info(current_thread());
 
        if (ut->uu_lowpri_window)

-               throttle_lowpri_io(TRUE);
+               throttle_lowpri_io(1);

 }
 
 
 }
 
 


@@ -102,6 +102,33 @@ vnode_pager_isSSD(vnode_t vp)
        return (FALSE);
 }
 
        return (FALSE);
 }
 

+#if CONFIG_IOSCHED
+void
+vnode_pager_issue_reprioritize_io(struct vnode *devvp, uint64_t blkno, uint32_t len, int priority)
+{
+       u_int32_t blocksize = 0;
+       dk_extent_t extent;
+        dk_set_tier_t set_tier;
+       int error = 0;
+
+       error = VNOP_IOCTL(devvp, DKIOCGETBLOCKSIZE, (caddr_t)&blocksize, 0, vfs_context_kernel());
+       if (error)
+               return;
+
+       memset(&extent, 0, sizeof(dk_extent_t));
+       memset(&set_tier, 0, sizeof(dk_set_tier_t));
+       
+       extent.offset = blkno * (u_int64_t) blocksize;
+       extent.length = len;
+
+       set_tier.extents = &extent; 
+       set_tier.extentsCount = 1;
+       set_tier.tier = priority;
+               
+       error = VNOP_IOCTL(devvp, DKIOCSETTIER, (caddr_t)&set_tier, 0, vfs_context_kernel());
+       return;
+}
+#endif

 
 uint32_t
 vnode_pager_isinuse(struct vnode *vp)
 
 uint32_t
 vnode_pager_isinuse(struct vnode *vp)


@@ -112,41 +139,57 @@ vnode_pager_isinuse(struct vnode *vp)
 }
 
 uint32_t
 }
 
 uint32_t

-vnode_pager_return_hard_throttle_limit(struct vnode *vp, uint32_t *limit, uint32_t hard_throttle)
+vnode_pager_return_throttle_io_limit(struct vnode *vp, uint32_t *limit)

 {
 {

-       return(cluster_hard_throttle_limit(vp, limit, hard_throttle));
+       return(cluster_throttle_io_limit(vp, limit));

 }
 
 vm_object_offset_t
 vnode_pager_get_filesize(struct vnode *vp)
 {
 }
 
 vm_object_offset_t
 vnode_pager_get_filesize(struct vnode *vp)
 {

-

        return (vm_object_offset_t) ubc_getsize(vp);
 }
 
        return (vm_object_offset_t) ubc_getsize(vp);
 }
 

+extern int safe_getpath(struct vnode *dvp, char *leafname, char *path, int _len, int *truncated_path);
+

 kern_return_t
 kern_return_t

-vnode_pager_get_pathname(
+vnode_pager_get_name(

        struct vnode    *vp,
        char            *pathname,
        struct vnode    *vp,
        char            *pathname,

-       vm_size_t       *length_p)
+       vm_size_t       pathname_len,
+       char            *filename,
+       vm_size_t       filename_len,
+       boolean_t       *truncated_path_p)

 {
 {

-       int     error, len;
-
-       len = (int) *length_p;
-       error = vn_getpath(vp, pathname, &len);
-       if (error != 0) {
-               return KERN_FAILURE;
+       *truncated_path_p = FALSE;
+       if (pathname != NULL) {
+               /* get the path name */
+               safe_getpath(vp, NULL,
+                            pathname, (int) pathname_len,
+                            truncated_path_p);
+       }
+       if ((pathname == NULL || *truncated_path_p) &&
+           filename != NULL) {
+               /* get the file name */
+               const char *name;
+
+               name = vnode_getname_printable(vp);
+               strlcpy(filename, name, (size_t) filename_len);
+               vnode_putname_printable(name);

        }
        }

-       *length_p = (vm_size_t) len;

        return KERN_SUCCESS;
 }
 
 kern_return_t
        return KERN_SUCCESS;
 }
 
 kern_return_t

-vnode_pager_get_filename(
+vnode_pager_get_mtime(

        struct vnode    *vp,
        struct vnode    *vp,

-       const char      **filename)
+       struct timespec *current_mtime,
+       struct timespec *cs_mtime)

 {
 {

-       *filename = vp->v_name;
+       vnode_mtime(vp, current_mtime, vfs_context_current());
+       if (cs_mtime != NULL) {
+               ubc_get_cs_mtime(vp, cs_mtime);
+       }

        return KERN_SUCCESS;
 }
 
        return KERN_SUCCESS;
 }
 


@@ -280,15 +323,17 @@ vnode_pageout(struct vnode *vp,
                 * just go ahead and call vnop_pageout since
                 * it has already sorted out the dirty ranges
                 */
                 * just go ahead and call vnop_pageout since
                 * it has already sorted out the dirty ranges
                 */

-               KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START, 
-                                     size, 1, 0, 0, 0);
+               KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
+                       (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START, 
+                       size, 1, 0, 0, 0);

 
                if ( (error_ret = VNOP_PAGEOUT(vp, upl, upl_offset, (off_t)f_offset,
                                               (size_t)size, flags, ctx)) )
                        result = PAGER_ERROR;
 
 
                if ( (error_ret = VNOP_PAGEOUT(vp, upl, upl_offset, (off_t)f_offset,
                                               (size_t)size, flags, ctx)) )
                        result = PAGER_ERROR;
 

-               KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END, 
-                                     size, 1, 0, 0, 0);
+               KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
+                       (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END, 
+                       size, 1, 0, 0, 0);

 
                goto out;
        }
 
                goto out;
        }


@@ -303,15 +348,17 @@ vnode_pageout(struct vnode *vp,
                         * via 'f_offset' and 'size' into a UPL... this allows the filesystem to first
                         * take any locks it needs, before effectively locking the pages into a UPL...
                         */
                         * via 'f_offset' and 'size' into a UPL... this allows the filesystem to first
                         * take any locks it needs, before effectively locking the pages into a UPL...
                         */

-                       KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START, 
-                                             size, (int)f_offset, 0, 0, 0);
+                       KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, 
+                               (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START, 
+                               size, (int)f_offset, 0, 0, 0);

 
                        if ( (error_ret = VNOP_PAGEOUT(vp, NULL, upl_offset, (off_t)f_offset,
                                                       size, flags, ctx)) ) {
                                result = PAGER_ERROR;
                        }
 
                        if ( (error_ret = VNOP_PAGEOUT(vp, NULL, upl_offset, (off_t)f_offset,
                                                       size, flags, ctx)) ) {
                                result = PAGER_ERROR;
                        }

-                       KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END, 
-                                             size, 0, 0, 0, 0);
+                       KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
+                               (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END, 
+                               size, 0, 0, 0, 0);

 
                        goto out;
                }
 
                        goto out;
                }


@@ -329,6 +376,30 @@ vnode_pageout(struct vnode *vp,
        } else 
                pl = ubc_upl_pageinfo(upl);
 
        } else 
                pl = ubc_upl_pageinfo(upl);
 

+       /*
+        * Ignore any non-present pages at the end of the
+        * UPL so that we aren't looking at a upl that 
+        * may already have been freed by the preceeding
+        * aborts/completions.
+        */
+       base_index = upl_offset / PAGE_SIZE;
+
+       for (pg_index = (upl_offset + isize) / PAGE_SIZE; pg_index > base_index;) {
+               if (upl_page_present(pl, --pg_index))
+                       break;
+               if (pg_index == base_index) {
+                       /*
+                        * no pages were returned, so release
+                        * our hold on the upl and leave
+                        */
+                       if ( !(flags & UPL_NOCOMMIT))
+                               ubc_upl_abort_range(upl, upl_offset, isize, UPL_ABORT_FREE_ON_EMPTY);
+
+                       goto out;
+               }
+       }
+       isize = ((pg_index + 1) - base_index) * PAGE_SIZE;
+

        /*
         * we come here for pageouts to 'real' files and
         * for msyncs...  the upl may not contain any
        /*
         * we come here for pageouts to 'real' files and
         * for msyncs...  the upl may not contain any


@@ -336,6 +407,7 @@ vnode_pageout(struct vnode *vp,
         * through it and find the 'runs' of dirty pages
         * to call VNOP_PAGEOUT on...
         */
         * through it and find the 'runs' of dirty pages
         * to call VNOP_PAGEOUT on...
         */

+

        if (ubc_getsize(vp) == 0) {
                /*
                 * if the file has been effectively deleted, then
        if (ubc_getsize(vp) == 0) {
                /*
                 * if the file has been effectively deleted, then


@@ -368,29 +440,6 @@ vnode_pageout(struct vnode *vp,
                }
                goto out;
        }
                }
                goto out;
        }

-       /*
-        * Ignore any non-present pages at the end of the
-        * UPL so that we aren't looking at a upl that 
-        * may already have been freed by the preceeding
-        * aborts/completions.
-        */
-       base_index = upl_offset / PAGE_SIZE;
-
-       for (pg_index = (upl_offset + isize) / PAGE_SIZE; pg_index > base_index;) {
-               if (upl_page_present(pl, --pg_index))
-                       break;
-               if (pg_index == base_index) {
-                       /*
-                        * no pages were returned, so release
-                        * our hold on the upl and leave
-                        */
-                       if ( !(flags & UPL_NOCOMMIT))
-                               ubc_upl_abort_range(upl, upl_offset, isize, UPL_ABORT_FREE_ON_EMPTY);
-
-                       goto out;
-               }
-       }
-       isize = ((pg_index + 1) - base_index) * PAGE_SIZE;

 
        offset = upl_offset;
        pg_index = base_index;
 
        offset = upl_offset;
        pg_index = base_index;


@@ -461,8 +510,9 @@ vnode_pageout(struct vnode *vp,
                }
                xsize = num_of_pages * PAGE_SIZE;
 
                }
                xsize = num_of_pages * PAGE_SIZE;
 

-               KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START, 
-                                     xsize, (int)f_offset, 0, 0, 0);
+               KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
+                       (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START, 
+                       xsize, (int)f_offset, 0, 0, 0);

 
                if ( (error = VNOP_PAGEOUT(vp, upl, offset, (off_t)f_offset,
                                           xsize, flags, ctx)) ) {
 
                if ( (error = VNOP_PAGEOUT(vp, upl, offset, (off_t)f_offset,
                                           xsize, flags, ctx)) ) {


@@ -470,8 +520,9 @@ vnode_pageout(struct vnode *vp,
                                error_ret = error;
                        result = PAGER_ERROR;
                }
                                error_ret = error;
                        result = PAGER_ERROR;
                }

-               KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END, 
-                                     xsize, 0, 0, 0, 0);
+               KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
+                       (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END, 
+                       xsize, 0, 0, 0, 0);

 
                f_offset += xsize;
                offset   += xsize;
 
                f_offset += xsize;
                offset   += xsize;


@@ -496,7 +547,6 @@ vnode_pagein(
        int                     flags,
        int                     *errorp)
 {
        int                     flags,
        int                     *errorp)
 {

-        struct uthread *ut;

         upl_page_info_t *pl;
        int             result = PAGER_SUCCESS;
        int             error = 0;
         upl_page_info_t *pl;
        int             result = PAGER_SUCCESS;
        int             error = 0;


@@ -506,10 +556,14 @@ vnode_pagein(
        int             first_pg;
         int             xsize;
        int             must_commit = 1;
        int             first_pg;
         int             xsize;
        int             must_commit = 1;

+       int             ignore_valid_page_check = 0;

 
        if (flags & UPL_NOCOMMIT)
                must_commit = 0;
 
 
        if (flags & UPL_NOCOMMIT)
                must_commit = 0;
 

+       if (flags & UPL_IGNORE_VALID_PAGE_CHECK)
+               ignore_valid_page_check = 1;
+

        if (UBCINFOEXISTS(vp) == 0) {
                result = PAGER_ERROR;
                error  = PAGER_ERROR;
        if (UBCINFOEXISTS(vp) == 0) {
                result = PAGER_ERROR;
                error  = PAGER_ERROR;


@@ -522,7 +576,7 @@ vnode_pagein(
        if (upl == (upl_t)NULL) {
                flags &= ~UPL_NOCOMMIT;
 
        if (upl == (upl_t)NULL) {
                flags &= ~UPL_NOCOMMIT;
 

-               if (size > (MAX_UPL_SIZE * PAGE_SIZE)) {
+               if (size > MAX_UPL_SIZE_BYTES) {

                        result = PAGER_ERROR;
                        error  = PAGER_ERROR;
                        goto out;
                        result = PAGER_ERROR;
                        error  = PAGER_ERROR;
                        goto out;


@@ -554,6 +608,8 @@ vnode_pagein(
                        error = PAGER_ABSENT;
                        goto out;
                }
                        error = PAGER_ABSENT;
                        goto out;
                }

+               ubc_upl_range_needed(upl, upl_offset / PAGE_SIZE, 1);
+

                upl_offset = 0;
                first_pg = 0;
                
                upl_offset = 0;
                first_pg = 0;
                


@@ -598,13 +654,19 @@ vnode_pagein(
                        if (upl_page_present(pl, last_pg))
                                break;
                }
                        if (upl_page_present(pl, last_pg))
                                break;
                }

-               /*
-                * skip over 'valid' pages... we don't want to issue I/O for these
-                */
-               for (start_pg = last_pg; last_pg < pages_in_upl; last_pg++) {
-                       if (!upl_valid_page(pl, last_pg))
-                               break;
+
+               if (ignore_valid_page_check == 1) {
+                       start_pg = last_pg;
+               } else {
+                       /*
+                        * skip over 'valid' pages... we don't want to issue I/O for these
+                        */
+                       for (start_pg = last_pg; last_pg < pages_in_upl; last_pg++) {
+                               if (!upl_valid_page(pl, last_pg))
+                                       break;
+                       }

                }
                }

+

                if (last_pg > start_pg) {
                        /*
                         * we've found a range of valid pages
                if (last_pg > start_pg) {
                        /*
                         * we've found a range of valid pages


@@ -641,7 +703,7 @@ vnode_pagein(
                 * 'cluster_io'
                 */
                for (start_pg = last_pg; last_pg < pages_in_upl; last_pg++) {
                 * 'cluster_io'
                 */
                for (start_pg = last_pg; last_pg < pages_in_upl; last_pg++) {

-                       if (upl_valid_page(pl, last_pg) || !upl_page_present(pl, last_pg))
+                       if (( !ignore_valid_page_check && upl_valid_page(pl, last_pg)) || !upl_page_present(pl, last_pg))

                                break;
                }
                if (last_pg > start_pg) {
                                break;
                }
                if (last_pg > start_pg) {


@@ -666,11 +728,9 @@ vnode_pagein(
                                        if(error == EAGAIN) {
                                                ubc_upl_abort_range(upl, (upl_offset_t) xoff, xsize, UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_RESTART);
                                        }
                                        if(error == EAGAIN) {
                                                ubc_upl_abort_range(upl, (upl_offset_t) xoff, xsize, UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_RESTART);
                                        }

-#if CONFIG_PROTECT

                                        if(error == EPERM) {
                                                ubc_upl_abort_range(upl, (upl_offset_t) xoff, xsize, UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_ERROR);
                                        }
                                        if(error == EPERM) {
                                                ubc_upl_abort_range(upl, (upl_offset_t) xoff, xsize, UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_ERROR);
                                        }

-#endif

                                }
                                result = PAGER_ERROR;
                                error  = PAGER_ERROR;
                                }
                                result = PAGER_ERROR;
                                error  = PAGER_ERROR;


@@ -682,39 +742,9 @@ out:
        if (errorp)
                *errorp = result;
 
        if (errorp)
                *errorp = result;
 

-       ut = get_bsdthread_info(current_thread());
-
-       if (ut->uu_lowpri_window) {
-               /*
-                * task is marked as a low priority I/O type
-                * and the I/O we issued while in this page fault
-                * collided with normal I/O operations... we'll
-                * delay in order to mitigate the impact of this
-                * task on the normal operation of the system
-                */
-               throttle_lowpri_io(TRUE);
-       }

        return (error);
 }
 
        return (error);
 }
 

-void
-vnode_pager_shutdown(void)
-{
-       int i;
-       vnode_t vp;
-
-       for(i = 0; i < MAX_BACKING_STORE; i++) {
-               vp = (vnode_t)(bs_port_table[i]).vp;
-               if (vp) {
-                       (bs_port_table[i]).vp = 0;
-
-                       /* get rid of macx_swapon() reference */
-                       vnode_rele(vp);
-               }
-       }
-}
-
-

 void *
 upl_get_internal_page_list(upl_t upl)
 {
 void *
 upl_get_internal_page_list(upl_t upl)
 {