xnu-344.tar.gz

[apple/xnu.git] / bsd / vm / vnode_pager.c

diff --git a/bsd/vm/vnode_pager.c b/bsd/vm/vnode_pager.c
index 5cdbf447fd56c9f61a528de33d8c8410333c63cc..387b0995f899b9b256b2a4bc92f5a36a27e69013 100644 (file)
--- a/bsd/vm/vnode_pager.c
+++ b/bsd/vm/vnode_pager.c
@@ -50,7 +50,6 @@
 
 #include <vm/vm_map.h>
 #include <vm/vm_kern.h>
 
 #include <vm/vm_map.h>
 #include <vm/vm_kern.h>

-#include <kern/parallel.h>

 #include <kern/zalloc.h>
 #include <kern/kalloc.h>
 #include <libkern/libkern.h>
 #include <kern/zalloc.h>
 #include <kern/kalloc.h>
 #include <libkern/libkern.h>


@@ -59,6 +58,7 @@
 #include <vm/vm_pageout.h>
 
 #include <kern/assert.h>
 #include <vm/vm_pageout.h>
 
 #include <kern/assert.h>

+#include <sys/kdebug.h>

 
 unsigned int vp_pagein=0;
 unsigned int vp_pgodirty=0;
 
 unsigned int vp_pagein=0;
 unsigned int vp_pgodirty=0;


@@ -102,18 +102,17 @@ vnode_pageout(struct vnode *vp,
 
        isize = (int)size;
 
 
        isize = (int)size;
 

-       if (isize < 0)
-               panic("-ve count in vnode_pageout");
-       if (isize == 0)
-               panic("vnode_pageout: size == 0\n");
-
+       if (isize <= 0) {
+               result = error = PAGER_ERROR;
+               goto out;
+       }

        UBCINFOCHECK("vnode_pageout", vp);
 
        if (UBCINVALID(vp)) {
        UBCINFOCHECK("vnode_pageout", vp);
 
        if (UBCINVALID(vp)) {

-               result = PAGER_ERROR;
-               error  = PAGER_ERROR;
+               result = error = PAGER_ERROR;
+

                if (upl && !(flags & UPL_NOCOMMIT))
                if (upl && !(flags & UPL_NOCOMMIT))

-                       ubc_upl_abort(upl, 0);
+                       ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY);

                goto out;
        }
        if (upl) {
                goto out;
        }
        if (upl) {


@@ -122,14 +121,31 @@ vnode_pageout(struct vnode *vp,
                 * just go ahead and call VOP_PAGEOUT
                 */
                dp_pgouts++;
                 * just go ahead and call VOP_PAGEOUT
                 */
                dp_pgouts++;

+
+               KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START, 
+                                     size, 1, 0, 0, 0);
+

                if (error = VOP_PAGEOUT(vp, upl, upl_offset, (off_t)f_offset,
                                        (size_t)size, p->p_ucred, flags))
                        result = error = PAGER_ERROR;
                if (error = VOP_PAGEOUT(vp, upl, upl_offset, (off_t)f_offset,
                                        (size_t)size, p->p_ucred, flags))
                        result = error = PAGER_ERROR;

+
+               KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END, 
+                                     size, 1, 0, 0, 0);
+

                goto out;
        }
        ubc_create_upl(vp, f_offset, isize, &vpupl, &pl, UPL_COPYOUT_FROM);
                goto out;
        }
        ubc_create_upl(vp, f_offset, isize, &vpupl, &pl, UPL_COPYOUT_FROM);

-       if (vpupl == (upl_t) 0)
-               return PAGER_ABSENT;
+
+       if (vpupl == (upl_t) 0) {
+               result = error = PAGER_ABSENT;
+               goto out;
+       }
+       /*
+        * if we get here, we've created the upl and
+        * are responsible for commiting/aborting it
+        * regardless of what the caller has passed in
+        */
+       flags &= ~UPL_NOCOMMIT;

 
        if (ubc_getsize(vp) == 0) {
                for (offset = 0; isize; isize -= PAGE_SIZE,
 
        if (ubc_getsize(vp) == 0) {
                for (offset = 0; isize; isize -= PAGE_SIZE,


@@ -223,11 +239,17 @@ vnode_pageout(struct vnode *vp,
                }
                xsize = num_of_pages * PAGE_SIZE;
 
                }
                xsize = num_of_pages * PAGE_SIZE;
 

-               /*  By defn callee will commit or abort upls */
+               KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START, 
+                                     xsize, 0, 0, 0, 0);
+

                if (error = VOP_PAGEOUT(vp, vpupl, (vm_offset_t)offset,
                                        (off_t)(f_offset + offset), xsize,
                if (error = VOP_PAGEOUT(vp, vpupl, (vm_offset_t)offset,
                                        (off_t)(f_offset + offset), xsize,

-                                       p->p_ucred, flags & ~UPL_NOCOMMIT))
+                                       p->p_ucred, flags))

                        result = error = PAGER_ERROR;
                        result = error = PAGER_ERROR;

+
+               KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END, 
+                                     xsize, 0, 0, 0, 0);
+

                offset += xsize;
                isize  -= xsize;
                pg_index += num_of_pages;
                offset += xsize;
                isize  -= xsize;
                pg_index += num_of_pages;


@@ -245,21 +267,26 @@ out:
 pager_return_t
 vnode_pagein(
        struct vnode            *vp,
 pager_return_t
 vnode_pagein(
        struct vnode            *vp,

-       upl_t                    pl,
-       vm_offset_t              pl_offset,
+       upl_t                   upl,
+       vm_offset_t             upl_offset,

        vm_object_offset_t      f_offset,
        vm_size_t               size,
        int                     flags,
        int                     *errorp)
 {
        vm_object_offset_t      f_offset,
        vm_size_t               size,
        int                     flags,
        int                     *errorp)
 {

-       int     result = PAGER_SUCCESS;
-       struct proc     *p = current_proc();
+        struct proc     *p = current_proc();
+        upl_page_info_t *pl;
+       int             result = PAGER_SUCCESS;

        int             error = 0;
        int             xfer_size;
        int             error = 0;
        int             xfer_size;

+        int             pages_in_upl;
+        int             start_pg;
+        int             last_pg;
+       int             first_pg;
+        int             xsize;
+       int             abort_needed = 1;

        boolean_t       funnel_state;
        boolean_t       funnel_state;

-       upl_t vpupl = NULL;
-       off_t   local_offset;
-       unsigned int  ioaddr;
+

 
        funnel_state = thread_funnel_set(kernel_flock, TRUE);
 
 
        funnel_state = thread_funnel_set(kernel_flock, TRUE);
 


@@ -268,50 +295,129 @@ vnode_pagein(
        if (UBCINVALID(vp)) {
                result = PAGER_ERROR;
                error  = PAGER_ERROR;
        if (UBCINVALID(vp)) {
                result = PAGER_ERROR;
                error  = PAGER_ERROR;

-               if (pl && !(flags & UPL_NOCOMMIT)) {
-                       ubc_upl_abort(pl, 0);
+               if (upl && !(flags & UPL_NOCOMMIT)) {
+                       ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_ERROR);

                }
                goto out;
        }
                }
                goto out;
        }

+       if (upl == (upl_t)NULL) {
+               if (size > (MAX_UPL_TRANSFER * PAGE_SIZE)) {
+                       result = PAGER_ERROR;
+                       error  = PAGER_ERROR;
+                       goto out;
+               }
+               ubc_create_upl(vp, f_offset, size, &upl, &pl, UPL_RET_ONLY_ABSENT);

 
 

-       if (pl) {
-               dp_pgins++;
-               if (error = VOP_PAGEIN(vp, pl, pl_offset, (off_t)f_offset,
-                                      size, p->p_ucred, flags)) {
-                       result = PAGER_ERROR;
+               if (upl == (upl_t)NULL) {
+                       result =  PAGER_ABSENT;
+                       error = PAGER_ABSENT;
+                       goto out;

                }
                }

+               upl_offset = 0;
+               /*
+                * if we get here, we've created the upl and
+                * are responsible for commiting/aborting it
+                * regardless of what the caller has passed in
+                */
+               flags &= ~UPL_NOCOMMIT;
+               
+               vp_pagein++;

        } else {
        } else {

+               pl = ubc_upl_pageinfo(upl);

 
 

-               local_offset = 0;
-               while (size) {
-                       if(size > 4096 && vp->v_tag == VT_NFS) {
-                               xfer_size =  4096;
-                               size = size - xfer_size;
-                       } else {
-                               xfer_size = size;
-                               size = 0;
-                       }
-                       ubc_create_upl(vp, f_offset + local_offset, xfer_size,
-                                      &vpupl, NULL, UPL_FLAGS_NONE);
-                       if (vpupl == (upl_t) 0) {
-                               result =  PAGER_ABSENT;
-                               error = PAGER_ABSENT;
-                               goto out;
-                       }
+               dp_pgins++;
+       }
+       pages_in_upl = size / PAGE_SIZE;
+       first_pg     = upl_offset / PAGE_SIZE;
+
+       /*
+        * before we start marching forward, we must make sure we end on 
+        * a present page, otherwise we will be working with a freed
+         * upl
+        */
+       for (last_pg = pages_in_upl - 1; last_pg >= first_pg; last_pg--) {
+               if (upl_page_present(pl, last_pg))
+                       break;
+       }
+       pages_in_upl = last_pg + 1;
+
+       for (last_pg = first_pg; last_pg < pages_in_upl;) {
+               /*
+                * scan the upl looking for the next
+                * page that is present.... if all of the 
+                * pages are absent, we're done
+                */
+               for (start_pg = last_pg; last_pg < pages_in_upl; last_pg++) {
+                       if (upl_page_present(pl, last_pg))
+                               break;
+               }
+               if (last_pg == pages_in_upl)
+                       break;
+
+               /*
+                * if we get here, we've sitting on a page 
+                * that is present... we want to skip over
+                * any range of 'valid' pages... if this takes
+                * us to the end of the request, than we're done
+                */
+               for (start_pg = last_pg; last_pg < pages_in_upl; last_pg++) {
+                       if (!upl_valid_page(pl, last_pg) || !upl_page_present(pl, last_pg))
+                               break;
+               }
+               if (last_pg > start_pg) {
+                       /*
+                        * we've found a range of valid pages
+                        * if we've got COMMIT responsibility
+                        * commit this range of pages back to the
+                        * cache unchanged
+                        */
+                       xsize = (last_pg - start_pg) * PAGE_SIZE;

 
 

-                       vp_pagein++;
+                       if (!(flags & UPL_NOCOMMIT))
+                               ubc_upl_abort_range(upl, start_pg * PAGE_SIZE, xsize, UPL_ABORT_FREE_ON_EMPTY);

 
 

-                       /*  By defn callee will commit or abort upls */
-                       if (error = VOP_PAGEIN(vp, vpupl, (vm_offset_t) 0,
-                                              (off_t)f_offset + local_offset,
-                                              xfer_size, p->p_ucred,
-                                              flags & ~UPL_NOCOMMIT)) {
+                       abort_needed = 0;
+               }
+               if (last_pg == pages_in_upl)
+                       break;
+
+               if (!upl_page_present(pl, last_pg))
+                       /*
+                        * if we found a range of valid pages 
+                        * terminated by a non-present page
+                        * than start over
+                        */
+                       continue;
+
+               /*
+                * scan from the found invalid page looking for a valid
+                * or non-present page before the end of the upl is reached, if we
+                * find one, then it will be the last page of the request to
+                * 'cluster_io'
+                */
+               for (start_pg = last_pg; last_pg < pages_in_upl; last_pg++) {
+                       if (upl_valid_page(pl, last_pg) || !upl_page_present(pl, last_pg))
+                               break;
+               }
+               if (last_pg > start_pg) {
+                       int xoff;
+
+                       xsize = (last_pg - start_pg) * PAGE_SIZE;
+                       xoff  = start_pg * PAGE_SIZE;
+
+                       if (error = VOP_PAGEIN(vp, upl, (vm_offset_t) xoff,
+                                              (off_t)f_offset + xoff,
+                                              xsize, p->p_ucred,
+                                              flags)) {

                                result = PAGER_ERROR;
                                error  = PAGER_ERROR;
                                result = PAGER_ERROR;
                                error  = PAGER_ERROR;

+

                        }
                        }

-                       local_offset += PAGE_SIZE_64;
+                       abort_needed = 0;

                }
                }

-       }
+        }
+       if (!(flags & UPL_NOCOMMIT) && abort_needed)
+               ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY);

 out:
        if (errorp)
                *errorp = result;
 out:
        if (errorp)
                *errorp = result;