xnu-344.21.73.tar.gz

[apple/xnu.git] / bsd / vm / vnode_pager.c

diff --git a/bsd/vm/vnode_pager.c b/bsd/vm/vnode_pager.c
index dede2616db02e947c38ee72769847a12df19671c..8b1be28436be2b6a042ede8aa7cca4a1dc781234 100644 (file)
--- a/bsd/vm/vnode_pager.c
+++ b/bsd/vm/vnode_pager.c
@@ -3,19 +3,22 @@
  *
  * @APPLE_LICENSE_HEADER_START@
  * 
  *
  * @APPLE_LICENSE_HEADER_START@
  * 

- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * Copyright (c) 1999-2003 Apple Computer, Inc.  All Rights Reserved.

  * 
  * 

- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this
+ * file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER

  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,

- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.

  * 
  * @APPLE_LICENSE_HEADER_END@
  */
  * 
  * @APPLE_LICENSE_HEADER_END@
  */


@@ -50,7 +53,6 @@
 
 #include <vm/vm_map.h>
 #include <vm/vm_kern.h>
 
 #include <vm/vm_map.h>
 #include <vm/vm_kern.h>

-#include <kern/parallel.h>

 #include <kern/zalloc.h>
 #include <kern/kalloc.h>
 #include <libkern/libkern.h>
 #include <kern/zalloc.h>
 #include <kern/kalloc.h>
 #include <libkern/libkern.h>


@@ -59,6 +61,7 @@
 #include <vm/vm_pageout.h>
 
 #include <kern/assert.h>
 #include <vm/vm_pageout.h>
 
 #include <kern/assert.h>

+#include <sys/kdebug.h>

 
 unsigned int vp_pagein=0;
 unsigned int vp_pgodirty=0;
 
 unsigned int vp_pagein=0;
 unsigned int vp_pgodirty=0;


@@ -89,82 +92,83 @@ vnode_pageout(struct vnode *vp,
        int             result = PAGER_SUCCESS;
        struct proc     *p = current_proc();
        int             error = 0;
        int             result = PAGER_SUCCESS;
        struct proc     *p = current_proc();
        int             error = 0;

-       int vp_size = 0;

        int blkno=0, s;
        int cnt, isize;
        int pg_index;
        int offset;
        struct buf *bp;
        boolean_t       funnel_state;
        int blkno=0, s;
        int cnt, isize;
        int pg_index;
        int offset;
        struct buf *bp;
        boolean_t       funnel_state;

-       int haveupl=0;

        upl_page_info_t *pl;
        upl_t vpupl = NULL;
 
        funnel_state = thread_funnel_set(kernel_flock, TRUE);
 
        upl_page_info_t *pl;
        upl_t vpupl = NULL;
 
        funnel_state = thread_funnel_set(kernel_flock, TRUE);
 

-       if (upl != (upl_t)NULL) {
-               haveupl = 1;
-       }

        isize = (int)size;
 
        isize = (int)size;
 

-       if (isize < 0)
-               panic("-ve count in vnode_pageout");
-       if (isize == 0)
-               panic("vnode_pageout: size == 0\n");
-
+       if (isize <= 0) {
+               result = error = PAGER_ERROR;
+               goto out;
+       }

        UBCINFOCHECK("vnode_pageout", vp);
 
        if (UBCINVALID(vp)) {
        UBCINFOCHECK("vnode_pageout", vp);
 
        if (UBCINVALID(vp)) {

-               result = PAGER_ERROR;
-               error  = PAGER_ERROR;
+               result = error = PAGER_ERROR;
+
+               if (upl && !(flags & UPL_NOCOMMIT))
+                       ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY);

                goto out;
        }
                goto out;
        }

-       if (haveupl) {
+       if (upl) {

                /*
                /*

-                * This is a pageout form the Default pager,
+                * This is a pageout from the Default pager,

                 * just go ahead and call VOP_PAGEOUT
                 */
                dp_pgouts++;
                 * just go ahead and call VOP_PAGEOUT
                 */
                dp_pgouts++;

-               if (error = VOP_PAGEOUT(vp, upl, upl_offset,
-                        (off_t)f_offset,(size_t)size, p->p_ucred, flags)) {
-                       result = PAGER_ERROR;
-                       error  = PAGER_ERROR;
-               }
+
+               KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START, 
+                                     size, 1, 0, 0, 0);
+
+               if (error = VOP_PAGEOUT(vp, upl, upl_offset, (off_t)f_offset,
+                                       (size_t)size, p->p_ucred, flags))
+                       result = error = PAGER_ERROR;
+
+               KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END, 
+                                     size, 1, 0, 0, 0);
+

                goto out;
        }
                goto out;
        }

-       ubc_create_upl( vp,
-                                       f_offset,
-                                       isize,
-                                       &vpupl,
-                                       &pl,
-                                       UPL_COPYOUT_FROM);
-       if (vpupl == (upl_t) 0)
-               return PAGER_ABSENT;
-
-       vp_size = ubc_getsize(vp);
-       if (vp_size == 0) {
-
-               while (isize) {
+       ubc_create_upl(vp, f_offset, isize, &vpupl, &pl, UPL_FOR_PAGEOUT | UPL_COPYOUT_FROM);
+
+       if (vpupl == (upl_t) 0) {
+               result = error = PAGER_ABSENT;
+               goto out;
+       }
+       /*
+        * if we get here, we've created the upl and
+        * are responsible for commiting/aborting it
+        * regardless of what the caller has passed in
+        */
+       flags &= ~UPL_NOCOMMIT;
+
+       if (ubc_getsize(vp) == 0) {
+               for (offset = 0; isize; isize -= PAGE_SIZE,
+                                       offset += PAGE_SIZE) {

                        blkno = ubc_offtoblk(vp, (off_t)f_offset);
                        blkno = ubc_offtoblk(vp, (off_t)f_offset);

-start0:
-                       if (bp = incore(vp, blkno)) {
-                               if (ISSET(bp->b_flags, B_BUSY)) {
-                                       SET(bp->b_flags, B_WANTED);
-                                       error = tsleep(bp, (PRIBIO + 1), "vnpgout", 0);
-                                       goto start0;
-                               } else {
-                                       bremfree(bp);
-                                       SET(bp->b_flags, (B_BUSY|B_INVAL));
-                               }
-                       }
-                       if (bp)
-                               brelse(bp);

                        f_offset += PAGE_SIZE;
                        f_offset += PAGE_SIZE;

-                       isize    -= PAGE_SIZE;
+                       if ((bp = incore(vp, blkno)) &&
+                           ISSET(bp->b_flags, B_BUSY)) {
+                               ubc_upl_abort_range(vpupl, offset, PAGE_SIZE,
+                                                   UPL_ABORT_FREE_ON_EMPTY);
+                               result = error = PAGER_ERROR;
+                               continue;
+                       } else if (bp) {
+                               bremfree(bp);
+                               SET(bp->b_flags, B_BUSY | B_INVAL);
+                               brelse(bp);
+                       }
+                       ubc_upl_commit_range(vpupl, offset, PAGE_SIZE,
+                                            UPL_COMMIT_FREE_ON_EMPTY);

                }
                }

-               ubc_upl_commit_range(vpupl, 0, size, UPL_COMMIT_FREE_ON_EMPTY);
-
-               error = 0;

                goto out;
        }
        pg_index = 0;
                goto out;
        }
        pg_index = 0;


@@ -176,8 +180,7 @@ start0:
 
                if ( !upl_valid_page(pl, pg_index)) {
                        ubc_upl_abort_range(vpupl, offset, PAGE_SIZE,
 
                if ( !upl_valid_page(pl, pg_index)) {
                        ubc_upl_abort_range(vpupl, offset, PAGE_SIZE,

-                                       UPL_ABORT_FREE_ON_EMPTY);
-                
+                                           UPL_ABORT_FREE_ON_EMPTY);

                        offset += PAGE_SIZE;
                        isize  -= PAGE_SIZE;
                        pg_index++;
                        offset += PAGE_SIZE;
                        isize  -= PAGE_SIZE;
                        pg_index++;


@@ -192,28 +195,32 @@ start0:
                         * We also get here from vm_object_terminate()
                         * So all you need to do in these
                         * cases is to invalidate incore buffer if it is there
                         * We also get here from vm_object_terminate()
                         * So all you need to do in these
                         * cases is to invalidate incore buffer if it is there

+                        * Note we must not sleep here if B_BUSY - that is
+                        * a lock inversion which causes deadlock.

                         */
                        blkno = ubc_offtoblk(vp, (off_t)(f_offset + offset));
                        s = splbio();
                        vp_pgoclean++;                  
                         */
                        blkno = ubc_offtoblk(vp, (off_t)(f_offset + offset));
                        s = splbio();
                        vp_pgoclean++;                  

-start:
-                       if (bp = incore(vp, blkno)) {
-                               if (ISSET(bp->b_flags, B_BUSY)) {
-                                       SET(bp->b_flags, B_WANTED);
-                                       error = tsleep(bp, (PRIBIO + 1), "vnpgout", 0);
-                                       goto start;
-                               } else {
-                                       bremfree(bp);
-                                       SET(bp->b_flags, (B_BUSY|B_INVAL));
-                               }
-                       }
-                       splx(s);
-                       if (bp)
+                       if ((bp = incore(vp, blkno)) &&
+                           ISSET(bp->b_flags, B_BUSY | B_NEEDCOMMIT)) {
+                               splx(s);
+                               ubc_upl_abort_range(vpupl, offset, PAGE_SIZE,
+                                                   UPL_ABORT_FREE_ON_EMPTY);
+                               result = error = PAGER_ERROR;
+                               offset += PAGE_SIZE;
+                               isize -= PAGE_SIZE;
+                               pg_index++;
+                               continue;
+                       } else if (bp) {
+                               bremfree(bp);
+                               SET(bp->b_flags, B_BUSY | B_INVAL );
+                               splx(s);

                                brelse(bp);
                                brelse(bp);

+                       } else
+                               splx(s);

 
                        ubc_upl_commit_range(vpupl, offset, PAGE_SIZE, 
 
                        ubc_upl_commit_range(vpupl, offset, PAGE_SIZE, 

-                                       UPL_COMMIT_FREE_ON_EMPTY);
-
+                                            UPL_COMMIT_FREE_ON_EMPTY);

                        offset += PAGE_SIZE;
                        isize  -= PAGE_SIZE;
                        pg_index++;
                        offset += PAGE_SIZE;
                        isize  -= PAGE_SIZE;
                        pg_index++;


@@ -235,13 +242,17 @@ start:
                }
                xsize = num_of_pages * PAGE_SIZE;
 
                }
                xsize = num_of_pages * PAGE_SIZE;
 

-               /*  By defn callee will commit or abort upls */
-               if (error = VOP_PAGEOUT(vp, vpupl, (vm_offset_t) offset,
-                                       (off_t)(f_offset + offset),
-                                       xsize, p->p_ucred, flags & ~UPL_NOCOMMIT)) {
-                       result = PAGER_ERROR;
-                       error  = PAGER_ERROR;
-               }
+               KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START, 
+                                     xsize, 0, 0, 0, 0);
+
+               if (error = VOP_PAGEOUT(vp, vpupl, (vm_offset_t)offset,
+                                       (off_t)(f_offset + offset), xsize,
+                                       p->p_ucred, flags))
+                       result = error = PAGER_ERROR;
+
+               KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END, 
+                                     xsize, 0, 0, 0, 0);
+

                offset += xsize;
                isize  -= xsize;
                pg_index += num_of_pages;
                offset += xsize;
                isize  -= xsize;
                pg_index += num_of_pages;


@@ -259,84 +270,160 @@ out:
 pager_return_t
 vnode_pagein(
        struct vnode            *vp,
 pager_return_t
 vnode_pagein(
        struct vnode            *vp,

-       upl_t                    pl,
-       vm_offset_t              pl_offset,
+       upl_t                   upl,
+       vm_offset_t             upl_offset,

        vm_object_offset_t      f_offset,
        vm_size_t               size,
        int                     flags,
        int                     *errorp)
 {
        vm_object_offset_t      f_offset,
        vm_size_t               size,
        int                     flags,
        int                     *errorp)
 {

-       int     result = PAGER_SUCCESS;
-       struct proc     *p = current_proc();
+        struct proc     *p = current_proc();
+        upl_page_info_t *pl;
+       int             result = PAGER_SUCCESS;

        int             error = 0;
        int             xfer_size;
        int             error = 0;
        int             xfer_size;

+        int             pages_in_upl;
+        int             start_pg;
+        int             last_pg;
+       int             first_pg;
+        int             xsize;
+       int             abort_needed = 1;

        boolean_t       funnel_state;
        boolean_t       funnel_state;

-       int haveupl=0;
-       upl_t vpupl = NULL;
-       off_t   local_offset;
-       unsigned int  ioaddr;

 
 

-       funnel_state = thread_funnel_set(kernel_flock, TRUE);

 
 

-#if 0
-       if(pl->page_list.npages >1 )
-               panic("vnode_pageout: Can't handle more than one page");
-#endif /* 0 */
+       funnel_state = thread_funnel_set(kernel_flock, TRUE);

 
 

-       if (pl != (upl_t)NULL) {
-               haveupl = 1;
-       }

        UBCINFOCHECK("vnode_pagein", vp);
 
        if (UBCINVALID(vp)) {
                result = PAGER_ERROR;
                error  = PAGER_ERROR;
        UBCINFOCHECK("vnode_pagein", vp);
 
        if (UBCINVALID(vp)) {
                result = PAGER_ERROR;
                error  = PAGER_ERROR;

+               if (upl && !(flags & UPL_NOCOMMIT)) {
+                       ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_ERROR);
+               }

                goto out;
        }
                goto out;
        }

+       if (upl == (upl_t)NULL) {
+               if (size > (MAX_UPL_TRANSFER * PAGE_SIZE)) {
+                       result = PAGER_ERROR;
+                       error  = PAGER_ERROR;
+                       goto out;
+               }
+               ubc_create_upl(vp, f_offset, size, &upl, &pl, UPL_RET_ONLY_ABSENT);

 
 

-       if (haveupl) {
-               dp_pgins++;
-               if (error = VOP_PAGEIN(vp, pl, pl_offset, (off_t)f_offset,
-                                size,p->p_ucred, flags)) {
-                       result = PAGER_ERROR;
+               if (upl == (upl_t)NULL) {
+                       result =  PAGER_ABSENT;
+                       error = PAGER_ABSENT;
+                       goto out;

                }
                }

+               upl_offset = 0;
+               /*
+                * if we get here, we've created the upl and
+                * are responsible for commiting/aborting it
+                * regardless of what the caller has passed in
+                */
+               flags &= ~UPL_NOCOMMIT;
+               
+               vp_pagein++;

        } else {
        } else {

+               pl = ubc_upl_pageinfo(upl);

 
 

-               local_offset = 0;
-               while (size) {
-                       if((size > 4096) && (vp->v_tag == VT_NFS)) {
-                               xfer_size =  4096;
-                               size = size - xfer_size;
-                       } else {
-                               xfer_size = size;
-                               size = 0;
-                       }
-                       ubc_create_upl( vp,
-                                                       f_offset+local_offset,
-                                                       xfer_size,
-                                                       &vpupl,
-                                                       NULL,
-                                                       UPL_FLAGS_NONE);
-                       if (vpupl == (upl_t) 0) {
-                               result =  PAGER_ABSENT;
-                               error = PAGER_ABSENT;
-                               goto out;
-                       }
+               dp_pgins++;
+       }
+       pages_in_upl = size / PAGE_SIZE;
+       first_pg     = upl_offset / PAGE_SIZE;
+
+       /*
+        * before we start marching forward, we must make sure we end on 
+        * a present page, otherwise we will be working with a freed
+         * upl
+        */
+       for (last_pg = pages_in_upl - 1; last_pg >= first_pg; last_pg--) {
+               if (upl_page_present(pl, last_pg))
+                       break;
+       }
+       pages_in_upl = last_pg + 1;
+
+       for (last_pg = first_pg; last_pg < pages_in_upl;) {
+               /*
+                * scan the upl looking for the next
+                * page that is present.... if all of the 
+                * pages are absent, we're done
+                */
+               for (start_pg = last_pg; last_pg < pages_in_upl; last_pg++) {
+                       if (upl_page_present(pl, last_pg))
+                               break;
+               }
+               if (last_pg == pages_in_upl)
+                       break;
+
+               /*
+                * if we get here, we've sitting on a page 
+                * that is present... we want to skip over
+                * any range of 'valid' pages... if this takes
+                * us to the end of the request, than we're done
+                */
+               for (start_pg = last_pg; last_pg < pages_in_upl; last_pg++) {
+                       if (!upl_valid_page(pl, last_pg) || !upl_page_present(pl, last_pg))
+                               break;
+               }
+               if (last_pg > start_pg) {
+                       /*
+                        * we've found a range of valid pages
+                        * if we've got COMMIT responsibility
+                        * commit this range of pages back to the
+                        * cache unchanged
+                        */
+                       xsize = (last_pg - start_pg) * PAGE_SIZE;

 
 

-                       vp_pagein++;
+                       if (!(flags & UPL_NOCOMMIT))
+                               ubc_upl_abort_range(upl, start_pg * PAGE_SIZE, xsize, UPL_ABORT_FREE_ON_EMPTY);

 
 

-                       /*  By defn callee will commit or abort upls */
-                       if (error = VOP_PAGEIN(vp, vpupl, (vm_offset_t) 0,
-                               (off_t)f_offset+local_offset, xfer_size,p->p_ucred, flags & ~UPL_NOCOMMIT)) {
+                       abort_needed = 0;
+               }
+               if (last_pg == pages_in_upl)
+                       break;
+
+               if (!upl_page_present(pl, last_pg))
+                       /*
+                        * if we found a range of valid pages 
+                        * terminated by a non-present page
+                        * than start over
+                        */
+                       continue;
+
+               /*
+                * scan from the found invalid page looking for a valid
+                * or non-present page before the end of the upl is reached, if we
+                * find one, then it will be the last page of the request to
+                * 'cluster_io'
+                */
+               for (start_pg = last_pg; last_pg < pages_in_upl; last_pg++) {
+                       if (upl_valid_page(pl, last_pg) || !upl_page_present(pl, last_pg))
+                               break;
+               }
+               if (last_pg > start_pg) {
+                       int xoff;
+
+                       xsize = (last_pg - start_pg) * PAGE_SIZE;
+                       xoff  = start_pg * PAGE_SIZE;
+
+                       if (error = VOP_PAGEIN(vp, upl, (vm_offset_t) xoff,
+                                              (off_t)f_offset + xoff,
+                                              xsize, p->p_ucred,
+                                              flags)) {

                                result = PAGER_ERROR;
                                error  = PAGER_ERROR;
                                result = PAGER_ERROR;
                                error  = PAGER_ERROR;

+

                        }
                        }

-                       local_offset += PAGE_SIZE_64;
+                       abort_needed = 0;

                }
                }

-       }
+        }
+       if (!(flags & UPL_NOCOMMIT) && abort_needed)
+               ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY);

 out:
        if (errorp)
 out:
        if (errorp)

-           *errorp = result;
+               *errorp = result;

        thread_funnel_set(kernel_flock, funnel_state);
 
        return (error);
        thread_funnel_set(kernel_flock, funnel_state);
 
        return (error);