*
* @APPLE_LICENSE_HEADER_START@
*
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License"). You may not use this file except in compliance with the
- * License. Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * Copyright (c) 1999-2003 Apple Computer, Inc. All Rights Reserved.
*
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this
+ * file.
+ *
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
*
* @APPLE_LICENSE_HEADER_END@
*/
#include <vm/vm_map.h>
#include <vm/vm_kern.h>
-#include <kern/parallel.h>
#include <kern/zalloc.h>
#include <kern/kalloc.h>
#include <libkern/libkern.h>
#include <vm/vm_pageout.h>
#include <kern/assert.h>
+#include <sys/kdebug.h>
unsigned int vp_pagein=0;
unsigned int vp_pgodirty=0;
unsigned int dp_pgouts=0; /* Default pager pageouts */
unsigned int dp_pgins=0; /* Default pager pageins */
+vm_object_offset_t
+vnode_pager_get_filesize(struct vnode *vp)
+{
+ if (UBCINVALID(vp)) {
+ return (vm_object_offset_t) 0;
+ }
+
+ return (vm_object_offset_t) ubc_getsize(vp);
+
+}
+
pager_return_t
vnode_pageout(struct vnode *vp,
upl_t upl,
int result = PAGER_SUCCESS;
struct proc *p = current_proc();
int error = 0;
- int vp_size = 0;
int blkno=0, s;
int cnt, isize;
int pg_index;
int offset;
struct buf *bp;
boolean_t funnel_state;
- int haveupl=0;
- void * object;
upl_page_info_t *pl;
- upl_t vpupl;
+ upl_t vpupl = NULL;
funnel_state = thread_funnel_set(kernel_flock, TRUE);
- if (upl != (upl_t)NULL) {
- haveupl = 1;
- }
isize = (int)size;
- if (isize < 0)
- panic("-ve count in vnode_pageout");
- if (isize == 0)
- panic("vnode_pageout: size == 0\n");
-
+ if (isize <= 0) {
+ result = error = PAGER_ERROR;
+ goto out;
+ }
UBCINFOCHECK("vnode_pageout", vp);
if (UBCINVALID(vp)) {
- result = PAGER_ERROR;
- error = PAGER_ERROR;
+ result = error = PAGER_ERROR;
+
+ if (upl && !(flags & UPL_NOCOMMIT))
+ ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY);
goto out;
}
- if (haveupl) {
+ if (upl) {
/*
- * This is a pageout form the Default pager,
+ * This is a pageout from the Default pager,
* just go ahead and call VOP_PAGEOUT
*/
dp_pgouts++;
- if (error = VOP_PAGEOUT(vp, upl, upl_offset,
- (off_t)f_offset,(size_t)size, p->p_ucred, flags)) {
- result = PAGER_ERROR;
- error = PAGER_ERROR;
- }
- goto out;
- }
- object = ubc_getobject(vp, UBC_PAGINGOP|UBC_NOREACTIVATE);
- if (object == NULL)
- panic("vnode_pageout: null object");
- vp_size = ubc_getsize(vp);
+ KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START,
+ size, 1, 0, 0, 0);
- vm_fault_list_request(object,
- f_offset, isize, &vpupl, NULL, 0,
- UPL_NO_SYNC | UPL_CLEAN_IN_PLACE | UPL_COPYOUT_FROM | UPL_SET_INTERNAL);
- if (vpupl == (upl_t) 0)
- panic("vnode_pageout: null upl returned");
+ if (error = VOP_PAGEOUT(vp, upl, upl_offset, (off_t)f_offset,
+ (size_t)size, p->p_ucred, flags))
+ result = error = PAGER_ERROR;
- pl = UPL_GET_INTERNAL_PAGE_LIST(vpupl);
+ KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END,
+ size, 1, 0, 0, 0);
- if (vp_size == 0) {
+ goto out;
+ }
+ ubc_create_upl(vp, f_offset, isize, &vpupl, &pl, UPL_COPYOUT_FROM);
- while (isize) {
+ if (vpupl == (upl_t) 0) {
+ result = error = PAGER_ABSENT;
+ goto out;
+ }
+ /*
+ * if we get here, we've created the upl and
+ * are responsible for commiting/aborting it
+ * regardless of what the caller has passed in
+ */
+ flags &= ~UPL_NOCOMMIT;
+
+ if (ubc_getsize(vp) == 0) {
+ for (offset = 0; isize; isize -= PAGE_SIZE,
+ offset += PAGE_SIZE) {
blkno = ubc_offtoblk(vp, (off_t)f_offset);
-start0:
- if (bp = incore(vp, blkno)) {
- if (ISSET(bp->b_flags, B_BUSY)) {
- SET(bp->b_flags, B_WANTED);
- error = tsleep(bp, (PRIBIO + 1), "vnpgout", 0);
- goto start0;
- } else {
- bremfree(bp);
- SET(bp->b_flags, (B_BUSY|B_INVAL));
- }
- }
- if (bp)
- brelse(bp);
f_offset += PAGE_SIZE;
- isize -= PAGE_SIZE;
+ if ((bp = incore(vp, blkno)) &&
+ ISSET(bp->b_flags, B_BUSY)) {
+ ubc_upl_abort_range(vpupl, offset, PAGE_SIZE,
+ UPL_ABORT_FREE_ON_EMPTY);
+ result = error = PAGER_ERROR;
+ continue;
+ } else if (bp) {
+ bremfree(bp);
+ SET(bp->b_flags, B_BUSY | B_INVAL);
+ brelse(bp);
+ }
+ ubc_upl_commit_range(vpupl, offset, PAGE_SIZE,
+ UPL_COMMIT_FREE_ON_EMPTY);
}
- kernel_upl_commit_range(vpupl, 0, size, UPL_COMMIT_FREE_ON_EMPTY,
- pl, MAX_UPL_TRANSFER);
-
- error = 0;
goto out;
}
pg_index = 0;
int num_of_pages;
if ( !upl_valid_page(pl, pg_index)) {
- kernel_upl_abort_range(vpupl, offset, PAGE_SIZE, UPL_ABORT_FREE_ON_EMPTY);
-
+ ubc_upl_abort_range(vpupl, offset, PAGE_SIZE,
+ UPL_ABORT_FREE_ON_EMPTY);
offset += PAGE_SIZE;
isize -= PAGE_SIZE;
pg_index++;
* We also get here from vm_object_terminate()
* So all you need to do in these
* cases is to invalidate incore buffer if it is there
+ * Note we must not sleep here if B_BUSY - that is
+ * a lock inversion which causes deadlock.
*/
blkno = ubc_offtoblk(vp, (off_t)(f_offset + offset));
s = splbio();
vp_pgoclean++;
-start:
- if (bp = incore(vp, blkno)) {
- if (ISSET(bp->b_flags, B_BUSY)) {
- SET(bp->b_flags, B_WANTED);
- error = tsleep(bp, (PRIBIO + 1), "vnpgout", 0);
- goto start;
- } else {
- bremfree(bp);
- SET(bp->b_flags, (B_BUSY|B_INVAL));
- }
- }
- splx(s);
- if (bp)
+ if ((bp = incore(vp, blkno)) &&
+ ISSET(bp->b_flags, B_BUSY | B_NEEDCOMMIT)) {
+ splx(s);
+ ubc_upl_abort_range(vpupl, offset, PAGE_SIZE,
+ UPL_ABORT_FREE_ON_EMPTY);
+ result = error = PAGER_ERROR;
+ offset += PAGE_SIZE;
+ isize -= PAGE_SIZE;
+ pg_index++;
+ continue;
+ } else if (bp) {
+ bremfree(bp);
+ SET(bp->b_flags, B_BUSY | B_INVAL );
+ splx(s);
brelse(bp);
+ } else
+ splx(s);
- kernel_upl_commit_range(vpupl, offset, PAGE_SIZE,
- UPL_COMMIT_FREE_ON_EMPTY, pl, MAX_UPL_TRANSFER);
-
+ ubc_upl_commit_range(vpupl, offset, PAGE_SIZE,
+ UPL_COMMIT_FREE_ON_EMPTY);
offset += PAGE_SIZE;
isize -= PAGE_SIZE;
pg_index++;
}
xsize = num_of_pages * PAGE_SIZE;
- /* By defn callee will commit or abort upls */
- if (error = VOP_PAGEOUT(vp, vpupl, (vm_offset_t) offset,
- (off_t)(f_offset + offset),
- xsize, p->p_ucred, flags & ~UPL_NOCOMMIT)) {
- result = PAGER_ERROR;
- error = PAGER_ERROR;
- }
+ KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START,
+ xsize, 0, 0, 0, 0);
+
+ if (error = VOP_PAGEOUT(vp, vpupl, (vm_offset_t)offset,
+ (off_t)(f_offset + offset), xsize,
+ p->p_ucred, flags))
+ result = error = PAGER_ERROR;
+
+ KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END,
+ xsize, 0, 0, 0, 0);
+
offset += xsize;
isize -= xsize;
pg_index += num_of_pages;
pager_return_t
vnode_pagein(
struct vnode *vp,
- upl_t pl,
- vm_offset_t pl_offset,
+ upl_t upl,
+ vm_offset_t upl_offset,
vm_object_offset_t f_offset,
vm_size_t size,
int flags,
int *errorp)
{
- int result = PAGER_SUCCESS;
- struct proc *p = current_proc();
+ struct proc *p = current_proc();
+ upl_page_info_t *pl;
+ int result = PAGER_SUCCESS;
int error = 0;
+ int xfer_size;
+ int pages_in_upl;
+ int start_pg;
+ int last_pg;
+ int first_pg;
+ int xsize;
+ int abort_needed = 1;
boolean_t funnel_state;
- int haveupl=0;
- void * object;
- upl_t vpupl;
- unsigned int ioaddr;
- funnel_state = thread_funnel_set(kernel_flock, TRUE);
-#if 0
- if(pl->page_list.npages >1 )
- panic("vnode_pageout: Can't handle more than one page");
-#endif /* 0 */
+ funnel_state = thread_funnel_set(kernel_flock, TRUE);
- if (pl != (upl_t)NULL) {
- haveupl = 1;
- }
UBCINFOCHECK("vnode_pagein", vp);
if (UBCINVALID(vp)) {
result = PAGER_ERROR;
error = PAGER_ERROR;
+ if (upl && !(flags & UPL_NOCOMMIT)) {
+ ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_ERROR);
+ }
goto out;
}
+ if (upl == (upl_t)NULL) {
+ if (size > (MAX_UPL_TRANSFER * PAGE_SIZE)) {
+ result = PAGER_ERROR;
+ error = PAGER_ERROR;
+ goto out;
+ }
+ ubc_create_upl(vp, f_offset, size, &upl, &pl, UPL_RET_ONLY_ABSENT);
- if (haveupl) {
- dp_pgins++;
- if (error = VOP_PAGEIN(vp, pl, pl_offset, (off_t)f_offset,
- size,p->p_ucred, flags)) {
- result = PAGER_ERROR;
+ if (upl == (upl_t)NULL) {
+ result = PAGER_ABSENT;
+ error = PAGER_ABSENT;
+ goto out;
}
+ upl_offset = 0;
+ /*
+ * if we get here, we've created the upl and
+ * are responsible for commiting/aborting it
+ * regardless of what the caller has passed in
+ */
+ flags &= ~UPL_NOCOMMIT;
+
+ vp_pagein++;
} else {
+ pl = ubc_upl_pageinfo(upl);
+
+ dp_pgins++;
+ }
+ pages_in_upl = size / PAGE_SIZE;
+ first_pg = upl_offset / PAGE_SIZE;
+
+ /*
+ * before we start marching forward, we must make sure we end on
+ * a present page, otherwise we will be working with a freed
+ * upl
+ */
+ for (last_pg = pages_in_upl - 1; last_pg >= first_pg; last_pg--) {
+ if (upl_page_present(pl, last_pg))
+ break;
+ }
+ pages_in_upl = last_pg + 1;
+
+ for (last_pg = first_pg; last_pg < pages_in_upl;) {
+ /*
+ * scan the upl looking for the next
+ * page that is present.... if all of the
+ * pages are absent, we're done
+ */
+ for (start_pg = last_pg; last_pg < pages_in_upl; last_pg++) {
+ if (upl_page_present(pl, last_pg))
+ break;
+ }
+ if (last_pg == pages_in_upl)
+ break;
+
+ /*
+ * if we get here, we've sitting on a page
+ * that is present... we want to skip over
+ * any range of 'valid' pages... if this takes
+ * us to the end of the request, than we're done
+ */
+ for (start_pg = last_pg; last_pg < pages_in_upl; last_pg++) {
+ if (!upl_valid_page(pl, last_pg) || !upl_page_present(pl, last_pg))
+ break;
+ }
+ if (last_pg > start_pg) {
+ /*
+ * we've found a range of valid pages
+ * if we've got COMMIT responsibility
+ * commit this range of pages back to the
+ * cache unchanged
+ */
+ xsize = (last_pg - start_pg) * PAGE_SIZE;
- object = ubc_getobject(vp, UBC_PAGINGOP|UBC_NOREACTIVATE);
- if (object == (void *)NULL)
- panic("vnode_pagein: null object");
- vm_fault_list_request(object, f_offset, size, &vpupl, NULL, 0,
- UPL_NO_SYNC | UPL_CLEAN_IN_PLACE | UPL_SET_INTERNAL);
+ if (!(flags & UPL_NOCOMMIT))
+ ubc_upl_abort_range(upl, start_pg * PAGE_SIZE, xsize, UPL_ABORT_FREE_ON_EMPTY);
- if (vpupl == (upl_t) 0)
- panic("vnode_pagein: null upl returned");
+ abort_needed = 0;
+ }
+ if (last_pg == pages_in_upl)
+ break;
+
+ if (!upl_page_present(pl, last_pg))
+ /*
+ * if we found a range of valid pages
+ * terminated by a non-present page
+ * than start over
+ */
+ continue;
- vp_pagein++;
+ /*
+ * scan from the found invalid page looking for a valid
+ * or non-present page before the end of the upl is reached, if we
+ * find one, then it will be the last page of the request to
+ * 'cluster_io'
+ */
+ for (start_pg = last_pg; last_pg < pages_in_upl; last_pg++) {
+ if (upl_valid_page(pl, last_pg) || !upl_page_present(pl, last_pg))
+ break;
+ }
+ if (last_pg > start_pg) {
+ int xoff;
- /* By defn callee will commit or abort upls */
- if (error = VOP_PAGEIN(vp, vpupl, (vm_offset_t) 0,
- (off_t)f_offset, size,p->p_ucred, flags & ~UPL_NOCOMMIT)) {
- result = PAGER_ERROR;
- error = PAGER_ERROR;
+ xsize = (last_pg - start_pg) * PAGE_SIZE;
+ xoff = start_pg * PAGE_SIZE;
+
+ if (error = VOP_PAGEIN(vp, upl, (vm_offset_t) xoff,
+ (off_t)f_offset + xoff,
+ xsize, p->p_ucred,
+ flags)) {
+ result = PAGER_ERROR;
+ error = PAGER_ERROR;
+
+ }
+ abort_needed = 0;
}
- }
+ }
+ if (!(flags & UPL_NOCOMMIT) && abort_needed)
+ ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY);
out:
if (errorp)
- *errorp = result;
+ *errorp = result;
thread_funnel_set(kernel_flock, funnel_state);
return (error);
}
}
-void *
+
+void *
upl_get_internal_page_list(upl_t upl)
{
- return(UPL_GET_INTERNAL_PAGE_LIST(upl));
+ return(UPL_GET_INTERNAL_PAGE_LIST(upl));
}
-
projects / apple / xnu.git / blobdiff