X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/0b4e3aa066abc0728aacb4bbeb86f53f9737156e..d7e50217d7adf6e52786a38bcaa4cd698cb9a79e:/bsd/vm/vnode_pager.c?ds=sidebyside diff --git a/bsd/vm/vnode_pager.c b/bsd/vm/vnode_pager.c index dede2616d..8b1be2843 100644 --- a/bsd/vm/vnode_pager.c +++ b/bsd/vm/vnode_pager.c @@ -3,19 +3,22 @@ * * @APPLE_LICENSE_HEADER_START@ * - * The contents of this file constitute Original Code as defined in and - * are subject to the Apple Public Source License Version 1.1 (the - * "License"). You may not use this file except in compliance with the - * License. Please obtain a copy of the License at - * http://www.apple.com/publicsource and read it before using this file. + * Copyright (c) 1999-2003 Apple Computer, Inc. All Rights Reserved. * - * This Original Code and all software distributed under the License are - * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this + * file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the - * License for the specific language governing rights and limitations - * under the License. + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. * * @APPLE_LICENSE_HEADER_END@ */ @@ -50,7 +53,6 @@ #include #include -#include #include #include #include @@ -59,6 +61,7 @@ #include #include +#include unsigned int vp_pagein=0; unsigned int vp_pgodirty=0; @@ -89,82 +92,83 @@ vnode_pageout(struct vnode *vp, int result = PAGER_SUCCESS; struct proc *p = current_proc(); int error = 0; - int vp_size = 0; int blkno=0, s; int cnt, isize; int pg_index; int offset; struct buf *bp; boolean_t funnel_state; - int haveupl=0; upl_page_info_t *pl; upl_t vpupl = NULL; funnel_state = thread_funnel_set(kernel_flock, TRUE); - if (upl != (upl_t)NULL) { - haveupl = 1; - } isize = (int)size; - if (isize < 0) - panic("-ve count in vnode_pageout"); - if (isize == 0) - panic("vnode_pageout: size == 0\n"); - + if (isize <= 0) { + result = error = PAGER_ERROR; + goto out; + } UBCINFOCHECK("vnode_pageout", vp); if (UBCINVALID(vp)) { - result = PAGER_ERROR; - error = PAGER_ERROR; + result = error = PAGER_ERROR; + + if (upl && !(flags & UPL_NOCOMMIT)) + ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY); goto out; } - if (haveupl) { + if (upl) { /* - * This is a pageout form the Default pager, + * This is a pageout from the Default pager, * just go ahead and call VOP_PAGEOUT */ dp_pgouts++; - if (error = VOP_PAGEOUT(vp, upl, upl_offset, - (off_t)f_offset,(size_t)size, p->p_ucred, flags)) { - result = PAGER_ERROR; - error = PAGER_ERROR; - } + + KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START, + size, 1, 0, 0, 0); + + if (error = VOP_PAGEOUT(vp, upl, upl_offset, (off_t)f_offset, + (size_t)size, p->p_ucred, flags)) + result = error = PAGER_ERROR; + + KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END, + size, 1, 0, 0, 0); + goto out; } - ubc_create_upl( vp, - f_offset, - isize, - &vpupl, - &pl, - UPL_COPYOUT_FROM); - if (vpupl == (upl_t) 0) - return PAGER_ABSENT; - - vp_size = ubc_getsize(vp); - if (vp_size == 0) { - - while (isize) { + ubc_create_upl(vp, f_offset, isize, &vpupl, &pl, UPL_FOR_PAGEOUT | UPL_COPYOUT_FROM); + + if (vpupl == (upl_t) 0) { + result = error = PAGER_ABSENT; + goto out; + } + /* + * if we get here, we've created the upl and + * are responsible for commiting/aborting it + * regardless of what the caller has passed in + */ + flags &= ~UPL_NOCOMMIT; + + if (ubc_getsize(vp) == 0) { + for (offset = 0; isize; isize -= PAGE_SIZE, + offset += PAGE_SIZE) { blkno = ubc_offtoblk(vp, (off_t)f_offset); -start0: - if (bp = incore(vp, blkno)) { - if (ISSET(bp->b_flags, B_BUSY)) { - SET(bp->b_flags, B_WANTED); - error = tsleep(bp, (PRIBIO + 1), "vnpgout", 0); - goto start0; - } else { - bremfree(bp); - SET(bp->b_flags, (B_BUSY|B_INVAL)); - } - } - if (bp) - brelse(bp); f_offset += PAGE_SIZE; - isize -= PAGE_SIZE; + if ((bp = incore(vp, blkno)) && + ISSET(bp->b_flags, B_BUSY)) { + ubc_upl_abort_range(vpupl, offset, PAGE_SIZE, + UPL_ABORT_FREE_ON_EMPTY); + result = error = PAGER_ERROR; + continue; + } else if (bp) { + bremfree(bp); + SET(bp->b_flags, B_BUSY | B_INVAL); + brelse(bp); + } + ubc_upl_commit_range(vpupl, offset, PAGE_SIZE, + UPL_COMMIT_FREE_ON_EMPTY); } - ubc_upl_commit_range(vpupl, 0, size, UPL_COMMIT_FREE_ON_EMPTY); - - error = 0; goto out; } pg_index = 0; @@ -176,8 +180,7 @@ start0: if ( !upl_valid_page(pl, pg_index)) { ubc_upl_abort_range(vpupl, offset, PAGE_SIZE, - UPL_ABORT_FREE_ON_EMPTY); - + UPL_ABORT_FREE_ON_EMPTY); offset += PAGE_SIZE; isize -= PAGE_SIZE; pg_index++; @@ -192,28 +195,32 @@ start0: * We also get here from vm_object_terminate() * So all you need to do in these * cases is to invalidate incore buffer if it is there + * Note we must not sleep here if B_BUSY - that is + * a lock inversion which causes deadlock. */ blkno = ubc_offtoblk(vp, (off_t)(f_offset + offset)); s = splbio(); vp_pgoclean++; -start: - if (bp = incore(vp, blkno)) { - if (ISSET(bp->b_flags, B_BUSY)) { - SET(bp->b_flags, B_WANTED); - error = tsleep(bp, (PRIBIO + 1), "vnpgout", 0); - goto start; - } else { - bremfree(bp); - SET(bp->b_flags, (B_BUSY|B_INVAL)); - } - } - splx(s); - if (bp) + if ((bp = incore(vp, blkno)) && + ISSET(bp->b_flags, B_BUSY | B_NEEDCOMMIT)) { + splx(s); + ubc_upl_abort_range(vpupl, offset, PAGE_SIZE, + UPL_ABORT_FREE_ON_EMPTY); + result = error = PAGER_ERROR; + offset += PAGE_SIZE; + isize -= PAGE_SIZE; + pg_index++; + continue; + } else if (bp) { + bremfree(bp); + SET(bp->b_flags, B_BUSY | B_INVAL ); + splx(s); brelse(bp); + } else + splx(s); ubc_upl_commit_range(vpupl, offset, PAGE_SIZE, - UPL_COMMIT_FREE_ON_EMPTY); - + UPL_COMMIT_FREE_ON_EMPTY); offset += PAGE_SIZE; isize -= PAGE_SIZE; pg_index++; @@ -235,13 +242,17 @@ start: } xsize = num_of_pages * PAGE_SIZE; - /* By defn callee will commit or abort upls */ - if (error = VOP_PAGEOUT(vp, vpupl, (vm_offset_t) offset, - (off_t)(f_offset + offset), - xsize, p->p_ucred, flags & ~UPL_NOCOMMIT)) { - result = PAGER_ERROR; - error = PAGER_ERROR; - } + KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START, + xsize, 0, 0, 0, 0); + + if (error = VOP_PAGEOUT(vp, vpupl, (vm_offset_t)offset, + (off_t)(f_offset + offset), xsize, + p->p_ucred, flags)) + result = error = PAGER_ERROR; + + KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END, + xsize, 0, 0, 0, 0); + offset += xsize; isize -= xsize; pg_index += num_of_pages; @@ -259,84 +270,160 @@ out: pager_return_t vnode_pagein( struct vnode *vp, - upl_t pl, - vm_offset_t pl_offset, + upl_t upl, + vm_offset_t upl_offset, vm_object_offset_t f_offset, vm_size_t size, int flags, int *errorp) { - int result = PAGER_SUCCESS; - struct proc *p = current_proc(); + struct proc *p = current_proc(); + upl_page_info_t *pl; + int result = PAGER_SUCCESS; int error = 0; int xfer_size; + int pages_in_upl; + int start_pg; + int last_pg; + int first_pg; + int xsize; + int abort_needed = 1; boolean_t funnel_state; - int haveupl=0; - upl_t vpupl = NULL; - off_t local_offset; - unsigned int ioaddr; - funnel_state = thread_funnel_set(kernel_flock, TRUE); -#if 0 - if(pl->page_list.npages >1 ) - panic("vnode_pageout: Can't handle more than one page"); -#endif /* 0 */ + funnel_state = thread_funnel_set(kernel_flock, TRUE); - if (pl != (upl_t)NULL) { - haveupl = 1; - } UBCINFOCHECK("vnode_pagein", vp); if (UBCINVALID(vp)) { result = PAGER_ERROR; error = PAGER_ERROR; + if (upl && !(flags & UPL_NOCOMMIT)) { + ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_ERROR); + } goto out; } + if (upl == (upl_t)NULL) { + if (size > (MAX_UPL_TRANSFER * PAGE_SIZE)) { + result = PAGER_ERROR; + error = PAGER_ERROR; + goto out; + } + ubc_create_upl(vp, f_offset, size, &upl, &pl, UPL_RET_ONLY_ABSENT); - if (haveupl) { - dp_pgins++; - if (error = VOP_PAGEIN(vp, pl, pl_offset, (off_t)f_offset, - size,p->p_ucred, flags)) { - result = PAGER_ERROR; + if (upl == (upl_t)NULL) { + result = PAGER_ABSENT; + error = PAGER_ABSENT; + goto out; } + upl_offset = 0; + /* + * if we get here, we've created the upl and + * are responsible for commiting/aborting it + * regardless of what the caller has passed in + */ + flags &= ~UPL_NOCOMMIT; + + vp_pagein++; } else { + pl = ubc_upl_pageinfo(upl); - local_offset = 0; - while (size) { - if((size > 4096) && (vp->v_tag == VT_NFS)) { - xfer_size = 4096; - size = size - xfer_size; - } else { - xfer_size = size; - size = 0; - } - ubc_create_upl( vp, - f_offset+local_offset, - xfer_size, - &vpupl, - NULL, - UPL_FLAGS_NONE); - if (vpupl == (upl_t) 0) { - result = PAGER_ABSENT; - error = PAGER_ABSENT; - goto out; - } + dp_pgins++; + } + pages_in_upl = size / PAGE_SIZE; + first_pg = upl_offset / PAGE_SIZE; + + /* + * before we start marching forward, we must make sure we end on + * a present page, otherwise we will be working with a freed + * upl + */ + for (last_pg = pages_in_upl - 1; last_pg >= first_pg; last_pg--) { + if (upl_page_present(pl, last_pg)) + break; + } + pages_in_upl = last_pg + 1; + + for (last_pg = first_pg; last_pg < pages_in_upl;) { + /* + * scan the upl looking for the next + * page that is present.... if all of the + * pages are absent, we're done + */ + for (start_pg = last_pg; last_pg < pages_in_upl; last_pg++) { + if (upl_page_present(pl, last_pg)) + break; + } + if (last_pg == pages_in_upl) + break; + + /* + * if we get here, we've sitting on a page + * that is present... we want to skip over + * any range of 'valid' pages... if this takes + * us to the end of the request, than we're done + */ + for (start_pg = last_pg; last_pg < pages_in_upl; last_pg++) { + if (!upl_valid_page(pl, last_pg) || !upl_page_present(pl, last_pg)) + break; + } + if (last_pg > start_pg) { + /* + * we've found a range of valid pages + * if we've got COMMIT responsibility + * commit this range of pages back to the + * cache unchanged + */ + xsize = (last_pg - start_pg) * PAGE_SIZE; - vp_pagein++; + if (!(flags & UPL_NOCOMMIT)) + ubc_upl_abort_range(upl, start_pg * PAGE_SIZE, xsize, UPL_ABORT_FREE_ON_EMPTY); - /* By defn callee will commit or abort upls */ - if (error = VOP_PAGEIN(vp, vpupl, (vm_offset_t) 0, - (off_t)f_offset+local_offset, xfer_size,p->p_ucred, flags & ~UPL_NOCOMMIT)) { + abort_needed = 0; + } + if (last_pg == pages_in_upl) + break; + + if (!upl_page_present(pl, last_pg)) + /* + * if we found a range of valid pages + * terminated by a non-present page + * than start over + */ + continue; + + /* + * scan from the found invalid page looking for a valid + * or non-present page before the end of the upl is reached, if we + * find one, then it will be the last page of the request to + * 'cluster_io' + */ + for (start_pg = last_pg; last_pg < pages_in_upl; last_pg++) { + if (upl_valid_page(pl, last_pg) || !upl_page_present(pl, last_pg)) + break; + } + if (last_pg > start_pg) { + int xoff; + + xsize = (last_pg - start_pg) * PAGE_SIZE; + xoff = start_pg * PAGE_SIZE; + + if (error = VOP_PAGEIN(vp, upl, (vm_offset_t) xoff, + (off_t)f_offset + xoff, + xsize, p->p_ucred, + flags)) { result = PAGER_ERROR; error = PAGER_ERROR; + } - local_offset += PAGE_SIZE_64; + abort_needed = 0; } - } + } + if (!(flags & UPL_NOCOMMIT) && abort_needed) + ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY); out: if (errorp) - *errorp = result; + *errorp = result; thread_funnel_set(kernel_flock, funnel_state); return (error);