X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/1c79356b52d46aa6b508fb032f5ae709b1f2897b..8ad349bb6ed4a0be06e34c92be0d98b92e078db4:/bsd/vm/vnode_pager.c diff --git a/bsd/vm/vnode_pager.c b/bsd/vm/vnode_pager.c index 6500724f2..70ae69b25 100644 --- a/bsd/vm/vnode_pager.c +++ b/bsd/vm/vnode_pager.c @@ -1,23 +1,31 @@ /* - * Copyright (c) 2000 Apple Computer, Inc. All rights reserved. - * - * @APPLE_LICENSE_HEADER_START@ - * - * The contents of this file constitute Original Code as defined in and - * are subject to the Apple Public Source License Version 1.1 (the - * "License"). You may not use this file except in compliance with the - * License. Please obtain a copy of the License at - * http://www.apple.com/publicsource and read it before using this file. + * Copyright (c) 2006 Apple Computer, Inc. All Rights Reserved. * - * This Original Code and all software distributed under the License are - * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the - * License for the specific language governing rights and limitations - * under the License. + * @APPLE_LICENSE_OSREFERENCE_HEADER_START@ * - * @APPLE_LICENSE_HEADER_END@ + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the + * License may not be used to create, or enable the creation or + * redistribution of, unlawful or unlicensed copies of an Apple operating + * system, or to circumvent, violate, or enable the circumvention or + * violation of, any terms of an Apple operating system software license + * agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this + * file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_LICENSE_OSREFERENCE_HEADER_END@ */ /* * Mach Operating System @@ -36,13 +44,15 @@ #include #include #include +#include #include +#include #include #include -#include +#include #include -#include -#include +#include /* needs internal due to fhandle_t */ +#include #include #include @@ -50,7 +60,6 @@ #include #include -#include #include #include #include @@ -59,6 +68,14 @@ #include #include +#include +#include + +#include +#include +#include + +#include unsigned int vp_pagein=0; unsigned int vp_pgodirty=0; @@ -66,6 +83,13 @@ unsigned int vp_pgoclean=0; unsigned int dp_pgouts=0; /* Default pager pageouts */ unsigned int dp_pgins=0; /* Default pager pageins */ +vm_object_offset_t +vnode_pager_get_filesize(struct vnode *vp) +{ + + return (vm_object_offset_t) ubc_getsize(vp); +} + pager_return_t vnode_pageout(struct vnode *vp, upl_t upl, @@ -75,103 +99,136 @@ vnode_pageout(struct vnode *vp, int flags, int *errorp) { - int result = PAGER_SUCCESS; struct proc *p = current_proc(); + int result = PAGER_SUCCESS; int error = 0; - int vp_size = 0; - int blkno=0, s; - int cnt, isize; + int error_ret = 0; + daddr64_t blkno; + int isize; int pg_index; + int base_index; int offset; - struct buf *bp; - boolean_t funnel_state; - int haveupl=0; - void * object; upl_page_info_t *pl; - upl_t vpupl; + struct vfs_context context; - funnel_state = thread_funnel_set(kernel_flock, TRUE); + context.vc_proc = p; + context.vc_ucred = kauth_cred_get(); - if (upl != (upl_t)NULL) { - haveupl = 1; - } isize = (int)size; - if (isize < 0) - panic("-ve count in vnode_pageout"); - if (isize == 0) - panic("vnode_pageout: size == 0\n"); - + if (isize <= 0) { + result = PAGER_ERROR; + error_ret = EINVAL; + goto out; + } UBCINFOCHECK("vnode_pageout", vp); if (UBCINVALID(vp)) { - result = PAGER_ERROR; - error = PAGER_ERROR; + result = PAGER_ERROR; + error_ret = EINVAL; + + if (upl && !(flags & UPL_NOCOMMIT)) + ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY); goto out; } - if (haveupl) { + if ( !(flags & UPL_VNODE_PAGER)) { /* - * This is a pageout form the Default pager, - * just go ahead and call VOP_PAGEOUT + * This is a pageout from the default pager, + * just go ahead and call vnop_pageout since + * it has already sorted out the dirty ranges */ dp_pgouts++; - if (error = VOP_PAGEOUT(vp, upl, upl_offset, - (off_t)f_offset,(size_t)size, p->p_ucred, flags)) { + + KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START, + size, 1, 0, 0, 0); + + if ( (error_ret = VNOP_PAGEOUT(vp, upl, upl_offset, (off_t)f_offset, + (size_t)size, flags, &context)) ) result = PAGER_ERROR; - error = PAGER_ERROR; - } + + KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END, + size, 1, 0, 0, 0); + goto out; } - - object = ubc_getobject(vp, UBC_PAGINGOP|UBC_NOREACTIVATE); - if (object == NULL) - panic("vnode_pageout: null object"); - vp_size = ubc_getsize(vp); - - vm_fault_list_request(object, - f_offset, isize, &vpupl, NULL, 0, - UPL_NO_SYNC | UPL_CLEAN_IN_PLACE | UPL_COPYOUT_FROM | UPL_SET_INTERNAL); - if (vpupl == (upl_t) 0) - panic("vnode_pageout: null upl returned"); - - pl = UPL_GET_INTERNAL_PAGE_LIST(vpupl); - - if (vp_size == 0) { - - while (isize) { - blkno = ubc_offtoblk(vp, (off_t)f_offset); -start0: - if (bp = incore(vp, blkno)) { - if (ISSET(bp->b_flags, B_BUSY)) { - SET(bp->b_flags, B_WANTED); - error = tsleep(bp, (PRIBIO + 1), "vnpgout", 0); - goto start0; - } else { - bremfree(bp); - SET(bp->b_flags, (B_BUSY|B_INVAL)); - } + /* + * we come here for pageouts to 'real' files and + * for msyncs... the upl may not contain any + * dirty pages.. it's our responsibility to sort + * through it and find the 'runs' of dirty pages + * to call VNOP_PAGEOUT on... + */ + pl = ubc_upl_pageinfo(upl); + + if (ubc_getsize(vp) == 0) { + /* + * if the file has been effectively deleted, then + * we need to go through the UPL and invalidate any + * buffer headers we might have that reference any + * of it's pages + */ + for (offset = upl_offset; isize; isize -= PAGE_SIZE, offset += PAGE_SIZE) { +#if NFSCLIENT + if (vp->v_tag == VT_NFS) + /* check with nfs if page is OK to drop */ + error = nfs_buf_page_inval(vp, (off_t)f_offset); + else +#endif + { + blkno = ubc_offtoblk(vp, (off_t)f_offset); + error = buf_invalblkno(vp, blkno, 0); + } + if (error) { + if ( !(flags & UPL_NOCOMMIT)) + ubc_upl_abort_range(upl, offset, PAGE_SIZE, UPL_ABORT_FREE_ON_EMPTY); + if (error_ret == 0) + error_ret = error; + result = PAGER_ERROR; + + } else if ( !(flags & UPL_NOCOMMIT)) { + ubc_upl_commit_range(upl, offset, PAGE_SIZE, UPL_COMMIT_FREE_ON_EMPTY); } - if (bp) - brelse(bp); f_offset += PAGE_SIZE; - isize -= PAGE_SIZE; } - kernel_upl_commit_range(vpupl, 0, size, UPL_COMMIT_FREE_ON_EMPTY, - pl, MAX_UPL_TRANSFER); - - error = 0; goto out; } - pg_index = 0; - offset = 0; + /* + * Ignore any non-present pages at the end of the + * UPL so that we aren't looking at a upl that + * may already have been freed by the preceeding + * aborts/completions. + */ + base_index = upl_offset / PAGE_SIZE; + + for (pg_index = (upl_offset + isize) / PAGE_SIZE; pg_index > base_index;) { + if (upl_page_present(pl, --pg_index)) + break; + if (pg_index == base_index) { + /* + * no pages were returned, so release + * our hold on the upl and leave + */ + if ( !(flags & UPL_NOCOMMIT)) + ubc_upl_abort_range(upl, upl_offset, isize, UPL_ABORT_FREE_ON_EMPTY); + + goto out; + } + } + isize = (pg_index + 1) * PAGE_SIZE; + + offset = upl_offset; + pg_index = base_index; while (isize) { int xsize; int num_of_pages; - if ( !upl_valid_page(pl, pg_index)) { - kernel_upl_abort_range(vpupl, offset, PAGE_SIZE, UPL_ABORT_FREE_ON_EMPTY); - + if ( !upl_page_present(pl, pg_index)) { + /* + * we asked for RET_ONLY_DIRTY, so it's possible + * to get back empty slots in the UPL + * just skip over them + */ offset += PAGE_SIZE; isize -= PAGE_SIZE; pg_index++; @@ -186,28 +243,31 @@ start0: * We also get here from vm_object_terminate() * So all you need to do in these * cases is to invalidate incore buffer if it is there + * Note we must not sleep here if the buffer is busy - that is + * a lock inversion which causes deadlock. */ - blkno = ubc_offtoblk(vp, (off_t)(f_offset + offset)); - s = splbio(); vp_pgoclean++; -start: - if (bp = incore(vp, blkno)) { - if (ISSET(bp->b_flags, B_BUSY)) { - SET(bp->b_flags, B_WANTED); - error = tsleep(bp, (PRIBIO + 1), "vnpgout", 0); - goto start; - } else { - bremfree(bp); - SET(bp->b_flags, (B_BUSY|B_INVAL)); - } - } - splx(s); - if (bp) - brelse(bp); - - kernel_upl_commit_range(vpupl, offset, PAGE_SIZE, - UPL_COMMIT_FREE_ON_EMPTY, pl, MAX_UPL_TRANSFER); +#if NFSCLIENT + if (vp->v_tag == VT_NFS) + /* check with nfs if page is OK to drop */ + error = nfs_buf_page_inval(vp, (off_t)(f_offset + offset)); + else +#endif + { + blkno = ubc_offtoblk(vp, (off_t)(f_offset + offset)); + error = buf_invalblkno(vp, blkno, 0); + } + if (error) { + if ( !(flags & UPL_NOCOMMIT)) + ubc_upl_abort_range(upl, offset, PAGE_SIZE, UPL_ABORT_FREE_ON_EMPTY); + if (error_ret == 0) + error_ret = error; + result = PAGER_ERROR; + + } else if ( !(flags & UPL_NOCOMMIT)) { + ubc_upl_commit_range(upl, offset, PAGE_SIZE, UPL_COMMIT_FREE_ON_EMPTY); + } offset += PAGE_SIZE; isize -= PAGE_SIZE; pg_index++; @@ -220,8 +280,6 @@ start: xsize = isize - PAGE_SIZE; while (xsize) { - if ( !upl_valid_page(pl, pg_index + num_of_pages)) - break; if ( !upl_dirty_page(pl, pg_index + num_of_pages)) break; num_of_pages++; @@ -229,123 +287,227 @@ start: } xsize = num_of_pages * PAGE_SIZE; - /* By defn callee will commit or abort upls */ - if (error = VOP_PAGEOUT(vp, vpupl, (vm_offset_t) offset, - (off_t)(f_offset + offset), - xsize, p->p_ucred, flags & ~UPL_NOCOMMIT)) { + KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START, + xsize, (int)(f_offset + offset), 0, 0, 0); + + if ( (error = VNOP_PAGEOUT(vp, upl, (vm_offset_t)offset, + (off_t)(f_offset + offset), xsize, + flags, &context)) ) { + if (error_ret == 0) + error_ret = error; result = PAGER_ERROR; - error = PAGER_ERROR; } + KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END, + xsize, 0, 0, 0, 0); + offset += xsize; isize -= xsize; pg_index += num_of_pages; } out: if (errorp) - *errorp = result; - - thread_funnel_set(kernel_flock, funnel_state); + *errorp = error_ret; - return (error); + return (result); } +void IOSleep(int); + pager_return_t vnode_pagein( struct vnode *vp, - upl_t pl, - vm_offset_t pl_offset, + upl_t upl, + vm_offset_t upl_offset, vm_object_offset_t f_offset, vm_size_t size, int flags, int *errorp) { - int result = PAGER_SUCCESS; - struct proc *p = current_proc(); + struct proc *p = current_proc(); + struct uthread *ut; + upl_page_info_t *pl; + int result = PAGER_SUCCESS; int error = 0; - boolean_t funnel_state; - int haveupl=0; - void * object; - upl_t vpupl; - unsigned int ioaddr; - - funnel_state = thread_funnel_set(kernel_flock, TRUE); + int pages_in_upl; + int start_pg; + int last_pg; + int first_pg; + int xsize; + int abort_needed = 1; -#if 0 - if(pl->page_list.npages >1 ) - panic("vnode_pageout: Can't handle more than one page"); -#endif /* 0 */ - if (pl != (upl_t)NULL) { - haveupl = 1; - } UBCINFOCHECK("vnode_pagein", vp); if (UBCINVALID(vp)) { result = PAGER_ERROR; error = PAGER_ERROR; + if (upl && !(flags & UPL_NOCOMMIT)) { + ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_ERROR); + } goto out; } + if (upl == (upl_t)NULL) { + if (size > (MAX_UPL_TRANSFER * PAGE_SIZE)) { + result = PAGER_ERROR; + error = PAGER_ERROR; + goto out; + } + ubc_create_upl(vp, f_offset, size, &upl, &pl, UPL_RET_ONLY_ABSENT | UPL_SET_LITE); - if (haveupl) { - dp_pgins++; - if (error = VOP_PAGEIN(vp, pl, pl_offset, (off_t)f_offset, - size,p->p_ucred, flags)) { - result = PAGER_ERROR; + if (upl == (upl_t)NULL) { + result = PAGER_ABSENT; + error = PAGER_ABSENT; + goto out; } + upl_offset = 0; + /* + * if we get here, we've created the upl and + * are responsible for commiting/aborting it + * regardless of what the caller has passed in + */ + flags &= ~UPL_NOCOMMIT; + + vp_pagein++; } else { + pl = ubc_upl_pageinfo(upl); - object = ubc_getobject(vp, UBC_PAGINGOP|UBC_NOREACTIVATE); - if (object == (void *)NULL) - panic("vnode_pagein: null object"); - vm_fault_list_request(object, f_offset, size, &vpupl, NULL, 0, - UPL_NO_SYNC | UPL_CLEAN_IN_PLACE | UPL_SET_INTERNAL); + dp_pgins++; + } + pages_in_upl = size / PAGE_SIZE; + first_pg = upl_offset / PAGE_SIZE; + + /* + * before we start marching forward, we must make sure we end on + * a present page, otherwise we will be working with a freed + * upl + */ + for (last_pg = pages_in_upl - 1; last_pg >= first_pg; last_pg--) { + if (upl_page_present(pl, last_pg)) + break; + } + pages_in_upl = last_pg + 1; - if (vpupl == (upl_t) 0) - panic("vnode_pagein: null upl returned"); + for (last_pg = first_pg; last_pg < pages_in_upl;) { + /* + * scan the upl looking for the next + * page that is present.... if all of the + * pages are absent, we're done + */ + for (start_pg = last_pg; last_pg < pages_in_upl; last_pg++) { + if (upl_page_present(pl, last_pg)) + break; + } + if (last_pg == pages_in_upl) + break; + + /* + * if we get here, we've sitting on a page + * that is present... we want to skip over + * any range of 'valid' pages... if this takes + * us to the end of the request, than we're done + */ + for (start_pg = last_pg; last_pg < pages_in_upl; last_pg++) { + if (!upl_valid_page(pl, last_pg) || !upl_page_present(pl, last_pg)) + break; + } + if (last_pg > start_pg) { + /* + * we've found a range of valid pages + * if we've got COMMIT responsibility + * commit this range of pages back to the + * cache unchanged + */ + xsize = (last_pg - start_pg) * PAGE_SIZE; - vp_pagein++; + if (!(flags & UPL_NOCOMMIT)) + ubc_upl_abort_range(upl, start_pg * PAGE_SIZE, xsize, UPL_ABORT_FREE_ON_EMPTY); - /* By defn callee will commit or abort upls */ - if (error = VOP_PAGEIN(vp, vpupl, (vm_offset_t) 0, - (off_t)f_offset, size,p->p_ucred, flags & ~UPL_NOCOMMIT)) { - result = PAGER_ERROR; - error = PAGER_ERROR; + abort_needed = 0; } - } + if (last_pg == pages_in_upl) + break; + + if (!upl_page_present(pl, last_pg)) + /* + * if we found a range of valid pages + * terminated by a non-present page + * than start over + */ + continue; + + /* + * scan from the found invalid page looking for a valid + * or non-present page before the end of the upl is reached, if we + * find one, then it will be the last page of the request to + * 'cluster_io' + */ + for (start_pg = last_pg; last_pg < pages_in_upl; last_pg++) { + if (upl_valid_page(pl, last_pg) || !upl_page_present(pl, last_pg)) + break; + } + if (last_pg > start_pg) { + int xoff; + struct vfs_context context; + + context.vc_proc = p; + context.vc_ucred = kauth_cred_get(); + xsize = (last_pg - start_pg) * PAGE_SIZE; + xoff = start_pg * PAGE_SIZE; + + if ( (error = VNOP_PAGEIN(vp, upl, (vm_offset_t) xoff, + (off_t)f_offset + xoff, + xsize, flags, &context)) ) { + result = PAGER_ERROR; + error = PAGER_ERROR; + + } + abort_needed = 0; + } + } + if (!(flags & UPL_NOCOMMIT) && abort_needed) + ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY); out: if (errorp) - *errorp = result; - thread_funnel_set(kernel_flock, funnel_state); + *errorp = result; + ut = get_bsdthread_info(current_thread()); + + if (ut->uu_lowpri_delay) { + /* + * task is marked as a low priority I/O type + * and the I/O we issued while in this system call + * collided with normal I/O operations... we'll + * delay in order to mitigate the impact of this + * task on the normal operation of the system + */ + IOSleep(ut->uu_lowpri_delay); + ut->uu_lowpri_delay = 0; + } return (error); } void -vnode_pager_shutdown() +vnode_pager_shutdown(void) { int i; - extern struct bs_map bs_port_table[]; - struct vnode *vp; + vnode_t vp; for(i = 0; i < MAX_BACKING_STORE; i++) { - vp = (struct vnode *)(bs_port_table[i]).vp; + vp = (vnode_t)(bs_port_table[i]).vp; if (vp) { (bs_port_table[i]).vp = 0; - ubc_rele(vp); - /* get rid of macx_swapon() namei() reference */ - vrele(vp); - /* get rid of macx_swapon() "extra" reference */ - vrele(vp); + /* get rid of macx_swapon() reference */ + vnode_rele(vp); } } } -void * + +void * upl_get_internal_page_list(upl_t upl) { - return(UPL_GET_INTERNAL_PAGE_LIST(upl)); + return(UPL_GET_INTERNAL_PAGE_LIST(upl)); } -