X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/99c3a10404e5d1ef94397ab4df5a8b74711fc4d3..c3c9b80d004dbbfdf763edeb97968c6997e3b45b:/bsd/vm/vnode_pager.c diff --git a/bsd/vm/vnode_pager.c b/bsd/vm/vnode_pager.c index 93441bcb8..fc3426fd5 100644 --- a/bsd/vm/vnode_pager.c +++ b/bsd/vm/vnode_pager.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved. + * Copyright (c) 2000-2020 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * + * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in @@ -11,10 +11,10 @@ * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. - * + * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. - * + * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, @@ -22,10 +22,10 @@ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. - * + * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ -/* +/* * Mach Operating System * Copyright (c) 1987 Carnegie-Mellon University * All rights reserved. The CMU software License Agreement specifies @@ -49,14 +49,13 @@ #include #include #include -#include /* needs internal due to fhandle_t */ +#include /* needs internal due to fhandle_t */ #include #include -#include /* For DKIOC calls */ +#include /* For DKIOC calls */ #include #include -#include #include #include #include @@ -65,7 +64,6 @@ #include #include #include -#include #include #include @@ -73,93 +71,145 @@ #include #include -#include - +#include #include #include #include #include +#include void -vnode_pager_throttle() +vnode_pager_throttle(void) { struct uthread *ut; ut = get_bsdthread_info(current_thread()); - if (ut->uu_lowpri_window) - throttle_lowpri_io(TRUE); + if (ut->uu_lowpri_window) { + throttle_lowpri_io(1); + } } - boolean_t vnode_pager_isSSD(vnode_t vp) { - if (vp->v_mount->mnt_kern_flag & MNTK_SSD) - return (TRUE); - return (FALSE); + return disk_conditioner_mount_is_ssd(vp->v_mount); +} + +#if CONFIG_IOSCHED +void +vnode_pager_issue_reprioritize_io(struct vnode *devvp, uint64_t blkno, uint32_t len, int priority) +{ + u_int32_t blocksize = 0; + dk_extent_t extent; + dk_set_tier_t set_tier; + int error = 0; + + error = VNOP_IOCTL(devvp, DKIOCGETBLOCKSIZE, (caddr_t)&blocksize, 0, vfs_context_kernel()); + if (error) { + return; + } + + memset(&extent, 0, sizeof(dk_extent_t)); + memset(&set_tier, 0, sizeof(dk_set_tier_t)); + + extent.offset = blkno * (u_int64_t) blocksize; + extent.length = len; + + set_tier.extents = &extent; + set_tier.extentsCount = 1; + set_tier.tier = (uint8_t)priority; + + error = VNOP_IOCTL(devvp, DKIOCSETTIER, (caddr_t)&set_tier, 0, vfs_context_kernel()); + return; } +#endif +void +vnode_pager_was_dirtied( + struct vnode *vp, + vm_object_offset_t s_offset, + vm_object_offset_t e_offset) +{ + cluster_update_state(vp, s_offset, e_offset, TRUE); +} uint32_t vnode_pager_isinuse(struct vnode *vp) { - if (vp->v_usecount > vp->v_kusecount) - return (1); - return (0); + if (vp->v_usecount > vp->v_kusecount) { + return 1; + } + return 0; } uint32_t -vnode_pager_return_hard_throttle_limit(struct vnode *vp, uint32_t *limit, uint32_t hard_throttle) +vnode_pager_return_throttle_io_limit(struct vnode *vp, uint32_t *limit) { - return(cluster_hard_throttle_limit(vp, limit, hard_throttle)); + return cluster_throttle_io_limit(vp, limit); } vm_object_offset_t vnode_pager_get_filesize(struct vnode *vp) { - return (vm_object_offset_t) ubc_getsize(vp); } +extern int safe_getpath(struct vnode *dvp, char *leafname, char *path, int _len, int *truncated_path); + kern_return_t -vnode_pager_get_pathname( - struct vnode *vp, - char *pathname, - vm_size_t *length_p) +vnode_pager_get_name( + struct vnode *vp, + char *pathname, + vm_size_t pathname_len, + char *filename, + vm_size_t filename_len, + boolean_t *truncated_path_p) { - int error, len; - - len = (int) *length_p; - error = vn_getpath(vp, pathname, &len); - if (error != 0) { - return KERN_FAILURE; + *truncated_path_p = FALSE; + if (pathname != NULL) { + /* get the path name */ + safe_getpath(vp, NULL, + pathname, (int) pathname_len, + truncated_path_p); + } + if ((pathname == NULL || *truncated_path_p) && + filename != NULL) { + /* get the file name */ + const char *name; + + name = vnode_getname_printable(vp); + strlcpy(filename, name, (size_t) filename_len); + vnode_putname_printable(name); } - *length_p = (vm_size_t) len; return KERN_SUCCESS; } kern_return_t -vnode_pager_get_filename( - struct vnode *vp, - const char **filename) +vnode_pager_get_mtime( + struct vnode *vp, + struct timespec *current_mtime, + struct timespec *cs_mtime) { - *filename = vp->v_name; + vnode_mtime(vp, current_mtime, vfs_context_current()); + if (cs_mtime != NULL) { + ubc_get_cs_mtime(vp, cs_mtime); + } return KERN_SUCCESS; } kern_return_t vnode_pager_get_cs_blobs( - struct vnode *vp, - void **blobs) + struct vnode *vp, + void **blobs) { *blobs = ubc_get_cs_blobs(vp); return KERN_SUCCESS; } -/* +/* * vnode_trim: * Used to call the DKIOCUNMAP ioctl on the underlying disk device for the specified vnode. * Trims the region at offset bytes into the file, for length bytes. @@ -169,15 +219,16 @@ vnode_pager_get_cs_blobs( * This function is non-idempotent in error cases; We cannot un-discard the blocks if only some of them * are successfully discarded. */ -u_int32_t vnode_trim ( - struct vnode *vp, - off_t offset, - size_t length) +u_int32_t +vnode_trim( + struct vnode *vp, + off_t offset, + size_t length) { - daddr64_t io_blockno; /* Block number corresponding to the start of the extent */ - size_t io_bytecount; /* Number of bytes in current extent for the specified range */ + daddr64_t io_blockno; /* Block number corresponding to the start of the extent */ + size_t io_bytecount; /* Number of bytes in current extent for the specified range */ size_t trimmed = 0; - off_t current_offset = offset; + off_t current_offset = offset; size_t remaining_length = length; int error = 0; u_int32_t blocksize = 0; @@ -195,7 +246,7 @@ u_int32_t vnode_trim ( goto trim_exit; } - /* + /* * We may not get the entire range from offset -> offset+length in a single * extent from the blockmap call. Keep looping/going until we are sure we've hit * the whole range or if we encounter an error. @@ -203,22 +254,22 @@ u_int32_t vnode_trim ( while (trimmed < length) { /* * VNOP_BLOCKMAP will tell us the logical to physical block number mapping for the - * specified offset. It returns blocks in contiguous chunks, so if the logical range is + * specified offset. It returns blocks in contiguous chunks, so if the logical range is * broken into multiple extents, it must be called multiple times, increasing the offset * in each call to ensure that the entire range is covered. */ - error = VNOP_BLOCKMAP (vp, current_offset, remaining_length, - &io_blockno, &io_bytecount, NULL, VNODE_READ, NULL); + error = VNOP_BLOCKMAP(vp, current_offset, remaining_length, + &io_blockno, &io_bytecount, NULL, VNODE_READ | VNODE_BLOCKMAP_NO_TRACK, NULL); if (error) { goto trim_exit; } - /* + /* * We have a contiguous run. Prepare & issue the ioctl for the device. * the DKIOCUNMAP ioctl takes offset in bytes from the start of the device. */ - memset (&extent, 0, sizeof(dk_extent_t)); - memset (&unmap, 0, sizeof(dk_unmap_t)); + memset(&extent, 0, sizeof(dk_extent_t)); + memset(&unmap, 0, sizeof(dk_unmap_t)); extent.offset = (uint64_t) io_blockno * (u_int64_t) blocksize; extent.length = io_bytecount; unmap.extents = &extent; @@ -235,33 +286,52 @@ u_int32_t vnode_trim ( trim_exit: return error; - } pager_return_t vnode_pageout(struct vnode *vp, - upl_t upl, - upl_offset_t upl_offset, - vm_object_offset_t f_offset, - upl_size_t size, - int flags, - int *errorp) + upl_t upl, + upl_offset_t upl_offset, + vm_object_offset_t f_offset, + upl_size_t size, + int flags, + int *errorp) { - int result = PAGER_SUCCESS; - int error = 0; - int error_ret = 0; + int result = PAGER_SUCCESS; + int error = 0; + int error_ret = 0; daddr64_t blkno; int isize; int pg_index; int base_index; upl_offset_t offset; upl_page_info_t *pl; - vfs_context_t ctx = vfs_context_current(); /* pager context */ + vfs_context_t ctx = vfs_context_current(); /* pager context */ isize = (int)size; + /* + * This call is non-blocking and does not ever fail but it can + * only be made when there is other explicit synchronization + * with reclaiming of the vnode which, in this path, is provided + * by the paging in progress counter. + * + * In addition, this may also be entered via explicit ubc_msync + * calls or vm_swapfile_io where the existing iocount provides + * the necessary synchronization. Ideally we would not take an + * additional iocount here in the cases where an explcit iocount + * has already been taken but this call doesn't cause a deadlock + * as other forms of vnode_get* might if this thread has already + * taken an iocount. + */ + error = vnode_getalways_from_pager(vp); + if (error != 0) { + /* This can't happen */ + panic("vnode_getalways returned %d for vp %p", error, vp); + } + if (isize <= 0) { - result = PAGER_ERROR; + result = PAGER_ERROR; error_ret = EINVAL; goto out; } @@ -270,32 +340,34 @@ vnode_pageout(struct vnode *vp, result = PAGER_ERROR; error_ret = EINVAL; - if (upl && !(flags & UPL_NOCOMMIT)) - ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY); + if (upl && !(flags & UPL_NOCOMMIT)) { + ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY); + } goto out; } - if ( !(flags & UPL_VNODE_PAGER)) { + if (!(flags & UPL_VNODE_PAGER)) { /* * This is a pageout from the default pager, * just go ahead and call vnop_pageout since * it has already sorted out the dirty ranges */ KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, - (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START, - size, 1, 0, 0, 0); + (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START, + size, 1, 0, 0, 0); - if ( (error_ret = VNOP_PAGEOUT(vp, upl, upl_offset, (off_t)f_offset, - (size_t)size, flags, ctx)) ) + if ((error_ret = VNOP_PAGEOUT(vp, upl, upl_offset, (off_t)f_offset, + (size_t)size, flags, ctx))) { result = PAGER_ERROR; + } KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, - (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END, - size, 1, 0, 0, 0); + (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END, + size, 1, 0, 0, 0); goto out; } if (upl == NULL) { - int request_flags; + int request_flags; if (vp->v_mount->mnt_vtable->vfc_vfsflags & VFC_VFSVNOP_PAGEOUTV2) { /* @@ -305,33 +377,61 @@ vnode_pageout(struct vnode *vp, * via 'f_offset' and 'size' into a UPL... this allows the filesystem to first * take any locks it needs, before effectively locking the pages into a UPL... */ - KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, - (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START, - size, (int)f_offset, 0, 0, 0); + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, + (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START, + size, (int)f_offset, 0, 0, 0); - if ( (error_ret = VNOP_PAGEOUT(vp, NULL, upl_offset, (off_t)f_offset, - size, flags, ctx)) ) { + if ((error_ret = VNOP_PAGEOUT(vp, NULL, upl_offset, (off_t)f_offset, + size, flags, ctx))) { result = PAGER_ERROR; } KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, - (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END, - size, 0, 0, 0, 0); + (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END, + size, 0, 0, 0, 0); goto out; } - if (flags & UPL_MSYNC) + if (flags & UPL_MSYNC) { request_flags = UPL_UBC_MSYNC | UPL_RET_ONLY_DIRTY; - else + } else { request_flags = UPL_UBC_PAGEOUT | UPL_RET_ONLY_DIRTY; - - if (ubc_create_upl(vp, f_offset, size, &upl, &pl, request_flags) != KERN_SUCCESS) { + } + + if (ubc_create_upl_kernel(vp, f_offset, size, &upl, &pl, request_flags, VM_KERN_MEMORY_FILE) != KERN_SUCCESS) { result = PAGER_ERROR; error_ret = EINVAL; goto out; } upl_offset = 0; - } else + } else { pl = ubc_upl_pageinfo(upl); + } + + /* + * Ignore any non-present pages at the end of the + * UPL so that we aren't looking at a upl that + * may already have been freed by the preceeding + * aborts/completions. + */ + base_index = upl_offset / PAGE_SIZE; + + for (pg_index = (upl_offset + isize) / PAGE_SIZE; pg_index > base_index;) { + if (upl_page_present(pl, --pg_index)) { + break; + } + if (pg_index == base_index) { + /* + * no pages were returned, so release + * our hold on the upl and leave + */ + if (!(flags & UPL_NOCOMMIT)) { + ubc_upl_abort_range(upl, upl_offset, isize, UPL_ABORT_FREE_ON_EMPTY); + } + + goto out; + } + } + isize = ((pg_index + 1) - base_index) * PAGE_SIZE; /* * we come here for pageouts to 'real' files and @@ -340,61 +440,40 @@ vnode_pageout(struct vnode *vp, * through it and find the 'runs' of dirty pages * to call VNOP_PAGEOUT on... */ + if (ubc_getsize(vp) == 0) { - /* + /* * if the file has been effectively deleted, then * we need to go through the UPL and invalidate any * buffer headers we might have that reference any * of it's pages */ for (offset = upl_offset; isize; isize -= PAGE_SIZE, offset += PAGE_SIZE) { -#if NFSCLIENT - if (vp->v_tag == VT_NFS) +#if CONFIG_NFS_CLIENT + if (vp->v_tag == VT_NFS) { /* check with nfs if page is OK to drop */ error = nfs_buf_page_inval(vp, (off_t)f_offset); - else -#endif + } else +#endif /* CONFIG_NFS_CLIENT */ { - blkno = ubc_offtoblk(vp, (off_t)f_offset); - error = buf_invalblkno(vp, blkno, 0); + blkno = ubc_offtoblk(vp, (off_t)f_offset); + error = buf_invalblkno(vp, blkno, 0); } if (error) { - if ( !(flags & UPL_NOCOMMIT)) - ubc_upl_abort_range(upl, offset, PAGE_SIZE, UPL_ABORT_FREE_ON_EMPTY); - if (error_ret == 0) - error_ret = error; + if (!(flags & UPL_NOCOMMIT)) { + ubc_upl_abort_range(upl, offset, PAGE_SIZE, UPL_ABORT_FREE_ON_EMPTY); + } + if (error_ret == 0) { + error_ret = error; + } result = PAGER_ERROR; - - } else if ( !(flags & UPL_NOCOMMIT)) { - ubc_upl_commit_range(upl, offset, PAGE_SIZE, UPL_COMMIT_FREE_ON_EMPTY); + } else if (!(flags & UPL_NOCOMMIT)) { + ubc_upl_commit_range(upl, offset, PAGE_SIZE, UPL_COMMIT_FREE_ON_EMPTY); } f_offset += PAGE_SIZE; } goto out; } - /* - * Ignore any non-present pages at the end of the - * UPL so that we aren't looking at a upl that - * may already have been freed by the preceeding - * aborts/completions. - */ - base_index = upl_offset / PAGE_SIZE; - - for (pg_index = (upl_offset + isize) / PAGE_SIZE; pg_index > base_index;) { - if (upl_page_present(pl, --pg_index)) - break; - if (pg_index == base_index) { - /* - * no pages were returned, so release - * our hold on the upl and leave - */ - if ( !(flags & UPL_NOCOMMIT)) - ubc_upl_abort_range(upl, upl_offset, isize, UPL_ABORT_FREE_ON_EMPTY); - - goto out; - } - } - isize = ((pg_index + 1) - base_index) * PAGE_SIZE; offset = upl_offset; pg_index = base_index; @@ -403,20 +482,20 @@ vnode_pageout(struct vnode *vp, int xsize; int num_of_pages; - if ( !upl_page_present(pl, pg_index)) { - /* + if (!upl_page_present(pl, pg_index)) { + /* * we asked for RET_ONLY_DIRTY, so it's possible * to get back empty slots in the UPL * just skip over them */ - f_offset += PAGE_SIZE; + f_offset += PAGE_SIZE; offset += PAGE_SIZE; isize -= PAGE_SIZE; pg_index++; continue; } - if ( !upl_dirty_page(pl, pg_index)) { + if (!upl_dirty_page(pl, pg_index)) { /* * if the page is not dirty and reached here it is * marked precious or it is due to invalidation in @@ -427,27 +506,28 @@ vnode_pageout(struct vnode *vp, * Note we must not sleep here if the buffer is busy - that is * a lock inversion which causes deadlock. */ -#if NFSCLIENT - if (vp->v_tag == VT_NFS) +#if CONFIG_NFS_CLIENT + if (vp->v_tag == VT_NFS) { /* check with nfs if page is OK to drop */ error = nfs_buf_page_inval(vp, (off_t)f_offset); - else -#endif + } else +#endif /* CONFIG_NFS_CLIENT */ { - blkno = ubc_offtoblk(vp, (off_t)f_offset); - error = buf_invalblkno(vp, blkno, 0); + blkno = ubc_offtoblk(vp, (off_t)f_offset); + error = buf_invalblkno(vp, blkno, 0); } if (error) { - if ( !(flags & UPL_NOCOMMIT)) - ubc_upl_abort_range(upl, offset, PAGE_SIZE, UPL_ABORT_FREE_ON_EMPTY); - if (error_ret == 0) - error_ret = error; + if (!(flags & UPL_NOCOMMIT)) { + ubc_upl_abort_range(upl, offset, PAGE_SIZE, UPL_ABORT_FREE_ON_EMPTY); + } + if (error_ret == 0) { + error_ret = error; + } result = PAGER_ERROR; - - } else if ( !(flags & UPL_NOCOMMIT)) { - ubc_upl_commit_range(upl, offset, PAGE_SIZE, UPL_COMMIT_FREE_ON_EMPTY); + } else if (!(flags & UPL_NOCOMMIT)) { + ubc_upl_commit_range(upl, offset, PAGE_SIZE, UPL_COMMIT_FREE_ON_EMPTY); } - f_offset += PAGE_SIZE; + f_offset += PAGE_SIZE; offset += PAGE_SIZE; isize -= PAGE_SIZE; pg_index++; @@ -458,77 +538,108 @@ vnode_pageout(struct vnode *vp, xsize = isize - PAGE_SIZE; while (xsize) { - if ( !upl_dirty_page(pl, pg_index + num_of_pages)) + if (!upl_dirty_page(pl, pg_index + num_of_pages)) { break; + } num_of_pages++; xsize -= PAGE_SIZE; } xsize = num_of_pages * PAGE_SIZE; KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, - (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START, - xsize, (int)f_offset, 0, 0, 0); + (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START, + xsize, (int)f_offset, 0, 0, 0); - if ( (error = VNOP_PAGEOUT(vp, upl, offset, (off_t)f_offset, - xsize, flags, ctx)) ) { - if (error_ret == 0) - error_ret = error; + if ((error = VNOP_PAGEOUT(vp, upl, offset, (off_t)f_offset, + xsize, flags, ctx))) { + if (error_ret == 0) { + error_ret = error; + } result = PAGER_ERROR; } KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, - (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END, - xsize, 0, 0, 0, 0); + (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END, + xsize, 0, 0, 0, 0); - f_offset += xsize; + f_offset += xsize; offset += xsize; isize -= xsize; pg_index += num_of_pages; } out: - if (errorp) + vnode_put_from_pager(vp); + + if (errorp) { *errorp = error_ret; + } - return (result); + return result; } pager_return_t vnode_pagein( - struct vnode *vp, - upl_t upl, - upl_offset_t upl_offset, - vm_object_offset_t f_offset, - upl_size_t size, - int flags, - int *errorp) + struct vnode *vp, + upl_t upl, + upl_offset_t upl_offset, + vm_object_offset_t f_offset, + upl_size_t size, + int flags, + int *errorp) { - upl_page_info_t *pl; - int result = PAGER_SUCCESS; - int error = 0; - int pages_in_upl; - int start_pg; - int last_pg; + upl_page_info_t *pl; + int result = PAGER_SUCCESS; + int error = 0; + int pages_in_upl; + int start_pg; + int last_pg; int first_pg; - int xsize; - int must_commit = 1; + int xsize; + int must_commit = 1; + int ignore_valid_page_check = 0; - if (flags & UPL_NOCOMMIT) - must_commit = 0; + if (flags & UPL_NOCOMMIT) { + must_commit = 0; + } + + if (flags & UPL_IGNORE_VALID_PAGE_CHECK) { + ignore_valid_page_check = 1; + } + + /* + * This call is non-blocking and does not ever fail but it can + * only be made when there is other explicit synchronization + * with reclaiming of the vnode which, in this path, is provided + * by the paging in progress counter. + * + * In addition, this may also be entered via vm_swapfile_io + * where the existing iocount provides the necessary synchronization. + * Ideally we would not take an additional iocount here in the cases + * where an explcit iocount has already been taken but this call + * doesn't cause a deadlock as other forms of vnode_get* might if + * this thread has already taken an iocount. + */ + error = vnode_getalways_from_pager(vp); + if (error != 0) { + /* This can't happen */ + panic("vnode_getalways returned %d for vp %p", error, vp); + } if (UBCINFOEXISTS(vp) == 0) { result = PAGER_ERROR; error = PAGER_ERROR; - if (upl && must_commit) + if (upl && must_commit) { ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_ERROR); + } goto out; } if (upl == (upl_t)NULL) { flags &= ~UPL_NOCOMMIT; - if (size > (MAX_UPL_SIZE * PAGE_SIZE)) { - result = PAGER_ERROR; + if (size > MAX_UPL_SIZE_BYTES) { + result = PAGER_ERROR; error = PAGER_ERROR; goto out; } @@ -541,21 +652,22 @@ vnode_pagein( * take any locks it needs, before effectively locking the pages into a UPL... * so we pass a NULL into the filesystem instead of a UPL pointer... the 'upl_offset' * is used to identify the "must have" page in the extent... the filesystem is free - * to clip the extent to better fit the underlying FS blocksize if it desires as + * to clip the extent to better fit the underlying FS blocksize if it desires as * long as it continues to include the "must have" page... 'f_offset' + 'upl_offset' * identifies that page */ - if ( (error = VNOP_PAGEIN(vp, NULL, upl_offset, (off_t)f_offset, - size, flags, vfs_context_current())) ) { + if ((error = VNOP_PAGEIN(vp, NULL, upl_offset, (off_t)f_offset, + size, flags, vfs_context_current()))) { + set_thread_pagein_error(current_thread(), error); result = PAGER_ERROR; error = PAGER_ERROR; } goto out; } - ubc_create_upl(vp, f_offset, size, &upl, &pl, UPL_UBC_PAGEIN | UPL_RET_ONLY_ABSENT); + ubc_create_upl_kernel(vp, f_offset, size, &upl, &pl, UPL_UBC_PAGEIN | UPL_RET_ONLY_ABSENT, VM_KERN_MEMORY_FILE); if (upl == (upl_t)NULL) { - result = PAGER_ABSENT; + result = PAGER_ABSENT; error = PAGER_ABSENT; goto out; } @@ -563,7 +675,7 @@ vnode_pagein( upl_offset = 0; first_pg = 0; - + /* * if we get here, we've created the upl and * are responsible for commiting/aborting it @@ -571,26 +683,28 @@ vnode_pagein( */ must_commit = 1; } else { - pl = ubc_upl_pageinfo(upl); + pl = ubc_upl_pageinfo(upl); first_pg = upl_offset / PAGE_SIZE; } pages_in_upl = size / PAGE_SIZE; DTRACE_VM2(pgpgin, int, pages_in_upl, (uint64_t *), NULL); /* - * before we start marching forward, we must make sure we end on + * before we start marching forward, we must make sure we end on * a present page, otherwise we will be working with a freed - * upl + * upl */ for (last_pg = pages_in_upl - 1; last_pg >= first_pg; last_pg--) { - if (upl_page_present(pl, last_pg)) + if (upl_page_present(pl, last_pg)) { break; + } if (last_pg == first_pg) { - /* + /* * empty UPL, no pages are present */ - if (must_commit) - ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY); + if (must_commit) { + ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY); + } goto out; } } @@ -598,48 +712,58 @@ vnode_pagein( last_pg = first_pg; while (last_pg < pages_in_upl) { - /* + /* * skip over missing pages... */ - for ( ; last_pg < pages_in_upl; last_pg++) { - if (upl_page_present(pl, last_pg)) - break; + for (; last_pg < pages_in_upl; last_pg++) { + if (upl_page_present(pl, last_pg)) { + break; + } } - /* - * skip over 'valid' pages... we don't want to issue I/O for these - */ - for (start_pg = last_pg; last_pg < pages_in_upl; last_pg++) { - if (!upl_valid_page(pl, last_pg)) - break; + + if (ignore_valid_page_check == 1) { + start_pg = last_pg; + } else { + /* + * skip over 'valid' pages... we don't want to issue I/O for these + */ + for (start_pg = last_pg; last_pg < pages_in_upl; last_pg++) { + if (!upl_valid_page(pl, last_pg)) { + break; + } + } } + if (last_pg > start_pg) { - /* + /* * we've found a range of valid pages * if we've got COMMIT responsibility * commit this range of pages back to the * cache unchanged */ - xsize = (last_pg - start_pg) * PAGE_SIZE; + xsize = (last_pg - start_pg) * PAGE_SIZE; - if (must_commit) - ubc_upl_abort_range(upl, start_pg * PAGE_SIZE, xsize, UPL_ABORT_FREE_ON_EMPTY); + if (must_commit) { + ubc_upl_abort_range(upl, start_pg * PAGE_SIZE, xsize, UPL_ABORT_FREE_ON_EMPTY); + } } - if (last_pg == pages_in_upl) - /* + if (last_pg == pages_in_upl) { + /* * we're done... all pages that were present - * have either had I/O issued on them or + * have either had I/O issued on them or * were aborted unchanged... */ - break; + break; + } if (!upl_page_present(pl, last_pg)) { - /* - * we found a range of valid pages + /* + * we found a range of valid pages * terminated by a missing page... * bump index to the next page and continue on */ - last_pg++; - continue; + last_pg++; + continue; } /* * scan from the found invalid page looking for a valid @@ -648,71 +772,54 @@ vnode_pagein( * 'cluster_io' */ for (start_pg = last_pg; last_pg < pages_in_upl; last_pg++) { - if (upl_valid_page(pl, last_pg) || !upl_page_present(pl, last_pg)) - break; + if ((!ignore_valid_page_check && upl_valid_page(pl, last_pg)) || !upl_page_present(pl, last_pg)) { + break; + } } if (last_pg > start_pg) { - int xoff; - xsize = (last_pg - start_pg) * PAGE_SIZE; + int xoff; + xsize = (last_pg - start_pg) * PAGE_SIZE; xoff = start_pg * PAGE_SIZE; - if ( (error = VNOP_PAGEIN(vp, upl, (upl_offset_t) xoff, - (off_t)f_offset + xoff, - xsize, flags, vfs_context_current())) ) { - /* + if ((error = VNOP_PAGEIN(vp, upl, (upl_offset_t) xoff, + (off_t)f_offset + xoff, + xsize, flags, vfs_context_current()))) { + /* * Usually this UPL will be aborted/committed by the lower cluster layer. * * a) In the case of decmpfs, however, we may return an error (EAGAIN) to avoid - * a deadlock with another thread already inflating the file. + * a deadlock with another thread already inflating the file. * * b) In the case of content protection, EPERM is a valid error and we should respect it. * * In those cases, we must take care of our UPL at this layer itself. */ if (must_commit) { - if(error == EAGAIN) { - ubc_upl_abort_range(upl, (upl_offset_t) xoff, xsize, UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_RESTART); + if (error == EAGAIN) { + ubc_upl_abort_range(upl, (upl_offset_t) xoff, xsize, UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_RESTART); } -#if CONFIG_PROTECT - if(error == EPERM) { - ubc_upl_abort_range(upl, (upl_offset_t) xoff, xsize, UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_ERROR); + if (error == EPERM) { + ubc_upl_abort_range(upl, (upl_offset_t) xoff, xsize, UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_ERROR); } -#endif } + set_thread_pagein_error(current_thread(), error); result = PAGER_ERROR; error = PAGER_ERROR; - } } - } + } out: - if (errorp) - *errorp = result; + vnode_put_from_pager(vp); - return (error); -} - -void -vnode_pager_shutdown(void) -{ - int i; - vnode_t vp; - - for(i = 0; i < MAX_BACKING_STORE; i++) { - vp = (vnode_t)(bs_port_table[i]).vp; - if (vp) { - (bs_port_table[i]).vp = 0; - - /* get rid of macx_swapon() reference */ - vnode_rele(vp); - } + if (errorp) { + *errorp = result; } -} + return error; +} void * upl_get_internal_page_list(upl_t upl) { - return(UPL_GET_INTERNAL_PAGE_LIST(upl)); - + return UPL_GET_INTERNAL_PAGE_LIST(upl); }