X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/6d2010ae8f7a6078e10b361c6962983bab233e0f..4d15aeb193b2c68f1d38666c317f8d3734f5f083:/bsd/vm/vnode_pager.c diff --git a/bsd/vm/vnode_pager.c b/bsd/vm/vnode_pager.c index d12a65652..b70717b92 100644 --- a/bsd/vm/vnode_pager.c +++ b/bsd/vm/vnode_pager.c @@ -90,7 +90,7 @@ vnode_pager_throttle() ut = get_bsdthread_info(current_thread()); if (ut->uu_lowpri_window) - throttle_lowpri_io(TRUE); + throttle_lowpri_io(1); } @@ -102,6 +102,33 @@ vnode_pager_isSSD(vnode_t vp) return (FALSE); } +#if CONFIG_IOSCHED +void +vnode_pager_issue_reprioritize_io(struct vnode *devvp, uint64_t blkno, uint32_t len, int priority) +{ + u_int32_t blocksize = 0; + dk_extent_t extent; + dk_set_tier_t set_tier; + int error = 0; + + error = VNOP_IOCTL(devvp, DKIOCGETBLOCKSIZE, (caddr_t)&blocksize, 0, vfs_context_kernel()); + if (error) + return; + + memset(&extent, 0, sizeof(dk_extent_t)); + memset(&set_tier, 0, sizeof(dk_set_tier_t)); + + extent.offset = blkno * (u_int64_t) blocksize; + extent.length = len; + + set_tier.extents = &extent; + set_tier.extentsCount = 1; + set_tier.tier = priority; + + error = VNOP_IOCTL(devvp, DKIOCSETTIER, (caddr_t)&set_tier, 0, vfs_context_kernel()); + return; +} +#endif uint32_t vnode_pager_isinuse(struct vnode *vp) @@ -112,41 +139,57 @@ vnode_pager_isinuse(struct vnode *vp) } uint32_t -vnode_pager_return_hard_throttle_limit(struct vnode *vp, uint32_t *limit, uint32_t hard_throttle) +vnode_pager_return_throttle_io_limit(struct vnode *vp, uint32_t *limit) { - return(cluster_hard_throttle_limit(vp, limit, hard_throttle)); + return(cluster_throttle_io_limit(vp, limit)); } vm_object_offset_t vnode_pager_get_filesize(struct vnode *vp) { - return (vm_object_offset_t) ubc_getsize(vp); } +extern int safe_getpath(struct vnode *dvp, char *leafname, char *path, int _len, int *truncated_path); + kern_return_t -vnode_pager_get_pathname( +vnode_pager_get_name( struct vnode *vp, char *pathname, - vm_size_t *length_p) + vm_size_t pathname_len, + char *filename, + vm_size_t filename_len, + boolean_t *truncated_path_p) { - int error, len; - - len = (int) *length_p; - error = vn_getpath(vp, pathname, &len); - if (error != 0) { - return KERN_FAILURE; + *truncated_path_p = FALSE; + if (pathname != NULL) { + /* get the path name */ + safe_getpath(vp, NULL, + pathname, (int) pathname_len, + truncated_path_p); + } + if ((pathname == NULL || *truncated_path_p) && + filename != NULL) { + /* get the file name */ + const char *name; + + name = vnode_getname_printable(vp); + strlcpy(filename, name, (size_t) filename_len); + vnode_putname_printable(name); } - *length_p = (vm_size_t) len; return KERN_SUCCESS; } kern_return_t -vnode_pager_get_filename( +vnode_pager_get_mtime( struct vnode *vp, - const char **filename) + struct timespec *current_mtime, + struct timespec *cs_mtime) { - *filename = vp->v_name; + vnode_mtime(vp, current_mtime, vfs_context_current()); + if (cs_mtime != NULL) { + ubc_get_cs_mtime(vp, cs_mtime); + } return KERN_SUCCESS; } @@ -280,15 +323,17 @@ vnode_pageout(struct vnode *vp, * just go ahead and call vnop_pageout since * it has already sorted out the dirty ranges */ - KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START, - size, 1, 0, 0, 0); + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, + (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START, + size, 1, 0, 0, 0); if ( (error_ret = VNOP_PAGEOUT(vp, upl, upl_offset, (off_t)f_offset, (size_t)size, flags, ctx)) ) result = PAGER_ERROR; - KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END, - size, 1, 0, 0, 0); + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, + (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END, + size, 1, 0, 0, 0); goto out; } @@ -303,15 +348,17 @@ vnode_pageout(struct vnode *vp, * via 'f_offset' and 'size' into a UPL... this allows the filesystem to first * take any locks it needs, before effectively locking the pages into a UPL... */ - KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START, - size, (int)f_offset, 0, 0, 0); + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, + (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START, + size, (int)f_offset, 0, 0, 0); if ( (error_ret = VNOP_PAGEOUT(vp, NULL, upl_offset, (off_t)f_offset, size, flags, ctx)) ) { result = PAGER_ERROR; } - KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END, - size, 0, 0, 0, 0); + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, + (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END, + size, 0, 0, 0, 0); goto out; } @@ -329,6 +376,30 @@ vnode_pageout(struct vnode *vp, } else pl = ubc_upl_pageinfo(upl); + /* + * Ignore any non-present pages at the end of the + * UPL so that we aren't looking at a upl that + * may already have been freed by the preceeding + * aborts/completions. + */ + base_index = upl_offset / PAGE_SIZE; + + for (pg_index = (upl_offset + isize) / PAGE_SIZE; pg_index > base_index;) { + if (upl_page_present(pl, --pg_index)) + break; + if (pg_index == base_index) { + /* + * no pages were returned, so release + * our hold on the upl and leave + */ + if ( !(flags & UPL_NOCOMMIT)) + ubc_upl_abort_range(upl, upl_offset, isize, UPL_ABORT_FREE_ON_EMPTY); + + goto out; + } + } + isize = ((pg_index + 1) - base_index) * PAGE_SIZE; + /* * we come here for pageouts to 'real' files and * for msyncs... the upl may not contain any @@ -336,6 +407,7 @@ vnode_pageout(struct vnode *vp, * through it and find the 'runs' of dirty pages * to call VNOP_PAGEOUT on... */ + if (ubc_getsize(vp) == 0) { /* * if the file has been effectively deleted, then @@ -368,29 +440,6 @@ vnode_pageout(struct vnode *vp, } goto out; } - /* - * Ignore any non-present pages at the end of the - * UPL so that we aren't looking at a upl that - * may already have been freed by the preceeding - * aborts/completions. - */ - base_index = upl_offset / PAGE_SIZE; - - for (pg_index = (upl_offset + isize) / PAGE_SIZE; pg_index > base_index;) { - if (upl_page_present(pl, --pg_index)) - break; - if (pg_index == base_index) { - /* - * no pages were returned, so release - * our hold on the upl and leave - */ - if ( !(flags & UPL_NOCOMMIT)) - ubc_upl_abort_range(upl, upl_offset, isize, UPL_ABORT_FREE_ON_EMPTY); - - goto out; - } - } - isize = ((pg_index + 1) - base_index) * PAGE_SIZE; offset = upl_offset; pg_index = base_index; @@ -461,8 +510,9 @@ vnode_pageout(struct vnode *vp, } xsize = num_of_pages * PAGE_SIZE; - KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START, - xsize, (int)f_offset, 0, 0, 0); + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, + (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START, + xsize, (int)f_offset, 0, 0, 0); if ( (error = VNOP_PAGEOUT(vp, upl, offset, (off_t)f_offset, xsize, flags, ctx)) ) { @@ -470,8 +520,9 @@ vnode_pageout(struct vnode *vp, error_ret = error; result = PAGER_ERROR; } - KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END, - xsize, 0, 0, 0, 0); + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, + (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END, + xsize, 0, 0, 0, 0); f_offset += xsize; offset += xsize; @@ -496,7 +547,6 @@ vnode_pagein( int flags, int *errorp) { - struct uthread *ut; upl_page_info_t *pl; int result = PAGER_SUCCESS; int error = 0; @@ -506,10 +556,14 @@ vnode_pagein( int first_pg; int xsize; int must_commit = 1; + int ignore_valid_page_check = 0; if (flags & UPL_NOCOMMIT) must_commit = 0; + if (flags & UPL_IGNORE_VALID_PAGE_CHECK) + ignore_valid_page_check = 1; + if (UBCINFOEXISTS(vp) == 0) { result = PAGER_ERROR; error = PAGER_ERROR; @@ -522,7 +576,7 @@ vnode_pagein( if (upl == (upl_t)NULL) { flags &= ~UPL_NOCOMMIT; - if (size > (MAX_UPL_SIZE * PAGE_SIZE)) { + if (size > MAX_UPL_SIZE_BYTES) { result = PAGER_ERROR; error = PAGER_ERROR; goto out; @@ -554,6 +608,8 @@ vnode_pagein( error = PAGER_ABSENT; goto out; } + ubc_upl_range_needed(upl, upl_offset / PAGE_SIZE, 1); + upl_offset = 0; first_pg = 0; @@ -598,13 +654,19 @@ vnode_pagein( if (upl_page_present(pl, last_pg)) break; } - /* - * skip over 'valid' pages... we don't want to issue I/O for these - */ - for (start_pg = last_pg; last_pg < pages_in_upl; last_pg++) { - if (!upl_valid_page(pl, last_pg)) - break; + + if (ignore_valid_page_check == 1) { + start_pg = last_pg; + } else { + /* + * skip over 'valid' pages... we don't want to issue I/O for these + */ + for (start_pg = last_pg; last_pg < pages_in_upl; last_pg++) { + if (!upl_valid_page(pl, last_pg)) + break; + } } + if (last_pg > start_pg) { /* * we've found a range of valid pages @@ -641,7 +703,7 @@ vnode_pagein( * 'cluster_io' */ for (start_pg = last_pg; last_pg < pages_in_upl; last_pg++) { - if (upl_valid_page(pl, last_pg) || !upl_page_present(pl, last_pg)) + if (( !ignore_valid_page_check && upl_valid_page(pl, last_pg)) || !upl_page_present(pl, last_pg)) break; } if (last_pg > start_pg) { @@ -666,11 +728,9 @@ vnode_pagein( if(error == EAGAIN) { ubc_upl_abort_range(upl, (upl_offset_t) xoff, xsize, UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_RESTART); } -#if CONFIG_PROTECT if(error == EPERM) { ubc_upl_abort_range(upl, (upl_offset_t) xoff, xsize, UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_ERROR); } -#endif } result = PAGER_ERROR; error = PAGER_ERROR; @@ -682,39 +742,9 @@ out: if (errorp) *errorp = result; - ut = get_bsdthread_info(current_thread()); - - if (ut->uu_lowpri_window) { - /* - * task is marked as a low priority I/O type - * and the I/O we issued while in this page fault - * collided with normal I/O operations... we'll - * delay in order to mitigate the impact of this - * task on the normal operation of the system - */ - throttle_lowpri_io(TRUE); - } return (error); } -void -vnode_pager_shutdown(void) -{ - int i; - vnode_t vp; - - for(i = 0; i < MAX_BACKING_STORE; i++) { - vp = (vnode_t)(bs_port_table[i]).vp; - if (vp) { - (bs_port_table[i]).vp = 0; - - /* get rid of macx_swapon() reference */ - vnode_rele(vp); - } - } -} - - void * upl_get_internal_page_list(upl_t upl) {