X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/2d21ac55c334faf3a56e5634905ed6987fc787d4..9d749ea394c01276fa19e397e70f46858e849c76:/osfmk/vm/memory_object.c diff --git a/osfmk/vm/memory_object.c b/osfmk/vm/memory_object.c index c84e776c6..d37eb4224 100644 --- a/osfmk/vm/memory_object.c +++ b/osfmk/vm/memory_object.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved. + * Copyright (c) 2000-2008 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -62,8 +62,6 @@ * External memory management interface control functions. */ -#include - /* * Interface dependencies: */ @@ -101,16 +99,14 @@ #include /* For kernel_map, vm_move */ #include /* For vm_map_pageable */ #include /* Needed by some vm_page.h macros */ +#include -#if MACH_PAGEMAP #include -#endif /* MACH_PAGEMAP */ #include - memory_object_default_t memory_manager_default = MEMORY_OBJECT_DEFAULT_NULL; -decl_mutex_data(, memory_manager_default_lock) +decl_lck_mtx_data(, memory_manager_default_lock) /* @@ -133,16 +129,16 @@ decl_mutex_data(, memory_manager_default_lock) #define memory_object_should_return_page(m, should_return) \ (should_return != MEMORY_OBJECT_RETURN_NONE && \ - (((m)->dirty || ((m)->dirty = pmap_is_modified((m)->phys_page))) || \ + (((m)->dirty || ((m)->dirty = pmap_is_modified(VM_PAGE_GET_PHYS_PAGE(m)))) || \ ((m)->precious && (should_return) == MEMORY_OBJECT_RETURN_ALL) || \ (should_return) == MEMORY_OBJECT_RETURN_ANYTHING)) typedef int memory_object_lock_result_t; -#define MEMORY_OBJECT_LOCK_RESULT_DONE 0 -#define MEMORY_OBJECT_LOCK_RESULT_MUST_BLOCK 1 -#define MEMORY_OBJECT_LOCK_RESULT_MUST_CLEAN 2 -#define MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN 3 +#define MEMORY_OBJECT_LOCK_RESULT_DONE 0 +#define MEMORY_OBJECT_LOCK_RESULT_MUST_BLOCK 1 +#define MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN 2 +#define MEMORY_OBJECT_LOCK_RESULT_MUST_FREE 3 memory_object_lock_result_t memory_object_lock_page( vm_page_t m, @@ -172,157 +168,107 @@ memory_object_lock_page( { XPR(XPR_MEMORY_OBJECT, "m_o_lock_page, page 0x%X rtn %d flush %d prot %d\n", - (integer_t)m, should_return, should_flush, prot, 0); + m, should_return, should_flush, prot, 0); - /* - * If we cannot change access to the page, - * either because a mapping is in progress - * (busy page) or because a mapping has been - * wired, then give up. - */ if (m->busy || m->cleaning) - return(MEMORY_OBJECT_LOCK_RESULT_MUST_BLOCK); + return (MEMORY_OBJECT_LOCK_RESULT_MUST_BLOCK); + + if (m->laundry) + vm_pageout_steal_laundry(m, FALSE); /* * Don't worry about pages for which the kernel * does not have any data. */ - if (m->absent || m->error || m->restart) { - if(m->error && should_flush) { - /* dump the page, pager wants us to */ - /* clean it up and there is no */ - /* relevant data to return */ - if(m->wire_count == 0) { - VM_PAGE_FREE(m); - return(MEMORY_OBJECT_LOCK_RESULT_DONE); - } - } else { - return(MEMORY_OBJECT_LOCK_RESULT_DONE); + if (m->error && should_flush && !VM_PAGE_WIRED(m)) { + /* + * dump the page, pager wants us to + * clean it up and there is no + * relevant data to return + */ + return (MEMORY_OBJECT_LOCK_RESULT_MUST_FREE); } + return (MEMORY_OBJECT_LOCK_RESULT_DONE); } - assert(!m->fictitious); - /* - * If the page is wired, just clean or return the page if needed. - * Wired pages don't get flushed or disconnected from the pmap. - */ - - if (m->wire_count != 0) { - if (memory_object_should_return_page(m, should_return)) { - if (m->dirty) - return(MEMORY_OBJECT_LOCK_RESULT_MUST_CLEAN); - else - return(MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN); - } - - return(MEMORY_OBJECT_LOCK_RESULT_DONE); - } - - /* - * If the page is to be flushed, allow - * that to be done as part of the protection. - */ - - if (should_flush) - prot = VM_PROT_ALL; - - /* - * Set the page lock. - * - * If we are decreasing permission, do it now; - * let the fault handler take care of increases - * (pmap_page_protect may not increase protection). - */ + if (VM_PAGE_WIRED(m)) { + /* + * The page is wired... just clean or return the page if needed. + * Wired pages don't get flushed or disconnected from the pmap. + */ + if (memory_object_should_return_page(m, should_return)) + return (MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN); - if (prot != VM_PROT_NO_CHANGE) { - pmap_page_protect(m->phys_page, VM_PROT_ALL & ~prot); + return (MEMORY_OBJECT_LOCK_RESULT_DONE); + } - PAGE_WAKEUP(m); + if (should_flush) { + /* + * must do the pmap_disconnect before determining the + * need to return the page... otherwise it's possible + * for the page to go from the clean to the dirty state + * after we've made our decision + */ + if (pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m)) & VM_MEM_MODIFIED) { + SET_PAGE_DIRTY(m, FALSE); + } + } else { + /* + * If we are decreasing permission, do it now; + * let the fault handler take care of increases + * (pmap_page_protect may not increase protection). + */ + if (prot != VM_PROT_NO_CHANGE) + pmap_page_protect(VM_PAGE_GET_PHYS_PAGE(m), VM_PROT_ALL & ~prot); } - /* - * Handle page returning. + * Handle returning dirty or precious pages */ if (memory_object_should_return_page(m, should_return)) { - /* - * If we weren't planning - * to flush the page anyway, - * we may need to remove the - * page from the pageout - * system and from physical - * maps now. + * we use to do a pmap_disconnect here in support + * of memory_object_lock_request, but that routine + * no longer requires this... in any event, in + * our world, it would turn into a big noop since + * we don't lock the page in any way and as soon + * as we drop the object lock, the page can be + * faulted back into an address space + * + * if (!should_flush) + * pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m)); */ - - vm_page_lockspin_queues(); - VM_PAGE_QUEUES_REMOVE(m); - vm_page_unlock_queues(); - - if (!should_flush) - pmap_disconnect(m->phys_page); - - if (m->dirty) - return(MEMORY_OBJECT_LOCK_RESULT_MUST_CLEAN); - else - return(MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN); + return (MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN); } /* - * Handle flushing + * Handle flushing clean pages */ - if (should_flush) { - VM_PAGE_FREE(m); - } else { - /* - * XXX Make clean but not flush a paging hint, - * and deactivate the pages. This is a hack - * because it overloads flush/clean with - * implementation-dependent meaning. This only - * happens to pages that are already clean. - */ + if (should_flush) + return (MEMORY_OBJECT_LOCK_RESULT_MUST_FREE); - if (vm_page_deactivate_hint && - (should_return != MEMORY_OBJECT_RETURN_NONE)) { - vm_page_lock_queues(); - vm_page_deactivate(m); - vm_page_unlock_queues(); - } - } + /* + * we use to deactivate clean pages at this point, + * but we do not believe that an msync should change + * the 'age' of a page in the cache... here is the + * original comment and code concerning this... + * + * XXX Make clean but not flush a paging hint, + * and deactivate the pages. This is a hack + * because it overloads flush/clean with + * implementation-dependent meaning. This only + * happens to pages that are already clean. + * + * if (vm_page_deactivate_hint && (should_return != MEMORY_OBJECT_RETURN_NONE)) + * return (MEMORY_OBJECT_LOCK_RESULT_MUST_DEACTIVATE); + */ - return(MEMORY_OBJECT_LOCK_RESULT_DONE); + return (MEMORY_OBJECT_LOCK_RESULT_DONE); } -#define LIST_REQ_PAGEOUT_PAGES(object, data_cnt, action, po, ro, ioerr, iosync) \ -MACRO_BEGIN \ - \ - register int upl_flags; \ - memory_object_t pager; \ - \ - if ((pager = (object)->pager) != MEMORY_OBJECT_NULL) { \ - vm_object_paging_begin(object); \ - vm_object_unlock(object); \ - \ - if (iosync) \ - upl_flags = UPL_MSYNC | UPL_IOSYNC; \ - else \ - upl_flags = UPL_MSYNC; \ - \ - (void) memory_object_data_return(pager, \ - po, \ - data_cnt, \ - ro, \ - ioerr, \ - (action) == MEMORY_OBJECT_LOCK_RESULT_MUST_CLEAN,\ - !should_flush, \ - upl_flags); \ - \ - vm_object_lock(object); \ - vm_object_paging_end(object); \ - } \ -MACRO_END + /* * Routine: memory_object_lock_request [user interface] @@ -361,16 +307,8 @@ memory_object_lock_request( vm_prot_t prot) { vm_object_t object; - __unused boolean_t should_flush; - should_flush = flags & MEMORY_OBJECT_DATA_FLUSH; - - XPR(XPR_MEMORY_OBJECT, - "m_o_lock_request, control 0x%X off 0x%X size 0x%X flags %X prot %X\n", - (integer_t)control, offset, size, - (((should_return&1)<<1)|should_flush), prot); - - /* + /* * Check for bogus arguments. */ object = memory_object_control_to_vm_object(control); @@ -388,10 +326,20 @@ memory_object_lock_request( */ vm_object_lock(object); vm_object_paging_begin(object); + + if (flags & MEMORY_OBJECT_DATA_FLUSH_ALL) { + if ((should_return != MEMORY_OBJECT_RETURN_NONE) || offset || object->copy) { + flags &= ~MEMORY_OBJECT_DATA_FLUSH_ALL; + flags |= MEMORY_OBJECT_DATA_FLUSH; + } + } offset -= object->paging_offset; - (void)vm_object_update(object, - offset, size, resid_offset, io_errno, should_return, flags, prot); + if (flags & MEMORY_OBJECT_DATA_FLUSH_ALL) + vm_object_reap_pages(object, REAP_DATA_FLUSH); + else + (void)vm_object_update(object, offset, size, resid_offset, + io_errno, should_return, flags, prot); vm_object_paging_end(object); vm_object_unlock(object); @@ -491,7 +439,7 @@ vm_object_sync( XPR(XPR_VM_OBJECT, "vm_o_sync, object 0x%X, offset 0x%X size 0x%x flush %d rtn %d\n", - (integer_t)object, offset, size, should_flush, should_return); + object, offset, size, should_flush, should_return); /* * Lock the object, and acquire a paging reference to @@ -501,9 +449,17 @@ vm_object_sync( vm_object_lock(object); vm_object_paging_begin(object); - if (should_flush) + if (should_flush) { flags = MEMORY_OBJECT_DATA_FLUSH; - else + /* + * This flush is from an msync(), not a truncate(), so the + * contents of the file are not affected. + * MEMORY_OBECT_DATA_NO_CHANGE lets vm_object_update() know + * that the data is not changed and that there's no need to + * push the old contents to a copy object. + */ + flags |= MEMORY_OBJECT_DATA_NO_CHANGE; + } else flags = 0; if (should_iosync) @@ -524,6 +480,41 @@ vm_object_sync( +#define LIST_REQ_PAGEOUT_PAGES(object, data_cnt, po, ro, ioerr, iosync) \ +MACRO_BEGIN \ + \ + int upl_flags; \ + memory_object_t pager; \ + \ + if (object->object_slid) { \ + panic("Objects with slid pages not allowed\n"); \ + } \ + \ + if ((pager = (object)->pager) != MEMORY_OBJECT_NULL) { \ + vm_object_paging_begin(object); \ + vm_object_unlock(object); \ + \ + if (iosync) \ + upl_flags = UPL_MSYNC | UPL_IOSYNC; \ + else \ + upl_flags = UPL_MSYNC; \ + \ + (void) memory_object_data_return(pager, \ + po, \ + (memory_object_cluster_size_t)data_cnt, \ + ro, \ + ioerr, \ + FALSE, \ + FALSE, \ + upl_flags); \ + \ + vm_object_lock(object); \ + vm_object_paging_end(object); \ + } \ +MACRO_END + +extern struct vnode * +vnode_pager_lookup_vnode(memory_object_t); static int vm_object_update_extent( @@ -539,127 +530,140 @@ vm_object_update_extent( { vm_page_t m; int retval = 0; - vm_size_t data_cnt = 0; vm_object_offset_t paging_offset = 0; - vm_object_offset_t last_offset = offset; + vm_object_offset_t next_offset = offset; memory_object_lock_result_t page_lock_result; - memory_object_lock_result_t pageout_action; - - pageout_action = MEMORY_OBJECT_LOCK_RESULT_DONE; + memory_object_cluster_size_t data_cnt = 0; + struct vm_page_delayed_work dw_array[DEFAULT_DELAYED_WORK_LIMIT]; + struct vm_page_delayed_work *dwp; + int dw_count; + int dw_limit; + int dirty_count; + + dwp = &dw_array[0]; + dw_count = 0; + dw_limit = DELAYED_WORK_LIMIT(DEFAULT_DELAYED_WORK_LIMIT); + dirty_count = 0; for (; offset < offset_end && object->resident_page_count; offset += PAGE_SIZE_64) { /* - * Limit the number of pages to be cleaned at once. + * Limit the number of pages to be cleaned at once to a contiguous + * run, or at most MAX_UPL_TRANSFER_BYTES */ - if (data_cnt >= PAGE_SIZE * MAX_UPL_TRANSFER) { - LIST_REQ_PAGEOUT_PAGES(object, data_cnt, - pageout_action, paging_offset, offset_resid, io_errno, should_iosync); - data_cnt = 0; - } + if (data_cnt) { + if ((data_cnt >= MAX_UPL_TRANSFER_BYTES) || (next_offset != offset)) { + if (dw_count) { + vm_page_do_delayed_work(object, VM_KERN_MEMORY_NONE, &dw_array[0], dw_count); + dwp = &dw_array[0]; + dw_count = 0; + } + LIST_REQ_PAGEOUT_PAGES(object, data_cnt, + paging_offset, offset_resid, io_errno, should_iosync); + data_cnt = 0; + } + } while ((m = vm_page_lookup(object, offset)) != VM_PAGE_NULL) { - page_lock_result = memory_object_lock_page(m, should_return, should_flush, prot); - - XPR(XPR_MEMORY_OBJECT, - "m_o_update: lock_page, obj 0x%X offset 0x%X result %d\n", - (integer_t)object, offset, page_lock_result, 0, 0); - - switch (page_lock_result) - { - case MEMORY_OBJECT_LOCK_RESULT_DONE: - /* - * End of a cluster of dirty pages. - */ - if (data_cnt) { - LIST_REQ_PAGEOUT_PAGES(object, - data_cnt, pageout_action, - paging_offset, offset_resid, io_errno, should_iosync); - data_cnt = 0; - continue; - } - break; - - case MEMORY_OBJECT_LOCK_RESULT_MUST_BLOCK: - /* - * Since it is necessary to block, - * clean any dirty pages now. - */ - if (data_cnt) { - LIST_REQ_PAGEOUT_PAGES(object, - data_cnt, pageout_action, - paging_offset, offset_resid, io_errno, should_iosync); - data_cnt = 0; - continue; - } - PAGE_SLEEP(object, m, THREAD_UNINT); - continue; - - case MEMORY_OBJECT_LOCK_RESULT_MUST_CLEAN: - case MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN: - /* - * The clean and return cases are similar. - * - * if this would form a discontiguous block, - * clean the old pages and start anew. - * - * Mark the page busy since we will unlock the - * object if we issue the LIST_REQ_PAGEOUT - */ - m->busy = TRUE; - if (data_cnt && - ((last_offset != offset) || (pageout_action != page_lock_result))) { - LIST_REQ_PAGEOUT_PAGES(object, - data_cnt, pageout_action, - paging_offset, offset_resid, io_errno, should_iosync); - data_cnt = 0; - } - m->busy = FALSE; - - if (m->cleaning) { - PAGE_SLEEP(object, m, THREAD_UNINT); - continue; - } - if (data_cnt == 0) { - pageout_action = page_lock_result; - paging_offset = offset; - } - data_cnt += PAGE_SIZE; - last_offset = offset + PAGE_SIZE_64; - - vm_page_lockspin_queues(); - /* - * Clean - */ - m->list_req_pending = TRUE; - m->cleaning = TRUE; - - if (should_flush) { - /* - * and add additional state - * for the flush - */ - m->busy = TRUE; - m->pageout = TRUE; - vm_page_wire(m); - } - vm_page_unlock_queues(); - - retval = 1; - break; + + dwp->dw_mask = 0; + + page_lock_result = memory_object_lock_page(m, should_return, should_flush, prot); + + if (data_cnt && page_lock_result != MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN) { + /* + * End of a run of dirty/precious pages. + */ + if (dw_count) { + vm_page_do_delayed_work(object, VM_KERN_MEMORY_NONE, &dw_array[0], dw_count); + dwp = &dw_array[0]; + dw_count = 0; + } + LIST_REQ_PAGEOUT_PAGES(object, data_cnt, + paging_offset, offset_resid, io_errno, should_iosync); + /* + * LIST_REQ_PAGEOUT_PAGES will drop the object lock which will + * allow the state of page 'm' to change... we need to re-lookup + * the current offset + */ + data_cnt = 0; + continue; + } + + switch (page_lock_result) { + + case MEMORY_OBJECT_LOCK_RESULT_DONE: + break; + + case MEMORY_OBJECT_LOCK_RESULT_MUST_FREE: + if (m->dirty == TRUE) + dirty_count++; + dwp->dw_mask |= DW_vm_page_free; + break; + + case MEMORY_OBJECT_LOCK_RESULT_MUST_BLOCK: + PAGE_SLEEP(object, m, THREAD_UNINT); + continue; + + case MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN: + if (data_cnt == 0) + paging_offset = offset; + + data_cnt += PAGE_SIZE; + next_offset = offset + PAGE_SIZE_64; + + /* + * wired pages shouldn't be flushed and + * since they aren't on any queue, + * no need to remove them + */ + if (!VM_PAGE_WIRED(m)) { + + if (should_flush) { + /* + * add additional state for the flush + */ + m->free_when_done = TRUE; + } + /* + * we use to remove the page from the queues at this + * point, but we do not believe that an msync + * should cause the 'age' of a page to be changed + * + * else + * dwp->dw_mask |= DW_VM_PAGE_QUEUES_REMOVE; + */ + } + retval = 1; + break; + } + if (dwp->dw_mask) { + VM_PAGE_ADD_DELAYED_WORK(dwp, m, dw_count); + + if (dw_count >= dw_limit) { + vm_page_do_delayed_work(object, VM_KERN_MEMORY_NONE, &dw_array[0], dw_count); + dwp = &dw_array[0]; + dw_count = 0; + } } break; } } + + if (object->pager) + task_update_logical_writes(current_task(), (dirty_count * PAGE_SIZE), TASK_WRITE_INVALIDATED, vnode_pager_lookup_vnode(object->pager)); /* * We have completed the scan for applicable pages. * Clean any pages that have been saved. */ + if (dw_count) + vm_page_do_delayed_work(object, VM_KERN_MEMORY_NONE, &dw_array[0], dw_count); + if (data_cnt) { - LIST_REQ_PAGEOUT_PAGES(object, - data_cnt, pageout_action, paging_offset, offset_resid, io_errno, should_iosync); + LIST_REQ_PAGEOUT_PAGES(object, data_cnt, + paging_offset, offset_resid, io_errno, should_iosync); } return (retval); } @@ -675,20 +679,21 @@ vm_object_update_extent( */ kern_return_t vm_object_update( - register vm_object_t object, - register vm_object_offset_t offset, - register vm_object_size_t size, - register vm_object_offset_t *resid_offset, - int *io_errno, - memory_object_return_t should_return, - int flags, - vm_prot_t protection) + vm_object_t object, + vm_object_offset_t offset, + vm_object_size_t size, + vm_object_offset_t *resid_offset, + int *io_errno, + memory_object_return_t should_return, + int flags, + vm_prot_t protection) { vm_object_t copy_object = VM_OBJECT_NULL; boolean_t data_returned = FALSE; boolean_t update_cow; boolean_t should_flush = (flags & MEMORY_OBJECT_DATA_FLUSH) ? TRUE : FALSE; boolean_t should_iosync = (flags & MEMORY_OBJECT_IO_SYNC) ? TRUE : FALSE; + vm_fault_return_t result; int num_of_extents; int n; #define MAX_EXTENTS 8 @@ -768,27 +773,27 @@ vm_object_update( /* * translate offset with respect to shadow's offset */ - copy_offset = (offset >= copy_object->shadow_offset) ? - (vm_map_offset_t)(offset - copy_object->shadow_offset) : + copy_offset = (offset >= copy_object->vo_shadow_offset) ? + (vm_map_offset_t)(offset - copy_object->vo_shadow_offset) : (vm_map_offset_t) 0; - if (copy_offset > copy_object->size) - copy_offset = copy_object->size; + if (copy_offset > copy_object->vo_size) + copy_offset = copy_object->vo_size; /* * clip size with respect to shadow offset */ - if (offset >= copy_object->shadow_offset) { + if (offset >= copy_object->vo_shadow_offset) { copy_size = size; - } else if (size >= copy_object->shadow_offset - offset) { - copy_size = size - (copy_object->shadow_offset - offset); + } else if (size >= copy_object->vo_shadow_offset - offset) { + copy_size = size - (copy_object->vo_shadow_offset - offset); } else { copy_size = 0; } - if (copy_offset + copy_size > copy_object->size) { - if (copy_object->size >= copy_offset) { - copy_size = copy_object->size - copy_offset; + if (copy_offset + copy_size > copy_object->vo_size) { + if (copy_object->vo_size >= copy_offset) { + copy_size = copy_object->vo_size - copy_offset; } else { copy_size = 0; } @@ -804,40 +809,54 @@ vm_object_update( fault_info.interruptible = THREAD_UNINT; fault_info.behavior = VM_BEHAVIOR_SEQUENTIAL; fault_info.user_tag = 0; + fault_info.pmap_options = 0; fault_info.lo_offset = copy_offset; fault_info.hi_offset = copy_size; fault_info.no_cache = FALSE; + fault_info.stealth = TRUE; + fault_info.io_sync = FALSE; + fault_info.cs_bypass = FALSE; + fault_info.mark_zf_absent = FALSE; + fault_info.batch_pmap_op = FALSE; vm_object_paging_begin(copy_object); for (i = copy_offset; i < copy_size; i += PAGE_SIZE) { RETRY_COW_OF_LOCK_REQUEST: - fault_info.cluster_size = copy_size - i; + fault_info.cluster_size = (vm_size_t) (copy_size - i); + assert(fault_info.cluster_size == copy_size - i); prot = VM_PROT_WRITE|VM_PROT_READ; - switch (vm_fault_page(copy_object, i, - VM_PROT_WRITE|VM_PROT_READ, - FALSE, - &prot, - &page, - &top_page, - (int *)0, - &error, - FALSE, - FALSE, &fault_info)) { - + page = VM_PAGE_NULL; + result = vm_fault_page(copy_object, i, + VM_PROT_WRITE|VM_PROT_READ, + FALSE, + FALSE, /* page not looked up */ + &prot, + &page, + &top_page, + (int *)0, + &error, + FALSE, + FALSE, &fault_info); + + switch (result) { case VM_FAULT_SUCCESS: if (top_page) { vm_fault_cleanup( - page->object, top_page); + VM_PAGE_OBJECT(page), top_page); vm_object_lock(copy_object); vm_object_paging_begin(copy_object); } - vm_page_lock_queues(); - if (!page->active && !page->inactive) - vm_page_deactivate(page); - vm_page_unlock_queues(); + if (( !VM_PAGE_NON_SPECULATIVE_PAGEABLE(page))) { + vm_page_lockspin_queues(); + + if (( !VM_PAGE_NON_SPECULATIVE_PAGEABLE(page))) { + vm_page_deactivate(page); + } + vm_page_unlock_queues(); + } PAGE_WAKEUP_DONE(page); break; case VM_FAULT_RETRY: @@ -856,17 +875,19 @@ vm_object_update( vm_object_lock(copy_object); vm_object_paging_begin(copy_object); goto RETRY_COW_OF_LOCK_REQUEST; - case VM_FAULT_FICTITIOUS_SHORTAGE: - vm_page_more_fictitious(); - prot = VM_PROT_WRITE|VM_PROT_READ; - vm_object_lock(copy_object); - vm_object_paging_begin(copy_object); - goto RETRY_COW_OF_LOCK_REQUEST; + case VM_FAULT_SUCCESS_NO_VM_PAGE: + /* success but no VM page: fail */ + vm_object_paging_end(copy_object); + vm_object_unlock(copy_object); + /*FALLTHROUGH*/ case VM_FAULT_MEMORY_ERROR: if (object != copy_object) vm_object_deallocate(copy_object); vm_object_lock(object); goto BYPASS_COW_COPYIN; + default: + panic("vm_object_update: unexpected error 0x%x" + " from vm_fault_page()\n", result); } } @@ -882,6 +903,7 @@ vm_object_update( } if (copy_object != VM_OBJECT_NULL && copy_object != object) { if ((flags & MEMORY_OBJECT_DATA_PURGE)) { + vm_object_lock_assert_exclusive(copy_object); copy_object->shadow_severed = TRUE; copy_object->shadowed = FALSE; copy_object->shadow = NULL; @@ -933,10 +955,10 @@ BYPASS_COW_COPYIN: num_of_extents = 0; e_mask = ~((vm_object_size_t)(EXTENT_SIZE - 1)); - m = (vm_page_t) queue_first(&object->memq); + m = (vm_page_t) vm_page_queue_first(&object->memq); - while (!queue_end(&object->memq, (queue_entry_t) m)) { - next = (vm_page_t) queue_next(&m->listq); + while (!vm_page_queue_end(&object->memq, (vm_page_queue_entry_t) m)) { + next = (vm_page_t) vm_page_queue_next(&m->listq); if ((m->offset >= start) && (m->offset < end)) { /* @@ -1023,79 +1045,17 @@ BYPASS_COW_COPYIN: } -/* - * Routine: memory_object_synchronize_completed [user interface] - * - * Tell kernel that previously synchronized data - * (memory_object_synchronize) has been queue or placed on the - * backing storage. - * - * Note: there may be multiple synchronize requests for a given - * memory object outstanding but they will not overlap. - */ - -kern_return_t -memory_object_synchronize_completed( - memory_object_control_t control, - memory_object_offset_t offset, - vm_offset_t length) -{ - vm_object_t object; - msync_req_t msr; - - object = memory_object_control_to_vm_object(control); - - XPR(XPR_MEMORY_OBJECT, - "m_o_sync_completed, object 0x%X, offset 0x%X length 0x%X\n", - (integer_t)object, offset, length, 0, 0); - - /* - * Look for bogus arguments - */ - - if (object == VM_OBJECT_NULL) - return (KERN_INVALID_ARGUMENT); - - vm_object_lock(object); - -/* - * search for sync request structure - */ - queue_iterate(&object->msr_q, msr, msync_req_t, msr_q) { - if (msr->offset == offset && msr->length == length) { - queue_remove(&object->msr_q, msr, msync_req_t, msr_q); - break; - } - }/* queue_iterate */ - - if (queue_end(&object->msr_q, (queue_entry_t)msr)) { - vm_object_unlock(object); - return KERN_INVALID_ARGUMENT; - } - - msr_lock(msr); - vm_object_unlock(object); - msr->flag = VM_MSYNC_DONE; - msr_unlock(msr); - thread_wakeup((event_t) msr); - - return KERN_SUCCESS; -}/* memory_object_synchronize_completed */ - static kern_return_t vm_object_set_attributes_common( vm_object_t object, boolean_t may_cache, - memory_object_copy_strategy_t copy_strategy, - boolean_t temporary, - boolean_t silent_overwrite, - boolean_t advisory_pageout) + memory_object_copy_strategy_t copy_strategy) { boolean_t object_became_ready; XPR(XPR_MEMORY_OBJECT, "m_o_set_attr_com, object 0x%X flg %x strat %d\n", - (integer_t)object, (may_cache&1)|((temporary&1)<1), copy_strategy, 0, 0); + object, (may_cache&1), copy_strategy, 0, 0); if (object == VM_OBJECT_NULL) return(KERN_INVALID_ARGUMENT); @@ -1112,15 +1072,8 @@ vm_object_set_attributes_common( return(KERN_INVALID_ARGUMENT); } -#if !ADVISORY_PAGEOUT - if (silent_overwrite || advisory_pageout) - return(KERN_INVALID_ARGUMENT); - -#endif /* !ADVISORY_PAGEOUT */ if (may_cache) may_cache = TRUE; - if (temporary) - temporary = TRUE; vm_object_lock(object); @@ -1131,9 +1084,6 @@ vm_object_set_attributes_common( object_became_ready = !object->pager_ready; object->copy_strategy = copy_strategy; object->can_persist = may_cache; - object->temporary = temporary; - object->silent_overwrite = silent_overwrite; - object->advisory_pageout = advisory_pageout; /* * Wake up anyone waiting for the ready attribute @@ -1150,6 +1100,18 @@ vm_object_set_attributes_common( return(KERN_SUCCESS); } + +kern_return_t +memory_object_synchronize_completed( + __unused memory_object_control_t control, + __unused memory_object_offset_t offset, + __unused memory_object_size_t length) +{ + panic("memory_object_synchronize_completed no longer supported\n"); + return(KERN_FAILURE); +} + + /* * Set the memory object attribute as provided. * @@ -1167,12 +1129,9 @@ memory_object_change_attributes( { vm_object_t object; kern_return_t result = KERN_SUCCESS; - boolean_t temporary; boolean_t may_cache; boolean_t invalidate; memory_object_copy_strategy_t copy_strategy; - boolean_t silent_overwrite; - boolean_t advisory_pageout; object = memory_object_control_to_vm_object(control); if (object == VM_OBJECT_NULL) @@ -1180,11 +1139,8 @@ memory_object_change_attributes( vm_object_lock(object); - temporary = object->temporary; may_cache = object->can_persist; copy_strategy = object->copy_strategy; - silent_overwrite = object->silent_overwrite; - advisory_pageout = object->advisory_pageout; #if notyet invalidate = object->invalidate; #endif @@ -1202,7 +1158,6 @@ memory_object_change_attributes( behave = (old_memory_object_behave_info_t) attributes; - temporary = behave->temporary; invalidate = behave->invalidate; copy_strategy = behave->copy_strategy; @@ -1220,11 +1175,8 @@ memory_object_change_attributes( behave = (memory_object_behave_info_t) attributes; - temporary = behave->temporary; invalidate = behave->invalidate; copy_strategy = behave->copy_strategy; - silent_overwrite = behave->silent_overwrite; - advisory_pageout = behave->advisory_pageout; break; } @@ -1274,7 +1226,6 @@ memory_object_change_attributes( copy_strategy = attr->copy_strategy; may_cache = attr->may_cache_object; - temporary = attr->temporary; break; } @@ -1289,9 +1240,6 @@ memory_object_change_attributes( if (copy_strategy == MEMORY_OBJECT_COPY_TEMPORARY) { copy_strategy = MEMORY_OBJECT_COPY_DELAY; - temporary = TRUE; - } else { - temporary = FALSE; } /* @@ -1300,10 +1248,7 @@ memory_object_change_attributes( */ return (vm_object_set_attributes_common(object, may_cache, - copy_strategy, - temporary, - silent_overwrite, - advisory_pageout)); + copy_strategy)); } kern_return_t @@ -1334,7 +1279,7 @@ memory_object_get_attributes( behave = (old_memory_object_behave_info_t) attributes; behave->copy_strategy = object->copy_strategy; - behave->temporary = object->temporary; + behave->temporary = FALSE; #if notyet /* remove when vm_msync complies and clean in place fini */ behave->invalidate = object->invalidate; #else @@ -1356,14 +1301,14 @@ memory_object_get_attributes( behave = (memory_object_behave_info_t) attributes; behave->copy_strategy = object->copy_strategy; - behave->temporary = object->temporary; + behave->temporary = FALSE; #if notyet /* remove when vm_msync complies and clean in place fini */ behave->invalidate = object->invalidate; #else behave->invalidate = FALSE; #endif - behave->advisory_pageout = object->advisory_pageout; - behave->silent_overwrite = object->silent_overwrite; + behave->advisory_pageout = FALSE; + behave->silent_overwrite = FALSE; *count = MEMORY_OBJECT_BEHAVE_INFO_COUNT; break; } @@ -1415,7 +1360,7 @@ memory_object_get_attributes( attr->copy_strategy = object->copy_strategy; attr->cluster_size = PAGE_SIZE; attr->may_cache_object = object->can_persist; - attr->temporary = object->temporary; + attr->temporary = FALSE; *count = MEMORY_OBJECT_ATTR_INFO_COUNT; break; @@ -1440,11 +1385,12 @@ memory_object_iopl_request( upl_t *upl_ptr, upl_page_info_array_t user_page_list, unsigned int *page_list_count, - int *flags) + upl_control_flags_t *flags, + vm_tag_t tag) { vm_object_t object; kern_return_t ret; - int caller_flags; + upl_control_flags_t caller_flags; caller_flags = *flags; @@ -1464,7 +1410,9 @@ memory_object_iopl_request( if(*upl_size == 0) { if(offset >= named_entry->size) return(KERN_INVALID_RIGHT); - *upl_size = named_entry->size - offset; + *upl_size = (upl_size_t)(named_entry->size - offset); + if (*upl_size != named_entry->size - offset) + return KERN_INVALID_ARGUMENT; } if(caller_flags & UPL_COPYOUT_FROM) { if((named_entry->protection & VM_PROT_READ) @@ -1485,50 +1433,15 @@ memory_object_iopl_request( /* offset from beginning of named entry offset in object */ offset = offset + named_entry->offset; - if(named_entry->is_sub_map) - return (KERN_INVALID_ARGUMENT); + if (named_entry->is_sub_map || + named_entry->is_copy) + return KERN_INVALID_ARGUMENT; named_entry_lock(named_entry); - if (named_entry->is_pager) { - object = vm_object_enter(named_entry->backing.pager, - named_entry->offset + named_entry->size, - named_entry->internal, - FALSE, - FALSE); - if (object == VM_OBJECT_NULL) { - named_entry_unlock(named_entry); - return(KERN_INVALID_OBJECT); - } - - /* JMM - drop reference on pager here? */ - - /* create an extra reference for the named entry */ - vm_object_lock(object); - vm_object_reference_locked(object); - named_entry->backing.object = object; - named_entry->is_pager = FALSE; - named_entry_unlock(named_entry); - - /* wait for object to be ready */ - while (!object->pager_ready) { - vm_object_wait(object, - VM_OBJECT_EVENT_PAGER_READY, - THREAD_UNINT); - vm_object_lock(object); - } - vm_object_unlock(object); - } else { - /* This is the case where we are going to map */ - /* an already mapped object. If the object is */ - /* not ready it is internal. An external */ - /* object cannot be mapped until it is ready */ - /* we can therefore avoid the ready check */ - /* in this case. */ - object = named_entry->backing.object; - vm_object_reference(object); - named_entry_unlock(named_entry); - } + object = named_entry->backing.object; + vm_object_reference(object); + named_entry_unlock(named_entry); } else if (ip_kotype(port) == IKOT_MEM_OBJ_CONTROL) { memory_object_control_t control; control = (memory_object_control_t) port; @@ -1545,8 +1458,6 @@ memory_object_iopl_request( return (KERN_INVALID_ARGUMENT); if (!object->private) { - if (*upl_size > (MAX_UPL_TRANSFER*PAGE_SIZE)) - *upl_size = (MAX_UPL_TRANSFER*PAGE_SIZE); if (object->phys_contiguous) { *flags = UPL_PHYS_CONTIG; } else { @@ -1562,7 +1473,8 @@ memory_object_iopl_request( upl_ptr, user_page_list, page_list_count, - caller_flags); + caller_flags, + tag); vm_object_deallocate(object); return ret; } @@ -1584,13 +1496,14 @@ memory_object_upl_request( upl_t *upl_ptr, upl_page_info_array_t user_page_list, unsigned int *page_list_count, - int cntrl_flags) + int cntrl_flags, + int tag) { vm_object_t object; object = memory_object_control_to_vm_object(control); if (object == VM_OBJECT_NULL) - return (KERN_INVALID_ARGUMENT); + return (KERN_TERMINATED); return vm_object_upl_request(object, offset, @@ -1598,7 +1511,8 @@ memory_object_upl_request( upl_ptr, user_page_list, page_list_count, - cntrl_flags); + (upl_control_flags_t)(unsigned int) cntrl_flags, + tag); } /* @@ -1621,7 +1535,8 @@ memory_object_super_upl_request( upl_t *upl, upl_page_info_t *user_page_list, unsigned int *page_list_count, - int cntrl_flags) + int cntrl_flags, + int tag) { vm_object_t object; @@ -1636,12 +1551,13 @@ memory_object_super_upl_request( upl, user_page_list, page_list_count, - cntrl_flags); + (upl_control_flags_t)(unsigned int) cntrl_flags, + tag); } kern_return_t memory_object_cluster_size(memory_object_control_t control, memory_object_offset_t *start, - vm_size_t *length, memory_object_fault_info_t fault_info) + vm_size_t *length, uint32_t *io_streaming, memory_object_fault_info_t fault_info) { vm_object_t object; @@ -1652,7 +1568,7 @@ memory_object_cluster_size(memory_object_control_t control, memory_object_offset *start -= object->paging_offset; - vm_object_cluster_size(object, (vm_object_offset_t *)start, length, (vm_object_fault_info_t)fault_info); + vm_object_cluster_size(object, (vm_object_offset_t *)start, length, (vm_object_fault_info_t)fault_info, io_streaming); *start += object->paging_offset; @@ -1660,12 +1576,6 @@ memory_object_cluster_size(memory_object_control_t control, memory_object_offset } -int vm_stat_discard_cleared_reply = 0; -int vm_stat_discard_cleared_unset = 0; -int vm_stat_discard_cleared_too_late = 0; - - - /* * Routine: host_default_memory_manager [interface] * Purpose: @@ -1691,7 +1601,7 @@ host_default_memory_manager( assert(host_priv == &realhost); new_manager = *default_manager; - mutex_lock(&memory_manager_default_lock); + lck_mtx_lock(&memory_manager_default_lock); current_manager = memory_manager_default; returned_manager = MEMORY_OBJECT_DEFAULT_NULL; @@ -1702,6 +1612,14 @@ host_default_memory_manager( returned_manager = current_manager; memory_object_default_reference(returned_manager); } else { + /* + * Only allow the kernel to change the value. + */ + extern task_t kernel_task; + if (current_task() != kernel_task) { + result = KERN_NO_ACCESS; + goto out; + } /* * If this is the first non-null manager, start @@ -1730,9 +1648,19 @@ host_default_memory_manager( */ thread_wakeup((event_t) &memory_manager_default); + + /* + * Now that we have a default pager for anonymous memory, + * reactivate all the throttled pages (i.e. dirty pages with + * no pager). + */ + if (current_manager == MEMORY_OBJECT_DEFAULT_NULL) + { + vm_page_reactivate_all_throttled(); + } } out: - mutex_unlock(&memory_manager_default_lock); + lck_mtx_unlock(&memory_manager_default_lock); *default_manager = returned_manager; return(result); @@ -1751,19 +1679,20 @@ memory_manager_default_reference(void) { memory_object_default_t current_manager; - mutex_lock(&memory_manager_default_lock); + lck_mtx_lock(&memory_manager_default_lock); current_manager = memory_manager_default; while (current_manager == MEMORY_OBJECT_DEFAULT_NULL) { wait_result_t res; - res = thread_sleep_mutex((event_t) &memory_manager_default, - &memory_manager_default_lock, - THREAD_UNINT); + res = lck_mtx_sleep(&memory_manager_default_lock, + LCK_SLEEP_DEFAULT, + (event_t) &memory_manager_default, + THREAD_UNINT); assert(res == THREAD_AWAKENED); current_manager = memory_manager_default; } memory_object_default_reference(current_manager); - mutex_unlock(&memory_manager_default_lock); + lck_mtx_unlock(&memory_manager_default_lock); return current_manager; } @@ -1785,18 +1714,18 @@ memory_manager_default_check(void) { memory_object_default_t current; - mutex_lock(&memory_manager_default_lock); + lck_mtx_lock(&memory_manager_default_lock); current = memory_manager_default; if (current == MEMORY_OBJECT_DEFAULT_NULL) { static boolean_t logged; /* initialized to 0 */ boolean_t complain = !logged; logged = TRUE; - mutex_unlock(&memory_manager_default_lock); + lck_mtx_unlock(&memory_manager_default_lock); if (complain) printf("Warning: No default memory manager\n"); return(KERN_FAILURE); } else { - mutex_unlock(&memory_manager_default_lock); + lck_mtx_unlock(&memory_manager_default_lock); return(KERN_SUCCESS); } } @@ -1805,7 +1734,7 @@ __private_extern__ void memory_manager_default_init(void) { memory_manager_default = MEMORY_OBJECT_DEFAULT_NULL; - mutex_init(&memory_manager_default_lock, 0); + lck_mtx_init(&memory_manager_default_lock, &vm_object_lck_grp, &vm_object_lck_attr); } @@ -1858,10 +1787,94 @@ memory_object_range_op( offset_beg, offset_end, ops, - range); + (uint32_t *) range); +} + + +void +memory_object_mark_used( + memory_object_control_t control) +{ + vm_object_t object; + + if (control == NULL) + return; + + object = memory_object_control_to_vm_object(control); + + if (object != VM_OBJECT_NULL) + vm_object_cache_remove(object); } +void +memory_object_mark_unused( + memory_object_control_t control, + __unused boolean_t rage) +{ + vm_object_t object; + + if (control == NULL) + return; + + object = memory_object_control_to_vm_object(control); + + if (object != VM_OBJECT_NULL) + vm_object_cache_add(object); +} + +void +memory_object_mark_io_tracking( + memory_object_control_t control) +{ + vm_object_t object; + + if (control == NULL) + return; + object = memory_object_control_to_vm_object(control); + + if (object != VM_OBJECT_NULL) { + vm_object_lock(object); + object->io_tracking = TRUE; + vm_object_unlock(object); + } +} + +#if CONFIG_SECLUDED_MEMORY +void +memory_object_mark_eligible_for_secluded( + memory_object_control_t control, + boolean_t eligible_for_secluded) +{ + vm_object_t object; + + if (control == NULL) + return; + object = memory_object_control_to_vm_object(control); + + if (object == VM_OBJECT_NULL) { + return; + } + + vm_object_lock(object); + if (eligible_for_secluded && + secluded_for_filecache && /* global boot-arg */ + !object->eligible_for_secluded) { + object->eligible_for_secluded = TRUE; + vm_page_secluded.eligible_for_secluded += object->resident_page_count; + } else if (!eligible_for_secluded && + object->eligible_for_secluded) { + object->eligible_for_secluded = FALSE; + vm_page_secluded.eligible_for_secluded -= object->resident_page_count; + if (object->resident_page_count) { + /* XXX FBDP TODO: flush pages from secluded queue? */ + // printf("FBDP TODO: flush %d pages from %p from secluded queue\n", object->resident_page_count, object); + } + } + vm_object_unlock(object); +} +#endif /* CONFIG_SECLUDED_MEMORY */ + kern_return_t memory_object_pages_resident( memory_object_control_t control, @@ -1899,6 +1912,37 @@ memory_object_signed( return KERN_SUCCESS; } +boolean_t +memory_object_is_signed( + memory_object_control_t control) +{ + boolean_t is_signed; + vm_object_t object; + + object = memory_object_control_to_vm_object(control); + if (object == VM_OBJECT_NULL) + return FALSE; + + vm_object_lock_shared(object); + is_signed = object->code_signed; + vm_object_unlock(object); + + return is_signed; +} + +boolean_t +memory_object_is_slid( + memory_object_control_t control) +{ + vm_object_t object = VM_OBJECT_NULL; + + object = memory_object_control_to_vm_object(control); + if (object == VM_OBJECT_NULL) + return FALSE; + + return object->object_slid; +} + static zone_t mem_obj_control_zone; __private_extern__ void @@ -1908,6 +1952,8 @@ memory_object_control_bootstrap(void) i = (vm_size_t) sizeof (struct memory_object_control); mem_obj_control_zone = zinit (i, 8192*i, 4096, "mem_obj_control"); + zone_change(mem_obj_control_zone, Z_CALLERACCT, FALSE); + zone_change(mem_obj_control_zone, Z_NOENCRYPT, TRUE); return; } @@ -1946,6 +1992,22 @@ memory_object_control_to_vm_object( return (control->moc_object); } +__private_extern__ vm_object_t +memory_object_to_vm_object( + memory_object_t mem_obj) +{ + memory_object_control_t mo_control; + + if (mem_obj == MEMORY_OBJECT_NULL) { + return VM_OBJECT_NULL; + } + mo_control = mem_obj->mo_control; + if (mo_control == NULL) { + return VM_OBJECT_NULL; + } + return memory_object_control_to_vm_object(mo_control); +} + memory_object_control_t convert_port_to_mo_control( __unused mach_port_t port) @@ -2082,7 +2144,7 @@ kern_return_t memory_object_data_return ( memory_object_t memory_object, memory_object_offset_t offset, - vm_size_t size, + memory_object_cluster_size_t size, memory_object_offset_t *resid_offset, int *io_error, boolean_t dirty, @@ -2106,7 +2168,7 @@ kern_return_t memory_object_data_initialize ( memory_object_t memory_object, memory_object_offset_t offset, - vm_size_t size + memory_object_cluster_size_t size ) { return (memory_object->mo_pager_ops->memory_object_data_initialize)( @@ -2120,7 +2182,7 @@ kern_return_t memory_object_data_unlock ( memory_object_t memory_object, memory_object_offset_t offset, - vm_size_t size, + memory_object_size_t size, vm_prot_t desired_access ) { @@ -2136,10 +2198,12 @@ kern_return_t memory_object_synchronize ( memory_object_t memory_object, memory_object_offset_t offset, - vm_size_t size, + memory_object_size_t size, vm_sync_t sync_flags ) { + panic("memory_object_syncrhonize no longer supported\n"); + return (memory_object->mo_pager_ops->memory_object_synchronize)( memory_object, offset, @@ -2147,27 +2211,59 @@ kern_return_t memory_object_synchronize sync_flags); } -/* Routine memory_object_unmap */ -kern_return_t memory_object_unmap + +/* + * memory_object_map() is called by VM (in vm_map_enter() and its variants) + * each time a "named" VM object gets mapped directly or indirectly + * (copy-on-write mapping). A "named" VM object has an extra reference held + * by the pager to keep it alive until the pager decides that the + * memory object (and its VM object) can be reclaimed. + * VM calls memory_object_last_unmap() (in vm_object_deallocate()) when all + * the mappings of that memory object have been removed. + * + * For a given VM object, calls to memory_object_map() and memory_object_unmap() + * are serialized (through object->mapping_in_progress), to ensure that the + * pager gets a consistent view of the mapping status of the memory object. + * + * This allows the pager to keep track of how many times a memory object + * has been mapped and with which protections, to decide when it can be + * reclaimed. + */ + +/* Routine memory_object_map */ +kern_return_t memory_object_map +( + memory_object_t memory_object, + vm_prot_t prot +) +{ + return (memory_object->mo_pager_ops->memory_object_map)( + memory_object, + prot); +} + +/* Routine memory_object_last_unmap */ +kern_return_t memory_object_last_unmap ( memory_object_t memory_object ) { - return (memory_object->mo_pager_ops->memory_object_unmap)( + return (memory_object->mo_pager_ops->memory_object_last_unmap)( memory_object); } -/* Routine memory_object_create */ -kern_return_t memory_object_create +/* Routine memory_object_data_reclaim */ +kern_return_t memory_object_data_reclaim ( - memory_object_default_t default_memory_manager, - vm_size_t new_memory_object_size, - memory_object_t *new_memory_object + memory_object_t memory_object, + boolean_t reclaim_backing_store ) { - return default_pager_memory_object_create(default_memory_manager, - new_memory_object_size, - new_memory_object); + if (memory_object->mo_pager_ops->memory_object_data_reclaim == NULL) + return KERN_NOT_SUPPORTED; + return (memory_object->mo_pager_ops->memory_object_data_reclaim)( + memory_object, + reclaim_backing_store); } upl_t