X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/7ddcb079202367355dddccdfa4318e57d50318be..22ba694c5857e62b5a553b1505dcf2e509177f28:/osfmk/vm/vm_resident.c?ds=inline diff --git a/osfmk/vm/vm_resident.c b/osfmk/vm/vm_resident.c index 0c0a34e04..a9fd4d0b2 100644 --- a/osfmk/vm/vm_resident.c +++ b/osfmk/vm/vm_resident.c @@ -90,14 +90,13 @@ #include #include #include +#include #include - -#include - #include +boolean_t hibernate_cleaning_in_progress = FALSE; boolean_t vm_page_free_verify = TRUE; uint32_t vm_lopage_free_count = 0; @@ -166,6 +165,13 @@ unsigned int vm_page_bucket_lock_count = 0; /* How big is array of locks? */ lck_spin_t *vm_page_bucket_locks; +#if VM_PAGE_BUCKETS_CHECK +boolean_t vm_page_buckets_check_ready = FALSE; +#if VM_PAGE_FAKE_BUCKETS +vm_page_bucket_t *vm_page_fake_buckets; /* decoy buckets */ +vm_map_offset_t vm_page_fake_buckets_start, vm_page_fake_buckets_end; +#endif /* VM_PAGE_FAKE_BUCKETS */ +#endif /* VM_PAGE_BUCKETS_CHECK */ #if MACH_PAGE_HASH_STATS /* This routine is only for debug. It is intended to be called by @@ -259,6 +265,8 @@ unsigned int vm_page_free_count_minimum; /* debugging */ zone_t vm_page_zone; vm_locks_array_t vm_page_locks; decl_lck_mtx_data(,vm_page_alloc_lock) +lck_mtx_ext_t vm_page_alloc_lock_ext; + unsigned int io_throttle_zero_fill; unsigned int vm_page_local_q_count = 0; @@ -266,6 +274,9 @@ unsigned int vm_page_local_q_soft_limit = 250; unsigned int vm_page_local_q_hard_limit = 500; struct vplq *vm_page_local_q = NULL; +/* N.B. Guard and fictitious pages must not + * be assigned a zero phys_page value. + */ /* * Fictitious pages don't have a physical address, * but we must initialize phys_page to something. @@ -290,17 +301,18 @@ ppnum_t vm_page_guard_addr = (ppnum_t) -2; * system (pageout daemon). These queues are * defined here, but are shared by the pageout * module. The inactive queue is broken into - * inactive and zf for convenience as the + * file backed and anonymous for convenience as the * pageout daemon often assignes a higher - * affinity to zf pages + * importance to anonymous pages (less likely to pick) */ queue_head_t vm_page_queue_active; queue_head_t vm_page_queue_inactive; -queue_head_t vm_page_queue_zf; /* inactive memory queue for zero fill */ +queue_head_t vm_page_queue_anonymous; /* inactive memory queue for anonymous pages */ queue_head_t vm_page_queue_throttled; unsigned int vm_page_active_count; unsigned int vm_page_inactive_count; +unsigned int vm_page_anonymous_count; unsigned int vm_page_throttled_count; unsigned int vm_page_speculative_count; unsigned int vm_page_wire_count; @@ -313,12 +325,22 @@ unsigned int vm_page_purgeable_count = 0; /* # of pages purgeable now */ unsigned int vm_page_purgeable_wired_count = 0; /* # of purgeable pages that are wired now */ uint64_t vm_page_purged_count = 0; /* total count of purged pages */ +unsigned int vm_page_external_count = 0; +unsigned int vm_page_internal_count = 0; +unsigned int vm_page_pageable_external_count = 0; +unsigned int vm_page_pageable_internal_count = 0; + #if DEVELOPMENT || DEBUG unsigned int vm_page_speculative_recreated = 0; unsigned int vm_page_speculative_created = 0; unsigned int vm_page_speculative_used = 0; #endif +queue_head_t vm_page_queue_cleaned; + +unsigned int vm_page_cleaned_count = 0; +unsigned int vm_pageout_enqueued_cleaned = 0; + uint64_t max_valid_dma_address = 0xffffffffffffffffULL; ppnum_t max_valid_low_ppnum = 0xffffffff; @@ -334,10 +356,12 @@ unsigned int vm_page_free_min = 0; unsigned int vm_page_throttle_limit = 0; uint32_t vm_page_creation_throttle = 0; unsigned int vm_page_inactive_target = 0; +unsigned int vm_page_anonymous_min = 0; unsigned int vm_page_inactive_min = 0; unsigned int vm_page_free_reserved = 0; unsigned int vm_page_throttle_count = 0; + /* * The VM system has a couple of heuristics for deciding * that pages are "uninteresting" and should be placed @@ -424,6 +448,9 @@ vm_page_init_lck_grp(void) lck_grp_init(&vm_page_lck_grp_alloc, "vm_page_alloc", &vm_page_lck_grp_attr); lck_grp_init(&vm_page_lck_grp_bucket, "vm_page_bucket", &vm_page_lck_grp_attr); lck_attr_setdefault(&vm_page_lck_attr); + lck_mtx_init_ext(&vm_page_alloc_lock, &vm_page_alloc_lock_ext, &vm_page_lck_grp_alloc, &vm_page_lck_attr); + + vm_compressor_init_locks(); } void @@ -448,6 +475,8 @@ vm_page_init_local_q() VPL_LOCK_INIT(lq, &vm_page_lck_grp_local, &vm_page_lck_attr); queue_init(&lq->vpl_queue); lq->vpl_count = 0; + lq->vpl_internal_count = 0; + lq->vpl_external_count = 0; } vm_page_local_q_count = num_cpus; @@ -513,6 +542,7 @@ vm_page_bootstrap( m->busy = TRUE; m->wanted = FALSE; m->tabled = FALSE; + m->hashed = FALSE; m->fictitious = FALSE; m->pmapped = FALSE; m->wpmapped = FALSE; @@ -528,17 +558,17 @@ vm_page_bootstrap( m->unusual = FALSE; m->encrypted = FALSE; m->encrypted_cleaning = FALSE; - m->list_req_pending = FALSE; - m->dump_cleaning = FALSE; m->cs_validated = FALSE; m->cs_tainted = FALSE; m->no_cache = FALSE; - m->zero_fill = FALSE; m->reusable = FALSE; m->slid = FALSE; + m->was_dirty = FALSE; + m->xpmapped = FALSE; + m->compressor = FALSE; + m->written_by_kernel = FALSE; m->__unused_object_bits = 0; - /* * Initialize the page queues. */ @@ -570,8 +600,9 @@ vm_page_bootstrap( queue_init(&vm_lopage_queue_free); queue_init(&vm_page_queue_active); queue_init(&vm_page_queue_inactive); + queue_init(&vm_page_queue_cleaned); queue_init(&vm_page_queue_throttled); - queue_init(&vm_page_queue_zf); + queue_init(&vm_page_queue_anonymous); for ( i = 0; i <= VM_PAGE_MAX_SPECULATIVE_AGE_Q; i++ ) { queue_init(&vm_page_queue_speculative[i].age_q); @@ -588,9 +619,8 @@ vm_page_bootstrap( /* * Steal memory for the map and zone subsystems. */ - - vm_map_steal_memory(); zone_steal_memory(); + vm_map_steal_memory(); /* * Allocate (and initialize) the virtual-to-physical @@ -635,6 +665,30 @@ vm_page_bootstrap( if (vm_page_hash_mask & vm_page_bucket_count) printf("vm_page_bootstrap: WARNING -- strange page hash\n"); +#if VM_PAGE_BUCKETS_CHECK +#if VM_PAGE_FAKE_BUCKETS + /* + * Allocate a decoy set of page buckets, to detect + * any stomping there. + */ + vm_page_fake_buckets = (vm_page_bucket_t *) + pmap_steal_memory(vm_page_bucket_count * + sizeof(vm_page_bucket_t)); + vm_page_fake_buckets_start = (vm_map_offset_t) vm_page_fake_buckets; + vm_page_fake_buckets_end = + vm_map_round_page((vm_page_fake_buckets_start + + (vm_page_bucket_count * + sizeof (vm_page_bucket_t))), + PAGE_MASK); + char *cp; + for (cp = (char *)vm_page_fake_buckets_start; + cp < (char *)vm_page_fake_buckets_end; + cp++) { + *cp = 0x5a; + } +#endif /* VM_PAGE_FAKE_BUCKETS */ +#endif /* VM_PAGE_BUCKETS_CHECK */ + vm_page_buckets = (vm_page_bucket_t *) pmap_steal_memory(vm_page_bucket_count * sizeof(vm_page_bucket_t)); @@ -656,6 +710,10 @@ vm_page_bootstrap( for (i = 0; i < vm_page_bucket_lock_count; i++) lck_spin_init(&vm_page_bucket_locks[i], &vm_page_lck_grp_bucket, &vm_page_lck_attr); +#if VM_PAGE_BUCKETS_CHECK + vm_page_buckets_check_ready = TRUE; +#endif /* VM_PAGE_BUCKETS_CHECK */ + /* * Machine-dependent code allocates the resident page table. * It uses vm_page_init to initialize the page frames. @@ -754,7 +812,7 @@ pmap_steal_memory( #endif pmap_enter(kernel_pmap, vaddr, phys_page, - VM_PROT_READ|VM_PROT_WRITE, + VM_PROT_READ|VM_PROT_WRITE, VM_PROT_NONE, VM_WIMG_USE_DEFAULT, FALSE); /* * Account for newly stolen memory @@ -806,7 +864,18 @@ pmap_startup( */ fill = 0; /* Assume no fill */ if (PE_parse_boot_argn("fill", &fillval, sizeof (fillval))) fill = 1; /* Set fill */ - +#if DEBUG + /* This slows down booting the DEBUG kernel, particularly on + * large memory systems, but is worthwhile in deterministically + * trapping uninitialized memory usage. + */ + if (fill == 0) { + fill = 1; + fillval = 0xDEB8F177; + } +#endif + if (fill) + kprintf("Filling vm_pages with pattern: 0x%x\n", fillval); // -debug code remove if (2 == vm_himemory_mode) { // free low -> high so high is preferred @@ -903,7 +972,7 @@ vm_page_module_init(void) zone_change(vm_page_zone, Z_EXPAND, FALSE); zone_change(vm_page_zone, Z_EXHAUST, TRUE); zone_change(vm_page_zone, Z_FOREIGN, TRUE); - + zone_change(vm_page_zone, Z_GZALLOC_EXEMPT, TRUE); /* * Adjust zone statistics to account for the real pages allocated * in vm_page_create(). [Q: is this really what we want?] @@ -911,8 +980,6 @@ vm_page_module_init(void) vm_page_zone->count += vm_page_pages; vm_page_zone->sum_count += vm_page_pages; vm_page_zone->cur_size += vm_page_pages * vm_page_zone->elem_size; - - lck_mtx_init(&vm_page_alloc_lock, &vm_page_lck_grp_alloc, &vm_page_lck_attr); } /* @@ -973,7 +1040,7 @@ vm_page_insert( vm_object_t object, vm_object_offset_t offset) { - vm_page_insert_internal(mem, object, offset, FALSE, TRUE); + vm_page_insert_internal(mem, object, offset, FALSE, TRUE, FALSE); } void @@ -982,7 +1049,8 @@ vm_page_insert_internal( vm_object_t object, vm_object_offset_t offset, boolean_t queues_lock_held, - boolean_t insert_in_hash) + boolean_t insert_in_hash, + boolean_t batch_pmap_op) { vm_page_bucket_t *bucket; lck_spin_t *bucket_lock; @@ -991,8 +1059,15 @@ vm_page_insert_internal( XPR(XPR_VM_PAGE, "vm_page_insert, object 0x%X offset 0x%X page 0x%X\n", object, offset, mem, 0,0); - +#if 0 + /* + * we may not hold the page queue lock + * so this check isn't safe to make + */ VM_PAGE_CHECK(mem); +#endif + + assert(page_aligned(offset)); if (object == vm_submap_object) { /* the vm_submap_object is only a placeholder for submaps */ @@ -1007,7 +1082,7 @@ vm_page_insert_internal( #endif /* DEBUG */ if (insert_in_hash == TRUE) { -#if DEBUG +#if DEBUG || VM_PAGE_CHECK_BUCKETS if (mem->tabled || mem->object != VM_OBJECT_NULL) panic("vm_page_insert: page %p for (obj=%p,off=0x%llx) " "already in (obj=%p,off=0x%llx)", @@ -1043,23 +1118,22 @@ vm_page_insert_internal( if (++bucket->cur_count > bucket->hi_count) bucket->hi_count = bucket->cur_count; #endif /* MACH_PAGE_HASH_STATS */ - + mem->hashed = TRUE; lck_spin_unlock(bucket_lock); } - { unsigned int cache_attr; + { + unsigned int cache_attr; cache_attr = object->wimg_bits & VM_WIMG_MASK; if (cache_attr != VM_WIMG_USE_DEFAULT) { - pmap_set_cache_attributes(mem->phys_page, cache_attr); - object->set_cache_attr = TRUE; + PMAP_SET_CACHE_ATTR(mem, object, cache_attr, batch_pmap_op); } } /* * Now link into the object's list of backed pages. */ - VM_PAGE_INSERT(mem, object); mem->tabled = TRUE; @@ -1073,7 +1147,28 @@ vm_page_insert_internal( } assert(object->resident_page_count >= object->wired_page_count); + if (object->internal) { + OSAddAtomic(1, &vm_page_internal_count); + } else { + OSAddAtomic(1, &vm_page_external_count); + } + + /* + * It wouldn't make sense to insert a "reusable" page in + * an object (the page would have been marked "reusable" only + * at the time of a madvise(MADV_FREE_REUSABLE) if it was already + * in the object at that time). + * But a page could be inserted in a "all_reusable" object, if + * something faults it in (a vm_read() from another task or a + * "use-after-free" issue in user space, for example). It can + * also happen if we're relocating a page from that object to + * a different physical page during a physically-contiguous + * allocation. + */ assert(!mem->reusable); + if (mem->object->all_reusable) { + OSAddAtomic(+1, &vm_page_stats_reusable.reusable_count); + } if (object->purgable == VM_PURGABLE_VOLATILE) { if (VM_PAGE_WIRED(mem)) { @@ -1118,9 +1213,15 @@ vm_page_replace( lck_spin_t *bucket_lock; int hash_id; +#if 0 + /* + * we don't hold the page queue lock + * so this check isn't safe to make + */ VM_PAGE_CHECK(mem); +#endif vm_object_lock_assert_exclusive(object); -#if DEBUG +#if DEBUG || VM_PAGE_CHECK_BUCKETS if (mem->tabled || mem->object != VM_OBJECT_NULL) panic("vm_page_replace: page %p for (obj=%p,off=0x%llx) " "already in (obj=%p,off=0x%llx)", @@ -1155,6 +1256,7 @@ vm_page_replace( * Remove old page from hash list */ *mp = m->next; + m->hashed = FALSE; found_m = m; break; @@ -1170,6 +1272,7 @@ vm_page_replace( * insert new page at head of hash list */ bucket->pages = mem; + mem->hashed = TRUE; lck_spin_unlock(bucket_lock); @@ -1181,7 +1284,7 @@ vm_page_replace( */ vm_page_free_unlocked(found_m, FALSE); } - vm_page_insert_internal(mem, object, offset, FALSE, FALSE); + vm_page_insert_internal(mem, object, offset, FALSE, FALSE, FALSE); } /* @@ -1211,8 +1314,14 @@ vm_page_remove( vm_object_lock_assert_exclusive(mem->object); assert(mem->tabled); assert(!mem->cleaning); + assert(!mem->laundry); +#if 0 + /* + * we don't hold the page queue lock + * so this check isn't safe to make + */ VM_PAGE_CHECK(mem); - +#endif if (remove_from_hash == TRUE) { /* * Remove from the object_object/offset hash table @@ -1239,7 +1348,7 @@ vm_page_remove( #if MACH_PAGE_HASH_STATS bucket->cur_count--; #endif /* MACH_PAGE_HASH_STATS */ - + mem->hashed = FALSE; lck_spin_unlock(bucket_lock); } /* @@ -1256,6 +1365,13 @@ vm_page_remove( assert(mem->object->resident_page_count > 0); mem->object->resident_page_count--; + if (mem->object->internal) { + assert(vm_page_internal_count); + OSAddAtomic(-1, &vm_page_internal_count); + } else { + assert(vm_page_external_count); + OSAddAtomic(-1, &vm_page_external_count); + } if (!mem->object->internal && (mem->object->objq.next || mem->object->objq.prev)) { if (mem->object->resident_page_count == 0) vm_object_cache_remove(mem->object); @@ -1389,7 +1505,13 @@ vm_page_lookup( lck_spin_lock(bucket_lock); for (mem = bucket->pages; mem != VM_PAGE_NULL; mem = mem->next) { +#if 0 + /* + * we don't hold the page queue lock + * so this check isn't safe to make + */ VM_PAGE_CHECK(mem); +#endif if ((mem->object == object) && (mem->offset == offset)) break; } @@ -1423,6 +1545,8 @@ vm_page_rename( vm_object_offset_t new_offset, boolean_t encrypted_ok) { + boolean_t internal_to_external, external_to_internal; + assert(mem->object != new_object); /* @@ -1453,8 +1577,38 @@ vm_page_rename( */ vm_page_lockspin_queues(); + internal_to_external = FALSE; + external_to_internal = FALSE; + + if (mem->local) { + /* + * it's much easier to get the vm_page_pageable_xxx accounting correct + * if we first move the page to the active queue... it's going to end + * up there anyway, and we don't do vm_page_rename's frequently enough + * for this to matter. + */ + VM_PAGE_QUEUES_REMOVE(mem); + vm_page_activate(mem); + } + if (mem->active || mem->inactive || mem->speculative) { + if (mem->object->internal && !new_object->internal) { + internal_to_external = TRUE; + } + if (!mem->object->internal && new_object->internal) { + external_to_internal = TRUE; + } + } + vm_page_remove(mem, TRUE); - vm_page_insert_internal(mem, new_object, new_offset, TRUE, TRUE); + vm_page_insert_internal(mem, new_object, new_offset, TRUE, TRUE, FALSE); + + if (internal_to_external) { + vm_page_pageable_internal_count--; + vm_page_pageable_external_count++; + } else if (external_to_internal) { + vm_page_pageable_external_count--; + vm_page_pageable_internal_count++; + } vm_page_unlock_queues(); } @@ -1636,6 +1790,10 @@ void vm_page_more_fictitious(void) vm_page_wait(THREAD_UNINT); return; } + + /* Increment zone page count. We account for all memory managed by the zone in z->page_count */ + OSAddAtomic64(1, &(vm_page_zone->page_count)); + zcram(vm_page_zone, addr, PAGE_SIZE); lck_mtx_unlock(&vm_page_alloc_lock); @@ -1932,9 +2090,9 @@ return_page_from_cpu_list: * it doesn't really matter. */ if ((vm_page_free_count < vm_page_free_min) || - ((vm_page_free_count < vm_page_free_target) && - ((vm_page_inactive_count + vm_page_speculative_count) < vm_page_inactive_min))) - thread_wakeup((event_t) &vm_page_free_wanted); + ((vm_page_free_count < vm_page_free_target) && + ((vm_page_inactive_count + vm_page_speculative_count) < vm_page_inactive_min))) + thread_wakeup((event_t) &vm_page_free_wanted); VM_CHECK_MEMORYSTATUS; @@ -1964,7 +2122,6 @@ vm_page_release( } // dbgLog(mem->phys_page, vm_page_free_count, vm_page_wire_count, 5); /* (TEST/DEBUG) */ - pmap_clear_noencrypt(mem->phys_page); lck_mtx_lock_spin(&vm_page_queue_free_lock); @@ -2098,8 +2255,12 @@ vm_page_wait( if (need_wakeup) thread_wakeup((event_t)&vm_page_free_wanted); - if (wait_result == THREAD_WAITING) + if (wait_result == THREAD_WAITING) { + VM_DEBUG_EVENT(vm_page_wait_block, VM_PAGE_WAIT_BLOCK, DBG_FUNC_START, + vm_page_free_wanted_privileged, vm_page_free_wanted, 0, 0); wait_result = thread_block(THREAD_CONTINUE_NULL); + VM_DEBUG_EVENT(vm_page_wait_block, VM_PAGE_WAIT_BLOCK, DBG_FUNC_END, 0, 0, 0, 0); + } return(wait_result == THREAD_AWAKENED); } else { @@ -2204,7 +2365,6 @@ vm_page_free_prepare_queues( VM_PAGE_CHECK(mem); assert(!mem->free); assert(!mem->cleaning); - assert(!mem->pageout); #if DEBUG lck_mtx_assert(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED); if (mem->free) @@ -2213,18 +2373,18 @@ vm_page_free_prepare_queues( if (mem->object) { vm_object_lock_assert_exclusive(mem->object); } - if (mem->laundry) { /* * We may have to free a page while it's being laundered * if we lost its pager (due to a forced unmount, for example). - * We need to call vm_pageout_throttle_up() before removing - * the page from its VM object, so that we can find out on - * which pageout queue the page is on. + * We need to call vm_pageout_steal_laundry() before removing + * the page from its VM object, so that we can remove it + * from its pageout queue and adjust the laundry accounting */ - vm_pageout_throttle_up(mem); + vm_pageout_steal_laundry(mem, TRUE); counter(++c_laundry_pages_freed); } + VM_PAGE_QUEUES_REMOVE(mem); /* clears local/active/inactive/throttled/speculative */ if (VM_PAGE_WIRED(mem)) { @@ -2268,8 +2428,6 @@ vm_page_free_prepare_object( mem->phys_page = vm_page_fictitious_addr; } if ( !mem->fictitious) { - if (mem->zero_fill == TRUE) - VM_ZF_COUNT_DECR(); vm_page_init(mem, mem->phys_page, mem->lopage); } } @@ -2315,196 +2473,163 @@ vm_page_free_unlocked( } } + /* * Free a list of pages. The list can be up to several hundred pages, * as blocked up by vm_pageout_scan(). * The big win is not having to take the free list lock once - * per page. We sort the incoming pages into n lists, one for - * each color. + * per page. */ void vm_page_free_list( - vm_page_t mem, + vm_page_t freeq, boolean_t prepare_object) { + vm_page_t mem; vm_page_t nxt; - int pg_count = 0; - int color; - int inuse_list_head = -1; + vm_page_t local_freeq; + int pg_count; - queue_head_t free_list[MAX_COLORS]; - int inuse[MAX_COLORS]; + while (freeq) { - for (color = 0; color < (signed) vm_colors; color++) { - queue_init(&free_list[color]); - } - - while (mem) { - assert(!mem->inactive); - assert(!mem->active); - assert(!mem->throttled); - assert(!mem->free); - assert(!mem->speculative); - assert(!VM_PAGE_WIRED(mem)); - assert(mem->pageq.prev == NULL); + pg_count = 0; + local_freeq = VM_PAGE_NULL; + mem = freeq; + + /* + * break up the processing into smaller chunks so + * that we can 'pipeline' the pages onto the + * free list w/o introducing too much + * contention on the global free queue lock + */ + while (mem && pg_count < 64) { + + assert(!mem->inactive); + assert(!mem->active); + assert(!mem->throttled); + assert(!mem->free); + assert(!mem->speculative); + assert(!VM_PAGE_WIRED(mem)); + assert(mem->pageq.prev == NULL); - nxt = (vm_page_t)(mem->pageq.next); + nxt = (vm_page_t)(mem->pageq.next); - if (prepare_object == TRUE) - vm_page_free_prepare_object(mem, TRUE); + if (vm_page_free_verify && !mem->fictitious && !mem->private) { + assert(pmap_verify_free(mem->phys_page)); + } + if (prepare_object == TRUE) + vm_page_free_prepare_object(mem, TRUE); - if (vm_page_free_verify && !mem->fictitious && !mem->private) { - assert(pmap_verify_free(mem->phys_page)); - } + if (!mem->fictitious) { + assert(mem->busy); - if (!mem->fictitious) { - assert(mem->busy); - if ((mem->lopage == TRUE || vm_lopage_refill == TRUE) && - vm_lopage_free_count < vm_lopage_free_limit && - mem->phys_page < max_valid_low_ppnum) { - mem->pageq.next = NULL; - vm_page_release(mem); - } else { + if ((mem->lopage == TRUE || vm_lopage_refill == TRUE) && + vm_lopage_free_count < vm_lopage_free_limit && + mem->phys_page < max_valid_low_ppnum) { + mem->pageq.next = NULL; + vm_page_release(mem); + } else { + /* + * IMPORTANT: we can't set the page "free" here + * because that would make the page eligible for + * a physically-contiguous allocation (see + * vm_page_find_contiguous()) right away (we don't + * hold the vm_page_queue_free lock). That would + * cause trouble because the page is not actually + * in the free queue yet... + */ + mem->pageq.next = (queue_entry_t)local_freeq; + local_freeq = mem; + pg_count++; - /* - * IMPORTANT: we can't set the page "free" here - * because that would make the page eligible for - * a physically-contiguous allocation (see - * vm_page_find_contiguous()) right away (we don't - * hold the vm_page_queue_free lock). That would - * cause trouble because the page is not actually - * in the free queue yet... - */ - color = mem->phys_page & vm_color_mask; - if (queue_empty(&free_list[color])) { - inuse[color] = inuse_list_head; - inuse_list_head = color; + pmap_clear_noencrypt(mem->phys_page); } - queue_enter_first(&free_list[color], - mem, - vm_page_t, - pageq); - pg_count++; - - pmap_clear_noencrypt(mem->phys_page); + } else { + assert(mem->phys_page == vm_page_fictitious_addr || + mem->phys_page == vm_page_guard_addr); + vm_page_release_fictitious(mem); } - } else { - assert(mem->phys_page == vm_page_fictitious_addr || - mem->phys_page == vm_page_guard_addr); - vm_page_release_fictitious(mem); + mem = nxt; } - mem = nxt; - } - if (pg_count) { - unsigned int avail_free_count; - unsigned int need_wakeup = 0; - unsigned int need_priv_wakeup = 0; + freeq = mem; + + if ( (mem = local_freeq) ) { + unsigned int avail_free_count; + unsigned int need_wakeup = 0; + unsigned int need_priv_wakeup = 0; - lck_mtx_lock_spin(&vm_page_queue_free_lock); + lck_mtx_lock_spin(&vm_page_queue_free_lock); - color = inuse_list_head; - - while( color != -1 ) { - vm_page_t first, last; - vm_page_t first_free; + while (mem) { + int color; + + nxt = (vm_page_t)(mem->pageq.next); - /* - * Now that we hold the vm_page_queue_free lock, - * it's safe to mark all pages in our local queue - * as "free"... - */ - queue_iterate(&free_list[color], - mem, - vm_page_t, - pageq) { assert(!mem->free); assert(mem->busy); mem->free = TRUE; - } - /* - * ... and insert our local queue at the head of - * the global free queue. - */ - first = (vm_page_t) queue_first(&free_list[color]); - last = (vm_page_t) queue_last(&free_list[color]); - first_free = (vm_page_t) queue_first(&vm_page_queue_free[color]); - if (queue_empty(&vm_page_queue_free[color])) { - queue_last(&vm_page_queue_free[color]) = - (queue_entry_t) last; - } else { - queue_prev(&first_free->pageq) = - (queue_entry_t) last; + color = mem->phys_page & vm_color_mask; + queue_enter_first(&vm_page_queue_free[color], + mem, + vm_page_t, + pageq); + mem = nxt; } - queue_first(&vm_page_queue_free[color]) = - (queue_entry_t) first; - queue_prev(&first->pageq) = - (queue_entry_t) &vm_page_queue_free[color]; - queue_next(&last->pageq) = - (queue_entry_t) first_free; - - /* next color */ - color = inuse[color]; - } - - vm_page_free_count += pg_count; - avail_free_count = vm_page_free_count; - - if (vm_page_free_wanted_privileged > 0 && - avail_free_count > 0) { - if (avail_free_count < vm_page_free_wanted_privileged) { - need_priv_wakeup = avail_free_count; - vm_page_free_wanted_privileged -= - avail_free_count; - avail_free_count = 0; - } else { - need_priv_wakeup = vm_page_free_wanted_privileged; - vm_page_free_wanted_privileged = 0; - avail_free_count -= - vm_page_free_wanted_privileged; + vm_page_free_count += pg_count; + avail_free_count = vm_page_free_count; + + if (vm_page_free_wanted_privileged > 0 && avail_free_count > 0) { + + if (avail_free_count < vm_page_free_wanted_privileged) { + need_priv_wakeup = avail_free_count; + vm_page_free_wanted_privileged -= avail_free_count; + avail_free_count = 0; + } else { + need_priv_wakeup = vm_page_free_wanted_privileged; + vm_page_free_wanted_privileged = 0; + avail_free_count -= vm_page_free_wanted_privileged; + } } - } + if (vm_page_free_wanted > 0 && avail_free_count > vm_page_free_reserved) { + unsigned int available_pages; - if (vm_page_free_wanted > 0 && - avail_free_count > vm_page_free_reserved) { - unsigned int available_pages; + available_pages = avail_free_count - vm_page_free_reserved; - available_pages = (avail_free_count - - vm_page_free_reserved); + if (available_pages >= vm_page_free_wanted) { + need_wakeup = vm_page_free_wanted; + vm_page_free_wanted = 0; + } else { + need_wakeup = available_pages; + vm_page_free_wanted -= available_pages; + } + } + lck_mtx_unlock(&vm_page_queue_free_lock); - if (available_pages >= vm_page_free_wanted) { - need_wakeup = vm_page_free_wanted; - vm_page_free_wanted = 0; - } else { - need_wakeup = available_pages; - vm_page_free_wanted -= available_pages; + if (need_priv_wakeup != 0) { + /* + * There shouldn't be that many VM-privileged threads, + * so let's wake them all up, even if we don't quite + * have enough pages to satisfy them all. + */ + thread_wakeup((event_t)&vm_page_free_wanted_privileged); + } + if (need_wakeup != 0 && vm_page_free_wanted == 0) { + /* + * We don't expect to have any more waiters + * after this, so let's wake them all up at + * once. + */ + thread_wakeup((event_t) &vm_page_free_count); + } else for (; need_wakeup != 0; need_wakeup--) { + /* + * Wake up one waiter per page we just released. + */ + thread_wakeup_one((event_t) &vm_page_free_count); } - } - lck_mtx_unlock(&vm_page_queue_free_lock); - if (need_priv_wakeup != 0) { - /* - * There shouldn't be that many VM-privileged threads, - * so let's wake them all up, even if we don't quite - * have enough pages to satisfy them all. - */ - thread_wakeup((event_t)&vm_page_free_wanted_privileged); + VM_CHECK_MEMORYSTATUS; } - if (need_wakeup != 0 && vm_page_free_wanted == 0) { - /* - * We don't expect to have any more waiters - * after this, so let's wake them all up at - * once. - */ - thread_wakeup((event_t) &vm_page_free_count); - } else for (; need_wakeup != 0; need_wakeup--) { - /* - * Wake up one waiter per page we just released. - */ - thread_wakeup_one((event_t) &vm_page_free_count); - } - - VM_CHECK_MEMORYSTATUS; } } @@ -2543,6 +2668,11 @@ vm_page_wire( lck_mtx_assert(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED); #endif if ( !VM_PAGE_WIRED(mem)) { + + if (mem->pageout_queue) { + mem->pageout = FALSE; + vm_pageout_throttle_up(mem); + } VM_PAGE_QUEUES_REMOVE(mem); if (mem->object) { @@ -2579,10 +2709,6 @@ vm_page_wire( if (mem->gobbled) vm_page_gobble_count--; mem->gobbled = FALSE; - if (mem->zero_fill == TRUE) { - mem->zero_fill = FALSE; - VM_ZF_COUNT_DECR(); - } VM_CHECK_MEMORYSTATUS; @@ -2728,7 +2854,15 @@ vm_page_deactivate_internal( vm_page_gobble_count--; m->gobbled = FALSE; } - if (m->private || m->fictitious || (VM_PAGE_WIRED(m))) + /* + * if this page is currently on the pageout queue, we can't do the + * VM_PAGE_QUEUES_REMOVE (which doesn't handle the pageout queue case) + * and we can't remove it manually since we would need the object lock + * (which is not required here) to decrement the activity_in_progress + * reference which is held on the object while the page is in the pageout queue... + * just let the normal laundry processing proceed + */ + if (m->pageout_queue || m->private || m->fictitious || m->compressor || (VM_PAGE_WIRED(m))) return; if (!m->absent && clear_hw_reference == TRUE) @@ -2740,9 +2874,6 @@ vm_page_deactivate_internal( if (!m->inactive) { VM_PAGE_QUEUES_REMOVE(m); - assert(!m->laundry); - assert(m->pageq.next == NULL && m->pageq.prev == NULL); - if (!VM_DYNAMIC_PAGING_ENABLED(memory_manager_default) && m->dirty && m->object->internal && (m->object->purgable == VM_PURGABLE_DENY || @@ -2764,6 +2895,59 @@ vm_page_deactivate_internal( } } +/* + * vm_page_enqueue_cleaned + * + * Put the page on the cleaned queue, mark it cleaned, etc. + * Being on the cleaned queue (and having m->clean_queue set) + * does ** NOT ** guarantee that the page is clean! + * + * Call with the queues lock held. + */ + +void vm_page_enqueue_cleaned(vm_page_t m) +{ + assert(m->phys_page != vm_page_guard_addr); +#if DEBUG + lck_mtx_assert(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED); +#endif + assert( !(m->absent && !m->unusual)); + + if (m->gobbled) { + assert( !VM_PAGE_WIRED(m)); + if (!m->private && !m->fictitious) + vm_page_wire_count--; + vm_page_gobble_count--; + m->gobbled = FALSE; + } + /* + * if this page is currently on the pageout queue, we can't do the + * VM_PAGE_QUEUES_REMOVE (which doesn't handle the pageout queue case) + * and we can't remove it manually since we would need the object lock + * (which is not required here) to decrement the activity_in_progress + * reference which is held on the object while the page is in the pageout queue... + * just let the normal laundry processing proceed + */ + if (m->clean_queue || m->pageout_queue || m->private || m->fictitious) + return; + + VM_PAGE_QUEUES_REMOVE(m); + + queue_enter(&vm_page_queue_cleaned, m, vm_page_t, pageq); + m->clean_queue = TRUE; + vm_page_cleaned_count++; + + m->inactive = TRUE; + vm_page_inactive_count++; + if (m->object->internal) { + vm_page_pageable_internal_count++; + } else { + vm_page_pageable_external_count++; + } + + vm_pageout_enqueued_cleaned++; +} + /* * vm_page_activate: * @@ -2772,6 +2956,12 @@ vm_page_deactivate_internal( * The page queues must be locked. */ +#if CONFIG_JETSAM +#if LATENCY_JETSAM +extern struct vm_page jetsam_latency_page[NUM_OF_JETSAM_LATENCY_TOKENS]; +#endif /* LATENCY_JETSAM */ +#endif /* CONFIG_JETSAM */ + void vm_page_activate( register vm_page_t m) @@ -2793,7 +2983,15 @@ vm_page_activate( vm_page_gobble_count--; m->gobbled = FALSE; } - if (m->private || m->fictitious) + /* + * if this page is currently on the pageout queue, we can't do the + * VM_PAGE_QUEUES_REMOVE (which doesn't handle the pageout queue case) + * and we can't remove it manually since we would need the object lock + * (which is not required here) to decrement the activity_in_progress + * reference which is held on the object while the page is in the pageout queue... + * just let the normal laundry processing proceed + */ + if (m->pageout_queue || m->private || m->fictitious || m->compressor) return; #if DEBUG @@ -2805,12 +3003,11 @@ vm_page_activate( DTRACE_VM2(pgrec, int, 1, (uint64_t *), NULL); DTRACE_VM2(pgfrec, int, 1, (uint64_t *), NULL); } - + VM_PAGE_QUEUES_REMOVE(m); if ( !VM_PAGE_WIRED(m)) { - assert(!m->laundry); - assert(m->pageq.next == NULL && m->pageq.prev == NULL); + if (!VM_DYNAMIC_PAGING_ENABLED(memory_manager_default) && m->dirty && m->object->internal && (m->object->purgable == VM_PURGABLE_DENY || @@ -2823,6 +3020,42 @@ vm_page_activate( queue_enter(&vm_page_queue_active, m, vm_page_t, pageq); m->active = TRUE; vm_page_active_count++; + if (m->object->internal) { + vm_page_pageable_internal_count++; + } else { + vm_page_pageable_external_count++; + } +#if LATENCY_JETSAM + if (jlp_init) { + uint64_t now = mach_absolute_time(); + uint64_t delta = now - jlp_time; + clock_sec_t jl_secs = 0; + clock_usec_t jl_usecs = 0; + vm_page_t jlp; + + absolutetime_to_microtime(delta, &jl_secs, &jl_usecs); + + jl_usecs += jl_secs * USEC_PER_SEC; + if (jl_usecs >= JETSAM_LATENCY_TOKEN_AGE) { + + jlp = &jetsam_latency_page[jlp_current]; + if (jlp->active) { + queue_remove(&vm_page_queue_active, jlp, vm_page_t, pageq); + } + queue_enter(&vm_page_queue_active, jlp, vm_page_t, pageq); + + jlp->active = TRUE; + + jlp->offset = now; + jlp_time = jlp->offset; + + if(++jlp_current == NUM_OF_JETSAM_LATENCY_TOKENS) { + jlp_current = 0; + } + + } + } +#endif /* LATENCY_JETSAM */ } m->reference = TRUE; m->no_cache = FALSE; @@ -2853,7 +3086,15 @@ vm_page_speculate( #endif assert( !(m->absent && !m->unusual)); - if (m->private || m->fictitious) + /* + * if this page is currently on the pageout queue, we can't do the + * VM_PAGE_QUEUES_REMOVE (which doesn't handle the pageout queue case) + * and we can't remove it manually since we would need the object lock + * (which is not required here) to decrement the activity_in_progress + * reference which is held on the object while the page is in the pageout queue... + * just let the normal laundry processing proceed + */ + if (m->pageout_queue || m->private || m->fictitious || m->compressor) return; VM_PAGE_QUEUES_REMOVE(m); @@ -2910,6 +3151,11 @@ vm_page_speculate( enqueue_tail(&aq->age_q, &m->pageq); m->speculative = TRUE; vm_page_speculative_count++; + if (m->object->internal) { + vm_page_pageable_internal_count++; + } else { + vm_page_pageable_external_count++; + } if (new == TRUE) { vm_object_lock_assert_exclusive(m->object); @@ -2974,19 +3220,21 @@ vm_page_lru( #if DEBUG lck_mtx_assert(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED); #endif - if (m->active || m->reference) - return; - - if (m->private || (VM_PAGE_WIRED(m))) + /* + * if this page is currently on the pageout queue, we can't do the + * VM_PAGE_QUEUES_REMOVE (which doesn't handle the pageout queue case) + * and we can't remove it manually since we would need the object lock + * (which is not required here) to decrement the activity_in_progress + * reference which is held on the object while the page is in the pageout queue... + * just let the normal laundry processing proceed + */ + if (m->pageout_queue || m->private || m->compressor || (VM_PAGE_WIRED(m))) return; m->no_cache = FALSE; VM_PAGE_QUEUES_REMOVE(m); - assert(!m->laundry); - assert(m->pageq.next == NULL && m->pageq.prev == NULL); - VM_PAGE_ENQUEUE_INACTIVE(m, FALSE); } @@ -2998,11 +3246,14 @@ vm_page_reactivate_all_throttled(void) vm_page_t first_active; vm_page_t m; int extra_active_count; + int extra_internal_count, extra_external_count; if (!VM_DYNAMIC_PAGING_ENABLED(memory_manager_default)) return; extra_active_count = 0; + extra_internal_count = 0; + extra_external_count = 0; vm_page_lock_queues(); if (! queue_empty(&vm_page_queue_throttled)) { /* @@ -3017,6 +3268,11 @@ vm_page_reactivate_all_throttled(void) assert(!VM_PAGE_WIRED(m)); extra_active_count++; + if (m->object->internal) { + extra_internal_count++; + } else { + extra_external_count++; + } m->throttled = FALSE; m->active = TRUE; @@ -3049,6 +3305,8 @@ vm_page_reactivate_all_throttled(void) * Adjust the global page counts. */ vm_page_active_count += extra_active_count; + vm_page_pageable_internal_count += extra_internal_count; + vm_page_pageable_external_count += extra_external_count; vm_page_throttled_count = 0; } assert(vm_page_throttled_count == 0); @@ -3137,7 +3395,11 @@ vm_page_reactivate_local(uint32_t lid, boolean_t force, boolean_t nolocks) * Adjust the global page counts. */ vm_page_active_count += lq->vpl_count; + vm_page_pageable_internal_count += lq->vpl_internal_count; + vm_page_pageable_external_count += lq->vpl_external_count; lq->vpl_count = 0; + lq->vpl_internal_count = 0; + lq->vpl_external_count = 0; } assert(queue_empty(&lq->vpl_queue)); @@ -3152,18 +3414,26 @@ vm_page_reactivate_local(uint32_t lid, boolean_t force, boolean_t nolocks) * * Zero-fill a part of the page. */ +#define PMAP_ZERO_PART_PAGE_IMPLEMENTED void vm_page_part_zero_fill( vm_page_t m, vm_offset_t m_pa, vm_size_t len) { - vm_page_t tmp; +#if 0 + /* + * we don't hold the page queue lock + * so this check isn't safe to make + */ VM_PAGE_CHECK(m); +#endif + #ifdef PMAP_ZERO_PART_PAGE_IMPLEMENTED pmap_zero_part_page(m->phys_page, m_pa, len); #else + vm_page_t tmp; while (1) { tmp = vm_page_grab(); if (tmp == VM_PAGE_NULL) { @@ -3198,8 +3468,13 @@ vm_page_zero_fill( XPR(XPR_VM_PAGE, "vm_page_zero_fill, object 0x%X offset 0x%X page 0x%X\n", m->object, m->offset, m, 0,0); - +#if 0 + /* + * we don't hold the page queue lock + * so this check isn't safe to make + */ VM_PAGE_CHECK(m); +#endif // dbgTrace(0xAEAEAEAE, m->phys_page, 0); /* (BRINGUP) */ pmap_zero_page(m->phys_page); @@ -3219,9 +3494,14 @@ vm_page_part_copy( vm_offset_t dst_pa, vm_size_t len) { +#if 0 + /* + * we don't hold the page queue lock + * so this check isn't safe to make + */ VM_PAGE_CHECK(src_m); VM_PAGE_CHECK(dst_m); - +#endif pmap_copy_part_page(src_m->phys_page, src_pa, dst_m->phys_page, dst_pa, len); } @@ -3249,9 +3529,15 @@ vm_page_copy( src_m->object, src_m->offset, dest_m->object, dest_m->offset, 0); - +#if 0 + /* + * we don't hold the page queue lock + * so this check isn't safe to make + */ VM_PAGE_CHECK(src_m); VM_PAGE_CHECK(dest_m); +#endif + vm_object_lock_assert_held(src_m->object); /* * ENCRYPTED SWAP: @@ -3281,7 +3567,7 @@ vm_page_copy( src_m->busy = TRUE; (void) vm_page_slide(src_m, 0); assert(src_m->busy); - if(!was_busy) { + if (!was_busy) { PAGE_WAKEUP_DONE(src_m); } } @@ -3345,14 +3631,10 @@ _vm_page_print( (p->unusual ? "" : "!"), (p->encrypted ? "" : "!"), (p->encrypted_cleaning ? "" : "!")); - printf(" %slist_req_pending, %sdump_cleaning, %scs_validated, %scs_tainted, %sno_cache\n", - (p->list_req_pending ? "" : "!"), - (p->dump_cleaning ? "" : "!"), + printf(" %scs_validated, %scs_tainted, %sno_cache\n", (p->cs_validated ? "" : "!"), (p->cs_tainted ? "" : "!"), (p->no_cache ? "" : "!")); - printf(" %szero_fill\n", - (p->zero_fill ? "" : "!")); printf("phys_page=0x%x\n", p->phys_page); } @@ -3496,6 +3778,9 @@ vm_page_queues_assert( vm_page_t mem, int val) { +#if DEBUG + lck_mtx_assert(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED); +#endif if (mem->free + mem->active + mem->inactive + mem->speculative + mem->throttled + mem->pageout_queue > (val)) { _vm_page_print(mem); @@ -3506,6 +3791,7 @@ vm_page_queues_assert( assert(!mem->inactive); assert(!mem->speculative); assert(!mem->throttled); + assert(!mem->pageout_queue); } } #endif /* MACH_ASSERT */ @@ -3602,6 +3888,7 @@ vm_page_find_contiguous( int yielded = 0; int dumped_run = 0; int stolen_pages = 0; + int compressed_pages = 0; #endif if (contig_pages == 0) @@ -3613,6 +3900,8 @@ vm_page_find_contiguous( #if DEBUG clock_get_system_microtime(&tv_start_sec, &tv_start_usec); #endif + PAGE_REPLACEMENT_ALLOWED(TRUE); + vm_page_lock_queues(); lck_mtx_lock(&vm_page_queue_free_lock); @@ -3663,10 +3952,9 @@ retry: RESET_STATE_OF_RUN(); } else if (VM_PAGE_WIRED(m) || m->gobbled || - m->encrypted || m->encrypted_cleaning || m->cs_validated || m->cs_tainted || - m->error || m->absent || m->pageout_queue || m->laundry || m->wanted || m->precious || - m->cleaning || m->overwriting || m->restart || m->unusual || m->list_req_pending || - m->pageout) { + m->encrypted_cleaning || + m->pageout_queue || m->laundry || m->wanted || + m->cleaning || m->overwriting || m->pageout) { /* * page is in a transient state * or a state we don't want to deal @@ -3675,9 +3963,10 @@ retry: */ RESET_STATE_OF_RUN(); - } else if (!m->free && !m->active && !m->inactive && !m->speculative && !m->throttled) { + } else if (!m->free && !m->active && !m->inactive && !m->speculative && !m->throttled && !m->compressor) { /* * page needs to be on one of our queues + * or it needs to belong to the compressor pool * in order for it to be stable behind the * locks we hold at this point... * if not, don't consider it which @@ -3728,7 +4017,7 @@ retry: * into a substitute page. */ #if VM_PAGE_FIND_CONTIGUOUS_CAN_STEAL - if (m->pmapped || m->dirty) { + if (m->pmapped || m->dirty || m->precious) { substitute_needed++; } #else @@ -3763,12 +4052,16 @@ retry: } did_consider: if (considered > MAX_CONSIDERED_BEFORE_YIELD && npages <= 1) { - + + PAGE_REPLACEMENT_ALLOWED(FALSE); + lck_mtx_unlock(&vm_page_queue_free_lock); vm_page_unlock_queues(); mutex_pause(0); + PAGE_REPLACEMENT_ALLOWED(TRUE); + vm_page_lock_queues(); lck_mtx_lock(&vm_page_queue_free_lock); @@ -3893,6 +4186,7 @@ did_consider: m1 = &vm_pages[cur_idx--]; assert(!m1->free); + if (m1->object == VM_OBJECT_NULL) { /* * page has already been removed from @@ -3904,6 +4198,8 @@ did_consider: assert(!m1->laundry); } else { vm_object_t object; + int refmod; + boolean_t disconnected, reusable; if (abort_run == TRUE) continue; @@ -3920,9 +4216,9 @@ did_consider: } if (locked_object == VM_OBJECT_NULL || (VM_PAGE_WIRED(m1) || m1->gobbled || - m1->encrypted || m1->encrypted_cleaning || m1->cs_validated || m1->cs_tainted || - m1->error || m1->absent || m1->pageout_queue || m1->laundry || m1->wanted || m1->precious || - m1->cleaning || m1->overwriting || m1->restart || m1->unusual || m1->list_req_pending || m1->busy)) { + m1->encrypted_cleaning || + m1->pageout_queue || m1->laundry || m1->wanted || + m1->cleaning || m1->overwriting || m1->pageout || m1->busy)) { if (locked_object) { vm_object_unlock(locked_object); @@ -3932,11 +4228,34 @@ did_consider: abort_run = TRUE; continue; } - if (m1->pmapped || m1->dirty) { - int refmod; - vm_object_offset_t offset; - - m2 = vm_page_grab(); + + disconnected = FALSE; + reusable = FALSE; + + if ((m1->reusable || + m1->object->all_reusable) && + m1->inactive && + !m1->dirty && + !m1->reference) { + /* reusable page... */ + refmod = pmap_disconnect(m1->phys_page); + disconnected = TRUE; + if (refmod == 0) { + /* + * ... not reused: can steal + * without relocating contents. + */ + reusable = TRUE; + } + } + + if ((m1->pmapped && + ! reusable) || + m1->dirty || + m1->precious) { + vm_object_offset_t offset; + + m2 = vm_page_grab(); if (m2 == VM_PAGE_NULL) { if (locked_object) { @@ -3947,19 +4266,81 @@ did_consider: abort_run = TRUE; continue; } - if (m1->pmapped) - refmod = pmap_disconnect(m1->phys_page); - else - refmod = 0; - vm_page_copy(m1, m2); - - m2->reference = m1->reference; - m2->dirty = m1->dirty; + if (! disconnected) { + if (m1->pmapped) + refmod = pmap_disconnect(m1->phys_page); + else + refmod = 0; + } + + /* copy the page's contents */ + pmap_copy_page(m1->phys_page, m2->phys_page); + /* copy the page's state */ + assert(!VM_PAGE_WIRED(m1)); + assert(!m1->free); + assert(!m1->pageout_queue); + assert(!m1->laundry); + m2->reference = m1->reference; + assert(!m1->gobbled); + assert(!m1->private); + m2->no_cache = m1->no_cache; + m2->xpmapped = m1->xpmapped; + assert(!m1->busy); + assert(!m1->wanted); + assert(!m1->fictitious); + m2->pmapped = m1->pmapped; /* should flush cache ? */ + m2->wpmapped = m1->wpmapped; + assert(!m1->pageout); + m2->absent = m1->absent; + m2->error = m1->error; + m2->dirty = m1->dirty; + assert(!m1->cleaning); + m2->precious = m1->precious; + m2->clustered = m1->clustered; + assert(!m1->overwriting); + m2->restart = m1->restart; + m2->unusual = m1->unusual; + m2->encrypted = m1->encrypted; + assert(!m1->encrypted_cleaning); + m2->cs_validated = m1->cs_validated; + m2->cs_tainted = m1->cs_tainted; + + /* + * If m1 had really been reusable, + * we would have just stolen it, so + * let's not propagate it's "reusable" + * bit and assert that m2 is not + * marked as "reusable". + */ + // m2->reusable = m1->reusable; + assert(!m2->reusable); + + assert(!m1->lopage); + m2->slid = m1->slid; + m2->was_dirty = m1->was_dirty; + m2->compressor = m1->compressor; + + /* + * page may need to be flushed if + * it is marshalled into a UPL + * that is going to be used by a device + * that doesn't support coherency + */ + m2->written_by_kernel = TRUE; + + /* + * make sure we clear the ref/mod state + * from the pmap layer... else we risk + * inheriting state from the last time + * this page was used... + */ + pmap_clear_refmod(m2->phys_page, VM_MEM_MODIFIED | VM_MEM_REFERENCED); if (refmod & VM_MEM_REFERENCED) m2->reference = TRUE; - if (refmod & VM_MEM_MODIFIED) - m2->dirty = TRUE; + if (refmod & VM_MEM_MODIFIED) { + SET_PAGE_DIRTY(m2, TRUE); + } offset = m1->offset; /* @@ -3972,25 +4353,31 @@ did_consider: vm_page_free_prepare(m1); /* - * make sure we clear the ref/mod state - * from the pmap layer... else we risk - * inheriting state from the last time - * this page was used... + * now put the substitute page + * on the object */ - pmap_clear_refmod(m2->phys_page, VM_MEM_MODIFIED | VM_MEM_REFERENCED); - /* - * now put the substitute page on the object - */ - vm_page_insert_internal(m2, locked_object, offset, TRUE, TRUE); + vm_page_insert_internal(m2, locked_object, offset, TRUE, TRUE, FALSE); - if (m2->reference) - vm_page_activate(m2); - else - vm_page_deactivate(m2); + if (m2->compressor) { + m2->pmapped = TRUE; + m2->wpmapped = TRUE; + PMAP_ENTER(kernel_pmap, m2->offset, m2, + VM_PROT_READ | VM_PROT_WRITE, VM_PROT_NONE, 0, TRUE); +#if MACH_ASSERT + compressed_pages++; +#endif + } else { + if (m2->reference) + vm_page_activate(m2); + else + vm_page_deactivate(m2); + } PAGE_WAKEUP_DONE(m2); } else { + assert(!m1->compressor); + /* * completely cleans up the state * of the page so that it is ready @@ -4077,6 +4464,8 @@ did_consider: assert(vm_page_verify_contiguous(m, npages)); } done_scanning: + PAGE_REPLACEMENT_ALLOWED(FALSE); + vm_page_unlock_queues(); #if DEBUG @@ -4093,10 +4482,10 @@ done_scanning: tv_end_sec -= 1000000; } if (vm_page_find_contig_debug) { - printf("%s(num=%d,low=%d): found %d pages at 0x%llx in %ld.%06ds... started at %d... scanned %d pages... yielded %d times... dumped run %d times... stole %d pages\n", - __func__, contig_pages, max_pnum, npages, (vm_object_offset_t)start_pnum << PAGE_SHIFT, - (long)tv_end_sec, tv_end_usec, orig_last_idx, - scanned, yielded, dumped_run, stolen_pages); + printf("%s(num=%d,low=%d): found %d pages at 0x%llx in %ld.%06ds... started at %d... scanned %d pages... yielded %d times... dumped run %d times... stole %d pages... stole %d compressed pages\n", + __func__, contig_pages, max_pnum, npages, (vm_object_offset_t)start_pnum << PAGE_SHIFT, + (long)tv_end_sec, tv_end_usec, orig_last_idx, + scanned, yielded, dumped_run, stolen_pages, compressed_pages); } #endif @@ -4143,9 +4532,9 @@ cpm_allocate( * determine need for wakeups */ if ((vm_page_free_count < vm_page_free_min) || - ((vm_page_free_count < vm_page_free_target) && - ((vm_page_inactive_count + vm_page_speculative_count) < vm_page_inactive_min))) - thread_wakeup((event_t) &vm_page_free_wanted); + ((vm_page_free_count < vm_page_free_target) && + ((vm_page_inactive_count + vm_page_speculative_count) < vm_page_inactive_min))) + thread_wakeup((event_t) &vm_page_free_wanted); VM_CHECK_MEMORYSTATUS; @@ -4190,7 +4579,6 @@ vm_page_do_delayed_work( int j; vm_page_t m; vm_page_t local_free_q = VM_PAGE_NULL; - boolean_t dropped_obj_lock = FALSE; /* * pageout_scan takes the vm_page_lock_queues first @@ -4218,28 +4606,11 @@ vm_page_do_delayed_work( mutex_pause(j); vm_page_lockspin_queues(); } - dropped_obj_lock = TRUE; } for (j = 0; j < dw_count; j++, dwp++) { m = dwp->dw_m; - if (dwp->dw_mask & DW_set_list_req_pending) { - m->list_req_pending = TRUE; - - if (dropped_obj_lock == TRUE) { - /* - * need to make sure anyone that might have - * blocked on busy == TRUE when we dropped - * the object lock gets a chance to re-evaluate - * its state since we have several places - * where we avoid potential deadlocks with - * the fileysystem by stealing pages with - * list_req_pending == TRUE and busy == TRUE - */ - dwp->dw_mask |= DW_PAGE_WAKEUP; - } - } if (dwp->dw_mask & DW_vm_pageout_throttle_up) vm_pageout_throttle_up(m); @@ -4272,25 +4643,51 @@ vm_page_do_delayed_work( } else if (dwp->dw_mask & DW_vm_page_speculate) vm_page_speculate(m, TRUE); + else if (dwp->dw_mask & DW_enqueue_cleaned) { + /* + * if we didn't hold the object lock and did this, + * we might disconnect the page, then someone might + * soft fault it back in, then we would put it on the + * cleaned queue, and so we would have a referenced (maybe even dirty) + * page on that queue, which we don't want + */ + int refmod_state = pmap_disconnect(m->phys_page); + + if ((refmod_state & VM_MEM_REFERENCED)) { + /* + * this page has been touched since it got cleaned; let's activate it + * if it hasn't already been + */ + vm_pageout_enqueued_cleaned++; + vm_pageout_cleaned_reactivated++; + vm_pageout_cleaned_commit_reactivated++; + + if (m->active == FALSE) + vm_page_activate(m); + } else { + m->reference = FALSE; + vm_page_enqueue_cleaned(m); + } + } else if (dwp->dw_mask & DW_vm_page_lru) vm_page_lru(m); - else if (dwp->dw_mask & DW_VM_PAGE_QUEUES_REMOVE) - VM_PAGE_QUEUES_REMOVE(m); - + else if (dwp->dw_mask & DW_VM_PAGE_QUEUES_REMOVE) { + if ( !m->pageout_queue) + VM_PAGE_QUEUES_REMOVE(m); + } if (dwp->dw_mask & DW_set_reference) m->reference = TRUE; else if (dwp->dw_mask & DW_clear_reference) m->reference = FALSE; if (dwp->dw_mask & DW_move_page) { - VM_PAGE_QUEUES_REMOVE(m); + if ( !m->pageout_queue) { + VM_PAGE_QUEUES_REMOVE(m); - assert(!m->laundry); - assert(m->object != kernel_object); - assert(m->pageq.next == NULL && - m->pageq.prev == NULL); + assert(m->object != kernel_object); - VM_PAGE_ENQUEUE_INACTIVE(m, FALSE); + VM_PAGE_ENQUEUE_INACTIVE(m, FALSE); + } } if (dwp->dw_mask & DW_clear_busy) m->busy = FALSE; @@ -4308,38 +4705,6 @@ vm_page_do_delayed_work( } - - - -void vm_check_memorystatus() -{ -#if CONFIG_EMBEDDED - static boolean_t in_critical = FALSE; - static unsigned int last_memorystatus = 0; - unsigned int pages_avail; - - if (!kern_memorystatus_delta) { - return; - } - - pages_avail = (vm_page_active_count + - vm_page_inactive_count + - vm_page_speculative_count + - vm_page_free_count + - (VM_DYNAMIC_PAGING_ENABLED(memory_manager_default) ? 0 : vm_page_purgeable_count)); - if ( (!in_critical && (pages_avail < kern_memorystatus_delta)) || - (pages_avail >= (last_memorystatus + kern_memorystatus_delta)) || - (last_memorystatus >= (pages_avail + kern_memorystatus_delta)) ) { - kern_memorystatus_level = pages_avail * 100 / atop_64(max_mem); - last_memorystatus = pages_avail; - - thread_wakeup((event_t)&kern_memorystatus_wakeup); - - in_critical = (pages_avail < kern_memorystatus_delta) ? TRUE : FALSE; - } -#endif -} - kern_return_t vm_page_alloc_list( int page_count, @@ -4407,14 +4772,19 @@ static vm_page_t hibernate_gobble_queue; extern boolean_t (* volatile consider_buffer_cache_collect)(int); static int hibernate_drain_pageout_queue(struct vm_pageout_queue *); -static int hibernate_flush_dirty_pages(void); +static int hibernate_flush_dirty_pages(int); static int hibernate_flush_queue(queue_head_t *, int); -static void hibernate_dirty_page(vm_page_t); void hibernate_flush_wait(void); void hibernate_mark_in_progress(void); void hibernate_clear_in_progress(void); +void hibernate_free_range(int, int); +void hibernate_hash_insert_page(vm_page_t); +uint32_t hibernate_mark_as_unneeded(addr64_t, addr64_t, hibernate_page_list_t *, hibernate_page_list_t *); +void hibernate_rebuild_vm_structs(void); +uint32_t hibernate_teardown_vm_structs(hibernate_page_list_t *, hibernate_page_list_t *); +ppnum_t hibernate_lookup_paddr(unsigned int); struct hibernate_statistics { int hibernate_considered; @@ -4423,6 +4793,7 @@ struct hibernate_statistics { int hibernate_skipped_cleaning; int hibernate_skipped_transient; int hibernate_skipped_precious; + int hibernate_skipped_external; int hibernate_queue_nolock; int hibernate_queue_paused; int hibernate_throttled; @@ -4437,9 +4808,11 @@ struct hibernate_statistics { int cd_found_cleaning; int cd_found_laundry; int cd_found_dirty; + int cd_found_xpmapped; int cd_local_free; int cd_total_free; int cd_vm_page_wire_count; + int cd_vm_struct_pages_unneeded; int cd_pages; int cd_discarded; int cd_count_wire; @@ -4454,7 +4827,7 @@ hibernate_drain_pageout_queue(struct vm_pageout_queue *q) vm_page_lock_queues(); - while (q->pgo_laundry) { + while ( !queue_empty(&q->pgo_pending) ) { q->pgo_draining = TRUE; @@ -4464,8 +4837,12 @@ hibernate_drain_pageout_queue(struct vm_pageout_queue *q) wait_result = thread_block(THREAD_CONTINUE_NULL); - if (wait_result == THREAD_TIMED_OUT) { + if (wait_result == THREAD_TIMED_OUT && !queue_empty(&q->pgo_pending)) { hibernate_stats.hibernate_drain_timeout++; + + if (q == &vm_pageout_queue_external) + return (0); + return (1); } vm_page_lock_queues(); @@ -4477,46 +4854,8 @@ hibernate_drain_pageout_queue(struct vm_pageout_queue *q) return (0); } -static void -hibernate_dirty_page(vm_page_t m) -{ - vm_object_t object = m->object; - struct vm_pageout_queue *q; - -#if DEBUG - lck_mtx_assert(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED); -#endif - vm_object_lock_assert_exclusive(object); - - /* - * protect the object from collapse - - * locking in the object's paging_offset. - */ - vm_object_paging_begin(object); - - m->list_req_pending = TRUE; - m->cleaning = TRUE; - m->busy = TRUE; - if (object->internal == TRUE) - q = &vm_pageout_queue_internal; - else - q = &vm_pageout_queue_external; - - /* - * pgo_laundry count is tied to the laundry bit - */ - m->laundry = TRUE; - q->pgo_laundry++; - - m->pageout_queue = TRUE; - queue_enter(&q->pgo_pending, m, vm_page_t, pageq); - - if (q->pgo_idle == TRUE) { - q->pgo_idle = FALSE; - thread_wakeup((event_t) &q->pgo_pending); - } -} +boolean_t hibernate_skip_external = FALSE; static int hibernate_flush_queue(queue_head_t *q, int qcount) @@ -4595,7 +4934,7 @@ hibernate_flush_queue(queue_head_t *q, int qcount) vm_pageout_scan_wants_object = VM_OBJECT_NULL; } } - if ( !m_object->alive || m->encrypted_cleaning || m->cleaning || m->busy || m->absent || m->error) { + if ( !m_object->alive || m->encrypted_cleaning || m->cleaning || m->laundry || m->busy || m->absent || m->error) { /* * page is not to be cleaned * put it back on the head of its queue @@ -4607,9 +4946,6 @@ hibernate_flush_queue(queue_head_t *q, int qcount) goto reenter_pg_on_q; } - if ( !m_object->pager_initialized && m_object->pager_created) - goto reenter_pg_on_q; - if (m_object->copy == VM_OBJECT_NULL) { if (m_object->purgable == VM_PURGABLE_VOLATILE || m_object->purgable == VM_PURGABLE_EMPTY) { /* @@ -4622,8 +4958,9 @@ hibernate_flush_queue(queue_head_t *q, int qcount) if ( !m->dirty && m->pmapped) { refmod_state = pmap_get_refmod(m->phys_page); - if ((refmod_state & VM_MEM_MODIFIED)) - m->dirty = TRUE; + if ((refmod_state & VM_MEM_MODIFIED)) { + SET_PAGE_DIRTY(m, FALSE); + } } else refmod_state = 0; @@ -4637,6 +4974,13 @@ hibernate_flush_queue(queue_head_t *q, int qcount) goto reenter_pg_on_q; } + + if (hibernate_skip_external == TRUE && !m_object->internal) { + + hibernate_stats.hibernate_skipped_external++; + + goto reenter_pg_on_q; + } tq = NULL; if (m_object->internal) { @@ -4655,28 +4999,36 @@ hibernate_flush_queue(queue_head_t *q, int qcount) } vm_pageout_scan_wants_object = VM_OBJECT_NULL; - tq->pgo_throttled = TRUE; - while (retval == 0) { + tq->pgo_throttled = TRUE; + assert_wait_timeout((event_t) &tq->pgo_laundry, THREAD_INTERRUPTIBLE, 1000, 1000*NSEC_PER_USEC); - vm_page_unlock_queues(); + vm_page_unlock_queues(); - wait_result = thread_block(THREAD_CONTINUE_NULL); + wait_result = thread_block(THREAD_CONTINUE_NULL); vm_page_lock_queues(); + if (wait_result != THREAD_TIMED_OUT) + break; + if (!VM_PAGE_Q_THROTTLED(tq)) + break; + if (hibernate_should_abort()) retval = 1; - if (wait_result != THREAD_TIMED_OUT) - break; - if (--wait_count == 0) { - hibernate_stats.hibernate_throttle_timeout++; - retval = 1; - } + + hibernate_stats.hibernate_throttle_timeout++; + + if (tq == eq) { + hibernate_skip_external = TRUE; + break; + } + retval = 1; + } } if (retval) break; @@ -4685,9 +5037,19 @@ hibernate_flush_queue(queue_head_t *q, int qcount) continue; } + /* + * we've already factored out pages in the laundry which + * means this page can't be on the pageout queue so it's + * safe to do the VM_PAGE_QUEUES_REMOVE + */ + assert(!m->pageout_queue); + VM_PAGE_QUEUES_REMOVE(m); - hibernate_dirty_page(m); + if (COMPRESSED_PAGER_IS_ACTIVE) + pmap_disconnect(m->phys_page); + + vm_pageout_cluster(m, FALSE); hibernate_stats.hibernate_found_dirty++; @@ -4708,7 +5070,7 @@ next_pg: vm_object_unlock(l_object); l_object = NULL; } - vm_pageout_scan_wants_object = VM_OBJECT_NULL; + vm_pageout_scan_wants_object = VM_OBJECT_NULL; vm_page_unlock_queues(); @@ -4719,7 +5081,7 @@ next_pg: static int -hibernate_flush_dirty_pages() +hibernate_flush_dirty_pages(int pass) { struct vm_speculative_age_q *aq; uint32_t i; @@ -4757,21 +5119,37 @@ hibernate_flush_dirty_pages() return (1); } } - if (hibernate_flush_queue(&vm_page_queue_active, vm_page_active_count)) + if (hibernate_flush_queue(&vm_page_queue_inactive, vm_page_inactive_count - vm_page_anonymous_count - vm_page_cleaned_count)) return (1); - if (hibernate_flush_queue(&vm_page_queue_inactive, vm_page_inactive_count - vm_zf_queue_count)) + if (hibernate_flush_queue(&vm_page_queue_anonymous, vm_page_anonymous_count)) return (1); - if (hibernate_flush_queue(&vm_page_queue_zf, vm_zf_queue_count)) + if (hibernate_flush_queue(&vm_page_queue_cleaned, vm_page_cleaned_count)) return (1); - if (hibernate_drain_pageout_queue(&vm_pageout_queue_internal)) return (1); - return (hibernate_drain_pageout_queue(&vm_pageout_queue_external)); -} + if (COMPRESSED_PAGER_IS_ACTIVE && pass == 1) + vm_compressor_record_warmup_start(); + + if (hibernate_flush_queue(&vm_page_queue_active, vm_page_active_count)) { + if (COMPRESSED_PAGER_IS_ACTIVE && pass == 1) + vm_compressor_record_warmup_end(); + return (1); + } + if (hibernate_drain_pageout_queue(&vm_pageout_queue_internal)) { + if (COMPRESSED_PAGER_IS_ACTIVE && pass == 1) + vm_compressor_record_warmup_end(); + return (1); + } + if (COMPRESSED_PAGER_IS_ACTIVE && pass == 1) + vm_compressor_record_warmup_end(); + + if (hibernate_skip_external == FALSE && hibernate_drain_pageout_queue(&vm_pageout_queue_external)) + return (1); + + return (0); +} -extern void IOSleep(unsigned int); -extern int sync_internal(void); int hibernate_flush_memory() @@ -4780,32 +5158,53 @@ hibernate_flush_memory() KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 3) | DBG_FUNC_START, vm_page_free_count, 0, 0, 0, 0); - IOSleep(2 * 1000); + hibernate_cleaning_in_progress = TRUE; + hibernate_skip_external = FALSE; + + if ((retval = hibernate_flush_dirty_pages(1)) == 0) { + + if (COMPRESSED_PAGER_IS_ACTIVE) { - KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 3) | DBG_FUNC_NONE, vm_page_free_count, 0, 0, 0, 0); + if ((retval = hibernate_flush_dirty_pages(2)) == 0) { - if ((retval = hibernate_flush_dirty_pages()) == 0) { - if (consider_buffer_cache_collect != NULL) { + KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 10) | DBG_FUNC_START, VM_PAGE_COMPRESSOR_COUNT, 0, 0, 0, 0); + + vm_compressor_flush(); + + KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 10) | DBG_FUNC_END, VM_PAGE_COMPRESSOR_COUNT, 0, 0, 0, 0); + } + } + if (retval == 0 && consider_buffer_cache_collect != NULL) { + unsigned int orig_wire_count; - KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 7) | DBG_FUNC_START, vm_page_wire_count, 0, 0, 0, 0); + KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 7) | DBG_FUNC_START, 0, 0, 0, 0, 0); + orig_wire_count = vm_page_wire_count; - sync_internal(); (void)(*consider_buffer_cache_collect)(1); consider_zone_gc(TRUE); - KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 7) | DBG_FUNC_END, vm_page_wire_count, 0, 0, 0, 0); + HIBLOG("hibernate_flush_memory: buffer_cache_gc freed up %d wired pages\n", orig_wire_count - vm_page_wire_count); + + KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 7) | DBG_FUNC_END, orig_wire_count - vm_page_wire_count, 0, 0, 0, 0); } } + hibernate_cleaning_in_progress = FALSE; + KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 3) | DBG_FUNC_END, vm_page_free_count, hibernate_stats.hibernate_found_dirty, retval, 0, 0); + if (retval && COMPRESSED_PAGER_IS_ACTIVE) + HIBLOG("hibernate_flush_memory() failed to finish - vm_page_compressor_count(%d)\n", VM_PAGE_COMPRESSOR_COUNT); + + HIBPRINT("hibernate_flush_memory() considered(%d) reentered_on_q(%d) found_dirty(%d)\n", hibernate_stats.hibernate_considered, hibernate_stats.hibernate_reentered_on_q, hibernate_stats.hibernate_found_dirty); - HIBPRINT(" skipped_cleaning(%d) skipped_transient(%d) skipped_precious(%d) queue_nolock(%d)\n", + HIBPRINT(" skipped_cleaning(%d) skipped_transient(%d) skipped_precious(%d) skipped_external(%d) queue_nolock(%d)\n", hibernate_stats.hibernate_skipped_cleaning, hibernate_stats.hibernate_skipped_transient, hibernate_stats.hibernate_skipped_precious, + hibernate_stats.hibernate_skipped_external, hibernate_stats.hibernate_queue_nolock); HIBPRINT(" queue_paused(%d) throttled(%d) throttle_timeout(%d) drained(%d) drain_timeout(%d)\n", hibernate_stats.hibernate_queue_paused, @@ -4892,7 +5291,7 @@ hibernate_free_gobble_pages(void) } static boolean_t -hibernate_consider_discard(vm_page_t m) +hibernate_consider_discard(vm_page_t m, boolean_t preflight) { vm_object_t object = NULL; int refmod_state; @@ -4904,39 +5303,39 @@ hibernate_consider_discard(vm_page_t m) panic("hibernate_consider_discard: private"); if (!vm_object_lock_try(m->object)) { - hibernate_stats.cd_lock_failed++; + if (!preflight) hibernate_stats.cd_lock_failed++; break; } object = m->object; if (VM_PAGE_WIRED(m)) { - hibernate_stats.cd_found_wired++; + if (!preflight) hibernate_stats.cd_found_wired++; break; } if (m->precious) { - hibernate_stats.cd_found_precious++; + if (!preflight) hibernate_stats.cd_found_precious++; break; } if (m->busy || !object->alive) { /* * Somebody is playing with this page. */ - hibernate_stats.cd_found_busy++; + if (!preflight) hibernate_stats.cd_found_busy++; break; } if (m->absent || m->unusual || m->error) { /* * If it's unusual in anyway, ignore it */ - hibernate_stats.cd_found_unusual++; + if (!preflight) hibernate_stats.cd_found_unusual++; break; } if (m->cleaning) { - hibernate_stats.cd_found_cleaning++; + if (!preflight) hibernate_stats.cd_found_cleaning++; break; } - if (m->laundry || m->list_req_pending) { - hibernate_stats.cd_found_laundry++; + if (m->laundry) { + if (!preflight) hibernate_stats.cd_found_laundry++; break; } if (!m->dirty) @@ -4945,8 +5344,9 @@ hibernate_consider_discard(vm_page_t m) if (refmod_state & VM_MEM_REFERENCED) m->reference = TRUE; - if (refmod_state & VM_MEM_MODIFIED) - m->dirty = TRUE; + if (refmod_state & VM_MEM_MODIFIED) { + SET_PAGE_DIRTY(m, FALSE); + } } /* @@ -4956,8 +5356,15 @@ hibernate_consider_discard(vm_page_t m) || (VM_PURGABLE_VOLATILE == object->purgable) || (VM_PURGABLE_EMPTY == object->purgable); - if (discard == FALSE) - hibernate_stats.cd_found_dirty++; + + if (discard == FALSE) { + if (!preflight) + hibernate_stats.cd_found_dirty++; + } else if (m->xpmapped && m->reference) { + if (!preflight) + hibernate_stats.cd_found_xpmapped++; + discard = FALSE; + } } while (FALSE); @@ -4977,6 +5384,15 @@ hibernate_discard_page(vm_page_t m) */ return; +#if DEBUG + vm_object_t object = m->object; + if (!vm_object_lock_try(m->object)) + panic("hibernate_discard_page(%p) !vm_object_lock_try", m); +#else + /* No need to lock page queue for token delete, hibernate_vm_unlock() + makes sure these locks are uncontended before sleep */ +#endif /* !DEBUG */ + if (m->pmapped == TRUE) { __unused int refmod_state = pmap_disconnect(m->phys_page); @@ -4995,13 +5411,53 @@ hibernate_discard_page(vm_page_t m) assert((m->object->objq.next != NULL) && (m->object->objq.prev != NULL)); purgeable_q_t old_queue = vm_purgeable_object_remove(m->object); assert(old_queue); - /* No need to lock page queue for token delete, hibernate_vm_unlock() - makes sure these locks are uncontended before sleep */ - vm_purgeable_token_delete_first(old_queue); + if (m->object->purgeable_when_ripe) { + vm_purgeable_token_delete_first(old_queue); + } m->object->purgable = VM_PURGABLE_EMPTY; } vm_page_free(m); + +#if DEBUG + vm_object_unlock(object); +#endif /* DEBUG */ +} + +/* + Grab locks for hibernate_page_list_setall() +*/ +void +hibernate_vm_lock_queues(void) +{ + vm_object_lock(compressor_object); + vm_page_lock_queues(); + lck_mtx_lock(&vm_page_queue_free_lock); + + if (vm_page_local_q) { + uint32_t i; + for (i = 0; i < vm_page_local_q_count; i++) { + struct vpl *lq; + lq = &vm_page_local_q[i].vpl_un.vpl; + VPL_LOCK(&lq->vpl_lock); + } + } +} + +void +hibernate_vm_unlock_queues(void) +{ + if (vm_page_local_q) { + uint32_t i; + for (i = 0; i < vm_page_local_q_count; i++) { + struct vpl *lq; + lq = &vm_page_local_q[i].vpl_un.vpl; + VPL_UNLOCK(&lq->vpl_lock); + } + } + lck_mtx_unlock(&vm_page_queue_free_lock); + vm_page_unlock_queues(); + vm_object_unlock(compressor_object); } /* @@ -5014,53 +5470,93 @@ void hibernate_page_list_setall(hibernate_page_list_t * page_list, hibernate_page_list_t * page_list_wired, hibernate_page_list_t * page_list_pal, + boolean_t preflight, + boolean_t will_discard, uint32_t * pagesOut) { uint64_t start, end, nsec; vm_page_t m; + vm_page_t next; uint32_t pages = page_list->page_count; - uint32_t count_zf = 0, count_throttled = 0; - uint32_t count_inactive = 0, count_active = 0, count_speculative = 0; + uint32_t count_anonymous = 0, count_throttled = 0, count_compressor = 0; + uint32_t count_inactive = 0, count_active = 0, count_speculative = 0, count_cleaned = 0; uint32_t count_wire = pages; uint32_t count_discard_active = 0; uint32_t count_discard_inactive = 0; + uint32_t count_discard_cleaned = 0; uint32_t count_discard_purgeable = 0; uint32_t count_discard_speculative = 0; + uint32_t count_discard_vm_struct_pages = 0; uint32_t i; uint32_t bank; hibernate_bitmap_t * bitmap; hibernate_bitmap_t * bitmap_wired; + boolean_t discard_all; + boolean_t discard; + + HIBLOG("hibernate_page_list_setall(preflight %d) start %p, %p\n", preflight, page_list, page_list_wired); + + if (preflight) { + page_list = NULL; + page_list_wired = NULL; + page_list_pal = NULL; + discard_all = FALSE; + } else { + discard_all = will_discard; + } +#if DEBUG + if (!preflight) + { + vm_page_lock_queues(); + if (vm_page_local_q) { + for (i = 0; i < vm_page_local_q_count; i++) { + struct vpl *lq; + lq = &vm_page_local_q[i].vpl_un.vpl; + VPL_LOCK(&lq->vpl_lock); + } + } + } +#endif /* DEBUG */ - HIBLOG("hibernate_page_list_setall start %p, %p\n", page_list, page_list_wired); KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 8) | DBG_FUNC_START, count_wire, 0, 0, 0, 0); clock_get_uptime(&start); - hibernate_page_list_zero(page_list); - hibernate_page_list_zero(page_list_wired); - hibernate_page_list_zero(page_list_pal); - - hibernate_stats.cd_vm_page_wire_count = vm_page_wire_count; - hibernate_stats.cd_pages = pages; + if (!preflight) { + hibernate_page_list_zero(page_list); + hibernate_page_list_zero(page_list_wired); + hibernate_page_list_zero(page_list_pal); + + hibernate_stats.cd_vm_page_wire_count = vm_page_wire_count; + hibernate_stats.cd_pages = pages; + } if (vm_page_local_q) { for (i = 0; i < vm_page_local_q_count; i++) - vm_page_reactivate_local(i, TRUE, TRUE); + vm_page_reactivate_local(i, TRUE, !preflight); + } + + if (preflight) { + vm_object_lock(compressor_object); + vm_page_lock_queues(); + lck_mtx_lock(&vm_page_queue_free_lock); } m = (vm_page_t) hibernate_gobble_queue; - while(m) + while (m) { pages--; count_wire--; - hibernate_page_bitset(page_list, TRUE, m->phys_page); - hibernate_page_bitset(page_list_wired, TRUE, m->phys_page); + if (!preflight) { + hibernate_page_bitset(page_list, TRUE, m->phys_page); + hibernate_page_bitset(page_list_wired, TRUE, m->phys_page); + } m = (vm_page_t) m->pageq.next; } - for( i = 0; i < real_ncpus; i++ ) + if (!preflight) for( i = 0; i < real_ncpus; i++ ) { if (cpu_data_ptr[i] && cpu_data_ptr[i]->cpu_processor) { @@ -5086,10 +5582,12 @@ hibernate_page_list_setall(hibernate_page_list_t * page_list, { pages--; count_wire--; - hibernate_page_bitset(page_list, TRUE, m->phys_page); - hibernate_page_bitset(page_list_wired, TRUE, m->phys_page); - - hibernate_stats.cd_total_free++; + if (!preflight) { + hibernate_page_bitset(page_list, TRUE, m->phys_page); + hibernate_page_bitset(page_list_wired, TRUE, m->phys_page); + + hibernate_stats.cd_total_free++; + } } } @@ -5100,136 +5598,226 @@ hibernate_page_list_setall(hibernate_page_list_t * page_list, { pages--; count_wire--; - hibernate_page_bitset(page_list, TRUE, m->phys_page); - hibernate_page_bitset(page_list_wired, TRUE, m->phys_page); - - hibernate_stats.cd_total_free++; + if (!preflight) { + hibernate_page_bitset(page_list, TRUE, m->phys_page); + hibernate_page_bitset(page_list_wired, TRUE, m->phys_page); + + hibernate_stats.cd_total_free++; + } } - queue_iterate( &vm_page_queue_throttled, - m, - vm_page_t, - pageq ) + m = (vm_page_t) queue_first(&vm_page_queue_throttled); + while (m && !queue_end(&vm_page_queue_throttled, (queue_entry_t)m)) { + next = (vm_page_t) m->pageq.next; + discard = FALSE; if ((kIOHibernateModeDiscardCleanInactive & gIOHibernateMode) - && hibernate_consider_discard(m)) + && hibernate_consider_discard(m, preflight)) { - hibernate_page_bitset(page_list, TRUE, m->phys_page); + if (!preflight) hibernate_page_bitset(page_list, TRUE, m->phys_page); count_discard_inactive++; + discard = discard_all; } else count_throttled++; count_wire--; - hibernate_page_bitset(page_list_wired, TRUE, m->phys_page); + if (!preflight) hibernate_page_bitset(page_list_wired, TRUE, m->phys_page); + + if (discard) hibernate_discard_page(m); + m = next; } - queue_iterate( &vm_page_queue_zf, - m, - vm_page_t, - pageq ) + m = (vm_page_t) queue_first(&vm_page_queue_anonymous); + while (m && !queue_end(&vm_page_queue_anonymous, (queue_entry_t)m)) { + next = (vm_page_t) m->pageq.next; + discard = FALSE; if ((kIOHibernateModeDiscardCleanInactive & gIOHibernateMode) - && hibernate_consider_discard(m)) + && hibernate_consider_discard(m, preflight)) { - hibernate_page_bitset(page_list, TRUE, m->phys_page); + if (!preflight) hibernate_page_bitset(page_list, TRUE, m->phys_page); if (m->dirty) count_discard_purgeable++; else count_discard_inactive++; + discard = discard_all; } else - count_zf++; + count_anonymous++; count_wire--; - hibernate_page_bitset(page_list_wired, TRUE, m->phys_page); + if (!preflight) hibernate_page_bitset(page_list_wired, TRUE, m->phys_page); + if (discard) hibernate_discard_page(m); + m = next; } - queue_iterate( &vm_page_queue_inactive, - m, - vm_page_t, - pageq ) + m = (vm_page_t) queue_first(&vm_page_queue_inactive); + while (m && !queue_end(&vm_page_queue_inactive, (queue_entry_t)m)) { + next = (vm_page_t) m->pageq.next; + discard = FALSE; if ((kIOHibernateModeDiscardCleanInactive & gIOHibernateMode) - && hibernate_consider_discard(m)) + && hibernate_consider_discard(m, preflight)) { - hibernate_page_bitset(page_list, TRUE, m->phys_page); + if (!preflight) hibernate_page_bitset(page_list, TRUE, m->phys_page); if (m->dirty) count_discard_purgeable++; else count_discard_inactive++; + discard = discard_all; } else count_inactive++; count_wire--; - hibernate_page_bitset(page_list_wired, TRUE, m->phys_page); + if (!preflight) hibernate_page_bitset(page_list_wired, TRUE, m->phys_page); + if (discard) hibernate_discard_page(m); + m = next; + } + + m = (vm_page_t) queue_first(&vm_page_queue_cleaned); + while (m && !queue_end(&vm_page_queue_cleaned, (queue_entry_t)m)) + { + next = (vm_page_t) m->pageq.next; + discard = FALSE; + if ((kIOHibernateModeDiscardCleanInactive & gIOHibernateMode) + && hibernate_consider_discard(m, preflight)) + { + if (!preflight) hibernate_page_bitset(page_list, TRUE, m->phys_page); + if (m->dirty) + count_discard_purgeable++; + else + count_discard_cleaned++; + discard = discard_all; + } + else + count_cleaned++; + count_wire--; + if (!preflight) hibernate_page_bitset(page_list_wired, TRUE, m->phys_page); + if (discard) hibernate_discard_page(m); + m = next; } for( i = 0; i <= VM_PAGE_MAX_SPECULATIVE_AGE_Q; i++ ) { - queue_iterate(&vm_page_queue_speculative[i].age_q, - m, - vm_page_t, - pageq) - { - if ((kIOHibernateModeDiscardCleanInactive & gIOHibernateMode) - && hibernate_consider_discard(m)) - { - hibernate_page_bitset(page_list, TRUE, m->phys_page); - count_discard_speculative++; - } - else - count_speculative++; - count_wire--; - hibernate_page_bitset(page_list_wired, TRUE, m->phys_page); - } + m = (vm_page_t) queue_first(&vm_page_queue_speculative[i].age_q); + while (m && !queue_end(&vm_page_queue_speculative[i].age_q, (queue_entry_t)m)) + { + next = (vm_page_t) m->pageq.next; + discard = FALSE; + if ((kIOHibernateModeDiscardCleanInactive & gIOHibernateMode) + && hibernate_consider_discard(m, preflight)) + { + if (!preflight) hibernate_page_bitset(page_list, TRUE, m->phys_page); + count_discard_speculative++; + discard = discard_all; + } + else + count_speculative++; + count_wire--; + if (!preflight) hibernate_page_bitset(page_list_wired, TRUE, m->phys_page); + if (discard) hibernate_discard_page(m); + m = next; + } } - queue_iterate( &vm_page_queue_active, - m, - vm_page_t, - pageq ) + m = (vm_page_t) queue_first(&vm_page_queue_active); + while (m && !queue_end(&vm_page_queue_active, (queue_entry_t)m)) { + next = (vm_page_t) m->pageq.next; + discard = FALSE; if ((kIOHibernateModeDiscardCleanActive & gIOHibernateMode) - && hibernate_consider_discard(m)) + && hibernate_consider_discard(m, preflight)) { - hibernate_page_bitset(page_list, TRUE, m->phys_page); + if (!preflight) hibernate_page_bitset(page_list, TRUE, m->phys_page); if (m->dirty) count_discard_purgeable++; else count_discard_active++; + discard = discard_all; } else count_active++; count_wire--; - hibernate_page_bitset(page_list_wired, TRUE, m->phys_page); + if (!preflight) hibernate_page_bitset(page_list_wired, TRUE, m->phys_page); + if (discard) hibernate_discard_page(m); + m = next; } - // pull wired from hibernate_bitmap - - bitmap = &page_list->bank_bitmap[0]; - bitmap_wired = &page_list_wired->bank_bitmap[0]; - for (bank = 0; bank < page_list->bank_count; bank++) + queue_iterate(&compressor_object->memq, m, vm_page_t, listq) { - for (i = 0; i < bitmap->bitmapwords; i++) - bitmap->bitmap[i] = bitmap->bitmap[i] | ~bitmap_wired->bitmap[i]; - bitmap = (hibernate_bitmap_t *) &bitmap->bitmap [bitmap->bitmapwords]; - bitmap_wired = (hibernate_bitmap_t *) &bitmap_wired->bitmap[bitmap_wired->bitmapwords]; + count_compressor++; + count_wire--; + if (!preflight) hibernate_page_bitset(page_list_wired, TRUE, m->phys_page); + } + + if (preflight == FALSE && discard_all == TRUE) { + KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 12) | DBG_FUNC_START, 0, 0, 0, 0, 0); + + HIBLOG("hibernate_teardown started\n"); + count_discard_vm_struct_pages = hibernate_teardown_vm_structs(page_list, page_list_wired); + HIBLOG("hibernate_teardown completed - discarded %d\n", count_discard_vm_struct_pages); + + pages -= count_discard_vm_struct_pages; + count_wire -= count_discard_vm_struct_pages; + + hibernate_stats.cd_vm_struct_pages_unneeded = count_discard_vm_struct_pages; + + KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 13) | DBG_FUNC_END, 0, 0, 0, 0, 0); + } + + if (!preflight) { + // pull wired from hibernate_bitmap + bitmap = &page_list->bank_bitmap[0]; + bitmap_wired = &page_list_wired->bank_bitmap[0]; + for (bank = 0; bank < page_list->bank_count; bank++) + { + for (i = 0; i < bitmap->bitmapwords; i++) + bitmap->bitmap[i] = bitmap->bitmap[i] | ~bitmap_wired->bitmap[i]; + bitmap = (hibernate_bitmap_t *) &bitmap->bitmap [bitmap->bitmapwords]; + bitmap_wired = (hibernate_bitmap_t *) &bitmap_wired->bitmap[bitmap_wired->bitmapwords]; + } } // machine dependent adjustments - hibernate_page_list_setall_machine(page_list, page_list_wired, &pages); + hibernate_page_list_setall_machine(page_list, page_list_wired, preflight, &pages); - hibernate_stats.cd_count_wire = count_wire; - hibernate_stats.cd_discarded = count_discard_active + count_discard_inactive + count_discard_purgeable + count_discard_speculative; + if (!preflight) { + hibernate_stats.cd_count_wire = count_wire; + hibernate_stats.cd_discarded = count_discard_active + count_discard_inactive + count_discard_purgeable + + count_discard_speculative + count_discard_cleaned + count_discard_vm_struct_pages; + } clock_get_uptime(&end); absolutetime_to_nanoseconds(end - start, &nsec); HIBLOG("hibernate_page_list_setall time: %qd ms\n", nsec / 1000000ULL); - HIBLOG("pages %d, wire %d, act %d, inact %d, spec %d, zf %d, throt %d, could discard act %d inact %d purgeable %d spec %d\n", - pages, count_wire, count_active, count_inactive, count_speculative, count_zf, count_throttled, - count_discard_active, count_discard_inactive, count_discard_purgeable, count_discard_speculative); + HIBLOG("pages %d, wire %d, act %d, inact %d, cleaned %d spec %d, zf %d, throt %d, compr %d, xpmapped %d\n %s discard act %d inact %d purgeable %d spec %d cleaned %d\n", + pages, count_wire, count_active, count_inactive, count_cleaned, count_speculative, count_anonymous, count_throttled, count_compressor, hibernate_stats.cd_found_xpmapped, + discard_all ? "did" : "could", + count_discard_active, count_discard_inactive, count_discard_purgeable, count_discard_speculative, count_discard_cleaned); - *pagesOut = pages - count_discard_active - count_discard_inactive - count_discard_purgeable - count_discard_speculative; + *pagesOut = pages - count_discard_active - count_discard_inactive - count_discard_purgeable - count_discard_speculative - count_discard_cleaned; + + if (preflight && will_discard) *pagesOut -= count_compressor + count_throttled + count_anonymous + count_inactive + count_cleaned + count_speculative + count_active; + +#if DEBUG + if (!preflight) + { + if (vm_page_local_q) { + for (i = 0; i < vm_page_local_q_count; i++) { + struct vpl *lq; + lq = &vm_page_local_q[i].vpl_un.vpl; + VPL_UNLOCK(&lq->vpl_lock); + } + } + vm_page_unlock_queues(); + } +#endif /* DEBUG */ + + if (preflight) { + lck_mtx_unlock(&vm_page_queue_free_lock); + vm_page_unlock_queues(); + vm_object_unlock(compressor_object); + } KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 8) | DBG_FUNC_END, count_wire, *pagesOut, 0, 0, 0); } @@ -5244,12 +5832,25 @@ hibernate_page_list_discard(hibernate_page_list_t * page_list) uint32_t count_discard_active = 0; uint32_t count_discard_inactive = 0; uint32_t count_discard_purgeable = 0; + uint32_t count_discard_cleaned = 0; uint32_t count_discard_speculative = 0; + +#if DEBUG + vm_page_lock_queues(); + if (vm_page_local_q) { + for (i = 0; i < vm_page_local_q_count; i++) { + struct vpl *lq; + lq = &vm_page_local_q[i].vpl_un.vpl; + VPL_LOCK(&lq->vpl_lock); + } + } +#endif /* DEBUG */ + clock_get_uptime(&start); - m = (vm_page_t) queue_first(&vm_page_queue_zf); - while (m && !queue_end(&vm_page_queue_zf, (queue_entry_t)m)) + m = (vm_page_t) queue_first(&vm_page_queue_anonymous); + while (m && !queue_end(&vm_page_queue_anonymous, (queue_entry_t)m)) { next = (vm_page_t) m->pageq.next; if (hibernate_page_bittst(page_list, m->phys_page)) @@ -5308,13 +5909,388 @@ hibernate_page_list_discard(hibernate_page_list_t * page_list) m = next; } + m = (vm_page_t) queue_first(&vm_page_queue_cleaned); + while (m && !queue_end(&vm_page_queue_cleaned, (queue_entry_t)m)) + { + next = (vm_page_t) m->pageq.next; + if (hibernate_page_bittst(page_list, m->phys_page)) + { + if (m->dirty) + count_discard_purgeable++; + else + count_discard_cleaned++; + hibernate_discard_page(m); + } + m = next; + } + +#if DEBUG + if (vm_page_local_q) { + for (i = 0; i < vm_page_local_q_count; i++) { + struct vpl *lq; + lq = &vm_page_local_q[i].vpl_un.vpl; + VPL_UNLOCK(&lq->vpl_lock); + } + } + vm_page_unlock_queues(); +#endif /* DEBUG */ + clock_get_uptime(&end); absolutetime_to_nanoseconds(end - start, &nsec); - HIBLOG("hibernate_page_list_discard time: %qd ms, discarded act %d inact %d purgeable %d spec %d\n", + HIBLOG("hibernate_page_list_discard time: %qd ms, discarded act %d inact %d purgeable %d spec %d cleaned %d\n", nsec / 1000000ULL, - count_discard_active, count_discard_inactive, count_discard_purgeable, count_discard_speculative); + count_discard_active, count_discard_inactive, count_discard_purgeable, count_discard_speculative, count_discard_cleaned); } +boolean_t hibernate_paddr_map_inited = FALSE; +boolean_t hibernate_rebuild_needed = FALSE; +unsigned int hibernate_teardown_last_valid_compact_indx = -1; +vm_page_t hibernate_rebuild_hash_list = NULL; + +unsigned int hibernate_teardown_found_tabled_pages = 0; +unsigned int hibernate_teardown_found_created_pages = 0; +unsigned int hibernate_teardown_found_free_pages = 0; +unsigned int hibernate_teardown_vm_page_free_count; + + +struct ppnum_mapping { + struct ppnum_mapping *ppnm_next; + ppnum_t ppnm_base_paddr; + unsigned int ppnm_sindx; + unsigned int ppnm_eindx; +}; + +struct ppnum_mapping *ppnm_head; +struct ppnum_mapping *ppnm_last_found = NULL; + + +void +hibernate_create_paddr_map() +{ + unsigned int i; + ppnum_t next_ppnum_in_run = 0; + struct ppnum_mapping *ppnm = NULL; + + if (hibernate_paddr_map_inited == FALSE) { + + for (i = 0; i < vm_pages_count; i++) { + + if (ppnm) + ppnm->ppnm_eindx = i; + + if (ppnm == NULL || vm_pages[i].phys_page != next_ppnum_in_run) { + + ppnm = kalloc(sizeof(struct ppnum_mapping)); + + ppnm->ppnm_next = ppnm_head; + ppnm_head = ppnm; + + ppnm->ppnm_sindx = i; + ppnm->ppnm_base_paddr = vm_pages[i].phys_page; + } + next_ppnum_in_run = vm_pages[i].phys_page + 1; + } + ppnm->ppnm_eindx++; + + hibernate_paddr_map_inited = TRUE; + } +} + +ppnum_t +hibernate_lookup_paddr(unsigned int indx) +{ + struct ppnum_mapping *ppnm = NULL; + + ppnm = ppnm_last_found; + + if (ppnm) { + if (indx >= ppnm->ppnm_sindx && indx < ppnm->ppnm_eindx) + goto done; + } + for (ppnm = ppnm_head; ppnm; ppnm = ppnm->ppnm_next) { + + if (indx >= ppnm->ppnm_sindx && indx < ppnm->ppnm_eindx) { + ppnm_last_found = ppnm; + break; + } + } + if (ppnm == NULL) + panic("hibernate_lookup_paddr of %d failed\n", indx); +done: + return (ppnm->ppnm_base_paddr + (indx - ppnm->ppnm_sindx)); +} + + +uint32_t +hibernate_mark_as_unneeded(addr64_t saddr, addr64_t eaddr, hibernate_page_list_t *page_list, hibernate_page_list_t *page_list_wired) +{ + addr64_t saddr_aligned; + addr64_t eaddr_aligned; + addr64_t addr; + ppnum_t paddr; + unsigned int mark_as_unneeded_pages = 0; + + saddr_aligned = (saddr + PAGE_MASK_64) & ~PAGE_MASK_64; + eaddr_aligned = eaddr & ~PAGE_MASK_64; + + for (addr = saddr_aligned; addr < eaddr_aligned; addr += PAGE_SIZE_64) { + + paddr = pmap_find_phys(kernel_pmap, addr); + + assert(paddr); + + hibernate_page_bitset(page_list, TRUE, paddr); + hibernate_page_bitset(page_list_wired, TRUE, paddr); + + mark_as_unneeded_pages++; + } + return (mark_as_unneeded_pages); +} + + +void +hibernate_hash_insert_page(vm_page_t mem) +{ + vm_page_bucket_t *bucket; + int hash_id; + + assert(mem->hashed); + assert(mem->object); + assert(mem->offset != (vm_object_offset_t) -1); + + /* + * Insert it into the object_object/offset hash table + */ + hash_id = vm_page_hash(mem->object, mem->offset); + bucket = &vm_page_buckets[hash_id]; + + mem->next = bucket->pages; + bucket->pages = mem; +} + + +void +hibernate_free_range(int sindx, int eindx) +{ + vm_page_t mem; + unsigned int color; + + while (sindx < eindx) { + mem = &vm_pages[sindx]; + + vm_page_init(mem, hibernate_lookup_paddr(sindx), FALSE); + + mem->lopage = FALSE; + mem->free = TRUE; + + color = mem->phys_page & vm_color_mask; + queue_enter_first(&vm_page_queue_free[color], + mem, + vm_page_t, + pageq); + vm_page_free_count++; + + sindx++; + } +} + + +extern void hibernate_rebuild_pmap_structs(void); + +void +hibernate_rebuild_vm_structs(void) +{ + int cindx, sindx, eindx; + vm_page_t mem, tmem, mem_next; + AbsoluteTime startTime, endTime; + uint64_t nsec; + + if (hibernate_rebuild_needed == FALSE) + return; + + KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 13) | DBG_FUNC_START, 0, 0, 0, 0, 0); + HIBLOG("hibernate_rebuild started\n"); + + clock_get_uptime(&startTime); + + hibernate_rebuild_pmap_structs(); + + bzero(&vm_page_buckets[0], vm_page_bucket_count * sizeof(vm_page_bucket_t)); + eindx = vm_pages_count; + + for (cindx = hibernate_teardown_last_valid_compact_indx; cindx >= 0; cindx--) { + + mem = &vm_pages[cindx]; + /* + * hibernate_teardown_vm_structs leaves the location where + * this vm_page_t must be located in "next". + */ + tmem = mem->next; + mem->next = NULL; + + sindx = (int)(tmem - &vm_pages[0]); + + if (mem != tmem) { + /* + * this vm_page_t was moved by hibernate_teardown_vm_structs, + * so move it back to its real location + */ + *tmem = *mem; + mem = tmem; + } + if (mem->hashed) + hibernate_hash_insert_page(mem); + /* + * the 'hole' between this vm_page_t and the previous + * vm_page_t we moved needs to be initialized as + * a range of free vm_page_t's + */ + hibernate_free_range(sindx + 1, eindx); + + eindx = sindx; + } + if (sindx) + hibernate_free_range(0, sindx); + + assert(vm_page_free_count == hibernate_teardown_vm_page_free_count); + + /* + * process the list of vm_page_t's that were entered in the hash, + * but were not located in the vm_pages arrary... these are + * vm_page_t's that were created on the fly (i.e. fictitious) + */ + for (mem = hibernate_rebuild_hash_list; mem; mem = mem_next) { + mem_next = mem->next; + + mem->next = NULL; + hibernate_hash_insert_page(mem); + } + hibernate_rebuild_hash_list = NULL; + + clock_get_uptime(&endTime); + SUB_ABSOLUTETIME(&endTime, &startTime); + absolutetime_to_nanoseconds(endTime, &nsec); + + HIBLOG("hibernate_rebuild completed - took %qd msecs\n", nsec / 1000000ULL); + + hibernate_rebuild_needed = FALSE; + + KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 13) | DBG_FUNC_END, 0, 0, 0, 0, 0); +} + + +extern void hibernate_teardown_pmap_structs(addr64_t *, addr64_t *); + +uint32_t +hibernate_teardown_vm_structs(hibernate_page_list_t *page_list, hibernate_page_list_t *page_list_wired) +{ + unsigned int i; + unsigned int compact_target_indx; + vm_page_t mem, mem_next; + vm_page_bucket_t *bucket; + unsigned int mark_as_unneeded_pages = 0; + unsigned int unneeded_vm_page_bucket_pages = 0; + unsigned int unneeded_vm_pages_pages = 0; + unsigned int unneeded_pmap_pages = 0; + addr64_t start_of_unneeded = 0; + addr64_t end_of_unneeded = 0; + + + if (hibernate_should_abort()) + return (0); + + HIBLOG("hibernate_teardown: wired_pages %d, free_pages %d, active_pages %d, inactive_pages %d, speculative_pages %d, cleaned_pages %d, compressor_pages %d\n", + vm_page_wire_count, vm_page_free_count, vm_page_active_count, vm_page_inactive_count, vm_page_speculative_count, + vm_page_cleaned_count, compressor_object->resident_page_count); + + for (i = 0; i < vm_page_bucket_count; i++) { + + bucket = &vm_page_buckets[i]; + + for (mem = bucket->pages; mem != VM_PAGE_NULL; mem = mem_next) { + assert(mem->hashed); + + mem_next = mem->next; + + if (mem < &vm_pages[0] || mem >= &vm_pages[vm_pages_count]) { + mem->next = hibernate_rebuild_hash_list; + hibernate_rebuild_hash_list = mem; + } + } + } + unneeded_vm_page_bucket_pages = hibernate_mark_as_unneeded((addr64_t)&vm_page_buckets[0], (addr64_t)&vm_page_buckets[vm_page_bucket_count], page_list, page_list_wired); + mark_as_unneeded_pages += unneeded_vm_page_bucket_pages; + + hibernate_teardown_vm_page_free_count = vm_page_free_count; + + compact_target_indx = 0; + + for (i = 0; i < vm_pages_count; i++) { + + mem = &vm_pages[i]; + + if (mem->free) { + unsigned int color; + + assert(mem->busy); + assert(!mem->lopage); + + color = mem->phys_page & vm_color_mask; + + queue_remove(&vm_page_queue_free[color], + mem, + vm_page_t, + pageq); + mem->pageq.next = NULL; + mem->pageq.prev = NULL; + + vm_page_free_count--; + + hibernate_teardown_found_free_pages++; + + if ( !vm_pages[compact_target_indx].free) + compact_target_indx = i; + } else { + /* + * record this vm_page_t's original location + * we need this even if it doesn't get moved + * as an indicator to the rebuild function that + * we don't have to move it + */ + mem->next = mem; + + if (vm_pages[compact_target_indx].free) { + /* + * we've got a hole to fill, so + * move this vm_page_t to it's new home + */ + vm_pages[compact_target_indx] = *mem; + mem->free = TRUE; + + hibernate_teardown_last_valid_compact_indx = compact_target_indx; + compact_target_indx++; + } else + hibernate_teardown_last_valid_compact_indx = i; + } + } + unneeded_vm_pages_pages = hibernate_mark_as_unneeded((addr64_t)&vm_pages[hibernate_teardown_last_valid_compact_indx+1], + (addr64_t)&vm_pages[vm_pages_count-1], page_list, page_list_wired); + mark_as_unneeded_pages += unneeded_vm_pages_pages; + + hibernate_teardown_pmap_structs(&start_of_unneeded, &end_of_unneeded); + + if (start_of_unneeded) { + unneeded_pmap_pages = hibernate_mark_as_unneeded(start_of_unneeded, end_of_unneeded, page_list, page_list_wired); + mark_as_unneeded_pages += unneeded_pmap_pages; + } + HIBLOG("hibernate_teardown: mark_as_unneeded_pages %d, %d, %d\n", unneeded_vm_page_bucket_pages, unneeded_vm_pages_pages, unneeded_pmap_pages); + + hibernate_rebuild_needed = TRUE; + + return (mark_as_unneeded_pages); +} + + #endif /* HIBERNATION */ /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ @@ -5368,63 +6344,75 @@ vm_page_info( } #endif /* MACH_VM_DEBUG */ -#include -#if MACH_KDB - -#include -#include -#define printf kdbprintf - -/* - * Routine: vm_page_print [exported] - */ +#if VM_PAGE_BUCKETS_CHECK void -vm_page_print( - db_addr_t db_addr) +vm_page_buckets_check(void) { - vm_page_t p; + unsigned int i; + vm_page_t p; + unsigned int p_hash; + vm_page_bucket_t *bucket; + lck_spin_t *bucket_lock; - p = (vm_page_t) (long) db_addr; + if (!vm_page_buckets_check_ready) { + return; + } - iprintf("page 0x%x\n", p); +#if HIBERNATION + if (hibernate_rebuild_needed || + hibernate_rebuild_hash_list) { + panic("BUCKET_CHECK: hibernation in progress: " + "rebuild_needed=%d rebuild_hash_list=%p\n", + hibernate_rebuild_needed, + hibernate_rebuild_hash_list); + } +#endif /* HIBERNATION */ + +#if VM_PAGE_FAKE_BUCKETS + char *cp; + for (cp = (char *) vm_page_fake_buckets_start; + cp < (char *) vm_page_fake_buckets_end; + cp++) { + if (*cp != 0x5a) { + panic("BUCKET_CHECK: corruption at %p in fake buckets " + "[0x%llx:0x%llx]\n", + cp, + vm_page_fake_buckets_start, + vm_page_fake_buckets_end); + } + } +#endif /* VM_PAGE_FAKE_BUCKETS */ - db_indent += 2; + for (i = 0; i < vm_page_bucket_count; i++) { + bucket = &vm_page_buckets[i]; + if (bucket->pages == VM_PAGE_NULL) { + continue; + } - iprintf("object=0x%x", p->object); - printf(", offset=0x%x", p->offset); - printf(", wire_count=%d", p->wire_count); + bucket_lock = &vm_page_bucket_locks[i / BUCKETS_PER_LOCK]; + lck_spin_lock(bucket_lock); + p = bucket->pages; + while (p != VM_PAGE_NULL) { + if (!p->hashed) { + panic("BUCKET_CHECK: page %p (%p,0x%llx) " + "hash %d in bucket %d at %p " + "is not hashed\n", + p, p->object, p->offset, + p_hash, i, bucket); + } + p_hash = vm_page_hash(p->object, p->offset); + if (p_hash != i) { + panic("BUCKET_CHECK: corruption in bucket %d " + "at %p: page %p object %p offset 0x%llx " + "hash %d\n", + i, bucket, p, p->object, p->offset, + p_hash); + } + p = p->next; + } + lck_spin_unlock(bucket_lock); + } - iprintf("%slocal, %sinactive, %sactive, %sthrottled, %sgobbled, %slaundry, %sfree, %sref, %sencrypted\n", - (p->local ? "" : "!"), - (p->inactive ? "" : "!"), - (p->active ? "" : "!"), - (p->throttled ? "" : "!"), - (p->gobbled ? "" : "!"), - (p->laundry ? "" : "!"), - (p->free ? "" : "!"), - (p->reference ? "" : "!"), - (p->encrypted ? "" : "!")); - iprintf("%sbusy, %swanted, %stabled, %sfictitious, %sprivate, %sprecious\n", - (p->busy ? "" : "!"), - (p->wanted ? "" : "!"), - (p->tabled ? "" : "!"), - (p->fictitious ? "" : "!"), - (p->private ? "" : "!"), - (p->precious ? "" : "!")); - iprintf("%sabsent, %serror, %sdirty, %scleaning, %spageout, %sclustered\n", - (p->absent ? "" : "!"), - (p->error ? "" : "!"), - (p->dirty ? "" : "!"), - (p->cleaning ? "" : "!"), - (p->pageout ? "" : "!"), - (p->clustered ? "" : "!")); - iprintf("%soverwriting, %srestart, %sunusual\n", - (p->overwriting ? "" : "!"), - (p->restart ? "" : "!"), - (p->unusual ? "" : "!")); - - iprintf("phys_page=0x%x", p->phys_page); - - db_indent -= 2; +// printf("BUCKET_CHECK: checked buckets\n"); } -#endif /* MACH_KDB */ +#endif /* VM_PAGE_BUCKETS_CHECK */