X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/6d2010ae8f7a6078e10b361c6962983bab233e0f..15129b1c8dbb3650c63b70adb1cad9af601c6c17:/osfmk/vm/vm_pageout.h diff --git a/osfmk/vm/vm_pageout.h b/osfmk/vm/vm_pageout.h index d8ddac6a7..7396460b2 100644 --- a/osfmk/vm/vm_pageout.h +++ b/osfmk/vm/vm_pageout.h @@ -87,29 +87,56 @@ #include +#define VM_PAGE_AVAILABLE_COUNT() ((unsigned int)(vm_page_cleaned_count)) + +/* externally manipulated counters */ +extern unsigned int vm_pageout_cleaned_reactivated, vm_pageout_cleaned_fault_reactivated, vm_pageout_cleaned_commit_reactivated; + +#if CONFIG_JETSAM +#define LATENCY_JETSAM FALSE +#if LATENCY_JETSAM +#define JETSAM_LATENCY_TOKEN_AGE 3000 /* 3ms */ +#define NUM_OF_JETSAM_LATENCY_TOKENS 1000 + +#define JETSAM_AGE_NOTIFY_CRITICAL 1500000 /* 1.5 secs */ + +extern boolean_t jlp_init; +extern uint64_t jlp_time, jlp_current; +extern unsigned int latency_jetsam_wakeup; +#endif /* LATENCY_JETSAM */ +#endif /* CONFIG_JETSAM */ + #if CONFIG_FREEZE -extern boolean_t vm_freeze_enabled; -#define VM_DYNAMIC_PAGING_ENABLED(port) ((vm_freeze_enabled == FALSE) && IP_VALID(port)) +extern boolean_t memorystatus_freeze_enabled; +#define VM_DYNAMIC_PAGING_ENABLED(port) ((COMPRESSED_PAGER_IS_ACTIVE || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_ACTIVE) || (memorystatus_freeze_enabled == FALSE && IP_VALID(port))) #else -#define VM_DYNAMIC_PAGING_ENABLED(port) IP_VALID(port) +#define VM_DYNAMIC_PAGING_ENABLED(port) (COMPRESSED_PAGER_IS_ACTIVE || IP_VALID(port)) #endif extern int vm_debug_events; -#define VMF_CHECK_ZFDELAY 0x100 -#define VMF_COWDELAY 0x101 -#define VMF_ZFDELAY 0x102 +#define VMF_CHECK_ZFDELAY 0x100 +#define VMF_COWDELAY 0x101 +#define VMF_ZFDELAY 0x102 +#define VMF_COMPRESSORDELAY 0x103 + +#define VM_PAGEOUT_SCAN 0x104 +#define VM_PAGEOUT_BALANCE 0x105 +#define VM_PAGEOUT_FREELIST 0x106 +#define VM_PAGEOUT_PURGEONE 0x107 +#define VM_PAGEOUT_CACHE_EVICT 0x108 +#define VM_PAGEOUT_THREAD_BLOCK 0x109 +#define VM_PAGEOUT_JETSAM 0x10A +#define VM_PAGEOUT_PAGE_TOKEN 0x10B -#define VM_PAGEOUT_SCAN 0x104 -#define VM_PAGEOUT_BALANCE 0x105 -#define VM_PAGEOUT_FREELIST 0x106 -#define VM_PAGEOUT_PURGEONE 0x107 -#define VM_PAGEOUT_CACHE_EVICT 0x108 -#define VM_PAGEOUT_THREAD_BLOCK 0x109 +#define VM_UPL_PAGE_WAIT 0x120 +#define VM_IOPL_PAGE_WAIT 0x121 +#define VM_PAGE_WAIT_BLOCK 0x122 -#define VM_UPL_PAGE_WAIT 0x120 -#define VM_IOPL_PAGE_WAIT 0x121 +#define VM_PRESSURE_EVENT 0x130 +#define VM_EXECVE 0x131 +#define VM_WAKEUP_COMPACTOR_SWAPPER 0x132 #define VM_DEBUG_EVENT(name, event, control, arg1, arg2, arg3, arg4) \ MACRO_BEGIN \ @@ -118,7 +145,7 @@ extern int vm_debug_events; } \ MACRO_END - +extern void inline memoryshot(unsigned int event, unsigned int control); extern kern_return_t vm_map_create_upl( vm_map_t map, @@ -154,25 +181,15 @@ extern vm_object_offset_t vm_page_get_offset(vm_page_t page); extern ppnum_t vm_page_get_phys_page(vm_page_t page); extern vm_page_t vm_page_get_next(vm_page_t page); +extern kern_return_t mach_vm_pressure_level_monitor(boolean_t wait_for_pressure, unsigned int *pressure_level); + #ifdef MACH_KERNEL_PRIVATE #include extern unsigned int vm_pageout_scan_event_counter; -extern unsigned int vm_zf_queue_count; - - -extern uint64_t vm_zf_count; - -#define VM_ZF_COUNT_INCR() \ - MACRO_BEGIN \ - OSAddAtomic64(1, (SInt64 *) &vm_zf_count); \ - MACRO_END \ +extern unsigned int vm_page_anonymous_count; -#define VM_ZF_COUNT_DECR() \ - MACRO_BEGIN \ - OSAddAtomic64(-1, (SInt64 *) &vm_zf_count); \ - MACRO_END \ /* * must hold the page queues lock to @@ -182,11 +199,14 @@ struct vm_pageout_queue { queue_head_t pgo_pending; /* laundry pages to be processed by pager's iothread */ unsigned int pgo_laundry; /* current count of laundry pages on queue or in flight */ unsigned int pgo_maxlaundry; + uint64_t pgo_tid; /* thread ID of I/O thread that services this queue */ + uint8_t pgo_lowpriority; /* iothread is set to use low priority I/O */ unsigned int pgo_idle:1, /* iothread is blocked waiting for work to do */ pgo_busy:1, /* iothread is currently processing request from pgo_pending */ pgo_throttled:1,/* vm_pageout_scan thread needs a wakeup when pgo_laundry drops */ pgo_draining:1, + pgo_inited:1, :0; }; @@ -208,7 +228,8 @@ extern void vm_pageout_object_terminate( vm_object_t object); extern void vm_pageout_cluster( - vm_page_t m); + vm_page_t m, + boolean_t pageout); extern void vm_pageout_initialize_page( vm_page_t m); @@ -328,6 +349,12 @@ extern void vector_upl_get_iostate_byindex(upl_t, uint32_t, upl_offset_t*, upl_s extern upl_t vector_upl_subupl_byindex(upl_t , uint32_t); extern upl_t vector_upl_subupl_byoffset(upl_t , upl_offset_t*, upl_size_t*); +extern void vm_object_set_pmap_cache_attr( + vm_object_t object, + upl_page_info_array_t user_page_list, + unsigned int num_pages, + boolean_t batch_pmap_op); + extern kern_return_t vm_object_iopl_request( vm_object_t object, vm_object_offset_t offset, @@ -381,13 +408,14 @@ extern void vm_page_decrypt( vm_page_t page, vm_map_offset_t kernel_map_offset); extern kern_return_t vm_paging_map_object( - vm_map_offset_t *address, vm_page_t page, vm_object_t object, vm_object_offset_t offset, - vm_map_size_t *size, vm_prot_t protection, - boolean_t can_unlock_object); + boolean_t can_unlock_object, + vm_map_size_t *size, /* IN/OUT */ + vm_map_offset_t *address, /* OUT */ + boolean_t *need_unmap); /* OUT */ extern void vm_paging_unmap_object( vm_object_t object, vm_map_offset_t start, @@ -399,7 +427,7 @@ decl_simple_lock_data(extern, vm_paging_lock) */ extern unsigned int vm_backing_store_low; -extern void vm_pageout_queue_steal( +extern void vm_pageout_steal_laundry( vm_page_t page, boolean_t queues_locked); @@ -455,10 +483,33 @@ struct vm_page_stats_reusable { uint64_t reuse_pages_failure; uint64_t can_reuse_success; uint64_t can_reuse_failure; + uint64_t reusable_reclaimed; }; extern struct vm_page_stats_reusable vm_page_stats_reusable; extern int hibernate_flush_memory(void); +extern void hibernate_create_paddr_map(void); + +extern int vm_compressor_mode; +extern int vm_compressor_thread_count; + +#define VM_PAGER_DEFAULT 0x1 /* Use default pager. */ +#define VM_PAGER_COMPRESSOR_NO_SWAP 0x2 /* In-core compressor only. */ +#define VM_PAGER_COMPRESSOR_WITH_SWAP 0x4 /* In-core compressor + swap backend. */ +#define VM_PAGER_FREEZER_DEFAULT 0x8 /* Freezer backed by default pager.*/ +#define VM_PAGER_FREEZER_COMPRESSOR_NO_SWAP 0x10 /* Freezer backed by in-core compressor only i.e. frozen data remain in-core compressed.*/ +#define VM_PAGER_FREEZER_COMPRESSOR_WITH_SWAP 0x20 /* Freezer backed by in-core compressor with swap support too.*/ + +#define VM_PAGER_MAX_MODES 6 /* Total number of vm compressor modes supported */ + +#define DEFAULT_PAGER_IS_ACTIVE ((vm_compressor_mode & VM_PAGER_DEFAULT) == VM_PAGER_DEFAULT) + +#define COMPRESSED_PAGER_IS_ACTIVE (vm_compressor_mode & (VM_PAGER_COMPRESSOR_NO_SWAP | VM_PAGER_COMPRESSOR_WITH_SWAP)) + +#define DEFAULT_FREEZER_IS_ACTIVE ((vm_compressor_mode & VM_PAGER_FREEZER_DEFAULT) == VM_PAGER_FREEZER_DEFAULT) + +#define DEFAULT_FREEZER_COMPRESSED_PAGER_IS_ACTIVE (vm_compressor_mode & (VM_PAGER_FREEZER_COMPRESSOR_NO_SWAP | VM_PAGER_FREEZER_COMPRESSOR_WITH_SWAP)) + #endif /* KERNEL_PRIVATE */