X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/c18c124eaa464aaaa5549e99e5a70fc9cbb50944..5ba3f43ea354af8ad55bea84372a2bc834d8757c:/osfmk/vm/vm_page.h?ds=sidebyside diff --git a/osfmk/vm/vm_page.h b/osfmk/vm/vm_page.h index 9c3d9f0ae..559af3f0b 100644 --- a/osfmk/vm/vm_page.h +++ b/osfmk/vm/vm_page.h @@ -68,69 +68,58 @@ #include #include - #include #include #include -#include -#include -#include -#include -#include +#if defined(__LP64__) -/* - * VM_PAGE_MIN_SPECULATIVE_AGE_Q through VM_PAGE_MAX_SPECULATIVE_AGE_Q - * represents a set of aging bins that are 'protected'... - * - * VM_PAGE_SPECULATIVE_AGED_Q is a list of the speculative pages that have - * not yet been 'claimed' but have been aged out of the protective bins - * this occurs in vm_page_speculate when it advances to the next bin - * and discovers that it is still occupied... at that point, all of the - * pages in that bin are moved to the VM_PAGE_SPECULATIVE_AGED_Q. the pages - * in that bin are all guaranteed to have reached at least the maximum age - * we allow for a protected page... they can be older if there is no - * memory pressure to pull them from the bin, or there are no new speculative pages - * being generated to push them out. - * this list is the one that vm_pageout_scan will prefer when looking - * for pages to move to the underweight free list - * - * VM_PAGE_MAX_SPECULATIVE_AGE_Q * VM_PAGE_SPECULATIVE_Q_AGE_MS - * defines the amount of time a speculative page is normally - * allowed to live in the 'protected' state (i.e. not available - * to be stolen if vm_pageout_scan is running and looking for - * pages)... however, if the total number of speculative pages - * in the protected state exceeds our limit (defined in vm_pageout.c) - * and there are none available in VM_PAGE_SPECULATIVE_AGED_Q, then - * vm_pageout_scan is allowed to steal pages from the protected - * bucket even if they are underage. - * - * vm_pageout_scan is also allowed to pull pages from a protected - * bin if the bin has reached the "age of consent" we've set +/* + * in order to make the size of a vm_page_t 64 bytes (cache line size for both arm64 and x86_64) + * we'll keep the next_m pointer packed... as long as the kernel virtual space where we allocate + * vm_page_t's from doesn't span more then 256 Gbytes, we're safe. There are live tests in the + * vm_page_t array allocation and the zone init code to determine if we can safely pack and unpack + * pointers from the 2 ends of these spaces */ -#define VM_PAGE_MAX_SPECULATIVE_AGE_Q 10 -#define VM_PAGE_MIN_SPECULATIVE_AGE_Q 1 -#define VM_PAGE_SPECULATIVE_AGED_Q 0 - -#define VM_PAGE_SPECULATIVE_Q_AGE_MS 500 +typedef uint32_t vm_page_packed_t; -struct vm_speculative_age_q { - /* - * memory queue for speculative pages via clustered pageins - */ - queue_head_t age_q; - mach_timespec_t age_ts; +struct vm_page_packed_queue_entry { + vm_page_packed_t next; /* next element */ + vm_page_packed_t prev; /* previous element */ }; +typedef struct vm_page_packed_queue_entry *vm_page_queue_t; +typedef struct vm_page_packed_queue_entry vm_page_queue_head_t; +typedef struct vm_page_packed_queue_entry vm_page_queue_chain_t; +typedef struct vm_page_packed_queue_entry *vm_page_queue_entry_t; +typedef vm_page_packed_t vm_page_object_t; -extern -struct vm_speculative_age_q vm_page_queue_speculative[]; +#else + +/* + * we can't do the packing trick on 32 bit architectures, so + * just turn the macros into noops. + */ +typedef struct vm_page *vm_page_packed_t; + +#define vm_page_queue_t queue_t +#define vm_page_queue_head_t queue_head_t +#define vm_page_queue_chain_t queue_chain_t +#define vm_page_queue_entry_t queue_entry_t + +#define vm_page_object_t vm_object_t +#endif + + +#include +#include +#include + +#include +#include -extern int speculative_steal_index; -extern int speculative_age_index; -extern unsigned int vm_page_speculative_q_age_ms; #define VM_PAGE_COMPRESSOR_COUNT (compressor_object->resident_page_count) @@ -161,76 +150,93 @@ extern unsigned int vm_page_speculative_q_age_ms; * change that field; holding either lock is sufficient to read.] */ +#define VM_PAGE_NULL ((vm_page_t) 0) -#if defined(__LP64__) +extern char vm_page_inactive_states[]; +extern char vm_page_pageable_states[]; +extern char vm_page_non_speculative_pageable_states[]; +extern char vm_page_active_or_inactive_states[]; -/* - * in order to make the size of a vm_page_t 64 bytes (cache line size for both arm64 and x86_64) - * we'll keep the next_m pointer packed... as long as the kernel virtual space where we allocate - * vm_page_t's from doesn't span more then 256 Gbytes, we're safe. There are live tests in the - * vm_page_t array allocation and the zone init code to determine if we can safely pack and unpack - * pointers from the 2 ends of these spaces - */ -typedef uint32_t vm_page_packed_t; -#define VM_PAGE_PACK_PTR(m) (!(m) ? (vm_page_packed_t)0 : ((vm_page_packed_t)((uintptr_t)(((uintptr_t)(m) - (uintptr_t) VM_MIN_KERNEL_AND_KEXT_ADDRESS)) >> 6))) -#define VM_PAGE_UNPACK_PTR(p) (!(p) ? VM_PAGE_NULL : ((vm_page_t)((((uintptr_t)(p)) << 6) + (uintptr_t) VM_MIN_KERNEL_AND_KEXT_ADDRESS))) +#define VM_PAGE_INACTIVE(m) (vm_page_inactive_states[m->vm_page_q_state]) +#define VM_PAGE_PAGEABLE(m) (vm_page_pageable_states[m->vm_page_q_state]) +#define VM_PAGE_NON_SPECULATIVE_PAGEABLE(m) (vm_page_non_speculative_pageable_states[m->vm_page_q_state]) +#define VM_PAGE_ACTIVE_OR_INACTIVE(m) (vm_page_active_or_inactive_states[m->vm_page_q_state]) -#else -/* - * we can't do the packing trick on 32 bit architectures, so - * just turn the macros into noops. - */ -typedef struct vm_page *vm_page_packed_t; +#define VM_PAGE_NOT_ON_Q 0 /* page is not present on any queue, nor is it wired... mainly a transient state */ +#define VM_PAGE_IS_WIRED 1 /* page is currently wired */ +#define VM_PAGE_USED_BY_COMPRESSOR 2 /* page is in use by the compressor to hold compressed data */ +#define VM_PAGE_ON_FREE_Q 3 /* page is on the main free queue */ +#define VM_PAGE_ON_FREE_LOCAL_Q 4 /* page is on one of the per-CPU free queues */ +#define VM_PAGE_ON_FREE_LOPAGE_Q 5 /* page is on the lopage pool free list */ +#define VM_PAGE_ON_THROTTLED_Q 6 /* page is on the throttled queue... we stash anonymous pages here when not paging */ +#define VM_PAGE_ON_PAGEOUT_Q 7 /* page is on one of the pageout queues (internal/external) awaiting processing */ +#define VM_PAGE_ON_SPECULATIVE_Q 8 /* page is on one of the speculative queues */ +#define VM_PAGE_ON_ACTIVE_LOCAL_Q 9 /* page has recently been created and is being held in one of the per-CPU local queues */ +#define VM_PAGE_ON_ACTIVE_Q 10 /* page is in global active queue */ +#define VM_PAGE_ON_INACTIVE_INTERNAL_Q 11 /* page is on the inactive internal queue a.k.a. anonymous queue */ +#define VM_PAGE_ON_INACTIVE_EXTERNAL_Q 12 /* page in on the inactive external queue a.k.a. file backed queue */ +#define VM_PAGE_ON_INACTIVE_CLEANED_Q 13 /* page has been cleaned to a backing file and is ready to be stolen */ +#define VM_PAGE_ON_SECLUDED_Q 14 /* page is on secluded queue */ +#define VM_PAGE_Q_STATE_LAST_VALID_VALUE 14 /* we currently use 4 bits for the state... don't let this go beyond 15 */ -#define VM_PAGE_PACK_PTR(m) ((vm_page_packed_t)(m)) -#define VM_PAGE_UNPACK_PTR(p) ((vm_page_t)(p)) +#define VM_PAGE_Q_STATE_ARRAY_SIZE (VM_PAGE_Q_STATE_LAST_VALID_VALUE+1) -#endif +#define pageq pageq_un.vm_page_pageq +#define snext pageq_un.vm_page_snext struct vm_page { - queue_chain_t pageq; /* queue info for FIFO */ - /* queue or free list (P) */ + union { + vm_page_queue_chain_t vm_page_pageq; /* queue info for FIFO queue or free list (P) */ + struct vm_page *vm_page_snext; + } pageq_un; - queue_chain_t listq; /* all pages in same object (O) */ + vm_page_queue_chain_t listq; /* all pages in same object (O) */ - vm_object_offset_t offset; /* offset into that object (O,P) */ - vm_object_t object; /* which object am I in (O&P) */ +#if CONFIG_BACKGROUND_QUEUE + vm_page_queue_chain_t vm_page_backgroundq; /* anonymous pages in the background pool (P) */ +#endif + + vm_object_offset_t offset; /* offset into that object (O,P) */ + vm_page_object_t vm_page_object; /* which object am I in (O&P) */ - vm_page_packed_t next_m; /* VP bucket link (O) */ /* * The following word of flags is protected * by the "page queues" lock. * * we use the 'wire_count' field to store the local * queue id if local queues are enabled... - * see the comments at 'VM_PAGE_QUEUES_REMOVE' as to + * see the comments at 'vm_page_queues_remove' as to * why this is safe to do */ #define local_id wire_count unsigned int wire_count:16, /* how many wired down maps use me? (O&P) */ - /* boolean_t */ active:1, /* page is in active list (P) */ - inactive:1, /* page is in inactive list (P) */ - clean_queue:1, /* page is in pre-cleaned list (P) */ - local:1, /* page is in one of the local queues (P) */ - speculative:1, /* page is in speculative list (P) */ - throttled:1, /* pager is not responding or doesn't exist(P) */ - free:1, /* page is on free list (P) */ - pageout_queue:1,/* page is on queue for pageout (P) */ - laundry:1, /* page is being cleaned now (P)*/ - reference:1, /* page has been used (P) */ + vm_page_q_state:4, /* which q is the page on (P) */ + + vm_page_in_background:1, + vm_page_on_backgroundq:1, + /* boolean_t */ gobbled:1, /* page used internally (P) */ - private:1, /* Page should not be returned to - * the free list (P) */ + laundry:1, /* page is being cleaned now (P)*/ no_cache:1, /* page is not to be cached and should * be reused ahead of other pages (P) */ + private:1, /* Page should not be returned to + * the free list (P) */ + reference:1, /* page has been used (P) */ - __unused_pageq_bits:3; /* 3 bits available here */ + __unused_pageq_bits:5; /* 5 bits available here */ - ppnum_t phys_page; /* Physical address of page, passed - * to pmap_enter (read-only) */ + /* + * MUST keep the 2 32 bit words used as bit fields + * separated since the compiler has a nasty habit + * of using 64 bit loads and stores on them as + * if they were a single 64 bit field... since + * they are protected by 2 different locks, this + * is a real problem + */ + vm_page_packed_t next_m; /* VP bucket link (O) */ /* * The following word of flags is protected @@ -255,9 +261,9 @@ struct vm_page { xpmapped:1, /* page has been entered with execute permission (O) or (O-shared AND pmap_page) */ - wpmapped:1, /* page has been entered at some + wpmapped:1, /* page has been entered at some * point into a pmap for write (O) */ - pageout:1, /* page wired & busy for pageout (O) */ + free_when_done:1, /* page is to be freed once cleaning is completed (O) */ absent:1, /* Data has been requested, but is * not yet available (O) */ error:1, /* Data manager was unable to provide @@ -274,33 +280,649 @@ struct vm_page { start again at top of chain */ unusual:1, /* Page is absent, error, restart or page locked */ - encrypted:1, /* encrypted for secure swap (O) */ - encrypted_cleaning:1, /* encrypting page */ cs_validated:1, /* code-signing: page was checked */ cs_tainted:1, /* code-signing: page is tainted */ cs_nx:1, /* code-signing: page is nx */ reusable:1, lopage:1, slid:1, - compressor:1, /* page owned by compressor pool */ written_by_kernel:1, /* page was written by kernel (i.e. decompressed) */ - __unused_object_bits:4; /* 5 bits available here */ + __unused_object_bits:7; /* 7 bits available here */ + +#if !defined(__arm__) && !defined(__arm64__) + ppnum_t phys_page; /* Physical address of page, passed + * to pmap_enter (read-only) */ +#endif }; -#define DEBUG_ENCRYPTED_SWAP 1 -#if DEBUG_ENCRYPTED_SWAP -#define ASSERT_PAGE_DECRYPTED(page) \ - MACRO_BEGIN \ - if ((page)->encrypted) { \ - panic("VM page %p should not be encrypted here\n", \ - (page)); \ - } \ - MACRO_END -#else /* DEBUG_ENCRYPTED_SWAP */ -#define ASSERT_PAGE_DECRYPTED(page) assert(!(page)->encrypted) -#endif /* DEBUG_ENCRYPTED_SWAP */ typedef struct vm_page *vm_page_t; +extern vm_page_t vm_pages; +extern vm_page_t vm_page_array_beginning_addr; +extern vm_page_t vm_page_array_ending_addr; + + +#if defined(__arm__) || defined(__arm64__) + +extern unsigned int vm_first_phys_ppnum; + +struct vm_page_with_ppnum { + struct vm_page vm_page_wo_ppnum; + + ppnum_t phys_page; +}; +typedef struct vm_page_with_ppnum *vm_page_with_ppnum_t; + + +static inline ppnum_t VM_PAGE_GET_PHYS_PAGE(vm_page_t m) +{ + if (m >= vm_page_array_beginning_addr && m < vm_page_array_ending_addr) + return ((ppnum_t)((uintptr_t)(m - vm_page_array_beginning_addr) + vm_first_phys_ppnum)); + else + return (((vm_page_with_ppnum_t)m)->phys_page); +} + +#define VM_PAGE_SET_PHYS_PAGE(m, ppnum) \ + MACRO_BEGIN \ + if ((m) < vm_page_array_beginning_addr || (m) >= vm_page_array_ending_addr) \ + ((vm_page_with_ppnum_t)(m))->phys_page = ppnum; \ + assert(ppnum == VM_PAGE_GET_PHYS_PAGE(m)); \ + MACRO_END + +#define VM_PAGE_GET_COLOR(m) (VM_PAGE_GET_PHYS_PAGE(m) & vm_color_mask) + +#else /* defined(__arm__) || defined(__arm64__) */ + + +struct vm_page_with_ppnum { + struct vm_page vm_page_with_ppnum; +}; +typedef struct vm_page_with_ppnum *vm_page_with_ppnum_t; + + +#define VM_PAGE_GET_PHYS_PAGE(page) (page)->phys_page +#define VM_PAGE_SET_PHYS_PAGE(page, ppnum) \ + MACRO_BEGIN \ + (page)->phys_page = ppnum; \ + MACRO_END + +#define VM_PAGE_GET_CLUMP(m) ((VM_PAGE_GET_PHYS_PAGE(m)) >> vm_clump_shift) +#define VM_PAGE_GET_COLOR(m) ((VM_PAGE_GET_CLUMP(m)) & vm_color_mask) + +#endif /* defined(__arm__) || defined(__arm64__) */ + + + +#if defined(__LP64__) + +#define VM_VPLQ_ALIGNMENT 128 +#define VM_PACKED_POINTER_ALIGNMENT 64 /* must be a power of 2 */ +#define VM_PACKED_POINTER_SHIFT 6 + +#define VM_PACKED_FROM_VM_PAGES_ARRAY 0x80000000 + +static inline vm_page_packed_t vm_page_pack_ptr(uintptr_t p) +{ + vm_page_packed_t packed_ptr; + + if (!p) + return ((vm_page_packed_t)0); + + if (p >= (uintptr_t)(vm_page_array_beginning_addr) && p < (uintptr_t)(vm_page_array_ending_addr)) { + packed_ptr = ((vm_page_packed_t)(((vm_page_t)p - vm_page_array_beginning_addr))); + assert(! (packed_ptr & VM_PACKED_FROM_VM_PAGES_ARRAY)); + packed_ptr |= VM_PACKED_FROM_VM_PAGES_ARRAY; + return packed_ptr; + } + + assert((p & (VM_PACKED_POINTER_ALIGNMENT - 1)) == 0); + + packed_ptr = ((vm_page_packed_t)(((uintptr_t)(p - (uintptr_t) VM_MIN_KERNEL_AND_KEXT_ADDRESS)) >> VM_PACKED_POINTER_SHIFT)); + assert(packed_ptr != 0); + assert(! (packed_ptr & VM_PACKED_FROM_VM_PAGES_ARRAY)); + return packed_ptr; +} + + +static inline uintptr_t vm_page_unpack_ptr(uintptr_t p) +{ + if (!p) + return ((uintptr_t)0); + + if (p & VM_PACKED_FROM_VM_PAGES_ARRAY) + return ((uintptr_t)(&vm_pages[(uint32_t)(p & ~VM_PACKED_FROM_VM_PAGES_ARRAY)])); + return (((p << VM_PACKED_POINTER_SHIFT) + (uintptr_t) VM_MIN_KERNEL_AND_KEXT_ADDRESS)); +} + + +#define VM_PAGE_PACK_PTR(p) vm_page_pack_ptr((uintptr_t)(p)) +#define VM_PAGE_UNPACK_PTR(p) vm_page_unpack_ptr((uintptr_t)(p)) + +#define VM_PAGE_OBJECT(p) ((vm_object_t)(VM_PAGE_UNPACK_PTR(p->vm_page_object))) +#define VM_PAGE_PACK_OBJECT(o) ((vm_page_object_t)(VM_PAGE_PACK_PTR(o))) + + +#define VM_PAGE_ZERO_PAGEQ_ENTRY(p) \ +MACRO_BEGIN \ + (p)->snext = 0; \ +MACRO_END + + +#define VM_PAGE_CONVERT_TO_QUEUE_ENTRY(p) VM_PAGE_PACK_PTR(p) + + +static __inline__ void +vm_page_enqueue_tail( + vm_page_queue_t que, + vm_page_queue_entry_t elt) +{ + vm_page_queue_entry_t old_tail; + + old_tail = (vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR(que->prev); + elt->next = VM_PAGE_PACK_PTR(que); + elt->prev = que->prev; + old_tail->next = VM_PAGE_PACK_PTR(elt); + que->prev = VM_PAGE_PACK_PTR(elt); +} + + +static __inline__ void +vm_page_remque( + vm_page_queue_entry_t elt) +{ + vm_page_queue_entry_t next_elt, prev_elt; + + next_elt = (vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR(elt->next); + + /* next_elt may equal prev_elt (and the queue head) if elt was the only element */ + prev_elt = (vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR(elt->prev); + + next_elt->prev = VM_PAGE_PACK_PTR(prev_elt); + prev_elt->next = VM_PAGE_PACK_PTR(next_elt); + + elt->next = 0; + elt->prev = 0; +} + + +/* + * Macro: vm_page_queue_init + * Function: + * Initialize the given queue. + * Header: + * void vm_page_queue_init(q) + * vm_page_queue_t q; \* MODIFIED *\ + */ +#define vm_page_queue_init(q) \ +MACRO_BEGIN \ + assert((((uintptr_t)q) & (VM_PACKED_POINTER_ALIGNMENT-1)) == 0); \ + assert((VM_PAGE_UNPACK_PTR(VM_PAGE_PACK_PTR((uintptr_t)q))) == (uintptr_t)q); \ + (q)->next = VM_PAGE_PACK_PTR(q); \ + (q)->prev = VM_PAGE_PACK_PTR(q); \ +MACRO_END + + +/* + * Macro: vm_page_queue_enter + * Function: + * Insert a new element at the tail of the queue. + * Header: + * void vm_page_queue_enter(q, elt, type, field) + * queue_t q; + * elt; + * is what's in our queue + * is the chain field in (*) + * Note: + * This should only be used with Method 2 queue iteration (element chains) + */ +#define vm_page_queue_enter(head, elt, type, field) \ +MACRO_BEGIN \ + vm_page_queue_entry_t __prev; \ + \ + __prev = ((vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR((head)->prev)); \ + if ((head) == __prev) { \ + (head)->next = VM_PAGE_PACK_PTR(elt); \ + } \ + else { \ + ((type)(void *)__prev)->field.next = VM_PAGE_PACK_PTR(elt); \ + } \ + (elt)->field.prev = VM_PAGE_PACK_PTR(__prev); \ + (elt)->field.next = VM_PAGE_PACK_PTR(head); \ + (head)->prev = VM_PAGE_PACK_PTR(elt); \ +MACRO_END + + +/* + * These are helper macros for vm_page_queue_enter_clump to assist + * with conditional compilation (release / debug / development) + */ +#if DEVELOPMENT || DEBUG + +#define __DEBUG_CHECK_BUDDIES(__check, __prev, __p, field) \ +MACRO_BEGIN \ + if(__check) { /* if first forward buddy.. */ \ + if(__prev) { /* ..and if a backward buddy was found, verify link consistency */ \ + assert(__p == (vm_page_t) VM_PAGE_UNPACK_PTR(__prev->next)); \ + assert(__prev == (vm_page_queue_entry_t) VM_PAGE_UNPACK_PTR(__p->field.prev)); \ + } \ + __check=0; \ + } \ +MACRO_END + +#define __DEBUG_VERIFY_LINKS(__i, __first, __n_free, __last_next) \ +MACRO_BEGIN \ + vm_page_queue_entry_t __tmp; \ + for(__i=0, __tmp=__first; __i<__n_free; __i++) \ + __tmp=(vm_page_queue_entry_t) VM_PAGE_UNPACK_PTR(__tmp->next); \ + assert(__tmp == __last_next); \ +MACRO_END + +#define __DEBUG_STAT_INCREMENT_INRANGE vm_clump_inrange++ +#define __DEBUG_STAT_INCREMENT_INSERTS vm_clump_inserts++ +#define __DEBUG_STAT_INCREMENT_PROMOTES(__n_free) vm_clump_promotes+=__n_free + +#else + +#define __DEBUG_CHECK_BUDDIES(__check, __prev, __p, field) __check=1 +#define __DEBUG_VERIFY_LINKS(__i, __first, __n_free, __last_next) +#define __DEBUG_STAT_INCREMENT_INRANGE +#define __DEBUG_STAT_INCREMENT_INSERTS +#define __DEBUG_STAT_INCREMENT_PROMOTES(__n_free) + +#endif /* if DEVELOPMENT || DEBUG */ + +/* + * Macro: vm_page_queue_enter_clump + * Function: + * Insert a new element into the free queue and clump pages within the same 16K boundary together + * + * Header: + * void vm_page_queue_enter_clump(q, elt, type, field) + * queue_t q; + * elt; + * is what's in our queue + * is the chain field in (*) + * Note: + * This should only be used with Method 2 queue iteration (element chains) + */ +#if defined(__x86_64__) +#define vm_page_queue_enter_clump(head, elt, type, field) \ +MACRO_BEGIN \ + ppnum_t __clump_num; \ + unsigned int __i, __n, __n_free=1, __check=1; \ + vm_page_queue_entry_t __prev=0, __next, __last, __last_next, __first, __first_prev, __head_next; \ + vm_page_t __p; \ + \ + /* if elt is part of vm_pages[] */ \ + if((elt) >= vm_page_array_beginning_addr && (elt) < vm_page_array_boundary) { \ + __first = __last = (vm_page_queue_entry_t) (elt); \ + __clump_num = VM_PAGE_GET_CLUMP(elt); \ + __n = VM_PAGE_GET_PHYS_PAGE(elt) & vm_clump_mask; \ + /* scan backward looking for a buddy page */ \ + for(__i=0, __p=(elt)-1; __i<__n && __p>=vm_page_array_beginning_addr; __i++, __p--) { \ + if(__p->vm_page_q_state == VM_PAGE_ON_FREE_Q && __clump_num == VM_PAGE_GET_CLUMP(__p)) { \ + if(__prev == 0) __prev = (vm_page_queue_entry_t) __p; \ + __first = (vm_page_queue_entry_t) __p; \ + __n_free++; \ + } \ + } \ + /* scan forward looking for a buddy page */ \ + for(__i=__n+1, __p=(elt)+1; __ivm_page_q_state == VM_PAGE_ON_FREE_Q && __clump_num == VM_PAGE_GET_CLUMP(__p)) { \ + __DEBUG_CHECK_BUDDIES(__check, __prev, __p, field); \ + if(__prev == 0) __prev = (vm_page_queue_entry_t) VM_PAGE_UNPACK_PTR(__p->field.prev); \ + __last = (vm_page_queue_entry_t) __p; \ + __n_free++; \ + } \ + } \ + __DEBUG_STAT_INCREMENT_INRANGE; \ + } \ + /* if elt is not part of vm_pages or if 1st page in clump, insert at tail */ \ + if(__prev == 0) __prev = (vm_page_queue_entry_t) VM_PAGE_UNPACK_PTR((head)->prev); \ + \ + /* insert the element */ \ + __next = (vm_page_queue_entry_t) VM_PAGE_UNPACK_PTR(__prev->next); \ + (elt)->field.next = __prev->next; \ + (elt)->field.prev = __next->prev; \ + __prev->next = __next->prev = VM_PAGE_PACK_PTR(elt); \ + __DEBUG_STAT_INCREMENT_INSERTS; \ + \ + /* check if clump needs to be promoted to head */ \ + if(__n_free >= vm_clump_promote_threshold && __n_free > 1) { \ + __first_prev = (vm_page_queue_entry_t) VM_PAGE_UNPACK_PTR(__first->prev); \ + if(__first_prev != (head)) { /* if not at head already */ \ + __last_next = (vm_page_queue_entry_t) VM_PAGE_UNPACK_PTR(__last->next); \ + /* verify that the links within the clump are consistent */ \ + __DEBUG_VERIFY_LINKS(__i, __first, __n_free, __last_next); \ + /* promote clump to head */ \ + __first_prev->next = __last->next; \ + __last_next->prev = __first->prev; \ + __first->prev = VM_PAGE_PACK_PTR(head); \ + __last->next = (head)->next; \ + __head_next = (vm_page_queue_entry_t) VM_PAGE_UNPACK_PTR((head)->next); \ + __head_next->prev = VM_PAGE_PACK_PTR(__last); \ + (head)->next = VM_PAGE_PACK_PTR(__first); \ + __DEBUG_STAT_INCREMENT_PROMOTES(__n_free); \ + } \ + } \ +MACRO_END +#endif + +/* + * Macro: vm_page_queue_enter_first + * Function: + * Insert a new element at the head of the queue. + * Header: + * void queue_enter_first(q, elt, type, field) + * queue_t q; + * elt; + * is what's in our queue + * is the chain field in (*) + * Note: + * This should only be used with Method 2 queue iteration (element chains) + */ +#define vm_page_queue_enter_first(head, elt, type, field) \ +MACRO_BEGIN \ + vm_page_queue_entry_t __next; \ + \ + __next = ((vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR((head)->next)); \ + if ((head) == __next) { \ + (head)->prev = VM_PAGE_PACK_PTR(elt); \ + } \ + else { \ + ((type)(void *)__next)->field.prev = VM_PAGE_PACK_PTR(elt); \ + } \ + (elt)->field.next = VM_PAGE_PACK_PTR(__next); \ + (elt)->field.prev = VM_PAGE_PACK_PTR(head); \ + (head)->next = VM_PAGE_PACK_PTR(elt); \ +MACRO_END + + +/* + * Macro: vm_page_queue_remove + * Function: + * Remove an arbitrary item from the queue. + * Header: + * void vm_page_queue_remove(q, qe, type, field) + * arguments as in vm_page_queue_enter + * Note: + * This should only be used with Method 2 queue iteration (element chains) + */ +#define vm_page_queue_remove(head, elt, type, field) \ +MACRO_BEGIN \ + vm_page_queue_entry_t __next, __prev; \ + \ + __next = ((vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR((elt)->field.next)); \ + __prev = ((vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR((elt)->field.prev)); \ + \ + if ((head) == __next) \ + (head)->prev = VM_PAGE_PACK_PTR(__prev); \ + else \ + ((type)(void *)__next)->field.prev = VM_PAGE_PACK_PTR(__prev); \ + \ + if ((head) == __prev) \ + (head)->next = VM_PAGE_PACK_PTR(__next); \ + else \ + ((type)(void *)__prev)->field.next = VM_PAGE_PACK_PTR(__next); \ + \ + (elt)->field.next = 0; \ + (elt)->field.prev = 0; \ +MACRO_END + + +/* + * Macro: vm_page_queue_remove_first + * Function: + * Remove and return the entry at the head of + * the queue. + * Header: + * vm_page_queue_remove_first(head, entry, type, field) + * entry is returned by reference + * Note: + * This should only be used with Method 2 queue iteration (element chains) + */ +#define vm_page_queue_remove_first(head, entry, type, field) \ +MACRO_BEGIN \ + vm_page_queue_entry_t __next; \ + \ + (entry) = (type)(void *) VM_PAGE_UNPACK_PTR(((head)->next)); \ + __next = ((vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR((entry)->field.next)); \ + \ + if ((head) == __next) \ + (head)->prev = VM_PAGE_PACK_PTR(head); \ + else \ + ((type)(void *)(__next))->field.prev = VM_PAGE_PACK_PTR(head); \ + (head)->next = VM_PAGE_PACK_PTR(__next); \ + \ + (entry)->field.next = 0; \ + (entry)->field.prev = 0; \ +MACRO_END + + +/* + * Macro: vm_page_queue_remove_first_with_clump + * Function: + * Remove and return the entry at the head of the free queue + * end is set to 1 to indicate that we just returned the last page in a clump + * + * Header: + * vm_page_queue_remove_first_with_clump(head, entry, type, field, end) + * entry is returned by reference + * end is returned by reference + * Note: + * This should only be used with Method 2 queue iteration (element chains) + */ +#if defined(__x86_64__) +#define vm_page_queue_remove_first_with_clump(head, entry, type, field, end) \ +MACRO_BEGIN \ + vm_page_queue_entry_t __next; \ + \ + (entry) = (type)(void *) VM_PAGE_UNPACK_PTR(((head)->next)); \ + __next = ((vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR((entry)->field.next)); \ + \ + (end)=0; \ + if ((head) == __next) { \ + (head)->prev = VM_PAGE_PACK_PTR(head); \ + (end)=1; \ + } \ + else { \ + ((type)(void *)(__next))->field.prev = VM_PAGE_PACK_PTR(head); \ + if(VM_PAGE_GET_CLUMP(entry) != VM_PAGE_GET_CLUMP(((type)(void *)(__next)))) (end)=1; \ + } \ + (head)->next = VM_PAGE_PACK_PTR(__next); \ + \ + (entry)->field.next = 0; \ + (entry)->field.prev = 0; \ + \ +MACRO_END +#endif + +/* + * Macro: vm_page_queue_end + * Function: + * Tests whether a new entry is really the end of + * the queue. + * Header: + * boolean_t vm_page_queue_end(q, qe) + * vm_page_queue_t q; + * vm_page_queue_entry_t qe; + */ +#define vm_page_queue_end(q, qe) ((q) == (qe)) + + +/* + * Macro: vm_page_queue_empty + * Function: + * Tests whether a queue is empty. + * Header: + * boolean_t vm_page_queue_empty(q) + * vm_page_queue_t q; + */ +#define vm_page_queue_empty(q) vm_page_queue_end((q), ((vm_page_queue_entry_t)vm_page_queue_first(q))) + + + +/* + * Macro: vm_page_queue_first + * Function: + * Returns the first entry in the queue, + * Header: + * uintpr_t vm_page_queue_first(q) + * vm_page_queue_t q; \* IN *\ + */ +#define vm_page_queue_first(q) (VM_PAGE_UNPACK_PTR((q)->next)) + + + +/* + * Macro: vm_page_queue_last + * Function: + * Returns the last entry in the queue. + * Header: + * vm_page_queue_entry_t queue_last(q) + * queue_t q; \* IN *\ + */ +#define vm_page_queue_last(q) (VM_PAGE_UNPACK_PTR((q)->prev)) + + + +/* + * Macro: vm_page_queue_next + * Function: + * Returns the entry after an item in the queue. + * Header: + * uintpr_t vm_page_queue_next(qc) + * vm_page_queue_t qc; + */ +#define vm_page_queue_next(qc) (VM_PAGE_UNPACK_PTR((qc)->next)) + + + +/* + * Macro: vm_page_queue_prev + * Function: + * Returns the entry before an item in the queue. + * Header: + * uinptr_t vm_page_queue_prev(qc) + * vm_page_queue_t qc; + */ +#define vm_page_queue_prev(qc) (VM_PAGE_UNPACK_PTR((qc)->prev)) + + + +/* + * Macro: vm_page_queue_iterate + * Function: + * iterate over each item in the queue. + * Generates a 'for' loop, setting elt to + * each item in turn (by reference). + * Header: + * vm_page_queue_iterate(q, elt, type, field) + * queue_t q; + * elt; + * is what's in our queue + * is the chain field in (*) + * Note: + * This should only be used with Method 2 queue iteration (element chains) + */ +#define vm_page_queue_iterate(head, elt, type, field) \ + for ((elt) = (type)(void *) vm_page_queue_first(head); \ + !vm_page_queue_end((head), (vm_page_queue_entry_t)(elt)); \ + (elt) = (type)(void *) vm_page_queue_next(&(elt)->field)) + +#else + +#define VM_VPLQ_ALIGNMENT 128 +#define VM_PACKED_POINTER_ALIGNMENT 4 +#define VM_PACKED_POINTER_SHIFT 0 + +#define VM_PACKED_FROM_VM_PAGES_ARRAY 0 + +#define VM_PAGE_PACK_PTR(p) (p) +#define VM_PAGE_UNPACK_PTR(p) ((uintptr_t)(p)) + +#define VM_PAGE_OBJECT(p) (vm_object_t)(p->vm_page_object) +#define VM_PAGE_PACK_OBJECT(o) ((vm_page_object_t)(VM_PAGE_PACK_PTR(o))) + + +#define VM_PAGE_ZERO_PAGEQ_ENTRY(p) \ +MACRO_BEGIN \ + (p)->pageq.next = 0; \ + (p)->pageq.prev = 0; \ +MACRO_END + +#define VM_PAGE_CONVERT_TO_QUEUE_ENTRY(p) ((queue_entry_t)(p)) + +#define vm_page_remque remque +#define vm_page_enqueue_tail enqueue_tail +#define vm_page_queue_init queue_init +#define vm_page_queue_enter queue_enter +#define vm_page_queue_enter_first queue_enter_first +#define vm_page_queue_remove queue_remove +#define vm_page_queue_remove_first queue_remove_first +#define vm_page_queue_end queue_end +#define vm_page_queue_empty queue_empty +#define vm_page_queue_first queue_first +#define vm_page_queue_last queue_last +#define vm_page_queue_next queue_next +#define vm_page_queue_prev queue_prev +#define vm_page_queue_iterate queue_iterate + +#endif + + + +/* + * VM_PAGE_MIN_SPECULATIVE_AGE_Q through VM_PAGE_MAX_SPECULATIVE_AGE_Q + * represents a set of aging bins that are 'protected'... + * + * VM_PAGE_SPECULATIVE_AGED_Q is a list of the speculative pages that have + * not yet been 'claimed' but have been aged out of the protective bins + * this occurs in vm_page_speculate when it advances to the next bin + * and discovers that it is still occupied... at that point, all of the + * pages in that bin are moved to the VM_PAGE_SPECULATIVE_AGED_Q. the pages + * in that bin are all guaranteed to have reached at least the maximum age + * we allow for a protected page... they can be older if there is no + * memory pressure to pull them from the bin, or there are no new speculative pages + * being generated to push them out. + * this list is the one that vm_pageout_scan will prefer when looking + * for pages to move to the underweight free list + * + * VM_PAGE_MAX_SPECULATIVE_AGE_Q * VM_PAGE_SPECULATIVE_Q_AGE_MS + * defines the amount of time a speculative page is normally + * allowed to live in the 'protected' state (i.e. not available + * to be stolen if vm_pageout_scan is running and looking for + * pages)... however, if the total number of speculative pages + * in the protected state exceeds our limit (defined in vm_pageout.c) + * and there are none available in VM_PAGE_SPECULATIVE_AGED_Q, then + * vm_pageout_scan is allowed to steal pages from the protected + * bucket even if they are underage. + * + * vm_pageout_scan is also allowed to pull pages from a protected + * bin if the bin has reached the "age of consent" we've set + */ +#define VM_PAGE_MAX_SPECULATIVE_AGE_Q 10 +#define VM_PAGE_MIN_SPECULATIVE_AGE_Q 1 +#define VM_PAGE_SPECULATIVE_AGED_Q 0 + +#define VM_PAGE_SPECULATIVE_Q_AGE_MS 500 + +struct vm_speculative_age_q { + /* + * memory queue for speculative pages via clustered pageins + */ + vm_page_queue_head_t age_q; + mach_timespec_t age_ts; +} __attribute__((aligned(VM_PACKED_POINTER_ALIGNMENT))); + + + +extern +struct vm_speculative_age_q vm_page_queue_speculative[]; + +extern int speculative_steal_index; +extern int speculative_age_index; +extern unsigned int vm_page_speculative_q_age_ms; typedef struct vm_locks_array { @@ -311,10 +933,16 @@ typedef struct vm_locks_array { } vm_locks_array_t; -#define VM_PAGE_WIRED(m) ((!(m)->local && (m)->wire_count)) -#define VM_PAGE_NULL ((vm_page_t) 0) -#define NEXT_PAGE(m) ((vm_page_t) (m)->pageq.next) -#define NEXT_PAGE_PTR(m) ((vm_page_t *) &(m)->pageq.next) +#if CONFIG_BACKGROUND_QUEUE +extern void vm_page_assign_background_state(vm_page_t mem); +extern void vm_page_update_background_state(vm_page_t mem); +extern void vm_page_add_to_backgroundq(vm_page_t mem, boolean_t first); +extern void vm_page_remove_from_backgroundq(vm_page_t mem); +#endif + +#define VM_PAGE_WIRED(m) ((m)->vm_page_q_state == VM_PAGE_IS_WIRED) +#define NEXT_PAGE(m) ((m)->snext) +#define NEXT_PAGE_PTR(m) (&(m)->snext) /* * XXX The unusual bit should not be necessary. Most of the bit @@ -324,11 +952,12 @@ typedef struct vm_locks_array { /* * For debugging, this macro can be defined to perform * some useful check on a page structure. + * INTENTIONALLY left as a no-op so that the + * current call-sites can be left intact for future uses. */ #define VM_PAGE_CHECK(mem) \ MACRO_BEGIN \ - VM_PAGE_QUEUES_ASSERT(mem, 1); \ MACRO_END /* Page coloring: @@ -397,10 +1026,10 @@ vm_map_size_t vm_global_no_user_wire_amount; #define VPL_LOCK_SPIN 1 struct vpl { + vm_page_queue_head_t vpl_queue; unsigned int vpl_count; unsigned int vpl_internal_count; unsigned int vpl_external_count; - queue_head_t vpl_queue; #ifdef VPL_LOCK_SPIN lck_spin_t vpl_lock; #else @@ -411,7 +1040,7 @@ struct vpl { struct vplq { union { - char cache_line_pad[128]; + char cache_line_pad[VM_VPLQ_ALIGNMENT]; struct vpl vpl; } vpl_un; }; @@ -427,19 +1056,52 @@ extern vm_locks_array_t vm_page_locks; extern -queue_head_t vm_page_queue_free[MAX_COLORS]; /* memory free queue */ +vm_page_queue_head_t vm_lopage_queue_free; /* low memory free queue */ +extern +vm_page_queue_head_t vm_page_queue_active; /* active memory queue */ +extern +vm_page_queue_head_t vm_page_queue_inactive; /* inactive memory queue for normal pages */ +#if CONFIG_SECLUDED_MEMORY +extern +vm_page_queue_head_t vm_page_queue_secluded; /* reclaimable pages secluded for Camera */ +#endif /* CONFIG_SECLUDED_MEMORY */ +extern +vm_page_queue_head_t vm_page_queue_cleaned; /* clean-queue inactive memory */ +extern +vm_page_queue_head_t vm_page_queue_anonymous; /* inactive memory queue for anonymous pages */ +extern +vm_page_queue_head_t vm_page_queue_throttled; /* memory queue for throttled pageout pages */ + +extern +queue_head_t vm_objects_wired; +extern +lck_spin_t vm_objects_wired_lock; + +#if CONFIG_BACKGROUND_QUEUE + +#define VM_PAGE_BACKGROUND_TARGET_MAX 50000 + +#define VM_PAGE_BG_DISABLED 0 +#define VM_PAGE_BG_LEVEL_1 1 + +extern +vm_page_queue_head_t vm_page_queue_background; extern -queue_head_t vm_lopage_queue_free; /* low memory free queue */ +uint64_t vm_page_background_promoted_count; extern -queue_head_t vm_page_queue_active; /* active memory queue */ +uint32_t vm_page_background_count; extern -queue_head_t vm_page_queue_inactive; /* inactive memory queue for normal pages */ +uint32_t vm_page_background_target; extern -queue_head_t vm_page_queue_cleaned; /* clean-queue inactive memory */ +uint32_t vm_page_background_internal_count; extern -queue_head_t vm_page_queue_anonymous; /* inactive memory queue for anonymous pages */ +uint32_t vm_page_background_external_count; extern -queue_head_t vm_page_queue_throttled; /* memory queue for throttled pageout pages */ +uint32_t vm_page_background_mode; +extern +uint32_t vm_page_background_exclude_external; + +#endif extern vm_offset_t first_phys_addr; /* physical address for first_page */ @@ -449,11 +1111,17 @@ vm_offset_t last_phys_addr; /* physical address for last_page */ extern unsigned int vm_page_free_count; /* How many pages are free? (sum of all colors) */ extern -unsigned int vm_page_fictitious_count;/* How many fictitious pages are free? */ -extern unsigned int vm_page_active_count; /* How many pages are active? */ extern unsigned int vm_page_inactive_count; /* How many pages are inactive? */ +#if CONFIG_SECLUDED_MEMORY +extern +unsigned int vm_page_secluded_count; /* How many pages are secluded? */ +extern +unsigned int vm_page_secluded_count_free; +extern +unsigned int vm_page_secluded_count_inuse; +#endif /* CONFIG_SECLUDED_MEMORY */ extern unsigned int vm_page_cleaned_count; /* How many pages are in the clean queue? */ extern @@ -482,16 +1150,23 @@ extern uint32_t vm_page_creation_throttle; /* When to throttle new page creation */ extern unsigned int vm_page_inactive_target;/* How many do we want inactive? */ +#if CONFIG_SECLUDED_MEMORY +extern +unsigned int vm_page_secluded_target;/* How many do we want secluded? */ +#endif /* CONFIG_SECLUDED_MEMORY */ extern unsigned int vm_page_anonymous_min; /* When it's ok to pre-clean */ extern -unsigned int vm_page_inactive_min; /* When do wakeup pageout */ +unsigned int vm_page_inactive_min; /* When to wakeup pageout */ extern unsigned int vm_page_free_reserved; /* How many pages reserved to do pageout */ extern unsigned int vm_page_throttle_count; /* Count of page allocations throttled */ extern unsigned int vm_page_gobble_count; +extern +unsigned int vm_page_stolen_count; /* Count of stolen pages not acccounted in zones */ + #if DEVELOPMENT || DEBUG extern @@ -510,11 +1185,15 @@ extern unsigned int vm_page_free_wanted; extern unsigned int vm_page_free_wanted_privileged; /* how many VM privileged threads are waiting for memory */ +#if CONFIG_SECLUDED_MEMORY +extern unsigned int vm_page_free_wanted_secluded; + /* how many threads are waiting for secluded memory */ +#endif /* CONFIG_SECLUDED_MEMORY */ -extern ppnum_t vm_page_fictitious_addr; +extern const ppnum_t vm_page_fictitious_addr; /* (fake) phys_addr of fictitious pages */ -extern ppnum_t vm_page_guard_addr; +extern const ppnum_t vm_page_guard_addr; /* (fake) phys_addr of guard pages */ @@ -552,6 +1231,10 @@ extern void vm_page_create( ppnum_t start, ppnum_t end); +extern vm_page_t kdp_vm_page_lookup( + vm_object_t object, + vm_object_offset_t offset); + extern vm_page_t vm_page_lookup( vm_object_t object, vm_object_offset_t offset); @@ -568,11 +1251,16 @@ extern void vm_page_more_fictitious(void); extern int vm_pool_low(void); extern vm_page_t vm_page_grab(void); +extern vm_page_t vm_page_grab_options(int flags); +#if CONFIG_SECLUDED_MEMORY +#define VM_PAGE_GRAB_SECLUDED 0x00000001 +#endif /* CONFIG_SECLUDED_MEMORY */ extern vm_page_t vm_page_grablo(void); extern void vm_page_release( - vm_page_t page); + vm_page_t page, + boolean_t page_queues_locked); extern boolean_t vm_page_wait( int interruptible ); @@ -581,10 +1269,6 @@ extern vm_page_t vm_page_alloc( vm_object_t object, vm_object_offset_t offset); -extern vm_page_t vm_page_alloclo( - vm_object_t object, - vm_object_offset_t offset); - extern vm_page_t vm_page_alloc_guard( vm_object_t object, vm_object_offset_t offset); @@ -630,21 +1314,29 @@ extern void vm_page_reactivate_local(uint32_t lid, boolean_t force, boolean_t n extern void vm_page_rename( vm_page_t page, vm_object_t new_object, - vm_object_offset_t new_offset, - boolean_t encrypted_ok); + vm_object_offset_t new_offset); extern void vm_page_insert( vm_page_t page, vm_object_t object, vm_object_offset_t offset); +extern void vm_page_insert_wired( + vm_page_t page, + vm_object_t object, + vm_object_offset_t offset, + vm_tag_t tag); + extern void vm_page_insert_internal( vm_page_t page, vm_object_t object, vm_object_offset_t offset, + vm_tag_t tag, boolean_t queues_lock_held, boolean_t insert_in_hash, - boolean_t batch_pmap_op); + boolean_t batch_pmap_op, + boolean_t delayed_accounting, + uint64_t *delayed_ledger_update); extern void vm_page_replace( vm_page_t mem, @@ -675,7 +1367,9 @@ extern void vm_page_part_copy( vm_size_t len); extern void vm_page_wire( - vm_page_t page); + vm_page_t page, + vm_tag_t tag, + boolean_t check_memorystatus); extern void vm_page_unwire( vm_page_t page, @@ -690,6 +1384,13 @@ extern void vm_page_validate_cs(vm_page_t page); extern void vm_page_validate_cs_mapped( vm_page_t page, const void *kaddr); +extern void vm_page_validate_cs_mapped_chunk( + vm_page_t page, + const void *kaddr, + vm_offset_t chunk_offset, + vm_size_t chunk_size, + boolean_t *validated, + unsigned *tainted); extern void vm_page_free_prepare_queues( vm_page_t page); @@ -714,15 +1415,21 @@ extern void memorystatus_pages_update(unsigned int pages_avail); memorystatus_pages_update( \ vm_page_pageable_external_count + \ vm_page_free_count + \ - (VM_DYNAMIC_PAGING_ENABLED(memory_manager_default) ? 0 : vm_page_purgeable_count) \ + (VM_DYNAMIC_PAGING_ENABLED() ? 0 : vm_page_purgeable_count) \ ); \ } while(0) #else /* CONFIG_JETSAM */ +#if CONFIG_EMBEDDED + +#define VM_CHECK_MEMORYSTATUS do {} while(0) + +#else /* CONFIG_EMBEDDED */ #define VM_CHECK_MEMORYSTATUS vm_pressure_response() +#endif /* CONFIG_EMBEDDED */ #endif /* CONFIG_JETSAM */ @@ -731,11 +1438,22 @@ extern void memorystatus_pages_update(unsigned int pages_avail); * protected by the object lock. */ +#if CONFIG_EMBEDDED #define SET_PAGE_DIRTY(m, set_pmap_modified) \ MACRO_BEGIN \ vm_page_t __page__ = (m); \ + if (__page__->dirty == FALSE && (set_pmap_modified)) { \ + pmap_set_modify(VM_PAGE_GET_PHYS_PAGE(__page__)); \ + } \ __page__->dirty = TRUE; \ MACRO_END +#else /* CONFIG_EMBEDDED */ +#define SET_PAGE_DIRTY(m, set_pmap_modified) \ + MACRO_BEGIN \ + vm_page_t __page__ = (m); \ + __page__->dirty = TRUE; \ + MACRO_END +#endif /* CONFIG_EMBEDDED */ #define PAGE_ASSERT_WAIT(m, interruptible) \ (((m)->wanted = TRUE), \ @@ -784,6 +1502,7 @@ extern void memorystatus_pages_update(unsigned int pages_avail); #define vm_page_queue_free_lock (vm_page_locks.vm_page_queue_free_lock2) #define vm_page_lock_queues() lck_mtx_lock(&vm_page_queue_lock) +#define vm_page_trylock_queues() lck_mtx_try_lock(&vm_page_queue_lock) #define vm_page_unlock_queues() lck_mtx_unlock(&vm_page_queue_lock) #define vm_page_lockspin_queues() lck_mtx_lock_spin(&vm_page_queue_lock) @@ -800,176 +1519,6 @@ extern void memorystatus_pages_update(unsigned int pages_avail); #define VPL_UNLOCK(vpl) lck_mtx_unlock(vpl) #endif -#if MACH_ASSERT -extern void vm_page_queues_assert(vm_page_t mem, int val); -#define VM_PAGE_QUEUES_ASSERT(mem, val) vm_page_queues_assert((mem), (val)) -#else -#define VM_PAGE_QUEUES_ASSERT(mem, val) -#endif - - -/* - * 'vm_fault_enter' will place newly created pages (zero-fill and COW) onto the - * local queues if they exist... its the only spot in the system where we add pages - * to those queues... once on those queues, those pages can only move to one of the - * global page queues or the free queues... they NEVER move from local q to local q. - * the 'local' state is stable when VM_PAGE_QUEUES_REMOVE is called since we're behind - * the global vm_page_queue_lock at this point... we still need to take the local lock - * in case this operation is being run on a different CPU then the local queue's identity, - * but we don't have to worry about the page moving to a global queue or becoming wired - * while we're grabbing the local lock since those operations would require the global - * vm_page_queue_lock to be held, and we already own it. - * - * this is why its safe to utilze the wire_count field in the vm_page_t as the local_id... - * 'wired' and local are ALWAYS mutually exclusive conditions. - */ - -#define VM_PAGE_QUEUES_REMOVE(mem) \ - MACRO_BEGIN \ - boolean_t was_pageable; \ - \ - VM_PAGE_QUEUES_ASSERT(mem, 1); \ - assert(!mem->pageout_queue); \ -/* \ - * if (mem->pageout_queue) \ - * NOTE: VM_PAGE_QUEUES_REMOVE does not deal with removing pages from the pageout queue... \ - * the caller is responsible for determing if the page is on that queue, and if so, must \ - * either first remove it (it needs both the page queues lock and the object lock to do \ - * this via vm_pageout_steal_laundry), or avoid the call to VM_PAGE_QUEUES_REMOVE \ - */ \ - if (mem->local) { \ - struct vpl *lq; \ - assert(mem->object != kernel_object); \ - assert(mem->object != compressor_object); \ - assert(!mem->inactive && !mem->speculative); \ - assert(!mem->active && !mem->throttled); \ - assert(!mem->clean_queue); \ - assert(!mem->fictitious); \ - lq = &vm_page_local_q[mem->local_id].vpl_un.vpl; \ - VPL_LOCK(&lq->vpl_lock); \ - queue_remove(&lq->vpl_queue, \ - mem, vm_page_t, pageq); \ - mem->local = FALSE; \ - mem->local_id = 0; \ - lq->vpl_count--; \ - if (mem->object->internal) { \ - lq->vpl_internal_count--; \ - } else { \ - lq->vpl_external_count--; \ - } \ - VPL_UNLOCK(&lq->vpl_lock); \ - was_pageable = FALSE; \ - } \ - \ - else if (mem->active) { \ - assert(mem->object != kernel_object); \ - assert(mem->object != compressor_object); \ - assert(!mem->inactive && !mem->speculative); \ - assert(!mem->clean_queue); \ - assert(!mem->throttled); \ - assert(!mem->fictitious); \ - queue_remove(&vm_page_queue_active, \ - mem, vm_page_t, pageq); \ - mem->active = FALSE; \ - vm_page_active_count--; \ - was_pageable = TRUE; \ - } \ - \ - else if (mem->inactive) { \ - assert(mem->object != kernel_object); \ - assert(mem->object != compressor_object); \ - assert(!mem->active && !mem->speculative); \ - assert(!mem->throttled); \ - assert(!mem->fictitious); \ - vm_page_inactive_count--; \ - if (mem->clean_queue) { \ - queue_remove(&vm_page_queue_cleaned, \ - mem, vm_page_t, pageq); \ - mem->clean_queue = FALSE; \ - vm_page_cleaned_count--; \ - } else { \ - if (mem->object->internal) { \ - queue_remove(&vm_page_queue_anonymous, \ - mem, vm_page_t, pageq); \ - vm_page_anonymous_count--; \ - } else { \ - queue_remove(&vm_page_queue_inactive, \ - mem, vm_page_t, pageq); \ - } \ - vm_purgeable_q_advance_all(); \ - } \ - mem->inactive = FALSE; \ - was_pageable = TRUE; \ - } \ - \ - else if (mem->throttled) { \ - assert(mem->object != compressor_object); \ - assert(!mem->active && !mem->inactive); \ - assert(!mem->speculative); \ - assert(!mem->fictitious); \ - queue_remove(&vm_page_queue_throttled, \ - mem, vm_page_t, pageq); \ - mem->throttled = FALSE; \ - vm_page_throttled_count--; \ - was_pageable = FALSE; \ - } \ - \ - else if (mem->speculative) { \ - assert(mem->object != compressor_object); \ - assert(!mem->active && !mem->inactive); \ - assert(!mem->throttled); \ - assert(!mem->fictitious); \ - remque(&mem->pageq); \ - mem->speculative = FALSE; \ - vm_page_speculative_count--; \ - was_pageable = TRUE; \ - } \ - \ - else if (mem->pageq.next || mem->pageq.prev) { \ - was_pageable = FALSE; \ - panic("VM_PAGE_QUEUES_REMOVE: unmarked page on Q"); \ - } else { \ - was_pageable = FALSE; \ - } \ - \ - mem->pageq.next = NULL; \ - mem->pageq.prev = NULL; \ - VM_PAGE_QUEUES_ASSERT(mem, 0); \ - if (was_pageable) { \ - if (mem->object->internal) { \ - vm_page_pageable_internal_count--; \ - } else { \ - vm_page_pageable_external_count--; \ - } \ - } \ - MACRO_END - - -#define VM_PAGE_ENQUEUE_INACTIVE(mem, first) \ - MACRO_BEGIN \ - VM_PAGE_QUEUES_ASSERT(mem, 0); \ - assert(!mem->fictitious); \ - assert(!mem->laundry); \ - assert(!mem->pageout_queue); \ - if (mem->object->internal) { \ - if (first == TRUE) \ - queue_enter_first(&vm_page_queue_anonymous, mem, vm_page_t, pageq); \ - else \ - queue_enter(&vm_page_queue_anonymous, mem, vm_page_t, pageq); \ - vm_page_anonymous_count++; \ - vm_page_pageable_internal_count++; \ - } else { \ - if (first == TRUE) \ - queue_enter_first(&vm_page_queue_inactive, mem, vm_page_t, pageq); \ - else \ - queue_enter(&vm_page_queue_inactive, mem, vm_page_t, pageq); \ - vm_page_pageable_external_count++; \ - } \ - mem->inactive = TRUE; \ - vm_page_inactive_count++; \ - token_new_pagecount++; \ - MACRO_END - #if DEVELOPMENT || DEBUG #define VM_PAGE_SPECULATIVE_USED_ADD() \ @@ -983,28 +1532,42 @@ extern void vm_page_queues_assert(vm_page_t mem, int val); #define VM_PAGE_CONSUME_CLUSTERED(mem) \ MACRO_BEGIN \ - pmap_lock_phys_page(mem->phys_page); \ + ppnum_t __phys_page; \ + __phys_page = VM_PAGE_GET_PHYS_PAGE(mem); \ + pmap_lock_phys_page(__phys_page); \ if (mem->clustered) { \ - assert(mem->object); \ - mem->object->pages_used++; \ + vm_object_t o; \ + o = VM_PAGE_OBJECT(mem); \ + assert(o); \ + o->pages_used++; \ mem->clustered = FALSE; \ VM_PAGE_SPECULATIVE_USED_ADD(); \ } \ - pmap_unlock_phys_page(mem->phys_page); \ + pmap_unlock_phys_page(__phys_page); \ MACRO_END #define VM_PAGE_COUNT_AS_PAGEIN(mem) \ MACRO_BEGIN \ + { \ + vm_object_t o; \ + o = VM_PAGE_OBJECT(mem); \ DTRACE_VM2(pgin, int, 1, (uint64_t *), NULL); \ current_task()->pageins++; \ - if (mem->object->internal) { \ + if (o->internal) { \ DTRACE_VM2(anonpgin, int, 1, (uint64_t *), NULL); \ } else { \ DTRACE_VM2(fspgin, int, 1, (uint64_t *), NULL); \ } \ + } \ MACRO_END +/* adjust for stolen pages accounted elsewhere */ +#define VM_PAGE_MOVE_STOLEN(page_count) \ + MACRO_BEGIN \ + vm_page_stolen_count -= (page_count); \ + vm_page_wire_count_initial -= (page_count); \ + MACRO_END #define DW_vm_page_unwire 0x01 #define DW_vm_page_wire 0x02 @@ -1028,7 +1591,7 @@ struct vm_page_delayed_work { int dw_mask; }; -void vm_page_do_delayed_work(vm_object_t object, struct vm_page_delayed_work *dwp, int dw_count); +void vm_page_do_delayed_work(vm_object_t object, vm_tag_t tag, struct vm_page_delayed_work *dwp, int dw_count); extern unsigned int vm_max_delayed_work_limit; @@ -1063,4 +1626,11 @@ extern vm_page_t vm_object_page_grab(vm_object_t); extern void vm_page_buckets_check(void); #endif /* VM_PAGE_BUCKETS_CHECK */ +extern void vm_page_queues_remove(vm_page_t mem, boolean_t remove_from_backgroundq); +extern void vm_page_remove_internal(vm_page_t page); +extern void vm_page_enqueue_inactive(vm_page_t mem, boolean_t first); +extern void vm_page_enqueue_active(vm_page_t mem, boolean_t first); +extern void vm_page_check_pageable_safe(vm_page_t page); + + #endif /* _VM_VM_PAGE_H_ */