2 * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University
34 * All Rights Reserved.
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
46 * Carnegie Mellon requests users of this software to return to
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
60 * Author: Avadis Tevanian, Jr., Michael Wayne Young
63 * Resident memory system definitions.
66 #ifndef _VM_VM_PAGE_H_
67 #define _VM_VM_PAGE_H_
71 #include <mach/boolean.h>
72 #include <mach/vm_prot.h>
73 #include <mach/vm_param.h>
74 #include <vm/vm_object.h>
75 #include <kern/queue.h>
76 #include <kern/lock.h>
78 #include <kern/macro_help.h>
79 #include <libkern/OSAtomic.h>
83 * VM_PAGE_MIN_SPECULATIVE_AGE_Q through VM_PAGE_MAX_SPECULATIVE_AGE_Q
84 * represents a set of aging bins that are 'protected'...
86 * VM_PAGE_SPECULATIVE_AGED_Q is a list of the speculative pages that have
87 * not yet been 'claimed' but have been aged out of the protective bins
88 * this occurs in vm_page_speculate when it advances to the next bin
89 * and discovers that it is still occupied... at that point, all of the
90 * pages in that bin are moved to the VM_PAGE_SPECULATIVE_AGED_Q. the pages
91 * in that bin are all guaranteed to have reached at least the maximum age
92 * we allow for a protected page... they can be older if there is no
93 * memory pressure to pull them from the bin, or there are no new speculative pages
94 * being generated to push them out.
95 * this list is the one that vm_pageout_scan will prefer when looking
96 * for pages to move to the underweight free list
98 * VM_PAGE_MAX_SPECULATIVE_AGE_Q * VM_PAGE_SPECULATIVE_Q_AGE_MS
99 * defines the amount of time a speculative page is normally
100 * allowed to live in the 'protected' state (i.e. not available
101 * to be stolen if vm_pageout_scan is running and looking for
102 * pages)... however, if the total number of speculative pages
103 * in the protected state exceeds our limit (defined in vm_pageout.c)
104 * and there are none available in VM_PAGE_SPECULATIVE_AGED_Q, then
105 * vm_pageout_scan is allowed to steal pages from the protected
106 * bucket even if they are underage.
108 * vm_pageout_scan is also allowed to pull pages from a protected
109 * bin if the bin has reached the "age of consent" we've set
111 #define VM_PAGE_MAX_SPECULATIVE_AGE_Q 10
112 #define VM_PAGE_MIN_SPECULATIVE_AGE_Q 1
113 #define VM_PAGE_SPECULATIVE_AGED_Q 0
115 #define VM_PAGE_SPECULATIVE_Q_AGE_MS 500
118 struct vm_speculative_age_q
{
120 * memory queue for speculative pages via clustered pageins
123 mach_timespec_t age_ts
;
128 struct vm_speculative_age_q vm_page_queue_speculative
[];
130 extern int speculative_steal_index
;
131 extern int speculative_age_index
;
135 * Management of resident (logical) pages.
137 * A small structure is kept for each resident
138 * page, indexed by page number. Each structure
139 * is an element of several lists:
141 * A hash table bucket used to quickly
142 * perform object/offset lookups
144 * A list of all pages for a given object,
145 * so they can be quickly deactivated at
146 * time of deallocation.
148 * An ordered list of pages due for pageout.
150 * In addition, the structure contains the object
151 * and offset to which this page belongs (for pageout),
152 * and sundry status bits.
154 * Fields in this structure are locked either by the lock on the
155 * object that the page belongs to (O) or by the lock on the page
156 * queues (P). [Some fields require that both locks be held to
157 * change that field; holding either lock is sufficient to read.]
161 queue_chain_t pageq
; /* queue info for FIFO */
162 /* queue or free list (P) */
164 queue_chain_t listq
; /* all pages in same object (O) */
165 struct vm_page
*next
; /* VP bucket link (O) */
167 vm_object_t object
; /* which object am I in (O&P) */
168 vm_object_offset_t offset
; /* offset into that object (O,P) */
171 * The following word of flags is protected
172 * by the "page queues" lock.
174 * we use the 'wire_count' field to store the local
175 * queue id if local queues are enabled...
176 * see the comments at 'VM_PAGE_QUEUES_REMOVE' as to
177 * why this is safe to do
179 #define local_id wire_count
180 unsigned int wire_count
:16, /* how many wired down maps use me? (O&P) */
181 /* boolean_t */ inactive
:1, /* page is in inactive list (P) */
182 active
:1, /* page is in active list (P) */
183 pageout_queue
:1,/* page is on queue for pageout (P) */
184 speculative
:1, /* page is on speculative list (P) */
185 laundry
:1, /* page is being cleaned now (P)*/
186 free
:1, /* page is on free list (P) */
187 reference
:1, /* page has been used (P) */
188 gobbled
:1, /* page used internally (P) */
189 private:1, /* Page should not be returned to
190 * the free list (P) */
191 throttled
:1, /* pager is not responding (P) */
193 __unused_pageq_bits
:5; /* 5 bits available here */
195 ppnum_t phys_page
; /* Physical address of page, passed
196 * to pmap_enter (read-only) */
199 * The following word of flags is protected
200 * by the "VM object" lock.
203 /* boolean_t */ busy
:1, /* page is in transit (O) */
204 wanted
:1, /* someone is waiting for page (O) */
205 tabled
:1, /* page is in VP table (O) */
206 fictitious
:1, /* Physical page doesn't exist (O) */
208 * IMPORTANT: the "pmapped" bit can be turned on while holding the
209 * VM object "shared" lock. See vm_fault_enter().
210 * This is OK as long as it's the only bit in this bit field that
211 * can be updated without holding the VM object "exclusive" lock.
213 pmapped
:1, /* page has been entered at some
214 * point into a pmap (O **shared**) */
215 wpmapped
:1, /* page has been entered at some
216 * point into a pmap for write (O) */
217 pageout
:1, /* page wired & busy for pageout (O) */
218 absent
:1, /* Data has been requested, but is
219 * not yet available (O) */
220 error
:1, /* Data manager was unable to provide
221 * data due to error (O) */
222 dirty
:1, /* Page must be cleaned (O) */
223 cleaning
:1, /* Page clean has begun (O) */
224 precious
:1, /* Page is precious; data must be
225 * returned even if clean (O) */
226 clustered
:1, /* page is not the faulted page (O) */
227 overwriting
:1, /* Request to unlock has been made
228 * without having data. (O)
229 * [See vm_fault_page_overwrite] */
230 restart
:1, /* Page was pushed higher in shadow
231 chain by copy_call-related pagers;
232 start again at top of chain */
233 unusual
:1, /* Page is absent, error, restart or
235 encrypted
:1, /* encrypted for secure swap (O) */
236 encrypted_cleaning
:1, /* encrypting page */
237 list_req_pending
:1, /* pagein/pageout alt mechanism */
238 /* allows creation of list */
239 /* requests on pages that are */
240 /* actively being paged. */
241 dump_cleaning
:1, /* set by the pageout daemon when */
242 /* a page being cleaned is */
243 /* encountered and targeted as */
244 /* a pageout candidate */
245 cs_validated
:1, /* code-signing: page was checked */
246 cs_tainted
:1, /* code-signing: page is tainted */
247 no_cache
:1, /* page is not to be cached and */
248 /* should be reused ahead of */
253 __unused_object_bits
:6; /* 6 bits available here */
256 unsigned int __unused_padding
; /* Pad structure explicitly
257 * to 8-byte multiple for LP64 */
261 #define DEBUG_ENCRYPTED_SWAP 1
262 #if DEBUG_ENCRYPTED_SWAP
263 #define ASSERT_PAGE_DECRYPTED(page) \
265 if ((page)->encrypted) { \
266 panic("VM page %p should not be encrypted here\n", \
270 #else /* DEBUG_ENCRYPTED_SWAP */
271 #define ASSERT_PAGE_DECRYPTED(page) assert(!(page)->encrypted)
272 #endif /* DEBUG_ENCRYPTED_SWAP */
274 typedef struct vm_page
*vm_page_t
;
277 typedef struct vm_locks_array
{
278 char pad
__attribute__ ((aligned (64)));
279 lck_mtx_t vm_page_queue_lock2
__attribute__ ((aligned (64)));
280 lck_mtx_t vm_page_queue_free_lock2
__attribute__ ((aligned (64)));
281 char pad2
__attribute__ ((aligned (64)));
285 #define VM_PAGE_WIRED(m) ((!(m)->local && (m)->wire_count))
286 #define VM_PAGE_NULL ((vm_page_t) 0)
287 #define NEXT_PAGE(m) ((vm_page_t) (m)->pageq.next)
288 #define NEXT_PAGE_PTR(m) ((vm_page_t *) &(m)->pageq.next)
291 * XXX The unusual bit should not be necessary. Most of the bit
292 * XXX fields above really want to be masks.
296 * For debugging, this macro can be defined to perform
297 * some useful check on a page structure.
300 #define VM_PAGE_CHECK(mem) \
302 VM_PAGE_QUEUES_ASSERT(mem, 1); \
307 * The free page list is actually n lists, one per color,
308 * where the number of colors is a function of the machine's
309 * cache geometry set at system initialization. To disable
310 * coloring, set vm_colors to 1 and vm_color_mask to 0.
311 * The boot-arg "colors" may be used to override vm_colors.
312 * Note that there is little harm in having more colors than needed.
315 #define MAX_COLORS 128
316 #define DEFAULT_COLORS 32
319 unsigned int vm_colors
; /* must be in range 1..MAX_COLORS */
321 unsigned int vm_color_mask
; /* must be (vm_colors-1) */
323 unsigned int vm_cache_geometry_colors
; /* optimal #colors based on cache geometry */
326 * Wired memory is a very limited resource and we can't let users exhaust it
327 * and deadlock the entire system. We enforce the following limits:
329 * vm_user_wire_limit (default: all memory minus vm_global_no_user_wire_amount)
330 * how much memory can be user-wired in one user task
332 * vm_global_user_wire_limit (default: same as vm_user_wire_limit)
333 * how much memory can be user-wired in all user tasks
335 * vm_global_no_user_wire_amount (default: VM_NOT_USER_WIREABLE)
336 * how much memory must remain user-unwired at any time
338 #define VM_NOT_USER_WIREABLE (64*1024*1024) /* 64MB */
340 vm_map_size_t vm_user_wire_limit
;
342 vm_map_size_t vm_global_user_wire_limit
;
344 vm_map_size_t vm_global_no_user_wire_amount
;
347 * Each pageable resident page falls into one of three lists:
350 * Available for allocation now. The free list is
351 * actually an array of lists, one per color.
353 * Not referenced in any map, but still has an
354 * object/offset-page mapping, and may be dirty.
355 * This is the list of pages that should be
356 * paged out next. There are actually two
357 * inactive lists, one for pages brought in from
358 * disk or other backing store, and another
359 * for "zero-filled" pages. See vm_pageout_scan()
360 * for the distinction and usage.
362 * A list of pages which have been placed in
363 * at least one physical map. This list is
364 * ordered, in LRU-like fashion.
368 #define VPL_LOCK_SPIN 1
371 unsigned int vpl_count
;
372 queue_head_t vpl_queue
;
377 lck_mtx_ext_t vpl_lock_ext
;
383 char cache_line_pad
[128];
388 unsigned int vm_page_local_q_count
;
390 struct vplq
*vm_page_local_q
;
392 unsigned int vm_page_local_q_soft_limit
;
394 unsigned int vm_page_local_q_hard_limit
;
396 vm_locks_array_t vm_page_locks
;
399 queue_head_t vm_page_queue_free
[MAX_COLORS
]; /* memory free queue */
401 queue_head_t vm_lopage_queue_free
; /* low memory free queue */
403 vm_page_t vm_page_queue_fictitious
; /* fictitious free queue */
405 queue_head_t vm_page_queue_active
; /* active memory queue */
407 queue_head_t vm_page_queue_inactive
; /* inactive memory queue for normal pages */
409 queue_head_t vm_page_queue_zf
; /* inactive memory queue for zero fill */
411 queue_head_t vm_page_queue_throttled
; /* memory queue for throttled pageout pages */
414 vm_offset_t first_phys_addr
; /* physical address for first_page */
416 vm_offset_t last_phys_addr
; /* physical address for last_page */
419 unsigned int vm_page_free_count
; /* How many pages are free? (sum of all colors) */
421 unsigned int vm_page_fictitious_count
;/* How many fictitious pages are free? */
423 unsigned int vm_page_active_count
; /* How many pages are active? */
425 unsigned int vm_page_inactive_count
; /* How many pages are inactive? */
427 unsigned int vm_page_throttled_count
;/* How many inactives are throttled */
429 unsigned int vm_page_speculative_count
; /* How many speculative pages are unclaimed? */
431 unsigned int vm_page_wire_count
; /* How many pages are wired? */
433 unsigned int vm_page_free_target
; /* How many do we want free? */
435 unsigned int vm_page_free_min
; /* When to wakeup pageout */
437 unsigned int vm_page_throttle_limit
; /* When to throttle new page creation */
439 uint32_t vm_page_creation_throttle
; /* When to throttle new page creation */
441 unsigned int vm_page_inactive_target
;/* How many do we want inactive? */
443 unsigned int vm_page_inactive_min
; /* When do wakeup pageout */
445 unsigned int vm_page_free_reserved
; /* How many pages reserved to do pageout */
447 unsigned int vm_page_throttle_count
; /* Count of page allocations throttled */
449 unsigned int vm_page_gobble_count
;
451 #if DEVELOPMENT || DEBUG
453 unsigned int vm_page_speculative_used
;
457 unsigned int vm_page_purgeable_count
;/* How many pages are purgeable now ? */
459 unsigned int vm_page_purgeable_wired_count
;/* How many purgeable pages are wired now ? */
461 uint64_t vm_page_purged_count
; /* How many pages got purged so far ? */
463 extern unsigned int vm_page_free_wanted
;
464 /* how many threads are waiting for memory */
466 extern unsigned int vm_page_free_wanted_privileged
;
467 /* how many VM privileged threads are waiting for memory */
469 extern ppnum_t vm_page_fictitious_addr
;
470 /* (fake) phys_addr of fictitious pages */
472 extern ppnum_t vm_page_guard_addr
;
473 /* (fake) phys_addr of guard pages */
476 extern boolean_t vm_page_deactivate_hint
;
479 0 = all pages avail ( default. )
480 1 = disable high mem ( cap max pages to 4G)
483 extern int vm_himemory_mode
;
485 extern boolean_t vm_lopage_needed
;
486 extern uint32_t vm_lopage_free_count
;
487 extern uint32_t vm_lopage_free_limit
;
488 extern uint32_t vm_lopage_lowater
;
489 extern boolean_t vm_lopage_refill
;
490 extern uint64_t max_valid_dma_address
;
491 extern ppnum_t max_valid_low_ppnum
;
494 * Prototypes for functions exported by this module.
496 extern void vm_page_bootstrap(
498 vm_offset_t
*endp
) __attribute__((section("__TEXT, initcode")));
500 extern void vm_page_module_init(void) __attribute__((section("__TEXT, initcode")));
502 extern void vm_page_init_local_q(void);
504 extern void vm_page_create(
508 extern vm_page_t
vm_page_lookup(
510 vm_object_offset_t offset
);
512 extern vm_page_t
vm_page_grab_fictitious(void);
514 extern vm_page_t
vm_page_grab_guard(void);
516 extern void vm_page_release_fictitious(
519 extern void vm_page_more_fictitious(void);
521 extern int vm_pool_low(void);
523 extern vm_page_t
vm_page_grab(void);
525 extern vm_page_t
vm_page_grablo(void);
527 extern void vm_page_release(
530 extern boolean_t
vm_page_wait(
533 extern vm_page_t
vm_page_alloc(
535 vm_object_offset_t offset
);
537 extern vm_page_t
vm_page_alloclo(
539 vm_object_offset_t offset
);
541 extern vm_page_t
vm_page_alloc_guard(
543 vm_object_offset_t offset
);
545 extern void vm_page_init(
550 extern void vm_page_free(
553 extern void vm_page_free_unlocked(
555 boolean_t remove_from_hash
);
557 extern void vm_page_activate(
560 extern void vm_page_deactivate(
563 extern void vm_page_deactivate_internal(
565 boolean_t clear_hw_reference
);
567 extern void vm_page_lru(
570 extern void vm_page_speculate(
574 extern void vm_page_speculate_ageit(
575 struct vm_speculative_age_q
*aq
);
577 extern void vm_page_reactivate_all_throttled(void);
579 extern void vm_page_reactivate_local(uint32_t lid
, boolean_t force
, boolean_t nolocks
);
581 extern void vm_page_rename(
583 vm_object_t new_object
,
584 vm_object_offset_t new_offset
,
585 boolean_t encrypted_ok
);
587 extern void vm_page_insert(
590 vm_object_offset_t offset
);
592 extern void vm_page_insert_internal(
595 vm_object_offset_t offset
,
596 boolean_t queues_lock_held
,
597 boolean_t insert_in_hash
);
599 extern void vm_page_replace(
602 vm_object_offset_t offset
);
604 extern void vm_page_remove(
606 boolean_t remove_from_hash
);
608 extern void vm_page_zero_fill(
611 extern void vm_page_part_zero_fill(
616 extern void vm_page_copy(
618 vm_page_t dest_page
);
620 extern void vm_page_part_copy(
627 extern void vm_page_wire(
630 extern void vm_page_unwire(
634 extern void vm_set_page_size(void);
636 extern void vm_page_gobble(
639 extern void vm_page_validate_cs(vm_page_t page
);
640 extern void vm_page_validate_cs_mapped(
644 extern void vm_page_free_prepare_queues(
647 extern void vm_page_free_prepare_object(
649 boolean_t remove_from_hash
);
652 * Functions implemented as macros. m->wanted and m->busy are
653 * protected by the object lock.
656 #define PAGE_ASSERT_WAIT(m, interruptible) \
657 (((m)->wanted = TRUE), \
658 assert_wait((event_t) (m), (interruptible)))
660 #define PAGE_SLEEP(o, m, interruptible) \
661 (((m)->wanted = TRUE), \
662 thread_sleep_vm_object((o), (m), (interruptible)))
664 #define PAGE_WAKEUP_DONE(m) \
668 (m)->wanted = FALSE; \
669 thread_wakeup((event_t) (m)); \
673 #define PAGE_WAKEUP(m) \
676 (m)->wanted = FALSE; \
677 thread_wakeup((event_t) (m)); \
681 #define VM_PAGE_FREE(p) \
683 vm_page_free_unlocked(p, TRUE); \
686 #define VM_PAGE_GRAB_FICTITIOUS(M) \
688 while ((M = vm_page_grab_fictitious()) == VM_PAGE_NULL) \
689 vm_page_more_fictitious(); \
692 #define VM_PAGE_WAIT() ((void)vm_page_wait(THREAD_UNINT))
694 #define vm_page_queue_lock (vm_page_locks.vm_page_queue_lock2)
695 #define vm_page_queue_free_lock (vm_page_locks.vm_page_queue_free_lock2)
697 #define vm_page_lock_queues() lck_mtx_lock(&vm_page_queue_lock)
698 #define vm_page_unlock_queues() lck_mtx_unlock(&vm_page_queue_lock)
700 #define vm_page_lockspin_queues() lck_mtx_lock_spin(&vm_page_queue_lock)
701 #define vm_page_trylockspin_queues() lck_mtx_try_lock_spin(&vm_page_queue_lock)
702 #define vm_page_lockconvert_queues() lck_mtx_convert_spin(&vm_page_queue_lock)
705 #define VPL_LOCK_INIT(vlq, vpl_grp, vpl_attr) lck_spin_init(&vlq->vpl_lock, vpl_grp, vpl_attr)
706 #define VPL_LOCK(vpl) lck_spin_lock(vpl)
707 #define VPL_UNLOCK(vpl) lck_spin_unlock(vpl)
709 #define VPL_LOCK_INIT(vlq, vpl_grp, vpl_attr) lck_mtx_init_ext(&vlq->vpl_lock, &vlq->vpl_lock_ext, vpl_grp, vpl_attr)
710 #define VPL_LOCK(vpl) lck_mtx_lock_spin(vpl)
711 #define VPL_UNLOCK(vpl) lck_mtx_unlock(vpl)
715 extern void vm_page_queues_assert(vm_page_t mem
, int val
);
716 #define VM_PAGE_QUEUES_ASSERT(mem, val) vm_page_queues_assert((mem), (val))
718 #define VM_PAGE_QUEUES_ASSERT(mem, val)
723 * 'vm_fault_enter' will place newly created pages (zero-fill and COW) onto the
724 * local queues if they exist... its the only spot in the system where we add pages
725 * to those queues... once on those queues, those pages can only move to one of the
726 * global page queues or the free queues... they NEVER move from local q to local q.
727 * the 'local' state is stable when VM_PAGE_QUEUES_REMOVE is called since we're behind
728 * the global vm_page_queue_lock at this point... we still need to take the local lock
729 * in case this operation is being run on a different CPU then the local queue's identity,
730 * but we don't have to worry about the page moving to a global queue or becoming wired
731 * while we're grabbing the local lock since those operations would require the global
732 * vm_page_queue_lock to be held, and we already own it.
734 * this is why its safe to utilze the wire_count field in the vm_page_t as the local_id...
735 * 'wired' and local are ALWAYS mutually exclusive conditions.
737 #define VM_PAGE_QUEUES_REMOVE(mem) \
739 VM_PAGE_QUEUES_ASSERT(mem, 1); \
740 assert(!mem->laundry); \
741 assert(!mem->pageout_queue); \
744 assert(mem->object != kernel_object); \
745 assert(!mem->inactive && !mem->speculative); \
746 assert(!mem->active && !mem->throttled); \
747 lq = &vm_page_local_q[mem->local_id].vpl_un.vpl; \
748 VPL_LOCK(&lq->vpl_lock); \
749 queue_remove(&lq->vpl_queue, \
750 mem, vm_page_t, pageq); \
751 mem->local = FALSE; \
754 VPL_UNLOCK(&lq->vpl_lock); \
757 assert(mem->object != kernel_object); \
758 assert(!mem->inactive && !mem->speculative); \
759 assert(!mem->throttled); \
760 queue_remove(&vm_page_queue_active, \
761 mem, vm_page_t, pageq); \
762 mem->active = FALSE; \
763 if (!mem->fictitious) { \
764 vm_page_active_count--; \
766 assert(mem->phys_page == \
767 vm_page_fictitious_addr); \
771 else if (mem->inactive) { \
772 assert(mem->object != kernel_object); \
773 assert(!mem->active && !mem->speculative); \
774 assert(!mem->throttled); \
775 if (mem->zero_fill) { \
776 queue_remove(&vm_page_queue_zf, \
777 mem, vm_page_t, pageq); \
778 vm_zf_queue_count--; \
780 queue_remove(&vm_page_queue_inactive, \
781 mem, vm_page_t, pageq); \
783 mem->inactive = FALSE; \
784 if (!mem->fictitious) { \
785 vm_page_inactive_count--; \
786 vm_purgeable_q_advance_all(); \
788 assert(mem->phys_page == \
789 vm_page_fictitious_addr); \
793 else if (mem->throttled) { \
794 assert(!mem->active && !mem->inactive); \
795 assert(!mem->speculative); \
796 queue_remove(&vm_page_queue_throttled, \
797 mem, vm_page_t, pageq); \
798 mem->throttled = FALSE; \
799 if (!mem->fictitious) \
800 vm_page_throttled_count--; \
803 else if (mem->speculative) { \
804 assert(!mem->active && !mem->inactive); \
805 assert(!mem->throttled); \
806 assert(!mem->fictitious); \
807 remque(&mem->pageq); \
808 mem->speculative = FALSE; \
809 vm_page_speculative_count--; \
811 mem->pageq.next = NULL; \
812 mem->pageq.prev = NULL; \
813 VM_PAGE_QUEUES_ASSERT(mem, 0); \
817 #if DEVELOPMENT || DEBUG
818 #define VM_PAGE_SPECULATIVE_USED_ADD() \
820 OSAddAtomic(1, &vm_page_speculative_used); \
823 #define VM_PAGE_SPECULATIVE_USED_ADD()
827 #define VM_PAGE_CONSUME_CLUSTERED(mem) \
829 if (mem->clustered) { \
830 assert(mem->object); \
831 mem->object->pages_used++; \
832 mem->clustered = FALSE; \
833 VM_PAGE_SPECULATIVE_USED_ADD(); \
837 #endif /* _VM_VM_PAGE_H_ */