]> git.saurik.com Git - apple/xnu.git/blame - osfmk/vm/vm_page.h
projects / apple / xnu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
xnu-1504.7.4.tar.gz
[apple/xnu.git] / osfmk / vm / vm_page.h
CommitLineData
1c79356b 1/*
2d21ac55 2 * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
1c79356b 3 *
2d21ac55 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
1c79356b 5 *
2d21ac55
A		 6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
8f6c56a5 14 *
2d21ac55
A		 15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
8f6c56a5
A		 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
2d21ac55
A		 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
8f6c56a5 25 *
2d21ac55 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
1c79356b
A		 27 */
28/*
29 * @OSF_COPYRIGHT@
30 */
31/*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56/*
57 */
58/*
59 * File: vm/vm_page.h
60 * Author: Avadis Tevanian, Jr., Michael Wayne Young
61 * Date: 1985
62 *
63 * Resident memory system definitions.
64 */
65
66#ifndef _VM_VM_PAGE_H_
67#define _VM_VM_PAGE_H_
68
91447636
A		 69#include <debug.h>
70
1c79356b
A		 71#include <mach/boolean.h>
72#include <mach/vm_prot.h>
73#include <mach/vm_param.h>
74#include <vm/vm_object.h>
75#include <kern/queue.h>
76#include <kern/lock.h>
77
78#include <kern/macro_help.h>
2d21ac55
A		 79#include <libkern/OSAtomic.h>
80
1c79356b 81
0b4e3aa0 82/*
2d21ac55
A		 83 * VM_PAGE_MIN_SPECULATIVE_AGE_Q through VM_PAGE_MAX_SPECULATIVE_AGE_Q
84 * represents a set of aging bins that are 'protected'...
85 *
86 * VM_PAGE_SPECULATIVE_AGED_Q is a list of the speculative pages that have
87 * not yet been 'claimed' but have been aged out of the protective bins
88 * this occurs in vm_page_speculate when it advances to the next bin
89 * and discovers that it is still occupied... at that point, all of the
90 * pages in that bin are moved to the VM_PAGE_SPECULATIVE_AGED_Q. the pages
91 * in that bin are all guaranteed to have reached at least the maximum age
92 * we allow for a protected page... they can be older if there is no
93 * memory pressure to pull them from the bin, or there are no new speculative pages
94 * being generated to push them out.
95 * this list is the one that vm_pageout_scan will prefer when looking
96 * for pages to move to the underweight free list
97 *
98 * VM_PAGE_MAX_SPECULATIVE_AGE_Q * VM_PAGE_SPECULATIVE_Q_AGE_MS
99 * defines the amount of time a speculative page is normally
100 * allowed to live in the 'protected' state (i.e. not available
101 * to be stolen if vm_pageout_scan is running and looking for
102 * pages)... however, if the total number of speculative pages
103 * in the protected state exceeds our limit (defined in vm_pageout.c)
104 * and there are none available in VM_PAGE_SPECULATIVE_AGED_Q, then
105 * vm_pageout_scan is allowed to steal pages from the protected
106 * bucket even if they are underage.
107 *
108 * vm_pageout_scan is also allowed to pull pages from a protected
109 * bin if the bin has reached the "age of consent" we've set
0b4e3aa0 110 */
2d21ac55
A		 111#define VM_PAGE_MAX_SPECULATIVE_AGE_Q 10
112#define VM_PAGE_MIN_SPECULATIVE_AGE_Q 1
113#define VM_PAGE_SPECULATIVE_AGED_Q 0
114
115#define VM_PAGE_SPECULATIVE_Q_AGE_MS 500
116
117
118struct vm_speculative_age_q {
119 /*
120 * memory queue for speculative pages via clustered pageins
121 */
122 queue_head_t age_q;
123 mach_timespec_t age_ts;
124};
0b4e3aa0
A		 125
126
2d21ac55
A		 127extern
128struct vm_speculative_age_q vm_page_queue_speculative[];
0b4e3aa0 129
2d21ac55
A		 130extern int speculative_steal_index;
131extern int speculative_age_index;
9bccf70c 132
0b4e3aa0 133
1c79356b
A		 134/*
135 * Management of resident (logical) pages.
136 *
137 * A small structure is kept for each resident
138 * page, indexed by page number. Each structure
139 * is an element of several lists:
140 *
141 * A hash table bucket used to quickly
142 * perform object/offset lookups
143 *
144 * A list of all pages for a given object,
145 * so they can be quickly deactivated at
146 * time of deallocation.
147 *
148 * An ordered list of pages due for pageout.
149 *
150 * In addition, the structure contains the object
151 * and offset to which this page belongs (for pageout),
152 * and sundry status bits.
153 *
154 * Fields in this structure are locked either by the lock on the
155 * object that the page belongs to (O) or by the lock on the page
156 * queues (P). [Some fields require that both locks be held to
157 * change that field; holding either lock is sufficient to read.]
158 */
159
160struct vm_page {
b0d623f7
A		 161 queue_chain_t pageq; /* queue info for FIFO */
162 /* queue or free list (P) */
163
1c79356b
A		 164 queue_chain_t listq; /* all pages in same object (O) */
165 struct vm_page *next; /* VP bucket link (O) */
166
167 vm_object_t object; /* which object am I in (O&P) */
168 vm_object_offset_t offset; /* offset into that object (O,P) */
169
91447636
A		 170 /*
171 * The following word of flags is protected
172 * by the "page queues" lock.
b0d623f7
A		 173 *
174 * we use the 'wire_count' field to store the local
175 * queue id if local queues are enabled...
176 * see the comments at 'VM_PAGE_QUEUES_REMOVE' as to
177 * why this is safe to do
91447636 178 */
b0d623f7 179#define local_id wire_count
1c79356b 180 unsigned int wire_count:16, /* how many wired down maps use me? (O&P) */
1c79356b
A		 181 /* boolean_t */ inactive:1, /* page is in inactive list (P) */
182 active:1, /* page is in active list (P) */
91447636 183 pageout_queue:1,/* page is on queue for pageout (P) */
2d21ac55 184 speculative:1, /* page is on speculative list (P) */
1c79356b
A		 185 laundry:1, /* page is being cleaned now (P)*/
186 free:1, /* page is on free list (P) */
187 reference:1, /* page has been used (P) */
0b4e3aa0
A		 188 gobbled:1, /* page used internally (P) */
189 private:1, /* Page should not be returned to
91447636 190 * the free list (P) */
2d21ac55 191 throttled:1, /* pager is not responding (P) */
b0d623f7 192 local:1,
2d21ac55 193 __unused_pageq_bits:5; /* 5 bits available here */
1c79356b 194
b0d623f7
A		 195 ppnum_t phys_page; /* Physical address of page, passed
196 * to pmap_enter (read-only) */
197
91447636
A		 198 /*
199 * The following word of flags is protected
200 * by the "VM object" lock.
201 */
1c79356b
A		 202 unsigned int
203 /* boolean_t */ busy:1, /* page is in transit (O) */
204 wanted:1, /* someone is waiting for page (O) */
205 tabled:1, /* page is in VP table (O) */
206 fictitious:1, /* Physical page doesn't exist (O) */
b0d623f7
A		 207 /*
208 * IMPORTANT: the "pmapped" bit can be turned on while holding the
209 * VM object "shared" lock. See vm_fault_enter().
210 * This is OK as long as it's the only bit in this bit field that
211 * can be updated without holding the VM object "exclusive" lock.
212 */
2d21ac55 213 pmapped:1, /* page has been entered at some
b0d623f7 214 * point into a pmap (O **shared**) */
4a3eedf9
A		 215 wpmapped:1, /* page has been entered at some
216 * point into a pmap for write (O) */
b0d623f7 217 pageout:1, /* page wired & busy for pageout (O) */
1c79356b
A		 218 absent:1, /* Data has been requested, but is
219 * not yet available (O) */
220 error:1, /* Data manager was unable to provide
221 * data due to error (O) */
222 dirty:1, /* Page must be cleaned (O) */
223 cleaning:1, /* Page clean has begun (O) */
224 precious:1, /* Page is precious; data must be
225 * returned even if clean (O) */
226 clustered:1, /* page is not the faulted page (O) */
227 overwriting:1, /* Request to unlock has been made
228 * without having data. (O)
229 * [See vm_fault_page_overwrite] */
230 restart:1, /* Page was pushed higher in shadow
231 chain by copy_call-related pagers;
232 start again at top of chain */
1c79356b
A		 233 unusual:1, /* Page is absent, error, restart or
234 page locked */
91447636 235 encrypted:1, /* encrypted for secure swap (O) */
2d21ac55 236 encrypted_cleaning:1, /* encrypting page */
1c79356b
A		 237 list_req_pending:1, /* pagein/pageout alt mechanism */
238 /* allows creation of list */
239 /* requests on pages that are */
240 /* actively being paged. */
2d21ac55 241 dump_cleaning:1, /* set by the pageout daemon when */
0b4e3aa0
A		 242 /* a page being cleaned is */
243 /* encountered and targeted as */
244 /* a pageout candidate */
2d21ac55
A		 245 cs_validated:1, /* code-signing: page was checked */
246 cs_tainted:1, /* code-signing: page is tainted */
247 no_cache:1, /* page is not to be cached and */
248 /* should be reused ahead of */
249 /* other pages */
2d21ac55 250 zero_fill:1,
b0d623f7
A		 251 reusable:1,
252 __unused_object_bits:7; /* 7 bits available here */
1c79356b 253
b0d623f7
A		 254#if __LP64__
255 unsigned int __unused_padding; /* Pad structure explicitly
256 * to 8-byte multiple for LP64 */
257#endif
1c79356b
A		 258};
259
91447636
A		 260#define DEBUG_ENCRYPTED_SWAP 1
261#if DEBUG_ENCRYPTED_SWAP
262#define ASSERT_PAGE_DECRYPTED(page) \
263 MACRO_BEGIN \
264 if ((page)->encrypted) { \
265 panic("VM page %p should not be encrypted here\n", \
266 (page)); \
267 } \
268 MACRO_END
269#else /* DEBUG_ENCRYPTED_SWAP */
270#define ASSERT_PAGE_DECRYPTED(page) assert(!(page)->encrypted)
271#endif /* DEBUG_ENCRYPTED_SWAP */
272
1c79356b
A		 273typedef struct vm_page *vm_page_t;
274
b0d623f7
A		 275
276typedef struct vm_locks_array {
277 char pad __attribute__ ((aligned (64)));
278 lck_mtx_t vm_page_queue_lock2 __attribute__ ((aligned (64)));
279 lck_mtx_t vm_page_queue_free_lock2 __attribute__ ((aligned (64)));
280 char pad2 __attribute__ ((aligned (64)));
281} vm_locks_array_t;
282
283
284#define VM_PAGE_WIRED(m) ((!(m)->local && (m)->wire_count))
1c79356b 285#define VM_PAGE_NULL ((vm_page_t) 0)
91447636 286#define NEXT_PAGE(m) ((vm_page_t) (m)->pageq.next)
e5568f75 287#define NEXT_PAGE_PTR(m) ((vm_page_t *) &(m)->pageq.next)
1c79356b
A		 288
289/*
290 * XXX The unusual bit should not be necessary. Most of the bit
291 * XXX fields above really want to be masks.
292 */
293
294/*
295 * For debugging, this macro can be defined to perform
296 * some useful check on a page structure.
297 */
298
b0d623f7
A		 299#define VM_PAGE_CHECK(mem) \
300 MACRO_BEGIN \
301 VM_PAGE_QUEUES_ASSERT(mem, 1); \
302 MACRO_END
2d21ac55
A		 303
304/* Page coloring:
305 *
306 * The free page list is actually n lists, one per color,
307 * where the number of colors is a function of the machine's
308 * cache geometry set at system initialization. To disable
309 * coloring, set vm_colors to 1 and vm_color_mask to 0.
310 * The boot-arg "colors" may be used to override vm_colors.
311 * Note that there is little harm in having more colors than needed.
312 */
313
314#define MAX_COLORS 128
315#define DEFAULT_COLORS 32
316
317extern
318unsigned int vm_colors; /* must be in range 1..MAX_COLORS */
319extern
320unsigned int vm_color_mask; /* must be (vm_colors-1) */
321extern
322unsigned int vm_cache_geometry_colors; /* optimal #colors based on cache geometry */
1c79356b 323
b0d623f7
A		 324/*
325 * Wired memory is a very limited resource and we can't let users exhaust it
326 * and deadlock the entire system. We enforce the following limits:
327 *
328 * vm_user_wire_limit (default: all memory minus vm_global_no_user_wire_amount)
329 * how much memory can be user-wired in one user task
330 *
331 * vm_global_user_wire_limit (default: same as vm_user_wire_limit)
332 * how much memory can be user-wired in all user tasks
333 *
334 * vm_global_no_user_wire_amount (default: VM_NOT_USER_WIREABLE)
335 * how much memory must remain user-unwired at any time
336 */
337#define VM_NOT_USER_WIREABLE (64*1024*1024) /* 64MB */
338extern
339vm_map_size_t vm_user_wire_limit;
340extern
341vm_map_size_t vm_global_user_wire_limit;
342extern
343vm_map_size_t vm_global_no_user_wire_amount;
344
1c79356b
A		 345/*
346 * Each pageable resident page falls into one of three lists:
347 *
348 * free
2d21ac55
A		 349 * Available for allocation now. The free list is
350 * actually an array of lists, one per color.
1c79356b
A		 351 * inactive
352 * Not referenced in any map, but still has an
353 * object/offset-page mapping, and may be dirty.
354 * This is the list of pages that should be
2d21ac55
A		 355 * paged out next. There are actually two
356 * inactive lists, one for pages brought in from
357 * disk or other backing store, and another
358 * for "zero-filled" pages. See vm_pageout_scan()
359 * for the distinction and usage.
1c79356b
A		 360 * active
361 * A list of pages which have been placed in
362 * at least one physical map. This list is
363 * ordered, in LRU-like fashion.
364 */
365
b0d623f7
A		 366
367#define VPL_LOCK_SPIN 1
368
369struct vpl {
370 unsigned int vpl_count;
371 queue_head_t vpl_queue;
372#ifdef VPL_LOCK_SPIN
373 lck_spin_t vpl_lock;
374#else
375 lck_mtx_t vpl_lock;
376 lck_mtx_ext_t vpl_lock_ext;
377#endif
378};
379
380struct vplq {
381 union {
382 char cache_line_pad[128];
383 struct vpl vpl;
384 } vpl_un;
385};
386extern
387unsigned int vm_page_local_q_count;
388extern
389struct vplq *vm_page_local_q;
390extern
391unsigned int vm_page_local_q_soft_limit;
392extern
393unsigned int vm_page_local_q_hard_limit;
394extern
395vm_locks_array_t vm_page_locks;
396
1c79356b 397extern
2d21ac55
A		 398queue_head_t vm_page_queue_free[MAX_COLORS]; /* memory free queue */
399extern
400queue_head_t vm_lopage_queue_free; /* low memory free queue */
1c79356b
A		 401extern
402vm_page_t vm_page_queue_fictitious; /* fictitious free queue */
403extern
404queue_head_t vm_page_queue_active; /* active memory queue */
405extern
2d21ac55
A		 406queue_head_t vm_page_queue_inactive; /* inactive memory queue for normal pages */
407extern
9bccf70c 408queue_head_t vm_page_queue_zf; /* inactive memory queue for zero fill */
b0d623f7 409extern
2d21ac55 410queue_head_t vm_page_queue_throttled; /* memory queue for throttled pageout pages */
1c79356b
A		 411
412extern
413vm_offset_t first_phys_addr; /* physical address for first_page */
414extern
415vm_offset_t last_phys_addr; /* physical address for last_page */
416
417extern
2d21ac55 418unsigned int vm_page_free_count; /* How many pages are free? (sum of all colors) */
1c79356b 419extern
91447636 420unsigned int vm_page_fictitious_count;/* How many fictitious pages are free? */
1c79356b 421extern
91447636 422unsigned int vm_page_active_count; /* How many pages are active? */
1c79356b 423extern
91447636 424unsigned int vm_page_inactive_count; /* How many pages are inactive? */
1c79356b 425extern
2d21ac55
A		 426unsigned int vm_page_throttled_count;/* How many inactives are throttled */
427extern
428unsigned int vm_page_speculative_count; /* How many speculative pages are unclaimed? */
429extern
91447636 430unsigned int vm_page_wire_count; /* How many pages are wired? */
1c79356b 431extern
91447636 432unsigned int vm_page_free_target; /* How many do we want free? */
1c79356b 433extern
91447636 434unsigned int vm_page_free_min; /* When to wakeup pageout */
1c79356b 435extern
b0d623f7
A		 436unsigned int vm_page_throttle_limit; /* When to throttle new page creation */
437extern
438uint32_t vm_page_creation_throttle; /* When to throttle new page creation */
439extern
91447636 440unsigned int vm_page_inactive_target;/* How many do we want inactive? */
1c79356b 441extern
2d21ac55
A		 442unsigned int vm_page_inactive_min; /* When do wakeup pageout */
443extern
91447636 444unsigned int vm_page_free_reserved; /* How many pages reserved to do pageout */
1c79356b 445extern
b0d623f7 446unsigned int vm_page_throttle_count; /* Count of page allocations throttled */
91447636
A		 447extern
448unsigned int vm_page_gobble_count;
449
b0d623f7 450#if DEVELOPMENT || DEBUG
2d21ac55
A		 451extern
452unsigned int vm_page_speculative_used;
b0d623f7
A		 453#endif
454
55e303ae 455extern
91447636 456unsigned int vm_page_purgeable_count;/* How many pages are purgeable now ? */
55e303ae 457extern
b0d623f7
A		 458unsigned int vm_page_purgeable_wired_count;/* How many purgeable pages are wired now ? */
459extern
91447636 460uint64_t vm_page_purged_count; /* How many pages got purged so far ? */
1c79356b 461
1c79356b
A		 462extern unsigned int vm_page_free_wanted;
463 /* how many threads are waiting for memory */
464
2d21ac55
A		 465extern unsigned int vm_page_free_wanted_privileged;
466 /* how many VM privileged threads are waiting for memory */
467
b0d623f7 468extern ppnum_t vm_page_fictitious_addr;
1c79356b
A		 469 /* (fake) phys_addr of fictitious pages */
470
b0d623f7 471extern ppnum_t vm_page_guard_addr;
2d21ac55
A		 472 /* (fake) phys_addr of guard pages */
473
474
91447636
A		 475extern boolean_t vm_page_deactivate_hint;
476
0c530ab8
A		 477// 0 = all pages avail, 1 = disable high mem, 2 = prefer himem
478extern int vm_himemory_mode;
479
480extern ppnum_t vm_lopage_poolend;
481extern int vm_lopage_poolsize;
482extern uint64_t max_valid_dma_address;
483
484
1c79356b
A		 485/*
486 * Prototypes for functions exported by this module.
487 */
488extern void vm_page_bootstrap(
489 vm_offset_t *startp,
2d21ac55 490 vm_offset_t *endp) __attribute__((section("__TEXT, initcode")));
1c79356b 491
2d21ac55
A		 492extern void vm_page_module_init(void) __attribute__((section("__TEXT, initcode")));
493
b0d623f7
A		 494extern void vm_page_init_local_q(void);
495
1c79356b 496extern void vm_page_create(
55e303ae
A		 497 ppnum_t start,
498 ppnum_t end);
1c79356b
A		 499
500extern vm_page_t vm_page_lookup(
501 vm_object_t object,
502 vm_object_offset_t offset);
503
504extern vm_page_t vm_page_grab_fictitious(void);
505
2d21ac55
A		 506extern vm_page_t vm_page_grab_guard(void);
507
1c79356b
A		 508extern void vm_page_release_fictitious(
509 vm_page_t page);
510
1c79356b
A		 511extern void vm_page_more_fictitious(void);
512
513extern int vm_pool_low(void);
514
515extern vm_page_t vm_page_grab(void);
516
0c530ab8
A		 517extern vm_page_t vm_page_grablo(void);
518
1c79356b
A		 519extern void vm_page_release(
520 vm_page_t page);
521
1c79356b
A		 522extern boolean_t vm_page_wait(
523 int interruptible );
524
525extern vm_page_t vm_page_alloc(
526 vm_object_t object,
527 vm_object_offset_t offset);
528
0c530ab8
A		 529extern vm_page_t vm_page_alloclo(
530 vm_object_t object,
531 vm_object_offset_t offset);
532
2d21ac55
A		 533extern vm_page_t vm_page_alloc_guard(
534 vm_object_t object,
535 vm_object_offset_t offset);
536
1c79356b
A		 537extern void vm_page_init(
538 vm_page_t page,
55e303ae 539 ppnum_t phys_page);
1c79356b
A		 540
541extern void vm_page_free(
b0d623f7 542 vm_page_t page);
1c79356b 543
b0d623f7
A		 544extern void vm_page_free_unlocked(
545 vm_page_t page,
546 boolean_t remove_from_hash);
2d21ac55 547
1c79356b
A		 548extern void vm_page_activate(
549 vm_page_t page);
550
551extern void vm_page_deactivate(
552 vm_page_t page);
553
b0d623f7
A		 554extern void vm_page_deactivate_internal(
555 vm_page_t page,
556 boolean_t clear_hw_reference);
557
2d21ac55
A		 558extern void vm_page_lru(
559 vm_page_t page);
560
561extern void vm_page_speculate(
562 vm_page_t page,
563 boolean_t new);
564
565extern void vm_page_speculate_ageit(
566 struct vm_speculative_age_q *aq);
567
b0d623f7
A		 568extern void vm_page_reactivate_all_throttled(void);
569
570extern void vm_page_reactivate_local(uint32_t lid, boolean_t force, boolean_t nolocks);
571
1c79356b
A		 572extern void vm_page_rename(
573 vm_page_t page,
574 vm_object_t new_object,
2d21ac55
A		 575 vm_object_offset_t new_offset,
576 boolean_t encrypted_ok);
1c79356b
A		 577
578extern void vm_page_insert(
579 vm_page_t page,
580 vm_object_t object,
581 vm_object_offset_t offset);
582
4a3eedf9 583extern void vm_page_insert_internal(
b0d623f7 584 vm_page_t page,
4a3eedf9
A		 585 vm_object_t object,
586 vm_object_offset_t offset,
b0d623f7
A		 587 boolean_t queues_lock_held,
588 boolean_t insert_in_hash);
4a3eedf9 589
1c79356b
A		 590extern void vm_page_replace(
591 vm_page_t mem,
592 vm_object_t object,
593 vm_object_offset_t offset);
594
595extern void vm_page_remove(
b0d623f7
A		 596 vm_page_t page,
597 boolean_t remove_from_hash);
1c79356b
A		 598
599extern void vm_page_zero_fill(
600 vm_page_t page);
601
602extern void vm_page_part_zero_fill(
603 vm_page_t m,
604 vm_offset_t m_pa,
605 vm_size_t len);
606
607extern void vm_page_copy(
608 vm_page_t src_page,
609 vm_page_t dest_page);
610
611extern void vm_page_part_copy(
612 vm_page_t src_m,
613 vm_offset_t src_pa,
614 vm_page_t dst_m,
615 vm_offset_t dst_pa,
616 vm_size_t len);
617
618extern void vm_page_wire(
619 vm_page_t page);
620
621extern void vm_page_unwire(
622 vm_page_t page);
623
624extern void vm_set_page_size(void);
625
626extern void vm_page_gobble(
627 vm_page_t page);
628
2d21ac55 629extern void vm_page_validate_cs(vm_page_t page);
593a1d5f
A		 630extern void vm_page_validate_cs_mapped(
631 vm_page_t page,
632 const void *kaddr);
2d21ac55 633
b0d623f7
A		 634extern void vm_page_free_prepare_queues(
635 vm_page_t page);
636
637extern void vm_page_free_prepare_object(
638 vm_page_t page,
639 boolean_t remove_from_hash);
640
1c79356b
A		 641/*
642 * Functions implemented as macros. m->wanted and m->busy are
643 * protected by the object lock.
644 */
645
646#define PAGE_ASSERT_WAIT(m, interruptible) \
9bccf70c
A		 647 (((m)->wanted = TRUE), \
648 assert_wait((event_t) (m), (interruptible)))
649
650#define PAGE_SLEEP(o, m, interruptible) \
651 (((m)->wanted = TRUE), \
652 thread_sleep_vm_object((o), (m), (interruptible)))
1c79356b
A		 653
654#define PAGE_WAKEUP_DONE(m) \
655 MACRO_BEGIN \
656 (m)->busy = FALSE; \
657 if ((m)->wanted) { \
658 (m)->wanted = FALSE; \
659 thread_wakeup((event_t) (m)); \
660 } \
661 MACRO_END
662
663#define PAGE_WAKEUP(m) \
664 MACRO_BEGIN \
665 if ((m)->wanted) { \
666 (m)->wanted = FALSE; \
667 thread_wakeup((event_t) (m)); \
668 } \
669 MACRO_END
670
671#define VM_PAGE_FREE(p) \
672 MACRO_BEGIN \
b0d623f7 673 vm_page_free_unlocked(p, TRUE); \
1c79356b
A		 674 MACRO_END
675
676#define VM_PAGE_GRAB_FICTITIOUS(M) \
677 MACRO_BEGIN \
678 while ((M = vm_page_grab_fictitious()) == VM_PAGE_NULL) \
679 vm_page_more_fictitious(); \
680 MACRO_END
681
1c79356b
A		 682#define VM_PAGE_WAIT() ((void)vm_page_wait(THREAD_UNINT))
683
b0d623f7
A		 684#define vm_page_queue_lock (vm_page_locks.vm_page_queue_lock2)
685#define vm_page_queue_free_lock (vm_page_locks.vm_page_queue_free_lock2)
686
687#define vm_page_lock_queues() lck_mtx_lock(&vm_page_queue_lock)
688#define vm_page_unlock_queues() lck_mtx_unlock(&vm_page_queue_lock)
689
690#define vm_page_lockspin_queues() lck_mtx_lock_spin(&vm_page_queue_lock)
691#define vm_page_trylockspin_queues() lck_mtx_try_lock_spin(&vm_page_queue_lock)
692#define vm_page_lockconvert_queues() lck_mtx_convert_spin(&vm_page_queue_lock)
693
694#ifdef VPL_LOCK_SPIN
695#define VPL_LOCK_INIT(vlq, vpl_grp, vpl_attr) lck_spin_init(&vlq->vpl_lock, vpl_grp, vpl_attr)
696#define VPL_LOCK(vpl) lck_spin_lock(vpl)
697#define VPL_UNLOCK(vpl) lck_spin_unlock(vpl)
698#else
699#define VPL_LOCK_INIT(vlq, vpl_grp, vpl_attr) lck_mtx_init_ext(&vlq->vpl_lock, &vlq->vpl_lock_ext, vpl_grp, vpl_attr)
700#define VPL_LOCK(vpl) lck_mtx_lock_spin(vpl)
701#define VPL_UNLOCK(vpl) lck_mtx_unlock(vpl)
702#endif
1c79356b 703
b0d623f7
A		 704#if MACH_ASSERT
705extern void vm_page_queues_assert(vm_page_t mem, int val);
706#define VM_PAGE_QUEUES_ASSERT(mem, val) vm_page_queues_assert((mem), (val))
707#else
708#define VM_PAGE_QUEUES_ASSERT(mem, val)
709#endif
2d21ac55 710
b0d623f7
A		 711
712/*
713 * 'vm_fault_enter' will place newly created pages (zero-fill and COW) onto the
714 * local queues if they exist... its the only spot in the system where we add pages
715 * to those queues... once on those queues, those pages can only move to one of the
716 * global page queues or the free queues... they NEVER move from local q to local q.
717 * the 'local' state is stable when VM_PAGE_QUEUES_REMOVE is called since we're behind
718 * the global vm_page_queue_lock at this point... we still need to take the local lock
719 * in case this operation is being run on a different CPU then the local queue's identity,
720 * but we don't have to worry about the page moving to a global queue or becoming wired
721 * while we're grabbing the local lock since those operations would require the global
722 * vm_page_queue_lock to be held, and we already own it.
723 *
724 * this is why its safe to utilze the wire_count field in the vm_page_t as the local_id...
725 * 'wired' and local are ALWAYS mutually exclusive conditions.
726 */
1c79356b
A		 727#define VM_PAGE_QUEUES_REMOVE(mem) \
728 MACRO_BEGIN \
b0d623f7 729 VM_PAGE_QUEUES_ASSERT(mem, 1); \
91447636 730 assert(!mem->laundry); \
b0d623f7
A		 731 assert(!mem->pageout_queue); \
732 if (mem->local) { \
733 struct vpl *lq; \
734 assert(mem->object != kernel_object); \
735 assert(!mem->inactive && !mem->speculative); \
736 assert(!mem->active && !mem->throttled); \
737 lq = &vm_page_local_q[mem->local_id].vpl_un.vpl; \
738 VPL_LOCK(&lq->vpl_lock); \
739 queue_remove(&lq->vpl_queue, \
740 mem, vm_page_t, pageq); \
741 mem->local = FALSE; \
742 mem->local_id = 0; \
743 lq->vpl_count--; \
744 VPL_UNLOCK(&lq->vpl_lock); \
745 } \
1c79356b 746 if (mem->active) { \
91447636 747 assert(mem->object != kernel_object); \
2d21ac55
A		 748 assert(!mem->inactive && !mem->speculative); \
749 assert(!mem->throttled); \
1c79356b
A		 750 queue_remove(&vm_page_queue_active, \
751 mem, vm_page_t, pageq); \
752 mem->active = FALSE; \
2d21ac55 753 if (!mem->fictitious) { \
1c79356b 754 vm_page_active_count--; \
2d21ac55
A		 755 } else { \
756 assert(mem->phys_page == \
757 vm_page_fictitious_addr); \
758 } \
1c79356b
A		 759 } \
760 \
2d21ac55 761 else if (mem->inactive) { \
91447636 762 assert(mem->object != kernel_object); \
2d21ac55
A		 763 assert(!mem->active && !mem->speculative); \
764 assert(!mem->throttled); \
9bccf70c
A		 765 if (mem->zero_fill) { \
766 queue_remove(&vm_page_queue_zf, \
767 mem, vm_page_t, pageq); \
2d21ac55 768 vm_zf_queue_count--; \
9bccf70c
A		 769 } else { \
770 queue_remove(&vm_page_queue_inactive, \
1c79356b 771 mem, vm_page_t, pageq); \
9bccf70c 772 } \
1c79356b 773 mem->inactive = FALSE; \
2d21ac55 774 if (!mem->fictitious) { \
1c79356b 775 vm_page_inactive_count--; \
cf7d32b8 776 vm_purgeable_q_advance_all(); \
2d21ac55
A		 777 } else { \
778 assert(mem->phys_page == \
779 vm_page_fictitious_addr); \
780 } \
781 } \
782 \
783 else if (mem->throttled) { \
784 assert(!mem->active && !mem->inactive); \
785 assert(!mem->speculative); \
786 queue_remove(&vm_page_queue_throttled, \
787 mem, vm_page_t, pageq); \
788 mem->throttled = FALSE; \
789 if (!mem->fictitious) \
790 vm_page_throttled_count--; \
791 } \
792 \
793 else if (mem->speculative) { \
794 assert(!mem->active && !mem->inactive); \
795 assert(!mem->throttled); \
796 assert(!mem->fictitious); \
797 remque(&mem->pageq); \
798 mem->speculative = FALSE; \
799 vm_page_speculative_count--; \
800 } \
801 mem->pageq.next = NULL; \
802 mem->pageq.prev = NULL; \
b0d623f7
A		 803 VM_PAGE_QUEUES_ASSERT(mem, 0); \
804 MACRO_END
805
806
807#if DEVELOPMENT || DEBUG
808#define VM_PAGE_SPECULATIVE_USED_ADD() \
809 MACRO_BEGIN \
810 OSAddAtomic(1, &vm_page_speculative_used); \
2d21ac55 811 MACRO_END
b0d623f7
A		 812#else
813#define VM_PAGE_SPECULATIVE_USED_ADD()
814#endif
2d21ac55
A		 815
816
817#define VM_PAGE_CONSUME_CLUSTERED(mem) \
818 MACRO_BEGIN \
819 if (mem->clustered) { \
820 assert(mem->object); \
821 mem->object->pages_used++; \
822 mem->clustered = FALSE; \
b0d623f7 823 VM_PAGE_SPECULATIVE_USED_ADD(); \
1c79356b
A		 824 } \
825 MACRO_END
826
827#endif /* _VM_VM_PAGE_H_ */
mirror of XNU