]> git.saurik.com Git - apple/xnu.git/blob - osfmk/vm/vm_page.h
projects / apple / xnu.git / blob
? search:


9c3d9f0aee9bfc86e4585c1adf803c18f329a359
[apple/xnu.git] / osfmk / vm / vm_page.h
1 /*
2 * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56 /*
57 */
58 /*
59 * File: vm/vm_page.h
60 * Author: Avadis Tevanian, Jr., Michael Wayne Young
61 * Date: 1985
62 *
63 * Resident memory system definitions.
64 */
65
66 #ifndef _VM_VM_PAGE_H_
67 #define _VM_VM_PAGE_H_
68
69 #include <debug.h>
70 #include <vm/vm_options.h>
71
72 #include <mach/boolean.h>
73 #include <mach/vm_prot.h>
74 #include <mach/vm_param.h>
75 #include <vm/vm_object.h>
76 #include <kern/queue.h>
77 #include <kern/locks.h>
78
79 #include <kern/macro_help.h>
80 #include <libkern/OSAtomic.h>
81
82
83 /*
84 * VM_PAGE_MIN_SPECULATIVE_AGE_Q through VM_PAGE_MAX_SPECULATIVE_AGE_Q
85 * represents a set of aging bins that are 'protected'...
86 *
87 * VM_PAGE_SPECULATIVE_AGED_Q is a list of the speculative pages that have
88 * not yet been 'claimed' but have been aged out of the protective bins
89 * this occurs in vm_page_speculate when it advances to the next bin
90 * and discovers that it is still occupied... at that point, all of the
91 * pages in that bin are moved to the VM_PAGE_SPECULATIVE_AGED_Q. the pages
92 * in that bin are all guaranteed to have reached at least the maximum age
93 * we allow for a protected page... they can be older if there is no
94 * memory pressure to pull them from the bin, or there are no new speculative pages
95 * being generated to push them out.
96 * this list is the one that vm_pageout_scan will prefer when looking
97 * for pages to move to the underweight free list
98 *
99 * VM_PAGE_MAX_SPECULATIVE_AGE_Q * VM_PAGE_SPECULATIVE_Q_AGE_MS
100 * defines the amount of time a speculative page is normally
101 * allowed to live in the 'protected' state (i.e. not available
102 * to be stolen if vm_pageout_scan is running and looking for
103 * pages)... however, if the total number of speculative pages
104 * in the protected state exceeds our limit (defined in vm_pageout.c)
105 * and there are none available in VM_PAGE_SPECULATIVE_AGED_Q, then
106 * vm_pageout_scan is allowed to steal pages from the protected
107 * bucket even if they are underage.
108 *
109 * vm_pageout_scan is also allowed to pull pages from a protected
110 * bin if the bin has reached the "age of consent" we've set
111 */
112 #define VM_PAGE_MAX_SPECULATIVE_AGE_Q 10
113 #define VM_PAGE_MIN_SPECULATIVE_AGE_Q 1
114 #define VM_PAGE_SPECULATIVE_AGED_Q 0
115
116 #define VM_PAGE_SPECULATIVE_Q_AGE_MS 500
117
118 struct vm_speculative_age_q {
119 /*
120 * memory queue for speculative pages via clustered pageins
121 */
122 queue_head_t age_q;
123 mach_timespec_t age_ts;
124 };
125
126
127
128 extern
129 struct vm_speculative_age_q vm_page_queue_speculative[];
130
131 extern int speculative_steal_index;
132 extern int speculative_age_index;
133 extern unsigned int vm_page_speculative_q_age_ms;
134
135
136 #define VM_PAGE_COMPRESSOR_COUNT (compressor_object->resident_page_count)
137
138 /*
139 * Management of resident (logical) pages.
140 *
141 * A small structure is kept for each resident
142 * page, indexed by page number. Each structure
143 * is an element of several lists:
144 *
145 * A hash table bucket used to quickly
146 * perform object/offset lookups
147 *
148 * A list of all pages for a given object,
149 * so they can be quickly deactivated at
150 * time of deallocation.
151 *
152 * An ordered list of pages due for pageout.
153 *
154 * In addition, the structure contains the object
155 * and offset to which this page belongs (for pageout),
156 * and sundry status bits.
157 *
158 * Fields in this structure are locked either by the lock on the
159 * object that the page belongs to (O) or by the lock on the page
160 * queues (P). [Some fields require that both locks be held to
161 * change that field; holding either lock is sufficient to read.]
162 */
163
164
165 #if defined(__LP64__)
166
167 /*
168 * in order to make the size of a vm_page_t 64 bytes (cache line size for both arm64 and x86_64)
169 * we'll keep the next_m pointer packed... as long as the kernel virtual space where we allocate
170 * vm_page_t's from doesn't span more then 256 Gbytes, we're safe. There are live tests in the
171 * vm_page_t array allocation and the zone init code to determine if we can safely pack and unpack
172 * pointers from the 2 ends of these spaces
173 */
174 typedef uint32_t vm_page_packed_t;
175
176 #define VM_PAGE_PACK_PTR(m) (!(m) ? (vm_page_packed_t)0 : ((vm_page_packed_t)((uintptr_t)(((uintptr_t)(m) - (uintptr_t) VM_MIN_KERNEL_AND_KEXT_ADDRESS)) >> 6)))
177 #define VM_PAGE_UNPACK_PTR(p) (!(p) ? VM_PAGE_NULL : ((vm_page_t)((((uintptr_t)(p)) << 6) + (uintptr_t) VM_MIN_KERNEL_AND_KEXT_ADDRESS)))
178
179 #else
180
181 /*
182 * we can't do the packing trick on 32 bit architectures, so
183 * just turn the macros into noops.
184 */
185 typedef struct vm_page *vm_page_packed_t;
186
187 #define VM_PAGE_PACK_PTR(m) ((vm_page_packed_t)(m))
188 #define VM_PAGE_UNPACK_PTR(p) ((vm_page_t)(p))
189
190 #endif
191
192
193 struct vm_page {
194 queue_chain_t pageq; /* queue info for FIFO */
195 /* queue or free list (P) */
196
197 queue_chain_t listq; /* all pages in same object (O) */
198
199 vm_object_offset_t offset; /* offset into that object (O,P) */
200 vm_object_t object; /* which object am I in (O&P) */
201
202 vm_page_packed_t next_m; /* VP bucket link (O) */
203 /*
204 * The following word of flags is protected
205 * by the "page queues" lock.
206 *
207 * we use the 'wire_count' field to store the local
208 * queue id if local queues are enabled...
209 * see the comments at 'VM_PAGE_QUEUES_REMOVE' as to
210 * why this is safe to do
211 */
212 #define local_id wire_count
213 unsigned int wire_count:16, /* how many wired down maps use me? (O&P) */
214 /* boolean_t */ active:1, /* page is in active list (P) */
215 inactive:1, /* page is in inactive list (P) */
216 clean_queue:1, /* page is in pre-cleaned list (P) */
217 local:1, /* page is in one of the local queues (P) */
218 speculative:1, /* page is in speculative list (P) */
219 throttled:1, /* pager is not responding or doesn't exist(P) */
220 free:1, /* page is on free list (P) */
221 pageout_queue:1,/* page is on queue for pageout (P) */
222 laundry:1, /* page is being cleaned now (P)*/
223 reference:1, /* page has been used (P) */
224 gobbled:1, /* page used internally (P) */
225 private:1, /* Page should not be returned to
226 * the free list (P) */
227 no_cache:1, /* page is not to be cached and should
228 * be reused ahead of other pages (P) */
229
230 __unused_pageq_bits:3; /* 3 bits available here */
231
232 ppnum_t phys_page; /* Physical address of page, passed
233 * to pmap_enter (read-only) */
234
235 /*
236 * The following word of flags is protected
237 * by the "VM object" lock.
238 */
239 unsigned int
240 /* boolean_t */ busy:1, /* page is in transit (O) */
241 wanted:1, /* someone is waiting for page (O) */
242 tabled:1, /* page is in VP table (O) */
243 hashed:1, /* page is in vm_page_buckets[]
244 (O) + the bucket lock */
245 fictitious:1, /* Physical page doesn't exist (O) */
246 /*
247 * IMPORTANT: the "pmapped", "xpmapped" and "clustered" bits can be modified while holding the
248 * VM object "shared" lock + the page lock provided through the pmap_lock_phys_page function.
249 * This is done in vm_fault_enter and the CONSUME_CLUSTERED macro.
250 * It's also ok to modify them behind just the VM object "exclusive" lock.
251 */
252 clustered:1, /* page is not the faulted page (O) or (O-shared AND pmap_page) */
253 pmapped:1, /* page has been entered at some
254 * point into a pmap (O) or (O-shared AND pmap_page) */
255 xpmapped:1, /* page has been entered with execute permission (O)
256 or (O-shared AND pmap_page) */
257
258 wpmapped:1, /* page has been entered at some
259 * point into a pmap for write (O) */
260 pageout:1, /* page wired & busy for pageout (O) */
261 absent:1, /* Data has been requested, but is
262 * not yet available (O) */
263 error:1, /* Data manager was unable to provide
264 * data due to error (O) */
265 dirty:1, /* Page must be cleaned (O) */
266 cleaning:1, /* Page clean has begun (O) */
267 precious:1, /* Page is precious; data must be
268 * returned even if clean (O) */
269 overwriting:1, /* Request to unlock has been made
270 * without having data. (O)
271 * [See vm_fault_page_overwrite] */
272 restart:1, /* Page was pushed higher in shadow
273 chain by copy_call-related pagers;
274 start again at top of chain */
275 unusual:1, /* Page is absent, error, restart or
276 page locked */
277 encrypted:1, /* encrypted for secure swap (O) */
278 encrypted_cleaning:1, /* encrypting page */
279 cs_validated:1, /* code-signing: page was checked */
280 cs_tainted:1, /* code-signing: page is tainted */
281 cs_nx:1, /* code-signing: page is nx */
282 reusable:1,
283 lopage:1,
284 slid:1,
285 compressor:1, /* page owned by compressor pool */
286 written_by_kernel:1, /* page was written by kernel (i.e. decompressed) */
287 __unused_object_bits:4; /* 5 bits available here */
288 };
289
290 #define DEBUG_ENCRYPTED_SWAP 1
291 #if DEBUG_ENCRYPTED_SWAP
292 #define ASSERT_PAGE_DECRYPTED(page) \
293 MACRO_BEGIN \
294 if ((page)->encrypted) { \
295 panic("VM page %p should not be encrypted here\n", \
296 (page)); \
297 } \
298 MACRO_END
299 #else /* DEBUG_ENCRYPTED_SWAP */
300 #define ASSERT_PAGE_DECRYPTED(page) assert(!(page)->encrypted)
301 #endif /* DEBUG_ENCRYPTED_SWAP */
302
303 typedef struct vm_page *vm_page_t;
304
305
306 typedef struct vm_locks_array {
307 char pad __attribute__ ((aligned (64)));
308 lck_mtx_t vm_page_queue_lock2 __attribute__ ((aligned (64)));
309 lck_mtx_t vm_page_queue_free_lock2 __attribute__ ((aligned (64)));
310 char pad2 __attribute__ ((aligned (64)));
311 } vm_locks_array_t;
312
313
314 #define VM_PAGE_WIRED(m) ((!(m)->local && (m)->wire_count))
315 #define VM_PAGE_NULL ((vm_page_t) 0)
316 #define NEXT_PAGE(m) ((vm_page_t) (m)->pageq.next)
317 #define NEXT_PAGE_PTR(m) ((vm_page_t *) &(m)->pageq.next)
318
319 /*
320 * XXX The unusual bit should not be necessary. Most of the bit
321 * XXX fields above really want to be masks.
322 */
323
324 /*
325 * For debugging, this macro can be defined to perform
326 * some useful check on a page structure.
327 */
328
329 #define VM_PAGE_CHECK(mem) \
330 MACRO_BEGIN \
331 VM_PAGE_QUEUES_ASSERT(mem, 1); \
332 MACRO_END
333
334 /* Page coloring:
335 *
336 * The free page list is actually n lists, one per color,
337 * where the number of colors is a function of the machine's
338 * cache geometry set at system initialization. To disable
339 * coloring, set vm_colors to 1 and vm_color_mask to 0.
340 * The boot-arg "colors" may be used to override vm_colors.
341 * Note that there is little harm in having more colors than needed.
342 */
343
344 #define MAX_COLORS 128
345 #define DEFAULT_COLORS 32
346
347 extern
348 unsigned int vm_colors; /* must be in range 1..MAX_COLORS */
349 extern
350 unsigned int vm_color_mask; /* must be (vm_colors-1) */
351 extern
352 unsigned int vm_cache_geometry_colors; /* optimal #colors based on cache geometry */
353
354 /*
355 * Wired memory is a very limited resource and we can't let users exhaust it
356 * and deadlock the entire system. We enforce the following limits:
357 *
358 * vm_user_wire_limit (default: all memory minus vm_global_no_user_wire_amount)
359 * how much memory can be user-wired in one user task
360 *
361 * vm_global_user_wire_limit (default: same as vm_user_wire_limit)
362 * how much memory can be user-wired in all user tasks
363 *
364 * vm_global_no_user_wire_amount (default: VM_NOT_USER_WIREABLE)
365 * how much memory must remain user-unwired at any time
366 */
367 #define VM_NOT_USER_WIREABLE (64*1024*1024) /* 64MB */
368 extern
369 vm_map_size_t vm_user_wire_limit;
370 extern
371 vm_map_size_t vm_global_user_wire_limit;
372 extern
373 vm_map_size_t vm_global_no_user_wire_amount;
374
375 /*
376 * Each pageable resident page falls into one of three lists:
377 *
378 * free
379 * Available for allocation now. The free list is
380 * actually an array of lists, one per color.
381 * inactive
382 * Not referenced in any map, but still has an
383 * object/offset-page mapping, and may be dirty.
384 * This is the list of pages that should be
385 * paged out next. There are actually two
386 * inactive lists, one for pages brought in from
387 * disk or other backing store, and another
388 * for "zero-filled" pages. See vm_pageout_scan()
389 * for the distinction and usage.
390 * active
391 * A list of pages which have been placed in
392 * at least one physical map. This list is
393 * ordered, in LRU-like fashion.
394 */
395
396
397 #define VPL_LOCK_SPIN 1
398
399 struct vpl {
400 unsigned int vpl_count;
401 unsigned int vpl_internal_count;
402 unsigned int vpl_external_count;
403 queue_head_t vpl_queue;
404 #ifdef VPL_LOCK_SPIN
405 lck_spin_t vpl_lock;
406 #else
407 lck_mtx_t vpl_lock;
408 lck_mtx_ext_t vpl_lock_ext;
409 #endif
410 };
411
412 struct vplq {
413 union {
414 char cache_line_pad[128];
415 struct vpl vpl;
416 } vpl_un;
417 };
418 extern
419 unsigned int vm_page_local_q_count;
420 extern
421 struct vplq *vm_page_local_q;
422 extern
423 unsigned int vm_page_local_q_soft_limit;
424 extern
425 unsigned int vm_page_local_q_hard_limit;
426 extern
427 vm_locks_array_t vm_page_locks;
428
429 extern
430 queue_head_t vm_page_queue_free[MAX_COLORS]; /* memory free queue */
431 extern
432 queue_head_t vm_lopage_queue_free; /* low memory free queue */
433 extern
434 queue_head_t vm_page_queue_active; /* active memory queue */
435 extern
436 queue_head_t vm_page_queue_inactive; /* inactive memory queue for normal pages */
437 extern
438 queue_head_t vm_page_queue_cleaned; /* clean-queue inactive memory */
439 extern
440 queue_head_t vm_page_queue_anonymous; /* inactive memory queue for anonymous pages */
441 extern
442 queue_head_t vm_page_queue_throttled; /* memory queue for throttled pageout pages */
443
444 extern
445 vm_offset_t first_phys_addr; /* physical address for first_page */
446 extern
447 vm_offset_t last_phys_addr; /* physical address for last_page */
448
449 extern
450 unsigned int vm_page_free_count; /* How many pages are free? (sum of all colors) */
451 extern
452 unsigned int vm_page_fictitious_count;/* How many fictitious pages are free? */
453 extern
454 unsigned int vm_page_active_count; /* How many pages are active? */
455 extern
456 unsigned int vm_page_inactive_count; /* How many pages are inactive? */
457 extern
458 unsigned int vm_page_cleaned_count; /* How many pages are in the clean queue? */
459 extern
460 unsigned int vm_page_throttled_count;/* How many inactives are throttled */
461 extern
462 unsigned int vm_page_speculative_count; /* How many speculative pages are unclaimed? */
463 extern unsigned int vm_page_pageable_internal_count;
464 extern unsigned int vm_page_pageable_external_count;
465 extern
466 unsigned int vm_page_xpmapped_external_count; /* How many pages are mapped executable? */
467 extern
468 unsigned int vm_page_external_count; /* How many pages are file-backed? */
469 extern
470 unsigned int vm_page_internal_count; /* How many pages are anonymous? */
471 extern
472 unsigned int vm_page_wire_count; /* How many pages are wired? */
473 extern
474 unsigned int vm_page_wire_count_initial; /* How many pages wired at startup */
475 extern
476 unsigned int vm_page_free_target; /* How many do we want free? */
477 extern
478 unsigned int vm_page_free_min; /* When to wakeup pageout */
479 extern
480 unsigned int vm_page_throttle_limit; /* When to throttle new page creation */
481 extern
482 uint32_t vm_page_creation_throttle; /* When to throttle new page creation */
483 extern
484 unsigned int vm_page_inactive_target;/* How many do we want inactive? */
485 extern
486 unsigned int vm_page_anonymous_min; /* When it's ok to pre-clean */
487 extern
488 unsigned int vm_page_inactive_min; /* When do wakeup pageout */
489 extern
490 unsigned int vm_page_free_reserved; /* How many pages reserved to do pageout */
491 extern
492 unsigned int vm_page_throttle_count; /* Count of page allocations throttled */
493 extern
494 unsigned int vm_page_gobble_count;
495
496 #if DEVELOPMENT || DEBUG
497 extern
498 unsigned int vm_page_speculative_used;
499 #endif
500
501 extern
502 unsigned int vm_page_purgeable_count;/* How many pages are purgeable now ? */
503 extern
504 unsigned int vm_page_purgeable_wired_count;/* How many purgeable pages are wired now ? */
505 extern
506 uint64_t vm_page_purged_count; /* How many pages got purged so far ? */
507
508 extern unsigned int vm_page_free_wanted;
509 /* how many threads are waiting for memory */
510
511 extern unsigned int vm_page_free_wanted_privileged;
512 /* how many VM privileged threads are waiting for memory */
513
514 extern ppnum_t vm_page_fictitious_addr;
515 /* (fake) phys_addr of fictitious pages */
516
517 extern ppnum_t vm_page_guard_addr;
518 /* (fake) phys_addr of guard pages */
519
520
521 extern boolean_t vm_page_deactivate_hint;
522
523 extern int vm_compressor_mode;
524
525 /*
526 0 = all pages avail ( default. )
527 1 = disable high mem ( cap max pages to 4G)
528 2 = prefer himem
529 */
530 extern int vm_himemory_mode;
531
532 extern boolean_t vm_lopage_needed;
533 extern uint32_t vm_lopage_free_count;
534 extern uint32_t vm_lopage_free_limit;
535 extern uint32_t vm_lopage_lowater;
536 extern boolean_t vm_lopage_refill;
537 extern uint64_t max_valid_dma_address;
538 extern ppnum_t max_valid_low_ppnum;
539
540 /*
541 * Prototypes for functions exported by this module.
542 */
543 extern void vm_page_bootstrap(
544 vm_offset_t *startp,
545 vm_offset_t *endp);
546
547 extern void vm_page_module_init(void);
548
549 extern void vm_page_init_local_q(void);
550
551 extern void vm_page_create(
552 ppnum_t start,
553 ppnum_t end);
554
555 extern vm_page_t vm_page_lookup(
556 vm_object_t object,
557 vm_object_offset_t offset);
558
559 extern vm_page_t vm_page_grab_fictitious(void);
560
561 extern vm_page_t vm_page_grab_guard(void);
562
563 extern void vm_page_release_fictitious(
564 vm_page_t page);
565
566 extern void vm_page_more_fictitious(void);
567
568 extern int vm_pool_low(void);
569
570 extern vm_page_t vm_page_grab(void);
571
572 extern vm_page_t vm_page_grablo(void);
573
574 extern void vm_page_release(
575 vm_page_t page);
576
577 extern boolean_t vm_page_wait(
578 int interruptible );
579
580 extern vm_page_t vm_page_alloc(
581 vm_object_t object,
582 vm_object_offset_t offset);
583
584 extern vm_page_t vm_page_alloclo(
585 vm_object_t object,
586 vm_object_offset_t offset);
587
588 extern vm_page_t vm_page_alloc_guard(
589 vm_object_t object,
590 vm_object_offset_t offset);
591
592 extern void vm_page_init(
593 vm_page_t page,
594 ppnum_t phys_page,
595 boolean_t lopage);
596
597 extern void vm_page_free(
598 vm_page_t page);
599
600 extern void vm_page_free_unlocked(
601 vm_page_t page,
602 boolean_t remove_from_hash);
603
604 extern void vm_page_activate(
605 vm_page_t page);
606
607 extern void vm_page_deactivate(
608 vm_page_t page);
609
610 extern void vm_page_deactivate_internal(
611 vm_page_t page,
612 boolean_t clear_hw_reference);
613
614 extern void vm_page_enqueue_cleaned(vm_page_t page);
615
616 extern void vm_page_lru(
617 vm_page_t page);
618
619 extern void vm_page_speculate(
620 vm_page_t page,
621 boolean_t new);
622
623 extern void vm_page_speculate_ageit(
624 struct vm_speculative_age_q *aq);
625
626 extern void vm_page_reactivate_all_throttled(void);
627
628 extern void vm_page_reactivate_local(uint32_t lid, boolean_t force, boolean_t nolocks);
629
630 extern void vm_page_rename(
631 vm_page_t page,
632 vm_object_t new_object,
633 vm_object_offset_t new_offset,
634 boolean_t encrypted_ok);
635
636 extern void vm_page_insert(
637 vm_page_t page,
638 vm_object_t object,
639 vm_object_offset_t offset);
640
641 extern void vm_page_insert_internal(
642 vm_page_t page,
643 vm_object_t object,
644 vm_object_offset_t offset,
645 boolean_t queues_lock_held,
646 boolean_t insert_in_hash,
647 boolean_t batch_pmap_op);
648
649 extern void vm_page_replace(
650 vm_page_t mem,
651 vm_object_t object,
652 vm_object_offset_t offset);
653
654 extern void vm_page_remove(
655 vm_page_t page,
656 boolean_t remove_from_hash);
657
658 extern void vm_page_zero_fill(
659 vm_page_t page);
660
661 extern void vm_page_part_zero_fill(
662 vm_page_t m,
663 vm_offset_t m_pa,
664 vm_size_t len);
665
666 extern void vm_page_copy(
667 vm_page_t src_page,
668 vm_page_t dest_page);
669
670 extern void vm_page_part_copy(
671 vm_page_t src_m,
672 vm_offset_t src_pa,
673 vm_page_t dst_m,
674 vm_offset_t dst_pa,
675 vm_size_t len);
676
677 extern void vm_page_wire(
678 vm_page_t page);
679
680 extern void vm_page_unwire(
681 vm_page_t page,
682 boolean_t queueit);
683
684 extern void vm_set_page_size(void);
685
686 extern void vm_page_gobble(
687 vm_page_t page);
688
689 extern void vm_page_validate_cs(vm_page_t page);
690 extern void vm_page_validate_cs_mapped(
691 vm_page_t page,
692 const void *kaddr);
693
694 extern void vm_page_free_prepare_queues(
695 vm_page_t page);
696
697 extern void vm_page_free_prepare_object(
698 vm_page_t page,
699 boolean_t remove_from_hash);
700
701 #if CONFIG_IOSCHED
702 extern wait_result_t vm_page_sleep(
703 vm_object_t object,
704 vm_page_t m,
705 int interruptible);
706 #endif
707
708 extern void vm_pressure_response(void);
709
710 #if CONFIG_JETSAM
711 extern void memorystatus_pages_update(unsigned int pages_avail);
712
713 #define VM_CHECK_MEMORYSTATUS do { \
714 memorystatus_pages_update( \
715 vm_page_pageable_external_count + \
716 vm_page_free_count + \
717 (VM_DYNAMIC_PAGING_ENABLED(memory_manager_default) ? 0 : vm_page_purgeable_count) \
718 ); \
719 } while(0)
720
721 #else /* CONFIG_JETSAM */
722
723
724 #define VM_CHECK_MEMORYSTATUS vm_pressure_response()
725
726
727 #endif /* CONFIG_JETSAM */
728
729 /*
730 * Functions implemented as macros. m->wanted and m->busy are
731 * protected by the object lock.
732 */
733
734 #define SET_PAGE_DIRTY(m, set_pmap_modified) \
735 MACRO_BEGIN \
736 vm_page_t __page__ = (m); \
737 __page__->dirty = TRUE; \
738 MACRO_END
739
740 #define PAGE_ASSERT_WAIT(m, interruptible) \
741 (((m)->wanted = TRUE), \
742 assert_wait((event_t) (m), (interruptible)))
743
744 #if CONFIG_IOSCHED
745 #define PAGE_SLEEP(o, m, interruptible) \
746 vm_page_sleep(o, m, interruptible)
747 #else
748 #define PAGE_SLEEP(o, m, interruptible) \
749 (((m)->wanted = TRUE), \
750 thread_sleep_vm_object((o), (m), (interruptible)))
751 #endif
752
753 #define PAGE_WAKEUP_DONE(m) \
754 MACRO_BEGIN \
755 (m)->busy = FALSE; \
756 if ((m)->wanted) { \
757 (m)->wanted = FALSE; \
758 thread_wakeup((event_t) (m)); \
759 } \
760 MACRO_END
761
762 #define PAGE_WAKEUP(m) \
763 MACRO_BEGIN \
764 if ((m)->wanted) { \
765 (m)->wanted = FALSE; \
766 thread_wakeup((event_t) (m)); \
767 } \
768 MACRO_END
769
770 #define VM_PAGE_FREE(p) \
771 MACRO_BEGIN \
772 vm_page_free_unlocked(p, TRUE); \
773 MACRO_END
774
775 #define VM_PAGE_GRAB_FICTITIOUS(M) \
776 MACRO_BEGIN \
777 while ((M = vm_page_grab_fictitious()) == VM_PAGE_NULL) \
778 vm_page_more_fictitious(); \
779 MACRO_END
780
781 #define VM_PAGE_WAIT() ((void)vm_page_wait(THREAD_UNINT))
782
783 #define vm_page_queue_lock (vm_page_locks.vm_page_queue_lock2)
784 #define vm_page_queue_free_lock (vm_page_locks.vm_page_queue_free_lock2)
785
786 #define vm_page_lock_queues() lck_mtx_lock(&vm_page_queue_lock)
787 #define vm_page_unlock_queues() lck_mtx_unlock(&vm_page_queue_lock)
788
789 #define vm_page_lockspin_queues() lck_mtx_lock_spin(&vm_page_queue_lock)
790 #define vm_page_trylockspin_queues() lck_mtx_try_lock_spin(&vm_page_queue_lock)
791 #define vm_page_lockconvert_queues() lck_mtx_convert_spin(&vm_page_queue_lock)
792
793 #ifdef VPL_LOCK_SPIN
794 #define VPL_LOCK_INIT(vlq, vpl_grp, vpl_attr) lck_spin_init(&vlq->vpl_lock, vpl_grp, vpl_attr)
795 #define VPL_LOCK(vpl) lck_spin_lock(vpl)
796 #define VPL_UNLOCK(vpl) lck_spin_unlock(vpl)
797 #else
798 #define VPL_LOCK_INIT(vlq, vpl_grp, vpl_attr) lck_mtx_init_ext(&vlq->vpl_lock, &vlq->vpl_lock_ext, vpl_grp, vpl_attr)
799 #define VPL_LOCK(vpl) lck_mtx_lock_spin(vpl)
800 #define VPL_UNLOCK(vpl) lck_mtx_unlock(vpl)
801 #endif
802
803 #if MACH_ASSERT
804 extern void vm_page_queues_assert(vm_page_t mem, int val);
805 #define VM_PAGE_QUEUES_ASSERT(mem, val) vm_page_queues_assert((mem), (val))
806 #else
807 #define VM_PAGE_QUEUES_ASSERT(mem, val)
808 #endif
809
810
811 /*
812 * 'vm_fault_enter' will place newly created pages (zero-fill and COW) onto the
813 * local queues if they exist... its the only spot in the system where we add pages
814 * to those queues... once on those queues, those pages can only move to one of the
815 * global page queues or the free queues... they NEVER move from local q to local q.
816 * the 'local' state is stable when VM_PAGE_QUEUES_REMOVE is called since we're behind
817 * the global vm_page_queue_lock at this point... we still need to take the local lock
818 * in case this operation is being run on a different CPU then the local queue's identity,
819 * but we don't have to worry about the page moving to a global queue or becoming wired
820 * while we're grabbing the local lock since those operations would require the global
821 * vm_page_queue_lock to be held, and we already own it.
822 *
823 * this is why its safe to utilze the wire_count field in the vm_page_t as the local_id...
824 * 'wired' and local are ALWAYS mutually exclusive conditions.
825 */
826
827 #define VM_PAGE_QUEUES_REMOVE(mem) \
828 MACRO_BEGIN \
829 boolean_t was_pageable; \
830 \
831 VM_PAGE_QUEUES_ASSERT(mem, 1); \
832 assert(!mem->pageout_queue); \
833 /* \
834 * if (mem->pageout_queue) \
835 * NOTE: VM_PAGE_QUEUES_REMOVE does not deal with removing pages from the pageout queue... \
836 * the caller is responsible for determing if the page is on that queue, and if so, must \
837 * either first remove it (it needs both the page queues lock and the object lock to do \
838 * this via vm_pageout_steal_laundry), or avoid the call to VM_PAGE_QUEUES_REMOVE \
839 */ \
840 if (mem->local) { \
841 struct vpl *lq; \
842 assert(mem->object != kernel_object); \
843 assert(mem->object != compressor_object); \
844 assert(!mem->inactive && !mem->speculative); \
845 assert(!mem->active && !mem->throttled); \
846 assert(!mem->clean_queue); \
847 assert(!mem->fictitious); \
848 lq = &vm_page_local_q[mem->local_id].vpl_un.vpl; \
849 VPL_LOCK(&lq->vpl_lock); \
850 queue_remove(&lq->vpl_queue, \
851 mem, vm_page_t, pageq); \
852 mem->local = FALSE; \
853 mem->local_id = 0; \
854 lq->vpl_count--; \
855 if (mem->object->internal) { \
856 lq->vpl_internal_count--; \
857 } else { \
858 lq->vpl_external_count--; \
859 } \
860 VPL_UNLOCK(&lq->vpl_lock); \
861 was_pageable = FALSE; \
862 } \
863 \
864 else if (mem->active) { \
865 assert(mem->object != kernel_object); \
866 assert(mem->object != compressor_object); \
867 assert(!mem->inactive && !mem->speculative); \
868 assert(!mem->clean_queue); \
869 assert(!mem->throttled); \
870 assert(!mem->fictitious); \
871 queue_remove(&vm_page_queue_active, \
872 mem, vm_page_t, pageq); \
873 mem->active = FALSE; \
874 vm_page_active_count--; \
875 was_pageable = TRUE; \
876 } \
877 \
878 else if (mem->inactive) { \
879 assert(mem->object != kernel_object); \
880 assert(mem->object != compressor_object); \
881 assert(!mem->active && !mem->speculative); \
882 assert(!mem->throttled); \
883 assert(!mem->fictitious); \
884 vm_page_inactive_count--; \
885 if (mem->clean_queue) { \
886 queue_remove(&vm_page_queue_cleaned, \
887 mem, vm_page_t, pageq); \
888 mem->clean_queue = FALSE; \
889 vm_page_cleaned_count--; \
890 } else { \
891 if (mem->object->internal) { \
892 queue_remove(&vm_page_queue_anonymous, \
893 mem, vm_page_t, pageq); \
894 vm_page_anonymous_count--; \
895 } else { \
896 queue_remove(&vm_page_queue_inactive, \
897 mem, vm_page_t, pageq); \
898 } \
899 vm_purgeable_q_advance_all(); \
900 } \
901 mem->inactive = FALSE; \
902 was_pageable = TRUE; \
903 } \
904 \
905 else if (mem->throttled) { \
906 assert(mem->object != compressor_object); \
907 assert(!mem->active && !mem->inactive); \
908 assert(!mem->speculative); \
909 assert(!mem->fictitious); \
910 queue_remove(&vm_page_queue_throttled, \
911 mem, vm_page_t, pageq); \
912 mem->throttled = FALSE; \
913 vm_page_throttled_count--; \
914 was_pageable = FALSE; \
915 } \
916 \
917 else if (mem->speculative) { \
918 assert(mem->object != compressor_object); \
919 assert(!mem->active && !mem->inactive); \
920 assert(!mem->throttled); \
921 assert(!mem->fictitious); \
922 remque(&mem->pageq); \
923 mem->speculative = FALSE; \
924 vm_page_speculative_count--; \
925 was_pageable = TRUE; \
926 } \
927 \
928 else if (mem->pageq.next || mem->pageq.prev) { \
929 was_pageable = FALSE; \
930 panic("VM_PAGE_QUEUES_REMOVE: unmarked page on Q"); \
931 } else { \
932 was_pageable = FALSE; \
933 } \
934 \
935 mem->pageq.next = NULL; \
936 mem->pageq.prev = NULL; \
937 VM_PAGE_QUEUES_ASSERT(mem, 0); \
938 if (was_pageable) { \
939 if (mem->object->internal) { \
940 vm_page_pageable_internal_count--; \
941 } else { \
942 vm_page_pageable_external_count--; \
943 } \
944 } \
945 MACRO_END
946
947
948 #define VM_PAGE_ENQUEUE_INACTIVE(mem, first) \
949 MACRO_BEGIN \
950 VM_PAGE_QUEUES_ASSERT(mem, 0); \
951 assert(!mem->fictitious); \
952 assert(!mem->laundry); \
953 assert(!mem->pageout_queue); \
954 if (mem->object->internal) { \
955 if (first == TRUE) \
956 queue_enter_first(&vm_page_queue_anonymous, mem, vm_page_t, pageq); \
957 else \
958 queue_enter(&vm_page_queue_anonymous, mem, vm_page_t, pageq); \
959 vm_page_anonymous_count++; \
960 vm_page_pageable_internal_count++; \
961 } else { \
962 if (first == TRUE) \
963 queue_enter_first(&vm_page_queue_inactive, mem, vm_page_t, pageq); \
964 else \
965 queue_enter(&vm_page_queue_inactive, mem, vm_page_t, pageq); \
966 vm_page_pageable_external_count++; \
967 } \
968 mem->inactive = TRUE; \
969 vm_page_inactive_count++; \
970 token_new_pagecount++; \
971 MACRO_END
972
973
974 #if DEVELOPMENT || DEBUG
975 #define VM_PAGE_SPECULATIVE_USED_ADD() \
976 MACRO_BEGIN \
977 OSAddAtomic(1, &vm_page_speculative_used); \
978 MACRO_END
979 #else
980 #define VM_PAGE_SPECULATIVE_USED_ADD()
981 #endif
982
983
984 #define VM_PAGE_CONSUME_CLUSTERED(mem) \
985 MACRO_BEGIN \
986 pmap_lock_phys_page(mem->phys_page); \
987 if (mem->clustered) { \
988 assert(mem->object); \
989 mem->object->pages_used++; \
990 mem->clustered = FALSE; \
991 VM_PAGE_SPECULATIVE_USED_ADD(); \
992 } \
993 pmap_unlock_phys_page(mem->phys_page); \
994 MACRO_END
995
996
997 #define VM_PAGE_COUNT_AS_PAGEIN(mem) \
998 MACRO_BEGIN \
999 DTRACE_VM2(pgin, int, 1, (uint64_t *), NULL); \
1000 current_task()->pageins++; \
1001 if (mem->object->internal) { \
1002 DTRACE_VM2(anonpgin, int, 1, (uint64_t *), NULL); \
1003 } else { \
1004 DTRACE_VM2(fspgin, int, 1, (uint64_t *), NULL); \
1005 } \
1006 MACRO_END
1007
1008
1009 #define DW_vm_page_unwire 0x01
1010 #define DW_vm_page_wire 0x02
1011 #define DW_vm_page_free 0x04
1012 #define DW_vm_page_activate 0x08
1013 #define DW_vm_page_deactivate_internal 0x10
1014 #define DW_vm_page_speculate 0x20
1015 #define DW_vm_page_lru 0x40
1016 #define DW_vm_pageout_throttle_up 0x80
1017 #define DW_PAGE_WAKEUP 0x100
1018 #define DW_clear_busy 0x200
1019 #define DW_clear_reference 0x400
1020 #define DW_set_reference 0x800
1021 #define DW_move_page 0x1000
1022 #define DW_VM_PAGE_QUEUES_REMOVE 0x2000
1023 #define DW_enqueue_cleaned 0x4000
1024 #define DW_vm_phantom_cache_update 0x8000
1025
1026 struct vm_page_delayed_work {
1027 vm_page_t dw_m;
1028 int dw_mask;
1029 };
1030
1031 void vm_page_do_delayed_work(vm_object_t object, struct vm_page_delayed_work *dwp, int dw_count);
1032
1033 extern unsigned int vm_max_delayed_work_limit;
1034
1035 #define DEFAULT_DELAYED_WORK_LIMIT 32
1036
1037 #define DELAYED_WORK_LIMIT(max) ((vm_max_delayed_work_limit >= max ? max : vm_max_delayed_work_limit))
1038
1039 /*
1040 * vm_page_do_delayed_work may need to drop the object lock...
1041 * if it does, we need the pages it's looking at to
1042 * be held stable via the busy bit, so if busy isn't already
1043 * set, we need to set it and ask vm_page_do_delayed_work
1044 * to clear it and wakeup anyone that might have blocked on
1045 * it once we're done processing the page.
1046 */
1047
1048 #define VM_PAGE_ADD_DELAYED_WORK(dwp, mem, dw_cnt) \
1049 MACRO_BEGIN \
1050 if (mem->busy == FALSE) { \
1051 mem->busy = TRUE; \
1052 if ( !(dwp->dw_mask & DW_vm_page_free)) \
1053 dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP); \
1054 } \
1055 dwp->dw_m = mem; \
1056 dwp++; \
1057 dw_cnt++; \
1058 MACRO_END
1059
1060 extern vm_page_t vm_object_page_grab(vm_object_t);
1061
1062 #if VM_PAGE_BUCKETS_CHECK
1063 extern void vm_page_buckets_check(void);
1064 #endif /* VM_PAGE_BUCKETS_CHECK */
1065
1066 #endif /* _VM_VM_PAGE_H_ */
mirror of XNU