]> git.saurik.com Git - apple/xnu.git/blame - osfmk/vm/vm_page.h
projects / apple / xnu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
xnu-4570.1.46.tar.gz
[apple/xnu.git] / osfmk / vm / vm_page.h
CommitLineData
1c79356b 1/*
2d21ac55 2 * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
1c79356b 3 *
2d21ac55 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
1c79356b 5 *
2d21ac55
A		 6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
8f6c56a5 14 *
2d21ac55
A		 15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
8f6c56a5
A		 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
2d21ac55
A		 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
8f6c56a5 25 *
2d21ac55 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
1c79356b
A		 27 */
28/*
29 * @OSF_COPYRIGHT@
30 */
31/*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56/*
57 */
58/*
59 * File: vm/vm_page.h
60 * Author: Avadis Tevanian, Jr., Michael Wayne Young
61 * Date: 1985
62 *
63 * Resident memory system definitions.
64 */
65
66#ifndef _VM_VM_PAGE_H_
67#define _VM_VM_PAGE_H_
68
91447636 69#include <debug.h>
15129b1c 70#include <vm/vm_options.h>
1c79356b
A		 71#include <mach/boolean.h>
72#include <mach/vm_prot.h>
73#include <mach/vm_param.h>
1c79356b 74
2d21ac55 75
39037602 76#if defined(__LP64__)
1c79356b 77
39037602
A		 78/*
79 * in order to make the size of a vm_page_t 64 bytes (cache line size for both arm64 and x86_64)
80 * we'll keep the next_m pointer packed... as long as the kernel virtual space where we allocate
81 * vm_page_t's from doesn't span more then 256 Gbytes, we're safe. There are live tests in the
82 * vm_page_t array allocation and the zone init code to determine if we can safely pack and unpack
83 * pointers from the 2 ends of these spaces
0b4e3aa0 84 */
39037602 85typedef uint32_t vm_page_packed_t;
2d21ac55 86
39037602
A		 87struct vm_page_packed_queue_entry {
88 vm_page_packed_t next; /* next element */
89 vm_page_packed_t prev; /* previous element */
2d21ac55 90};
0b4e3aa0 91
39037602
A		 92typedef struct vm_page_packed_queue_entry *vm_page_queue_t;
93typedef struct vm_page_packed_queue_entry vm_page_queue_head_t;
94typedef struct vm_page_packed_queue_entry vm_page_queue_chain_t;
95typedef struct vm_page_packed_queue_entry *vm_page_queue_entry_t;
0b4e3aa0 96
39037602 97typedef vm_page_packed_t vm_page_object_t;
6d2010ae 98
39037602
A		 99#else
100
101/*
102 * we can't do the packing trick on 32 bit architectures, so
103 * just turn the macros into noops.
104 */
105typedef struct vm_page *vm_page_packed_t;
106
107#define vm_page_queue_t queue_t
108#define vm_page_queue_head_t queue_head_t
109#define vm_page_queue_chain_t queue_chain_t
110#define vm_page_queue_entry_t queue_entry_t
111
112#define vm_page_object_t vm_object_t
113#endif
114
115
116#include <vm/vm_object.h>
117#include <kern/queue.h>
118#include <kern/locks.h>
119
120#include <kern/macro_help.h>
121#include <libkern/OSAtomic.h>
0b4e3aa0 122
9bccf70c 123
0b4e3aa0 124
39236c6e
A		 125#define VM_PAGE_COMPRESSOR_COUNT (compressor_object->resident_page_count)
126
1c79356b
A		 127/*
128 * Management of resident (logical) pages.
129 *
130 * A small structure is kept for each resident
131 * page, indexed by page number. Each structure
132 * is an element of several lists:
133 *
134 * A hash table bucket used to quickly
135 * perform object/offset lookups
136 *
137 * A list of all pages for a given object,
138 * so they can be quickly deactivated at
139 * time of deallocation.
140 *
141 * An ordered list of pages due for pageout.
142 *
143 * In addition, the structure contains the object
144 * and offset to which this page belongs (for pageout),
145 * and sundry status bits.
146 *
147 * Fields in this structure are locked either by the lock on the
148 * object that the page belongs to (O) or by the lock on the page
149 * queues (P). [Some fields require that both locks be held to
150 * change that field; holding either lock is sufficient to read.]
151 */
152
39037602 153#define VM_PAGE_NULL ((vm_page_t) 0)
fe8ab488 154
39037602
A		 155extern char vm_page_inactive_states[];
156extern char vm_page_pageable_states[];
157extern char vm_page_non_speculative_pageable_states[];
158extern char vm_page_active_or_inactive_states[];
fe8ab488 159
fe8ab488 160
39037602
A		 161#define VM_PAGE_INACTIVE(m) (vm_page_inactive_states[m->vm_page_q_state])
162#define VM_PAGE_PAGEABLE(m) (vm_page_pageable_states[m->vm_page_q_state])
163#define VM_PAGE_NON_SPECULATIVE_PAGEABLE(m) (vm_page_non_speculative_pageable_states[m->vm_page_q_state])
164#define VM_PAGE_ACTIVE_OR_INACTIVE(m) (vm_page_active_or_inactive_states[m->vm_page_q_state])
fe8ab488 165
fe8ab488 166
39037602
A		 167#define VM_PAGE_NOT_ON_Q 0 /* page is not present on any queue, nor is it wired... mainly a transient state */
168#define VM_PAGE_IS_WIRED 1 /* page is currently wired */
169#define VM_PAGE_USED_BY_COMPRESSOR 2 /* page is in use by the compressor to hold compressed data */
170#define VM_PAGE_ON_FREE_Q 3 /* page is on the main free queue */
171#define VM_PAGE_ON_FREE_LOCAL_Q 4 /* page is on one of the per-CPU free queues */
172#define VM_PAGE_ON_FREE_LOPAGE_Q 5 /* page is on the lopage pool free list */
173#define VM_PAGE_ON_THROTTLED_Q 6 /* page is on the throttled queue... we stash anonymous pages here when not paging */
174#define VM_PAGE_ON_PAGEOUT_Q 7 /* page is on one of the pageout queues (internal/external) awaiting processing */
175#define VM_PAGE_ON_SPECULATIVE_Q 8 /* page is on one of the speculative queues */
176#define VM_PAGE_ON_ACTIVE_LOCAL_Q 9 /* page has recently been created and is being held in one of the per-CPU local queues */
177#define VM_PAGE_ON_ACTIVE_Q 10 /* page is in global active queue */
178#define VM_PAGE_ON_INACTIVE_INTERNAL_Q 11 /* page is on the inactive internal queue a.k.a. anonymous queue */
179#define VM_PAGE_ON_INACTIVE_EXTERNAL_Q 12 /* page in on the inactive external queue a.k.a. file backed queue */
180#define VM_PAGE_ON_INACTIVE_CLEANED_Q 13 /* page has been cleaned to a backing file and is ready to be stolen */
181#define VM_PAGE_ON_SECLUDED_Q 14 /* page is on secluded queue */
182#define VM_PAGE_Q_STATE_LAST_VALID_VALUE 14 /* we currently use 4 bits for the state... don't let this go beyond 15 */
fe8ab488 183
39037602 184#define VM_PAGE_Q_STATE_ARRAY_SIZE (VM_PAGE_Q_STATE_LAST_VALID_VALUE+1)
fe8ab488 185
fe8ab488 186
39037602
A		 187#define pageq pageq_un.vm_page_pageq
188#define snext pageq_un.vm_page_snext
fe8ab488 189
1c79356b 190struct vm_page {
39037602
A		 191 union {
192 vm_page_queue_chain_t vm_page_pageq; /* queue info for FIFO queue or free list (P) */
193 struct vm_page *vm_page_snext;
194 } pageq_un;
b0d623f7 195
39037602 196 vm_page_queue_chain_t listq; /* all pages in same object (O) */
1c79356b 197
39037602
A		 198#if CONFIG_BACKGROUND_QUEUE
199 vm_page_queue_chain_t vm_page_backgroundq; /* anonymous pages in the background pool (P) */
200#endif
201
202 vm_object_offset_t offset; /* offset into that object (O,P) */
203 vm_page_object_t vm_page_object; /* which object am I in (O&P) */
1c79356b 204
91447636
A		 205 /*
206 * The following word of flags is protected
207 * by the "page queues" lock.
b0d623f7
A		 208 *
209 * we use the 'wire_count' field to store the local
210 * queue id if local queues are enabled...
3e170ce0 211 * see the comments at 'vm_page_queues_remove' as to
b0d623f7 212 * why this is safe to do
91447636 213 */
b0d623f7 214#define local_id wire_count
1c79356b 215 unsigned int wire_count:16, /* how many wired down maps use me? (O&P) */
39037602
A		 216 vm_page_q_state:4, /* which q is the page on (P) */
217
218 vm_page_in_background:1,
219 vm_page_on_backgroundq:1,
220 /* boolean_t */
0b4e3aa0 221 gobbled:1, /* page used internally (P) */
39037602 222 laundry:1, /* page is being cleaned now (P)*/
6d2010ae
A		 223 no_cache:1, /* page is not to be cached and should
224 * be reused ahead of other pages (P) */
39037602
A		 225 private:1, /* Page should not be returned to
226 * the free list (P) */
227 reference:1, /* page has been used (P) */
fe8ab488 228
39037602 229 __unused_pageq_bits:5; /* 5 bits available here */
1c79356b 230
39037602
A		 231 /*
232 * MUST keep the 2 32 bit words used as bit fields
233 * separated since the compiler has a nasty habit
234 * of using 64 bit loads and stores on them as
235 * if they were a single 64 bit field... since
236 * they are protected by 2 different locks, this
237 * is a real problem
238 */
239 vm_page_packed_t next_m; /* VP bucket link (O) */
b0d623f7 240
91447636
A		 241 /*
242 * The following word of flags is protected
243 * by the "VM object" lock.
244 */
1c79356b
A		 245 unsigned int
246 /* boolean_t */ busy:1, /* page is in transit (O) */
247 wanted:1, /* someone is waiting for page (O) */
248 tabled:1, /* page is in VP table (O) */
15129b1c
A		 249 hashed:1, /* page is in vm_page_buckets[]
250 (O) + the bucket lock */
1c79356b 251 fictitious:1, /* Physical page doesn't exist (O) */
b0d623f7 252 /*
fe8ab488
A		 253 * IMPORTANT: the "pmapped", "xpmapped" and "clustered" bits can be modified while holding the
254 * VM object "shared" lock + the page lock provided through the pmap_lock_phys_page function.
255 * This is done in vm_fault_enter and the CONSUME_CLUSTERED macro.
256 * It's also ok to modify them behind just the VM object "exclusive" lock.
b0d623f7 257 */
fe8ab488 258 clustered:1, /* page is not the faulted page (O) or (O-shared AND pmap_page) */
2d21ac55 259 pmapped:1, /* page has been entered at some
fe8ab488
A		 260 * point into a pmap (O) or (O-shared AND pmap_page) */
261 xpmapped:1, /* page has been entered with execute permission (O)
262 or (O-shared AND pmap_page) */
263
39037602 264 wpmapped:1, /* page has been entered at some
4a3eedf9 265 * point into a pmap for write (O) */
39037602 266 free_when_done:1, /* page is to be freed once cleaning is completed (O) */
1c79356b
A		 267 absent:1, /* Data has been requested, but is
268 * not yet available (O) */
269 error:1, /* Data manager was unable to provide
270 * data due to error (O) */
271 dirty:1, /* Page must be cleaned (O) */
272 cleaning:1, /* Page clean has begun (O) */
273 precious:1, /* Page is precious; data must be
274 * returned even if clean (O) */
1c79356b
A		 275 overwriting:1, /* Request to unlock has been made
276 * without having data. (O)
277 * [See vm_fault_page_overwrite] */
278 restart:1, /* Page was pushed higher in shadow
279 chain by copy_call-related pagers;
280 start again at top of chain */
1c79356b
A		 281 unusual:1, /* Page is absent, error, restart or
282 page locked */
2d21ac55
A		 283 cs_validated:1, /* code-signing: page was checked */
284 cs_tainted:1, /* code-signing: page is tainted */
c18c124e 285 cs_nx:1, /* code-signing: page is nx */
b0d623f7 286 reusable:1,
0b4c1975 287 lopage:1,
6d2010ae 288 slid:1,
15129b1c 289 written_by_kernel:1, /* page was written by kernel (i.e. decompressed) */
5ba3f43e 290 __unused_object_bits:7; /* 7 bits available here */
39037602 291
5ba3f43e 292#if !defined(__arm__) && !defined(__arm64__)
39037602
A		 293 ppnum_t phys_page; /* Physical address of page, passed
294 * to pmap_enter (read-only) */
5ba3f43e 295#endif
1c79356b
A		 296};
297
39037602
A		 298
299typedef struct vm_page *vm_page_t;
300extern vm_page_t vm_pages;
301extern vm_page_t vm_page_array_beginning_addr;
302extern vm_page_t vm_page_array_ending_addr;
303
304
5ba3f43e
A		 305#if defined(__arm__) || defined(__arm64__)
306
307extern unsigned int vm_first_phys_ppnum;
308
309struct vm_page_with_ppnum {
310 struct vm_page vm_page_wo_ppnum;
311
312 ppnum_t phys_page;
313};
314typedef struct vm_page_with_ppnum *vm_page_with_ppnum_t;
315
316
317static inline ppnum_t VM_PAGE_GET_PHYS_PAGE(vm_page_t m)
318{
319 if (m >= vm_page_array_beginning_addr && m < vm_page_array_ending_addr)
320 return ((ppnum_t)((uintptr_t)(m - vm_page_array_beginning_addr) + vm_first_phys_ppnum));
321 else
322 return (((vm_page_with_ppnum_t)m)->phys_page);
323}
324
325#define VM_PAGE_SET_PHYS_PAGE(m, ppnum) \
326 MACRO_BEGIN \
327 if ((m) < vm_page_array_beginning_addr || (m) >= vm_page_array_ending_addr) \
328 ((vm_page_with_ppnum_t)(m))->phys_page = ppnum; \
329 assert(ppnum == VM_PAGE_GET_PHYS_PAGE(m)); \
330 MACRO_END
331
332#define VM_PAGE_GET_COLOR(m) (VM_PAGE_GET_PHYS_PAGE(m) & vm_color_mask)
333
334#else /* defined(__arm__) || defined(__arm64__) */
39037602
A		 335
336
337struct vm_page_with_ppnum {
338 struct vm_page vm_page_with_ppnum;
339};
340typedef struct vm_page_with_ppnum *vm_page_with_ppnum_t;
341
342
343#define VM_PAGE_GET_PHYS_PAGE(page) (page)->phys_page
344#define VM_PAGE_SET_PHYS_PAGE(page, ppnum) \
345 MACRO_BEGIN \
346 (page)->phys_page = ppnum; \
347 MACRO_END
348
5ba3f43e
A		 349#define VM_PAGE_GET_CLUMP(m) ((VM_PAGE_GET_PHYS_PAGE(m)) >> vm_clump_shift)
350#define VM_PAGE_GET_COLOR(m) ((VM_PAGE_GET_CLUMP(m)) & vm_color_mask)
39037602 351
5ba3f43e 352#endif /* defined(__arm__) || defined(__arm64__) */
91447636 353
39037602
A		 354
355
356#if defined(__LP64__)
357
358#define VM_VPLQ_ALIGNMENT 128
359#define VM_PACKED_POINTER_ALIGNMENT 64 /* must be a power of 2 */
360#define VM_PACKED_POINTER_SHIFT 6
361
362#define VM_PACKED_FROM_VM_PAGES_ARRAY 0x80000000
363
364static inline vm_page_packed_t vm_page_pack_ptr(uintptr_t p)
365{
366 vm_page_packed_t packed_ptr;
367
368 if (!p)
369 return ((vm_page_packed_t)0);
370
371 if (p >= (uintptr_t)(vm_page_array_beginning_addr) && p < (uintptr_t)(vm_page_array_ending_addr)) {
372 packed_ptr = ((vm_page_packed_t)(((vm_page_t)p - vm_page_array_beginning_addr)));
373 assert(! (packed_ptr & VM_PACKED_FROM_VM_PAGES_ARRAY));
374 packed_ptr |= VM_PACKED_FROM_VM_PAGES_ARRAY;
375 return packed_ptr;
376 }
377
378 assert((p & (VM_PACKED_POINTER_ALIGNMENT - 1)) == 0);
379
380 packed_ptr = ((vm_page_packed_t)(((uintptr_t)(p - (uintptr_t) VM_MIN_KERNEL_AND_KEXT_ADDRESS)) >> VM_PACKED_POINTER_SHIFT));
381 assert(packed_ptr != 0);
382 assert(! (packed_ptr & VM_PACKED_FROM_VM_PAGES_ARRAY));
383 return packed_ptr;
384}
385
386
387static inline uintptr_t vm_page_unpack_ptr(uintptr_t p)
388{
389 if (!p)
390 return ((uintptr_t)0);
391
392 if (p & VM_PACKED_FROM_VM_PAGES_ARRAY)
393 return ((uintptr_t)(&vm_pages[(uint32_t)(p & ~VM_PACKED_FROM_VM_PAGES_ARRAY)]));
394 return (((p << VM_PACKED_POINTER_SHIFT) + (uintptr_t) VM_MIN_KERNEL_AND_KEXT_ADDRESS));
395}
396
397
398#define VM_PAGE_PACK_PTR(p) vm_page_pack_ptr((uintptr_t)(p))
399#define VM_PAGE_UNPACK_PTR(p) vm_page_unpack_ptr((uintptr_t)(p))
400
401#define VM_PAGE_OBJECT(p) ((vm_object_t)(VM_PAGE_UNPACK_PTR(p->vm_page_object)))
402#define VM_PAGE_PACK_OBJECT(o) ((vm_page_object_t)(VM_PAGE_PACK_PTR(o)))
403
404
405#define VM_PAGE_ZERO_PAGEQ_ENTRY(p) \
406MACRO_BEGIN \
407 (p)->snext = 0; \
408MACRO_END
409
410
411#define VM_PAGE_CONVERT_TO_QUEUE_ENTRY(p) VM_PAGE_PACK_PTR(p)
412
413
414static __inline__ void
415vm_page_enqueue_tail(
416 vm_page_queue_t que,
417 vm_page_queue_entry_t elt)
418{
419 vm_page_queue_entry_t old_tail;
420
421 old_tail = (vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR(que->prev);
422 elt->next = VM_PAGE_PACK_PTR(que);
423 elt->prev = que->prev;
424 old_tail->next = VM_PAGE_PACK_PTR(elt);
425 que->prev = VM_PAGE_PACK_PTR(elt);
426}
427
428
429static __inline__ void
430vm_page_remque(
431 vm_page_queue_entry_t elt)
432{
433 vm_page_queue_entry_t next_elt, prev_elt;
434
435 next_elt = (vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR(elt->next);
436
437 /* next_elt may equal prev_elt (and the queue head) if elt was the only element */
438 prev_elt = (vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR(elt->prev);
439
440 next_elt->prev = VM_PAGE_PACK_PTR(prev_elt);
441 prev_elt->next = VM_PAGE_PACK_PTR(next_elt);
442
443 elt->next = 0;
444 elt->prev = 0;
445}
446
447
448/*
449 * Macro: vm_page_queue_init
450 * Function:
451 * Initialize the given queue.
452 * Header:
453 * void vm_page_queue_init(q)
454 * vm_page_queue_t q; \* MODIFIED *\
455 */
456#define vm_page_queue_init(q) \
457MACRO_BEGIN \
458 assert((((uintptr_t)q) & (VM_PACKED_POINTER_ALIGNMENT-1)) == 0); \
459 assert((VM_PAGE_UNPACK_PTR(VM_PAGE_PACK_PTR((uintptr_t)q))) == (uintptr_t)q); \
460 (q)->next = VM_PAGE_PACK_PTR(q); \
461 (q)->prev = VM_PAGE_PACK_PTR(q); \
462MACRO_END
463
464
465/*
466 * Macro: vm_page_queue_enter
467 * Function:
468 * Insert a new element at the tail of the queue.
469 * Header:
470 * void vm_page_queue_enter(q, elt, type, field)
471 * queue_t q;
472 * <type> elt;
473 * <type> is what's in our queue
474 * <field> is the chain field in (*<type>)
475 * Note:
476 * This should only be used with Method 2 queue iteration (element chains)
477 */
478#define vm_page_queue_enter(head, elt, type, field) \
479MACRO_BEGIN \
480 vm_page_queue_entry_t __prev; \
481 \
482 __prev = ((vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR((head)->prev)); \
483 if ((head) == __prev) { \
484 (head)->next = VM_PAGE_PACK_PTR(elt); \
485 } \
486 else { \
487 ((type)(void *)__prev)->field.next = VM_PAGE_PACK_PTR(elt); \
488 } \
489 (elt)->field.prev = VM_PAGE_PACK_PTR(__prev); \
490 (elt)->field.next = VM_PAGE_PACK_PTR(head); \
491 (head)->prev = VM_PAGE_PACK_PTR(elt); \
492MACRO_END
493
494
5ba3f43e
A		 495/*
496 * These are helper macros for vm_page_queue_enter_clump to assist
497 * with conditional compilation (release / debug / development)
498 */
499#if DEVELOPMENT || DEBUG
500
501#define __DEBUG_CHECK_BUDDIES(__check, __prev, __p, field) \
502MACRO_BEGIN \
503 if(__check) { /* if first forward buddy.. */ \
504 if(__prev) { /* ..and if a backward buddy was found, verify link consistency */ \
505 assert(__p == (vm_page_t) VM_PAGE_UNPACK_PTR(__prev->next)); \
506 assert(__prev == (vm_page_queue_entry_t) VM_PAGE_UNPACK_PTR(__p->field.prev)); \
507 } \
508 __check=0; \
509 } \
510MACRO_END
511
512#define __DEBUG_VERIFY_LINKS(__i, __first, __n_free, __last_next) \
513MACRO_BEGIN \
514 vm_page_queue_entry_t __tmp; \
515 for(__i=0, __tmp=__first; __i<__n_free; __i++) \
516 __tmp=(vm_page_queue_entry_t) VM_PAGE_UNPACK_PTR(__tmp->next); \
517 assert(__tmp == __last_next); \
518MACRO_END
519
520#define __DEBUG_STAT_INCREMENT_INRANGE vm_clump_inrange++
521#define __DEBUG_STAT_INCREMENT_INSERTS vm_clump_inserts++
522#define __DEBUG_STAT_INCREMENT_PROMOTES(__n_free) vm_clump_promotes+=__n_free
523
524#else
525
526#define __DEBUG_CHECK_BUDDIES(__check, __prev, __p, field) __check=1
527#define __DEBUG_VERIFY_LINKS(__i, __first, __n_free, __last_next)
528#define __DEBUG_STAT_INCREMENT_INRANGE
529#define __DEBUG_STAT_INCREMENT_INSERTS
530#define __DEBUG_STAT_INCREMENT_PROMOTES(__n_free)
531
532#endif /* if DEVELOPMENT || DEBUG */
533
534/*
535 * Macro: vm_page_queue_enter_clump
536 * Function:
537 * Insert a new element into the free queue and clump pages within the same 16K boundary together
538 *
539 * Header:
540 * void vm_page_queue_enter_clump(q, elt, type, field)
541 * queue_t q;
542 * <type> elt;
543 * <type> is what's in our queue
544 * <field> is the chain field in (*<type>)
545 * Note:
546 * This should only be used with Method 2 queue iteration (element chains)
547 */
548#if defined(__x86_64__)
549#define vm_page_queue_enter_clump(head, elt, type, field) \
550MACRO_BEGIN \
551 ppnum_t __clump_num; \
552 unsigned int __i, __n, __n_free=1, __check=1; \
553 vm_page_queue_entry_t __prev=0, __next, __last, __last_next, __first, __first_prev, __head_next; \
554 vm_page_t __p; \
555 \
556 /* if elt is part of vm_pages[] */ \
557 if((elt) >= vm_page_array_beginning_addr && (elt) < vm_page_array_boundary) { \
558 __first = __last = (vm_page_queue_entry_t) (elt); \
559 __clump_num = VM_PAGE_GET_CLUMP(elt); \
560 __n = VM_PAGE_GET_PHYS_PAGE(elt) & vm_clump_mask; \
561 /* scan backward looking for a buddy page */ \
562 for(__i=0, __p=(elt)-1; __i<__n && __p>=vm_page_array_beginning_addr; __i++, __p--) { \
563 if(__p->vm_page_q_state == VM_PAGE_ON_FREE_Q && __clump_num == VM_PAGE_GET_CLUMP(__p)) { \
564 if(__prev == 0) __prev = (vm_page_queue_entry_t) __p; \
565 __first = (vm_page_queue_entry_t) __p; \
566 __n_free++; \
567 } \
568 } \
569 /* scan forward looking for a buddy page */ \
570 for(__i=__n+1, __p=(elt)+1; __i<vm_clump_size && __p<vm_page_array_boundary; __i++, __p++) { \
571 if(__p->vm_page_q_state == VM_PAGE_ON_FREE_Q && __clump_num == VM_PAGE_GET_CLUMP(__p)) { \
572 __DEBUG_CHECK_BUDDIES(__check, __prev, __p, field); \
573 if(__prev == 0) __prev = (vm_page_queue_entry_t) VM_PAGE_UNPACK_PTR(__p->field.prev); \
574 __last = (vm_page_queue_entry_t) __p; \
575 __n_free++; \
576 } \
577 } \
578 __DEBUG_STAT_INCREMENT_INRANGE; \
579 } \
580 /* if elt is not part of vm_pages or if 1st page in clump, insert at tail */ \
581 if(__prev == 0) __prev = (vm_page_queue_entry_t) VM_PAGE_UNPACK_PTR((head)->prev); \
582 \
583 /* insert the element */ \
584 __next = (vm_page_queue_entry_t) VM_PAGE_UNPACK_PTR(__prev->next); \
585 (elt)->field.next = __prev->next; \
586 (elt)->field.prev = __next->prev; \
587 __prev->next = __next->prev = VM_PAGE_PACK_PTR(elt); \
588 __DEBUG_STAT_INCREMENT_INSERTS; \
589 \
590 /* check if clump needs to be promoted to head */ \
591 if(__n_free >= vm_clump_promote_threshold && __n_free > 1) { \
592 __first_prev = (vm_page_queue_entry_t) VM_PAGE_UNPACK_PTR(__first->prev); \
593 if(__first_prev != (head)) { /* if not at head already */ \
594 __last_next = (vm_page_queue_entry_t) VM_PAGE_UNPACK_PTR(__last->next); \
595 /* verify that the links within the clump are consistent */ \
596 __DEBUG_VERIFY_LINKS(__i, __first, __n_free, __last_next); \
597 /* promote clump to head */ \
598 __first_prev->next = __last->next; \
599 __last_next->prev = __first->prev; \
600 __first->prev = VM_PAGE_PACK_PTR(head); \
601 __last->next = (head)->next; \
602 __head_next = (vm_page_queue_entry_t) VM_PAGE_UNPACK_PTR((head)->next); \
603 __head_next->prev = VM_PAGE_PACK_PTR(__last); \
604 (head)->next = VM_PAGE_PACK_PTR(__first); \
605 __DEBUG_STAT_INCREMENT_PROMOTES(__n_free); \
606 } \
607 } \
608MACRO_END
609#endif
610
39037602
A		 611/*
612 * Macro: vm_page_queue_enter_first
613 * Function:
614 * Insert a new element at the head of the queue.
615 * Header:
616 * void queue_enter_first(q, elt, type, field)
617 * queue_t q;
618 * <type> elt;
619 * <type> is what's in our queue
620 * <field> is the chain field in (*<type>)
621 * Note:
622 * This should only be used with Method 2 queue iteration (element chains)
623 */
624#define vm_page_queue_enter_first(head, elt, type, field) \
625MACRO_BEGIN \
626 vm_page_queue_entry_t __next; \
627 \
628 __next = ((vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR((head)->next)); \
629 if ((head) == __next) { \
630 (head)->prev = VM_PAGE_PACK_PTR(elt); \
631 } \
632 else { \
633 ((type)(void *)__next)->field.prev = VM_PAGE_PACK_PTR(elt); \
634 } \
635 (elt)->field.next = VM_PAGE_PACK_PTR(__next); \
636 (elt)->field.prev = VM_PAGE_PACK_PTR(head); \
637 (head)->next = VM_PAGE_PACK_PTR(elt); \
638MACRO_END
639
640
641/*
642 * Macro: vm_page_queue_remove
643 * Function:
644 * Remove an arbitrary item from the queue.
645 * Header:
646 * void vm_page_queue_remove(q, qe, type, field)
647 * arguments as in vm_page_queue_enter
648 * Note:
649 * This should only be used with Method 2 queue iteration (element chains)
650 */
651#define vm_page_queue_remove(head, elt, type, field) \
652MACRO_BEGIN \
653 vm_page_queue_entry_t __next, __prev; \
654 \
655 __next = ((vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR((elt)->field.next)); \
656 __prev = ((vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR((elt)->field.prev)); \
657 \
658 if ((head) == __next) \
659 (head)->prev = VM_PAGE_PACK_PTR(__prev); \
660 else \
661 ((type)(void *)__next)->field.prev = VM_PAGE_PACK_PTR(__prev); \
662 \
663 if ((head) == __prev) \
664 (head)->next = VM_PAGE_PACK_PTR(__next); \
665 else \
666 ((type)(void *)__prev)->field.next = VM_PAGE_PACK_PTR(__next); \
667 \
668 (elt)->field.next = 0; \
669 (elt)->field.prev = 0; \
670MACRO_END
671
672
673/*
674 * Macro: vm_page_queue_remove_first
675 * Function:
676 * Remove and return the entry at the head of
677 * the queue.
678 * Header:
679 * vm_page_queue_remove_first(head, entry, type, field)
680 * entry is returned by reference
681 * Note:
682 * This should only be used with Method 2 queue iteration (element chains)
683 */
684#define vm_page_queue_remove_first(head, entry, type, field) \
685MACRO_BEGIN \
686 vm_page_queue_entry_t __next; \
687 \
688 (entry) = (type)(void *) VM_PAGE_UNPACK_PTR(((head)->next)); \
689 __next = ((vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR((entry)->field.next)); \
690 \
691 if ((head) == __next) \
692 (head)->prev = VM_PAGE_PACK_PTR(head); \
693 else \
694 ((type)(void *)(__next))->field.prev = VM_PAGE_PACK_PTR(head); \
695 (head)->next = VM_PAGE_PACK_PTR(__next); \
696 \
697 (entry)->field.next = 0; \
698 (entry)->field.prev = 0; \
699MACRO_END
700
701
5ba3f43e
A		 702/*
703 * Macro: vm_page_queue_remove_first_with_clump
704 * Function:
705 * Remove and return the entry at the head of the free queue
706 * end is set to 1 to indicate that we just returned the last page in a clump
707 *
708 * Header:
709 * vm_page_queue_remove_first_with_clump(head, entry, type, field, end)
710 * entry is returned by reference
711 * end is returned by reference
712 * Note:
713 * This should only be used with Method 2 queue iteration (element chains)
714 */
715#if defined(__x86_64__)
716#define vm_page_queue_remove_first_with_clump(head, entry, type, field, end) \
717MACRO_BEGIN \
718 vm_page_queue_entry_t __next; \
719 \
720 (entry) = (type)(void *) VM_PAGE_UNPACK_PTR(((head)->next)); \
721 __next = ((vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR((entry)->field.next)); \
722 \
723 (end)=0; \
724 if ((head) == __next) { \
725 (head)->prev = VM_PAGE_PACK_PTR(head); \
726 (end)=1; \
727 } \
728 else { \
729 ((type)(void *)(__next))->field.prev = VM_PAGE_PACK_PTR(head); \
730 if(VM_PAGE_GET_CLUMP(entry) != VM_PAGE_GET_CLUMP(((type)(void *)(__next)))) (end)=1; \
731 } \
732 (head)->next = VM_PAGE_PACK_PTR(__next); \
733 \
734 (entry)->field.next = 0; \
735 (entry)->field.prev = 0; \
736 \
737MACRO_END
738#endif
739
39037602
A		 740/*
741 * Macro: vm_page_queue_end
742 * Function:
743 * Tests whether a new entry is really the end of
744 * the queue.
745 * Header:
746 * boolean_t vm_page_queue_end(q, qe)
747 * vm_page_queue_t q;
748 * vm_page_queue_entry_t qe;
749 */
750#define vm_page_queue_end(q, qe) ((q) == (qe))
751
752
753/*
754 * Macro: vm_page_queue_empty
755 * Function:
756 * Tests whether a queue is empty.
757 * Header:
758 * boolean_t vm_page_queue_empty(q)
759 * vm_page_queue_t q;
760 */
761#define vm_page_queue_empty(q) vm_page_queue_end((q), ((vm_page_queue_entry_t)vm_page_queue_first(q)))
762
763
764
765/*
766 * Macro: vm_page_queue_first
767 * Function:
768 * Returns the first entry in the queue,
769 * Header:
770 * uintpr_t vm_page_queue_first(q)
771 * vm_page_queue_t q; \* IN *\
772 */
773#define vm_page_queue_first(q) (VM_PAGE_UNPACK_PTR((q)->next))
774
775
776
777/*
778 * Macro: vm_page_queue_last
779 * Function:
780 * Returns the last entry in the queue.
781 * Header:
782 * vm_page_queue_entry_t queue_last(q)
783 * queue_t q; \* IN *\
784 */
785#define vm_page_queue_last(q) (VM_PAGE_UNPACK_PTR((q)->prev))
786
787
788
789/*
790 * Macro: vm_page_queue_next
791 * Function:
792 * Returns the entry after an item in the queue.
793 * Header:
794 * uintpr_t vm_page_queue_next(qc)
795 * vm_page_queue_t qc;
796 */
797#define vm_page_queue_next(qc) (VM_PAGE_UNPACK_PTR((qc)->next))
798
799
800
801/*
802 * Macro: vm_page_queue_prev
803 * Function:
804 * Returns the entry before an item in the queue.
805 * Header:
806 * uinptr_t vm_page_queue_prev(qc)
807 * vm_page_queue_t qc;
808 */
809#define vm_page_queue_prev(qc) (VM_PAGE_UNPACK_PTR((qc)->prev))
810
811
812
813/*
814 * Macro: vm_page_queue_iterate
815 * Function:
816 * iterate over each item in the queue.
817 * Generates a 'for' loop, setting elt to
818 * each item in turn (by reference).
819 * Header:
820 * vm_page_queue_iterate(q, elt, type, field)
821 * queue_t q;
822 * <type> elt;
823 * <type> is what's in our queue
824 * <field> is the chain field in (*<type>)
825 * Note:
826 * This should only be used with Method 2 queue iteration (element chains)
827 */
828#define vm_page_queue_iterate(head, elt, type, field) \
829 for ((elt) = (type)(void *) vm_page_queue_first(head); \
830 !vm_page_queue_end((head), (vm_page_queue_entry_t)(elt)); \
831 (elt) = (type)(void *) vm_page_queue_next(&(elt)->field))
832
833#else
834
835#define VM_VPLQ_ALIGNMENT 128
836#define VM_PACKED_POINTER_ALIGNMENT 4
837#define VM_PACKED_POINTER_SHIFT 0
838
839#define VM_PACKED_FROM_VM_PAGES_ARRAY 0
840
841#define VM_PAGE_PACK_PTR(p) (p)
842#define VM_PAGE_UNPACK_PTR(p) ((uintptr_t)(p))
843
844#define VM_PAGE_OBJECT(p) (vm_object_t)(p->vm_page_object)
845#define VM_PAGE_PACK_OBJECT(o) ((vm_page_object_t)(VM_PAGE_PACK_PTR(o)))
846
847
848#define VM_PAGE_ZERO_PAGEQ_ENTRY(p) \
849MACRO_BEGIN \
850 (p)->pageq.next = 0; \
851 (p)->pageq.prev = 0; \
852MACRO_END
853
854#define VM_PAGE_CONVERT_TO_QUEUE_ENTRY(p) ((queue_entry_t)(p))
855
856#define vm_page_remque remque
857#define vm_page_enqueue_tail enqueue_tail
858#define vm_page_queue_init queue_init
859#define vm_page_queue_enter queue_enter
860#define vm_page_queue_enter_first queue_enter_first
861#define vm_page_queue_remove queue_remove
862#define vm_page_queue_remove_first queue_remove_first
863#define vm_page_queue_end queue_end
864#define vm_page_queue_empty queue_empty
865#define vm_page_queue_first queue_first
866#define vm_page_queue_last queue_last
867#define vm_page_queue_next queue_next
868#define vm_page_queue_prev queue_prev
869#define vm_page_queue_iterate queue_iterate
870
871#endif
872
873
874
875/*
876 * VM_PAGE_MIN_SPECULATIVE_AGE_Q through VM_PAGE_MAX_SPECULATIVE_AGE_Q
877 * represents a set of aging bins that are 'protected'...
878 *
879 * VM_PAGE_SPECULATIVE_AGED_Q is a list of the speculative pages that have
880 * not yet been 'claimed' but have been aged out of the protective bins
881 * this occurs in vm_page_speculate when it advances to the next bin
882 * and discovers that it is still occupied... at that point, all of the
883 * pages in that bin are moved to the VM_PAGE_SPECULATIVE_AGED_Q. the pages
884 * in that bin are all guaranteed to have reached at least the maximum age
885 * we allow for a protected page... they can be older if there is no
886 * memory pressure to pull them from the bin, or there are no new speculative pages
887 * being generated to push them out.
888 * this list is the one that vm_pageout_scan will prefer when looking
889 * for pages to move to the underweight free list
890 *
891 * VM_PAGE_MAX_SPECULATIVE_AGE_Q * VM_PAGE_SPECULATIVE_Q_AGE_MS
892 * defines the amount of time a speculative page is normally
893 * allowed to live in the 'protected' state (i.e. not available
894 * to be stolen if vm_pageout_scan is running and looking for
895 * pages)... however, if the total number of speculative pages
896 * in the protected state exceeds our limit (defined in vm_pageout.c)
897 * and there are none available in VM_PAGE_SPECULATIVE_AGED_Q, then
898 * vm_pageout_scan is allowed to steal pages from the protected
899 * bucket even if they are underage.
900 *
901 * vm_pageout_scan is also allowed to pull pages from a protected
902 * bin if the bin has reached the "age of consent" we've set
903 */
904#define VM_PAGE_MAX_SPECULATIVE_AGE_Q 10
905#define VM_PAGE_MIN_SPECULATIVE_AGE_Q 1
906#define VM_PAGE_SPECULATIVE_AGED_Q 0
907
908#define VM_PAGE_SPECULATIVE_Q_AGE_MS 500
909
910struct vm_speculative_age_q {
911 /*
912 * memory queue for speculative pages via clustered pageins
913 */
914 vm_page_queue_head_t age_q;
915 mach_timespec_t age_ts;
916} __attribute__((aligned(VM_PACKED_POINTER_ALIGNMENT)));
917
918
919
920extern
921struct vm_speculative_age_q vm_page_queue_speculative[];
922
923extern int speculative_steal_index;
924extern int speculative_age_index;
925extern unsigned int vm_page_speculative_q_age_ms;
1c79356b 926
b0d623f7
A		 927
928typedef struct vm_locks_array {
929 char pad __attribute__ ((aligned (64)));
930 lck_mtx_t vm_page_queue_lock2 __attribute__ ((aligned (64)));
931 lck_mtx_t vm_page_queue_free_lock2 __attribute__ ((aligned (64)));
932 char pad2 __attribute__ ((aligned (64)));
933} vm_locks_array_t;
934
935
39037602
A		 936#if CONFIG_BACKGROUND_QUEUE
937extern void vm_page_assign_background_state(vm_page_t mem);
938extern void vm_page_update_background_state(vm_page_t mem);
939extern void vm_page_add_to_backgroundq(vm_page_t mem, boolean_t first);
940extern void vm_page_remove_from_backgroundq(vm_page_t mem);
941#endif
942
943#define VM_PAGE_WIRED(m) ((m)->vm_page_q_state == VM_PAGE_IS_WIRED)
944#define NEXT_PAGE(m) ((m)->snext)
945#define NEXT_PAGE_PTR(m) (&(m)->snext)
1c79356b
A		 946
947/*
948 * XXX The unusual bit should not be necessary. Most of the bit
949 * XXX fields above really want to be masks.
950 */
951
952/*
953 * For debugging, this macro can be defined to perform
954 * some useful check on a page structure.
39037602
A		 955 * INTENTIONALLY left as a no-op so that the
956 * current call-sites can be left intact for future uses.
1c79356b
A		 957 */
958
b0d623f7
A		 959#define VM_PAGE_CHECK(mem) \
960 MACRO_BEGIN \
b0d623f7 961 MACRO_END
2d21ac55
A		 962
963/* Page coloring:
964 *
965 * The free page list is actually n lists, one per color,
966 * where the number of colors is a function of the machine's
967 * cache geometry set at system initialization. To disable
968 * coloring, set vm_colors to 1 and vm_color_mask to 0.
969 * The boot-arg "colors" may be used to override vm_colors.
970 * Note that there is little harm in having more colors than needed.
971 */
972
973#define MAX_COLORS 128
974#define DEFAULT_COLORS 32
975
976extern
977unsigned int vm_colors; /* must be in range 1..MAX_COLORS */
978extern
979unsigned int vm_color_mask; /* must be (vm_colors-1) */
980extern
981unsigned int vm_cache_geometry_colors; /* optimal #colors based on cache geometry */
1c79356b 982
b0d623f7
A		 983/*
984 * Wired memory is a very limited resource and we can't let users exhaust it
985 * and deadlock the entire system. We enforce the following limits:
986 *
987 * vm_user_wire_limit (default: all memory minus vm_global_no_user_wire_amount)
988 * how much memory can be user-wired in one user task
989 *
990 * vm_global_user_wire_limit (default: same as vm_user_wire_limit)
991 * how much memory can be user-wired in all user tasks
992 *
993 * vm_global_no_user_wire_amount (default: VM_NOT_USER_WIREABLE)
994 * how much memory must remain user-unwired at any time
995 */
996#define VM_NOT_USER_WIREABLE (64*1024*1024) /* 64MB */
997extern
998vm_map_size_t vm_user_wire_limit;
999extern
1000vm_map_size_t vm_global_user_wire_limit;
1001extern
1002vm_map_size_t vm_global_no_user_wire_amount;
1003
1c79356b
A		 1004/*
1005 * Each pageable resident page falls into one of three lists:
1006 *
1007 * free
2d21ac55
A		 1008 * Available for allocation now. The free list is
1009 * actually an array of lists, one per color.
1c79356b
A		 1010 * inactive
1011 * Not referenced in any map, but still has an
1012 * object/offset-page mapping, and may be dirty.
1013 * This is the list of pages that should be
2d21ac55
A		 1014 * paged out next. There are actually two
1015 * inactive lists, one for pages brought in from
1016 * disk or other backing store, and another
1017 * for "zero-filled" pages. See vm_pageout_scan()
1018 * for the distinction and usage.
1c79356b
A		 1019 * active
1020 * A list of pages which have been placed in
1021 * at least one physical map. This list is
1022 * ordered, in LRU-like fashion.
1023 */
1024
b0d623f7
A		 1025
1026#define VPL_LOCK_SPIN 1
1027
1028struct vpl {
39037602 1029 vm_page_queue_head_t vpl_queue;
b0d623f7 1030 unsigned int vpl_count;
39236c6e
A		 1031 unsigned int vpl_internal_count;
1032 unsigned int vpl_external_count;
b0d623f7
A		 1033#ifdef VPL_LOCK_SPIN
1034 lck_spin_t vpl_lock;
1035#else
1036 lck_mtx_t vpl_lock;
1037 lck_mtx_ext_t vpl_lock_ext;
1038#endif
1039};
1040
1041struct vplq {
1042 union {
39037602 1043 char cache_line_pad[VM_VPLQ_ALIGNMENT];
b0d623f7
A		 1044 struct vpl vpl;
1045 } vpl_un;
1046};
1047extern
1048unsigned int vm_page_local_q_count;
1049extern
1050struct vplq *vm_page_local_q;
1051extern
1052unsigned int vm_page_local_q_soft_limit;
1053extern
1054unsigned int vm_page_local_q_hard_limit;
1055extern
1056vm_locks_array_t vm_page_locks;
1057
1c79356b 1058extern
39037602 1059vm_page_queue_head_t vm_lopage_queue_free; /* low memory free queue */
2d21ac55 1060extern
39037602 1061vm_page_queue_head_t vm_page_queue_active; /* active memory queue */
1c79356b 1062extern
39037602
A		 1063vm_page_queue_head_t vm_page_queue_inactive; /* inactive memory queue for normal pages */
1064#if CONFIG_SECLUDED_MEMORY
1c79356b 1065extern
39037602
A		 1066vm_page_queue_head_t vm_page_queue_secluded; /* reclaimable pages secluded for Camera */
1067#endif /* CONFIG_SECLUDED_MEMORY */
2d21ac55 1068extern
39037602 1069vm_page_queue_head_t vm_page_queue_cleaned; /* clean-queue inactive memory */
316670eb 1070extern
39037602 1071vm_page_queue_head_t vm_page_queue_anonymous; /* inactive memory queue for anonymous pages */
b0d623f7 1072extern
39037602 1073vm_page_queue_head_t vm_page_queue_throttled; /* memory queue for throttled pageout pages */
1c79356b 1074
3e170ce0
A		 1075extern
1076queue_head_t vm_objects_wired;
1077extern
1078lck_spin_t vm_objects_wired_lock;
1079
39037602
A		 1080#if CONFIG_BACKGROUND_QUEUE
1081
1082#define VM_PAGE_BACKGROUND_TARGET_MAX 50000
1083
1084#define VM_PAGE_BG_DISABLED 0
1085#define VM_PAGE_BG_LEVEL_1 1
39037602
A		 1086
1087extern
1088vm_page_queue_head_t vm_page_queue_background;
1089extern
1090uint64_t vm_page_background_promoted_count;
1091extern
1092uint32_t vm_page_background_count;
1093extern
39037602
A		 1094uint32_t vm_page_background_target;
1095extern
1096uint32_t vm_page_background_internal_count;
1097extern
1098uint32_t vm_page_background_external_count;
1099extern
1100uint32_t vm_page_background_mode;
1101extern
1102uint32_t vm_page_background_exclude_external;
1103
1104#endif
3e170ce0 1105
1c79356b
A		 1106extern
1107vm_offset_t first_phys_addr; /* physical address for first_page */
1108extern
1109vm_offset_t last_phys_addr; /* physical address for last_page */
1110
1111extern
2d21ac55 1112unsigned int vm_page_free_count; /* How many pages are free? (sum of all colors) */
1c79356b 1113extern
91447636 1114unsigned int vm_page_active_count; /* How many pages are active? */
1c79356b 1115extern
91447636 1116unsigned int vm_page_inactive_count; /* How many pages are inactive? */
39037602
A		 1117#if CONFIG_SECLUDED_MEMORY
1118extern
1119unsigned int vm_page_secluded_count; /* How many pages are secluded? */
1120extern
1121unsigned int vm_page_secluded_count_free;
1122extern
1123unsigned int vm_page_secluded_count_inuse;
1124#endif /* CONFIG_SECLUDED_MEMORY */
1c79356b 1125extern
316670eb
A		 1126unsigned int vm_page_cleaned_count; /* How many pages are in the clean queue? */
1127extern
2d21ac55
A		 1128unsigned int vm_page_throttled_count;/* How many inactives are throttled */
1129extern
1130unsigned int vm_page_speculative_count; /* How many speculative pages are unclaimed? */
39236c6e
A		 1131extern unsigned int vm_page_pageable_internal_count;
1132extern unsigned int vm_page_pageable_external_count;
1133extern
fe8ab488
A		 1134unsigned int vm_page_xpmapped_external_count; /* How many pages are mapped executable? */
1135extern
39236c6e
A		 1136unsigned int vm_page_external_count; /* How many pages are file-backed? */
1137extern
1138unsigned int vm_page_internal_count; /* How many pages are anonymous? */
2d21ac55 1139extern
db609669
A		 1140unsigned int vm_page_wire_count; /* How many pages are wired? */
1141extern
1142unsigned int vm_page_wire_count_initial; /* How many pages wired at startup */
1c79356b 1143extern
91447636 1144unsigned int vm_page_free_target; /* How many do we want free? */
1c79356b 1145extern
91447636 1146unsigned int vm_page_free_min; /* When to wakeup pageout */
1c79356b 1147extern
b0d623f7
A		 1148unsigned int vm_page_throttle_limit; /* When to throttle new page creation */
1149extern
1150uint32_t vm_page_creation_throttle; /* When to throttle new page creation */
1151extern
91447636 1152unsigned int vm_page_inactive_target;/* How many do we want inactive? */
39037602
A		 1153#if CONFIG_SECLUDED_MEMORY
1154extern
1155unsigned int vm_page_secluded_target;/* How many do we want secluded? */
1156#endif /* CONFIG_SECLUDED_MEMORY */
1c79356b 1157extern
316670eb
A		 1158unsigned int vm_page_anonymous_min; /* When it's ok to pre-clean */
1159extern
39037602 1160unsigned int vm_page_inactive_min; /* When to wakeup pageout */
2d21ac55 1161extern
91447636 1162unsigned int vm_page_free_reserved; /* How many pages reserved to do pageout */
1c79356b 1163extern
b0d623f7 1164unsigned int vm_page_throttle_count; /* Count of page allocations throttled */
91447636
A		 1165extern
1166unsigned int vm_page_gobble_count;
3e170ce0
A		 1167extern
1168unsigned int vm_page_stolen_count; /* Count of stolen pages not acccounted in zones */
1169
91447636 1170
b0d623f7 1171#if DEVELOPMENT || DEBUG
2d21ac55
A		 1172extern
1173unsigned int vm_page_speculative_used;
b0d623f7
A		 1174#endif
1175
55e303ae 1176extern
91447636 1177unsigned int vm_page_purgeable_count;/* How many pages are purgeable now ? */
55e303ae 1178extern
b0d623f7
A		 1179unsigned int vm_page_purgeable_wired_count;/* How many purgeable pages are wired now ? */
1180extern
91447636 1181uint64_t vm_page_purged_count; /* How many pages got purged so far ? */
1c79356b 1182
1c79356b
A		 1183extern unsigned int vm_page_free_wanted;
1184 /* how many threads are waiting for memory */
1185
2d21ac55
A		 1186extern unsigned int vm_page_free_wanted_privileged;
1187 /* how many VM privileged threads are waiting for memory */
39037602
A		 1188#if CONFIG_SECLUDED_MEMORY
1189extern unsigned int vm_page_free_wanted_secluded;
1190 /* how many threads are waiting for secluded memory */
1191#endif /* CONFIG_SECLUDED_MEMORY */
2d21ac55 1192
5ba3f43e 1193extern const ppnum_t vm_page_fictitious_addr;
1c79356b
A		 1194 /* (fake) phys_addr of fictitious pages */
1195
5ba3f43e 1196extern const ppnum_t vm_page_guard_addr;
2d21ac55
A		 1197 /* (fake) phys_addr of guard pages */
1198
1199
91447636
A		 1200extern boolean_t vm_page_deactivate_hint;
1201
39236c6e
A		 1202extern int vm_compressor_mode;
1203
0b4c1975
A		 1204/*
1205 0 = all pages avail ( default. )
1206 1 = disable high mem ( cap max pages to 4G)
1207 2 = prefer himem
1208*/
0c530ab8
A		 1209extern int vm_himemory_mode;
1210
0b4c1975
A		 1211extern boolean_t vm_lopage_needed;
1212extern uint32_t vm_lopage_free_count;
1213extern uint32_t vm_lopage_free_limit;
1214extern uint32_t vm_lopage_lowater;
1215extern boolean_t vm_lopage_refill;
0c530ab8 1216extern uint64_t max_valid_dma_address;
0b4c1975 1217extern ppnum_t max_valid_low_ppnum;
0c530ab8 1218
1c79356b
A		 1219/*
1220 * Prototypes for functions exported by this module.
1221 */
1222extern void vm_page_bootstrap(
1223 vm_offset_t *startp,
39236c6e 1224 vm_offset_t *endp);
1c79356b 1225
39236c6e 1226extern void vm_page_module_init(void);
2d21ac55 1227
b0d623f7
A		 1228extern void vm_page_init_local_q(void);
1229
1c79356b 1230extern void vm_page_create(
55e303ae
A		 1231 ppnum_t start,
1232 ppnum_t end);
1c79356b 1233
3e170ce0
A		 1234extern vm_page_t kdp_vm_page_lookup(
1235 vm_object_t object,
1236 vm_object_offset_t offset);
1237
1c79356b
A		 1238extern vm_page_t vm_page_lookup(
1239 vm_object_t object,
1240 vm_object_offset_t offset);
1241
1242extern vm_page_t vm_page_grab_fictitious(void);
1243
2d21ac55
A		 1244extern vm_page_t vm_page_grab_guard(void);
1245
1c79356b
A		 1246extern void vm_page_release_fictitious(
1247 vm_page_t page);
1248
1c79356b
A		 1249extern void vm_page_more_fictitious(void);
1250
1251extern int vm_pool_low(void);
1252
1253extern vm_page_t vm_page_grab(void);
39037602
A		 1254extern vm_page_t vm_page_grab_options(int flags);
1255#if CONFIG_SECLUDED_MEMORY
1256#define VM_PAGE_GRAB_SECLUDED 0x00000001
1257#endif /* CONFIG_SECLUDED_MEMORY */
1c79356b 1258
0c530ab8
A		 1259extern vm_page_t vm_page_grablo(void);
1260
1c79356b 1261extern void vm_page_release(
39037602
A		 1262 vm_page_t page,
1263 boolean_t page_queues_locked);
1c79356b 1264
1c79356b
A		 1265extern boolean_t vm_page_wait(
1266 int interruptible );
1267
1268extern vm_page_t vm_page_alloc(
1269 vm_object_t object,
1270 vm_object_offset_t offset);
1271
2d21ac55
A		 1272extern vm_page_t vm_page_alloc_guard(
1273 vm_object_t object,
1274 vm_object_offset_t offset);
1275
1c79356b
A		 1276extern void vm_page_init(
1277 vm_page_t page,
0b4c1975 1278 ppnum_t phys_page,
6d2010ae 1279 boolean_t lopage);
1c79356b
A		 1280
1281extern void vm_page_free(
b0d623f7 1282 vm_page_t page);
1c79356b 1283
b0d623f7
A		 1284extern void vm_page_free_unlocked(
1285 vm_page_t page,
1286 boolean_t remove_from_hash);
2d21ac55 1287
1c79356b
A		 1288extern void vm_page_activate(
1289 vm_page_t page);
1290
1291extern void vm_page_deactivate(
1292 vm_page_t page);
1293
b0d623f7
A		 1294extern void vm_page_deactivate_internal(
1295 vm_page_t page,
1296 boolean_t clear_hw_reference);
1297
316670eb
A		 1298extern void vm_page_enqueue_cleaned(vm_page_t page);
1299
2d21ac55
A		 1300extern void vm_page_lru(
1301 vm_page_t page);
1302
1303extern void vm_page_speculate(
1304 vm_page_t page,
1305 boolean_t new);
1306
1307extern void vm_page_speculate_ageit(
1308 struct vm_speculative_age_q *aq);
1309
b0d623f7
A		 1310extern void vm_page_reactivate_all_throttled(void);
1311
1312extern void vm_page_reactivate_local(uint32_t lid, boolean_t force, boolean_t nolocks);
1313
1c79356b
A		 1314extern void vm_page_rename(
1315 vm_page_t page,
1316 vm_object_t new_object,
5ba3f43e 1317 vm_object_offset_t new_offset);
1c79356b
A		 1318
1319extern void vm_page_insert(
1320 vm_page_t page,
1321 vm_object_t object,
1322 vm_object_offset_t offset);
1323
3e170ce0
A		 1324extern void vm_page_insert_wired(
1325 vm_page_t page,
1326 vm_object_t object,
1327 vm_object_offset_t offset,
1328 vm_tag_t tag);
1329
4a3eedf9 1330extern void vm_page_insert_internal(
b0d623f7 1331 vm_page_t page,
4a3eedf9
A		 1332 vm_object_t object,
1333 vm_object_offset_t offset,
3e170ce0 1334 vm_tag_t tag,
b0d623f7 1335 boolean_t queues_lock_held,
316670eb 1336 boolean_t insert_in_hash,
3e170ce0
A		 1337 boolean_t batch_pmap_op,
1338 boolean_t delayed_accounting,
1339 uint64_t *delayed_ledger_update);
4a3eedf9 1340
1c79356b
A		 1341extern void vm_page_replace(
1342 vm_page_t mem,
1343 vm_object_t object,
1344 vm_object_offset_t offset);
1345
1346extern void vm_page_remove(
b0d623f7
A		 1347 vm_page_t page,
1348 boolean_t remove_from_hash);
1c79356b
A		 1349
1350extern void vm_page_zero_fill(
1351 vm_page_t page);
1352
1353extern void vm_page_part_zero_fill(
1354 vm_page_t m,
1355 vm_offset_t m_pa,
1356 vm_size_t len);
1357
1358extern void vm_page_copy(
1359 vm_page_t src_page,
1360 vm_page_t dest_page);
1361
1362extern void vm_page_part_copy(
1363 vm_page_t src_m,
1364 vm_offset_t src_pa,
1365 vm_page_t dst_m,
1366 vm_offset_t dst_pa,
1367 vm_size_t len);
1368
1369extern void vm_page_wire(
3e170ce0
A		 1370 vm_page_t page,
1371 vm_tag_t tag,
1372 boolean_t check_memorystatus);
1c79356b
A		 1373
1374extern void vm_page_unwire(
0b4c1975
A		 1375 vm_page_t page,
1376 boolean_t queueit);
1c79356b
A		 1377
1378extern void vm_set_page_size(void);
1379
1380extern void vm_page_gobble(
1381 vm_page_t page);
1382
2d21ac55 1383extern void vm_page_validate_cs(vm_page_t page);
593a1d5f
A		 1384extern void vm_page_validate_cs_mapped(
1385 vm_page_t page,
1386 const void *kaddr);
3e170ce0
A		 1387extern void vm_page_validate_cs_mapped_chunk(
1388 vm_page_t page,
1389 const void *kaddr,
1390 vm_offset_t chunk_offset,
39037602 1391 vm_size_t chunk_size,
3e170ce0
A		 1392 boolean_t *validated,
1393 unsigned *tainted);
2d21ac55 1394
b0d623f7
A		 1395extern void vm_page_free_prepare_queues(
1396 vm_page_t page);
1397
1398extern void vm_page_free_prepare_object(
1399 vm_page_t page,
1400 boolean_t remove_from_hash);
1401
fe8ab488
A		 1402#if CONFIG_IOSCHED
1403extern wait_result_t vm_page_sleep(
1404 vm_object_t object,
1405 vm_page_t m,
1406 int interruptible);
1407#endif
1408
1409extern void vm_pressure_response(void);
1410
316670eb 1411#if CONFIG_JETSAM
39236c6e 1412extern void memorystatus_pages_update(unsigned int pages_avail);
316670eb
A		 1413
1414#define VM_CHECK_MEMORYSTATUS do { \
39236c6e 1415 memorystatus_pages_update( \
fe8ab488 1416 vm_page_pageable_external_count + \
39236c6e 1417 vm_page_free_count + \
39037602 1418 (VM_DYNAMIC_PAGING_ENABLED() ? 0 : vm_page_purgeable_count) \
316670eb
A		 1419 ); \
1420 } while(0)
39236c6e
A		 1421
1422#else /* CONFIG_JETSAM */
1423
5ba3f43e
A		 1424#if CONFIG_EMBEDDED
1425
1426#define VM_CHECK_MEMORYSTATUS do {} while(0)
1427
1428#else /* CONFIG_EMBEDDED */
39236c6e 1429
39236c6e
A		 1430#define VM_CHECK_MEMORYSTATUS vm_pressure_response()
1431
5ba3f43e 1432#endif /* CONFIG_EMBEDDED */
39236c6e
A		 1433
1434#endif /* CONFIG_JETSAM */
6d2010ae 1435
1c79356b
A		 1436/*
1437 * Functions implemented as macros. m->wanted and m->busy are
1438 * protected by the object lock.
1439 */
1440
5ba3f43e
A		 1441#if CONFIG_EMBEDDED
1442#define SET_PAGE_DIRTY(m, set_pmap_modified) \
1443 MACRO_BEGIN \
1444 vm_page_t __page__ = (m); \
1445 if (__page__->dirty == FALSE && (set_pmap_modified)) { \
1446 pmap_set_modify(VM_PAGE_GET_PHYS_PAGE(__page__)); \
1447 } \
1448 __page__->dirty = TRUE; \
1449 MACRO_END
1450#else /* CONFIG_EMBEDDED */
316670eb
A		 1451#define SET_PAGE_DIRTY(m, set_pmap_modified) \
1452 MACRO_BEGIN \
1453 vm_page_t __page__ = (m); \
316670eb
A		 1454 __page__->dirty = TRUE; \
1455 MACRO_END
5ba3f43e 1456#endif /* CONFIG_EMBEDDED */
316670eb 1457
1c79356b 1458#define PAGE_ASSERT_WAIT(m, interruptible) \
9bccf70c
A		 1459 (((m)->wanted = TRUE), \
1460 assert_wait((event_t) (m), (interruptible)))
1461
fe8ab488 1462#if CONFIG_IOSCHED
9bccf70c 1463#define PAGE_SLEEP(o, m, interruptible) \
fe8ab488
A		 1464 vm_page_sleep(o, m, interruptible)
1465#else
1466#define PAGE_SLEEP(o, m, interruptible) \
1467 (((m)->wanted = TRUE), \
1468 thread_sleep_vm_object((o), (m), (interruptible)))
1469#endif
1c79356b
A		 1470
1471#define PAGE_WAKEUP_DONE(m) \
1472 MACRO_BEGIN \
1473 (m)->busy = FALSE; \
1474 if ((m)->wanted) { \
1475 (m)->wanted = FALSE; \
1476 thread_wakeup((event_t) (m)); \
1477 } \
1478 MACRO_END
1479
1480#define PAGE_WAKEUP(m) \
1481 MACRO_BEGIN \
1482 if ((m)->wanted) { \
1483 (m)->wanted = FALSE; \
1484 thread_wakeup((event_t) (m)); \
1485 } \
1486 MACRO_END
1487
1488#define VM_PAGE_FREE(p) \
1489 MACRO_BEGIN \
b0d623f7 1490 vm_page_free_unlocked(p, TRUE); \
1c79356b
A		 1491 MACRO_END
1492
1493#define VM_PAGE_GRAB_FICTITIOUS(M) \
1494 MACRO_BEGIN \
1495 while ((M = vm_page_grab_fictitious()) == VM_PAGE_NULL) \
1496 vm_page_more_fictitious(); \
1497 MACRO_END
1498
1c79356b
A		 1499#define VM_PAGE_WAIT() ((void)vm_page_wait(THREAD_UNINT))
1500
b0d623f7
A		 1501#define vm_page_queue_lock (vm_page_locks.vm_page_queue_lock2)
1502#define vm_page_queue_free_lock (vm_page_locks.vm_page_queue_free_lock2)
1503
1504#define vm_page_lock_queues() lck_mtx_lock(&vm_page_queue_lock)
39037602 1505#define vm_page_trylock_queues() lck_mtx_try_lock(&vm_page_queue_lock)
b0d623f7
A		 1506#define vm_page_unlock_queues() lck_mtx_unlock(&vm_page_queue_lock)
1507
1508#define vm_page_lockspin_queues() lck_mtx_lock_spin(&vm_page_queue_lock)
1509#define vm_page_trylockspin_queues() lck_mtx_try_lock_spin(&vm_page_queue_lock)
1510#define vm_page_lockconvert_queues() lck_mtx_convert_spin(&vm_page_queue_lock)
1511
1512#ifdef VPL_LOCK_SPIN
1513#define VPL_LOCK_INIT(vlq, vpl_grp, vpl_attr) lck_spin_init(&vlq->vpl_lock, vpl_grp, vpl_attr)
1514#define VPL_LOCK(vpl) lck_spin_lock(vpl)
1515#define VPL_UNLOCK(vpl) lck_spin_unlock(vpl)
1516#else
1517#define VPL_LOCK_INIT(vlq, vpl_grp, vpl_attr) lck_mtx_init_ext(&vlq->vpl_lock, &vlq->vpl_lock_ext, vpl_grp, vpl_attr)
1518#define VPL_LOCK(vpl) lck_mtx_lock_spin(vpl)
1519#define VPL_UNLOCK(vpl) lck_mtx_unlock(vpl)
1520#endif
1c79356b 1521
2d21ac55 1522
b0d623f7
A		 1523#if DEVELOPMENT || DEBUG
1524#define VM_PAGE_SPECULATIVE_USED_ADD() \
1525 MACRO_BEGIN \
1526 OSAddAtomic(1, &vm_page_speculative_used); \
2d21ac55 1527 MACRO_END
b0d623f7
A		 1528#else
1529#define VM_PAGE_SPECULATIVE_USED_ADD()
1530#endif
2d21ac55
A		 1531
1532
1533#define VM_PAGE_CONSUME_CLUSTERED(mem) \
1534 MACRO_BEGIN \
39037602
A		 1535 ppnum_t __phys_page; \
1536 __phys_page = VM_PAGE_GET_PHYS_PAGE(mem); \
1537 pmap_lock_phys_page(__phys_page); \
2d21ac55 1538 if (mem->clustered) { \
39037602
A		 1539 vm_object_t o; \
1540 o = VM_PAGE_OBJECT(mem); \
1541 assert(o); \
1542 o->pages_used++; \
2d21ac55 1543 mem->clustered = FALSE; \
b0d623f7 1544 VM_PAGE_SPECULATIVE_USED_ADD(); \
1c79356b 1545 } \
39037602 1546 pmap_unlock_phys_page(__phys_page); \
1c79356b
A		 1547 MACRO_END
1548
6d2010ae 1549
fe8ab488
A		 1550#define VM_PAGE_COUNT_AS_PAGEIN(mem) \
1551 MACRO_BEGIN \
39037602
A		 1552 { \
1553 vm_object_t o; \
1554 o = VM_PAGE_OBJECT(mem); \
fe8ab488
A		 1555 DTRACE_VM2(pgin, int, 1, (uint64_t *), NULL); \
1556 current_task()->pageins++; \
39037602 1557 if (o->internal) { \
fe8ab488
A		 1558 DTRACE_VM2(anonpgin, int, 1, (uint64_t *), NULL); \
1559 } else { \
1560 DTRACE_VM2(fspgin, int, 1, (uint64_t *), NULL); \
1561 } \
39037602 1562 } \
fe8ab488
A		 1563 MACRO_END
1564
3e170ce0
A		 1565/* adjust for stolen pages accounted elsewhere */
1566#define VM_PAGE_MOVE_STOLEN(page_count) \
1567 MACRO_BEGIN \
1568 vm_page_stolen_count -= (page_count); \
1569 vm_page_wire_count_initial -= (page_count); \
1570 MACRO_END
6d2010ae
A		 1571
1572#define DW_vm_page_unwire 0x01
1573#define DW_vm_page_wire 0x02
1574#define DW_vm_page_free 0x04
1575#define DW_vm_page_activate 0x08
1576#define DW_vm_page_deactivate_internal 0x10
1577#define DW_vm_page_speculate 0x20
1578#define DW_vm_page_lru 0x40
1579#define DW_vm_pageout_throttle_up 0x80
1580#define DW_PAGE_WAKEUP 0x100
1581#define DW_clear_busy 0x200
1582#define DW_clear_reference 0x400
1583#define DW_set_reference 0x800
1584#define DW_move_page 0x1000
1585#define DW_VM_PAGE_QUEUES_REMOVE 0x2000
316670eb 1586#define DW_enqueue_cleaned 0x4000
fe8ab488 1587#define DW_vm_phantom_cache_update 0x8000
6d2010ae
A		 1588
1589struct vm_page_delayed_work {
1590 vm_page_t dw_m;
1591 int dw_mask;
1592};
1593
3e170ce0 1594void vm_page_do_delayed_work(vm_object_t object, vm_tag_t tag, struct vm_page_delayed_work *dwp, int dw_count);
6d2010ae
A		 1595
1596extern unsigned int vm_max_delayed_work_limit;
1597
1598#define DEFAULT_DELAYED_WORK_LIMIT 32
1599
1600#define DELAYED_WORK_LIMIT(max) ((vm_max_delayed_work_limit >= max ? max : vm_max_delayed_work_limit))
1601
1602/*
1603 * vm_page_do_delayed_work may need to drop the object lock...
1604 * if it does, we need the pages it's looking at to
1605 * be held stable via the busy bit, so if busy isn't already
1606 * set, we need to set it and ask vm_page_do_delayed_work
1607 * to clear it and wakeup anyone that might have blocked on
1608 * it once we're done processing the page.
6d2010ae
A		 1609 */
1610
1611#define VM_PAGE_ADD_DELAYED_WORK(dwp, mem, dw_cnt) \
1612 MACRO_BEGIN \
1613 if (mem->busy == FALSE) { \
1614 mem->busy = TRUE; \
1615 if ( !(dwp->dw_mask & DW_vm_page_free)) \
1616 dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP); \
1617 } \
6d2010ae
A		 1618 dwp->dw_m = mem; \
1619 dwp++; \
316670eb 1620 dw_cnt++; \
6d2010ae
A		 1621 MACRO_END
1622
1623extern vm_page_t vm_object_page_grab(vm_object_t);
1624
15129b1c
A		 1625#if VM_PAGE_BUCKETS_CHECK
1626extern void vm_page_buckets_check(void);
1627#endif /* VM_PAGE_BUCKETS_CHECK */
6d2010ae 1628
39037602 1629extern void vm_page_queues_remove(vm_page_t mem, boolean_t remove_from_backgroundq);
3e170ce0
A		 1630extern void vm_page_remove_internal(vm_page_t page);
1631extern void vm_page_enqueue_inactive(vm_page_t mem, boolean_t first);
39037602 1632extern void vm_page_enqueue_active(vm_page_t mem, boolean_t first);
3e170ce0
A		 1633extern void vm_page_check_pageable_safe(vm_page_t page);
1634
1635
1c79356b 1636#endif /* _VM_VM_PAGE_H_ */
mirror of XNU