]> git.saurik.com Git - apple/xnu.git/blame - osfmk/vm/vm_page.h
projects / apple / xnu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
xnu-6153.141.1.tar.gz
[apple/xnu.git] / osfmk / vm / vm_page.h
CommitLineData
1c79356b 1/*
2d21ac55 2 * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
1c79356b 3 *
2d21ac55 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
0a7de745 5 *
2d21ac55
A		 6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
0a7de745 14 *
2d21ac55
A		 15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
0a7de745 17 *
2d21ac55
A		 18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
8f6c56a5
A		 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
2d21ac55
A		 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
0a7de745 25 *
2d21ac55 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
1c79356b
A		 27 */
28/*
29 * @OSF_COPYRIGHT@
30 */
0a7de745 31/*
1c79356b
A		 32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University
34 * All Rights Reserved.
0a7de745 35 *
1c79356b
A		 36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
0a7de745 41 *
1c79356b
A		 42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
0a7de745 45 *
1c79356b 46 * Carnegie Mellon requests users of this software to return to
0a7de745 47 *
1c79356b
A		 48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
0a7de745 52 *
1c79356b
A		 53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56/*
57 */
58/*
59 * File: vm/vm_page.h
60 * Author: Avadis Tevanian, Jr., Michael Wayne Young
61 * Date: 1985
62 *
63 * Resident memory system definitions.
64 */
65
0a7de745 66#ifndef _VM_VM_PAGE_H_
1c79356b
A		 67#define _VM_VM_PAGE_H_
68
91447636 69#include <debug.h>
15129b1c 70#include <vm/vm_options.h>
1c79356b
A		 71#include <mach/boolean.h>
72#include <mach/vm_prot.h>
73#include <mach/vm_param.h>
1c79356b 74
2d21ac55 75
39037602 76#if defined(__LP64__)
1c79356b 77
39037602
A		 78/*
79 * in order to make the size of a vm_page_t 64 bytes (cache line size for both arm64 and x86_64)
80 * we'll keep the next_m pointer packed... as long as the kernel virtual space where we allocate
81 * vm_page_t's from doesn't span more then 256 Gbytes, we're safe. There are live tests in the
82 * vm_page_t array allocation and the zone init code to determine if we can safely pack and unpack
83 * pointers from the 2 ends of these spaces
0b4e3aa0 84 */
0a7de745 85typedef uint32_t vm_page_packed_t;
2d21ac55 86
39037602 87struct vm_page_packed_queue_entry {
0a7de745
A		 88 vm_page_packed_t next; /* next element */
89 vm_page_packed_t prev; /* previous element */
2d21ac55 90};
0b4e3aa0 91
0a7de745
A		 92typedef struct vm_page_packed_queue_entry *vm_page_queue_t;
93typedef struct vm_page_packed_queue_entry vm_page_queue_head_t;
94typedef struct vm_page_packed_queue_entry vm_page_queue_chain_t;
95typedef struct vm_page_packed_queue_entry *vm_page_queue_entry_t;
0b4e3aa0 96
0a7de745 97typedef vm_page_packed_t vm_page_object_t;
6d2010ae 98
39037602
A		 99#else
100
101/*
0a7de745 102 * we can't do the packing trick on 32 bit architectures, so
39037602
A		 103 * just turn the macros into noops.
104 */
0a7de745 105typedef struct vm_page *vm_page_packed_t;
39037602 106
0a7de745
A		 107#define vm_page_queue_t queue_t
108#define vm_page_queue_head_t queue_head_t
109#define vm_page_queue_chain_t queue_chain_t
110#define vm_page_queue_entry_t queue_entry_t
39037602 111
0a7de745 112#define vm_page_object_t vm_object_t
39037602
A		 113#endif
114
115
116#include <vm/vm_object.h>
117#include <kern/queue.h>
118#include <kern/locks.h>
119
120#include <kern/macro_help.h>
121#include <libkern/OSAtomic.h>
0b4e3aa0 122
9bccf70c 123
0b4e3aa0 124
0a7de745 125#define VM_PAGE_COMPRESSOR_COUNT (compressor_object->resident_page_count)
39236c6e 126
1c79356b
A		 127/*
128 * Management of resident (logical) pages.
129 *
130 * A small structure is kept for each resident
131 * page, indexed by page number. Each structure
132 * is an element of several lists:
133 *
134 * A hash table bucket used to quickly
135 * perform object/offset lookups
136 *
137 * A list of all pages for a given object,
138 * so they can be quickly deactivated at
139 * time of deallocation.
140 *
141 * An ordered list of pages due for pageout.
142 *
143 * In addition, the structure contains the object
144 * and offset to which this page belongs (for pageout),
145 * and sundry status bits.
146 *
147 * Fields in this structure are locked either by the lock on the
148 * object that the page belongs to (O) or by the lock on the page
149 * queues (P). [Some fields require that both locks be held to
150 * change that field; holding either lock is sufficient to read.]
151 */
152
0a7de745 153#define VM_PAGE_NULL ((vm_page_t) 0)
fe8ab488 154
0a7de745
A		 155extern char vm_page_inactive_states[];
156extern char vm_page_pageable_states[];
157extern char vm_page_non_speculative_pageable_states[];
158extern char vm_page_active_or_inactive_states[];
fe8ab488 159
fe8ab488 160
0a7de745
A		 161#define VM_PAGE_INACTIVE(m) (vm_page_inactive_states[m->vmp_q_state])
162#define VM_PAGE_PAGEABLE(m) (vm_page_pageable_states[m->vmp_q_state])
163#define VM_PAGE_NON_SPECULATIVE_PAGEABLE(m) (vm_page_non_speculative_pageable_states[m->vmp_q_state])
164#define VM_PAGE_ACTIVE_OR_INACTIVE(m) (vm_page_active_or_inactive_states[m->vmp_q_state])
fe8ab488 165
fe8ab488 166
0a7de745
A		 167#define VM_PAGE_NOT_ON_Q 0 /* page is not present on any queue, nor is it wired... mainly a transient state */
168#define VM_PAGE_IS_WIRED 1 /* page is currently wired */
169#define VM_PAGE_USED_BY_COMPRESSOR 2 /* page is in use by the compressor to hold compressed data */
170#define VM_PAGE_ON_FREE_Q 3 /* page is on the main free queue */
171#define VM_PAGE_ON_FREE_LOCAL_Q 4 /* page is on one of the per-CPU free queues */
172#define VM_PAGE_ON_FREE_LOPAGE_Q 5 /* page is on the lopage pool free list */
173#define VM_PAGE_ON_THROTTLED_Q 6 /* page is on the throttled queue... we stash anonymous pages here when not paging */
174#define VM_PAGE_ON_PAGEOUT_Q 7 /* page is on one of the pageout queues (internal/external) awaiting processing */
175#define VM_PAGE_ON_SPECULATIVE_Q 8 /* page is on one of the speculative queues */
176#define VM_PAGE_ON_ACTIVE_LOCAL_Q 9 /* page has recently been created and is being held in one of the per-CPU local queues */
177#define VM_PAGE_ON_ACTIVE_Q 10 /* page is in global active queue */
178#define VM_PAGE_ON_INACTIVE_INTERNAL_Q 11 /* page is on the inactive internal queue a.k.a. anonymous queue */
179#define VM_PAGE_ON_INACTIVE_EXTERNAL_Q 12 /* page in on the inactive external queue a.k.a. file backed queue */
180#define VM_PAGE_ON_INACTIVE_CLEANED_Q 13 /* page has been cleaned to a backing file and is ready to be stolen */
181#define VM_PAGE_ON_SECLUDED_Q 14 /* page is on secluded queue */
182#define VM_PAGE_Q_STATE_LAST_VALID_VALUE 14 /* we currently use 4 bits for the state... don't let this go beyond 15 */
fe8ab488 183
0a7de745 184#define VM_PAGE_Q_STATE_ARRAY_SIZE (VM_PAGE_Q_STATE_LAST_VALID_VALUE+1)
fe8ab488 185
fe8ab488 186
d9a64523
A		 187/*
188 * The structure itself. See the block comment above for what (O) and (P) mean.
189 */
190#define vmp_pageq vmp_q_un.vmp_q_pageq
191#define vmp_snext vmp_q_un.vmp_q_snext
fe8ab488 192
1c79356b 193struct vm_page {
39037602 194 union {
d9a64523
A		 195 vm_page_queue_chain_t vmp_q_pageq; /* queue info for FIFO queue or free list (P) */
196 struct vm_page *vmp_q_snext;
197 } vmp_q_un;
b0d623f7 198
d9a64523 199 vm_page_queue_chain_t vmp_listq; /* all pages in same object (O) */
1c79356b 200
39037602 201#if CONFIG_BACKGROUND_QUEUE
0a7de745 202 vm_page_queue_chain_t vmp_backgroundq; /* anonymous pages in the background pool (P) */
39037602
A		 203#endif
204
d9a64523
A		 205 vm_object_offset_t vmp_offset; /* offset into that object (O,P) */
206 vm_page_object_t vmp_object; /* which object am I in (O&P) */
1c79356b 207
91447636 208 /*
d9a64523 209 * The following word of flags is always protected by the "page queues" lock.
b0d623f7 210 *
d9a64523
A		 211 * We use 'vmp_wire_count' to store the local queue id if local queues are enabled.
212 * See the comments at 'vm_page_queues_remove' as to why this is safe to do.
91447636 213 */
d9a64523
A		 214#define vmp_local_id vmp_wire_count
215 unsigned int vmp_wire_count:16, /* how many wired down maps use me? (O&P) */
0a7de745
A		 216 vmp_q_state:4, /* which q is the page on (P) */
217 vmp_in_background:1,
218 vmp_on_backgroundq:1,
219 vmp_gobbled:1, /* page used internally (P) */
220 vmp_laundry:1, /* page is being cleaned now (P)*/
221 vmp_no_cache:1, /* page is not to be cached and should */
d9a64523 222 /* be reused ahead of other pages (P) */
0a7de745
A		 223 vmp_private:1, /* Page should not be returned to the free list (P) */
224 vmp_reference:1, /* page has been used (P) */
225 vmp_unused_page_bits:5;
1c79356b 226
39037602
A		 227 /*
228 * MUST keep the 2 32 bit words used as bit fields
229 * separated since the compiler has a nasty habit
0a7de745 230 * of using 64 bit loads and stores on them as
39037602
A		 231 * if they were a single 64 bit field... since
232 * they are protected by 2 different locks, this
233 * is a real problem
234 */
d9a64523 235 vm_page_packed_t vmp_next_m; /* VP bucket link (O) */
b0d623f7 236
91447636 237 /*
d9a64523 238 * The following word of flags is protected by the "VM object" lock.
0a7de745 239 *
d9a64523 240 * IMPORTANT: the "vmp_pmapped", "vmp_xpmapped" and "vmp_clustered" bits can be modified while holding the
fe8ab488 241 * VM object "shared" lock + the page lock provided through the pmap_lock_phys_page function.
d9a64523 242 * This is done in vm_fault_enter() and the CONSUME_CLUSTERED macro.
fe8ab488 243 * It's also ok to modify them behind just the VM object "exclusive" lock.
b0d623f7 244 */
d9a64523 245 unsigned int vmp_busy:1, /* page is in transit (O) */
0a7de745
A		 246 vmp_wanted:1, /* someone is waiting for page (O) */
247 vmp_tabled:1, /* page is in VP table (O) */
248 vmp_hashed:1, /* page is in vm_page_buckets[] (O) + the bucket lock */
249 vmp_fictitious:1, /* Physical page doesn't exist (O) */
250 vmp_clustered:1, /* page is not the faulted page (O) or (O-shared AND pmap_page) */
251 vmp_pmapped:1, /* page has at some time been entered into a pmap (O) or */
d9a64523 252 /* (O-shared AND pmap_page) */
0a7de745 253 vmp_xpmapped:1, /* page has been entered with execute permission (O) or */
d9a64523 254 /* (O-shared AND pmap_page) */
0a7de745
A		 255 vmp_wpmapped:1, /* page has been entered at some point into a pmap for write (O) */
256 vmp_free_when_done:1, /* page is to be freed once cleaning is completed (O) */
257 vmp_absent:1, /* Data has been requested, but is not yet available (O) */
258 vmp_error:1, /* Data manager was unable to provide data due to error (O) */
259 vmp_dirty:1, /* Page must be cleaned (O) */
260 vmp_cleaning:1, /* Page clean has begun (O) */
261 vmp_precious:1, /* Page is precious; data must be returned even if clean (O) */
262 vmp_overwriting:1, /* Request to unlock has been made without having data. (O) */
d9a64523 263 /* [See vm_fault_page_overwrite] */
0a7de745 264 vmp_restart:1, /* Page was pushed higher in shadow chain by copy_call-related pagers */
d9a64523 265 /* start again at top of chain */
0a7de745
A		 266 vmp_unusual:1, /* Page is absent, error, restart or page locked */
267 vmp_cs_validated:1, /* code-signing: page was checked */
268 vmp_cs_tainted:1, /* code-signing: page is tainted */
269 vmp_cs_nx:1, /* code-signing: page is nx */
270 vmp_reusable:1,
271 vmp_lopage:1,
272 vmp_written_by_kernel:1, /* page was written by kernel (i.e. decompressed) */
273 vmp_unused_object_bits:8;
39037602 274
5ba3f43e 275#if !defined(__arm__) && !defined(__arm64__)
d9a64523 276 ppnum_t vmp_phys_page; /* Physical page number of the page */
5ba3f43e 277#endif
1c79356b
A		 278};
279
39037602 280
0a7de745
A		 281typedef struct vm_page *vm_page_t;
282extern vm_page_t vm_pages;
283extern vm_page_t vm_page_array_beginning_addr;
284extern vm_page_t vm_page_array_ending_addr;
39037602
A		 285
286
5ba3f43e
A		 287#if defined(__arm__) || defined(__arm64__)
288
0a7de745 289extern unsigned int vm_first_phys_ppnum;
5ba3f43e
A		 290
291struct vm_page_with_ppnum {
0a7de745 292 struct vm_page vm_page_wo_ppnum;
5ba3f43e 293
0a7de745 294 ppnum_t vmp_phys_page;
5ba3f43e
A		 295};
296typedef struct vm_page_with_ppnum *vm_page_with_ppnum_t;
297
298
0a7de745
A		 299static inline ppnum_t
300VM_PAGE_GET_PHYS_PAGE(vm_page_t m)
5ba3f43e 301{
0a7de745
A		 302 if (m >= vm_page_array_beginning_addr && m < vm_page_array_ending_addr) {
303 return (ppnum_t)((uintptr_t)(m - vm_page_array_beginning_addr) + vm_first_phys_ppnum);
304 } else {
305 return ((vm_page_with_ppnum_t)m)->vmp_phys_page;
306 }
5ba3f43e
A		 307}
308
0a7de745
A		 309#define VM_PAGE_SET_PHYS_PAGE(m, ppnum) \
310 MACRO_BEGIN \
311 if ((m) < vm_page_array_beginning_addr || (m) >= vm_page_array_ending_addr) \
312 ((vm_page_with_ppnum_t)(m))->vmp_phys_page = ppnum; \
313 assert(ppnum == VM_PAGE_GET_PHYS_PAGE(m)); \
5ba3f43e
A		 314 MACRO_END
315
316#define VM_PAGE_GET_COLOR(m) (VM_PAGE_GET_PHYS_PAGE(m) & vm_color_mask)
317
0a7de745 318#else /* defined(__arm__) || defined(__arm64__) */
39037602
A		 319
320
321struct vm_page_with_ppnum {
0a7de745 322 struct vm_page vm_page_with_ppnum;
39037602
A		 323};
324typedef struct vm_page_with_ppnum *vm_page_with_ppnum_t;
325
326
0a7de745
A		 327#define VM_PAGE_GET_PHYS_PAGE(page) (page)->vmp_phys_page
328#define VM_PAGE_SET_PHYS_PAGE(page, ppnum) \
329 MACRO_BEGIN \
330 (page)->vmp_phys_page = ppnum; \
39037602
A		 331 MACRO_END
332
5ba3f43e
A		 333#define VM_PAGE_GET_CLUMP(m) ((VM_PAGE_GET_PHYS_PAGE(m)) >> vm_clump_shift)
334#define VM_PAGE_GET_COLOR(m) ((VM_PAGE_GET_CLUMP(m)) & vm_color_mask)
39037602 335
0a7de745 336#endif /* defined(__arm__) || defined(__arm64__) */
91447636 337
39037602
A		 338
339
340#if defined(__LP64__)
341
0a7de745
A		 342#define VM_VPLQ_ALIGNMENT 128
343#define VM_PACKED_POINTER_ALIGNMENT 64 /* must be a power of 2 */
344#define VM_PACKED_POINTER_SHIFT 6
39037602 345
0a7de745 346#define VM_PACKED_FROM_VM_PAGES_ARRAY 0x80000000
39037602 347
0a7de745
A		 348static inline vm_page_packed_t
349vm_page_pack_ptr(uintptr_t p)
39037602
A		 350{
351 vm_page_packed_t packed_ptr;
352
0a7de745
A		 353 if (!p) {
354 return (vm_page_packed_t)0;
355 }
39037602
A		 356
357 if (p >= (uintptr_t)(vm_page_array_beginning_addr) && p < (uintptr_t)(vm_page_array_ending_addr)) {
358 packed_ptr = ((vm_page_packed_t)(((vm_page_t)p - vm_page_array_beginning_addr)));
0a7de745 359 assert(!(packed_ptr & VM_PACKED_FROM_VM_PAGES_ARRAY));
39037602
A		 360 packed_ptr |= VM_PACKED_FROM_VM_PAGES_ARRAY;
361 return packed_ptr;
362 }
363
364 assert((p & (VM_PACKED_POINTER_ALIGNMENT - 1)) == 0);
365
366 packed_ptr = ((vm_page_packed_t)(((uintptr_t)(p - (uintptr_t) VM_MIN_KERNEL_AND_KEXT_ADDRESS)) >> VM_PACKED_POINTER_SHIFT));
367 assert(packed_ptr != 0);
0a7de745 368 assert(!(packed_ptr & VM_PACKED_FROM_VM_PAGES_ARRAY));
39037602
A		 369 return packed_ptr;
370}
371
372
0a7de745
A		 373static inline uintptr_t
374vm_page_unpack_ptr(uintptr_t p)
39037602 375{
cb323159
A		 376 extern unsigned int vm_pages_count;
377
0a7de745
A		 378 if (!p) {
379 return (uintptr_t)0;
380 }
39037602 381
0a7de745 382 if (p & VM_PACKED_FROM_VM_PAGES_ARRAY) {
cb323159 383 assert((uint32_t)(p & ~VM_PACKED_FROM_VM_PAGES_ARRAY) < vm_pages_count);
0a7de745
A		 384 return (uintptr_t)(&vm_pages[(uint32_t)(p & ~VM_PACKED_FROM_VM_PAGES_ARRAY)]);
385 }
386 return (p << VM_PACKED_POINTER_SHIFT) + (uintptr_t) VM_MIN_KERNEL_AND_KEXT_ADDRESS;
39037602
A		 387}
388
389
0a7de745
A		 390#define VM_PAGE_PACK_PTR(p) vm_page_pack_ptr((uintptr_t)(p))
391#define VM_PAGE_UNPACK_PTR(p) vm_page_unpack_ptr((uintptr_t)(p))
39037602 392
0a7de745
A		 393#define VM_PAGE_OBJECT(p) ((vm_object_t)(VM_PAGE_UNPACK_PTR(p->vmp_object)))
394#define VM_PAGE_PACK_OBJECT(o) ((vm_page_object_t)(VM_PAGE_PACK_PTR(o)))
39037602
A		 395
396
0a7de745
A		 397#define VM_PAGE_ZERO_PAGEQ_ENTRY(p) \
398MACRO_BEGIN \
399 (p)->vmp_snext = 0; \
39037602
A		 400MACRO_END
401
402
0a7de745 403#define VM_PAGE_CONVERT_TO_QUEUE_ENTRY(p) VM_PAGE_PACK_PTR(p)
39037602
A		 404
405
406static __inline__ void
407vm_page_enqueue_tail(
0a7de745
A		 408 vm_page_queue_t que,
409 vm_page_queue_entry_t elt)
39037602 410{
0a7de745 411 vm_page_queue_entry_t old_tail;
39037602
A		 412
413 old_tail = (vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR(que->prev);
414 elt->next = VM_PAGE_PACK_PTR(que);
415 elt->prev = que->prev;
0a7de745 416 que->prev = old_tail->next = VM_PAGE_PACK_PTR(elt);
39037602
A		 417}
418
419
420static __inline__ void
421vm_page_remque(
422 vm_page_queue_entry_t elt)
423{
0a7de745
A		 424 vm_page_queue_entry_t next;
425 vm_page_queue_entry_t prev;
426 vm_page_packed_t next_pck = elt->next;
427 vm_page_packed_t prev_pck = elt->prev;
39037602 428
0a7de745 429 next = (vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR(next_pck);
39037602 430
0a7de745
A		 431 /* next may equal prev (and the queue head) if elt was the only element */
432 prev = (vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR(prev_pck);
39037602 433
0a7de745
A		 434 next->prev = prev_pck;
435 prev->next = next_pck;
39037602
A		 436
437 elt->next = 0;
438 elt->prev = 0;
439}
440
441
442/*
443 * Macro: vm_page_queue_init
444 * Function:
445 * Initialize the given queue.
446 * Header:
447 * void vm_page_queue_init(q)
448 * vm_page_queue_t q; \* MODIFIED *\
449 */
0a7de745
A		 450#define vm_page_queue_init(q) \
451MACRO_BEGIN \
452 assert((((uintptr_t)q) & (VM_PACKED_POINTER_ALIGNMENT-1)) == 0); \
453 assert((VM_PAGE_UNPACK_PTR(VM_PAGE_PACK_PTR((uintptr_t)q))) == (uintptr_t)q); \
454 (q)->next = VM_PAGE_PACK_PTR(q); \
455 (q)->prev = VM_PAGE_PACK_PTR(q); \
39037602
A		 456MACRO_END
457
458
459/*
0a7de745
A		 460 * Macro: vm_page_queue_enter
461 * Function:
462 * Insert a new element at the tail of the vm_page queue.
463 * Header:
464 * void vm_page_queue_enter(q, elt, field)
465 * queue_t q;
466 * vm_page_t elt;
467 * <field> is the list field in vm_page_t
468 *
469 * This macro's arguments have to match the generic "queue_enter()" macro which is
470 * what is used for this on 32 bit kernels.
39037602 471 */
0a7de745
A		 472#define vm_page_queue_enter(head, elt, field) \
473MACRO_BEGIN \
474 vm_page_packed_t __pck_elt = VM_PAGE_PACK_PTR(elt); \
475 vm_page_packed_t __pck_head = VM_PAGE_PACK_PTR(head); \
476 vm_page_packed_t __pck_prev = (head)->prev; \
477 \
478 if (__pck_head == __pck_prev) { \
479 (head)->next = __pck_elt; \
480 } else { \
481 vm_page_t __prev; \
482 __prev = (vm_page_t)VM_PAGE_UNPACK_PTR(__pck_prev); \
483 __prev->field.next = __pck_elt; \
484 } \
485 (elt)->field.prev = __pck_prev; \
486 (elt)->field.next = __pck_head; \
487 (head)->prev = __pck_elt; \
39037602
A		 488MACRO_END
489
490
0a7de745 491#if defined(__x86_64__)
5ba3f43e
A		 492/*
493 * These are helper macros for vm_page_queue_enter_clump to assist
494 * with conditional compilation (release / debug / development)
495 */
496#if DEVELOPMENT || DEBUG
497
0a7de745
A		 498#define __DEBUG_CHECK_BUDDIES(__prev, __p, field) \
499MACRO_BEGIN \
500 if (__prev != NULL) { \
501 assert(__p == (vm_page_t)VM_PAGE_UNPACK_PTR(__prev->next)); \
502 assert(__prev == (vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR(__p->field.prev)); \
503 } \
5ba3f43e
A		 504MACRO_END
505
0a7de745
A		 506#define __DEBUG_VERIFY_LINKS(__first, __n_free, __last_next) \
507MACRO_BEGIN \
508 unsigned int __i; \
509 vm_page_queue_entry_t __tmp; \
510 for (__i = 0, __tmp = __first; __i < __n_free; __i++) { \
511 __tmp = (vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR(__tmp->next); \
512 } \
513 assert(__tmp == __last_next); \
5ba3f43e
A		 514MACRO_END
515
516#define __DEBUG_STAT_INCREMENT_INRANGE vm_clump_inrange++
517#define __DEBUG_STAT_INCREMENT_INSERTS vm_clump_inserts++
518#define __DEBUG_STAT_INCREMENT_PROMOTES(__n_free) vm_clump_promotes+=__n_free
519
520#else
521
0a7de745
A		 522#define __DEBUG_CHECK_BUDDIES(__prev, __p, field)
523#define __DEBUG_VERIFY_LINKS(__first, __n_free, __last_next)
5ba3f43e
A		 524#define __DEBUG_STAT_INCREMENT_INRANGE
525#define __DEBUG_STAT_INCREMENT_INSERTS
526#define __DEBUG_STAT_INCREMENT_PROMOTES(__n_free)
527
528#endif /* if DEVELOPMENT || DEBUG */
529
530/*
0a7de745 531 * Insert a new page into a free queue and clump pages within the same 16K boundary together
5ba3f43e 532 */
0a7de745
A		 533static inline void
534vm_page_queue_enter_clump(
535 vm_page_queue_t head,
536 vm_page_t elt)
537{
538 vm_page_queue_entry_t first; /* first page in the clump */
539 vm_page_queue_entry_t last; /* last page in the clump */
540 vm_page_queue_entry_t prev = NULL;
541 vm_page_queue_entry_t next;
542 uint_t n_free = 1;
543 extern unsigned int vm_pages_count;
544 extern unsigned int vm_clump_size, vm_clump_mask, vm_clump_shift, vm_clump_promote_threshold;
545 extern unsigned long vm_clump_allocs, vm_clump_inserts, vm_clump_inrange, vm_clump_promotes;
546
547 /*
548 * If elt is part of the vm_pages[] array, find its neighboring buddies in the array.
549 */
550 if (vm_page_array_beginning_addr <= elt && elt < &vm_pages[vm_pages_count]) {
551 vm_page_t p;
552 uint_t i;
553 uint_t n;
554 ppnum_t clump_num;
555
556 first = last = (vm_page_queue_entry_t)elt;
557 clump_num = VM_PAGE_GET_CLUMP(elt);
558 n = VM_PAGE_GET_PHYS_PAGE(elt) & vm_clump_mask;
559
560 /*
561 * Check for preceeding vm_pages[] entries in the same chunk
562 */
563 for (i = 0, p = elt - 1; i < n && vm_page_array_beginning_addr <= p; i++, p--) {
564 if (p->vmp_q_state == VM_PAGE_ON_FREE_Q && clump_num == VM_PAGE_GET_CLUMP(p)) {
565 if (prev == NULL) {
566 prev = (vm_page_queue_entry_t)p;
567 }
568 first = (vm_page_queue_entry_t)p;
569 n_free++;
570 }
571 }
572
573 /*
574 * Check the following vm_pages[] entries in the same chunk
575 */
576 for (i = n + 1, p = elt + 1; i < vm_clump_size && p < &vm_pages[vm_pages_count]; i++, p++) {
577 if (p->vmp_q_state == VM_PAGE_ON_FREE_Q && clump_num == VM_PAGE_GET_CLUMP(p)) {
578 if (last == (vm_page_queue_entry_t)elt) { /* first one only */
579 __DEBUG_CHECK_BUDDIES(prev, p, vmp_pageq);
580 }
581
582 if (prev == NULL) {
583 prev = (vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR(p->vmp_pageq.prev);
584 }
585 last = (vm_page_queue_entry_t)p;
586 n_free++;
587 }
588 }
589 __DEBUG_STAT_INCREMENT_INRANGE;
590 }
591
592 /* if elt is not part of vm_pages or if 1st page in clump, insert at tail */
593 if (prev == NULL) {
594 prev = (vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR(head->prev);
595 }
596
597 /* insert the element */
598 next = (vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR(prev->next);
599 elt->vmp_pageq.next = prev->next;
600 elt->vmp_pageq.prev = next->prev;
601 prev->next = next->prev = VM_PAGE_PACK_PTR(elt);
602 __DEBUG_STAT_INCREMENT_INSERTS;
603
604 /*
605 * Check if clump needs to be promoted to head.
606 */
607 if (n_free >= vm_clump_promote_threshold && n_free > 1) {
608 vm_page_queue_entry_t first_prev;
609
610 first_prev = (vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR(first->prev);
611
612 /* If not at head already */
613 if (first_prev != head) {
614 vm_page_queue_entry_t last_next;
615 vm_page_queue_entry_t head_next;
616
617 last_next = (vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR(last->next);
618
619 /* verify that the links within the clump are consistent */
620 __DEBUG_VERIFY_LINKS(first, n_free, last_next);
621
622 /* promote clump to head */
623 first_prev->next = last->next;
624 last_next->prev = first->prev;
625 first->prev = VM_PAGE_PACK_PTR(head);
626 last->next = head->next;
627
628 head_next = (vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR(head->next);
629 head_next->prev = VM_PAGE_PACK_PTR(last);
630 head->next = VM_PAGE_PACK_PTR(first);
631 __DEBUG_STAT_INCREMENT_PROMOTES(n_free);
632 }
633 }
634}
5ba3f43e
A		 635#endif
636
39037602 637/*
0a7de745
A		 638 * Macro: vm_page_queue_enter_first
639 * Function:
640 * Insert a new element at the head of the vm_page queue.
641 * Header:
642 * void queue_enter_first(q, elt, , field)
643 * queue_t q;
644 * vm_page_t elt;
645 * <field> is the linkage field in vm_page
646 *
647 * This macro's arguments have to match the generic "queue_enter_first()" macro which is
648 * what is used for this on 32 bit kernels.
39037602 649 */
0a7de745
A		 650#define vm_page_queue_enter_first(head, elt, field) \
651MACRO_BEGIN \
652 vm_page_packed_t __pck_next = (head)->next; \
653 vm_page_packed_t __pck_head = VM_PAGE_PACK_PTR(head); \
654 vm_page_packed_t __pck_elt = VM_PAGE_PACK_PTR(elt); \
655 \
656 if (__pck_head == __pck_next) { \
657 (head)->prev = __pck_elt; \
658 } else { \
659 vm_page_t __next; \
660 __next = (vm_page_t)VM_PAGE_UNPACK_PTR(__pck_next); \
661 __next->field.prev = __pck_elt; \
662 } \
663 \
664 (elt)->field.next = __pck_next; \
665 (elt)->field.prev = __pck_head; \
666 (head)->next = __pck_elt; \
39037602
A		 667MACRO_END
668
669
670/*
0a7de745
A		 671 * Macro: vm_page_queue_remove
672 * Function:
673 * Remove an arbitrary page from a vm_page queue.
674 * Header:
675 * void vm_page_queue_remove(q, qe, field)
676 * arguments as in vm_page_queue_enter
677 *
678 * This macro's arguments have to match the generic "queue_enter()" macro which is
679 * what is used for this on 32 bit kernels.
39037602 680 */
0a7de745
A		 681#define vm_page_queue_remove(head, elt, field) \
682MACRO_BEGIN \
683 vm_page_packed_t __pck_next = (elt)->field.next; \
684 vm_page_packed_t __pck_prev = (elt)->field.prev; \
685 vm_page_t __next = (vm_page_t)VM_PAGE_UNPACK_PTR(__pck_next); \
686 vm_page_t __prev = (vm_page_t)VM_PAGE_UNPACK_PTR(__pck_prev); \
687 \
688 if ((void *)(head) == (void *)__next) { \
689 (head)->prev = __pck_prev; \
690 } else { \
691 __next->field.prev = __pck_prev; \
692 } \
693 \
694 if ((void *)(head) == (void *)__prev) { \
695 (head)->next = __pck_next; \
696 } else { \
697 __prev->field.next = __pck_next; \
698 } \
699 \
700 (elt)->field.next = 0; \
701 (elt)->field.prev = 0; \
39037602
A		 702MACRO_END
703
704
705/*
0a7de745
A		 706 * Macro: vm_page_queue_remove_first
707 *
708 * Function:
709 * Remove and return the entry at the head of a vm_page queue.
710 *
711 * Header:
712 * vm_page_queue_remove_first(head, entry, field)
713 * N.B. entry is returned by reference
714 *
715 * This macro's arguments have to match the generic "queue_remove_first()" macro which is
716 * what is used for this on 32 bit kernels.
39037602 717 */
0a7de745
A		 718#define vm_page_queue_remove_first(head, entry, field) \
719MACRO_BEGIN \
720 vm_page_packed_t __pck_head = VM_PAGE_PACK_PTR(head); \
721 vm_page_packed_t __pck_next; \
722 vm_page_t __next; \
723 \
724 (entry) = (vm_page_t)VM_PAGE_UNPACK_PTR((head)->next); \
725 __pck_next = (entry)->field.next; \
726 __next = (vm_page_t)VM_PAGE_UNPACK_PTR(__pck_next); \
727 \
728 if (__pck_head == __pck_next) { \
729 (head)->prev = __pck_head; \
730 } else { \
731 __next->field.prev = __pck_head; \
732 } \
733 \
734 (head)->next = __pck_next; \
735 (entry)->field.next = 0; \
736 (entry)->field.prev = 0; \
39037602
A		 737MACRO_END
738
739
0a7de745 740#if defined(__x86_64__)
5ba3f43e 741/*
0a7de745
A		 742 * Macro: vm_page_queue_remove_first_with_clump
743 * Function:
744 * Remove and return the entry at the head of the free queue
745 * end is set to 1 to indicate that we just returned the last page in a clump
5ba3f43e 746 *
0a7de745
A		 747 * Header:
748 * vm_page_queue_remove_first_with_clump(head, entry, end)
749 * entry is returned by reference
750 * end is returned by reference
5ba3f43e 751 */
0a7de745
A		 752#define vm_page_queue_remove_first_with_clump(head, entry, end) \
753MACRO_BEGIN \
754 vm_page_packed_t __pck_head = VM_PAGE_PACK_PTR(head); \
755 vm_page_packed_t __pck_next; \
756 vm_page_t __next; \
757 \
758 (entry) = (vm_page_t)VM_PAGE_UNPACK_PTR((head)->next); \
759 __pck_next = (entry)->vmp_pageq.next; \
760 __next = (vm_page_t)VM_PAGE_UNPACK_PTR(__pck_next); \
761 \
762 (end) = 0; \
763 if (__pck_head == __pck_next) { \
764 (head)->prev = __pck_head; \
765 (end) = 1; \
766 } else { \
767 __next->vmp_pageq.prev = __pck_head; \
768 if (VM_PAGE_GET_CLUMP(entry) != VM_PAGE_GET_CLUMP(__next)) { \
769 (end) = 1; \
770 } \
771 } \
772 \
773 (head)->next = __pck_next; \
774 (entry)->vmp_pageq.next = 0; \
775 (entry)->vmp_pageq.prev = 0; \
5ba3f43e
A		 776MACRO_END
777#endif
778
39037602
A		 779/*
780 * Macro: vm_page_queue_end
781 * Function:
782 * Tests whether a new entry is really the end of
783 * the queue.
784 * Header:
785 * boolean_t vm_page_queue_end(q, qe)
786 * vm_page_queue_t q;
787 * vm_page_queue_entry_t qe;
788 */
0a7de745 789#define vm_page_queue_end(q, qe) ((q) == (qe))
39037602
A		 790
791
792/*
793 * Macro: vm_page_queue_empty
794 * Function:
795 * Tests whether a queue is empty.
796 * Header:
797 * boolean_t vm_page_queue_empty(q)
798 * vm_page_queue_t q;
799 */
0a7de745 800#define vm_page_queue_empty(q) vm_page_queue_end((q), ((vm_page_queue_entry_t)vm_page_queue_first(q)))
39037602
A		 801
802
803
804/*
805 * Macro: vm_page_queue_first
806 * Function:
807 * Returns the first entry in the queue,
808 * Header:
809 * uintpr_t vm_page_queue_first(q)
810 * vm_page_queue_t q; \* IN *\
811 */
0a7de745 812#define vm_page_queue_first(q) (VM_PAGE_UNPACK_PTR((q)->next))
39037602
A		 813
814
815
816/*
817 * Macro: vm_page_queue_last
818 * Function:
819 * Returns the last entry in the queue.
820 * Header:
821 * vm_page_queue_entry_t queue_last(q)
822 * queue_t q; \* IN *\
823 */
0a7de745 824#define vm_page_queue_last(q) (VM_PAGE_UNPACK_PTR((q)->prev))
39037602
A		 825
826
827
828/*
829 * Macro: vm_page_queue_next
830 * Function:
831 * Returns the entry after an item in the queue.
832 * Header:
833 * uintpr_t vm_page_queue_next(qc)
834 * vm_page_queue_t qc;
835 */
0a7de745 836#define vm_page_queue_next(qc) (VM_PAGE_UNPACK_PTR((qc)->next))
39037602
A		 837
838
839
840/*
841 * Macro: vm_page_queue_prev
842 * Function:
843 * Returns the entry before an item in the queue.
844 * Header:
845 * uinptr_t vm_page_queue_prev(qc)
846 * vm_page_queue_t qc;
847 */
0a7de745 848#define vm_page_queue_prev(qc) (VM_PAGE_UNPACK_PTR((qc)->prev))
39037602
A		 849
850
851
852/*
853 * Macro: vm_page_queue_iterate
854 * Function:
0a7de745 855 * iterate over each item in a vm_page queue.
39037602
A		 856 * Generates a 'for' loop, setting elt to
857 * each item in turn (by reference).
858 * Header:
0a7de745 859 * vm_page_queue_iterate(q, elt, field)
39037602 860 * queue_t q;
0a7de745
A		 861 * vm_page_t elt;
862 * <field> is the chain field in vm_page_t
39037602 863 */
0a7de745
A		 864#define vm_page_queue_iterate(head, elt, field) \
865 for ((elt) = (vm_page_t)vm_page_queue_first(head); \
866 !vm_page_queue_end((head), (vm_page_queue_entry_t)(elt)); \
867 (elt) = (vm_page_t)vm_page_queue_next(&(elt)->field)) \
39037602
A		 868
869#else
870
0a7de745
A		 871#define VM_VPLQ_ALIGNMENT 128
872#define VM_PACKED_POINTER_ALIGNMENT 4
873#define VM_PACKED_POINTER_SHIFT 0
39037602 874
0a7de745 875#define VM_PACKED_FROM_VM_PAGES_ARRAY 0
39037602 876
0a7de745
A		 877#define VM_PAGE_PACK_PTR(p) (p)
878#define VM_PAGE_UNPACK_PTR(p) ((uintptr_t)(p))
39037602 879
0a7de745
A		 880#define VM_PAGE_OBJECT(p) (vm_object_t)(p->vmp_object)
881#define VM_PAGE_PACK_OBJECT(o) ((vm_page_object_t)(VM_PAGE_PACK_PTR(o)))
39037602
A		 882
883
0a7de745
A		 884#define VM_PAGE_ZERO_PAGEQ_ENTRY(p) \
885MACRO_BEGIN \
886 (p)->vmp_pageq.next = 0; \
887 (p)->vmp_pageq.prev = 0; \
39037602
A		 888MACRO_END
889
0a7de745
A		 890#define VM_PAGE_CONVERT_TO_QUEUE_ENTRY(p) ((queue_entry_t)(p))
891
892#define vm_page_remque remque
893#define vm_page_enqueue_tail enqueue_tail
894#define vm_page_queue_init queue_init
895#define vm_page_queue_enter(h, e, f) queue_enter(h, e, vm_page_t, f)
896#define vm_page_queue_enter_first(h, e, f) queue_enter_first(h, e, vm_page_t, f)
897#define vm_page_queue_remove(h, e, f) queue_remove(h, e, vm_page_t, f)
898#define vm_page_queue_remove_first(h, e, f) queue_remove_first(h, e, vm_page_t, f)
899#define vm_page_queue_end queue_end
900#define vm_page_queue_empty queue_empty
901#define vm_page_queue_first queue_first
902#define vm_page_queue_last queue_last
903#define vm_page_queue_next queue_next
904#define vm_page_queue_prev queue_prev
905#define vm_page_queue_iterate(h, e, f) queue_iterate(h, e, vm_page_t, f)
39037602
A		 906
907#endif
908
909
910
0a7de745 911/*
39037602
A		 912 * VM_PAGE_MIN_SPECULATIVE_AGE_Q through VM_PAGE_MAX_SPECULATIVE_AGE_Q
913 * represents a set of aging bins that are 'protected'...
914 *
915 * VM_PAGE_SPECULATIVE_AGED_Q is a list of the speculative pages that have
916 * not yet been 'claimed' but have been aged out of the protective bins
0a7de745 917 * this occurs in vm_page_speculate when it advances to the next bin
39037602
A		 918 * and discovers that it is still occupied... at that point, all of the
919 * pages in that bin are moved to the VM_PAGE_SPECULATIVE_AGED_Q. the pages
920 * in that bin are all guaranteed to have reached at least the maximum age
921 * we allow for a protected page... they can be older if there is no
922 * memory pressure to pull them from the bin, or there are no new speculative pages
923 * being generated to push them out.
0a7de745 924 * this list is the one that vm_pageout_scan will prefer when looking
39037602 925 * for pages to move to the underweight free list
0a7de745 926 *
39037602
A		 927 * VM_PAGE_MAX_SPECULATIVE_AGE_Q * VM_PAGE_SPECULATIVE_Q_AGE_MS
928 * defines the amount of time a speculative page is normally
929 * allowed to live in the 'protected' state (i.e. not available
930 * to be stolen if vm_pageout_scan is running and looking for
931 * pages)... however, if the total number of speculative pages
932 * in the protected state exceeds our limit (defined in vm_pageout.c)
933 * and there are none available in VM_PAGE_SPECULATIVE_AGED_Q, then
934 * vm_pageout_scan is allowed to steal pages from the protected
935 * bucket even if they are underage.
936 *
937 * vm_pageout_scan is also allowed to pull pages from a protected
938 * bin if the bin has reached the "age of consent" we've set
939 */
0a7de745
A		 940#define VM_PAGE_MAX_SPECULATIVE_AGE_Q 10
941#define VM_PAGE_MIN_SPECULATIVE_AGE_Q 1
942#define VM_PAGE_SPECULATIVE_AGED_Q 0
39037602 943
0a7de745 944#define VM_PAGE_SPECULATIVE_Q_AGE_MS 500
39037602
A		 945
946struct vm_speculative_age_q {
947 /*
948 * memory queue for speculative pages via clustered pageins
949 */
0a7de745
A		 950 vm_page_queue_head_t age_q;
951 mach_timespec_t age_ts;
39037602
A		 952} __attribute__((aligned(VM_PACKED_POINTER_ALIGNMENT)));
953
954
955
956extern
0a7de745 957struct vm_speculative_age_q vm_page_queue_speculative[];
39037602 958
0a7de745
A		 959extern int speculative_steal_index;
960extern int speculative_age_index;
961extern unsigned int vm_page_speculative_q_age_ms;
1c79356b 962
b0d623f7
A		 963
964typedef struct vm_locks_array {
0a7de745
A		 965 char pad __attribute__ ((aligned(64)));
966 lck_mtx_t vm_page_queue_lock2 __attribute__ ((aligned(64)));
967 lck_mtx_t vm_page_queue_free_lock2 __attribute__ ((aligned(64)));
968 char pad2 __attribute__ ((aligned(64)));
b0d623f7
A		 969} vm_locks_array_t;
970
971
39037602
A		 972#if CONFIG_BACKGROUND_QUEUE
973extern void vm_page_assign_background_state(vm_page_t mem);
0a7de745
A		 974extern void vm_page_update_background_state(vm_page_t mem);
975extern void vm_page_add_to_backgroundq(vm_page_t mem, boolean_t first);
976extern void vm_page_remove_from_backgroundq(vm_page_t mem);
39037602
A		 977#endif
978
0a7de745
A		 979#define VM_PAGE_WIRED(m) ((m)->vmp_q_state == VM_PAGE_IS_WIRED)
980#define NEXT_PAGE(m) ((m)->vmp_snext)
981#define NEXT_PAGE_PTR(m) (&(m)->vmp_snext)
1c79356b
A		 982
983/*
984 * XXX The unusual bit should not be necessary. Most of the bit
985 * XXX fields above really want to be masks.
986 */
987
988/*
989 * For debugging, this macro can be defined to perform
990 * some useful check on a page structure.
39037602
A		 991 * INTENTIONALLY left as a no-op so that the
992 * current call-sites can be left intact for future uses.
1c79356b
A		 993 */
994
0a7de745
A		 995#define VM_PAGE_CHECK(mem) \
996 MACRO_BEGIN \
b0d623f7 997 MACRO_END
2d21ac55
A		 998
999/* Page coloring:
1000 *
1001 * The free page list is actually n lists, one per color,
1002 * where the number of colors is a function of the machine's
1003 * cache geometry set at system initialization. To disable
1004 * coloring, set vm_colors to 1 and vm_color_mask to 0.
1005 * The boot-arg "colors" may be used to override vm_colors.
1006 * Note that there is little harm in having more colors than needed.
1007 */
0a7de745 1008
2d21ac55 1009#define MAX_COLORS 128
0a7de745 1010#define DEFAULT_COLORS 32
2d21ac55
A		 1011
1012extern
0a7de745 1013unsigned int vm_colors; /* must be in range 1..MAX_COLORS */
2d21ac55 1014extern
0a7de745 1015unsigned int vm_color_mask; /* must be (vm_colors-1) */
2d21ac55 1016extern
0a7de745 1017unsigned int vm_cache_geometry_colors; /* optimal #colors based on cache geometry */
1c79356b 1018
b0d623f7
A		 1019/*
1020 * Wired memory is a very limited resource and we can't let users exhaust it
1021 * and deadlock the entire system. We enforce the following limits:
0a7de745 1022 *
4ba76501 1023 * vm_per_task_user_wire_limit
0a7de745 1024 * how much memory can be user-wired in one user task
b0d623f7 1025 *
4ba76501 1026 * vm_global_user_wire_limit (default: same as vm_per_task_user_wire_limit)
0a7de745 1027 * how much memory can be user-wired in all user tasks
b0d623f7 1028 *
4ba76501
A		 1029 * These values are set to defaults based on the number of pages managed
1030 * by the VM system. They can be overriden via sysctls.
1031 * See kmem_set_user_wire_limits for details on the default values.
1032 *
1033 * Regardless of the amount of memory in the system, we never reserve
1034 * more than VM_NOT_USER_WIREABLE_MAX bytes as unlockable.
b0d623f7 1035 */
4ba76501
A		 1036#if defined(__LP64__)
1037#define VM_NOT_USER_WIREABLE_MAX (32ULL*1024*1024*1024) /* 32GB */
1038#else
1039#define VM_NOT_USER_WIREABLE_MAX (1UL*1024*1024*1024) /* 1GB */
1040#endif /* __LP64__ */
b0d623f7 1041extern
4ba76501 1042vm_map_size_t vm_per_task_user_wire_limit;
b0d623f7 1043extern
0a7de745 1044vm_map_size_t vm_global_user_wire_limit;
b0d623f7 1045
1c79356b
A		 1046/*
1047 * Each pageable resident page falls into one of three lists:
1048 *
0a7de745 1049 * free
2d21ac55
A		 1050 * Available for allocation now. The free list is
1051 * actually an array of lists, one per color.
1c79356b
A		 1052 * inactive
1053 * Not referenced in any map, but still has an
1054 * object/offset-page mapping, and may be dirty.
1055 * This is the list of pages that should be
2d21ac55
A		 1056 * paged out next. There are actually two
1057 * inactive lists, one for pages brought in from
1058 * disk or other backing store, and another
1059 * for "zero-filled" pages. See vm_pageout_scan()
1060 * for the distinction and usage.
1c79356b
A		 1061 * active
1062 * A list of pages which have been placed in
1063 * at least one physical map. This list is
1064 * ordered, in LRU-like fashion.
1065 */
1066
b0d623f7
A		 1067
1068#define VPL_LOCK_SPIN 1
1069
1070struct vpl {
0a7de745
A		 1071 vm_page_queue_head_t vpl_queue;
1072 unsigned int vpl_count;
1073 unsigned int vpl_internal_count;
1074 unsigned int vpl_external_count;
1075#ifdef VPL_LOCK_SPIN
1076 lck_spin_t vpl_lock;
b0d623f7 1077#else
0a7de745
A		 1078 lck_mtx_t vpl_lock;
1079 lck_mtx_ext_t vpl_lock_ext;
b0d623f7
A		 1080#endif
1081};
1082
0a7de745 1083struct vplq {
b0d623f7 1084 union {
39037602 1085 char cache_line_pad[VM_VPLQ_ALIGNMENT];
b0d623f7
A		 1086 struct vpl vpl;
1087 } vpl_un;
1088};
1089extern
0a7de745 1090unsigned int vm_page_local_q_count;
b0d623f7 1091extern
0a7de745 1092struct vplq *vm_page_local_q;
b0d623f7 1093extern
0a7de745 1094unsigned int vm_page_local_q_soft_limit;
b0d623f7 1095extern
0a7de745 1096unsigned int vm_page_local_q_hard_limit;
b0d623f7
A		 1097extern
1098vm_locks_array_t vm_page_locks;
1099
1c79356b 1100extern
0a7de745 1101vm_page_queue_head_t vm_lopage_queue_free; /* low memory free queue */
2d21ac55 1102extern
0a7de745 1103vm_page_queue_head_t vm_page_queue_active; /* active memory queue */
1c79356b 1104extern
0a7de745 1105vm_page_queue_head_t vm_page_queue_inactive; /* inactive memory queue for normal pages */
39037602 1106#if CONFIG_SECLUDED_MEMORY
1c79356b 1107extern
0a7de745 1108vm_page_queue_head_t vm_page_queue_secluded; /* reclaimable pages secluded for Camera */
39037602 1109#endif /* CONFIG_SECLUDED_MEMORY */
2d21ac55 1110extern
39037602 1111vm_page_queue_head_t vm_page_queue_cleaned; /* clean-queue inactive memory */
316670eb 1112extern
0a7de745 1113vm_page_queue_head_t vm_page_queue_anonymous; /* inactive memory queue for anonymous pages */
b0d623f7 1114extern
0a7de745 1115vm_page_queue_head_t vm_page_queue_throttled; /* memory queue for throttled pageout pages */
1c79356b 1116
3e170ce0 1117extern
0a7de745 1118queue_head_t vm_objects_wired;
3e170ce0 1119extern
0a7de745 1120lck_spin_t vm_objects_wired_lock;
3e170ce0 1121
39037602
A		 1122#if CONFIG_BACKGROUND_QUEUE
1123
0a7de745 1124#define VM_PAGE_BACKGROUND_TARGET_MAX 50000
39037602 1125
0a7de745
A		 1126#define VM_PAGE_BG_DISABLED 0
1127#define VM_PAGE_BG_LEVEL_1 1
39037602
A		 1128
1129extern
0a7de745 1130vm_page_queue_head_t vm_page_queue_background;
39037602 1131extern
0a7de745 1132uint64_t vm_page_background_promoted_count;
39037602 1133extern
0a7de745 1134uint32_t vm_page_background_count;
39037602 1135extern
0a7de745 1136uint32_t vm_page_background_target;
39037602 1137extern
0a7de745 1138uint32_t vm_page_background_internal_count;
39037602 1139extern
0a7de745 1140uint32_t vm_page_background_external_count;
39037602 1141extern
0a7de745 1142uint32_t vm_page_background_mode;
39037602 1143extern
0a7de745 1144uint32_t vm_page_background_exclude_external;
39037602
A		 1145
1146#endif
3e170ce0 1147
1c79356b 1148extern
0a7de745 1149vm_offset_t first_phys_addr; /* physical address for first_page */
1c79356b 1150extern
0a7de745 1151vm_offset_t last_phys_addr; /* physical address for last_page */
1c79356b
A		 1152
1153extern
0a7de745 1154unsigned int vm_page_free_count; /* How many pages are free? (sum of all colors) */
1c79356b 1155extern
0a7de745 1156unsigned int vm_page_active_count; /* How many pages are active? */
1c79356b 1157extern
0a7de745 1158unsigned int vm_page_inactive_count; /* How many pages are inactive? */
39037602
A		 1159#if CONFIG_SECLUDED_MEMORY
1160extern
0a7de745 1161unsigned int vm_page_secluded_count; /* How many pages are secluded? */
39037602 1162extern
cb323159 1163unsigned int vm_page_secluded_count_free; /* how many of them are free? */
39037602 1164extern
cb323159
A		 1165unsigned int vm_page_secluded_count_inuse; /* how many of them are in use? */
1166/*
1167 * We keep filling the secluded pool with new eligible pages and
1168 * we can overshoot our target by a lot.
1169 * When there's memory pressure, vm_pageout_scan() will re-balance the queues,
1170 * pushing the extra secluded pages to the active or free queue.
1171 * Since these "over target" secluded pages are actually "available", jetsam
1172 * should consider them as such, so make them visible to jetsam via the
1173 * "vm_page_secluded_count_over_target" counter and update it whenever we
1174 * update vm_page_secluded_count or vm_page_secluded_target.
1175 */
1176extern
1177unsigned int vm_page_secluded_count_over_target;
1178#define VM_PAGE_SECLUDED_COUNT_OVER_TARGET_UPDATE() \
1179 MACRO_BEGIN \
1180 if (vm_page_secluded_count > vm_page_secluded_target) { \
1181 vm_page_secluded_count_over_target = \
1182 (vm_page_secluded_count - vm_page_secluded_target); \
1183 } else { \
1184 vm_page_secluded_count_over_target = 0; \
1185 } \
1186 MACRO_END
1187#define VM_PAGE_SECLUDED_COUNT_OVER_TARGET() vm_page_secluded_count_over_target
1188#else /* CONFIG_SECLUDED_MEMORY */
1189#define VM_PAGE_SECLUDED_COUNT_OVER_TARGET_UPDATE() \
1190 MACRO_BEGIN \
1191 MACRO_END
1192#define VM_PAGE_SECLUDED_COUNT_OVER_TARGET() 0
39037602 1193#endif /* CONFIG_SECLUDED_MEMORY */
1c79356b 1194extern
316670eb
A		 1195unsigned int vm_page_cleaned_count; /* How many pages are in the clean queue? */
1196extern
0a7de745
A		 1197unsigned int vm_page_throttled_count;/* How many inactives are throttled */
1198extern
1199unsigned int vm_page_speculative_count; /* How many speculative pages are unclaimed? */
1200extern unsigned int vm_page_pageable_internal_count;
1201extern unsigned int vm_page_pageable_external_count;
2d21ac55 1202extern
0a7de745 1203unsigned int vm_page_xpmapped_external_count; /* How many pages are mapped executable? */
39236c6e 1204extern
0a7de745 1205unsigned int vm_page_external_count; /* How many pages are file-backed? */
fe8ab488 1206extern
0a7de745 1207unsigned int vm_page_internal_count; /* How many pages are anonymous? */
39236c6e 1208extern
0a7de745 1209unsigned int vm_page_wire_count; /* How many pages are wired? */
2d21ac55 1210extern
0a7de745 1211unsigned int vm_page_wire_count_initial; /* How many pages wired at startup */
db609669 1212extern
0a7de745 1213unsigned int vm_page_wire_count_on_boot; /* even earlier than _initial */
1c79356b 1214extern
0a7de745 1215unsigned int vm_page_free_target; /* How many do we want free? */
1c79356b 1216extern
0a7de745 1217unsigned int vm_page_free_min; /* When to wakeup pageout */
1c79356b 1218extern
0a7de745 1219unsigned int vm_page_throttle_limit; /* When to throttle new page creation */
b0d623f7 1220extern
0a7de745 1221unsigned int vm_page_inactive_target;/* How many do we want inactive? */
39037602
A		 1222#if CONFIG_SECLUDED_MEMORY
1223extern
0a7de745 1224unsigned int vm_page_secluded_target;/* How many do we want secluded? */
39037602 1225#endif /* CONFIG_SECLUDED_MEMORY */
1c79356b 1226extern
0a7de745 1227unsigned int vm_page_anonymous_min; /* When it's ok to pre-clean */
316670eb 1228extern
0a7de745 1229unsigned int vm_page_free_reserved; /* How many pages reserved to do pageout */
1c79356b 1230extern
0a7de745 1231unsigned int vm_page_gobble_count;
3e170ce0 1232extern
0a7de745 1233unsigned int vm_page_stolen_count; /* Count of stolen pages not acccounted in zones */
cb323159
A		 1234extern
1235unsigned int vm_page_kern_lpage_count; /* Count of large pages used in early boot */
3e170ce0 1236
91447636 1237
b0d623f7 1238#if DEVELOPMENT || DEBUG
2d21ac55 1239extern
0a7de745 1240unsigned int vm_page_speculative_used;
b0d623f7
A		 1241#endif
1242
55e303ae 1243extern
0a7de745 1244unsigned int vm_page_purgeable_count;/* How many pages are purgeable now ? */
55e303ae 1245extern
0a7de745 1246unsigned int vm_page_purgeable_wired_count;/* How many purgeable pages are wired now ? */
b0d623f7 1247extern
0a7de745 1248uint64_t vm_page_purged_count; /* How many pages got purged so far ? */
1c79356b 1249
0a7de745
A		 1250extern unsigned int vm_page_free_wanted;
1251/* how many threads are waiting for memory */
1c79356b 1252
0a7de745
A		 1253extern unsigned int vm_page_free_wanted_privileged;
1254/* how many VM privileged threads are waiting for memory */
39037602 1255#if CONFIG_SECLUDED_MEMORY
0a7de745
A		 1256extern unsigned int vm_page_free_wanted_secluded;
1257/* how many threads are waiting for secluded memory */
39037602 1258#endif /* CONFIG_SECLUDED_MEMORY */
2d21ac55 1259
0a7de745
A		 1260extern const ppnum_t vm_page_fictitious_addr;
1261/* (fake) phys_addr of fictitious pages */
1c79356b 1262
0a7de745
A		 1263extern const ppnum_t vm_page_guard_addr;
1264/* (fake) phys_addr of guard pages */
2d21ac55
A		 1265
1266
0a7de745 1267extern boolean_t vm_page_deactivate_hint;
91447636 1268
0a7de745 1269extern int vm_compressor_mode;
39236c6e 1270
0b4c1975 1271/*
0a7de745
A		 1272 * Defaults to true, so highest memory is used first.
1273 */
1274extern boolean_t vm_himemory_mode;
1275
1276extern boolean_t vm_lopage_needed;
1277extern uint32_t vm_lopage_free_count;
1278extern uint32_t vm_lopage_free_limit;
1279extern uint32_t vm_lopage_lowater;
1280extern boolean_t vm_lopage_refill;
1281extern uint64_t max_valid_dma_address;
1282extern ppnum_t max_valid_low_ppnum;
0c530ab8 1283
1c79356b
A		 1284/*
1285 * Prototypes for functions exported by this module.
1286 */
0a7de745
A		 1287extern void vm_page_bootstrap(
1288 vm_offset_t *startp,
1289 vm_offset_t *endp);
1290
1291extern void vm_page_module_init(void);
1c79356b 1292
0a7de745 1293extern void vm_page_init_local_q(void);
b0d623f7 1294
0a7de745
A		 1295extern void vm_page_create(
1296 ppnum_t start,
1297 ppnum_t end);
1c79356b 1298
0a7de745
A		 1299extern vm_page_t kdp_vm_page_lookup(
1300 vm_object_t object,
1301 vm_object_offset_t offset);
3e170ce0 1302
0a7de745
A		 1303extern vm_page_t vm_page_lookup(
1304 vm_object_t object,
1305 vm_object_offset_t offset);
1c79356b 1306
0a7de745 1307extern vm_page_t vm_page_grab_fictitious(void);
1c79356b 1308
0a7de745 1309extern vm_page_t vm_page_grab_guard(void);
2d21ac55 1310
0a7de745
A		 1311extern void vm_page_release_fictitious(
1312 vm_page_t page);
1c79356b 1313
0a7de745 1314extern void vm_free_delayed_pages(void);
1c79356b 1315
0a7de745 1316extern void vm_page_more_fictitious(void);
1c79356b 1317
0a7de745
A		 1318extern int vm_pool_low(void);
1319
1320extern vm_page_t vm_page_grab(void);
1321extern vm_page_t vm_page_grab_options(int flags);
1322
1323#define VM_PAGE_GRAB_OPTIONS_NONE 0x00000000
39037602 1324#if CONFIG_SECLUDED_MEMORY
0a7de745 1325#define VM_PAGE_GRAB_SECLUDED 0x00000001
39037602 1326#endif /* CONFIG_SECLUDED_MEMORY */
0a7de745 1327#define VM_PAGE_GRAB_Q_LOCK_HELD 0x00000002
1c79356b 1328
0a7de745 1329extern vm_page_t vm_page_grablo(void);
0c530ab8 1330
0a7de745
A		 1331extern void vm_page_release(
1332 vm_page_t page,
1333 boolean_t page_queues_locked);
1c79356b 1334
0a7de745
A		 1335extern boolean_t vm_page_wait(
1336 int interruptible );
1c79356b 1337
0a7de745
A		 1338extern vm_page_t vm_page_alloc(
1339 vm_object_t object,
1340 vm_object_offset_t offset);
1c79356b 1341
0a7de745
A		 1342extern vm_page_t vm_page_alloc_guard(
1343 vm_object_t object,
1344 vm_object_offset_t offset);
2d21ac55 1345
0a7de745
A		 1346extern void vm_page_init(
1347 vm_page_t page,
1348 ppnum_t phys_page,
1349 boolean_t lopage);
1c79356b 1350
0a7de745
A		 1351extern void vm_page_free(
1352 vm_page_t page);
1c79356b 1353
0a7de745
A		 1354extern void vm_page_free_unlocked(
1355 vm_page_t page,
1356 boolean_t remove_from_hash);
2d21ac55 1357
d9a64523 1358extern void vm_page_balance_inactive(
0a7de745
A		 1359 int max_to_move);
1360
1361extern void vm_page_activate(
1362 vm_page_t page);
1363
1364extern void vm_page_deactivate(
1365 vm_page_t page);
1366
1367extern void vm_page_deactivate_internal(
1368 vm_page_t page,
1369 boolean_t clear_hw_reference);
1370
1371extern void vm_page_enqueue_cleaned(vm_page_t page);
1372
1373extern void vm_page_lru(
1374 vm_page_t page);
1375
1376extern void vm_page_speculate(
1377 vm_page_t page,
1378 boolean_t new);
1379
1380extern void vm_page_speculate_ageit(
1381 struct vm_speculative_age_q *aq);
1382
1383extern void vm_page_reactivate_all_throttled(void);
1384
1385extern void vm_page_reactivate_local(uint32_t lid, boolean_t force, boolean_t nolocks);
1386
1387extern void vm_page_rename(
1388 vm_page_t page,
1389 vm_object_t new_object,
1390 vm_object_offset_t new_offset);
1391
1392extern void vm_page_insert(
1393 vm_page_t page,
1394 vm_object_t object,
1395 vm_object_offset_t offset);
1396
1397extern void vm_page_insert_wired(
1398 vm_page_t page,
1399 vm_object_t object,
1400 vm_object_offset_t offset,
1401 vm_tag_t tag);
1402
1403extern void vm_page_insert_internal(
1404 vm_page_t page,
1405 vm_object_t object,
1406 vm_object_offset_t offset,
1407 vm_tag_t tag,
1408 boolean_t queues_lock_held,
1409 boolean_t insert_in_hash,
1410 boolean_t batch_pmap_op,
1411 boolean_t delayed_accounting,
1412 uint64_t *delayed_ledger_update);
1413
1414extern void vm_page_replace(
1415 vm_page_t mem,
1416 vm_object_t object,
1417 vm_object_offset_t offset);
1418
1419extern void vm_page_remove(
1420 vm_page_t page,
1421 boolean_t remove_from_hash);
1422
1423extern void vm_page_zero_fill(
1424 vm_page_t page);
1425
1426extern void vm_page_part_zero_fill(
1427 vm_page_t m,
1428 vm_offset_t m_pa,
1429 vm_size_t len);
1430
1431extern void vm_page_copy(
1432 vm_page_t src_page,
1433 vm_page_t dest_page);
1434
1435extern void vm_page_part_copy(
1436 vm_page_t src_m,
1437 vm_offset_t src_pa,
1438 vm_page_t dst_m,
1439 vm_offset_t dst_pa,
1440 vm_size_t len);
1441
1442extern void vm_page_wire(
1443 vm_page_t page,
1444 vm_tag_t tag,
1445 boolean_t check_memorystatus);
1446
1447extern void vm_page_unwire(
1448 vm_page_t page,
1449 boolean_t queueit);
1450
1451extern void vm_set_page_size(void);
1452
1453extern void vm_page_gobble(
1454 vm_page_t page);
1455
1456extern void vm_page_validate_cs(vm_page_t page);
1457extern void vm_page_validate_cs_mapped(
1458 vm_page_t page,
1459 const void *kaddr);
1460extern void vm_page_validate_cs_mapped_slow(
1461 vm_page_t page,
1462 const void *kaddr);
1463extern void vm_page_validate_cs_mapped_chunk(
1464 vm_page_t page,
1465 const void *kaddr,
1466 vm_offset_t chunk_offset,
1467 vm_size_t chunk_size,
1468 boolean_t *validated,
1469 unsigned *tainted);
1470
1471extern void vm_page_free_prepare_queues(
1472 vm_page_t page);
1473
1474extern void vm_page_free_prepare_object(
1475 vm_page_t page,
1476 boolean_t remove_from_hash);
b0d623f7 1477
fe8ab488 1478#if CONFIG_IOSCHED
0a7de745
A		 1479extern wait_result_t vm_page_sleep(
1480 vm_object_t object,
1481 vm_page_t m,
1482 int interruptible);
fe8ab488
A		 1483#endif
1484
1485extern void vm_pressure_response(void);
1486
316670eb 1487#if CONFIG_JETSAM
39236c6e 1488extern void memorystatus_pages_update(unsigned int pages_avail);
316670eb
A		 1489
1490#define VM_CHECK_MEMORYSTATUS do { \
0a7de745
A		 1491 memorystatus_pages_update( \
1492 vm_page_pageable_external_count + \
1493 vm_page_free_count + \
cb323159 1494 VM_PAGE_SECLUDED_COUNT_OVER_TARGET() + \
0a7de745
A		 1495 (VM_DYNAMIC_PAGING_ENABLED() ? 0 : vm_page_purgeable_count) \
1496 ); \
316670eb 1497 } while(0)
39236c6e
A		 1498
1499#else /* CONFIG_JETSAM */
1500
5ba3f43e
A		 1501#if CONFIG_EMBEDDED
1502
1503#define VM_CHECK_MEMORYSTATUS do {} while(0)
1504
1505#else /* CONFIG_EMBEDDED */
39236c6e 1506
0a7de745 1507#define VM_CHECK_MEMORYSTATUS vm_pressure_response()
39236c6e 1508
5ba3f43e 1509#endif /* CONFIG_EMBEDDED */
39236c6e
A		 1510
1511#endif /* CONFIG_JETSAM */
6d2010ae 1512
1c79356b 1513/*
d9a64523
A		 1514 * Functions implemented as macros. m->vmp_wanted and m->vmp_busy are
1515 * protected by the object lock.
1c79356b
A		 1516 */
1517
5ba3f43e 1518#if CONFIG_EMBEDDED
0a7de745
A		 1519#define SET_PAGE_DIRTY(m, set_pmap_modified) \
1520 MACRO_BEGIN \
1521 vm_page_t __page__ = (m); \
1522 if (__page__->vmp_pmapped == TRUE && \
1523 __page__->vmp_wpmapped == TRUE && \
1524 __page__->vmp_dirty == FALSE && \
1525 (set_pmap_modified)) { \
1526 pmap_set_modify(VM_PAGE_GET_PHYS_PAGE(__page__)); \
1527 } \
1528 __page__->vmp_dirty = TRUE; \
1529 MACRO_END
5ba3f43e 1530#else /* CONFIG_EMBEDDED */
0a7de745
A		 1531#define SET_PAGE_DIRTY(m, set_pmap_modified) \
1532 MACRO_BEGIN \
1533 vm_page_t __page__ = (m); \
1534 __page__->vmp_dirty = TRUE; \
1535 MACRO_END
5ba3f43e 1536#endif /* CONFIG_EMBEDDED */
316670eb 1537
0a7de745
A		 1538#define PAGE_ASSERT_WAIT(m, interruptible) \
1539 (((m)->vmp_wanted = TRUE), \
1540 assert_wait((event_t) (m), (interruptible)))
9bccf70c 1541
fe8ab488 1542#if CONFIG_IOSCHED
0a7de745
A		 1543#define PAGE_SLEEP(o, m, interruptible) \
1544 vm_page_sleep(o, m, interruptible)
fe8ab488 1545#else
0a7de745
A		 1546#define PAGE_SLEEP(o, m, interruptible) \
1547 (((m)->vmp_wanted = TRUE), \
fe8ab488
A		 1548 thread_sleep_vm_object((o), (m), (interruptible)))
1549#endif
1c79356b 1550
0a7de745
A		 1551#define PAGE_WAKEUP_DONE(m) \
1552 MACRO_BEGIN \
1553 (m)->vmp_busy = FALSE; \
1554 if ((m)->vmp_wanted) { \
1555 (m)->vmp_wanted = FALSE; \
1556 thread_wakeup((event_t) (m)); \
1557 } \
1558 MACRO_END
1559
1560#define PAGE_WAKEUP(m) \
1561 MACRO_BEGIN \
1562 if ((m)->vmp_wanted) { \
1563 (m)->vmp_wanted = FALSE; \
1564 thread_wakeup((event_t) (m)); \
1565 } \
1566 MACRO_END
1567
1568#define VM_PAGE_FREE(p) \
1569 MACRO_BEGIN \
1570 vm_page_free_unlocked(p, TRUE); \
1571 MACRO_END
1572
1573#define VM_PAGE_GRAB_FICTITIOUS(M) \
1574 MACRO_BEGIN \
1575 while ((M = vm_page_grab_fictitious()) == VM_PAGE_NULL) \
1576 vm_page_more_fictitious(); \
1577 MACRO_END
1578
1579#define VM_PAGE_WAIT() ((void)vm_page_wait(THREAD_UNINT))
1c79356b 1580
b0d623f7
A		 1581#define vm_page_queue_lock (vm_page_locks.vm_page_queue_lock2)
1582#define vm_page_queue_free_lock (vm_page_locks.vm_page_queue_free_lock2)
1583
0a7de745 1584#define vm_page_lock_queues() lck_mtx_lock(&vm_page_queue_lock)
39037602 1585#define vm_page_trylock_queues() lck_mtx_try_lock(&vm_page_queue_lock)
0a7de745 1586#define vm_page_unlock_queues() lck_mtx_unlock(&vm_page_queue_lock)
b0d623f7 1587
0a7de745
A		 1588#define vm_page_lockspin_queues() lck_mtx_lock_spin(&vm_page_queue_lock)
1589#define vm_page_trylockspin_queues() lck_mtx_try_lock_spin(&vm_page_queue_lock)
1590#define vm_page_lockconvert_queues() lck_mtx_convert_spin(&vm_page_queue_lock)
1591
1592#ifdef VPL_LOCK_SPIN
1593extern lck_grp_t vm_page_lck_grp_local;
b0d623f7 1594
b0d623f7 1595#define VPL_LOCK_INIT(vlq, vpl_grp, vpl_attr) lck_spin_init(&vlq->vpl_lock, vpl_grp, vpl_attr)
0a7de745 1596#define VPL_LOCK(vpl) lck_spin_lock_grp(vpl, &vm_page_lck_grp_local)
b0d623f7
A		 1597#define VPL_UNLOCK(vpl) lck_spin_unlock(vpl)
1598#else
1599#define VPL_LOCK_INIT(vlq, vpl_grp, vpl_attr) lck_mtx_init_ext(&vlq->vpl_lock, &vlq->vpl_lock_ext, vpl_grp, vpl_attr)
1600#define VPL_LOCK(vpl) lck_mtx_lock_spin(vpl)
1601#define VPL_UNLOCK(vpl) lck_mtx_unlock(vpl)
1602#endif
1c79356b 1603
2d21ac55 1604
b0d623f7 1605#if DEVELOPMENT || DEBUG
0a7de745
A		 1606#define VM_PAGE_SPECULATIVE_USED_ADD() \
1607 MACRO_BEGIN \
1608 OSAddAtomic(1, &vm_page_speculative_used); \
2d21ac55 1609 MACRO_END
b0d623f7 1610#else
0a7de745 1611#define VM_PAGE_SPECULATIVE_USED_ADD()
b0d623f7 1612#endif
2d21ac55
A		 1613
1614
0a7de745
A		 1615#define VM_PAGE_CONSUME_CLUSTERED(mem) \
1616 MACRO_BEGIN \
1617 ppnum_t __phys_page; \
1618 __phys_page = VM_PAGE_GET_PHYS_PAGE(mem); \
1619 pmap_lock_phys_page(__phys_page); \
1620 if (mem->vmp_clustered) { \
1621 vm_object_t o; \
1622 o = VM_PAGE_OBJECT(mem); \
1623 assert(o); \
1624 o->pages_used++; \
1625 mem->vmp_clustered = FALSE; \
1626 VM_PAGE_SPECULATIVE_USED_ADD(); \
1627 } \
1628 pmap_unlock_phys_page(__phys_page); \
1c79356b
A		 1629 MACRO_END
1630
6d2010ae 1631
0a7de745
A		 1632#define VM_PAGE_COUNT_AS_PAGEIN(mem) \
1633 MACRO_BEGIN \
1634 { \
1635 vm_object_t o; \
1636 o = VM_PAGE_OBJECT(mem); \
1637 DTRACE_VM2(pgin, int, 1, (uint64_t *), NULL); \
1638 current_task()->pageins++; \
1639 if (o->internal) { \
1640 DTRACE_VM2(anonpgin, int, 1, (uint64_t *), NULL); \
1641 } else { \
1642 DTRACE_VM2(fspgin, int, 1, (uint64_t *), NULL); \
1643 } \
1644 } \
fe8ab488
A		 1645 MACRO_END
1646
3e170ce0 1647/* adjust for stolen pages accounted elsewhere */
0a7de745
A		 1648#define VM_PAGE_MOVE_STOLEN(page_count) \
1649 MACRO_BEGIN \
1650 vm_page_stolen_count -= (page_count); \
1651 vm_page_wire_count_initial -= (page_count); \
3e170ce0 1652 MACRO_END
0a7de745
A		 1653
1654#define DW_vm_page_unwire 0x01
1655#define DW_vm_page_wire 0x02
1656#define DW_vm_page_free 0x04
1657#define DW_vm_page_activate 0x08
1658#define DW_vm_page_deactivate_internal 0x10
1659#define DW_vm_page_speculate 0x20
1660#define DW_vm_page_lru 0x40
1661#define DW_vm_pageout_throttle_up 0x80
1662#define DW_PAGE_WAKEUP 0x100
1663#define DW_clear_busy 0x200
1664#define DW_clear_reference 0x400
1665#define DW_set_reference 0x800
1666#define DW_move_page 0x1000
1667#define DW_VM_PAGE_QUEUES_REMOVE 0x2000
1668#define DW_enqueue_cleaned 0x4000
1669#define DW_vm_phantom_cache_update 0x8000
6d2010ae
A		 1670
1671struct vm_page_delayed_work {
0a7de745
A		 1672 vm_page_t dw_m;
1673 int dw_mask;
6d2010ae
A		 1674};
1675
3e170ce0 1676void vm_page_do_delayed_work(vm_object_t object, vm_tag_t tag, struct vm_page_delayed_work *dwp, int dw_count);
6d2010ae
A		 1677
1678extern unsigned int vm_max_delayed_work_limit;
1679
0a7de745 1680#define DEFAULT_DELAYED_WORK_LIMIT 32
6d2010ae 1681
0a7de745 1682#define DELAYED_WORK_LIMIT(max) ((vm_max_delayed_work_limit >= max ? max : vm_max_delayed_work_limit))
6d2010ae
A		 1683
1684/*
1685 * vm_page_do_delayed_work may need to drop the object lock...
1686 * if it does, we need the pages it's looking at to
1687 * be held stable via the busy bit, so if busy isn't already
1688 * set, we need to set it and ask vm_page_do_delayed_work
1689 * to clear it and wakeup anyone that might have blocked on
1690 * it once we're done processing the page.
6d2010ae
A		 1691 */
1692
0a7de745
A		 1693#define VM_PAGE_ADD_DELAYED_WORK(dwp, mem, dw_cnt) \
1694 MACRO_BEGIN \
1695 if (mem->vmp_busy == FALSE) { \
1696 mem->vmp_busy = TRUE; \
1697 if ( !(dwp->dw_mask & DW_vm_page_free)) \
1698 dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP); \
1699 } \
1700 dwp->dw_m = mem; \
1701 dwp++; \
1702 dw_cnt++; \
6d2010ae
A		 1703 MACRO_END
1704
1705extern vm_page_t vm_object_page_grab(vm_object_t);
1706
15129b1c
A		 1707#if VM_PAGE_BUCKETS_CHECK
1708extern void vm_page_buckets_check(void);
1709#endif /* VM_PAGE_BUCKETS_CHECK */
6d2010ae 1710
39037602 1711extern void vm_page_queues_remove(vm_page_t mem, boolean_t remove_from_backgroundq);
3e170ce0
A		 1712extern void vm_page_remove_internal(vm_page_t page);
1713extern void vm_page_enqueue_inactive(vm_page_t mem, boolean_t first);
39037602 1714extern void vm_page_enqueue_active(vm_page_t mem, boolean_t first);
3e170ce0
A		 1715extern void vm_page_check_pageable_safe(vm_page_t page);
1716
d9a64523
A		 1717#if CONFIG_SECLUDED_MEMORY
1718extern uint64_t secluded_shutoff_trigger;
1719extern void start_secluded_suppression(task_t);
1720extern void stop_secluded_suppression(task_t);
1721#endif /* CONFIG_SECLUDED_MEMORY */
1722
3e170ce0 1723
0a7de745 1724#endif /* _VM_VM_PAGE_H_ */
mirror of XNU