]> git.saurik.com Git - apple/xnu.git/blame - osfmk/vm/vm_object.h
projects / apple / xnu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
xnu-7195.101.1.tar.gz
[apple/xnu.git] / osfmk / vm / vm_object.h
CommitLineData
1c79356b 1/*
f427ee49 2 * Copyright (c) 2000-2020 Apple Inc. All rights reserved.
1c79356b 3 *
2d21ac55 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
0a7de745 5 *
2d21ac55
A		 6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
0a7de745 14 *
2d21ac55
A		 15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
0a7de745 17 *
2d21ac55
A		 18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
8f6c56a5
A		 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
2d21ac55
A		 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
0a7de745 25 *
2d21ac55 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
1c79356b
A		 27 */
28/*
29 * @OSF_COPYRIGHT@
30 */
0a7de745 31/*
1c79356b
A		 32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
34 * All Rights Reserved.
0a7de745 35 *
1c79356b
A		 36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
0a7de745 41 *
1c79356b
A		 42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
0a7de745 45 *
1c79356b 46 * Carnegie Mellon requests users of this software to return to
0a7de745 47 *
1c79356b
A		 48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
0a7de745 52 *
1c79356b
A		 53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56/*
57 */
58/*
59 * File: vm_object.h
60 * Author: Avadis Tevanian, Jr., Michael Wayne Young
61 * Date: 1985
62 *
63 * Virtual memory object module definitions.
64 */
65
0a7de745 66#ifndef _VM_VM_OBJECT_H_
1c79356b
A		 67#define _VM_VM_OBJECT_H_
68
6d2010ae
A		 69#include <debug.h>
70#include <mach_assert.h>
1c79356b 71#include <mach_pagemap.h>
1c79356b
A		 72
73#include <mach/kern_return.h>
74#include <mach/boolean.h>
75#include <mach/memory_object_types.h>
76#include <mach/port.h>
77#include <mach/vm_prot.h>
91447636 78#include <mach/vm_param.h>
1c79356b
A		 79#include <mach/machine/vm_types.h>
80#include <kern/queue.h>
2d21ac55 81#include <kern/locks.h>
1c79356b 82#include <kern/assert.h>
0b4e3aa0 83#include <kern/misc_protos.h>
1c79356b
A		 84#include <kern/macro_help.h>
85#include <ipc/ipc_types.h>
86#include <vm/pmap.h>
1c79356b 87
1c79356b 88#include <vm/vm_external.h>
1c79356b 89
b0d623f7 90#include <vm/vm_options.h>
39037602 91#include <vm/vm_page.h>
b0d623f7 92
fe8ab488
A		 93#if VM_OBJECT_TRACKING
94#include <libkern/OSDebug.h>
95#include <kern/btlog.h>
96extern void vm_object_tracking_init(void);
97extern boolean_t vm_object_tracking_inited;
98extern btlog_t *vm_object_tracking_btlog;
0a7de745 99#define VM_OBJECT_TRACKING_NUM_RECORDS 50000
fe8ab488 100#define VM_OBJECT_TRACKING_BTDEPTH 7
0a7de745
A		 101#define VM_OBJECT_TRACKING_OP_CREATED 1
102#define VM_OBJECT_TRACKING_OP_MODIFIED 2
103#define VM_OBJECT_TRACKING_OP_TRUESHARE 3
fe8ab488
A		 104#endif /* VM_OBJECT_TRACKING */
105
91447636 106struct vm_page;
1c79356b
A		 107
108/*
109 * Types defined:
110 *
111 * vm_object_t Virtual memory object.
2d21ac55 112 * vm_object_fault_info_t Used to determine cluster size.
1c79356b
A		 113 */
114
2d21ac55 115struct vm_object_fault_info {
0a7de745
A		 116 int interruptible;
117 uint32_t user_tag;
118 vm_size_t cluster_size;
119 vm_behavior_t behavior;
f427ee49
A		 120 vm_object_offset_t lo_offset;
121 vm_object_offset_t hi_offset;
6d2010ae 122 unsigned int
0a7de745
A		 123 /* boolean_t */ no_cache:1,
124 /* boolean_t */ stealth:1,
125 /* boolean_t */ io_sync:1,
6d2010ae 126 /* boolean_t */ cs_bypass:1,
d9a64523 127 /* boolean_t */ pmap_cs_associated:1,
0a7de745 128 /* boolean_t */ mark_zf_absent:1,
316670eb 129 /* boolean_t */ batch_pmap_op:1,
cb323159
A		 130 /* boolean_t */ resilient_media:1,
131 /* boolean_t */ no_copy_on_read:1,
132 __vm_object_fault_info_unused_bits:23;
0a7de745 133 int pmap_options;
2d21ac55
A		 134};
135
136
0a7de745
A		 137#define vo_size vo_un1.vou_size
138#define vo_cache_pages_to_scan vo_un1.vou_cache_pages_to_scan
139#define vo_shadow_offset vo_un2.vou_shadow_offset
140#define vo_cache_ts vo_un2.vou_cache_ts
141#define vo_owner vo_un2.vou_owner
2d21ac55 142
1c79356b 143struct vm_object {
39037602
A		 144 /*
145 * on 64 bit systems we pack the pointers hung off the memq.
146 * those pointers have to be able to point back to the memq.
147 * the packed pointers are required to be on a 64 byte boundary
148 * which means 2 things for the vm_object... (1) the memq
149 * struct has to be the first element of the structure so that
150 * we can control it's alignment... (2) the vm_object must be
151 * aligned on a 64 byte boundary... for static vm_object's
152 * this is accomplished via the 'aligned' attribute... for
153 * vm_object's in the zone pool, this is accomplished by
154 * rounding the size of the vm_object element to the nearest
155 * 64 byte size before creating the zone.
156 */
0a7de745
A		 157 vm_page_queue_head_t memq; /* Resident memory - must be first */
158 lck_rw_t Lock; /* Synchronization */
1c79356b 159
39037602 160#if DEVELOPMENT || DEBUG
0a7de745 161 thread_t Lock_owner;
39037602 162#endif
6d2010ae 163 union {
0a7de745
A		 164 vm_object_size_t vou_size; /* Object size (only valid if internal) */
165 int vou_cache_pages_to_scan; /* pages yet to be visited in an
166 * external object in cache
167 */
6d2010ae
A		 168 } vo_un1;
169
0a7de745
A		 170 struct vm_page *memq_hint;
171 int ref_count; /* Number of references */
172 unsigned int resident_page_count;
173 /* number of resident pages */
174 unsigned int wired_page_count; /* number of wired pages
175 * use VM_OBJECT_WIRED_PAGE_UPDATE macros to update */
176 unsigned int reusable_page_count;
177
178 struct vm_object *copy; /* Object that should receive
179 * a copy of my changed pages,
180 * for copy_delay, or just the
181 * temporary object that
182 * shadows this object, for
183 * copy_call.
184 */
185 struct vm_object *shadow; /* My shadow */
94ff46dc 186 memory_object_t pager; /* Where to get data */
6d2010ae
A		 187
188 union {
0a7de745
A		 189 vm_object_offset_t vou_shadow_offset; /* Offset into shadow */
190 clock_sec_t vou_cache_ts; /* age of an external object
191 * present in cache
192 */
193 task_t vou_owner; /* If the object is purgeable
194 * or has a "ledger_tag", this
195 * is the task that owns it.
196 */
6d2010ae 197 } vo_un2;
1c79356b 198
0a7de745
A		 199 vm_object_offset_t paging_offset; /* Offset into memory object */
200 memory_object_control_t pager_control; /* Where data comes back */
1c79356b
A		 201
202 memory_object_copy_strategy_t
0a7de745 203 copy_strategy; /* How to handle data copy */
1c79356b 204
fe8ab488
A		 205#if __LP64__
206 /*
207 * Some user processes (mostly VirtualMachine software) take a large
208 * number of UPLs (via IOMemoryDescriptors) to wire pages in large
209 * VM objects and overflow the 16-bit "activity_in_progress" counter.
210 * Since we never enforced any limit there, let's give them 32 bits
211 * for backwards compatibility's sake.
212 */
0a7de745
A		 213 unsigned int paging_in_progress:16,
214 __object1_unused_bits:16;
215 unsigned int activity_in_progress;
fe8ab488
A		 216#else /* __LP64__ */
217 /*
218 * On 32-bit platforms, enlarging "activity_in_progress" would increase
219 * the size of "struct vm_object". Since we don't know of any actual
220 * overflow of these counters on these platforms, let's keep the
221 * counters as 16-bit integers.
222 */
0a7de745
A		 223 unsigned short paging_in_progress;
224 unsigned short activity_in_progress;
fe8ab488 225#endif /* __LP64__ */
0a7de745
A		 226 /* The memory object ports are
227 * being used (e.g., for pagein
228 * or pageout) -- don't change
229 * any of these fields (i.e.,
230 * don't collapse, destroy or
231 * terminate)
232 */
b0d623f7 233
1c79356b 234 unsigned int
0a7de745
A		 235 /* boolean_t array */ all_wanted:11, /* Bit array of "want to be
236 * awakened" notations. See
237 * VM_OBJECT_EVENT_* items
238 * below */
239 /* boolean_t */ pager_created:1, /* Has pager been created? */
240 /* boolean_t */ pager_initialized:1, /* Are fields ready to use? */
241 /* boolean_t */ pager_ready:1, /* Will pager take requests? */
242
243 /* boolean_t */ pager_trusted:1, /* The pager for this object
244 * is trusted. This is true for
245 * all internal objects (backed
246 * by the default pager)
247 */
248 /* boolean_t */ can_persist:1, /* The kernel may keep the data
249 * for this object (and rights
250 * to the memory object) after
251 * all address map references
252 * are deallocated?
253 */
254 /* boolean_t */ internal:1, /* Created by the kernel (and
255 * therefore, managed by the
256 * default memory manger)
257 */
258 /* boolean_t */ private:1, /* magic device_pager object,
259 * holds private pages only */
260 /* boolean_t */ pageout:1, /* pageout object. contains
261 * private pages that refer to
262 * a real memory object. */
263 /* boolean_t */ alive:1, /* Not yet terminated */
264
265 /* boolean_t */ purgable:2, /* Purgable state. See
266 * VM_PURGABLE_*
267 */
268 /* boolean_t */ purgeable_only_by_kernel:1,
269 /* boolean_t */ purgeable_when_ripe:1, /* Purgeable when a token
270 * becomes ripe.
271 */
272 /* boolean_t */ shadowed:1, /* Shadow may exist */
273 /* boolean_t */ true_share:1,
274 /* This object is mapped
275 * in more than one place
276 * and hence cannot be
277 * coalesced */
278 /* boolean_t */ terminating:1,
279 /* Allows vm_object_lookup
280 * and vm_object_deallocate
281 * to special case their
282 * behavior when they are
283 * called as a result of
284 * page cleaning during
285 * object termination
286 */
287 /* boolean_t */ named:1, /* An enforces an internal
288 * naming convention, by
289 * calling the right routines
290 * for allocation and
291 * destruction, UBC references
292 * against the vm_object are
293 * checked.
294 */
295 /* boolean_t */ shadow_severed:1,
296 /* When a permanent object
297 * backing a COW goes away
298 * unexpectedly. This bit
299 * allows vm_fault to return
300 * an error rather than a
301 * zero filled page.
302 */
303 /* boolean_t */ phys_contiguous:1,
304 /* Memory is wired and
305 * guaranteed physically
306 * contiguous. However
307 * it is not device memory
308 * and obeys normal virtual
309 * memory rules w.r.t pmap
310 * access bits.
311 */
312 /* boolean_t */ nophyscache:1,
313 /* When mapped at the
314 * pmap level, don't allow
315 * primary caching. (for
316 * I/O)
317 */
318 /* boolean_t */ _object5_unused_bits:1;
319
320 queue_chain_t cached_list; /* Attachment point for the
321 * list of objects cached as a
322 * result of their can_persist
323 * value
324 */
325 /*
326 * the following fields are not protected by any locks
327 * they are updated via atomic compare and swap
328 */
329 vm_object_offset_t last_alloc; /* last allocation offset */
94ff46dc
A		 330 vm_offset_t cow_hint; /* last page present in */
331 /* shadow but not in object */
0a7de745
A		 332 int sequential; /* sequential access size */
333
334 uint32_t pages_created;
335 uint32_t pages_used;
2d21ac55 336 /* hold object lock when altering */
0a7de745
A		 337 unsigned int
338 wimg_bits:8, /* cache WIMG bits */
339 code_signed:1, /* pages are signed and should be
340 * validated; the signatures are stored
341 * with the pager */
342 transposed:1, /* object was transposed with another */
343 mapping_in_progress:1, /* pager being mapped/unmapped */
344 phantom_isssd:1,
345 volatile_empty:1,
346 volatile_fault:1,
347 all_reusable:1,
348 blocked_access:1,
349 set_cache_attr:1,
350 object_is_shared_cache:1,
351 purgeable_queue_type:2,
352 purgeable_queue_group:3,
353 io_tracking:1,
354 no_tag_update:1, /* */
39037602 355#if CONFIG_SECLUDED_MEMORY
0a7de745
A		 356 eligible_for_secluded:1,
357 can_grab_secluded:1,
39037602 358#else /* CONFIG_SECLUDED_MEMORY */
0a7de745 359 __object3_unused_bits:2,
39037602 360#endif /* CONFIG_SECLUDED_MEMORY */
d9a64523 361#if VM_OBJECT_ACCESS_TRACKING
0a7de745 362 access_tracking:1,
d9a64523 363#else /* VM_OBJECT_ACCESS_TRACKING */
0a7de745 364 __unused_access_tracking:1,
d9a64523 365#endif /* VM_OBJECT_ACCESS_TRACKING */
cb323159
A		 366 vo_ledger_tag:3,
367 vo_no_footprint:1;
d9a64523
A		 368
369#if VM_OBJECT_ACCESS_TRACKING
0a7de745
A		 370 uint32_t access_tracking_reads;
371 uint32_t access_tracking_writes;
d9a64523 372#endif /* VM_OBJECT_ACCESS_TRACKING */
6d2010ae 373
0a7de745 374 uint8_t scan_collisions;
94ff46dc 375 uint8_t __object4_unused_bits[1];
0a7de745 376 vm_tag_t wire_tag;
3e170ce0 377
fe8ab488 378#if CONFIG_PHANTOM_CACHE
0a7de745 379 uint32_t phantom_object_id;
fe8ab488
A		 380#endif
381#if CONFIG_IOSCHED || UPL_DEBUG
0a7de745 382 queue_head_t uplq; /* List of outstanding upls */
fe8ab488 383#endif
2d21ac55 384
0a7de745 385#ifdef VM_PIP_DEBUG
0c530ab8
A		 386/*
387 * Keep track of the stack traces for the first holders
388 * of a "paging_in_progress" reference for this VM object.
389 */
0a7de745
A		 390#define VM_PIP_DEBUG_STACK_FRAMES 25 /* depth of each stack trace */
391#define VM_PIP_DEBUG_MAX_REFS 10 /* track that many references */
0c530ab8
A		 392 struct __pip_backtrace {
393 void *pip_retaddr[VM_PIP_DEBUG_STACK_FRAMES];
394 } pip_holders[VM_PIP_DEBUG_MAX_REFS];
0a7de745 395#endif /* VM_PIP_DEBUG */
2d21ac55 396
0a7de745
A		 397 queue_chain_t objq; /* object queue - currently used for purgable queues */
398 queue_chain_t task_objq; /* objects owned by task - protected by task lock */
fe8ab488 399
d9a64523 400#if !VM_TAG_ACTIVE_UPDATE
0a7de745 401 queue_chain_t wired_objq;
d9a64523
A		 402#endif /* !VM_TAG_ACTIVE_UPDATE */
403
fe8ab488
A		 404#if DEBUG
405 void *purgeable_owner_bt[16];
406 task_t vo_purgeable_volatilizer; /* who made it volatile? */
407 void *purgeable_volatilizer_bt[16];
408#endif /* DEBUG */
1c79356b
A		 409};
410
0a7de745
A		 411#define VM_OBJECT_PURGEABLE_FAULT_ERROR(object) \
412 ((object)->volatile_fault && \
413 ((object)->purgable == VM_PURGABLE_VOLATILE || \
b0d623f7
A		 414 (object)->purgable == VM_PURGABLE_EMPTY))
415
d9a64523
A		 416#if VM_OBJECT_ACCESS_TRACKING
417extern uint64_t vm_object_access_tracking_reads;
418extern uint64_t vm_object_access_tracking_writes;
419extern void vm_object_access_tracking(vm_object_t object,
0a7de745
A		 420 int *access_tracking,
421 uint32_t *access_tracking_reads,
422 uint32_t *acess_tracking_writes);
d9a64523
A		 423#endif /* VM_OBJECT_ACCESS_TRACKING */
424
39236c6e 425extern
0a7de745 426vm_object_t kernel_object; /* the single kernel object */
1c79356b 427
39236c6e 428extern
0a7de745 429vm_object_t compressor_object; /* the single compressor object */
39236c6e 430
c3c9b80d
A		 431extern
432vm_object_t retired_pages_object; /* holds VM pages which should never be used */
433
39236c6e 434extern
0a7de745
A		 435unsigned int vm_object_absent_max; /* maximum number of absent pages
436 * at a time for each object */
1c79356b 437
0a7de745
A		 438# define VM_MSYNC_INITIALIZED 0
439# define VM_MSYNC_SYNCHRONIZING 1
440# define VM_MSYNC_DONE 2
1c79356b 441
b0d623f7 442
0a7de745
A		 443extern lck_grp_t vm_map_lck_grp;
444extern lck_attr_t vm_map_lck_attr;
b0d623f7 445
5ba3f43e
A		 446#ifndef VM_TAG_ACTIVE_UPDATE
447#error VM_TAG_ACTIVE_UPDATE
448#endif
1c79356b 449
d9a64523
A		 450#if VM_TAG_ACTIVE_UPDATE
451#define VM_OBJECT_WIRED_ENQUEUE(object) panic("VM_OBJECT_WIRED_ENQUEUE")
452#define VM_OBJECT_WIRED_DEQUEUE(object) panic("VM_OBJECT_WIRED_DEQUEUE")
453#else /* VM_TAG_ACTIVE_UPDATE */
0a7de745
A		 454#define VM_OBJECT_WIRED_ENQUEUE(object) \
455 MACRO_BEGIN \
456 lck_spin_lock_grp(&vm_objects_wired_lock, &vm_page_lck_grp_bucket); \
457 assert(!(object)->wired_objq.next); \
458 assert(!(object)->wired_objq.prev); \
459 queue_enter(&vm_objects_wired, (object), \
460 vm_object_t, wired_objq); \
461 lck_spin_unlock(&vm_objects_wired_lock); \
d9a64523 462 MACRO_END
0a7de745
A		 463#define VM_OBJECT_WIRED_DEQUEUE(object) \
464 MACRO_BEGIN \
465 if ((object)->wired_objq.next) { \
466 lck_spin_lock_grp(&vm_objects_wired_lock, &vm_page_lck_grp_bucket); \
467 queue_remove(&vm_objects_wired, (object), \
468 vm_object_t, wired_objq); \
469 lck_spin_unlock(&vm_objects_wired_lock); \
470 } \
d9a64523
A		 471 MACRO_END
472#endif /* VM_TAG_ACTIVE_UPDATE */
473
0a7de745
A		 474#define VM_OBJECT_WIRED(object, tag) \
475 MACRO_BEGIN \
476 assert(VM_KERN_MEMORY_NONE != (tag)); \
477 assert(VM_KERN_MEMORY_NONE == (object)->wire_tag); \
478 (object)->wire_tag = (tag); \
479 if (!VM_TAG_ACTIVE_UPDATE) { \
480 VM_OBJECT_WIRED_ENQUEUE((object)); \
481 } \
3e170ce0
A		 482 MACRO_END
483
0a7de745
A		 484#define VM_OBJECT_UNWIRED(object) \
485 MACRO_BEGIN \
486 if (!VM_TAG_ACTIVE_UPDATE) { \
487 VM_OBJECT_WIRED_DEQUEUE((object)); \
488 } \
489 if (VM_KERN_MEMORY_NONE != (object)->wire_tag) { \
5ba3f43e 490 vm_tag_update_size((object)->wire_tag, -ptoa_64((object)->wired_page_count)); \
0a7de745
A		 491 (object)->wire_tag = VM_KERN_MEMORY_NONE; \
492 } \
5ba3f43e
A		 493 MACRO_END
494
495// These two macros start & end a C block
0a7de745
A		 496#define VM_OBJECT_WIRED_PAGE_UPDATE_START(object) \
497 MACRO_BEGIN \
498 { \
5ba3f43e
A		 499 int64_t __wireddelta = 0; vm_tag_t __waswired = (object)->wire_tag;
500
0a7de745
A		 501#define VM_OBJECT_WIRED_PAGE_UPDATE_END(object, tag) \
502 if (__wireddelta) { \
503 boolean_t __overflow __assert_only = \
504 os_add_overflow((object)->wired_page_count, __wireddelta, \
505 &(object)->wired_page_count); \
506 assert(!__overflow); \
507 if (!(object)->pageout && !(object)->no_tag_update) { \
508 if (__wireddelta > 0) { \
509 assert (VM_KERN_MEMORY_NONE != (tag)); \
510 if (VM_KERN_MEMORY_NONE == __waswired) { \
511 VM_OBJECT_WIRED((object), (tag)); \
512 } \
513 vm_tag_update_size((object)->wire_tag, ptoa_64(__wireddelta)); \
514 } else if (VM_KERN_MEMORY_NONE != __waswired) { \
515 assert (VM_KERN_MEMORY_NONE != (object)->wire_tag); \
516 vm_tag_update_size((object)->wire_tag, ptoa_64(__wireddelta)); \
517 if (!(object)->wired_page_count) { \
518 VM_OBJECT_UNWIRED((object)); \
519 } \
520 } \
521 } \
522 } \
523 } \
3e170ce0
A		 524 MACRO_END
525
5ba3f43e
A		 526#define VM_OBJECT_WIRED_PAGE_COUNT(object, delta) \
527 __wireddelta += delta; \
528
529#define VM_OBJECT_WIRED_PAGE_ADD(object, m) \
d9a64523 530 if (!(m)->vmp_private && !(m)->vmp_fictitious) __wireddelta++;
5ba3f43e
A		 531
532#define VM_OBJECT_WIRED_PAGE_REMOVE(object, m) \
d9a64523 533 if (!(m)->vmp_private && !(m)->vmp_fictitious) __wireddelta--;
5ba3f43e
A		 534
535
3e170ce0 536
0a7de745
A		 537#define OBJECT_LOCK_SHARED 0
538#define OBJECT_LOCK_EXCLUSIVE 1
39037602 539
0a7de745 540extern lck_grp_t vm_object_lck_grp;
0a7de745
A		 541extern lck_attr_t vm_object_lck_attr;
542extern lck_attr_t kernel_object_lck_attr;
543extern lck_attr_t compressor_object_lck_attr;
39037602 544
0a7de745 545extern vm_object_t vm_pageout_scan_wants_object;
39037602 546
0a7de745 547extern void vm_object_lock(vm_object_t);
f427ee49 548extern bool vm_object_lock_check_contended(vm_object_t);
0a7de745
A		 549extern boolean_t vm_object_lock_try(vm_object_t);
550extern boolean_t _vm_object_lock_try(vm_object_t);
551extern boolean_t vm_object_lock_avoid(vm_object_t);
552extern void vm_object_lock_shared(vm_object_t);
553extern boolean_t vm_object_lock_yield_shared(vm_object_t);
554extern boolean_t vm_object_lock_try_shared(vm_object_t);
555extern void vm_object_unlock(vm_object_t);
556extern boolean_t vm_object_lock_upgrade(vm_object_t);
39037602
A		 557
558/*
559 * Object locking macros
560 */
561
0a7de745
A		 562#define vm_object_lock_init(object) \
563 lck_rw_init(&(object)->Lock, &vm_object_lck_grp, \
564 (((object) == kernel_object || \
565 (object) == vm_submap_object) ? \
566 &kernel_object_lck_attr : \
567 (((object) == compressor_object) ? \
568 &compressor_object_lck_attr : \
569 &vm_object_lck_attr)))
570#define vm_object_lock_destroy(object) lck_rw_destroy(&(object)->Lock, &vm_object_lck_grp)
39037602 571
0a7de745 572#define vm_object_lock_try_scan(object) _vm_object_lock_try(object)
39037602
A		 573
574/*
575 * CAUTION: the following vm_object_lock_assert_held*() macros merely
576 * check if anyone is holding the lock, but the holder may not necessarily
577 * be the caller...
578 */
579#if MACH_ASSERT || DEBUG
580#define vm_object_lock_assert_held(object) \
581 lck_rw_assert(&(object)->Lock, LCK_RW_ASSERT_HELD)
582#define vm_object_lock_assert_shared(object) \
583 lck_rw_assert(&(object)->Lock, LCK_RW_ASSERT_SHARED)
584#define vm_object_lock_assert_exclusive(object) \
585 lck_rw_assert(&(object)->Lock, LCK_RW_ASSERT_EXCLUSIVE)
586#define vm_object_lock_assert_notheld(object) \
587 lck_rw_assert(&(object)->Lock, LCK_RW_ASSERT_NOTHELD)
0a7de745 588#else /* MACH_ASSERT || DEBUG */
39037602
A		 589#define vm_object_lock_assert_held(object)
590#define vm_object_lock_assert_shared(object)
591#define vm_object_lock_assert_exclusive(object)
592#define vm_object_lock_assert_notheld(object)
593#endif /* MACH_ASSERT || DEBUG */
594
595
1c79356b
A		 596/*
597 * Declare procedures that operate on VM objects.
598 */
599
0a7de745 600__private_extern__ void vm_object_bootstrap(void);
1c79356b 601
0a7de745 602__private_extern__ void vm_object_reaper_init(void);
8f6c56a5 603
0a7de745 604__private_extern__ vm_object_t vm_object_allocate(vm_object_size_t size);
1c79356b 605
91447636 606__private_extern__ void _vm_object_allocate(vm_object_size_t size,
0a7de745 607 vm_object_t object);
91447636 608
0a7de745
A		 609#define vm_object_reference_locked(object) \
610 MACRO_BEGIN \
611 vm_object_t RLObject = (object); \
612 vm_object_lock_assert_exclusive(object); \
613 assert((RLObject)->ref_count > 0); \
614 (RLObject)->ref_count++; \
615 assert((RLObject)->ref_count > 1); \
2d21ac55
A		 616 MACRO_END
617
618
c3c9b80d
A		 619#define vm_object_reference_shared(object) \
620 MACRO_BEGIN \
621 vm_object_t RLObject = (object); \
622 vm_object_lock_assert_shared(object); \
623 assert((RLObject)->ref_count > 0); \
0a7de745 624 OSAddAtomic(1, &(RLObject)->ref_count); \
c3c9b80d 625 assert((RLObject)->ref_count > 0); \
2d21ac55 626 MACRO_END
0b4e3aa0
A		 627
628
0a7de745
A		 629__private_extern__ void vm_object_reference(
630 vm_object_t object);
0b4e3aa0 631
0a7de745 632#if !MACH_ASSERT
0b4e3aa0 633
0a7de745
A		 634#define vm_object_reference(object) \
635MACRO_BEGIN \
636 vm_object_t RObject = (object); \
637 if (RObject) { \
638 vm_object_lock_shared(RObject); \
639 vm_object_reference_shared(RObject); \
640 vm_object_unlock(RObject); \
641 } \
1c79356b 642MACRO_END
0b4e3aa0 643
0a7de745 644#endif /* MACH_ASSERT */
1c79356b 645
0a7de745
A		 646__private_extern__ void vm_object_deallocate(
647 vm_object_t object);
1c79356b 648
0b4e3aa0 649__private_extern__ kern_return_t vm_object_release_name(
0a7de745
A		 650 vm_object_t object,
651 int flags);
652
653__private_extern__ void vm_object_pmap_protect(
654 vm_object_t object,
655 vm_object_offset_t offset,
656 vm_object_size_t size,
657 pmap_t pmap,
f427ee49 658 vm_map_size_t pmap_page_size,
0a7de745
A		 659 vm_map_offset_t pmap_start,
660 vm_prot_t prot);
661
662__private_extern__ void vm_object_pmap_protect_options(
663 vm_object_t object,
664 vm_object_offset_t offset,
665 vm_object_size_t size,
666 pmap_t pmap,
f427ee49 667 vm_map_size_t pmap_page_size,
0a7de745
A		 668 vm_map_offset_t pmap_start,
669 vm_prot_t prot,
670 int options);
671
672__private_extern__ void vm_object_page_remove(
673 vm_object_t object,
674 vm_object_offset_t start,
675 vm_object_offset_t end);
676
677__private_extern__ void vm_object_deactivate_pages(
678 vm_object_t object,
679 vm_object_offset_t offset,
680 vm_object_size_t size,
681 boolean_t kill_page,
682 boolean_t reusable_page,
683 struct pmap *pmap,
f427ee49 684/* XXX TODO4K: need pmap_page_size here too? */
0a7de745
A		 685 vm_map_offset_t pmap_offset);
686
687__private_extern__ void vm_object_reuse_pages(
688 vm_object_t object,
689 vm_object_offset_t start_offset,
690 vm_object_offset_t end_offset,
691 boolean_t allow_partial_reuse);
692
693__private_extern__ uint64_t vm_object_purge(
694 vm_object_t object,
695 int flags);
91447636
A		 696
697__private_extern__ kern_return_t vm_object_purgable_control(
0a7de745
A		 698 vm_object_t object,
699 vm_purgable_t control,
700 int *state);
91447636 701
39236c6e 702__private_extern__ kern_return_t vm_object_get_page_counts(
0a7de745
A		 703 vm_object_t object,
704 vm_object_offset_t offset,
705 vm_object_size_t size,
706 unsigned int *resident_page_count,
707 unsigned int *dirty_page_count);
708
709__private_extern__ boolean_t vm_object_coalesce(
710 vm_object_t prev_object,
711 vm_object_t next_object,
712 vm_object_offset_t prev_offset,
713 vm_object_offset_t next_offset,
714 vm_object_size_t prev_size,
715 vm_object_size_t next_size);
716
717__private_extern__ boolean_t vm_object_shadow(
718 vm_object_t *object,
719 vm_object_offset_t *offset,
720 vm_object_size_t length);
721
722__private_extern__ void vm_object_collapse(
723 vm_object_t object,
724 vm_object_offset_t offset,
725 boolean_t can_bypass);
726
727__private_extern__ boolean_t vm_object_copy_quickly(
728 vm_object_t *_object,
729 vm_object_offset_t src_offset,
730 vm_object_size_t size,
731 boolean_t *_src_needs_copy,
732 boolean_t *_dst_needs_copy);
733
734__private_extern__ kern_return_t vm_object_copy_strategically(
735 vm_object_t src_object,
736 vm_object_offset_t src_offset,
737 vm_object_size_t size,
738 vm_object_t *dst_object,
739 vm_object_offset_t *dst_offset,
740 boolean_t *dst_needs_copy);
741
742__private_extern__ kern_return_t vm_object_copy_slowly(
743 vm_object_t src_object,
744 vm_object_offset_t src_offset,
745 vm_object_size_t size,
746 boolean_t interruptible,
747 vm_object_t *_result_object);
748
749__private_extern__ vm_object_t vm_object_copy_delayed(
750 vm_object_t src_object,
751 vm_object_offset_t src_offset,
752 vm_object_size_t size,
753 boolean_t src_object_shared);
754
755
756
757__private_extern__ kern_return_t vm_object_destroy(
758 vm_object_t object,
759 kern_return_t reason);
760
761__private_extern__ void vm_object_pager_create(
762 vm_object_t object);
763
764__private_extern__ void vm_object_compressor_pager_create(
765 vm_object_t object);
766
767__private_extern__ void vm_object_page_map(
768 vm_object_t object,
769 vm_object_offset_t offset,
770 vm_object_size_t size,
771 vm_object_offset_t (*map_fn)
772 (void *, vm_object_offset_t),
773 void *map_fn_data);
1c79356b 774
0b4e3aa0 775__private_extern__ kern_return_t vm_object_upl_request(
0a7de745
A		 776 vm_object_t object,
777 vm_object_offset_t offset,
778 upl_size_t size,
779 upl_t *upl,
780 upl_page_info_t *page_info,
781 unsigned int *count,
782 upl_control_flags_t flags,
783 vm_tag_t tag);
0b4e3aa0 784
91447636 785__private_extern__ kern_return_t vm_object_transpose(
0a7de745
A		 786 vm_object_t object1,
787 vm_object_t object2,
788 vm_object_size_t transpose_size);
91447636 789
0b4e3aa0 790__private_extern__ boolean_t vm_object_sync(
0a7de745
A		 791 vm_object_t object,
792 vm_object_offset_t offset,
793 vm_object_size_t size,
794 boolean_t should_flush,
795 boolean_t should_return,
796 boolean_t should_iosync);
1c79356b 797
0b4e3aa0 798__private_extern__ kern_return_t vm_object_update(
0a7de745
A		 799 vm_object_t object,
800 vm_object_offset_t offset,
801 vm_object_size_t size,
802 vm_object_offset_t *error_offset,
803 int *io_errno,
804 memory_object_return_t should_return,
805 int flags,
806 vm_prot_t prot);
1c79356b 807
0b4e3aa0 808__private_extern__ kern_return_t vm_object_lock_request(
0a7de745
A		 809 vm_object_t object,
810 vm_object_offset_t offset,
811 vm_object_size_t size,
812 memory_object_return_t should_return,
813 int flags,
814 vm_prot_t prot);
1c79356b 815
1c79356b 816
1c79356b 817
0a7de745
A		 818__private_extern__ vm_object_t vm_object_memory_object_associate(
819 memory_object_t pager,
820 vm_object_t object,
821 vm_object_size_t size,
822 boolean_t check_named);
1c79356b
A		 823
824
0a7de745
A		 825__private_extern__ void vm_object_cluster_size(
826 vm_object_t object,
827 vm_object_offset_t *start,
828 vm_size_t *length,
829 vm_object_fault_info_t fault_info,
830 uint32_t *io_streaming);
91447636
A		 831
832__private_extern__ kern_return_t vm_object_populate_with_private(
0a7de745
A		 833 vm_object_t object,
834 vm_object_offset_t offset,
835 ppnum_t phys_page,
836 vm_size_t size);
91447636 837
6d2010ae 838__private_extern__ void vm_object_change_wimg_mode(
0a7de745
A		 839 vm_object_t object,
840 unsigned int wimg_mode);
6d2010ae 841
0c530ab8 842extern kern_return_t adjust_vm_object_cache(
91447636
A		 843 vm_size_t oval,
844 vm_size_t nval);
845
0c530ab8 846extern kern_return_t vm_object_page_op(
0a7de745
A		 847 vm_object_t object,
848 vm_object_offset_t offset,
849 int ops,
850 ppnum_t *phys_entry,
851 int *flags);
0c530ab8
A		 852
853extern kern_return_t vm_object_range_op(
0a7de745
A		 854 vm_object_t object,
855 vm_object_offset_t offset_beg,
856 vm_object_offset_t offset_end,
0c530ab8 857 int ops,
0a7de745 858 uint32_t *range);
b0d623f7
A		 859
860
0a7de745
A		 861__private_extern__ void vm_object_reap_pages(
862 vm_object_t object,
863 int reap_type);
864#define REAP_REAP 0
865#define REAP_TERMINATE 1
866#define REAP_PURGEABLE 2
867#define REAP_DATA_FLUSH 3
b0d623f7 868
6d2010ae 869#if CONFIG_FREEZE
6d2010ae 870
cb323159 871__private_extern__ uint32_t
3e170ce0 872vm_object_compressed_freezer_pageout(
cb323159 873 vm_object_t object, uint32_t dirty_budget);
6d2010ae 874
3e170ce0
A		 875__private_extern__ void
876vm_object_compressed_freezer_done(
877 void);
878
6d2010ae 879#endif /* CONFIG_FREEZE */
0c530ab8 880
3e170ce0
A		 881__private_extern__ void
882vm_object_pageout(
883 vm_object_t object);
884
fe8ab488
A		 885#if CONFIG_IOSCHED
886struct io_reprioritize_req {
0a7de745
A		 887 uint64_t blkno;
888 uint32_t len;
889 int priority;
890 struct vnode *devvp;
891 queue_chain_t io_reprioritize_list;
fe8ab488
A		 892};
893typedef struct io_reprioritize_req *io_reprioritize_req_t;
894
895extern void vm_io_reprioritize_init(void);
896#endif
897
1c79356b
A		 898/*
899 * Event waiting handling
900 */
901
0a7de745
A		 902#define VM_OBJECT_EVENT_INITIALIZED 0
903#define VM_OBJECT_EVENT_PAGER_READY 1
904#define VM_OBJECT_EVENT_PAGING_IN_PROGRESS 2
905#define VM_OBJECT_EVENT_MAPPING_IN_PROGRESS 3
906#define VM_OBJECT_EVENT_LOCK_IN_PROGRESS 4
907#define VM_OBJECT_EVENT_UNCACHING 5
908#define VM_OBJECT_EVENT_COPY_CALL 6
909#define VM_OBJECT_EVENT_CACHING 7
910#define VM_OBJECT_EVENT_UNBLOCKED 8
911#define VM_OBJECT_EVENT_PAGING_ONLY_IN_PROGRESS 9
1c79356b 912
39037602 913#define VM_OBJECT_EVENT_MAX 10 /* 11 bits in "all_wanted", so 0->10 */
1c79356b 914
39037602
A		 915static __inline__ wait_result_t
916vm_object_assert_wait(
0a7de745
A		 917 vm_object_t object,
918 int event,
919 wait_interrupt_t interruptible)
39037602
A		 920{
921 wait_result_t wr;
922
923 vm_object_lock_assert_exclusive(object);
924 assert(event >= 0 && event <= VM_OBJECT_EVENT_MAX);
925
926 object->all_wanted |= 1 << event;
f427ee49 927 wr = assert_wait((event_t)((vm_offset_t)object + (vm_offset_t)event),
0a7de745 928 interruptible);
39037602
A		 929 return wr;
930}
931
932static __inline__ wait_result_t
933vm_object_wait(
0a7de745
A		 934 vm_object_t object,
935 int event,
936 wait_interrupt_t interruptible)
39037602
A		 937{
938 wait_result_t wr;
939
940 vm_object_assert_wait(object, event, interruptible);
941 vm_object_unlock(object);
942 wr = thread_block(THREAD_CONTINUE_NULL);
943 return wr;
944}
945
946static __inline__ wait_result_t
947thread_sleep_vm_object(
0a7de745
A		 948 vm_object_t object,
949 event_t event,
950 wait_interrupt_t interruptible)
39037602
A		 951{
952 wait_result_t wr;
953
954#if DEVELOPMENT || DEBUG
0a7de745 955 if (object->Lock_owner != current_thread()) {
39037602 956 panic("thread_sleep_vm_object: now owner - %p\n", object);
0a7de745 957 }
39037602
A		 958 object->Lock_owner = 0;
959#endif
0a7de745
A		 960 wr = lck_rw_sleep(&object->Lock,
961 LCK_SLEEP_PROMOTED_PRI,
962 event,
963 interruptible);
39037602
A		 964#if DEVELOPMENT || DEBUG
965 object->Lock_owner = current_thread();
966#endif
967 return wr;
968}
9bccf70c 969
39037602
A		 970static __inline__ wait_result_t
971vm_object_sleep(
0a7de745
A		 972 vm_object_t object,
973 int event,
974 wait_interrupt_t interruptible)
39037602
A		 975{
976 wait_result_t wr;
977
978 vm_object_lock_assert_exclusive(object);
979 assert(event >= 0 && event <= VM_OBJECT_EVENT_MAX);
980
981 object->all_wanted |= 1 << event;
982 wr = thread_sleep_vm_object(object,
f427ee49 983 (event_t)((vm_offset_t)object + (vm_offset_t)event),
0a7de745 984 interruptible);
39037602
A		 985 return wr;
986}
987
988static __inline__ void
989vm_object_wakeup(
0a7de745
A		 990 vm_object_t object,
991 int event)
39037602
A		 992{
993 vm_object_lock_assert_exclusive(object);
994 assert(event >= 0 && event <= VM_OBJECT_EVENT_MAX);
995
0a7de745 996 if (object->all_wanted & (1 << event)) {
f427ee49 997 thread_wakeup((event_t)((vm_offset_t)object + (vm_offset_t)event));
0a7de745 998 }
39037602
A		 999 object->all_wanted &= ~(1 << event);
1000}
1001
1002static __inline__ void
1003vm_object_set_wanted(
0a7de745
A		 1004 vm_object_t object,
1005 int event)
39037602
A		 1006{
1007 vm_object_lock_assert_exclusive(object);
1008 assert(event >= 0 && event <= VM_OBJECT_EVENT_MAX);
1c79356b 1009
39037602
A		 1010 object->all_wanted |= (1 << event);
1011}
1c79356b 1012
39037602
A		 1013static __inline__ int
1014vm_object_wanted(
0a7de745
A		 1015 vm_object_t object,
1016 int event)
39037602
A		 1017{
1018 vm_object_lock_assert_held(object);
1019 assert(event >= 0 && event <= VM_OBJECT_EVENT_MAX);
1c79356b 1020
39037602
A		 1021 return object->all_wanted & (1 << event);
1022}
1c79356b
A		 1023
1024/*
1025 * Routines implemented as macros
1026 */
0c530ab8 1027#ifdef VM_PIP_DEBUG
2d21ac55 1028#include <libkern/OSDebug.h>
0a7de745
A		 1029#define VM_PIP_DEBUG_BEGIN(object) \
1030 MACRO_BEGIN \
1031 int pip = ((object)->paging_in_progress + \
1032 (object)->activity_in_progress); \
1033 if (pip < VM_PIP_DEBUG_MAX_REFS) { \
1034 (void) OSBacktrace(&(object)->pip_holders[pip].pip_retaddr[0], \
1035 VM_PIP_DEBUG_STACK_FRAMES); \
1036 } \
0c530ab8 1037 MACRO_END
0a7de745 1038#else /* VM_PIP_DEBUG */
0c530ab8 1039#define VM_PIP_DEBUG_BEGIN(object)
0a7de745
A		 1040#endif /* VM_PIP_DEBUG */
1041
1042#define vm_object_activity_begin(object) \
1043 MACRO_BEGIN \
1044 vm_object_lock_assert_exclusive((object)); \
1045 VM_PIP_DEBUG_BEGIN((object)); \
1046 (object)->activity_in_progress++; \
1047 if ((object)->activity_in_progress == 0) { \
1048 panic("vm_object_activity_begin(%p): overflow\n", (object));\
1049 } \
b0d623f7
A		 1050 MACRO_END
1051
0a7de745
A		 1052#define vm_object_activity_end(object) \
1053 MACRO_BEGIN \
1054 vm_object_lock_assert_exclusive((object)); \
1055 if ((object)->activity_in_progress == 0) { \
1056 panic("vm_object_activity_end(%p): underflow\n", (object));\
1057 } \
1058 (object)->activity_in_progress--; \
1059 if ((object)->paging_in_progress == 0 && \
1060 (object)->activity_in_progress == 0) \
1061 vm_object_wakeup((object), \
1062 VM_OBJECT_EVENT_PAGING_IN_PROGRESS); \
b0d623f7
A		 1063 MACRO_END
1064
0a7de745
A		 1065#define vm_object_paging_begin(object) \
1066 MACRO_BEGIN \
1067 vm_object_lock_assert_exclusive((object)); \
1068 VM_PIP_DEBUG_BEGIN((object)); \
1069 (object)->paging_in_progress++; \
1070 if ((object)->paging_in_progress == 0) { \
1071 panic("vm_object_paging_begin(%p): overflow\n", (object));\
1072 } \
1c79356b
A		 1073 MACRO_END
1074
0a7de745
A		 1075#define vm_object_paging_end(object) \
1076 MACRO_BEGIN \
1077 vm_object_lock_assert_exclusive((object)); \
1078 if ((object)->paging_in_progress == 0) { \
1079 panic("vm_object_paging_end(%p): underflow\n", (object));\
1080 } \
1081 (object)->paging_in_progress--; \
1082 if ((object)->paging_in_progress == 0) { \
1083 vm_object_wakeup((object), \
1084 VM_OBJECT_EVENT_PAGING_ONLY_IN_PROGRESS); \
1085 if ((object)->activity_in_progress == 0) \
1086 vm_object_wakeup((object), \
1087 VM_OBJECT_EVENT_PAGING_IN_PROGRESS); \
1088 } \
1c79356b
A		 1089 MACRO_END
1090
0a7de745
A		 1091#define vm_object_paging_wait(object, interruptible) \
1092 MACRO_BEGIN \
1093 vm_object_lock_assert_exclusive((object)); \
1094 while ((object)->paging_in_progress != 0 || \
1095 (object)->activity_in_progress != 0) { \
1096 wait_result_t _wr; \
1097 \
1098 _wr = vm_object_sleep((object), \
1099 VM_OBJECT_EVENT_PAGING_IN_PROGRESS, \
1100 (interruptible)); \
1101 \
1102 /*XXX if ((interruptible) && (_wr != THREAD_AWAKENED))*/ \
1103 /*XXX break; */ \
1104 } \
1c79356b
A		 1105 MACRO_END
1106
0a7de745
A		 1107#define vm_object_paging_only_wait(object, interruptible) \
1108 MACRO_BEGIN \
1109 vm_object_lock_assert_exclusive((object)); \
1110 while ((object)->paging_in_progress != 0) { \
1111 wait_result_t _wr; \
1112 \
1113 _wr = vm_object_sleep((object), \
1114 VM_OBJECT_EVENT_PAGING_ONLY_IN_PROGRESS,\
1115 (interruptible)); \
1116 \
1117 /*XXX if ((interruptible) && (_wr != THREAD_AWAKENED))*/ \
1118 /*XXX break; */ \
1119 } \
b0d623f7
A		 1120 MACRO_END
1121
1c79356b 1122
0a7de745
A		 1123#define vm_object_mapping_begin(object) \
1124 MACRO_BEGIN \
1125 vm_object_lock_assert_exclusive((object)); \
1126 assert(! (object)->mapping_in_progress); \
1127 (object)->mapping_in_progress = TRUE; \
593a1d5f
A		 1128 MACRO_END
1129
0a7de745
A		 1130#define vm_object_mapping_end(object) \
1131 MACRO_BEGIN \
1132 vm_object_lock_assert_exclusive((object)); \
1133 assert((object)->mapping_in_progress); \
1134 (object)->mapping_in_progress = FALSE; \
1135 vm_object_wakeup((object), \
1136 VM_OBJECT_EVENT_MAPPING_IN_PROGRESS); \
593a1d5f
A		 1137 MACRO_END
1138
0a7de745
A		 1139#define vm_object_mapping_wait(object, interruptible) \
1140 MACRO_BEGIN \
1141 vm_object_lock_assert_exclusive((object)); \
1142 while ((object)->mapping_in_progress) { \
1143 wait_result_t _wr; \
1144 \
1145 _wr = vm_object_sleep((object), \
1146 VM_OBJECT_EVENT_MAPPING_IN_PROGRESS, \
1147 (interruptible)); \
1148 /*XXX if ((interruptible) && (_wr != THREAD_AWAKENED))*/ \
1149 /*XXX break; */ \
1150 } \
1151 assert(!(object)->mapping_in_progress); \
593a1d5f
A		 1152 MACRO_END
1153
1154
1c79356b 1155
91447636
A		 1156#define vm_object_round_page(x) (((vm_object_offset_t)(x) + PAGE_MASK) & ~((signed)PAGE_MASK))
1157#define vm_object_trunc_page(x) ((vm_object_offset_t)(x) & ~((signed)PAGE_MASK))
1158
0a7de745
A		 1159extern void vm_object_cache_add(vm_object_t);
1160extern void vm_object_cache_remove(vm_object_t);
1161extern int vm_object_cache_evict(int, int);
6d2010ae 1162
d9a64523 1163#define VM_OBJECT_OWNER_DISOWNED ((task_t) -1)
0a7de745
A		 1164#define VM_OBJECT_OWNER(object) \
1165 ((((object)->purgable == VM_PURGABLE_DENY && \
1166 (object)->vo_ledger_tag == 0) || \
1167 (object)->vo_owner == TASK_NULL) \
1168 ? TASK_NULL /* not owned */ \
1169 : (((object)->vo_owner == VM_OBJECT_OWNER_DISOWNED) \
1170 ? kernel_task /* disowned -> kernel */ \
1171 : (object)->vo_owner)) /* explicit owner */ \
1172
1173extern void vm_object_ledger_tag_ledgers(
d9a64523
A		 1174 vm_object_t object,
1175 int *ledger_idx_volatile,
1176 int *ledger_idx_nonvolatile,
1177 int *ledger_idx_volatile_compressed,
1178 int *ledger_idx_nonvolatile_compressed,
1179 boolean_t *do_footprint);
1180extern kern_return_t vm_object_ownership_change(
1181 vm_object_t object,
cb323159
A		 1182 int new_ledger_tag,
1183 task_t new_owner,
1184 int new_ledger_flags,
d9a64523
A		 1185 boolean_t task_objq_locked);
1186
f427ee49
A		 1187// LP64todo: all the current tools are 32bit, obviously never worked for 64b
1188// so probably should be a real 32b ID vs. ptr.
1189// Current users just check for equality
1190#define VM_OBJECT_ID(o) ((uint32_t)(uintptr_t)VM_KERNEL_ADDRPERM((o)))
1191
0a7de745 1192#endif /* _VM_VM_OBJECT_H_ */
mirror of XNU