]> git.saurik.com Git - apple/xnu.git/blame - osfmk/vm/vm_object.h
projects / apple / xnu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
xnu-4903.241.1.tar.gz
[apple/xnu.git] / osfmk / vm / vm_object.h
CommitLineData
1c79356b 1/*
2d21ac55 2 * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
1c79356b 3 *
2d21ac55 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
1c79356b 5 *
2d21ac55
A		 6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
8f6c56a5 14 *
2d21ac55
A		 15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
8f6c56a5
A		 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
2d21ac55
A		 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
8f6c56a5 25 *
2d21ac55 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
1c79356b
A		 27 */
28/*
29 * @OSF_COPYRIGHT@
30 */
31/*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56/*
57 */
58/*
59 * File: vm_object.h
60 * Author: Avadis Tevanian, Jr., Michael Wayne Young
61 * Date: 1985
62 *
63 * Virtual memory object module definitions.
64 */
65
66#ifndef _VM_VM_OBJECT_H_
67#define _VM_VM_OBJECT_H_
68
6d2010ae
A		 69#include <debug.h>
70#include <mach_assert.h>
1c79356b
A		 71#include <mach_pagemap.h>
72#include <task_swapper.h>
73
74#include <mach/kern_return.h>
75#include <mach/boolean.h>
76#include <mach/memory_object_types.h>
77#include <mach/port.h>
78#include <mach/vm_prot.h>
91447636 79#include <mach/vm_param.h>
1c79356b
A		 80#include <mach/machine/vm_types.h>
81#include <kern/queue.h>
2d21ac55 82#include <kern/locks.h>
1c79356b 83#include <kern/assert.h>
0b4e3aa0 84#include <kern/misc_protos.h>
1c79356b
A		 85#include <kern/macro_help.h>
86#include <ipc/ipc_types.h>
87#include <vm/pmap.h>
1c79356b 88
1c79356b 89#include <vm/vm_external.h>
1c79356b 90
b0d623f7 91#include <vm/vm_options.h>
39037602 92#include <vm/vm_page.h>
b0d623f7 93
fe8ab488
A		 94#if VM_OBJECT_TRACKING
95#include <libkern/OSDebug.h>
96#include <kern/btlog.h>
97extern void vm_object_tracking_init(void);
98extern boolean_t vm_object_tracking_inited;
99extern btlog_t *vm_object_tracking_btlog;
39037602 100#define VM_OBJECT_TRACKING_NUM_RECORDS 50000
fe8ab488
A		 101#define VM_OBJECT_TRACKING_BTDEPTH 7
102#define VM_OBJECT_TRACKING_OP_CREATED 1
103#define VM_OBJECT_TRACKING_OP_MODIFIED 2
104#define VM_OBJECT_TRACKING_OP_TRUESHARE 3
105#endif /* VM_OBJECT_TRACKING */
106
91447636 107struct vm_page;
1c79356b
A		 108
109/*
110 * Types defined:
111 *
112 * vm_object_t Virtual memory object.
2d21ac55 113 * vm_object_fault_info_t Used to determine cluster size.
1c79356b
A		 114 */
115
2d21ac55
A		 116struct vm_object_fault_info {
117 int interruptible;
118 uint32_t user_tag;
119 vm_size_t cluster_size;
120 vm_behavior_t behavior;
121 vm_map_offset_t lo_offset;
122 vm_map_offset_t hi_offset;
6d2010ae
A		 123 unsigned int
124 /* boolean_t */ no_cache:1,
125 /* boolean_t */ stealth:1,
126 /* boolean_t */ io_sync:1,
127 /* boolean_t */ cs_bypass:1,
d9a64523 128 /* boolean_t */ pmap_cs_associated:1,
6d2010ae 129 /* boolean_t */ mark_zf_absent:1,
316670eb 130 /* boolean_t */ batch_pmap_op:1,
d9a64523 131 __vm_object_fault_info_unused_bits:25;
fe8ab488 132 int pmap_options;
2d21ac55
A		 133};
134
135
39236c6e
A		 136#define vo_size vo_un1.vou_size
137#define vo_cache_pages_to_scan vo_un1.vou_cache_pages_to_scan
138#define vo_shadow_offset vo_un2.vou_shadow_offset
139#define vo_cache_ts vo_un2.vou_cache_ts
d9a64523 140#define vo_owner vo_un2.vou_owner
2d21ac55 141
1c79356b 142struct vm_object {
39037602
A		 143 /*
144 * on 64 bit systems we pack the pointers hung off the memq.
145 * those pointers have to be able to point back to the memq.
146 * the packed pointers are required to be on a 64 byte boundary
147 * which means 2 things for the vm_object... (1) the memq
148 * struct has to be the first element of the structure so that
149 * we can control it's alignment... (2) the vm_object must be
150 * aligned on a 64 byte boundary... for static vm_object's
151 * this is accomplished via the 'aligned' attribute... for
152 * vm_object's in the zone pool, this is accomplished by
153 * rounding the size of the vm_object element to the nearest
154 * 64 byte size before creating the zone.
155 */
156 vm_page_queue_head_t memq; /* Resident memory - must be first */
2d21ac55 157 lck_rw_t Lock; /* Synchronization */
1c79356b 158
39037602
A		 159#if DEVELOPMENT || DEBUG
160 thread_t Lock_owner;
161#endif
6d2010ae
A		 162 union {
163 vm_object_size_t vou_size; /* Object size (only valid if internal) */
164 int vou_cache_pages_to_scan; /* pages yet to be visited in an
165 * external object in cache
166 */
167 } vo_un1;
168
91447636 169 struct vm_page *memq_hint;
1c79356b 170 int ref_count; /* Number of references */
1c79356b
A		 171 unsigned int resident_page_count;
172 /* number of resident pages */
d9a64523 173 unsigned int wired_page_count; /* number of wired pages
5ba3f43e 174 use VM_OBJECT_WIRED_PAGE_UPDATE macros to update */
b0d623f7 175 unsigned int reusable_page_count;
1c79356b
A		 176
177 struct vm_object *copy; /* Object that should receive
178 * a copy of my changed pages,
179 * for copy_delay, or just the
180 * temporary object that
181 * shadows this object, for
182 * copy_call.
183 */
184 struct vm_object *shadow; /* My shadow */
6d2010ae
A		 185
186 union {
187 vm_object_offset_t vou_shadow_offset; /* Offset into shadow */
39236c6e
A		 188 clock_sec_t vou_cache_ts; /* age of an external object
189 * present in cache
190 */
d9a64523
A		 191 task_t vou_owner; /* If the object is purgeable
192 * or has a "ledger_tag", this
193 * is the task that owns it.
194 */
6d2010ae 195 } vo_un2;
1c79356b 196
0b4e3aa0 197 memory_object_t pager; /* Where to get data */
1c79356b 198 vm_object_offset_t paging_offset; /* Offset into memory object */
91447636 199 memory_object_control_t pager_control; /* Where data comes back */
1c79356b
A		 200
201 memory_object_copy_strategy_t
202 copy_strategy; /* How to handle data copy */
203
fe8ab488
A		 204#if __LP64__
205 /*
206 * Some user processes (mostly VirtualMachine software) take a large
207 * number of UPLs (via IOMemoryDescriptors) to wire pages in large
208 * VM objects and overflow the 16-bit "activity_in_progress" counter.
209 * Since we never enforced any limit there, let's give them 32 bits
210 * for backwards compatibility's sake.
211 */
212 unsigned int paging_in_progress:16,
213 __object1_unused_bits:16;
214 unsigned int activity_in_progress;
215#else /* __LP64__ */
216 /*
217 * On 32-bit platforms, enlarging "activity_in_progress" would increase
218 * the size of "struct vm_object". Since we don't know of any actual
219 * overflow of these counters on these platforms, let's keep the
220 * counters as 16-bit integers.
221 */
222 unsigned short paging_in_progress;
223 unsigned short activity_in_progress;
224#endif /* __LP64__ */
1c79356b
A		 225 /* The memory object ports are
226 * being used (e.g., for pagein
227 * or pageout) -- don't change
228 * any of these fields (i.e.,
229 * don't collapse, destroy or
230 * terminate)
231 */
b0d623f7 232
1c79356b
A		 233 unsigned int
234 /* boolean_t array */ all_wanted:11, /* Bit array of "want to be
235 * awakened" notations. See
236 * VM_OBJECT_EVENT_* items
237 * below */
238 /* boolean_t */ pager_created:1, /* Has pager been created? */
239 /* boolean_t */ pager_initialized:1, /* Are fields ready to use? */
240 /* boolean_t */ pager_ready:1, /* Will pager take requests? */
241
242 /* boolean_t */ pager_trusted:1,/* The pager for this object
243 * is trusted. This is true for
244 * all internal objects (backed
245 * by the default pager)
246 */
247 /* boolean_t */ can_persist:1, /* The kernel may keep the data
248 * for this object (and rights
249 * to the memory object) after
250 * all address map references
251 * are deallocated?
252 */
253 /* boolean_t */ internal:1, /* Created by the kernel (and
254 * therefore, managed by the
255 * default memory manger)
256 */
1c79356b
A		 257 /* boolean_t */ private:1, /* magic device_pager object,
258 * holds private pages only */
259 /* boolean_t */ pageout:1, /* pageout object. contains
260 * private pages that refer to
261 * a real memory object. */
262 /* boolean_t */ alive:1, /* Not yet terminated */
263
91447636 264 /* boolean_t */ purgable:2, /* Purgable state. See
2d21ac55 265 * VM_PURGABLE_*
1c79356b 266 */
5ba3f43e 267 /* boolean_t */ purgeable_only_by_kernel:1,
39236c6e
A		 268 /* boolean_t */ purgeable_when_ripe:1, /* Purgeable when a token
269 * becomes ripe.
270 */
1c79356b 271 /* boolean_t */ shadowed:1, /* Shadow may exist */
1c79356b
A		 272 /* boolean_t */ true_share:1,
273 /* This object is mapped
274 * in more than one place
275 * and hence cannot be
276 * coalesced */
277 /* boolean_t */ terminating:1,
278 /* Allows vm_object_lookup
279 * and vm_object_deallocate
280 * to special case their
281 * behavior when they are
282 * called as a result of
283 * page cleaning during
284 * object termination
285 */
286 /* boolean_t */ named:1, /* An enforces an internal
287 * naming convention, by
288 * calling the right routines
289 * for allocation and
290 * destruction, UBC references
291 * against the vm_object are
292 * checked.
293 */
294 /* boolean_t */ shadow_severed:1,
295 /* When a permanent object
296 * backing a COW goes away
297 * unexpectedly. This bit
298 * allows vm_fault to return
299 * an error rather than a
300 * zero filled page.
301 */
0b4e3aa0 302 /* boolean_t */ phys_contiguous:1,
1c79356b
A		 303 /* Memory is wired and
304 * guaranteed physically
305 * contiguous. However
306 * it is not device memory
307 * and obeys normal virtual
308 * memory rules w.r.t pmap
309 * access bits.
310 */
5ba3f43e 311 /* boolean_t */ nophyscache:1,
0b4e3aa0
A		 312 /* When mapped at the
313 * pmap level, don't allow
314 * primary caching. (for
315 * I/O)
316 */
5ba3f43e 317 /* boolean_t */ _object5_unused_bits:1;
1c79356b
A		 318
319 queue_chain_t cached_list; /* Attachment point for the
320 * list of objects cached as a
321 * result of their can_persist
322 * value
323 */
2d21ac55
A		 324 /*
325 * the following fields are not protected by any locks
326 * they are updated via atomic compare and swap
327 */
1c79356b 328 vm_object_offset_t last_alloc; /* last allocation offset */
2d21ac55
A		 329 int sequential; /* sequential access size */
330
331 uint32_t pages_created;
332 uint32_t pages_used;
55e303ae 333 vm_offset_t cow_hint; /* last page present in */
0b4e3aa0 334 /* shadow but not in object */
2d21ac55
A		 335 /* hold object lock when altering */
336 unsigned int
337 wimg_bits:8, /* cache WIMG bits */
338 code_signed:1, /* pages are signed and should be
339 validated; the signatures are stored
340 with the pager */
b0d623f7 341 transposed:1, /* object was transposed with another */
593a1d5f 342 mapping_in_progress:1, /* pager being mapped/unmapped */
fe8ab488 343 phantom_isssd:1,
b0d623f7
A		 344 volatile_empty:1,
345 volatile_fault:1,
346 all_reusable:1,
347 blocked_access:1,
6d2010ae 348 set_cache_attr:1,
d9a64523 349 object_is_shared_cache:1,
39236c6e
A		 350 purgeable_queue_type:2,
351 purgeable_queue_group:3,
fe8ab488 352 io_tracking:1,
5ba3f43e 353 no_tag_update:1, /* */
39037602
A		 354#if CONFIG_SECLUDED_MEMORY
355 eligible_for_secluded:1,
356 can_grab_secluded:1,
357#else /* CONFIG_SECLUDED_MEMORY */
358 __object3_unused_bits:2,
359#endif /* CONFIG_SECLUDED_MEMORY */
d9a64523
A		 360#if VM_OBJECT_ACCESS_TRACKING
361 access_tracking:1,
362#else /* VM_OBJECT_ACCESS_TRACKING */
363 __unused_access_tracking:1,
364#endif /* VM_OBJECT_ACCESS_TRACKING */
365 vo_ledger_tag:2,
366 __object2_unused_bits:2; /* for expansion */
367
368#if VM_OBJECT_ACCESS_TRACKING
369 uint32_t access_tracking_reads;
370 uint32_t access_tracking_writes;
371#endif /* VM_OBJECT_ACCESS_TRACKING */
6d2010ae 372
3e170ce0
A		 373 uint8_t scan_collisions;
374 vm_tag_t wire_tag;
375 uint8_t __object4_unused_bits[2];
376
fe8ab488
A		 377#if CONFIG_PHANTOM_CACHE
378 uint32_t phantom_object_id;
379#endif
380#if CONFIG_IOSCHED || UPL_DEBUG
1c79356b 381 queue_head_t uplq; /* List of outstanding upls */
fe8ab488 382#endif
2d21ac55 383
0c530ab8
A		 384#ifdef VM_PIP_DEBUG
385/*
386 * Keep track of the stack traces for the first holders
387 * of a "paging_in_progress" reference for this VM object.
388 */
389#define VM_PIP_DEBUG_STACK_FRAMES 25 /* depth of each stack trace */
390#define VM_PIP_DEBUG_MAX_REFS 10 /* track that many references */
391 struct __pip_backtrace {
392 void *pip_retaddr[VM_PIP_DEBUG_STACK_FRAMES];
393 } pip_holders[VM_PIP_DEBUG_MAX_REFS];
394#endif /* VM_PIP_DEBUG */
2d21ac55 395
6d2010ae 396 queue_chain_t objq; /* object queue - currently used for purgable queues */
a39ff7e2 397 queue_chain_t task_objq; /* objects owned by task - protected by task lock */
fe8ab488 398
d9a64523
A		 399#if !VM_TAG_ACTIVE_UPDATE
400 queue_chain_t wired_objq;
401#endif /* !VM_TAG_ACTIVE_UPDATE */
402
fe8ab488
A		 403#if DEBUG
404 void *purgeable_owner_bt[16];
405 task_t vo_purgeable_volatilizer; /* who made it volatile? */
406 void *purgeable_volatilizer_bt[16];
407#endif /* DEBUG */
1c79356b
A		 408};
409
d9a64523
A		 410/* values for object->vo_ledger_tag */
411#define VM_OBJECT_LEDGER_TAG_NONE 0
412#define VM_OBJECT_LEDGER_TAG_NETWORK 1
413#define VM_OBJECT_LEDGER_TAG_MEDIA 2
414#define VM_OBJECT_LEDGER_TAG_RESERVED 3
415
b0d623f7
A		 416#define VM_OBJECT_PURGEABLE_FAULT_ERROR(object) \
417 ((object)->volatile_fault && \
418 ((object)->purgable == VM_PURGABLE_VOLATILE || \
419 (object)->purgable == VM_PURGABLE_EMPTY))
420
d9a64523
A		 421#if VM_OBJECT_ACCESS_TRACKING
422extern uint64_t vm_object_access_tracking_reads;
423extern uint64_t vm_object_access_tracking_writes;
424extern void vm_object_access_tracking(vm_object_t object,
425 int *access_tracking,
426 uint32_t *access_tracking_reads,
427 uint32_t *acess_tracking_writes);
428#endif /* VM_OBJECT_ACCESS_TRACKING */
429
39236c6e 430extern
1c79356b
A		 431vm_object_t kernel_object; /* the single kernel object */
432
39236c6e
A		 433extern
434vm_object_t compressor_object; /* the single compressor object */
435
436extern
91447636 437unsigned int vm_object_absent_max; /* maximum number of absent pages
1c79356b
A		 438 at a time for each object */
439
440# define VM_MSYNC_INITIALIZED 0
441# define VM_MSYNC_SYNCHRONIZING 1
442# define VM_MSYNC_DONE 2
443
b0d623f7
A		 444
445extern lck_grp_t vm_map_lck_grp;
446extern lck_attr_t vm_map_lck_attr;
447
5ba3f43e
A		 448#ifndef VM_TAG_ACTIVE_UPDATE
449#error VM_TAG_ACTIVE_UPDATE
450#endif
1c79356b 451
d9a64523
A		 452#if VM_TAG_ACTIVE_UPDATE
453#define VM_OBJECT_WIRED_ENQUEUE(object) panic("VM_OBJECT_WIRED_ENQUEUE")
454#define VM_OBJECT_WIRED_DEQUEUE(object) panic("VM_OBJECT_WIRED_DEQUEUE")
455#else /* VM_TAG_ACTIVE_UPDATE */
456#define VM_OBJECT_WIRED_ENQUEUE(object) \
457 MACRO_BEGIN \
458 lck_spin_lock(&vm_objects_wired_lock); \
459 assert(!(object)->wired_objq.next); \
460 assert(!(object)->wired_objq.prev); \
461 queue_enter(&vm_objects_wired, (object), \
462 vm_object_t, wired_objq); \
463 lck_spin_unlock(&vm_objects_wired_lock); \
464 MACRO_END
465#define VM_OBJECT_WIRED_DEQUEUE(object) \
466 MACRO_BEGIN \
467 if ((object)->wired_objq.next) { \
468 lck_spin_lock(&vm_objects_wired_lock); \
469 queue_remove(&vm_objects_wired, (object), \
470 vm_object_t, wired_objq); \
471 lck_spin_unlock(&vm_objects_wired_lock); \
472 } \
473 MACRO_END
474#endif /* VM_TAG_ACTIVE_UPDATE */
475
5ba3f43e 476#define VM_OBJECT_WIRED(object, tag) \
3e170ce0 477 MACRO_BEGIN \
5ba3f43e
A		 478 assert(VM_KERN_MEMORY_NONE != (tag)); \
479 assert(VM_KERN_MEMORY_NONE == (object)->wire_tag); \
480 (object)->wire_tag = (tag); \
d9a64523
A		 481 if (!VM_TAG_ACTIVE_UPDATE) { \
482 VM_OBJECT_WIRED_ENQUEUE((object)); \
3e170ce0
A		 483 } \
484 MACRO_END
485
5ba3f43e
A		 486#define VM_OBJECT_UNWIRED(object) \
487 MACRO_BEGIN \
d9a64523
A		 488 if (!VM_TAG_ACTIVE_UPDATE) { \
489 VM_OBJECT_WIRED_DEQUEUE((object)); \
5ba3f43e
A		 490 } \
491 if (VM_KERN_MEMORY_NONE != (object)->wire_tag) { \
492 vm_tag_update_size((object)->wire_tag, -ptoa_64((object)->wired_page_count)); \
493 (object)->wire_tag = VM_KERN_MEMORY_NONE; \
494 } \
495 MACRO_END
496
497// These two macros start & end a C block
498#define VM_OBJECT_WIRED_PAGE_UPDATE_START(object) \
499 MACRO_BEGIN \
500 { \
501 int64_t __wireddelta = 0; vm_tag_t __waswired = (object)->wire_tag;
502
503#define VM_OBJECT_WIRED_PAGE_UPDATE_END(object, tag) \
504 if (__wireddelta) { \
505 boolean_t __overflow __assert_only = \
506 os_add_overflow((object)->wired_page_count, __wireddelta, \
d9a64523 507 &(object)->wired_page_count); \
5ba3f43e
A		 508 assert(!__overflow); \
509 if (!(object)->pageout && !(object)->no_tag_update) { \
510 if (__wireddelta > 0) { \
511 assert (VM_KERN_MEMORY_NONE != (tag)); \
512 if (VM_KERN_MEMORY_NONE == __waswired) { \
513 VM_OBJECT_WIRED((object), (tag)); \
514 } \
515 vm_tag_update_size((object)->wire_tag, ptoa_64(__wireddelta)); \
516 } else if (VM_KERN_MEMORY_NONE != __waswired) { \
517 assert (VM_KERN_MEMORY_NONE != (object)->wire_tag); \
518 vm_tag_update_size((object)->wire_tag, ptoa_64(__wireddelta)); \
519 if (!(object)->wired_page_count) { \
520 VM_OBJECT_UNWIRED((object)); \
521 } \
522 } \
523 } \
524 } \
525 } \
3e170ce0
A		 526 MACRO_END
527
5ba3f43e
A		 528#define VM_OBJECT_WIRED_PAGE_COUNT(object, delta) \
529 __wireddelta += delta; \
530
531#define VM_OBJECT_WIRED_PAGE_ADD(object, m) \
d9a64523 532 if (!(m)->vmp_private && !(m)->vmp_fictitious) __wireddelta++;
5ba3f43e
A		 533
534#define VM_OBJECT_WIRED_PAGE_REMOVE(object, m) \
d9a64523 535 if (!(m)->vmp_private && !(m)->vmp_fictitious) __wireddelta--;
5ba3f43e
A		 536
537
3e170ce0 538
39037602
A		 539#define OBJECT_LOCK_SHARED 0
540#define OBJECT_LOCK_EXCLUSIVE 1
541
542extern lck_grp_t vm_object_lck_grp;
543extern lck_grp_attr_t vm_object_lck_grp_attr;
544extern lck_attr_t vm_object_lck_attr;
545extern lck_attr_t kernel_object_lck_attr;
546extern lck_attr_t compressor_object_lck_attr;
547
548extern vm_object_t vm_pageout_scan_wants_object;
549
550extern void vm_object_lock(vm_object_t);
551extern boolean_t vm_object_lock_try(vm_object_t);
552extern boolean_t _vm_object_lock_try(vm_object_t);
553extern boolean_t vm_object_lock_avoid(vm_object_t);
554extern void vm_object_lock_shared(vm_object_t);
5ba3f43e 555extern boolean_t vm_object_lock_yield_shared(vm_object_t);
39037602
A		 556extern boolean_t vm_object_lock_try_shared(vm_object_t);
557extern void vm_object_unlock(vm_object_t);
558extern boolean_t vm_object_lock_upgrade(vm_object_t);
559
560/*
561 * Object locking macros
562 */
563
564#define vm_object_lock_init(object) \
565 lck_rw_init(&(object)->Lock, &vm_object_lck_grp, \
566 (((object) == kernel_object || \
567 (object) == vm_submap_object) ? \
568 &kernel_object_lck_attr : \
569 (((object) == compressor_object) ? \
570 &compressor_object_lck_attr : \
571 &vm_object_lck_attr)))
572#define vm_object_lock_destroy(object) lck_rw_destroy(&(object)->Lock, &vm_object_lck_grp)
573
574#define vm_object_lock_try_scan(object) _vm_object_lock_try(object)
575
576/*
577 * CAUTION: the following vm_object_lock_assert_held*() macros merely
578 * check if anyone is holding the lock, but the holder may not necessarily
579 * be the caller...
580 */
581#if MACH_ASSERT || DEBUG
582#define vm_object_lock_assert_held(object) \
583 lck_rw_assert(&(object)->Lock, LCK_RW_ASSERT_HELD)
584#define vm_object_lock_assert_shared(object) \
585 lck_rw_assert(&(object)->Lock, LCK_RW_ASSERT_SHARED)
586#define vm_object_lock_assert_exclusive(object) \
587 lck_rw_assert(&(object)->Lock, LCK_RW_ASSERT_EXCLUSIVE)
588#define vm_object_lock_assert_notheld(object) \
589 lck_rw_assert(&(object)->Lock, LCK_RW_ASSERT_NOTHELD)
590#else /* MACH_ASSERT || DEBUG */
591#define vm_object_lock_assert_held(object)
592#define vm_object_lock_assert_shared(object)
593#define vm_object_lock_assert_exclusive(object)
594#define vm_object_lock_assert_notheld(object)
595#endif /* MACH_ASSERT || DEBUG */
596
597
1c79356b
A		 598/*
599 * Declare procedures that operate on VM objects.
600 */
601
39236c6e 602__private_extern__ void vm_object_bootstrap(void);
1c79356b 603
0b4e3aa0 604__private_extern__ void vm_object_init(void);
1c79356b 605
2d21ac55
A		 606__private_extern__ void vm_object_init_lck_grp(void);
607
8f6c56a5
A		 608__private_extern__ void vm_object_reaper_init(void);
609
3e170ce0 610__private_extern__ vm_object_t vm_object_allocate(vm_object_size_t size);
1c79356b 611
91447636
A		 612__private_extern__ void _vm_object_allocate(vm_object_size_t size,
613 vm_object_t object);
614
0b4e3aa0
A		 615#if TASK_SWAPPER
616
617__private_extern__ void vm_object_res_reference(
618 vm_object_t object);
619__private_extern__ void vm_object_res_deallocate(
620 vm_object_t object);
621#define VM_OBJ_RES_INCR(object) (object)->res_count++
622#define VM_OBJ_RES_DECR(object) (object)->res_count--
623
624#else /* TASK_SWAPPER */
625
626#define VM_OBJ_RES_INCR(object)
627#define VM_OBJ_RES_DECR(object)
628#define vm_object_res_reference(object)
629#define vm_object_res_deallocate(object)
630
631#endif /* TASK_SWAPPER */
632
633#define vm_object_reference_locked(object) \
2d21ac55
A		 634 MACRO_BEGIN \
635 vm_object_t RLObject = (object); \
636 vm_object_lock_assert_exclusive(object); \
637 assert((RLObject)->ref_count > 0); \
638 (RLObject)->ref_count++; \
639 assert((RLObject)->ref_count > 1); \
640 vm_object_res_reference(RLObject); \
641 MACRO_END
642
643
644#define vm_object_reference_shared(object) \
645 MACRO_BEGIN \
646 vm_object_t RLObject = (object); \
647 vm_object_lock_assert_shared(object); \
648 assert((RLObject)->ref_count > 0); \
b0d623f7 649 OSAddAtomic(1, &(RLObject)->ref_count); \
316670eb 650 assert((RLObject)->ref_count > 0); \
2d21ac55
A		 651 /* XXX we would need an atomic version of the following ... */ \
652 vm_object_res_reference(RLObject); \
653 MACRO_END
0b4e3aa0
A		 654
655
0b4e3aa0 656__private_extern__ void vm_object_reference(
1c79356b 657 vm_object_t object);
0b4e3aa0 658
91447636 659#if !MACH_ASSERT
0b4e3aa0 660
1c79356b
A		 661#define vm_object_reference(object) \
662MACRO_BEGIN \
0b4e3aa0
A		 663 vm_object_t RObject = (object); \
664 if (RObject) { \
b0d623f7
A		 665 vm_object_lock_shared(RObject); \
666 vm_object_reference_shared(RObject); \
0b4e3aa0 667 vm_object_unlock(RObject); \
1c79356b
A		 668 } \
669MACRO_END
0b4e3aa0 670
1c79356b
A		 671#endif /* MACH_ASSERT */
672
0b4e3aa0 673__private_extern__ void vm_object_deallocate(
1c79356b
A		 674 vm_object_t object);
675
0b4e3aa0
A		 676__private_extern__ kern_return_t vm_object_release_name(
677 vm_object_t object,
678 int flags);
679
680__private_extern__ void vm_object_pmap_protect(
1c79356b
A		 681 vm_object_t object,
682 vm_object_offset_t offset,
91447636 683 vm_object_size_t size,
1c79356b 684 pmap_t pmap,
91447636 685 vm_map_offset_t pmap_start,
1c79356b
A		 686 vm_prot_t prot);
687
39236c6e
A		 688__private_extern__ void vm_object_pmap_protect_options(
689 vm_object_t object,
690 vm_object_offset_t offset,
691 vm_object_size_t size,
692 pmap_t pmap,
693 vm_map_offset_t pmap_start,
694 vm_prot_t prot,
695 int options);
696
0b4e3aa0 697__private_extern__ void vm_object_page_remove(
1c79356b
A		 698 vm_object_t object,
699 vm_object_offset_t start,
700 vm_object_offset_t end);
701
0b4e3aa0
A		 702__private_extern__ void vm_object_deactivate_pages(
703 vm_object_t object,
704 vm_object_offset_t offset,
705 vm_object_size_t size,
b0d623f7 706 boolean_t kill_page,
3e170ce0
A		 707 boolean_t reusable_page,
708 struct pmap *pmap,
709 vm_map_offset_t pmap_offset);
b0d623f7
A		 710
711__private_extern__ void vm_object_reuse_pages(
712 vm_object_t object,
713 vm_object_offset_t start_offset,
714 vm_object_offset_t end_offset,
715 boolean_t allow_partial_reuse);
0b4e3aa0 716
a39ff7e2 717__private_extern__ uint64_t vm_object_purge(
fe8ab488
A		 718 vm_object_t object,
719 int flags);
91447636
A		 720
721__private_extern__ kern_return_t vm_object_purgable_control(
722 vm_object_t object,
723 vm_purgable_t control,
724 int *state);
725
39236c6e
A		 726__private_extern__ kern_return_t vm_object_get_page_counts(
727 vm_object_t object,
728 vm_object_offset_t offset,
729 vm_object_size_t size,
730 unsigned int *resident_page_count,
731 unsigned int *dirty_page_count);
732
0b4e3aa0 733__private_extern__ boolean_t vm_object_coalesce(
1c79356b
A		 734 vm_object_t prev_object,
735 vm_object_t next_object,
736 vm_object_offset_t prev_offset,
737 vm_object_offset_t next_offset,
738 vm_object_size_t prev_size,
739 vm_object_size_t next_size);
740
0b4e3aa0 741__private_extern__ boolean_t vm_object_shadow(
1c79356b
A		 742 vm_object_t *object,
743 vm_object_offset_t *offset,
744 vm_object_size_t length);
745
0b4e3aa0 746__private_extern__ void vm_object_collapse(
55e303ae 747 vm_object_t object,
0c530ab8
A		 748 vm_object_offset_t offset,
749 boolean_t can_bypass);
1c79356b 750
0b4e3aa0 751__private_extern__ boolean_t vm_object_copy_quickly(
1c79356b
A		 752 vm_object_t *_object,
753 vm_object_offset_t src_offset,
754 vm_object_size_t size,
755 boolean_t *_src_needs_copy,
756 boolean_t *_dst_needs_copy);
757
0b4e3aa0 758__private_extern__ kern_return_t vm_object_copy_strategically(
1c79356b
A		 759 vm_object_t src_object,
760 vm_object_offset_t src_offset,
761 vm_object_size_t size,
762 vm_object_t *dst_object,
763 vm_object_offset_t *dst_offset,
764 boolean_t *dst_needs_copy);
765
0b4e3aa0 766__private_extern__ kern_return_t vm_object_copy_slowly(
1c79356b
A		 767 vm_object_t src_object,
768 vm_object_offset_t src_offset,
769 vm_object_size_t size,
b0d623f7 770 boolean_t interruptible,
1c79356b
A		 771 vm_object_t *_result_object);
772
0b4e3aa0
A		 773__private_extern__ vm_object_t vm_object_copy_delayed(
774 vm_object_t src_object,
775 vm_object_offset_t src_offset,
2d21ac55
A		 776 vm_object_size_t size,
777 boolean_t src_object_shared);
0b4e3aa0 778
1c79356b 779
1c79356b 780
0b4e3aa0
A		 781__private_extern__ kern_return_t vm_object_destroy(
782 vm_object_t object,
783 kern_return_t reason);
1c79356b 784
0b4e3aa0
A		 785__private_extern__ void vm_object_pager_create(
786 vm_object_t object);
787
39236c6e
A		 788__private_extern__ void vm_object_compressor_pager_create(
789 vm_object_t object);
790
0b4e3aa0 791__private_extern__ void vm_object_page_map(
1c79356b
A		 792 vm_object_t object,
793 vm_object_offset_t offset,
794 vm_object_size_t size,
795 vm_object_offset_t (*map_fn)
796 (void *, vm_object_offset_t),
797 void *map_fn_data);
798
0b4e3aa0
A		 799__private_extern__ kern_return_t vm_object_upl_request(
800 vm_object_t object,
801 vm_object_offset_t offset,
91447636 802 upl_size_t size,
0b4e3aa0
A		 803 upl_t *upl,
804 upl_page_info_t *page_info,
805 unsigned int *count,
5ba3f43e
A		 806 upl_control_flags_t flags,
807 vm_tag_t tag);
0b4e3aa0 808
91447636
A		 809__private_extern__ kern_return_t vm_object_transpose(
810 vm_object_t object1,
811 vm_object_t object2,
812 vm_object_size_t transpose_size);
813
0b4e3aa0
A		 814__private_extern__ boolean_t vm_object_sync(
815 vm_object_t object,
816 vm_object_offset_t offset,
91447636 817 vm_object_size_t size,
0b4e3aa0 818 boolean_t should_flush,
91447636
A		 819 boolean_t should_return,
820 boolean_t should_iosync);
1c79356b 821
0b4e3aa0
A		 822__private_extern__ kern_return_t vm_object_update(
823 vm_object_t object,
824 vm_object_offset_t offset,
91447636
A		 825 vm_object_size_t size,
826 vm_object_offset_t *error_offset,
827 int *io_errno,
0b4e3aa0
A		 828 memory_object_return_t should_return,
829 int flags,
830 vm_prot_t prot);
1c79356b 831
0b4e3aa0
A		 832__private_extern__ kern_return_t vm_object_lock_request(
833 vm_object_t object,
834 vm_object_offset_t offset,
835 vm_object_size_t size,
836 memory_object_return_t should_return,
837 int flags,
838 vm_prot_t prot);
1c79356b 839
1c79356b 840
1c79356b 841
5ba3f43e 842__private_extern__ vm_object_t vm_object_memory_object_associate(
0b4e3aa0 843 memory_object_t pager,
5ba3f43e 844 vm_object_t object,
1c79356b 845 vm_object_size_t size,
1c79356b
A		 846 boolean_t check_named);
847
848
2d21ac55
A		 849__private_extern__ void vm_object_cluster_size(
850 vm_object_t object,
851 vm_object_offset_t *start,
852 vm_size_t *length,
b0d623f7
A		 853 vm_object_fault_info_t fault_info,
854 uint32_t *io_streaming);
91447636
A		 855
856__private_extern__ kern_return_t vm_object_populate_with_private(
857 vm_object_t object,
858 vm_object_offset_t offset,
859 ppnum_t phys_page,
860 vm_size_t size);
861
6d2010ae
A		 862__private_extern__ void vm_object_change_wimg_mode(
863 vm_object_t object,
864 unsigned int wimg_mode);
865
0c530ab8 866extern kern_return_t adjust_vm_object_cache(
91447636
A		 867 vm_size_t oval,
868 vm_size_t nval);
869
0c530ab8
A		 870extern kern_return_t vm_object_page_op(
871 vm_object_t object,
872 vm_object_offset_t offset,
873 int ops,
874 ppnum_t *phys_entry,
875 int *flags);
876
877extern kern_return_t vm_object_range_op(
878 vm_object_t object,
879 vm_object_offset_t offset_beg,
880 vm_object_offset_t offset_end,
881 int ops,
b0d623f7
A		 882 uint32_t *range);
883
884
885__private_extern__ void vm_object_reap_pages(
886 vm_object_t object,
887 int reap_type);
888#define REAP_REAP 0
889#define REAP_TERMINATE 1
890#define REAP_PURGEABLE 2
891#define REAP_DATA_FLUSH 3
892
6d2010ae 893#if CONFIG_FREEZE
6d2010ae 894
316670eb 895__private_extern__ void
3e170ce0 896vm_object_compressed_freezer_pageout(
316670eb 897 vm_object_t object);
6d2010ae 898
3e170ce0
A		 899__private_extern__ void
900vm_object_compressed_freezer_done(
901 void);
902
6d2010ae 903#endif /* CONFIG_FREEZE */
0c530ab8 904
3e170ce0
A		 905__private_extern__ void
906vm_object_pageout(
907 vm_object_t object);
908
fe8ab488
A		 909#if CONFIG_IOSCHED
910struct io_reprioritize_req {
911 uint64_t blkno;
912 uint32_t len;
913 int priority;
914 struct vnode *devvp;
915 queue_chain_t io_reprioritize_list;
916};
917typedef struct io_reprioritize_req *io_reprioritize_req_t;
918
919extern void vm_io_reprioritize_init(void);
920#endif
921
1c79356b
A		 922/*
923 * Event waiting handling
924 */
925
926#define VM_OBJECT_EVENT_INITIALIZED 0
927#define VM_OBJECT_EVENT_PAGER_READY 1
928#define VM_OBJECT_EVENT_PAGING_IN_PROGRESS 2
593a1d5f 929#define VM_OBJECT_EVENT_MAPPING_IN_PROGRESS 3
1c79356b
A		 930#define VM_OBJECT_EVENT_LOCK_IN_PROGRESS 4
931#define VM_OBJECT_EVENT_UNCACHING 5
932#define VM_OBJECT_EVENT_COPY_CALL 6
933#define VM_OBJECT_EVENT_CACHING 7
b0d623f7
A		 934#define VM_OBJECT_EVENT_UNBLOCKED 8
935#define VM_OBJECT_EVENT_PAGING_ONLY_IN_PROGRESS 9
1c79356b 936
39037602 937#define VM_OBJECT_EVENT_MAX 10 /* 11 bits in "all_wanted", so 0->10 */
1c79356b 938
39037602
A		 939static __inline__ wait_result_t
940vm_object_assert_wait(
941 vm_object_t object,
942 int event,
943 wait_interrupt_t interruptible)
944{
945 wait_result_t wr;
946
947 vm_object_lock_assert_exclusive(object);
948 assert(event >= 0 && event <= VM_OBJECT_EVENT_MAX);
949
950 object->all_wanted |= 1 << event;
951 wr = assert_wait((event_t)((vm_offset_t)object + event),
952 interruptible);
953 return wr;
954}
955
956static __inline__ wait_result_t
957vm_object_wait(
958 vm_object_t object,
959 int event,
960 wait_interrupt_t interruptible)
961{
962 wait_result_t wr;
963
964 vm_object_assert_wait(object, event, interruptible);
965 vm_object_unlock(object);
966 wr = thread_block(THREAD_CONTINUE_NULL);
967 return wr;
968}
969
970static __inline__ wait_result_t
971thread_sleep_vm_object(
972 vm_object_t object,
973 event_t event,
974 wait_interrupt_t interruptible)
975{
976 wait_result_t wr;
977
978#if DEVELOPMENT || DEBUG
979 if (object->Lock_owner != current_thread())
980 panic("thread_sleep_vm_object: now owner - %p\n", object);
981 object->Lock_owner = 0;
982#endif
983 wr = lck_rw_sleep(&object->Lock,
984 LCK_SLEEP_PROMOTED_PRI,
985 event,
986 interruptible);
987#if DEVELOPMENT || DEBUG
988 object->Lock_owner = current_thread();
989#endif
990 return wr;
991}
9bccf70c 992
39037602
A		 993static __inline__ wait_result_t
994vm_object_sleep(
995 vm_object_t object,
996 int event,
997 wait_interrupt_t interruptible)
998{
999 wait_result_t wr;
1000
1001 vm_object_lock_assert_exclusive(object);
1002 assert(event >= 0 && event <= VM_OBJECT_EVENT_MAX);
1003
1004 object->all_wanted |= 1 << event;
1005 wr = thread_sleep_vm_object(object,
1006 (event_t)((vm_offset_t)object + event),
1007 interruptible);
1008 return wr;
1009}
1010
1011static __inline__ void
1012vm_object_wakeup(
1013 vm_object_t object,
1014 int event)
1015{
1016 vm_object_lock_assert_exclusive(object);
1017 assert(event >= 0 && event <= VM_OBJECT_EVENT_MAX);
1018
1019 if (object->all_wanted & (1 << event))
1020 thread_wakeup((event_t)((vm_offset_t)object + event));
1021 object->all_wanted &= ~(1 << event);
1022}
1023
1024static __inline__ void
1025vm_object_set_wanted(
1026 vm_object_t object,
1027 int event)
1028{
1029 vm_object_lock_assert_exclusive(object);
1030 assert(event >= 0 && event <= VM_OBJECT_EVENT_MAX);
1c79356b 1031
39037602
A		 1032 object->all_wanted |= (1 << event);
1033}
1c79356b 1034
39037602
A		 1035static __inline__ int
1036vm_object_wanted(
1037 vm_object_t object,
1038 int event)
1039{
1040 vm_object_lock_assert_held(object);
1041 assert(event >= 0 && event <= VM_OBJECT_EVENT_MAX);
1c79356b 1042
39037602
A		 1043 return object->all_wanted & (1 << event);
1044}
1c79356b
A		 1045
1046/*
1047 * Routines implemented as macros
1048 */
0c530ab8 1049#ifdef VM_PIP_DEBUG
2d21ac55 1050#include <libkern/OSDebug.h>
0c530ab8
A		 1051#define VM_PIP_DEBUG_BEGIN(object) \
1052 MACRO_BEGIN \
b0d623f7
A		 1053 int pip = ((object)->paging_in_progress + \
1054 (object)->activity_in_progress); \
1055 if (pip < VM_PIP_DEBUG_MAX_REFS) { \
2d21ac55 1056 (void) OSBacktrace(&(object)->pip_holders[pip].pip_retaddr[0], \
0c530ab8
A		 1057 VM_PIP_DEBUG_STACK_FRAMES); \
1058 } \
1059 MACRO_END
1060#else /* VM_PIP_DEBUG */
1061#define VM_PIP_DEBUG_BEGIN(object)
1062#endif /* VM_PIP_DEBUG */
1c79356b 1063
b0d623f7
A		 1064#define vm_object_activity_begin(object) \
1065 MACRO_BEGIN \
1066 vm_object_lock_assert_exclusive((object)); \
b0d623f7
A		 1067 VM_PIP_DEBUG_BEGIN((object)); \
1068 (object)->activity_in_progress++; \
fe8ab488
A		 1069 if ((object)->activity_in_progress == 0) { \
1070 panic("vm_object_activity_begin(%p): overflow\n", (object));\
1071 } \
b0d623f7
A		 1072 MACRO_END
1073
1074#define vm_object_activity_end(object) \
1075 MACRO_BEGIN \
1076 vm_object_lock_assert_exclusive((object)); \
fe8ab488
A		 1077 if ((object)->activity_in_progress == 0) { \
1078 panic("vm_object_activity_end(%p): underflow\n", (object));\
1079 } \
b0d623f7
A		 1080 (object)->activity_in_progress--; \
1081 if ((object)->paging_in_progress == 0 && \
1082 (object)->activity_in_progress == 0) \
1083 vm_object_wakeup((object), \
1084 VM_OBJECT_EVENT_PAGING_IN_PROGRESS); \
1085 MACRO_END
1086
1087#define vm_object_paging_begin(object) \
1c79356b 1088 MACRO_BEGIN \
2d21ac55 1089 vm_object_lock_assert_exclusive((object)); \
0c530ab8 1090 VM_PIP_DEBUG_BEGIN((object)); \
1c79356b 1091 (object)->paging_in_progress++; \
fe8ab488
A		 1092 if ((object)->paging_in_progress == 0) { \
1093 panic("vm_object_paging_begin(%p): overflow\n", (object));\
1094 } \
1c79356b
A		 1095 MACRO_END
1096
b0d623f7 1097#define vm_object_paging_end(object) \
1c79356b 1098 MACRO_BEGIN \
2d21ac55 1099 vm_object_lock_assert_exclusive((object)); \
fe8ab488
A		 1100 if ((object)->paging_in_progress == 0) { \
1101 panic("vm_object_paging_end(%p): underflow\n", (object));\
1102 } \
b0d623f7
A		 1103 (object)->paging_in_progress--; \
1104 if ((object)->paging_in_progress == 0) { \
1105 vm_object_wakeup((object), \
1106 VM_OBJECT_EVENT_PAGING_ONLY_IN_PROGRESS); \
1107 if ((object)->activity_in_progress == 0) \
1108 vm_object_wakeup((object), \
1109 VM_OBJECT_EVENT_PAGING_IN_PROGRESS); \
1c79356b
A		 1110 } \
1111 MACRO_END
1112
1113#define vm_object_paging_wait(object, interruptible) \
1114 MACRO_BEGIN \
2d21ac55 1115 vm_object_lock_assert_exclusive((object)); \
b0d623f7
A		 1116 while ((object)->paging_in_progress != 0 || \
1117 (object)->activity_in_progress != 0) { \
9bccf70c
A		 1118 wait_result_t _wr; \
1119 \
1120 _wr = vm_object_sleep((object), \
1c79356b
A		 1121 VM_OBJECT_EVENT_PAGING_IN_PROGRESS, \
1122 (interruptible)); \
1c79356b 1123 \
9bccf70c 1124 /*XXX if ((interruptible) && (_wr != THREAD_AWAKENED))*/\
1c79356b
A		 1125 /*XXX break; */ \
1126 } \
1127 MACRO_END
1128
b0d623f7
A		 1129#define vm_object_paging_only_wait(object, interruptible) \
1130 MACRO_BEGIN \
1131 vm_object_lock_assert_exclusive((object)); \
1132 while ((object)->paging_in_progress != 0) { \
1133 wait_result_t _wr; \
1134 \
1135 _wr = vm_object_sleep((object), \
1136 VM_OBJECT_EVENT_PAGING_ONLY_IN_PROGRESS,\
1137 (interruptible)); \
1138 \
1139 /*XXX if ((interruptible) && (_wr != THREAD_AWAKENED))*/\
1140 /*XXX break; */ \
1141 } \
1142 MACRO_END
1143
1c79356b 1144
593a1d5f
A		 1145#define vm_object_mapping_begin(object) \
1146 MACRO_BEGIN \
1147 vm_object_lock_assert_exclusive((object)); \
1148 assert(! (object)->mapping_in_progress); \
1149 (object)->mapping_in_progress = TRUE; \
1150 MACRO_END
1151
1152#define vm_object_mapping_end(object) \
1153 MACRO_BEGIN \
1154 vm_object_lock_assert_exclusive((object)); \
1155 assert((object)->mapping_in_progress); \
1156 (object)->mapping_in_progress = FALSE; \
1157 vm_object_wakeup((object), \
1158 VM_OBJECT_EVENT_MAPPING_IN_PROGRESS); \
1159 MACRO_END
1160
1161#define vm_object_mapping_wait(object, interruptible) \
1162 MACRO_BEGIN \
1163 vm_object_lock_assert_exclusive((object)); \
1164 while ((object)->mapping_in_progress) { \
1165 wait_result_t _wr; \
1166 \
1167 _wr = vm_object_sleep((object), \
1168 VM_OBJECT_EVENT_MAPPING_IN_PROGRESS, \
1169 (interruptible)); \
1170 /*XXX if ((interruptible) && (_wr != THREAD_AWAKENED))*/\
1171 /*XXX break; */ \
1172 } \
1173 assert(!(object)->mapping_in_progress); \
1174 MACRO_END
1175
1176
1c79356b 1177
91447636
A		 1178#define vm_object_round_page(x) (((vm_object_offset_t)(x) + PAGE_MASK) & ~((signed)PAGE_MASK))
1179#define vm_object_trunc_page(x) ((vm_object_offset_t)(x) & ~((signed)PAGE_MASK))
1180
6d2010ae
A		 1181extern void vm_object_cache_add(vm_object_t);
1182extern void vm_object_cache_remove(vm_object_t);
1183extern int vm_object_cache_evict(int, int);
1184
d9a64523
A		 1185#define VM_OBJECT_OWNER_DISOWNED ((task_t) -1)
1186#define VM_OBJECT_OWNER(object) \
1187 ((((object)->purgable == VM_PURGABLE_DENY && \
1188 (object)->vo_ledger_tag == 0) || \
1189 (object)->vo_owner == TASK_NULL) \
1190 ? TASK_NULL /* not owned */ \
1191 : (((object)->vo_owner == VM_OBJECT_OWNER_DISOWNED) \
1192 ? kernel_task /* disowned -> kernel */ \
1193 : (object)->vo_owner)) /* explicit owner */ \
1194
1195extern void vm_object_ledger_tag_ledgers(
1196 vm_object_t object,
1197 int *ledger_idx_volatile,
1198 int *ledger_idx_nonvolatile,
1199 int *ledger_idx_volatile_compressed,
1200 int *ledger_idx_nonvolatile_compressed,
1201 boolean_t *do_footprint);
1202extern kern_return_t vm_object_ownership_change(
1203 vm_object_t object,
1204 int ledger_tag,
1205 task_t owner,
1206 boolean_t task_objq_locked);
1207
1c79356b 1208#endif /* _VM_VM_OBJECT_H_ */
mirror of XNU