]> git.saurik.com Git - apple/xnu.git/blame - osfmk/vm/vm_object.h
projects / apple / xnu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
xnu-4570.51.1.tar.gz
[apple/xnu.git] / osfmk / vm / vm_object.h
CommitLineData
1c79356b 1/*
2d21ac55 2 * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
1c79356b 3 *
2d21ac55 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
1c79356b 5 *
2d21ac55
A		 6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
8f6c56a5 14 *
2d21ac55
A		 15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
8f6c56a5
A		 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
2d21ac55
A		 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
8f6c56a5 25 *
2d21ac55 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
1c79356b
A		 27 */
28/*
29 * @OSF_COPYRIGHT@
30 */
31/*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56/*
57 */
58/*
59 * File: vm_object.h
60 * Author: Avadis Tevanian, Jr., Michael Wayne Young
61 * Date: 1985
62 *
63 * Virtual memory object module definitions.
64 */
65
66#ifndef _VM_VM_OBJECT_H_
67#define _VM_VM_OBJECT_H_
68
6d2010ae
A		 69#include <debug.h>
70#include <mach_assert.h>
1c79356b
A		 71#include <mach_pagemap.h>
72#include <task_swapper.h>
73
74#include <mach/kern_return.h>
75#include <mach/boolean.h>
76#include <mach/memory_object_types.h>
77#include <mach/port.h>
78#include <mach/vm_prot.h>
91447636 79#include <mach/vm_param.h>
1c79356b
A		 80#include <mach/machine/vm_types.h>
81#include <kern/queue.h>
2d21ac55 82#include <kern/locks.h>
1c79356b 83#include <kern/assert.h>
0b4e3aa0 84#include <kern/misc_protos.h>
1c79356b
A		 85#include <kern/macro_help.h>
86#include <ipc/ipc_types.h>
87#include <vm/pmap.h>
1c79356b 88
1c79356b 89#include <vm/vm_external.h>
1c79356b 90
b0d623f7 91#include <vm/vm_options.h>
39037602 92#include <vm/vm_page.h>
b0d623f7 93
fe8ab488
A		 94#if VM_OBJECT_TRACKING
95#include <libkern/OSDebug.h>
96#include <kern/btlog.h>
97extern void vm_object_tracking_init(void);
98extern boolean_t vm_object_tracking_inited;
99extern btlog_t *vm_object_tracking_btlog;
39037602 100#define VM_OBJECT_TRACKING_NUM_RECORDS 50000
fe8ab488
A		 101#define VM_OBJECT_TRACKING_BTDEPTH 7
102#define VM_OBJECT_TRACKING_OP_CREATED 1
103#define VM_OBJECT_TRACKING_OP_MODIFIED 2
104#define VM_OBJECT_TRACKING_OP_TRUESHARE 3
105#endif /* VM_OBJECT_TRACKING */
106
91447636 107struct vm_page;
39236c6e 108struct vm_shared_region_slide_info;
1c79356b
A		 109
110/*
111 * Types defined:
112 *
113 * vm_object_t Virtual memory object.
2d21ac55 114 * vm_object_fault_info_t Used to determine cluster size.
1c79356b
A		 115 */
116
2d21ac55
A		 117struct vm_object_fault_info {
118 int interruptible;
119 uint32_t user_tag;
120 vm_size_t cluster_size;
121 vm_behavior_t behavior;
122 vm_map_offset_t lo_offset;
123 vm_map_offset_t hi_offset;
6d2010ae
A		 124 unsigned int
125 /* boolean_t */ no_cache:1,
126 /* boolean_t */ stealth:1,
127 /* boolean_t */ io_sync:1,
128 /* boolean_t */ cs_bypass:1,
129 /* boolean_t */ mark_zf_absent:1,
316670eb
A		 130 /* boolean_t */ batch_pmap_op:1,
131 __vm_object_fault_info_unused_bits:26;
fe8ab488 132 int pmap_options;
2d21ac55
A		 133};
134
135
39236c6e
A		 136#define vo_size vo_un1.vou_size
137#define vo_cache_pages_to_scan vo_un1.vou_cache_pages_to_scan
138#define vo_shadow_offset vo_un2.vou_shadow_offset
139#define vo_cache_ts vo_un2.vou_cache_ts
140#define vo_purgeable_owner vo_un2.vou_purgeable_owner
141#define vo_slide_info vo_un2.vou_slide_info
2d21ac55 142
1c79356b 143struct vm_object {
39037602
A		 144 /*
145 * on 64 bit systems we pack the pointers hung off the memq.
146 * those pointers have to be able to point back to the memq.
147 * the packed pointers are required to be on a 64 byte boundary
148 * which means 2 things for the vm_object... (1) the memq
149 * struct has to be the first element of the structure so that
150 * we can control it's alignment... (2) the vm_object must be
151 * aligned on a 64 byte boundary... for static vm_object's
152 * this is accomplished via the 'aligned' attribute... for
153 * vm_object's in the zone pool, this is accomplished by
154 * rounding the size of the vm_object element to the nearest
155 * 64 byte size before creating the zone.
156 */
157 vm_page_queue_head_t memq; /* Resident memory - must be first */
2d21ac55 158 lck_rw_t Lock; /* Synchronization */
1c79356b 159
39037602
A		 160#if DEVELOPMENT || DEBUG
161 thread_t Lock_owner;
162#endif
6d2010ae
A		 163 union {
164 vm_object_size_t vou_size; /* Object size (only valid if internal) */
165 int vou_cache_pages_to_scan; /* pages yet to be visited in an
166 * external object in cache
167 */
168 } vo_un1;
169
91447636 170 struct vm_page *memq_hint;
1c79356b 171 int ref_count; /* Number of references */
1c79356b
A		 172 unsigned int resident_page_count;
173 /* number of resident pages */
5ba3f43e
A		 174 const unsigned int wired_page_count; /* number of wired pages
175 use VM_OBJECT_WIRED_PAGE_UPDATE macros to update */
b0d623f7 176 unsigned int reusable_page_count;
1c79356b
A		 177
178 struct vm_object *copy; /* Object that should receive
179 * a copy of my changed pages,
180 * for copy_delay, or just the
181 * temporary object that
182 * shadows this object, for
183 * copy_call.
184 */
185 struct vm_object *shadow; /* My shadow */
6d2010ae
A		 186
187 union {
188 vm_object_offset_t vou_shadow_offset; /* Offset into shadow */
39236c6e
A		 189 clock_sec_t vou_cache_ts; /* age of an external object
190 * present in cache
191 */
192 task_t vou_purgeable_owner; /* If the purg'a'ble bits below are set
193 * to volatile/emtpy, this is the task
194 * that owns this purgeable object.
6d2010ae 195 */
39236c6e 196 struct vm_shared_region_slide_info *vou_slide_info;
6d2010ae 197 } vo_un2;
1c79356b 198
0b4e3aa0 199 memory_object_t pager; /* Where to get data */
1c79356b 200 vm_object_offset_t paging_offset; /* Offset into memory object */
91447636 201 memory_object_control_t pager_control; /* Where data comes back */
1c79356b
A		 202
203 memory_object_copy_strategy_t
204 copy_strategy; /* How to handle data copy */
205
fe8ab488
A		 206#if __LP64__
207 /*
208 * Some user processes (mostly VirtualMachine software) take a large
209 * number of UPLs (via IOMemoryDescriptors) to wire pages in large
210 * VM objects and overflow the 16-bit "activity_in_progress" counter.
211 * Since we never enforced any limit there, let's give them 32 bits
212 * for backwards compatibility's sake.
213 */
214 unsigned int paging_in_progress:16,
215 __object1_unused_bits:16;
216 unsigned int activity_in_progress;
217#else /* __LP64__ */
218 /*
219 * On 32-bit platforms, enlarging "activity_in_progress" would increase
220 * the size of "struct vm_object". Since we don't know of any actual
221 * overflow of these counters on these platforms, let's keep the
222 * counters as 16-bit integers.
223 */
224 unsigned short paging_in_progress;
225 unsigned short activity_in_progress;
226#endif /* __LP64__ */
1c79356b
A		 227 /* The memory object ports are
228 * being used (e.g., for pagein
229 * or pageout) -- don't change
230 * any of these fields (i.e.,
231 * don't collapse, destroy or
232 * terminate)
233 */
b0d623f7 234
1c79356b
A		 235 unsigned int
236 /* boolean_t array */ all_wanted:11, /* Bit array of "want to be
237 * awakened" notations. See
238 * VM_OBJECT_EVENT_* items
239 * below */
240 /* boolean_t */ pager_created:1, /* Has pager been created? */
241 /* boolean_t */ pager_initialized:1, /* Are fields ready to use? */
242 /* boolean_t */ pager_ready:1, /* Will pager take requests? */
243
244 /* boolean_t */ pager_trusted:1,/* The pager for this object
245 * is trusted. This is true for
246 * all internal objects (backed
247 * by the default pager)
248 */
249 /* boolean_t */ can_persist:1, /* The kernel may keep the data
250 * for this object (and rights
251 * to the memory object) after
252 * all address map references
253 * are deallocated?
254 */
255 /* boolean_t */ internal:1, /* Created by the kernel (and
256 * therefore, managed by the
257 * default memory manger)
258 */
1c79356b
A		 259 /* boolean_t */ private:1, /* magic device_pager object,
260 * holds private pages only */
261 /* boolean_t */ pageout:1, /* pageout object. contains
262 * private pages that refer to
263 * a real memory object. */
264 /* boolean_t */ alive:1, /* Not yet terminated */
265
91447636 266 /* boolean_t */ purgable:2, /* Purgable state. See
2d21ac55 267 * VM_PURGABLE_*
1c79356b 268 */
5ba3f43e 269 /* boolean_t */ purgeable_only_by_kernel:1,
39236c6e
A		 270 /* boolean_t */ purgeable_when_ripe:1, /* Purgeable when a token
271 * becomes ripe.
272 */
1c79356b 273 /* boolean_t */ shadowed:1, /* Shadow may exist */
1c79356b
A		 274 /* boolean_t */ true_share:1,
275 /* This object is mapped
276 * in more than one place
277 * and hence cannot be
278 * coalesced */
279 /* boolean_t */ terminating:1,
280 /* Allows vm_object_lookup
281 * and vm_object_deallocate
282 * to special case their
283 * behavior when they are
284 * called as a result of
285 * page cleaning during
286 * object termination
287 */
288 /* boolean_t */ named:1, /* An enforces an internal
289 * naming convention, by
290 * calling the right routines
291 * for allocation and
292 * destruction, UBC references
293 * against the vm_object are
294 * checked.
295 */
296 /* boolean_t */ shadow_severed:1,
297 /* When a permanent object
298 * backing a COW goes away
299 * unexpectedly. This bit
300 * allows vm_fault to return
301 * an error rather than a
302 * zero filled page.
303 */
0b4e3aa0 304 /* boolean_t */ phys_contiguous:1,
1c79356b
A		 305 /* Memory is wired and
306 * guaranteed physically
307 * contiguous. However
308 * it is not device memory
309 * and obeys normal virtual
310 * memory rules w.r.t pmap
311 * access bits.
312 */
5ba3f43e 313 /* boolean_t */ nophyscache:1,
0b4e3aa0
A		 314 /* When mapped at the
315 * pmap level, don't allow
316 * primary caching. (for
317 * I/O)
318 */
5ba3f43e 319 /* boolean_t */ _object5_unused_bits:1;
1c79356b
A		 320
321 queue_chain_t cached_list; /* Attachment point for the
322 * list of objects cached as a
323 * result of their can_persist
324 * value
325 */
2d21ac55
A		 326 /*
327 * the following fields are not protected by any locks
328 * they are updated via atomic compare and swap
329 */
1c79356b 330 vm_object_offset_t last_alloc; /* last allocation offset */
2d21ac55
A		 331 int sequential; /* sequential access size */
332
333 uint32_t pages_created;
334 uint32_t pages_used;
55e303ae 335 vm_offset_t cow_hint; /* last page present in */
0b4e3aa0 336 /* shadow but not in object */
2d21ac55
A		 337 /* hold object lock when altering */
338 unsigned int
339 wimg_bits:8, /* cache WIMG bits */
340 code_signed:1, /* pages are signed and should be
341 validated; the signatures are stored
342 with the pager */
b0d623f7 343 transposed:1, /* object was transposed with another */
593a1d5f 344 mapping_in_progress:1, /* pager being mapped/unmapped */
fe8ab488 345 phantom_isssd:1,
b0d623f7
A		 346 volatile_empty:1,
347 volatile_fault:1,
348 all_reusable:1,
349 blocked_access:1,
6d2010ae 350 set_cache_attr:1,
39236c6e
A		 351 object_slid:1,
352 purgeable_queue_type:2,
353 purgeable_queue_group:3,
fe8ab488 354 io_tracking:1,
5ba3f43e 355 no_tag_update:1, /* */
39037602
A		 356#if CONFIG_SECLUDED_MEMORY
357 eligible_for_secluded:1,
358 can_grab_secluded:1,
359#else /* CONFIG_SECLUDED_MEMORY */
360 __object3_unused_bits:2,
361#endif /* CONFIG_SECLUDED_MEMORY */
362 __object2_unused_bits:5; /* for expansion */
6d2010ae 363
3e170ce0
A		 364 uint8_t scan_collisions;
365 vm_tag_t wire_tag;
366 uint8_t __object4_unused_bits[2];
367
fe8ab488
A		 368#if CONFIG_PHANTOM_CACHE
369 uint32_t phantom_object_id;
370#endif
371#if CONFIG_IOSCHED || UPL_DEBUG
1c79356b 372 queue_head_t uplq; /* List of outstanding upls */
fe8ab488 373#endif
2d21ac55 374
0c530ab8
A		 375#ifdef VM_PIP_DEBUG
376/*
377 * Keep track of the stack traces for the first holders
378 * of a "paging_in_progress" reference for this VM object.
379 */
380#define VM_PIP_DEBUG_STACK_FRAMES 25 /* depth of each stack trace */
381#define VM_PIP_DEBUG_MAX_REFS 10 /* track that many references */
382 struct __pip_backtrace {
383 void *pip_retaddr[VM_PIP_DEBUG_STACK_FRAMES];
384 } pip_holders[VM_PIP_DEBUG_MAX_REFS];
385#endif /* VM_PIP_DEBUG */
2d21ac55 386
6d2010ae 387 queue_chain_t objq; /* object queue - currently used for purgable queues */
a39ff7e2 388 queue_chain_t task_objq; /* objects owned by task - protected by task lock */
fe8ab488
A		 389
390#if DEBUG
391 void *purgeable_owner_bt[16];
392 task_t vo_purgeable_volatilizer; /* who made it volatile? */
393 void *purgeable_volatilizer_bt[16];
394#endif /* DEBUG */
1c79356b
A		 395};
396
b0d623f7
A		 397#define VM_OBJECT_PURGEABLE_FAULT_ERROR(object) \
398 ((object)->volatile_fault && \
399 ((object)->purgable == VM_PURGABLE_VOLATILE || \
400 (object)->purgable == VM_PURGABLE_EMPTY))
401
39236c6e 402extern
1c79356b
A		 403vm_object_t kernel_object; /* the single kernel object */
404
39236c6e
A		 405extern
406vm_object_t compressor_object; /* the single compressor object */
407
408extern
91447636 409unsigned int vm_object_absent_max; /* maximum number of absent pages
1c79356b
A		 410 at a time for each object */
411
412# define VM_MSYNC_INITIALIZED 0
413# define VM_MSYNC_SYNCHRONIZING 1
414# define VM_MSYNC_DONE 2
415
b0d623f7
A		 416
417extern lck_grp_t vm_map_lck_grp;
418extern lck_attr_t vm_map_lck_attr;
419
5ba3f43e
A		 420#ifndef VM_TAG_ACTIVE_UPDATE
421#error VM_TAG_ACTIVE_UPDATE
422#endif
1c79356b 423
5ba3f43e 424#define VM_OBJECT_WIRED(object, tag) \
3e170ce0 425 MACRO_BEGIN \
5ba3f43e
A		 426 assert(VM_KERN_MEMORY_NONE != (tag)); \
427 assert(VM_KERN_MEMORY_NONE == (object)->wire_tag); \
428 (object)->wire_tag = (tag); \
429 if (!VM_TAG_ACTIVE_UPDATE \
430 && ((object)->purgable == VM_PURGABLE_DENY)) \
3e170ce0
A		 431 { \
432 lck_spin_lock(&vm_objects_wired_lock); \
433 assert(!(object)->objq.next); \
39037602 434 assert(!(object)->objq.prev); \
3e170ce0
A		 435 queue_enter(&vm_objects_wired, (object), vm_object_t, objq); \
436 lck_spin_unlock(&vm_objects_wired_lock); \
437 } \
438 MACRO_END
439
5ba3f43e
A		 440#define VM_OBJECT_UNWIRED(object) \
441 MACRO_BEGIN \
442 if (!VM_TAG_ACTIVE_UPDATE \
443 && ((object)->purgable == VM_PURGABLE_DENY) && (object)->objq.next) \
444 { \
445 lck_spin_lock(&vm_objects_wired_lock); \
446 queue_remove(&vm_objects_wired, (object), vm_object_t, objq); \
447 lck_spin_unlock(&vm_objects_wired_lock); \
448 } \
449 if (VM_KERN_MEMORY_NONE != (object)->wire_tag) { \
450 vm_tag_update_size((object)->wire_tag, -ptoa_64((object)->wired_page_count)); \
451 (object)->wire_tag = VM_KERN_MEMORY_NONE; \
452 } \
453 MACRO_END
454
455// These two macros start & end a C block
456#define VM_OBJECT_WIRED_PAGE_UPDATE_START(object) \
457 MACRO_BEGIN \
458 { \
459 int64_t __wireddelta = 0; vm_tag_t __waswired = (object)->wire_tag;
460
461#define VM_OBJECT_WIRED_PAGE_UPDATE_END(object, tag) \
462 if (__wireddelta) { \
463 boolean_t __overflow __assert_only = \
464 os_add_overflow((object)->wired_page_count, __wireddelta, \
465 (unsigned int *)(uintptr_t)&(object)->wired_page_count); \
466 assert(!__overflow); \
467 if (!(object)->pageout && !(object)->no_tag_update) { \
468 if (__wireddelta > 0) { \
469 assert (VM_KERN_MEMORY_NONE != (tag)); \
470 if (VM_KERN_MEMORY_NONE == __waswired) { \
471 VM_OBJECT_WIRED((object), (tag)); \
472 } \
473 vm_tag_update_size((object)->wire_tag, ptoa_64(__wireddelta)); \
474 } else if (VM_KERN_MEMORY_NONE != __waswired) { \
475 assert (VM_KERN_MEMORY_NONE != (object)->wire_tag); \
476 vm_tag_update_size((object)->wire_tag, ptoa_64(__wireddelta)); \
477 if (!(object)->wired_page_count) { \
478 VM_OBJECT_UNWIRED((object)); \
479 } \
480 } \
481 } \
482 } \
483 } \
3e170ce0
A		 484 MACRO_END
485
5ba3f43e
A		 486#define VM_OBJECT_WIRED_PAGE_COUNT(object, delta) \
487 __wireddelta += delta; \
488
489#define VM_OBJECT_WIRED_PAGE_ADD(object, m) \
490 if (!m->private && !m->fictitious) __wireddelta++;
491
492#define VM_OBJECT_WIRED_PAGE_REMOVE(object, m) \
493 if (!m->private && !m->fictitious) __wireddelta--;
494
495
3e170ce0 496
39037602
A		 497#define OBJECT_LOCK_SHARED 0
498#define OBJECT_LOCK_EXCLUSIVE 1
499
500extern lck_grp_t vm_object_lck_grp;
501extern lck_grp_attr_t vm_object_lck_grp_attr;
502extern lck_attr_t vm_object_lck_attr;
503extern lck_attr_t kernel_object_lck_attr;
504extern lck_attr_t compressor_object_lck_attr;
505
506extern vm_object_t vm_pageout_scan_wants_object;
507
508extern void vm_object_lock(vm_object_t);
509extern boolean_t vm_object_lock_try(vm_object_t);
510extern boolean_t _vm_object_lock_try(vm_object_t);
511extern boolean_t vm_object_lock_avoid(vm_object_t);
512extern void vm_object_lock_shared(vm_object_t);
5ba3f43e 513extern boolean_t vm_object_lock_yield_shared(vm_object_t);
39037602
A		 514extern boolean_t vm_object_lock_try_shared(vm_object_t);
515extern void vm_object_unlock(vm_object_t);
516extern boolean_t vm_object_lock_upgrade(vm_object_t);
517
518/*
519 * Object locking macros
520 */
521
522#define vm_object_lock_init(object) \
523 lck_rw_init(&(object)->Lock, &vm_object_lck_grp, \
524 (((object) == kernel_object || \
525 (object) == vm_submap_object) ? \
526 &kernel_object_lck_attr : \
527 (((object) == compressor_object) ? \
528 &compressor_object_lck_attr : \
529 &vm_object_lck_attr)))
530#define vm_object_lock_destroy(object) lck_rw_destroy(&(object)->Lock, &vm_object_lck_grp)
531
532#define vm_object_lock_try_scan(object) _vm_object_lock_try(object)
533
534/*
535 * CAUTION: the following vm_object_lock_assert_held*() macros merely
536 * check if anyone is holding the lock, but the holder may not necessarily
537 * be the caller...
538 */
539#if MACH_ASSERT || DEBUG
540#define vm_object_lock_assert_held(object) \
541 lck_rw_assert(&(object)->Lock, LCK_RW_ASSERT_HELD)
542#define vm_object_lock_assert_shared(object) \
543 lck_rw_assert(&(object)->Lock, LCK_RW_ASSERT_SHARED)
544#define vm_object_lock_assert_exclusive(object) \
545 lck_rw_assert(&(object)->Lock, LCK_RW_ASSERT_EXCLUSIVE)
546#define vm_object_lock_assert_notheld(object) \
547 lck_rw_assert(&(object)->Lock, LCK_RW_ASSERT_NOTHELD)
548#else /* MACH_ASSERT || DEBUG */
549#define vm_object_lock_assert_held(object)
550#define vm_object_lock_assert_shared(object)
551#define vm_object_lock_assert_exclusive(object)
552#define vm_object_lock_assert_notheld(object)
553#endif /* MACH_ASSERT || DEBUG */
554
555
1c79356b
A		 556/*
557 * Declare procedures that operate on VM objects.
558 */
559
39236c6e 560__private_extern__ void vm_object_bootstrap(void);
1c79356b 561
0b4e3aa0 562__private_extern__ void vm_object_init(void);
1c79356b 563
2d21ac55
A		 564__private_extern__ void vm_object_init_lck_grp(void);
565
8f6c56a5
A		 566__private_extern__ void vm_object_reaper_init(void);
567
3e170ce0 568__private_extern__ vm_object_t vm_object_allocate(vm_object_size_t size);
1c79356b 569
91447636
A		 570__private_extern__ void _vm_object_allocate(vm_object_size_t size,
571 vm_object_t object);
572
0b4e3aa0
A		 573#if TASK_SWAPPER
574
575__private_extern__ void vm_object_res_reference(
576 vm_object_t object);
577__private_extern__ void vm_object_res_deallocate(
578 vm_object_t object);
579#define VM_OBJ_RES_INCR(object) (object)->res_count++
580#define VM_OBJ_RES_DECR(object) (object)->res_count--
581
582#else /* TASK_SWAPPER */
583
584#define VM_OBJ_RES_INCR(object)
585#define VM_OBJ_RES_DECR(object)
586#define vm_object_res_reference(object)
587#define vm_object_res_deallocate(object)
588
589#endif /* TASK_SWAPPER */
590
591#define vm_object_reference_locked(object) \
2d21ac55
A		 592 MACRO_BEGIN \
593 vm_object_t RLObject = (object); \
594 vm_object_lock_assert_exclusive(object); \
595 assert((RLObject)->ref_count > 0); \
596 (RLObject)->ref_count++; \
597 assert((RLObject)->ref_count > 1); \
598 vm_object_res_reference(RLObject); \
599 MACRO_END
600
601
602#define vm_object_reference_shared(object) \
603 MACRO_BEGIN \
604 vm_object_t RLObject = (object); \
605 vm_object_lock_assert_shared(object); \
606 assert((RLObject)->ref_count > 0); \
b0d623f7 607 OSAddAtomic(1, &(RLObject)->ref_count); \
316670eb 608 assert((RLObject)->ref_count > 0); \
2d21ac55
A		 609 /* XXX we would need an atomic version of the following ... */ \
610 vm_object_res_reference(RLObject); \
611 MACRO_END
0b4e3aa0
A		 612
613
0b4e3aa0 614__private_extern__ void vm_object_reference(
1c79356b 615 vm_object_t object);
0b4e3aa0 616
91447636 617#if !MACH_ASSERT
0b4e3aa0 618
1c79356b
A		 619#define vm_object_reference(object) \
620MACRO_BEGIN \
0b4e3aa0
A		 621 vm_object_t RObject = (object); \
622 if (RObject) { \
b0d623f7
A		 623 vm_object_lock_shared(RObject); \
624 vm_object_reference_shared(RObject); \
0b4e3aa0 625 vm_object_unlock(RObject); \
1c79356b
A		 626 } \
627MACRO_END
0b4e3aa0 628
1c79356b
A		 629#endif /* MACH_ASSERT */
630
0b4e3aa0 631__private_extern__ void vm_object_deallocate(
1c79356b
A		 632 vm_object_t object);
633
0b4e3aa0
A		 634__private_extern__ kern_return_t vm_object_release_name(
635 vm_object_t object,
636 int flags);
637
638__private_extern__ void vm_object_pmap_protect(
1c79356b
A		 639 vm_object_t object,
640 vm_object_offset_t offset,
91447636 641 vm_object_size_t size,
1c79356b 642 pmap_t pmap,
91447636 643 vm_map_offset_t pmap_start,
1c79356b
A		 644 vm_prot_t prot);
645
39236c6e
A		 646__private_extern__ void vm_object_pmap_protect_options(
647 vm_object_t object,
648 vm_object_offset_t offset,
649 vm_object_size_t size,
650 pmap_t pmap,
651 vm_map_offset_t pmap_start,
652 vm_prot_t prot,
653 int options);
654
0b4e3aa0 655__private_extern__ void vm_object_page_remove(
1c79356b
A		 656 vm_object_t object,
657 vm_object_offset_t start,
658 vm_object_offset_t end);
659
0b4e3aa0
A		 660__private_extern__ void vm_object_deactivate_pages(
661 vm_object_t object,
662 vm_object_offset_t offset,
663 vm_object_size_t size,
b0d623f7 664 boolean_t kill_page,
3e170ce0
A		 665 boolean_t reusable_page,
666 struct pmap *pmap,
667 vm_map_offset_t pmap_offset);
b0d623f7
A		 668
669__private_extern__ void vm_object_reuse_pages(
670 vm_object_t object,
671 vm_object_offset_t start_offset,
672 vm_object_offset_t end_offset,
673 boolean_t allow_partial_reuse);
0b4e3aa0 674
a39ff7e2 675__private_extern__ uint64_t vm_object_purge(
fe8ab488
A		 676 vm_object_t object,
677 int flags);
91447636
A		 678
679__private_extern__ kern_return_t vm_object_purgable_control(
680 vm_object_t object,
681 vm_purgable_t control,
682 int *state);
683
39236c6e
A		 684__private_extern__ kern_return_t vm_object_get_page_counts(
685 vm_object_t object,
686 vm_object_offset_t offset,
687 vm_object_size_t size,
688 unsigned int *resident_page_count,
689 unsigned int *dirty_page_count);
690
0b4e3aa0 691__private_extern__ boolean_t vm_object_coalesce(
1c79356b
A		 692 vm_object_t prev_object,
693 vm_object_t next_object,
694 vm_object_offset_t prev_offset,
695 vm_object_offset_t next_offset,
696 vm_object_size_t prev_size,
697 vm_object_size_t next_size);
698
0b4e3aa0 699__private_extern__ boolean_t vm_object_shadow(
1c79356b
A		 700 vm_object_t *object,
701 vm_object_offset_t *offset,
702 vm_object_size_t length);
703
0b4e3aa0 704__private_extern__ void vm_object_collapse(
55e303ae 705 vm_object_t object,
0c530ab8
A		 706 vm_object_offset_t offset,
707 boolean_t can_bypass);
1c79356b 708
0b4e3aa0 709__private_extern__ boolean_t vm_object_copy_quickly(
1c79356b
A		 710 vm_object_t *_object,
711 vm_object_offset_t src_offset,
712 vm_object_size_t size,
713 boolean_t *_src_needs_copy,
714 boolean_t *_dst_needs_copy);
715
0b4e3aa0 716__private_extern__ kern_return_t vm_object_copy_strategically(
1c79356b
A		 717 vm_object_t src_object,
718 vm_object_offset_t src_offset,
719 vm_object_size_t size,
720 vm_object_t *dst_object,
721 vm_object_offset_t *dst_offset,
722 boolean_t *dst_needs_copy);
723
0b4e3aa0 724__private_extern__ kern_return_t vm_object_copy_slowly(
1c79356b
A		 725 vm_object_t src_object,
726 vm_object_offset_t src_offset,
727 vm_object_size_t size,
b0d623f7 728 boolean_t interruptible,
1c79356b
A		 729 vm_object_t *_result_object);
730
0b4e3aa0
A		 731__private_extern__ vm_object_t vm_object_copy_delayed(
732 vm_object_t src_object,
733 vm_object_offset_t src_offset,
2d21ac55
A		 734 vm_object_size_t size,
735 boolean_t src_object_shared);
0b4e3aa0 736
1c79356b 737
1c79356b 738
0b4e3aa0
A		 739__private_extern__ kern_return_t vm_object_destroy(
740 vm_object_t object,
741 kern_return_t reason);
1c79356b 742
0b4e3aa0
A		 743__private_extern__ void vm_object_pager_create(
744 vm_object_t object);
745
39236c6e
A		 746__private_extern__ void vm_object_compressor_pager_create(
747 vm_object_t object);
748
0b4e3aa0 749__private_extern__ void vm_object_page_map(
1c79356b
A		 750 vm_object_t object,
751 vm_object_offset_t offset,
752 vm_object_size_t size,
753 vm_object_offset_t (*map_fn)
754 (void *, vm_object_offset_t),
755 void *map_fn_data);
756
0b4e3aa0
A		 757__private_extern__ kern_return_t vm_object_upl_request(
758 vm_object_t object,
759 vm_object_offset_t offset,
91447636 760 upl_size_t size,
0b4e3aa0
A		 761 upl_t *upl,
762 upl_page_info_t *page_info,
763 unsigned int *count,
5ba3f43e
A		 764 upl_control_flags_t flags,
765 vm_tag_t tag);
0b4e3aa0 766
91447636
A		 767__private_extern__ kern_return_t vm_object_transpose(
768 vm_object_t object1,
769 vm_object_t object2,
770 vm_object_size_t transpose_size);
771
0b4e3aa0
A		 772__private_extern__ boolean_t vm_object_sync(
773 vm_object_t object,
774 vm_object_offset_t offset,
91447636 775 vm_object_size_t size,
0b4e3aa0 776 boolean_t should_flush,
91447636
A		 777 boolean_t should_return,
778 boolean_t should_iosync);
1c79356b 779
0b4e3aa0
A		 780__private_extern__ kern_return_t vm_object_update(
781 vm_object_t object,
782 vm_object_offset_t offset,
91447636
A		 783 vm_object_size_t size,
784 vm_object_offset_t *error_offset,
785 int *io_errno,
0b4e3aa0
A		 786 memory_object_return_t should_return,
787 int flags,
788 vm_prot_t prot);
1c79356b 789
0b4e3aa0
A		 790__private_extern__ kern_return_t vm_object_lock_request(
791 vm_object_t object,
792 vm_object_offset_t offset,
793 vm_object_size_t size,
794 memory_object_return_t should_return,
795 int flags,
796 vm_prot_t prot);
1c79356b 797
1c79356b 798
1c79356b 799
5ba3f43e 800__private_extern__ vm_object_t vm_object_memory_object_associate(
0b4e3aa0 801 memory_object_t pager,
5ba3f43e 802 vm_object_t object,
1c79356b 803 vm_object_size_t size,
1c79356b
A		 804 boolean_t check_named);
805
806
2d21ac55
A		 807__private_extern__ void vm_object_cluster_size(
808 vm_object_t object,
809 vm_object_offset_t *start,
810 vm_size_t *length,
b0d623f7
A		 811 vm_object_fault_info_t fault_info,
812 uint32_t *io_streaming);
91447636
A		 813
814__private_extern__ kern_return_t vm_object_populate_with_private(
815 vm_object_t object,
816 vm_object_offset_t offset,
817 ppnum_t phys_page,
818 vm_size_t size);
819
6d2010ae
A		 820__private_extern__ void vm_object_change_wimg_mode(
821 vm_object_t object,
822 unsigned int wimg_mode);
823
0c530ab8 824extern kern_return_t adjust_vm_object_cache(
91447636
A		 825 vm_size_t oval,
826 vm_size_t nval);
827
0c530ab8
A		 828extern kern_return_t vm_object_page_op(
829 vm_object_t object,
830 vm_object_offset_t offset,
831 int ops,
832 ppnum_t *phys_entry,
833 int *flags);
834
835extern kern_return_t vm_object_range_op(
836 vm_object_t object,
837 vm_object_offset_t offset_beg,
838 vm_object_offset_t offset_end,
839 int ops,
b0d623f7
A		 840 uint32_t *range);
841
842
843__private_extern__ void vm_object_reap_pages(
844 vm_object_t object,
845 int reap_type);
846#define REAP_REAP 0
847#define REAP_TERMINATE 1
848#define REAP_PURGEABLE 2
849#define REAP_DATA_FLUSH 3
850
6d2010ae 851#if CONFIG_FREEZE
6d2010ae 852
316670eb 853__private_extern__ void
3e170ce0 854vm_object_compressed_freezer_pageout(
316670eb 855 vm_object_t object);
6d2010ae 856
3e170ce0
A		 857__private_extern__ void
858vm_object_compressed_freezer_done(
859 void);
860
6d2010ae 861#endif /* CONFIG_FREEZE */
0c530ab8 862
3e170ce0
A		 863__private_extern__ void
864vm_object_pageout(
865 vm_object_t object);
866
fe8ab488
A		 867#if CONFIG_IOSCHED
868struct io_reprioritize_req {
869 uint64_t blkno;
870 uint32_t len;
871 int priority;
872 struct vnode *devvp;
873 queue_chain_t io_reprioritize_list;
874};
875typedef struct io_reprioritize_req *io_reprioritize_req_t;
876
877extern void vm_io_reprioritize_init(void);
878#endif
879
1c79356b
A		 880/*
881 * Event waiting handling
882 */
883
884#define VM_OBJECT_EVENT_INITIALIZED 0
885#define VM_OBJECT_EVENT_PAGER_READY 1
886#define VM_OBJECT_EVENT_PAGING_IN_PROGRESS 2
593a1d5f 887#define VM_OBJECT_EVENT_MAPPING_IN_PROGRESS 3
1c79356b
A		 888#define VM_OBJECT_EVENT_LOCK_IN_PROGRESS 4
889#define VM_OBJECT_EVENT_UNCACHING 5
890#define VM_OBJECT_EVENT_COPY_CALL 6
891#define VM_OBJECT_EVENT_CACHING 7
b0d623f7
A		 892#define VM_OBJECT_EVENT_UNBLOCKED 8
893#define VM_OBJECT_EVENT_PAGING_ONLY_IN_PROGRESS 9
1c79356b 894
39037602 895#define VM_OBJECT_EVENT_MAX 10 /* 11 bits in "all_wanted", so 0->10 */
1c79356b 896
39037602
A		 897static __inline__ wait_result_t
898vm_object_assert_wait(
899 vm_object_t object,
900 int event,
901 wait_interrupt_t interruptible)
902{
903 wait_result_t wr;
904
905 vm_object_lock_assert_exclusive(object);
906 assert(event >= 0 && event <= VM_OBJECT_EVENT_MAX);
907
908 object->all_wanted |= 1 << event;
909 wr = assert_wait((event_t)((vm_offset_t)object + event),
910 interruptible);
911 return wr;
912}
913
914static __inline__ wait_result_t
915vm_object_wait(
916 vm_object_t object,
917 int event,
918 wait_interrupt_t interruptible)
919{
920 wait_result_t wr;
921
922 vm_object_assert_wait(object, event, interruptible);
923 vm_object_unlock(object);
924 wr = thread_block(THREAD_CONTINUE_NULL);
925 return wr;
926}
927
928static __inline__ wait_result_t
929thread_sleep_vm_object(
930 vm_object_t object,
931 event_t event,
932 wait_interrupt_t interruptible)
933{
934 wait_result_t wr;
935
936#if DEVELOPMENT || DEBUG
937 if (object->Lock_owner != current_thread())
938 panic("thread_sleep_vm_object: now owner - %p\n", object);
939 object->Lock_owner = 0;
940#endif
941 wr = lck_rw_sleep(&object->Lock,
942 LCK_SLEEP_PROMOTED_PRI,
943 event,
944 interruptible);
945#if DEVELOPMENT || DEBUG
946 object->Lock_owner = current_thread();
947#endif
948 return wr;
949}
9bccf70c 950
39037602
A		 951static __inline__ wait_result_t
952vm_object_sleep(
953 vm_object_t object,
954 int event,
955 wait_interrupt_t interruptible)
956{
957 wait_result_t wr;
958
959 vm_object_lock_assert_exclusive(object);
960 assert(event >= 0 && event <= VM_OBJECT_EVENT_MAX);
961
962 object->all_wanted |= 1 << event;
963 wr = thread_sleep_vm_object(object,
964 (event_t)((vm_offset_t)object + event),
965 interruptible);
966 return wr;
967}
968
969static __inline__ void
970vm_object_wakeup(
971 vm_object_t object,
972 int event)
973{
974 vm_object_lock_assert_exclusive(object);
975 assert(event >= 0 && event <= VM_OBJECT_EVENT_MAX);
976
977 if (object->all_wanted & (1 << event))
978 thread_wakeup((event_t)((vm_offset_t)object + event));
979 object->all_wanted &= ~(1 << event);
980}
981
982static __inline__ void
983vm_object_set_wanted(
984 vm_object_t object,
985 int event)
986{
987 vm_object_lock_assert_exclusive(object);
988 assert(event >= 0 && event <= VM_OBJECT_EVENT_MAX);
1c79356b 989
39037602
A		 990 object->all_wanted |= (1 << event);
991}
1c79356b 992
39037602
A		 993static __inline__ int
994vm_object_wanted(
995 vm_object_t object,
996 int event)
997{
998 vm_object_lock_assert_held(object);
999 assert(event >= 0 && event <= VM_OBJECT_EVENT_MAX);
1c79356b 1000
39037602
A		 1001 return object->all_wanted & (1 << event);
1002}
1c79356b
A		 1003
1004/*
1005 * Routines implemented as macros
1006 */
0c530ab8 1007#ifdef VM_PIP_DEBUG
2d21ac55 1008#include <libkern/OSDebug.h>
0c530ab8
A		 1009#define VM_PIP_DEBUG_BEGIN(object) \
1010 MACRO_BEGIN \
b0d623f7
A		 1011 int pip = ((object)->paging_in_progress + \
1012 (object)->activity_in_progress); \
1013 if (pip < VM_PIP_DEBUG_MAX_REFS) { \
2d21ac55 1014 (void) OSBacktrace(&(object)->pip_holders[pip].pip_retaddr[0], \
0c530ab8
A		 1015 VM_PIP_DEBUG_STACK_FRAMES); \
1016 } \
1017 MACRO_END
1018#else /* VM_PIP_DEBUG */
1019#define VM_PIP_DEBUG_BEGIN(object)
1020#endif /* VM_PIP_DEBUG */
1c79356b 1021
b0d623f7
A		 1022#define vm_object_activity_begin(object) \
1023 MACRO_BEGIN \
1024 vm_object_lock_assert_exclusive((object)); \
b0d623f7
A		 1025 VM_PIP_DEBUG_BEGIN((object)); \
1026 (object)->activity_in_progress++; \
fe8ab488
A		 1027 if ((object)->activity_in_progress == 0) { \
1028 panic("vm_object_activity_begin(%p): overflow\n", (object));\
1029 } \
b0d623f7
A		 1030 MACRO_END
1031
1032#define vm_object_activity_end(object) \
1033 MACRO_BEGIN \
1034 vm_object_lock_assert_exclusive((object)); \
fe8ab488
A		 1035 if ((object)->activity_in_progress == 0) { \
1036 panic("vm_object_activity_end(%p): underflow\n", (object));\
1037 } \
b0d623f7
A		 1038 (object)->activity_in_progress--; \
1039 if ((object)->paging_in_progress == 0 && \
1040 (object)->activity_in_progress == 0) \
1041 vm_object_wakeup((object), \
1042 VM_OBJECT_EVENT_PAGING_IN_PROGRESS); \
1043 MACRO_END
1044
1045#define vm_object_paging_begin(object) \
1c79356b 1046 MACRO_BEGIN \
2d21ac55 1047 vm_object_lock_assert_exclusive((object)); \
0c530ab8 1048 VM_PIP_DEBUG_BEGIN((object)); \
1c79356b 1049 (object)->paging_in_progress++; \
fe8ab488
A		 1050 if ((object)->paging_in_progress == 0) { \
1051 panic("vm_object_paging_begin(%p): overflow\n", (object));\
1052 } \
1c79356b
A		 1053 MACRO_END
1054
b0d623f7 1055#define vm_object_paging_end(object) \
1c79356b 1056 MACRO_BEGIN \
2d21ac55 1057 vm_object_lock_assert_exclusive((object)); \
fe8ab488
A		 1058 if ((object)->paging_in_progress == 0) { \
1059 panic("vm_object_paging_end(%p): underflow\n", (object));\
1060 } \
b0d623f7
A		 1061 (object)->paging_in_progress--; \
1062 if ((object)->paging_in_progress == 0) { \
1063 vm_object_wakeup((object), \
1064 VM_OBJECT_EVENT_PAGING_ONLY_IN_PROGRESS); \
1065 if ((object)->activity_in_progress == 0) \
1066 vm_object_wakeup((object), \
1067 VM_OBJECT_EVENT_PAGING_IN_PROGRESS); \
1c79356b
A		 1068 } \
1069 MACRO_END
1070
1071#define vm_object_paging_wait(object, interruptible) \
1072 MACRO_BEGIN \
2d21ac55 1073 vm_object_lock_assert_exclusive((object)); \
b0d623f7
A		 1074 while ((object)->paging_in_progress != 0 || \
1075 (object)->activity_in_progress != 0) { \
9bccf70c
A		 1076 wait_result_t _wr; \
1077 \
1078 _wr = vm_object_sleep((object), \
1c79356b
A		 1079 VM_OBJECT_EVENT_PAGING_IN_PROGRESS, \
1080 (interruptible)); \
1c79356b 1081 \
9bccf70c 1082 /*XXX if ((interruptible) && (_wr != THREAD_AWAKENED))*/\
1c79356b
A		 1083 /*XXX break; */ \
1084 } \
1085 MACRO_END
1086
b0d623f7
A		 1087#define vm_object_paging_only_wait(object, interruptible) \
1088 MACRO_BEGIN \
1089 vm_object_lock_assert_exclusive((object)); \
1090 while ((object)->paging_in_progress != 0) { \
1091 wait_result_t _wr; \
1092 \
1093 _wr = vm_object_sleep((object), \
1094 VM_OBJECT_EVENT_PAGING_ONLY_IN_PROGRESS,\
1095 (interruptible)); \
1096 \
1097 /*XXX if ((interruptible) && (_wr != THREAD_AWAKENED))*/\
1098 /*XXX break; */ \
1099 } \
1100 MACRO_END
1101
1c79356b 1102
593a1d5f
A		 1103#define vm_object_mapping_begin(object) \
1104 MACRO_BEGIN \
1105 vm_object_lock_assert_exclusive((object)); \
1106 assert(! (object)->mapping_in_progress); \
1107 (object)->mapping_in_progress = TRUE; \
1108 MACRO_END
1109
1110#define vm_object_mapping_end(object) \
1111 MACRO_BEGIN \
1112 vm_object_lock_assert_exclusive((object)); \
1113 assert((object)->mapping_in_progress); \
1114 (object)->mapping_in_progress = FALSE; \
1115 vm_object_wakeup((object), \
1116 VM_OBJECT_EVENT_MAPPING_IN_PROGRESS); \
1117 MACRO_END
1118
1119#define vm_object_mapping_wait(object, interruptible) \
1120 MACRO_BEGIN \
1121 vm_object_lock_assert_exclusive((object)); \
1122 while ((object)->mapping_in_progress) { \
1123 wait_result_t _wr; \
1124 \
1125 _wr = vm_object_sleep((object), \
1126 VM_OBJECT_EVENT_MAPPING_IN_PROGRESS, \
1127 (interruptible)); \
1128 /*XXX if ((interruptible) && (_wr != THREAD_AWAKENED))*/\
1129 /*XXX break; */ \
1130 } \
1131 assert(!(object)->mapping_in_progress); \
1132 MACRO_END
1133
1134
1c79356b 1135
91447636
A		 1136#define vm_object_round_page(x) (((vm_object_offset_t)(x) + PAGE_MASK) & ~((signed)PAGE_MASK))
1137#define vm_object_trunc_page(x) ((vm_object_offset_t)(x) & ~((signed)PAGE_MASK))
1138
6d2010ae
A		 1139extern void vm_object_cache_add(vm_object_t);
1140extern void vm_object_cache_remove(vm_object_t);
1141extern int vm_object_cache_evict(int, int);
1142
1c79356b 1143#endif /* _VM_VM_OBJECT_H_ */
mirror of XNU