]> git.saurik.com Git - apple/xnu.git/blame - osfmk/vm/vm_object.h
projects / apple / xnu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
xnu-3248.20.55.tar.gz
[apple/xnu.git] / osfmk / vm / vm_object.h
CommitLineData
1c79356b 1/*
2d21ac55 2 * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
1c79356b 3 *
2d21ac55 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
1c79356b 5 *
2d21ac55
A		 6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
8f6c56a5 14 *
2d21ac55
A		 15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
8f6c56a5
A		 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
2d21ac55
A		 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
8f6c56a5 25 *
2d21ac55 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
1c79356b
A		 27 */
28/*
29 * @OSF_COPYRIGHT@
30 */
31/*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56/*
57 */
58/*
59 * File: vm_object.h
60 * Author: Avadis Tevanian, Jr., Michael Wayne Young
61 * Date: 1985
62 *
63 * Virtual memory object module definitions.
64 */
65
66#ifndef _VM_VM_OBJECT_H_
67#define _VM_VM_OBJECT_H_
68
6d2010ae
A		 69#include <debug.h>
70#include <mach_assert.h>
1c79356b
A		 71#include <mach_pagemap.h>
72#include <task_swapper.h>
73
74#include <mach/kern_return.h>
75#include <mach/boolean.h>
76#include <mach/memory_object_types.h>
77#include <mach/port.h>
78#include <mach/vm_prot.h>
91447636 79#include <mach/vm_param.h>
1c79356b
A		 80#include <mach/machine/vm_types.h>
81#include <kern/queue.h>
2d21ac55 82#include <kern/locks.h>
1c79356b 83#include <kern/assert.h>
0b4e3aa0 84#include <kern/misc_protos.h>
1c79356b
A		 85#include <kern/macro_help.h>
86#include <ipc/ipc_types.h>
87#include <vm/pmap.h>
1c79356b 88
1c79356b 89#include <vm/vm_external.h>
1c79356b 90
b0d623f7
A		 91#include <vm/vm_options.h>
92
fe8ab488
A		 93#if VM_OBJECT_TRACKING
94#include <libkern/OSDebug.h>
95#include <kern/btlog.h>
96extern void vm_object_tracking_init(void);
97extern boolean_t vm_object_tracking_inited;
98extern btlog_t *vm_object_tracking_btlog;
99#define VM_OBJECT_TRACKING_BTDEPTH 7
100#define VM_OBJECT_TRACKING_OP_CREATED 1
101#define VM_OBJECT_TRACKING_OP_MODIFIED 2
102#define VM_OBJECT_TRACKING_OP_TRUESHARE 3
103#endif /* VM_OBJECT_TRACKING */
104
91447636 105struct vm_page;
39236c6e 106struct vm_shared_region_slide_info;
1c79356b
A		 107
108/*
109 * Types defined:
110 *
111 * vm_object_t Virtual memory object.
2d21ac55 112 * vm_object_fault_info_t Used to determine cluster size.
1c79356b
A		 113 */
114
2d21ac55
A		 115struct vm_object_fault_info {
116 int interruptible;
117 uint32_t user_tag;
118 vm_size_t cluster_size;
119 vm_behavior_t behavior;
120 vm_map_offset_t lo_offset;
121 vm_map_offset_t hi_offset;
6d2010ae
A		 122 unsigned int
123 /* boolean_t */ no_cache:1,
124 /* boolean_t */ stealth:1,
125 /* boolean_t */ io_sync:1,
126 /* boolean_t */ cs_bypass:1,
127 /* boolean_t */ mark_zf_absent:1,
316670eb
A		 128 /* boolean_t */ batch_pmap_op:1,
129 __vm_object_fault_info_unused_bits:26;
fe8ab488 130 int pmap_options;
2d21ac55
A		 131};
132
133
39236c6e
A		 134#define vo_size vo_un1.vou_size
135#define vo_cache_pages_to_scan vo_un1.vou_cache_pages_to_scan
136#define vo_shadow_offset vo_un2.vou_shadow_offset
137#define vo_cache_ts vo_un2.vou_cache_ts
138#define vo_purgeable_owner vo_un2.vou_purgeable_owner
139#define vo_slide_info vo_un2.vou_slide_info
2d21ac55 140
1c79356b
A		 141struct vm_object {
142 queue_head_t memq; /* Resident memory */
2d21ac55 143 lck_rw_t Lock; /* Synchronization */
1c79356b 144
6d2010ae
A		 145 union {
146 vm_object_size_t vou_size; /* Object size (only valid if internal) */
147 int vou_cache_pages_to_scan; /* pages yet to be visited in an
148 * external object in cache
149 */
150 } vo_un1;
151
91447636 152 struct vm_page *memq_hint;
1c79356b
A		 153 int ref_count; /* Number of references */
154#if TASK_SWAPPER
155 int res_count; /* Residency references (swap)*/
156#endif /* TASK_SWAPPER */
157 unsigned int resident_page_count;
158 /* number of resident pages */
b0d623f7
A		 159 unsigned int wired_page_count; /* number of wired pages */
160 unsigned int reusable_page_count;
1c79356b
A		 161
162 struct vm_object *copy; /* Object that should receive
163 * a copy of my changed pages,
164 * for copy_delay, or just the
165 * temporary object that
166 * shadows this object, for
167 * copy_call.
168 */
169 struct vm_object *shadow; /* My shadow */
6d2010ae
A		 170
171 union {
172 vm_object_offset_t vou_shadow_offset; /* Offset into shadow */
39236c6e
A		 173 clock_sec_t vou_cache_ts; /* age of an external object
174 * present in cache
175 */
176 task_t vou_purgeable_owner; /* If the purg'a'ble bits below are set
177 * to volatile/emtpy, this is the task
178 * that owns this purgeable object.
6d2010ae 179 */
39236c6e 180 struct vm_shared_region_slide_info *vou_slide_info;
6d2010ae 181 } vo_un2;
1c79356b 182
0b4e3aa0 183 memory_object_t pager; /* Where to get data */
1c79356b 184 vm_object_offset_t paging_offset; /* Offset into memory object */
91447636 185 memory_object_control_t pager_control; /* Where data comes back */
1c79356b
A		 186
187 memory_object_copy_strategy_t
188 copy_strategy; /* How to handle data copy */
189
fe8ab488
A		 190#if __LP64__
191 /*
192 * Some user processes (mostly VirtualMachine software) take a large
193 * number of UPLs (via IOMemoryDescriptors) to wire pages in large
194 * VM objects and overflow the 16-bit "activity_in_progress" counter.
195 * Since we never enforced any limit there, let's give them 32 bits
196 * for backwards compatibility's sake.
197 */
198 unsigned int paging_in_progress:16,
199 __object1_unused_bits:16;
200 unsigned int activity_in_progress;
201#else /* __LP64__ */
202 /*
203 * On 32-bit platforms, enlarging "activity_in_progress" would increase
204 * the size of "struct vm_object". Since we don't know of any actual
205 * overflow of these counters on these platforms, let's keep the
206 * counters as 16-bit integers.
207 */
208 unsigned short paging_in_progress;
209 unsigned short activity_in_progress;
210#endif /* __LP64__ */
1c79356b
A		 211 /* The memory object ports are
212 * being used (e.g., for pagein
213 * or pageout) -- don't change
214 * any of these fields (i.e.,
215 * don't collapse, destroy or
216 * terminate)
217 */
b0d623f7 218
1c79356b
A		 219 unsigned int
220 /* boolean_t array */ all_wanted:11, /* Bit array of "want to be
221 * awakened" notations. See
222 * VM_OBJECT_EVENT_* items
223 * below */
224 /* boolean_t */ pager_created:1, /* Has pager been created? */
225 /* boolean_t */ pager_initialized:1, /* Are fields ready to use? */
226 /* boolean_t */ pager_ready:1, /* Will pager take requests? */
227
228 /* boolean_t */ pager_trusted:1,/* The pager for this object
229 * is trusted. This is true for
230 * all internal objects (backed
231 * by the default pager)
232 */
233 /* boolean_t */ can_persist:1, /* The kernel may keep the data
234 * for this object (and rights
235 * to the memory object) after
236 * all address map references
237 * are deallocated?
238 */
239 /* boolean_t */ internal:1, /* Created by the kernel (and
240 * therefore, managed by the
241 * default memory manger)
242 */
243 /* boolean_t */ temporary:1, /* Permanent objects may be
244 * changed externally by the
245 * memory manager, and changes
246 * made in memory must be
247 * reflected back to the memory
248 * manager. Temporary objects
249 * lack both of these
250 * characteristics.
251 */
252 /* boolean_t */ private:1, /* magic device_pager object,
253 * holds private pages only */
254 /* boolean_t */ pageout:1, /* pageout object. contains
255 * private pages that refer to
256 * a real memory object. */
257 /* boolean_t */ alive:1, /* Not yet terminated */
258
91447636 259 /* boolean_t */ purgable:2, /* Purgable state. See
2d21ac55 260 * VM_PURGABLE_*
1c79356b 261 */
39236c6e
A		 262 /* boolean_t */ purgeable_when_ripe:1, /* Purgeable when a token
263 * becomes ripe.
264 */
1c79356b 265 /* boolean_t */ shadowed:1, /* Shadow may exist */
1c79356b
A		 266 /* boolean_t */ advisory_pageout:1,
267 /* Instead of sending page
268 * via OOL, just notify
269 * pager that the kernel
270 * wants to discard it, page
271 * remains in object */
272 /* boolean_t */ true_share:1,
273 /* This object is mapped
274 * in more than one place
275 * and hence cannot be
276 * coalesced */
277 /* boolean_t */ terminating:1,
278 /* Allows vm_object_lookup
279 * and vm_object_deallocate
280 * to special case their
281 * behavior when they are
282 * called as a result of
283 * page cleaning during
284 * object termination
285 */
286 /* boolean_t */ named:1, /* An enforces an internal
287 * naming convention, by
288 * calling the right routines
289 * for allocation and
290 * destruction, UBC references
291 * against the vm_object are
292 * checked.
293 */
294 /* boolean_t */ shadow_severed:1,
295 /* When a permanent object
296 * backing a COW goes away
297 * unexpectedly. This bit
298 * allows vm_fault to return
299 * an error rather than a
300 * zero filled page.
301 */
0b4e3aa0 302 /* boolean_t */ phys_contiguous:1,
1c79356b
A		 303 /* Memory is wired and
304 * guaranteed physically
305 * contiguous. However
306 * it is not device memory
307 * and obeys normal virtual
308 * memory rules w.r.t pmap
309 * access bits.
310 */
0b4e3aa0
A		 311 /* boolean_t */ nophyscache:1;
312 /* When mapped at the
313 * pmap level, don't allow
314 * primary caching. (for
315 * I/O)
316 */
1c79356b
A		 317
318
319
320 queue_chain_t cached_list; /* Attachment point for the
321 * list of objects cached as a
322 * result of their can_persist
323 * value
324 */
325
326 queue_head_t msr_q; /* memory object synchronise
327 request queue */
328
2d21ac55
A		 329 /*
330 * the following fields are not protected by any locks
331 * they are updated via atomic compare and swap
332 */
1c79356b 333 vm_object_offset_t last_alloc; /* last allocation offset */
2d21ac55
A		 334 int sequential; /* sequential access size */
335
336 uint32_t pages_created;
337 uint32_t pages_used;
1c79356b
A		 338#if MACH_PAGEMAP
339 vm_external_map_t existence_map; /* bitmap of pages written to
340 * backing storage */
341#endif /* MACH_PAGEMAP */
55e303ae 342 vm_offset_t cow_hint; /* last page present in */
0b4e3aa0 343 /* shadow but not in object */
1c79356b
A		 344#if MACH_ASSERT
345 struct vm_object *paging_object; /* object which pages to be
346 * swapped out are temporary
347 * put in current object
348 */
349#endif
2d21ac55
A		 350 /* hold object lock when altering */
351 unsigned int
352 wimg_bits:8, /* cache WIMG bits */
353 code_signed:1, /* pages are signed and should be
354 validated; the signatures are stored
355 with the pager */
b0d623f7
A		 356 hashed:1, /* object/pager entered in hash */
357 transposed:1, /* object was transposed with another */
593a1d5f 358 mapping_in_progress:1, /* pager being mapped/unmapped */
fe8ab488 359 phantom_isssd:1,
b0d623f7
A		 360 volatile_empty:1,
361 volatile_fault:1,
362 all_reusable:1,
363 blocked_access:1,
6d2010ae 364 set_cache_attr:1,
39236c6e
A		 365 object_slid:1,
366 purgeable_queue_type:2,
367 purgeable_queue_group:3,
fe8ab488
A		 368 io_tracking:1,
369 __object2_unused_bits:7; /* for expansion */
6d2010ae 370
3e170ce0
A		 371 uint8_t scan_collisions;
372 vm_tag_t wire_tag;
373 uint8_t __object4_unused_bits[2];
374
fe8ab488
A		 375#if CONFIG_PHANTOM_CACHE
376 uint32_t phantom_object_id;
377#endif
378#if CONFIG_IOSCHED || UPL_DEBUG
1c79356b 379 queue_head_t uplq; /* List of outstanding upls */
fe8ab488 380#endif
2d21ac55 381
0c530ab8
A		 382#ifdef VM_PIP_DEBUG
383/*
384 * Keep track of the stack traces for the first holders
385 * of a "paging_in_progress" reference for this VM object.
386 */
387#define VM_PIP_DEBUG_STACK_FRAMES 25 /* depth of each stack trace */
388#define VM_PIP_DEBUG_MAX_REFS 10 /* track that many references */
389 struct __pip_backtrace {
390 void *pip_retaddr[VM_PIP_DEBUG_STACK_FRAMES];
391 } pip_holders[VM_PIP_DEBUG_MAX_REFS];
392#endif /* VM_PIP_DEBUG */
2d21ac55 393
6d2010ae 394 queue_chain_t objq; /* object queue - currently used for purgable queues */
fe8ab488
A		 395
396#if DEBUG
397 void *purgeable_owner_bt[16];
398 task_t vo_purgeable_volatilizer; /* who made it volatile? */
399 void *purgeable_volatilizer_bt[16];
400#endif /* DEBUG */
1c79356b
A		 401};
402
b0d623f7
A		 403#define VM_OBJECT_PURGEABLE_FAULT_ERROR(object) \
404 ((object)->volatile_fault && \
405 ((object)->purgable == VM_PURGABLE_VOLATILE || \
406 (object)->purgable == VM_PURGABLE_EMPTY))
407
39236c6e 408extern
1c79356b
A		 409vm_object_t kernel_object; /* the single kernel object */
410
39236c6e
A		 411extern
412vm_object_t compressor_object; /* the single compressor object */
413
414extern
91447636 415unsigned int vm_object_absent_max; /* maximum number of absent pages
1c79356b
A		 416 at a time for each object */
417
418# define VM_MSYNC_INITIALIZED 0
419# define VM_MSYNC_SYNCHRONIZING 1
420# define VM_MSYNC_DONE 2
421
422struct msync_req {
423 queue_chain_t msr_q; /* object request queue */
424 queue_chain_t req_q; /* vm_msync request queue */
425 unsigned int flag;
426 vm_object_offset_t offset;
427 vm_object_size_t length;
428 vm_object_t object; /* back pointer */
b0d623f7 429 decl_lck_mtx_data(, msync_req_lock) /* Lock for this structure */
1c79356b
A		 430};
431
432typedef struct msync_req *msync_req_t;
433#define MSYNC_REQ_NULL ((msync_req_t) 0)
434
b0d623f7
A		 435
436extern lck_grp_t vm_map_lck_grp;
437extern lck_attr_t vm_map_lck_attr;
438
1c79356b
A		 439/*
440 * Macros to allocate and free msync_reqs
441 */
442#define msync_req_alloc(msr) \
b0d623f7 443 MACRO_BEGIN \
1c79356b 444 (msr) = (msync_req_t)kalloc(sizeof(struct msync_req)); \
b0d623f7
A		 445 lck_mtx_init(&(msr)->msync_req_lock, &vm_map_lck_grp, &vm_map_lck_attr); \
446 msr->flag = VM_MSYNC_INITIALIZED; \
447 MACRO_END
1c79356b
A		 448
449#define msync_req_free(msr) \
39236c6e
A		 450 MACRO_BEGIN \
451 lck_mtx_destroy(&(msr)->msync_req_lock, &vm_map_lck_grp); \
452 kfree((msr), sizeof(struct msync_req)); \
453 MACRO_END
1c79356b 454
b0d623f7
A		 455#define msr_lock(msr) lck_mtx_lock(&(msr)->msync_req_lock)
456#define msr_unlock(msr) lck_mtx_unlock(&(msr)->msync_req_lock)
1c79356b 457
3e170ce0
A		 458#define VM_OBJECT_WIRED(object) \
459 MACRO_BEGIN \
460 if ((object)->purgable == VM_PURGABLE_DENY) \
461 { \
462 lck_spin_lock(&vm_objects_wired_lock); \
463 assert(!(object)->objq.next); \
464 queue_enter(&vm_objects_wired, (object), vm_object_t, objq); \
465 lck_spin_unlock(&vm_objects_wired_lock); \
466 } \
467 MACRO_END
468
469#define VM_OBJECT_UNWIRED(object) \
470 MACRO_BEGIN \
471 (object)->wire_tag = VM_KERN_MEMORY_NONE; \
472 if (((object)->purgable == VM_PURGABLE_DENY) && (object)->objq.next) \
473 { \
474 lck_spin_lock(&vm_objects_wired_lock); \
475 queue_remove(&vm_objects_wired, (object), vm_object_t, objq); \
476 lck_spin_unlock(&vm_objects_wired_lock); \
477 } \
478 MACRO_END
479
480
1c79356b
A		 481/*
482 * Declare procedures that operate on VM objects.
483 */
484
39236c6e 485__private_extern__ void vm_object_bootstrap(void);
1c79356b 486
0b4e3aa0 487__private_extern__ void vm_object_init(void);
1c79356b 488
2d21ac55
A		 489__private_extern__ void vm_object_init_lck_grp(void);
490
8f6c56a5
A		 491__private_extern__ void vm_object_reaper_init(void);
492
3e170ce0 493__private_extern__ vm_object_t vm_object_allocate(vm_object_size_t size);
1c79356b 494
91447636
A		 495__private_extern__ void _vm_object_allocate(vm_object_size_t size,
496 vm_object_t object);
497
0b4e3aa0
A		 498#if TASK_SWAPPER
499
500__private_extern__ void vm_object_res_reference(
501 vm_object_t object);
502__private_extern__ void vm_object_res_deallocate(
503 vm_object_t object);
504#define VM_OBJ_RES_INCR(object) (object)->res_count++
505#define VM_OBJ_RES_DECR(object) (object)->res_count--
506
507#else /* TASK_SWAPPER */
508
509#define VM_OBJ_RES_INCR(object)
510#define VM_OBJ_RES_DECR(object)
511#define vm_object_res_reference(object)
512#define vm_object_res_deallocate(object)
513
514#endif /* TASK_SWAPPER */
515
516#define vm_object_reference_locked(object) \
2d21ac55
A		 517 MACRO_BEGIN \
518 vm_object_t RLObject = (object); \
519 vm_object_lock_assert_exclusive(object); \
520 assert((RLObject)->ref_count > 0); \
521 (RLObject)->ref_count++; \
522 assert((RLObject)->ref_count > 1); \
523 vm_object_res_reference(RLObject); \
524 MACRO_END
525
526
527#define vm_object_reference_shared(object) \
528 MACRO_BEGIN \
529 vm_object_t RLObject = (object); \
530 vm_object_lock_assert_shared(object); \
531 assert((RLObject)->ref_count > 0); \
b0d623f7 532 OSAddAtomic(1, &(RLObject)->ref_count); \
316670eb 533 assert((RLObject)->ref_count > 0); \
2d21ac55
A		 534 /* XXX we would need an atomic version of the following ... */ \
535 vm_object_res_reference(RLObject); \
536 MACRO_END
0b4e3aa0
A		 537
538
0b4e3aa0 539__private_extern__ void vm_object_reference(
1c79356b 540 vm_object_t object);
0b4e3aa0 541
91447636 542#if !MACH_ASSERT
0b4e3aa0 543
1c79356b
A		 544#define vm_object_reference(object) \
545MACRO_BEGIN \
0b4e3aa0
A		 546 vm_object_t RObject = (object); \
547 if (RObject) { \
b0d623f7
A		 548 vm_object_lock_shared(RObject); \
549 vm_object_reference_shared(RObject); \
0b4e3aa0 550 vm_object_unlock(RObject); \
1c79356b
A		 551 } \
552MACRO_END
0b4e3aa0 553
1c79356b
A		 554#endif /* MACH_ASSERT */
555
0b4e3aa0 556__private_extern__ void vm_object_deallocate(
1c79356b
A		 557 vm_object_t object);
558
0b4e3aa0
A		 559__private_extern__ kern_return_t vm_object_release_name(
560 vm_object_t object,
561 int flags);
562
563__private_extern__ void vm_object_pmap_protect(
1c79356b
A		 564 vm_object_t object,
565 vm_object_offset_t offset,
91447636 566 vm_object_size_t size,
1c79356b 567 pmap_t pmap,
91447636 568 vm_map_offset_t pmap_start,
1c79356b
A		 569 vm_prot_t prot);
570
39236c6e
A		 571__private_extern__ void vm_object_pmap_protect_options(
572 vm_object_t object,
573 vm_object_offset_t offset,
574 vm_object_size_t size,
575 pmap_t pmap,
576 vm_map_offset_t pmap_start,
577 vm_prot_t prot,
578 int options);
579
0b4e3aa0 580__private_extern__ void vm_object_page_remove(
1c79356b
A		 581 vm_object_t object,
582 vm_object_offset_t start,
583 vm_object_offset_t end);
584
0b4e3aa0
A		 585__private_extern__ void vm_object_deactivate_pages(
586 vm_object_t object,
587 vm_object_offset_t offset,
588 vm_object_size_t size,
b0d623f7 589 boolean_t kill_page,
3e170ce0
A		 590 boolean_t reusable_page,
591 struct pmap *pmap,
592 vm_map_offset_t pmap_offset);
b0d623f7
A		 593
594__private_extern__ void vm_object_reuse_pages(
595 vm_object_t object,
596 vm_object_offset_t start_offset,
597 vm_object_offset_t end_offset,
598 boolean_t allow_partial_reuse);
0b4e3aa0 599
b0d623f7 600__private_extern__ void vm_object_purge(
fe8ab488
A		 601 vm_object_t object,
602 int flags);
91447636
A		 603
604__private_extern__ kern_return_t vm_object_purgable_control(
605 vm_object_t object,
606 vm_purgable_t control,
607 int *state);
608
39236c6e
A		 609__private_extern__ kern_return_t vm_object_get_page_counts(
610 vm_object_t object,
611 vm_object_offset_t offset,
612 vm_object_size_t size,
613 unsigned int *resident_page_count,
614 unsigned int *dirty_page_count);
615
0b4e3aa0 616__private_extern__ boolean_t vm_object_coalesce(
1c79356b
A		 617 vm_object_t prev_object,
618 vm_object_t next_object,
619 vm_object_offset_t prev_offset,
620 vm_object_offset_t next_offset,
621 vm_object_size_t prev_size,
622 vm_object_size_t next_size);
623
0b4e3aa0 624__private_extern__ boolean_t vm_object_shadow(
1c79356b
A		 625 vm_object_t *object,
626 vm_object_offset_t *offset,
627 vm_object_size_t length);
628
0b4e3aa0 629__private_extern__ void vm_object_collapse(
55e303ae 630 vm_object_t object,
0c530ab8
A		 631 vm_object_offset_t offset,
632 boolean_t can_bypass);
1c79356b 633
0b4e3aa0 634__private_extern__ boolean_t vm_object_copy_quickly(
1c79356b
A		 635 vm_object_t *_object,
636 vm_object_offset_t src_offset,
637 vm_object_size_t size,
638 boolean_t *_src_needs_copy,
639 boolean_t *_dst_needs_copy);
640
0b4e3aa0 641__private_extern__ kern_return_t vm_object_copy_strategically(
1c79356b
A		 642 vm_object_t src_object,
643 vm_object_offset_t src_offset,
644 vm_object_size_t size,
645 vm_object_t *dst_object,
646 vm_object_offset_t *dst_offset,
647 boolean_t *dst_needs_copy);
648
0b4e3aa0 649__private_extern__ kern_return_t vm_object_copy_slowly(
1c79356b
A		 650 vm_object_t src_object,
651 vm_object_offset_t src_offset,
652 vm_object_size_t size,
b0d623f7 653 boolean_t interruptible,
1c79356b
A		 654 vm_object_t *_result_object);
655
0b4e3aa0
A		 656__private_extern__ vm_object_t vm_object_copy_delayed(
657 vm_object_t src_object,
658 vm_object_offset_t src_offset,
2d21ac55
A		 659 vm_object_size_t size,
660 boolean_t src_object_shared);
0b4e3aa0 661
1c79356b 662
1c79356b 663
0b4e3aa0
A		 664__private_extern__ kern_return_t vm_object_destroy(
665 vm_object_t object,
666 kern_return_t reason);
1c79356b 667
0b4e3aa0
A		 668__private_extern__ void vm_object_pager_create(
669 vm_object_t object);
670
39236c6e
A		 671__private_extern__ void vm_object_compressor_pager_create(
672 vm_object_t object);
673
0b4e3aa0 674__private_extern__ void vm_object_page_map(
1c79356b
A		 675 vm_object_t object,
676 vm_object_offset_t offset,
677 vm_object_size_t size,
678 vm_object_offset_t (*map_fn)
679 (void *, vm_object_offset_t),
680 void *map_fn_data);
681
0b4e3aa0
A		 682__private_extern__ kern_return_t vm_object_upl_request(
683 vm_object_t object,
684 vm_object_offset_t offset,
91447636 685 upl_size_t size,
0b4e3aa0
A		 686 upl_t *upl,
687 upl_page_info_t *page_info,
688 unsigned int *count,
3e170ce0 689 upl_control_flags_t flags);
0b4e3aa0 690
91447636
A		 691__private_extern__ kern_return_t vm_object_transpose(
692 vm_object_t object1,
693 vm_object_t object2,
694 vm_object_size_t transpose_size);
695
0b4e3aa0
A		 696__private_extern__ boolean_t vm_object_sync(
697 vm_object_t object,
698 vm_object_offset_t offset,
91447636 699 vm_object_size_t size,
0b4e3aa0 700 boolean_t should_flush,
91447636
A		 701 boolean_t should_return,
702 boolean_t should_iosync);
1c79356b 703
0b4e3aa0
A		 704__private_extern__ kern_return_t vm_object_update(
705 vm_object_t object,
706 vm_object_offset_t offset,
91447636
A		 707 vm_object_size_t size,
708 vm_object_offset_t *error_offset,
709 int *io_errno,
0b4e3aa0
A		 710 memory_object_return_t should_return,
711 int flags,
712 vm_prot_t prot);
1c79356b 713
0b4e3aa0
A		 714__private_extern__ kern_return_t vm_object_lock_request(
715 vm_object_t object,
716 vm_object_offset_t offset,
717 vm_object_size_t size,
718 memory_object_return_t should_return,
719 int flags,
720 vm_prot_t prot);
1c79356b 721
1c79356b 722
1c79356b 723
0b4e3aa0
A		 724__private_extern__ vm_object_t vm_object_enter(
725 memory_object_t pager,
1c79356b
A		 726 vm_object_size_t size,
727 boolean_t internal,
728 boolean_t init,
729 boolean_t check_named);
730
731
2d21ac55
A		 732__private_extern__ void vm_object_cluster_size(
733 vm_object_t object,
734 vm_object_offset_t *start,
735 vm_size_t *length,
b0d623f7
A		 736 vm_object_fault_info_t fault_info,
737 uint32_t *io_streaming);
91447636
A		 738
739__private_extern__ kern_return_t vm_object_populate_with_private(
740 vm_object_t object,
741 vm_object_offset_t offset,
742 ppnum_t phys_page,
743 vm_size_t size);
744
6d2010ae
A		 745__private_extern__ void vm_object_change_wimg_mode(
746 vm_object_t object,
747 unsigned int wimg_mode);
748
0c530ab8 749extern kern_return_t adjust_vm_object_cache(
91447636
A		 750 vm_size_t oval,
751 vm_size_t nval);
752
0c530ab8
A		 753extern kern_return_t vm_object_page_op(
754 vm_object_t object,
755 vm_object_offset_t offset,
756 int ops,
757 ppnum_t *phys_entry,
758 int *flags);
759
760extern kern_return_t vm_object_range_op(
761 vm_object_t object,
762 vm_object_offset_t offset_beg,
763 vm_object_offset_t offset_end,
764 int ops,
b0d623f7
A		 765 uint32_t *range);
766
767
768__private_extern__ void vm_object_reap_pages(
769 vm_object_t object,
770 int reap_type);
771#define REAP_REAP 0
772#define REAP_TERMINATE 1
773#define REAP_PURGEABLE 2
774#define REAP_DATA_FLUSH 3
775
6d2010ae 776#if CONFIG_FREEZE
316670eb 777struct default_freezer_handle;
6d2010ae
A		 778
779__private_extern__ kern_return_t
780vm_object_pack(
316670eb
A		 781 unsigned int *purgeable_count,
782 unsigned int *wired_count,
783 unsigned int *clean_count,
784 unsigned int *dirty_count,
785 unsigned int dirty_budget,
786 boolean_t *shared,
787 vm_object_t src_object,
788 struct default_freezer_handle *df_handle);
6d2010ae
A		 789
790__private_extern__ void
791vm_object_pack_pages(
316670eb
A		 792 unsigned int *wired_count,
793 unsigned int *clean_count,
794 unsigned int *dirty_count,
795 unsigned int dirty_budget,
796 vm_object_t src_object,
797 struct default_freezer_handle *df_handle);
6d2010ae 798
316670eb 799__private_extern__ void
3e170ce0 800vm_object_compressed_freezer_pageout(
316670eb 801 vm_object_t object);
6d2010ae 802
3e170ce0
A		 803__private_extern__ void
804vm_object_compressed_freezer_done(
805 void);
806
316670eb
A		 807__private_extern__ kern_return_t
808vm_object_pagein(
809 vm_object_t object);
6d2010ae 810#endif /* CONFIG_FREEZE */
0c530ab8 811
3e170ce0
A		 812__private_extern__ void
813vm_object_pageout(
814 vm_object_t object);
815
fe8ab488
A		 816#if CONFIG_IOSCHED
817struct io_reprioritize_req {
818 uint64_t blkno;
819 uint32_t len;
820 int priority;
821 struct vnode *devvp;
822 queue_chain_t io_reprioritize_list;
823};
824typedef struct io_reprioritize_req *io_reprioritize_req_t;
825
826extern void vm_io_reprioritize_init(void);
827#endif
828
1c79356b
A		 829/*
830 * Event waiting handling
831 */
832
833#define VM_OBJECT_EVENT_INITIALIZED 0
834#define VM_OBJECT_EVENT_PAGER_READY 1
835#define VM_OBJECT_EVENT_PAGING_IN_PROGRESS 2
593a1d5f 836#define VM_OBJECT_EVENT_MAPPING_IN_PROGRESS 3
1c79356b
A		 837#define VM_OBJECT_EVENT_LOCK_IN_PROGRESS 4
838#define VM_OBJECT_EVENT_UNCACHING 5
839#define VM_OBJECT_EVENT_COPY_CALL 6
840#define VM_OBJECT_EVENT_CACHING 7
b0d623f7
A		 841#define VM_OBJECT_EVENT_UNBLOCKED 8
842#define VM_OBJECT_EVENT_PAGING_ONLY_IN_PROGRESS 9
1c79356b
A		 843
844#define vm_object_assert_wait(object, event, interruptible) \
9bccf70c
A		 845 (((object)->all_wanted |= 1 << (event)), \
846 assert_wait((event_t)((vm_offset_t)(object)+(event)),(interruptible)))
1c79356b
A		 847
848#define vm_object_wait(object, event, interruptible) \
9bccf70c
A		 849 (vm_object_assert_wait((object),(event),(interruptible)), \
850 vm_object_unlock(object), \
851 thread_block(THREAD_CONTINUE_NULL)) \
852
853#define thread_sleep_vm_object(object, event, interruptible) \
fe8ab488 854 lck_rw_sleep(&(object)->Lock, LCK_SLEEP_PROMOTED_PRI, (event_t)(event), (interruptible))
9bccf70c
A		 855
856#define vm_object_sleep(object, event, interruptible) \
857 (((object)->all_wanted |= 1 << (event)), \
858 thread_sleep_vm_object((object), \
859 ((vm_offset_t)(object)+(event)), (interruptible)))
1c79356b
A		 860
861#define vm_object_wakeup(object, event) \
862 MACRO_BEGIN \
863 if ((object)->all_wanted & (1 << (event))) \
864 thread_wakeup((event_t)((vm_offset_t)(object) + (event))); \
865 (object)->all_wanted &= ~(1 << (event)); \
866 MACRO_END
867
868#define vm_object_set_wanted(object, event) \
869 MACRO_BEGIN \
870 ((object)->all_wanted |= (1 << (event))); \
871 MACRO_END
872
873#define vm_object_wanted(object, event) \
874 ((object)->all_wanted & (1 << (event)))
875
876/*
877 * Routines implemented as macros
878 */
0c530ab8 879#ifdef VM_PIP_DEBUG
2d21ac55 880#include <libkern/OSDebug.h>
0c530ab8
A		 881#define VM_PIP_DEBUG_BEGIN(object) \
882 MACRO_BEGIN \
b0d623f7
A		 883 int pip = ((object)->paging_in_progress + \
884 (object)->activity_in_progress); \
885 if (pip < VM_PIP_DEBUG_MAX_REFS) { \
2d21ac55 886 (void) OSBacktrace(&(object)->pip_holders[pip].pip_retaddr[0], \
0c530ab8
A		 887 VM_PIP_DEBUG_STACK_FRAMES); \
888 } \
889 MACRO_END
890#else /* VM_PIP_DEBUG */
891#define VM_PIP_DEBUG_BEGIN(object)
892#endif /* VM_PIP_DEBUG */
1c79356b 893
b0d623f7
A		 894#define vm_object_activity_begin(object) \
895 MACRO_BEGIN \
896 vm_object_lock_assert_exclusive((object)); \
b0d623f7
A		 897 VM_PIP_DEBUG_BEGIN((object)); \
898 (object)->activity_in_progress++; \
fe8ab488
A		 899 if ((object)->activity_in_progress == 0) { \
900 panic("vm_object_activity_begin(%p): overflow\n", (object));\
901 } \
b0d623f7
A		 902 MACRO_END
903
904#define vm_object_activity_end(object) \
905 MACRO_BEGIN \
906 vm_object_lock_assert_exclusive((object)); \
fe8ab488
A		 907 if ((object)->activity_in_progress == 0) { \
908 panic("vm_object_activity_end(%p): underflow\n", (object));\
909 } \
b0d623f7
A		 910 (object)->activity_in_progress--; \
911 if ((object)->paging_in_progress == 0 && \
912 (object)->activity_in_progress == 0) \
913 vm_object_wakeup((object), \
914 VM_OBJECT_EVENT_PAGING_IN_PROGRESS); \
915 MACRO_END
916
917#define vm_object_paging_begin(object) \
1c79356b 918 MACRO_BEGIN \
2d21ac55 919 vm_object_lock_assert_exclusive((object)); \
0c530ab8 920 VM_PIP_DEBUG_BEGIN((object)); \
1c79356b 921 (object)->paging_in_progress++; \
fe8ab488
A		 922 if ((object)->paging_in_progress == 0) { \
923 panic("vm_object_paging_begin(%p): overflow\n", (object));\
924 } \
1c79356b
A		 925 MACRO_END
926
b0d623f7 927#define vm_object_paging_end(object) \
1c79356b 928 MACRO_BEGIN \
2d21ac55 929 vm_object_lock_assert_exclusive((object)); \
fe8ab488
A		 930 if ((object)->paging_in_progress == 0) { \
931 panic("vm_object_paging_end(%p): underflow\n", (object));\
932 } \
b0d623f7
A		 933 (object)->paging_in_progress--; \
934 if ((object)->paging_in_progress == 0) { \
935 vm_object_wakeup((object), \
936 VM_OBJECT_EVENT_PAGING_ONLY_IN_PROGRESS); \
937 if ((object)->activity_in_progress == 0) \
938 vm_object_wakeup((object), \
939 VM_OBJECT_EVENT_PAGING_IN_PROGRESS); \
1c79356b
A		 940 } \
941 MACRO_END
942
943#define vm_object_paging_wait(object, interruptible) \
944 MACRO_BEGIN \
2d21ac55 945 vm_object_lock_assert_exclusive((object)); \
b0d623f7
A		 946 while ((object)->paging_in_progress != 0 || \
947 (object)->activity_in_progress != 0) { \
9bccf70c
A		 948 wait_result_t _wr; \
949 \
950 _wr = vm_object_sleep((object), \
1c79356b
A		 951 VM_OBJECT_EVENT_PAGING_IN_PROGRESS, \
952 (interruptible)); \
1c79356b 953 \
9bccf70c 954 /*XXX if ((interruptible) && (_wr != THREAD_AWAKENED))*/\
1c79356b
A		 955 /*XXX break; */ \
956 } \
957 MACRO_END
958
b0d623f7
A		 959#define vm_object_paging_only_wait(object, interruptible) \
960 MACRO_BEGIN \
961 vm_object_lock_assert_exclusive((object)); \
962 while ((object)->paging_in_progress != 0) { \
963 wait_result_t _wr; \
964 \
965 _wr = vm_object_sleep((object), \
966 VM_OBJECT_EVENT_PAGING_ONLY_IN_PROGRESS,\
967 (interruptible)); \
968 \
969 /*XXX if ((interruptible) && (_wr != THREAD_AWAKENED))*/\
970 /*XXX break; */ \
971 } \
972 MACRO_END
973
1c79356b 974
593a1d5f
A		 975#define vm_object_mapping_begin(object) \
976 MACRO_BEGIN \
977 vm_object_lock_assert_exclusive((object)); \
978 assert(! (object)->mapping_in_progress); \
979 (object)->mapping_in_progress = TRUE; \
980 MACRO_END
981
982#define vm_object_mapping_end(object) \
983 MACRO_BEGIN \
984 vm_object_lock_assert_exclusive((object)); \
985 assert((object)->mapping_in_progress); \
986 (object)->mapping_in_progress = FALSE; \
987 vm_object_wakeup((object), \
988 VM_OBJECT_EVENT_MAPPING_IN_PROGRESS); \
989 MACRO_END
990
991#define vm_object_mapping_wait(object, interruptible) \
992 MACRO_BEGIN \
993 vm_object_lock_assert_exclusive((object)); \
994 while ((object)->mapping_in_progress) { \
995 wait_result_t _wr; \
996 \
997 _wr = vm_object_sleep((object), \
998 VM_OBJECT_EVENT_MAPPING_IN_PROGRESS, \
999 (interruptible)); \
1000 /*XXX if ((interruptible) && (_wr != THREAD_AWAKENED))*/\
1001 /*XXX break; */ \
1002 } \
1003 assert(!(object)->mapping_in_progress); \
1004 MACRO_END
1005
1006
1c79356b 1007
2d21ac55
A		 1008#define OBJECT_LOCK_SHARED 0
1009#define OBJECT_LOCK_EXCLUSIVE 1
1010
1011extern lck_grp_t vm_object_lck_grp;
1012extern lck_grp_attr_t vm_object_lck_grp_attr;
1013extern lck_attr_t vm_object_lck_attr;
1014extern lck_attr_t kernel_object_lck_attr;
39236c6e 1015extern lck_attr_t compressor_object_lck_attr;
2d21ac55
A		 1016
1017extern vm_object_t vm_pageout_scan_wants_object;
1018
1019extern void vm_object_lock(vm_object_t);
1020extern boolean_t vm_object_lock_try(vm_object_t);
b0d623f7
A		 1021extern boolean_t _vm_object_lock_try(vm_object_t);
1022extern boolean_t vm_object_lock_avoid(vm_object_t);
2d21ac55
A		 1023extern void vm_object_lock_shared(vm_object_t);
1024extern boolean_t vm_object_lock_try_shared(vm_object_t);
1c79356b
A		 1025
1026/*
1027 * Object locking macros
1028 */
1029
2d21ac55
A		 1030#define vm_object_lock_init(object) \
1031 lck_rw_init(&(object)->Lock, &vm_object_lck_grp, \
1032 (((object) == kernel_object || \
1033 (object) == vm_submap_object) ? \
1034 &kernel_object_lck_attr : \
39236c6e
A		 1035 (((object) == compressor_object) ? \
1036 &compressor_object_lck_attr : \
1037 &vm_object_lck_attr)))
2d21ac55
A		 1038#define vm_object_lock_destroy(object) lck_rw_destroy(&(object)->Lock, &vm_object_lck_grp)
1039
1040#define vm_object_unlock(object) lck_rw_done(&(object)->Lock)
1041#define vm_object_lock_upgrade(object) lck_rw_lock_shared_to_exclusive(&(object)->Lock)
b0d623f7 1042#define vm_object_lock_try_scan(object) _vm_object_lock_try(object)
2d21ac55
A		 1043
1044/*
1045 * CAUTION: the following vm_object_lock_assert_held*() macros merely
1046 * check if anyone is holding the lock, but the holder may not necessarily
1047 * be the caller...
1048 */
6d2010ae 1049#if MACH_ASSERT || DEBUG
2d21ac55
A		 1050#define vm_object_lock_assert_held(object) \
1051 lck_rw_assert(&(object)->Lock, LCK_RW_ASSERT_HELD)
1052#define vm_object_lock_assert_shared(object) \
1053 lck_rw_assert(&(object)->Lock, LCK_RW_ASSERT_SHARED)
1054#define vm_object_lock_assert_exclusive(object) \
1055 lck_rw_assert(&(object)->Lock, LCK_RW_ASSERT_EXCLUSIVE)
3e170ce0
A		 1056#define vm_object_lock_assert_notheld(object) \
1057 lck_rw_assert(&(object)->Lock, LCK_RW_ASSERT_NOTHELD)
6d2010ae 1058#else /* MACH_ASSERT || DEBUG */
2d21ac55
A		 1059#define vm_object_lock_assert_held(object)
1060#define vm_object_lock_assert_shared(object)
1061#define vm_object_lock_assert_exclusive(object)
3e170ce0 1062#define vm_object_lock_assert_notheld(object)
6d2010ae 1063#endif /* MACH_ASSERT || DEBUG */
1c79356b 1064
91447636
A		 1065#define vm_object_round_page(x) (((vm_object_offset_t)(x) + PAGE_MASK) & ~((signed)PAGE_MASK))
1066#define vm_object_trunc_page(x) ((vm_object_offset_t)(x) & ~((signed)PAGE_MASK))
1067
6d2010ae
A		 1068extern void vm_object_cache_add(vm_object_t);
1069extern void vm_object_cache_remove(vm_object_t);
1070extern int vm_object_cache_evict(int, int);
1071
1c79356b 1072#endif /* _VM_VM_OBJECT_H_ */
mirror of XNU