]> git.saurik.com Git - apple/xnu.git/blame - osfmk/vm/vm_object.h
projects / apple / xnu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
xnu-2422.90.20.tar.gz
[apple/xnu.git] / osfmk / vm / vm_object.h
CommitLineData
1c79356b 1/*
2d21ac55 2 * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
1c79356b 3 *
2d21ac55 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
1c79356b 5 *
2d21ac55
A		 6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
8f6c56a5 14 *
2d21ac55
A		 15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
8f6c56a5
A		 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
2d21ac55
A		 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
8f6c56a5 25 *
2d21ac55 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
1c79356b
A		 27 */
28/*
29 * @OSF_COPYRIGHT@
30 */
31/*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56/*
57 */
58/*
59 * File: vm_object.h
60 * Author: Avadis Tevanian, Jr., Michael Wayne Young
61 * Date: 1985
62 *
63 * Virtual memory object module definitions.
64 */
65
66#ifndef _VM_VM_OBJECT_H_
67#define _VM_VM_OBJECT_H_
68
6d2010ae
A		 69#include <debug.h>
70#include <mach_assert.h>
1c79356b
A		 71#include <mach_pagemap.h>
72#include <task_swapper.h>
73
74#include <mach/kern_return.h>
75#include <mach/boolean.h>
76#include <mach/memory_object_types.h>
77#include <mach/port.h>
78#include <mach/vm_prot.h>
91447636 79#include <mach/vm_param.h>
1c79356b
A		 80#include <mach/machine/vm_types.h>
81#include <kern/queue.h>
82#include <kern/lock.h>
2d21ac55 83#include <kern/locks.h>
1c79356b 84#include <kern/assert.h>
0b4e3aa0 85#include <kern/misc_protos.h>
1c79356b
A		 86#include <kern/macro_help.h>
87#include <ipc/ipc_types.h>
88#include <vm/pmap.h>
1c79356b 89
1c79356b 90#include <vm/vm_external.h>
1c79356b 91
b0d623f7
A		 92#include <vm/vm_options.h>
93
91447636 94struct vm_page;
39236c6e 95struct vm_shared_region_slide_info;
1c79356b
A		 96
97/*
98 * Types defined:
99 *
100 * vm_object_t Virtual memory object.
2d21ac55 101 * vm_object_fault_info_t Used to determine cluster size.
1c79356b
A		 102 */
103
2d21ac55
A		 104struct vm_object_fault_info {
105 int interruptible;
106 uint32_t user_tag;
107 vm_size_t cluster_size;
108 vm_behavior_t behavior;
109 vm_map_offset_t lo_offset;
110 vm_map_offset_t hi_offset;
6d2010ae
A		 111 unsigned int
112 /* boolean_t */ no_cache:1,
113 /* boolean_t */ stealth:1,
114 /* boolean_t */ io_sync:1,
115 /* boolean_t */ cs_bypass:1,
116 /* boolean_t */ mark_zf_absent:1,
316670eb
A		 117 /* boolean_t */ batch_pmap_op:1,
118 __vm_object_fault_info_unused_bits:26;
2d21ac55
A		 119};
120
121
39236c6e
A		 122#define vo_size vo_un1.vou_size
123#define vo_cache_pages_to_scan vo_un1.vou_cache_pages_to_scan
124#define vo_shadow_offset vo_un2.vou_shadow_offset
125#define vo_cache_ts vo_un2.vou_cache_ts
126#define vo_purgeable_owner vo_un2.vou_purgeable_owner
127#define vo_slide_info vo_un2.vou_slide_info
2d21ac55 128
1c79356b
A		 129struct vm_object {
130 queue_head_t memq; /* Resident memory */
2d21ac55 131 lck_rw_t Lock; /* Synchronization */
1c79356b 132
6d2010ae
A		 133 union {
134 vm_object_size_t vou_size; /* Object size (only valid if internal) */
135 int vou_cache_pages_to_scan; /* pages yet to be visited in an
136 * external object in cache
137 */
138 } vo_un1;
139
91447636 140 struct vm_page *memq_hint;
1c79356b
A		 141 int ref_count; /* Number of references */
142#if TASK_SWAPPER
143 int res_count; /* Residency references (swap)*/
144#endif /* TASK_SWAPPER */
145 unsigned int resident_page_count;
146 /* number of resident pages */
b0d623f7
A		 147 unsigned int wired_page_count; /* number of wired pages */
148 unsigned int reusable_page_count;
1c79356b
A		 149
150 struct vm_object *copy; /* Object that should receive
151 * a copy of my changed pages,
152 * for copy_delay, or just the
153 * temporary object that
154 * shadows this object, for
155 * copy_call.
156 */
157 struct vm_object *shadow; /* My shadow */
6d2010ae
A		 158
159 union {
160 vm_object_offset_t vou_shadow_offset; /* Offset into shadow */
39236c6e
A		 161 clock_sec_t vou_cache_ts; /* age of an external object
162 * present in cache
163 */
164 task_t vou_purgeable_owner; /* If the purg'a'ble bits below are set
165 * to volatile/emtpy, this is the task
166 * that owns this purgeable object.
6d2010ae 167 */
39236c6e 168 struct vm_shared_region_slide_info *vou_slide_info;
6d2010ae 169 } vo_un2;
1c79356b 170
0b4e3aa0 171 memory_object_t pager; /* Where to get data */
1c79356b 172 vm_object_offset_t paging_offset; /* Offset into memory object */
91447636 173 memory_object_control_t pager_control; /* Where data comes back */
1c79356b
A		 174
175 memory_object_copy_strategy_t
176 copy_strategy; /* How to handle data copy */
177
b0d623f7 178 short paging_in_progress;
1c79356b
A		 179 /* The memory object ports are
180 * being used (e.g., for pagein
181 * or pageout) -- don't change
182 * any of these fields (i.e.,
183 * don't collapse, destroy or
184 * terminate)
185 */
b0d623f7
A		 186 short activity_in_progress;
187
1c79356b
A		 188 unsigned int
189 /* boolean_t array */ all_wanted:11, /* Bit array of "want to be
190 * awakened" notations. See
191 * VM_OBJECT_EVENT_* items
192 * below */
193 /* boolean_t */ pager_created:1, /* Has pager been created? */
194 /* boolean_t */ pager_initialized:1, /* Are fields ready to use? */
195 /* boolean_t */ pager_ready:1, /* Will pager take requests? */
196
197 /* boolean_t */ pager_trusted:1,/* The pager for this object
198 * is trusted. This is true for
199 * all internal objects (backed
200 * by the default pager)
201 */
202 /* boolean_t */ can_persist:1, /* The kernel may keep the data
203 * for this object (and rights
204 * to the memory object) after
205 * all address map references
206 * are deallocated?
207 */
208 /* boolean_t */ internal:1, /* Created by the kernel (and
209 * therefore, managed by the
210 * default memory manger)
211 */
212 /* boolean_t */ temporary:1, /* Permanent objects may be
213 * changed externally by the
214 * memory manager, and changes
215 * made in memory must be
216 * reflected back to the memory
217 * manager. Temporary objects
218 * lack both of these
219 * characteristics.
220 */
221 /* boolean_t */ private:1, /* magic device_pager object,
222 * holds private pages only */
223 /* boolean_t */ pageout:1, /* pageout object. contains
224 * private pages that refer to
225 * a real memory object. */
226 /* boolean_t */ alive:1, /* Not yet terminated */
227
91447636 228 /* boolean_t */ purgable:2, /* Purgable state. See
2d21ac55 229 * VM_PURGABLE_*
1c79356b 230 */
39236c6e
A		 231 /* boolean_t */ purgeable_when_ripe:1, /* Purgeable when a token
232 * becomes ripe.
233 */
1c79356b 234 /* boolean_t */ shadowed:1, /* Shadow may exist */
1c79356b
A		 235 /* boolean_t */ advisory_pageout:1,
236 /* Instead of sending page
237 * via OOL, just notify
238 * pager that the kernel
239 * wants to discard it, page
240 * remains in object */
241 /* boolean_t */ true_share:1,
242 /* This object is mapped
243 * in more than one place
244 * and hence cannot be
245 * coalesced */
246 /* boolean_t */ terminating:1,
247 /* Allows vm_object_lookup
248 * and vm_object_deallocate
249 * to special case their
250 * behavior when they are
251 * called as a result of
252 * page cleaning during
253 * object termination
254 */
255 /* boolean_t */ named:1, /* An enforces an internal
256 * naming convention, by
257 * calling the right routines
258 * for allocation and
259 * destruction, UBC references
260 * against the vm_object are
261 * checked.
262 */
263 /* boolean_t */ shadow_severed:1,
264 /* When a permanent object
265 * backing a COW goes away
266 * unexpectedly. This bit
267 * allows vm_fault to return
268 * an error rather than a
269 * zero filled page.
270 */
0b4e3aa0 271 /* boolean_t */ phys_contiguous:1,
1c79356b
A		 272 /* Memory is wired and
273 * guaranteed physically
274 * contiguous. However
275 * it is not device memory
276 * and obeys normal virtual
277 * memory rules w.r.t pmap
278 * access bits.
279 */
0b4e3aa0
A		 280 /* boolean_t */ nophyscache:1;
281 /* When mapped at the
282 * pmap level, don't allow
283 * primary caching. (for
284 * I/O)
285 */
1c79356b
A		 286
287
288
289 queue_chain_t cached_list; /* Attachment point for the
290 * list of objects cached as a
291 * result of their can_persist
292 * value
293 */
294
295 queue_head_t msr_q; /* memory object synchronise
296 request queue */
297
2d21ac55
A		 298 /*
299 * the following fields are not protected by any locks
300 * they are updated via atomic compare and swap
301 */
1c79356b 302 vm_object_offset_t last_alloc; /* last allocation offset */
2d21ac55
A		 303 int sequential; /* sequential access size */
304
305 uint32_t pages_created;
306 uint32_t pages_used;
1c79356b
A		 307#if MACH_PAGEMAP
308 vm_external_map_t existence_map; /* bitmap of pages written to
309 * backing storage */
310#endif /* MACH_PAGEMAP */
55e303ae 311 vm_offset_t cow_hint; /* last page present in */
0b4e3aa0 312 /* shadow but not in object */
1c79356b
A		 313#if MACH_ASSERT
314 struct vm_object *paging_object; /* object which pages to be
315 * swapped out are temporary
316 * put in current object
317 */
318#endif
2d21ac55
A		 319 /* hold object lock when altering */
320 unsigned int
321 wimg_bits:8, /* cache WIMG bits */
322 code_signed:1, /* pages are signed and should be
323 validated; the signatures are stored
324 with the pager */
b0d623f7
A		 325 hashed:1, /* object/pager entered in hash */
326 transposed:1, /* object was transposed with another */
593a1d5f 327 mapping_in_progress:1, /* pager being mapped/unmapped */
b0d623f7
A		 328 volatile_empty:1,
329 volatile_fault:1,
330 all_reusable:1,
331 blocked_access:1,
6d2010ae 332 set_cache_attr:1,
39236c6e
A		 333 object_slid:1,
334 purgeable_queue_type:2,
335 purgeable_queue_group:3,
336 __object2_unused_bits:9; /* for expansion */
6d2010ae
A		 337
338 uint32_t scan_collisions;
2d21ac55 339
b0d623f7 340#if UPL_DEBUG
1c79356b 341 queue_head_t uplq; /* List of outstanding upls */
2d21ac55
A		 342#endif /* UPL_DEBUG */
343
0c530ab8
A		 344#ifdef VM_PIP_DEBUG
345/*
346 * Keep track of the stack traces for the first holders
347 * of a "paging_in_progress" reference for this VM object.
348 */
349#define VM_PIP_DEBUG_STACK_FRAMES 25 /* depth of each stack trace */
350#define VM_PIP_DEBUG_MAX_REFS 10 /* track that many references */
351 struct __pip_backtrace {
352 void *pip_retaddr[VM_PIP_DEBUG_STACK_FRAMES];
353 } pip_holders[VM_PIP_DEBUG_MAX_REFS];
354#endif /* VM_PIP_DEBUG */
2d21ac55 355
6d2010ae 356 queue_chain_t objq; /* object queue - currently used for purgable queues */
1c79356b
A		 357};
358
b0d623f7
A		 359#define VM_OBJECT_PURGEABLE_FAULT_ERROR(object) \
360 ((object)->volatile_fault && \
361 ((object)->purgable == VM_PURGABLE_VOLATILE || \
362 (object)->purgable == VM_PURGABLE_EMPTY))
363
91447636
A		 364#define VM_PAGE_REMOVE(page) \
365 MACRO_BEGIN \
366 vm_page_t __page = (page); \
367 vm_object_t __object = __page->object; \
368 if (__page == __object->memq_hint) { \
369 vm_page_t __new_hint; \
370 queue_entry_t __qe; \
371 __qe = queue_next(&__page->listq); \
372 if (queue_end(&__object->memq, __qe)) { \
373 __qe = queue_prev(&__page->listq); \
374 if (queue_end(&__object->memq, __qe)) { \
375 __qe = NULL; \
376 } \
377 } \
378 __new_hint = (vm_page_t) __qe; \
379 __object->memq_hint = __new_hint; \
380 } \
381 queue_remove(&__object->memq, __page, vm_page_t, listq); \
382 MACRO_END
383
384#define VM_PAGE_INSERT(page, object) \
385 MACRO_BEGIN \
386 vm_page_t __page = (page); \
387 vm_object_t __object = (object); \
388 queue_enter(&__object->memq, __page, vm_page_t, listq); \
389 __object->memq_hint = __page; \
390 MACRO_END
391
39236c6e 392extern
1c79356b
A		 393vm_object_t kernel_object; /* the single kernel object */
394
39236c6e
A		 395extern
396vm_object_t compressor_object; /* the single compressor object */
397
398extern
91447636 399unsigned int vm_object_absent_max; /* maximum number of absent pages
1c79356b
A		 400 at a time for each object */
401
402# define VM_MSYNC_INITIALIZED 0
403# define VM_MSYNC_SYNCHRONIZING 1
404# define VM_MSYNC_DONE 2
405
406struct msync_req {
407 queue_chain_t msr_q; /* object request queue */
408 queue_chain_t req_q; /* vm_msync request queue */
409 unsigned int flag;
410 vm_object_offset_t offset;
411 vm_object_size_t length;
412 vm_object_t object; /* back pointer */
b0d623f7 413 decl_lck_mtx_data(, msync_req_lock) /* Lock for this structure */
1c79356b
A		 414};
415
416typedef struct msync_req *msync_req_t;
417#define MSYNC_REQ_NULL ((msync_req_t) 0)
418
b0d623f7
A		 419
420extern lck_grp_t vm_map_lck_grp;
421extern lck_attr_t vm_map_lck_attr;
422
1c79356b
A		 423/*
424 * Macros to allocate and free msync_reqs
425 */
426#define msync_req_alloc(msr) \
b0d623f7 427 MACRO_BEGIN \
1c79356b 428 (msr) = (msync_req_t)kalloc(sizeof(struct msync_req)); \
b0d623f7
A		 429 lck_mtx_init(&(msr)->msync_req_lock, &vm_map_lck_grp, &vm_map_lck_attr); \
430 msr->flag = VM_MSYNC_INITIALIZED; \
431 MACRO_END
1c79356b
A		 432
433#define msync_req_free(msr) \
39236c6e
A		 434 MACRO_BEGIN \
435 lck_mtx_destroy(&(msr)->msync_req_lock, &vm_map_lck_grp); \
436 kfree((msr), sizeof(struct msync_req)); \
437 MACRO_END
1c79356b 438
b0d623f7
A		 439#define msr_lock(msr) lck_mtx_lock(&(msr)->msync_req_lock)
440#define msr_unlock(msr) lck_mtx_unlock(&(msr)->msync_req_lock)
1c79356b
A		 441
442/*
443 * Declare procedures that operate on VM objects.
444 */
445
39236c6e 446__private_extern__ void vm_object_bootstrap(void);
1c79356b 447
0b4e3aa0 448__private_extern__ void vm_object_init(void);
1c79356b 449
2d21ac55
A		 450__private_extern__ void vm_object_init_lck_grp(void);
451
8f6c56a5
A		 452__private_extern__ void vm_object_reaper_init(void);
453
0b4e3aa0 454__private_extern__ vm_object_t vm_object_allocate(
1c79356b
A		 455 vm_object_size_t size);
456
91447636
A		 457__private_extern__ void _vm_object_allocate(vm_object_size_t size,
458 vm_object_t object);
459
0b4e3aa0
A		 460#if TASK_SWAPPER
461
462__private_extern__ void vm_object_res_reference(
463 vm_object_t object);
464__private_extern__ void vm_object_res_deallocate(
465 vm_object_t object);
466#define VM_OBJ_RES_INCR(object) (object)->res_count++
467#define VM_OBJ_RES_DECR(object) (object)->res_count--
468
469#else /* TASK_SWAPPER */
470
471#define VM_OBJ_RES_INCR(object)
472#define VM_OBJ_RES_DECR(object)
473#define vm_object_res_reference(object)
474#define vm_object_res_deallocate(object)
475
476#endif /* TASK_SWAPPER */
477
478#define vm_object_reference_locked(object) \
2d21ac55
A		 479 MACRO_BEGIN \
480 vm_object_t RLObject = (object); \
481 vm_object_lock_assert_exclusive(object); \
482 assert((RLObject)->ref_count > 0); \
483 (RLObject)->ref_count++; \
484 assert((RLObject)->ref_count > 1); \
485 vm_object_res_reference(RLObject); \
486 MACRO_END
487
488
489#define vm_object_reference_shared(object) \
490 MACRO_BEGIN \
491 vm_object_t RLObject = (object); \
492 vm_object_lock_assert_shared(object); \
493 assert((RLObject)->ref_count > 0); \
b0d623f7 494 OSAddAtomic(1, &(RLObject)->ref_count); \
316670eb 495 assert((RLObject)->ref_count > 0); \
2d21ac55
A		 496 /* XXX we would need an atomic version of the following ... */ \
497 vm_object_res_reference(RLObject); \
498 MACRO_END
0b4e3aa0
A		 499
500
0b4e3aa0 501__private_extern__ void vm_object_reference(
1c79356b 502 vm_object_t object);
0b4e3aa0 503
91447636 504#if !MACH_ASSERT
0b4e3aa0 505
1c79356b
A		 506#define vm_object_reference(object) \
507MACRO_BEGIN \
0b4e3aa0
A		 508 vm_object_t RObject = (object); \
509 if (RObject) { \
b0d623f7
A		 510 vm_object_lock_shared(RObject); \
511 vm_object_reference_shared(RObject); \
0b4e3aa0 512 vm_object_unlock(RObject); \
1c79356b
A		 513 } \
514MACRO_END
0b4e3aa0 515
1c79356b
A		 516#endif /* MACH_ASSERT */
517
0b4e3aa0 518__private_extern__ void vm_object_deallocate(
1c79356b
A		 519 vm_object_t object);
520
0b4e3aa0
A		 521__private_extern__ kern_return_t vm_object_release_name(
522 vm_object_t object,
523 int flags);
524
525__private_extern__ void vm_object_pmap_protect(
1c79356b
A		 526 vm_object_t object,
527 vm_object_offset_t offset,
91447636 528 vm_object_size_t size,
1c79356b 529 pmap_t pmap,
91447636 530 vm_map_offset_t pmap_start,
1c79356b
A		 531 vm_prot_t prot);
532
39236c6e
A		 533__private_extern__ void vm_object_pmap_protect_options(
534 vm_object_t object,
535 vm_object_offset_t offset,
536 vm_object_size_t size,
537 pmap_t pmap,
538 vm_map_offset_t pmap_start,
539 vm_prot_t prot,
540 int options);
541
0b4e3aa0 542__private_extern__ void vm_object_page_remove(
1c79356b
A		 543 vm_object_t object,
544 vm_object_offset_t start,
545 vm_object_offset_t end);
546
0b4e3aa0
A		 547__private_extern__ void vm_object_deactivate_pages(
548 vm_object_t object,
549 vm_object_offset_t offset,
550 vm_object_size_t size,
b0d623f7
A		 551 boolean_t kill_page,
552 boolean_t reusable_page);
553
554__private_extern__ void vm_object_reuse_pages(
555 vm_object_t object,
556 vm_object_offset_t start_offset,
557 vm_object_offset_t end_offset,
558 boolean_t allow_partial_reuse);
0b4e3aa0 559
b0d623f7 560__private_extern__ void vm_object_purge(
91447636
A		 561 vm_object_t object);
562
563__private_extern__ kern_return_t vm_object_purgable_control(
564 vm_object_t object,
565 vm_purgable_t control,
566 int *state);
567
39236c6e
A		 568__private_extern__ kern_return_t vm_object_get_page_counts(
569 vm_object_t object,
570 vm_object_offset_t offset,
571 vm_object_size_t size,
572 unsigned int *resident_page_count,
573 unsigned int *dirty_page_count);
574
0b4e3aa0 575__private_extern__ boolean_t vm_object_coalesce(
1c79356b
A		 576 vm_object_t prev_object,
577 vm_object_t next_object,
578 vm_object_offset_t prev_offset,
579 vm_object_offset_t next_offset,
580 vm_object_size_t prev_size,
581 vm_object_size_t next_size);
582
0b4e3aa0 583__private_extern__ boolean_t vm_object_shadow(
1c79356b
A		 584 vm_object_t *object,
585 vm_object_offset_t *offset,
586 vm_object_size_t length);
587
0b4e3aa0 588__private_extern__ void vm_object_collapse(
55e303ae 589 vm_object_t object,
0c530ab8
A		 590 vm_object_offset_t offset,
591 boolean_t can_bypass);
1c79356b 592
0b4e3aa0 593__private_extern__ boolean_t vm_object_copy_quickly(
1c79356b
A		 594 vm_object_t *_object,
595 vm_object_offset_t src_offset,
596 vm_object_size_t size,
597 boolean_t *_src_needs_copy,
598 boolean_t *_dst_needs_copy);
599
0b4e3aa0 600__private_extern__ kern_return_t vm_object_copy_strategically(
1c79356b
A		 601 vm_object_t src_object,
602 vm_object_offset_t src_offset,
603 vm_object_size_t size,
604 vm_object_t *dst_object,
605 vm_object_offset_t *dst_offset,
606 boolean_t *dst_needs_copy);
607
0b4e3aa0 608__private_extern__ kern_return_t vm_object_copy_slowly(
1c79356b
A		 609 vm_object_t src_object,
610 vm_object_offset_t src_offset,
611 vm_object_size_t size,
b0d623f7 612 boolean_t interruptible,
1c79356b
A		 613 vm_object_t *_result_object);
614
0b4e3aa0
A		 615__private_extern__ vm_object_t vm_object_copy_delayed(
616 vm_object_t src_object,
617 vm_object_offset_t src_offset,
2d21ac55
A		 618 vm_object_size_t size,
619 boolean_t src_object_shared);
0b4e3aa0 620
1c79356b 621
1c79356b 622
0b4e3aa0
A		 623__private_extern__ kern_return_t vm_object_destroy(
624 vm_object_t object,
625 kern_return_t reason);
1c79356b 626
0b4e3aa0
A		 627__private_extern__ void vm_object_pager_create(
628 vm_object_t object);
629
39236c6e
A		 630__private_extern__ void vm_object_compressor_pager_create(
631 vm_object_t object);
632
0b4e3aa0 633__private_extern__ void vm_object_page_map(
1c79356b
A		 634 vm_object_t object,
635 vm_object_offset_t offset,
636 vm_object_size_t size,
637 vm_object_offset_t (*map_fn)
638 (void *, vm_object_offset_t),
639 void *map_fn_data);
640
0b4e3aa0
A		 641__private_extern__ kern_return_t vm_object_upl_request(
642 vm_object_t object,
643 vm_object_offset_t offset,
91447636 644 upl_size_t size,
0b4e3aa0
A		 645 upl_t *upl,
646 upl_page_info_t *page_info,
647 unsigned int *count,
648 int flags);
649
91447636
A		 650__private_extern__ kern_return_t vm_object_transpose(
651 vm_object_t object1,
652 vm_object_t object2,
653 vm_object_size_t transpose_size);
654
0b4e3aa0
A		 655__private_extern__ boolean_t vm_object_sync(
656 vm_object_t object,
657 vm_object_offset_t offset,
91447636 658 vm_object_size_t size,
0b4e3aa0 659 boolean_t should_flush,
91447636
A		 660 boolean_t should_return,
661 boolean_t should_iosync);
1c79356b 662
0b4e3aa0
A		 663__private_extern__ kern_return_t vm_object_update(
664 vm_object_t object,
665 vm_object_offset_t offset,
91447636
A		 666 vm_object_size_t size,
667 vm_object_offset_t *error_offset,
668 int *io_errno,
0b4e3aa0
A		 669 memory_object_return_t should_return,
670 int flags,
671 vm_prot_t prot);
1c79356b 672
0b4e3aa0
A		 673__private_extern__ kern_return_t vm_object_lock_request(
674 vm_object_t object,
675 vm_object_offset_t offset,
676 vm_object_size_t size,
677 memory_object_return_t should_return,
678 int flags,
679 vm_prot_t prot);
1c79356b 680
1c79356b 681
1c79356b 682
0b4e3aa0
A		 683__private_extern__ vm_object_t vm_object_enter(
684 memory_object_t pager,
1c79356b
A		 685 vm_object_size_t size,
686 boolean_t internal,
687 boolean_t init,
688 boolean_t check_named);
689
690
2d21ac55
A		 691__private_extern__ void vm_object_cluster_size(
692 vm_object_t object,
693 vm_object_offset_t *start,
694 vm_size_t *length,
b0d623f7
A		 695 vm_object_fault_info_t fault_info,
696 uint32_t *io_streaming);
91447636
A		 697
698__private_extern__ kern_return_t vm_object_populate_with_private(
699 vm_object_t object,
700 vm_object_offset_t offset,
701 ppnum_t phys_page,
702 vm_size_t size);
703
6d2010ae
A		 704__private_extern__ void vm_object_change_wimg_mode(
705 vm_object_t object,
706 unsigned int wimg_mode);
707
0c530ab8 708extern kern_return_t adjust_vm_object_cache(
91447636
A		 709 vm_size_t oval,
710 vm_size_t nval);
711
0c530ab8
A		 712extern kern_return_t vm_object_page_op(
713 vm_object_t object,
714 vm_object_offset_t offset,
715 int ops,
716 ppnum_t *phys_entry,
717 int *flags);
718
719extern kern_return_t vm_object_range_op(
720 vm_object_t object,
721 vm_object_offset_t offset_beg,
722 vm_object_offset_t offset_end,
723 int ops,
b0d623f7
A		 724 uint32_t *range);
725
726
727__private_extern__ void vm_object_reap_pages(
728 vm_object_t object,
729 int reap_type);
730#define REAP_REAP 0
731#define REAP_TERMINATE 1
732#define REAP_PURGEABLE 2
733#define REAP_DATA_FLUSH 3
734
6d2010ae 735#if CONFIG_FREEZE
316670eb 736struct default_freezer_handle;
6d2010ae
A		 737
738__private_extern__ kern_return_t
739vm_object_pack(
316670eb
A		 740 unsigned int *purgeable_count,
741 unsigned int *wired_count,
742 unsigned int *clean_count,
743 unsigned int *dirty_count,
744 unsigned int dirty_budget,
745 boolean_t *shared,
746 vm_object_t src_object,
747 struct default_freezer_handle *df_handle);
6d2010ae
A		 748
749__private_extern__ void
750vm_object_pack_pages(
316670eb
A		 751 unsigned int *wired_count,
752 unsigned int *clean_count,
753 unsigned int *dirty_count,
754 unsigned int dirty_budget,
755 vm_object_t src_object,
756 struct default_freezer_handle *df_handle);
6d2010ae 757
316670eb
A		 758__private_extern__ void
759vm_object_pageout(
760 vm_object_t object);
6d2010ae 761
316670eb
A		 762__private_extern__ kern_return_t
763vm_object_pagein(
764 vm_object_t object);
6d2010ae 765#endif /* CONFIG_FREEZE */
0c530ab8 766
1c79356b
A		 767/*
768 * Event waiting handling
769 */
770
771#define VM_OBJECT_EVENT_INITIALIZED 0
772#define VM_OBJECT_EVENT_PAGER_READY 1
773#define VM_OBJECT_EVENT_PAGING_IN_PROGRESS 2
593a1d5f 774#define VM_OBJECT_EVENT_MAPPING_IN_PROGRESS 3
1c79356b
A		 775#define VM_OBJECT_EVENT_LOCK_IN_PROGRESS 4
776#define VM_OBJECT_EVENT_UNCACHING 5
777#define VM_OBJECT_EVENT_COPY_CALL 6
778#define VM_OBJECT_EVENT_CACHING 7
b0d623f7
A		 779#define VM_OBJECT_EVENT_UNBLOCKED 8
780#define VM_OBJECT_EVENT_PAGING_ONLY_IN_PROGRESS 9
1c79356b
A		 781
782#define vm_object_assert_wait(object, event, interruptible) \
9bccf70c
A		 783 (((object)->all_wanted |= 1 << (event)), \
784 assert_wait((event_t)((vm_offset_t)(object)+(event)),(interruptible)))
1c79356b
A		 785
786#define vm_object_wait(object, event, interruptible) \
9bccf70c
A		 787 (vm_object_assert_wait((object),(event),(interruptible)), \
788 vm_object_unlock(object), \
789 thread_block(THREAD_CONTINUE_NULL)) \
790
791#define thread_sleep_vm_object(object, event, interruptible) \
2d21ac55 792 lck_rw_sleep(&(object)->Lock, LCK_SLEEP_DEFAULT, (event_t)(event), (interruptible))
9bccf70c
A		 793
794#define vm_object_sleep(object, event, interruptible) \
795 (((object)->all_wanted |= 1 << (event)), \
796 thread_sleep_vm_object((object), \
797 ((vm_offset_t)(object)+(event)), (interruptible)))
1c79356b
A		 798
799#define vm_object_wakeup(object, event) \
800 MACRO_BEGIN \
801 if ((object)->all_wanted & (1 << (event))) \
802 thread_wakeup((event_t)((vm_offset_t)(object) + (event))); \
803 (object)->all_wanted &= ~(1 << (event)); \
804 MACRO_END
805
806#define vm_object_set_wanted(object, event) \
807 MACRO_BEGIN \
808 ((object)->all_wanted |= (1 << (event))); \
809 MACRO_END
810
811#define vm_object_wanted(object, event) \
812 ((object)->all_wanted & (1 << (event)))
813
814/*
815 * Routines implemented as macros
816 */
0c530ab8 817#ifdef VM_PIP_DEBUG
2d21ac55 818#include <libkern/OSDebug.h>
0c530ab8
A		 819#define VM_PIP_DEBUG_BEGIN(object) \
820 MACRO_BEGIN \
b0d623f7
A		 821 int pip = ((object)->paging_in_progress + \
822 (object)->activity_in_progress); \
823 if (pip < VM_PIP_DEBUG_MAX_REFS) { \
2d21ac55 824 (void) OSBacktrace(&(object)->pip_holders[pip].pip_retaddr[0], \
0c530ab8
A		 825 VM_PIP_DEBUG_STACK_FRAMES); \
826 } \
827 MACRO_END
828#else /* VM_PIP_DEBUG */
829#define VM_PIP_DEBUG_BEGIN(object)
830#endif /* VM_PIP_DEBUG */
1c79356b 831
b0d623f7
A		 832#define vm_object_activity_begin(object) \
833 MACRO_BEGIN \
834 vm_object_lock_assert_exclusive((object)); \
835 assert((object)->paging_in_progress >= 0); \
836 VM_PIP_DEBUG_BEGIN((object)); \
837 (object)->activity_in_progress++; \
838 MACRO_END
839
840#define vm_object_activity_end(object) \
841 MACRO_BEGIN \
842 vm_object_lock_assert_exclusive((object)); \
843 assert((object)->activity_in_progress > 0); \
844 (object)->activity_in_progress--; \
845 if ((object)->paging_in_progress == 0 && \
846 (object)->activity_in_progress == 0) \
847 vm_object_wakeup((object), \
848 VM_OBJECT_EVENT_PAGING_IN_PROGRESS); \
849 MACRO_END
850
851#define vm_object_paging_begin(object) \
1c79356b 852 MACRO_BEGIN \
2d21ac55 853 vm_object_lock_assert_exclusive((object)); \
0c530ab8
A		 854 assert((object)->paging_in_progress >= 0); \
855 VM_PIP_DEBUG_BEGIN((object)); \
1c79356b
A		 856 (object)->paging_in_progress++; \
857 MACRO_END
858
b0d623f7 859#define vm_object_paging_end(object) \
1c79356b 860 MACRO_BEGIN \
2d21ac55 861 vm_object_lock_assert_exclusive((object)); \
0c530ab8 862 assert((object)->paging_in_progress > 0); \
b0d623f7
A		 863 (object)->paging_in_progress--; \
864 if ((object)->paging_in_progress == 0) { \
865 vm_object_wakeup((object), \
866 VM_OBJECT_EVENT_PAGING_ONLY_IN_PROGRESS); \
867 if ((object)->activity_in_progress == 0) \
868 vm_object_wakeup((object), \
869 VM_OBJECT_EVENT_PAGING_IN_PROGRESS); \
1c79356b
A		 870 } \
871 MACRO_END
872
873#define vm_object_paging_wait(object, interruptible) \
874 MACRO_BEGIN \
2d21ac55 875 vm_object_lock_assert_exclusive((object)); \
b0d623f7
A		 876 while ((object)->paging_in_progress != 0 || \
877 (object)->activity_in_progress != 0) { \
9bccf70c
A		 878 wait_result_t _wr; \
879 \
880 _wr = vm_object_sleep((object), \
1c79356b
A		 881 VM_OBJECT_EVENT_PAGING_IN_PROGRESS, \
882 (interruptible)); \
1c79356b 883 \
9bccf70c 884 /*XXX if ((interruptible) && (_wr != THREAD_AWAKENED))*/\
1c79356b
A		 885 /*XXX break; */ \
886 } \
887 MACRO_END
888
b0d623f7
A		 889#define vm_object_paging_only_wait(object, interruptible) \
890 MACRO_BEGIN \
891 vm_object_lock_assert_exclusive((object)); \
892 while ((object)->paging_in_progress != 0) { \
893 wait_result_t _wr; \
894 \
895 _wr = vm_object_sleep((object), \
896 VM_OBJECT_EVENT_PAGING_ONLY_IN_PROGRESS,\
897 (interruptible)); \
898 \
899 /*XXX if ((interruptible) && (_wr != THREAD_AWAKENED))*/\
900 /*XXX break; */ \
901 } \
902 MACRO_END
903
1c79356b 904
593a1d5f
A		 905#define vm_object_mapping_begin(object) \
906 MACRO_BEGIN \
907 vm_object_lock_assert_exclusive((object)); \
908 assert(! (object)->mapping_in_progress); \
909 (object)->mapping_in_progress = TRUE; \
910 MACRO_END
911
912#define vm_object_mapping_end(object) \
913 MACRO_BEGIN \
914 vm_object_lock_assert_exclusive((object)); \
915 assert((object)->mapping_in_progress); \
916 (object)->mapping_in_progress = FALSE; \
917 vm_object_wakeup((object), \
918 VM_OBJECT_EVENT_MAPPING_IN_PROGRESS); \
919 MACRO_END
920
921#define vm_object_mapping_wait(object, interruptible) \
922 MACRO_BEGIN \
923 vm_object_lock_assert_exclusive((object)); \
924 while ((object)->mapping_in_progress) { \
925 wait_result_t _wr; \
926 \
927 _wr = vm_object_sleep((object), \
928 VM_OBJECT_EVENT_MAPPING_IN_PROGRESS, \
929 (interruptible)); \
930 /*XXX if ((interruptible) && (_wr != THREAD_AWAKENED))*/\
931 /*XXX break; */ \
932 } \
933 assert(!(object)->mapping_in_progress); \
934 MACRO_END
935
936
1c79356b 937
2d21ac55
A		 938#define OBJECT_LOCK_SHARED 0
939#define OBJECT_LOCK_EXCLUSIVE 1
940
941extern lck_grp_t vm_object_lck_grp;
942extern lck_grp_attr_t vm_object_lck_grp_attr;
943extern lck_attr_t vm_object_lck_attr;
944extern lck_attr_t kernel_object_lck_attr;
39236c6e 945extern lck_attr_t compressor_object_lck_attr;
2d21ac55
A		 946
947extern vm_object_t vm_pageout_scan_wants_object;
948
949extern void vm_object_lock(vm_object_t);
950extern boolean_t vm_object_lock_try(vm_object_t);
b0d623f7
A		 951extern boolean_t _vm_object_lock_try(vm_object_t);
952extern boolean_t vm_object_lock_avoid(vm_object_t);
2d21ac55
A		 953extern void vm_object_lock_shared(vm_object_t);
954extern boolean_t vm_object_lock_try_shared(vm_object_t);
1c79356b
A		 955
956/*
957 * Object locking macros
958 */
959
2d21ac55
A		 960#define vm_object_lock_init(object) \
961 lck_rw_init(&(object)->Lock, &vm_object_lck_grp, \
962 (((object) == kernel_object || \
963 (object) == vm_submap_object) ? \
964 &kernel_object_lck_attr : \
39236c6e
A		 965 (((object) == compressor_object) ? \
966 &compressor_object_lck_attr : \
967 &vm_object_lck_attr)))
2d21ac55
A		 968#define vm_object_lock_destroy(object) lck_rw_destroy(&(object)->Lock, &vm_object_lck_grp)
969
970#define vm_object_unlock(object) lck_rw_done(&(object)->Lock)
971#define vm_object_lock_upgrade(object) lck_rw_lock_shared_to_exclusive(&(object)->Lock)
b0d623f7 972#define vm_object_lock_try_scan(object) _vm_object_lock_try(object)
2d21ac55
A		 973
974/*
975 * CAUTION: the following vm_object_lock_assert_held*() macros merely
976 * check if anyone is holding the lock, but the holder may not necessarily
977 * be the caller...
978 */
6d2010ae 979#if MACH_ASSERT || DEBUG
2d21ac55
A		 980#define vm_object_lock_assert_held(object) \
981 lck_rw_assert(&(object)->Lock, LCK_RW_ASSERT_HELD)
982#define vm_object_lock_assert_shared(object) \
983 lck_rw_assert(&(object)->Lock, LCK_RW_ASSERT_SHARED)
984#define vm_object_lock_assert_exclusive(object) \
985 lck_rw_assert(&(object)->Lock, LCK_RW_ASSERT_EXCLUSIVE)
6d2010ae 986#else /* MACH_ASSERT || DEBUG */
2d21ac55
A		 987#define vm_object_lock_assert_held(object)
988#define vm_object_lock_assert_shared(object)
989#define vm_object_lock_assert_exclusive(object)
6d2010ae 990#endif /* MACH_ASSERT || DEBUG */
1c79356b 991
91447636
A		 992#define vm_object_round_page(x) (((vm_object_offset_t)(x) + PAGE_MASK) & ~((signed)PAGE_MASK))
993#define vm_object_trunc_page(x) ((vm_object_offset_t)(x) & ~((signed)PAGE_MASK))
994
6d2010ae
A		 995extern void vm_object_cache_add(vm_object_t);
996extern void vm_object_cache_remove(vm_object_t);
997extern int vm_object_cache_evict(int, int);
998
1c79356b 999#endif /* _VM_VM_OBJECT_H_ */
mirror of XNU