]> git.saurik.com Git - apple/xnu.git/blob - osfmk/vm/vm_object.h
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-2050.22.13.tar.gz
[apple/xnu.git] / osfmk / vm / vm_object.h
1 /*
2 * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56 /*
57 */
58 /*
59 * File: vm_object.h
60 * Author: Avadis Tevanian, Jr., Michael Wayne Young
61 * Date: 1985
62 *
63 * Virtual memory object module definitions.
64 */
65
66 #ifndef _VM_VM_OBJECT_H_
67 #define _VM_VM_OBJECT_H_
68
69 #include <debug.h>
70 #include <mach_assert.h>
71 #include <mach_pagemap.h>
72 #include <task_swapper.h>
73
74 #include <mach/kern_return.h>
75 #include <mach/boolean.h>
76 #include <mach/memory_object_types.h>
77 #include <mach/port.h>
78 #include <mach/vm_prot.h>
79 #include <mach/vm_param.h>
80 #include <mach/machine/vm_types.h>
81 #include <kern/queue.h>
82 #include <kern/lock.h>
83 #include <kern/locks.h>
84 #include <kern/assert.h>
85 #include <kern/misc_protos.h>
86 #include <kern/macro_help.h>
87 #include <ipc/ipc_types.h>
88 #include <vm/pmap.h>
89
90 #if MACH_PAGEMAP
91 #include <vm/vm_external.h>
92 #endif /* MACH_PAGEMAP */
93
94 #include <vm/vm_options.h>
95
96 struct vm_page;
97
98 /*
99 * Types defined:
100 *
101 * vm_object_t Virtual memory object.
102 * vm_object_fault_info_t Used to determine cluster size.
103 */
104
105 struct vm_object_fault_info {
106 int interruptible;
107 uint32_t user_tag;
108 vm_size_t cluster_size;
109 vm_behavior_t behavior;
110 vm_map_offset_t lo_offset;
111 vm_map_offset_t hi_offset;
112 unsigned int
113 /* boolean_t */ no_cache:1,
114 /* boolean_t */ stealth:1,
115 /* boolean_t */ io_sync:1,
116 /* boolean_t */ cs_bypass:1,
117 /* boolean_t */ mark_zf_absent:1,
118 /* boolean_t */ batch_pmap_op:1,
119 __vm_object_fault_info_unused_bits:26;
120 };
121
122
123 #define vo_size vo_un1.vou_size
124 #define vo_cache_pages_to_scan vo_un1.vou_cache_pages_to_scan
125 #define vo_shadow_offset vo_un2.vou_shadow_offset
126 #define vo_cache_ts vo_un2.vou_cache_ts
127
128 struct vm_object {
129 queue_head_t memq; /* Resident memory */
130 lck_rw_t Lock; /* Synchronization */
131
132 union {
133 vm_object_size_t vou_size; /* Object size (only valid if internal) */
134 int vou_cache_pages_to_scan; /* pages yet to be visited in an
135 * external object in cache
136 */
137 } vo_un1;
138
139 struct vm_page *memq_hint;
140 int ref_count; /* Number of references */
141 #if TASK_SWAPPER
142 int res_count; /* Residency references (swap)*/
143 #endif /* TASK_SWAPPER */
144 unsigned int resident_page_count;
145 /* number of resident pages */
146 unsigned int wired_page_count; /* number of wired pages */
147 unsigned int reusable_page_count;
148
149 struct vm_object *copy; /* Object that should receive
150 * a copy of my changed pages,
151 * for copy_delay, or just the
152 * temporary object that
153 * shadows this object, for
154 * copy_call.
155 */
156 struct vm_object *shadow; /* My shadow */
157
158 union {
159 vm_object_offset_t vou_shadow_offset; /* Offset into shadow */
160 clock_sec_t vou_cache_ts; /* age of an external object
161 * present in cache
162 */
163 } vo_un2;
164
165 memory_object_t pager; /* Where to get data */
166 vm_object_offset_t paging_offset; /* Offset into memory object */
167 memory_object_control_t pager_control; /* Where data comes back */
168
169 memory_object_copy_strategy_t
170 copy_strategy; /* How to handle data copy */
171
172 short paging_in_progress;
173 /* The memory object ports are
174 * being used (e.g., for pagein
175 * or pageout) -- don't change
176 * any of these fields (i.e.,
177 * don't collapse, destroy or
178 * terminate)
179 */
180 short activity_in_progress;
181
182 unsigned int
183 /* boolean_t array */ all_wanted:11, /* Bit array of "want to be
184 * awakened" notations. See
185 * VM_OBJECT_EVENT_* items
186 * below */
187 /* boolean_t */ pager_created:1, /* Has pager been created? */
188 /* boolean_t */ pager_initialized:1, /* Are fields ready to use? */
189 /* boolean_t */ pager_ready:1, /* Will pager take requests? */
190
191 /* boolean_t */ pager_trusted:1,/* The pager for this object
192 * is trusted. This is true for
193 * all internal objects (backed
194 * by the default pager)
195 */
196 /* boolean_t */ can_persist:1, /* The kernel may keep the data
197 * for this object (and rights
198 * to the memory object) after
199 * all address map references
200 * are deallocated?
201 */
202 /* boolean_t */ internal:1, /* Created by the kernel (and
203 * therefore, managed by the
204 * default memory manger)
205 */
206 /* boolean_t */ temporary:1, /* Permanent objects may be
207 * changed externally by the
208 * memory manager, and changes
209 * made in memory must be
210 * reflected back to the memory
211 * manager. Temporary objects
212 * lack both of these
213 * characteristics.
214 */
215 /* boolean_t */ private:1, /* magic device_pager object,
216 * holds private pages only */
217 /* boolean_t */ pageout:1, /* pageout object. contains
218 * private pages that refer to
219 * a real memory object. */
220 /* boolean_t */ alive:1, /* Not yet terminated */
221
222 /* boolean_t */ purgable:2, /* Purgable state. See
223 * VM_PURGABLE_*
224 */
225 /* boolean_t */ shadowed:1, /* Shadow may exist */
226 /* boolean_t */ silent_overwrite:1,
227 /* Allow full page overwrite
228 * without data_request if
229 * page is absent */
230 /* boolean_t */ advisory_pageout:1,
231 /* Instead of sending page
232 * via OOL, just notify
233 * pager that the kernel
234 * wants to discard it, page
235 * remains in object */
236 /* boolean_t */ true_share:1,
237 /* This object is mapped
238 * in more than one place
239 * and hence cannot be
240 * coalesced */
241 /* boolean_t */ terminating:1,
242 /* Allows vm_object_lookup
243 * and vm_object_deallocate
244 * to special case their
245 * behavior when they are
246 * called as a result of
247 * page cleaning during
248 * object termination
249 */
250 /* boolean_t */ named:1, /* An enforces an internal
251 * naming convention, by
252 * calling the right routines
253 * for allocation and
254 * destruction, UBC references
255 * against the vm_object are
256 * checked.
257 */
258 /* boolean_t */ shadow_severed:1,
259 /* When a permanent object
260 * backing a COW goes away
261 * unexpectedly. This bit
262 * allows vm_fault to return
263 * an error rather than a
264 * zero filled page.
265 */
266 /* boolean_t */ phys_contiguous:1,
267 /* Memory is wired and
268 * guaranteed physically
269 * contiguous. However
270 * it is not device memory
271 * and obeys normal virtual
272 * memory rules w.r.t pmap
273 * access bits.
274 */
275 /* boolean_t */ nophyscache:1;
276 /* When mapped at the
277 * pmap level, don't allow
278 * primary caching. (for
279 * I/O)
280 */
281
282
283
284 queue_chain_t cached_list; /* Attachment point for the
285 * list of objects cached as a
286 * result of their can_persist
287 * value
288 */
289
290 queue_head_t msr_q; /* memory object synchronise
291 request queue */
292
293 /*
294 * the following fields are not protected by any locks
295 * they are updated via atomic compare and swap
296 */
297 vm_object_offset_t last_alloc; /* last allocation offset */
298 int sequential; /* sequential access size */
299
300 uint32_t pages_created;
301 uint32_t pages_used;
302 #if MACH_PAGEMAP
303 vm_external_map_t existence_map; /* bitmap of pages written to
304 * backing storage */
305 #endif /* MACH_PAGEMAP */
306 vm_offset_t cow_hint; /* last page present in */
307 /* shadow but not in object */
308 #if MACH_ASSERT
309 struct vm_object *paging_object; /* object which pages to be
310 * swapped out are temporary
311 * put in current object
312 */
313 #endif
314 /* hold object lock when altering */
315 unsigned int
316 wimg_bits:8, /* cache WIMG bits */
317 code_signed:1, /* pages are signed and should be
318 validated; the signatures are stored
319 with the pager */
320 hashed:1, /* object/pager entered in hash */
321 transposed:1, /* object was transposed with another */
322 mapping_in_progress:1, /* pager being mapped/unmapped */
323 volatile_empty:1,
324 volatile_fault:1,
325 all_reusable:1,
326 blocked_access:1,
327 set_cache_attr:1,
328 __object2_unused_bits:15; /* for expansion */
329
330 uint32_t scan_collisions;
331
332 #if UPL_DEBUG
333 queue_head_t uplq; /* List of outstanding upls */
334 #endif /* UPL_DEBUG */
335
336 #ifdef VM_PIP_DEBUG
337 /*
338 * Keep track of the stack traces for the first holders
339 * of a "paging_in_progress" reference for this VM object.
340 */
341 #define VM_PIP_DEBUG_STACK_FRAMES 25 /* depth of each stack trace */
342 #define VM_PIP_DEBUG_MAX_REFS 10 /* track that many references */
343 struct __pip_backtrace {
344 void *pip_retaddr[VM_PIP_DEBUG_STACK_FRAMES];
345 } pip_holders[VM_PIP_DEBUG_MAX_REFS];
346 #endif /* VM_PIP_DEBUG */
347
348 queue_chain_t objq; /* object queue - currently used for purgable queues */
349 };
350
351 #define VM_OBJECT_PURGEABLE_FAULT_ERROR(object) \
352 ((object)->volatile_fault && \
353 ((object)->purgable == VM_PURGABLE_VOLATILE || \
354 (object)->purgable == VM_PURGABLE_EMPTY))
355
356 #define VM_PAGE_REMOVE(page) \
357 MACRO_BEGIN \
358 vm_page_t __page = (page); \
359 vm_object_t __object = __page->object; \
360 if (__page == __object->memq_hint) { \
361 vm_page_t __new_hint; \
362 queue_entry_t __qe; \
363 __qe = queue_next(&__page->listq); \
364 if (queue_end(&__object->memq, __qe)) { \
365 __qe = queue_prev(&__page->listq); \
366 if (queue_end(&__object->memq, __qe)) { \
367 __qe = NULL; \
368 } \
369 } \
370 __new_hint = (vm_page_t) __qe; \
371 __object->memq_hint = __new_hint; \
372 } \
373 queue_remove(&__object->memq, __page, vm_page_t, listq); \
374 MACRO_END
375
376 #define VM_PAGE_INSERT(page, object) \
377 MACRO_BEGIN \
378 vm_page_t __page = (page); \
379 vm_object_t __object = (object); \
380 queue_enter(&__object->memq, __page, vm_page_t, listq); \
381 __object->memq_hint = __page; \
382 MACRO_END
383
384 __private_extern__
385 vm_object_t kernel_object; /* the single kernel object */
386
387 __private_extern__
388 unsigned int vm_object_absent_max; /* maximum number of absent pages
389 at a time for each object */
390
391 # define VM_MSYNC_INITIALIZED 0
392 # define VM_MSYNC_SYNCHRONIZING 1
393 # define VM_MSYNC_DONE 2
394
395 struct msync_req {
396 queue_chain_t msr_q; /* object request queue */
397 queue_chain_t req_q; /* vm_msync request queue */
398 unsigned int flag;
399 vm_object_offset_t offset;
400 vm_object_size_t length;
401 vm_object_t object; /* back pointer */
402 decl_lck_mtx_data(, msync_req_lock) /* Lock for this structure */
403 };
404
405 typedef struct msync_req *msync_req_t;
406 #define MSYNC_REQ_NULL ((msync_req_t) 0)
407
408
409 extern lck_grp_t vm_map_lck_grp;
410 extern lck_attr_t vm_map_lck_attr;
411
412 /*
413 * Macros to allocate and free msync_reqs
414 */
415 #define msync_req_alloc(msr) \
416 MACRO_BEGIN \
417 (msr) = (msync_req_t)kalloc(sizeof(struct msync_req)); \
418 lck_mtx_init(&(msr)->msync_req_lock, &vm_map_lck_grp, &vm_map_lck_attr); \
419 msr->flag = VM_MSYNC_INITIALIZED; \
420 MACRO_END
421
422 #define msync_req_free(msr) \
423 (kfree((msr), sizeof(struct msync_req)))
424
425 #define msr_lock(msr) lck_mtx_lock(&(msr)->msync_req_lock)
426 #define msr_unlock(msr) lck_mtx_unlock(&(msr)->msync_req_lock)
427
428 /*
429 * Declare procedures that operate on VM objects.
430 */
431
432 __private_extern__ void vm_object_bootstrap(void) __attribute__((section("__TEXT, initcode")));
433
434 __private_extern__ void vm_object_init(void);
435
436 __private_extern__ void vm_object_init_lck_grp(void);
437
438 __private_extern__ void vm_object_reaper_init(void);
439
440 __private_extern__ vm_object_t vm_object_allocate(
441 vm_object_size_t size);
442
443 __private_extern__ void _vm_object_allocate(vm_object_size_t size,
444 vm_object_t object);
445
446 #if TASK_SWAPPER
447
448 __private_extern__ void vm_object_res_reference(
449 vm_object_t object);
450 __private_extern__ void vm_object_res_deallocate(
451 vm_object_t object);
452 #define VM_OBJ_RES_INCR(object) (object)->res_count++
453 #define VM_OBJ_RES_DECR(object) (object)->res_count--
454
455 #else /* TASK_SWAPPER */
456
457 #define VM_OBJ_RES_INCR(object)
458 #define VM_OBJ_RES_DECR(object)
459 #define vm_object_res_reference(object)
460 #define vm_object_res_deallocate(object)
461
462 #endif /* TASK_SWAPPER */
463
464 #define vm_object_reference_locked(object) \
465 MACRO_BEGIN \
466 vm_object_t RLObject = (object); \
467 vm_object_lock_assert_exclusive(object); \
468 assert((RLObject)->ref_count > 0); \
469 (RLObject)->ref_count++; \
470 assert((RLObject)->ref_count > 1); \
471 vm_object_res_reference(RLObject); \
472 MACRO_END
473
474
475 #define vm_object_reference_shared(object) \
476 MACRO_BEGIN \
477 vm_object_t RLObject = (object); \
478 vm_object_lock_assert_shared(object); \
479 assert((RLObject)->ref_count > 0); \
480 OSAddAtomic(1, &(RLObject)->ref_count); \
481 assert((RLObject)->ref_count > 0); \
482 /* XXX we would need an atomic version of the following ... */ \
483 vm_object_res_reference(RLObject); \
484 MACRO_END
485
486
487 __private_extern__ void vm_object_reference(
488 vm_object_t object);
489
490 #if !MACH_ASSERT
491
492 #define vm_object_reference(object) \
493 MACRO_BEGIN \
494 vm_object_t RObject = (object); \
495 if (RObject) { \
496 vm_object_lock_shared(RObject); \
497 vm_object_reference_shared(RObject); \
498 vm_object_unlock(RObject); \
499 } \
500 MACRO_END
501
502 #endif /* MACH_ASSERT */
503
504 __private_extern__ void vm_object_deallocate(
505 vm_object_t object);
506
507 __private_extern__ kern_return_t vm_object_release_name(
508 vm_object_t object,
509 int flags);
510
511 __private_extern__ void vm_object_pmap_protect(
512 vm_object_t object,
513 vm_object_offset_t offset,
514 vm_object_size_t size,
515 pmap_t pmap,
516 vm_map_offset_t pmap_start,
517 vm_prot_t prot);
518
519 __private_extern__ void vm_object_page_remove(
520 vm_object_t object,
521 vm_object_offset_t start,
522 vm_object_offset_t end);
523
524 __private_extern__ void vm_object_deactivate_pages(
525 vm_object_t object,
526 vm_object_offset_t offset,
527 vm_object_size_t size,
528 boolean_t kill_page,
529 boolean_t reusable_page);
530
531 __private_extern__ void vm_object_reuse_pages(
532 vm_object_t object,
533 vm_object_offset_t start_offset,
534 vm_object_offset_t end_offset,
535 boolean_t allow_partial_reuse);
536
537 __private_extern__ void vm_object_purge(
538 vm_object_t object);
539
540 __private_extern__ kern_return_t vm_object_purgable_control(
541 vm_object_t object,
542 vm_purgable_t control,
543 int *state);
544
545 __private_extern__ boolean_t vm_object_coalesce(
546 vm_object_t prev_object,
547 vm_object_t next_object,
548 vm_object_offset_t prev_offset,
549 vm_object_offset_t next_offset,
550 vm_object_size_t prev_size,
551 vm_object_size_t next_size);
552
553 __private_extern__ boolean_t vm_object_shadow(
554 vm_object_t *object,
555 vm_object_offset_t *offset,
556 vm_object_size_t length);
557
558 __private_extern__ void vm_object_collapse(
559 vm_object_t object,
560 vm_object_offset_t offset,
561 boolean_t can_bypass);
562
563 __private_extern__ boolean_t vm_object_copy_quickly(
564 vm_object_t *_object,
565 vm_object_offset_t src_offset,
566 vm_object_size_t size,
567 boolean_t *_src_needs_copy,
568 boolean_t *_dst_needs_copy);
569
570 __private_extern__ kern_return_t vm_object_copy_strategically(
571 vm_object_t src_object,
572 vm_object_offset_t src_offset,
573 vm_object_size_t size,
574 vm_object_t *dst_object,
575 vm_object_offset_t *dst_offset,
576 boolean_t *dst_needs_copy);
577
578 __private_extern__ kern_return_t vm_object_copy_slowly(
579 vm_object_t src_object,
580 vm_object_offset_t src_offset,
581 vm_object_size_t size,
582 boolean_t interruptible,
583 vm_object_t *_result_object);
584
585 __private_extern__ vm_object_t vm_object_copy_delayed(
586 vm_object_t src_object,
587 vm_object_offset_t src_offset,
588 vm_object_size_t size,
589 boolean_t src_object_shared);
590
591
592
593 __private_extern__ kern_return_t vm_object_destroy(
594 vm_object_t object,
595 kern_return_t reason);
596
597 __private_extern__ void vm_object_pager_create(
598 vm_object_t object);
599
600 __private_extern__ void vm_object_page_map(
601 vm_object_t object,
602 vm_object_offset_t offset,
603 vm_object_size_t size,
604 vm_object_offset_t (*map_fn)
605 (void *, vm_object_offset_t),
606 void *map_fn_data);
607
608 __private_extern__ kern_return_t vm_object_upl_request(
609 vm_object_t object,
610 vm_object_offset_t offset,
611 upl_size_t size,
612 upl_t *upl,
613 upl_page_info_t *page_info,
614 unsigned int *count,
615 int flags);
616
617 __private_extern__ kern_return_t vm_object_transpose(
618 vm_object_t object1,
619 vm_object_t object2,
620 vm_object_size_t transpose_size);
621
622 __private_extern__ boolean_t vm_object_sync(
623 vm_object_t object,
624 vm_object_offset_t offset,
625 vm_object_size_t size,
626 boolean_t should_flush,
627 boolean_t should_return,
628 boolean_t should_iosync);
629
630 __private_extern__ kern_return_t vm_object_update(
631 vm_object_t object,
632 vm_object_offset_t offset,
633 vm_object_size_t size,
634 vm_object_offset_t *error_offset,
635 int *io_errno,
636 memory_object_return_t should_return,
637 int flags,
638 vm_prot_t prot);
639
640 __private_extern__ kern_return_t vm_object_lock_request(
641 vm_object_t object,
642 vm_object_offset_t offset,
643 vm_object_size_t size,
644 memory_object_return_t should_return,
645 int flags,
646 vm_prot_t prot);
647
648
649
650 __private_extern__ vm_object_t vm_object_enter(
651 memory_object_t pager,
652 vm_object_size_t size,
653 boolean_t internal,
654 boolean_t init,
655 boolean_t check_named);
656
657
658 __private_extern__ void vm_object_cluster_size(
659 vm_object_t object,
660 vm_object_offset_t *start,
661 vm_size_t *length,
662 vm_object_fault_info_t fault_info,
663 uint32_t *io_streaming);
664
665 __private_extern__ kern_return_t vm_object_populate_with_private(
666 vm_object_t object,
667 vm_object_offset_t offset,
668 ppnum_t phys_page,
669 vm_size_t size);
670
671 __private_extern__ void vm_object_change_wimg_mode(
672 vm_object_t object,
673 unsigned int wimg_mode);
674
675 extern kern_return_t adjust_vm_object_cache(
676 vm_size_t oval,
677 vm_size_t nval);
678
679 extern kern_return_t vm_object_page_op(
680 vm_object_t object,
681 vm_object_offset_t offset,
682 int ops,
683 ppnum_t *phys_entry,
684 int *flags);
685
686 extern kern_return_t vm_object_range_op(
687 vm_object_t object,
688 vm_object_offset_t offset_beg,
689 vm_object_offset_t offset_end,
690 int ops,
691 uint32_t *range);
692
693
694 __private_extern__ void vm_object_reap_pages(
695 vm_object_t object,
696 int reap_type);
697 #define REAP_REAP 0
698 #define REAP_TERMINATE 1
699 #define REAP_PURGEABLE 2
700 #define REAP_DATA_FLUSH 3
701
702 #if CONFIG_FREEZE
703 struct default_freezer_handle;
704
705 __private_extern__ kern_return_t
706 vm_object_pack(
707 unsigned int *purgeable_count,
708 unsigned int *wired_count,
709 unsigned int *clean_count,
710 unsigned int *dirty_count,
711 unsigned int dirty_budget,
712 boolean_t *shared,
713 vm_object_t src_object,
714 struct default_freezer_handle *df_handle);
715
716 __private_extern__ void
717 vm_object_pack_pages(
718 unsigned int *wired_count,
719 unsigned int *clean_count,
720 unsigned int *dirty_count,
721 unsigned int dirty_budget,
722 vm_object_t src_object,
723 struct default_freezer_handle *df_handle);
724
725 __private_extern__ void
726 vm_object_pageout(
727 vm_object_t object);
728
729 __private_extern__ kern_return_t
730 vm_object_pagein(
731 vm_object_t object);
732 #endif /* CONFIG_FREEZE */
733
734 /*
735 * Event waiting handling
736 */
737
738 #define VM_OBJECT_EVENT_INITIALIZED 0
739 #define VM_OBJECT_EVENT_PAGER_READY 1
740 #define VM_OBJECT_EVENT_PAGING_IN_PROGRESS 2
741 #define VM_OBJECT_EVENT_MAPPING_IN_PROGRESS 3
742 #define VM_OBJECT_EVENT_LOCK_IN_PROGRESS 4
743 #define VM_OBJECT_EVENT_UNCACHING 5
744 #define VM_OBJECT_EVENT_COPY_CALL 6
745 #define VM_OBJECT_EVENT_CACHING 7
746 #define VM_OBJECT_EVENT_UNBLOCKED 8
747 #define VM_OBJECT_EVENT_PAGING_ONLY_IN_PROGRESS 9
748
749 #define vm_object_assert_wait(object, event, interruptible) \
750 (((object)->all_wanted |= 1 << (event)), \
751 assert_wait((event_t)((vm_offset_t)(object)+(event)),(interruptible)))
752
753 #define vm_object_wait(object, event, interruptible) \
754 (vm_object_assert_wait((object),(event),(interruptible)), \
755 vm_object_unlock(object), \
756 thread_block(THREAD_CONTINUE_NULL)) \
757
758 #define thread_sleep_vm_object(object, event, interruptible) \
759 lck_rw_sleep(&(object)->Lock, LCK_SLEEP_DEFAULT, (event_t)(event), (interruptible))
760
761 #define vm_object_sleep(object, event, interruptible) \
762 (((object)->all_wanted |= 1 << (event)), \
763 thread_sleep_vm_object((object), \
764 ((vm_offset_t)(object)+(event)), (interruptible)))
765
766 #define vm_object_wakeup(object, event) \
767 MACRO_BEGIN \
768 if ((object)->all_wanted & (1 << (event))) \
769 thread_wakeup((event_t)((vm_offset_t)(object) + (event))); \
770 (object)->all_wanted &= ~(1 << (event)); \
771 MACRO_END
772
773 #define vm_object_set_wanted(object, event) \
774 MACRO_BEGIN \
775 ((object)->all_wanted |= (1 << (event))); \
776 MACRO_END
777
778 #define vm_object_wanted(object, event) \
779 ((object)->all_wanted & (1 << (event)))
780
781 /*
782 * Routines implemented as macros
783 */
784 #ifdef VM_PIP_DEBUG
785 #include <libkern/OSDebug.h>
786 #define VM_PIP_DEBUG_BEGIN(object) \
787 MACRO_BEGIN \
788 int pip = ((object)->paging_in_progress + \
789 (object)->activity_in_progress); \
790 if (pip < VM_PIP_DEBUG_MAX_REFS) { \
791 (void) OSBacktrace(&(object)->pip_holders[pip].pip_retaddr[0], \
792 VM_PIP_DEBUG_STACK_FRAMES); \
793 } \
794 MACRO_END
795 #else /* VM_PIP_DEBUG */
796 #define VM_PIP_DEBUG_BEGIN(object)
797 #endif /* VM_PIP_DEBUG */
798
799 #define vm_object_activity_begin(object) \
800 MACRO_BEGIN \
801 vm_object_lock_assert_exclusive((object)); \
802 assert((object)->paging_in_progress >= 0); \
803 VM_PIP_DEBUG_BEGIN((object)); \
804 (object)->activity_in_progress++; \
805 MACRO_END
806
807 #define vm_object_activity_end(object) \
808 MACRO_BEGIN \
809 vm_object_lock_assert_exclusive((object)); \
810 assert((object)->activity_in_progress > 0); \
811 (object)->activity_in_progress--; \
812 if ((object)->paging_in_progress == 0 && \
813 (object)->activity_in_progress == 0) \
814 vm_object_wakeup((object), \
815 VM_OBJECT_EVENT_PAGING_IN_PROGRESS); \
816 MACRO_END
817
818 #define vm_object_paging_begin(object) \
819 MACRO_BEGIN \
820 vm_object_lock_assert_exclusive((object)); \
821 assert((object)->paging_in_progress >= 0); \
822 VM_PIP_DEBUG_BEGIN((object)); \
823 (object)->paging_in_progress++; \
824 MACRO_END
825
826 #define vm_object_paging_end(object) \
827 MACRO_BEGIN \
828 vm_object_lock_assert_exclusive((object)); \
829 assert((object)->paging_in_progress > 0); \
830 (object)->paging_in_progress--; \
831 if ((object)->paging_in_progress == 0) { \
832 vm_object_wakeup((object), \
833 VM_OBJECT_EVENT_PAGING_ONLY_IN_PROGRESS); \
834 if ((object)->activity_in_progress == 0) \
835 vm_object_wakeup((object), \
836 VM_OBJECT_EVENT_PAGING_IN_PROGRESS); \
837 } \
838 MACRO_END
839
840 #define vm_object_paging_wait(object, interruptible) \
841 MACRO_BEGIN \
842 vm_object_lock_assert_exclusive((object)); \
843 while ((object)->paging_in_progress != 0 || \
844 (object)->activity_in_progress != 0) { \
845 wait_result_t _wr; \
846 \
847 _wr = vm_object_sleep((object), \
848 VM_OBJECT_EVENT_PAGING_IN_PROGRESS, \
849 (interruptible)); \
850 \
851 /*XXX if ((interruptible) && (_wr != THREAD_AWAKENED))*/\
852 /*XXX break; */ \
853 } \
854 MACRO_END
855
856 #define vm_object_paging_only_wait(object, interruptible) \
857 MACRO_BEGIN \
858 vm_object_lock_assert_exclusive((object)); \
859 while ((object)->paging_in_progress != 0) { \
860 wait_result_t _wr; \
861 \
862 _wr = vm_object_sleep((object), \
863 VM_OBJECT_EVENT_PAGING_ONLY_IN_PROGRESS,\
864 (interruptible)); \
865 \
866 /*XXX if ((interruptible) && (_wr != THREAD_AWAKENED))*/\
867 /*XXX break; */ \
868 } \
869 MACRO_END
870
871
872 #define vm_object_mapping_begin(object) \
873 MACRO_BEGIN \
874 vm_object_lock_assert_exclusive((object)); \
875 assert(! (object)->mapping_in_progress); \
876 (object)->mapping_in_progress = TRUE; \
877 MACRO_END
878
879 #define vm_object_mapping_end(object) \
880 MACRO_BEGIN \
881 vm_object_lock_assert_exclusive((object)); \
882 assert((object)->mapping_in_progress); \
883 (object)->mapping_in_progress = FALSE; \
884 vm_object_wakeup((object), \
885 VM_OBJECT_EVENT_MAPPING_IN_PROGRESS); \
886 MACRO_END
887
888 #define vm_object_mapping_wait(object, interruptible) \
889 MACRO_BEGIN \
890 vm_object_lock_assert_exclusive((object)); \
891 while ((object)->mapping_in_progress) { \
892 wait_result_t _wr; \
893 \
894 _wr = vm_object_sleep((object), \
895 VM_OBJECT_EVENT_MAPPING_IN_PROGRESS, \
896 (interruptible)); \
897 /*XXX if ((interruptible) && (_wr != THREAD_AWAKENED))*/\
898 /*XXX break; */ \
899 } \
900 assert(!(object)->mapping_in_progress); \
901 MACRO_END
902
903
904
905 #define OBJECT_LOCK_SHARED 0
906 #define OBJECT_LOCK_EXCLUSIVE 1
907
908 extern lck_grp_t vm_object_lck_grp;
909 extern lck_grp_attr_t vm_object_lck_grp_attr;
910 extern lck_attr_t vm_object_lck_attr;
911 extern lck_attr_t kernel_object_lck_attr;
912
913 extern vm_object_t vm_pageout_scan_wants_object;
914
915 extern void vm_object_lock(vm_object_t);
916 extern boolean_t vm_object_lock_try(vm_object_t);
917 extern boolean_t _vm_object_lock_try(vm_object_t);
918 extern boolean_t vm_object_lock_avoid(vm_object_t);
919 extern void vm_object_lock_shared(vm_object_t);
920 extern boolean_t vm_object_lock_try_shared(vm_object_t);
921
922 /*
923 * Object locking macros
924 */
925
926 #define vm_object_lock_init(object) \
927 lck_rw_init(&(object)->Lock, &vm_object_lck_grp, \
928 (((object) == kernel_object || \
929 (object) == vm_submap_object) ? \
930 &kernel_object_lck_attr : \
931 &vm_object_lck_attr))
932 #define vm_object_lock_destroy(object) lck_rw_destroy(&(object)->Lock, &vm_object_lck_grp)
933
934 #define vm_object_unlock(object) lck_rw_done(&(object)->Lock)
935 #define vm_object_lock_upgrade(object) lck_rw_lock_shared_to_exclusive(&(object)->Lock)
936 #define vm_object_lock_try_scan(object) _vm_object_lock_try(object)
937
938 /*
939 * CAUTION: the following vm_object_lock_assert_held*() macros merely
940 * check if anyone is holding the lock, but the holder may not necessarily
941 * be the caller...
942 */
943 #if MACH_ASSERT || DEBUG
944 #define vm_object_lock_assert_held(object) \
945 lck_rw_assert(&(object)->Lock, LCK_RW_ASSERT_HELD)
946 #define vm_object_lock_assert_shared(object) \
947 lck_rw_assert(&(object)->Lock, LCK_RW_ASSERT_SHARED)
948 #define vm_object_lock_assert_exclusive(object) \
949 lck_rw_assert(&(object)->Lock, LCK_RW_ASSERT_EXCLUSIVE)
950 #else /* MACH_ASSERT || DEBUG */
951 #define vm_object_lock_assert_held(object)
952 #define vm_object_lock_assert_shared(object)
953 #define vm_object_lock_assert_exclusive(object)
954 #endif /* MACH_ASSERT || DEBUG */
955
956 #define vm_object_round_page(x) (((vm_object_offset_t)(x) + PAGE_MASK) & ~((signed)PAGE_MASK))
957 #define vm_object_trunc_page(x) ((vm_object_offset_t)(x) & ~((signed)PAGE_MASK))
958
959 extern void vm_object_cache_add(vm_object_t);
960 extern void vm_object_cache_remove(vm_object_t);
961 extern int vm_object_cache_evict(int, int);
962
963 #endif /* _VM_VM_OBJECT_H_ */
mirror of XNU