]> git.saurik.com Git - apple/xnu.git/blob - osfmk/vm/vm_object.h
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-1699.24.23.tar.gz
[apple/xnu.git] / osfmk / vm / vm_object.h
1 /*
2 * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56 /*
57 */
58 /*
59 * File: vm_object.h
60 * Author: Avadis Tevanian, Jr., Michael Wayne Young
61 * Date: 1985
62 *
63 * Virtual memory object module definitions.
64 */
65
66 #ifndef _VM_VM_OBJECT_H_
67 #define _VM_VM_OBJECT_H_
68
69 #include <debug.h>
70 #include <mach_assert.h>
71 #include <mach_pagemap.h>
72 #include <task_swapper.h>
73
74 #include <mach/kern_return.h>
75 #include <mach/boolean.h>
76 #include <mach/memory_object_types.h>
77 #include <mach/port.h>
78 #include <mach/vm_prot.h>
79 #include <mach/vm_param.h>
80 #include <mach/machine/vm_types.h>
81 #include <kern/queue.h>
82 #include <kern/lock.h>
83 #include <kern/locks.h>
84 #include <kern/assert.h>
85 #include <kern/misc_protos.h>
86 #include <kern/macro_help.h>
87 #include <ipc/ipc_types.h>
88 #include <vm/pmap.h>
89
90 #if MACH_PAGEMAP
91 #include <vm/vm_external.h>
92 #endif /* MACH_PAGEMAP */
93
94 #include <vm/vm_options.h>
95
96 struct vm_page;
97
98 /*
99 * Types defined:
100 *
101 * vm_object_t Virtual memory object.
102 * vm_object_fault_info_t Used to determine cluster size.
103 */
104
105 struct vm_object_fault_info {
106 int interruptible;
107 uint32_t user_tag;
108 vm_size_t cluster_size;
109 vm_behavior_t behavior;
110 vm_map_offset_t lo_offset;
111 vm_map_offset_t hi_offset;
112 unsigned int
113 /* boolean_t */ no_cache:1,
114 /* boolean_t */ stealth:1,
115 /* boolean_t */ io_sync:1,
116 /* boolean_t */ cs_bypass:1,
117 /* boolean_t */ mark_zf_absent:1,
118 __vm_object_fault_info_unused_bits:27;
119 };
120
121
122 #define vo_size vo_un1.vou_size
123 #define vo_cache_pages_to_scan vo_un1.vou_cache_pages_to_scan
124 #define vo_shadow_offset vo_un2.vou_shadow_offset
125 #define vo_cache_ts vo_un2.vou_cache_ts
126
127 struct vm_object {
128 queue_head_t memq; /* Resident memory */
129 lck_rw_t Lock; /* Synchronization */
130
131 union {
132 vm_object_size_t vou_size; /* Object size (only valid if internal) */
133 int vou_cache_pages_to_scan; /* pages yet to be visited in an
134 * external object in cache
135 */
136 } vo_un1;
137
138 struct vm_page *memq_hint;
139 int ref_count; /* Number of references */
140 #if TASK_SWAPPER
141 int res_count; /* Residency references (swap)*/
142 #endif /* TASK_SWAPPER */
143 unsigned int resident_page_count;
144 /* number of resident pages */
145 unsigned int wired_page_count; /* number of wired pages */
146 unsigned int reusable_page_count;
147
148 struct vm_object *copy; /* Object that should receive
149 * a copy of my changed pages,
150 * for copy_delay, or just the
151 * temporary object that
152 * shadows this object, for
153 * copy_call.
154 */
155 struct vm_object *shadow; /* My shadow */
156
157 union {
158 vm_object_offset_t vou_shadow_offset; /* Offset into shadow */
159 clock_sec_t vou_cache_ts; /* age of an external object
160 * present in cache
161 */
162 } vo_un2;
163
164 memory_object_t pager; /* Where to get data */
165 vm_object_offset_t paging_offset; /* Offset into memory object */
166 memory_object_control_t pager_control; /* Where data comes back */
167
168 memory_object_copy_strategy_t
169 copy_strategy; /* How to handle data copy */
170
171 short paging_in_progress;
172 /* The memory object ports are
173 * being used (e.g., for pagein
174 * or pageout) -- don't change
175 * any of these fields (i.e.,
176 * don't collapse, destroy or
177 * terminate)
178 */
179 short activity_in_progress;
180
181 unsigned int
182 /* boolean_t array */ all_wanted:11, /* Bit array of "want to be
183 * awakened" notations. See
184 * VM_OBJECT_EVENT_* items
185 * below */
186 /* boolean_t */ pager_created:1, /* Has pager been created? */
187 /* boolean_t */ pager_initialized:1, /* Are fields ready to use? */
188 /* boolean_t */ pager_ready:1, /* Will pager take requests? */
189
190 /* boolean_t */ pager_trusted:1,/* The pager for this object
191 * is trusted. This is true for
192 * all internal objects (backed
193 * by the default pager)
194 */
195 /* boolean_t */ can_persist:1, /* The kernel may keep the data
196 * for this object (and rights
197 * to the memory object) after
198 * all address map references
199 * are deallocated?
200 */
201 /* boolean_t */ internal:1, /* Created by the kernel (and
202 * therefore, managed by the
203 * default memory manger)
204 */
205 /* boolean_t */ temporary:1, /* Permanent objects may be
206 * changed externally by the
207 * memory manager, and changes
208 * made in memory must be
209 * reflected back to the memory
210 * manager. Temporary objects
211 * lack both of these
212 * characteristics.
213 */
214 /* boolean_t */ private:1, /* magic device_pager object,
215 * holds private pages only */
216 /* boolean_t */ pageout:1, /* pageout object. contains
217 * private pages that refer to
218 * a real memory object. */
219 /* boolean_t */ alive:1, /* Not yet terminated */
220
221 /* boolean_t */ purgable:2, /* Purgable state. See
222 * VM_PURGABLE_*
223 */
224 /* boolean_t */ shadowed:1, /* Shadow may exist */
225 /* boolean_t */ silent_overwrite:1,
226 /* Allow full page overwrite
227 * without data_request if
228 * page is absent */
229 /* boolean_t */ advisory_pageout:1,
230 /* Instead of sending page
231 * via OOL, just notify
232 * pager that the kernel
233 * wants to discard it, page
234 * remains in object */
235 /* boolean_t */ true_share:1,
236 /* This object is mapped
237 * in more than one place
238 * and hence cannot be
239 * coalesced */
240 /* boolean_t */ terminating:1,
241 /* Allows vm_object_lookup
242 * and vm_object_deallocate
243 * to special case their
244 * behavior when they are
245 * called as a result of
246 * page cleaning during
247 * object termination
248 */
249 /* boolean_t */ named:1, /* An enforces an internal
250 * naming convention, by
251 * calling the right routines
252 * for allocation and
253 * destruction, UBC references
254 * against the vm_object are
255 * checked.
256 */
257 /* boolean_t */ shadow_severed:1,
258 /* When a permanent object
259 * backing a COW goes away
260 * unexpectedly. This bit
261 * allows vm_fault to return
262 * an error rather than a
263 * zero filled page.
264 */
265 /* boolean_t */ phys_contiguous:1,
266 /* Memory is wired and
267 * guaranteed physically
268 * contiguous. However
269 * it is not device memory
270 * and obeys normal virtual
271 * memory rules w.r.t pmap
272 * access bits.
273 */
274 /* boolean_t */ nophyscache:1;
275 /* When mapped at the
276 * pmap level, don't allow
277 * primary caching. (for
278 * I/O)
279 */
280
281
282
283 queue_chain_t cached_list; /* Attachment point for the
284 * list of objects cached as a
285 * result of their can_persist
286 * value
287 */
288
289 queue_head_t msr_q; /* memory object synchronise
290 request queue */
291
292 /*
293 * the following fields are not protected by any locks
294 * they are updated via atomic compare and swap
295 */
296 vm_object_offset_t last_alloc; /* last allocation offset */
297 int sequential; /* sequential access size */
298
299 uint32_t pages_created;
300 uint32_t pages_used;
301 #if MACH_PAGEMAP
302 vm_external_map_t existence_map; /* bitmap of pages written to
303 * backing storage */
304 #endif /* MACH_PAGEMAP */
305 vm_offset_t cow_hint; /* last page present in */
306 /* shadow but not in object */
307 #if MACH_ASSERT
308 struct vm_object *paging_object; /* object which pages to be
309 * swapped out are temporary
310 * put in current object
311 */
312 #endif
313 /* hold object lock when altering */
314 unsigned int
315 wimg_bits:8, /* cache WIMG bits */
316 code_signed:1, /* pages are signed and should be
317 validated; the signatures are stored
318 with the pager */
319 hashed:1, /* object/pager entered in hash */
320 transposed:1, /* object was transposed with another */
321 mapping_in_progress:1, /* pager being mapped/unmapped */
322 volatile_empty:1,
323 volatile_fault:1,
324 all_reusable:1,
325 blocked_access:1,
326 set_cache_attr:1,
327 __object2_unused_bits:15; /* for expansion */
328
329 uint32_t scan_collisions;
330
331 #if UPL_DEBUG
332 queue_head_t uplq; /* List of outstanding upls */
333 #endif /* UPL_DEBUG */
334
335 #ifdef VM_PIP_DEBUG
336 /*
337 * Keep track of the stack traces for the first holders
338 * of a "paging_in_progress" reference for this VM object.
339 */
340 #define VM_PIP_DEBUG_STACK_FRAMES 25 /* depth of each stack trace */
341 #define VM_PIP_DEBUG_MAX_REFS 10 /* track that many references */
342 struct __pip_backtrace {
343 void *pip_retaddr[VM_PIP_DEBUG_STACK_FRAMES];
344 } pip_holders[VM_PIP_DEBUG_MAX_REFS];
345 #endif /* VM_PIP_DEBUG */
346
347 queue_chain_t objq; /* object queue - currently used for purgable queues */
348 };
349
350 #define VM_OBJECT_PURGEABLE_FAULT_ERROR(object) \
351 ((object)->volatile_fault && \
352 ((object)->purgable == VM_PURGABLE_VOLATILE || \
353 (object)->purgable == VM_PURGABLE_EMPTY))
354
355 #define VM_PAGE_REMOVE(page) \
356 MACRO_BEGIN \
357 vm_page_t __page = (page); \
358 vm_object_t __object = __page->object; \
359 if (__page == __object->memq_hint) { \
360 vm_page_t __new_hint; \
361 queue_entry_t __qe; \
362 __qe = queue_next(&__page->listq); \
363 if (queue_end(&__object->memq, __qe)) { \
364 __qe = queue_prev(&__page->listq); \
365 if (queue_end(&__object->memq, __qe)) { \
366 __qe = NULL; \
367 } \
368 } \
369 __new_hint = (vm_page_t) __qe; \
370 __object->memq_hint = __new_hint; \
371 } \
372 queue_remove(&__object->memq, __page, vm_page_t, listq); \
373 MACRO_END
374
375 #define VM_PAGE_INSERT(page, object) \
376 MACRO_BEGIN \
377 vm_page_t __page = (page); \
378 vm_object_t __object = (object); \
379 queue_enter(&__object->memq, __page, vm_page_t, listq); \
380 __object->memq_hint = __page; \
381 MACRO_END
382
383 __private_extern__
384 vm_object_t kernel_object; /* the single kernel object */
385
386 __private_extern__
387 unsigned int vm_object_absent_max; /* maximum number of absent pages
388 at a time for each object */
389
390 # define VM_MSYNC_INITIALIZED 0
391 # define VM_MSYNC_SYNCHRONIZING 1
392 # define VM_MSYNC_DONE 2
393
394 struct msync_req {
395 queue_chain_t msr_q; /* object request queue */
396 queue_chain_t req_q; /* vm_msync request queue */
397 unsigned int flag;
398 vm_object_offset_t offset;
399 vm_object_size_t length;
400 vm_object_t object; /* back pointer */
401 decl_lck_mtx_data(, msync_req_lock) /* Lock for this structure */
402 };
403
404 typedef struct msync_req *msync_req_t;
405 #define MSYNC_REQ_NULL ((msync_req_t) 0)
406
407
408 extern lck_grp_t vm_map_lck_grp;
409 extern lck_attr_t vm_map_lck_attr;
410
411 /*
412 * Macros to allocate and free msync_reqs
413 */
414 #define msync_req_alloc(msr) \
415 MACRO_BEGIN \
416 (msr) = (msync_req_t)kalloc(sizeof(struct msync_req)); \
417 lck_mtx_init(&(msr)->msync_req_lock, &vm_map_lck_grp, &vm_map_lck_attr); \
418 msr->flag = VM_MSYNC_INITIALIZED; \
419 MACRO_END
420
421 #define msync_req_free(msr) \
422 (kfree((msr), sizeof(struct msync_req)))
423
424 #define msr_lock(msr) lck_mtx_lock(&(msr)->msync_req_lock)
425 #define msr_unlock(msr) lck_mtx_unlock(&(msr)->msync_req_lock)
426
427 /*
428 * Declare procedures that operate on VM objects.
429 */
430
431 __private_extern__ void vm_object_bootstrap(void) __attribute__((section("__TEXT, initcode")));
432
433 __private_extern__ void vm_object_init(void);
434
435 __private_extern__ void vm_object_init_lck_grp(void);
436
437 __private_extern__ void vm_object_reaper_init(void);
438
439 __private_extern__ vm_object_t vm_object_allocate(
440 vm_object_size_t size);
441
442 __private_extern__ void _vm_object_allocate(vm_object_size_t size,
443 vm_object_t object);
444
445 #if TASK_SWAPPER
446
447 __private_extern__ void vm_object_res_reference(
448 vm_object_t object);
449 __private_extern__ void vm_object_res_deallocate(
450 vm_object_t object);
451 #define VM_OBJ_RES_INCR(object) (object)->res_count++
452 #define VM_OBJ_RES_DECR(object) (object)->res_count--
453
454 #else /* TASK_SWAPPER */
455
456 #define VM_OBJ_RES_INCR(object)
457 #define VM_OBJ_RES_DECR(object)
458 #define vm_object_res_reference(object)
459 #define vm_object_res_deallocate(object)
460
461 #endif /* TASK_SWAPPER */
462
463 #define vm_object_reference_locked(object) \
464 MACRO_BEGIN \
465 vm_object_t RLObject = (object); \
466 vm_object_lock_assert_exclusive(object); \
467 assert((RLObject)->ref_count > 0); \
468 (RLObject)->ref_count++; \
469 assert((RLObject)->ref_count > 1); \
470 vm_object_res_reference(RLObject); \
471 MACRO_END
472
473
474 #define vm_object_reference_shared(object) \
475 MACRO_BEGIN \
476 vm_object_t RLObject = (object); \
477 vm_object_lock_assert_shared(object); \
478 assert((RLObject)->ref_count > 0); \
479 OSAddAtomic(1, &(RLObject)->ref_count); \
480 assert((RLObject)->ref_count > 1); \
481 /* XXX we would need an atomic version of the following ... */ \
482 vm_object_res_reference(RLObject); \
483 MACRO_END
484
485
486 __private_extern__ void vm_object_reference(
487 vm_object_t object);
488
489 #if !MACH_ASSERT
490
491 #define vm_object_reference(object) \
492 MACRO_BEGIN \
493 vm_object_t RObject = (object); \
494 if (RObject) { \
495 vm_object_lock_shared(RObject); \
496 vm_object_reference_shared(RObject); \
497 vm_object_unlock(RObject); \
498 } \
499 MACRO_END
500
501 #endif /* MACH_ASSERT */
502
503 __private_extern__ void vm_object_deallocate(
504 vm_object_t object);
505
506 __private_extern__ kern_return_t vm_object_release_name(
507 vm_object_t object,
508 int flags);
509
510 __private_extern__ void vm_object_pmap_protect(
511 vm_object_t object,
512 vm_object_offset_t offset,
513 vm_object_size_t size,
514 pmap_t pmap,
515 vm_map_offset_t pmap_start,
516 vm_prot_t prot);
517
518 __private_extern__ void vm_object_page_remove(
519 vm_object_t object,
520 vm_object_offset_t start,
521 vm_object_offset_t end);
522
523 __private_extern__ void vm_object_deactivate_pages(
524 vm_object_t object,
525 vm_object_offset_t offset,
526 vm_object_size_t size,
527 boolean_t kill_page,
528 boolean_t reusable_page);
529
530 __private_extern__ void vm_object_reuse_pages(
531 vm_object_t object,
532 vm_object_offset_t start_offset,
533 vm_object_offset_t end_offset,
534 boolean_t allow_partial_reuse);
535
536 __private_extern__ void vm_object_purge(
537 vm_object_t object);
538
539 __private_extern__ kern_return_t vm_object_purgable_control(
540 vm_object_t object,
541 vm_purgable_t control,
542 int *state);
543
544 __private_extern__ boolean_t vm_object_coalesce(
545 vm_object_t prev_object,
546 vm_object_t next_object,
547 vm_object_offset_t prev_offset,
548 vm_object_offset_t next_offset,
549 vm_object_size_t prev_size,
550 vm_object_size_t next_size);
551
552 __private_extern__ boolean_t vm_object_shadow(
553 vm_object_t *object,
554 vm_object_offset_t *offset,
555 vm_object_size_t length);
556
557 __private_extern__ void vm_object_collapse(
558 vm_object_t object,
559 vm_object_offset_t offset,
560 boolean_t can_bypass);
561
562 __private_extern__ boolean_t vm_object_copy_quickly(
563 vm_object_t *_object,
564 vm_object_offset_t src_offset,
565 vm_object_size_t size,
566 boolean_t *_src_needs_copy,
567 boolean_t *_dst_needs_copy);
568
569 __private_extern__ kern_return_t vm_object_copy_strategically(
570 vm_object_t src_object,
571 vm_object_offset_t src_offset,
572 vm_object_size_t size,
573 vm_object_t *dst_object,
574 vm_object_offset_t *dst_offset,
575 boolean_t *dst_needs_copy);
576
577 __private_extern__ kern_return_t vm_object_copy_slowly(
578 vm_object_t src_object,
579 vm_object_offset_t src_offset,
580 vm_object_size_t size,
581 boolean_t interruptible,
582 vm_object_t *_result_object);
583
584 __private_extern__ vm_object_t vm_object_copy_delayed(
585 vm_object_t src_object,
586 vm_object_offset_t src_offset,
587 vm_object_size_t size,
588 boolean_t src_object_shared);
589
590
591
592 __private_extern__ kern_return_t vm_object_destroy(
593 vm_object_t object,
594 kern_return_t reason);
595
596 __private_extern__ void vm_object_pager_create(
597 vm_object_t object);
598
599 __private_extern__ void vm_object_page_map(
600 vm_object_t object,
601 vm_object_offset_t offset,
602 vm_object_size_t size,
603 vm_object_offset_t (*map_fn)
604 (void *, vm_object_offset_t),
605 void *map_fn_data);
606
607 __private_extern__ kern_return_t vm_object_upl_request(
608 vm_object_t object,
609 vm_object_offset_t offset,
610 upl_size_t size,
611 upl_t *upl,
612 upl_page_info_t *page_info,
613 unsigned int *count,
614 int flags);
615
616 __private_extern__ kern_return_t vm_object_transpose(
617 vm_object_t object1,
618 vm_object_t object2,
619 vm_object_size_t transpose_size);
620
621 __private_extern__ boolean_t vm_object_sync(
622 vm_object_t object,
623 vm_object_offset_t offset,
624 vm_object_size_t size,
625 boolean_t should_flush,
626 boolean_t should_return,
627 boolean_t should_iosync);
628
629 __private_extern__ kern_return_t vm_object_update(
630 vm_object_t object,
631 vm_object_offset_t offset,
632 vm_object_size_t size,
633 vm_object_offset_t *error_offset,
634 int *io_errno,
635 memory_object_return_t should_return,
636 int flags,
637 vm_prot_t prot);
638
639 __private_extern__ kern_return_t vm_object_lock_request(
640 vm_object_t object,
641 vm_object_offset_t offset,
642 vm_object_size_t size,
643 memory_object_return_t should_return,
644 int flags,
645 vm_prot_t prot);
646
647
648
649 __private_extern__ vm_object_t vm_object_enter(
650 memory_object_t pager,
651 vm_object_size_t size,
652 boolean_t internal,
653 boolean_t init,
654 boolean_t check_named);
655
656
657 __private_extern__ void vm_object_cluster_size(
658 vm_object_t object,
659 vm_object_offset_t *start,
660 vm_size_t *length,
661 vm_object_fault_info_t fault_info,
662 uint32_t *io_streaming);
663
664 __private_extern__ kern_return_t vm_object_populate_with_private(
665 vm_object_t object,
666 vm_object_offset_t offset,
667 ppnum_t phys_page,
668 vm_size_t size);
669
670 __private_extern__ void vm_object_change_wimg_mode(
671 vm_object_t object,
672 unsigned int wimg_mode);
673
674 extern kern_return_t adjust_vm_object_cache(
675 vm_size_t oval,
676 vm_size_t nval);
677
678 extern kern_return_t vm_object_page_op(
679 vm_object_t object,
680 vm_object_offset_t offset,
681 int ops,
682 ppnum_t *phys_entry,
683 int *flags);
684
685 extern kern_return_t vm_object_range_op(
686 vm_object_t object,
687 vm_object_offset_t offset_beg,
688 vm_object_offset_t offset_end,
689 int ops,
690 uint32_t *range);
691
692
693 __private_extern__ void vm_object_reap_pages(
694 vm_object_t object,
695 int reap_type);
696 #define REAP_REAP 0
697 #define REAP_TERMINATE 1
698 #define REAP_PURGEABLE 2
699 #define REAP_DATA_FLUSH 3
700
701 #if CONFIG_FREEZE
702
703 __private_extern__ kern_return_t
704 vm_object_pack(
705 unsigned int *purgeable_count,
706 unsigned int *wired_count,
707 unsigned int *clean_count,
708 unsigned int *dirty_count,
709 boolean_t *shared,
710 vm_object_t src_object,
711 vm_object_t dst_object,
712 void **table,
713 vm_object_offset_t *offset);
714
715 __private_extern__ void
716 vm_object_pack_pages(
717 unsigned int *wired_count,
718 unsigned int *clean_count,
719 unsigned int *dirty_count,
720 vm_object_t src_object,
721 vm_object_t dst_object,
722 void **table,
723 vm_object_offset_t *offset);
724
725 __private_extern__ void vm_object_pageout(
726 vm_object_t object);
727
728 __private_extern__ kern_return_t vm_object_pagein(
729 vm_object_t object);
730
731 __private_extern__ void vm_object_unpack(
732 vm_object_t object,
733 void **table);
734
735 #endif /* CONFIG_FREEZE */
736
737 /*
738 * Event waiting handling
739 */
740
741 #define VM_OBJECT_EVENT_INITIALIZED 0
742 #define VM_OBJECT_EVENT_PAGER_READY 1
743 #define VM_OBJECT_EVENT_PAGING_IN_PROGRESS 2
744 #define VM_OBJECT_EVENT_MAPPING_IN_PROGRESS 3
745 #define VM_OBJECT_EVENT_LOCK_IN_PROGRESS 4
746 #define VM_OBJECT_EVENT_UNCACHING 5
747 #define VM_OBJECT_EVENT_COPY_CALL 6
748 #define VM_OBJECT_EVENT_CACHING 7
749 #define VM_OBJECT_EVENT_UNBLOCKED 8
750 #define VM_OBJECT_EVENT_PAGING_ONLY_IN_PROGRESS 9
751
752 #define vm_object_assert_wait(object, event, interruptible) \
753 (((object)->all_wanted |= 1 << (event)), \
754 assert_wait((event_t)((vm_offset_t)(object)+(event)),(interruptible)))
755
756 #define vm_object_wait(object, event, interruptible) \
757 (vm_object_assert_wait((object),(event),(interruptible)), \
758 vm_object_unlock(object), \
759 thread_block(THREAD_CONTINUE_NULL)) \
760
761 #define thread_sleep_vm_object(object, event, interruptible) \
762 lck_rw_sleep(&(object)->Lock, LCK_SLEEP_DEFAULT, (event_t)(event), (interruptible))
763
764 #define vm_object_sleep(object, event, interruptible) \
765 (((object)->all_wanted |= 1 << (event)), \
766 thread_sleep_vm_object((object), \
767 ((vm_offset_t)(object)+(event)), (interruptible)))
768
769 #define vm_object_wakeup(object, event) \
770 MACRO_BEGIN \
771 if ((object)->all_wanted & (1 << (event))) \
772 thread_wakeup((event_t)((vm_offset_t)(object) + (event))); \
773 (object)->all_wanted &= ~(1 << (event)); \
774 MACRO_END
775
776 #define vm_object_set_wanted(object, event) \
777 MACRO_BEGIN \
778 ((object)->all_wanted |= (1 << (event))); \
779 MACRO_END
780
781 #define vm_object_wanted(object, event) \
782 ((object)->all_wanted & (1 << (event)))
783
784 /*
785 * Routines implemented as macros
786 */
787 #ifdef VM_PIP_DEBUG
788 #include <libkern/OSDebug.h>
789 #define VM_PIP_DEBUG_BEGIN(object) \
790 MACRO_BEGIN \
791 int pip = ((object)->paging_in_progress + \
792 (object)->activity_in_progress); \
793 if (pip < VM_PIP_DEBUG_MAX_REFS) { \
794 (void) OSBacktrace(&(object)->pip_holders[pip].pip_retaddr[0], \
795 VM_PIP_DEBUG_STACK_FRAMES); \
796 } \
797 MACRO_END
798 #else /* VM_PIP_DEBUG */
799 #define VM_PIP_DEBUG_BEGIN(object)
800 #endif /* VM_PIP_DEBUG */
801
802 #define vm_object_activity_begin(object) \
803 MACRO_BEGIN \
804 vm_object_lock_assert_exclusive((object)); \
805 assert((object)->paging_in_progress >= 0); \
806 VM_PIP_DEBUG_BEGIN((object)); \
807 (object)->activity_in_progress++; \
808 MACRO_END
809
810 #define vm_object_activity_end(object) \
811 MACRO_BEGIN \
812 vm_object_lock_assert_exclusive((object)); \
813 assert((object)->activity_in_progress > 0); \
814 (object)->activity_in_progress--; \
815 if ((object)->paging_in_progress == 0 && \
816 (object)->activity_in_progress == 0) \
817 vm_object_wakeup((object), \
818 VM_OBJECT_EVENT_PAGING_IN_PROGRESS); \
819 MACRO_END
820
821 #define vm_object_paging_begin(object) \
822 MACRO_BEGIN \
823 vm_object_lock_assert_exclusive((object)); \
824 assert((object)->paging_in_progress >= 0); \
825 VM_PIP_DEBUG_BEGIN((object)); \
826 (object)->paging_in_progress++; \
827 MACRO_END
828
829 #define vm_object_paging_end(object) \
830 MACRO_BEGIN \
831 vm_object_lock_assert_exclusive((object)); \
832 assert((object)->paging_in_progress > 0); \
833 (object)->paging_in_progress--; \
834 if ((object)->paging_in_progress == 0) { \
835 vm_object_wakeup((object), \
836 VM_OBJECT_EVENT_PAGING_ONLY_IN_PROGRESS); \
837 if ((object)->activity_in_progress == 0) \
838 vm_object_wakeup((object), \
839 VM_OBJECT_EVENT_PAGING_IN_PROGRESS); \
840 } \
841 MACRO_END
842
843 #define vm_object_paging_wait(object, interruptible) \
844 MACRO_BEGIN \
845 vm_object_lock_assert_exclusive((object)); \
846 while ((object)->paging_in_progress != 0 || \
847 (object)->activity_in_progress != 0) { \
848 wait_result_t _wr; \
849 \
850 _wr = vm_object_sleep((object), \
851 VM_OBJECT_EVENT_PAGING_IN_PROGRESS, \
852 (interruptible)); \
853 \
854 /*XXX if ((interruptible) && (_wr != THREAD_AWAKENED))*/\
855 /*XXX break; */ \
856 } \
857 MACRO_END
858
859 #define vm_object_paging_only_wait(object, interruptible) \
860 MACRO_BEGIN \
861 vm_object_lock_assert_exclusive((object)); \
862 while ((object)->paging_in_progress != 0) { \
863 wait_result_t _wr; \
864 \
865 _wr = vm_object_sleep((object), \
866 VM_OBJECT_EVENT_PAGING_ONLY_IN_PROGRESS,\
867 (interruptible)); \
868 \
869 /*XXX if ((interruptible) && (_wr != THREAD_AWAKENED))*/\
870 /*XXX break; */ \
871 } \
872 MACRO_END
873
874
875 #define vm_object_mapping_begin(object) \
876 MACRO_BEGIN \
877 vm_object_lock_assert_exclusive((object)); \
878 assert(! (object)->mapping_in_progress); \
879 (object)->mapping_in_progress = TRUE; \
880 MACRO_END
881
882 #define vm_object_mapping_end(object) \
883 MACRO_BEGIN \
884 vm_object_lock_assert_exclusive((object)); \
885 assert((object)->mapping_in_progress); \
886 (object)->mapping_in_progress = FALSE; \
887 vm_object_wakeup((object), \
888 VM_OBJECT_EVENT_MAPPING_IN_PROGRESS); \
889 MACRO_END
890
891 #define vm_object_mapping_wait(object, interruptible) \
892 MACRO_BEGIN \
893 vm_object_lock_assert_exclusive((object)); \
894 while ((object)->mapping_in_progress) { \
895 wait_result_t _wr; \
896 \
897 _wr = vm_object_sleep((object), \
898 VM_OBJECT_EVENT_MAPPING_IN_PROGRESS, \
899 (interruptible)); \
900 /*XXX if ((interruptible) && (_wr != THREAD_AWAKENED))*/\
901 /*XXX break; */ \
902 } \
903 assert(!(object)->mapping_in_progress); \
904 MACRO_END
905
906
907
908 #define OBJECT_LOCK_SHARED 0
909 #define OBJECT_LOCK_EXCLUSIVE 1
910
911 extern lck_grp_t vm_object_lck_grp;
912 extern lck_grp_attr_t vm_object_lck_grp_attr;
913 extern lck_attr_t vm_object_lck_attr;
914 extern lck_attr_t kernel_object_lck_attr;
915
916 extern vm_object_t vm_pageout_scan_wants_object;
917
918 extern void vm_object_lock(vm_object_t);
919 extern boolean_t vm_object_lock_try(vm_object_t);
920 extern boolean_t _vm_object_lock_try(vm_object_t);
921 extern boolean_t vm_object_lock_avoid(vm_object_t);
922 extern void vm_object_lock_shared(vm_object_t);
923 extern boolean_t vm_object_lock_try_shared(vm_object_t);
924
925 /*
926 * Object locking macros
927 */
928
929 #define vm_object_lock_init(object) \
930 lck_rw_init(&(object)->Lock, &vm_object_lck_grp, \
931 (((object) == kernel_object || \
932 (object) == vm_submap_object) ? \
933 &kernel_object_lck_attr : \
934 &vm_object_lck_attr))
935 #define vm_object_lock_destroy(object) lck_rw_destroy(&(object)->Lock, &vm_object_lck_grp)
936
937 #define vm_object_unlock(object) lck_rw_done(&(object)->Lock)
938 #define vm_object_lock_upgrade(object) lck_rw_lock_shared_to_exclusive(&(object)->Lock)
939 #define vm_object_lock_try_scan(object) _vm_object_lock_try(object)
940
941 /*
942 * CAUTION: the following vm_object_lock_assert_held*() macros merely
943 * check if anyone is holding the lock, but the holder may not necessarily
944 * be the caller...
945 */
946 #if MACH_ASSERT || DEBUG
947 #define vm_object_lock_assert_held(object) \
948 lck_rw_assert(&(object)->Lock, LCK_RW_ASSERT_HELD)
949 #define vm_object_lock_assert_shared(object) \
950 lck_rw_assert(&(object)->Lock, LCK_RW_ASSERT_SHARED)
951 #define vm_object_lock_assert_exclusive(object) \
952 lck_rw_assert(&(object)->Lock, LCK_RW_ASSERT_EXCLUSIVE)
953 #else /* MACH_ASSERT || DEBUG */
954 #define vm_object_lock_assert_held(object)
955 #define vm_object_lock_assert_shared(object)
956 #define vm_object_lock_assert_exclusive(object)
957 #endif /* MACH_ASSERT || DEBUG */
958
959 #define vm_object_round_page(x) (((vm_object_offset_t)(x) + PAGE_MASK) & ~((signed)PAGE_MASK))
960 #define vm_object_trunc_page(x) ((vm_object_offset_t)(x) & ~((signed)PAGE_MASK))
961
962 extern void vm_object_cache_add(vm_object_t);
963 extern void vm_object_cache_remove(vm_object_t);
964 extern int vm_object_cache_evict(int, int);
965
966 #endif /* _VM_VM_OBJECT_H_ */
mirror of XNU