]> git.saurik.com Git - apple/xnu.git/blob - osfmk/vm/vm_object.h
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-1504.3.12.tar.gz
[apple/xnu.git] / osfmk / vm / vm_object.h
1 /*
2 * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56 /*
57 */
58 /*
59 * File: vm_object.h
60 * Author: Avadis Tevanian, Jr., Michael Wayne Young
61 * Date: 1985
62 *
63 * Virtual memory object module definitions.
64 */
65
66 #ifndef _VM_VM_OBJECT_H_
67 #define _VM_VM_OBJECT_H_
68
69 #include <mach_pagemap.h>
70 #include <task_swapper.h>
71
72 #include <mach/kern_return.h>
73 #include <mach/boolean.h>
74 #include <mach/memory_object_types.h>
75 #include <mach/port.h>
76 #include <mach/vm_prot.h>
77 #include <mach/vm_param.h>
78 #include <mach/machine/vm_types.h>
79 #include <kern/queue.h>
80 #include <kern/lock.h>
81 #include <kern/locks.h>
82 #include <kern/assert.h>
83 #include <kern/misc_protos.h>
84 #include <kern/macro_help.h>
85 #include <ipc/ipc_types.h>
86 #include <vm/pmap.h>
87
88 #if MACH_PAGEMAP
89 #include <vm/vm_external.h>
90 #endif /* MACH_PAGEMAP */
91
92 #include <vm/vm_options.h>
93
94 struct vm_page;
95
96 /*
97 * Types defined:
98 *
99 * vm_object_t Virtual memory object.
100 * vm_object_fault_info_t Used to determine cluster size.
101 */
102
103 struct vm_object_fault_info {
104 int interruptible;
105 uint32_t user_tag;
106 vm_size_t cluster_size;
107 vm_behavior_t behavior;
108 vm_map_offset_t lo_offset;
109 vm_map_offset_t hi_offset;
110 boolean_t no_cache;
111 boolean_t stealth;
112 };
113
114
115
116 struct vm_object {
117 queue_head_t memq; /* Resident memory */
118 lck_rw_t Lock; /* Synchronization */
119
120 vm_object_size_t size; /* Object size (only valid
121 * if internal)
122 */
123 struct vm_page *memq_hint;
124 int ref_count; /* Number of references */
125 #if TASK_SWAPPER
126 int res_count; /* Residency references (swap)*/
127 #endif /* TASK_SWAPPER */
128 unsigned int resident_page_count;
129 /* number of resident pages */
130 unsigned int wired_page_count; /* number of wired pages */
131 unsigned int reusable_page_count;
132
133 struct vm_object *copy; /* Object that should receive
134 * a copy of my changed pages,
135 * for copy_delay, or just the
136 * temporary object that
137 * shadows this object, for
138 * copy_call.
139 */
140 struct vm_object *shadow; /* My shadow */
141 vm_object_offset_t shadow_offset; /* Offset into shadow */
142
143 memory_object_t pager; /* Where to get data */
144 vm_object_offset_t paging_offset; /* Offset into memory object */
145 memory_object_control_t pager_control; /* Where data comes back */
146
147 memory_object_copy_strategy_t
148 copy_strategy; /* How to handle data copy */
149
150 short paging_in_progress;
151 /* The memory object ports are
152 * being used (e.g., for pagein
153 * or pageout) -- don't change
154 * any of these fields (i.e.,
155 * don't collapse, destroy or
156 * terminate)
157 */
158 short activity_in_progress;
159
160 unsigned int
161 /* boolean_t array */ all_wanted:11, /* Bit array of "want to be
162 * awakened" notations. See
163 * VM_OBJECT_EVENT_* items
164 * below */
165 /* boolean_t */ pager_created:1, /* Has pager been created? */
166 /* boolean_t */ pager_initialized:1, /* Are fields ready to use? */
167 /* boolean_t */ pager_ready:1, /* Will pager take requests? */
168
169 /* boolean_t */ pager_trusted:1,/* The pager for this object
170 * is trusted. This is true for
171 * all internal objects (backed
172 * by the default pager)
173 */
174 /* boolean_t */ can_persist:1, /* The kernel may keep the data
175 * for this object (and rights
176 * to the memory object) after
177 * all address map references
178 * are deallocated?
179 */
180 /* boolean_t */ internal:1, /* Created by the kernel (and
181 * therefore, managed by the
182 * default memory manger)
183 */
184 /* boolean_t */ temporary:1, /* Permanent objects may be
185 * changed externally by the
186 * memory manager, and changes
187 * made in memory must be
188 * reflected back to the memory
189 * manager. Temporary objects
190 * lack both of these
191 * characteristics.
192 */
193 /* boolean_t */ private:1, /* magic device_pager object,
194 * holds private pages only */
195 /* boolean_t */ pageout:1, /* pageout object. contains
196 * private pages that refer to
197 * a real memory object. */
198 /* boolean_t */ alive:1, /* Not yet terminated */
199
200 /* boolean_t */ purgable:2, /* Purgable state. See
201 * VM_PURGABLE_*
202 */
203 /* boolean_t */ shadowed:1, /* Shadow may exist */
204 /* boolean_t */ silent_overwrite:1,
205 /* Allow full page overwrite
206 * without data_request if
207 * page is absent */
208 /* boolean_t */ advisory_pageout:1,
209 /* Instead of sending page
210 * via OOL, just notify
211 * pager that the kernel
212 * wants to discard it, page
213 * remains in object */
214 /* boolean_t */ true_share:1,
215 /* This object is mapped
216 * in more than one place
217 * and hence cannot be
218 * coalesced */
219 /* boolean_t */ terminating:1,
220 /* Allows vm_object_lookup
221 * and vm_object_deallocate
222 * to special case their
223 * behavior when they are
224 * called as a result of
225 * page cleaning during
226 * object termination
227 */
228 /* boolean_t */ named:1, /* An enforces an internal
229 * naming convention, by
230 * calling the right routines
231 * for allocation and
232 * destruction, UBC references
233 * against the vm_object are
234 * checked.
235 */
236 /* boolean_t */ shadow_severed:1,
237 /* When a permanent object
238 * backing a COW goes away
239 * unexpectedly. This bit
240 * allows vm_fault to return
241 * an error rather than a
242 * zero filled page.
243 */
244 /* boolean_t */ phys_contiguous:1,
245 /* Memory is wired and
246 * guaranteed physically
247 * contiguous. However
248 * it is not device memory
249 * and obeys normal virtual
250 * memory rules w.r.t pmap
251 * access bits.
252 */
253 /* boolean_t */ nophyscache:1;
254 /* When mapped at the
255 * pmap level, don't allow
256 * primary caching. (for
257 * I/O)
258 */
259
260
261
262 queue_chain_t cached_list; /* Attachment point for the
263 * list of objects cached as a
264 * result of their can_persist
265 * value
266 */
267
268 queue_head_t msr_q; /* memory object synchronise
269 request queue */
270
271 /*
272 * the following fields are not protected by any locks
273 * they are updated via atomic compare and swap
274 */
275 vm_object_offset_t last_alloc; /* last allocation offset */
276 int sequential; /* sequential access size */
277
278 uint32_t pages_created;
279 uint32_t pages_used;
280 #if MACH_PAGEMAP
281 vm_external_map_t existence_map; /* bitmap of pages written to
282 * backing storage */
283 #endif /* MACH_PAGEMAP */
284 vm_offset_t cow_hint; /* last page present in */
285 /* shadow but not in object */
286 #if MACH_ASSERT
287 struct vm_object *paging_object; /* object which pages to be
288 * swapped out are temporary
289 * put in current object
290 */
291 #endif
292 /* hold object lock when altering */
293 unsigned int
294 wimg_bits:8, /* cache WIMG bits */
295 code_signed:1, /* pages are signed and should be
296 validated; the signatures are stored
297 with the pager */
298 hashed:1, /* object/pager entered in hash */
299 transposed:1, /* object was transposed with another */
300 mapping_in_progress:1, /* pager being mapped/unmapped */
301 volatile_empty:1,
302 volatile_fault:1,
303 all_reusable:1,
304 blocked_access:1,
305 __object2_unused_bits:16; /* for expansion */
306
307 #if UPL_DEBUG
308 queue_head_t uplq; /* List of outstanding upls */
309 #endif /* UPL_DEBUG */
310
311 #ifdef VM_PIP_DEBUG
312 /*
313 * Keep track of the stack traces for the first holders
314 * of a "paging_in_progress" reference for this VM object.
315 */
316 #define VM_PIP_DEBUG_STACK_FRAMES 25 /* depth of each stack trace */
317 #define VM_PIP_DEBUG_MAX_REFS 10 /* track that many references */
318 struct __pip_backtrace {
319 void *pip_retaddr[VM_PIP_DEBUG_STACK_FRAMES];
320 } pip_holders[VM_PIP_DEBUG_MAX_REFS];
321 #endif /* VM_PIP_DEBUG */
322
323 queue_chain_t objq; /* object queue - currently used for purgable queues */
324 };
325
326 #define VM_OBJECT_PURGEABLE_FAULT_ERROR(object) \
327 ((object)->volatile_fault && \
328 ((object)->purgable == VM_PURGABLE_VOLATILE || \
329 (object)->purgable == VM_PURGABLE_EMPTY))
330
331 #define VM_PAGE_REMOVE(page) \
332 MACRO_BEGIN \
333 vm_page_t __page = (page); \
334 vm_object_t __object = __page->object; \
335 if (__page == __object->memq_hint) { \
336 vm_page_t __new_hint; \
337 queue_entry_t __qe; \
338 __qe = queue_next(&__page->listq); \
339 if (queue_end(&__object->memq, __qe)) { \
340 __qe = queue_prev(&__page->listq); \
341 if (queue_end(&__object->memq, __qe)) { \
342 __qe = NULL; \
343 } \
344 } \
345 __new_hint = (vm_page_t) __qe; \
346 __object->memq_hint = __new_hint; \
347 } \
348 queue_remove(&__object->memq, __page, vm_page_t, listq); \
349 MACRO_END
350
351 #define VM_PAGE_INSERT(page, object) \
352 MACRO_BEGIN \
353 vm_page_t __page = (page); \
354 vm_object_t __object = (object); \
355 queue_enter(&__object->memq, __page, vm_page_t, listq); \
356 __object->memq_hint = __page; \
357 MACRO_END
358
359 __private_extern__
360 vm_object_t kernel_object; /* the single kernel object */
361
362 __private_extern__
363 unsigned int vm_object_absent_max; /* maximum number of absent pages
364 at a time for each object */
365
366 # define VM_MSYNC_INITIALIZED 0
367 # define VM_MSYNC_SYNCHRONIZING 1
368 # define VM_MSYNC_DONE 2
369
370 struct msync_req {
371 queue_chain_t msr_q; /* object request queue */
372 queue_chain_t req_q; /* vm_msync request queue */
373 unsigned int flag;
374 vm_object_offset_t offset;
375 vm_object_size_t length;
376 vm_object_t object; /* back pointer */
377 decl_lck_mtx_data(, msync_req_lock) /* Lock for this structure */
378 };
379
380 typedef struct msync_req *msync_req_t;
381 #define MSYNC_REQ_NULL ((msync_req_t) 0)
382
383
384 extern lck_grp_t vm_map_lck_grp;
385 extern lck_attr_t vm_map_lck_attr;
386
387 /*
388 * Macros to allocate and free msync_reqs
389 */
390 #define msync_req_alloc(msr) \
391 MACRO_BEGIN \
392 (msr) = (msync_req_t)kalloc(sizeof(struct msync_req)); \
393 lck_mtx_init(&(msr)->msync_req_lock, &vm_map_lck_grp, &vm_map_lck_attr); \
394 msr->flag = VM_MSYNC_INITIALIZED; \
395 MACRO_END
396
397 #define msync_req_free(msr) \
398 (kfree((msr), sizeof(struct msync_req)))
399
400 #define msr_lock(msr) lck_mtx_lock(&(msr)->msync_req_lock)
401 #define msr_unlock(msr) lck_mtx_unlock(&(msr)->msync_req_lock)
402
403 /*
404 * Declare procedures that operate on VM objects.
405 */
406
407 __private_extern__ void vm_object_bootstrap(void) __attribute__((section("__TEXT, initcode")));
408
409 __private_extern__ void vm_object_init(void);
410
411 __private_extern__ void vm_object_init_lck_grp(void);
412
413 __private_extern__ void vm_object_reaper_init(void);
414
415 __private_extern__ vm_object_t vm_object_allocate(
416 vm_object_size_t size);
417
418 __private_extern__ void _vm_object_allocate(vm_object_size_t size,
419 vm_object_t object);
420
421 #if TASK_SWAPPER
422
423 __private_extern__ void vm_object_res_reference(
424 vm_object_t object);
425 __private_extern__ void vm_object_res_deallocate(
426 vm_object_t object);
427 #define VM_OBJ_RES_INCR(object) (object)->res_count++
428 #define VM_OBJ_RES_DECR(object) (object)->res_count--
429
430 #else /* TASK_SWAPPER */
431
432 #define VM_OBJ_RES_INCR(object)
433 #define VM_OBJ_RES_DECR(object)
434 #define vm_object_res_reference(object)
435 #define vm_object_res_deallocate(object)
436
437 #endif /* TASK_SWAPPER */
438
439 #define vm_object_reference_locked(object) \
440 MACRO_BEGIN \
441 vm_object_t RLObject = (object); \
442 vm_object_lock_assert_exclusive(object); \
443 assert((RLObject)->ref_count > 0); \
444 (RLObject)->ref_count++; \
445 assert((RLObject)->ref_count > 1); \
446 vm_object_res_reference(RLObject); \
447 MACRO_END
448
449
450 #define vm_object_reference_shared(object) \
451 MACRO_BEGIN \
452 vm_object_t RLObject = (object); \
453 vm_object_lock_assert_shared(object); \
454 assert((RLObject)->ref_count > 0); \
455 OSAddAtomic(1, &(RLObject)->ref_count); \
456 assert((RLObject)->ref_count > 1); \
457 /* XXX we would need an atomic version of the following ... */ \
458 vm_object_res_reference(RLObject); \
459 MACRO_END
460
461
462 __private_extern__ void vm_object_reference(
463 vm_object_t object);
464
465 #if !MACH_ASSERT
466
467 #define vm_object_reference(object) \
468 MACRO_BEGIN \
469 vm_object_t RObject = (object); \
470 if (RObject) { \
471 vm_object_lock_shared(RObject); \
472 vm_object_reference_shared(RObject); \
473 vm_object_unlock(RObject); \
474 } \
475 MACRO_END
476
477 #endif /* MACH_ASSERT */
478
479 __private_extern__ void vm_object_deallocate(
480 vm_object_t object);
481
482 __private_extern__ kern_return_t vm_object_release_name(
483 vm_object_t object,
484 int flags);
485
486 __private_extern__ void vm_object_pmap_protect(
487 vm_object_t object,
488 vm_object_offset_t offset,
489 vm_object_size_t size,
490 pmap_t pmap,
491 vm_map_offset_t pmap_start,
492 vm_prot_t prot);
493
494 __private_extern__ void vm_object_page_remove(
495 vm_object_t object,
496 vm_object_offset_t start,
497 vm_object_offset_t end);
498
499 __private_extern__ void vm_object_deactivate_pages(
500 vm_object_t object,
501 vm_object_offset_t offset,
502 vm_object_size_t size,
503 boolean_t kill_page,
504 boolean_t reusable_page);
505
506 __private_extern__ void vm_object_reuse_pages(
507 vm_object_t object,
508 vm_object_offset_t start_offset,
509 vm_object_offset_t end_offset,
510 boolean_t allow_partial_reuse);
511
512 __private_extern__ void vm_object_purge(
513 vm_object_t object);
514
515 __private_extern__ kern_return_t vm_object_purgable_control(
516 vm_object_t object,
517 vm_purgable_t control,
518 int *state);
519
520 __private_extern__ boolean_t vm_object_coalesce(
521 vm_object_t prev_object,
522 vm_object_t next_object,
523 vm_object_offset_t prev_offset,
524 vm_object_offset_t next_offset,
525 vm_object_size_t prev_size,
526 vm_object_size_t next_size);
527
528 __private_extern__ boolean_t vm_object_shadow(
529 vm_object_t *object,
530 vm_object_offset_t *offset,
531 vm_object_size_t length);
532
533 __private_extern__ void vm_object_collapse(
534 vm_object_t object,
535 vm_object_offset_t offset,
536 boolean_t can_bypass);
537
538 __private_extern__ boolean_t vm_object_copy_quickly(
539 vm_object_t *_object,
540 vm_object_offset_t src_offset,
541 vm_object_size_t size,
542 boolean_t *_src_needs_copy,
543 boolean_t *_dst_needs_copy);
544
545 __private_extern__ kern_return_t vm_object_copy_strategically(
546 vm_object_t src_object,
547 vm_object_offset_t src_offset,
548 vm_object_size_t size,
549 vm_object_t *dst_object,
550 vm_object_offset_t *dst_offset,
551 boolean_t *dst_needs_copy);
552
553 __private_extern__ kern_return_t vm_object_copy_slowly(
554 vm_object_t src_object,
555 vm_object_offset_t src_offset,
556 vm_object_size_t size,
557 boolean_t interruptible,
558 vm_object_t *_result_object);
559
560 __private_extern__ vm_object_t vm_object_copy_delayed(
561 vm_object_t src_object,
562 vm_object_offset_t src_offset,
563 vm_object_size_t size,
564 boolean_t src_object_shared);
565
566
567
568 __private_extern__ kern_return_t vm_object_destroy(
569 vm_object_t object,
570 kern_return_t reason);
571
572 __private_extern__ void vm_object_pager_create(
573 vm_object_t object);
574
575 __private_extern__ void vm_object_page_map(
576 vm_object_t object,
577 vm_object_offset_t offset,
578 vm_object_size_t size,
579 vm_object_offset_t (*map_fn)
580 (void *, vm_object_offset_t),
581 void *map_fn_data);
582
583 __private_extern__ kern_return_t vm_object_upl_request(
584 vm_object_t object,
585 vm_object_offset_t offset,
586 upl_size_t size,
587 upl_t *upl,
588 upl_page_info_t *page_info,
589 unsigned int *count,
590 int flags);
591
592 __private_extern__ kern_return_t vm_object_transpose(
593 vm_object_t object1,
594 vm_object_t object2,
595 vm_object_size_t transpose_size);
596
597 __private_extern__ boolean_t vm_object_sync(
598 vm_object_t object,
599 vm_object_offset_t offset,
600 vm_object_size_t size,
601 boolean_t should_flush,
602 boolean_t should_return,
603 boolean_t should_iosync);
604
605 __private_extern__ kern_return_t vm_object_update(
606 vm_object_t object,
607 vm_object_offset_t offset,
608 vm_object_size_t size,
609 vm_object_offset_t *error_offset,
610 int *io_errno,
611 memory_object_return_t should_return,
612 int flags,
613 vm_prot_t prot);
614
615 __private_extern__ kern_return_t vm_object_lock_request(
616 vm_object_t object,
617 vm_object_offset_t offset,
618 vm_object_size_t size,
619 memory_object_return_t should_return,
620 int flags,
621 vm_prot_t prot);
622
623
624
625 __private_extern__ vm_object_t vm_object_enter(
626 memory_object_t pager,
627 vm_object_size_t size,
628 boolean_t internal,
629 boolean_t init,
630 boolean_t check_named);
631
632
633 __private_extern__ void vm_object_cluster_size(
634 vm_object_t object,
635 vm_object_offset_t *start,
636 vm_size_t *length,
637 vm_object_fault_info_t fault_info,
638 uint32_t *io_streaming);
639
640 __private_extern__ kern_return_t vm_object_populate_with_private(
641 vm_object_t object,
642 vm_object_offset_t offset,
643 ppnum_t phys_page,
644 vm_size_t size);
645
646 extern kern_return_t adjust_vm_object_cache(
647 vm_size_t oval,
648 vm_size_t nval);
649
650 extern kern_return_t vm_object_page_op(
651 vm_object_t object,
652 vm_object_offset_t offset,
653 int ops,
654 ppnum_t *phys_entry,
655 int *flags);
656
657 extern kern_return_t vm_object_range_op(
658 vm_object_t object,
659 vm_object_offset_t offset_beg,
660 vm_object_offset_t offset_end,
661 int ops,
662 uint32_t *range);
663
664
665 __private_extern__ void vm_object_reap_pages(
666 vm_object_t object,
667 int reap_type);
668 #define REAP_REAP 0
669 #define REAP_TERMINATE 1
670 #define REAP_PURGEABLE 2
671 #define REAP_DATA_FLUSH 3
672
673
674 /*
675 * Event waiting handling
676 */
677
678 #define VM_OBJECT_EVENT_INITIALIZED 0
679 #define VM_OBJECT_EVENT_PAGER_READY 1
680 #define VM_OBJECT_EVENT_PAGING_IN_PROGRESS 2
681 #define VM_OBJECT_EVENT_MAPPING_IN_PROGRESS 3
682 #define VM_OBJECT_EVENT_LOCK_IN_PROGRESS 4
683 #define VM_OBJECT_EVENT_UNCACHING 5
684 #define VM_OBJECT_EVENT_COPY_CALL 6
685 #define VM_OBJECT_EVENT_CACHING 7
686 #define VM_OBJECT_EVENT_UNBLOCKED 8
687 #define VM_OBJECT_EVENT_PAGING_ONLY_IN_PROGRESS 9
688
689 #define vm_object_assert_wait(object, event, interruptible) \
690 (((object)->all_wanted |= 1 << (event)), \
691 assert_wait((event_t)((vm_offset_t)(object)+(event)),(interruptible)))
692
693 #define vm_object_wait(object, event, interruptible) \
694 (vm_object_assert_wait((object),(event),(interruptible)), \
695 vm_object_unlock(object), \
696 thread_block(THREAD_CONTINUE_NULL)) \
697
698 #define thread_sleep_vm_object(object, event, interruptible) \
699 lck_rw_sleep(&(object)->Lock, LCK_SLEEP_DEFAULT, (event_t)(event), (interruptible))
700
701 #define vm_object_sleep(object, event, interruptible) \
702 (((object)->all_wanted |= 1 << (event)), \
703 thread_sleep_vm_object((object), \
704 ((vm_offset_t)(object)+(event)), (interruptible)))
705
706 #define vm_object_wakeup(object, event) \
707 MACRO_BEGIN \
708 if ((object)->all_wanted & (1 << (event))) \
709 thread_wakeup((event_t)((vm_offset_t)(object) + (event))); \
710 (object)->all_wanted &= ~(1 << (event)); \
711 MACRO_END
712
713 #define vm_object_set_wanted(object, event) \
714 MACRO_BEGIN \
715 ((object)->all_wanted |= (1 << (event))); \
716 MACRO_END
717
718 #define vm_object_wanted(object, event) \
719 ((object)->all_wanted & (1 << (event)))
720
721 /*
722 * Routines implemented as macros
723 */
724 #ifdef VM_PIP_DEBUG
725 #include <libkern/OSDebug.h>
726 #define VM_PIP_DEBUG_BEGIN(object) \
727 MACRO_BEGIN \
728 int pip = ((object)->paging_in_progress + \
729 (object)->activity_in_progress); \
730 if (pip < VM_PIP_DEBUG_MAX_REFS) { \
731 (void) OSBacktrace(&(object)->pip_holders[pip].pip_retaddr[0], \
732 VM_PIP_DEBUG_STACK_FRAMES); \
733 } \
734 MACRO_END
735 #else /* VM_PIP_DEBUG */
736 #define VM_PIP_DEBUG_BEGIN(object)
737 #endif /* VM_PIP_DEBUG */
738
739 #define vm_object_activity_begin(object) \
740 MACRO_BEGIN \
741 vm_object_lock_assert_exclusive((object)); \
742 assert((object)->paging_in_progress >= 0); \
743 VM_PIP_DEBUG_BEGIN((object)); \
744 (object)->activity_in_progress++; \
745 MACRO_END
746
747 #define vm_object_activity_end(object) \
748 MACRO_BEGIN \
749 vm_object_lock_assert_exclusive((object)); \
750 assert((object)->activity_in_progress > 0); \
751 (object)->activity_in_progress--; \
752 if ((object)->paging_in_progress == 0 && \
753 (object)->activity_in_progress == 0) \
754 vm_object_wakeup((object), \
755 VM_OBJECT_EVENT_PAGING_IN_PROGRESS); \
756 MACRO_END
757
758 #define vm_object_paging_begin(object) \
759 MACRO_BEGIN \
760 vm_object_lock_assert_exclusive((object)); \
761 assert((object)->paging_in_progress >= 0); \
762 VM_PIP_DEBUG_BEGIN((object)); \
763 (object)->paging_in_progress++; \
764 MACRO_END
765
766 #define vm_object_paging_end(object) \
767 MACRO_BEGIN \
768 vm_object_lock_assert_exclusive((object)); \
769 assert((object)->paging_in_progress > 0); \
770 (object)->paging_in_progress--; \
771 if ((object)->paging_in_progress == 0) { \
772 vm_object_wakeup((object), \
773 VM_OBJECT_EVENT_PAGING_ONLY_IN_PROGRESS); \
774 if ((object)->activity_in_progress == 0) \
775 vm_object_wakeup((object), \
776 VM_OBJECT_EVENT_PAGING_IN_PROGRESS); \
777 } \
778 MACRO_END
779
780 #define vm_object_paging_wait(object, interruptible) \
781 MACRO_BEGIN \
782 vm_object_lock_assert_exclusive((object)); \
783 while ((object)->paging_in_progress != 0 || \
784 (object)->activity_in_progress != 0) { \
785 wait_result_t _wr; \
786 \
787 _wr = vm_object_sleep((object), \
788 VM_OBJECT_EVENT_PAGING_IN_PROGRESS, \
789 (interruptible)); \
790 \
791 /*XXX if ((interruptible) && (_wr != THREAD_AWAKENED))*/\
792 /*XXX break; */ \
793 } \
794 MACRO_END
795
796 #define vm_object_paging_only_wait(object, interruptible) \
797 MACRO_BEGIN \
798 vm_object_lock_assert_exclusive((object)); \
799 while ((object)->paging_in_progress != 0) { \
800 wait_result_t _wr; \
801 \
802 _wr = vm_object_sleep((object), \
803 VM_OBJECT_EVENT_PAGING_ONLY_IN_PROGRESS,\
804 (interruptible)); \
805 \
806 /*XXX if ((interruptible) && (_wr != THREAD_AWAKENED))*/\
807 /*XXX break; */ \
808 } \
809 MACRO_END
810
811
812 #define vm_object_mapping_begin(object) \
813 MACRO_BEGIN \
814 vm_object_lock_assert_exclusive((object)); \
815 assert(! (object)->mapping_in_progress); \
816 (object)->mapping_in_progress = TRUE; \
817 MACRO_END
818
819 #define vm_object_mapping_end(object) \
820 MACRO_BEGIN \
821 vm_object_lock_assert_exclusive((object)); \
822 assert((object)->mapping_in_progress); \
823 (object)->mapping_in_progress = FALSE; \
824 vm_object_wakeup((object), \
825 VM_OBJECT_EVENT_MAPPING_IN_PROGRESS); \
826 MACRO_END
827
828 #define vm_object_mapping_wait(object, interruptible) \
829 MACRO_BEGIN \
830 vm_object_lock_assert_exclusive((object)); \
831 while ((object)->mapping_in_progress) { \
832 wait_result_t _wr; \
833 \
834 _wr = vm_object_sleep((object), \
835 VM_OBJECT_EVENT_MAPPING_IN_PROGRESS, \
836 (interruptible)); \
837 /*XXX if ((interruptible) && (_wr != THREAD_AWAKENED))*/\
838 /*XXX break; */ \
839 } \
840 assert(!(object)->mapping_in_progress); \
841 MACRO_END
842
843
844
845 #define OBJECT_LOCK_SHARED 0
846 #define OBJECT_LOCK_EXCLUSIVE 1
847
848 extern lck_grp_t vm_object_lck_grp;
849 extern lck_grp_attr_t vm_object_lck_grp_attr;
850 extern lck_attr_t vm_object_lck_attr;
851 extern lck_attr_t kernel_object_lck_attr;
852
853 extern vm_object_t vm_pageout_scan_wants_object;
854
855 extern void vm_object_lock(vm_object_t);
856 extern boolean_t vm_object_lock_try(vm_object_t);
857 extern boolean_t _vm_object_lock_try(vm_object_t);
858 extern boolean_t vm_object_lock_avoid(vm_object_t);
859 extern void vm_object_lock_shared(vm_object_t);
860 extern boolean_t vm_object_lock_try_shared(vm_object_t);
861
862 /*
863 * Object locking macros
864 */
865
866 #define vm_object_lock_init(object) \
867 lck_rw_init(&(object)->Lock, &vm_object_lck_grp, \
868 (((object) == kernel_object || \
869 (object) == vm_submap_object) ? \
870 &kernel_object_lck_attr : \
871 &vm_object_lck_attr))
872 #define vm_object_lock_destroy(object) lck_rw_destroy(&(object)->Lock, &vm_object_lck_grp)
873
874 #define vm_object_unlock(object) lck_rw_done(&(object)->Lock)
875 #define vm_object_lock_upgrade(object) lck_rw_lock_shared_to_exclusive(&(object)->Lock)
876 #define vm_object_lock_try_scan(object) _vm_object_lock_try(object)
877
878 /*
879 * CAUTION: the following vm_object_lock_assert_held*() macros merely
880 * check if anyone is holding the lock, but the holder may not necessarily
881 * be the caller...
882 */
883 #if DEBUG
884 #define vm_object_lock_assert_held(object) \
885 lck_rw_assert(&(object)->Lock, LCK_RW_ASSERT_HELD)
886 #define vm_object_lock_assert_shared(object) \
887 lck_rw_assert(&(object)->Lock, LCK_RW_ASSERT_SHARED)
888 #define vm_object_lock_assert_exclusive(object) \
889 lck_rw_assert(&(object)->Lock, LCK_RW_ASSERT_EXCLUSIVE)
890 #else /* DEBUG */
891 #define vm_object_lock_assert_held(object)
892 #define vm_object_lock_assert_shared(object)
893 #define vm_object_lock_assert_exclusive(object)
894 #endif /* DEBUG */
895
896 #define vm_object_round_page(x) (((vm_object_offset_t)(x) + PAGE_MASK) & ~((signed)PAGE_MASK))
897 #define vm_object_trunc_page(x) ((vm_object_offset_t)(x) & ~((signed)PAGE_MASK))
898
899 #endif /* _VM_VM_OBJECT_H_ */
mirror of XNU