2 * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
4 * @APPLE_LICENSE_HEADER_START@
6 * The contents of this file constitute Original Code as defined in and
7 * are subject to the Apple Public Source License Version 1.1 (the
8 * "License"). You may not use this file except in compliance with the
9 * License. Please obtain a copy of the License at
10 * http://www.apple.com/publicsource and read it before using this file.
12 * This Original Code and all software distributed under the License are
13 * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
14 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
15 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
17 * License for the specific language governing rights and limitations
20 * @APPLE_LICENSE_HEADER_END@
26 * Mach Operating System
27 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
28 * All Rights Reserved.
30 * Permission to use, copy, modify and distribute this software and its
31 * documentation is hereby granted, provided that both the copyright
32 * notice and this permission notice appear in all copies of the
33 * software, derivative works or modified versions, and any portions
34 * thereof, and that both notices appear in supporting documentation.
36 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
37 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
38 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
40 * Carnegie Mellon requests users of this software to return to
42 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
43 * School of Computer Science
44 * Carnegie Mellon University
45 * Pittsburgh PA 15213-3890
47 * any improvements or extensions that they make and grant Carnegie Mellon
48 * the rights to redistribute these changes.
54 * Author: Avadis Tevanian, Jr., Michael Wayne Young
57 * Virtual memory object module definitions.
60 #ifndef _VM_VM_OBJECT_H_
61 #define _VM_VM_OBJECT_H_
63 #include <mach_pagemap.h>
64 #include <task_swapper.h>
66 #include <mach/kern_return.h>
67 #include <mach/boolean.h>
68 #include <mach/memory_object_types.h>
69 #include <mach/port.h>
70 #include <mach/vm_prot.h>
71 #include <mach/vm_param.h>
72 #include <mach/machine/vm_types.h>
73 #include <kern/queue.h>
74 #include <kern/lock.h>
75 #include <kern/assert.h>
76 #include <kern/misc_protos.h>
77 #include <kern/macro_help.h>
78 #include <ipc/ipc_types.h>
82 #include <vm/vm_external.h>
83 #endif /* MACH_PAGEMAP */
90 * vm_object_t Virtual memory object.
94 queue_head_t memq
; /* Resident memory */
95 decl_mutex_data(, Lock
) /* Synchronization */
97 vm_object_size_t size
; /* Object size (only valid
100 struct vm_page
*memq_hint
;
101 int ref_count
; /* Number of references */
103 int res_count
; /* Residency references (swap)*/
104 #endif /* TASK_SWAPPER */
105 unsigned int resident_page_count
;
106 /* number of resident pages */
108 struct vm_object
*copy
; /* Object that should receive
109 * a copy of my changed pages,
110 * for copy_delay, or just the
111 * temporary object that
112 * shadows this object, for
115 struct vm_object
*shadow
; /* My shadow */
116 vm_object_offset_t shadow_offset
; /* Offset into shadow */
118 memory_object_t pager
; /* Where to get data */
119 vm_object_offset_t paging_offset
; /* Offset into memory object */
120 memory_object_control_t pager_control
; /* Where data comes back */
122 memory_object_copy_strategy_t
123 copy_strategy
; /* How to handle data copy */
125 unsigned int absent_count
; /* The number of pages that
126 * have been requested but
127 * not filled. That is, the
128 * number of pages for which
129 * the "absent" attribute is
133 int paging_in_progress
;
134 /* The memory object ports are
135 * being used (e.g., for pagein
136 * or pageout) -- don't change
137 * any of these fields (i.e.,
138 * don't collapse, destroy or
142 /* boolean_t array */ all_wanted
:11, /* Bit array of "want to be
143 * awakened" notations. See
144 * VM_OBJECT_EVENT_* items
146 /* boolean_t */ pager_created
:1, /* Has pager been created? */
147 /* boolean_t */ pager_initialized
:1, /* Are fields ready to use? */
148 /* boolean_t */ pager_ready
:1, /* Will pager take requests? */
150 /* boolean_t */ pager_trusted
:1,/* The pager for this object
151 * is trusted. This is true for
152 * all internal objects (backed
153 * by the default pager)
155 /* boolean_t */ can_persist
:1, /* The kernel may keep the data
156 * for this object (and rights
157 * to the memory object) after
158 * all address map references
161 /* boolean_t */ internal
:1, /* Created by the kernel (and
162 * therefore, managed by the
163 * default memory manger)
165 /* boolean_t */ temporary
:1, /* Permanent objects may be
166 * changed externally by the
167 * memory manager, and changes
168 * made in memory must be
169 * reflected back to the memory
170 * manager. Temporary objects
174 /* boolean_t */ private:1, /* magic device_pager object,
175 * holds private pages only */
176 /* boolean_t */ pageout
:1, /* pageout object. contains
177 * private pages that refer to
178 * a real memory object. */
179 /* boolean_t */ alive
:1, /* Not yet terminated */
181 /* boolean_t */ purgable
:2, /* Purgable state. See
182 * VM_OBJECT_PURGABLE_*
185 /* boolean_t */ shadowed
:1, /* Shadow may exist */
186 /* boolean_t */ silent_overwrite
:1,
187 /* Allow full page overwrite
188 * without data_request if
190 /* boolean_t */ advisory_pageout
:1,
191 /* Instead of sending page
192 * via OOL, just notify
193 * pager that the kernel
194 * wants to discard it, page
195 * remains in object */
196 /* boolean_t */ true_share
:1,
197 /* This object is mapped
198 * in more than one place
199 * and hence cannot be
201 /* boolean_t */ terminating
:1,
202 /* Allows vm_object_lookup
203 * and vm_object_deallocate
204 * to special case their
205 * behavior when they are
206 * called as a result of
207 * page cleaning during
210 /* boolean_t */ named
:1, /* An enforces an internal
211 * naming convention, by
212 * calling the right routines
214 * destruction, UBC references
215 * against the vm_object are
218 /* boolean_t */ shadow_severed
:1,
219 /* When a permanent object
220 * backing a COW goes away
221 * unexpectedly. This bit
222 * allows vm_fault to return
223 * an error rather than a
226 /* boolean_t */ phys_contiguous
:1,
227 /* Memory is wired and
228 * guaranteed physically
229 * contiguous. However
230 * it is not device memory
231 * and obeys normal virtual
232 * memory rules w.r.t pmap
235 /* boolean_t */ nophyscache
:1;
236 /* When mapped at the
237 * pmap level, don't allow
238 * primary caching. (for
244 queue_chain_t cached_list
; /* Attachment point for the
245 * list of objects cached as a
246 * result of their can_persist
250 queue_head_t msr_q
; /* memory object synchronise
253 vm_object_offset_t last_alloc
; /* last allocation offset */
254 vm_object_offset_t sequential
; /* sequential access size */
255 vm_size_t cluster_size
; /* size of paging cluster */
257 vm_external_map_t existence_map
; /* bitmap of pages written to
259 #endif /* MACH_PAGEMAP */
260 vm_offset_t cow_hint
; /* last page present in */
261 /* shadow but not in object */
263 struct vm_object
*paging_object
; /* object which pages to be
264 * swapped out are temporary
265 * put in current object
268 /* hold object lock when altering */
269 unsigned int /* cache WIMG bits */
270 wimg_bits
:8, /* wimg plus some expansion*/
273 queue_head_t uplq
; /* List of outstanding upls */
276 * Keep track of the stack traces for the first holders
277 * of a "paging_in_progress" reference for this VM object.
279 #define VM_PIP_DEBUG_STACK_FRAMES 25 /* depth of each stack trace */
280 #define VM_PIP_DEBUG_MAX_REFS 10 /* track that many references */
281 struct __pip_backtrace
{
282 void *pip_retaddr
[VM_PIP_DEBUG_STACK_FRAMES
];
283 } pip_holders
[VM_PIP_DEBUG_MAX_REFS
];
284 #endif /* VM_PIP_DEBUG */
285 #endif /* UPL_DEBUG */
288 #define VM_PAGE_REMOVE(page) \
290 vm_page_t __page = (page); \
291 vm_object_t __object = __page->object; \
292 if (__page == __object->memq_hint) { \
293 vm_page_t __new_hint; \
294 queue_entry_t __qe; \
295 __qe = queue_next(&__page->listq); \
296 if (queue_end(&__object->memq, __qe)) { \
297 __qe = queue_prev(&__page->listq); \
298 if (queue_end(&__object->memq, __qe)) { \
302 __new_hint = (vm_page_t) __qe; \
303 __object->memq_hint = __new_hint; \
305 queue_remove(&__object->memq, __page, vm_page_t, listq); \
308 #define VM_PAGE_INSERT(page, object) \
310 vm_page_t __page = (page); \
311 vm_object_t __object = (object); \
312 queue_enter(&__object->memq, __page, vm_page_t, listq); \
313 __object->memq_hint = __page; \
317 vm_object_t kernel_object
; /* the single kernel object */
320 unsigned int vm_object_absent_max
; /* maximum number of absent pages
321 at a time for each object */
323 # define VM_MSYNC_INITIALIZED 0
324 # define VM_MSYNC_SYNCHRONIZING 1
325 # define VM_MSYNC_DONE 2
328 queue_chain_t msr_q
; /* object request queue */
329 queue_chain_t req_q
; /* vm_msync request queue */
331 vm_object_offset_t offset
;
332 vm_object_size_t length
;
333 vm_object_t object
; /* back pointer */
334 decl_mutex_data(, msync_req_lock
) /* Lock for this structure */
337 typedef struct msync_req
*msync_req_t
;
338 #define MSYNC_REQ_NULL ((msync_req_t) 0)
341 * Macros to allocate and free msync_reqs
343 #define msync_req_alloc(msr) \
345 (msr) = (msync_req_t)kalloc(sizeof(struct msync_req)); \
346 mutex_init(&(msr)->msync_req_lock, 0); \
347 msr->flag = VM_MSYNC_INITIALIZED; \
350 #define msync_req_free(msr) \
351 (kfree((msr), sizeof(struct msync_req)))
353 #define msr_lock(msr) mutex_lock(&(msr)->msync_req_lock)
354 #define msr_unlock(msr) mutex_unlock(&(msr)->msync_req_lock)
357 * Declare procedures that operate on VM objects.
360 __private_extern__
void vm_object_bootstrap(void);
362 __private_extern__
void vm_object_init(void);
364 __private_extern__
void vm_object_reaper_init(void);
366 __private_extern__ vm_object_t
vm_object_allocate(
367 vm_object_size_t size
);
369 __private_extern__
void _vm_object_allocate(vm_object_size_t size
,
374 __private_extern__
void vm_object_res_reference(
376 __private_extern__
void vm_object_res_deallocate(
378 #define VM_OBJ_RES_INCR(object) (object)->res_count++
379 #define VM_OBJ_RES_DECR(object) (object)->res_count--
381 #else /* TASK_SWAPPER */
383 #define VM_OBJ_RES_INCR(object)
384 #define VM_OBJ_RES_DECR(object)
385 #define vm_object_res_reference(object)
386 #define vm_object_res_deallocate(object)
388 #endif /* TASK_SWAPPER */
390 #define vm_object_reference_locked(object) \
392 vm_object_t RLObject = (object); \
393 assert((RLObject)->ref_count > 0); \
394 (RLObject)->ref_count++; \
395 vm_object_res_reference(RLObject); \
399 __private_extern__
void vm_object_reference(
404 #define vm_object_reference(object) \
406 vm_object_t RObject = (object); \
408 vm_object_lock(RObject); \
409 vm_object_reference_locked(RObject); \
410 vm_object_unlock(RObject); \
414 #endif /* MACH_ASSERT */
416 __private_extern__
void vm_object_deallocate(
419 __private_extern__ kern_return_t
vm_object_release_name(
423 __private_extern__
void vm_object_pmap_protect(
425 vm_object_offset_t offset
,
426 vm_object_size_t size
,
428 vm_map_offset_t pmap_start
,
431 __private_extern__
void vm_object_page_remove(
433 vm_object_offset_t start
,
434 vm_object_offset_t end
);
436 __private_extern__
void vm_object_deactivate_pages(
438 vm_object_offset_t offset
,
439 vm_object_size_t size
,
440 boolean_t kill_page
);
442 __private_extern__
unsigned int vm_object_purge(
445 __private_extern__ kern_return_t
vm_object_purgable_control(
447 vm_purgable_t control
,
450 __private_extern__ boolean_t
vm_object_coalesce(
451 vm_object_t prev_object
,
452 vm_object_t next_object
,
453 vm_object_offset_t prev_offset
,
454 vm_object_offset_t next_offset
,
455 vm_object_size_t prev_size
,
456 vm_object_size_t next_size
);
458 __private_extern__ boolean_t
vm_object_shadow(
460 vm_object_offset_t
*offset
,
461 vm_object_size_t length
);
463 __private_extern__
void vm_object_collapse(
465 vm_object_offset_t offset
,
466 boolean_t can_bypass
);
468 __private_extern__ boolean_t
vm_object_copy_quickly(
469 vm_object_t
*_object
,
470 vm_object_offset_t src_offset
,
471 vm_object_size_t size
,
472 boolean_t
*_src_needs_copy
,
473 boolean_t
*_dst_needs_copy
);
475 __private_extern__ kern_return_t
vm_object_copy_strategically(
476 vm_object_t src_object
,
477 vm_object_offset_t src_offset
,
478 vm_object_size_t size
,
479 vm_object_t
*dst_object
,
480 vm_object_offset_t
*dst_offset
,
481 boolean_t
*dst_needs_copy
);
483 __private_extern__ kern_return_t
vm_object_copy_slowly(
484 vm_object_t src_object
,
485 vm_object_offset_t src_offset
,
486 vm_object_size_t size
,
488 vm_object_t
*_result_object
);
490 __private_extern__ vm_object_t
vm_object_copy_delayed(
491 vm_object_t src_object
,
492 vm_object_offset_t src_offset
,
493 vm_object_size_t size
);
497 __private_extern__ kern_return_t
vm_object_destroy(
499 kern_return_t reason
);
501 __private_extern__
void vm_object_pager_create(
504 __private_extern__
void vm_object_page_map(
506 vm_object_offset_t offset
,
507 vm_object_size_t size
,
508 vm_object_offset_t (*map_fn
)
509 (void *, vm_object_offset_t
),
512 __private_extern__ kern_return_t
vm_object_upl_request(
514 vm_object_offset_t offset
,
517 upl_page_info_t
*page_info
,
521 __private_extern__ kern_return_t
vm_object_transpose(
524 vm_object_size_t transpose_size
);
526 __private_extern__ boolean_t
vm_object_sync(
528 vm_object_offset_t offset
,
529 vm_object_size_t size
,
530 boolean_t should_flush
,
531 boolean_t should_return
,
532 boolean_t should_iosync
);
534 __private_extern__ kern_return_t
vm_object_update(
536 vm_object_offset_t offset
,
537 vm_object_size_t size
,
538 vm_object_offset_t
*error_offset
,
540 memory_object_return_t should_return
,
544 __private_extern__ kern_return_t
vm_object_lock_request(
546 vm_object_offset_t offset
,
547 vm_object_size_t size
,
548 memory_object_return_t should_return
,
554 __private_extern__ vm_object_t
vm_object_enter(
555 memory_object_t pager
,
556 vm_object_size_t size
,
559 boolean_t check_named
);
563 * Purgable object state.
566 #define VM_OBJECT_NONPURGABLE 0 /* not a purgable object */
567 #define VM_OBJECT_PURGABLE_NONVOLATILE 1 /* non-volatile purgable object */
568 #define VM_OBJECT_PURGABLE_VOLATILE 2 /* volatile (but intact) purgable object */
569 #define VM_OBJECT_PURGABLE_EMPTY 3 /* volatile purgable object that has been emptied */
571 __private_extern__ kern_return_t
vm_object_populate_with_private(
573 vm_object_offset_t offset
,
577 extern kern_return_t
adjust_vm_object_cache(
581 extern kern_return_t
vm_object_page_op(
583 vm_object_offset_t offset
,
588 extern kern_return_t
vm_object_range_op(
590 vm_object_offset_t offset_beg
,
591 vm_object_offset_t offset_end
,
596 * Event waiting handling
599 #define VM_OBJECT_EVENT_INITIALIZED 0
600 #define VM_OBJECT_EVENT_PAGER_READY 1
601 #define VM_OBJECT_EVENT_PAGING_IN_PROGRESS 2
602 #define VM_OBJECT_EVENT_ABSENT_COUNT 3
603 #define VM_OBJECT_EVENT_LOCK_IN_PROGRESS 4
604 #define VM_OBJECT_EVENT_UNCACHING 5
605 #define VM_OBJECT_EVENT_COPY_CALL 6
606 #define VM_OBJECT_EVENT_CACHING 7
608 #define vm_object_assert_wait(object, event, interruptible) \
609 (((object)->all_wanted |= 1 << (event)), \
610 assert_wait((event_t)((vm_offset_t)(object)+(event)),(interruptible)))
612 #define vm_object_wait(object, event, interruptible) \
613 (vm_object_assert_wait((object),(event),(interruptible)), \
614 vm_object_unlock(object), \
615 thread_block(THREAD_CONTINUE_NULL)) \
617 #define thread_sleep_vm_object(object, event, interruptible) \
618 thread_sleep_mutex((event_t)(event), &(object)->Lock, (interruptible))
620 #define vm_object_sleep(object, event, interruptible) \
621 (((object)->all_wanted |= 1 << (event)), \
622 thread_sleep_vm_object((object), \
623 ((vm_offset_t)(object)+(event)), (interruptible)))
625 #define vm_object_wakeup(object, event) \
627 if ((object)->all_wanted & (1 << (event))) \
628 thread_wakeup((event_t)((vm_offset_t)(object) + (event))); \
629 (object)->all_wanted &= ~(1 << (event)); \
632 #define vm_object_set_wanted(object, event) \
634 ((object)->all_wanted |= (1 << (event))); \
637 #define vm_object_wanted(object, event) \
638 ((object)->all_wanted & (1 << (event)))
641 * Routines implemented as macros
644 extern unsigned OSBacktrace(void **bt
, unsigned maxAddrs
);
645 #define VM_PIP_DEBUG_BEGIN(object) \
647 if ((object)->paging_in_progress < VM_PIP_DEBUG_MAX_REFS) { \
648 int pip = (object)->paging_in_progress; \
649 (void) OSBacktrace(&(object)->pip_holders[pip].retaddr[0], \
650 VM_PIP_DEBUG_STACK_FRAMES); \
653 #else /* VM_PIP_DEBUG */
654 #define VM_PIP_DEBUG_BEGIN(object)
655 #endif /* VM_PIP_DEBUG */
657 #define vm_object_paging_begin(object) \
659 assert((object)->paging_in_progress >= 0); \
660 VM_PIP_DEBUG_BEGIN((object)); \
661 (object)->paging_in_progress++; \
664 #define vm_object_paging_end(object) \
666 assert((object)->paging_in_progress > 0); \
667 if (--(object)->paging_in_progress == 0) { \
668 vm_object_wakeup(object, \
669 VM_OBJECT_EVENT_PAGING_IN_PROGRESS); \
673 #define vm_object_paging_wait(object, interruptible) \
675 while ((object)->paging_in_progress != 0) { \
678 _wr = vm_object_sleep((object), \
679 VM_OBJECT_EVENT_PAGING_IN_PROGRESS, \
682 /*XXX if ((interruptible) && (_wr != THREAD_AWAKENED))*/\
687 #define vm_object_absent_assert_wait(object, interruptible) \
689 vm_object_assert_wait( (object), \
690 VM_OBJECT_EVENT_ABSENT_COUNT, \
695 #define vm_object_absent_release(object) \
697 (object)->absent_count--; \
698 vm_object_wakeup((object), \
699 VM_OBJECT_EVENT_ABSENT_COUNT); \
703 * Object locking macros
706 #define vm_object_lock_init(object) mutex_init(&(object)->Lock, 0)
707 #define vm_object_lock(object) mutex_lock(&(object)->Lock)
708 #define vm_object_unlock(object) mutex_unlock(&(object)->Lock)
709 #define vm_object_lock_try(object) mutex_try(&(object)->Lock)
711 #define vm_object_round_page(x) (((vm_object_offset_t)(x) + PAGE_MASK) & ~((signed)PAGE_MASK))
712 #define vm_object_trunc_page(x) ((vm_object_offset_t)(x) & ~((signed)PAGE_MASK))
714 #endif /* _VM_VM_OBJECT_H_ */