2 * Copyright (c) 2000-2019 Apple Inc. All rights reserved.
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
34 * All Rights Reserved.
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
46 * Carnegie Mellon requests users of this software to return to
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
59 * File: vm/memory_object.c
60 * Author: Michael Wayne Young
62 * External memory management interface control functions.
66 * Interface dependencies:
69 #include <mach/std_types.h> /* For pointer_t */
70 #include <mach/mach_types.h>
73 #include <mach/kern_return.h>
74 #include <mach/memory_object.h>
75 #include <mach/memory_object_default.h>
76 #include <mach/memory_object_control_server.h>
77 #include <mach/host_priv_server.h>
78 #include <mach/boolean.h>
79 #include <mach/vm_prot.h>
80 #include <mach/message.h>
83 * Implementation dependencies:
85 #include <string.h> /* For memcpy() */
87 #include <kern/host.h>
88 #include <kern/thread.h> /* For current_thread() */
89 #include <kern/ipc_mig.h>
90 #include <kern/misc_protos.h>
92 #include <vm/vm_object.h>
93 #include <vm/vm_fault.h>
94 #include <vm/memory_object.h>
95 #include <vm/vm_page.h>
96 #include <vm/vm_pageout.h>
97 #include <vm/pmap.h> /* For pmap_clear_modify */
98 #include <vm/vm_kern.h> /* For kernel_map, vm_move */
99 #include <vm/vm_map.h> /* For vm_map_pageable */
100 #include <vm/vm_purgeable_internal.h> /* Needed by some vm_page.h macros */
101 #include <vm/vm_shared_region.h>
103 #include <vm/vm_external.h>
105 #include <vm/vm_protos.h>
107 memory_object_default_t memory_manager_default
= MEMORY_OBJECT_DEFAULT_NULL
;
108 decl_lck_mtx_data(, memory_manager_default_lock
);
112 * Routine: memory_object_should_return_page
115 * Determine whether the given page should be returned,
116 * based on the page's state and on the given return policy.
118 * We should return the page if one of the following is true:
120 * 1. Page is dirty and should_return is not RETURN_NONE.
121 * 2. Page is precious and should_return is RETURN_ALL.
122 * 3. Should_return is RETURN_ANYTHING.
124 * As a side effect, m->vmp_dirty will be made consistent
125 * with pmap_is_modified(m), if should_return is not
126 * MEMORY_OBJECT_RETURN_NONE.
129 #define memory_object_should_return_page(m, should_return) \
130 (should_return != MEMORY_OBJECT_RETURN_NONE && \
131 (((m)->vmp_dirty || ((m)->vmp_dirty = pmap_is_modified(VM_PAGE_GET_PHYS_PAGE(m)))) || \
132 ((m)->vmp_precious && (should_return) == MEMORY_OBJECT_RETURN_ALL) || \
133 (should_return) == MEMORY_OBJECT_RETURN_ANYTHING))
135 typedef int memory_object_lock_result_t
;
137 #define MEMORY_OBJECT_LOCK_RESULT_DONE 0
138 #define MEMORY_OBJECT_LOCK_RESULT_MUST_BLOCK 1
139 #define MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN 2
140 #define MEMORY_OBJECT_LOCK_RESULT_MUST_FREE 3
142 memory_object_lock_result_t
memory_object_lock_page(
144 memory_object_return_t should_return
,
145 boolean_t should_flush
,
149 * Routine: memory_object_lock_page
152 * Perform the appropriate lock operations on the
153 * given page. See the description of
154 * "memory_object_lock_request" for the meanings
157 * Returns an indication that the operation
158 * completed, blocked, or that the page must
161 memory_object_lock_result_t
162 memory_object_lock_page(
164 memory_object_return_t should_return
,
165 boolean_t should_flush
,
168 if (m
->vmp_busy
|| m
->vmp_cleaning
) {
169 return MEMORY_OBJECT_LOCK_RESULT_MUST_BLOCK
;
172 if (m
->vmp_laundry
) {
173 vm_pageout_steal_laundry(m
, FALSE
);
177 * Don't worry about pages for which the kernel
178 * does not have any data.
180 if (m
->vmp_absent
|| m
->vmp_error
|| m
->vmp_restart
) {
181 if (m
->vmp_error
&& should_flush
&& !VM_PAGE_WIRED(m
)) {
183 * dump the page, pager wants us to
184 * clean it up and there is no
185 * relevant data to return
187 return MEMORY_OBJECT_LOCK_RESULT_MUST_FREE
;
189 return MEMORY_OBJECT_LOCK_RESULT_DONE
;
191 assert(!m
->vmp_fictitious
);
193 if (VM_PAGE_WIRED(m
)) {
195 * The page is wired... just clean or return the page if needed.
196 * Wired pages don't get flushed or disconnected from the pmap.
198 if (memory_object_should_return_page(m
, should_return
)) {
199 return MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN
;
202 return MEMORY_OBJECT_LOCK_RESULT_DONE
;
207 * must do the pmap_disconnect before determining the
208 * need to return the page... otherwise it's possible
209 * for the page to go from the clean to the dirty state
210 * after we've made our decision
212 if (pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m
)) & VM_MEM_MODIFIED
) {
213 SET_PAGE_DIRTY(m
, FALSE
);
217 * If we are decreasing permission, do it now;
218 * let the fault handler take care of increases
219 * (pmap_page_protect may not increase protection).
221 if (prot
!= VM_PROT_NO_CHANGE
) {
222 pmap_page_protect(VM_PAGE_GET_PHYS_PAGE(m
), VM_PROT_ALL
& ~prot
);
226 * Handle returning dirty or precious pages
228 if (memory_object_should_return_page(m
, should_return
)) {
230 * we use to do a pmap_disconnect here in support
231 * of memory_object_lock_request, but that routine
232 * no longer requires this... in any event, in
233 * our world, it would turn into a big noop since
234 * we don't lock the page in any way and as soon
235 * as we drop the object lock, the page can be
236 * faulted back into an address space
239 * pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m));
241 return MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN
;
245 * Handle flushing clean pages
248 return MEMORY_OBJECT_LOCK_RESULT_MUST_FREE
;
252 * we use to deactivate clean pages at this point,
253 * but we do not believe that an msync should change
254 * the 'age' of a page in the cache... here is the
255 * original comment and code concerning this...
257 * XXX Make clean but not flush a paging hint,
258 * and deactivate the pages. This is a hack
259 * because it overloads flush/clean with
260 * implementation-dependent meaning. This only
261 * happens to pages that are already clean.
263 * if (vm_page_deactivate_hint && (should_return != MEMORY_OBJECT_RETURN_NONE))
264 * return (MEMORY_OBJECT_LOCK_RESULT_MUST_DEACTIVATE);
267 return MEMORY_OBJECT_LOCK_RESULT_DONE
;
273 * Routine: memory_object_lock_request [user interface]
276 * Control use of the data associated with the given
277 * memory object. For each page in the given range,
278 * perform the following operations, in order:
279 * 1) restrict access to the page (disallow
280 * forms specified by "prot");
281 * 2) return data to the manager (if "should_return"
282 * is RETURN_DIRTY and the page is dirty, or
283 * "should_return" is RETURN_ALL and the page
284 * is either dirty or precious); and,
285 * 3) flush the cached copy (if "should_flush"
287 * The set of pages is defined by a starting offset
288 * ("offset") and size ("size"). Only pages with the
289 * same page alignment as the starting offset are
292 * A single acknowledgement is sent (to the "reply_to"
293 * port) when these actions are complete. If successful,
294 * the naked send right for reply_to is consumed.
298 memory_object_lock_request(
299 memory_object_control_t control
,
300 memory_object_offset_t offset
,
301 memory_object_size_t size
,
302 memory_object_offset_t
* resid_offset
,
304 memory_object_return_t should_return
,
311 * Check for bogus arguments.
313 object
= memory_object_control_to_vm_object(control
);
314 if (object
== VM_OBJECT_NULL
) {
315 return KERN_INVALID_ARGUMENT
;
318 if ((prot
& ~VM_PROT_ALL
) != 0 && prot
!= VM_PROT_NO_CHANGE
) {
319 return KERN_INVALID_ARGUMENT
;
322 size
= round_page_64(size
);
325 * Lock the object, and acquire a paging reference to
326 * prevent the memory_object reference from being released.
328 vm_object_lock(object
);
329 vm_object_paging_begin(object
);
331 if (flags
& MEMORY_OBJECT_DATA_FLUSH_ALL
) {
332 if ((should_return
!= MEMORY_OBJECT_RETURN_NONE
) || offset
|| object
->copy
) {
333 flags
&= ~MEMORY_OBJECT_DATA_FLUSH_ALL
;
334 flags
|= MEMORY_OBJECT_DATA_FLUSH
;
337 offset
-= object
->paging_offset
;
339 if (flags
& MEMORY_OBJECT_DATA_FLUSH_ALL
) {
340 vm_object_reap_pages(object
, REAP_DATA_FLUSH
);
342 (void)vm_object_update(object
, offset
, size
, resid_offset
,
343 io_errno
, should_return
, flags
, prot
);
346 vm_object_paging_end(object
);
347 vm_object_unlock(object
);
353 * memory_object_release_name: [interface]
355 * Enforces name semantic on memory_object reference count decrement
356 * This routine should not be called unless the caller holds a name
357 * reference gained through the memory_object_named_create or the
358 * memory_object_rename call.
359 * If the TERMINATE_IDLE flag is set, the call will return if the
360 * reference count is not 1. i.e. idle with the only remaining reference
362 * If the decision is made to proceed the name field flag is set to
363 * false and the reference count is decremented. If the RESPECT_CACHE
364 * flag is set and the reference count has gone to zero, the
365 * memory_object is checked to see if it is cacheable otherwise when
366 * the reference count is zero, it is simply terminated.
370 memory_object_release_name(
371 memory_object_control_t control
,
376 object
= memory_object_control_to_vm_object(control
);
377 if (object
== VM_OBJECT_NULL
) {
378 return KERN_INVALID_ARGUMENT
;
381 return vm_object_release_name(object
, flags
);
387 * Routine: memory_object_destroy [user interface]
389 * Shut down a memory object, despite the
390 * presence of address map (or other) references
394 memory_object_destroy(
395 memory_object_control_t control
,
396 kern_return_t reason
)
400 object
= memory_object_control_to_vm_object(control
);
401 if (object
== VM_OBJECT_NULL
) {
402 return KERN_INVALID_ARGUMENT
;
405 return vm_object_destroy(object
, reason
);
409 * Routine: vm_object_sync
411 * Kernel internal function to synch out pages in a given
412 * range within an object to its memory manager. Much the
413 * same as memory_object_lock_request but page protection
416 * If the should_flush and should_return flags are true pages
417 * are flushed, that is dirty & precious pages are written to
418 * the memory manager and then discarded. If should_return
419 * is false, only precious pages are returned to the memory
422 * If should flush is false and should_return true, the memory
423 * manager's copy of the pages is updated. If should_return
424 * is also false, only the precious pages are updated. This
425 * last option is of limited utility.
428 * FALSE if no pages were returned to the pager
435 vm_object_offset_t offset
,
436 vm_object_size_t size
,
437 boolean_t should_flush
,
438 boolean_t should_return
,
439 boolean_t should_iosync
)
445 * Lock the object, and acquire a paging reference to
446 * prevent the memory_object and control ports from
449 vm_object_lock(object
);
450 vm_object_paging_begin(object
);
453 flags
= MEMORY_OBJECT_DATA_FLUSH
;
455 * This flush is from an msync(), not a truncate(), so the
456 * contents of the file are not affected.
457 * MEMORY_OBECT_DATA_NO_CHANGE lets vm_object_update() know
458 * that the data is not changed and that there's no need to
459 * push the old contents to a copy object.
461 flags
|= MEMORY_OBJECT_DATA_NO_CHANGE
;
467 flags
|= MEMORY_OBJECT_IO_SYNC
;
470 rv
= vm_object_update(object
, offset
, (vm_object_size_t
)size
, NULL
, NULL
,
472 MEMORY_OBJECT_RETURN_ALL
:
473 MEMORY_OBJECT_RETURN_NONE
,
478 vm_object_paging_end(object
);
479 vm_object_unlock(object
);
485 #define LIST_REQ_PAGEOUT_PAGES(object, data_cnt, po, ro, ioerr, iosync) \
489 memory_object_t pager; \
491 if ((pager = (object)->pager) != MEMORY_OBJECT_NULL) { \
492 vm_object_paging_begin(object); \
493 vm_object_unlock(object); \
496 upl_flags = UPL_MSYNC | UPL_IOSYNC; \
498 upl_flags = UPL_MSYNC; \
500 (void) memory_object_data_return(pager, \
502 (memory_object_cluster_size_t)data_cnt, \
509 vm_object_lock(object); \
510 vm_object_paging_end(object); \
514 extern struct vnode
*
515 vnode_pager_lookup_vnode(memory_object_t
);
518 vm_object_update_extent(
520 vm_object_offset_t offset
,
521 vm_object_offset_t offset_end
,
522 vm_object_offset_t
*offset_resid
,
524 boolean_t should_flush
,
525 memory_object_return_t should_return
,
526 boolean_t should_iosync
,
531 vm_object_offset_t paging_offset
= 0;
532 vm_object_offset_t next_offset
= offset
;
533 memory_object_lock_result_t page_lock_result
;
534 memory_object_cluster_size_t data_cnt
= 0;
535 struct vm_page_delayed_work dw_array
[DEFAULT_DELAYED_WORK_LIMIT
];
536 struct vm_page_delayed_work
*dwp
;
543 dw_limit
= DELAYED_WORK_LIMIT(DEFAULT_DELAYED_WORK_LIMIT
);
547 offset
< offset_end
&& object
->resident_page_count
;
548 offset
+= PAGE_SIZE_64
) {
550 * Limit the number of pages to be cleaned at once to a contiguous
551 * run, or at most MAX_UPL_TRANSFER_BYTES
554 if ((data_cnt
>= MAX_UPL_TRANSFER_BYTES
) || (next_offset
!= offset
)) {
556 vm_page_do_delayed_work(object
, VM_KERN_MEMORY_NONE
, &dw_array
[0], dw_count
);
560 LIST_REQ_PAGEOUT_PAGES(object
, data_cnt
,
561 paging_offset
, offset_resid
, io_errno
, should_iosync
);
565 while ((m
= vm_page_lookup(object
, offset
)) != VM_PAGE_NULL
) {
568 page_lock_result
= memory_object_lock_page(m
, should_return
, should_flush
, prot
);
570 if (data_cnt
&& page_lock_result
!= MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN
) {
572 * End of a run of dirty/precious pages.
575 vm_page_do_delayed_work(object
, VM_KERN_MEMORY_NONE
, &dw_array
[0], dw_count
);
579 LIST_REQ_PAGEOUT_PAGES(object
, data_cnt
,
580 paging_offset
, offset_resid
, io_errno
, should_iosync
);
582 * LIST_REQ_PAGEOUT_PAGES will drop the object lock which will
583 * allow the state of page 'm' to change... we need to re-lookup
590 switch (page_lock_result
) {
591 case MEMORY_OBJECT_LOCK_RESULT_DONE
:
594 case MEMORY_OBJECT_LOCK_RESULT_MUST_FREE
:
595 if (m
->vmp_dirty
== TRUE
) {
598 dwp
->dw_mask
|= DW_vm_page_free
;
601 case MEMORY_OBJECT_LOCK_RESULT_MUST_BLOCK
:
602 PAGE_SLEEP(object
, m
, THREAD_UNINT
);
605 case MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN
:
607 paging_offset
= offset
;
610 data_cnt
+= PAGE_SIZE
;
611 next_offset
= offset
+ PAGE_SIZE_64
;
614 * wired pages shouldn't be flushed and
615 * since they aren't on any queue,
616 * no need to remove them
618 if (!VM_PAGE_WIRED(m
)) {
621 * add additional state for the flush
623 m
->vmp_free_when_done
= TRUE
;
626 * we use to remove the page from the queues at this
627 * point, but we do not believe that an msync
628 * should cause the 'age' of a page to be changed
631 * dwp->dw_mask |= DW_VM_PAGE_QUEUES_REMOVE;
638 VM_PAGE_ADD_DELAYED_WORK(dwp
, m
, dw_count
);
640 if (dw_count
>= dw_limit
) {
641 vm_page_do_delayed_work(object
, VM_KERN_MEMORY_NONE
, &dw_array
[0], dw_count
);
651 task_update_logical_writes(current_task(), (dirty_count
* PAGE_SIZE
), TASK_WRITE_INVALIDATED
, vnode_pager_lookup_vnode(object
->pager
));
654 * We have completed the scan for applicable pages.
655 * Clean any pages that have been saved.
658 vm_page_do_delayed_work(object
, VM_KERN_MEMORY_NONE
, &dw_array
[0], dw_count
);
662 LIST_REQ_PAGEOUT_PAGES(object
, data_cnt
,
663 paging_offset
, offset_resid
, io_errno
, should_iosync
);
671 * Routine: vm_object_update
673 * Work function for m_o_lock_request(), vm_o_sync().
675 * Called with object locked and paging ref taken.
680 vm_object_offset_t offset
,
681 vm_object_size_t size
,
682 vm_object_offset_t
*resid_offset
,
684 memory_object_return_t should_return
,
686 vm_prot_t protection
)
688 vm_object_t copy_object
= VM_OBJECT_NULL
;
689 boolean_t data_returned
= FALSE
;
690 boolean_t update_cow
;
691 boolean_t should_flush
= (flags
& MEMORY_OBJECT_DATA_FLUSH
) ? TRUE
: FALSE
;
692 boolean_t should_iosync
= (flags
& MEMORY_OBJECT_IO_SYNC
) ? TRUE
: FALSE
;
693 vm_fault_return_t result
;
696 #define MAX_EXTENTS 8
697 #define EXTENT_SIZE (1024 * 1024 * 256)
698 #define RESIDENT_LIMIT (1024 * 32)
700 vm_object_offset_t e_base
;
701 vm_object_offset_t e_min
;
702 vm_object_offset_t e_max
;
703 } extents
[MAX_EXTENTS
];
706 * To avoid blocking while scanning for pages, save
707 * dirty pages to be cleaned all at once.
709 * XXXO A similar strategy could be used to limit the
710 * number of times that a scan must be restarted for
711 * other reasons. Those pages that would require blocking
712 * could be temporarily collected in another list, or
713 * their offsets could be recorded in a small array.
717 * XXX NOTE: May want to consider converting this to a page list
718 * XXX vm_map_copy interface. Need to understand object
719 * XXX coalescing implications before doing so.
722 update_cow
= ((flags
& MEMORY_OBJECT_DATA_FLUSH
)
723 && (!(flags
& MEMORY_OBJECT_DATA_NO_CHANGE
) &&
724 !(flags
& MEMORY_OBJECT_DATA_PURGE
)))
725 || (flags
& MEMORY_OBJECT_COPY_SYNC
);
727 if (update_cow
|| (flags
& (MEMORY_OBJECT_DATA_PURGE
| MEMORY_OBJECT_DATA_SYNC
))) {
730 while ((copy_object
= object
->copy
) != VM_OBJECT_NULL
) {
732 * need to do a try here since we're swimming upstream
733 * against the normal lock ordering... however, we need
734 * to hold the object stable until we gain control of the
735 * copy object so we have to be careful how we approach this
737 if (vm_object_lock_try(copy_object
)) {
739 * we 'won' the lock on the copy object...
740 * no need to hold the object lock any longer...
741 * take a real reference on the copy object because
742 * we're going to call vm_fault_page on it which may
743 * under certain conditions drop the lock and the paging
744 * reference we're about to take... the reference
745 * will keep the copy object from going away if that happens
747 vm_object_unlock(object
);
748 vm_object_reference_locked(copy_object
);
751 vm_object_unlock(object
);
754 mutex_pause(collisions
);
756 vm_object_lock(object
);
759 if ((copy_object
!= VM_OBJECT_NULL
&& update_cow
) || (flags
& MEMORY_OBJECT_DATA_SYNC
)) {
761 vm_map_size_t copy_size
;
762 vm_map_offset_t copy_offset
;
766 kern_return_t error
= 0;
767 struct vm_object_fault_info fault_info
= {};
769 if (copy_object
!= VM_OBJECT_NULL
) {
771 * translate offset with respect to shadow's offset
773 copy_offset
= (offset
>= copy_object
->vo_shadow_offset
) ?
774 (vm_map_offset_t
)(offset
- copy_object
->vo_shadow_offset
) :
777 if (copy_offset
> copy_object
->vo_size
) {
778 copy_offset
= copy_object
->vo_size
;
782 * clip size with respect to shadow offset
784 if (offset
>= copy_object
->vo_shadow_offset
) {
786 } else if (size
>= copy_object
->vo_shadow_offset
- offset
) {
787 copy_size
= size
- (copy_object
->vo_shadow_offset
- offset
);
792 if (copy_offset
+ copy_size
> copy_object
->vo_size
) {
793 if (copy_object
->vo_size
>= copy_offset
) {
794 copy_size
= copy_object
->vo_size
- copy_offset
;
799 copy_size
+= copy_offset
;
801 copy_object
= object
;
803 copy_size
= offset
+ size
;
804 copy_offset
= offset
;
806 fault_info
.interruptible
= THREAD_UNINT
;
807 fault_info
.behavior
= VM_BEHAVIOR_SEQUENTIAL
;
808 fault_info
.lo_offset
= copy_offset
;
809 fault_info
.hi_offset
= copy_size
;
810 fault_info
.stealth
= TRUE
;
811 assert(fault_info
.cs_bypass
== FALSE
);
812 assert(fault_info
.pmap_cs_associated
== FALSE
);
814 vm_object_paging_begin(copy_object
);
816 for (i
= copy_offset
; i
< copy_size
; i
+= PAGE_SIZE
) {
817 RETRY_COW_OF_LOCK_REQUEST
:
818 fault_info
.cluster_size
= (vm_size_t
) (copy_size
- i
);
819 assert(fault_info
.cluster_size
== copy_size
- i
);
821 prot
= VM_PROT_WRITE
| VM_PROT_READ
;
823 result
= vm_fault_page(copy_object
, i
,
824 VM_PROT_WRITE
| VM_PROT_READ
,
826 FALSE
, /* page not looked up */
836 case VM_FAULT_SUCCESS
:
839 VM_PAGE_OBJECT(page
), top_page
);
840 vm_object_lock(copy_object
);
841 vm_object_paging_begin(copy_object
);
843 if ((!VM_PAGE_NON_SPECULATIVE_PAGEABLE(page
))) {
844 vm_page_lockspin_queues();
846 if ((!VM_PAGE_NON_SPECULATIVE_PAGEABLE(page
))) {
847 vm_page_deactivate(page
);
849 vm_page_unlock_queues();
851 PAGE_WAKEUP_DONE(page
);
854 prot
= VM_PROT_WRITE
| VM_PROT_READ
;
855 vm_object_lock(copy_object
);
856 vm_object_paging_begin(copy_object
);
857 goto RETRY_COW_OF_LOCK_REQUEST
;
858 case VM_FAULT_INTERRUPTED
:
859 prot
= VM_PROT_WRITE
| VM_PROT_READ
;
860 vm_object_lock(copy_object
);
861 vm_object_paging_begin(copy_object
);
862 goto RETRY_COW_OF_LOCK_REQUEST
;
863 case VM_FAULT_MEMORY_SHORTAGE
:
865 prot
= VM_PROT_WRITE
| VM_PROT_READ
;
866 vm_object_lock(copy_object
);
867 vm_object_paging_begin(copy_object
);
868 goto RETRY_COW_OF_LOCK_REQUEST
;
869 case VM_FAULT_SUCCESS_NO_VM_PAGE
:
870 /* success but no VM page: fail */
871 vm_object_paging_end(copy_object
);
872 vm_object_unlock(copy_object
);
874 case VM_FAULT_MEMORY_ERROR
:
875 if (object
!= copy_object
) {
876 vm_object_deallocate(copy_object
);
878 vm_object_lock(object
);
879 goto BYPASS_COW_COPYIN
;
881 panic("vm_object_update: unexpected error 0x%x"
882 " from vm_fault_page()\n", result
);
885 vm_object_paging_end(copy_object
);
887 if ((flags
& (MEMORY_OBJECT_DATA_SYNC
| MEMORY_OBJECT_COPY_SYNC
))) {
888 if (copy_object
!= VM_OBJECT_NULL
&& copy_object
!= object
) {
889 vm_object_unlock(copy_object
);
890 vm_object_deallocate(copy_object
);
891 vm_object_lock(object
);
895 if (copy_object
!= VM_OBJECT_NULL
&& copy_object
!= object
) {
896 if ((flags
& MEMORY_OBJECT_DATA_PURGE
)) {
897 vm_object_lock_assert_exclusive(copy_object
);
898 copy_object
->shadow_severed
= TRUE
;
899 copy_object
->shadowed
= FALSE
;
900 copy_object
->shadow
= NULL
;
902 * delete the ref the COW was holding on the target object
904 vm_object_deallocate(object
);
906 vm_object_unlock(copy_object
);
907 vm_object_deallocate(copy_object
);
908 vm_object_lock(object
);
913 * when we have a really large range to check relative
914 * to the number of actual resident pages, we'd like
915 * to use the resident page list to drive our checks
916 * however, the object lock will get dropped while processing
917 * the page which means the resident queue can change which
918 * means we can't walk the queue as we process the pages
919 * we also want to do the processing in offset order to allow
920 * 'runs' of pages to be collected if we're being told to
921 * flush to disk... the resident page queue is NOT ordered.
923 * a temporary solution (until we figure out how to deal with
924 * large address spaces more generically) is to pre-flight
925 * the resident page queue (if it's small enough) and develop
926 * a collection of extents (that encompass actual resident pages)
927 * to visit. This will at least allow us to deal with some of the
928 * more pathological cases in a more efficient manner. The current
929 * worst case (a single resident page at the end of an extremely large
930 * range) can take minutes to complete for ranges in the terrabyte
931 * category... since this routine is called when truncating a file,
932 * and we currently support files up to 16 Tbytes in size, this
933 * is not a theoretical problem
936 if ((object
->resident_page_count
< RESIDENT_LIMIT
) &&
937 (atop_64(size
) > (unsigned)(object
->resident_page_count
/ (8 * MAX_EXTENTS
)))) {
939 vm_object_offset_t start
;
940 vm_object_offset_t end
;
941 vm_object_size_t e_mask
;
947 e_mask
= ~((vm_object_size_t
)(EXTENT_SIZE
- 1));
949 m
= (vm_page_t
) vm_page_queue_first(&object
->memq
);
951 while (!vm_page_queue_end(&object
->memq
, (vm_page_queue_entry_t
) m
)) {
952 next
= (vm_page_t
) vm_page_queue_next(&m
->vmp_listq
);
954 if ((m
->vmp_offset
>= start
) && (m
->vmp_offset
< end
)) {
956 * this is a page we're interested in
957 * try to fit it into a current extent
959 for (n
= 0; n
< num_of_extents
; n
++) {
960 if ((m
->vmp_offset
& e_mask
) == extents
[n
].e_base
) {
962 * use (PAGE_SIZE - 1) to determine the
963 * max offset so that we don't wrap if
964 * we're at the last page of the space
966 if (m
->vmp_offset
< extents
[n
].e_min
) {
967 extents
[n
].e_min
= m
->vmp_offset
;
968 } else if ((m
->vmp_offset
+ (PAGE_SIZE
- 1)) > extents
[n
].e_max
) {
969 extents
[n
].e_max
= m
->vmp_offset
+ (PAGE_SIZE
- 1);
974 if (n
== num_of_extents
) {
976 * didn't find a current extent that can encompass
979 if (n
< MAX_EXTENTS
) {
981 * if we still have room,
982 * create a new extent
984 extents
[n
].e_base
= m
->vmp_offset
& e_mask
;
985 extents
[n
].e_min
= m
->vmp_offset
;
986 extents
[n
].e_max
= m
->vmp_offset
+ (PAGE_SIZE
- 1);
991 * no room to create a new extent...
992 * fall back to a single extent based
993 * on the min and max page offsets
994 * we find in the range we're interested in...
995 * first, look through the extent list and
996 * develop the overall min and max for the
997 * pages we've looked at up to this point
999 for (n
= 1; n
< num_of_extents
; n
++) {
1000 if (extents
[n
].e_min
< extents
[0].e_min
) {
1001 extents
[0].e_min
= extents
[n
].e_min
;
1003 if (extents
[n
].e_max
> extents
[0].e_max
) {
1004 extents
[0].e_max
= extents
[n
].e_max
;
1008 * now setup to run through the remaining pages
1009 * to determine the overall min and max
1010 * offset for the specified range
1012 extents
[0].e_base
= 0;
1017 * by continuing, we'll reprocess the
1018 * page that forced us to abandon trying
1019 * to develop multiple extents
1028 extents
[0].e_min
= offset
;
1029 extents
[0].e_max
= offset
+ (size
- 1);
1033 for (n
= 0; n
< num_of_extents
; n
++) {
1034 if (vm_object_update_extent(object
, extents
[n
].e_min
, extents
[n
].e_max
, resid_offset
, io_errno
,
1035 should_flush
, should_return
, should_iosync
, protection
)) {
1036 data_returned
= TRUE
;
1039 return data_returned
;
1043 static kern_return_t
1044 vm_object_set_attributes_common(
1046 boolean_t may_cache
,
1047 memory_object_copy_strategy_t copy_strategy
)
1049 boolean_t object_became_ready
;
1051 if (object
== VM_OBJECT_NULL
) {
1052 return KERN_INVALID_ARGUMENT
;
1056 * Verify the attributes of importance
1059 switch (copy_strategy
) {
1060 case MEMORY_OBJECT_COPY_NONE
:
1061 case MEMORY_OBJECT_COPY_DELAY
:
1064 return KERN_INVALID_ARGUMENT
;
1071 vm_object_lock(object
);
1074 * Copy the attributes
1076 assert(!object
->internal
);
1077 object_became_ready
= !object
->pager_ready
;
1078 object
->copy_strategy
= copy_strategy
;
1079 object
->can_persist
= may_cache
;
1082 * Wake up anyone waiting for the ready attribute
1083 * to become asserted.
1086 if (object_became_ready
) {
1087 object
->pager_ready
= TRUE
;
1088 vm_object_wakeup(object
, VM_OBJECT_EVENT_PAGER_READY
);
1091 vm_object_unlock(object
);
1093 return KERN_SUCCESS
;
1098 memory_object_synchronize_completed(
1099 __unused memory_object_control_t control
,
1100 __unused memory_object_offset_t offset
,
1101 __unused memory_object_size_t length
)
1103 panic("memory_object_synchronize_completed no longer supported\n");
1104 return KERN_FAILURE
;
1109 * Set the memory object attribute as provided.
1111 * XXX This routine cannot be completed until the vm_msync, clean
1112 * in place, and cluster work is completed. See ifdef notyet
1113 * below and note that vm_object_set_attributes_common()
1114 * may have to be expanded.
1117 memory_object_change_attributes(
1118 memory_object_control_t control
,
1119 memory_object_flavor_t flavor
,
1120 memory_object_info_t attributes
,
1121 mach_msg_type_number_t count
)
1124 kern_return_t result
= KERN_SUCCESS
;
1125 boolean_t may_cache
;
1126 boolean_t invalidate
;
1127 memory_object_copy_strategy_t copy_strategy
;
1129 object
= memory_object_control_to_vm_object(control
);
1130 if (object
== VM_OBJECT_NULL
) {
1131 return KERN_INVALID_ARGUMENT
;
1134 vm_object_lock(object
);
1136 may_cache
= object
->can_persist
;
1137 copy_strategy
= object
->copy_strategy
;
1139 invalidate
= object
->invalidate
;
1141 vm_object_unlock(object
);
1144 case OLD_MEMORY_OBJECT_BEHAVIOR_INFO
:
1146 old_memory_object_behave_info_t behave
;
1148 if (count
!= OLD_MEMORY_OBJECT_BEHAVE_INFO_COUNT
) {
1149 result
= KERN_INVALID_ARGUMENT
;
1153 behave
= (old_memory_object_behave_info_t
) attributes
;
1155 invalidate
= behave
->invalidate
;
1156 copy_strategy
= behave
->copy_strategy
;
1161 case MEMORY_OBJECT_BEHAVIOR_INFO
:
1163 memory_object_behave_info_t behave
;
1165 if (count
!= MEMORY_OBJECT_BEHAVE_INFO_COUNT
) {
1166 result
= KERN_INVALID_ARGUMENT
;
1170 behave
= (memory_object_behave_info_t
) attributes
;
1172 invalidate
= behave
->invalidate
;
1173 copy_strategy
= behave
->copy_strategy
;
1177 case MEMORY_OBJECT_PERFORMANCE_INFO
:
1179 memory_object_perf_info_t perf
;
1181 if (count
!= MEMORY_OBJECT_PERF_INFO_COUNT
) {
1182 result
= KERN_INVALID_ARGUMENT
;
1186 perf
= (memory_object_perf_info_t
) attributes
;
1188 may_cache
= perf
->may_cache
;
1193 case OLD_MEMORY_OBJECT_ATTRIBUTE_INFO
:
1195 old_memory_object_attr_info_t attr
;
1197 if (count
!= OLD_MEMORY_OBJECT_ATTR_INFO_COUNT
) {
1198 result
= KERN_INVALID_ARGUMENT
;
1202 attr
= (old_memory_object_attr_info_t
) attributes
;
1204 may_cache
= attr
->may_cache
;
1205 copy_strategy
= attr
->copy_strategy
;
1210 case MEMORY_OBJECT_ATTRIBUTE_INFO
:
1212 memory_object_attr_info_t attr
;
1214 if (count
!= MEMORY_OBJECT_ATTR_INFO_COUNT
) {
1215 result
= KERN_INVALID_ARGUMENT
;
1219 attr
= (memory_object_attr_info_t
) attributes
;
1221 copy_strategy
= attr
->copy_strategy
;
1222 may_cache
= attr
->may_cache_object
;
1228 result
= KERN_INVALID_ARGUMENT
;
1232 if (result
!= KERN_SUCCESS
) {
1236 if (copy_strategy
== MEMORY_OBJECT_COPY_TEMPORARY
) {
1237 copy_strategy
= MEMORY_OBJECT_COPY_DELAY
;
1241 * XXX may_cache may become a tri-valued variable to handle
1242 * XXX uncache if not in use.
1244 return vm_object_set_attributes_common(object
,
1250 memory_object_get_attributes(
1251 memory_object_control_t control
,
1252 memory_object_flavor_t flavor
,
1253 memory_object_info_t attributes
, /* pointer to OUT array */
1254 mach_msg_type_number_t
*count
) /* IN/OUT */
1256 kern_return_t ret
= KERN_SUCCESS
;
1259 object
= memory_object_control_to_vm_object(control
);
1260 if (object
== VM_OBJECT_NULL
) {
1261 return KERN_INVALID_ARGUMENT
;
1264 vm_object_lock(object
);
1267 case OLD_MEMORY_OBJECT_BEHAVIOR_INFO
:
1269 old_memory_object_behave_info_t behave
;
1271 if (*count
< OLD_MEMORY_OBJECT_BEHAVE_INFO_COUNT
) {
1272 ret
= KERN_INVALID_ARGUMENT
;
1276 behave
= (old_memory_object_behave_info_t
) attributes
;
1277 behave
->copy_strategy
= object
->copy_strategy
;
1278 behave
->temporary
= FALSE
;
1279 #if notyet /* remove when vm_msync complies and clean in place fini */
1280 behave
->invalidate
= object
->invalidate
;
1282 behave
->invalidate
= FALSE
;
1285 *count
= OLD_MEMORY_OBJECT_BEHAVE_INFO_COUNT
;
1289 case MEMORY_OBJECT_BEHAVIOR_INFO
:
1291 memory_object_behave_info_t behave
;
1293 if (*count
< MEMORY_OBJECT_BEHAVE_INFO_COUNT
) {
1294 ret
= KERN_INVALID_ARGUMENT
;
1298 behave
= (memory_object_behave_info_t
) attributes
;
1299 behave
->copy_strategy
= object
->copy_strategy
;
1300 behave
->temporary
= FALSE
;
1301 #if notyet /* remove when vm_msync complies and clean in place fini */
1302 behave
->invalidate
= object
->invalidate
;
1304 behave
->invalidate
= FALSE
;
1306 behave
->advisory_pageout
= FALSE
;
1307 behave
->silent_overwrite
= FALSE
;
1308 *count
= MEMORY_OBJECT_BEHAVE_INFO_COUNT
;
1312 case MEMORY_OBJECT_PERFORMANCE_INFO
:
1314 memory_object_perf_info_t perf
;
1316 if (*count
< MEMORY_OBJECT_PERF_INFO_COUNT
) {
1317 ret
= KERN_INVALID_ARGUMENT
;
1321 perf
= (memory_object_perf_info_t
) attributes
;
1322 perf
->cluster_size
= PAGE_SIZE
;
1323 perf
->may_cache
= object
->can_persist
;
1325 *count
= MEMORY_OBJECT_PERF_INFO_COUNT
;
1329 case OLD_MEMORY_OBJECT_ATTRIBUTE_INFO
:
1331 old_memory_object_attr_info_t attr
;
1333 if (*count
< OLD_MEMORY_OBJECT_ATTR_INFO_COUNT
) {
1334 ret
= KERN_INVALID_ARGUMENT
;
1338 attr
= (old_memory_object_attr_info_t
) attributes
;
1339 attr
->may_cache
= object
->can_persist
;
1340 attr
->copy_strategy
= object
->copy_strategy
;
1342 *count
= OLD_MEMORY_OBJECT_ATTR_INFO_COUNT
;
1346 case MEMORY_OBJECT_ATTRIBUTE_INFO
:
1348 memory_object_attr_info_t attr
;
1350 if (*count
< MEMORY_OBJECT_ATTR_INFO_COUNT
) {
1351 ret
= KERN_INVALID_ARGUMENT
;
1355 attr
= (memory_object_attr_info_t
) attributes
;
1356 attr
->copy_strategy
= object
->copy_strategy
;
1357 attr
->cluster_size
= PAGE_SIZE
;
1358 attr
->may_cache_object
= object
->can_persist
;
1359 attr
->temporary
= FALSE
;
1361 *count
= MEMORY_OBJECT_ATTR_INFO_COUNT
;
1366 ret
= KERN_INVALID_ARGUMENT
;
1370 vm_object_unlock(object
);
1377 memory_object_iopl_request(
1379 memory_object_offset_t offset
,
1380 upl_size_t
*upl_size
,
1382 upl_page_info_array_t user_page_list
,
1383 unsigned int *page_list_count
,
1384 upl_control_flags_t
*flags
,
1389 upl_control_flags_t caller_flags
;
1391 caller_flags
= *flags
;
1393 if (caller_flags
& ~UPL_VALID_FLAGS
) {
1395 * For forward compatibility's sake,
1396 * reject any unknown flag.
1398 return KERN_INVALID_VALUE
;
1401 if (ip_kotype(port
) == IKOT_NAMED_ENTRY
) {
1402 vm_named_entry_t named_entry
;
1404 named_entry
= (vm_named_entry_t
)port
->ip_kobject
;
1405 /* a few checks to make sure user is obeying rules */
1406 if (*upl_size
== 0) {
1407 if (offset
>= named_entry
->size
) {
1408 return KERN_INVALID_RIGHT
;
1410 *upl_size
= (upl_size_t
)(named_entry
->size
- offset
);
1411 if (*upl_size
!= named_entry
->size
- offset
) {
1412 return KERN_INVALID_ARGUMENT
;
1415 if (caller_flags
& UPL_COPYOUT_FROM
) {
1416 if ((named_entry
->protection
& VM_PROT_READ
)
1418 return KERN_INVALID_RIGHT
;
1421 if ((named_entry
->protection
&
1422 (VM_PROT_READ
| VM_PROT_WRITE
))
1423 != (VM_PROT_READ
| VM_PROT_WRITE
)) {
1424 return KERN_INVALID_RIGHT
;
1427 if (named_entry
->size
< (offset
+ *upl_size
)) {
1428 return KERN_INVALID_ARGUMENT
;
1431 /* the callers parameter offset is defined to be the */
1432 /* offset from beginning of named entry offset in object */
1433 offset
= offset
+ named_entry
->offset
;
1435 if (named_entry
->is_sub_map
||
1436 named_entry
->is_copy
) {
1437 return KERN_INVALID_ARGUMENT
;
1440 named_entry_lock(named_entry
);
1442 object
= named_entry
->backing
.object
;
1443 vm_object_reference(object
);
1444 named_entry_unlock(named_entry
);
1445 } else if (ip_kotype(port
) == IKOT_MEM_OBJ_CONTROL
) {
1446 memory_object_control_t control
;
1447 control
= (memory_object_control_t
) port
;
1448 if (control
== NULL
) {
1449 return KERN_INVALID_ARGUMENT
;
1451 object
= memory_object_control_to_vm_object(control
);
1452 if (object
== VM_OBJECT_NULL
) {
1453 return KERN_INVALID_ARGUMENT
;
1455 vm_object_reference(object
);
1457 return KERN_INVALID_ARGUMENT
;
1459 if (object
== VM_OBJECT_NULL
) {
1460 return KERN_INVALID_ARGUMENT
;
1463 if (!object
->private) {
1464 if (object
->phys_contiguous
) {
1465 *flags
= UPL_PHYS_CONTIG
;
1470 *flags
= UPL_DEV_MEMORY
| UPL_PHYS_CONTIG
;
1473 ret
= vm_object_iopl_request(object
,
1481 vm_object_deallocate(object
);
1486 * Routine: memory_object_upl_request [interface]
1488 * Cause the population of a portion of a vm_object.
1489 * Depending on the nature of the request, the pages
1490 * returned may be contain valid data or be uninitialized.
1495 memory_object_upl_request(
1496 memory_object_control_t control
,
1497 memory_object_offset_t offset
,
1500 upl_page_info_array_t user_page_list
,
1501 unsigned int *page_list_count
,
1507 object
= memory_object_control_to_vm_object(control
);
1508 if (object
== VM_OBJECT_NULL
) {
1509 return KERN_TERMINATED
;
1512 return vm_object_upl_request(object
,
1518 (upl_control_flags_t
)(unsigned int) cntrl_flags
,
1523 * Routine: memory_object_super_upl_request [interface]
1525 * Cause the population of a portion of a vm_object
1526 * in much the same way as memory_object_upl_request.
1527 * Depending on the nature of the request, the pages
1528 * returned may be contain valid data or be uninitialized.
1529 * However, the region may be expanded up to the super
1530 * cluster size provided.
1534 memory_object_super_upl_request(
1535 memory_object_control_t control
,
1536 memory_object_offset_t offset
,
1538 upl_size_t super_cluster
,
1540 upl_page_info_t
*user_page_list
,
1541 unsigned int *page_list_count
,
1547 object
= memory_object_control_to_vm_object(control
);
1548 if (object
== VM_OBJECT_NULL
) {
1549 return KERN_INVALID_ARGUMENT
;
1552 return vm_object_super_upl_request(object
,
1559 (upl_control_flags_t
)(unsigned int) cntrl_flags
,
1564 memory_object_cluster_size(
1565 memory_object_control_t control
,
1566 memory_object_offset_t
*start
,
1568 uint32_t *io_streaming
,
1569 memory_object_fault_info_t mo_fault_info
)
1572 vm_object_fault_info_t fault_info
;
1574 object
= memory_object_control_to_vm_object(control
);
1576 if (object
== VM_OBJECT_NULL
|| object
->paging_offset
> *start
) {
1577 return KERN_INVALID_ARGUMENT
;
1580 *start
-= object
->paging_offset
;
1582 fault_info
= (vm_object_fault_info_t
)(uintptr_t) mo_fault_info
;
1583 vm_object_cluster_size(object
,
1584 (vm_object_offset_t
*)start
,
1589 *start
+= object
->paging_offset
;
1591 return KERN_SUCCESS
;
1596 * Routine: host_default_memory_manager [interface]
1598 * set/get the default memory manager port and default cluster
1601 * If successful, consumes the supplied naked send right.
1604 host_default_memory_manager(
1605 host_priv_t host_priv
,
1606 memory_object_default_t
*default_manager
,
1607 __unused memory_object_cluster_size_t cluster_size
)
1609 memory_object_default_t current_manager
;
1610 memory_object_default_t new_manager
;
1611 memory_object_default_t returned_manager
;
1612 kern_return_t result
= KERN_SUCCESS
;
1614 if (host_priv
== HOST_PRIV_NULL
) {
1615 return KERN_INVALID_HOST
;
1618 assert(host_priv
== &realhost
);
1620 new_manager
= *default_manager
;
1621 lck_mtx_lock(&memory_manager_default_lock
);
1622 current_manager
= memory_manager_default
;
1623 returned_manager
= MEMORY_OBJECT_DEFAULT_NULL
;
1625 if (new_manager
== MEMORY_OBJECT_DEFAULT_NULL
) {
1627 * Retrieve the current value.
1629 returned_manager
= current_manager
;
1630 memory_object_default_reference(returned_manager
);
1633 * Only allow the kernel to change the value.
1635 extern task_t kernel_task
;
1636 if (current_task() != kernel_task
) {
1637 result
= KERN_NO_ACCESS
;
1642 * If this is the first non-null manager, start
1643 * up the internal pager support.
1645 if (current_manager
== MEMORY_OBJECT_DEFAULT_NULL
) {
1646 result
= vm_pageout_internal_start();
1647 if (result
!= KERN_SUCCESS
) {
1653 * Retrieve the current value,
1654 * and replace it with the supplied value.
1655 * We return the old reference to the caller
1656 * but we have to take a reference on the new
1659 returned_manager
= current_manager
;
1660 memory_manager_default
= new_manager
;
1661 memory_object_default_reference(new_manager
);
1664 * In case anyone's been waiting for a memory
1665 * manager to be established, wake them up.
1668 thread_wakeup((event_t
) &memory_manager_default
);
1671 * Now that we have a default pager for anonymous memory,
1672 * reactivate all the throttled pages (i.e. dirty pages with
1675 if (current_manager
== MEMORY_OBJECT_DEFAULT_NULL
) {
1676 vm_page_reactivate_all_throttled();
1680 lck_mtx_unlock(&memory_manager_default_lock
);
1682 *default_manager
= returned_manager
;
1687 * Routine: memory_manager_default_reference
1689 * Returns a naked send right for the default
1690 * memory manager. The returned right is always
1691 * valid (not IP_NULL or IP_DEAD).
1694 __private_extern__ memory_object_default_t
1695 memory_manager_default_reference(void)
1697 memory_object_default_t current_manager
;
1699 lck_mtx_lock(&memory_manager_default_lock
);
1700 current_manager
= memory_manager_default
;
1701 while (current_manager
== MEMORY_OBJECT_DEFAULT_NULL
) {
1704 res
= lck_mtx_sleep(&memory_manager_default_lock
,
1706 (event_t
) &memory_manager_default
,
1708 assert(res
== THREAD_AWAKENED
);
1709 current_manager
= memory_manager_default
;
1711 memory_object_default_reference(current_manager
);
1712 lck_mtx_unlock(&memory_manager_default_lock
);
1714 return current_manager
;
1718 * Routine: memory_manager_default_check
1721 * Check whether a default memory manager has been set
1722 * up yet, or not. Returns KERN_SUCCESS if dmm exists,
1723 * and KERN_FAILURE if dmm does not exist.
1725 * If there is no default memory manager, log an error,
1726 * but only the first time.
1729 __private_extern__ kern_return_t
1730 memory_manager_default_check(void)
1732 memory_object_default_t current
;
1734 lck_mtx_lock(&memory_manager_default_lock
);
1735 current
= memory_manager_default
;
1736 if (current
== MEMORY_OBJECT_DEFAULT_NULL
) {
1737 static boolean_t logged
; /* initialized to 0 */
1738 boolean_t complain
= !logged
;
1740 lck_mtx_unlock(&memory_manager_default_lock
);
1742 printf("Warning: No default memory manager\n");
1744 return KERN_FAILURE
;
1746 lck_mtx_unlock(&memory_manager_default_lock
);
1747 return KERN_SUCCESS
;
1751 __private_extern__
void
1752 memory_manager_default_init(void)
1754 memory_manager_default
= MEMORY_OBJECT_DEFAULT_NULL
;
1755 lck_mtx_init(&memory_manager_default_lock
, &vm_object_lck_grp
, &vm_object_lck_attr
);
1760 /* Allow manipulation of individual page state. This is actually part of */
1761 /* the UPL regimen but takes place on the object rather than on a UPL */
1764 memory_object_page_op(
1765 memory_object_control_t control
,
1766 memory_object_offset_t offset
,
1768 ppnum_t
*phys_entry
,
1773 object
= memory_object_control_to_vm_object(control
);
1774 if (object
== VM_OBJECT_NULL
) {
1775 return KERN_INVALID_ARGUMENT
;
1778 return vm_object_page_op(object
, offset
, ops
, phys_entry
, flags
);
1782 * memory_object_range_op offers performance enhancement over
1783 * memory_object_page_op for page_op functions which do not require page
1784 * level state to be returned from the call. Page_op was created to provide
1785 * a low-cost alternative to page manipulation via UPLs when only a single
1786 * page was involved. The range_op call establishes the ability in the _op
1787 * family of functions to work on multiple pages where the lack of page level
1788 * state handling allows the caller to avoid the overhead of the upl structures.
1792 memory_object_range_op(
1793 memory_object_control_t control
,
1794 memory_object_offset_t offset_beg
,
1795 memory_object_offset_t offset_end
,
1801 object
= memory_object_control_to_vm_object(control
);
1802 if (object
== VM_OBJECT_NULL
) {
1803 return KERN_INVALID_ARGUMENT
;
1806 return vm_object_range_op(object
,
1810 (uint32_t *) range
);
1815 memory_object_mark_used(
1816 memory_object_control_t control
)
1820 if (control
== NULL
) {
1824 object
= memory_object_control_to_vm_object(control
);
1826 if (object
!= VM_OBJECT_NULL
) {
1827 vm_object_cache_remove(object
);
1833 memory_object_mark_unused(
1834 memory_object_control_t control
,
1835 __unused boolean_t rage
)
1839 if (control
== NULL
) {
1843 object
= memory_object_control_to_vm_object(control
);
1845 if (object
!= VM_OBJECT_NULL
) {
1846 vm_object_cache_add(object
);
1851 memory_object_mark_io_tracking(
1852 memory_object_control_t control
)
1856 if (control
== NULL
) {
1859 object
= memory_object_control_to_vm_object(control
);
1861 if (object
!= VM_OBJECT_NULL
) {
1862 vm_object_lock(object
);
1863 object
->io_tracking
= TRUE
;
1864 vm_object_unlock(object
);
1869 memory_object_mark_trusted(
1870 memory_object_control_t control
)
1874 if (control
== NULL
) {
1877 object
= memory_object_control_to_vm_object(control
);
1879 if (object
!= VM_OBJECT_NULL
) {
1880 vm_object_lock(object
);
1881 object
->pager_trusted
= TRUE
;
1882 vm_object_unlock(object
);
1886 #if CONFIG_SECLUDED_MEMORY
1888 memory_object_mark_eligible_for_secluded(
1889 memory_object_control_t control
,
1890 boolean_t eligible_for_secluded
)
1894 if (control
== NULL
) {
1897 object
= memory_object_control_to_vm_object(control
);
1899 if (object
== VM_OBJECT_NULL
) {
1903 vm_object_lock(object
);
1904 if (eligible_for_secluded
&&
1905 secluded_for_filecache
&& /* global boot-arg */
1906 !object
->eligible_for_secluded
) {
1907 object
->eligible_for_secluded
= TRUE
;
1908 vm_page_secluded
.eligible_for_secluded
+= object
->resident_page_count
;
1909 } else if (!eligible_for_secluded
&&
1910 object
->eligible_for_secluded
) {
1911 object
->eligible_for_secluded
= FALSE
;
1912 vm_page_secluded
.eligible_for_secluded
-= object
->resident_page_count
;
1913 if (object
->resident_page_count
) {
1914 /* XXX FBDP TODO: flush pages from secluded queue? */
1915 // printf("FBDP TODO: flush %d pages from %p from secluded queue\n", object->resident_page_count, object);
1918 vm_object_unlock(object
);
1920 #endif /* CONFIG_SECLUDED_MEMORY */
1923 memory_object_pages_resident(
1924 memory_object_control_t control
,
1925 boolean_t
* has_pages_resident
)
1929 *has_pages_resident
= FALSE
;
1931 object
= memory_object_control_to_vm_object(control
);
1932 if (object
== VM_OBJECT_NULL
) {
1933 return KERN_INVALID_ARGUMENT
;
1936 if (object
->resident_page_count
) {
1937 *has_pages_resident
= TRUE
;
1940 return KERN_SUCCESS
;
1944 memory_object_signed(
1945 memory_object_control_t control
,
1946 boolean_t is_signed
)
1950 object
= memory_object_control_to_vm_object(control
);
1951 if (object
== VM_OBJECT_NULL
) {
1952 return KERN_INVALID_ARGUMENT
;
1955 vm_object_lock(object
);
1956 object
->code_signed
= is_signed
;
1957 vm_object_unlock(object
);
1959 return KERN_SUCCESS
;
1963 memory_object_is_signed(
1964 memory_object_control_t control
)
1966 boolean_t is_signed
;
1969 object
= memory_object_control_to_vm_object(control
);
1970 if (object
== VM_OBJECT_NULL
) {
1974 vm_object_lock_shared(object
);
1975 is_signed
= object
->code_signed
;
1976 vm_object_unlock(object
);
1982 memory_object_is_shared_cache(
1983 memory_object_control_t control
)
1985 vm_object_t object
= VM_OBJECT_NULL
;
1987 object
= memory_object_control_to_vm_object(control
);
1988 if (object
== VM_OBJECT_NULL
) {
1992 return object
->object_is_shared_cache
;
1995 static zone_t mem_obj_control_zone
;
1997 __private_extern__
void
1998 memory_object_control_bootstrap(void)
2002 i
= (vm_size_t
) sizeof(struct memory_object_control
);
2003 mem_obj_control_zone
= zinit(i
, 8192 * i
, 4096, "mem_obj_control");
2004 zone_change(mem_obj_control_zone
, Z_CALLERACCT
, FALSE
);
2005 zone_change(mem_obj_control_zone
, Z_NOENCRYPT
, TRUE
);
2009 __private_extern__ memory_object_control_t
2010 memory_object_control_allocate(
2013 memory_object_control_t control
;
2015 control
= (memory_object_control_t
)zalloc(mem_obj_control_zone
);
2016 if (control
!= MEMORY_OBJECT_CONTROL_NULL
) {
2017 control
->moc_object
= object
;
2018 control
->moc_ikot
= IKOT_MEM_OBJ_CONTROL
; /* fake ip_kotype */
2023 __private_extern__
void
2024 memory_object_control_collapse(
2025 memory_object_control_t control
,
2028 assert((control
->moc_object
!= VM_OBJECT_NULL
) &&
2029 (control
->moc_object
!= object
));
2030 control
->moc_object
= object
;
2033 __private_extern__ vm_object_t
2034 memory_object_control_to_vm_object(
2035 memory_object_control_t control
)
2037 if (control
== MEMORY_OBJECT_CONTROL_NULL
||
2038 control
->moc_ikot
!= IKOT_MEM_OBJ_CONTROL
) {
2039 return VM_OBJECT_NULL
;
2042 return control
->moc_object
;
2045 __private_extern__ vm_object_t
2046 memory_object_to_vm_object(
2047 memory_object_t mem_obj
)
2049 memory_object_control_t mo_control
;
2051 if (mem_obj
== MEMORY_OBJECT_NULL
) {
2052 return VM_OBJECT_NULL
;
2054 mo_control
= mem_obj
->mo_control
;
2055 if (mo_control
== NULL
) {
2056 return VM_OBJECT_NULL
;
2058 return memory_object_control_to_vm_object(mo_control
);
2061 memory_object_control_t
2062 convert_port_to_mo_control(
2063 __unused mach_port_t port
)
2065 return MEMORY_OBJECT_CONTROL_NULL
;
2070 convert_mo_control_to_port(
2071 __unused memory_object_control_t control
)
2073 return MACH_PORT_NULL
;
2077 memory_object_control_reference(
2078 __unused memory_object_control_t control
)
2084 * We only every issue one of these references, so kill it
2085 * when that gets released (should switch the real reference
2086 * counting in true port-less EMMI).
2089 memory_object_control_deallocate(
2090 memory_object_control_t control
)
2092 zfree(mem_obj_control_zone
, control
);
2096 memory_object_control_disable(
2097 memory_object_control_t control
)
2099 assert(control
->moc_object
!= VM_OBJECT_NULL
);
2100 control
->moc_object
= VM_OBJECT_NULL
;
2104 memory_object_default_reference(
2105 memory_object_default_t dmm
)
2107 ipc_port_make_send(dmm
);
2111 memory_object_default_deallocate(
2112 memory_object_default_t dmm
)
2114 ipc_port_release_send(dmm
);
2118 convert_port_to_memory_object(
2119 __unused mach_port_t port
)
2121 return MEMORY_OBJECT_NULL
;
2126 convert_memory_object_to_port(
2127 __unused memory_object_t object
)
2129 return MACH_PORT_NULL
;
2133 /* Routine memory_object_reference */
2135 memory_object_reference(
2136 memory_object_t memory_object
)
2138 (memory_object
->mo_pager_ops
->memory_object_reference
)(
2142 /* Routine memory_object_deallocate */
2144 memory_object_deallocate(
2145 memory_object_t memory_object
)
2147 (memory_object
->mo_pager_ops
->memory_object_deallocate
)(
2152 /* Routine memory_object_init */
2156 memory_object_t memory_object
,
2157 memory_object_control_t memory_control
,
2158 memory_object_cluster_size_t memory_object_page_size
2161 return (memory_object
->mo_pager_ops
->memory_object_init
)(
2164 memory_object_page_size
);
2167 /* Routine memory_object_terminate */
2169 memory_object_terminate
2171 memory_object_t memory_object
2174 return (memory_object
->mo_pager_ops
->memory_object_terminate
)(
2178 /* Routine memory_object_data_request */
2180 memory_object_data_request
2182 memory_object_t memory_object
,
2183 memory_object_offset_t offset
,
2184 memory_object_cluster_size_t length
,
2185 vm_prot_t desired_access
,
2186 memory_object_fault_info_t fault_info
2189 return (memory_object
->mo_pager_ops
->memory_object_data_request
)(
2197 /* Routine memory_object_data_return */
2199 memory_object_data_return
2201 memory_object_t memory_object
,
2202 memory_object_offset_t offset
,
2203 memory_object_cluster_size_t size
,
2204 memory_object_offset_t
*resid_offset
,
2207 boolean_t kernel_copy
,
2211 return (memory_object
->mo_pager_ops
->memory_object_data_return
)(
2222 /* Routine memory_object_data_initialize */
2224 memory_object_data_initialize
2226 memory_object_t memory_object
,
2227 memory_object_offset_t offset
,
2228 memory_object_cluster_size_t size
2231 return (memory_object
->mo_pager_ops
->memory_object_data_initialize
)(
2237 /* Routine memory_object_data_unlock */
2239 memory_object_data_unlock
2241 memory_object_t memory_object
,
2242 memory_object_offset_t offset
,
2243 memory_object_size_t size
,
2244 vm_prot_t desired_access
2247 return (memory_object
->mo_pager_ops
->memory_object_data_unlock
)(
2254 /* Routine memory_object_synchronize */
2256 memory_object_synchronize
2258 memory_object_t memory_object
,
2259 memory_object_offset_t offset
,
2260 memory_object_size_t size
,
2261 vm_sync_t sync_flags
2264 panic("memory_object_syncrhonize no longer supported\n");
2266 return (memory_object
->mo_pager_ops
->memory_object_synchronize
)(
2275 * memory_object_map() is called by VM (in vm_map_enter() and its variants)
2276 * each time a "named" VM object gets mapped directly or indirectly
2277 * (copy-on-write mapping). A "named" VM object has an extra reference held
2278 * by the pager to keep it alive until the pager decides that the
2279 * memory object (and its VM object) can be reclaimed.
2280 * VM calls memory_object_last_unmap() (in vm_object_deallocate()) when all
2281 * the mappings of that memory object have been removed.
2283 * For a given VM object, calls to memory_object_map() and memory_object_unmap()
2284 * are serialized (through object->mapping_in_progress), to ensure that the
2285 * pager gets a consistent view of the mapping status of the memory object.
2287 * This allows the pager to keep track of how many times a memory object
2288 * has been mapped and with which protections, to decide when it can be
2292 /* Routine memory_object_map */
2296 memory_object_t memory_object
,
2300 return (memory_object
->mo_pager_ops
->memory_object_map
)(
2305 /* Routine memory_object_last_unmap */
2307 memory_object_last_unmap
2309 memory_object_t memory_object
2312 return (memory_object
->mo_pager_ops
->memory_object_last_unmap
)(
2316 /* Routine memory_object_data_reclaim */
2318 memory_object_data_reclaim
2320 memory_object_t memory_object
,
2321 boolean_t reclaim_backing_store
2324 if (memory_object
->mo_pager_ops
->memory_object_data_reclaim
== NULL
) {
2325 return KERN_NOT_SUPPORTED
;
2327 return (memory_object
->mo_pager_ops
->memory_object_data_reclaim
)(
2329 reclaim_backing_store
);
2333 convert_port_to_upl(
2339 if (!ip_active(port
) || (ip_kotype(port
) != IKOT_UPL
)) {
2343 upl
= (upl_t
) port
->ip_kobject
;
2346 upl
->ref_count
+= 1;
2352 convert_upl_to_port(
2355 return MACH_PORT_NULL
;
2358 __private_extern__
void
2360 __unused ipc_port_t port
,
2361 __unused mach_port_mscount_t mscount
)