]> git.saurik.com Git - apple/xnu.git/blob - osfmk/vm/memory_object.c
projects / apple / xnu.git / blob
? search:


de7baff299a1821cc16e4cf6e17ac9bc2c7c1c41
[apple/xnu.git] / osfmk / vm / memory_object.c
1 /*
2 * Copyright (c) 2000-2008 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56 /*
57 */
58 /*
59 * File: vm/memory_object.c
60 * Author: Michael Wayne Young
61 *
62 * External memory management interface control functions.
63 */
64
65 #include <advisory_pageout.h>
66
67 /*
68 * Interface dependencies:
69 */
70
71 #include <mach/std_types.h> /* For pointer_t */
72 #include <mach/mach_types.h>
73
74 #include <mach/mig.h>
75 #include <mach/kern_return.h>
76 #include <mach/memory_object.h>
77 #include <mach/memory_object_default.h>
78 #include <mach/memory_object_control_server.h>
79 #include <mach/host_priv_server.h>
80 #include <mach/boolean.h>
81 #include <mach/vm_prot.h>
82 #include <mach/message.h>
83
84 /*
85 * Implementation dependencies:
86 */
87 #include <string.h> /* For memcpy() */
88
89 #include <kern/xpr.h>
90 #include <kern/host.h>
91 #include <kern/thread.h> /* For current_thread() */
92 #include <kern/ipc_mig.h>
93 #include <kern/misc_protos.h>
94
95 #include <vm/vm_object.h>
96 #include <vm/vm_fault.h>
97 #include <vm/memory_object.h>
98 #include <vm/vm_page.h>
99 #include <vm/vm_pageout.h>
100 #include <vm/pmap.h> /* For pmap_clear_modify */
101 #include <vm/vm_kern.h> /* For kernel_map, vm_move */
102 #include <vm/vm_map.h> /* For vm_map_pageable */
103 #include <vm/vm_purgeable_internal.h> /* Needed by some vm_page.h macros */
104 #include <vm/vm_shared_region.h>
105
106 #if MACH_PAGEMAP
107 #include <vm/vm_external.h>
108 #endif /* MACH_PAGEMAP */
109
110 #include <vm/vm_protos.h>
111
112
113 memory_object_default_t memory_manager_default = MEMORY_OBJECT_DEFAULT_NULL;
114 decl_lck_mtx_data(, memory_manager_default_lock)
115
116
117 /*
118 * Routine: memory_object_should_return_page
119 *
120 * Description:
121 * Determine whether the given page should be returned,
122 * based on the page's state and on the given return policy.
123 *
124 * We should return the page if one of the following is true:
125 *
126 * 1. Page is dirty and should_return is not RETURN_NONE.
127 * 2. Page is precious and should_return is RETURN_ALL.
128 * 3. Should_return is RETURN_ANYTHING.
129 *
130 * As a side effect, m->dirty will be made consistent
131 * with pmap_is_modified(m), if should_return is not
132 * MEMORY_OBJECT_RETURN_NONE.
133 */
134
135 #define memory_object_should_return_page(m, should_return) \
136 (should_return != MEMORY_OBJECT_RETURN_NONE && \
137 (((m)->dirty || ((m)->dirty = pmap_is_modified((m)->phys_page))) || \
138 ((m)->precious && (should_return) == MEMORY_OBJECT_RETURN_ALL) || \
139 (should_return) == MEMORY_OBJECT_RETURN_ANYTHING))
140
141 typedef int memory_object_lock_result_t;
142
143 #define MEMORY_OBJECT_LOCK_RESULT_DONE 0
144 #define MEMORY_OBJECT_LOCK_RESULT_MUST_BLOCK 1
145 #define MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN 2
146 #define MEMORY_OBJECT_LOCK_RESULT_MUST_FREE 3
147
148 memory_object_lock_result_t memory_object_lock_page(
149 vm_page_t m,
150 memory_object_return_t should_return,
151 boolean_t should_flush,
152 vm_prot_t prot);
153
154 /*
155 * Routine: memory_object_lock_page
156 *
157 * Description:
158 * Perform the appropriate lock operations on the
159 * given page. See the description of
160 * "memory_object_lock_request" for the meanings
161 * of the arguments.
162 *
163 * Returns an indication that the operation
164 * completed, blocked, or that the page must
165 * be cleaned.
166 */
167 memory_object_lock_result_t
168 memory_object_lock_page(
169 vm_page_t m,
170 memory_object_return_t should_return,
171 boolean_t should_flush,
172 vm_prot_t prot)
173 {
174 XPR(XPR_MEMORY_OBJECT,
175 "m_o_lock_page, page 0x%X rtn %d flush %d prot %d\n",
176 m, should_return, should_flush, prot, 0);
177
178
179 if (m->busy || m->cleaning) {
180 if (m->list_req_pending &&
181 should_return == MEMORY_OBJECT_RETURN_NONE &&
182 should_flush == TRUE) {
183
184 if (m->absent) {
185 /*
186 * this is the list_req_pending | absent | busy case
187 * which originates from vm_fault_page.
188 * Combine that with should_flush == TRUE and we
189 * have a case where we need to toss the page from
190 * the object.
191 */
192 if (!VM_PAGE_WIRED(m)) {
193 return (MEMORY_OBJECT_LOCK_RESULT_MUST_FREE);
194 } else {
195 return (MEMORY_OBJECT_LOCK_RESULT_DONE);
196 }
197 }
198 if (m->pageout || m->cleaning) {
199 /*
200 * if pageout is set, page was earmarked by vm_pageout_scan
201 * to be cleaned and stolen... if cleaning is set, we're
202 * pre-cleaning pages for a hibernate...
203 * in either case, we're going
204 * to take it back since we are being asked to
205 * flush the page w/o cleaning it (i.e. we don't
206 * care that it's dirty, we want it gone from
207 * the cache) and we don't want to stall
208 * waiting for it to be cleaned for 2 reasons...
209 * 1 - no use paging it out since we're probably
210 * shrinking the file at this point or we no
211 * longer care about the data in the page
212 * 2 - if we stall, we may casue a deadlock in
213 * the FS trying to acquire its locks
214 * on the VNOP_PAGEOUT path presuming that
215 * those locks are already held on the truncate
216 * path before calling through to this function
217 *
218 * so undo all of the state that vm_pageout_scan
219 * hung on this page
220 */
221
222 vm_pageout_queue_steal(m, FALSE);
223 PAGE_WAKEUP_DONE(m);
224 } else {
225 panic("list_req_pending on page %p without absent/pageout/cleaning set\n", m);
226 }
227 } else
228 return (MEMORY_OBJECT_LOCK_RESULT_MUST_BLOCK);
229 }
230 /*
231 * Don't worry about pages for which the kernel
232 * does not have any data.
233 */
234 if (m->absent || m->error || m->restart) {
235 if (m->error && should_flush && !VM_PAGE_WIRED(m)) {
236 /*
237 * dump the page, pager wants us to
238 * clean it up and there is no
239 * relevant data to return
240 */
241 return (MEMORY_OBJECT_LOCK_RESULT_MUST_FREE);
242 }
243 return (MEMORY_OBJECT_LOCK_RESULT_DONE);
244 }
245 assert(!m->fictitious);
246
247 if (VM_PAGE_WIRED(m)) {
248 /*
249 * The page is wired... just clean or return the page if needed.
250 * Wired pages don't get flushed or disconnected from the pmap.
251 */
252 if (memory_object_should_return_page(m, should_return))
253 return (MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN);
254
255 return (MEMORY_OBJECT_LOCK_RESULT_DONE);
256 }
257
258 if (should_flush) {
259 /*
260 * must do the pmap_disconnect before determining the
261 * need to return the page... otherwise it's possible
262 * for the page to go from the clean to the dirty state
263 * after we've made our decision
264 */
265 if (pmap_disconnect(m->phys_page) & VM_MEM_MODIFIED)
266 m->dirty = TRUE;
267 } else {
268 /*
269 * If we are decreasing permission, do it now;
270 * let the fault handler take care of increases
271 * (pmap_page_protect may not increase protection).
272 */
273 if (prot != VM_PROT_NO_CHANGE)
274 pmap_page_protect(m->phys_page, VM_PROT_ALL & ~prot);
275 }
276 /*
277 * Handle returning dirty or precious pages
278 */
279 if (memory_object_should_return_page(m, should_return)) {
280 /*
281 * we use to do a pmap_disconnect here in support
282 * of memory_object_lock_request, but that routine
283 * no longer requires this... in any event, in
284 * our world, it would turn into a big noop since
285 * we don't lock the page in any way and as soon
286 * as we drop the object lock, the page can be
287 * faulted back into an address space
288 *
289 * if (!should_flush)
290 * pmap_disconnect(m->phys_page);
291 */
292 return (MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN);
293 }
294
295 /*
296 * Handle flushing clean pages
297 */
298 if (should_flush)
299 return (MEMORY_OBJECT_LOCK_RESULT_MUST_FREE);
300
301 /*
302 * we use to deactivate clean pages at this point,
303 * but we do not believe that an msync should change
304 * the 'age' of a page in the cache... here is the
305 * original comment and code concerning this...
306 *
307 * XXX Make clean but not flush a paging hint,
308 * and deactivate the pages. This is a hack
309 * because it overloads flush/clean with
310 * implementation-dependent meaning. This only
311 * happens to pages that are already clean.
312 *
313 * if (vm_page_deactivate_hint && (should_return != MEMORY_OBJECT_RETURN_NONE))
314 * return (MEMORY_OBJECT_LOCK_RESULT_MUST_DEACTIVATE);
315 */
316
317 return (MEMORY_OBJECT_LOCK_RESULT_DONE);
318 }
319
320
321
322 /*
323 * Routine: memory_object_lock_request [user interface]
324 *
325 * Description:
326 * Control use of the data associated with the given
327 * memory object. For each page in the given range,
328 * perform the following operations, in order:
329 * 1) restrict access to the page (disallow
330 * forms specified by "prot");
331 * 2) return data to the manager (if "should_return"
332 * is RETURN_DIRTY and the page is dirty, or
333 * "should_return" is RETURN_ALL and the page
334 * is either dirty or precious); and,
335 * 3) flush the cached copy (if "should_flush"
336 * is asserted).
337 * The set of pages is defined by a starting offset
338 * ("offset") and size ("size"). Only pages with the
339 * same page alignment as the starting offset are
340 * considered.
341 *
342 * A single acknowledgement is sent (to the "reply_to"
343 * port) when these actions are complete. If successful,
344 * the naked send right for reply_to is consumed.
345 */
346
347 kern_return_t
348 memory_object_lock_request(
349 memory_object_control_t control,
350 memory_object_offset_t offset,
351 memory_object_size_t size,
352 memory_object_offset_t * resid_offset,
353 int * io_errno,
354 memory_object_return_t should_return,
355 int flags,
356 vm_prot_t prot)
357 {
358 vm_object_t object;
359
360 /*
361 * Check for bogus arguments.
362 */
363 object = memory_object_control_to_vm_object(control);
364 if (object == VM_OBJECT_NULL)
365 return (KERN_INVALID_ARGUMENT);
366
367 if ((prot & ~VM_PROT_ALL) != 0 && prot != VM_PROT_NO_CHANGE)
368 return (KERN_INVALID_ARGUMENT);
369
370 size = round_page_64(size);
371
372 /*
373 * Lock the object, and acquire a paging reference to
374 * prevent the memory_object reference from being released.
375 */
376 vm_object_lock(object);
377 vm_object_paging_begin(object);
378
379 if (flags & MEMORY_OBJECT_DATA_FLUSH_ALL) {
380 if ((should_return != MEMORY_OBJECT_RETURN_NONE) || offset || object->copy) {
381 flags &= ~MEMORY_OBJECT_DATA_FLUSH_ALL;
382 flags |= MEMORY_OBJECT_DATA_FLUSH;
383 }
384 }
385 offset -= object->paging_offset;
386
387 if (flags & MEMORY_OBJECT_DATA_FLUSH_ALL)
388 vm_object_reap_pages(object, REAP_DATA_FLUSH);
389 else
390 (void)vm_object_update(object, offset, size, resid_offset,
391 io_errno, should_return, flags, prot);
392
393 vm_object_paging_end(object);
394 vm_object_unlock(object);
395
396 return (KERN_SUCCESS);
397 }
398
399 /*
400 * memory_object_release_name: [interface]
401 *
402 * Enforces name semantic on memory_object reference count decrement
403 * This routine should not be called unless the caller holds a name
404 * reference gained through the memory_object_named_create or the
405 * memory_object_rename call.
406 * If the TERMINATE_IDLE flag is set, the call will return if the
407 * reference count is not 1. i.e. idle with the only remaining reference
408 * being the name.
409 * If the decision is made to proceed the name field flag is set to
410 * false and the reference count is decremented. If the RESPECT_CACHE
411 * flag is set and the reference count has gone to zero, the
412 * memory_object is checked to see if it is cacheable otherwise when
413 * the reference count is zero, it is simply terminated.
414 */
415
416 kern_return_t
417 memory_object_release_name(
418 memory_object_control_t control,
419 int flags)
420 {
421 vm_object_t object;
422
423 object = memory_object_control_to_vm_object(control);
424 if (object == VM_OBJECT_NULL)
425 return (KERN_INVALID_ARGUMENT);
426
427 return vm_object_release_name(object, flags);
428 }
429
430
431
432 /*
433 * Routine: memory_object_destroy [user interface]
434 * Purpose:
435 * Shut down a memory object, despite the
436 * presence of address map (or other) references
437 * to the vm_object.
438 */
439 kern_return_t
440 memory_object_destroy(
441 memory_object_control_t control,
442 kern_return_t reason)
443 {
444 vm_object_t object;
445
446 object = memory_object_control_to_vm_object(control);
447 if (object == VM_OBJECT_NULL)
448 return (KERN_INVALID_ARGUMENT);
449
450 return (vm_object_destroy(object, reason));
451 }
452
453 /*
454 * Routine: vm_object_sync
455 *
456 * Kernel internal function to synch out pages in a given
457 * range within an object to its memory manager. Much the
458 * same as memory_object_lock_request but page protection
459 * is not changed.
460 *
461 * If the should_flush and should_return flags are true pages
462 * are flushed, that is dirty & precious pages are written to
463 * the memory manager and then discarded. If should_return
464 * is false, only precious pages are returned to the memory
465 * manager.
466 *
467 * If should flush is false and should_return true, the memory
468 * manager's copy of the pages is updated. If should_return
469 * is also false, only the precious pages are updated. This
470 * last option is of limited utility.
471 *
472 * Returns:
473 * FALSE if no pages were returned to the pager
474 * TRUE otherwise.
475 */
476
477 boolean_t
478 vm_object_sync(
479 vm_object_t object,
480 vm_object_offset_t offset,
481 vm_object_size_t size,
482 boolean_t should_flush,
483 boolean_t should_return,
484 boolean_t should_iosync)
485 {
486 boolean_t rv;
487 int flags;
488
489 XPR(XPR_VM_OBJECT,
490 "vm_o_sync, object 0x%X, offset 0x%X size 0x%x flush %d rtn %d\n",
491 object, offset, size, should_flush, should_return);
492
493 /*
494 * Lock the object, and acquire a paging reference to
495 * prevent the memory_object and control ports from
496 * being destroyed.
497 */
498 vm_object_lock(object);
499 vm_object_paging_begin(object);
500
501 if (should_flush)
502 flags = MEMORY_OBJECT_DATA_FLUSH;
503 else
504 flags = 0;
505
506 if (should_iosync)
507 flags |= MEMORY_OBJECT_IO_SYNC;
508
509 rv = vm_object_update(object, offset, (vm_object_size_t)size, NULL, NULL,
510 (should_return) ?
511 MEMORY_OBJECT_RETURN_ALL :
512 MEMORY_OBJECT_RETURN_NONE,
513 flags,
514 VM_PROT_NO_CHANGE);
515
516
517 vm_object_paging_end(object);
518 vm_object_unlock(object);
519 return rv;
520 }
521
522
523
524 #define LIST_REQ_PAGEOUT_PAGES(object, data_cnt, po, ro, ioerr, iosync) \
525 MACRO_BEGIN \
526 \
527 int upl_flags; \
528 memory_object_t pager; \
529 \
530 if (object == slide_info.slide_object) { \
531 panic("Objects with slid pages not allowed\n"); \
532 } \
533 \
534 if ((pager = (object)->pager) != MEMORY_OBJECT_NULL) { \
535 vm_object_paging_begin(object); \
536 vm_object_unlock(object); \
537 \
538 if (iosync) \
539 upl_flags = UPL_MSYNC | UPL_IOSYNC; \
540 else \
541 upl_flags = UPL_MSYNC; \
542 \
543 (void) memory_object_data_return(pager, \
544 po, \
545 (memory_object_cluster_size_t)data_cnt, \
546 ro, \
547 ioerr, \
548 FALSE, \
549 FALSE, \
550 upl_flags); \
551 \
552 vm_object_lock(object); \
553 vm_object_paging_end(object); \
554 } \
555 MACRO_END
556
557
558
559 static int
560 vm_object_update_extent(
561 vm_object_t object,
562 vm_object_offset_t offset,
563 vm_object_offset_t offset_end,
564 vm_object_offset_t *offset_resid,
565 int *io_errno,
566 boolean_t should_flush,
567 memory_object_return_t should_return,
568 boolean_t should_iosync,
569 vm_prot_t prot)
570 {
571 vm_page_t m;
572 int retval = 0;
573 vm_object_offset_t paging_offset = 0;
574 vm_object_offset_t next_offset = offset;
575 memory_object_lock_result_t page_lock_result;
576 memory_object_cluster_size_t data_cnt = 0;
577 struct vm_page_delayed_work dw_array[DEFAULT_DELAYED_WORK_LIMIT];
578 struct vm_page_delayed_work *dwp;
579 int dw_count;
580 int dw_limit;
581
582 dwp = &dw_array[0];
583 dw_count = 0;
584 dw_limit = DELAYED_WORK_LIMIT(DEFAULT_DELAYED_WORK_LIMIT);
585
586 for (;
587 offset < offset_end && object->resident_page_count;
588 offset += PAGE_SIZE_64) {
589
590 /*
591 * Limit the number of pages to be cleaned at once to a contiguous
592 * run, or at most MAX_UPL_TRANSFER size
593 */
594 if (data_cnt) {
595 if ((data_cnt >= PAGE_SIZE * MAX_UPL_TRANSFER) || (next_offset != offset)) {
596
597 if (dw_count) {
598 vm_page_do_delayed_work(object, &dw_array[0], dw_count);
599 dwp = &dw_array[0];
600 dw_count = 0;
601 }
602 LIST_REQ_PAGEOUT_PAGES(object, data_cnt,
603 paging_offset, offset_resid, io_errno, should_iosync);
604 data_cnt = 0;
605 }
606 }
607 while ((m = vm_page_lookup(object, offset)) != VM_PAGE_NULL) {
608
609 dwp->dw_mask = 0;
610
611 page_lock_result = memory_object_lock_page(m, should_return, should_flush, prot);
612
613 if (data_cnt && page_lock_result != MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN) {
614 /*
615 * End of a run of dirty/precious pages.
616 */
617 if (dw_count) {
618 vm_page_do_delayed_work(object, &dw_array[0], dw_count);
619 dwp = &dw_array[0];
620 dw_count = 0;
621 }
622 LIST_REQ_PAGEOUT_PAGES(object, data_cnt,
623 paging_offset, offset_resid, io_errno, should_iosync);
624 /*
625 * LIST_REQ_PAGEOUT_PAGES will drop the object lock which will
626 * allow the state of page 'm' to change... we need to re-lookup
627 * the current offset
628 */
629 data_cnt = 0;
630 continue;
631 }
632
633 switch (page_lock_result) {
634
635 case MEMORY_OBJECT_LOCK_RESULT_DONE:
636 break;
637
638 case MEMORY_OBJECT_LOCK_RESULT_MUST_FREE:
639 dwp->dw_mask |= DW_vm_page_free;
640 break;
641
642 case MEMORY_OBJECT_LOCK_RESULT_MUST_BLOCK:
643 PAGE_SLEEP(object, m, THREAD_UNINT);
644 continue;
645
646 case MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN:
647 if (data_cnt == 0)
648 paging_offset = offset;
649
650 data_cnt += PAGE_SIZE;
651 next_offset = offset + PAGE_SIZE_64;
652
653 /*
654 * Clean
655 */
656 m->list_req_pending = TRUE;
657 m->cleaning = TRUE;
658
659 /*
660 * wired pages shouldn't be flushed and
661 * since they aren't on any queue,
662 * no need to remove them
663 */
664 if (!VM_PAGE_WIRED(m)) {
665
666 if (should_flush) {
667 /*
668 * add additional state for the flush
669 */
670 m->busy = TRUE;
671 m->pageout = TRUE;
672
673 dwp->dw_mask |= DW_vm_page_wire;
674 }
675 /*
676 * we use to remove the page from the queues at this
677 * point, but we do not believe that an msync
678 * should cause the 'age' of a page to be changed
679 *
680 * else
681 * dwp->dw_mask |= DW_VM_PAGE_QUEUES_REMOVE;
682 */
683 }
684 retval = 1;
685 break;
686 }
687 if (dwp->dw_mask) {
688 VM_PAGE_ADD_DELAYED_WORK(dwp, m, dw_count);
689
690 if (dw_count >= dw_limit) {
691 vm_page_do_delayed_work(object, &dw_array[0], dw_count);
692 dwp = &dw_array[0];
693 dw_count = 0;
694 }
695 }
696 break;
697 }
698 }
699 /*
700 * We have completed the scan for applicable pages.
701 * Clean any pages that have been saved.
702 */
703 if (dw_count)
704 vm_page_do_delayed_work(object, &dw_array[0], dw_count);
705
706 if (data_cnt) {
707 LIST_REQ_PAGEOUT_PAGES(object, data_cnt,
708 paging_offset, offset_resid, io_errno, should_iosync);
709 }
710 return (retval);
711 }
712
713
714
715 /*
716 * Routine: vm_object_update
717 * Description:
718 * Work function for m_o_lock_request(), vm_o_sync().
719 *
720 * Called with object locked and paging ref taken.
721 */
722 kern_return_t
723 vm_object_update(
724 vm_object_t object,
725 vm_object_offset_t offset,
726 vm_object_size_t size,
727 vm_object_offset_t *resid_offset,
728 int *io_errno,
729 memory_object_return_t should_return,
730 int flags,
731 vm_prot_t protection)
732 {
733 vm_object_t copy_object = VM_OBJECT_NULL;
734 boolean_t data_returned = FALSE;
735 boolean_t update_cow;
736 boolean_t should_flush = (flags & MEMORY_OBJECT_DATA_FLUSH) ? TRUE : FALSE;
737 boolean_t should_iosync = (flags & MEMORY_OBJECT_IO_SYNC) ? TRUE : FALSE;
738 vm_fault_return_t result;
739 int num_of_extents;
740 int n;
741 #define MAX_EXTENTS 8
742 #define EXTENT_SIZE (1024 * 1024 * 256)
743 #define RESIDENT_LIMIT (1024 * 32)
744 struct extent {
745 vm_object_offset_t e_base;
746 vm_object_offset_t e_min;
747 vm_object_offset_t e_max;
748 } extents[MAX_EXTENTS];
749
750 /*
751 * To avoid blocking while scanning for pages, save
752 * dirty pages to be cleaned all at once.
753 *
754 * XXXO A similar strategy could be used to limit the
755 * number of times that a scan must be restarted for
756 * other reasons. Those pages that would require blocking
757 * could be temporarily collected in another list, or
758 * their offsets could be recorded in a small array.
759 */
760
761 /*
762 * XXX NOTE: May want to consider converting this to a page list
763 * XXX vm_map_copy interface. Need to understand object
764 * XXX coalescing implications before doing so.
765 */
766
767 update_cow = ((flags & MEMORY_OBJECT_DATA_FLUSH)
768 && (!(flags & MEMORY_OBJECT_DATA_NO_CHANGE) &&
769 !(flags & MEMORY_OBJECT_DATA_PURGE)))
770 || (flags & MEMORY_OBJECT_COPY_SYNC);
771
772 if (update_cow || (flags & (MEMORY_OBJECT_DATA_PURGE | MEMORY_OBJECT_DATA_SYNC))) {
773 int collisions = 0;
774
775 while ((copy_object = object->copy) != VM_OBJECT_NULL) {
776 /*
777 * need to do a try here since we're swimming upstream
778 * against the normal lock ordering... however, we need
779 * to hold the object stable until we gain control of the
780 * copy object so we have to be careful how we approach this
781 */
782 if (vm_object_lock_try(copy_object)) {
783 /*
784 * we 'won' the lock on the copy object...
785 * no need to hold the object lock any longer...
786 * take a real reference on the copy object because
787 * we're going to call vm_fault_page on it which may
788 * under certain conditions drop the lock and the paging
789 * reference we're about to take... the reference
790 * will keep the copy object from going away if that happens
791 */
792 vm_object_unlock(object);
793 vm_object_reference_locked(copy_object);
794 break;
795 }
796 vm_object_unlock(object);
797
798 collisions++;
799 mutex_pause(collisions);
800
801 vm_object_lock(object);
802 }
803 }
804 if ((copy_object != VM_OBJECT_NULL && update_cow) || (flags & MEMORY_OBJECT_DATA_SYNC)) {
805 vm_map_size_t i;
806 vm_map_size_t copy_size;
807 vm_map_offset_t copy_offset;
808 vm_prot_t prot;
809 vm_page_t page;
810 vm_page_t top_page;
811 kern_return_t error = 0;
812 struct vm_object_fault_info fault_info;
813
814 if (copy_object != VM_OBJECT_NULL) {
815 /*
816 * translate offset with respect to shadow's offset
817 */
818 copy_offset = (offset >= copy_object->vo_shadow_offset) ?
819 (vm_map_offset_t)(offset - copy_object->vo_shadow_offset) :
820 (vm_map_offset_t) 0;
821
822 if (copy_offset > copy_object->vo_size)
823 copy_offset = copy_object->vo_size;
824
825 /*
826 * clip size with respect to shadow offset
827 */
828 if (offset >= copy_object->vo_shadow_offset) {
829 copy_size = size;
830 } else if (size >= copy_object->vo_shadow_offset - offset) {
831 copy_size = size - (copy_object->vo_shadow_offset - offset);
832 } else {
833 copy_size = 0;
834 }
835
836 if (copy_offset + copy_size > copy_object->vo_size) {
837 if (copy_object->vo_size >= copy_offset) {
838 copy_size = copy_object->vo_size - copy_offset;
839 } else {
840 copy_size = 0;
841 }
842 }
843 copy_size+=copy_offset;
844
845 } else {
846 copy_object = object;
847
848 copy_size = offset + size;
849 copy_offset = offset;
850 }
851 fault_info.interruptible = THREAD_UNINT;
852 fault_info.behavior = VM_BEHAVIOR_SEQUENTIAL;
853 fault_info.user_tag = 0;
854 fault_info.lo_offset = copy_offset;
855 fault_info.hi_offset = copy_size;
856 fault_info.no_cache = FALSE;
857 fault_info.stealth = TRUE;
858 fault_info.io_sync = FALSE;
859 fault_info.cs_bypass = FALSE;
860 fault_info.mark_zf_absent = FALSE;
861
862 vm_object_paging_begin(copy_object);
863
864 for (i = copy_offset; i < copy_size; i += PAGE_SIZE) {
865 RETRY_COW_OF_LOCK_REQUEST:
866 fault_info.cluster_size = (vm_size_t) (copy_size - i);
867 assert(fault_info.cluster_size == copy_size - i);
868
869 prot = VM_PROT_WRITE|VM_PROT_READ;
870 result = vm_fault_page(copy_object, i,
871 VM_PROT_WRITE|VM_PROT_READ,
872 FALSE,
873 &prot,
874 &page,
875 &top_page,
876 (int *)0,
877 &error,
878 FALSE,
879 FALSE, &fault_info);
880
881 switch (result) {
882 case VM_FAULT_SUCCESS:
883 if (top_page) {
884 vm_fault_cleanup(
885 page->object, top_page);
886 vm_object_lock(copy_object);
887 vm_object_paging_begin(copy_object);
888 }
889 if (!page->active &&
890 !page->inactive &&
891 !page->throttled) {
892 vm_page_lockspin_queues();
893 if (!page->active &&
894 !page->inactive &&
895 !page->throttled)
896 vm_page_deactivate(page);
897 vm_page_unlock_queues();
898 }
899 PAGE_WAKEUP_DONE(page);
900 break;
901 case VM_FAULT_RETRY:
902 prot = VM_PROT_WRITE|VM_PROT_READ;
903 vm_object_lock(copy_object);
904 vm_object_paging_begin(copy_object);
905 goto RETRY_COW_OF_LOCK_REQUEST;
906 case VM_FAULT_INTERRUPTED:
907 prot = VM_PROT_WRITE|VM_PROT_READ;
908 vm_object_lock(copy_object);
909 vm_object_paging_begin(copy_object);
910 goto RETRY_COW_OF_LOCK_REQUEST;
911 case VM_FAULT_MEMORY_SHORTAGE:
912 VM_PAGE_WAIT();
913 prot = VM_PROT_WRITE|VM_PROT_READ;
914 vm_object_lock(copy_object);
915 vm_object_paging_begin(copy_object);
916 goto RETRY_COW_OF_LOCK_REQUEST;
917 case VM_FAULT_SUCCESS_NO_VM_PAGE:
918 /* success but no VM page: fail */
919 vm_object_paging_end(copy_object);
920 vm_object_unlock(copy_object);
921 /*FALLTHROUGH*/
922 case VM_FAULT_MEMORY_ERROR:
923 if (object != copy_object)
924 vm_object_deallocate(copy_object);
925 vm_object_lock(object);
926 goto BYPASS_COW_COPYIN;
927 default:
928 panic("vm_object_update: unexpected error 0x%x"
929 " from vm_fault_page()\n", result);
930 }
931
932 }
933 vm_object_paging_end(copy_object);
934 }
935 if ((flags & (MEMORY_OBJECT_DATA_SYNC | MEMORY_OBJECT_COPY_SYNC))) {
936 if (copy_object != VM_OBJECT_NULL && copy_object != object) {
937 vm_object_unlock(copy_object);
938 vm_object_deallocate(copy_object);
939 vm_object_lock(object);
940 }
941 return KERN_SUCCESS;
942 }
943 if (copy_object != VM_OBJECT_NULL && copy_object != object) {
944 if ((flags & MEMORY_OBJECT_DATA_PURGE)) {
945 copy_object->shadow_severed = TRUE;
946 copy_object->shadowed = FALSE;
947 copy_object->shadow = NULL;
948 /*
949 * delete the ref the COW was holding on the target object
950 */
951 vm_object_deallocate(object);
952 }
953 vm_object_unlock(copy_object);
954 vm_object_deallocate(copy_object);
955 vm_object_lock(object);
956 }
957 BYPASS_COW_COPYIN:
958
959 /*
960 * when we have a really large range to check relative
961 * to the number of actual resident pages, we'd like
962 * to use the resident page list to drive our checks
963 * however, the object lock will get dropped while processing
964 * the page which means the resident queue can change which
965 * means we can't walk the queue as we process the pages
966 * we also want to do the processing in offset order to allow
967 * 'runs' of pages to be collected if we're being told to
968 * flush to disk... the resident page queue is NOT ordered.
969 *
970 * a temporary solution (until we figure out how to deal with
971 * large address spaces more generically) is to pre-flight
972 * the resident page queue (if it's small enough) and develop
973 * a collection of extents (that encompass actual resident pages)
974 * to visit. This will at least allow us to deal with some of the
975 * more pathological cases in a more efficient manner. The current
976 * worst case (a single resident page at the end of an extremely large
977 * range) can take minutes to complete for ranges in the terrabyte
978 * category... since this routine is called when truncating a file,
979 * and we currently support files up to 16 Tbytes in size, this
980 * is not a theoretical problem
981 */
982
983 if ((object->resident_page_count < RESIDENT_LIMIT) &&
984 (atop_64(size) > (unsigned)(object->resident_page_count/(8 * MAX_EXTENTS)))) {
985 vm_page_t next;
986 vm_object_offset_t start;
987 vm_object_offset_t end;
988 vm_object_size_t e_mask;
989 vm_page_t m;
990
991 start = offset;
992 end = offset + size;
993 num_of_extents = 0;
994 e_mask = ~((vm_object_size_t)(EXTENT_SIZE - 1));
995
996 m = (vm_page_t) queue_first(&object->memq);
997
998 while (!queue_end(&object->memq, (queue_entry_t) m)) {
999 next = (vm_page_t) queue_next(&m->listq);
1000
1001 if ((m->offset >= start) && (m->offset < end)) {
1002 /*
1003 * this is a page we're interested in
1004 * try to fit it into a current extent
1005 */
1006 for (n = 0; n < num_of_extents; n++) {
1007 if ((m->offset & e_mask) == extents[n].e_base) {
1008 /*
1009 * use (PAGE_SIZE - 1) to determine the
1010 * max offset so that we don't wrap if
1011 * we're at the last page of the space
1012 */
1013 if (m->offset < extents[n].e_min)
1014 extents[n].e_min = m->offset;
1015 else if ((m->offset + (PAGE_SIZE - 1)) > extents[n].e_max)
1016 extents[n].e_max = m->offset + (PAGE_SIZE - 1);
1017 break;
1018 }
1019 }
1020 if (n == num_of_extents) {
1021 /*
1022 * didn't find a current extent that can encompass
1023 * this page
1024 */
1025 if (n < MAX_EXTENTS) {
1026 /*
1027 * if we still have room,
1028 * create a new extent
1029 */
1030 extents[n].e_base = m->offset & e_mask;
1031 extents[n].e_min = m->offset;
1032 extents[n].e_max = m->offset + (PAGE_SIZE - 1);
1033
1034 num_of_extents++;
1035 } else {
1036 /*
1037 * no room to create a new extent...
1038 * fall back to a single extent based
1039 * on the min and max page offsets
1040 * we find in the range we're interested in...
1041 * first, look through the extent list and
1042 * develop the overall min and max for the
1043 * pages we've looked at up to this point
1044 */
1045 for (n = 1; n < num_of_extents; n++) {
1046 if (extents[n].e_min < extents[0].e_min)
1047 extents[0].e_min = extents[n].e_min;
1048 if (extents[n].e_max > extents[0].e_max)
1049 extents[0].e_max = extents[n].e_max;
1050 }
1051 /*
1052 * now setup to run through the remaining pages
1053 * to determine the overall min and max
1054 * offset for the specified range
1055 */
1056 extents[0].e_base = 0;
1057 e_mask = 0;
1058 num_of_extents = 1;
1059
1060 /*
1061 * by continuing, we'll reprocess the
1062 * page that forced us to abandon trying
1063 * to develop multiple extents
1064 */
1065 continue;
1066 }
1067 }
1068 }
1069 m = next;
1070 }
1071 } else {
1072 extents[0].e_min = offset;
1073 extents[0].e_max = offset + (size - 1);
1074
1075 num_of_extents = 1;
1076 }
1077 for (n = 0; n < num_of_extents; n++) {
1078 if (vm_object_update_extent(object, extents[n].e_min, extents[n].e_max, resid_offset, io_errno,
1079 should_flush, should_return, should_iosync, protection))
1080 data_returned = TRUE;
1081 }
1082 return (data_returned);
1083 }
1084
1085
1086 /*
1087 * Routine: memory_object_synchronize_completed [user interface]
1088 *
1089 * Tell kernel that previously synchronized data
1090 * (memory_object_synchronize) has been queue or placed on the
1091 * backing storage.
1092 *
1093 * Note: there may be multiple synchronize requests for a given
1094 * memory object outstanding but they will not overlap.
1095 */
1096
1097 kern_return_t
1098 memory_object_synchronize_completed(
1099 memory_object_control_t control,
1100 memory_object_offset_t offset,
1101 memory_object_size_t length)
1102 {
1103 vm_object_t object;
1104 msync_req_t msr;
1105
1106 object = memory_object_control_to_vm_object(control);
1107
1108 XPR(XPR_MEMORY_OBJECT,
1109 "m_o_sync_completed, object 0x%X, offset 0x%X length 0x%X\n",
1110 object, offset, length, 0, 0);
1111
1112 /*
1113 * Look for bogus arguments
1114 */
1115
1116 if (object == VM_OBJECT_NULL)
1117 return (KERN_INVALID_ARGUMENT);
1118
1119 vm_object_lock(object);
1120
1121 /*
1122 * search for sync request structure
1123 */
1124 queue_iterate(&object->msr_q, msr, msync_req_t, msr_q) {
1125 if (msr->offset == offset && msr->length == length) {
1126 queue_remove(&object->msr_q, msr, msync_req_t, msr_q);
1127 break;
1128 }
1129 }/* queue_iterate */
1130
1131 if (queue_end(&object->msr_q, (queue_entry_t)msr)) {
1132 vm_object_unlock(object);
1133 return KERN_INVALID_ARGUMENT;
1134 }
1135
1136 msr_lock(msr);
1137 vm_object_unlock(object);
1138 msr->flag = VM_MSYNC_DONE;
1139 msr_unlock(msr);
1140 thread_wakeup((event_t) msr);
1141
1142 return KERN_SUCCESS;
1143 }/* memory_object_synchronize_completed */
1144
1145 static kern_return_t
1146 vm_object_set_attributes_common(
1147 vm_object_t object,
1148 boolean_t may_cache,
1149 memory_object_copy_strategy_t copy_strategy,
1150 boolean_t temporary,
1151 boolean_t silent_overwrite,
1152 boolean_t advisory_pageout)
1153 {
1154 boolean_t object_became_ready;
1155
1156 XPR(XPR_MEMORY_OBJECT,
1157 "m_o_set_attr_com, object 0x%X flg %x strat %d\n",
1158 object, (may_cache&1)|((temporary&1)<1), copy_strategy, 0, 0);
1159
1160 if (object == VM_OBJECT_NULL)
1161 return(KERN_INVALID_ARGUMENT);
1162
1163 /*
1164 * Verify the attributes of importance
1165 */
1166
1167 switch(copy_strategy) {
1168 case MEMORY_OBJECT_COPY_NONE:
1169 case MEMORY_OBJECT_COPY_DELAY:
1170 break;
1171 default:
1172 return(KERN_INVALID_ARGUMENT);
1173 }
1174
1175 #if !ADVISORY_PAGEOUT
1176 if (silent_overwrite || advisory_pageout)
1177 return(KERN_INVALID_ARGUMENT);
1178
1179 #endif /* !ADVISORY_PAGEOUT */
1180 if (may_cache)
1181 may_cache = TRUE;
1182 if (temporary)
1183 temporary = TRUE;
1184
1185 vm_object_lock(object);
1186
1187 /*
1188 * Copy the attributes
1189 */
1190 assert(!object->internal);
1191 object_became_ready = !object->pager_ready;
1192 object->copy_strategy = copy_strategy;
1193 object->can_persist = may_cache;
1194 object->temporary = temporary;
1195 object->silent_overwrite = silent_overwrite;
1196 object->advisory_pageout = advisory_pageout;
1197
1198 /*
1199 * Wake up anyone waiting for the ready attribute
1200 * to become asserted.
1201 */
1202
1203 if (object_became_ready) {
1204 object->pager_ready = TRUE;
1205 vm_object_wakeup(object, VM_OBJECT_EVENT_PAGER_READY);
1206 }
1207
1208 vm_object_unlock(object);
1209
1210 return(KERN_SUCCESS);
1211 }
1212
1213 /*
1214 * Set the memory object attribute as provided.
1215 *
1216 * XXX This routine cannot be completed until the vm_msync, clean
1217 * in place, and cluster work is completed. See ifdef notyet
1218 * below and note that vm_object_set_attributes_common()
1219 * may have to be expanded.
1220 */
1221 kern_return_t
1222 memory_object_change_attributes(
1223 memory_object_control_t control,
1224 memory_object_flavor_t flavor,
1225 memory_object_info_t attributes,
1226 mach_msg_type_number_t count)
1227 {
1228 vm_object_t object;
1229 kern_return_t result = KERN_SUCCESS;
1230 boolean_t temporary;
1231 boolean_t may_cache;
1232 boolean_t invalidate;
1233 memory_object_copy_strategy_t copy_strategy;
1234 boolean_t silent_overwrite;
1235 boolean_t advisory_pageout;
1236
1237 object = memory_object_control_to_vm_object(control);
1238 if (object == VM_OBJECT_NULL)
1239 return (KERN_INVALID_ARGUMENT);
1240
1241 vm_object_lock(object);
1242
1243 temporary = object->temporary;
1244 may_cache = object->can_persist;
1245 copy_strategy = object->copy_strategy;
1246 silent_overwrite = object->silent_overwrite;
1247 advisory_pageout = object->advisory_pageout;
1248 #if notyet
1249 invalidate = object->invalidate;
1250 #endif
1251 vm_object_unlock(object);
1252
1253 switch (flavor) {
1254 case OLD_MEMORY_OBJECT_BEHAVIOR_INFO:
1255 {
1256 old_memory_object_behave_info_t behave;
1257
1258 if (count != OLD_MEMORY_OBJECT_BEHAVE_INFO_COUNT) {
1259 result = KERN_INVALID_ARGUMENT;
1260 break;
1261 }
1262
1263 behave = (old_memory_object_behave_info_t) attributes;
1264
1265 temporary = behave->temporary;
1266 invalidate = behave->invalidate;
1267 copy_strategy = behave->copy_strategy;
1268
1269 break;
1270 }
1271
1272 case MEMORY_OBJECT_BEHAVIOR_INFO:
1273 {
1274 memory_object_behave_info_t behave;
1275
1276 if (count != MEMORY_OBJECT_BEHAVE_INFO_COUNT) {
1277 result = KERN_INVALID_ARGUMENT;
1278 break;
1279 }
1280
1281 behave = (memory_object_behave_info_t) attributes;
1282
1283 temporary = behave->temporary;
1284 invalidate = behave->invalidate;
1285 copy_strategy = behave->copy_strategy;
1286 silent_overwrite = behave->silent_overwrite;
1287 advisory_pageout = behave->advisory_pageout;
1288 break;
1289 }
1290
1291 case MEMORY_OBJECT_PERFORMANCE_INFO:
1292 {
1293 memory_object_perf_info_t perf;
1294
1295 if (count != MEMORY_OBJECT_PERF_INFO_COUNT) {
1296 result = KERN_INVALID_ARGUMENT;
1297 break;
1298 }
1299
1300 perf = (memory_object_perf_info_t) attributes;
1301
1302 may_cache = perf->may_cache;
1303
1304 break;
1305 }
1306
1307 case OLD_MEMORY_OBJECT_ATTRIBUTE_INFO:
1308 {
1309 old_memory_object_attr_info_t attr;
1310
1311 if (count != OLD_MEMORY_OBJECT_ATTR_INFO_COUNT) {
1312 result = KERN_INVALID_ARGUMENT;
1313 break;
1314 }
1315
1316 attr = (old_memory_object_attr_info_t) attributes;
1317
1318 may_cache = attr->may_cache;
1319 copy_strategy = attr->copy_strategy;
1320
1321 break;
1322 }
1323
1324 case MEMORY_OBJECT_ATTRIBUTE_INFO:
1325 {
1326 memory_object_attr_info_t attr;
1327
1328 if (count != MEMORY_OBJECT_ATTR_INFO_COUNT) {
1329 result = KERN_INVALID_ARGUMENT;
1330 break;
1331 }
1332
1333 attr = (memory_object_attr_info_t) attributes;
1334
1335 copy_strategy = attr->copy_strategy;
1336 may_cache = attr->may_cache_object;
1337 temporary = attr->temporary;
1338
1339 break;
1340 }
1341
1342 default:
1343 result = KERN_INVALID_ARGUMENT;
1344 break;
1345 }
1346
1347 if (result != KERN_SUCCESS)
1348 return(result);
1349
1350 if (copy_strategy == MEMORY_OBJECT_COPY_TEMPORARY) {
1351 copy_strategy = MEMORY_OBJECT_COPY_DELAY;
1352 temporary = TRUE;
1353 } else {
1354 temporary = FALSE;
1355 }
1356
1357 /*
1358 * XXX may_cache may become a tri-valued variable to handle
1359 * XXX uncache if not in use.
1360 */
1361 return (vm_object_set_attributes_common(object,
1362 may_cache,
1363 copy_strategy,
1364 temporary,
1365 silent_overwrite,
1366 advisory_pageout));
1367 }
1368
1369 kern_return_t
1370 memory_object_get_attributes(
1371 memory_object_control_t control,
1372 memory_object_flavor_t flavor,
1373 memory_object_info_t attributes, /* pointer to OUT array */
1374 mach_msg_type_number_t *count) /* IN/OUT */
1375 {
1376 kern_return_t ret = KERN_SUCCESS;
1377 vm_object_t object;
1378
1379 object = memory_object_control_to_vm_object(control);
1380 if (object == VM_OBJECT_NULL)
1381 return (KERN_INVALID_ARGUMENT);
1382
1383 vm_object_lock(object);
1384
1385 switch (flavor) {
1386 case OLD_MEMORY_OBJECT_BEHAVIOR_INFO:
1387 {
1388 old_memory_object_behave_info_t behave;
1389
1390 if (*count < OLD_MEMORY_OBJECT_BEHAVE_INFO_COUNT) {
1391 ret = KERN_INVALID_ARGUMENT;
1392 break;
1393 }
1394
1395 behave = (old_memory_object_behave_info_t) attributes;
1396 behave->copy_strategy = object->copy_strategy;
1397 behave->temporary = object->temporary;
1398 #if notyet /* remove when vm_msync complies and clean in place fini */
1399 behave->invalidate = object->invalidate;
1400 #else
1401 behave->invalidate = FALSE;
1402 #endif
1403
1404 *count = OLD_MEMORY_OBJECT_BEHAVE_INFO_COUNT;
1405 break;
1406 }
1407
1408 case MEMORY_OBJECT_BEHAVIOR_INFO:
1409 {
1410 memory_object_behave_info_t behave;
1411
1412 if (*count < MEMORY_OBJECT_BEHAVE_INFO_COUNT) {
1413 ret = KERN_INVALID_ARGUMENT;
1414 break;
1415 }
1416
1417 behave = (memory_object_behave_info_t) attributes;
1418 behave->copy_strategy = object->copy_strategy;
1419 behave->temporary = object->temporary;
1420 #if notyet /* remove when vm_msync complies and clean in place fini */
1421 behave->invalidate = object->invalidate;
1422 #else
1423 behave->invalidate = FALSE;
1424 #endif
1425 behave->advisory_pageout = object->advisory_pageout;
1426 behave->silent_overwrite = object->silent_overwrite;
1427 *count = MEMORY_OBJECT_BEHAVE_INFO_COUNT;
1428 break;
1429 }
1430
1431 case MEMORY_OBJECT_PERFORMANCE_INFO:
1432 {
1433 memory_object_perf_info_t perf;
1434
1435 if (*count < MEMORY_OBJECT_PERF_INFO_COUNT) {
1436 ret = KERN_INVALID_ARGUMENT;
1437 break;
1438 }
1439
1440 perf = (memory_object_perf_info_t) attributes;
1441 perf->cluster_size = PAGE_SIZE;
1442 perf->may_cache = object->can_persist;
1443
1444 *count = MEMORY_OBJECT_PERF_INFO_COUNT;
1445 break;
1446 }
1447
1448 case OLD_MEMORY_OBJECT_ATTRIBUTE_INFO:
1449 {
1450 old_memory_object_attr_info_t attr;
1451
1452 if (*count < OLD_MEMORY_OBJECT_ATTR_INFO_COUNT) {
1453 ret = KERN_INVALID_ARGUMENT;
1454 break;
1455 }
1456
1457 attr = (old_memory_object_attr_info_t) attributes;
1458 attr->may_cache = object->can_persist;
1459 attr->copy_strategy = object->copy_strategy;
1460
1461 *count = OLD_MEMORY_OBJECT_ATTR_INFO_COUNT;
1462 break;
1463 }
1464
1465 case MEMORY_OBJECT_ATTRIBUTE_INFO:
1466 {
1467 memory_object_attr_info_t attr;
1468
1469 if (*count < MEMORY_OBJECT_ATTR_INFO_COUNT) {
1470 ret = KERN_INVALID_ARGUMENT;
1471 break;
1472 }
1473
1474 attr = (memory_object_attr_info_t) attributes;
1475 attr->copy_strategy = object->copy_strategy;
1476 attr->cluster_size = PAGE_SIZE;
1477 attr->may_cache_object = object->can_persist;
1478 attr->temporary = object->temporary;
1479
1480 *count = MEMORY_OBJECT_ATTR_INFO_COUNT;
1481 break;
1482 }
1483
1484 default:
1485 ret = KERN_INVALID_ARGUMENT;
1486 break;
1487 }
1488
1489 vm_object_unlock(object);
1490
1491 return(ret);
1492 }
1493
1494
1495 kern_return_t
1496 memory_object_iopl_request(
1497 ipc_port_t port,
1498 memory_object_offset_t offset,
1499 upl_size_t *upl_size,
1500 upl_t *upl_ptr,
1501 upl_page_info_array_t user_page_list,
1502 unsigned int *page_list_count,
1503 int *flags)
1504 {
1505 vm_object_t object;
1506 kern_return_t ret;
1507 int caller_flags;
1508
1509 caller_flags = *flags;
1510
1511 if (caller_flags & ~UPL_VALID_FLAGS) {
1512 /*
1513 * For forward compatibility's sake,
1514 * reject any unknown flag.
1515 */
1516 return KERN_INVALID_VALUE;
1517 }
1518
1519 if (ip_kotype(port) == IKOT_NAMED_ENTRY) {
1520 vm_named_entry_t named_entry;
1521
1522 named_entry = (vm_named_entry_t)port->ip_kobject;
1523 /* a few checks to make sure user is obeying rules */
1524 if(*upl_size == 0) {
1525 if(offset >= named_entry->size)
1526 return(KERN_INVALID_RIGHT);
1527 *upl_size = (upl_size_t)(named_entry->size - offset);
1528 if (*upl_size != named_entry->size - offset)
1529 return KERN_INVALID_ARGUMENT;
1530 }
1531 if(caller_flags & UPL_COPYOUT_FROM) {
1532 if((named_entry->protection & VM_PROT_READ)
1533 != VM_PROT_READ) {
1534 return(KERN_INVALID_RIGHT);
1535 }
1536 } else {
1537 if((named_entry->protection &
1538 (VM_PROT_READ | VM_PROT_WRITE))
1539 != (VM_PROT_READ | VM_PROT_WRITE)) {
1540 return(KERN_INVALID_RIGHT);
1541 }
1542 }
1543 if(named_entry->size < (offset + *upl_size))
1544 return(KERN_INVALID_ARGUMENT);
1545
1546 /* the callers parameter offset is defined to be the */
1547 /* offset from beginning of named entry offset in object */
1548 offset = offset + named_entry->offset;
1549
1550 if(named_entry->is_sub_map)
1551 return (KERN_INVALID_ARGUMENT);
1552
1553 named_entry_lock(named_entry);
1554
1555 if (named_entry->is_pager) {
1556 object = vm_object_enter(named_entry->backing.pager,
1557 named_entry->offset + named_entry->size,
1558 named_entry->internal,
1559 FALSE,
1560 FALSE);
1561 if (object == VM_OBJECT_NULL) {
1562 named_entry_unlock(named_entry);
1563 return(KERN_INVALID_OBJECT);
1564 }
1565
1566 /* JMM - drop reference on pager here? */
1567
1568 /* create an extra reference for the named entry */
1569 vm_object_lock(object);
1570 vm_object_reference_locked(object);
1571 named_entry->backing.object = object;
1572 named_entry->is_pager = FALSE;
1573 named_entry_unlock(named_entry);
1574
1575 /* wait for object to be ready */
1576 while (!object->pager_ready) {
1577 vm_object_wait(object,
1578 VM_OBJECT_EVENT_PAGER_READY,
1579 THREAD_UNINT);
1580 vm_object_lock(object);
1581 }
1582 vm_object_unlock(object);
1583 } else {
1584 /* This is the case where we are going to map */
1585 /* an already mapped object. If the object is */
1586 /* not ready it is internal. An external */
1587 /* object cannot be mapped until it is ready */
1588 /* we can therefore avoid the ready check */
1589 /* in this case. */
1590 object = named_entry->backing.object;
1591 vm_object_reference(object);
1592 named_entry_unlock(named_entry);
1593 }
1594 } else if (ip_kotype(port) == IKOT_MEM_OBJ_CONTROL) {
1595 memory_object_control_t control;
1596 control = (memory_object_control_t) port;
1597 if (control == NULL)
1598 return (KERN_INVALID_ARGUMENT);
1599 object = memory_object_control_to_vm_object(control);
1600 if (object == VM_OBJECT_NULL)
1601 return (KERN_INVALID_ARGUMENT);
1602 vm_object_reference(object);
1603 } else {
1604 return KERN_INVALID_ARGUMENT;
1605 }
1606 if (object == VM_OBJECT_NULL)
1607 return (KERN_INVALID_ARGUMENT);
1608
1609 if (!object->private) {
1610 if (*upl_size > (MAX_UPL_TRANSFER*PAGE_SIZE))
1611 *upl_size = (MAX_UPL_TRANSFER*PAGE_SIZE);
1612 if (object->phys_contiguous) {
1613 *flags = UPL_PHYS_CONTIG;
1614 } else {
1615 *flags = 0;
1616 }
1617 } else {
1618 *flags = UPL_DEV_MEMORY | UPL_PHYS_CONTIG;
1619 }
1620
1621 ret = vm_object_iopl_request(object,
1622 offset,
1623 *upl_size,
1624 upl_ptr,
1625 user_page_list,
1626 page_list_count,
1627 caller_flags);
1628 vm_object_deallocate(object);
1629 return ret;
1630 }
1631
1632 /*
1633 * Routine: memory_object_upl_request [interface]
1634 * Purpose:
1635 * Cause the population of a portion of a vm_object.
1636 * Depending on the nature of the request, the pages
1637 * returned may be contain valid data or be uninitialized.
1638 *
1639 */
1640
1641 kern_return_t
1642 memory_object_upl_request(
1643 memory_object_control_t control,
1644 memory_object_offset_t offset,
1645 upl_size_t size,
1646 upl_t *upl_ptr,
1647 upl_page_info_array_t user_page_list,
1648 unsigned int *page_list_count,
1649 int cntrl_flags)
1650 {
1651 vm_object_t object;
1652
1653 object = memory_object_control_to_vm_object(control);
1654 if (object == VM_OBJECT_NULL)
1655 return (KERN_TERMINATED);
1656
1657 return vm_object_upl_request(object,
1658 offset,
1659 size,
1660 upl_ptr,
1661 user_page_list,
1662 page_list_count,
1663 cntrl_flags);
1664 }
1665
1666 /*
1667 * Routine: memory_object_super_upl_request [interface]
1668 * Purpose:
1669 * Cause the population of a portion of a vm_object
1670 * in much the same way as memory_object_upl_request.
1671 * Depending on the nature of the request, the pages
1672 * returned may be contain valid data or be uninitialized.
1673 * However, the region may be expanded up to the super
1674 * cluster size provided.
1675 */
1676
1677 kern_return_t
1678 memory_object_super_upl_request(
1679 memory_object_control_t control,
1680 memory_object_offset_t offset,
1681 upl_size_t size,
1682 upl_size_t super_cluster,
1683 upl_t *upl,
1684 upl_page_info_t *user_page_list,
1685 unsigned int *page_list_count,
1686 int cntrl_flags)
1687 {
1688 vm_object_t object;
1689
1690 object = memory_object_control_to_vm_object(control);
1691 if (object == VM_OBJECT_NULL)
1692 return (KERN_INVALID_ARGUMENT);
1693
1694 return vm_object_super_upl_request(object,
1695 offset,
1696 size,
1697 super_cluster,
1698 upl,
1699 user_page_list,
1700 page_list_count,
1701 cntrl_flags);
1702 }
1703
1704 kern_return_t
1705 memory_object_cluster_size(memory_object_control_t control, memory_object_offset_t *start,
1706 vm_size_t *length, uint32_t *io_streaming, memory_object_fault_info_t fault_info)
1707 {
1708 vm_object_t object;
1709
1710 object = memory_object_control_to_vm_object(control);
1711
1712 if (object == VM_OBJECT_NULL || object->paging_offset > *start)
1713 return (KERN_INVALID_ARGUMENT);
1714
1715 *start -= object->paging_offset;
1716
1717 vm_object_cluster_size(object, (vm_object_offset_t *)start, length, (vm_object_fault_info_t)fault_info, io_streaming);
1718
1719 *start += object->paging_offset;
1720
1721 return (KERN_SUCCESS);
1722 }
1723
1724
1725 int vm_stat_discard_cleared_reply = 0;
1726 int vm_stat_discard_cleared_unset = 0;
1727 int vm_stat_discard_cleared_too_late = 0;
1728
1729
1730
1731 /*
1732 * Routine: host_default_memory_manager [interface]
1733 * Purpose:
1734 * set/get the default memory manager port and default cluster
1735 * size.
1736 *
1737 * If successful, consumes the supplied naked send right.
1738 */
1739 kern_return_t
1740 host_default_memory_manager(
1741 host_priv_t host_priv,
1742 memory_object_default_t *default_manager,
1743 __unused memory_object_cluster_size_t cluster_size)
1744 {
1745 memory_object_default_t current_manager;
1746 memory_object_default_t new_manager;
1747 memory_object_default_t returned_manager;
1748 kern_return_t result = KERN_SUCCESS;
1749
1750 if (host_priv == HOST_PRIV_NULL)
1751 return(KERN_INVALID_HOST);
1752
1753 assert(host_priv == &realhost);
1754
1755 new_manager = *default_manager;
1756 lck_mtx_lock(&memory_manager_default_lock);
1757 current_manager = memory_manager_default;
1758 returned_manager = MEMORY_OBJECT_DEFAULT_NULL;
1759
1760 if (new_manager == MEMORY_OBJECT_DEFAULT_NULL) {
1761 /*
1762 * Retrieve the current value.
1763 */
1764 returned_manager = current_manager;
1765 memory_object_default_reference(returned_manager);
1766 } else {
1767
1768 /*
1769 * If this is the first non-null manager, start
1770 * up the internal pager support.
1771 */
1772 if (current_manager == MEMORY_OBJECT_DEFAULT_NULL) {
1773 result = vm_pageout_internal_start();
1774 if (result != KERN_SUCCESS)
1775 goto out;
1776 }
1777
1778 /*
1779 * Retrieve the current value,
1780 * and replace it with the supplied value.
1781 * We return the old reference to the caller
1782 * but we have to take a reference on the new
1783 * one.
1784 */
1785 returned_manager = current_manager;
1786 memory_manager_default = new_manager;
1787 memory_object_default_reference(new_manager);
1788
1789 /*
1790 * In case anyone's been waiting for a memory
1791 * manager to be established, wake them up.
1792 */
1793
1794 thread_wakeup((event_t) &memory_manager_default);
1795
1796 #ifndef CONFIG_FREEZE
1797 /*
1798 * Now that we have a default pager for anonymous memory,
1799 * reactivate all the throttled pages (i.e. dirty pages with
1800 * no pager).
1801 */
1802 if (current_manager == MEMORY_OBJECT_DEFAULT_NULL)
1803 {
1804 vm_page_reactivate_all_throttled();
1805 }
1806 #endif
1807 }
1808 out:
1809 lck_mtx_unlock(&memory_manager_default_lock);
1810
1811 *default_manager = returned_manager;
1812 return(result);
1813 }
1814
1815 /*
1816 * Routine: memory_manager_default_reference
1817 * Purpose:
1818 * Returns a naked send right for the default
1819 * memory manager. The returned right is always
1820 * valid (not IP_NULL or IP_DEAD).
1821 */
1822
1823 __private_extern__ memory_object_default_t
1824 memory_manager_default_reference(void)
1825 {
1826 memory_object_default_t current_manager;
1827
1828 lck_mtx_lock(&memory_manager_default_lock);
1829 current_manager = memory_manager_default;
1830 while (current_manager == MEMORY_OBJECT_DEFAULT_NULL) {
1831 wait_result_t res;
1832
1833 res = lck_mtx_sleep(&memory_manager_default_lock,
1834 LCK_SLEEP_DEFAULT,
1835 (event_t) &memory_manager_default,
1836 THREAD_UNINT);
1837 assert(res == THREAD_AWAKENED);
1838 current_manager = memory_manager_default;
1839 }
1840 memory_object_default_reference(current_manager);
1841 lck_mtx_unlock(&memory_manager_default_lock);
1842
1843 return current_manager;
1844 }
1845
1846 /*
1847 * Routine: memory_manager_default_check
1848 *
1849 * Purpose:
1850 * Check whether a default memory manager has been set
1851 * up yet, or not. Returns KERN_SUCCESS if dmm exists,
1852 * and KERN_FAILURE if dmm does not exist.
1853 *
1854 * If there is no default memory manager, log an error,
1855 * but only the first time.
1856 *
1857 */
1858 __private_extern__ kern_return_t
1859 memory_manager_default_check(void)
1860 {
1861 memory_object_default_t current;
1862
1863 lck_mtx_lock(&memory_manager_default_lock);
1864 current = memory_manager_default;
1865 if (current == MEMORY_OBJECT_DEFAULT_NULL) {
1866 static boolean_t logged; /* initialized to 0 */
1867 boolean_t complain = !logged;
1868 logged = TRUE;
1869 lck_mtx_unlock(&memory_manager_default_lock);
1870 if (complain)
1871 printf("Warning: No default memory manager\n");
1872 return(KERN_FAILURE);
1873 } else {
1874 lck_mtx_unlock(&memory_manager_default_lock);
1875 return(KERN_SUCCESS);
1876 }
1877 }
1878
1879 __private_extern__ void
1880 memory_manager_default_init(void)
1881 {
1882 memory_manager_default = MEMORY_OBJECT_DEFAULT_NULL;
1883 lck_mtx_init(&memory_manager_default_lock, &vm_object_lck_grp, &vm_object_lck_attr);
1884 }
1885
1886
1887
1888 /* Allow manipulation of individual page state. This is actually part of */
1889 /* the UPL regimen but takes place on the object rather than on a UPL */
1890
1891 kern_return_t
1892 memory_object_page_op(
1893 memory_object_control_t control,
1894 memory_object_offset_t offset,
1895 int ops,
1896 ppnum_t *phys_entry,
1897 int *flags)
1898 {
1899 vm_object_t object;
1900
1901 object = memory_object_control_to_vm_object(control);
1902 if (object == VM_OBJECT_NULL)
1903 return (KERN_INVALID_ARGUMENT);
1904
1905 return vm_object_page_op(object, offset, ops, phys_entry, flags);
1906 }
1907
1908 /*
1909 * memory_object_range_op offers performance enhancement over
1910 * memory_object_page_op for page_op functions which do not require page
1911 * level state to be returned from the call. Page_op was created to provide
1912 * a low-cost alternative to page manipulation via UPLs when only a single
1913 * page was involved. The range_op call establishes the ability in the _op
1914 * family of functions to work on multiple pages where the lack of page level
1915 * state handling allows the caller to avoid the overhead of the upl structures.
1916 */
1917
1918 kern_return_t
1919 memory_object_range_op(
1920 memory_object_control_t control,
1921 memory_object_offset_t offset_beg,
1922 memory_object_offset_t offset_end,
1923 int ops,
1924 int *range)
1925 {
1926 vm_object_t object;
1927
1928 object = memory_object_control_to_vm_object(control);
1929 if (object == VM_OBJECT_NULL)
1930 return (KERN_INVALID_ARGUMENT);
1931
1932 return vm_object_range_op(object,
1933 offset_beg,
1934 offset_end,
1935 ops,
1936 (uint32_t *) range);
1937 }
1938
1939
1940 void
1941 memory_object_mark_used(
1942 memory_object_control_t control)
1943 {
1944 vm_object_t object;
1945
1946 if (control == NULL)
1947 return;
1948
1949 object = memory_object_control_to_vm_object(control);
1950
1951 if (object != VM_OBJECT_NULL)
1952 vm_object_cache_remove(object);
1953 }
1954
1955
1956 void
1957 memory_object_mark_unused(
1958 memory_object_control_t control,
1959 __unused boolean_t rage)
1960 {
1961 vm_object_t object;
1962
1963 if (control == NULL)
1964 return;
1965
1966 object = memory_object_control_to_vm_object(control);
1967
1968 if (object != VM_OBJECT_NULL)
1969 vm_object_cache_add(object);
1970 }
1971
1972
1973 kern_return_t
1974 memory_object_pages_resident(
1975 memory_object_control_t control,
1976 boolean_t * has_pages_resident)
1977 {
1978 vm_object_t object;
1979
1980 *has_pages_resident = FALSE;
1981
1982 object = memory_object_control_to_vm_object(control);
1983 if (object == VM_OBJECT_NULL)
1984 return (KERN_INVALID_ARGUMENT);
1985
1986 if (object->resident_page_count)
1987 *has_pages_resident = TRUE;
1988
1989 return (KERN_SUCCESS);
1990 }
1991
1992 kern_return_t
1993 memory_object_signed(
1994 memory_object_control_t control,
1995 boolean_t is_signed)
1996 {
1997 vm_object_t object;
1998
1999 object = memory_object_control_to_vm_object(control);
2000 if (object == VM_OBJECT_NULL)
2001 return KERN_INVALID_ARGUMENT;
2002
2003 vm_object_lock(object);
2004 object->code_signed = is_signed;
2005 vm_object_unlock(object);
2006
2007 return KERN_SUCCESS;
2008 }
2009
2010 boolean_t
2011 memory_object_is_slid(
2012 memory_object_control_t control)
2013 {
2014 vm_object_t object = VM_OBJECT_NULL;
2015 vm_object_t slide_object = slide_info.slide_object;
2016
2017 object = memory_object_control_to_vm_object(control);
2018 if (object == VM_OBJECT_NULL)
2019 return FALSE;
2020
2021 return (object == slide_object);
2022 }
2023
2024 static zone_t mem_obj_control_zone;
2025
2026 __private_extern__ void
2027 memory_object_control_bootstrap(void)
2028 {
2029 int i;
2030
2031 i = (vm_size_t) sizeof (struct memory_object_control);
2032 mem_obj_control_zone = zinit (i, 8192*i, 4096, "mem_obj_control");
2033 zone_change(mem_obj_control_zone, Z_CALLERACCT, FALSE);
2034 zone_change(mem_obj_control_zone, Z_NOENCRYPT, TRUE);
2035 return;
2036 }
2037
2038 __private_extern__ memory_object_control_t
2039 memory_object_control_allocate(
2040 vm_object_t object)
2041 {
2042 memory_object_control_t control;
2043
2044 control = (memory_object_control_t)zalloc(mem_obj_control_zone);
2045 if (control != MEMORY_OBJECT_CONTROL_NULL) {
2046 control->moc_object = object;
2047 control->moc_ikot = IKOT_MEM_OBJ_CONTROL; /* fake ip_kotype */
2048 }
2049 return (control);
2050 }
2051
2052 __private_extern__ void
2053 memory_object_control_collapse(
2054 memory_object_control_t control,
2055 vm_object_t object)
2056 {
2057 assert((control->moc_object != VM_OBJECT_NULL) &&
2058 (control->moc_object != object));
2059 control->moc_object = object;
2060 }
2061
2062 __private_extern__ vm_object_t
2063 memory_object_control_to_vm_object(
2064 memory_object_control_t control)
2065 {
2066 if (control == MEMORY_OBJECT_CONTROL_NULL ||
2067 control->moc_ikot != IKOT_MEM_OBJ_CONTROL)
2068 return VM_OBJECT_NULL;
2069
2070 return (control->moc_object);
2071 }
2072
2073 memory_object_control_t
2074 convert_port_to_mo_control(
2075 __unused mach_port_t port)
2076 {
2077 return MEMORY_OBJECT_CONTROL_NULL;
2078 }
2079
2080
2081 mach_port_t
2082 convert_mo_control_to_port(
2083 __unused memory_object_control_t control)
2084 {
2085 return MACH_PORT_NULL;
2086 }
2087
2088 void
2089 memory_object_control_reference(
2090 __unused memory_object_control_t control)
2091 {
2092 return;
2093 }
2094
2095 /*
2096 * We only every issue one of these references, so kill it
2097 * when that gets released (should switch the real reference
2098 * counting in true port-less EMMI).
2099 */
2100 void
2101 memory_object_control_deallocate(
2102 memory_object_control_t control)
2103 {
2104 zfree(mem_obj_control_zone, control);
2105 }
2106
2107 void
2108 memory_object_control_disable(
2109 memory_object_control_t control)
2110 {
2111 assert(control->moc_object != VM_OBJECT_NULL);
2112 control->moc_object = VM_OBJECT_NULL;
2113 }
2114
2115 void
2116 memory_object_default_reference(
2117 memory_object_default_t dmm)
2118 {
2119 ipc_port_make_send(dmm);
2120 }
2121
2122 void
2123 memory_object_default_deallocate(
2124 memory_object_default_t dmm)
2125 {
2126 ipc_port_release_send(dmm);
2127 }
2128
2129 memory_object_t
2130 convert_port_to_memory_object(
2131 __unused mach_port_t port)
2132 {
2133 return (MEMORY_OBJECT_NULL);
2134 }
2135
2136
2137 mach_port_t
2138 convert_memory_object_to_port(
2139 __unused memory_object_t object)
2140 {
2141 return (MACH_PORT_NULL);
2142 }
2143
2144
2145 /* Routine memory_object_reference */
2146 void memory_object_reference(
2147 memory_object_t memory_object)
2148 {
2149 (memory_object->mo_pager_ops->memory_object_reference)(
2150 memory_object);
2151 }
2152
2153 /* Routine memory_object_deallocate */
2154 void memory_object_deallocate(
2155 memory_object_t memory_object)
2156 {
2157 (memory_object->mo_pager_ops->memory_object_deallocate)(
2158 memory_object);
2159 }
2160
2161
2162 /* Routine memory_object_init */
2163 kern_return_t memory_object_init
2164 (
2165 memory_object_t memory_object,
2166 memory_object_control_t memory_control,
2167 memory_object_cluster_size_t memory_object_page_size
2168 )
2169 {
2170 return (memory_object->mo_pager_ops->memory_object_init)(
2171 memory_object,
2172 memory_control,
2173 memory_object_page_size);
2174 }
2175
2176 /* Routine memory_object_terminate */
2177 kern_return_t memory_object_terminate
2178 (
2179 memory_object_t memory_object
2180 )
2181 {
2182 return (memory_object->mo_pager_ops->memory_object_terminate)(
2183 memory_object);
2184 }
2185
2186 /* Routine memory_object_data_request */
2187 kern_return_t memory_object_data_request
2188 (
2189 memory_object_t memory_object,
2190 memory_object_offset_t offset,
2191 memory_object_cluster_size_t length,
2192 vm_prot_t desired_access,
2193 memory_object_fault_info_t fault_info
2194 )
2195 {
2196 return (memory_object->mo_pager_ops->memory_object_data_request)(
2197 memory_object,
2198 offset,
2199 length,
2200 desired_access,
2201 fault_info);
2202 }
2203
2204 /* Routine memory_object_data_return */
2205 kern_return_t memory_object_data_return
2206 (
2207 memory_object_t memory_object,
2208 memory_object_offset_t offset,
2209 memory_object_cluster_size_t size,
2210 memory_object_offset_t *resid_offset,
2211 int *io_error,
2212 boolean_t dirty,
2213 boolean_t kernel_copy,
2214 int upl_flags
2215 )
2216 {
2217 return (memory_object->mo_pager_ops->memory_object_data_return)(
2218 memory_object,
2219 offset,
2220 size,
2221 resid_offset,
2222 io_error,
2223 dirty,
2224 kernel_copy,
2225 upl_flags);
2226 }
2227
2228 /* Routine memory_object_data_initialize */
2229 kern_return_t memory_object_data_initialize
2230 (
2231 memory_object_t memory_object,
2232 memory_object_offset_t offset,
2233 memory_object_cluster_size_t size
2234 )
2235 {
2236 return (memory_object->mo_pager_ops->memory_object_data_initialize)(
2237 memory_object,
2238 offset,
2239 size);
2240 }
2241
2242 /* Routine memory_object_data_unlock */
2243 kern_return_t memory_object_data_unlock
2244 (
2245 memory_object_t memory_object,
2246 memory_object_offset_t offset,
2247 memory_object_size_t size,
2248 vm_prot_t desired_access
2249 )
2250 {
2251 return (memory_object->mo_pager_ops->memory_object_data_unlock)(
2252 memory_object,
2253 offset,
2254 size,
2255 desired_access);
2256 }
2257
2258 /* Routine memory_object_synchronize */
2259 kern_return_t memory_object_synchronize
2260 (
2261 memory_object_t memory_object,
2262 memory_object_offset_t offset,
2263 memory_object_size_t size,
2264 vm_sync_t sync_flags
2265 )
2266 {
2267 return (memory_object->mo_pager_ops->memory_object_synchronize)(
2268 memory_object,
2269 offset,
2270 size,
2271 sync_flags);
2272 }
2273
2274
2275 /*
2276 * memory_object_map() is called by VM (in vm_map_enter() and its variants)
2277 * each time a "named" VM object gets mapped directly or indirectly
2278 * (copy-on-write mapping). A "named" VM object has an extra reference held
2279 * by the pager to keep it alive until the pager decides that the
2280 * memory object (and its VM object) can be reclaimed.
2281 * VM calls memory_object_last_unmap() (in vm_object_deallocate()) when all
2282 * the mappings of that memory object have been removed.
2283 *
2284 * For a given VM object, calls to memory_object_map() and memory_object_unmap()
2285 * are serialized (through object->mapping_in_progress), to ensure that the
2286 * pager gets a consistent view of the mapping status of the memory object.
2287 *
2288 * This allows the pager to keep track of how many times a memory object
2289 * has been mapped and with which protections, to decide when it can be
2290 * reclaimed.
2291 */
2292
2293 /* Routine memory_object_map */
2294 kern_return_t memory_object_map
2295 (
2296 memory_object_t memory_object,
2297 vm_prot_t prot
2298 )
2299 {
2300 return (memory_object->mo_pager_ops->memory_object_map)(
2301 memory_object,
2302 prot);
2303 }
2304
2305 /* Routine memory_object_last_unmap */
2306 kern_return_t memory_object_last_unmap
2307 (
2308 memory_object_t memory_object
2309 )
2310 {
2311 return (memory_object->mo_pager_ops->memory_object_last_unmap)(
2312 memory_object);
2313 }
2314
2315 /* Routine memory_object_data_reclaim */
2316 kern_return_t memory_object_data_reclaim
2317 (
2318 memory_object_t memory_object,
2319 boolean_t reclaim_backing_store
2320 )
2321 {
2322 if (memory_object->mo_pager_ops->memory_object_data_reclaim == NULL)
2323 return KERN_NOT_SUPPORTED;
2324 return (memory_object->mo_pager_ops->memory_object_data_reclaim)(
2325 memory_object,
2326 reclaim_backing_store);
2327 }
2328
2329 /* Routine memory_object_create */
2330 kern_return_t memory_object_create
2331 (
2332 memory_object_default_t default_memory_manager,
2333 vm_size_t new_memory_object_size,
2334 memory_object_t *new_memory_object
2335 )
2336 {
2337 return default_pager_memory_object_create(default_memory_manager,
2338 new_memory_object_size,
2339 new_memory_object);
2340 }
2341
2342 upl_t
2343 convert_port_to_upl(
2344 ipc_port_t port)
2345 {
2346 upl_t upl;
2347
2348 ip_lock(port);
2349 if (!ip_active(port) || (ip_kotype(port) != IKOT_UPL)) {
2350 ip_unlock(port);
2351 return (upl_t)NULL;
2352 }
2353 upl = (upl_t) port->ip_kobject;
2354 ip_unlock(port);
2355 upl_lock(upl);
2356 upl->ref_count+=1;
2357 upl_unlock(upl);
2358 return upl;
2359 }
2360
2361 mach_port_t
2362 convert_upl_to_port(
2363 __unused upl_t upl)
2364 {
2365 return MACH_PORT_NULL;
2366 }
2367
2368 __private_extern__ void
2369 upl_no_senders(
2370 __unused ipc_port_t port,
2371 __unused mach_port_mscount_t mscount)
2372 {
2373 return;
2374 }
mirror of XNU