]> git.saurik.com Git - apple/xnu.git/blob - osfmk/vm/memory_object.c
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-2782.20.48.tar.gz
[apple/xnu.git] / osfmk / vm / memory_object.c
1 /*
2 * Copyright (c) 2000-2008 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56 /*
57 */
58 /*
59 * File: vm/memory_object.c
60 * Author: Michael Wayne Young
61 *
62 * External memory management interface control functions.
63 */
64
65 /*
66 * Interface dependencies:
67 */
68
69 #include <mach/std_types.h> /* For pointer_t */
70 #include <mach/mach_types.h>
71
72 #include <mach/mig.h>
73 #include <mach/kern_return.h>
74 #include <mach/memory_object.h>
75 #include <mach/memory_object_default.h>
76 #include <mach/memory_object_control_server.h>
77 #include <mach/host_priv_server.h>
78 #include <mach/boolean.h>
79 #include <mach/vm_prot.h>
80 #include <mach/message.h>
81
82 /*
83 * Implementation dependencies:
84 */
85 #include <string.h> /* For memcpy() */
86
87 #include <kern/xpr.h>
88 #include <kern/host.h>
89 #include <kern/thread.h> /* For current_thread() */
90 #include <kern/ipc_mig.h>
91 #include <kern/misc_protos.h>
92
93 #include <vm/vm_object.h>
94 #include <vm/vm_fault.h>
95 #include <vm/memory_object.h>
96 #include <vm/vm_page.h>
97 #include <vm/vm_pageout.h>
98 #include <vm/pmap.h> /* For pmap_clear_modify */
99 #include <vm/vm_kern.h> /* For kernel_map, vm_move */
100 #include <vm/vm_map.h> /* For vm_map_pageable */
101 #include <vm/vm_purgeable_internal.h> /* Needed by some vm_page.h macros */
102 #include <vm/vm_shared_region.h>
103
104 #include <vm/vm_external.h>
105
106 #include <vm/vm_protos.h>
107
108
109 memory_object_default_t memory_manager_default = MEMORY_OBJECT_DEFAULT_NULL;
110 decl_lck_mtx_data(, memory_manager_default_lock)
111
112
113 /*
114 * Routine: memory_object_should_return_page
115 *
116 * Description:
117 * Determine whether the given page should be returned,
118 * based on the page's state and on the given return policy.
119 *
120 * We should return the page if one of the following is true:
121 *
122 * 1. Page is dirty and should_return is not RETURN_NONE.
123 * 2. Page is precious and should_return is RETURN_ALL.
124 * 3. Should_return is RETURN_ANYTHING.
125 *
126 * As a side effect, m->dirty will be made consistent
127 * with pmap_is_modified(m), if should_return is not
128 * MEMORY_OBJECT_RETURN_NONE.
129 */
130
131 #define memory_object_should_return_page(m, should_return) \
132 (should_return != MEMORY_OBJECT_RETURN_NONE && \
133 (((m)->dirty || ((m)->dirty = pmap_is_modified((m)->phys_page))) || \
134 ((m)->precious && (should_return) == MEMORY_OBJECT_RETURN_ALL) || \
135 (should_return) == MEMORY_OBJECT_RETURN_ANYTHING))
136
137 typedef int memory_object_lock_result_t;
138
139 #define MEMORY_OBJECT_LOCK_RESULT_DONE 0
140 #define MEMORY_OBJECT_LOCK_RESULT_MUST_BLOCK 1
141 #define MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN 2
142 #define MEMORY_OBJECT_LOCK_RESULT_MUST_FREE 3
143
144 memory_object_lock_result_t memory_object_lock_page(
145 vm_page_t m,
146 memory_object_return_t should_return,
147 boolean_t should_flush,
148 vm_prot_t prot);
149
150 /*
151 * Routine: memory_object_lock_page
152 *
153 * Description:
154 * Perform the appropriate lock operations on the
155 * given page. See the description of
156 * "memory_object_lock_request" for the meanings
157 * of the arguments.
158 *
159 * Returns an indication that the operation
160 * completed, blocked, or that the page must
161 * be cleaned.
162 */
163 memory_object_lock_result_t
164 memory_object_lock_page(
165 vm_page_t m,
166 memory_object_return_t should_return,
167 boolean_t should_flush,
168 vm_prot_t prot)
169 {
170 XPR(XPR_MEMORY_OBJECT,
171 "m_o_lock_page, page 0x%X rtn %d flush %d prot %d\n",
172 m, should_return, should_flush, prot, 0);
173
174
175 if (m->busy || m->cleaning)
176 return (MEMORY_OBJECT_LOCK_RESULT_MUST_BLOCK);
177
178 if (m->laundry)
179 vm_pageout_steal_laundry(m, FALSE);
180
181 /*
182 * Don't worry about pages for which the kernel
183 * does not have any data.
184 */
185 if (m->absent || m->error || m->restart) {
186 if (m->error && should_flush && !VM_PAGE_WIRED(m)) {
187 /*
188 * dump the page, pager wants us to
189 * clean it up and there is no
190 * relevant data to return
191 */
192 return (MEMORY_OBJECT_LOCK_RESULT_MUST_FREE);
193 }
194 return (MEMORY_OBJECT_LOCK_RESULT_DONE);
195 }
196 assert(!m->fictitious);
197
198 if (VM_PAGE_WIRED(m)) {
199 /*
200 * The page is wired... just clean or return the page if needed.
201 * Wired pages don't get flushed or disconnected from the pmap.
202 */
203 if (memory_object_should_return_page(m, should_return))
204 return (MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN);
205
206 return (MEMORY_OBJECT_LOCK_RESULT_DONE);
207 }
208
209 if (should_flush) {
210 /*
211 * must do the pmap_disconnect before determining the
212 * need to return the page... otherwise it's possible
213 * for the page to go from the clean to the dirty state
214 * after we've made our decision
215 */
216 if (pmap_disconnect(m->phys_page) & VM_MEM_MODIFIED) {
217 SET_PAGE_DIRTY(m, FALSE);
218 }
219 } else {
220 /*
221 * If we are decreasing permission, do it now;
222 * let the fault handler take care of increases
223 * (pmap_page_protect may not increase protection).
224 */
225 if (prot != VM_PROT_NO_CHANGE)
226 pmap_page_protect(m->phys_page, VM_PROT_ALL & ~prot);
227 }
228 /*
229 * Handle returning dirty or precious pages
230 */
231 if (memory_object_should_return_page(m, should_return)) {
232 /*
233 * we use to do a pmap_disconnect here in support
234 * of memory_object_lock_request, but that routine
235 * no longer requires this... in any event, in
236 * our world, it would turn into a big noop since
237 * we don't lock the page in any way and as soon
238 * as we drop the object lock, the page can be
239 * faulted back into an address space
240 *
241 * if (!should_flush)
242 * pmap_disconnect(m->phys_page);
243 */
244 return (MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN);
245 }
246
247 /*
248 * Handle flushing clean pages
249 */
250 if (should_flush)
251 return (MEMORY_OBJECT_LOCK_RESULT_MUST_FREE);
252
253 /*
254 * we use to deactivate clean pages at this point,
255 * but we do not believe that an msync should change
256 * the 'age' of a page in the cache... here is the
257 * original comment and code concerning this...
258 *
259 * XXX Make clean but not flush a paging hint,
260 * and deactivate the pages. This is a hack
261 * because it overloads flush/clean with
262 * implementation-dependent meaning. This only
263 * happens to pages that are already clean.
264 *
265 * if (vm_page_deactivate_hint && (should_return != MEMORY_OBJECT_RETURN_NONE))
266 * return (MEMORY_OBJECT_LOCK_RESULT_MUST_DEACTIVATE);
267 */
268
269 return (MEMORY_OBJECT_LOCK_RESULT_DONE);
270 }
271
272
273
274 /*
275 * Routine: memory_object_lock_request [user interface]
276 *
277 * Description:
278 * Control use of the data associated with the given
279 * memory object. For each page in the given range,
280 * perform the following operations, in order:
281 * 1) restrict access to the page (disallow
282 * forms specified by "prot");
283 * 2) return data to the manager (if "should_return"
284 * is RETURN_DIRTY and the page is dirty, or
285 * "should_return" is RETURN_ALL and the page
286 * is either dirty or precious); and,
287 * 3) flush the cached copy (if "should_flush"
288 * is asserted).
289 * The set of pages is defined by a starting offset
290 * ("offset") and size ("size"). Only pages with the
291 * same page alignment as the starting offset are
292 * considered.
293 *
294 * A single acknowledgement is sent (to the "reply_to"
295 * port) when these actions are complete. If successful,
296 * the naked send right for reply_to is consumed.
297 */
298
299 kern_return_t
300 memory_object_lock_request(
301 memory_object_control_t control,
302 memory_object_offset_t offset,
303 memory_object_size_t size,
304 memory_object_offset_t * resid_offset,
305 int * io_errno,
306 memory_object_return_t should_return,
307 int flags,
308 vm_prot_t prot)
309 {
310 vm_object_t object;
311
312 /*
313 * Check for bogus arguments.
314 */
315 object = memory_object_control_to_vm_object(control);
316 if (object == VM_OBJECT_NULL)
317 return (KERN_INVALID_ARGUMENT);
318
319 if ((prot & ~VM_PROT_ALL) != 0 && prot != VM_PROT_NO_CHANGE)
320 return (KERN_INVALID_ARGUMENT);
321
322 size = round_page_64(size);
323
324 /*
325 * Lock the object, and acquire a paging reference to
326 * prevent the memory_object reference from being released.
327 */
328 vm_object_lock(object);
329 vm_object_paging_begin(object);
330
331 if (flags & MEMORY_OBJECT_DATA_FLUSH_ALL) {
332 if ((should_return != MEMORY_OBJECT_RETURN_NONE) || offset || object->copy) {
333 flags &= ~MEMORY_OBJECT_DATA_FLUSH_ALL;
334 flags |= MEMORY_OBJECT_DATA_FLUSH;
335 }
336 }
337 offset -= object->paging_offset;
338
339 if (flags & MEMORY_OBJECT_DATA_FLUSH_ALL)
340 vm_object_reap_pages(object, REAP_DATA_FLUSH);
341 else
342 (void)vm_object_update(object, offset, size, resid_offset,
343 io_errno, should_return, flags, prot);
344
345 vm_object_paging_end(object);
346 vm_object_unlock(object);
347
348 return (KERN_SUCCESS);
349 }
350
351 /*
352 * memory_object_release_name: [interface]
353 *
354 * Enforces name semantic on memory_object reference count decrement
355 * This routine should not be called unless the caller holds a name
356 * reference gained through the memory_object_named_create or the
357 * memory_object_rename call.
358 * If the TERMINATE_IDLE flag is set, the call will return if the
359 * reference count is not 1. i.e. idle with the only remaining reference
360 * being the name.
361 * If the decision is made to proceed the name field flag is set to
362 * false and the reference count is decremented. If the RESPECT_CACHE
363 * flag is set and the reference count has gone to zero, the
364 * memory_object is checked to see if it is cacheable otherwise when
365 * the reference count is zero, it is simply terminated.
366 */
367
368 kern_return_t
369 memory_object_release_name(
370 memory_object_control_t control,
371 int flags)
372 {
373 vm_object_t object;
374
375 object = memory_object_control_to_vm_object(control);
376 if (object == VM_OBJECT_NULL)
377 return (KERN_INVALID_ARGUMENT);
378
379 return vm_object_release_name(object, flags);
380 }
381
382
383
384 /*
385 * Routine: memory_object_destroy [user interface]
386 * Purpose:
387 * Shut down a memory object, despite the
388 * presence of address map (or other) references
389 * to the vm_object.
390 */
391 kern_return_t
392 memory_object_destroy(
393 memory_object_control_t control,
394 kern_return_t reason)
395 {
396 vm_object_t object;
397
398 object = memory_object_control_to_vm_object(control);
399 if (object == VM_OBJECT_NULL)
400 return (KERN_INVALID_ARGUMENT);
401
402 return (vm_object_destroy(object, reason));
403 }
404
405 /*
406 * Routine: vm_object_sync
407 *
408 * Kernel internal function to synch out pages in a given
409 * range within an object to its memory manager. Much the
410 * same as memory_object_lock_request but page protection
411 * is not changed.
412 *
413 * If the should_flush and should_return flags are true pages
414 * are flushed, that is dirty & precious pages are written to
415 * the memory manager and then discarded. If should_return
416 * is false, only precious pages are returned to the memory
417 * manager.
418 *
419 * If should flush is false and should_return true, the memory
420 * manager's copy of the pages is updated. If should_return
421 * is also false, only the precious pages are updated. This
422 * last option is of limited utility.
423 *
424 * Returns:
425 * FALSE if no pages were returned to the pager
426 * TRUE otherwise.
427 */
428
429 boolean_t
430 vm_object_sync(
431 vm_object_t object,
432 vm_object_offset_t offset,
433 vm_object_size_t size,
434 boolean_t should_flush,
435 boolean_t should_return,
436 boolean_t should_iosync)
437 {
438 boolean_t rv;
439 int flags;
440
441 XPR(XPR_VM_OBJECT,
442 "vm_o_sync, object 0x%X, offset 0x%X size 0x%x flush %d rtn %d\n",
443 object, offset, size, should_flush, should_return);
444
445 /*
446 * Lock the object, and acquire a paging reference to
447 * prevent the memory_object and control ports from
448 * being destroyed.
449 */
450 vm_object_lock(object);
451 vm_object_paging_begin(object);
452
453 if (should_flush) {
454 flags = MEMORY_OBJECT_DATA_FLUSH;
455 /*
456 * This flush is from an msync(), not a truncate(), so the
457 * contents of the file are not affected.
458 * MEMORY_OBECT_DATA_NO_CHANGE lets vm_object_update() know
459 * that the data is not changed and that there's no need to
460 * push the old contents to a copy object.
461 */
462 flags |= MEMORY_OBJECT_DATA_NO_CHANGE;
463 } else
464 flags = 0;
465
466 if (should_iosync)
467 flags |= MEMORY_OBJECT_IO_SYNC;
468
469 rv = vm_object_update(object, offset, (vm_object_size_t)size, NULL, NULL,
470 (should_return) ?
471 MEMORY_OBJECT_RETURN_ALL :
472 MEMORY_OBJECT_RETURN_NONE,
473 flags,
474 VM_PROT_NO_CHANGE);
475
476
477 vm_object_paging_end(object);
478 vm_object_unlock(object);
479 return rv;
480 }
481
482
483
484 #define LIST_REQ_PAGEOUT_PAGES(object, data_cnt, po, ro, ioerr, iosync) \
485 MACRO_BEGIN \
486 \
487 int upl_flags; \
488 memory_object_t pager; \
489 \
490 if (object->object_slid) { \
491 panic("Objects with slid pages not allowed\n"); \
492 } \
493 \
494 if ((pager = (object)->pager) != MEMORY_OBJECT_NULL) { \
495 vm_object_paging_begin(object); \
496 vm_object_unlock(object); \
497 \
498 if (iosync) \
499 upl_flags = UPL_MSYNC | UPL_IOSYNC; \
500 else \
501 upl_flags = UPL_MSYNC; \
502 \
503 (void) memory_object_data_return(pager, \
504 po, \
505 (memory_object_cluster_size_t)data_cnt, \
506 ro, \
507 ioerr, \
508 FALSE, \
509 FALSE, \
510 upl_flags); \
511 \
512 vm_object_lock(object); \
513 vm_object_paging_end(object); \
514 } \
515 MACRO_END
516
517
518
519 static int
520 vm_object_update_extent(
521 vm_object_t object,
522 vm_object_offset_t offset,
523 vm_object_offset_t offset_end,
524 vm_object_offset_t *offset_resid,
525 int *io_errno,
526 boolean_t should_flush,
527 memory_object_return_t should_return,
528 boolean_t should_iosync,
529 vm_prot_t prot)
530 {
531 vm_page_t m;
532 int retval = 0;
533 vm_object_offset_t paging_offset = 0;
534 vm_object_offset_t next_offset = offset;
535 memory_object_lock_result_t page_lock_result;
536 memory_object_cluster_size_t data_cnt = 0;
537 struct vm_page_delayed_work dw_array[DEFAULT_DELAYED_WORK_LIMIT];
538 struct vm_page_delayed_work *dwp;
539 int dw_count;
540 int dw_limit;
541
542 dwp = &dw_array[0];
543 dw_count = 0;
544 dw_limit = DELAYED_WORK_LIMIT(DEFAULT_DELAYED_WORK_LIMIT);
545
546 for (;
547 offset < offset_end && object->resident_page_count;
548 offset += PAGE_SIZE_64) {
549
550 /*
551 * Limit the number of pages to be cleaned at once to a contiguous
552 * run, or at most MAX_UPL_TRANSFER_BYTES
553 */
554 if (data_cnt) {
555 if ((data_cnt >= MAX_UPL_TRANSFER_BYTES) || (next_offset != offset)) {
556
557 if (dw_count) {
558 vm_page_do_delayed_work(object, &dw_array[0], dw_count);
559 dwp = &dw_array[0];
560 dw_count = 0;
561 }
562 LIST_REQ_PAGEOUT_PAGES(object, data_cnt,
563 paging_offset, offset_resid, io_errno, should_iosync);
564 data_cnt = 0;
565 }
566 }
567 while ((m = vm_page_lookup(object, offset)) != VM_PAGE_NULL) {
568
569 dwp->dw_mask = 0;
570
571 page_lock_result = memory_object_lock_page(m, should_return, should_flush, prot);
572
573 if (data_cnt && page_lock_result != MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN) {
574 /*
575 * End of a run of dirty/precious pages.
576 */
577 if (dw_count) {
578 vm_page_do_delayed_work(object, &dw_array[0], dw_count);
579 dwp = &dw_array[0];
580 dw_count = 0;
581 }
582 LIST_REQ_PAGEOUT_PAGES(object, data_cnt,
583 paging_offset, offset_resid, io_errno, should_iosync);
584 /*
585 * LIST_REQ_PAGEOUT_PAGES will drop the object lock which will
586 * allow the state of page 'm' to change... we need to re-lookup
587 * the current offset
588 */
589 data_cnt = 0;
590 continue;
591 }
592
593 switch (page_lock_result) {
594
595 case MEMORY_OBJECT_LOCK_RESULT_DONE:
596 break;
597
598 case MEMORY_OBJECT_LOCK_RESULT_MUST_FREE:
599 dwp->dw_mask |= DW_vm_page_free;
600 break;
601
602 case MEMORY_OBJECT_LOCK_RESULT_MUST_BLOCK:
603 PAGE_SLEEP(object, m, THREAD_UNINT);
604 continue;
605
606 case MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN:
607 if (data_cnt == 0)
608 paging_offset = offset;
609
610 data_cnt += PAGE_SIZE;
611 next_offset = offset + PAGE_SIZE_64;
612
613 /*
614 * wired pages shouldn't be flushed and
615 * since they aren't on any queue,
616 * no need to remove them
617 */
618 if (!VM_PAGE_WIRED(m)) {
619
620 if (should_flush) {
621 /*
622 * add additional state for the flush
623 */
624 m->pageout = TRUE;
625 }
626 /*
627 * we use to remove the page from the queues at this
628 * point, but we do not believe that an msync
629 * should cause the 'age' of a page to be changed
630 *
631 * else
632 * dwp->dw_mask |= DW_VM_PAGE_QUEUES_REMOVE;
633 */
634 }
635 retval = 1;
636 break;
637 }
638 if (dwp->dw_mask) {
639 VM_PAGE_ADD_DELAYED_WORK(dwp, m, dw_count);
640
641 if (dw_count >= dw_limit) {
642 vm_page_do_delayed_work(object, &dw_array[0], dw_count);
643 dwp = &dw_array[0];
644 dw_count = 0;
645 }
646 }
647 break;
648 }
649 }
650 /*
651 * We have completed the scan for applicable pages.
652 * Clean any pages that have been saved.
653 */
654 if (dw_count)
655 vm_page_do_delayed_work(object, &dw_array[0], dw_count);
656
657 if (data_cnt) {
658 LIST_REQ_PAGEOUT_PAGES(object, data_cnt,
659 paging_offset, offset_resid, io_errno, should_iosync);
660 }
661 return (retval);
662 }
663
664
665
666 /*
667 * Routine: vm_object_update
668 * Description:
669 * Work function for m_o_lock_request(), vm_o_sync().
670 *
671 * Called with object locked and paging ref taken.
672 */
673 kern_return_t
674 vm_object_update(
675 vm_object_t object,
676 vm_object_offset_t offset,
677 vm_object_size_t size,
678 vm_object_offset_t *resid_offset,
679 int *io_errno,
680 memory_object_return_t should_return,
681 int flags,
682 vm_prot_t protection)
683 {
684 vm_object_t copy_object = VM_OBJECT_NULL;
685 boolean_t data_returned = FALSE;
686 boolean_t update_cow;
687 boolean_t should_flush = (flags & MEMORY_OBJECT_DATA_FLUSH) ? TRUE : FALSE;
688 boolean_t should_iosync = (flags & MEMORY_OBJECT_IO_SYNC) ? TRUE : FALSE;
689 vm_fault_return_t result;
690 int num_of_extents;
691 int n;
692 #define MAX_EXTENTS 8
693 #define EXTENT_SIZE (1024 * 1024 * 256)
694 #define RESIDENT_LIMIT (1024 * 32)
695 struct extent {
696 vm_object_offset_t e_base;
697 vm_object_offset_t e_min;
698 vm_object_offset_t e_max;
699 } extents[MAX_EXTENTS];
700
701 /*
702 * To avoid blocking while scanning for pages, save
703 * dirty pages to be cleaned all at once.
704 *
705 * XXXO A similar strategy could be used to limit the
706 * number of times that a scan must be restarted for
707 * other reasons. Those pages that would require blocking
708 * could be temporarily collected in another list, or
709 * their offsets could be recorded in a small array.
710 */
711
712 /*
713 * XXX NOTE: May want to consider converting this to a page list
714 * XXX vm_map_copy interface. Need to understand object
715 * XXX coalescing implications before doing so.
716 */
717
718 update_cow = ((flags & MEMORY_OBJECT_DATA_FLUSH)
719 && (!(flags & MEMORY_OBJECT_DATA_NO_CHANGE) &&
720 !(flags & MEMORY_OBJECT_DATA_PURGE)))
721 || (flags & MEMORY_OBJECT_COPY_SYNC);
722
723 if (update_cow || (flags & (MEMORY_OBJECT_DATA_PURGE | MEMORY_OBJECT_DATA_SYNC))) {
724 int collisions = 0;
725
726 while ((copy_object = object->copy) != VM_OBJECT_NULL) {
727 /*
728 * need to do a try here since we're swimming upstream
729 * against the normal lock ordering... however, we need
730 * to hold the object stable until we gain control of the
731 * copy object so we have to be careful how we approach this
732 */
733 if (vm_object_lock_try(copy_object)) {
734 /*
735 * we 'won' the lock on the copy object...
736 * no need to hold the object lock any longer...
737 * take a real reference on the copy object because
738 * we're going to call vm_fault_page on it which may
739 * under certain conditions drop the lock and the paging
740 * reference we're about to take... the reference
741 * will keep the copy object from going away if that happens
742 */
743 vm_object_unlock(object);
744 vm_object_reference_locked(copy_object);
745 break;
746 }
747 vm_object_unlock(object);
748
749 collisions++;
750 mutex_pause(collisions);
751
752 vm_object_lock(object);
753 }
754 }
755 if ((copy_object != VM_OBJECT_NULL && update_cow) || (flags & MEMORY_OBJECT_DATA_SYNC)) {
756 vm_map_size_t i;
757 vm_map_size_t copy_size;
758 vm_map_offset_t copy_offset;
759 vm_prot_t prot;
760 vm_page_t page;
761 vm_page_t top_page;
762 kern_return_t error = 0;
763 struct vm_object_fault_info fault_info;
764
765 if (copy_object != VM_OBJECT_NULL) {
766 /*
767 * translate offset with respect to shadow's offset
768 */
769 copy_offset = (offset >= copy_object->vo_shadow_offset) ?
770 (vm_map_offset_t)(offset - copy_object->vo_shadow_offset) :
771 (vm_map_offset_t) 0;
772
773 if (copy_offset > copy_object->vo_size)
774 copy_offset = copy_object->vo_size;
775
776 /*
777 * clip size with respect to shadow offset
778 */
779 if (offset >= copy_object->vo_shadow_offset) {
780 copy_size = size;
781 } else if (size >= copy_object->vo_shadow_offset - offset) {
782 copy_size = size - (copy_object->vo_shadow_offset - offset);
783 } else {
784 copy_size = 0;
785 }
786
787 if (copy_offset + copy_size > copy_object->vo_size) {
788 if (copy_object->vo_size >= copy_offset) {
789 copy_size = copy_object->vo_size - copy_offset;
790 } else {
791 copy_size = 0;
792 }
793 }
794 copy_size+=copy_offset;
795
796 } else {
797 copy_object = object;
798
799 copy_size = offset + size;
800 copy_offset = offset;
801 }
802 fault_info.interruptible = THREAD_UNINT;
803 fault_info.behavior = VM_BEHAVIOR_SEQUENTIAL;
804 fault_info.user_tag = 0;
805 fault_info.pmap_options = 0;
806 fault_info.lo_offset = copy_offset;
807 fault_info.hi_offset = copy_size;
808 fault_info.no_cache = FALSE;
809 fault_info.stealth = TRUE;
810 fault_info.io_sync = FALSE;
811 fault_info.cs_bypass = FALSE;
812 fault_info.mark_zf_absent = FALSE;
813 fault_info.batch_pmap_op = FALSE;
814
815 vm_object_paging_begin(copy_object);
816
817 for (i = copy_offset; i < copy_size; i += PAGE_SIZE) {
818 RETRY_COW_OF_LOCK_REQUEST:
819 fault_info.cluster_size = (vm_size_t) (copy_size - i);
820 assert(fault_info.cluster_size == copy_size - i);
821
822 prot = VM_PROT_WRITE|VM_PROT_READ;
823 page = VM_PAGE_NULL;
824 result = vm_fault_page(copy_object, i,
825 VM_PROT_WRITE|VM_PROT_READ,
826 FALSE,
827 FALSE, /* page not looked up */
828 &prot,
829 &page,
830 &top_page,
831 (int *)0,
832 &error,
833 FALSE,
834 FALSE, &fault_info);
835
836 switch (result) {
837 case VM_FAULT_SUCCESS:
838 if (top_page) {
839 vm_fault_cleanup(
840 page->object, top_page);
841 vm_object_lock(copy_object);
842 vm_object_paging_begin(copy_object);
843 }
844 if (!page->active &&
845 !page->inactive &&
846 !page->throttled) {
847 vm_page_lockspin_queues();
848 if (!page->active &&
849 !page->inactive &&
850 !page->throttled)
851 vm_page_deactivate(page);
852 vm_page_unlock_queues();
853 }
854 PAGE_WAKEUP_DONE(page);
855 break;
856 case VM_FAULT_RETRY:
857 prot = VM_PROT_WRITE|VM_PROT_READ;
858 vm_object_lock(copy_object);
859 vm_object_paging_begin(copy_object);
860 goto RETRY_COW_OF_LOCK_REQUEST;
861 case VM_FAULT_INTERRUPTED:
862 prot = VM_PROT_WRITE|VM_PROT_READ;
863 vm_object_lock(copy_object);
864 vm_object_paging_begin(copy_object);
865 goto RETRY_COW_OF_LOCK_REQUEST;
866 case VM_FAULT_MEMORY_SHORTAGE:
867 VM_PAGE_WAIT();
868 prot = VM_PROT_WRITE|VM_PROT_READ;
869 vm_object_lock(copy_object);
870 vm_object_paging_begin(copy_object);
871 goto RETRY_COW_OF_LOCK_REQUEST;
872 case VM_FAULT_SUCCESS_NO_VM_PAGE:
873 /* success but no VM page: fail */
874 vm_object_paging_end(copy_object);
875 vm_object_unlock(copy_object);
876 /*FALLTHROUGH*/
877 case VM_FAULT_MEMORY_ERROR:
878 if (object != copy_object)
879 vm_object_deallocate(copy_object);
880 vm_object_lock(object);
881 goto BYPASS_COW_COPYIN;
882 default:
883 panic("vm_object_update: unexpected error 0x%x"
884 " from vm_fault_page()\n", result);
885 }
886
887 }
888 vm_object_paging_end(copy_object);
889 }
890 if ((flags & (MEMORY_OBJECT_DATA_SYNC | MEMORY_OBJECT_COPY_SYNC))) {
891 if (copy_object != VM_OBJECT_NULL && copy_object != object) {
892 vm_object_unlock(copy_object);
893 vm_object_deallocate(copy_object);
894 vm_object_lock(object);
895 }
896 return KERN_SUCCESS;
897 }
898 if (copy_object != VM_OBJECT_NULL && copy_object != object) {
899 if ((flags & MEMORY_OBJECT_DATA_PURGE)) {
900 copy_object->shadow_severed = TRUE;
901 copy_object->shadowed = FALSE;
902 copy_object->shadow = NULL;
903 /*
904 * delete the ref the COW was holding on the target object
905 */
906 vm_object_deallocate(object);
907 }
908 vm_object_unlock(copy_object);
909 vm_object_deallocate(copy_object);
910 vm_object_lock(object);
911 }
912 BYPASS_COW_COPYIN:
913
914 /*
915 * when we have a really large range to check relative
916 * to the number of actual resident pages, we'd like
917 * to use the resident page list to drive our checks
918 * however, the object lock will get dropped while processing
919 * the page which means the resident queue can change which
920 * means we can't walk the queue as we process the pages
921 * we also want to do the processing in offset order to allow
922 * 'runs' of pages to be collected if we're being told to
923 * flush to disk... the resident page queue is NOT ordered.
924 *
925 * a temporary solution (until we figure out how to deal with
926 * large address spaces more generically) is to pre-flight
927 * the resident page queue (if it's small enough) and develop
928 * a collection of extents (that encompass actual resident pages)
929 * to visit. This will at least allow us to deal with some of the
930 * more pathological cases in a more efficient manner. The current
931 * worst case (a single resident page at the end of an extremely large
932 * range) can take minutes to complete for ranges in the terrabyte
933 * category... since this routine is called when truncating a file,
934 * and we currently support files up to 16 Tbytes in size, this
935 * is not a theoretical problem
936 */
937
938 if ((object->resident_page_count < RESIDENT_LIMIT) &&
939 (atop_64(size) > (unsigned)(object->resident_page_count/(8 * MAX_EXTENTS)))) {
940 vm_page_t next;
941 vm_object_offset_t start;
942 vm_object_offset_t end;
943 vm_object_size_t e_mask;
944 vm_page_t m;
945
946 start = offset;
947 end = offset + size;
948 num_of_extents = 0;
949 e_mask = ~((vm_object_size_t)(EXTENT_SIZE - 1));
950
951 m = (vm_page_t) queue_first(&object->memq);
952
953 while (!queue_end(&object->memq, (queue_entry_t) m)) {
954 next = (vm_page_t) queue_next(&m->listq);
955
956 if ((m->offset >= start) && (m->offset < end)) {
957 /*
958 * this is a page we're interested in
959 * try to fit it into a current extent
960 */
961 for (n = 0; n < num_of_extents; n++) {
962 if ((m->offset & e_mask) == extents[n].e_base) {
963 /*
964 * use (PAGE_SIZE - 1) to determine the
965 * max offset so that we don't wrap if
966 * we're at the last page of the space
967 */
968 if (m->offset < extents[n].e_min)
969 extents[n].e_min = m->offset;
970 else if ((m->offset + (PAGE_SIZE - 1)) > extents[n].e_max)
971 extents[n].e_max = m->offset + (PAGE_SIZE - 1);
972 break;
973 }
974 }
975 if (n == num_of_extents) {
976 /*
977 * didn't find a current extent that can encompass
978 * this page
979 */
980 if (n < MAX_EXTENTS) {
981 /*
982 * if we still have room,
983 * create a new extent
984 */
985 extents[n].e_base = m->offset & e_mask;
986 extents[n].e_min = m->offset;
987 extents[n].e_max = m->offset + (PAGE_SIZE - 1);
988
989 num_of_extents++;
990 } else {
991 /*
992 * no room to create a new extent...
993 * fall back to a single extent based
994 * on the min and max page offsets
995 * we find in the range we're interested in...
996 * first, look through the extent list and
997 * develop the overall min and max for the
998 * pages we've looked at up to this point
999 */
1000 for (n = 1; n < num_of_extents; n++) {
1001 if (extents[n].e_min < extents[0].e_min)
1002 extents[0].e_min = extents[n].e_min;
1003 if (extents[n].e_max > extents[0].e_max)
1004 extents[0].e_max = extents[n].e_max;
1005 }
1006 /*
1007 * now setup to run through the remaining pages
1008 * to determine the overall min and max
1009 * offset for the specified range
1010 */
1011 extents[0].e_base = 0;
1012 e_mask = 0;
1013 num_of_extents = 1;
1014
1015 /*
1016 * by continuing, we'll reprocess the
1017 * page that forced us to abandon trying
1018 * to develop multiple extents
1019 */
1020 continue;
1021 }
1022 }
1023 }
1024 m = next;
1025 }
1026 } else {
1027 extents[0].e_min = offset;
1028 extents[0].e_max = offset + (size - 1);
1029
1030 num_of_extents = 1;
1031 }
1032 for (n = 0; n < num_of_extents; n++) {
1033 if (vm_object_update_extent(object, extents[n].e_min, extents[n].e_max, resid_offset, io_errno,
1034 should_flush, should_return, should_iosync, protection))
1035 data_returned = TRUE;
1036 }
1037 return (data_returned);
1038 }
1039
1040
1041 /*
1042 * Routine: memory_object_synchronize_completed [user interface]
1043 *
1044 * Tell kernel that previously synchronized data
1045 * (memory_object_synchronize) has been queue or placed on the
1046 * backing storage.
1047 *
1048 * Note: there may be multiple synchronize requests for a given
1049 * memory object outstanding but they will not overlap.
1050 */
1051
1052 kern_return_t
1053 memory_object_synchronize_completed(
1054 memory_object_control_t control,
1055 memory_object_offset_t offset,
1056 memory_object_size_t length)
1057 {
1058 vm_object_t object;
1059 msync_req_t msr;
1060
1061 object = memory_object_control_to_vm_object(control);
1062
1063 XPR(XPR_MEMORY_OBJECT,
1064 "m_o_sync_completed, object 0x%X, offset 0x%X length 0x%X\n",
1065 object, offset, length, 0, 0);
1066
1067 /*
1068 * Look for bogus arguments
1069 */
1070
1071 if (object == VM_OBJECT_NULL)
1072 return (KERN_INVALID_ARGUMENT);
1073
1074 vm_object_lock(object);
1075
1076 /*
1077 * search for sync request structure
1078 */
1079 queue_iterate(&object->msr_q, msr, msync_req_t, msr_q) {
1080 if (msr->offset == offset && msr->length == length) {
1081 queue_remove(&object->msr_q, msr, msync_req_t, msr_q);
1082 break;
1083 }
1084 }/* queue_iterate */
1085
1086 if (queue_end(&object->msr_q, (queue_entry_t)msr)) {
1087 vm_object_unlock(object);
1088 return KERN_INVALID_ARGUMENT;
1089 }
1090
1091 msr_lock(msr);
1092 vm_object_unlock(object);
1093 msr->flag = VM_MSYNC_DONE;
1094 msr_unlock(msr);
1095 thread_wakeup((event_t) msr);
1096
1097 return KERN_SUCCESS;
1098 }/* memory_object_synchronize_completed */
1099
1100 static kern_return_t
1101 vm_object_set_attributes_common(
1102 vm_object_t object,
1103 boolean_t may_cache,
1104 memory_object_copy_strategy_t copy_strategy,
1105 boolean_t temporary,
1106 __unused boolean_t silent_overwrite,
1107 boolean_t advisory_pageout)
1108 {
1109 boolean_t object_became_ready;
1110
1111 XPR(XPR_MEMORY_OBJECT,
1112 "m_o_set_attr_com, object 0x%X flg %x strat %d\n",
1113 object, (may_cache&1)|((temporary&1)<1), copy_strategy, 0, 0);
1114
1115 if (object == VM_OBJECT_NULL)
1116 return(KERN_INVALID_ARGUMENT);
1117
1118 /*
1119 * Verify the attributes of importance
1120 */
1121
1122 switch(copy_strategy) {
1123 case MEMORY_OBJECT_COPY_NONE:
1124 case MEMORY_OBJECT_COPY_DELAY:
1125 break;
1126 default:
1127 return(KERN_INVALID_ARGUMENT);
1128 }
1129
1130 if (may_cache)
1131 may_cache = TRUE;
1132 if (temporary)
1133 temporary = TRUE;
1134
1135 vm_object_lock(object);
1136
1137 /*
1138 * Copy the attributes
1139 */
1140 assert(!object->internal);
1141 object_became_ready = !object->pager_ready;
1142 object->copy_strategy = copy_strategy;
1143 object->can_persist = may_cache;
1144 object->temporary = temporary;
1145 // object->silent_overwrite = silent_overwrite;
1146 object->advisory_pageout = advisory_pageout;
1147
1148 /*
1149 * Wake up anyone waiting for the ready attribute
1150 * to become asserted.
1151 */
1152
1153 if (object_became_ready) {
1154 object->pager_ready = TRUE;
1155 vm_object_wakeup(object, VM_OBJECT_EVENT_PAGER_READY);
1156 }
1157
1158 vm_object_unlock(object);
1159
1160 return(KERN_SUCCESS);
1161 }
1162
1163 /*
1164 * Set the memory object attribute as provided.
1165 *
1166 * XXX This routine cannot be completed until the vm_msync, clean
1167 * in place, and cluster work is completed. See ifdef notyet
1168 * below and note that vm_object_set_attributes_common()
1169 * may have to be expanded.
1170 */
1171 kern_return_t
1172 memory_object_change_attributes(
1173 memory_object_control_t control,
1174 memory_object_flavor_t flavor,
1175 memory_object_info_t attributes,
1176 mach_msg_type_number_t count)
1177 {
1178 vm_object_t object;
1179 kern_return_t result = KERN_SUCCESS;
1180 boolean_t temporary;
1181 boolean_t may_cache;
1182 boolean_t invalidate;
1183 memory_object_copy_strategy_t copy_strategy;
1184 boolean_t silent_overwrite;
1185 boolean_t advisory_pageout;
1186
1187 object = memory_object_control_to_vm_object(control);
1188 if (object == VM_OBJECT_NULL)
1189 return (KERN_INVALID_ARGUMENT);
1190
1191 vm_object_lock(object);
1192
1193 temporary = object->temporary;
1194 may_cache = object->can_persist;
1195 copy_strategy = object->copy_strategy;
1196 // silent_overwrite = object->silent_overwrite;
1197 silent_overwrite = FALSE;
1198 advisory_pageout = object->advisory_pageout;
1199 #if notyet
1200 invalidate = object->invalidate;
1201 #endif
1202 vm_object_unlock(object);
1203
1204 switch (flavor) {
1205 case OLD_MEMORY_OBJECT_BEHAVIOR_INFO:
1206 {
1207 old_memory_object_behave_info_t behave;
1208
1209 if (count != OLD_MEMORY_OBJECT_BEHAVE_INFO_COUNT) {
1210 result = KERN_INVALID_ARGUMENT;
1211 break;
1212 }
1213
1214 behave = (old_memory_object_behave_info_t) attributes;
1215
1216 temporary = behave->temporary;
1217 invalidate = behave->invalidate;
1218 copy_strategy = behave->copy_strategy;
1219
1220 break;
1221 }
1222
1223 case MEMORY_OBJECT_BEHAVIOR_INFO:
1224 {
1225 memory_object_behave_info_t behave;
1226
1227 if (count != MEMORY_OBJECT_BEHAVE_INFO_COUNT) {
1228 result = KERN_INVALID_ARGUMENT;
1229 break;
1230 }
1231
1232 behave = (memory_object_behave_info_t) attributes;
1233
1234 temporary = behave->temporary;
1235 invalidate = behave->invalidate;
1236 copy_strategy = behave->copy_strategy;
1237 silent_overwrite = behave->silent_overwrite;
1238 advisory_pageout = behave->advisory_pageout;
1239 break;
1240 }
1241
1242 case MEMORY_OBJECT_PERFORMANCE_INFO:
1243 {
1244 memory_object_perf_info_t perf;
1245
1246 if (count != MEMORY_OBJECT_PERF_INFO_COUNT) {
1247 result = KERN_INVALID_ARGUMENT;
1248 break;
1249 }
1250
1251 perf = (memory_object_perf_info_t) attributes;
1252
1253 may_cache = perf->may_cache;
1254
1255 break;
1256 }
1257
1258 case OLD_MEMORY_OBJECT_ATTRIBUTE_INFO:
1259 {
1260 old_memory_object_attr_info_t attr;
1261
1262 if (count != OLD_MEMORY_OBJECT_ATTR_INFO_COUNT) {
1263 result = KERN_INVALID_ARGUMENT;
1264 break;
1265 }
1266
1267 attr = (old_memory_object_attr_info_t) attributes;
1268
1269 may_cache = attr->may_cache;
1270 copy_strategy = attr->copy_strategy;
1271
1272 break;
1273 }
1274
1275 case MEMORY_OBJECT_ATTRIBUTE_INFO:
1276 {
1277 memory_object_attr_info_t attr;
1278
1279 if (count != MEMORY_OBJECT_ATTR_INFO_COUNT) {
1280 result = KERN_INVALID_ARGUMENT;
1281 break;
1282 }
1283
1284 attr = (memory_object_attr_info_t) attributes;
1285
1286 copy_strategy = attr->copy_strategy;
1287 may_cache = attr->may_cache_object;
1288 temporary = attr->temporary;
1289
1290 break;
1291 }
1292
1293 default:
1294 result = KERN_INVALID_ARGUMENT;
1295 break;
1296 }
1297
1298 if (result != KERN_SUCCESS)
1299 return(result);
1300
1301 if (copy_strategy == MEMORY_OBJECT_COPY_TEMPORARY) {
1302 copy_strategy = MEMORY_OBJECT_COPY_DELAY;
1303 temporary = TRUE;
1304 } else {
1305 temporary = FALSE;
1306 }
1307
1308 /*
1309 * XXX may_cache may become a tri-valued variable to handle
1310 * XXX uncache if not in use.
1311 */
1312 return (vm_object_set_attributes_common(object,
1313 may_cache,
1314 copy_strategy,
1315 temporary,
1316 silent_overwrite,
1317 advisory_pageout));
1318 }
1319
1320 kern_return_t
1321 memory_object_get_attributes(
1322 memory_object_control_t control,
1323 memory_object_flavor_t flavor,
1324 memory_object_info_t attributes, /* pointer to OUT array */
1325 mach_msg_type_number_t *count) /* IN/OUT */
1326 {
1327 kern_return_t ret = KERN_SUCCESS;
1328 vm_object_t object;
1329
1330 object = memory_object_control_to_vm_object(control);
1331 if (object == VM_OBJECT_NULL)
1332 return (KERN_INVALID_ARGUMENT);
1333
1334 vm_object_lock(object);
1335
1336 switch (flavor) {
1337 case OLD_MEMORY_OBJECT_BEHAVIOR_INFO:
1338 {
1339 old_memory_object_behave_info_t behave;
1340
1341 if (*count < OLD_MEMORY_OBJECT_BEHAVE_INFO_COUNT) {
1342 ret = KERN_INVALID_ARGUMENT;
1343 break;
1344 }
1345
1346 behave = (old_memory_object_behave_info_t) attributes;
1347 behave->copy_strategy = object->copy_strategy;
1348 behave->temporary = object->temporary;
1349 #if notyet /* remove when vm_msync complies and clean in place fini */
1350 behave->invalidate = object->invalidate;
1351 #else
1352 behave->invalidate = FALSE;
1353 #endif
1354
1355 *count = OLD_MEMORY_OBJECT_BEHAVE_INFO_COUNT;
1356 break;
1357 }
1358
1359 case MEMORY_OBJECT_BEHAVIOR_INFO:
1360 {
1361 memory_object_behave_info_t behave;
1362
1363 if (*count < MEMORY_OBJECT_BEHAVE_INFO_COUNT) {
1364 ret = KERN_INVALID_ARGUMENT;
1365 break;
1366 }
1367
1368 behave = (memory_object_behave_info_t) attributes;
1369 behave->copy_strategy = object->copy_strategy;
1370 behave->temporary = object->temporary;
1371 #if notyet /* remove when vm_msync complies and clean in place fini */
1372 behave->invalidate = object->invalidate;
1373 #else
1374 behave->invalidate = FALSE;
1375 #endif
1376 behave->advisory_pageout = object->advisory_pageout;
1377 // behave->silent_overwrite = object->silent_overwrite;
1378 behave->silent_overwrite = FALSE;
1379 *count = MEMORY_OBJECT_BEHAVE_INFO_COUNT;
1380 break;
1381 }
1382
1383 case MEMORY_OBJECT_PERFORMANCE_INFO:
1384 {
1385 memory_object_perf_info_t perf;
1386
1387 if (*count < MEMORY_OBJECT_PERF_INFO_COUNT) {
1388 ret = KERN_INVALID_ARGUMENT;
1389 break;
1390 }
1391
1392 perf = (memory_object_perf_info_t) attributes;
1393 perf->cluster_size = PAGE_SIZE;
1394 perf->may_cache = object->can_persist;
1395
1396 *count = MEMORY_OBJECT_PERF_INFO_COUNT;
1397 break;
1398 }
1399
1400 case OLD_MEMORY_OBJECT_ATTRIBUTE_INFO:
1401 {
1402 old_memory_object_attr_info_t attr;
1403
1404 if (*count < OLD_MEMORY_OBJECT_ATTR_INFO_COUNT) {
1405 ret = KERN_INVALID_ARGUMENT;
1406 break;
1407 }
1408
1409 attr = (old_memory_object_attr_info_t) attributes;
1410 attr->may_cache = object->can_persist;
1411 attr->copy_strategy = object->copy_strategy;
1412
1413 *count = OLD_MEMORY_OBJECT_ATTR_INFO_COUNT;
1414 break;
1415 }
1416
1417 case MEMORY_OBJECT_ATTRIBUTE_INFO:
1418 {
1419 memory_object_attr_info_t attr;
1420
1421 if (*count < MEMORY_OBJECT_ATTR_INFO_COUNT) {
1422 ret = KERN_INVALID_ARGUMENT;
1423 break;
1424 }
1425
1426 attr = (memory_object_attr_info_t) attributes;
1427 attr->copy_strategy = object->copy_strategy;
1428 attr->cluster_size = PAGE_SIZE;
1429 attr->may_cache_object = object->can_persist;
1430 attr->temporary = object->temporary;
1431
1432 *count = MEMORY_OBJECT_ATTR_INFO_COUNT;
1433 break;
1434 }
1435
1436 default:
1437 ret = KERN_INVALID_ARGUMENT;
1438 break;
1439 }
1440
1441 vm_object_unlock(object);
1442
1443 return(ret);
1444 }
1445
1446
1447 kern_return_t
1448 memory_object_iopl_request(
1449 ipc_port_t port,
1450 memory_object_offset_t offset,
1451 upl_size_t *upl_size,
1452 upl_t *upl_ptr,
1453 upl_page_info_array_t user_page_list,
1454 unsigned int *page_list_count,
1455 int *flags)
1456 {
1457 vm_object_t object;
1458 kern_return_t ret;
1459 int caller_flags;
1460
1461 caller_flags = *flags;
1462
1463 if (caller_flags & ~UPL_VALID_FLAGS) {
1464 /*
1465 * For forward compatibility's sake,
1466 * reject any unknown flag.
1467 */
1468 return KERN_INVALID_VALUE;
1469 }
1470
1471 if (ip_kotype(port) == IKOT_NAMED_ENTRY) {
1472 vm_named_entry_t named_entry;
1473
1474 named_entry = (vm_named_entry_t)port->ip_kobject;
1475 /* a few checks to make sure user is obeying rules */
1476 if(*upl_size == 0) {
1477 if(offset >= named_entry->size)
1478 return(KERN_INVALID_RIGHT);
1479 *upl_size = (upl_size_t)(named_entry->size - offset);
1480 if (*upl_size != named_entry->size - offset)
1481 return KERN_INVALID_ARGUMENT;
1482 }
1483 if(caller_flags & UPL_COPYOUT_FROM) {
1484 if((named_entry->protection & VM_PROT_READ)
1485 != VM_PROT_READ) {
1486 return(KERN_INVALID_RIGHT);
1487 }
1488 } else {
1489 if((named_entry->protection &
1490 (VM_PROT_READ | VM_PROT_WRITE))
1491 != (VM_PROT_READ | VM_PROT_WRITE)) {
1492 return(KERN_INVALID_RIGHT);
1493 }
1494 }
1495 if(named_entry->size < (offset + *upl_size))
1496 return(KERN_INVALID_ARGUMENT);
1497
1498 /* the callers parameter offset is defined to be the */
1499 /* offset from beginning of named entry offset in object */
1500 offset = offset + named_entry->offset;
1501
1502 if (named_entry->is_sub_map ||
1503 named_entry->is_copy)
1504 return KERN_INVALID_ARGUMENT;
1505
1506 named_entry_lock(named_entry);
1507
1508 if (named_entry->is_pager) {
1509 object = vm_object_enter(named_entry->backing.pager,
1510 named_entry->offset + named_entry->size,
1511 named_entry->internal,
1512 FALSE,
1513 FALSE);
1514 if (object == VM_OBJECT_NULL) {
1515 named_entry_unlock(named_entry);
1516 return(KERN_INVALID_OBJECT);
1517 }
1518
1519 /* JMM - drop reference on pager here? */
1520
1521 /* create an extra reference for the named entry */
1522 vm_object_lock(object);
1523 vm_object_reference_locked(object);
1524 named_entry->backing.object = object;
1525 named_entry->is_pager = FALSE;
1526 named_entry_unlock(named_entry);
1527
1528 /* wait for object to be ready */
1529 while (!object->pager_ready) {
1530 vm_object_wait(object,
1531 VM_OBJECT_EVENT_PAGER_READY,
1532 THREAD_UNINT);
1533 vm_object_lock(object);
1534 }
1535 vm_object_unlock(object);
1536 } else {
1537 /* This is the case where we are going to map */
1538 /* an already mapped object. If the object is */
1539 /* not ready it is internal. An external */
1540 /* object cannot be mapped until it is ready */
1541 /* we can therefore avoid the ready check */
1542 /* in this case. */
1543 object = named_entry->backing.object;
1544 vm_object_reference(object);
1545 named_entry_unlock(named_entry);
1546 }
1547 } else if (ip_kotype(port) == IKOT_MEM_OBJ_CONTROL) {
1548 memory_object_control_t control;
1549 control = (memory_object_control_t) port;
1550 if (control == NULL)
1551 return (KERN_INVALID_ARGUMENT);
1552 object = memory_object_control_to_vm_object(control);
1553 if (object == VM_OBJECT_NULL)
1554 return (KERN_INVALID_ARGUMENT);
1555 vm_object_reference(object);
1556 } else {
1557 return KERN_INVALID_ARGUMENT;
1558 }
1559 if (object == VM_OBJECT_NULL)
1560 return (KERN_INVALID_ARGUMENT);
1561
1562 if (!object->private) {
1563 if (object->phys_contiguous) {
1564 *flags = UPL_PHYS_CONTIG;
1565 } else {
1566 *flags = 0;
1567 }
1568 } else {
1569 *flags = UPL_DEV_MEMORY | UPL_PHYS_CONTIG;
1570 }
1571
1572 ret = vm_object_iopl_request(object,
1573 offset,
1574 *upl_size,
1575 upl_ptr,
1576 user_page_list,
1577 page_list_count,
1578 caller_flags);
1579 vm_object_deallocate(object);
1580 return ret;
1581 }
1582
1583 /*
1584 * Routine: memory_object_upl_request [interface]
1585 * Purpose:
1586 * Cause the population of a portion of a vm_object.
1587 * Depending on the nature of the request, the pages
1588 * returned may be contain valid data or be uninitialized.
1589 *
1590 */
1591
1592 kern_return_t
1593 memory_object_upl_request(
1594 memory_object_control_t control,
1595 memory_object_offset_t offset,
1596 upl_size_t size,
1597 upl_t *upl_ptr,
1598 upl_page_info_array_t user_page_list,
1599 unsigned int *page_list_count,
1600 int cntrl_flags)
1601 {
1602 vm_object_t object;
1603
1604 object = memory_object_control_to_vm_object(control);
1605 if (object == VM_OBJECT_NULL)
1606 return (KERN_TERMINATED);
1607
1608 return vm_object_upl_request(object,
1609 offset,
1610 size,
1611 upl_ptr,
1612 user_page_list,
1613 page_list_count,
1614 cntrl_flags);
1615 }
1616
1617 /*
1618 * Routine: memory_object_super_upl_request [interface]
1619 * Purpose:
1620 * Cause the population of a portion of a vm_object
1621 * in much the same way as memory_object_upl_request.
1622 * Depending on the nature of the request, the pages
1623 * returned may be contain valid data or be uninitialized.
1624 * However, the region may be expanded up to the super
1625 * cluster size provided.
1626 */
1627
1628 kern_return_t
1629 memory_object_super_upl_request(
1630 memory_object_control_t control,
1631 memory_object_offset_t offset,
1632 upl_size_t size,
1633 upl_size_t super_cluster,
1634 upl_t *upl,
1635 upl_page_info_t *user_page_list,
1636 unsigned int *page_list_count,
1637 int cntrl_flags)
1638 {
1639 vm_object_t object;
1640
1641 object = memory_object_control_to_vm_object(control);
1642 if (object == VM_OBJECT_NULL)
1643 return (KERN_INVALID_ARGUMENT);
1644
1645 return vm_object_super_upl_request(object,
1646 offset,
1647 size,
1648 super_cluster,
1649 upl,
1650 user_page_list,
1651 page_list_count,
1652 cntrl_flags);
1653 }
1654
1655 kern_return_t
1656 memory_object_cluster_size(memory_object_control_t control, memory_object_offset_t *start,
1657 vm_size_t *length, uint32_t *io_streaming, memory_object_fault_info_t fault_info)
1658 {
1659 vm_object_t object;
1660
1661 object = memory_object_control_to_vm_object(control);
1662
1663 if (object == VM_OBJECT_NULL || object->paging_offset > *start)
1664 return (KERN_INVALID_ARGUMENT);
1665
1666 *start -= object->paging_offset;
1667
1668 vm_object_cluster_size(object, (vm_object_offset_t *)start, length, (vm_object_fault_info_t)fault_info, io_streaming);
1669
1670 *start += object->paging_offset;
1671
1672 return (KERN_SUCCESS);
1673 }
1674
1675
1676 int vm_stat_discard_cleared_reply = 0;
1677 int vm_stat_discard_cleared_unset = 0;
1678 int vm_stat_discard_cleared_too_late = 0;
1679
1680
1681
1682 /*
1683 * Routine: host_default_memory_manager [interface]
1684 * Purpose:
1685 * set/get the default memory manager port and default cluster
1686 * size.
1687 *
1688 * If successful, consumes the supplied naked send right.
1689 */
1690 kern_return_t
1691 host_default_memory_manager(
1692 host_priv_t host_priv,
1693 memory_object_default_t *default_manager,
1694 __unused memory_object_cluster_size_t cluster_size)
1695 {
1696 memory_object_default_t current_manager;
1697 memory_object_default_t new_manager;
1698 memory_object_default_t returned_manager;
1699 kern_return_t result = KERN_SUCCESS;
1700
1701 if (host_priv == HOST_PRIV_NULL)
1702 return(KERN_INVALID_HOST);
1703
1704 assert(host_priv == &realhost);
1705
1706 new_manager = *default_manager;
1707 lck_mtx_lock(&memory_manager_default_lock);
1708 current_manager = memory_manager_default;
1709 returned_manager = MEMORY_OBJECT_DEFAULT_NULL;
1710
1711 if (new_manager == MEMORY_OBJECT_DEFAULT_NULL) {
1712 /*
1713 * Retrieve the current value.
1714 */
1715 returned_manager = current_manager;
1716 memory_object_default_reference(returned_manager);
1717 } else {
1718
1719 /*
1720 * If this is the first non-null manager, start
1721 * up the internal pager support.
1722 */
1723 if (current_manager == MEMORY_OBJECT_DEFAULT_NULL) {
1724 result = vm_pageout_internal_start();
1725 if (result != KERN_SUCCESS)
1726 goto out;
1727 }
1728
1729 /*
1730 * Retrieve the current value,
1731 * and replace it with the supplied value.
1732 * We return the old reference to the caller
1733 * but we have to take a reference on the new
1734 * one.
1735 */
1736 returned_manager = current_manager;
1737 memory_manager_default = new_manager;
1738 memory_object_default_reference(new_manager);
1739
1740 /*
1741 * In case anyone's been waiting for a memory
1742 * manager to be established, wake them up.
1743 */
1744
1745 thread_wakeup((event_t) &memory_manager_default);
1746
1747 /*
1748 * Now that we have a default pager for anonymous memory,
1749 * reactivate all the throttled pages (i.e. dirty pages with
1750 * no pager).
1751 */
1752 if (current_manager == MEMORY_OBJECT_DEFAULT_NULL)
1753 {
1754 vm_page_reactivate_all_throttled();
1755 }
1756 }
1757 out:
1758 lck_mtx_unlock(&memory_manager_default_lock);
1759
1760 *default_manager = returned_manager;
1761 return(result);
1762 }
1763
1764 /*
1765 * Routine: memory_manager_default_reference
1766 * Purpose:
1767 * Returns a naked send right for the default
1768 * memory manager. The returned right is always
1769 * valid (not IP_NULL or IP_DEAD).
1770 */
1771
1772 __private_extern__ memory_object_default_t
1773 memory_manager_default_reference(void)
1774 {
1775 memory_object_default_t current_manager;
1776
1777 lck_mtx_lock(&memory_manager_default_lock);
1778 current_manager = memory_manager_default;
1779 while (current_manager == MEMORY_OBJECT_DEFAULT_NULL) {
1780 wait_result_t res;
1781
1782 res = lck_mtx_sleep(&memory_manager_default_lock,
1783 LCK_SLEEP_DEFAULT,
1784 (event_t) &memory_manager_default,
1785 THREAD_UNINT);
1786 assert(res == THREAD_AWAKENED);
1787 current_manager = memory_manager_default;
1788 }
1789 memory_object_default_reference(current_manager);
1790 lck_mtx_unlock(&memory_manager_default_lock);
1791
1792 return current_manager;
1793 }
1794
1795 /*
1796 * Routine: memory_manager_default_check
1797 *
1798 * Purpose:
1799 * Check whether a default memory manager has been set
1800 * up yet, or not. Returns KERN_SUCCESS if dmm exists,
1801 * and KERN_FAILURE if dmm does not exist.
1802 *
1803 * If there is no default memory manager, log an error,
1804 * but only the first time.
1805 *
1806 */
1807 __private_extern__ kern_return_t
1808 memory_manager_default_check(void)
1809 {
1810 memory_object_default_t current;
1811
1812 lck_mtx_lock(&memory_manager_default_lock);
1813 current = memory_manager_default;
1814 if (current == MEMORY_OBJECT_DEFAULT_NULL) {
1815 static boolean_t logged; /* initialized to 0 */
1816 boolean_t complain = !logged;
1817 logged = TRUE;
1818 lck_mtx_unlock(&memory_manager_default_lock);
1819 if (complain)
1820 printf("Warning: No default memory manager\n");
1821 return(KERN_FAILURE);
1822 } else {
1823 lck_mtx_unlock(&memory_manager_default_lock);
1824 return(KERN_SUCCESS);
1825 }
1826 }
1827
1828 __private_extern__ void
1829 memory_manager_default_init(void)
1830 {
1831 memory_manager_default = MEMORY_OBJECT_DEFAULT_NULL;
1832 lck_mtx_init(&memory_manager_default_lock, &vm_object_lck_grp, &vm_object_lck_attr);
1833 }
1834
1835
1836
1837 /* Allow manipulation of individual page state. This is actually part of */
1838 /* the UPL regimen but takes place on the object rather than on a UPL */
1839
1840 kern_return_t
1841 memory_object_page_op(
1842 memory_object_control_t control,
1843 memory_object_offset_t offset,
1844 int ops,
1845 ppnum_t *phys_entry,
1846 int *flags)
1847 {
1848 vm_object_t object;
1849
1850 object = memory_object_control_to_vm_object(control);
1851 if (object == VM_OBJECT_NULL)
1852 return (KERN_INVALID_ARGUMENT);
1853
1854 return vm_object_page_op(object, offset, ops, phys_entry, flags);
1855 }
1856
1857 /*
1858 * memory_object_range_op offers performance enhancement over
1859 * memory_object_page_op for page_op functions which do not require page
1860 * level state to be returned from the call. Page_op was created to provide
1861 * a low-cost alternative to page manipulation via UPLs when only a single
1862 * page was involved. The range_op call establishes the ability in the _op
1863 * family of functions to work on multiple pages where the lack of page level
1864 * state handling allows the caller to avoid the overhead of the upl structures.
1865 */
1866
1867 kern_return_t
1868 memory_object_range_op(
1869 memory_object_control_t control,
1870 memory_object_offset_t offset_beg,
1871 memory_object_offset_t offset_end,
1872 int ops,
1873 int *range)
1874 {
1875 vm_object_t object;
1876
1877 object = memory_object_control_to_vm_object(control);
1878 if (object == VM_OBJECT_NULL)
1879 return (KERN_INVALID_ARGUMENT);
1880
1881 return vm_object_range_op(object,
1882 offset_beg,
1883 offset_end,
1884 ops,
1885 (uint32_t *) range);
1886 }
1887
1888
1889 void
1890 memory_object_mark_used(
1891 memory_object_control_t control)
1892 {
1893 vm_object_t object;
1894
1895 if (control == NULL)
1896 return;
1897
1898 object = memory_object_control_to_vm_object(control);
1899
1900 if (object != VM_OBJECT_NULL)
1901 vm_object_cache_remove(object);
1902 }
1903
1904
1905 void
1906 memory_object_mark_unused(
1907 memory_object_control_t control,
1908 __unused boolean_t rage)
1909 {
1910 vm_object_t object;
1911
1912 if (control == NULL)
1913 return;
1914
1915 object = memory_object_control_to_vm_object(control);
1916
1917 if (object != VM_OBJECT_NULL)
1918 vm_object_cache_add(object);
1919 }
1920
1921 void
1922 memory_object_mark_io_tracking(
1923 memory_object_control_t control)
1924 {
1925 vm_object_t object;
1926
1927 if (control == NULL)
1928 return;
1929 object = memory_object_control_to_vm_object(control);
1930
1931 if (object != VM_OBJECT_NULL) {
1932 vm_object_lock(object);
1933 object->io_tracking = TRUE;
1934 vm_object_unlock(object);
1935 }
1936 }
1937
1938 kern_return_t
1939 memory_object_pages_resident(
1940 memory_object_control_t control,
1941 boolean_t * has_pages_resident)
1942 {
1943 vm_object_t object;
1944
1945 *has_pages_resident = FALSE;
1946
1947 object = memory_object_control_to_vm_object(control);
1948 if (object == VM_OBJECT_NULL)
1949 return (KERN_INVALID_ARGUMENT);
1950
1951 if (object->resident_page_count)
1952 *has_pages_resident = TRUE;
1953
1954 return (KERN_SUCCESS);
1955 }
1956
1957 kern_return_t
1958 memory_object_signed(
1959 memory_object_control_t control,
1960 boolean_t is_signed)
1961 {
1962 vm_object_t object;
1963
1964 object = memory_object_control_to_vm_object(control);
1965 if (object == VM_OBJECT_NULL)
1966 return KERN_INVALID_ARGUMENT;
1967
1968 vm_object_lock(object);
1969 object->code_signed = is_signed;
1970 vm_object_unlock(object);
1971
1972 return KERN_SUCCESS;
1973 }
1974
1975 boolean_t
1976 memory_object_is_signed(
1977 memory_object_control_t control)
1978 {
1979 boolean_t is_signed;
1980 vm_object_t object;
1981
1982 object = memory_object_control_to_vm_object(control);
1983 if (object == VM_OBJECT_NULL)
1984 return FALSE;
1985
1986 vm_object_lock_shared(object);
1987 is_signed = object->code_signed;
1988 vm_object_unlock(object);
1989
1990 return is_signed;
1991 }
1992
1993 boolean_t
1994 memory_object_is_slid(
1995 memory_object_control_t control)
1996 {
1997 vm_object_t object = VM_OBJECT_NULL;
1998
1999 object = memory_object_control_to_vm_object(control);
2000 if (object == VM_OBJECT_NULL)
2001 return FALSE;
2002
2003 return object->object_slid;
2004 }
2005
2006 static zone_t mem_obj_control_zone;
2007
2008 __private_extern__ void
2009 memory_object_control_bootstrap(void)
2010 {
2011 int i;
2012
2013 i = (vm_size_t) sizeof (struct memory_object_control);
2014 mem_obj_control_zone = zinit (i, 8192*i, 4096, "mem_obj_control");
2015 zone_change(mem_obj_control_zone, Z_CALLERACCT, FALSE);
2016 zone_change(mem_obj_control_zone, Z_NOENCRYPT, TRUE);
2017 return;
2018 }
2019
2020 __private_extern__ memory_object_control_t
2021 memory_object_control_allocate(
2022 vm_object_t object)
2023 {
2024 memory_object_control_t control;
2025
2026 control = (memory_object_control_t)zalloc(mem_obj_control_zone);
2027 if (control != MEMORY_OBJECT_CONTROL_NULL) {
2028 control->moc_object = object;
2029 control->moc_ikot = IKOT_MEM_OBJ_CONTROL; /* fake ip_kotype */
2030 }
2031 return (control);
2032 }
2033
2034 __private_extern__ void
2035 memory_object_control_collapse(
2036 memory_object_control_t control,
2037 vm_object_t object)
2038 {
2039 assert((control->moc_object != VM_OBJECT_NULL) &&
2040 (control->moc_object != object));
2041 control->moc_object = object;
2042 }
2043
2044 __private_extern__ vm_object_t
2045 memory_object_control_to_vm_object(
2046 memory_object_control_t control)
2047 {
2048 if (control == MEMORY_OBJECT_CONTROL_NULL ||
2049 control->moc_ikot != IKOT_MEM_OBJ_CONTROL)
2050 return VM_OBJECT_NULL;
2051
2052 return (control->moc_object);
2053 }
2054
2055 memory_object_control_t
2056 convert_port_to_mo_control(
2057 __unused mach_port_t port)
2058 {
2059 return MEMORY_OBJECT_CONTROL_NULL;
2060 }
2061
2062
2063 mach_port_t
2064 convert_mo_control_to_port(
2065 __unused memory_object_control_t control)
2066 {
2067 return MACH_PORT_NULL;
2068 }
2069
2070 void
2071 memory_object_control_reference(
2072 __unused memory_object_control_t control)
2073 {
2074 return;
2075 }
2076
2077 /*
2078 * We only every issue one of these references, so kill it
2079 * when that gets released (should switch the real reference
2080 * counting in true port-less EMMI).
2081 */
2082 void
2083 memory_object_control_deallocate(
2084 memory_object_control_t control)
2085 {
2086 zfree(mem_obj_control_zone, control);
2087 }
2088
2089 void
2090 memory_object_control_disable(
2091 memory_object_control_t control)
2092 {
2093 assert(control->moc_object != VM_OBJECT_NULL);
2094 control->moc_object = VM_OBJECT_NULL;
2095 }
2096
2097 void
2098 memory_object_default_reference(
2099 memory_object_default_t dmm)
2100 {
2101 ipc_port_make_send(dmm);
2102 }
2103
2104 void
2105 memory_object_default_deallocate(
2106 memory_object_default_t dmm)
2107 {
2108 ipc_port_release_send(dmm);
2109 }
2110
2111 memory_object_t
2112 convert_port_to_memory_object(
2113 __unused mach_port_t port)
2114 {
2115 return (MEMORY_OBJECT_NULL);
2116 }
2117
2118
2119 mach_port_t
2120 convert_memory_object_to_port(
2121 __unused memory_object_t object)
2122 {
2123 return (MACH_PORT_NULL);
2124 }
2125
2126
2127 /* Routine memory_object_reference */
2128 void memory_object_reference(
2129 memory_object_t memory_object)
2130 {
2131 (memory_object->mo_pager_ops->memory_object_reference)(
2132 memory_object);
2133 }
2134
2135 /* Routine memory_object_deallocate */
2136 void memory_object_deallocate(
2137 memory_object_t memory_object)
2138 {
2139 (memory_object->mo_pager_ops->memory_object_deallocate)(
2140 memory_object);
2141 }
2142
2143
2144 /* Routine memory_object_init */
2145 kern_return_t memory_object_init
2146 (
2147 memory_object_t memory_object,
2148 memory_object_control_t memory_control,
2149 memory_object_cluster_size_t memory_object_page_size
2150 )
2151 {
2152 return (memory_object->mo_pager_ops->memory_object_init)(
2153 memory_object,
2154 memory_control,
2155 memory_object_page_size);
2156 }
2157
2158 /* Routine memory_object_terminate */
2159 kern_return_t memory_object_terminate
2160 (
2161 memory_object_t memory_object
2162 )
2163 {
2164 return (memory_object->mo_pager_ops->memory_object_terminate)(
2165 memory_object);
2166 }
2167
2168 /* Routine memory_object_data_request */
2169 kern_return_t memory_object_data_request
2170 (
2171 memory_object_t memory_object,
2172 memory_object_offset_t offset,
2173 memory_object_cluster_size_t length,
2174 vm_prot_t desired_access,
2175 memory_object_fault_info_t fault_info
2176 )
2177 {
2178 return (memory_object->mo_pager_ops->memory_object_data_request)(
2179 memory_object,
2180 offset,
2181 length,
2182 desired_access,
2183 fault_info);
2184 }
2185
2186 /* Routine memory_object_data_return */
2187 kern_return_t memory_object_data_return
2188 (
2189 memory_object_t memory_object,
2190 memory_object_offset_t offset,
2191 memory_object_cluster_size_t size,
2192 memory_object_offset_t *resid_offset,
2193 int *io_error,
2194 boolean_t dirty,
2195 boolean_t kernel_copy,
2196 int upl_flags
2197 )
2198 {
2199 return (memory_object->mo_pager_ops->memory_object_data_return)(
2200 memory_object,
2201 offset,
2202 size,
2203 resid_offset,
2204 io_error,
2205 dirty,
2206 kernel_copy,
2207 upl_flags);
2208 }
2209
2210 /* Routine memory_object_data_initialize */
2211 kern_return_t memory_object_data_initialize
2212 (
2213 memory_object_t memory_object,
2214 memory_object_offset_t offset,
2215 memory_object_cluster_size_t size
2216 )
2217 {
2218 return (memory_object->mo_pager_ops->memory_object_data_initialize)(
2219 memory_object,
2220 offset,
2221 size);
2222 }
2223
2224 /* Routine memory_object_data_unlock */
2225 kern_return_t memory_object_data_unlock
2226 (
2227 memory_object_t memory_object,
2228 memory_object_offset_t offset,
2229 memory_object_size_t size,
2230 vm_prot_t desired_access
2231 )
2232 {
2233 return (memory_object->mo_pager_ops->memory_object_data_unlock)(
2234 memory_object,
2235 offset,
2236 size,
2237 desired_access);
2238 }
2239
2240 /* Routine memory_object_synchronize */
2241 kern_return_t memory_object_synchronize
2242 (
2243 memory_object_t memory_object,
2244 memory_object_offset_t offset,
2245 memory_object_size_t size,
2246 vm_sync_t sync_flags
2247 )
2248 {
2249 return (memory_object->mo_pager_ops->memory_object_synchronize)(
2250 memory_object,
2251 offset,
2252 size,
2253 sync_flags);
2254 }
2255
2256
2257 /*
2258 * memory_object_map() is called by VM (in vm_map_enter() and its variants)
2259 * each time a "named" VM object gets mapped directly or indirectly
2260 * (copy-on-write mapping). A "named" VM object has an extra reference held
2261 * by the pager to keep it alive until the pager decides that the
2262 * memory object (and its VM object) can be reclaimed.
2263 * VM calls memory_object_last_unmap() (in vm_object_deallocate()) when all
2264 * the mappings of that memory object have been removed.
2265 *
2266 * For a given VM object, calls to memory_object_map() and memory_object_unmap()
2267 * are serialized (through object->mapping_in_progress), to ensure that the
2268 * pager gets a consistent view of the mapping status of the memory object.
2269 *
2270 * This allows the pager to keep track of how many times a memory object
2271 * has been mapped and with which protections, to decide when it can be
2272 * reclaimed.
2273 */
2274
2275 /* Routine memory_object_map */
2276 kern_return_t memory_object_map
2277 (
2278 memory_object_t memory_object,
2279 vm_prot_t prot
2280 )
2281 {
2282 return (memory_object->mo_pager_ops->memory_object_map)(
2283 memory_object,
2284 prot);
2285 }
2286
2287 /* Routine memory_object_last_unmap */
2288 kern_return_t memory_object_last_unmap
2289 (
2290 memory_object_t memory_object
2291 )
2292 {
2293 return (memory_object->mo_pager_ops->memory_object_last_unmap)(
2294 memory_object);
2295 }
2296
2297 /* Routine memory_object_data_reclaim */
2298 kern_return_t memory_object_data_reclaim
2299 (
2300 memory_object_t memory_object,
2301 boolean_t reclaim_backing_store
2302 )
2303 {
2304 if (memory_object->mo_pager_ops->memory_object_data_reclaim == NULL)
2305 return KERN_NOT_SUPPORTED;
2306 return (memory_object->mo_pager_ops->memory_object_data_reclaim)(
2307 memory_object,
2308 reclaim_backing_store);
2309 }
2310
2311 /* Routine memory_object_create */
2312 kern_return_t memory_object_create
2313 (
2314 memory_object_default_t default_memory_manager,
2315 vm_size_t new_memory_object_size,
2316 memory_object_t *new_memory_object
2317 )
2318 {
2319 return default_pager_memory_object_create(default_memory_manager,
2320 new_memory_object_size,
2321 new_memory_object);
2322 }
2323
2324 upl_t
2325 convert_port_to_upl(
2326 ipc_port_t port)
2327 {
2328 upl_t upl;
2329
2330 ip_lock(port);
2331 if (!ip_active(port) || (ip_kotype(port) != IKOT_UPL)) {
2332 ip_unlock(port);
2333 return (upl_t)NULL;
2334 }
2335 upl = (upl_t) port->ip_kobject;
2336 ip_unlock(port);
2337 upl_lock(upl);
2338 upl->ref_count+=1;
2339 upl_unlock(upl);
2340 return upl;
2341 }
2342
2343 mach_port_t
2344 convert_upl_to_port(
2345 __unused upl_t upl)
2346 {
2347 return MACH_PORT_NULL;
2348 }
2349
2350 __private_extern__ void
2351 upl_no_senders(
2352 __unused ipc_port_t port,
2353 __unused mach_port_mscount_t mscount)
2354 {
2355 return;
2356 }
mirror of XNU