]> git.saurik.com Git - apple/xnu.git/blob - osfmk/vm/vm_user.c
projects / apple / xnu.git / blob
? search:


024140fb53562e8137fb76b73c08ee3894973366
[apple/xnu.git] / osfmk / vm / vm_user.c
1 /*
2 * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56 /*
57 */
58 /*
59 * File: vm/vm_user.c
60 * Author: Avadis Tevanian, Jr., Michael Wayne Young
61 *
62 * User-exported virtual memory functions.
63 */
64
65 /*
66 * There are three implementations of the "XXX_allocate" functionality in
67 * the kernel: mach_vm_allocate (for any task on the platform), vm_allocate
68 * (for a task with the same address space size, especially the current task),
69 * and vm32_vm_allocate (for the specific case of a 32-bit task). vm_allocate
70 * in the kernel should only be used on the kernel_task. vm32_vm_allocate only
71 * makes sense on platforms where a user task can either be 32 or 64, or the kernel
72 * task can be 32 or 64. mach_vm_allocate makes sense everywhere, and is preferred
73 * for new code.
74 *
75 * The entrypoints into the kernel are more complex. All platforms support a
76 * mach_vm_allocate-style API (subsystem 4800) which operates with the largest
77 * size types for the platform. On platforms that only support U32/K32,
78 * subsystem 4800 is all you need. On platforms that support both U32 and U64,
79 * subsystem 3800 is used disambiguate the size of parameters, and they will
80 * always be 32-bit and call into the vm32_vm_allocate APIs. On non-U32/K32 platforms,
81 * the MIG glue should never call into vm_allocate directly, because the calling
82 * task and kernel_task are unlikely to use the same size parameters
83 *
84 * New VM call implementations should be added here and to mach_vm.defs
85 * (subsystem 4800), and use mach_vm_* "wide" types.
86 */
87
88 #include <debug.h>
89
90 #include <vm_cpm.h>
91 #include <mach/boolean.h>
92 #include <mach/kern_return.h>
93 #include <mach/mach_types.h> /* to get vm_address_t */
94 #include <mach/memory_object.h>
95 #include <mach/std_types.h> /* to get pointer_t */
96 #include <mach/upl.h>
97 #include <mach/vm_attributes.h>
98 #include <mach/vm_param.h>
99 #include <mach/vm_statistics.h>
100 #include <mach/mach_syscalls.h>
101
102 #include <mach/host_priv_server.h>
103 #include <mach/mach_vm_server.h>
104 #include <mach/vm_map_server.h>
105
106 #include <kern/host.h>
107 #include <kern/kalloc.h>
108 #include <kern/task.h>
109 #include <kern/misc_protos.h>
110 #include <vm/vm_fault.h>
111 #include <vm/vm_map.h>
112 #include <vm/vm_object.h>
113 #include <vm/vm_page.h>
114 #include <vm/memory_object.h>
115 #include <vm/vm_pageout.h>
116 #include <vm/vm_protos.h>
117 #include <vm/vm_purgeable_internal.h>
118
119 vm_size_t upl_offset_to_pagelist = 0;
120
121 #if VM_CPM
122 #include <vm/cpm.h>
123 #endif /* VM_CPM */
124
125 ipc_port_t dynamic_pager_control_port=NULL;
126
127 /*
128 * mach_vm_allocate allocates "zero fill" memory in the specfied
129 * map.
130 */
131 kern_return_t
132 mach_vm_allocate(
133 vm_map_t map,
134 mach_vm_offset_t *addr,
135 mach_vm_size_t size,
136 int flags)
137 {
138 vm_map_offset_t map_addr;
139 vm_map_size_t map_size;
140 kern_return_t result;
141 boolean_t anywhere;
142
143 /* filter out any kernel-only flags */
144 if (flags & ~VM_FLAGS_USER_ALLOCATE)
145 return KERN_INVALID_ARGUMENT;
146
147 if (map == VM_MAP_NULL)
148 return(KERN_INVALID_ARGUMENT);
149 if (size == 0) {
150 *addr = 0;
151 return(KERN_SUCCESS);
152 }
153
154 anywhere = ((VM_FLAGS_ANYWHERE & flags) != 0);
155 if (anywhere) {
156 /*
157 * No specific address requested, so start candidate address
158 * search at the minimum address in the map. However, if that
159 * minimum is 0, bump it up by PAGE_SIZE. We want to limit
160 * allocations of PAGEZERO to explicit requests since its
161 * normal use is to catch dereferences of NULL and many
162 * applications also treat pointers with a value of 0 as
163 * special and suddenly having address 0 contain useable
164 * memory would tend to confuse those applications.
165 */
166 map_addr = vm_map_min(map);
167 if (map_addr == 0)
168 map_addr += VM_MAP_PAGE_SIZE(map);
169 } else
170 map_addr = vm_map_trunc_page(*addr,
171 VM_MAP_PAGE_MASK(map));
172 map_size = vm_map_round_page(size,
173 VM_MAP_PAGE_MASK(map));
174 if (map_size == 0) {
175 return(KERN_INVALID_ARGUMENT);
176 }
177
178 result = vm_map_enter(
179 map,
180 &map_addr,
181 map_size,
182 (vm_map_offset_t)0,
183 flags,
184 VM_OBJECT_NULL,
185 (vm_object_offset_t)0,
186 FALSE,
187 VM_PROT_DEFAULT,
188 VM_PROT_ALL,
189 VM_INHERIT_DEFAULT);
190
191 *addr = map_addr;
192 return(result);
193 }
194
195 /*
196 * vm_allocate
197 * Legacy routine that allocates "zero fill" memory in the specfied
198 * map (which is limited to the same size as the kernel).
199 */
200 kern_return_t
201 vm_allocate(
202 vm_map_t map,
203 vm_offset_t *addr,
204 vm_size_t size,
205 int flags)
206 {
207 vm_map_offset_t map_addr;
208 vm_map_size_t map_size;
209 kern_return_t result;
210 boolean_t anywhere;
211
212 /* filter out any kernel-only flags */
213 if (flags & ~VM_FLAGS_USER_ALLOCATE)
214 return KERN_INVALID_ARGUMENT;
215
216 if (map == VM_MAP_NULL)
217 return(KERN_INVALID_ARGUMENT);
218 if (size == 0) {
219 *addr = 0;
220 return(KERN_SUCCESS);
221 }
222
223 anywhere = ((VM_FLAGS_ANYWHERE & flags) != 0);
224 if (anywhere) {
225 /*
226 * No specific address requested, so start candidate address
227 * search at the minimum address in the map. However, if that
228 * minimum is 0, bump it up by PAGE_SIZE. We want to limit
229 * allocations of PAGEZERO to explicit requests since its
230 * normal use is to catch dereferences of NULL and many
231 * applications also treat pointers with a value of 0 as
232 * special and suddenly having address 0 contain useable
233 * memory would tend to confuse those applications.
234 */
235 map_addr = vm_map_min(map);
236 if (map_addr == 0)
237 map_addr += VM_MAP_PAGE_SIZE(map);
238 } else
239 map_addr = vm_map_trunc_page(*addr,
240 VM_MAP_PAGE_MASK(map));
241 map_size = vm_map_round_page(size,
242 VM_MAP_PAGE_MASK(map));
243 if (map_size == 0) {
244 return(KERN_INVALID_ARGUMENT);
245 }
246
247 result = vm_map_enter(
248 map,
249 &map_addr,
250 map_size,
251 (vm_map_offset_t)0,
252 flags,
253 VM_OBJECT_NULL,
254 (vm_object_offset_t)0,
255 FALSE,
256 VM_PROT_DEFAULT,
257 VM_PROT_ALL,
258 VM_INHERIT_DEFAULT);
259
260 *addr = CAST_DOWN(vm_offset_t, map_addr);
261 return(result);
262 }
263
264 /*
265 * mach_vm_deallocate -
266 * deallocates the specified range of addresses in the
267 * specified address map.
268 */
269 kern_return_t
270 mach_vm_deallocate(
271 vm_map_t map,
272 mach_vm_offset_t start,
273 mach_vm_size_t size)
274 {
275 if ((map == VM_MAP_NULL) || (start + size < start))
276 return(KERN_INVALID_ARGUMENT);
277
278 if (size == (mach_vm_offset_t) 0)
279 return(KERN_SUCCESS);
280
281 return(vm_map_remove(map,
282 vm_map_trunc_page(start,
283 VM_MAP_PAGE_MASK(map)),
284 vm_map_round_page(start+size,
285 VM_MAP_PAGE_MASK(map)),
286 VM_MAP_NO_FLAGS));
287 }
288
289 /*
290 * vm_deallocate -
291 * deallocates the specified range of addresses in the
292 * specified address map (limited to addresses the same
293 * size as the kernel).
294 */
295 kern_return_t
296 vm_deallocate(
297 register vm_map_t map,
298 vm_offset_t start,
299 vm_size_t size)
300 {
301 if ((map == VM_MAP_NULL) || (start + size < start))
302 return(KERN_INVALID_ARGUMENT);
303
304 if (size == (vm_offset_t) 0)
305 return(KERN_SUCCESS);
306
307 return(vm_map_remove(map,
308 vm_map_trunc_page(start,
309 VM_MAP_PAGE_MASK(map)),
310 vm_map_round_page(start+size,
311 VM_MAP_PAGE_MASK(map)),
312 VM_MAP_NO_FLAGS));
313 }
314
315 /*
316 * mach_vm_inherit -
317 * Sets the inheritance of the specified range in the
318 * specified map.
319 */
320 kern_return_t
321 mach_vm_inherit(
322 vm_map_t map,
323 mach_vm_offset_t start,
324 mach_vm_size_t size,
325 vm_inherit_t new_inheritance)
326 {
327 if ((map == VM_MAP_NULL) || (start + size < start) ||
328 (new_inheritance > VM_INHERIT_LAST_VALID))
329 return(KERN_INVALID_ARGUMENT);
330
331 if (size == 0)
332 return KERN_SUCCESS;
333
334 return(vm_map_inherit(map,
335 vm_map_trunc_page(start,
336 VM_MAP_PAGE_MASK(map)),
337 vm_map_round_page(start+size,
338 VM_MAP_PAGE_MASK(map)),
339 new_inheritance));
340 }
341
342 /*
343 * vm_inherit -
344 * Sets the inheritance of the specified range in the
345 * specified map (range limited to addresses
346 */
347 kern_return_t
348 vm_inherit(
349 register vm_map_t map,
350 vm_offset_t start,
351 vm_size_t size,
352 vm_inherit_t new_inheritance)
353 {
354 if ((map == VM_MAP_NULL) || (start + size < start) ||
355 (new_inheritance > VM_INHERIT_LAST_VALID))
356 return(KERN_INVALID_ARGUMENT);
357
358 if (size == 0)
359 return KERN_SUCCESS;
360
361 return(vm_map_inherit(map,
362 vm_map_trunc_page(start,
363 VM_MAP_PAGE_MASK(map)),
364 vm_map_round_page(start+size,
365 VM_MAP_PAGE_MASK(map)),
366 new_inheritance));
367 }
368
369 /*
370 * mach_vm_protect -
371 * Sets the protection of the specified range in the
372 * specified map.
373 */
374
375 kern_return_t
376 mach_vm_protect(
377 vm_map_t map,
378 mach_vm_offset_t start,
379 mach_vm_size_t size,
380 boolean_t set_maximum,
381 vm_prot_t new_protection)
382 {
383 if ((map == VM_MAP_NULL) || (start + size < start) ||
384 (new_protection & ~(VM_PROT_ALL | VM_PROT_COPY)))
385 return(KERN_INVALID_ARGUMENT);
386
387 if (size == 0)
388 return KERN_SUCCESS;
389
390 return(vm_map_protect(map,
391 vm_map_trunc_page(start,
392 VM_MAP_PAGE_MASK(map)),
393 vm_map_round_page(start+size,
394 VM_MAP_PAGE_MASK(map)),
395 new_protection,
396 set_maximum));
397 }
398
399 /*
400 * vm_protect -
401 * Sets the protection of the specified range in the
402 * specified map. Addressability of the range limited
403 * to the same size as the kernel.
404 */
405
406 kern_return_t
407 vm_protect(
408 vm_map_t map,
409 vm_offset_t start,
410 vm_size_t size,
411 boolean_t set_maximum,
412 vm_prot_t new_protection)
413 {
414 if ((map == VM_MAP_NULL) || (start + size < start) ||
415 (new_protection & ~(VM_PROT_ALL | VM_PROT_COPY)))
416 return(KERN_INVALID_ARGUMENT);
417
418 if (size == 0)
419 return KERN_SUCCESS;
420
421 return(vm_map_protect(map,
422 vm_map_trunc_page(start,
423 VM_MAP_PAGE_MASK(map)),
424 vm_map_round_page(start+size,
425 VM_MAP_PAGE_MASK(map)),
426 new_protection,
427 set_maximum));
428 }
429
430 /*
431 * mach_vm_machine_attributes -
432 * Handle machine-specific attributes for a mapping, such
433 * as cachability, migrability, etc.
434 */
435 kern_return_t
436 mach_vm_machine_attribute(
437 vm_map_t map,
438 mach_vm_address_t addr,
439 mach_vm_size_t size,
440 vm_machine_attribute_t attribute,
441 vm_machine_attribute_val_t* value) /* IN/OUT */
442 {
443 if ((map == VM_MAP_NULL) || (addr + size < addr))
444 return(KERN_INVALID_ARGUMENT);
445
446 if (size == 0)
447 return KERN_SUCCESS;
448
449 return vm_map_machine_attribute(
450 map,
451 vm_map_trunc_page(addr,
452 VM_MAP_PAGE_MASK(map)),
453 vm_map_round_page(addr+size,
454 VM_MAP_PAGE_MASK(map)),
455 attribute,
456 value);
457 }
458
459 /*
460 * vm_machine_attribute -
461 * Handle machine-specific attributes for a mapping, such
462 * as cachability, migrability, etc. Limited addressability
463 * (same range limits as for the native kernel map).
464 */
465 kern_return_t
466 vm_machine_attribute(
467 vm_map_t map,
468 vm_address_t addr,
469 vm_size_t size,
470 vm_machine_attribute_t attribute,
471 vm_machine_attribute_val_t* value) /* IN/OUT */
472 {
473 if ((map == VM_MAP_NULL) || (addr + size < addr))
474 return(KERN_INVALID_ARGUMENT);
475
476 if (size == 0)
477 return KERN_SUCCESS;
478
479 return vm_map_machine_attribute(
480 map,
481 vm_map_trunc_page(addr,
482 VM_MAP_PAGE_MASK(map)),
483 vm_map_round_page(addr+size,
484 VM_MAP_PAGE_MASK(map)),
485 attribute,
486 value);
487 }
488
489 /*
490 * mach_vm_read -
491 * Read/copy a range from one address space and return it to the caller.
492 *
493 * It is assumed that the address for the returned memory is selected by
494 * the IPC implementation as part of receiving the reply to this call.
495 * If IPC isn't used, the caller must deal with the vm_map_copy_t object
496 * that gets returned.
497 *
498 * JMM - because of mach_msg_type_number_t, this call is limited to a
499 * single 4GB region at this time.
500 *
501 */
502 kern_return_t
503 mach_vm_read(
504 vm_map_t map,
505 mach_vm_address_t addr,
506 mach_vm_size_t size,
507 pointer_t *data,
508 mach_msg_type_number_t *data_size)
509 {
510 kern_return_t error;
511 vm_map_copy_t ipc_address;
512
513 if (map == VM_MAP_NULL)
514 return(KERN_INVALID_ARGUMENT);
515
516 if ((mach_msg_type_number_t) size != size)
517 return KERN_INVALID_ARGUMENT;
518
519 error = vm_map_copyin(map,
520 (vm_map_address_t)addr,
521 (vm_map_size_t)size,
522 FALSE, /* src_destroy */
523 &ipc_address);
524
525 if (KERN_SUCCESS == error) {
526 *data = (pointer_t) ipc_address;
527 *data_size = (mach_msg_type_number_t) size;
528 assert(*data_size == size);
529 }
530 return(error);
531 }
532
533 /*
534 * vm_read -
535 * Read/copy a range from one address space and return it to the caller.
536 * Limited addressability (same range limits as for the native kernel map).
537 *
538 * It is assumed that the address for the returned memory is selected by
539 * the IPC implementation as part of receiving the reply to this call.
540 * If IPC isn't used, the caller must deal with the vm_map_copy_t object
541 * that gets returned.
542 */
543 kern_return_t
544 vm_read(
545 vm_map_t map,
546 vm_address_t addr,
547 vm_size_t size,
548 pointer_t *data,
549 mach_msg_type_number_t *data_size)
550 {
551 kern_return_t error;
552 vm_map_copy_t ipc_address;
553
554 if (map == VM_MAP_NULL)
555 return(KERN_INVALID_ARGUMENT);
556
557 if (size > (unsigned)(mach_msg_type_number_t) -1) {
558 /*
559 * The kernel could handle a 64-bit "size" value, but
560 * it could not return the size of the data in "*data_size"
561 * without overflowing.
562 * Let's reject this "size" as invalid.
563 */
564 return KERN_INVALID_ARGUMENT;
565 }
566
567 error = vm_map_copyin(map,
568 (vm_map_address_t)addr,
569 (vm_map_size_t)size,
570 FALSE, /* src_destroy */
571 &ipc_address);
572
573 if (KERN_SUCCESS == error) {
574 *data = (pointer_t) ipc_address;
575 *data_size = (mach_msg_type_number_t) size;
576 assert(*data_size == size);
577 }
578 return(error);
579 }
580
581 /*
582 * mach_vm_read_list -
583 * Read/copy a list of address ranges from specified map.
584 *
585 * MIG does not know how to deal with a returned array of
586 * vm_map_copy_t structures, so we have to do the copyout
587 * manually here.
588 */
589 kern_return_t
590 mach_vm_read_list(
591 vm_map_t map,
592 mach_vm_read_entry_t data_list,
593 natural_t count)
594 {
595 mach_msg_type_number_t i;
596 kern_return_t error;
597 vm_map_copy_t copy;
598
599 if (map == VM_MAP_NULL ||
600 count > VM_MAP_ENTRY_MAX)
601 return(KERN_INVALID_ARGUMENT);
602
603 error = KERN_SUCCESS;
604 for(i=0; i<count; i++) {
605 vm_map_address_t map_addr;
606 vm_map_size_t map_size;
607
608 map_addr = (vm_map_address_t)(data_list[i].address);
609 map_size = (vm_map_size_t)(data_list[i].size);
610
611 if(map_size != 0) {
612 error = vm_map_copyin(map,
613 map_addr,
614 map_size,
615 FALSE, /* src_destroy */
616 &copy);
617 if (KERN_SUCCESS == error) {
618 error = vm_map_copyout(
619 current_task()->map,
620 &map_addr,
621 copy);
622 if (KERN_SUCCESS == error) {
623 data_list[i].address = map_addr;
624 continue;
625 }
626 vm_map_copy_discard(copy);
627 }
628 }
629 data_list[i].address = (mach_vm_address_t)0;
630 data_list[i].size = (mach_vm_size_t)0;
631 }
632 return(error);
633 }
634
635 /*
636 * vm_read_list -
637 * Read/copy a list of address ranges from specified map.
638 *
639 * MIG does not know how to deal with a returned array of
640 * vm_map_copy_t structures, so we have to do the copyout
641 * manually here.
642 *
643 * The source and destination ranges are limited to those
644 * that can be described with a vm_address_t (i.e. same
645 * size map as the kernel).
646 *
647 * JMM - If the result of the copyout is an address range
648 * that cannot be described with a vm_address_t (i.e. the
649 * caller had a larger address space but used this call
650 * anyway), it will result in a truncated address being
651 * returned (and a likely confused caller).
652 */
653
654 kern_return_t
655 vm_read_list(
656 vm_map_t map,
657 vm_read_entry_t data_list,
658 natural_t count)
659 {
660 mach_msg_type_number_t i;
661 kern_return_t error;
662 vm_map_copy_t copy;
663
664 if (map == VM_MAP_NULL ||
665 count > VM_MAP_ENTRY_MAX)
666 return(KERN_INVALID_ARGUMENT);
667
668 error = KERN_SUCCESS;
669 for(i=0; i<count; i++) {
670 vm_map_address_t map_addr;
671 vm_map_size_t map_size;
672
673 map_addr = (vm_map_address_t)(data_list[i].address);
674 map_size = (vm_map_size_t)(data_list[i].size);
675
676 if(map_size != 0) {
677 error = vm_map_copyin(map,
678 map_addr,
679 map_size,
680 FALSE, /* src_destroy */
681 &copy);
682 if (KERN_SUCCESS == error) {
683 error = vm_map_copyout(current_task()->map,
684 &map_addr,
685 copy);
686 if (KERN_SUCCESS == error) {
687 data_list[i].address =
688 CAST_DOWN(vm_offset_t, map_addr);
689 continue;
690 }
691 vm_map_copy_discard(copy);
692 }
693 }
694 data_list[i].address = (mach_vm_address_t)0;
695 data_list[i].size = (mach_vm_size_t)0;
696 }
697 return(error);
698 }
699
700 /*
701 * mach_vm_read_overwrite -
702 * Overwrite a range of the current map with data from the specified
703 * map/address range.
704 *
705 * In making an assumption that the current thread is local, it is
706 * no longer cluster-safe without a fully supportive local proxy
707 * thread/task (but we don't support cluster's anymore so this is moot).
708 */
709
710 kern_return_t
711 mach_vm_read_overwrite(
712 vm_map_t map,
713 mach_vm_address_t address,
714 mach_vm_size_t size,
715 mach_vm_address_t data,
716 mach_vm_size_t *data_size)
717 {
718 kern_return_t error;
719 vm_map_copy_t copy;
720
721 if (map == VM_MAP_NULL)
722 return(KERN_INVALID_ARGUMENT);
723
724 error = vm_map_copyin(map, (vm_map_address_t)address,
725 (vm_map_size_t)size, FALSE, &copy);
726
727 if (KERN_SUCCESS == error) {
728 error = vm_map_copy_overwrite(current_thread()->map,
729 (vm_map_address_t)data,
730 copy, FALSE);
731 if (KERN_SUCCESS == error) {
732 *data_size = size;
733 return error;
734 }
735 vm_map_copy_discard(copy);
736 }
737 return(error);
738 }
739
740 /*
741 * vm_read_overwrite -
742 * Overwrite a range of the current map with data from the specified
743 * map/address range.
744 *
745 * This routine adds the additional limitation that the source and
746 * destination ranges must be describable with vm_address_t values
747 * (i.e. the same size address spaces as the kernel, or at least the
748 * the ranges are in that first portion of the respective address
749 * spaces).
750 */
751
752 kern_return_t
753 vm_read_overwrite(
754 vm_map_t map,
755 vm_address_t address,
756 vm_size_t size,
757 vm_address_t data,
758 vm_size_t *data_size)
759 {
760 kern_return_t error;
761 vm_map_copy_t copy;
762
763 if (map == VM_MAP_NULL)
764 return(KERN_INVALID_ARGUMENT);
765
766 error = vm_map_copyin(map, (vm_map_address_t)address,
767 (vm_map_size_t)size, FALSE, &copy);
768
769 if (KERN_SUCCESS == error) {
770 error = vm_map_copy_overwrite(current_thread()->map,
771 (vm_map_address_t)data,
772 copy, FALSE);
773 if (KERN_SUCCESS == error) {
774 *data_size = size;
775 return error;
776 }
777 vm_map_copy_discard(copy);
778 }
779 return(error);
780 }
781
782
783 /*
784 * mach_vm_write -
785 * Overwrite the specified address range with the data provided
786 * (from the current map).
787 */
788 kern_return_t
789 mach_vm_write(
790 vm_map_t map,
791 mach_vm_address_t address,
792 pointer_t data,
793 __unused mach_msg_type_number_t size)
794 {
795 if (map == VM_MAP_NULL)
796 return KERN_INVALID_ARGUMENT;
797
798 return vm_map_copy_overwrite(map, (vm_map_address_t)address,
799 (vm_map_copy_t) data, FALSE /* interruptible XXX */);
800 }
801
802 /*
803 * vm_write -
804 * Overwrite the specified address range with the data provided
805 * (from the current map).
806 *
807 * The addressability of the range of addresses to overwrite is
808 * limited bu the use of a vm_address_t (same size as kernel map).
809 * Either the target map is also small, or the range is in the
810 * low addresses within it.
811 */
812 kern_return_t
813 vm_write(
814 vm_map_t map,
815 vm_address_t address,
816 pointer_t data,
817 __unused mach_msg_type_number_t size)
818 {
819 if (map == VM_MAP_NULL)
820 return KERN_INVALID_ARGUMENT;
821
822 return vm_map_copy_overwrite(map, (vm_map_address_t)address,
823 (vm_map_copy_t) data, FALSE /* interruptible XXX */);
824 }
825
826 /*
827 * mach_vm_copy -
828 * Overwrite one range of the specified map with the contents of
829 * another range within that same map (i.e. both address ranges
830 * are "over there").
831 */
832 kern_return_t
833 mach_vm_copy(
834 vm_map_t map,
835 mach_vm_address_t source_address,
836 mach_vm_size_t size,
837 mach_vm_address_t dest_address)
838 {
839 vm_map_copy_t copy;
840 kern_return_t kr;
841
842 if (map == VM_MAP_NULL)
843 return KERN_INVALID_ARGUMENT;
844
845 kr = vm_map_copyin(map, (vm_map_address_t)source_address,
846 (vm_map_size_t)size, FALSE, &copy);
847
848 if (KERN_SUCCESS == kr) {
849 kr = vm_map_copy_overwrite(map,
850 (vm_map_address_t)dest_address,
851 copy, FALSE /* interruptible XXX */);
852
853 if (KERN_SUCCESS != kr)
854 vm_map_copy_discard(copy);
855 }
856 return kr;
857 }
858
859 kern_return_t
860 vm_copy(
861 vm_map_t map,
862 vm_address_t source_address,
863 vm_size_t size,
864 vm_address_t dest_address)
865 {
866 vm_map_copy_t copy;
867 kern_return_t kr;
868
869 if (map == VM_MAP_NULL)
870 return KERN_INVALID_ARGUMENT;
871
872 kr = vm_map_copyin(map, (vm_map_address_t)source_address,
873 (vm_map_size_t)size, FALSE, &copy);
874
875 if (KERN_SUCCESS == kr) {
876 kr = vm_map_copy_overwrite(map,
877 (vm_map_address_t)dest_address,
878 copy, FALSE /* interruptible XXX */);
879
880 if (KERN_SUCCESS != kr)
881 vm_map_copy_discard(copy);
882 }
883 return kr;
884 }
885
886 /*
887 * mach_vm_map -
888 * Map some range of an object into an address space.
889 *
890 * The object can be one of several types of objects:
891 * NULL - anonymous memory
892 * a named entry - a range within another address space
893 * or a range within a memory object
894 * a whole memory object
895 *
896 */
897 kern_return_t
898 mach_vm_map(
899 vm_map_t target_map,
900 mach_vm_offset_t *address,
901 mach_vm_size_t initial_size,
902 mach_vm_offset_t mask,
903 int flags,
904 ipc_port_t port,
905 vm_object_offset_t offset,
906 boolean_t copy,
907 vm_prot_t cur_protection,
908 vm_prot_t max_protection,
909 vm_inherit_t inheritance)
910 {
911 kern_return_t kr;
912 vm_map_offset_t vmmaddr;
913
914 vmmaddr = (vm_map_offset_t) *address;
915
916 /* filter out any kernel-only flags */
917 if (flags & ~VM_FLAGS_USER_MAP)
918 return KERN_INVALID_ARGUMENT;
919
920 kr = vm_map_enter_mem_object(target_map,
921 &vmmaddr,
922 initial_size,
923 mask,
924 flags,
925 port,
926 offset,
927 copy,
928 cur_protection,
929 max_protection,
930 inheritance);
931
932 *address = vmmaddr;
933 return kr;
934 }
935
936
937 /* legacy interface */
938 kern_return_t
939 vm_map_64(
940 vm_map_t target_map,
941 vm_offset_t *address,
942 vm_size_t size,
943 vm_offset_t mask,
944 int flags,
945 ipc_port_t port,
946 vm_object_offset_t offset,
947 boolean_t copy,
948 vm_prot_t cur_protection,
949 vm_prot_t max_protection,
950 vm_inherit_t inheritance)
951 {
952 mach_vm_address_t map_addr;
953 mach_vm_size_t map_size;
954 mach_vm_offset_t map_mask;
955 kern_return_t kr;
956
957 map_addr = (mach_vm_address_t)*address;
958 map_size = (mach_vm_size_t)size;
959 map_mask = (mach_vm_offset_t)mask;
960
961 kr = mach_vm_map(target_map, &map_addr, map_size, map_mask, flags,
962 port, offset, copy,
963 cur_protection, max_protection, inheritance);
964 *address = CAST_DOWN(vm_offset_t, map_addr);
965 return kr;
966 }
967
968 /* temporary, until world build */
969 kern_return_t
970 vm_map(
971 vm_map_t target_map,
972 vm_offset_t *address,
973 vm_size_t size,
974 vm_offset_t mask,
975 int flags,
976 ipc_port_t port,
977 vm_offset_t offset,
978 boolean_t copy,
979 vm_prot_t cur_protection,
980 vm_prot_t max_protection,
981 vm_inherit_t inheritance)
982 {
983 mach_vm_address_t map_addr;
984 mach_vm_size_t map_size;
985 mach_vm_offset_t map_mask;
986 vm_object_offset_t obj_offset;
987 kern_return_t kr;
988
989 map_addr = (mach_vm_address_t)*address;
990 map_size = (mach_vm_size_t)size;
991 map_mask = (mach_vm_offset_t)mask;
992 obj_offset = (vm_object_offset_t)offset;
993
994 kr = mach_vm_map(target_map, &map_addr, map_size, map_mask, flags,
995 port, obj_offset, copy,
996 cur_protection, max_protection, inheritance);
997 *address = CAST_DOWN(vm_offset_t, map_addr);
998 return kr;
999 }
1000
1001 /*
1002 * mach_vm_remap -
1003 * Remap a range of memory from one task into another,
1004 * to another address range within the same task, or
1005 * over top of itself (with altered permissions and/or
1006 * as an in-place copy of itself).
1007 */
1008
1009 kern_return_t
1010 mach_vm_remap(
1011 vm_map_t target_map,
1012 mach_vm_offset_t *address,
1013 mach_vm_size_t size,
1014 mach_vm_offset_t mask,
1015 int flags,
1016 vm_map_t src_map,
1017 mach_vm_offset_t memory_address,
1018 boolean_t copy,
1019 vm_prot_t *cur_protection,
1020 vm_prot_t *max_protection,
1021 vm_inherit_t inheritance)
1022 {
1023 vm_map_offset_t map_addr;
1024 kern_return_t kr;
1025
1026 if (VM_MAP_NULL == target_map || VM_MAP_NULL == src_map)
1027 return KERN_INVALID_ARGUMENT;
1028
1029 /* filter out any kernel-only flags */
1030 if (flags & ~VM_FLAGS_USER_REMAP)
1031 return KERN_INVALID_ARGUMENT;
1032
1033 map_addr = (vm_map_offset_t)*address;
1034
1035 kr = vm_map_remap(target_map,
1036 &map_addr,
1037 size,
1038 mask,
1039 flags,
1040 src_map,
1041 memory_address,
1042 copy,
1043 cur_protection,
1044 max_protection,
1045 inheritance);
1046 *address = map_addr;
1047 return kr;
1048 }
1049
1050 /*
1051 * vm_remap -
1052 * Remap a range of memory from one task into another,
1053 * to another address range within the same task, or
1054 * over top of itself (with altered permissions and/or
1055 * as an in-place copy of itself).
1056 *
1057 * The addressability of the source and target address
1058 * range is limited by the size of vm_address_t (in the
1059 * kernel context).
1060 */
1061 kern_return_t
1062 vm_remap(
1063 vm_map_t target_map,
1064 vm_offset_t *address,
1065 vm_size_t size,
1066 vm_offset_t mask,
1067 int flags,
1068 vm_map_t src_map,
1069 vm_offset_t memory_address,
1070 boolean_t copy,
1071 vm_prot_t *cur_protection,
1072 vm_prot_t *max_protection,
1073 vm_inherit_t inheritance)
1074 {
1075 vm_map_offset_t map_addr;
1076 kern_return_t kr;
1077
1078 if (VM_MAP_NULL == target_map || VM_MAP_NULL == src_map)
1079 return KERN_INVALID_ARGUMENT;
1080
1081 /* filter out any kernel-only flags */
1082 if (flags & ~VM_FLAGS_USER_REMAP)
1083 return KERN_INVALID_ARGUMENT;
1084
1085 map_addr = (vm_map_offset_t)*address;
1086
1087 kr = vm_map_remap(target_map,
1088 &map_addr,
1089 size,
1090 mask,
1091 flags,
1092 src_map,
1093 memory_address,
1094 copy,
1095 cur_protection,
1096 max_protection,
1097 inheritance);
1098 *address = CAST_DOWN(vm_offset_t, map_addr);
1099 return kr;
1100 }
1101
1102 /*
1103 * NOTE: these routine (and this file) will no longer require mach_host_server.h
1104 * when mach_vm_wire and vm_wire are changed to use ledgers.
1105 */
1106 #include <mach/mach_host_server.h>
1107 /*
1108 * mach_vm_wire
1109 * Specify that the range of the virtual address space
1110 * of the target task must not cause page faults for
1111 * the indicated accesses.
1112 *
1113 * [ To unwire the pages, specify VM_PROT_NONE. ]
1114 */
1115 kern_return_t
1116 mach_vm_wire(
1117 host_priv_t host_priv,
1118 vm_map_t map,
1119 mach_vm_offset_t start,
1120 mach_vm_size_t size,
1121 vm_prot_t access)
1122 {
1123 kern_return_t rc;
1124
1125 if (host_priv == HOST_PRIV_NULL)
1126 return KERN_INVALID_HOST;
1127
1128 assert(host_priv == &realhost);
1129
1130 if (map == VM_MAP_NULL)
1131 return KERN_INVALID_TASK;
1132
1133 if (access & ~VM_PROT_ALL || (start + size < start))
1134 return KERN_INVALID_ARGUMENT;
1135
1136 if (access != VM_PROT_NONE) {
1137 rc = vm_map_wire(map,
1138 vm_map_trunc_page(start,
1139 VM_MAP_PAGE_MASK(map)),
1140 vm_map_round_page(start+size,
1141 VM_MAP_PAGE_MASK(map)),
1142 access,
1143 TRUE);
1144 } else {
1145 rc = vm_map_unwire(map,
1146 vm_map_trunc_page(start,
1147 VM_MAP_PAGE_MASK(map)),
1148 vm_map_round_page(start+size,
1149 VM_MAP_PAGE_MASK(map)),
1150 TRUE);
1151 }
1152 return rc;
1153 }
1154
1155 /*
1156 * vm_wire -
1157 * Specify that the range of the virtual address space
1158 * of the target task must not cause page faults for
1159 * the indicated accesses.
1160 *
1161 * [ To unwire the pages, specify VM_PROT_NONE. ]
1162 */
1163 kern_return_t
1164 vm_wire(
1165 host_priv_t host_priv,
1166 register vm_map_t map,
1167 vm_offset_t start,
1168 vm_size_t size,
1169 vm_prot_t access)
1170 {
1171 kern_return_t rc;
1172
1173 if (host_priv == HOST_PRIV_NULL)
1174 return KERN_INVALID_HOST;
1175
1176 assert(host_priv == &realhost);
1177
1178 if (map == VM_MAP_NULL)
1179 return KERN_INVALID_TASK;
1180
1181 if ((access & ~VM_PROT_ALL) || (start + size < start))
1182 return KERN_INVALID_ARGUMENT;
1183
1184 if (size == 0) {
1185 rc = KERN_SUCCESS;
1186 } else if (access != VM_PROT_NONE) {
1187 rc = vm_map_wire(map,
1188 vm_map_trunc_page(start,
1189 VM_MAP_PAGE_MASK(map)),
1190 vm_map_round_page(start+size,
1191 VM_MAP_PAGE_MASK(map)),
1192 access,
1193 TRUE);
1194 } else {
1195 rc = vm_map_unwire(map,
1196 vm_map_trunc_page(start,
1197 VM_MAP_PAGE_MASK(map)),
1198 vm_map_round_page(start+size,
1199 VM_MAP_PAGE_MASK(map)),
1200 TRUE);
1201 }
1202 return rc;
1203 }
1204
1205 /*
1206 * vm_msync
1207 *
1208 * Synchronises the memory range specified with its backing store
1209 * image by either flushing or cleaning the contents to the appropriate
1210 * memory manager.
1211 *
1212 * interpretation of sync_flags
1213 * VM_SYNC_INVALIDATE - discard pages, only return precious
1214 * pages to manager.
1215 *
1216 * VM_SYNC_INVALIDATE & (VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS)
1217 * - discard pages, write dirty or precious
1218 * pages back to memory manager.
1219 *
1220 * VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS
1221 * - write dirty or precious pages back to
1222 * the memory manager.
1223 *
1224 * VM_SYNC_CONTIGUOUS - does everything normally, but if there
1225 * is a hole in the region, and we would
1226 * have returned KERN_SUCCESS, return
1227 * KERN_INVALID_ADDRESS instead.
1228 *
1229 * RETURNS
1230 * KERN_INVALID_TASK Bad task parameter
1231 * KERN_INVALID_ARGUMENT both sync and async were specified.
1232 * KERN_SUCCESS The usual.
1233 * KERN_INVALID_ADDRESS There was a hole in the region.
1234 */
1235
1236 kern_return_t
1237 mach_vm_msync(
1238 vm_map_t map,
1239 mach_vm_address_t address,
1240 mach_vm_size_t size,
1241 vm_sync_t sync_flags)
1242 {
1243
1244 if (map == VM_MAP_NULL)
1245 return(KERN_INVALID_TASK);
1246
1247 return vm_map_msync(map, (vm_map_address_t)address,
1248 (vm_map_size_t)size, sync_flags);
1249 }
1250
1251 /*
1252 * vm_msync
1253 *
1254 * Synchronises the memory range specified with its backing store
1255 * image by either flushing or cleaning the contents to the appropriate
1256 * memory manager.
1257 *
1258 * interpretation of sync_flags
1259 * VM_SYNC_INVALIDATE - discard pages, only return precious
1260 * pages to manager.
1261 *
1262 * VM_SYNC_INVALIDATE & (VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS)
1263 * - discard pages, write dirty or precious
1264 * pages back to memory manager.
1265 *
1266 * VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS
1267 * - write dirty or precious pages back to
1268 * the memory manager.
1269 *
1270 * VM_SYNC_CONTIGUOUS - does everything normally, but if there
1271 * is a hole in the region, and we would
1272 * have returned KERN_SUCCESS, return
1273 * KERN_INVALID_ADDRESS instead.
1274 *
1275 * The addressability of the range is limited to that which can
1276 * be described by a vm_address_t.
1277 *
1278 * RETURNS
1279 * KERN_INVALID_TASK Bad task parameter
1280 * KERN_INVALID_ARGUMENT both sync and async were specified.
1281 * KERN_SUCCESS The usual.
1282 * KERN_INVALID_ADDRESS There was a hole in the region.
1283 */
1284
1285 kern_return_t
1286 vm_msync(
1287 vm_map_t map,
1288 vm_address_t address,
1289 vm_size_t size,
1290 vm_sync_t sync_flags)
1291 {
1292
1293 if (map == VM_MAP_NULL)
1294 return(KERN_INVALID_TASK);
1295
1296 return vm_map_msync(map, (vm_map_address_t)address,
1297 (vm_map_size_t)size, sync_flags);
1298 }
1299
1300
1301 int
1302 vm_toggle_entry_reuse(int toggle, int *old_value)
1303 {
1304 vm_map_t map = current_map();
1305
1306 if(toggle == VM_TOGGLE_GETVALUE && old_value != NULL){
1307 *old_value = map->disable_vmentry_reuse;
1308 } else if(toggle == VM_TOGGLE_SET){
1309 vm_map_lock(map);
1310 map->disable_vmentry_reuse = TRUE;
1311 if (map->first_free == vm_map_to_entry(map)) {
1312 map->highest_entry_end = vm_map_min(map);
1313 } else {
1314 map->highest_entry_end = map->first_free->vme_end;
1315 }
1316 vm_map_unlock(map);
1317 } else if (toggle == VM_TOGGLE_CLEAR){
1318 vm_map_lock(map);
1319 map->disable_vmentry_reuse = FALSE;
1320 vm_map_unlock(map);
1321 } else
1322 return KERN_INVALID_ARGUMENT;
1323
1324 return KERN_SUCCESS;
1325 }
1326
1327 /*
1328 * mach_vm_behavior_set
1329 *
1330 * Sets the paging behavior attribute for the specified range
1331 * in the specified map.
1332 *
1333 * This routine will fail with KERN_INVALID_ADDRESS if any address
1334 * in [start,start+size) is not a valid allocated memory region.
1335 */
1336 kern_return_t
1337 mach_vm_behavior_set(
1338 vm_map_t map,
1339 mach_vm_offset_t start,
1340 mach_vm_size_t size,
1341 vm_behavior_t new_behavior)
1342 {
1343 if ((map == VM_MAP_NULL) || (start + size < start))
1344 return(KERN_INVALID_ARGUMENT);
1345
1346 if (size == 0)
1347 return KERN_SUCCESS;
1348
1349 return(vm_map_behavior_set(map,
1350 vm_map_trunc_page(start,
1351 VM_MAP_PAGE_MASK(map)),
1352 vm_map_round_page(start+size,
1353 VM_MAP_PAGE_MASK(map)),
1354 new_behavior));
1355 }
1356
1357 /*
1358 * vm_behavior_set
1359 *
1360 * Sets the paging behavior attribute for the specified range
1361 * in the specified map.
1362 *
1363 * This routine will fail with KERN_INVALID_ADDRESS if any address
1364 * in [start,start+size) is not a valid allocated memory region.
1365 *
1366 * This routine is potentially limited in addressibility by the
1367 * use of vm_offset_t (if the map provided is larger than the
1368 * kernel's).
1369 */
1370 kern_return_t
1371 vm_behavior_set(
1372 vm_map_t map,
1373 vm_offset_t start,
1374 vm_size_t size,
1375 vm_behavior_t new_behavior)
1376 {
1377 if ((map == VM_MAP_NULL) || (start + size < start))
1378 return(KERN_INVALID_ARGUMENT);
1379
1380 if (size == 0)
1381 return KERN_SUCCESS;
1382
1383 return(vm_map_behavior_set(map,
1384 vm_map_trunc_page(start,
1385 VM_MAP_PAGE_MASK(map)),
1386 vm_map_round_page(start+size,
1387 VM_MAP_PAGE_MASK(map)),
1388 new_behavior));
1389 }
1390
1391 /*
1392 * mach_vm_region:
1393 *
1394 * User call to obtain information about a region in
1395 * a task's address map. Currently, only one flavor is
1396 * supported.
1397 *
1398 * XXX The reserved and behavior fields cannot be filled
1399 * in until the vm merge from the IK is completed, and
1400 * vm_reserve is implemented.
1401 *
1402 * XXX Dependency: syscall_vm_region() also supports only one flavor.
1403 */
1404
1405 kern_return_t
1406 mach_vm_region(
1407 vm_map_t map,
1408 mach_vm_offset_t *address, /* IN/OUT */
1409 mach_vm_size_t *size, /* OUT */
1410 vm_region_flavor_t flavor, /* IN */
1411 vm_region_info_t info, /* OUT */
1412 mach_msg_type_number_t *count, /* IN/OUT */
1413 mach_port_t *object_name) /* OUT */
1414 {
1415 vm_map_offset_t map_addr;
1416 vm_map_size_t map_size;
1417 kern_return_t kr;
1418
1419 if (VM_MAP_NULL == map)
1420 return KERN_INVALID_ARGUMENT;
1421
1422 map_addr = (vm_map_offset_t)*address;
1423 map_size = (vm_map_size_t)*size;
1424
1425 /* legacy conversion */
1426 if (VM_REGION_BASIC_INFO == flavor)
1427 flavor = VM_REGION_BASIC_INFO_64;
1428
1429 kr = vm_map_region(map,
1430 &map_addr, &map_size,
1431 flavor, info, count,
1432 object_name);
1433
1434 *address = map_addr;
1435 *size = map_size;
1436 return kr;
1437 }
1438
1439 /*
1440 * vm_region_64 and vm_region:
1441 *
1442 * User call to obtain information about a region in
1443 * a task's address map. Currently, only one flavor is
1444 * supported.
1445 *
1446 * XXX The reserved and behavior fields cannot be filled
1447 * in until the vm merge from the IK is completed, and
1448 * vm_reserve is implemented.
1449 *
1450 * XXX Dependency: syscall_vm_region() also supports only one flavor.
1451 */
1452
1453 kern_return_t
1454 vm_region_64(
1455 vm_map_t map,
1456 vm_offset_t *address, /* IN/OUT */
1457 vm_size_t *size, /* OUT */
1458 vm_region_flavor_t flavor, /* IN */
1459 vm_region_info_t info, /* OUT */
1460 mach_msg_type_number_t *count, /* IN/OUT */
1461 mach_port_t *object_name) /* OUT */
1462 {
1463 vm_map_offset_t map_addr;
1464 vm_map_size_t map_size;
1465 kern_return_t kr;
1466
1467 if (VM_MAP_NULL == map)
1468 return KERN_INVALID_ARGUMENT;
1469
1470 map_addr = (vm_map_offset_t)*address;
1471 map_size = (vm_map_size_t)*size;
1472
1473 /* legacy conversion */
1474 if (VM_REGION_BASIC_INFO == flavor)
1475 flavor = VM_REGION_BASIC_INFO_64;
1476
1477 kr = vm_map_region(map,
1478 &map_addr, &map_size,
1479 flavor, info, count,
1480 object_name);
1481
1482 *address = CAST_DOWN(vm_offset_t, map_addr);
1483 *size = CAST_DOWN(vm_size_t, map_size);
1484
1485 if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS)
1486 return KERN_INVALID_ADDRESS;
1487 return kr;
1488 }
1489
1490 kern_return_t
1491 vm_region(
1492 vm_map_t map,
1493 vm_address_t *address, /* IN/OUT */
1494 vm_size_t *size, /* OUT */
1495 vm_region_flavor_t flavor, /* IN */
1496 vm_region_info_t info, /* OUT */
1497 mach_msg_type_number_t *count, /* IN/OUT */
1498 mach_port_t *object_name) /* OUT */
1499 {
1500 vm_map_address_t map_addr;
1501 vm_map_size_t map_size;
1502 kern_return_t kr;
1503
1504 if (VM_MAP_NULL == map)
1505 return KERN_INVALID_ARGUMENT;
1506
1507 map_addr = (vm_map_address_t)*address;
1508 map_size = (vm_map_size_t)*size;
1509
1510 kr = vm_map_region(map,
1511 &map_addr, &map_size,
1512 flavor, info, count,
1513 object_name);
1514
1515 *address = CAST_DOWN(vm_address_t, map_addr);
1516 *size = CAST_DOWN(vm_size_t, map_size);
1517
1518 if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS)
1519 return KERN_INVALID_ADDRESS;
1520 return kr;
1521 }
1522
1523 /*
1524 * vm_region_recurse: A form of vm_region which follows the
1525 * submaps in a target map
1526 *
1527 */
1528 kern_return_t
1529 mach_vm_region_recurse(
1530 vm_map_t map,
1531 mach_vm_address_t *address,
1532 mach_vm_size_t *size,
1533 uint32_t *depth,
1534 vm_region_recurse_info_t info,
1535 mach_msg_type_number_t *infoCnt)
1536 {
1537 vm_map_address_t map_addr;
1538 vm_map_size_t map_size;
1539 kern_return_t kr;
1540
1541 if (VM_MAP_NULL == map)
1542 return KERN_INVALID_ARGUMENT;
1543
1544 map_addr = (vm_map_address_t)*address;
1545 map_size = (vm_map_size_t)*size;
1546
1547 kr = vm_map_region_recurse_64(
1548 map,
1549 &map_addr,
1550 &map_size,
1551 depth,
1552 (vm_region_submap_info_64_t)info,
1553 infoCnt);
1554
1555 *address = map_addr;
1556 *size = map_size;
1557 return kr;
1558 }
1559
1560 /*
1561 * vm_region_recurse: A form of vm_region which follows the
1562 * submaps in a target map
1563 *
1564 */
1565 kern_return_t
1566 vm_region_recurse_64(
1567 vm_map_t map,
1568 vm_address_t *address,
1569 vm_size_t *size,
1570 uint32_t *depth,
1571 vm_region_recurse_info_64_t info,
1572 mach_msg_type_number_t *infoCnt)
1573 {
1574 vm_map_address_t map_addr;
1575 vm_map_size_t map_size;
1576 kern_return_t kr;
1577
1578 if (VM_MAP_NULL == map)
1579 return KERN_INVALID_ARGUMENT;
1580
1581 map_addr = (vm_map_address_t)*address;
1582 map_size = (vm_map_size_t)*size;
1583
1584 kr = vm_map_region_recurse_64(
1585 map,
1586 &map_addr,
1587 &map_size,
1588 depth,
1589 (vm_region_submap_info_64_t)info,
1590 infoCnt);
1591
1592 *address = CAST_DOWN(vm_address_t, map_addr);
1593 *size = CAST_DOWN(vm_size_t, map_size);
1594
1595 if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS)
1596 return KERN_INVALID_ADDRESS;
1597 return kr;
1598 }
1599
1600 kern_return_t
1601 vm_region_recurse(
1602 vm_map_t map,
1603 vm_offset_t *address, /* IN/OUT */
1604 vm_size_t *size, /* OUT */
1605 natural_t *depth, /* IN/OUT */
1606 vm_region_recurse_info_t info32, /* IN/OUT */
1607 mach_msg_type_number_t *infoCnt) /* IN/OUT */
1608 {
1609 vm_region_submap_info_data_64_t info64;
1610 vm_region_submap_info_t info;
1611 vm_map_address_t map_addr;
1612 vm_map_size_t map_size;
1613 kern_return_t kr;
1614
1615 if (VM_MAP_NULL == map || *infoCnt < VM_REGION_SUBMAP_INFO_COUNT)
1616 return KERN_INVALID_ARGUMENT;
1617
1618
1619 map_addr = (vm_map_address_t)*address;
1620 map_size = (vm_map_size_t)*size;
1621 info = (vm_region_submap_info_t)info32;
1622 *infoCnt = VM_REGION_SUBMAP_INFO_COUNT_64;
1623
1624 kr = vm_map_region_recurse_64(map, &map_addr,&map_size,
1625 depth, &info64, infoCnt);
1626
1627 info->protection = info64.protection;
1628 info->max_protection = info64.max_protection;
1629 info->inheritance = info64.inheritance;
1630 info->offset = (uint32_t)info64.offset; /* trouble-maker */
1631 info->user_tag = info64.user_tag;
1632 info->pages_resident = info64.pages_resident;
1633 info->pages_shared_now_private = info64.pages_shared_now_private;
1634 info->pages_swapped_out = info64.pages_swapped_out;
1635 info->pages_dirtied = info64.pages_dirtied;
1636 info->ref_count = info64.ref_count;
1637 info->shadow_depth = info64.shadow_depth;
1638 info->external_pager = info64.external_pager;
1639 info->share_mode = info64.share_mode;
1640 info->is_submap = info64.is_submap;
1641 info->behavior = info64.behavior;
1642 info->object_id = info64.object_id;
1643 info->user_wired_count = info64.user_wired_count;
1644
1645 *address = CAST_DOWN(vm_address_t, map_addr);
1646 *size = CAST_DOWN(vm_size_t, map_size);
1647 *infoCnt = VM_REGION_SUBMAP_INFO_COUNT;
1648
1649 if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS)
1650 return KERN_INVALID_ADDRESS;
1651 return kr;
1652 }
1653
1654 kern_return_t
1655 mach_vm_purgable_control(
1656 vm_map_t map,
1657 mach_vm_offset_t address,
1658 vm_purgable_t control,
1659 int *state)
1660 {
1661 if (VM_MAP_NULL == map)
1662 return KERN_INVALID_ARGUMENT;
1663
1664 return vm_map_purgable_control(map,
1665 vm_map_trunc_page(address, PAGE_MASK),
1666 control,
1667 state);
1668 }
1669
1670 kern_return_t
1671 vm_purgable_control(
1672 vm_map_t map,
1673 vm_offset_t address,
1674 vm_purgable_t control,
1675 int *state)
1676 {
1677 if (VM_MAP_NULL == map)
1678 return KERN_INVALID_ARGUMENT;
1679
1680 return vm_map_purgable_control(map,
1681 vm_map_trunc_page(address, PAGE_MASK),
1682 control,
1683 state);
1684 }
1685
1686
1687 /*
1688 * Ordinarily, the right to allocate CPM is restricted
1689 * to privileged applications (those that can gain access
1690 * to the host priv port). Set this variable to zero if
1691 * you want to let any application allocate CPM.
1692 */
1693 unsigned int vm_allocate_cpm_privileged = 0;
1694
1695 /*
1696 * Allocate memory in the specified map, with the caveat that
1697 * the memory is physically contiguous. This call may fail
1698 * if the system can't find sufficient contiguous memory.
1699 * This call may cause or lead to heart-stopping amounts of
1700 * paging activity.
1701 *
1702 * Memory obtained from this call should be freed in the
1703 * normal way, viz., via vm_deallocate.
1704 */
1705 kern_return_t
1706 vm_allocate_cpm(
1707 host_priv_t host_priv,
1708 vm_map_t map,
1709 vm_address_t *addr,
1710 vm_size_t size,
1711 int flags)
1712 {
1713 vm_map_address_t map_addr;
1714 vm_map_size_t map_size;
1715 kern_return_t kr;
1716
1717 if (vm_allocate_cpm_privileged && HOST_PRIV_NULL == host_priv)
1718 return KERN_INVALID_HOST;
1719
1720 if (VM_MAP_NULL == map)
1721 return KERN_INVALID_ARGUMENT;
1722
1723 map_addr = (vm_map_address_t)*addr;
1724 map_size = (vm_map_size_t)size;
1725
1726 kr = vm_map_enter_cpm(map,
1727 &map_addr,
1728 map_size,
1729 flags);
1730
1731 *addr = CAST_DOWN(vm_address_t, map_addr);
1732 return kr;
1733 }
1734
1735
1736 kern_return_t
1737 mach_vm_page_query(
1738 vm_map_t map,
1739 mach_vm_offset_t offset,
1740 int *disposition,
1741 int *ref_count)
1742 {
1743 if (VM_MAP_NULL == map)
1744 return KERN_INVALID_ARGUMENT;
1745
1746 return vm_map_page_query_internal(
1747 map,
1748 vm_map_trunc_page(offset, PAGE_MASK),
1749 disposition, ref_count);
1750 }
1751
1752 kern_return_t
1753 vm_map_page_query(
1754 vm_map_t map,
1755 vm_offset_t offset,
1756 int *disposition,
1757 int *ref_count)
1758 {
1759 if (VM_MAP_NULL == map)
1760 return KERN_INVALID_ARGUMENT;
1761
1762 return vm_map_page_query_internal(
1763 map,
1764 vm_map_trunc_page(offset, PAGE_MASK),
1765 disposition, ref_count);
1766 }
1767
1768 kern_return_t
1769 mach_vm_page_info(
1770 vm_map_t map,
1771 mach_vm_address_t address,
1772 vm_page_info_flavor_t flavor,
1773 vm_page_info_t info,
1774 mach_msg_type_number_t *count)
1775 {
1776 kern_return_t kr;
1777
1778 if (map == VM_MAP_NULL) {
1779 return KERN_INVALID_ARGUMENT;
1780 }
1781
1782 kr = vm_map_page_info(map, address, flavor, info, count);
1783 return kr;
1784 }
1785
1786 /* map a (whole) upl into an address space */
1787 kern_return_t
1788 vm_upl_map(
1789 vm_map_t map,
1790 upl_t upl,
1791 vm_address_t *dst_addr)
1792 {
1793 vm_map_offset_t map_addr;
1794 kern_return_t kr;
1795
1796 if (VM_MAP_NULL == map)
1797 return KERN_INVALID_ARGUMENT;
1798
1799 kr = vm_map_enter_upl(map, upl, &map_addr);
1800 *dst_addr = CAST_DOWN(vm_address_t, map_addr);
1801 return kr;
1802 }
1803
1804 kern_return_t
1805 vm_upl_unmap(
1806 vm_map_t map,
1807 upl_t upl)
1808 {
1809 if (VM_MAP_NULL == map)
1810 return KERN_INVALID_ARGUMENT;
1811
1812 return (vm_map_remove_upl(map, upl));
1813 }
1814
1815 /* Retrieve a upl for an object underlying an address range in a map */
1816
1817 kern_return_t
1818 vm_map_get_upl(
1819 vm_map_t map,
1820 vm_map_offset_t map_offset,
1821 upl_size_t *upl_size,
1822 upl_t *upl,
1823 upl_page_info_array_t page_list,
1824 unsigned int *count,
1825 int *flags,
1826 int force_data_sync)
1827 {
1828 int map_flags;
1829 kern_return_t kr;
1830
1831 if (VM_MAP_NULL == map)
1832 return KERN_INVALID_ARGUMENT;
1833
1834 map_flags = *flags & ~UPL_NOZEROFILL;
1835 if (force_data_sync)
1836 map_flags |= UPL_FORCE_DATA_SYNC;
1837
1838 kr = vm_map_create_upl(map,
1839 map_offset,
1840 upl_size,
1841 upl,
1842 page_list,
1843 count,
1844 &map_flags);
1845
1846 *flags = (map_flags & ~UPL_FORCE_DATA_SYNC);
1847 return kr;
1848 }
1849
1850 /*
1851 * mach_make_memory_entry_64
1852 *
1853 * Think of it as a two-stage vm_remap() operation. First
1854 * you get a handle. Second, you get map that handle in
1855 * somewhere else. Rather than doing it all at once (and
1856 * without needing access to the other whole map).
1857 */
1858
1859 kern_return_t
1860 mach_make_memory_entry_64(
1861 vm_map_t target_map,
1862 memory_object_size_t *size,
1863 memory_object_offset_t offset,
1864 vm_prot_t permission,
1865 ipc_port_t *object_handle,
1866 ipc_port_t parent_handle)
1867 {
1868 vm_map_version_t version;
1869 vm_named_entry_t parent_entry;
1870 vm_named_entry_t user_entry;
1871 ipc_port_t user_handle;
1872 kern_return_t kr;
1873 vm_map_t real_map;
1874
1875 /* needed for call to vm_map_lookup_locked */
1876 boolean_t wired;
1877 vm_object_offset_t obj_off;
1878 vm_prot_t prot;
1879 struct vm_object_fault_info fault_info;
1880 vm_object_t object;
1881 vm_object_t shadow_object;
1882
1883 /* needed for direct map entry manipulation */
1884 vm_map_entry_t map_entry;
1885 vm_map_entry_t next_entry;
1886 vm_map_t local_map;
1887 vm_map_t original_map = target_map;
1888 vm_map_size_t total_size;
1889 vm_map_size_t map_size;
1890 vm_map_offset_t map_offset;
1891 vm_map_offset_t local_offset;
1892 vm_object_size_t mappable_size;
1893
1894 /*
1895 * Stash the offset in the page for use by vm_map_enter_mem_object()
1896 * in the VM_FLAGS_RETURN_DATA_ADDR/MAP_MEM_USE_DATA_ADDR case.
1897 */
1898 vm_object_offset_t offset_in_page;
1899
1900 unsigned int access;
1901 vm_prot_t protections;
1902 vm_prot_t original_protections, mask_protections;
1903 unsigned int wimg_mode;
1904
1905 boolean_t force_shadow = FALSE;
1906 boolean_t use_data_addr;
1907
1908 if (((permission & 0x00FF0000) &
1909 ~(MAP_MEM_ONLY |
1910 MAP_MEM_NAMED_CREATE |
1911 MAP_MEM_PURGABLE |
1912 MAP_MEM_NAMED_REUSE |
1913 MAP_MEM_USE_DATA_ADDR |
1914 MAP_MEM_VM_COPY |
1915 MAP_MEM_VM_SHARE))) {
1916 /*
1917 * Unknown flag: reject for forward compatibility.
1918 */
1919 return KERN_INVALID_VALUE;
1920 }
1921
1922 if (parent_handle != IP_NULL &&
1923 ip_kotype(parent_handle) == IKOT_NAMED_ENTRY) {
1924 parent_entry = (vm_named_entry_t) parent_handle->ip_kobject;
1925 } else {
1926 parent_entry = NULL;
1927 }
1928
1929 if (parent_entry && parent_entry->is_copy) {
1930 return KERN_INVALID_ARGUMENT;
1931 }
1932
1933 original_protections = permission & VM_PROT_ALL;
1934 protections = original_protections;
1935 mask_protections = permission & VM_PROT_IS_MASK;
1936 access = GET_MAP_MEM(permission);
1937 use_data_addr = ((permission & MAP_MEM_USE_DATA_ADDR) != 0);
1938
1939 user_handle = IP_NULL;
1940 user_entry = NULL;
1941
1942 map_offset = vm_map_trunc_page(offset, PAGE_MASK);
1943
1944 if (permission & MAP_MEM_ONLY) {
1945 boolean_t parent_is_object;
1946
1947 map_size = vm_map_round_page(*size, PAGE_MASK);
1948
1949 if (use_data_addr || parent_entry == NULL) {
1950 return KERN_INVALID_ARGUMENT;
1951 }
1952
1953 parent_is_object = !(parent_entry->is_sub_map ||
1954 parent_entry->is_pager);
1955 object = parent_entry->backing.object;
1956 if(parent_is_object && object != VM_OBJECT_NULL)
1957 wimg_mode = object->wimg_bits;
1958 else
1959 wimg_mode = VM_WIMG_USE_DEFAULT;
1960 if((access != GET_MAP_MEM(parent_entry->protection)) &&
1961 !(parent_entry->protection & VM_PROT_WRITE)) {
1962 return KERN_INVALID_RIGHT;
1963 }
1964 if(access == MAP_MEM_IO) {
1965 SET_MAP_MEM(access, parent_entry->protection);
1966 wimg_mode = VM_WIMG_IO;
1967 } else if (access == MAP_MEM_COPYBACK) {
1968 SET_MAP_MEM(access, parent_entry->protection);
1969 wimg_mode = VM_WIMG_USE_DEFAULT;
1970 } else if (access == MAP_MEM_INNERWBACK) {
1971 SET_MAP_MEM(access, parent_entry->protection);
1972 wimg_mode = VM_WIMG_INNERWBACK;
1973 } else if (access == MAP_MEM_WTHRU) {
1974 SET_MAP_MEM(access, parent_entry->protection);
1975 wimg_mode = VM_WIMG_WTHRU;
1976 } else if (access == MAP_MEM_WCOMB) {
1977 SET_MAP_MEM(access, parent_entry->protection);
1978 wimg_mode = VM_WIMG_WCOMB;
1979 }
1980 if (parent_is_object && object &&
1981 (access != MAP_MEM_NOOP) &&
1982 (!(object->nophyscache))) {
1983
1984 if (object->wimg_bits != wimg_mode) {
1985 vm_object_lock(object);
1986 vm_object_change_wimg_mode(object, wimg_mode);
1987 vm_object_unlock(object);
1988 }
1989 }
1990 if (object_handle)
1991 *object_handle = IP_NULL;
1992 return KERN_SUCCESS;
1993 } else if (permission & MAP_MEM_NAMED_CREATE) {
1994 map_size = vm_map_round_page(*size, PAGE_MASK);
1995
1996 if (use_data_addr) {
1997 return KERN_INVALID_ARGUMENT;
1998 }
1999
2000 kr = mach_memory_entry_allocate(&user_entry, &user_handle);
2001 if (kr != KERN_SUCCESS) {
2002 return KERN_FAILURE;
2003 }
2004
2005 /*
2006 * Force the creation of the VM object now.
2007 */
2008 if (map_size > (vm_map_size_t) ANON_MAX_SIZE) {
2009 /*
2010 * LP64todo - for now, we can only allocate 4GB-4096
2011 * internal objects because the default pager can't
2012 * page bigger ones. Remove this when it can.
2013 */
2014 kr = KERN_FAILURE;
2015 goto make_mem_done;
2016 }
2017
2018 object = vm_object_allocate(map_size);
2019 assert(object != VM_OBJECT_NULL);
2020
2021 if (permission & MAP_MEM_PURGABLE) {
2022 if (! (permission & VM_PROT_WRITE)) {
2023 /* if we can't write, we can't purge */
2024 vm_object_deallocate(object);
2025 kr = KERN_INVALID_ARGUMENT;
2026 goto make_mem_done;
2027 }
2028 object->purgable = VM_PURGABLE_NONVOLATILE;
2029 assert(object->vo_purgeable_owner == NULL);
2030 assert(object->resident_page_count == 0);
2031 assert(object->wired_page_count == 0);
2032 vm_object_lock(object);
2033 vm_purgeable_nonvolatile_enqueue(object,
2034 current_task());
2035 vm_object_unlock(object);
2036 }
2037
2038 /*
2039 * The VM object is brand new and nobody else knows about it,
2040 * so we don't need to lock it.
2041 */
2042
2043 wimg_mode = object->wimg_bits;
2044 if (access == MAP_MEM_IO) {
2045 wimg_mode = VM_WIMG_IO;
2046 } else if (access == MAP_MEM_COPYBACK) {
2047 wimg_mode = VM_WIMG_USE_DEFAULT;
2048 } else if (access == MAP_MEM_INNERWBACK) {
2049 wimg_mode = VM_WIMG_INNERWBACK;
2050 } else if (access == MAP_MEM_WTHRU) {
2051 wimg_mode = VM_WIMG_WTHRU;
2052 } else if (access == MAP_MEM_WCOMB) {
2053 wimg_mode = VM_WIMG_WCOMB;
2054 }
2055 if (access != MAP_MEM_NOOP) {
2056 object->wimg_bits = wimg_mode;
2057 }
2058 /* the object has no pages, so no WIMG bits to update here */
2059
2060 /*
2061 * XXX
2062 * We use this path when we want to make sure that
2063 * nobody messes with the object (coalesce, for
2064 * example) before we map it.
2065 * We might want to use these objects for transposition via
2066 * vm_object_transpose() too, so we don't want any copy or
2067 * shadow objects either...
2068 */
2069 object->copy_strategy = MEMORY_OBJECT_COPY_NONE;
2070 object->true_share = TRUE;
2071
2072 user_entry->backing.object = object;
2073 user_entry->internal = TRUE;
2074 user_entry->is_sub_map = FALSE;
2075 user_entry->is_pager = FALSE;
2076 user_entry->offset = 0;
2077 user_entry->data_offset = 0;
2078 user_entry->protection = protections;
2079 SET_MAP_MEM(access, user_entry->protection);
2080 user_entry->size = map_size;
2081
2082 /* user_object pager and internal fields are not used */
2083 /* when the object field is filled in. */
2084
2085 *size = CAST_DOWN(vm_size_t, map_size);
2086 *object_handle = user_handle;
2087 return KERN_SUCCESS;
2088 }
2089
2090 if (permission & MAP_MEM_VM_COPY) {
2091 vm_map_copy_t copy;
2092
2093 if (target_map == VM_MAP_NULL) {
2094 return KERN_INVALID_TASK;
2095 }
2096
2097 if (use_data_addr) {
2098 map_size = (vm_map_round_page(offset + *size,
2099 PAGE_MASK) -
2100 map_offset);
2101 offset_in_page = offset - map_offset;
2102 } else {
2103 map_size = vm_map_round_page(*size, PAGE_MASK);
2104 offset_in_page = 0;
2105 }
2106
2107 kr = vm_map_copyin(target_map,
2108 map_offset,
2109 map_size,
2110 FALSE,
2111 &copy);
2112 if (kr != KERN_SUCCESS) {
2113 return kr;
2114 }
2115
2116 kr = mach_memory_entry_allocate(&user_entry, &user_handle);
2117 if (kr != KERN_SUCCESS) {
2118 vm_map_copy_discard(copy);
2119 return KERN_FAILURE;
2120 }
2121
2122 user_entry->backing.copy = copy;
2123 user_entry->internal = FALSE;
2124 user_entry->is_sub_map = FALSE;
2125 user_entry->is_pager = FALSE;
2126 user_entry->is_copy = TRUE;
2127 user_entry->offset = 0;
2128 user_entry->protection = protections;
2129 user_entry->size = map_size;
2130 user_entry->data_offset = offset_in_page;
2131
2132 *size = CAST_DOWN(vm_size_t, map_size);
2133 *object_handle = user_handle;
2134 return KERN_SUCCESS;
2135 }
2136
2137 if (permission & MAP_MEM_VM_SHARE) {
2138 vm_map_copy_t copy;
2139 vm_prot_t cur_prot, max_prot;
2140
2141 if (target_map == VM_MAP_NULL) {
2142 return KERN_INVALID_TASK;
2143 }
2144
2145 if (use_data_addr) {
2146 map_size = (vm_map_round_page(offset + *size,
2147 PAGE_MASK) -
2148 map_offset);
2149 offset_in_page = offset - map_offset;
2150 } else {
2151 map_size = vm_map_round_page(*size, PAGE_MASK);
2152 offset_in_page = 0;
2153 }
2154
2155 kr = vm_map_copy_extract(target_map,
2156 map_offset,
2157 map_size,
2158 &copy,
2159 &cur_prot,
2160 &max_prot);
2161 if (kr != KERN_SUCCESS) {
2162 return kr;
2163 }
2164
2165 if (mask_protections) {
2166 /*
2167 * We just want as much of "original_protections"
2168 * as we can get out of the actual "cur_prot".
2169 */
2170 protections &= cur_prot;
2171 if (protections == VM_PROT_NONE) {
2172 /* no access at all: fail */
2173 vm_map_copy_discard(copy);
2174 return KERN_PROTECTION_FAILURE;
2175 }
2176 } else {
2177 /*
2178 * We want exactly "original_protections"
2179 * out of "cur_prot".
2180 */
2181 if ((cur_prot & protections) != protections) {
2182 vm_map_copy_discard(copy);
2183 return KERN_PROTECTION_FAILURE;
2184 }
2185 }
2186
2187 kr = mach_memory_entry_allocate(&user_entry, &user_handle);
2188 if (kr != KERN_SUCCESS) {
2189 vm_map_copy_discard(copy);
2190 return KERN_FAILURE;
2191 }
2192
2193 user_entry->backing.copy = copy;
2194 user_entry->internal = FALSE;
2195 user_entry->is_sub_map = FALSE;
2196 user_entry->is_pager = FALSE;
2197 user_entry->is_copy = TRUE;
2198 user_entry->offset = 0;
2199 user_entry->protection = protections;
2200 user_entry->size = map_size;
2201 user_entry->data_offset = offset_in_page;
2202
2203 *size = CAST_DOWN(vm_size_t, map_size);
2204 *object_handle = user_handle;
2205 return KERN_SUCCESS;
2206 }
2207
2208 if (parent_entry == NULL ||
2209 (permission & MAP_MEM_NAMED_REUSE)) {
2210
2211 if (use_data_addr) {
2212 map_size = vm_map_round_page(offset + *size, PAGE_MASK) - map_offset;
2213 offset_in_page = offset - map_offset;
2214 } else {
2215 map_size = vm_map_round_page(*size, PAGE_MASK);
2216 offset_in_page = 0;
2217 }
2218
2219 /* Create a named object based on address range within the task map */
2220 /* Go find the object at given address */
2221
2222 if (target_map == VM_MAP_NULL) {
2223 return KERN_INVALID_TASK;
2224 }
2225
2226 redo_lookup:
2227 protections = original_protections;
2228 vm_map_lock_read(target_map);
2229
2230 /* get the object associated with the target address */
2231 /* note we check the permission of the range against */
2232 /* that requested by the caller */
2233
2234 kr = vm_map_lookup_locked(&target_map, map_offset,
2235 protections | mask_protections,
2236 OBJECT_LOCK_EXCLUSIVE, &version,
2237 &object, &obj_off, &prot, &wired,
2238 &fault_info,
2239 &real_map);
2240 if (kr != KERN_SUCCESS) {
2241 vm_map_unlock_read(target_map);
2242 goto make_mem_done;
2243 }
2244 if (mask_protections) {
2245 /*
2246 * The caller asked us to use the "protections" as
2247 * a mask, so restrict "protections" to what this
2248 * mapping actually allows.
2249 */
2250 protections &= prot;
2251 }
2252 if (((prot & protections) != protections)
2253 || (object == kernel_object)) {
2254 kr = KERN_INVALID_RIGHT;
2255 vm_object_unlock(object);
2256 vm_map_unlock_read(target_map);
2257 if(real_map != target_map)
2258 vm_map_unlock_read(real_map);
2259 if(object == kernel_object) {
2260 printf("Warning: Attempt to create a named"
2261 " entry from the kernel_object\n");
2262 }
2263 goto make_mem_done;
2264 }
2265
2266 /* We have an object, now check to see if this object */
2267 /* is suitable. If not, create a shadow and share that */
2268
2269 /*
2270 * We have to unlock the VM object to avoid deadlocking with
2271 * a VM map lock (the lock ordering is map, the object), if we
2272 * need to modify the VM map to create a shadow object. Since
2273 * we might release the VM map lock below anyway, we have
2274 * to release the VM map lock now.
2275 * XXX FBDP There must be a way to avoid this double lookup...
2276 *
2277 * Take an extra reference on the VM object to make sure it's
2278 * not going to disappear.
2279 */
2280 vm_object_reference_locked(object); /* extra ref to hold obj */
2281 vm_object_unlock(object);
2282
2283 local_map = original_map;
2284 local_offset = map_offset;
2285 if(target_map != local_map) {
2286 vm_map_unlock_read(target_map);
2287 if(real_map != target_map)
2288 vm_map_unlock_read(real_map);
2289 vm_map_lock_read(local_map);
2290 target_map = local_map;
2291 real_map = local_map;
2292 }
2293 while(TRUE) {
2294 if(!vm_map_lookup_entry(local_map,
2295 local_offset, &map_entry)) {
2296 kr = KERN_INVALID_ARGUMENT;
2297 vm_map_unlock_read(target_map);
2298 if(real_map != target_map)
2299 vm_map_unlock_read(real_map);
2300 vm_object_deallocate(object); /* release extra ref */
2301 object = VM_OBJECT_NULL;
2302 goto make_mem_done;
2303 }
2304 if(!(map_entry->is_sub_map)) {
2305 if(map_entry->object.vm_object != object) {
2306 kr = KERN_INVALID_ARGUMENT;
2307 vm_map_unlock_read(target_map);
2308 if(real_map != target_map)
2309 vm_map_unlock_read(real_map);
2310 vm_object_deallocate(object); /* release extra ref */
2311 object = VM_OBJECT_NULL;
2312 goto make_mem_done;
2313 }
2314 break;
2315 } else {
2316 vm_map_t tmap;
2317 tmap = local_map;
2318 local_map = map_entry->object.sub_map;
2319
2320 vm_map_lock_read(local_map);
2321 vm_map_unlock_read(tmap);
2322 target_map = local_map;
2323 real_map = local_map;
2324 local_offset = local_offset - map_entry->vme_start;
2325 local_offset += map_entry->offset;
2326 }
2327 }
2328
2329 /*
2330 * We found the VM map entry, lock the VM object again.
2331 */
2332 vm_object_lock(object);
2333 if(map_entry->wired_count) {
2334 /* JMM - The check below should be reworked instead. */
2335 object->true_share = TRUE;
2336 }
2337 if (mask_protections) {
2338 /*
2339 * The caller asked us to use the "protections" as
2340 * a mask, so restrict "protections" to what this
2341 * mapping actually allows.
2342 */
2343 protections &= map_entry->max_protection;
2344 }
2345 if(((map_entry->max_protection) & protections) != protections) {
2346 kr = KERN_INVALID_RIGHT;
2347 vm_object_unlock(object);
2348 vm_map_unlock_read(target_map);
2349 if(real_map != target_map)
2350 vm_map_unlock_read(real_map);
2351 vm_object_deallocate(object);
2352 object = VM_OBJECT_NULL;
2353 goto make_mem_done;
2354 }
2355
2356 mappable_size = fault_info.hi_offset - obj_off;
2357 total_size = map_entry->vme_end - map_entry->vme_start;
2358 if(map_size > mappable_size) {
2359 /* try to extend mappable size if the entries */
2360 /* following are from the same object and are */
2361 /* compatible */
2362 next_entry = map_entry->vme_next;
2363 /* lets see if the next map entry is still */
2364 /* pointing at this object and is contiguous */
2365 while(map_size > mappable_size) {
2366 if((next_entry->object.vm_object == object) &&
2367 (next_entry->vme_start ==
2368 next_entry->vme_prev->vme_end) &&
2369 (next_entry->offset ==
2370 next_entry->vme_prev->offset +
2371 (next_entry->vme_prev->vme_end -
2372 next_entry->vme_prev->vme_start))) {
2373 if (mask_protections) {
2374 /*
2375 * The caller asked us to use
2376 * the "protections" as a mask,
2377 * so restrict "protections" to
2378 * what this mapping actually
2379 * allows.
2380 */
2381 protections &= next_entry->max_protection;
2382 }
2383 if ((next_entry->wired_count) &&
2384 (map_entry->wired_count == 0)) {
2385 break;
2386 }
2387 if(((next_entry->max_protection)
2388 & protections) != protections) {
2389 break;
2390 }
2391 if (next_entry->needs_copy !=
2392 map_entry->needs_copy)
2393 break;
2394 mappable_size += next_entry->vme_end
2395 - next_entry->vme_start;
2396 total_size += next_entry->vme_end
2397 - next_entry->vme_start;
2398 next_entry = next_entry->vme_next;
2399 } else {
2400 break;
2401 }
2402
2403 }
2404 }
2405
2406 if (vm_map_entry_should_cow_for_true_share(map_entry) &&
2407 object->vo_size > map_size &&
2408 map_size != 0) {
2409 /*
2410 * Set up the targeted range for copy-on-write to
2411 * limit the impact of "true_share"/"copy_delay" to
2412 * that range instead of the entire VM object...
2413 */
2414
2415 vm_object_unlock(object);
2416 if (vm_map_lock_read_to_write(target_map)) {
2417 vm_object_deallocate(object);
2418 target_map = original_map;
2419 goto redo_lookup;
2420 }
2421
2422 vm_map_clip_start(target_map,
2423 map_entry,
2424 vm_map_trunc_page(offset,
2425 VM_MAP_PAGE_MASK(target_map)));
2426 vm_map_clip_end(target_map,
2427 map_entry,
2428 (vm_map_round_page(offset + map_size,
2429 VM_MAP_PAGE_MASK(target_map))));
2430 force_shadow = TRUE;
2431
2432 if ((map_entry->vme_end - offset) < map_size) {
2433 map_size = map_entry->vme_end - offset;
2434 }
2435 total_size = map_entry->vme_end - map_entry->vme_start;
2436
2437 vm_map_lock_write_to_read(target_map);
2438 vm_object_lock(object);
2439 }
2440
2441 if (object->internal) {
2442 /* vm_map_lookup_locked will create a shadow if */
2443 /* needs_copy is set but does not check for the */
2444 /* other two conditions shown. It is important to */
2445 /* set up an object which will not be pulled from */
2446 /* under us. */
2447
2448 if (force_shadow ||
2449 ((map_entry->needs_copy ||
2450 object->shadowed ||
2451 (object->vo_size > total_size &&
2452 (map_entry->offset != 0 ||
2453 object->vo_size >
2454 vm_map_round_page(total_size,
2455 VM_MAP_PAGE_MASK(target_map)))))
2456 && !object->true_share)) {
2457 /*
2458 * We have to unlock the VM object before
2459 * trying to upgrade the VM map lock, to
2460 * honor lock ordering (map then object).
2461 * Otherwise, we would deadlock if another
2462 * thread holds a read lock on the VM map and
2463 * is trying to acquire the VM object's lock.
2464 * We still hold an extra reference on the
2465 * VM object, guaranteeing that it won't
2466 * disappear.
2467 */
2468 vm_object_unlock(object);
2469
2470 if (vm_map_lock_read_to_write(target_map)) {
2471 /*
2472 * We couldn't upgrade our VM map lock
2473 * from "read" to "write" and we lost
2474 * our "read" lock.
2475 * Start all over again...
2476 */
2477 vm_object_deallocate(object); /* extra ref */
2478 target_map = original_map;
2479 goto redo_lookup;
2480 }
2481 #if 00
2482 vm_object_lock(object);
2483 #endif
2484
2485 /*
2486 * JMM - We need to avoid coming here when the object
2487 * is wired by anybody, not just the current map. Why
2488 * couldn't we use the standard vm_object_copy_quickly()
2489 * approach here?
2490 */
2491
2492 /* create a shadow object */
2493 vm_object_shadow(&map_entry->object.vm_object,
2494 &map_entry->offset, total_size);
2495 shadow_object = map_entry->object.vm_object;
2496 #if 00
2497 vm_object_unlock(object);
2498 #endif
2499
2500 prot = map_entry->protection & ~VM_PROT_WRITE;
2501
2502 if (override_nx(target_map, map_entry->alias) && prot)
2503 prot |= VM_PROT_EXECUTE;
2504
2505 vm_object_pmap_protect(
2506 object, map_entry->offset,
2507 total_size,
2508 ((map_entry->is_shared
2509 || target_map->mapped_in_other_pmaps)
2510 ? PMAP_NULL :
2511 target_map->pmap),
2512 map_entry->vme_start,
2513 prot);
2514 total_size -= (map_entry->vme_end
2515 - map_entry->vme_start);
2516 next_entry = map_entry->vme_next;
2517 map_entry->needs_copy = FALSE;
2518
2519 vm_object_lock(shadow_object);
2520 while (total_size) {
2521 assert((next_entry->wired_count == 0) ||
2522 (map_entry->wired_count));
2523
2524 if(next_entry->object.vm_object == object) {
2525 vm_object_reference_locked(shadow_object);
2526 next_entry->object.vm_object
2527 = shadow_object;
2528 vm_object_deallocate(object);
2529 next_entry->offset
2530 = next_entry->vme_prev->offset +
2531 (next_entry->vme_prev->vme_end
2532 - next_entry->vme_prev->vme_start);
2533 next_entry->needs_copy = FALSE;
2534 } else {
2535 panic("mach_make_memory_entry_64:"
2536 " map entries out of sync\n");
2537 }
2538 total_size -=
2539 next_entry->vme_end
2540 - next_entry->vme_start;
2541 next_entry = next_entry->vme_next;
2542 }
2543
2544 /*
2545 * Transfer our extra reference to the
2546 * shadow object.
2547 */
2548 vm_object_reference_locked(shadow_object);
2549 vm_object_deallocate(object); /* extra ref */
2550 object = shadow_object;
2551
2552 obj_off = (local_offset - map_entry->vme_start)
2553 + map_entry->offset;
2554
2555 vm_map_lock_write_to_read(target_map);
2556 }
2557 }
2558
2559 /* note: in the future we can (if necessary) allow for */
2560 /* memory object lists, this will better support */
2561 /* fragmentation, but is it necessary? The user should */
2562 /* be encouraged to create address space oriented */
2563 /* shared objects from CLEAN memory regions which have */
2564 /* a known and defined history. i.e. no inheritence */
2565 /* share, make this call before making the region the */
2566 /* target of ipc's, etc. The code above, protecting */
2567 /* against delayed copy, etc. is mostly defensive. */
2568
2569 wimg_mode = object->wimg_bits;
2570 if(!(object->nophyscache)) {
2571 if(access == MAP_MEM_IO) {
2572 wimg_mode = VM_WIMG_IO;
2573 } else if (access == MAP_MEM_COPYBACK) {
2574 wimg_mode = VM_WIMG_USE_DEFAULT;
2575 } else if (access == MAP_MEM_INNERWBACK) {
2576 wimg_mode = VM_WIMG_INNERWBACK;
2577 } else if (access == MAP_MEM_WTHRU) {
2578 wimg_mode = VM_WIMG_WTHRU;
2579 } else if (access == MAP_MEM_WCOMB) {
2580 wimg_mode = VM_WIMG_WCOMB;
2581 }
2582 }
2583
2584 #if VM_OBJECT_TRACKING_OP_TRUESHARE
2585 if (!object->true_share &&
2586 vm_object_tracking_inited) {
2587 void *bt[VM_OBJECT_TRACKING_BTDEPTH];
2588 int num = 0;
2589
2590 num = OSBacktrace(bt,
2591 VM_OBJECT_TRACKING_BTDEPTH);
2592 btlog_add_entry(vm_object_tracking_btlog,
2593 object,
2594 VM_OBJECT_TRACKING_OP_TRUESHARE,
2595 bt,
2596 num);
2597 }
2598 #endif /* VM_OBJECT_TRACKING_OP_TRUESHARE */
2599
2600 object->true_share = TRUE;
2601 if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC)
2602 object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
2603
2604 /*
2605 * The memory entry now points to this VM object and we
2606 * need to hold a reference on the VM object. Use the extra
2607 * reference we took earlier to keep the object alive when we
2608 * had to unlock it.
2609 */
2610
2611 vm_map_unlock_read(target_map);
2612 if(real_map != target_map)
2613 vm_map_unlock_read(real_map);
2614
2615 if (object->wimg_bits != wimg_mode)
2616 vm_object_change_wimg_mode(object, wimg_mode);
2617
2618 /* the size of mapped entry that overlaps with our region */
2619 /* which is targeted for share. */
2620 /* (entry_end - entry_start) - */
2621 /* offset of our beg addr within entry */
2622 /* it corresponds to this: */
2623
2624 if(map_size > mappable_size)
2625 map_size = mappable_size;
2626
2627 if (permission & MAP_MEM_NAMED_REUSE) {
2628 /*
2629 * Compare what we got with the "parent_entry".
2630 * If they match, re-use the "parent_entry" instead
2631 * of creating a new one.
2632 */
2633 if (parent_entry != NULL &&
2634 parent_entry->backing.object == object &&
2635 parent_entry->internal == object->internal &&
2636 parent_entry->is_sub_map == FALSE &&
2637 parent_entry->is_pager == FALSE &&
2638 parent_entry->offset == obj_off &&
2639 parent_entry->protection == protections &&
2640 parent_entry->size == map_size &&
2641 ((!use_data_addr && (parent_entry->data_offset == 0)) ||
2642 (use_data_addr && (parent_entry->data_offset == offset_in_page)))) {
2643 /*
2644 * We have a match: re-use "parent_entry".
2645 */
2646 /* release our extra reference on object */
2647 vm_object_unlock(object);
2648 vm_object_deallocate(object);
2649 /* parent_entry->ref_count++; XXX ? */
2650 /* Get an extra send-right on handle */
2651 ipc_port_copy_send(parent_handle);
2652
2653 *size = CAST_DOWN(vm_size_t, map_size);
2654 *object_handle = parent_handle;
2655 return KERN_SUCCESS;
2656 } else {
2657 /*
2658 * No match: we need to create a new entry.
2659 * fall through...
2660 */
2661 }
2662 }
2663
2664 vm_object_unlock(object);
2665 if (mach_memory_entry_allocate(&user_entry, &user_handle)
2666 != KERN_SUCCESS) {
2667 /* release our unused reference on the object */
2668 vm_object_deallocate(object);
2669 return KERN_FAILURE;
2670 }
2671
2672 user_entry->backing.object = object;
2673 user_entry->internal = object->internal;
2674 user_entry->is_sub_map = FALSE;
2675 user_entry->is_pager = FALSE;
2676 user_entry->offset = obj_off;
2677 user_entry->data_offset = offset_in_page;
2678 user_entry->protection = protections;
2679 SET_MAP_MEM(GET_MAP_MEM(permission), user_entry->protection);
2680 user_entry->size = map_size;
2681
2682 /* user_object pager and internal fields are not used */
2683 /* when the object field is filled in. */
2684
2685 *size = CAST_DOWN(vm_size_t, map_size);
2686 *object_handle = user_handle;
2687 return KERN_SUCCESS;
2688
2689 } else {
2690 /* The new object will be base on an existing named object */
2691 if (parent_entry == NULL) {
2692 kr = KERN_INVALID_ARGUMENT;
2693 goto make_mem_done;
2694 }
2695
2696 if (use_data_addr) {
2697 /*
2698 * submaps and pagers should only be accessible from within
2699 * the kernel, which shouldn't use the data address flag, so can fail here.
2700 */
2701 if (parent_entry->is_pager || parent_entry->is_sub_map) {
2702 panic("Shouldn't be using data address with a parent entry that is a submap or pager.");
2703 }
2704 /*
2705 * Account for offset to data in parent entry and
2706 * compute our own offset to data.
2707 */
2708 if((offset + *size + parent_entry->data_offset) > parent_entry->size) {
2709 kr = KERN_INVALID_ARGUMENT;
2710 goto make_mem_done;
2711 }
2712
2713 map_offset = vm_map_trunc_page(offset + parent_entry->data_offset, PAGE_MASK);
2714 offset_in_page = (offset + parent_entry->data_offset) - map_offset;
2715 map_size = vm_map_round_page(offset + parent_entry->data_offset + *size, PAGE_MASK) - map_offset;
2716 } else {
2717 map_size = vm_map_round_page(*size, PAGE_MASK);
2718 offset_in_page = 0;
2719
2720 if((offset + map_size) > parent_entry->size) {
2721 kr = KERN_INVALID_ARGUMENT;
2722 goto make_mem_done;
2723 }
2724 }
2725
2726 if (mask_protections) {
2727 /*
2728 * The caller asked us to use the "protections" as
2729 * a mask, so restrict "protections" to what this
2730 * mapping actually allows.
2731 */
2732 protections &= parent_entry->protection;
2733 }
2734 if((protections & parent_entry->protection) != protections) {
2735 kr = KERN_PROTECTION_FAILURE;
2736 goto make_mem_done;
2737 }
2738
2739 if (mach_memory_entry_allocate(&user_entry, &user_handle)
2740 != KERN_SUCCESS) {
2741 kr = KERN_FAILURE;
2742 goto make_mem_done;
2743 }
2744
2745 user_entry->size = map_size;
2746 user_entry->offset = parent_entry->offset + map_offset;
2747 user_entry->data_offset = offset_in_page;
2748 user_entry->is_sub_map = parent_entry->is_sub_map;
2749 user_entry->is_pager = parent_entry->is_pager;
2750 user_entry->is_copy = parent_entry->is_copy;
2751 user_entry->internal = parent_entry->internal;
2752 user_entry->protection = protections;
2753
2754 if(access != MAP_MEM_NOOP) {
2755 SET_MAP_MEM(access, user_entry->protection);
2756 }
2757
2758 if(parent_entry->is_sub_map) {
2759 user_entry->backing.map = parent_entry->backing.map;
2760 vm_map_lock(user_entry->backing.map);
2761 user_entry->backing.map->ref_count++;
2762 vm_map_unlock(user_entry->backing.map);
2763 }
2764 else if (parent_entry->is_pager) {
2765 user_entry->backing.pager = parent_entry->backing.pager;
2766 /* JMM - don't we need a reference here? */
2767 } else {
2768 object = parent_entry->backing.object;
2769 assert(object != VM_OBJECT_NULL);
2770 user_entry->backing.object = object;
2771 /* we now point to this object, hold on */
2772 vm_object_reference(object);
2773 vm_object_lock(object);
2774 #if VM_OBJECT_TRACKING_OP_TRUESHARE
2775 if (!object->true_share &&
2776 vm_object_tracking_inited) {
2777 void *bt[VM_OBJECT_TRACKING_BTDEPTH];
2778 int num = 0;
2779
2780 num = OSBacktrace(bt,
2781 VM_OBJECT_TRACKING_BTDEPTH);
2782 btlog_add_entry(vm_object_tracking_btlog,
2783 object,
2784 VM_OBJECT_TRACKING_OP_TRUESHARE,
2785 bt,
2786 num);
2787 }
2788 #endif /* VM_OBJECT_TRACKING_OP_TRUESHARE */
2789
2790 object->true_share = TRUE;
2791 if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC)
2792 object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
2793 vm_object_unlock(object);
2794 }
2795 *size = CAST_DOWN(vm_size_t, map_size);
2796 *object_handle = user_handle;
2797 return KERN_SUCCESS;
2798 }
2799
2800 make_mem_done:
2801 if (user_handle != IP_NULL) {
2802 /*
2803 * Releasing "user_handle" causes the kernel object
2804 * associated with it ("user_entry" here) to also be
2805 * released and freed.
2806 */
2807 mach_memory_entry_port_release(user_handle);
2808 }
2809 return kr;
2810 }
2811
2812 kern_return_t
2813 _mach_make_memory_entry(
2814 vm_map_t target_map,
2815 memory_object_size_t *size,
2816 memory_object_offset_t offset,
2817 vm_prot_t permission,
2818 ipc_port_t *object_handle,
2819 ipc_port_t parent_entry)
2820 {
2821 memory_object_size_t mo_size;
2822 kern_return_t kr;
2823
2824 mo_size = (memory_object_size_t)*size;
2825 kr = mach_make_memory_entry_64(target_map, &mo_size,
2826 (memory_object_offset_t)offset, permission, object_handle,
2827 parent_entry);
2828 *size = mo_size;
2829 return kr;
2830 }
2831
2832 kern_return_t
2833 mach_make_memory_entry(
2834 vm_map_t target_map,
2835 vm_size_t *size,
2836 vm_offset_t offset,
2837 vm_prot_t permission,
2838 ipc_port_t *object_handle,
2839 ipc_port_t parent_entry)
2840 {
2841 memory_object_size_t mo_size;
2842 kern_return_t kr;
2843
2844 mo_size = (memory_object_size_t)*size;
2845 kr = mach_make_memory_entry_64(target_map, &mo_size,
2846 (memory_object_offset_t)offset, permission, object_handle,
2847 parent_entry);
2848 *size = CAST_DOWN(vm_size_t, mo_size);
2849 return kr;
2850 }
2851
2852 /*
2853 * task_wire
2854 *
2855 * Set or clear the map's wiring_required flag. This flag, if set,
2856 * will cause all future virtual memory allocation to allocate
2857 * user wired memory. Unwiring pages wired down as a result of
2858 * this routine is done with the vm_wire interface.
2859 */
2860 kern_return_t
2861 task_wire(
2862 vm_map_t map,
2863 boolean_t must_wire)
2864 {
2865 if (map == VM_MAP_NULL)
2866 return(KERN_INVALID_ARGUMENT);
2867
2868 if (must_wire)
2869 map->wiring_required = TRUE;
2870 else
2871 map->wiring_required = FALSE;
2872
2873 return(KERN_SUCCESS);
2874 }
2875
2876 __private_extern__ kern_return_t
2877 mach_memory_entry_allocate(
2878 vm_named_entry_t *user_entry_p,
2879 ipc_port_t *user_handle_p)
2880 {
2881 vm_named_entry_t user_entry;
2882 ipc_port_t user_handle;
2883 ipc_port_t previous;
2884
2885 user_entry = (vm_named_entry_t) kalloc(sizeof *user_entry);
2886 if (user_entry == NULL)
2887 return KERN_FAILURE;
2888
2889 named_entry_lock_init(user_entry);
2890
2891 user_handle = ipc_port_alloc_kernel();
2892 if (user_handle == IP_NULL) {
2893 kfree(user_entry, sizeof *user_entry);
2894 return KERN_FAILURE;
2895 }
2896 ip_lock(user_handle);
2897
2898 /* make a sonce right */
2899 user_handle->ip_sorights++;
2900 ip_reference(user_handle);
2901
2902 user_handle->ip_destination = IP_NULL;
2903 user_handle->ip_receiver_name = MACH_PORT_NULL;
2904 user_handle->ip_receiver = ipc_space_kernel;
2905
2906 /* make a send right */
2907 user_handle->ip_mscount++;
2908 user_handle->ip_srights++;
2909 ip_reference(user_handle);
2910
2911 ipc_port_nsrequest(user_handle, 1, user_handle, &previous);
2912 /* nsrequest unlocks user_handle */
2913
2914 user_entry->backing.pager = NULL;
2915 user_entry->is_sub_map = FALSE;
2916 user_entry->is_pager = FALSE;
2917 user_entry->is_copy = FALSE;
2918 user_entry->internal = FALSE;
2919 user_entry->size = 0;
2920 user_entry->offset = 0;
2921 user_entry->data_offset = 0;
2922 user_entry->protection = VM_PROT_NONE;
2923 user_entry->ref_count = 1;
2924
2925 ipc_kobject_set(user_handle, (ipc_kobject_t) user_entry,
2926 IKOT_NAMED_ENTRY);
2927
2928 *user_entry_p = user_entry;
2929 *user_handle_p = user_handle;
2930
2931 return KERN_SUCCESS;
2932 }
2933
2934 /*
2935 * mach_memory_object_memory_entry_64
2936 *
2937 * Create a named entry backed by the provided pager.
2938 *
2939 * JMM - we need to hold a reference on the pager -
2940 * and release it when the named entry is destroyed.
2941 */
2942 kern_return_t
2943 mach_memory_object_memory_entry_64(
2944 host_t host,
2945 boolean_t internal,
2946 vm_object_offset_t size,
2947 vm_prot_t permission,
2948 memory_object_t pager,
2949 ipc_port_t *entry_handle)
2950 {
2951 unsigned int access;
2952 vm_named_entry_t user_entry;
2953 ipc_port_t user_handle;
2954
2955 if (host == HOST_NULL)
2956 return(KERN_INVALID_HOST);
2957
2958 if (mach_memory_entry_allocate(&user_entry, &user_handle)
2959 != KERN_SUCCESS) {
2960 return KERN_FAILURE;
2961 }
2962
2963 user_entry->backing.pager = pager;
2964 user_entry->size = size;
2965 user_entry->offset = 0;
2966 user_entry->protection = permission & VM_PROT_ALL;
2967 access = GET_MAP_MEM(permission);
2968 SET_MAP_MEM(access, user_entry->protection);
2969 user_entry->internal = internal;
2970 user_entry->is_sub_map = FALSE;
2971 user_entry->is_pager = TRUE;
2972 assert(user_entry->ref_count == 1);
2973
2974 *entry_handle = user_handle;
2975 return KERN_SUCCESS;
2976 }
2977
2978 kern_return_t
2979 mach_memory_object_memory_entry(
2980 host_t host,
2981 boolean_t internal,
2982 vm_size_t size,
2983 vm_prot_t permission,
2984 memory_object_t pager,
2985 ipc_port_t *entry_handle)
2986 {
2987 return mach_memory_object_memory_entry_64( host, internal,
2988 (vm_object_offset_t)size, permission, pager, entry_handle);
2989 }
2990
2991
2992 kern_return_t
2993 mach_memory_entry_purgable_control(
2994 ipc_port_t entry_port,
2995 vm_purgable_t control,
2996 int *state)
2997 {
2998 kern_return_t kr;
2999 vm_named_entry_t mem_entry;
3000 vm_object_t object;
3001
3002 if (entry_port == IP_NULL ||
3003 ip_kotype(entry_port) != IKOT_NAMED_ENTRY) {
3004 return KERN_INVALID_ARGUMENT;
3005 }
3006 if (control != VM_PURGABLE_SET_STATE &&
3007 control != VM_PURGABLE_GET_STATE)
3008 return(KERN_INVALID_ARGUMENT);
3009
3010 if (control == VM_PURGABLE_SET_STATE &&
3011 (((*state & ~(VM_PURGABLE_ALL_MASKS)) != 0) ||
3012 ((*state & VM_PURGABLE_STATE_MASK) > VM_PURGABLE_STATE_MASK)))
3013 return(KERN_INVALID_ARGUMENT);
3014
3015 mem_entry = (vm_named_entry_t) entry_port->ip_kobject;
3016
3017 named_entry_lock(mem_entry);
3018
3019 if (mem_entry->is_sub_map ||
3020 mem_entry->is_pager ||
3021 mem_entry->is_copy) {
3022 named_entry_unlock(mem_entry);
3023 return KERN_INVALID_ARGUMENT;
3024 }
3025
3026 object = mem_entry->backing.object;
3027 if (object == VM_OBJECT_NULL) {
3028 named_entry_unlock(mem_entry);
3029 return KERN_INVALID_ARGUMENT;
3030 }
3031
3032 vm_object_lock(object);
3033
3034 /* check that named entry covers entire object ? */
3035 if (mem_entry->offset != 0 || object->vo_size != mem_entry->size) {
3036 vm_object_unlock(object);
3037 named_entry_unlock(mem_entry);
3038 return KERN_INVALID_ARGUMENT;
3039 }
3040
3041 named_entry_unlock(mem_entry);
3042
3043 kr = vm_object_purgable_control(object, control, state);
3044
3045 vm_object_unlock(object);
3046
3047 return kr;
3048 }
3049
3050 kern_return_t
3051 mach_memory_entry_get_page_counts(
3052 ipc_port_t entry_port,
3053 unsigned int *resident_page_count,
3054 unsigned int *dirty_page_count)
3055 {
3056 kern_return_t kr;
3057 vm_named_entry_t mem_entry;
3058 vm_object_t object;
3059 vm_object_offset_t offset;
3060 vm_object_size_t size;
3061
3062 if (entry_port == IP_NULL ||
3063 ip_kotype(entry_port) != IKOT_NAMED_ENTRY) {
3064 return KERN_INVALID_ARGUMENT;
3065 }
3066
3067 mem_entry = (vm_named_entry_t) entry_port->ip_kobject;
3068
3069 named_entry_lock(mem_entry);
3070
3071 if (mem_entry->is_sub_map ||
3072 mem_entry->is_pager ||
3073 mem_entry->is_copy) {
3074 named_entry_unlock(mem_entry);
3075 return KERN_INVALID_ARGUMENT;
3076 }
3077
3078 object = mem_entry->backing.object;
3079 if (object == VM_OBJECT_NULL) {
3080 named_entry_unlock(mem_entry);
3081 return KERN_INVALID_ARGUMENT;
3082 }
3083
3084 vm_object_lock(object);
3085
3086 offset = mem_entry->offset;
3087 size = mem_entry->size;
3088
3089 named_entry_unlock(mem_entry);
3090
3091 kr = vm_object_get_page_counts(object, offset, size, resident_page_count, dirty_page_count);
3092
3093 vm_object_unlock(object);
3094
3095 return kr;
3096 }
3097
3098 /*
3099 * mach_memory_entry_port_release:
3100 *
3101 * Release a send right on a named entry port. This is the correct
3102 * way to destroy a named entry. When the last right on the port is
3103 * released, ipc_kobject_destroy() will call mach_destroy_memory_entry().
3104 */
3105 void
3106 mach_memory_entry_port_release(
3107 ipc_port_t port)
3108 {
3109 assert(ip_kotype(port) == IKOT_NAMED_ENTRY);
3110 ipc_port_release_send(port);
3111 }
3112
3113 /*
3114 * mach_destroy_memory_entry:
3115 *
3116 * Drops a reference on a memory entry and destroys the memory entry if
3117 * there are no more references on it.
3118 * NOTE: This routine should not be called to destroy a memory entry from the
3119 * kernel, as it will not release the Mach port associated with the memory
3120 * entry. The proper way to destroy a memory entry in the kernel is to
3121 * call mach_memort_entry_port_release() to release the kernel's send-right on
3122 * the memory entry's port. When the last send right is released, the memory
3123 * entry will be destroyed via ipc_kobject_destroy().
3124 */
3125 void
3126 mach_destroy_memory_entry(
3127 ipc_port_t port)
3128 {
3129 vm_named_entry_t named_entry;
3130 #if MACH_ASSERT
3131 assert(ip_kotype(port) == IKOT_NAMED_ENTRY);
3132 #endif /* MACH_ASSERT */
3133 named_entry = (vm_named_entry_t)port->ip_kobject;
3134
3135 named_entry_lock(named_entry);
3136 named_entry->ref_count -= 1;
3137
3138 if(named_entry->ref_count == 0) {
3139 if (named_entry->is_sub_map) {
3140 vm_map_deallocate(named_entry->backing.map);
3141 } else if (named_entry->is_pager) {
3142 /* JMM - need to drop reference on pager in that case */
3143 } else if (named_entry->is_copy) {
3144 vm_map_copy_discard(named_entry->backing.copy);
3145 } else {
3146 /* release the VM object we've been pointing to */
3147 vm_object_deallocate(named_entry->backing.object);
3148 }
3149
3150 named_entry_unlock(named_entry);
3151 named_entry_lock_destroy(named_entry);
3152
3153 kfree((void *) port->ip_kobject,
3154 sizeof (struct vm_named_entry));
3155 } else
3156 named_entry_unlock(named_entry);
3157 }
3158
3159 /* Allow manipulation of individual page state. This is actually part of */
3160 /* the UPL regimen but takes place on the memory entry rather than on a UPL */
3161
3162 kern_return_t
3163 mach_memory_entry_page_op(
3164 ipc_port_t entry_port,
3165 vm_object_offset_t offset,
3166 int ops,
3167 ppnum_t *phys_entry,
3168 int *flags)
3169 {
3170 vm_named_entry_t mem_entry;
3171 vm_object_t object;
3172 kern_return_t kr;
3173
3174 if (entry_port == IP_NULL ||
3175 ip_kotype(entry_port) != IKOT_NAMED_ENTRY) {
3176 return KERN_INVALID_ARGUMENT;
3177 }
3178
3179 mem_entry = (vm_named_entry_t) entry_port->ip_kobject;
3180
3181 named_entry_lock(mem_entry);
3182
3183 if (mem_entry->is_sub_map ||
3184 mem_entry->is_pager ||
3185 mem_entry->is_copy) {
3186 named_entry_unlock(mem_entry);
3187 return KERN_INVALID_ARGUMENT;
3188 }
3189
3190 object = mem_entry->backing.object;
3191 if (object == VM_OBJECT_NULL) {
3192 named_entry_unlock(mem_entry);
3193 return KERN_INVALID_ARGUMENT;
3194 }
3195
3196 vm_object_reference(object);
3197 named_entry_unlock(mem_entry);
3198
3199 kr = vm_object_page_op(object, offset, ops, phys_entry, flags);
3200
3201 vm_object_deallocate(object);
3202
3203 return kr;
3204 }
3205
3206 /*
3207 * mach_memory_entry_range_op offers performance enhancement over
3208 * mach_memory_entry_page_op for page_op functions which do not require page
3209 * level state to be returned from the call. Page_op was created to provide
3210 * a low-cost alternative to page manipulation via UPLs when only a single
3211 * page was involved. The range_op call establishes the ability in the _op
3212 * family of functions to work on multiple pages where the lack of page level
3213 * state handling allows the caller to avoid the overhead of the upl structures.
3214 */
3215
3216 kern_return_t
3217 mach_memory_entry_range_op(
3218 ipc_port_t entry_port,
3219 vm_object_offset_t offset_beg,
3220 vm_object_offset_t offset_end,
3221 int ops,
3222 int *range)
3223 {
3224 vm_named_entry_t mem_entry;
3225 vm_object_t object;
3226 kern_return_t kr;
3227
3228 if (entry_port == IP_NULL ||
3229 ip_kotype(entry_port) != IKOT_NAMED_ENTRY) {
3230 return KERN_INVALID_ARGUMENT;
3231 }
3232
3233 mem_entry = (vm_named_entry_t) entry_port->ip_kobject;
3234
3235 named_entry_lock(mem_entry);
3236
3237 if (mem_entry->is_sub_map ||
3238 mem_entry->is_pager ||
3239 mem_entry->is_copy) {
3240 named_entry_unlock(mem_entry);
3241 return KERN_INVALID_ARGUMENT;
3242 }
3243
3244 object = mem_entry->backing.object;
3245 if (object == VM_OBJECT_NULL) {
3246 named_entry_unlock(mem_entry);
3247 return KERN_INVALID_ARGUMENT;
3248 }
3249
3250 vm_object_reference(object);
3251 named_entry_unlock(mem_entry);
3252
3253 kr = vm_object_range_op(object,
3254 offset_beg,
3255 offset_end,
3256 ops,
3257 (uint32_t *) range);
3258
3259 vm_object_deallocate(object);
3260
3261 return kr;
3262 }
3263
3264
3265 kern_return_t
3266 set_dp_control_port(
3267 host_priv_t host_priv,
3268 ipc_port_t control_port)
3269 {
3270 if (host_priv == HOST_PRIV_NULL)
3271 return (KERN_INVALID_HOST);
3272
3273 if (IP_VALID(dynamic_pager_control_port))
3274 ipc_port_release_send(dynamic_pager_control_port);
3275
3276 dynamic_pager_control_port = control_port;
3277 return KERN_SUCCESS;
3278 }
3279
3280 kern_return_t
3281 get_dp_control_port(
3282 host_priv_t host_priv,
3283 ipc_port_t *control_port)
3284 {
3285 if (host_priv == HOST_PRIV_NULL)
3286 return (KERN_INVALID_HOST);
3287
3288 *control_port = ipc_port_copy_send(dynamic_pager_control_port);
3289 return KERN_SUCCESS;
3290
3291 }
3292
3293 /* ******* Temporary Internal calls to UPL for BSD ***** */
3294
3295 extern int kernel_upl_map(
3296 vm_map_t map,
3297 upl_t upl,
3298 vm_offset_t *dst_addr);
3299
3300 extern int kernel_upl_unmap(
3301 vm_map_t map,
3302 upl_t upl);
3303
3304 extern int kernel_upl_commit(
3305 upl_t upl,
3306 upl_page_info_t *pl,
3307 mach_msg_type_number_t count);
3308
3309 extern int kernel_upl_commit_range(
3310 upl_t upl,
3311 upl_offset_t offset,
3312 upl_size_t size,
3313 int flags,
3314 upl_page_info_array_t pl,
3315 mach_msg_type_number_t count);
3316
3317 extern int kernel_upl_abort(
3318 upl_t upl,
3319 int abort_type);
3320
3321 extern int kernel_upl_abort_range(
3322 upl_t upl,
3323 upl_offset_t offset,
3324 upl_size_t size,
3325 int abort_flags);
3326
3327
3328 kern_return_t
3329 kernel_upl_map(
3330 vm_map_t map,
3331 upl_t upl,
3332 vm_offset_t *dst_addr)
3333 {
3334 return vm_upl_map(map, upl, dst_addr);
3335 }
3336
3337
3338 kern_return_t
3339 kernel_upl_unmap(
3340 vm_map_t map,
3341 upl_t upl)
3342 {
3343 return vm_upl_unmap(map, upl);
3344 }
3345
3346 kern_return_t
3347 kernel_upl_commit(
3348 upl_t upl,
3349 upl_page_info_t *pl,
3350 mach_msg_type_number_t count)
3351 {
3352 kern_return_t kr;
3353
3354 kr = upl_commit(upl, pl, count);
3355 upl_deallocate(upl);
3356 return kr;
3357 }
3358
3359
3360 kern_return_t
3361 kernel_upl_commit_range(
3362 upl_t upl,
3363 upl_offset_t offset,
3364 upl_size_t size,
3365 int flags,
3366 upl_page_info_array_t pl,
3367 mach_msg_type_number_t count)
3368 {
3369 boolean_t finished = FALSE;
3370 kern_return_t kr;
3371
3372 if (flags & UPL_COMMIT_FREE_ON_EMPTY)
3373 flags |= UPL_COMMIT_NOTIFY_EMPTY;
3374
3375 if (flags & UPL_COMMIT_KERNEL_ONLY_FLAGS) {
3376 return KERN_INVALID_ARGUMENT;
3377 }
3378
3379 kr = upl_commit_range(upl, offset, size, flags, pl, count, &finished);
3380
3381 if ((flags & UPL_COMMIT_NOTIFY_EMPTY) && finished)
3382 upl_deallocate(upl);
3383
3384 return kr;
3385 }
3386
3387 kern_return_t
3388 kernel_upl_abort_range(
3389 upl_t upl,
3390 upl_offset_t offset,
3391 upl_size_t size,
3392 int abort_flags)
3393 {
3394 kern_return_t kr;
3395 boolean_t finished = FALSE;
3396
3397 if (abort_flags & UPL_COMMIT_FREE_ON_EMPTY)
3398 abort_flags |= UPL_COMMIT_NOTIFY_EMPTY;
3399
3400 kr = upl_abort_range(upl, offset, size, abort_flags, &finished);
3401
3402 if ((abort_flags & UPL_COMMIT_FREE_ON_EMPTY) && finished)
3403 upl_deallocate(upl);
3404
3405 return kr;
3406 }
3407
3408 kern_return_t
3409 kernel_upl_abort(
3410 upl_t upl,
3411 int abort_type)
3412 {
3413 kern_return_t kr;
3414
3415 kr = upl_abort(upl, abort_type);
3416 upl_deallocate(upl);
3417 return kr;
3418 }
3419
3420 /*
3421 * Now a kernel-private interface (for BootCache
3422 * use only). Need a cleaner way to create an
3423 * empty vm_map() and return a handle to it.
3424 */
3425
3426 kern_return_t
3427 vm_region_object_create(
3428 __unused vm_map_t target_map,
3429 vm_size_t size,
3430 ipc_port_t *object_handle)
3431 {
3432 vm_named_entry_t user_entry;
3433 ipc_port_t user_handle;
3434
3435 vm_map_t new_map;
3436
3437 if (mach_memory_entry_allocate(&user_entry, &user_handle)
3438 != KERN_SUCCESS) {
3439 return KERN_FAILURE;
3440 }
3441
3442 /* Create a named object based on a submap of specified size */
3443
3444 new_map = vm_map_create(PMAP_NULL, VM_MAP_MIN_ADDRESS,
3445 vm_map_round_page(size,
3446 VM_MAP_PAGE_MASK(target_map)),
3447 TRUE);
3448 vm_map_set_page_shift(new_map, VM_MAP_PAGE_SHIFT(target_map));
3449
3450 user_entry->backing.map = new_map;
3451 user_entry->internal = TRUE;
3452 user_entry->is_sub_map = TRUE;
3453 user_entry->offset = 0;
3454 user_entry->protection = VM_PROT_ALL;
3455 user_entry->size = size;
3456 assert(user_entry->ref_count == 1);
3457
3458 *object_handle = user_handle;
3459 return KERN_SUCCESS;
3460
3461 }
3462
3463 ppnum_t vm_map_get_phys_page( /* forward */
3464 vm_map_t map,
3465 vm_offset_t offset);
3466
3467 ppnum_t
3468 vm_map_get_phys_page(
3469 vm_map_t map,
3470 vm_offset_t addr)
3471 {
3472 vm_object_offset_t offset;
3473 vm_object_t object;
3474 vm_map_offset_t map_offset;
3475 vm_map_entry_t entry;
3476 ppnum_t phys_page = 0;
3477
3478 map_offset = vm_map_trunc_page(addr, PAGE_MASK);
3479
3480 vm_map_lock(map);
3481 while (vm_map_lookup_entry(map, map_offset, &entry)) {
3482
3483 if (entry->object.vm_object == VM_OBJECT_NULL) {
3484 vm_map_unlock(map);
3485 return (ppnum_t) 0;
3486 }
3487 if (entry->is_sub_map) {
3488 vm_map_t old_map;
3489 vm_map_lock(entry->object.sub_map);
3490 old_map = map;
3491 map = entry->object.sub_map;
3492 map_offset = entry->offset + (map_offset - entry->vme_start);
3493 vm_map_unlock(old_map);
3494 continue;
3495 }
3496 if (entry->object.vm_object->phys_contiguous) {
3497 /* These are not standard pageable memory mappings */
3498 /* If they are not present in the object they will */
3499 /* have to be picked up from the pager through the */
3500 /* fault mechanism. */
3501 if(entry->object.vm_object->vo_shadow_offset == 0) {
3502 /* need to call vm_fault */
3503 vm_map_unlock(map);
3504 vm_fault(map, map_offset, VM_PROT_NONE,
3505 FALSE, THREAD_UNINT, NULL, 0);
3506 vm_map_lock(map);
3507 continue;
3508 }
3509 offset = entry->offset + (map_offset - entry->vme_start);
3510 phys_page = (ppnum_t)
3511 ((entry->object.vm_object->vo_shadow_offset
3512 + offset) >> PAGE_SHIFT);
3513 break;
3514
3515 }
3516 offset = entry->offset + (map_offset - entry->vme_start);
3517 object = entry->object.vm_object;
3518 vm_object_lock(object);
3519 while (TRUE) {
3520 vm_page_t dst_page = vm_page_lookup(object,offset);
3521 if(dst_page == VM_PAGE_NULL) {
3522 if(object->shadow) {
3523 vm_object_t old_object;
3524 vm_object_lock(object->shadow);
3525 old_object = object;
3526 offset = offset + object->vo_shadow_offset;
3527 object = object->shadow;
3528 vm_object_unlock(old_object);
3529 } else {
3530 vm_object_unlock(object);
3531 break;
3532 }
3533 } else {
3534 phys_page = (ppnum_t)(dst_page->phys_page);
3535 vm_object_unlock(object);
3536 break;
3537 }
3538 }
3539 break;
3540
3541 }
3542
3543 vm_map_unlock(map);
3544 return phys_page;
3545 }
3546
3547
3548
3549 kern_return_t kernel_object_iopl_request( /* forward */
3550 vm_named_entry_t named_entry,
3551 memory_object_offset_t offset,
3552 upl_size_t *upl_size,
3553 upl_t *upl_ptr,
3554 upl_page_info_array_t user_page_list,
3555 unsigned int *page_list_count,
3556 int *flags);
3557
3558 kern_return_t
3559 kernel_object_iopl_request(
3560 vm_named_entry_t named_entry,
3561 memory_object_offset_t offset,
3562 upl_size_t *upl_size,
3563 upl_t *upl_ptr,
3564 upl_page_info_array_t user_page_list,
3565 unsigned int *page_list_count,
3566 int *flags)
3567 {
3568 vm_object_t object;
3569 kern_return_t ret;
3570
3571 int caller_flags;
3572
3573 caller_flags = *flags;
3574
3575 if (caller_flags & ~UPL_VALID_FLAGS) {
3576 /*
3577 * For forward compatibility's sake,
3578 * reject any unknown flag.
3579 */
3580 return KERN_INVALID_VALUE;
3581 }
3582
3583 /* a few checks to make sure user is obeying rules */
3584 if(*upl_size == 0) {
3585 if(offset >= named_entry->size)
3586 return(KERN_INVALID_RIGHT);
3587 *upl_size = (upl_size_t) (named_entry->size - offset);
3588 if (*upl_size != named_entry->size - offset)
3589 return KERN_INVALID_ARGUMENT;
3590 }
3591 if(caller_flags & UPL_COPYOUT_FROM) {
3592 if((named_entry->protection & VM_PROT_READ)
3593 != VM_PROT_READ) {
3594 return(KERN_INVALID_RIGHT);
3595 }
3596 } else {
3597 if((named_entry->protection &
3598 (VM_PROT_READ | VM_PROT_WRITE))
3599 != (VM_PROT_READ | VM_PROT_WRITE)) {
3600 return(KERN_INVALID_RIGHT);
3601 }
3602 }
3603 if(named_entry->size < (offset + *upl_size))
3604 return(KERN_INVALID_ARGUMENT);
3605
3606 /* the callers parameter offset is defined to be the */
3607 /* offset from beginning of named entry offset in object */
3608 offset = offset + named_entry->offset;
3609
3610 if (named_entry->is_sub_map ||
3611 named_entry->is_copy)
3612 return KERN_INVALID_ARGUMENT;
3613
3614 named_entry_lock(named_entry);
3615
3616 if (named_entry->is_pager) {
3617 object = vm_object_enter(named_entry->backing.pager,
3618 named_entry->offset + named_entry->size,
3619 named_entry->internal,
3620 FALSE,
3621 FALSE);
3622 if (object == VM_OBJECT_NULL) {
3623 named_entry_unlock(named_entry);
3624 return(KERN_INVALID_OBJECT);
3625 }
3626
3627 /* JMM - drop reference on the pager here? */
3628
3629 /* create an extra reference for the object */
3630 vm_object_lock(object);
3631 vm_object_reference_locked(object);
3632 named_entry->backing.object = object;
3633 named_entry->is_pager = FALSE;
3634 named_entry_unlock(named_entry);
3635
3636 /* wait for object (if any) to be ready */
3637 if (!named_entry->internal) {
3638 while (!object->pager_ready) {
3639 vm_object_wait(object,
3640 VM_OBJECT_EVENT_PAGER_READY,
3641 THREAD_UNINT);
3642 vm_object_lock(object);
3643 }
3644 }
3645 vm_object_unlock(object);
3646
3647 } else {
3648 /* This is the case where we are going to operate */
3649 /* an an already known object. If the object is */
3650 /* not ready it is internal. An external */
3651 /* object cannot be mapped until it is ready */
3652 /* we can therefore avoid the ready check */
3653 /* in this case. */
3654 object = named_entry->backing.object;
3655 vm_object_reference(object);
3656 named_entry_unlock(named_entry);
3657 }
3658
3659 if (!object->private) {
3660 if (*upl_size > MAX_UPL_TRANSFER_BYTES)
3661 *upl_size = MAX_UPL_TRANSFER_BYTES;
3662 if (object->phys_contiguous) {
3663 *flags = UPL_PHYS_CONTIG;
3664 } else {
3665 *flags = 0;
3666 }
3667 } else {
3668 *flags = UPL_DEV_MEMORY | UPL_PHYS_CONTIG;
3669 }
3670
3671 ret = vm_object_iopl_request(object,
3672 offset,
3673 *upl_size,
3674 upl_ptr,
3675 user_page_list,
3676 page_list_count,
3677 caller_flags);
3678 vm_object_deallocate(object);
3679 return ret;
3680 }
mirror of XNU