]> git.saurik.com Git - apple/xnu.git/blob - osfmk/vm/vm_user.c
projects / apple / xnu.git / blob
? search:


37d3cbd0c5051f7a02cffac71eb7d5b1815a0355
[apple/xnu.git] / osfmk / vm / vm_user.c
1 /*
2 * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56 /*
57 */
58 /*
59 * File: vm/vm_user.c
60 * Author: Avadis Tevanian, Jr., Michael Wayne Young
61 *
62 * User-exported virtual memory functions.
63 */
64
65 /*
66 * There are three implementations of the "XXX_allocate" functionality in
67 * the kernel: mach_vm_allocate (for any task on the platform), vm_allocate
68 * (for a task with the same address space size, especially the current task),
69 * and vm32_vm_allocate (for the specific case of a 32-bit task). vm_allocate
70 * in the kernel should only be used on the kernel_task. vm32_vm_allocate only
71 * makes sense on platforms where a user task can either be 32 or 64, or the kernel
72 * task can be 32 or 64. mach_vm_allocate makes sense everywhere, and is preferred
73 * for new code.
74 *
75 * The entrypoints into the kernel are more complex. All platforms support a
76 * mach_vm_allocate-style API (subsystem 4800) which operates with the largest
77 * size types for the platform. On platforms that only support U32/K32,
78 * subsystem 4800 is all you need. On platforms that support both U32 and U64,
79 * subsystem 3800 is used disambiguate the size of parameters, and they will
80 * always be 32-bit and call into the vm32_vm_allocate APIs. On non-U32/K32 platforms,
81 * the MIG glue should never call into vm_allocate directly, because the calling
82 * task and kernel_task are unlikely to use the same size parameters
83 *
84 * New VM call implementations should be added here and to mach_vm.defs
85 * (subsystem 4800), and use mach_vm_* "wide" types.
86 */
87
88 #include <debug.h>
89
90 #include <vm_cpm.h>
91 #include <mach/boolean.h>
92 #include <mach/kern_return.h>
93 #include <mach/mach_types.h> /* to get vm_address_t */
94 #include <mach/memory_object.h>
95 #include <mach/std_types.h> /* to get pointer_t */
96 #include <mach/upl.h>
97 #include <mach/vm_attributes.h>
98 #include <mach/vm_param.h>
99 #include <mach/vm_statistics.h>
100 #include <mach/mach_syscalls.h>
101 #include <mach/sdt.h>
102
103 #include <mach/host_priv_server.h>
104 #include <mach/mach_vm_server.h>
105 #include <mach/vm_map_server.h>
106
107 #include <kern/host.h>
108 #include <kern/kalloc.h>
109 #include <kern/task.h>
110 #include <kern/misc_protos.h>
111 #include <vm/vm_fault.h>
112 #include <vm/vm_map.h>
113 #include <vm/vm_object.h>
114 #include <vm/vm_page.h>
115 #include <vm/memory_object.h>
116 #include <vm/vm_pageout.h>
117 #include <vm/vm_protos.h>
118 #include <vm/vm_purgeable_internal.h>
119
120 vm_size_t upl_offset_to_pagelist = 0;
121
122 #if VM_CPM
123 #include <vm/cpm.h>
124 #endif /* VM_CPM */
125
126 ipc_port_t dynamic_pager_control_port=NULL;
127
128 /*
129 * mach_vm_allocate allocates "zero fill" memory in the specfied
130 * map.
131 */
132 kern_return_t
133 mach_vm_allocate(
134 vm_map_t map,
135 mach_vm_offset_t *addr,
136 mach_vm_size_t size,
137 int flags)
138 {
139 vm_map_offset_t map_addr;
140 vm_map_size_t map_size;
141 kern_return_t result;
142 boolean_t anywhere;
143
144 /* filter out any kernel-only flags */
145 if (flags & ~VM_FLAGS_USER_ALLOCATE)
146 return KERN_INVALID_ARGUMENT;
147
148 if (map == VM_MAP_NULL)
149 return(KERN_INVALID_ARGUMENT);
150 if (size == 0) {
151 *addr = 0;
152 return(KERN_SUCCESS);
153 }
154
155 anywhere = ((VM_FLAGS_ANYWHERE & flags) != 0);
156 if (anywhere) {
157 /*
158 * No specific address requested, so start candidate address
159 * search at the minimum address in the map. However, if that
160 * minimum is 0, bump it up by PAGE_SIZE. We want to limit
161 * allocations of PAGEZERO to explicit requests since its
162 * normal use is to catch dereferences of NULL and many
163 * applications also treat pointers with a value of 0 as
164 * special and suddenly having address 0 contain useable
165 * memory would tend to confuse those applications.
166 */
167 map_addr = vm_map_min(map);
168 if (map_addr == 0)
169 map_addr += VM_MAP_PAGE_SIZE(map);
170 } else
171 map_addr = vm_map_trunc_page(*addr,
172 VM_MAP_PAGE_MASK(map));
173 map_size = vm_map_round_page(size,
174 VM_MAP_PAGE_MASK(map));
175 if (map_size == 0) {
176 return(KERN_INVALID_ARGUMENT);
177 }
178
179 result = vm_map_enter(
180 map,
181 &map_addr,
182 map_size,
183 (vm_map_offset_t)0,
184 flags,
185 VM_OBJECT_NULL,
186 (vm_object_offset_t)0,
187 FALSE,
188 VM_PROT_DEFAULT,
189 VM_PROT_ALL,
190 VM_INHERIT_DEFAULT);
191
192 *addr = map_addr;
193 return(result);
194 }
195
196 /*
197 * vm_allocate
198 * Legacy routine that allocates "zero fill" memory in the specfied
199 * map (which is limited to the same size as the kernel).
200 */
201 kern_return_t
202 vm_allocate(
203 vm_map_t map,
204 vm_offset_t *addr,
205 vm_size_t size,
206 int flags)
207 {
208 vm_map_offset_t map_addr;
209 vm_map_size_t map_size;
210 kern_return_t result;
211 boolean_t anywhere;
212
213 /* filter out any kernel-only flags */
214 if (flags & ~VM_FLAGS_USER_ALLOCATE)
215 return KERN_INVALID_ARGUMENT;
216
217 if (map == VM_MAP_NULL)
218 return(KERN_INVALID_ARGUMENT);
219 if (size == 0) {
220 *addr = 0;
221 return(KERN_SUCCESS);
222 }
223
224 anywhere = ((VM_FLAGS_ANYWHERE & flags) != 0);
225 if (anywhere) {
226 /*
227 * No specific address requested, so start candidate address
228 * search at the minimum address in the map. However, if that
229 * minimum is 0, bump it up by PAGE_SIZE. We want to limit
230 * allocations of PAGEZERO to explicit requests since its
231 * normal use is to catch dereferences of NULL and many
232 * applications also treat pointers with a value of 0 as
233 * special and suddenly having address 0 contain useable
234 * memory would tend to confuse those applications.
235 */
236 map_addr = vm_map_min(map);
237 if (map_addr == 0)
238 map_addr += VM_MAP_PAGE_SIZE(map);
239 } else
240 map_addr = vm_map_trunc_page(*addr,
241 VM_MAP_PAGE_MASK(map));
242 map_size = vm_map_round_page(size,
243 VM_MAP_PAGE_MASK(map));
244 if (map_size == 0) {
245 return(KERN_INVALID_ARGUMENT);
246 }
247
248 result = vm_map_enter(
249 map,
250 &map_addr,
251 map_size,
252 (vm_map_offset_t)0,
253 flags,
254 VM_OBJECT_NULL,
255 (vm_object_offset_t)0,
256 FALSE,
257 VM_PROT_DEFAULT,
258 VM_PROT_ALL,
259 VM_INHERIT_DEFAULT);
260
261 *addr = CAST_DOWN(vm_offset_t, map_addr);
262 return(result);
263 }
264
265 /*
266 * mach_vm_deallocate -
267 * deallocates the specified range of addresses in the
268 * specified address map.
269 */
270 kern_return_t
271 mach_vm_deallocate(
272 vm_map_t map,
273 mach_vm_offset_t start,
274 mach_vm_size_t size)
275 {
276 if ((map == VM_MAP_NULL) || (start + size < start))
277 return(KERN_INVALID_ARGUMENT);
278
279 if (size == (mach_vm_offset_t) 0)
280 return(KERN_SUCCESS);
281
282 return(vm_map_remove(map,
283 vm_map_trunc_page(start,
284 VM_MAP_PAGE_MASK(map)),
285 vm_map_round_page(start+size,
286 VM_MAP_PAGE_MASK(map)),
287 VM_MAP_NO_FLAGS));
288 }
289
290 /*
291 * vm_deallocate -
292 * deallocates the specified range of addresses in the
293 * specified address map (limited to addresses the same
294 * size as the kernel).
295 */
296 kern_return_t
297 vm_deallocate(
298 vm_map_t map,
299 vm_offset_t start,
300 vm_size_t size)
301 {
302 if ((map == VM_MAP_NULL) || (start + size < start))
303 return(KERN_INVALID_ARGUMENT);
304
305 if (size == (vm_offset_t) 0)
306 return(KERN_SUCCESS);
307
308 return(vm_map_remove(map,
309 vm_map_trunc_page(start,
310 VM_MAP_PAGE_MASK(map)),
311 vm_map_round_page(start+size,
312 VM_MAP_PAGE_MASK(map)),
313 VM_MAP_NO_FLAGS));
314 }
315
316 /*
317 * mach_vm_inherit -
318 * Sets the inheritance of the specified range in the
319 * specified map.
320 */
321 kern_return_t
322 mach_vm_inherit(
323 vm_map_t map,
324 mach_vm_offset_t start,
325 mach_vm_size_t size,
326 vm_inherit_t new_inheritance)
327 {
328 if ((map == VM_MAP_NULL) || (start + size < start) ||
329 (new_inheritance > VM_INHERIT_LAST_VALID))
330 return(KERN_INVALID_ARGUMENT);
331
332 if (size == 0)
333 return KERN_SUCCESS;
334
335 return(vm_map_inherit(map,
336 vm_map_trunc_page(start,
337 VM_MAP_PAGE_MASK(map)),
338 vm_map_round_page(start+size,
339 VM_MAP_PAGE_MASK(map)),
340 new_inheritance));
341 }
342
343 /*
344 * vm_inherit -
345 * Sets the inheritance of the specified range in the
346 * specified map (range limited to addresses
347 */
348 kern_return_t
349 vm_inherit(
350 vm_map_t map,
351 vm_offset_t start,
352 vm_size_t size,
353 vm_inherit_t new_inheritance)
354 {
355 if ((map == VM_MAP_NULL) || (start + size < start) ||
356 (new_inheritance > VM_INHERIT_LAST_VALID))
357 return(KERN_INVALID_ARGUMENT);
358
359 if (size == 0)
360 return KERN_SUCCESS;
361
362 return(vm_map_inherit(map,
363 vm_map_trunc_page(start,
364 VM_MAP_PAGE_MASK(map)),
365 vm_map_round_page(start+size,
366 VM_MAP_PAGE_MASK(map)),
367 new_inheritance));
368 }
369
370 /*
371 * mach_vm_protect -
372 * Sets the protection of the specified range in the
373 * specified map.
374 */
375
376 kern_return_t
377 mach_vm_protect(
378 vm_map_t map,
379 mach_vm_offset_t start,
380 mach_vm_size_t size,
381 boolean_t set_maximum,
382 vm_prot_t new_protection)
383 {
384 if ((map == VM_MAP_NULL) || (start + size < start) ||
385 (new_protection & ~(VM_PROT_ALL | VM_PROT_COPY)))
386 return(KERN_INVALID_ARGUMENT);
387
388 if (size == 0)
389 return KERN_SUCCESS;
390
391 return(vm_map_protect(map,
392 vm_map_trunc_page(start,
393 VM_MAP_PAGE_MASK(map)),
394 vm_map_round_page(start+size,
395 VM_MAP_PAGE_MASK(map)),
396 new_protection,
397 set_maximum));
398 }
399
400 /*
401 * vm_protect -
402 * Sets the protection of the specified range in the
403 * specified map. Addressability of the range limited
404 * to the same size as the kernel.
405 */
406
407 kern_return_t
408 vm_protect(
409 vm_map_t map,
410 vm_offset_t start,
411 vm_size_t size,
412 boolean_t set_maximum,
413 vm_prot_t new_protection)
414 {
415 if ((map == VM_MAP_NULL) || (start + size < start) ||
416 (new_protection & ~(VM_PROT_ALL | VM_PROT_COPY)))
417 return(KERN_INVALID_ARGUMENT);
418
419 if (size == 0)
420 return KERN_SUCCESS;
421
422 return(vm_map_protect(map,
423 vm_map_trunc_page(start,
424 VM_MAP_PAGE_MASK(map)),
425 vm_map_round_page(start+size,
426 VM_MAP_PAGE_MASK(map)),
427 new_protection,
428 set_maximum));
429 }
430
431 /*
432 * mach_vm_machine_attributes -
433 * Handle machine-specific attributes for a mapping, such
434 * as cachability, migrability, etc.
435 */
436 kern_return_t
437 mach_vm_machine_attribute(
438 vm_map_t map,
439 mach_vm_address_t addr,
440 mach_vm_size_t size,
441 vm_machine_attribute_t attribute,
442 vm_machine_attribute_val_t* value) /* IN/OUT */
443 {
444 if ((map == VM_MAP_NULL) || (addr + size < addr))
445 return(KERN_INVALID_ARGUMENT);
446
447 if (size == 0)
448 return KERN_SUCCESS;
449
450 return vm_map_machine_attribute(
451 map,
452 vm_map_trunc_page(addr,
453 VM_MAP_PAGE_MASK(map)),
454 vm_map_round_page(addr+size,
455 VM_MAP_PAGE_MASK(map)),
456 attribute,
457 value);
458 }
459
460 /*
461 * vm_machine_attribute -
462 * Handle machine-specific attributes for a mapping, such
463 * as cachability, migrability, etc. Limited addressability
464 * (same range limits as for the native kernel map).
465 */
466 kern_return_t
467 vm_machine_attribute(
468 vm_map_t map,
469 vm_address_t addr,
470 vm_size_t size,
471 vm_machine_attribute_t attribute,
472 vm_machine_attribute_val_t* value) /* IN/OUT */
473 {
474 if ((map == VM_MAP_NULL) || (addr + size < addr))
475 return(KERN_INVALID_ARGUMENT);
476
477 if (size == 0)
478 return KERN_SUCCESS;
479
480 return vm_map_machine_attribute(
481 map,
482 vm_map_trunc_page(addr,
483 VM_MAP_PAGE_MASK(map)),
484 vm_map_round_page(addr+size,
485 VM_MAP_PAGE_MASK(map)),
486 attribute,
487 value);
488 }
489
490 /*
491 * mach_vm_read -
492 * Read/copy a range from one address space and return it to the caller.
493 *
494 * It is assumed that the address for the returned memory is selected by
495 * the IPC implementation as part of receiving the reply to this call.
496 * If IPC isn't used, the caller must deal with the vm_map_copy_t object
497 * that gets returned.
498 *
499 * JMM - because of mach_msg_type_number_t, this call is limited to a
500 * single 4GB region at this time.
501 *
502 */
503 kern_return_t
504 mach_vm_read(
505 vm_map_t map,
506 mach_vm_address_t addr,
507 mach_vm_size_t size,
508 pointer_t *data,
509 mach_msg_type_number_t *data_size)
510 {
511 kern_return_t error;
512 vm_map_copy_t ipc_address;
513
514 if (map == VM_MAP_NULL)
515 return(KERN_INVALID_ARGUMENT);
516
517 if ((mach_msg_type_number_t) size != size)
518 return KERN_INVALID_ARGUMENT;
519
520 error = vm_map_copyin(map,
521 (vm_map_address_t)addr,
522 (vm_map_size_t)size,
523 FALSE, /* src_destroy */
524 &ipc_address);
525
526 if (KERN_SUCCESS == error) {
527 *data = (pointer_t) ipc_address;
528 *data_size = (mach_msg_type_number_t) size;
529 assert(*data_size == size);
530 }
531 return(error);
532 }
533
534 /*
535 * vm_read -
536 * Read/copy a range from one address space and return it to the caller.
537 * Limited addressability (same range limits as for the native kernel map).
538 *
539 * It is assumed that the address for the returned memory is selected by
540 * the IPC implementation as part of receiving the reply to this call.
541 * If IPC isn't used, the caller must deal with the vm_map_copy_t object
542 * that gets returned.
543 */
544 kern_return_t
545 vm_read(
546 vm_map_t map,
547 vm_address_t addr,
548 vm_size_t size,
549 pointer_t *data,
550 mach_msg_type_number_t *data_size)
551 {
552 kern_return_t error;
553 vm_map_copy_t ipc_address;
554
555 if (map == VM_MAP_NULL)
556 return(KERN_INVALID_ARGUMENT);
557
558 if (size > (unsigned)(mach_msg_type_number_t) -1) {
559 /*
560 * The kernel could handle a 64-bit "size" value, but
561 * it could not return the size of the data in "*data_size"
562 * without overflowing.
563 * Let's reject this "size" as invalid.
564 */
565 return KERN_INVALID_ARGUMENT;
566 }
567
568 error = vm_map_copyin(map,
569 (vm_map_address_t)addr,
570 (vm_map_size_t)size,
571 FALSE, /* src_destroy */
572 &ipc_address);
573
574 if (KERN_SUCCESS == error) {
575 *data = (pointer_t) ipc_address;
576 *data_size = (mach_msg_type_number_t) size;
577 assert(*data_size == size);
578 }
579 return(error);
580 }
581
582 /*
583 * mach_vm_read_list -
584 * Read/copy a list of address ranges from specified map.
585 *
586 * MIG does not know how to deal with a returned array of
587 * vm_map_copy_t structures, so we have to do the copyout
588 * manually here.
589 */
590 kern_return_t
591 mach_vm_read_list(
592 vm_map_t map,
593 mach_vm_read_entry_t data_list,
594 natural_t count)
595 {
596 mach_msg_type_number_t i;
597 kern_return_t error;
598 vm_map_copy_t copy;
599
600 if (map == VM_MAP_NULL ||
601 count > VM_MAP_ENTRY_MAX)
602 return(KERN_INVALID_ARGUMENT);
603
604 error = KERN_SUCCESS;
605 for(i=0; i<count; i++) {
606 vm_map_address_t map_addr;
607 vm_map_size_t map_size;
608
609 map_addr = (vm_map_address_t)(data_list[i].address);
610 map_size = (vm_map_size_t)(data_list[i].size);
611
612 if(map_size != 0) {
613 error = vm_map_copyin(map,
614 map_addr,
615 map_size,
616 FALSE, /* src_destroy */
617 &copy);
618 if (KERN_SUCCESS == error) {
619 error = vm_map_copyout(
620 current_task()->map,
621 &map_addr,
622 copy);
623 if (KERN_SUCCESS == error) {
624 data_list[i].address = map_addr;
625 continue;
626 }
627 vm_map_copy_discard(copy);
628 }
629 }
630 data_list[i].address = (mach_vm_address_t)0;
631 data_list[i].size = (mach_vm_size_t)0;
632 }
633 return(error);
634 }
635
636 /*
637 * vm_read_list -
638 * Read/copy a list of address ranges from specified map.
639 *
640 * MIG does not know how to deal with a returned array of
641 * vm_map_copy_t structures, so we have to do the copyout
642 * manually here.
643 *
644 * The source and destination ranges are limited to those
645 * that can be described with a vm_address_t (i.e. same
646 * size map as the kernel).
647 *
648 * JMM - If the result of the copyout is an address range
649 * that cannot be described with a vm_address_t (i.e. the
650 * caller had a larger address space but used this call
651 * anyway), it will result in a truncated address being
652 * returned (and a likely confused caller).
653 */
654
655 kern_return_t
656 vm_read_list(
657 vm_map_t map,
658 vm_read_entry_t data_list,
659 natural_t count)
660 {
661 mach_msg_type_number_t i;
662 kern_return_t error;
663 vm_map_copy_t copy;
664
665 if (map == VM_MAP_NULL ||
666 count > VM_MAP_ENTRY_MAX)
667 return(KERN_INVALID_ARGUMENT);
668
669 error = KERN_SUCCESS;
670 for(i=0; i<count; i++) {
671 vm_map_address_t map_addr;
672 vm_map_size_t map_size;
673
674 map_addr = (vm_map_address_t)(data_list[i].address);
675 map_size = (vm_map_size_t)(data_list[i].size);
676
677 if(map_size != 0) {
678 error = vm_map_copyin(map,
679 map_addr,
680 map_size,
681 FALSE, /* src_destroy */
682 &copy);
683 if (KERN_SUCCESS == error) {
684 error = vm_map_copyout(current_task()->map,
685 &map_addr,
686 copy);
687 if (KERN_SUCCESS == error) {
688 data_list[i].address =
689 CAST_DOWN(vm_offset_t, map_addr);
690 continue;
691 }
692 vm_map_copy_discard(copy);
693 }
694 }
695 data_list[i].address = (mach_vm_address_t)0;
696 data_list[i].size = (mach_vm_size_t)0;
697 }
698 return(error);
699 }
700
701 /*
702 * mach_vm_read_overwrite -
703 * Overwrite a range of the current map with data from the specified
704 * map/address range.
705 *
706 * In making an assumption that the current thread is local, it is
707 * no longer cluster-safe without a fully supportive local proxy
708 * thread/task (but we don't support cluster's anymore so this is moot).
709 */
710
711 kern_return_t
712 mach_vm_read_overwrite(
713 vm_map_t map,
714 mach_vm_address_t address,
715 mach_vm_size_t size,
716 mach_vm_address_t data,
717 mach_vm_size_t *data_size)
718 {
719 kern_return_t error;
720 vm_map_copy_t copy;
721
722 if (map == VM_MAP_NULL)
723 return(KERN_INVALID_ARGUMENT);
724
725 error = vm_map_copyin(map, (vm_map_address_t)address,
726 (vm_map_size_t)size, FALSE, &copy);
727
728 if (KERN_SUCCESS == error) {
729 error = vm_map_copy_overwrite(current_thread()->map,
730 (vm_map_address_t)data,
731 copy, FALSE);
732 if (KERN_SUCCESS == error) {
733 *data_size = size;
734 return error;
735 }
736 vm_map_copy_discard(copy);
737 }
738 return(error);
739 }
740
741 /*
742 * vm_read_overwrite -
743 * Overwrite a range of the current map with data from the specified
744 * map/address range.
745 *
746 * This routine adds the additional limitation that the source and
747 * destination ranges must be describable with vm_address_t values
748 * (i.e. the same size address spaces as the kernel, or at least the
749 * the ranges are in that first portion of the respective address
750 * spaces).
751 */
752
753 kern_return_t
754 vm_read_overwrite(
755 vm_map_t map,
756 vm_address_t address,
757 vm_size_t size,
758 vm_address_t data,
759 vm_size_t *data_size)
760 {
761 kern_return_t error;
762 vm_map_copy_t copy;
763
764 if (map == VM_MAP_NULL)
765 return(KERN_INVALID_ARGUMENT);
766
767 error = vm_map_copyin(map, (vm_map_address_t)address,
768 (vm_map_size_t)size, FALSE, &copy);
769
770 if (KERN_SUCCESS == error) {
771 error = vm_map_copy_overwrite(current_thread()->map,
772 (vm_map_address_t)data,
773 copy, FALSE);
774 if (KERN_SUCCESS == error) {
775 *data_size = size;
776 return error;
777 }
778 vm_map_copy_discard(copy);
779 }
780 return(error);
781 }
782
783
784 /*
785 * mach_vm_write -
786 * Overwrite the specified address range with the data provided
787 * (from the current map).
788 */
789 kern_return_t
790 mach_vm_write(
791 vm_map_t map,
792 mach_vm_address_t address,
793 pointer_t data,
794 __unused mach_msg_type_number_t size)
795 {
796 if (map == VM_MAP_NULL)
797 return KERN_INVALID_ARGUMENT;
798
799 return vm_map_copy_overwrite(map, (vm_map_address_t)address,
800 (vm_map_copy_t) data, FALSE /* interruptible XXX */);
801 }
802
803 /*
804 * vm_write -
805 * Overwrite the specified address range with the data provided
806 * (from the current map).
807 *
808 * The addressability of the range of addresses to overwrite is
809 * limited bu the use of a vm_address_t (same size as kernel map).
810 * Either the target map is also small, or the range is in the
811 * low addresses within it.
812 */
813 kern_return_t
814 vm_write(
815 vm_map_t map,
816 vm_address_t address,
817 pointer_t data,
818 __unused mach_msg_type_number_t size)
819 {
820 if (map == VM_MAP_NULL)
821 return KERN_INVALID_ARGUMENT;
822
823 return vm_map_copy_overwrite(map, (vm_map_address_t)address,
824 (vm_map_copy_t) data, FALSE /* interruptible XXX */);
825 }
826
827 /*
828 * mach_vm_copy -
829 * Overwrite one range of the specified map with the contents of
830 * another range within that same map (i.e. both address ranges
831 * are "over there").
832 */
833 kern_return_t
834 mach_vm_copy(
835 vm_map_t map,
836 mach_vm_address_t source_address,
837 mach_vm_size_t size,
838 mach_vm_address_t dest_address)
839 {
840 vm_map_copy_t copy;
841 kern_return_t kr;
842
843 if (map == VM_MAP_NULL)
844 return KERN_INVALID_ARGUMENT;
845
846 kr = vm_map_copyin(map, (vm_map_address_t)source_address,
847 (vm_map_size_t)size, FALSE, &copy);
848
849 if (KERN_SUCCESS == kr) {
850 kr = vm_map_copy_overwrite(map,
851 (vm_map_address_t)dest_address,
852 copy, FALSE /* interruptible XXX */);
853
854 if (KERN_SUCCESS != kr)
855 vm_map_copy_discard(copy);
856 }
857 return kr;
858 }
859
860 kern_return_t
861 vm_copy(
862 vm_map_t map,
863 vm_address_t source_address,
864 vm_size_t size,
865 vm_address_t dest_address)
866 {
867 vm_map_copy_t copy;
868 kern_return_t kr;
869
870 if (map == VM_MAP_NULL)
871 return KERN_INVALID_ARGUMENT;
872
873 kr = vm_map_copyin(map, (vm_map_address_t)source_address,
874 (vm_map_size_t)size, FALSE, &copy);
875
876 if (KERN_SUCCESS == kr) {
877 kr = vm_map_copy_overwrite(map,
878 (vm_map_address_t)dest_address,
879 copy, FALSE /* interruptible XXX */);
880
881 if (KERN_SUCCESS != kr)
882 vm_map_copy_discard(copy);
883 }
884 return kr;
885 }
886
887 /*
888 * mach_vm_map -
889 * Map some range of an object into an address space.
890 *
891 * The object can be one of several types of objects:
892 * NULL - anonymous memory
893 * a named entry - a range within another address space
894 * or a range within a memory object
895 * a whole memory object
896 *
897 */
898 kern_return_t
899 mach_vm_map(
900 vm_map_t target_map,
901 mach_vm_offset_t *address,
902 mach_vm_size_t initial_size,
903 mach_vm_offset_t mask,
904 int flags,
905 ipc_port_t port,
906 vm_object_offset_t offset,
907 boolean_t copy,
908 vm_prot_t cur_protection,
909 vm_prot_t max_protection,
910 vm_inherit_t inheritance)
911 {
912 kern_return_t kr;
913 vm_map_offset_t vmmaddr;
914
915 vmmaddr = (vm_map_offset_t) *address;
916
917 /* filter out any kernel-only flags */
918 if (flags & ~VM_FLAGS_USER_MAP)
919 return KERN_INVALID_ARGUMENT;
920
921 kr = vm_map_enter_mem_object(target_map,
922 &vmmaddr,
923 initial_size,
924 mask,
925 flags,
926 port,
927 offset,
928 copy,
929 cur_protection,
930 max_protection,
931 inheritance);
932
933 *address = vmmaddr;
934 return kr;
935 }
936
937
938 /* legacy interface */
939 kern_return_t
940 vm_map_64(
941 vm_map_t target_map,
942 vm_offset_t *address,
943 vm_size_t size,
944 vm_offset_t mask,
945 int flags,
946 ipc_port_t port,
947 vm_object_offset_t offset,
948 boolean_t copy,
949 vm_prot_t cur_protection,
950 vm_prot_t max_protection,
951 vm_inherit_t inheritance)
952 {
953 mach_vm_address_t map_addr;
954 mach_vm_size_t map_size;
955 mach_vm_offset_t map_mask;
956 kern_return_t kr;
957
958 map_addr = (mach_vm_address_t)*address;
959 map_size = (mach_vm_size_t)size;
960 map_mask = (mach_vm_offset_t)mask;
961
962 kr = mach_vm_map(target_map, &map_addr, map_size, map_mask, flags,
963 port, offset, copy,
964 cur_protection, max_protection, inheritance);
965 *address = CAST_DOWN(vm_offset_t, map_addr);
966 return kr;
967 }
968
969 /* temporary, until world build */
970 kern_return_t
971 vm_map(
972 vm_map_t target_map,
973 vm_offset_t *address,
974 vm_size_t size,
975 vm_offset_t mask,
976 int flags,
977 ipc_port_t port,
978 vm_offset_t offset,
979 boolean_t copy,
980 vm_prot_t cur_protection,
981 vm_prot_t max_protection,
982 vm_inherit_t inheritance)
983 {
984 mach_vm_address_t map_addr;
985 mach_vm_size_t map_size;
986 mach_vm_offset_t map_mask;
987 vm_object_offset_t obj_offset;
988 kern_return_t kr;
989
990 map_addr = (mach_vm_address_t)*address;
991 map_size = (mach_vm_size_t)size;
992 map_mask = (mach_vm_offset_t)mask;
993 obj_offset = (vm_object_offset_t)offset;
994
995 kr = mach_vm_map(target_map, &map_addr, map_size, map_mask, flags,
996 port, obj_offset, copy,
997 cur_protection, max_protection, inheritance);
998 *address = CAST_DOWN(vm_offset_t, map_addr);
999 return kr;
1000 }
1001
1002 /*
1003 * mach_vm_remap -
1004 * Remap a range of memory from one task into another,
1005 * to another address range within the same task, or
1006 * over top of itself (with altered permissions and/or
1007 * as an in-place copy of itself).
1008 */
1009
1010 kern_return_t
1011 mach_vm_remap(
1012 vm_map_t target_map,
1013 mach_vm_offset_t *address,
1014 mach_vm_size_t size,
1015 mach_vm_offset_t mask,
1016 int flags,
1017 vm_map_t src_map,
1018 mach_vm_offset_t memory_address,
1019 boolean_t copy,
1020 vm_prot_t *cur_protection,
1021 vm_prot_t *max_protection,
1022 vm_inherit_t inheritance)
1023 {
1024 vm_map_offset_t map_addr;
1025 kern_return_t kr;
1026
1027 if (VM_MAP_NULL == target_map || VM_MAP_NULL == src_map)
1028 return KERN_INVALID_ARGUMENT;
1029
1030 /* filter out any kernel-only flags */
1031 if (flags & ~VM_FLAGS_USER_REMAP)
1032 return KERN_INVALID_ARGUMENT;
1033
1034 map_addr = (vm_map_offset_t)*address;
1035
1036 kr = vm_map_remap(target_map,
1037 &map_addr,
1038 size,
1039 mask,
1040 flags,
1041 src_map,
1042 memory_address,
1043 copy,
1044 cur_protection,
1045 max_protection,
1046 inheritance);
1047 *address = map_addr;
1048 return kr;
1049 }
1050
1051 /*
1052 * vm_remap -
1053 * Remap a range of memory from one task into another,
1054 * to another address range within the same task, or
1055 * over top of itself (with altered permissions and/or
1056 * as an in-place copy of itself).
1057 *
1058 * The addressability of the source and target address
1059 * range is limited by the size of vm_address_t (in the
1060 * kernel context).
1061 */
1062 kern_return_t
1063 vm_remap(
1064 vm_map_t target_map,
1065 vm_offset_t *address,
1066 vm_size_t size,
1067 vm_offset_t mask,
1068 int flags,
1069 vm_map_t src_map,
1070 vm_offset_t memory_address,
1071 boolean_t copy,
1072 vm_prot_t *cur_protection,
1073 vm_prot_t *max_protection,
1074 vm_inherit_t inheritance)
1075 {
1076 vm_map_offset_t map_addr;
1077 kern_return_t kr;
1078
1079 if (VM_MAP_NULL == target_map || VM_MAP_NULL == src_map)
1080 return KERN_INVALID_ARGUMENT;
1081
1082 /* filter out any kernel-only flags */
1083 if (flags & ~VM_FLAGS_USER_REMAP)
1084 return KERN_INVALID_ARGUMENT;
1085
1086 map_addr = (vm_map_offset_t)*address;
1087
1088 kr = vm_map_remap(target_map,
1089 &map_addr,
1090 size,
1091 mask,
1092 flags,
1093 src_map,
1094 memory_address,
1095 copy,
1096 cur_protection,
1097 max_protection,
1098 inheritance);
1099 *address = CAST_DOWN(vm_offset_t, map_addr);
1100 return kr;
1101 }
1102
1103 /*
1104 * NOTE: these routine (and this file) will no longer require mach_host_server.h
1105 * when mach_vm_wire and vm_wire are changed to use ledgers.
1106 */
1107 #include <mach/mach_host_server.h>
1108 /*
1109 * mach_vm_wire
1110 * Specify that the range of the virtual address space
1111 * of the target task must not cause page faults for
1112 * the indicated accesses.
1113 *
1114 * [ To unwire the pages, specify VM_PROT_NONE. ]
1115 */
1116 kern_return_t
1117 mach_vm_wire(
1118 host_priv_t host_priv,
1119 vm_map_t map,
1120 mach_vm_offset_t start,
1121 mach_vm_size_t size,
1122 vm_prot_t access)
1123 {
1124 kern_return_t rc;
1125
1126 if (host_priv == HOST_PRIV_NULL)
1127 return KERN_INVALID_HOST;
1128
1129 assert(host_priv == &realhost);
1130
1131 if (map == VM_MAP_NULL)
1132 return KERN_INVALID_TASK;
1133
1134 if (access & ~VM_PROT_ALL || (start + size < start))
1135 return KERN_INVALID_ARGUMENT;
1136
1137 if (access != VM_PROT_NONE) {
1138 rc = vm_map_wire(map,
1139 vm_map_trunc_page(start,
1140 VM_MAP_PAGE_MASK(map)),
1141 vm_map_round_page(start+size,
1142 VM_MAP_PAGE_MASK(map)),
1143 access | VM_PROT_MEMORY_TAG_MAKE(VM_KERN_MEMORY_MLOCK),
1144 TRUE);
1145 } else {
1146 rc = vm_map_unwire(map,
1147 vm_map_trunc_page(start,
1148 VM_MAP_PAGE_MASK(map)),
1149 vm_map_round_page(start+size,
1150 VM_MAP_PAGE_MASK(map)),
1151 TRUE);
1152 }
1153 return rc;
1154 }
1155
1156 /*
1157 * vm_wire -
1158 * Specify that the range of the virtual address space
1159 * of the target task must not cause page faults for
1160 * the indicated accesses.
1161 *
1162 * [ To unwire the pages, specify VM_PROT_NONE. ]
1163 */
1164 kern_return_t
1165 vm_wire(
1166 host_priv_t host_priv,
1167 vm_map_t map,
1168 vm_offset_t start,
1169 vm_size_t size,
1170 vm_prot_t access)
1171 {
1172 kern_return_t rc;
1173
1174 if (host_priv == HOST_PRIV_NULL)
1175 return KERN_INVALID_HOST;
1176
1177 assert(host_priv == &realhost);
1178
1179 if (map == VM_MAP_NULL)
1180 return KERN_INVALID_TASK;
1181
1182 if ((access & ~VM_PROT_ALL) || (start + size < start))
1183 return KERN_INVALID_ARGUMENT;
1184
1185 if (size == 0) {
1186 rc = KERN_SUCCESS;
1187 } else if (access != VM_PROT_NONE) {
1188 rc = vm_map_wire(map,
1189 vm_map_trunc_page(start,
1190 VM_MAP_PAGE_MASK(map)),
1191 vm_map_round_page(start+size,
1192 VM_MAP_PAGE_MASK(map)),
1193 access | VM_PROT_MEMORY_TAG_MAKE(VM_KERN_MEMORY_OSFMK),
1194 TRUE);
1195 } else {
1196 rc = vm_map_unwire(map,
1197 vm_map_trunc_page(start,
1198 VM_MAP_PAGE_MASK(map)),
1199 vm_map_round_page(start+size,
1200 VM_MAP_PAGE_MASK(map)),
1201 TRUE);
1202 }
1203 return rc;
1204 }
1205
1206 /*
1207 * vm_msync
1208 *
1209 * Synchronises the memory range specified with its backing store
1210 * image by either flushing or cleaning the contents to the appropriate
1211 * memory manager.
1212 *
1213 * interpretation of sync_flags
1214 * VM_SYNC_INVALIDATE - discard pages, only return precious
1215 * pages to manager.
1216 *
1217 * VM_SYNC_INVALIDATE & (VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS)
1218 * - discard pages, write dirty or precious
1219 * pages back to memory manager.
1220 *
1221 * VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS
1222 * - write dirty or precious pages back to
1223 * the memory manager.
1224 *
1225 * VM_SYNC_CONTIGUOUS - does everything normally, but if there
1226 * is a hole in the region, and we would
1227 * have returned KERN_SUCCESS, return
1228 * KERN_INVALID_ADDRESS instead.
1229 *
1230 * RETURNS
1231 * KERN_INVALID_TASK Bad task parameter
1232 * KERN_INVALID_ARGUMENT both sync and async were specified.
1233 * KERN_SUCCESS The usual.
1234 * KERN_INVALID_ADDRESS There was a hole in the region.
1235 */
1236
1237 kern_return_t
1238 mach_vm_msync(
1239 vm_map_t map,
1240 mach_vm_address_t address,
1241 mach_vm_size_t size,
1242 vm_sync_t sync_flags)
1243 {
1244
1245 if (map == VM_MAP_NULL)
1246 return(KERN_INVALID_TASK);
1247
1248 return vm_map_msync(map, (vm_map_address_t)address,
1249 (vm_map_size_t)size, sync_flags);
1250 }
1251
1252 /*
1253 * vm_msync
1254 *
1255 * Synchronises the memory range specified with its backing store
1256 * image by either flushing or cleaning the contents to the appropriate
1257 * memory manager.
1258 *
1259 * interpretation of sync_flags
1260 * VM_SYNC_INVALIDATE - discard pages, only return precious
1261 * pages to manager.
1262 *
1263 * VM_SYNC_INVALIDATE & (VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS)
1264 * - discard pages, write dirty or precious
1265 * pages back to memory manager.
1266 *
1267 * VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS
1268 * - write dirty or precious pages back to
1269 * the memory manager.
1270 *
1271 * VM_SYNC_CONTIGUOUS - does everything normally, but if there
1272 * is a hole in the region, and we would
1273 * have returned KERN_SUCCESS, return
1274 * KERN_INVALID_ADDRESS instead.
1275 *
1276 * The addressability of the range is limited to that which can
1277 * be described by a vm_address_t.
1278 *
1279 * RETURNS
1280 * KERN_INVALID_TASK Bad task parameter
1281 * KERN_INVALID_ARGUMENT both sync and async were specified.
1282 * KERN_SUCCESS The usual.
1283 * KERN_INVALID_ADDRESS There was a hole in the region.
1284 */
1285
1286 kern_return_t
1287 vm_msync(
1288 vm_map_t map,
1289 vm_address_t address,
1290 vm_size_t size,
1291 vm_sync_t sync_flags)
1292 {
1293
1294 if (map == VM_MAP_NULL)
1295 return(KERN_INVALID_TASK);
1296
1297 return vm_map_msync(map, (vm_map_address_t)address,
1298 (vm_map_size_t)size, sync_flags);
1299 }
1300
1301
1302 int
1303 vm_toggle_entry_reuse(int toggle, int *old_value)
1304 {
1305 vm_map_t map = current_map();
1306
1307 assert(!map->is_nested_map);
1308 if(toggle == VM_TOGGLE_GETVALUE && old_value != NULL){
1309 *old_value = map->disable_vmentry_reuse;
1310 } else if(toggle == VM_TOGGLE_SET){
1311 vm_map_entry_t map_to_entry;
1312
1313 vm_map_lock(map);
1314 vm_map_disable_hole_optimization(map);
1315 map->disable_vmentry_reuse = TRUE;
1316 __IGNORE_WCASTALIGN(map_to_entry = vm_map_to_entry(map));
1317 if (map->first_free == map_to_entry) {
1318 map->highest_entry_end = vm_map_min(map);
1319 } else {
1320 map->highest_entry_end = map->first_free->vme_end;
1321 }
1322 vm_map_unlock(map);
1323 } else if (toggle == VM_TOGGLE_CLEAR){
1324 vm_map_lock(map);
1325 map->disable_vmentry_reuse = FALSE;
1326 vm_map_unlock(map);
1327 } else
1328 return KERN_INVALID_ARGUMENT;
1329
1330 return KERN_SUCCESS;
1331 }
1332
1333 /*
1334 * mach_vm_behavior_set
1335 *
1336 * Sets the paging behavior attribute for the specified range
1337 * in the specified map.
1338 *
1339 * This routine will fail with KERN_INVALID_ADDRESS if any address
1340 * in [start,start+size) is not a valid allocated memory region.
1341 */
1342 kern_return_t
1343 mach_vm_behavior_set(
1344 vm_map_t map,
1345 mach_vm_offset_t start,
1346 mach_vm_size_t size,
1347 vm_behavior_t new_behavior)
1348 {
1349 vm_map_offset_t align_mask;
1350
1351 if ((map == VM_MAP_NULL) || (start + size < start))
1352 return(KERN_INVALID_ARGUMENT);
1353
1354 if (size == 0)
1355 return KERN_SUCCESS;
1356
1357 switch (new_behavior) {
1358 case VM_BEHAVIOR_REUSABLE:
1359 case VM_BEHAVIOR_REUSE:
1360 case VM_BEHAVIOR_CAN_REUSE:
1361 /*
1362 * Align to the hardware page size, to allow
1363 * malloc() to maximize the amount of re-usability,
1364 * even on systems with larger software page size.
1365 */
1366 align_mask = PAGE_MASK;
1367 break;
1368 default:
1369 align_mask = VM_MAP_PAGE_MASK(map);
1370 break;
1371 }
1372
1373 return vm_map_behavior_set(map,
1374 vm_map_trunc_page(start, align_mask),
1375 vm_map_round_page(start+size, align_mask),
1376 new_behavior);
1377 }
1378
1379 /*
1380 * vm_behavior_set
1381 *
1382 * Sets the paging behavior attribute for the specified range
1383 * in the specified map.
1384 *
1385 * This routine will fail with KERN_INVALID_ADDRESS if any address
1386 * in [start,start+size) is not a valid allocated memory region.
1387 *
1388 * This routine is potentially limited in addressibility by the
1389 * use of vm_offset_t (if the map provided is larger than the
1390 * kernel's).
1391 */
1392 kern_return_t
1393 vm_behavior_set(
1394 vm_map_t map,
1395 vm_offset_t start,
1396 vm_size_t size,
1397 vm_behavior_t new_behavior)
1398 {
1399 if (start + size < start)
1400 return KERN_INVALID_ARGUMENT;
1401
1402 return mach_vm_behavior_set(map,
1403 (mach_vm_offset_t) start,
1404 (mach_vm_size_t) size,
1405 new_behavior);
1406 }
1407
1408 /*
1409 * mach_vm_region:
1410 *
1411 * User call to obtain information about a region in
1412 * a task's address map. Currently, only one flavor is
1413 * supported.
1414 *
1415 * XXX The reserved and behavior fields cannot be filled
1416 * in until the vm merge from the IK is completed, and
1417 * vm_reserve is implemented.
1418 *
1419 * XXX Dependency: syscall_vm_region() also supports only one flavor.
1420 */
1421
1422 kern_return_t
1423 mach_vm_region(
1424 vm_map_t map,
1425 mach_vm_offset_t *address, /* IN/OUT */
1426 mach_vm_size_t *size, /* OUT */
1427 vm_region_flavor_t flavor, /* IN */
1428 vm_region_info_t info, /* OUT */
1429 mach_msg_type_number_t *count, /* IN/OUT */
1430 mach_port_t *object_name) /* OUT */
1431 {
1432 vm_map_offset_t map_addr;
1433 vm_map_size_t map_size;
1434 kern_return_t kr;
1435
1436 if (VM_MAP_NULL == map)
1437 return KERN_INVALID_ARGUMENT;
1438
1439 map_addr = (vm_map_offset_t)*address;
1440 map_size = (vm_map_size_t)*size;
1441
1442 /* legacy conversion */
1443 if (VM_REGION_BASIC_INFO == flavor)
1444 flavor = VM_REGION_BASIC_INFO_64;
1445
1446 kr = vm_map_region(map,
1447 &map_addr, &map_size,
1448 flavor, info, count,
1449 object_name);
1450
1451 *address = map_addr;
1452 *size = map_size;
1453 return kr;
1454 }
1455
1456 /*
1457 * vm_region_64 and vm_region:
1458 *
1459 * User call to obtain information about a region in
1460 * a task's address map. Currently, only one flavor is
1461 * supported.
1462 *
1463 * XXX The reserved and behavior fields cannot be filled
1464 * in until the vm merge from the IK is completed, and
1465 * vm_reserve is implemented.
1466 *
1467 * XXX Dependency: syscall_vm_region() also supports only one flavor.
1468 */
1469
1470 kern_return_t
1471 vm_region_64(
1472 vm_map_t map,
1473 vm_offset_t *address, /* IN/OUT */
1474 vm_size_t *size, /* OUT */
1475 vm_region_flavor_t flavor, /* IN */
1476 vm_region_info_t info, /* OUT */
1477 mach_msg_type_number_t *count, /* IN/OUT */
1478 mach_port_t *object_name) /* OUT */
1479 {
1480 vm_map_offset_t map_addr;
1481 vm_map_size_t map_size;
1482 kern_return_t kr;
1483
1484 if (VM_MAP_NULL == map)
1485 return KERN_INVALID_ARGUMENT;
1486
1487 map_addr = (vm_map_offset_t)*address;
1488 map_size = (vm_map_size_t)*size;
1489
1490 /* legacy conversion */
1491 if (VM_REGION_BASIC_INFO == flavor)
1492 flavor = VM_REGION_BASIC_INFO_64;
1493
1494 kr = vm_map_region(map,
1495 &map_addr, &map_size,
1496 flavor, info, count,
1497 object_name);
1498
1499 *address = CAST_DOWN(vm_offset_t, map_addr);
1500 *size = CAST_DOWN(vm_size_t, map_size);
1501
1502 if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS)
1503 return KERN_INVALID_ADDRESS;
1504 return kr;
1505 }
1506
1507 kern_return_t
1508 vm_region(
1509 vm_map_t map,
1510 vm_address_t *address, /* IN/OUT */
1511 vm_size_t *size, /* OUT */
1512 vm_region_flavor_t flavor, /* IN */
1513 vm_region_info_t info, /* OUT */
1514 mach_msg_type_number_t *count, /* IN/OUT */
1515 mach_port_t *object_name) /* OUT */
1516 {
1517 vm_map_address_t map_addr;
1518 vm_map_size_t map_size;
1519 kern_return_t kr;
1520
1521 if (VM_MAP_NULL == map)
1522 return KERN_INVALID_ARGUMENT;
1523
1524 map_addr = (vm_map_address_t)*address;
1525 map_size = (vm_map_size_t)*size;
1526
1527 kr = vm_map_region(map,
1528 &map_addr, &map_size,
1529 flavor, info, count,
1530 object_name);
1531
1532 *address = CAST_DOWN(vm_address_t, map_addr);
1533 *size = CAST_DOWN(vm_size_t, map_size);
1534
1535 if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS)
1536 return KERN_INVALID_ADDRESS;
1537 return kr;
1538 }
1539
1540 /*
1541 * vm_region_recurse: A form of vm_region which follows the
1542 * submaps in a target map
1543 *
1544 */
1545 kern_return_t
1546 mach_vm_region_recurse(
1547 vm_map_t map,
1548 mach_vm_address_t *address,
1549 mach_vm_size_t *size,
1550 uint32_t *depth,
1551 vm_region_recurse_info_t info,
1552 mach_msg_type_number_t *infoCnt)
1553 {
1554 vm_map_address_t map_addr;
1555 vm_map_size_t map_size;
1556 kern_return_t kr;
1557
1558 if (VM_MAP_NULL == map)
1559 return KERN_INVALID_ARGUMENT;
1560
1561 map_addr = (vm_map_address_t)*address;
1562 map_size = (vm_map_size_t)*size;
1563
1564 kr = vm_map_region_recurse_64(
1565 map,
1566 &map_addr,
1567 &map_size,
1568 depth,
1569 (vm_region_submap_info_64_t)info,
1570 infoCnt);
1571
1572 *address = map_addr;
1573 *size = map_size;
1574 return kr;
1575 }
1576
1577 /*
1578 * vm_region_recurse: A form of vm_region which follows the
1579 * submaps in a target map
1580 *
1581 */
1582 kern_return_t
1583 vm_region_recurse_64(
1584 vm_map_t map,
1585 vm_address_t *address,
1586 vm_size_t *size,
1587 uint32_t *depth,
1588 vm_region_recurse_info_64_t info,
1589 mach_msg_type_number_t *infoCnt)
1590 {
1591 vm_map_address_t map_addr;
1592 vm_map_size_t map_size;
1593 kern_return_t kr;
1594
1595 if (VM_MAP_NULL == map)
1596 return KERN_INVALID_ARGUMENT;
1597
1598 map_addr = (vm_map_address_t)*address;
1599 map_size = (vm_map_size_t)*size;
1600
1601 kr = vm_map_region_recurse_64(
1602 map,
1603 &map_addr,
1604 &map_size,
1605 depth,
1606 (vm_region_submap_info_64_t)info,
1607 infoCnt);
1608
1609 *address = CAST_DOWN(vm_address_t, map_addr);
1610 *size = CAST_DOWN(vm_size_t, map_size);
1611
1612 if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS)
1613 return KERN_INVALID_ADDRESS;
1614 return kr;
1615 }
1616
1617 kern_return_t
1618 vm_region_recurse(
1619 vm_map_t map,
1620 vm_offset_t *address, /* IN/OUT */
1621 vm_size_t *size, /* OUT */
1622 natural_t *depth, /* IN/OUT */
1623 vm_region_recurse_info_t info32, /* IN/OUT */
1624 mach_msg_type_number_t *infoCnt) /* IN/OUT */
1625 {
1626 vm_region_submap_info_data_64_t info64;
1627 vm_region_submap_info_t info;
1628 vm_map_address_t map_addr;
1629 vm_map_size_t map_size;
1630 kern_return_t kr;
1631
1632 if (VM_MAP_NULL == map || *infoCnt < VM_REGION_SUBMAP_INFO_COUNT)
1633 return KERN_INVALID_ARGUMENT;
1634
1635
1636 map_addr = (vm_map_address_t)*address;
1637 map_size = (vm_map_size_t)*size;
1638 info = (vm_region_submap_info_t)info32;
1639 *infoCnt = VM_REGION_SUBMAP_INFO_COUNT_64;
1640
1641 kr = vm_map_region_recurse_64(map, &map_addr,&map_size,
1642 depth, &info64, infoCnt);
1643
1644 info->protection = info64.protection;
1645 info->max_protection = info64.max_protection;
1646 info->inheritance = info64.inheritance;
1647 info->offset = (uint32_t)info64.offset; /* trouble-maker */
1648 info->user_tag = info64.user_tag;
1649 info->pages_resident = info64.pages_resident;
1650 info->pages_shared_now_private = info64.pages_shared_now_private;
1651 info->pages_swapped_out = info64.pages_swapped_out;
1652 info->pages_dirtied = info64.pages_dirtied;
1653 info->ref_count = info64.ref_count;
1654 info->shadow_depth = info64.shadow_depth;
1655 info->external_pager = info64.external_pager;
1656 info->share_mode = info64.share_mode;
1657 info->is_submap = info64.is_submap;
1658 info->behavior = info64.behavior;
1659 info->object_id = info64.object_id;
1660 info->user_wired_count = info64.user_wired_count;
1661
1662 *address = CAST_DOWN(vm_address_t, map_addr);
1663 *size = CAST_DOWN(vm_size_t, map_size);
1664 *infoCnt = VM_REGION_SUBMAP_INFO_COUNT;
1665
1666 if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS)
1667 return KERN_INVALID_ADDRESS;
1668 return kr;
1669 }
1670
1671 kern_return_t
1672 mach_vm_purgable_control(
1673 vm_map_t map,
1674 mach_vm_offset_t address,
1675 vm_purgable_t control,
1676 int *state)
1677 {
1678 if (VM_MAP_NULL == map)
1679 return KERN_INVALID_ARGUMENT;
1680
1681 return vm_map_purgable_control(map,
1682 vm_map_trunc_page(address, PAGE_MASK),
1683 control,
1684 state);
1685 }
1686
1687 kern_return_t
1688 vm_purgable_control(
1689 vm_map_t map,
1690 vm_offset_t address,
1691 vm_purgable_t control,
1692 int *state)
1693 {
1694 if (VM_MAP_NULL == map)
1695 return KERN_INVALID_ARGUMENT;
1696
1697 return vm_map_purgable_control(map,
1698 vm_map_trunc_page(address, PAGE_MASK),
1699 control,
1700 state);
1701 }
1702
1703
1704 /*
1705 * Ordinarily, the right to allocate CPM is restricted
1706 * to privileged applications (those that can gain access
1707 * to the host priv port). Set this variable to zero if
1708 * you want to let any application allocate CPM.
1709 */
1710 unsigned int vm_allocate_cpm_privileged = 0;
1711
1712 /*
1713 * Allocate memory in the specified map, with the caveat that
1714 * the memory is physically contiguous. This call may fail
1715 * if the system can't find sufficient contiguous memory.
1716 * This call may cause or lead to heart-stopping amounts of
1717 * paging activity.
1718 *
1719 * Memory obtained from this call should be freed in the
1720 * normal way, viz., via vm_deallocate.
1721 */
1722 kern_return_t
1723 vm_allocate_cpm(
1724 host_priv_t host_priv,
1725 vm_map_t map,
1726 vm_address_t *addr,
1727 vm_size_t size,
1728 int flags)
1729 {
1730 vm_map_address_t map_addr;
1731 vm_map_size_t map_size;
1732 kern_return_t kr;
1733
1734 if (vm_allocate_cpm_privileged && HOST_PRIV_NULL == host_priv)
1735 return KERN_INVALID_HOST;
1736
1737 if (VM_MAP_NULL == map)
1738 return KERN_INVALID_ARGUMENT;
1739
1740 map_addr = (vm_map_address_t)*addr;
1741 map_size = (vm_map_size_t)size;
1742
1743 kr = vm_map_enter_cpm(map,
1744 &map_addr,
1745 map_size,
1746 flags);
1747
1748 *addr = CAST_DOWN(vm_address_t, map_addr);
1749 return kr;
1750 }
1751
1752
1753 kern_return_t
1754 mach_vm_page_query(
1755 vm_map_t map,
1756 mach_vm_offset_t offset,
1757 int *disposition,
1758 int *ref_count)
1759 {
1760 if (VM_MAP_NULL == map)
1761 return KERN_INVALID_ARGUMENT;
1762
1763 return vm_map_page_query_internal(
1764 map,
1765 vm_map_trunc_page(offset, PAGE_MASK),
1766 disposition, ref_count);
1767 }
1768
1769 kern_return_t
1770 vm_map_page_query(
1771 vm_map_t map,
1772 vm_offset_t offset,
1773 int *disposition,
1774 int *ref_count)
1775 {
1776 if (VM_MAP_NULL == map)
1777 return KERN_INVALID_ARGUMENT;
1778
1779 return vm_map_page_query_internal(
1780 map,
1781 vm_map_trunc_page(offset, PAGE_MASK),
1782 disposition, ref_count);
1783 }
1784
1785 kern_return_t
1786 mach_vm_page_info(
1787 vm_map_t map,
1788 mach_vm_address_t address,
1789 vm_page_info_flavor_t flavor,
1790 vm_page_info_t info,
1791 mach_msg_type_number_t *count)
1792 {
1793 kern_return_t kr;
1794
1795 if (map == VM_MAP_NULL) {
1796 return KERN_INVALID_ARGUMENT;
1797 }
1798
1799 kr = vm_map_page_info(map, address, flavor, info, count);
1800 return kr;
1801 }
1802
1803 /* map a (whole) upl into an address space */
1804 kern_return_t
1805 vm_upl_map(
1806 vm_map_t map,
1807 upl_t upl,
1808 vm_address_t *dst_addr)
1809 {
1810 vm_map_offset_t map_addr;
1811 kern_return_t kr;
1812
1813 if (VM_MAP_NULL == map)
1814 return KERN_INVALID_ARGUMENT;
1815
1816 kr = vm_map_enter_upl(map, upl, &map_addr);
1817 *dst_addr = CAST_DOWN(vm_address_t, map_addr);
1818 return kr;
1819 }
1820
1821 kern_return_t
1822 vm_upl_unmap(
1823 vm_map_t map,
1824 upl_t upl)
1825 {
1826 if (VM_MAP_NULL == map)
1827 return KERN_INVALID_ARGUMENT;
1828
1829 return (vm_map_remove_upl(map, upl));
1830 }
1831
1832 /* Retrieve a upl for an object underlying an address range in a map */
1833
1834 kern_return_t
1835 vm_map_get_upl(
1836 vm_map_t map,
1837 vm_map_offset_t map_offset,
1838 upl_size_t *upl_size,
1839 upl_t *upl,
1840 upl_page_info_array_t page_list,
1841 unsigned int *count,
1842 upl_control_flags_t *flags,
1843 int force_data_sync)
1844 {
1845 upl_control_flags_t map_flags;
1846 kern_return_t kr;
1847
1848 if (VM_MAP_NULL == map)
1849 return KERN_INVALID_ARGUMENT;
1850
1851 map_flags = *flags & ~UPL_NOZEROFILL;
1852 if (force_data_sync)
1853 map_flags |= UPL_FORCE_DATA_SYNC;
1854
1855 kr = vm_map_create_upl(map,
1856 map_offset,
1857 upl_size,
1858 upl,
1859 page_list,
1860 count,
1861 &map_flags);
1862
1863 *flags = (map_flags & ~UPL_FORCE_DATA_SYNC);
1864 return kr;
1865 }
1866
1867
1868 /*
1869 * mach_make_memory_entry_64
1870 *
1871 * Think of it as a two-stage vm_remap() operation. First
1872 * you get a handle. Second, you get map that handle in
1873 * somewhere else. Rather than doing it all at once (and
1874 * without needing access to the other whole map).
1875 */
1876
1877 kern_return_t
1878 mach_make_memory_entry_64(
1879 vm_map_t target_map,
1880 memory_object_size_t *size,
1881 memory_object_offset_t offset,
1882 vm_prot_t permission,
1883 ipc_port_t *object_handle,
1884 ipc_port_t parent_handle)
1885 {
1886 vm_map_version_t version;
1887 vm_named_entry_t parent_entry;
1888 vm_named_entry_t user_entry;
1889 ipc_port_t user_handle;
1890 kern_return_t kr;
1891 vm_map_t real_map;
1892
1893 /* needed for call to vm_map_lookup_locked */
1894 boolean_t wired;
1895 boolean_t iskernel;
1896 vm_object_offset_t obj_off;
1897 vm_prot_t prot;
1898 struct vm_object_fault_info fault_info;
1899 vm_object_t object;
1900 vm_object_t shadow_object;
1901
1902 /* needed for direct map entry manipulation */
1903 vm_map_entry_t map_entry;
1904 vm_map_entry_t next_entry;
1905 vm_map_t local_map;
1906 vm_map_t original_map = target_map;
1907 vm_map_size_t total_size, map_size;
1908 vm_map_offset_t map_start, map_end;
1909 vm_map_offset_t local_offset;
1910 vm_object_size_t mappable_size;
1911
1912 /*
1913 * Stash the offset in the page for use by vm_map_enter_mem_object()
1914 * in the VM_FLAGS_RETURN_DATA_ADDR/MAP_MEM_USE_DATA_ADDR case.
1915 */
1916 vm_object_offset_t offset_in_page;
1917
1918 unsigned int access;
1919 vm_prot_t protections;
1920 vm_prot_t original_protections, mask_protections;
1921 unsigned int wimg_mode;
1922
1923 boolean_t force_shadow = FALSE;
1924 boolean_t use_data_addr;
1925 boolean_t use_4K_compat;
1926
1927 if (((permission & 0x00FF0000) &
1928 ~(MAP_MEM_ONLY |
1929 MAP_MEM_NAMED_CREATE |
1930 MAP_MEM_GRAB_SECLUDED | /* XXX FBDP TODO: restrict usage? */
1931 MAP_MEM_PURGABLE |
1932 MAP_MEM_NAMED_REUSE |
1933 MAP_MEM_USE_DATA_ADDR |
1934 MAP_MEM_VM_COPY |
1935 MAP_MEM_4K_DATA_ADDR |
1936 MAP_MEM_VM_SHARE))) {
1937 /*
1938 * Unknown flag: reject for forward compatibility.
1939 */
1940 return KERN_INVALID_VALUE;
1941 }
1942
1943 if (parent_handle != IP_NULL &&
1944 ip_kotype(parent_handle) == IKOT_NAMED_ENTRY) {
1945 parent_entry = (vm_named_entry_t) parent_handle->ip_kobject;
1946 } else {
1947 parent_entry = NULL;
1948 }
1949
1950 if (parent_entry && parent_entry->is_copy) {
1951 return KERN_INVALID_ARGUMENT;
1952 }
1953
1954 original_protections = permission & VM_PROT_ALL;
1955 protections = original_protections;
1956 mask_protections = permission & VM_PROT_IS_MASK;
1957 access = GET_MAP_MEM(permission);
1958 use_data_addr = ((permission & MAP_MEM_USE_DATA_ADDR) != 0);
1959 use_4K_compat = ((permission & MAP_MEM_4K_DATA_ADDR) != 0);
1960
1961 user_handle = IP_NULL;
1962 user_entry = NULL;
1963
1964 map_start = vm_map_trunc_page(offset, PAGE_MASK);
1965
1966 if (permission & MAP_MEM_ONLY) {
1967 boolean_t parent_is_object;
1968
1969 map_end = vm_map_round_page(offset + *size, PAGE_MASK);
1970 map_size = map_end - map_start;
1971
1972 if (use_data_addr || use_4K_compat || parent_entry == NULL) {
1973 return KERN_INVALID_ARGUMENT;
1974 }
1975
1976 parent_is_object = !(parent_entry->is_sub_map ||
1977 parent_entry->is_pager);
1978 object = parent_entry->backing.object;
1979 if(parent_is_object && object != VM_OBJECT_NULL)
1980 wimg_mode = object->wimg_bits;
1981 else
1982 wimg_mode = VM_WIMG_USE_DEFAULT;
1983 if((access != GET_MAP_MEM(parent_entry->protection)) &&
1984 !(parent_entry->protection & VM_PROT_WRITE)) {
1985 return KERN_INVALID_RIGHT;
1986 }
1987 if(access == MAP_MEM_IO) {
1988 SET_MAP_MEM(access, parent_entry->protection);
1989 wimg_mode = VM_WIMG_IO;
1990 } else if (access == MAP_MEM_COPYBACK) {
1991 SET_MAP_MEM(access, parent_entry->protection);
1992 wimg_mode = VM_WIMG_USE_DEFAULT;
1993 } else if (access == MAP_MEM_INNERWBACK) {
1994 SET_MAP_MEM(access, parent_entry->protection);
1995 wimg_mode = VM_WIMG_INNERWBACK;
1996 } else if (access == MAP_MEM_WTHRU) {
1997 SET_MAP_MEM(access, parent_entry->protection);
1998 wimg_mode = VM_WIMG_WTHRU;
1999 } else if (access == MAP_MEM_WCOMB) {
2000 SET_MAP_MEM(access, parent_entry->protection);
2001 wimg_mode = VM_WIMG_WCOMB;
2002 }
2003 if (parent_is_object && object &&
2004 (access != MAP_MEM_NOOP) &&
2005 (!(object->nophyscache))) {
2006
2007 if (object->wimg_bits != wimg_mode) {
2008 vm_object_lock(object);
2009 vm_object_change_wimg_mode(object, wimg_mode);
2010 vm_object_unlock(object);
2011 }
2012 }
2013 if (object_handle)
2014 *object_handle = IP_NULL;
2015 return KERN_SUCCESS;
2016 } else if (permission & MAP_MEM_NAMED_CREATE) {
2017 map_end = vm_map_round_page(offset + *size, PAGE_MASK);
2018 map_size = map_end - map_start;
2019
2020 if (use_data_addr || use_4K_compat) {
2021 return KERN_INVALID_ARGUMENT;
2022 }
2023
2024 kr = mach_memory_entry_allocate(&user_entry, &user_handle);
2025 if (kr != KERN_SUCCESS) {
2026 return KERN_FAILURE;
2027 }
2028
2029 /*
2030 * Force the creation of the VM object now.
2031 */
2032 if (map_size > (vm_map_size_t) ANON_MAX_SIZE) {
2033 /*
2034 * LP64todo - for now, we can only allocate 4GB-4096
2035 * internal objects because the default pager can't
2036 * page bigger ones. Remove this when it can.
2037 */
2038 kr = KERN_FAILURE;
2039 goto make_mem_done;
2040 }
2041
2042 object = vm_object_allocate(map_size);
2043 assert(object != VM_OBJECT_NULL);
2044
2045 if (permission & MAP_MEM_PURGABLE) {
2046 if (! (permission & VM_PROT_WRITE)) {
2047 /* if we can't write, we can't purge */
2048 vm_object_deallocate(object);
2049 kr = KERN_INVALID_ARGUMENT;
2050 goto make_mem_done;
2051 }
2052 object->purgable = VM_PURGABLE_NONVOLATILE;
2053 assert(object->vo_purgeable_owner == NULL);
2054 assert(object->resident_page_count == 0);
2055 assert(object->wired_page_count == 0);
2056 vm_object_lock(object);
2057 vm_purgeable_nonvolatile_enqueue(object,
2058 current_task());
2059 vm_object_unlock(object);
2060 }
2061
2062 #if CONFIG_SECLUDED_MEMORY
2063 if (secluded_for_iokit && /* global boot-arg */
2064 ((permission & MAP_MEM_GRAB_SECLUDED)
2065 #if 11
2066 /* XXX FBDP for my testing only */
2067 || (secluded_for_fbdp && map_size == 97550336)
2068 #endif
2069 )) {
2070 #if 11
2071 if (!(permission & MAP_MEM_GRAB_SECLUDED) &&
2072 secluded_for_fbdp) {
2073 printf("FBDP: object %p size %lld can grab secluded\n", object, (uint64_t) map_size);
2074 }
2075 #endif
2076 object->can_grab_secluded = TRUE;
2077 assert(!object->eligible_for_secluded);
2078 }
2079 #endif /* CONFIG_SECLUDED_MEMORY */
2080
2081 /*
2082 * The VM object is brand new and nobody else knows about it,
2083 * so we don't need to lock it.
2084 */
2085
2086 wimg_mode = object->wimg_bits;
2087 if (access == MAP_MEM_IO) {
2088 wimg_mode = VM_WIMG_IO;
2089 } else if (access == MAP_MEM_COPYBACK) {
2090 wimg_mode = VM_WIMG_USE_DEFAULT;
2091 } else if (access == MAP_MEM_INNERWBACK) {
2092 wimg_mode = VM_WIMG_INNERWBACK;
2093 } else if (access == MAP_MEM_WTHRU) {
2094 wimg_mode = VM_WIMG_WTHRU;
2095 } else if (access == MAP_MEM_WCOMB) {
2096 wimg_mode = VM_WIMG_WCOMB;
2097 }
2098 if (access != MAP_MEM_NOOP) {
2099 object->wimg_bits = wimg_mode;
2100 }
2101 /* the object has no pages, so no WIMG bits to update here */
2102
2103 /*
2104 * XXX
2105 * We use this path when we want to make sure that
2106 * nobody messes with the object (coalesce, for
2107 * example) before we map it.
2108 * We might want to use these objects for transposition via
2109 * vm_object_transpose() too, so we don't want any copy or
2110 * shadow objects either...
2111 */
2112 object->copy_strategy = MEMORY_OBJECT_COPY_NONE;
2113 object->true_share = TRUE;
2114
2115 user_entry->backing.object = object;
2116 user_entry->internal = TRUE;
2117 user_entry->is_sub_map = FALSE;
2118 user_entry->is_pager = FALSE;
2119 user_entry->offset = 0;
2120 user_entry->data_offset = 0;
2121 user_entry->protection = protections;
2122 SET_MAP_MEM(access, user_entry->protection);
2123 user_entry->size = map_size;
2124
2125 /* user_object pager and internal fields are not used */
2126 /* when the object field is filled in. */
2127
2128 *size = CAST_DOWN(vm_size_t, (user_entry->size -
2129 user_entry->data_offset));
2130 *object_handle = user_handle;
2131 return KERN_SUCCESS;
2132 }
2133
2134 if (permission & MAP_MEM_VM_COPY) {
2135 vm_map_copy_t copy;
2136
2137 if (target_map == VM_MAP_NULL) {
2138 return KERN_INVALID_TASK;
2139 }
2140
2141 map_end = vm_map_round_page(offset + *size, PAGE_MASK);
2142 map_size = map_end - map_start;
2143 if (use_data_addr || use_4K_compat) {
2144 offset_in_page = offset - map_start;
2145 if (use_4K_compat)
2146 offset_in_page &= ~((signed)(0xFFF));
2147 } else {
2148 offset_in_page = 0;
2149 }
2150
2151 kr = vm_map_copyin_internal(target_map,
2152 map_start,
2153 map_size,
2154 VM_MAP_COPYIN_ENTRY_LIST,
2155 &copy);
2156 if (kr != KERN_SUCCESS) {
2157 return kr;
2158 }
2159
2160 kr = mach_memory_entry_allocate(&user_entry, &user_handle);
2161 if (kr != KERN_SUCCESS) {
2162 vm_map_copy_discard(copy);
2163 return KERN_FAILURE;
2164 }
2165
2166 user_entry->backing.copy = copy;
2167 user_entry->internal = FALSE;
2168 user_entry->is_sub_map = FALSE;
2169 user_entry->is_pager = FALSE;
2170 user_entry->is_copy = TRUE;
2171 user_entry->offset = 0;
2172 user_entry->protection = protections;
2173 user_entry->size = map_size;
2174 user_entry->data_offset = offset_in_page;
2175
2176 *size = CAST_DOWN(vm_size_t, (user_entry->size -
2177 user_entry->data_offset));
2178 *object_handle = user_handle;
2179 return KERN_SUCCESS;
2180 }
2181
2182 if (permission & MAP_MEM_VM_SHARE) {
2183 vm_map_copy_t copy;
2184 vm_prot_t cur_prot, max_prot;
2185
2186 if (target_map == VM_MAP_NULL) {
2187 return KERN_INVALID_TASK;
2188 }
2189
2190 map_end = vm_map_round_page(offset + *size, PAGE_MASK);
2191 map_size = map_end - map_start;
2192 if (use_data_addr || use_4K_compat) {
2193 offset_in_page = offset - map_start;
2194 if (use_4K_compat)
2195 offset_in_page &= ~((signed)(0xFFF));
2196 } else {
2197 offset_in_page = 0;
2198 }
2199
2200 cur_prot = VM_PROT_ALL;
2201 kr = vm_map_copy_extract(target_map,
2202 map_start,
2203 map_size,
2204 &copy,
2205 &cur_prot,
2206 &max_prot);
2207 if (kr != KERN_SUCCESS) {
2208 return kr;
2209 }
2210
2211 if (mask_protections) {
2212 /*
2213 * We just want as much of "original_protections"
2214 * as we can get out of the actual "cur_prot".
2215 */
2216 protections &= cur_prot;
2217 if (protections == VM_PROT_NONE) {
2218 /* no access at all: fail */
2219 vm_map_copy_discard(copy);
2220 return KERN_PROTECTION_FAILURE;
2221 }
2222 } else {
2223 /*
2224 * We want exactly "original_protections"
2225 * out of "cur_prot".
2226 */
2227 if ((cur_prot & protections) != protections) {
2228 vm_map_copy_discard(copy);
2229 return KERN_PROTECTION_FAILURE;
2230 }
2231 }
2232
2233 kr = mach_memory_entry_allocate(&user_entry, &user_handle);
2234 if (kr != KERN_SUCCESS) {
2235 vm_map_copy_discard(copy);
2236 return KERN_FAILURE;
2237 }
2238
2239 user_entry->backing.copy = copy;
2240 user_entry->internal = FALSE;
2241 user_entry->is_sub_map = FALSE;
2242 user_entry->is_pager = FALSE;
2243 user_entry->is_copy = TRUE;
2244 user_entry->offset = 0;
2245 user_entry->protection = protections;
2246 user_entry->size = map_size;
2247 user_entry->data_offset = offset_in_page;
2248
2249 *size = CAST_DOWN(vm_size_t, (user_entry->size -
2250 user_entry->data_offset));
2251 *object_handle = user_handle;
2252 return KERN_SUCCESS;
2253 }
2254
2255 if (parent_entry == NULL ||
2256 (permission & MAP_MEM_NAMED_REUSE)) {
2257
2258 map_end = vm_map_round_page(offset + *size, PAGE_MASK);
2259 map_size = map_end - map_start;
2260 if (use_data_addr || use_4K_compat) {
2261 offset_in_page = offset - map_start;
2262 if (use_4K_compat)
2263 offset_in_page &= ~((signed)(0xFFF));
2264 } else {
2265 offset_in_page = 0;
2266 }
2267
2268 /* Create a named object based on address range within the task map */
2269 /* Go find the object at given address */
2270
2271 if (target_map == VM_MAP_NULL) {
2272 return KERN_INVALID_TASK;
2273 }
2274
2275 redo_lookup:
2276 protections = original_protections;
2277 vm_map_lock_read(target_map);
2278
2279 /* get the object associated with the target address */
2280 /* note we check the permission of the range against */
2281 /* that requested by the caller */
2282
2283 kr = vm_map_lookup_locked(&target_map, map_start,
2284 protections | mask_protections,
2285 OBJECT_LOCK_EXCLUSIVE, &version,
2286 &object, &obj_off, &prot, &wired,
2287 &fault_info,
2288 &real_map);
2289 if (kr != KERN_SUCCESS) {
2290 vm_map_unlock_read(target_map);
2291 goto make_mem_done;
2292 }
2293 if (mask_protections) {
2294 /*
2295 * The caller asked us to use the "protections" as
2296 * a mask, so restrict "protections" to what this
2297 * mapping actually allows.
2298 */
2299 protections &= prot;
2300 }
2301
2302 if (((prot & protections) != protections)
2303 || (object == kernel_object)) {
2304 kr = KERN_INVALID_RIGHT;
2305 vm_object_unlock(object);
2306 vm_map_unlock_read(target_map);
2307 if(real_map != target_map)
2308 vm_map_unlock_read(real_map);
2309 if(object == kernel_object) {
2310 printf("Warning: Attempt to create a named"
2311 " entry from the kernel_object\n");
2312 }
2313 goto make_mem_done;
2314 }
2315
2316 /* We have an object, now check to see if this object */
2317 /* is suitable. If not, create a shadow and share that */
2318
2319 /*
2320 * We have to unlock the VM object to avoid deadlocking with
2321 * a VM map lock (the lock ordering is map, the object), if we
2322 * need to modify the VM map to create a shadow object. Since
2323 * we might release the VM map lock below anyway, we have
2324 * to release the VM map lock now.
2325 * XXX FBDP There must be a way to avoid this double lookup...
2326 *
2327 * Take an extra reference on the VM object to make sure it's
2328 * not going to disappear.
2329 */
2330 vm_object_reference_locked(object); /* extra ref to hold obj */
2331 vm_object_unlock(object);
2332
2333 local_map = original_map;
2334 local_offset = map_start;
2335 if(target_map != local_map) {
2336 vm_map_unlock_read(target_map);
2337 if(real_map != target_map)
2338 vm_map_unlock_read(real_map);
2339 vm_map_lock_read(local_map);
2340 target_map = local_map;
2341 real_map = local_map;
2342 }
2343 while(TRUE) {
2344 if(!vm_map_lookup_entry(local_map,
2345 local_offset, &map_entry)) {
2346 kr = KERN_INVALID_ARGUMENT;
2347 vm_map_unlock_read(target_map);
2348 if(real_map != target_map)
2349 vm_map_unlock_read(real_map);
2350 vm_object_deallocate(object); /* release extra ref */
2351 object = VM_OBJECT_NULL;
2352 goto make_mem_done;
2353 }
2354 iskernel = (local_map->pmap == kernel_pmap);
2355 if(!(map_entry->is_sub_map)) {
2356 if (VME_OBJECT(map_entry) != object) {
2357 kr = KERN_INVALID_ARGUMENT;
2358 vm_map_unlock_read(target_map);
2359 if(real_map != target_map)
2360 vm_map_unlock_read(real_map);
2361 vm_object_deallocate(object); /* release extra ref */
2362 object = VM_OBJECT_NULL;
2363 goto make_mem_done;
2364 }
2365 break;
2366 } else {
2367 vm_map_t tmap;
2368 tmap = local_map;
2369 local_map = VME_SUBMAP(map_entry);
2370
2371 vm_map_lock_read(local_map);
2372 vm_map_unlock_read(tmap);
2373 target_map = local_map;
2374 real_map = local_map;
2375 local_offset = local_offset - map_entry->vme_start;
2376 local_offset += VME_OFFSET(map_entry);
2377 }
2378 }
2379
2380 /*
2381 * We found the VM map entry, lock the VM object again.
2382 */
2383 vm_object_lock(object);
2384 if(map_entry->wired_count) {
2385 /* JMM - The check below should be reworked instead. */
2386 object->true_share = TRUE;
2387 }
2388 if (mask_protections) {
2389 /*
2390 * The caller asked us to use the "protections" as
2391 * a mask, so restrict "protections" to what this
2392 * mapping actually allows.
2393 */
2394 protections &= map_entry->max_protection;
2395 }
2396 if(((map_entry->max_protection) & protections) != protections) {
2397 kr = KERN_INVALID_RIGHT;
2398 vm_object_unlock(object);
2399 vm_map_unlock_read(target_map);
2400 if(real_map != target_map)
2401 vm_map_unlock_read(real_map);
2402 vm_object_deallocate(object);
2403 object = VM_OBJECT_NULL;
2404 goto make_mem_done;
2405 }
2406
2407 mappable_size = fault_info.hi_offset - obj_off;
2408 total_size = map_entry->vme_end - map_entry->vme_start;
2409 if(map_size > mappable_size) {
2410 /* try to extend mappable size if the entries */
2411 /* following are from the same object and are */
2412 /* compatible */
2413 next_entry = map_entry->vme_next;
2414 /* lets see if the next map entry is still */
2415 /* pointing at this object and is contiguous */
2416 while(map_size > mappable_size) {
2417 if ((VME_OBJECT(next_entry) == object) &&
2418 (next_entry->vme_start ==
2419 next_entry->vme_prev->vme_end) &&
2420 (VME_OFFSET(next_entry) ==
2421 (VME_OFFSET(next_entry->vme_prev) +
2422 (next_entry->vme_prev->vme_end -
2423 next_entry->vme_prev->vme_start)))) {
2424 if (mask_protections) {
2425 /*
2426 * The caller asked us to use
2427 * the "protections" as a mask,
2428 * so restrict "protections" to
2429 * what this mapping actually
2430 * allows.
2431 */
2432 protections &= next_entry->max_protection;
2433 }
2434 if ((next_entry->wired_count) &&
2435 (map_entry->wired_count == 0)) {
2436 break;
2437 }
2438 if(((next_entry->max_protection)
2439 & protections) != protections) {
2440 break;
2441 }
2442 if (next_entry->needs_copy !=
2443 map_entry->needs_copy)
2444 break;
2445 mappable_size += next_entry->vme_end
2446 - next_entry->vme_start;
2447 total_size += next_entry->vme_end
2448 - next_entry->vme_start;
2449 next_entry = next_entry->vme_next;
2450 } else {
2451 break;
2452 }
2453
2454 }
2455 }
2456
2457 /* vm_map_entry_should_cow_for_true_share() checks for malloc tags,
2458 * never true in kernel */
2459 if (!iskernel && vm_map_entry_should_cow_for_true_share(map_entry) &&
2460 object->vo_size > map_size &&
2461 map_size != 0) {
2462 /*
2463 * Set up the targeted range for copy-on-write to
2464 * limit the impact of "true_share"/"copy_delay" to
2465 * that range instead of the entire VM object...
2466 */
2467
2468 vm_object_unlock(object);
2469 if (vm_map_lock_read_to_write(target_map)) {
2470 vm_object_deallocate(object);
2471 target_map = original_map;
2472 goto redo_lookup;
2473 }
2474
2475 vm_map_clip_start(target_map,
2476 map_entry,
2477 vm_map_trunc_page(map_start,
2478 VM_MAP_PAGE_MASK(target_map)));
2479 vm_map_clip_end(target_map,
2480 map_entry,
2481 (vm_map_round_page(map_end,
2482 VM_MAP_PAGE_MASK(target_map))));
2483 force_shadow = TRUE;
2484
2485 if ((map_entry->vme_end - offset) < map_size) {
2486 map_size = map_entry->vme_end - map_start;
2487 }
2488 total_size = map_entry->vme_end - map_entry->vme_start;
2489
2490 vm_map_lock_write_to_read(target_map);
2491 vm_object_lock(object);
2492 }
2493
2494 if (object->internal) {
2495 /* vm_map_lookup_locked will create a shadow if */
2496 /* needs_copy is set but does not check for the */
2497 /* other two conditions shown. It is important to */
2498 /* set up an object which will not be pulled from */
2499 /* under us. */
2500
2501 if (force_shadow ||
2502 ((map_entry->needs_copy ||
2503 object->shadowed ||
2504 (object->vo_size > total_size &&
2505 (VME_OFFSET(map_entry) != 0 ||
2506 object->vo_size >
2507 vm_map_round_page(total_size,
2508 VM_MAP_PAGE_MASK(target_map)))))
2509 && !object->true_share)) {
2510 /*
2511 * We have to unlock the VM object before
2512 * trying to upgrade the VM map lock, to
2513 * honor lock ordering (map then object).
2514 * Otherwise, we would deadlock if another
2515 * thread holds a read lock on the VM map and
2516 * is trying to acquire the VM object's lock.
2517 * We still hold an extra reference on the
2518 * VM object, guaranteeing that it won't
2519 * disappear.
2520 */
2521 vm_object_unlock(object);
2522
2523 if (vm_map_lock_read_to_write(target_map)) {
2524 /*
2525 * We couldn't upgrade our VM map lock
2526 * from "read" to "write" and we lost
2527 * our "read" lock.
2528 * Start all over again...
2529 */
2530 vm_object_deallocate(object); /* extra ref */
2531 target_map = original_map;
2532 goto redo_lookup;
2533 }
2534 #if 00
2535 vm_object_lock(object);
2536 #endif
2537
2538 /*
2539 * JMM - We need to avoid coming here when the object
2540 * is wired by anybody, not just the current map. Why
2541 * couldn't we use the standard vm_object_copy_quickly()
2542 * approach here?
2543 */
2544
2545 /* create a shadow object */
2546 VME_OBJECT_SHADOW(map_entry, total_size);
2547 shadow_object = VME_OBJECT(map_entry);
2548 #if 00
2549 vm_object_unlock(object);
2550 #endif
2551
2552 prot = map_entry->protection & ~VM_PROT_WRITE;
2553
2554 if (override_nx(target_map,
2555 VME_ALIAS(map_entry))
2556 && prot)
2557 prot |= VM_PROT_EXECUTE;
2558
2559 vm_object_pmap_protect(
2560 object, VME_OFFSET(map_entry),
2561 total_size,
2562 ((map_entry->is_shared
2563 || target_map->mapped_in_other_pmaps)
2564 ? PMAP_NULL :
2565 target_map->pmap),
2566 map_entry->vme_start,
2567 prot);
2568 total_size -= (map_entry->vme_end
2569 - map_entry->vme_start);
2570 next_entry = map_entry->vme_next;
2571 map_entry->needs_copy = FALSE;
2572
2573 vm_object_lock(shadow_object);
2574 while (total_size) {
2575 assert((next_entry->wired_count == 0) ||
2576 (map_entry->wired_count));
2577
2578 if (VME_OBJECT(next_entry) == object) {
2579 vm_object_reference_locked(shadow_object);
2580 VME_OBJECT_SET(next_entry,
2581 shadow_object);
2582 vm_object_deallocate(object);
2583 VME_OFFSET_SET(
2584 next_entry,
2585 (VME_OFFSET(next_entry->vme_prev) +
2586 (next_entry->vme_prev->vme_end
2587 - next_entry->vme_prev->vme_start)));
2588 next_entry->needs_copy = FALSE;
2589 } else {
2590 panic("mach_make_memory_entry_64:"
2591 " map entries out of sync\n");
2592 }
2593 total_size -=
2594 next_entry->vme_end
2595 - next_entry->vme_start;
2596 next_entry = next_entry->vme_next;
2597 }
2598
2599 /*
2600 * Transfer our extra reference to the
2601 * shadow object.
2602 */
2603 vm_object_reference_locked(shadow_object);
2604 vm_object_deallocate(object); /* extra ref */
2605 object = shadow_object;
2606
2607 obj_off = ((local_offset - map_entry->vme_start)
2608 + VME_OFFSET(map_entry));
2609
2610 vm_map_lock_write_to_read(target_map);
2611 }
2612 }
2613
2614 /* note: in the future we can (if necessary) allow for */
2615 /* memory object lists, this will better support */
2616 /* fragmentation, but is it necessary? The user should */
2617 /* be encouraged to create address space oriented */
2618 /* shared objects from CLEAN memory regions which have */
2619 /* a known and defined history. i.e. no inheritence */
2620 /* share, make this call before making the region the */
2621 /* target of ipc's, etc. The code above, protecting */
2622 /* against delayed copy, etc. is mostly defensive. */
2623
2624 wimg_mode = object->wimg_bits;
2625 if(!(object->nophyscache)) {
2626 if(access == MAP_MEM_IO) {
2627 wimg_mode = VM_WIMG_IO;
2628 } else if (access == MAP_MEM_COPYBACK) {
2629 wimg_mode = VM_WIMG_USE_DEFAULT;
2630 } else if (access == MAP_MEM_INNERWBACK) {
2631 wimg_mode = VM_WIMG_INNERWBACK;
2632 } else if (access == MAP_MEM_WTHRU) {
2633 wimg_mode = VM_WIMG_WTHRU;
2634 } else if (access == MAP_MEM_WCOMB) {
2635 wimg_mode = VM_WIMG_WCOMB;
2636 }
2637 }
2638
2639 #if VM_OBJECT_TRACKING_OP_TRUESHARE
2640 if (!object->true_share &&
2641 vm_object_tracking_inited) {
2642 void *bt[VM_OBJECT_TRACKING_BTDEPTH];
2643 int num = 0;
2644
2645 num = OSBacktrace(bt,
2646 VM_OBJECT_TRACKING_BTDEPTH);
2647 btlog_add_entry(vm_object_tracking_btlog,
2648 object,
2649 VM_OBJECT_TRACKING_OP_TRUESHARE,
2650 bt,
2651 num);
2652 }
2653 #endif /* VM_OBJECT_TRACKING_OP_TRUESHARE */
2654
2655 vm_object_lock_assert_exclusive(object);
2656 object->true_share = TRUE;
2657 if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC)
2658 object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
2659
2660 /*
2661 * The memory entry now points to this VM object and we
2662 * need to hold a reference on the VM object. Use the extra
2663 * reference we took earlier to keep the object alive when we
2664 * had to unlock it.
2665 */
2666
2667 vm_map_unlock_read(target_map);
2668 if(real_map != target_map)
2669 vm_map_unlock_read(real_map);
2670
2671 if (object->wimg_bits != wimg_mode)
2672 vm_object_change_wimg_mode(object, wimg_mode);
2673
2674 /* the size of mapped entry that overlaps with our region */
2675 /* which is targeted for share. */
2676 /* (entry_end - entry_start) - */
2677 /* offset of our beg addr within entry */
2678 /* it corresponds to this: */
2679
2680 if(map_size > mappable_size)
2681 map_size = mappable_size;
2682
2683 if (permission & MAP_MEM_NAMED_REUSE) {
2684 /*
2685 * Compare what we got with the "parent_entry".
2686 * If they match, re-use the "parent_entry" instead
2687 * of creating a new one.
2688 */
2689 if (parent_entry != NULL &&
2690 parent_entry->backing.object == object &&
2691 parent_entry->internal == object->internal &&
2692 parent_entry->is_sub_map == FALSE &&
2693 parent_entry->is_pager == FALSE &&
2694 parent_entry->offset == obj_off &&
2695 parent_entry->protection == protections &&
2696 parent_entry->size == map_size &&
2697 ((!(use_data_addr || use_4K_compat) &&
2698 (parent_entry->data_offset == 0)) ||
2699 ((use_data_addr || use_4K_compat) &&
2700 (parent_entry->data_offset == offset_in_page)))) {
2701 /*
2702 * We have a match: re-use "parent_entry".
2703 */
2704 /* release our extra reference on object */
2705 vm_object_unlock(object);
2706 vm_object_deallocate(object);
2707 /* parent_entry->ref_count++; XXX ? */
2708 /* Get an extra send-right on handle */
2709 ipc_port_copy_send(parent_handle);
2710
2711 *size = CAST_DOWN(vm_size_t,
2712 (parent_entry->size -
2713 parent_entry->data_offset));
2714 *object_handle = parent_handle;
2715 return KERN_SUCCESS;
2716 } else {
2717 /*
2718 * No match: we need to create a new entry.
2719 * fall through...
2720 */
2721 }
2722 }
2723
2724 vm_object_unlock(object);
2725 if (mach_memory_entry_allocate(&user_entry, &user_handle)
2726 != KERN_SUCCESS) {
2727 /* release our unused reference on the object */
2728 vm_object_deallocate(object);
2729 return KERN_FAILURE;
2730 }
2731
2732 user_entry->backing.object = object;
2733 user_entry->internal = object->internal;
2734 user_entry->is_sub_map = FALSE;
2735 user_entry->is_pager = FALSE;
2736 user_entry->offset = obj_off;
2737 user_entry->data_offset = offset_in_page;
2738 user_entry->protection = protections;
2739 SET_MAP_MEM(GET_MAP_MEM(permission), user_entry->protection);
2740 user_entry->size = map_size;
2741
2742 /* user_object pager and internal fields are not used */
2743 /* when the object field is filled in. */
2744
2745 *size = CAST_DOWN(vm_size_t, (user_entry->size -
2746 user_entry->data_offset));
2747 *object_handle = user_handle;
2748 return KERN_SUCCESS;
2749
2750 } else {
2751 /* The new object will be base on an existing named object */
2752 if (parent_entry == NULL) {
2753 kr = KERN_INVALID_ARGUMENT;
2754 goto make_mem_done;
2755 }
2756
2757 if (use_data_addr || use_4K_compat) {
2758 /*
2759 * submaps and pagers should only be accessible from within
2760 * the kernel, which shouldn't use the data address flag, so can fail here.
2761 */
2762 if (parent_entry->is_pager || parent_entry->is_sub_map) {
2763 panic("Shouldn't be using data address with a parent entry that is a submap or pager.");
2764 }
2765 /*
2766 * Account for offset to data in parent entry and
2767 * compute our own offset to data.
2768 */
2769 if((offset + *size + parent_entry->data_offset) > parent_entry->size) {
2770 kr = KERN_INVALID_ARGUMENT;
2771 goto make_mem_done;
2772 }
2773
2774 map_start = vm_map_trunc_page(offset + parent_entry->data_offset, PAGE_MASK);
2775 offset_in_page = (offset + parent_entry->data_offset) - map_start;
2776 if (use_4K_compat)
2777 offset_in_page &= ~((signed)(0xFFF));
2778 map_end = vm_map_round_page(offset + parent_entry->data_offset + *size, PAGE_MASK);
2779 map_size = map_end - map_start;
2780 } else {
2781 map_end = vm_map_round_page(offset + *size, PAGE_MASK);
2782 map_size = map_end - map_start;
2783 offset_in_page = 0;
2784
2785 if((offset + map_size) > parent_entry->size) {
2786 kr = KERN_INVALID_ARGUMENT;
2787 goto make_mem_done;
2788 }
2789 }
2790
2791 if (mask_protections) {
2792 /*
2793 * The caller asked us to use the "protections" as
2794 * a mask, so restrict "protections" to what this
2795 * mapping actually allows.
2796 */
2797 protections &= parent_entry->protection;
2798 }
2799 if((protections & parent_entry->protection) != protections) {
2800 kr = KERN_PROTECTION_FAILURE;
2801 goto make_mem_done;
2802 }
2803
2804 if (mach_memory_entry_allocate(&user_entry, &user_handle)
2805 != KERN_SUCCESS) {
2806 kr = KERN_FAILURE;
2807 goto make_mem_done;
2808 }
2809
2810 user_entry->size = map_size;
2811 user_entry->offset = parent_entry->offset + map_start;
2812 user_entry->data_offset = offset_in_page;
2813 user_entry->is_sub_map = parent_entry->is_sub_map;
2814 user_entry->is_pager = parent_entry->is_pager;
2815 user_entry->is_copy = parent_entry->is_copy;
2816 user_entry->internal = parent_entry->internal;
2817 user_entry->protection = protections;
2818
2819 if(access != MAP_MEM_NOOP) {
2820 SET_MAP_MEM(access, user_entry->protection);
2821 }
2822
2823 if(parent_entry->is_sub_map) {
2824 user_entry->backing.map = parent_entry->backing.map;
2825 vm_map_lock(user_entry->backing.map);
2826 user_entry->backing.map->ref_count++;
2827 vm_map_unlock(user_entry->backing.map);
2828 }
2829 else if (parent_entry->is_pager) {
2830 user_entry->backing.pager = parent_entry->backing.pager;
2831 /* JMM - don't we need a reference here? */
2832 } else {
2833 object = parent_entry->backing.object;
2834 assert(object != VM_OBJECT_NULL);
2835 user_entry->backing.object = object;
2836 /* we now point to this object, hold on */
2837 vm_object_lock(object);
2838 vm_object_reference_locked(object);
2839 #if VM_OBJECT_TRACKING_OP_TRUESHARE
2840 if (!object->true_share &&
2841 vm_object_tracking_inited) {
2842 void *bt[VM_OBJECT_TRACKING_BTDEPTH];
2843 int num = 0;
2844
2845 num = OSBacktrace(bt,
2846 VM_OBJECT_TRACKING_BTDEPTH);
2847 btlog_add_entry(vm_object_tracking_btlog,
2848 object,
2849 VM_OBJECT_TRACKING_OP_TRUESHARE,
2850 bt,
2851 num);
2852 }
2853 #endif /* VM_OBJECT_TRACKING_OP_TRUESHARE */
2854
2855 object->true_share = TRUE;
2856 if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC)
2857 object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
2858 vm_object_unlock(object);
2859 }
2860 *size = CAST_DOWN(vm_size_t, (user_entry->size -
2861 user_entry->data_offset));
2862 *object_handle = user_handle;
2863 return KERN_SUCCESS;
2864 }
2865
2866 make_mem_done:
2867 if (user_handle != IP_NULL) {
2868 /*
2869 * Releasing "user_handle" causes the kernel object
2870 * associated with it ("user_entry" here) to also be
2871 * released and freed.
2872 */
2873 mach_memory_entry_port_release(user_handle);
2874 }
2875 return kr;
2876 }
2877
2878 kern_return_t
2879 _mach_make_memory_entry(
2880 vm_map_t target_map,
2881 memory_object_size_t *size,
2882 memory_object_offset_t offset,
2883 vm_prot_t permission,
2884 ipc_port_t *object_handle,
2885 ipc_port_t parent_entry)
2886 {
2887 memory_object_size_t mo_size;
2888 kern_return_t kr;
2889
2890 mo_size = (memory_object_size_t)*size;
2891 kr = mach_make_memory_entry_64(target_map, &mo_size,
2892 (memory_object_offset_t)offset, permission, object_handle,
2893 parent_entry);
2894 *size = mo_size;
2895 return kr;
2896 }
2897
2898 kern_return_t
2899 mach_make_memory_entry(
2900 vm_map_t target_map,
2901 vm_size_t *size,
2902 vm_offset_t offset,
2903 vm_prot_t permission,
2904 ipc_port_t *object_handle,
2905 ipc_port_t parent_entry)
2906 {
2907 memory_object_size_t mo_size;
2908 kern_return_t kr;
2909
2910 mo_size = (memory_object_size_t)*size;
2911 kr = mach_make_memory_entry_64(target_map, &mo_size,
2912 (memory_object_offset_t)offset, permission, object_handle,
2913 parent_entry);
2914 *size = CAST_DOWN(vm_size_t, mo_size);
2915 return kr;
2916 }
2917
2918 /*
2919 * task_wire
2920 *
2921 * Set or clear the map's wiring_required flag. This flag, if set,
2922 * will cause all future virtual memory allocation to allocate
2923 * user wired memory. Unwiring pages wired down as a result of
2924 * this routine is done with the vm_wire interface.
2925 */
2926 kern_return_t
2927 task_wire(
2928 vm_map_t map,
2929 boolean_t must_wire)
2930 {
2931 if (map == VM_MAP_NULL)
2932 return(KERN_INVALID_ARGUMENT);
2933
2934 if (must_wire)
2935 map->wiring_required = TRUE;
2936 else
2937 map->wiring_required = FALSE;
2938
2939 return(KERN_SUCCESS);
2940 }
2941
2942 __private_extern__ kern_return_t
2943 mach_memory_entry_allocate(
2944 vm_named_entry_t *user_entry_p,
2945 ipc_port_t *user_handle_p)
2946 {
2947 vm_named_entry_t user_entry;
2948 ipc_port_t user_handle;
2949 ipc_port_t previous;
2950
2951 user_entry = (vm_named_entry_t) kalloc(sizeof *user_entry);
2952 if (user_entry == NULL)
2953 return KERN_FAILURE;
2954
2955 named_entry_lock_init(user_entry);
2956
2957 user_handle = ipc_port_alloc_kernel();
2958 if (user_handle == IP_NULL) {
2959 kfree(user_entry, sizeof *user_entry);
2960 return KERN_FAILURE;
2961 }
2962 ip_lock(user_handle);
2963
2964 /* make a sonce right */
2965 user_handle->ip_sorights++;
2966 ip_reference(user_handle);
2967
2968 user_handle->ip_destination = IP_NULL;
2969 user_handle->ip_receiver_name = MACH_PORT_NULL;
2970 user_handle->ip_receiver = ipc_space_kernel;
2971
2972 /* make a send right */
2973 user_handle->ip_mscount++;
2974 user_handle->ip_srights++;
2975 ip_reference(user_handle);
2976
2977 ipc_port_nsrequest(user_handle, 1, user_handle, &previous);
2978 /* nsrequest unlocks user_handle */
2979
2980 user_entry->backing.pager = NULL;
2981 user_entry->is_sub_map = FALSE;
2982 user_entry->is_pager = FALSE;
2983 user_entry->is_copy = FALSE;
2984 user_entry->internal = FALSE;
2985 user_entry->size = 0;
2986 user_entry->offset = 0;
2987 user_entry->data_offset = 0;
2988 user_entry->protection = VM_PROT_NONE;
2989 user_entry->ref_count = 1;
2990
2991 ipc_kobject_set(user_handle, (ipc_kobject_t) user_entry,
2992 IKOT_NAMED_ENTRY);
2993
2994 *user_entry_p = user_entry;
2995 *user_handle_p = user_handle;
2996
2997 return KERN_SUCCESS;
2998 }
2999
3000 /*
3001 * mach_memory_object_memory_entry_64
3002 *
3003 * Create a named entry backed by the provided pager.
3004 *
3005 * JMM - we need to hold a reference on the pager -
3006 * and release it when the named entry is destroyed.
3007 */
3008 kern_return_t
3009 mach_memory_object_memory_entry_64(
3010 host_t host,
3011 boolean_t internal,
3012 vm_object_offset_t size,
3013 vm_prot_t permission,
3014 memory_object_t pager,
3015 ipc_port_t *entry_handle)
3016 {
3017 unsigned int access;
3018 vm_named_entry_t user_entry;
3019 ipc_port_t user_handle;
3020
3021 if (host == HOST_NULL)
3022 return(KERN_INVALID_HOST);
3023
3024 if (mach_memory_entry_allocate(&user_entry, &user_handle)
3025 != KERN_SUCCESS) {
3026 return KERN_FAILURE;
3027 }
3028
3029 user_entry->backing.pager = pager;
3030 user_entry->size = size;
3031 user_entry->offset = 0;
3032 user_entry->protection = permission & VM_PROT_ALL;
3033 access = GET_MAP_MEM(permission);
3034 SET_MAP_MEM(access, user_entry->protection);
3035 user_entry->internal = internal;
3036 user_entry->is_sub_map = FALSE;
3037 user_entry->is_pager = TRUE;
3038 assert(user_entry->ref_count == 1);
3039
3040 *entry_handle = user_handle;
3041 return KERN_SUCCESS;
3042 }
3043
3044 kern_return_t
3045 mach_memory_object_memory_entry(
3046 host_t host,
3047 boolean_t internal,
3048 vm_size_t size,
3049 vm_prot_t permission,
3050 memory_object_t pager,
3051 ipc_port_t *entry_handle)
3052 {
3053 return mach_memory_object_memory_entry_64( host, internal,
3054 (vm_object_offset_t)size, permission, pager, entry_handle);
3055 }
3056
3057
3058 kern_return_t
3059 mach_memory_entry_purgable_control(
3060 ipc_port_t entry_port,
3061 vm_purgable_t control,
3062 int *state)
3063 {
3064 kern_return_t kr;
3065 vm_named_entry_t mem_entry;
3066 vm_object_t object;
3067
3068 if (entry_port == IP_NULL ||
3069 ip_kotype(entry_port) != IKOT_NAMED_ENTRY) {
3070 return KERN_INVALID_ARGUMENT;
3071 }
3072 if (control != VM_PURGABLE_SET_STATE &&
3073 control != VM_PURGABLE_GET_STATE)
3074 return(KERN_INVALID_ARGUMENT);
3075
3076 if (control == VM_PURGABLE_SET_STATE &&
3077 (((*state & ~(VM_PURGABLE_ALL_MASKS)) != 0) ||
3078 ((*state & VM_PURGABLE_STATE_MASK) > VM_PURGABLE_STATE_MASK)))
3079 return(KERN_INVALID_ARGUMENT);
3080
3081 mem_entry = (vm_named_entry_t) entry_port->ip_kobject;
3082
3083 named_entry_lock(mem_entry);
3084
3085 if (mem_entry->is_sub_map ||
3086 mem_entry->is_pager ||
3087 mem_entry->is_copy) {
3088 named_entry_unlock(mem_entry);
3089 return KERN_INVALID_ARGUMENT;
3090 }
3091
3092 object = mem_entry->backing.object;
3093 if (object == VM_OBJECT_NULL) {
3094 named_entry_unlock(mem_entry);
3095 return KERN_INVALID_ARGUMENT;
3096 }
3097
3098 vm_object_lock(object);
3099
3100 /* check that named entry covers entire object ? */
3101 if (mem_entry->offset != 0 || object->vo_size != mem_entry->size) {
3102 vm_object_unlock(object);
3103 named_entry_unlock(mem_entry);
3104 return KERN_INVALID_ARGUMENT;
3105 }
3106
3107 named_entry_unlock(mem_entry);
3108
3109 kr = vm_object_purgable_control(object, control, state);
3110
3111 vm_object_unlock(object);
3112
3113 return kr;
3114 }
3115
3116 kern_return_t
3117 mach_memory_entry_get_page_counts(
3118 ipc_port_t entry_port,
3119 unsigned int *resident_page_count,
3120 unsigned int *dirty_page_count)
3121 {
3122 kern_return_t kr;
3123 vm_named_entry_t mem_entry;
3124 vm_object_t object;
3125 vm_object_offset_t offset;
3126 vm_object_size_t size;
3127
3128 if (entry_port == IP_NULL ||
3129 ip_kotype(entry_port) != IKOT_NAMED_ENTRY) {
3130 return KERN_INVALID_ARGUMENT;
3131 }
3132
3133 mem_entry = (vm_named_entry_t) entry_port->ip_kobject;
3134
3135 named_entry_lock(mem_entry);
3136
3137 if (mem_entry->is_sub_map ||
3138 mem_entry->is_pager ||
3139 mem_entry->is_copy) {
3140 named_entry_unlock(mem_entry);
3141 return KERN_INVALID_ARGUMENT;
3142 }
3143
3144 object = mem_entry->backing.object;
3145 if (object == VM_OBJECT_NULL) {
3146 named_entry_unlock(mem_entry);
3147 return KERN_INVALID_ARGUMENT;
3148 }
3149
3150 vm_object_lock(object);
3151
3152 offset = mem_entry->offset;
3153 size = mem_entry->size;
3154
3155 named_entry_unlock(mem_entry);
3156
3157 kr = vm_object_get_page_counts(object, offset, size, resident_page_count, dirty_page_count);
3158
3159 vm_object_unlock(object);
3160
3161 return kr;
3162 }
3163
3164 /*
3165 * mach_memory_entry_port_release:
3166 *
3167 * Release a send right on a named entry port. This is the correct
3168 * way to destroy a named entry. When the last right on the port is
3169 * released, ipc_kobject_destroy() will call mach_destroy_memory_entry().
3170 */
3171 void
3172 mach_memory_entry_port_release(
3173 ipc_port_t port)
3174 {
3175 assert(ip_kotype(port) == IKOT_NAMED_ENTRY);
3176 ipc_port_release_send(port);
3177 }
3178
3179 /*
3180 * mach_destroy_memory_entry:
3181 *
3182 * Drops a reference on a memory entry and destroys the memory entry if
3183 * there are no more references on it.
3184 * NOTE: This routine should not be called to destroy a memory entry from the
3185 * kernel, as it will not release the Mach port associated with the memory
3186 * entry. The proper way to destroy a memory entry in the kernel is to
3187 * call mach_memort_entry_port_release() to release the kernel's send-right on
3188 * the memory entry's port. When the last send right is released, the memory
3189 * entry will be destroyed via ipc_kobject_destroy().
3190 */
3191 void
3192 mach_destroy_memory_entry(
3193 ipc_port_t port)
3194 {
3195 vm_named_entry_t named_entry;
3196 #if MACH_ASSERT
3197 assert(ip_kotype(port) == IKOT_NAMED_ENTRY);
3198 #endif /* MACH_ASSERT */
3199 named_entry = (vm_named_entry_t)port->ip_kobject;
3200
3201 named_entry_lock(named_entry);
3202 named_entry->ref_count -= 1;
3203
3204 if(named_entry->ref_count == 0) {
3205 if (named_entry->is_sub_map) {
3206 vm_map_deallocate(named_entry->backing.map);
3207 } else if (named_entry->is_pager) {
3208 /* JMM - need to drop reference on pager in that case */
3209 } else if (named_entry->is_copy) {
3210 vm_map_copy_discard(named_entry->backing.copy);
3211 } else {
3212 /* release the VM object we've been pointing to */
3213 vm_object_deallocate(named_entry->backing.object);
3214 }
3215
3216 named_entry_unlock(named_entry);
3217 named_entry_lock_destroy(named_entry);
3218
3219 kfree((void *) port->ip_kobject,
3220 sizeof (struct vm_named_entry));
3221 } else
3222 named_entry_unlock(named_entry);
3223 }
3224
3225 /* Allow manipulation of individual page state. This is actually part of */
3226 /* the UPL regimen but takes place on the memory entry rather than on a UPL */
3227
3228 kern_return_t
3229 mach_memory_entry_page_op(
3230 ipc_port_t entry_port,
3231 vm_object_offset_t offset,
3232 int ops,
3233 ppnum_t *phys_entry,
3234 int *flags)
3235 {
3236 vm_named_entry_t mem_entry;
3237 vm_object_t object;
3238 kern_return_t kr;
3239
3240 if (entry_port == IP_NULL ||
3241 ip_kotype(entry_port) != IKOT_NAMED_ENTRY) {
3242 return KERN_INVALID_ARGUMENT;
3243 }
3244
3245 mem_entry = (vm_named_entry_t) entry_port->ip_kobject;
3246
3247 named_entry_lock(mem_entry);
3248
3249 if (mem_entry->is_sub_map ||
3250 mem_entry->is_pager ||
3251 mem_entry->is_copy) {
3252 named_entry_unlock(mem_entry);
3253 return KERN_INVALID_ARGUMENT;
3254 }
3255
3256 object = mem_entry->backing.object;
3257 if (object == VM_OBJECT_NULL) {
3258 named_entry_unlock(mem_entry);
3259 return KERN_INVALID_ARGUMENT;
3260 }
3261
3262 vm_object_reference(object);
3263 named_entry_unlock(mem_entry);
3264
3265 kr = vm_object_page_op(object, offset, ops, phys_entry, flags);
3266
3267 vm_object_deallocate(object);
3268
3269 return kr;
3270 }
3271
3272 /*
3273 * mach_memory_entry_range_op offers performance enhancement over
3274 * mach_memory_entry_page_op for page_op functions which do not require page
3275 * level state to be returned from the call. Page_op was created to provide
3276 * a low-cost alternative to page manipulation via UPLs when only a single
3277 * page was involved. The range_op call establishes the ability in the _op
3278 * family of functions to work on multiple pages where the lack of page level
3279 * state handling allows the caller to avoid the overhead of the upl structures.
3280 */
3281
3282 kern_return_t
3283 mach_memory_entry_range_op(
3284 ipc_port_t entry_port,
3285 vm_object_offset_t offset_beg,
3286 vm_object_offset_t offset_end,
3287 int ops,
3288 int *range)
3289 {
3290 vm_named_entry_t mem_entry;
3291 vm_object_t object;
3292 kern_return_t kr;
3293
3294 if (entry_port == IP_NULL ||
3295 ip_kotype(entry_port) != IKOT_NAMED_ENTRY) {
3296 return KERN_INVALID_ARGUMENT;
3297 }
3298
3299 mem_entry = (vm_named_entry_t) entry_port->ip_kobject;
3300
3301 named_entry_lock(mem_entry);
3302
3303 if (mem_entry->is_sub_map ||
3304 mem_entry->is_pager ||
3305 mem_entry->is_copy) {
3306 named_entry_unlock(mem_entry);
3307 return KERN_INVALID_ARGUMENT;
3308 }
3309
3310 object = mem_entry->backing.object;
3311 if (object == VM_OBJECT_NULL) {
3312 named_entry_unlock(mem_entry);
3313 return KERN_INVALID_ARGUMENT;
3314 }
3315
3316 vm_object_reference(object);
3317 named_entry_unlock(mem_entry);
3318
3319 kr = vm_object_range_op(object,
3320 offset_beg,
3321 offset_end,
3322 ops,
3323 (uint32_t *) range);
3324
3325 vm_object_deallocate(object);
3326
3327 return kr;
3328 }
3329
3330
3331 kern_return_t
3332 set_dp_control_port(
3333 host_priv_t host_priv,
3334 ipc_port_t control_port)
3335 {
3336 if (host_priv == HOST_PRIV_NULL)
3337 return (KERN_INVALID_HOST);
3338
3339 if (IP_VALID(dynamic_pager_control_port))
3340 ipc_port_release_send(dynamic_pager_control_port);
3341
3342 dynamic_pager_control_port = control_port;
3343 return KERN_SUCCESS;
3344 }
3345
3346 kern_return_t
3347 get_dp_control_port(
3348 host_priv_t host_priv,
3349 ipc_port_t *control_port)
3350 {
3351 if (host_priv == HOST_PRIV_NULL)
3352 return (KERN_INVALID_HOST);
3353
3354 *control_port = ipc_port_copy_send(dynamic_pager_control_port);
3355 return KERN_SUCCESS;
3356
3357 }
3358
3359 /* ******* Temporary Internal calls to UPL for BSD ***** */
3360
3361 extern int kernel_upl_map(
3362 vm_map_t map,
3363 upl_t upl,
3364 vm_offset_t *dst_addr);
3365
3366 extern int kernel_upl_unmap(
3367 vm_map_t map,
3368 upl_t upl);
3369
3370 extern int kernel_upl_commit(
3371 upl_t upl,
3372 upl_page_info_t *pl,
3373 mach_msg_type_number_t count);
3374
3375 extern int kernel_upl_commit_range(
3376 upl_t upl,
3377 upl_offset_t offset,
3378 upl_size_t size,
3379 int flags,
3380 upl_page_info_array_t pl,
3381 mach_msg_type_number_t count);
3382
3383 extern int kernel_upl_abort(
3384 upl_t upl,
3385 int abort_type);
3386
3387 extern int kernel_upl_abort_range(
3388 upl_t upl,
3389 upl_offset_t offset,
3390 upl_size_t size,
3391 int abort_flags);
3392
3393
3394 kern_return_t
3395 kernel_upl_map(
3396 vm_map_t map,
3397 upl_t upl,
3398 vm_offset_t *dst_addr)
3399 {
3400 return vm_upl_map(map, upl, dst_addr);
3401 }
3402
3403
3404 kern_return_t
3405 kernel_upl_unmap(
3406 vm_map_t map,
3407 upl_t upl)
3408 {
3409 return vm_upl_unmap(map, upl);
3410 }
3411
3412 kern_return_t
3413 kernel_upl_commit(
3414 upl_t upl,
3415 upl_page_info_t *pl,
3416 mach_msg_type_number_t count)
3417 {
3418 kern_return_t kr;
3419
3420 kr = upl_commit(upl, pl, count);
3421 upl_deallocate(upl);
3422 return kr;
3423 }
3424
3425
3426 kern_return_t
3427 kernel_upl_commit_range(
3428 upl_t upl,
3429 upl_offset_t offset,
3430 upl_size_t size,
3431 int flags,
3432 upl_page_info_array_t pl,
3433 mach_msg_type_number_t count)
3434 {
3435 boolean_t finished = FALSE;
3436 kern_return_t kr;
3437
3438 if (flags & UPL_COMMIT_FREE_ON_EMPTY)
3439 flags |= UPL_COMMIT_NOTIFY_EMPTY;
3440
3441 if (flags & UPL_COMMIT_KERNEL_ONLY_FLAGS) {
3442 return KERN_INVALID_ARGUMENT;
3443 }
3444
3445 kr = upl_commit_range(upl, offset, size, flags, pl, count, &finished);
3446
3447 if ((flags & UPL_COMMIT_NOTIFY_EMPTY) && finished)
3448 upl_deallocate(upl);
3449
3450 return kr;
3451 }
3452
3453 kern_return_t
3454 kernel_upl_abort_range(
3455 upl_t upl,
3456 upl_offset_t offset,
3457 upl_size_t size,
3458 int abort_flags)
3459 {
3460 kern_return_t kr;
3461 boolean_t finished = FALSE;
3462
3463 if (abort_flags & UPL_COMMIT_FREE_ON_EMPTY)
3464 abort_flags |= UPL_COMMIT_NOTIFY_EMPTY;
3465
3466 kr = upl_abort_range(upl, offset, size, abort_flags, &finished);
3467
3468 if ((abort_flags & UPL_COMMIT_FREE_ON_EMPTY) && finished)
3469 upl_deallocate(upl);
3470
3471 return kr;
3472 }
3473
3474 kern_return_t
3475 kernel_upl_abort(
3476 upl_t upl,
3477 int abort_type)
3478 {
3479 kern_return_t kr;
3480
3481 kr = upl_abort(upl, abort_type);
3482 upl_deallocate(upl);
3483 return kr;
3484 }
3485
3486 /*
3487 * Now a kernel-private interface (for BootCache
3488 * use only). Need a cleaner way to create an
3489 * empty vm_map() and return a handle to it.
3490 */
3491
3492 kern_return_t
3493 vm_region_object_create(
3494 __unused vm_map_t target_map,
3495 vm_size_t size,
3496 ipc_port_t *object_handle)
3497 {
3498 vm_named_entry_t user_entry;
3499 ipc_port_t user_handle;
3500
3501 vm_map_t new_map;
3502
3503 if (mach_memory_entry_allocate(&user_entry, &user_handle)
3504 != KERN_SUCCESS) {
3505 return KERN_FAILURE;
3506 }
3507
3508 /* Create a named object based on a submap of specified size */
3509
3510 new_map = vm_map_create(PMAP_NULL, VM_MAP_MIN_ADDRESS,
3511 vm_map_round_page(size,
3512 VM_MAP_PAGE_MASK(target_map)),
3513 TRUE);
3514 vm_map_set_page_shift(new_map, VM_MAP_PAGE_SHIFT(target_map));
3515
3516 user_entry->backing.map = new_map;
3517 user_entry->internal = TRUE;
3518 user_entry->is_sub_map = TRUE;
3519 user_entry->offset = 0;
3520 user_entry->protection = VM_PROT_ALL;
3521 user_entry->size = size;
3522 assert(user_entry->ref_count == 1);
3523
3524 *object_handle = user_handle;
3525 return KERN_SUCCESS;
3526
3527 }
3528
3529 ppnum_t vm_map_get_phys_page( /* forward */
3530 vm_map_t map,
3531 vm_offset_t offset);
3532
3533 ppnum_t
3534 vm_map_get_phys_page(
3535 vm_map_t map,
3536 vm_offset_t addr)
3537 {
3538 vm_object_offset_t offset;
3539 vm_object_t object;
3540 vm_map_offset_t map_offset;
3541 vm_map_entry_t entry;
3542 ppnum_t phys_page = 0;
3543
3544 map_offset = vm_map_trunc_page(addr, PAGE_MASK);
3545
3546 vm_map_lock(map);
3547 while (vm_map_lookup_entry(map, map_offset, &entry)) {
3548
3549 if (VME_OBJECT(entry) == VM_OBJECT_NULL) {
3550 vm_map_unlock(map);
3551 return (ppnum_t) 0;
3552 }
3553 if (entry->is_sub_map) {
3554 vm_map_t old_map;
3555 vm_map_lock(VME_SUBMAP(entry));
3556 old_map = map;
3557 map = VME_SUBMAP(entry);
3558 map_offset = (VME_OFFSET(entry) +
3559 (map_offset - entry->vme_start));
3560 vm_map_unlock(old_map);
3561 continue;
3562 }
3563 if (VME_OBJECT(entry)->phys_contiguous) {
3564 /* These are not standard pageable memory mappings */
3565 /* If they are not present in the object they will */
3566 /* have to be picked up from the pager through the */
3567 /* fault mechanism. */
3568 if (VME_OBJECT(entry)->vo_shadow_offset == 0) {
3569 /* need to call vm_fault */
3570 vm_map_unlock(map);
3571 vm_fault(map, map_offset, VM_PROT_NONE,
3572 FALSE, THREAD_UNINT, NULL, 0);
3573 vm_map_lock(map);
3574 continue;
3575 }
3576 offset = (VME_OFFSET(entry) +
3577 (map_offset - entry->vme_start));
3578 phys_page = (ppnum_t)
3579 ((VME_OBJECT(entry)->vo_shadow_offset
3580 + offset) >> PAGE_SHIFT);
3581 break;
3582
3583 }
3584 offset = (VME_OFFSET(entry) + (map_offset - entry->vme_start));
3585 object = VME_OBJECT(entry);
3586 vm_object_lock(object);
3587 while (TRUE) {
3588 vm_page_t dst_page = vm_page_lookup(object,offset);
3589 if(dst_page == VM_PAGE_NULL) {
3590 if(object->shadow) {
3591 vm_object_t old_object;
3592 vm_object_lock(object->shadow);
3593 old_object = object;
3594 offset = offset + object->vo_shadow_offset;
3595 object = object->shadow;
3596 vm_object_unlock(old_object);
3597 } else {
3598 vm_object_unlock(object);
3599 break;
3600 }
3601 } else {
3602 phys_page = (ppnum_t)(VM_PAGE_GET_PHYS_PAGE(dst_page));
3603 vm_object_unlock(object);
3604 break;
3605 }
3606 }
3607 break;
3608
3609 }
3610
3611 vm_map_unlock(map);
3612 return phys_page;
3613 }
3614
3615
3616 #if 0
3617 kern_return_t kernel_object_iopl_request( /* forward */
3618 vm_named_entry_t named_entry,
3619 memory_object_offset_t offset,
3620 upl_size_t *upl_size,
3621 upl_t *upl_ptr,
3622 upl_page_info_array_t user_page_list,
3623 unsigned int *page_list_count,
3624 int *flags);
3625
3626 kern_return_t
3627 kernel_object_iopl_request(
3628 vm_named_entry_t named_entry,
3629 memory_object_offset_t offset,
3630 upl_size_t *upl_size,
3631 upl_t *upl_ptr,
3632 upl_page_info_array_t user_page_list,
3633 unsigned int *page_list_count,
3634 int *flags)
3635 {
3636 vm_object_t object;
3637 kern_return_t ret;
3638
3639 int caller_flags;
3640
3641 caller_flags = *flags;
3642
3643 if (caller_flags & ~UPL_VALID_FLAGS) {
3644 /*
3645 * For forward compatibility's sake,
3646 * reject any unknown flag.
3647 */
3648 return KERN_INVALID_VALUE;
3649 }
3650
3651 /* a few checks to make sure user is obeying rules */
3652 if(*upl_size == 0) {
3653 if(offset >= named_entry->size)
3654 return(KERN_INVALID_RIGHT);
3655 *upl_size = (upl_size_t) (named_entry->size - offset);
3656 if (*upl_size != named_entry->size - offset)
3657 return KERN_INVALID_ARGUMENT;
3658 }
3659 if(caller_flags & UPL_COPYOUT_FROM) {
3660 if((named_entry->protection & VM_PROT_READ)
3661 != VM_PROT_READ) {
3662 return(KERN_INVALID_RIGHT);
3663 }
3664 } else {
3665 if((named_entry->protection &
3666 (VM_PROT_READ | VM_PROT_WRITE))
3667 != (VM_PROT_READ | VM_PROT_WRITE)) {
3668 return(KERN_INVALID_RIGHT);
3669 }
3670 }
3671 if(named_entry->size < (offset + *upl_size))
3672 return(KERN_INVALID_ARGUMENT);
3673
3674 /* the callers parameter offset is defined to be the */
3675 /* offset from beginning of named entry offset in object */
3676 offset = offset + named_entry->offset;
3677
3678 if (named_entry->is_sub_map ||
3679 named_entry->is_copy)
3680 return KERN_INVALID_ARGUMENT;
3681
3682 named_entry_lock(named_entry);
3683
3684 if (named_entry->is_pager) {
3685 object = vm_object_enter(named_entry->backing.pager,
3686 named_entry->offset + named_entry->size,
3687 named_entry->internal,
3688 FALSE,
3689 FALSE);
3690 if (object == VM_OBJECT_NULL) {
3691 named_entry_unlock(named_entry);
3692 return(KERN_INVALID_OBJECT);
3693 }
3694
3695 /* JMM - drop reference on the pager here? */
3696
3697 /* create an extra reference for the object */
3698 vm_object_lock(object);
3699 vm_object_reference_locked(object);
3700 named_entry->backing.object = object;
3701 named_entry->is_pager = FALSE;
3702 named_entry_unlock(named_entry);
3703
3704 /* wait for object (if any) to be ready */
3705 if (!named_entry->internal) {
3706 while (!object->pager_ready) {
3707 vm_object_wait(object,
3708 VM_OBJECT_EVENT_PAGER_READY,
3709 THREAD_UNINT);
3710 vm_object_lock(object);
3711 }
3712 }
3713 vm_object_unlock(object);
3714
3715 } else {
3716 /* This is the case where we are going to operate */
3717 /* an an already known object. If the object is */
3718 /* not ready it is internal. An external */
3719 /* object cannot be mapped until it is ready */
3720 /* we can therefore avoid the ready check */
3721 /* in this case. */
3722 object = named_entry->backing.object;
3723 vm_object_reference(object);
3724 named_entry_unlock(named_entry);
3725 }
3726
3727 if (!object->private) {
3728 if (*upl_size > MAX_UPL_TRANSFER_BYTES)
3729 *upl_size = MAX_UPL_TRANSFER_BYTES;
3730 if (object->phys_contiguous) {
3731 *flags = UPL_PHYS_CONTIG;
3732 } else {
3733 *flags = 0;
3734 }
3735 } else {
3736 *flags = UPL_DEV_MEMORY | UPL_PHYS_CONTIG;
3737 }
3738
3739 ret = vm_object_iopl_request(object,
3740 offset,
3741 *upl_size,
3742 upl_ptr,
3743 user_page_list,
3744 page_list_count,
3745 (upl_control_flags_t)(unsigned int)caller_flags);
3746 vm_object_deallocate(object);
3747 return ret;
3748 }
3749 #endif
mirror of XNU