]> git.saurik.com Git - apple/xnu.git/blob - osfmk/vm/vm_user.c
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-1504.9.26.tar.gz
[apple/xnu.git] / osfmk / vm / vm_user.c
1 /*
2 * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56 /*
57 */
58 /*
59 * File: vm/vm_user.c
60 * Author: Avadis Tevanian, Jr., Michael Wayne Young
61 *
62 * User-exported virtual memory functions.
63 */
64
65 /*
66 * There are three implementations of the "XXX_allocate" functionality in
67 * the kernel: mach_vm_allocate (for any task on the platform), vm_allocate
68 * (for a task with the same address space size, especially the current task),
69 * and vm32_vm_allocate (for the specific case of a 32-bit task). vm_allocate
70 * in the kernel should only be used on the kernel_task. vm32_vm_allocate only
71 * makes sense on platforms where a user task can either be 32 or 64, or the kernel
72 * task can be 32 or 64. mach_vm_allocate makes sense everywhere, and is preferred
73 * for new code.
74 *
75 * The entrypoints into the kernel are more complex. All platforms support a
76 * mach_vm_allocate-style API (subsystem 4800) which operates with the largest
77 * size types for the platform. On platforms that only support U32/K32,
78 * subsystem 4800 is all you need. On platforms that support both U32 and U64,
79 * subsystem 3800 is used disambiguate the size of parameters, and they will
80 * always be 32-bit and call into the vm32_vm_allocate APIs. On non-U32/K32 platforms,
81 * the MIG glue should never call into vm_allocate directly, because the calling
82 * task and kernel_task are unlikely to use the same size parameters
83 *
84 * New VM call implementations should be added here and to mach_vm.defs
85 * (subsystem 4800), and use mach_vm_* "wide" types.
86 */
87
88 #include <debug.h>
89
90 #include <vm_cpm.h>
91 #include <mach/boolean.h>
92 #include <mach/kern_return.h>
93 #include <mach/mach_types.h> /* to get vm_address_t */
94 #include <mach/memory_object.h>
95 #include <mach/std_types.h> /* to get pointer_t */
96 #include <mach/upl.h>
97 #include <mach/vm_attributes.h>
98 #include <mach/vm_param.h>
99 #include <mach/vm_statistics.h>
100 #include <mach/mach_syscalls.h>
101
102 #include <mach/host_priv_server.h>
103 #include <mach/mach_vm_server.h>
104 #include <mach/vm_map_server.h>
105
106 #include <kern/host.h>
107 #include <kern/kalloc.h>
108 #include <kern/task.h>
109 #include <kern/misc_protos.h>
110 #include <vm/vm_fault.h>
111 #include <vm/vm_map.h>
112 #include <vm/vm_object.h>
113 #include <vm/vm_page.h>
114 #include <vm/memory_object.h>
115 #include <vm/vm_pageout.h>
116 #include <vm/vm_protos.h>
117
118 vm_size_t upl_offset_to_pagelist = 0;
119
120 #if VM_CPM
121 #include <vm/cpm.h>
122 #endif /* VM_CPM */
123
124 ipc_port_t dynamic_pager_control_port=NULL;
125
126 /*
127 * mach_vm_allocate allocates "zero fill" memory in the specfied
128 * map.
129 */
130 kern_return_t
131 mach_vm_allocate(
132 vm_map_t map,
133 mach_vm_offset_t *addr,
134 mach_vm_size_t size,
135 int flags)
136 {
137 vm_map_offset_t map_addr;
138 vm_map_size_t map_size;
139 kern_return_t result;
140 boolean_t anywhere;
141
142 /* filter out any kernel-only flags */
143 if (flags & ~VM_FLAGS_USER_ALLOCATE)
144 return KERN_INVALID_ARGUMENT;
145
146 if (map == VM_MAP_NULL)
147 return(KERN_INVALID_ARGUMENT);
148 if (size == 0) {
149 *addr = 0;
150 return(KERN_SUCCESS);
151 }
152
153 anywhere = ((VM_FLAGS_ANYWHERE & flags) != 0);
154 if (anywhere) {
155 /*
156 * No specific address requested, so start candidate address
157 * search at the minimum address in the map. However, if that
158 * minimum is 0, bump it up by PAGE_SIZE. We want to limit
159 * allocations of PAGEZERO to explicit requests since its
160 * normal use is to catch dereferences of NULL and many
161 * applications also treat pointers with a value of 0 as
162 * special and suddenly having address 0 contain useable
163 * memory would tend to confuse those applications.
164 */
165 map_addr = vm_map_min(map);
166 if (map_addr == 0)
167 map_addr += PAGE_SIZE;
168 } else
169 map_addr = vm_map_trunc_page(*addr);
170 map_size = vm_map_round_page(size);
171 if (map_size == 0) {
172 return(KERN_INVALID_ARGUMENT);
173 }
174
175 result = vm_map_enter(
176 map,
177 &map_addr,
178 map_size,
179 (vm_map_offset_t)0,
180 flags,
181 VM_OBJECT_NULL,
182 (vm_object_offset_t)0,
183 FALSE,
184 VM_PROT_DEFAULT,
185 VM_PROT_ALL,
186 VM_INHERIT_DEFAULT);
187
188 *addr = map_addr;
189 return(result);
190 }
191
192 /*
193 * vm_allocate
194 * Legacy routine that allocates "zero fill" memory in the specfied
195 * map (which is limited to the same size as the kernel).
196 */
197 kern_return_t
198 vm_allocate(
199 vm_map_t map,
200 vm_offset_t *addr,
201 vm_size_t size,
202 int flags)
203 {
204 vm_map_offset_t map_addr;
205 vm_map_size_t map_size;
206 kern_return_t result;
207 boolean_t anywhere;
208
209 /* filter out any kernel-only flags */
210 if (flags & ~VM_FLAGS_USER_ALLOCATE)
211 return KERN_INVALID_ARGUMENT;
212
213 if (map == VM_MAP_NULL)
214 return(KERN_INVALID_ARGUMENT);
215 if (size == 0) {
216 *addr = 0;
217 return(KERN_SUCCESS);
218 }
219
220 anywhere = ((VM_FLAGS_ANYWHERE & flags) != 0);
221 if (anywhere) {
222 /*
223 * No specific address requested, so start candidate address
224 * search at the minimum address in the map. However, if that
225 * minimum is 0, bump it up by PAGE_SIZE. We want to limit
226 * allocations of PAGEZERO to explicit requests since its
227 * normal use is to catch dereferences of NULL and many
228 * applications also treat pointers with a value of 0 as
229 * special and suddenly having address 0 contain useable
230 * memory would tend to confuse those applications.
231 */
232 map_addr = vm_map_min(map);
233 if (map_addr == 0)
234 map_addr += PAGE_SIZE;
235 } else
236 map_addr = vm_map_trunc_page(*addr);
237 map_size = vm_map_round_page(size);
238 if (map_size == 0) {
239 return(KERN_INVALID_ARGUMENT);
240 }
241
242 result = vm_map_enter(
243 map,
244 &map_addr,
245 map_size,
246 (vm_map_offset_t)0,
247 flags,
248 VM_OBJECT_NULL,
249 (vm_object_offset_t)0,
250 FALSE,
251 VM_PROT_DEFAULT,
252 VM_PROT_ALL,
253 VM_INHERIT_DEFAULT);
254
255 *addr = CAST_DOWN(vm_offset_t, map_addr);
256 return(result);
257 }
258
259 /*
260 * mach_vm_deallocate -
261 * deallocates the specified range of addresses in the
262 * specified address map.
263 */
264 kern_return_t
265 mach_vm_deallocate(
266 vm_map_t map,
267 mach_vm_offset_t start,
268 mach_vm_size_t size)
269 {
270 if ((map == VM_MAP_NULL) || (start + size < start))
271 return(KERN_INVALID_ARGUMENT);
272
273 if (size == (mach_vm_offset_t) 0)
274 return(KERN_SUCCESS);
275
276 return(vm_map_remove(map, vm_map_trunc_page(start),
277 vm_map_round_page(start+size), VM_MAP_NO_FLAGS));
278 }
279
280 /*
281 * vm_deallocate -
282 * deallocates the specified range of addresses in the
283 * specified address map (limited to addresses the same
284 * size as the kernel).
285 */
286 kern_return_t
287 vm_deallocate(
288 register vm_map_t map,
289 vm_offset_t start,
290 vm_size_t size)
291 {
292 if ((map == VM_MAP_NULL) || (start + size < start))
293 return(KERN_INVALID_ARGUMENT);
294
295 if (size == (vm_offset_t) 0)
296 return(KERN_SUCCESS);
297
298 return(vm_map_remove(map, vm_map_trunc_page(start),
299 vm_map_round_page(start+size), VM_MAP_NO_FLAGS));
300 }
301
302 /*
303 * mach_vm_inherit -
304 * Sets the inheritance of the specified range in the
305 * specified map.
306 */
307 kern_return_t
308 mach_vm_inherit(
309 vm_map_t map,
310 mach_vm_offset_t start,
311 mach_vm_size_t size,
312 vm_inherit_t new_inheritance)
313 {
314 if ((map == VM_MAP_NULL) || (start + size < start) ||
315 (new_inheritance > VM_INHERIT_LAST_VALID))
316 return(KERN_INVALID_ARGUMENT);
317
318 if (size == 0)
319 return KERN_SUCCESS;
320
321 return(vm_map_inherit(map,
322 vm_map_trunc_page(start),
323 vm_map_round_page(start+size),
324 new_inheritance));
325 }
326
327 /*
328 * vm_inherit -
329 * Sets the inheritance of the specified range in the
330 * specified map (range limited to addresses
331 */
332 kern_return_t
333 vm_inherit(
334 register vm_map_t map,
335 vm_offset_t start,
336 vm_size_t size,
337 vm_inherit_t new_inheritance)
338 {
339 if ((map == VM_MAP_NULL) || (start + size < start) ||
340 (new_inheritance > VM_INHERIT_LAST_VALID))
341 return(KERN_INVALID_ARGUMENT);
342
343 if (size == 0)
344 return KERN_SUCCESS;
345
346 return(vm_map_inherit(map,
347 vm_map_trunc_page(start),
348 vm_map_round_page(start+size),
349 new_inheritance));
350 }
351
352 /*
353 * mach_vm_protect -
354 * Sets the protection of the specified range in the
355 * specified map.
356 */
357
358 kern_return_t
359 mach_vm_protect(
360 vm_map_t map,
361 mach_vm_offset_t start,
362 mach_vm_size_t size,
363 boolean_t set_maximum,
364 vm_prot_t new_protection)
365 {
366 if ((map == VM_MAP_NULL) || (start + size < start) ||
367 (new_protection & ~(VM_PROT_ALL | VM_PROT_COPY)))
368 return(KERN_INVALID_ARGUMENT);
369
370 if (size == 0)
371 return KERN_SUCCESS;
372
373 return(vm_map_protect(map,
374 vm_map_trunc_page(start),
375 vm_map_round_page(start+size),
376 new_protection,
377 set_maximum));
378 }
379
380 /*
381 * vm_protect -
382 * Sets the protection of the specified range in the
383 * specified map. Addressability of the range limited
384 * to the same size as the kernel.
385 */
386
387 kern_return_t
388 vm_protect(
389 vm_map_t map,
390 vm_offset_t start,
391 vm_size_t size,
392 boolean_t set_maximum,
393 vm_prot_t new_protection)
394 {
395 if ((map == VM_MAP_NULL) || (start + size < start) ||
396 (new_protection & ~(VM_PROT_ALL | VM_PROT_COPY)))
397 return(KERN_INVALID_ARGUMENT);
398
399 if (size == 0)
400 return KERN_SUCCESS;
401
402 return(vm_map_protect(map,
403 vm_map_trunc_page(start),
404 vm_map_round_page(start+size),
405 new_protection,
406 set_maximum));
407 }
408
409 /*
410 * mach_vm_machine_attributes -
411 * Handle machine-specific attributes for a mapping, such
412 * as cachability, migrability, etc.
413 */
414 kern_return_t
415 mach_vm_machine_attribute(
416 vm_map_t map,
417 mach_vm_address_t addr,
418 mach_vm_size_t size,
419 vm_machine_attribute_t attribute,
420 vm_machine_attribute_val_t* value) /* IN/OUT */
421 {
422 if ((map == VM_MAP_NULL) || (addr + size < addr))
423 return(KERN_INVALID_ARGUMENT);
424
425 if (size == 0)
426 return KERN_SUCCESS;
427
428 return vm_map_machine_attribute(map,
429 vm_map_trunc_page(addr),
430 vm_map_round_page(addr+size),
431 attribute,
432 value);
433 }
434
435 /*
436 * vm_machine_attribute -
437 * Handle machine-specific attributes for a mapping, such
438 * as cachability, migrability, etc. Limited addressability
439 * (same range limits as for the native kernel map).
440 */
441 kern_return_t
442 vm_machine_attribute(
443 vm_map_t map,
444 vm_address_t addr,
445 vm_size_t size,
446 vm_machine_attribute_t attribute,
447 vm_machine_attribute_val_t* value) /* IN/OUT */
448 {
449 if ((map == VM_MAP_NULL) || (addr + size < addr))
450 return(KERN_INVALID_ARGUMENT);
451
452 if (size == 0)
453 return KERN_SUCCESS;
454
455 return vm_map_machine_attribute(map,
456 vm_map_trunc_page(addr),
457 vm_map_round_page(addr+size),
458 attribute,
459 value);
460 }
461
462 /*
463 * mach_vm_read -
464 * Read/copy a range from one address space and return it to the caller.
465 *
466 * It is assumed that the address for the returned memory is selected by
467 * the IPC implementation as part of receiving the reply to this call.
468 * If IPC isn't used, the caller must deal with the vm_map_copy_t object
469 * that gets returned.
470 *
471 * JMM - because of mach_msg_type_number_t, this call is limited to a
472 * single 4GB region at this time.
473 *
474 */
475 kern_return_t
476 mach_vm_read(
477 vm_map_t map,
478 mach_vm_address_t addr,
479 mach_vm_size_t size,
480 pointer_t *data,
481 mach_msg_type_number_t *data_size)
482 {
483 kern_return_t error;
484 vm_map_copy_t ipc_address;
485
486 if (map == VM_MAP_NULL)
487 return(KERN_INVALID_ARGUMENT);
488
489 if ((mach_msg_type_number_t) size != size)
490 return KERN_INVALID_ARGUMENT;
491
492 error = vm_map_copyin(map,
493 (vm_map_address_t)addr,
494 (vm_map_size_t)size,
495 FALSE, /* src_destroy */
496 &ipc_address);
497
498 if (KERN_SUCCESS == error) {
499 *data = (pointer_t) ipc_address;
500 *data_size = (mach_msg_type_number_t) size;
501 assert(*data_size == size);
502 }
503 return(error);
504 }
505
506 /*
507 * vm_read -
508 * Read/copy a range from one address space and return it to the caller.
509 * Limited addressability (same range limits as for the native kernel map).
510 *
511 * It is assumed that the address for the returned memory is selected by
512 * the IPC implementation as part of receiving the reply to this call.
513 * If IPC isn't used, the caller must deal with the vm_map_copy_t object
514 * that gets returned.
515 */
516 kern_return_t
517 vm_read(
518 vm_map_t map,
519 vm_address_t addr,
520 vm_size_t size,
521 pointer_t *data,
522 mach_msg_type_number_t *data_size)
523 {
524 kern_return_t error;
525 vm_map_copy_t ipc_address;
526
527 if (map == VM_MAP_NULL)
528 return(KERN_INVALID_ARGUMENT);
529
530 if (size > (unsigned)(mach_msg_type_number_t) -1) {
531 /*
532 * The kernel could handle a 64-bit "size" value, but
533 * it could not return the size of the data in "*data_size"
534 * without overflowing.
535 * Let's reject this "size" as invalid.
536 */
537 return KERN_INVALID_ARGUMENT;
538 }
539
540 error = vm_map_copyin(map,
541 (vm_map_address_t)addr,
542 (vm_map_size_t)size,
543 FALSE, /* src_destroy */
544 &ipc_address);
545
546 if (KERN_SUCCESS == error) {
547 *data = (pointer_t) ipc_address;
548 *data_size = (mach_msg_type_number_t) size;
549 assert(*data_size == size);
550 }
551 return(error);
552 }
553
554 /*
555 * mach_vm_read_list -
556 * Read/copy a list of address ranges from specified map.
557 *
558 * MIG does not know how to deal with a returned array of
559 * vm_map_copy_t structures, so we have to do the copyout
560 * manually here.
561 */
562 kern_return_t
563 mach_vm_read_list(
564 vm_map_t map,
565 mach_vm_read_entry_t data_list,
566 natural_t count)
567 {
568 mach_msg_type_number_t i;
569 kern_return_t error;
570 vm_map_copy_t copy;
571
572 if (map == VM_MAP_NULL ||
573 count > VM_MAP_ENTRY_MAX)
574 return(KERN_INVALID_ARGUMENT);
575
576 error = KERN_SUCCESS;
577 for(i=0; i<count; i++) {
578 vm_map_address_t map_addr;
579 vm_map_size_t map_size;
580
581 map_addr = (vm_map_address_t)(data_list[i].address);
582 map_size = (vm_map_size_t)(data_list[i].size);
583
584 if(map_size != 0) {
585 error = vm_map_copyin(map,
586 map_addr,
587 map_size,
588 FALSE, /* src_destroy */
589 &copy);
590 if (KERN_SUCCESS == error) {
591 error = vm_map_copyout(
592 current_task()->map,
593 &map_addr,
594 copy);
595 if (KERN_SUCCESS == error) {
596 data_list[i].address = map_addr;
597 continue;
598 }
599 vm_map_copy_discard(copy);
600 }
601 }
602 data_list[i].address = (mach_vm_address_t)0;
603 data_list[i].size = (mach_vm_size_t)0;
604 }
605 return(error);
606 }
607
608 /*
609 * vm_read_list -
610 * Read/copy a list of address ranges from specified map.
611 *
612 * MIG does not know how to deal with a returned array of
613 * vm_map_copy_t structures, so we have to do the copyout
614 * manually here.
615 *
616 * The source and destination ranges are limited to those
617 * that can be described with a vm_address_t (i.e. same
618 * size map as the kernel).
619 *
620 * JMM - If the result of the copyout is an address range
621 * that cannot be described with a vm_address_t (i.e. the
622 * caller had a larger address space but used this call
623 * anyway), it will result in a truncated address being
624 * returned (and a likely confused caller).
625 */
626
627 kern_return_t
628 vm_read_list(
629 vm_map_t map,
630 vm_read_entry_t data_list,
631 natural_t count)
632 {
633 mach_msg_type_number_t i;
634 kern_return_t error;
635 vm_map_copy_t copy;
636
637 if (map == VM_MAP_NULL ||
638 count > VM_MAP_ENTRY_MAX)
639 return(KERN_INVALID_ARGUMENT);
640
641 error = KERN_SUCCESS;
642 for(i=0; i<count; i++) {
643 vm_map_address_t map_addr;
644 vm_map_size_t map_size;
645
646 map_addr = (vm_map_address_t)(data_list[i].address);
647 map_size = (vm_map_size_t)(data_list[i].size);
648
649 if(map_size != 0) {
650 error = vm_map_copyin(map,
651 map_addr,
652 map_size,
653 FALSE, /* src_destroy */
654 &copy);
655 if (KERN_SUCCESS == error) {
656 error = vm_map_copyout(current_task()->map,
657 &map_addr,
658 copy);
659 if (KERN_SUCCESS == error) {
660 data_list[i].address =
661 CAST_DOWN(vm_offset_t, map_addr);
662 continue;
663 }
664 vm_map_copy_discard(copy);
665 }
666 }
667 data_list[i].address = (mach_vm_address_t)0;
668 data_list[i].size = (mach_vm_size_t)0;
669 }
670 return(error);
671 }
672
673 /*
674 * mach_vm_read_overwrite -
675 * Overwrite a range of the current map with data from the specified
676 * map/address range.
677 *
678 * In making an assumption that the current thread is local, it is
679 * no longer cluster-safe without a fully supportive local proxy
680 * thread/task (but we don't support cluster's anymore so this is moot).
681 */
682
683 kern_return_t
684 mach_vm_read_overwrite(
685 vm_map_t map,
686 mach_vm_address_t address,
687 mach_vm_size_t size,
688 mach_vm_address_t data,
689 mach_vm_size_t *data_size)
690 {
691 kern_return_t error;
692 vm_map_copy_t copy;
693
694 if (map == VM_MAP_NULL)
695 return(KERN_INVALID_ARGUMENT);
696
697 error = vm_map_copyin(map, (vm_map_address_t)address,
698 (vm_map_size_t)size, FALSE, &copy);
699
700 if (KERN_SUCCESS == error) {
701 error = vm_map_copy_overwrite(current_thread()->map,
702 (vm_map_address_t)data,
703 copy, FALSE);
704 if (KERN_SUCCESS == error) {
705 *data_size = size;
706 return error;
707 }
708 vm_map_copy_discard(copy);
709 }
710 return(error);
711 }
712
713 /*
714 * vm_read_overwrite -
715 * Overwrite a range of the current map with data from the specified
716 * map/address range.
717 *
718 * This routine adds the additional limitation that the source and
719 * destination ranges must be describable with vm_address_t values
720 * (i.e. the same size address spaces as the kernel, or at least the
721 * the ranges are in that first portion of the respective address
722 * spaces).
723 */
724
725 kern_return_t
726 vm_read_overwrite(
727 vm_map_t map,
728 vm_address_t address,
729 vm_size_t size,
730 vm_address_t data,
731 vm_size_t *data_size)
732 {
733 kern_return_t error;
734 vm_map_copy_t copy;
735
736 if (map == VM_MAP_NULL)
737 return(KERN_INVALID_ARGUMENT);
738
739 error = vm_map_copyin(map, (vm_map_address_t)address,
740 (vm_map_size_t)size, FALSE, &copy);
741
742 if (KERN_SUCCESS == error) {
743 error = vm_map_copy_overwrite(current_thread()->map,
744 (vm_map_address_t)data,
745 copy, FALSE);
746 if (KERN_SUCCESS == error) {
747 *data_size = size;
748 return error;
749 }
750 vm_map_copy_discard(copy);
751 }
752 return(error);
753 }
754
755
756 /*
757 * mach_vm_write -
758 * Overwrite the specified address range with the data provided
759 * (from the current map).
760 */
761 kern_return_t
762 mach_vm_write(
763 vm_map_t map,
764 mach_vm_address_t address,
765 pointer_t data,
766 __unused mach_msg_type_number_t size)
767 {
768 if (map == VM_MAP_NULL)
769 return KERN_INVALID_ARGUMENT;
770
771 return vm_map_copy_overwrite(map, (vm_map_address_t)address,
772 (vm_map_copy_t) data, FALSE /* interruptible XXX */);
773 }
774
775 /*
776 * vm_write -
777 * Overwrite the specified address range with the data provided
778 * (from the current map).
779 *
780 * The addressability of the range of addresses to overwrite is
781 * limited bu the use of a vm_address_t (same size as kernel map).
782 * Either the target map is also small, or the range is in the
783 * low addresses within it.
784 */
785 kern_return_t
786 vm_write(
787 vm_map_t map,
788 vm_address_t address,
789 pointer_t data,
790 __unused mach_msg_type_number_t size)
791 {
792 if (map == VM_MAP_NULL)
793 return KERN_INVALID_ARGUMENT;
794
795 return vm_map_copy_overwrite(map, (vm_map_address_t)address,
796 (vm_map_copy_t) data, FALSE /* interruptible XXX */);
797 }
798
799 /*
800 * mach_vm_copy -
801 * Overwrite one range of the specified map with the contents of
802 * another range within that same map (i.e. both address ranges
803 * are "over there").
804 */
805 kern_return_t
806 mach_vm_copy(
807 vm_map_t map,
808 mach_vm_address_t source_address,
809 mach_vm_size_t size,
810 mach_vm_address_t dest_address)
811 {
812 vm_map_copy_t copy;
813 kern_return_t kr;
814
815 if (map == VM_MAP_NULL)
816 return KERN_INVALID_ARGUMENT;
817
818 kr = vm_map_copyin(map, (vm_map_address_t)source_address,
819 (vm_map_size_t)size, FALSE, &copy);
820
821 if (KERN_SUCCESS == kr) {
822 kr = vm_map_copy_overwrite(map,
823 (vm_map_address_t)dest_address,
824 copy, FALSE /* interruptible XXX */);
825
826 if (KERN_SUCCESS != kr)
827 vm_map_copy_discard(copy);
828 }
829 return kr;
830 }
831
832 kern_return_t
833 vm_copy(
834 vm_map_t map,
835 vm_address_t source_address,
836 vm_size_t size,
837 vm_address_t dest_address)
838 {
839 vm_map_copy_t copy;
840 kern_return_t kr;
841
842 if (map == VM_MAP_NULL)
843 return KERN_INVALID_ARGUMENT;
844
845 kr = vm_map_copyin(map, (vm_map_address_t)source_address,
846 (vm_map_size_t)size, FALSE, &copy);
847
848 if (KERN_SUCCESS == kr) {
849 kr = vm_map_copy_overwrite(map,
850 (vm_map_address_t)dest_address,
851 copy, FALSE /* interruptible XXX */);
852
853 if (KERN_SUCCESS != kr)
854 vm_map_copy_discard(copy);
855 }
856 return kr;
857 }
858
859 /*
860 * mach_vm_map -
861 * Map some range of an object into an address space.
862 *
863 * The object can be one of several types of objects:
864 * NULL - anonymous memory
865 * a named entry - a range within another address space
866 * or a range within a memory object
867 * a whole memory object
868 *
869 */
870 kern_return_t
871 mach_vm_map(
872 vm_map_t target_map,
873 mach_vm_offset_t *address,
874 mach_vm_size_t initial_size,
875 mach_vm_offset_t mask,
876 int flags,
877 ipc_port_t port,
878 vm_object_offset_t offset,
879 boolean_t copy,
880 vm_prot_t cur_protection,
881 vm_prot_t max_protection,
882 vm_inherit_t inheritance)
883 {
884 /* filter out any kernel-only flags */
885 if (flags & ~VM_FLAGS_USER_MAP)
886 return KERN_INVALID_ARGUMENT;
887
888 return vm_map_enter_mem_object(target_map,
889 address,
890 initial_size,
891 mask,
892 flags,
893 port,
894 offset,
895 copy,
896 cur_protection,
897 max_protection,
898 inheritance);
899 }
900
901
902 /* legacy interface */
903 kern_return_t
904 vm_map_64(
905 vm_map_t target_map,
906 vm_offset_t *address,
907 vm_size_t size,
908 vm_offset_t mask,
909 int flags,
910 ipc_port_t port,
911 vm_object_offset_t offset,
912 boolean_t copy,
913 vm_prot_t cur_protection,
914 vm_prot_t max_protection,
915 vm_inherit_t inheritance)
916 {
917 mach_vm_address_t map_addr;
918 mach_vm_size_t map_size;
919 mach_vm_offset_t map_mask;
920 kern_return_t kr;
921
922 map_addr = (mach_vm_address_t)*address;
923 map_size = (mach_vm_size_t)size;
924 map_mask = (mach_vm_offset_t)mask;
925
926 kr = mach_vm_map(target_map, &map_addr, map_size, map_mask, flags,
927 port, offset, copy,
928 cur_protection, max_protection, inheritance);
929 *address = CAST_DOWN(vm_offset_t, map_addr);
930 return kr;
931 }
932
933 /* temporary, until world build */
934 kern_return_t
935 vm_map(
936 vm_map_t target_map,
937 vm_offset_t *address,
938 vm_size_t size,
939 vm_offset_t mask,
940 int flags,
941 ipc_port_t port,
942 vm_offset_t offset,
943 boolean_t copy,
944 vm_prot_t cur_protection,
945 vm_prot_t max_protection,
946 vm_inherit_t inheritance)
947 {
948 mach_vm_address_t map_addr;
949 mach_vm_size_t map_size;
950 mach_vm_offset_t map_mask;
951 vm_object_offset_t obj_offset;
952 kern_return_t kr;
953
954 map_addr = (mach_vm_address_t)*address;
955 map_size = (mach_vm_size_t)size;
956 map_mask = (mach_vm_offset_t)mask;
957 obj_offset = (vm_object_offset_t)offset;
958
959 kr = mach_vm_map(target_map, &map_addr, map_size, map_mask, flags,
960 port, obj_offset, copy,
961 cur_protection, max_protection, inheritance);
962 *address = CAST_DOWN(vm_offset_t, map_addr);
963 return kr;
964 }
965
966 /*
967 * mach_vm_remap -
968 * Remap a range of memory from one task into another,
969 * to another address range within the same task, or
970 * over top of itself (with altered permissions and/or
971 * as an in-place copy of itself).
972 */
973
974 kern_return_t
975 mach_vm_remap(
976 vm_map_t target_map,
977 mach_vm_offset_t *address,
978 mach_vm_size_t size,
979 mach_vm_offset_t mask,
980 boolean_t anywhere,
981 vm_map_t src_map,
982 mach_vm_offset_t memory_address,
983 boolean_t copy,
984 vm_prot_t *cur_protection,
985 vm_prot_t *max_protection,
986 vm_inherit_t inheritance)
987 {
988 vm_map_offset_t map_addr;
989 kern_return_t kr;
990
991 if (VM_MAP_NULL == target_map || VM_MAP_NULL == src_map)
992 return KERN_INVALID_ARGUMENT;
993
994 map_addr = (vm_map_offset_t)*address;
995
996 kr = vm_map_remap(target_map,
997 &map_addr,
998 size,
999 mask,
1000 anywhere,
1001 src_map,
1002 memory_address,
1003 copy,
1004 cur_protection,
1005 max_protection,
1006 inheritance);
1007 *address = map_addr;
1008 return kr;
1009 }
1010
1011 /*
1012 * vm_remap -
1013 * Remap a range of memory from one task into another,
1014 * to another address range within the same task, or
1015 * over top of itself (with altered permissions and/or
1016 * as an in-place copy of itself).
1017 *
1018 * The addressability of the source and target address
1019 * range is limited by the size of vm_address_t (in the
1020 * kernel context).
1021 */
1022 kern_return_t
1023 vm_remap(
1024 vm_map_t target_map,
1025 vm_offset_t *address,
1026 vm_size_t size,
1027 vm_offset_t mask,
1028 boolean_t anywhere,
1029 vm_map_t src_map,
1030 vm_offset_t memory_address,
1031 boolean_t copy,
1032 vm_prot_t *cur_protection,
1033 vm_prot_t *max_protection,
1034 vm_inherit_t inheritance)
1035 {
1036 vm_map_offset_t map_addr;
1037 kern_return_t kr;
1038
1039 if (VM_MAP_NULL == target_map || VM_MAP_NULL == src_map)
1040 return KERN_INVALID_ARGUMENT;
1041
1042 map_addr = (vm_map_offset_t)*address;
1043
1044 kr = vm_map_remap(target_map,
1045 &map_addr,
1046 size,
1047 mask,
1048 anywhere,
1049 src_map,
1050 memory_address,
1051 copy,
1052 cur_protection,
1053 max_protection,
1054 inheritance);
1055 *address = CAST_DOWN(vm_offset_t, map_addr);
1056 return kr;
1057 }
1058
1059 /*
1060 * NOTE: these routine (and this file) will no longer require mach_host_server.h
1061 * when mach_vm_wire and vm_wire are changed to use ledgers.
1062 */
1063 #include <mach/mach_host_server.h>
1064 /*
1065 * mach_vm_wire
1066 * Specify that the range of the virtual address space
1067 * of the target task must not cause page faults for
1068 * the indicated accesses.
1069 *
1070 * [ To unwire the pages, specify VM_PROT_NONE. ]
1071 */
1072 kern_return_t
1073 mach_vm_wire(
1074 host_priv_t host_priv,
1075 vm_map_t map,
1076 mach_vm_offset_t start,
1077 mach_vm_size_t size,
1078 vm_prot_t access)
1079 {
1080 kern_return_t rc;
1081
1082 if (host_priv == HOST_PRIV_NULL)
1083 return KERN_INVALID_HOST;
1084
1085 assert(host_priv == &realhost);
1086
1087 if (map == VM_MAP_NULL)
1088 return KERN_INVALID_TASK;
1089
1090 if (access & ~VM_PROT_ALL || (start + size < start))
1091 return KERN_INVALID_ARGUMENT;
1092
1093 if (access != VM_PROT_NONE) {
1094 rc = vm_map_wire(map, vm_map_trunc_page(start),
1095 vm_map_round_page(start+size), access, TRUE);
1096 } else {
1097 rc = vm_map_unwire(map, vm_map_trunc_page(start),
1098 vm_map_round_page(start+size), TRUE);
1099 }
1100 return rc;
1101 }
1102
1103 /*
1104 * vm_wire -
1105 * Specify that the range of the virtual address space
1106 * of the target task must not cause page faults for
1107 * the indicated accesses.
1108 *
1109 * [ To unwire the pages, specify VM_PROT_NONE. ]
1110 */
1111 kern_return_t
1112 vm_wire(
1113 host_priv_t host_priv,
1114 register vm_map_t map,
1115 vm_offset_t start,
1116 vm_size_t size,
1117 vm_prot_t access)
1118 {
1119 kern_return_t rc;
1120
1121 if (host_priv == HOST_PRIV_NULL)
1122 return KERN_INVALID_HOST;
1123
1124 assert(host_priv == &realhost);
1125
1126 if (map == VM_MAP_NULL)
1127 return KERN_INVALID_TASK;
1128
1129 if ((access & ~VM_PROT_ALL) || (start + size < start))
1130 return KERN_INVALID_ARGUMENT;
1131
1132 if (size == 0) {
1133 rc = KERN_SUCCESS;
1134 } else if (access != VM_PROT_NONE) {
1135 rc = vm_map_wire(map, vm_map_trunc_page(start),
1136 vm_map_round_page(start+size), access, TRUE);
1137 } else {
1138 rc = vm_map_unwire(map, vm_map_trunc_page(start),
1139 vm_map_round_page(start+size), TRUE);
1140 }
1141 return rc;
1142 }
1143
1144 /*
1145 * vm_msync
1146 *
1147 * Synchronises the memory range specified with its backing store
1148 * image by either flushing or cleaning the contents to the appropriate
1149 * memory manager.
1150 *
1151 * interpretation of sync_flags
1152 * VM_SYNC_INVALIDATE - discard pages, only return precious
1153 * pages to manager.
1154 *
1155 * VM_SYNC_INVALIDATE & (VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS)
1156 * - discard pages, write dirty or precious
1157 * pages back to memory manager.
1158 *
1159 * VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS
1160 * - write dirty or precious pages back to
1161 * the memory manager.
1162 *
1163 * VM_SYNC_CONTIGUOUS - does everything normally, but if there
1164 * is a hole in the region, and we would
1165 * have returned KERN_SUCCESS, return
1166 * KERN_INVALID_ADDRESS instead.
1167 *
1168 * RETURNS
1169 * KERN_INVALID_TASK Bad task parameter
1170 * KERN_INVALID_ARGUMENT both sync and async were specified.
1171 * KERN_SUCCESS The usual.
1172 * KERN_INVALID_ADDRESS There was a hole in the region.
1173 */
1174
1175 kern_return_t
1176 mach_vm_msync(
1177 vm_map_t map,
1178 mach_vm_address_t address,
1179 mach_vm_size_t size,
1180 vm_sync_t sync_flags)
1181 {
1182
1183 if (map == VM_MAP_NULL)
1184 return(KERN_INVALID_TASK);
1185
1186 return vm_map_msync(map, (vm_map_address_t)address,
1187 (vm_map_size_t)size, sync_flags);
1188 }
1189
1190 /*
1191 * vm_msync
1192 *
1193 * Synchronises the memory range specified with its backing store
1194 * image by either flushing or cleaning the contents to the appropriate
1195 * memory manager.
1196 *
1197 * interpretation of sync_flags
1198 * VM_SYNC_INVALIDATE - discard pages, only return precious
1199 * pages to manager.
1200 *
1201 * VM_SYNC_INVALIDATE & (VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS)
1202 * - discard pages, write dirty or precious
1203 * pages back to memory manager.
1204 *
1205 * VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS
1206 * - write dirty or precious pages back to
1207 * the memory manager.
1208 *
1209 * VM_SYNC_CONTIGUOUS - does everything normally, but if there
1210 * is a hole in the region, and we would
1211 * have returned KERN_SUCCESS, return
1212 * KERN_INVALID_ADDRESS instead.
1213 *
1214 * The addressability of the range is limited to that which can
1215 * be described by a vm_address_t.
1216 *
1217 * RETURNS
1218 * KERN_INVALID_TASK Bad task parameter
1219 * KERN_INVALID_ARGUMENT both sync and async were specified.
1220 * KERN_SUCCESS The usual.
1221 * KERN_INVALID_ADDRESS There was a hole in the region.
1222 */
1223
1224 kern_return_t
1225 vm_msync(
1226 vm_map_t map,
1227 vm_address_t address,
1228 vm_size_t size,
1229 vm_sync_t sync_flags)
1230 {
1231
1232 if (map == VM_MAP_NULL)
1233 return(KERN_INVALID_TASK);
1234
1235 return vm_map_msync(map, (vm_map_address_t)address,
1236 (vm_map_size_t)size, sync_flags);
1237 }
1238
1239
1240 /*
1241 * mach_vm_behavior_set
1242 *
1243 * Sets the paging behavior attribute for the specified range
1244 * in the specified map.
1245 *
1246 * This routine will fail with KERN_INVALID_ADDRESS if any address
1247 * in [start,start+size) is not a valid allocated memory region.
1248 */
1249 kern_return_t
1250 mach_vm_behavior_set(
1251 vm_map_t map,
1252 mach_vm_offset_t start,
1253 mach_vm_size_t size,
1254 vm_behavior_t new_behavior)
1255 {
1256 if ((map == VM_MAP_NULL) || (start + size < start))
1257 return(KERN_INVALID_ARGUMENT);
1258
1259 if (size == 0)
1260 return KERN_SUCCESS;
1261
1262 return(vm_map_behavior_set(map, vm_map_trunc_page(start),
1263 vm_map_round_page(start+size), new_behavior));
1264 }
1265
1266 /*
1267 * vm_behavior_set
1268 *
1269 * Sets the paging behavior attribute for the specified range
1270 * in the specified map.
1271 *
1272 * This routine will fail with KERN_INVALID_ADDRESS if any address
1273 * in [start,start+size) is not a valid allocated memory region.
1274 *
1275 * This routine is potentially limited in addressibility by the
1276 * use of vm_offset_t (if the map provided is larger than the
1277 * kernel's).
1278 */
1279 kern_return_t
1280 vm_behavior_set(
1281 vm_map_t map,
1282 vm_offset_t start,
1283 vm_size_t size,
1284 vm_behavior_t new_behavior)
1285 {
1286 if ((map == VM_MAP_NULL) || (start + size < start))
1287 return(KERN_INVALID_ARGUMENT);
1288
1289 if (size == 0)
1290 return KERN_SUCCESS;
1291
1292 return(vm_map_behavior_set(map, vm_map_trunc_page(start),
1293 vm_map_round_page(start+size), new_behavior));
1294 }
1295
1296 /*
1297 * mach_vm_region:
1298 *
1299 * User call to obtain information about a region in
1300 * a task's address map. Currently, only one flavor is
1301 * supported.
1302 *
1303 * XXX The reserved and behavior fields cannot be filled
1304 * in until the vm merge from the IK is completed, and
1305 * vm_reserve is implemented.
1306 *
1307 * XXX Dependency: syscall_vm_region() also supports only one flavor.
1308 */
1309
1310 kern_return_t
1311 mach_vm_region(
1312 vm_map_t map,
1313 mach_vm_offset_t *address, /* IN/OUT */
1314 mach_vm_size_t *size, /* OUT */
1315 vm_region_flavor_t flavor, /* IN */
1316 vm_region_info_t info, /* OUT */
1317 mach_msg_type_number_t *count, /* IN/OUT */
1318 mach_port_t *object_name) /* OUT */
1319 {
1320 vm_map_offset_t map_addr;
1321 vm_map_size_t map_size;
1322 kern_return_t kr;
1323
1324 if (VM_MAP_NULL == map)
1325 return KERN_INVALID_ARGUMENT;
1326
1327 map_addr = (vm_map_offset_t)*address;
1328 map_size = (vm_map_size_t)*size;
1329
1330 /* legacy conversion */
1331 if (VM_REGION_BASIC_INFO == flavor)
1332 flavor = VM_REGION_BASIC_INFO_64;
1333
1334 kr = vm_map_region(map,
1335 &map_addr, &map_size,
1336 flavor, info, count,
1337 object_name);
1338
1339 *address = map_addr;
1340 *size = map_size;
1341 return kr;
1342 }
1343
1344 /*
1345 * vm_region_64 and vm_region:
1346 *
1347 * User call to obtain information about a region in
1348 * a task's address map. Currently, only one flavor is
1349 * supported.
1350 *
1351 * XXX The reserved and behavior fields cannot be filled
1352 * in until the vm merge from the IK is completed, and
1353 * vm_reserve is implemented.
1354 *
1355 * XXX Dependency: syscall_vm_region() also supports only one flavor.
1356 */
1357
1358 kern_return_t
1359 vm_region_64(
1360 vm_map_t map,
1361 vm_offset_t *address, /* IN/OUT */
1362 vm_size_t *size, /* OUT */
1363 vm_region_flavor_t flavor, /* IN */
1364 vm_region_info_t info, /* OUT */
1365 mach_msg_type_number_t *count, /* IN/OUT */
1366 mach_port_t *object_name) /* OUT */
1367 {
1368 vm_map_offset_t map_addr;
1369 vm_map_size_t map_size;
1370 kern_return_t kr;
1371
1372 if (VM_MAP_NULL == map)
1373 return KERN_INVALID_ARGUMENT;
1374
1375 map_addr = (vm_map_offset_t)*address;
1376 map_size = (vm_map_size_t)*size;
1377
1378 /* legacy conversion */
1379 if (VM_REGION_BASIC_INFO == flavor)
1380 flavor = VM_REGION_BASIC_INFO_64;
1381
1382 kr = vm_map_region(map,
1383 &map_addr, &map_size,
1384 flavor, info, count,
1385 object_name);
1386
1387 *address = CAST_DOWN(vm_offset_t, map_addr);
1388 *size = CAST_DOWN(vm_size_t, map_size);
1389
1390 if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS)
1391 return KERN_INVALID_ADDRESS;
1392 return kr;
1393 }
1394
1395 kern_return_t
1396 vm_region(
1397 vm_map_t map,
1398 vm_address_t *address, /* IN/OUT */
1399 vm_size_t *size, /* OUT */
1400 vm_region_flavor_t flavor, /* IN */
1401 vm_region_info_t info, /* OUT */
1402 mach_msg_type_number_t *count, /* IN/OUT */
1403 mach_port_t *object_name) /* OUT */
1404 {
1405 vm_map_address_t map_addr;
1406 vm_map_size_t map_size;
1407 kern_return_t kr;
1408
1409 if (VM_MAP_NULL == map)
1410 return KERN_INVALID_ARGUMENT;
1411
1412 map_addr = (vm_map_address_t)*address;
1413 map_size = (vm_map_size_t)*size;
1414
1415 kr = vm_map_region(map,
1416 &map_addr, &map_size,
1417 flavor, info, count,
1418 object_name);
1419
1420 *address = CAST_DOWN(vm_address_t, map_addr);
1421 *size = CAST_DOWN(vm_size_t, map_size);
1422
1423 if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS)
1424 return KERN_INVALID_ADDRESS;
1425 return kr;
1426 }
1427
1428 /*
1429 * vm_region_recurse: A form of vm_region which follows the
1430 * submaps in a target map
1431 *
1432 */
1433 kern_return_t
1434 mach_vm_region_recurse(
1435 vm_map_t map,
1436 mach_vm_address_t *address,
1437 mach_vm_size_t *size,
1438 uint32_t *depth,
1439 vm_region_recurse_info_t info,
1440 mach_msg_type_number_t *infoCnt)
1441 {
1442 vm_map_address_t map_addr;
1443 vm_map_size_t map_size;
1444 kern_return_t kr;
1445
1446 if (VM_MAP_NULL == map)
1447 return KERN_INVALID_ARGUMENT;
1448
1449 map_addr = (vm_map_address_t)*address;
1450 map_size = (vm_map_size_t)*size;
1451
1452 kr = vm_map_region_recurse_64(
1453 map,
1454 &map_addr,
1455 &map_size,
1456 depth,
1457 (vm_region_submap_info_64_t)info,
1458 infoCnt);
1459
1460 *address = map_addr;
1461 *size = map_size;
1462 return kr;
1463 }
1464
1465 /*
1466 * vm_region_recurse: A form of vm_region which follows the
1467 * submaps in a target map
1468 *
1469 */
1470 kern_return_t
1471 vm_region_recurse_64(
1472 vm_map_t map,
1473 vm_address_t *address,
1474 vm_size_t *size,
1475 uint32_t *depth,
1476 vm_region_recurse_info_64_t info,
1477 mach_msg_type_number_t *infoCnt)
1478 {
1479 vm_map_address_t map_addr;
1480 vm_map_size_t map_size;
1481 kern_return_t kr;
1482
1483 if (VM_MAP_NULL == map)
1484 return KERN_INVALID_ARGUMENT;
1485
1486 map_addr = (vm_map_address_t)*address;
1487 map_size = (vm_map_size_t)*size;
1488
1489 kr = vm_map_region_recurse_64(
1490 map,
1491 &map_addr,
1492 &map_size,
1493 depth,
1494 (vm_region_submap_info_64_t)info,
1495 infoCnt);
1496
1497 *address = CAST_DOWN(vm_address_t, map_addr);
1498 *size = CAST_DOWN(vm_size_t, map_size);
1499
1500 if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS)
1501 return KERN_INVALID_ADDRESS;
1502 return kr;
1503 }
1504
1505 kern_return_t
1506 vm_region_recurse(
1507 vm_map_t map,
1508 vm_offset_t *address, /* IN/OUT */
1509 vm_size_t *size, /* OUT */
1510 natural_t *depth, /* IN/OUT */
1511 vm_region_recurse_info_t info32, /* IN/OUT */
1512 mach_msg_type_number_t *infoCnt) /* IN/OUT */
1513 {
1514 vm_region_submap_info_data_64_t info64;
1515 vm_region_submap_info_t info;
1516 vm_map_address_t map_addr;
1517 vm_map_size_t map_size;
1518 kern_return_t kr;
1519
1520 if (VM_MAP_NULL == map || *infoCnt < VM_REGION_SUBMAP_INFO_COUNT)
1521 return KERN_INVALID_ARGUMENT;
1522
1523
1524 map_addr = (vm_map_address_t)*address;
1525 map_size = (vm_map_size_t)*size;
1526 info = (vm_region_submap_info_t)info32;
1527 *infoCnt = VM_REGION_SUBMAP_INFO_COUNT_64;
1528
1529 kr = vm_map_region_recurse_64(map, &map_addr,&map_size,
1530 depth, &info64, infoCnt);
1531
1532 info->protection = info64.protection;
1533 info->max_protection = info64.max_protection;
1534 info->inheritance = info64.inheritance;
1535 info->offset = (uint32_t)info64.offset; /* trouble-maker */
1536 info->user_tag = info64.user_tag;
1537 info->pages_resident = info64.pages_resident;
1538 info->pages_shared_now_private = info64.pages_shared_now_private;
1539 info->pages_swapped_out = info64.pages_swapped_out;
1540 info->pages_dirtied = info64.pages_dirtied;
1541 info->ref_count = info64.ref_count;
1542 info->shadow_depth = info64.shadow_depth;
1543 info->external_pager = info64.external_pager;
1544 info->share_mode = info64.share_mode;
1545 info->is_submap = info64.is_submap;
1546 info->behavior = info64.behavior;
1547 info->object_id = info64.object_id;
1548 info->user_wired_count = info64.user_wired_count;
1549
1550 *address = CAST_DOWN(vm_address_t, map_addr);
1551 *size = CAST_DOWN(vm_size_t, map_size);
1552 *infoCnt = VM_REGION_SUBMAP_INFO_COUNT;
1553
1554 if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS)
1555 return KERN_INVALID_ADDRESS;
1556 return kr;
1557 }
1558
1559 kern_return_t
1560 mach_vm_purgable_control(
1561 vm_map_t map,
1562 mach_vm_offset_t address,
1563 vm_purgable_t control,
1564 int *state)
1565 {
1566 if (VM_MAP_NULL == map)
1567 return KERN_INVALID_ARGUMENT;
1568
1569 return vm_map_purgable_control(map,
1570 vm_map_trunc_page(address),
1571 control,
1572 state);
1573 }
1574
1575 kern_return_t
1576 vm_purgable_control(
1577 vm_map_t map,
1578 vm_offset_t address,
1579 vm_purgable_t control,
1580 int *state)
1581 {
1582 if (VM_MAP_NULL == map)
1583 return KERN_INVALID_ARGUMENT;
1584
1585 return vm_map_purgable_control(map,
1586 vm_map_trunc_page(address),
1587 control,
1588 state);
1589 }
1590
1591
1592 /*
1593 * Ordinarily, the right to allocate CPM is restricted
1594 * to privileged applications (those that can gain access
1595 * to the host priv port). Set this variable to zero if
1596 * you want to let any application allocate CPM.
1597 */
1598 unsigned int vm_allocate_cpm_privileged = 0;
1599
1600 /*
1601 * Allocate memory in the specified map, with the caveat that
1602 * the memory is physically contiguous. This call may fail
1603 * if the system can't find sufficient contiguous memory.
1604 * This call may cause or lead to heart-stopping amounts of
1605 * paging activity.
1606 *
1607 * Memory obtained from this call should be freed in the
1608 * normal way, viz., via vm_deallocate.
1609 */
1610 kern_return_t
1611 vm_allocate_cpm(
1612 host_priv_t host_priv,
1613 vm_map_t map,
1614 vm_address_t *addr,
1615 vm_size_t size,
1616 int flags)
1617 {
1618 vm_map_address_t map_addr;
1619 vm_map_size_t map_size;
1620 kern_return_t kr;
1621
1622 if (vm_allocate_cpm_privileged && HOST_PRIV_NULL == host_priv)
1623 return KERN_INVALID_HOST;
1624
1625 if (VM_MAP_NULL == map)
1626 return KERN_INVALID_ARGUMENT;
1627
1628 map_addr = (vm_map_address_t)*addr;
1629 map_size = (vm_map_size_t)size;
1630
1631 kr = vm_map_enter_cpm(map,
1632 &map_addr,
1633 map_size,
1634 flags);
1635
1636 *addr = CAST_DOWN(vm_address_t, map_addr);
1637 return kr;
1638 }
1639
1640
1641 kern_return_t
1642 mach_vm_page_query(
1643 vm_map_t map,
1644 mach_vm_offset_t offset,
1645 int *disposition,
1646 int *ref_count)
1647 {
1648 if (VM_MAP_NULL == map)
1649 return KERN_INVALID_ARGUMENT;
1650
1651 return vm_map_page_query_internal(map,
1652 vm_map_trunc_page(offset),
1653 disposition, ref_count);
1654 }
1655
1656 kern_return_t
1657 vm_map_page_query(
1658 vm_map_t map,
1659 vm_offset_t offset,
1660 int *disposition,
1661 int *ref_count)
1662 {
1663 if (VM_MAP_NULL == map)
1664 return KERN_INVALID_ARGUMENT;
1665
1666 return vm_map_page_query_internal(map,
1667 vm_map_trunc_page(offset),
1668 disposition, ref_count);
1669 }
1670
1671 kern_return_t
1672 mach_vm_page_info(
1673 vm_map_t map,
1674 mach_vm_address_t address,
1675 vm_page_info_flavor_t flavor,
1676 vm_page_info_t info,
1677 mach_msg_type_number_t *count)
1678 {
1679 kern_return_t kr;
1680
1681 if (map == VM_MAP_NULL) {
1682 return KERN_INVALID_ARGUMENT;
1683 }
1684
1685 kr = vm_map_page_info(map, address, flavor, info, count);
1686 return kr;
1687 }
1688
1689 /* map a (whole) upl into an address space */
1690 kern_return_t
1691 vm_upl_map(
1692 vm_map_t map,
1693 upl_t upl,
1694 vm_address_t *dst_addr)
1695 {
1696 vm_map_offset_t map_addr;
1697 kern_return_t kr;
1698
1699 if (VM_MAP_NULL == map)
1700 return KERN_INVALID_ARGUMENT;
1701
1702 kr = vm_map_enter_upl(map, upl, &map_addr);
1703 *dst_addr = CAST_DOWN(vm_address_t, map_addr);
1704 return kr;
1705 }
1706
1707 kern_return_t
1708 vm_upl_unmap(
1709 vm_map_t map,
1710 upl_t upl)
1711 {
1712 if (VM_MAP_NULL == map)
1713 return KERN_INVALID_ARGUMENT;
1714
1715 return (vm_map_remove_upl(map, upl));
1716 }
1717
1718 /* Retrieve a upl for an object underlying an address range in a map */
1719
1720 kern_return_t
1721 vm_map_get_upl(
1722 vm_map_t map,
1723 vm_map_offset_t map_offset,
1724 upl_size_t *upl_size,
1725 upl_t *upl,
1726 upl_page_info_array_t page_list,
1727 unsigned int *count,
1728 int *flags,
1729 int force_data_sync)
1730 {
1731 int map_flags;
1732 kern_return_t kr;
1733
1734 if (VM_MAP_NULL == map)
1735 return KERN_INVALID_ARGUMENT;
1736
1737 map_flags = *flags & ~UPL_NOZEROFILL;
1738 if (force_data_sync)
1739 map_flags |= UPL_FORCE_DATA_SYNC;
1740
1741 kr = vm_map_create_upl(map,
1742 map_offset,
1743 upl_size,
1744 upl,
1745 page_list,
1746 count,
1747 &map_flags);
1748
1749 *flags = (map_flags & ~UPL_FORCE_DATA_SYNC);
1750 return kr;
1751 }
1752
1753 /*
1754 * mach_make_memory_entry_64
1755 *
1756 * Think of it as a two-stage vm_remap() operation. First
1757 * you get a handle. Second, you get map that handle in
1758 * somewhere else. Rather than doing it all at once (and
1759 * without needing access to the other whole map).
1760 */
1761
1762 kern_return_t
1763 mach_make_memory_entry_64(
1764 vm_map_t target_map,
1765 memory_object_size_t *size,
1766 memory_object_offset_t offset,
1767 vm_prot_t permission,
1768 ipc_port_t *object_handle,
1769 ipc_port_t parent_handle)
1770 {
1771 vm_map_version_t version;
1772 vm_named_entry_t parent_entry;
1773 vm_named_entry_t user_entry;
1774 ipc_port_t user_handle;
1775 kern_return_t kr;
1776 vm_map_t real_map;
1777
1778 /* needed for call to vm_map_lookup_locked */
1779 boolean_t wired;
1780 vm_object_offset_t obj_off;
1781 vm_prot_t prot;
1782 struct vm_object_fault_info fault_info;
1783 vm_object_t object;
1784 vm_object_t shadow_object;
1785
1786 /* needed for direct map entry manipulation */
1787 vm_map_entry_t map_entry;
1788 vm_map_entry_t next_entry;
1789 vm_map_t local_map;
1790 vm_map_t original_map = target_map;
1791 vm_map_size_t total_size;
1792 vm_map_size_t map_size;
1793 vm_map_offset_t map_offset;
1794 vm_map_offset_t local_offset;
1795 vm_object_size_t mappable_size;
1796
1797 unsigned int access;
1798 vm_prot_t protections;
1799 unsigned int wimg_mode;
1800 boolean_t cache_attr = FALSE;
1801
1802 if (((permission & 0x00FF0000) &
1803 ~(MAP_MEM_ONLY |
1804 MAP_MEM_NAMED_CREATE |
1805 MAP_MEM_PURGABLE |
1806 MAP_MEM_NAMED_REUSE))) {
1807 /*
1808 * Unknown flag: reject for forward compatibility.
1809 */
1810 return KERN_INVALID_VALUE;
1811 }
1812
1813 if (parent_handle != IP_NULL &&
1814 ip_kotype(parent_handle) == IKOT_NAMED_ENTRY) {
1815 parent_entry = (vm_named_entry_t) parent_handle->ip_kobject;
1816 } else {
1817 parent_entry = NULL;
1818 }
1819
1820 protections = permission & VM_PROT_ALL;
1821 access = GET_MAP_MEM(permission);
1822
1823 user_handle = IP_NULL;
1824 user_entry = NULL;
1825
1826 map_offset = vm_map_trunc_page(offset);
1827 map_size = vm_map_round_page(*size);
1828
1829 if (permission & MAP_MEM_ONLY) {
1830 boolean_t parent_is_object;
1831
1832 if (parent_entry == NULL) {
1833 return KERN_INVALID_ARGUMENT;
1834 }
1835
1836 parent_is_object = !(parent_entry->is_sub_map || parent_entry->is_pager);
1837 object = parent_entry->backing.object;
1838 if(parent_is_object && object != VM_OBJECT_NULL)
1839 wimg_mode = object->wimg_bits;
1840 else
1841 wimg_mode = VM_WIMG_DEFAULT;
1842 if((access != GET_MAP_MEM(parent_entry->protection)) &&
1843 !(parent_entry->protection & VM_PROT_WRITE)) {
1844 return KERN_INVALID_RIGHT;
1845 }
1846 if(access == MAP_MEM_IO) {
1847 SET_MAP_MEM(access, parent_entry->protection);
1848 wimg_mode = VM_WIMG_IO;
1849 } else if (access == MAP_MEM_COPYBACK) {
1850 SET_MAP_MEM(access, parent_entry->protection);
1851 wimg_mode = VM_WIMG_DEFAULT;
1852 } else if (access == MAP_MEM_WTHRU) {
1853 SET_MAP_MEM(access, parent_entry->protection);
1854 wimg_mode = VM_WIMG_WTHRU;
1855 } else if (access == MAP_MEM_WCOMB) {
1856 SET_MAP_MEM(access, parent_entry->protection);
1857 wimg_mode = VM_WIMG_WCOMB;
1858 }
1859 if(parent_is_object && object &&
1860 (access != MAP_MEM_NOOP) &&
1861 (!(object->nophyscache))) {
1862 if(object->wimg_bits != wimg_mode) {
1863 vm_page_t p;
1864 if ((wimg_mode == VM_WIMG_IO)
1865 || (wimg_mode == VM_WIMG_WCOMB))
1866 cache_attr = TRUE;
1867 else
1868 cache_attr = FALSE;
1869 vm_object_lock(object);
1870 vm_object_paging_wait(object, THREAD_UNINT);
1871 object->wimg_bits = wimg_mode;
1872 queue_iterate(&object->memq,
1873 p, vm_page_t, listq) {
1874 if (!p->fictitious) {
1875 if (p->pmapped)
1876 pmap_disconnect(p->phys_page);
1877 if (cache_attr)
1878 pmap_sync_page_attributes_phys(p->phys_page);
1879 }
1880 }
1881 vm_object_unlock(object);
1882 }
1883 }
1884 if (object_handle)
1885 *object_handle = IP_NULL;
1886 return KERN_SUCCESS;
1887 }
1888
1889 if(permission & MAP_MEM_NAMED_CREATE) {
1890 kr = mach_memory_entry_allocate(&user_entry, &user_handle);
1891 if (kr != KERN_SUCCESS) {
1892 return KERN_FAILURE;
1893 }
1894
1895 /*
1896 * Force the creation of the VM object now.
1897 */
1898 if (map_size > (vm_map_size_t) ANON_MAX_SIZE) {
1899 /*
1900 * LP64todo - for now, we can only allocate 4GB-4096
1901 * internal objects because the default pager can't
1902 * page bigger ones. Remove this when it can.
1903 */
1904 kr = KERN_FAILURE;
1905 goto make_mem_done;
1906 }
1907
1908 object = vm_object_allocate(map_size);
1909 assert(object != VM_OBJECT_NULL);
1910
1911 if (permission & MAP_MEM_PURGABLE) {
1912 if (! (permission & VM_PROT_WRITE)) {
1913 /* if we can't write, we can't purge */
1914 vm_object_deallocate(object);
1915 kr = KERN_INVALID_ARGUMENT;
1916 goto make_mem_done;
1917 }
1918 object->purgable = VM_PURGABLE_NONVOLATILE;
1919 }
1920
1921 /*
1922 * The VM object is brand new and nobody else knows about it,
1923 * so we don't need to lock it.
1924 */
1925
1926 wimg_mode = object->wimg_bits;
1927 if (access == MAP_MEM_IO) {
1928 wimg_mode = VM_WIMG_IO;
1929 } else if (access == MAP_MEM_COPYBACK) {
1930 wimg_mode = VM_WIMG_DEFAULT;
1931 } else if (access == MAP_MEM_WTHRU) {
1932 wimg_mode = VM_WIMG_WTHRU;
1933 } else if (access == MAP_MEM_WCOMB) {
1934 wimg_mode = VM_WIMG_WCOMB;
1935 }
1936 if (access != MAP_MEM_NOOP) {
1937 object->wimg_bits = wimg_mode;
1938 }
1939 /* the object has no pages, so no WIMG bits to update here */
1940
1941 /*
1942 * XXX
1943 * We use this path when we want to make sure that
1944 * nobody messes with the object (coalesce, for
1945 * example) before we map it.
1946 * We might want to use these objects for transposition via
1947 * vm_object_transpose() too, so we don't want any copy or
1948 * shadow objects either...
1949 */
1950 object->copy_strategy = MEMORY_OBJECT_COPY_NONE;
1951
1952 user_entry->backing.object = object;
1953 user_entry->internal = TRUE;
1954 user_entry->is_sub_map = FALSE;
1955 user_entry->is_pager = FALSE;
1956 user_entry->offset = 0;
1957 user_entry->protection = protections;
1958 SET_MAP_MEM(access, user_entry->protection);
1959 user_entry->size = map_size;
1960
1961 /* user_object pager and internal fields are not used */
1962 /* when the object field is filled in. */
1963
1964 *size = CAST_DOWN(vm_size_t, map_size);
1965 *object_handle = user_handle;
1966 return KERN_SUCCESS;
1967 }
1968
1969 if (parent_entry == NULL ||
1970 (permission & MAP_MEM_NAMED_REUSE)) {
1971
1972 /* Create a named object based on address range within the task map */
1973 /* Go find the object at given address */
1974
1975 if (target_map == VM_MAP_NULL) {
1976 return KERN_INVALID_TASK;
1977 }
1978
1979 redo_lookup:
1980 vm_map_lock_read(target_map);
1981
1982 /* get the object associated with the target address */
1983 /* note we check the permission of the range against */
1984 /* that requested by the caller */
1985
1986 kr = vm_map_lookup_locked(&target_map, map_offset,
1987 protections, OBJECT_LOCK_EXCLUSIVE, &version,
1988 &object, &obj_off, &prot, &wired,
1989 &fault_info,
1990 &real_map);
1991 if (kr != KERN_SUCCESS) {
1992 vm_map_unlock_read(target_map);
1993 goto make_mem_done;
1994 }
1995 if (((prot & protections) != protections)
1996 || (object == kernel_object)) {
1997 kr = KERN_INVALID_RIGHT;
1998 vm_object_unlock(object);
1999 vm_map_unlock_read(target_map);
2000 if(real_map != target_map)
2001 vm_map_unlock_read(real_map);
2002 if(object == kernel_object) {
2003 printf("Warning: Attempt to create a named"
2004 " entry from the kernel_object\n");
2005 }
2006 goto make_mem_done;
2007 }
2008
2009 /* We have an object, now check to see if this object */
2010 /* is suitable. If not, create a shadow and share that */
2011
2012 /*
2013 * We have to unlock the VM object to avoid deadlocking with
2014 * a VM map lock (the lock ordering is map, the object), if we
2015 * need to modify the VM map to create a shadow object. Since
2016 * we might release the VM map lock below anyway, we have
2017 * to release the VM map lock now.
2018 * XXX FBDP There must be a way to avoid this double lookup...
2019 *
2020 * Take an extra reference on the VM object to make sure it's
2021 * not going to disappear.
2022 */
2023 vm_object_reference_locked(object); /* extra ref to hold obj */
2024 vm_object_unlock(object);
2025
2026 local_map = original_map;
2027 local_offset = map_offset;
2028 if(target_map != local_map) {
2029 vm_map_unlock_read(target_map);
2030 if(real_map != target_map)
2031 vm_map_unlock_read(real_map);
2032 vm_map_lock_read(local_map);
2033 target_map = local_map;
2034 real_map = local_map;
2035 }
2036 while(TRUE) {
2037 if(!vm_map_lookup_entry(local_map,
2038 local_offset, &map_entry)) {
2039 kr = KERN_INVALID_ARGUMENT;
2040 vm_map_unlock_read(target_map);
2041 if(real_map != target_map)
2042 vm_map_unlock_read(real_map);
2043 vm_object_deallocate(object); /* release extra ref */
2044 object = VM_OBJECT_NULL;
2045 goto make_mem_done;
2046 }
2047 if(!(map_entry->is_sub_map)) {
2048 if(map_entry->object.vm_object != object) {
2049 kr = KERN_INVALID_ARGUMENT;
2050 vm_map_unlock_read(target_map);
2051 if(real_map != target_map)
2052 vm_map_unlock_read(real_map);
2053 vm_object_deallocate(object); /* release extra ref */
2054 object = VM_OBJECT_NULL;
2055 goto make_mem_done;
2056 }
2057 break;
2058 } else {
2059 vm_map_t tmap;
2060 tmap = local_map;
2061 local_map = map_entry->object.sub_map;
2062
2063 vm_map_lock_read(local_map);
2064 vm_map_unlock_read(tmap);
2065 target_map = local_map;
2066 real_map = local_map;
2067 local_offset = local_offset - map_entry->vme_start;
2068 local_offset += map_entry->offset;
2069 }
2070 }
2071
2072 /*
2073 * We found the VM map entry, lock the VM object again.
2074 */
2075 vm_object_lock(object);
2076 if(map_entry->wired_count) {
2077 /* JMM - The check below should be reworked instead. */
2078 object->true_share = TRUE;
2079 }
2080 if(((map_entry->max_protection) & protections) != protections) {
2081 kr = KERN_INVALID_RIGHT;
2082 vm_object_unlock(object);
2083 vm_map_unlock_read(target_map);
2084 if(real_map != target_map)
2085 vm_map_unlock_read(real_map);
2086 vm_object_deallocate(object);
2087 object = VM_OBJECT_NULL;
2088 goto make_mem_done;
2089 }
2090
2091 mappable_size = fault_info.hi_offset - obj_off;
2092 total_size = map_entry->vme_end - map_entry->vme_start;
2093 if(map_size > mappable_size) {
2094 /* try to extend mappable size if the entries */
2095 /* following are from the same object and are */
2096 /* compatible */
2097 next_entry = map_entry->vme_next;
2098 /* lets see if the next map entry is still */
2099 /* pointing at this object and is contiguous */
2100 while(map_size > mappable_size) {
2101 if((next_entry->object.vm_object == object) &&
2102 (next_entry->vme_start ==
2103 next_entry->vme_prev->vme_end) &&
2104 (next_entry->offset ==
2105 next_entry->vme_prev->offset +
2106 (next_entry->vme_prev->vme_end -
2107 next_entry->vme_prev->vme_start))) {
2108 if(((next_entry->max_protection)
2109 & protections) != protections) {
2110 break;
2111 }
2112 if (next_entry->needs_copy !=
2113 map_entry->needs_copy)
2114 break;
2115 mappable_size += next_entry->vme_end
2116 - next_entry->vme_start;
2117 total_size += next_entry->vme_end
2118 - next_entry->vme_start;
2119 next_entry = next_entry->vme_next;
2120 } else {
2121 break;
2122 }
2123
2124 }
2125 }
2126
2127 if(object->internal) {
2128 /* vm_map_lookup_locked will create a shadow if */
2129 /* needs_copy is set but does not check for the */
2130 /* other two conditions shown. It is important to */
2131 /* set up an object which will not be pulled from */
2132 /* under us. */
2133
2134 if ((map_entry->needs_copy || object->shadowed ||
2135 (object->size > total_size))
2136 && !object->true_share) {
2137 /*
2138 * We have to unlock the VM object before
2139 * trying to upgrade the VM map lock, to
2140 * honor lock ordering (map then object).
2141 * Otherwise, we would deadlock if another
2142 * thread holds a read lock on the VM map and
2143 * is trying to acquire the VM object's lock.
2144 * We still hold an extra reference on the
2145 * VM object, guaranteeing that it won't
2146 * disappear.
2147 */
2148 vm_object_unlock(object);
2149
2150 if (vm_map_lock_read_to_write(target_map)) {
2151 /*
2152 * We couldn't upgrade our VM map lock
2153 * from "read" to "write" and we lost
2154 * our "read" lock.
2155 * Start all over again...
2156 */
2157 vm_object_deallocate(object); /* extra ref */
2158 target_map = original_map;
2159 goto redo_lookup;
2160 }
2161 vm_object_lock(object);
2162
2163 /*
2164 * JMM - We need to avoid coming here when the object
2165 * is wired by anybody, not just the current map. Why
2166 * couldn't we use the standard vm_object_copy_quickly()
2167 * approach here?
2168 */
2169
2170 /* create a shadow object */
2171 vm_object_shadow(&map_entry->object.vm_object,
2172 &map_entry->offset, total_size);
2173 shadow_object = map_entry->object.vm_object;
2174 vm_object_unlock(object);
2175
2176 prot = map_entry->protection & ~VM_PROT_WRITE;
2177
2178 if (override_nx(target_map, map_entry->alias) && prot)
2179 prot |= VM_PROT_EXECUTE;
2180
2181 vm_object_pmap_protect(
2182 object, map_entry->offset,
2183 total_size,
2184 ((map_entry->is_shared
2185 || target_map->mapped)
2186 ? PMAP_NULL :
2187 target_map->pmap),
2188 map_entry->vme_start,
2189 prot);
2190 total_size -= (map_entry->vme_end
2191 - map_entry->vme_start);
2192 next_entry = map_entry->vme_next;
2193 map_entry->needs_copy = FALSE;
2194
2195 vm_object_lock(shadow_object);
2196 while (total_size) {
2197 if(next_entry->object.vm_object == object) {
2198 vm_object_reference_locked(shadow_object);
2199 next_entry->object.vm_object
2200 = shadow_object;
2201 vm_object_deallocate(object);
2202 next_entry->offset
2203 = next_entry->vme_prev->offset +
2204 (next_entry->vme_prev->vme_end
2205 - next_entry->vme_prev->vme_start);
2206 next_entry->needs_copy = FALSE;
2207 } else {
2208 panic("mach_make_memory_entry_64:"
2209 " map entries out of sync\n");
2210 }
2211 total_size -=
2212 next_entry->vme_end
2213 - next_entry->vme_start;
2214 next_entry = next_entry->vme_next;
2215 }
2216
2217 /*
2218 * Transfer our extra reference to the
2219 * shadow object.
2220 */
2221 vm_object_reference_locked(shadow_object);
2222 vm_object_deallocate(object); /* extra ref */
2223 object = shadow_object;
2224
2225 obj_off = (local_offset - map_entry->vme_start)
2226 + map_entry->offset;
2227
2228 vm_map_lock_write_to_read(target_map);
2229 }
2230 }
2231
2232 /* note: in the future we can (if necessary) allow for */
2233 /* memory object lists, this will better support */
2234 /* fragmentation, but is it necessary? The user should */
2235 /* be encouraged to create address space oriented */
2236 /* shared objects from CLEAN memory regions which have */
2237 /* a known and defined history. i.e. no inheritence */
2238 /* share, make this call before making the region the */
2239 /* target of ipc's, etc. The code above, protecting */
2240 /* against delayed copy, etc. is mostly defensive. */
2241
2242 wimg_mode = object->wimg_bits;
2243 if(!(object->nophyscache)) {
2244 if(access == MAP_MEM_IO) {
2245 wimg_mode = VM_WIMG_IO;
2246 } else if (access == MAP_MEM_COPYBACK) {
2247 wimg_mode = VM_WIMG_USE_DEFAULT;
2248 } else if (access == MAP_MEM_WTHRU) {
2249 wimg_mode = VM_WIMG_WTHRU;
2250 } else if (access == MAP_MEM_WCOMB) {
2251 wimg_mode = VM_WIMG_WCOMB;
2252 }
2253 }
2254
2255 object->true_share = TRUE;
2256 if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC)
2257 object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
2258
2259 /*
2260 * The memory entry now points to this VM object and we
2261 * need to hold a reference on the VM object. Use the extra
2262 * reference we took earlier to keep the object alive when we
2263 * had to unlock it.
2264 */
2265
2266 vm_map_unlock_read(target_map);
2267 if(real_map != target_map)
2268 vm_map_unlock_read(real_map);
2269
2270 if(object->wimg_bits != wimg_mode) {
2271 vm_page_t p;
2272
2273 vm_object_paging_wait(object, THREAD_UNINT);
2274
2275 if ((wimg_mode == VM_WIMG_IO)
2276 || (wimg_mode == VM_WIMG_WCOMB))
2277 cache_attr = TRUE;
2278 else
2279 cache_attr = FALSE;
2280
2281 queue_iterate(&object->memq,
2282 p, vm_page_t, listq) {
2283 if (!p->fictitious) {
2284 if (p->pmapped)
2285 pmap_disconnect(p->phys_page);
2286 if (cache_attr)
2287 pmap_sync_page_attributes_phys(p->phys_page);
2288 }
2289 }
2290 object->wimg_bits = wimg_mode;
2291 }
2292
2293 /* the size of mapped entry that overlaps with our region */
2294 /* which is targeted for share. */
2295 /* (entry_end - entry_start) - */
2296 /* offset of our beg addr within entry */
2297 /* it corresponds to this: */
2298
2299 if(map_size > mappable_size)
2300 map_size = mappable_size;
2301
2302 if (permission & MAP_MEM_NAMED_REUSE) {
2303 /*
2304 * Compare what we got with the "parent_entry".
2305 * If they match, re-use the "parent_entry" instead
2306 * of creating a new one.
2307 */
2308 if (parent_entry != NULL &&
2309 parent_entry->backing.object == object &&
2310 parent_entry->internal == object->internal &&
2311 parent_entry->is_sub_map == FALSE &&
2312 parent_entry->is_pager == FALSE &&
2313 parent_entry->offset == obj_off &&
2314 parent_entry->protection == protections &&
2315 parent_entry->size == map_size) {
2316 /*
2317 * We have a match: re-use "parent_entry".
2318 */
2319 /* release our extra reference on object */
2320 vm_object_unlock(object);
2321 vm_object_deallocate(object);
2322 /* parent_entry->ref_count++; XXX ? */
2323 /* Get an extra send-right on handle */
2324 ipc_port_copy_send(parent_handle);
2325 *object_handle = parent_handle;
2326 return KERN_SUCCESS;
2327 } else {
2328 /*
2329 * No match: we need to create a new entry.
2330 * fall through...
2331 */
2332 }
2333 }
2334
2335 vm_object_unlock(object);
2336 if (mach_memory_entry_allocate(&user_entry, &user_handle)
2337 != KERN_SUCCESS) {
2338 /* release our unused reference on the object */
2339 vm_object_deallocate(object);
2340 return KERN_FAILURE;
2341 }
2342
2343 user_entry->backing.object = object;
2344 user_entry->internal = object->internal;
2345 user_entry->is_sub_map = FALSE;
2346 user_entry->is_pager = FALSE;
2347 user_entry->offset = obj_off;
2348 user_entry->protection = permission;
2349 user_entry->size = map_size;
2350
2351 /* user_object pager and internal fields are not used */
2352 /* when the object field is filled in. */
2353
2354 *size = CAST_DOWN(vm_size_t, map_size);
2355 *object_handle = user_handle;
2356 return KERN_SUCCESS;
2357
2358 } else {
2359 /* The new object will be base on an existing named object */
2360
2361 if (parent_entry == NULL) {
2362 kr = KERN_INVALID_ARGUMENT;
2363 goto make_mem_done;
2364 }
2365 if((offset + map_size) > parent_entry->size) {
2366 kr = KERN_INVALID_ARGUMENT;
2367 goto make_mem_done;
2368 }
2369
2370 if((protections & parent_entry->protection) != protections) {
2371 kr = KERN_PROTECTION_FAILURE;
2372 goto make_mem_done;
2373 }
2374
2375 if (mach_memory_entry_allocate(&user_entry, &user_handle)
2376 != KERN_SUCCESS) {
2377 kr = KERN_FAILURE;
2378 goto make_mem_done;
2379 }
2380
2381 user_entry->size = map_size;
2382 user_entry->offset = parent_entry->offset + map_offset;
2383 user_entry->is_sub_map = parent_entry->is_sub_map;
2384 user_entry->is_pager = parent_entry->is_pager;
2385 user_entry->internal = parent_entry->internal;
2386 user_entry->protection = protections;
2387
2388 if(access != MAP_MEM_NOOP) {
2389 SET_MAP_MEM(access, user_entry->protection);
2390 }
2391
2392 if(parent_entry->is_sub_map) {
2393 user_entry->backing.map = parent_entry->backing.map;
2394 vm_map_lock(user_entry->backing.map);
2395 user_entry->backing.map->ref_count++;
2396 vm_map_unlock(user_entry->backing.map);
2397 }
2398 else if (parent_entry->is_pager) {
2399 user_entry->backing.pager = parent_entry->backing.pager;
2400 /* JMM - don't we need a reference here? */
2401 } else {
2402 object = parent_entry->backing.object;
2403 assert(object != VM_OBJECT_NULL);
2404 user_entry->backing.object = object;
2405 /* we now point to this object, hold on */
2406 vm_object_reference(object);
2407 vm_object_lock(object);
2408 object->true_share = TRUE;
2409 if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC)
2410 object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
2411 vm_object_unlock(object);
2412 }
2413 *size = CAST_DOWN(vm_size_t, map_size);
2414 *object_handle = user_handle;
2415 return KERN_SUCCESS;
2416 }
2417
2418 make_mem_done:
2419 if (user_handle != IP_NULL) {
2420 /*
2421 * Releasing "user_handle" causes the kernel object
2422 * associated with it ("user_entry" here) to also be
2423 * released and freed.
2424 */
2425 mach_memory_entry_port_release(user_handle);
2426 }
2427 return kr;
2428 }
2429
2430 kern_return_t
2431 _mach_make_memory_entry(
2432 vm_map_t target_map,
2433 memory_object_size_t *size,
2434 memory_object_offset_t offset,
2435 vm_prot_t permission,
2436 ipc_port_t *object_handle,
2437 ipc_port_t parent_entry)
2438 {
2439 memory_object_size_t mo_size;
2440 kern_return_t kr;
2441
2442 mo_size = (memory_object_size_t)*size;
2443 kr = mach_make_memory_entry_64(target_map, &mo_size,
2444 (memory_object_offset_t)offset, permission, object_handle,
2445 parent_entry);
2446 *size = mo_size;
2447 return kr;
2448 }
2449
2450 kern_return_t
2451 mach_make_memory_entry(
2452 vm_map_t target_map,
2453 vm_size_t *size,
2454 vm_offset_t offset,
2455 vm_prot_t permission,
2456 ipc_port_t *object_handle,
2457 ipc_port_t parent_entry)
2458 {
2459 memory_object_size_t mo_size;
2460 kern_return_t kr;
2461
2462 mo_size = (memory_object_size_t)*size;
2463 kr = mach_make_memory_entry_64(target_map, &mo_size,
2464 (memory_object_offset_t)offset, permission, object_handle,
2465 parent_entry);
2466 *size = CAST_DOWN(vm_size_t, mo_size);
2467 return kr;
2468 }
2469
2470 /*
2471 * task_wire
2472 *
2473 * Set or clear the map's wiring_required flag. This flag, if set,
2474 * will cause all future virtual memory allocation to allocate
2475 * user wired memory. Unwiring pages wired down as a result of
2476 * this routine is done with the vm_wire interface.
2477 */
2478 kern_return_t
2479 task_wire(
2480 vm_map_t map,
2481 boolean_t must_wire)
2482 {
2483 if (map == VM_MAP_NULL)
2484 return(KERN_INVALID_ARGUMENT);
2485
2486 if (must_wire)
2487 map->wiring_required = TRUE;
2488 else
2489 map->wiring_required = FALSE;
2490
2491 return(KERN_SUCCESS);
2492 }
2493
2494 __private_extern__ kern_return_t
2495 mach_memory_entry_allocate(
2496 vm_named_entry_t *user_entry_p,
2497 ipc_port_t *user_handle_p)
2498 {
2499 vm_named_entry_t user_entry;
2500 ipc_port_t user_handle;
2501 ipc_port_t previous;
2502
2503 user_entry = (vm_named_entry_t) kalloc(sizeof *user_entry);
2504 if (user_entry == NULL)
2505 return KERN_FAILURE;
2506
2507 named_entry_lock_init(user_entry);
2508
2509 user_handle = ipc_port_alloc_kernel();
2510 if (user_handle == IP_NULL) {
2511 kfree(user_entry, sizeof *user_entry);
2512 return KERN_FAILURE;
2513 }
2514 ip_lock(user_handle);
2515
2516 /* make a sonce right */
2517 user_handle->ip_sorights++;
2518 ip_reference(user_handle);
2519
2520 user_handle->ip_destination = IP_NULL;
2521 user_handle->ip_receiver_name = MACH_PORT_NULL;
2522 user_handle->ip_receiver = ipc_space_kernel;
2523
2524 /* make a send right */
2525 user_handle->ip_mscount++;
2526 user_handle->ip_srights++;
2527 ip_reference(user_handle);
2528
2529 ipc_port_nsrequest(user_handle, 1, user_handle, &previous);
2530 /* nsrequest unlocks user_handle */
2531
2532 user_entry->backing.pager = NULL;
2533 user_entry->is_sub_map = FALSE;
2534 user_entry->is_pager = FALSE;
2535 user_entry->internal = FALSE;
2536 user_entry->size = 0;
2537 user_entry->offset = 0;
2538 user_entry->protection = VM_PROT_NONE;
2539 user_entry->ref_count = 1;
2540
2541 ipc_kobject_set(user_handle, (ipc_kobject_t) user_entry,
2542 IKOT_NAMED_ENTRY);
2543
2544 *user_entry_p = user_entry;
2545 *user_handle_p = user_handle;
2546
2547 return KERN_SUCCESS;
2548 }
2549
2550 /*
2551 * mach_memory_object_memory_entry_64
2552 *
2553 * Create a named entry backed by the provided pager.
2554 *
2555 * JMM - we need to hold a reference on the pager -
2556 * and release it when the named entry is destroyed.
2557 */
2558 kern_return_t
2559 mach_memory_object_memory_entry_64(
2560 host_t host,
2561 boolean_t internal,
2562 vm_object_offset_t size,
2563 vm_prot_t permission,
2564 memory_object_t pager,
2565 ipc_port_t *entry_handle)
2566 {
2567 unsigned int access;
2568 vm_named_entry_t user_entry;
2569 ipc_port_t user_handle;
2570
2571 if (host == HOST_NULL)
2572 return(KERN_INVALID_HOST);
2573
2574 if (mach_memory_entry_allocate(&user_entry, &user_handle)
2575 != KERN_SUCCESS) {
2576 return KERN_FAILURE;
2577 }
2578
2579 user_entry->backing.pager = pager;
2580 user_entry->size = size;
2581 user_entry->offset = 0;
2582 user_entry->protection = permission & VM_PROT_ALL;
2583 access = GET_MAP_MEM(permission);
2584 SET_MAP_MEM(access, user_entry->protection);
2585 user_entry->internal = internal;
2586 user_entry->is_sub_map = FALSE;
2587 user_entry->is_pager = TRUE;
2588 assert(user_entry->ref_count == 1);
2589
2590 *entry_handle = user_handle;
2591 return KERN_SUCCESS;
2592 }
2593
2594 kern_return_t
2595 mach_memory_object_memory_entry(
2596 host_t host,
2597 boolean_t internal,
2598 vm_size_t size,
2599 vm_prot_t permission,
2600 memory_object_t pager,
2601 ipc_port_t *entry_handle)
2602 {
2603 return mach_memory_object_memory_entry_64( host, internal,
2604 (vm_object_offset_t)size, permission, pager, entry_handle);
2605 }
2606
2607
2608 kern_return_t
2609 mach_memory_entry_purgable_control(
2610 ipc_port_t entry_port,
2611 vm_purgable_t control,
2612 int *state)
2613 {
2614 kern_return_t kr;
2615 vm_named_entry_t mem_entry;
2616 vm_object_t object;
2617
2618 if (entry_port == IP_NULL ||
2619 ip_kotype(entry_port) != IKOT_NAMED_ENTRY) {
2620 return KERN_INVALID_ARGUMENT;
2621 }
2622 if (control != VM_PURGABLE_SET_STATE &&
2623 control != VM_PURGABLE_GET_STATE)
2624 return(KERN_INVALID_ARGUMENT);
2625
2626 if (control == VM_PURGABLE_SET_STATE &&
2627 (((*state & ~(VM_PURGABLE_ALL_MASKS)) != 0) ||
2628 ((*state & VM_PURGABLE_STATE_MASK) > VM_PURGABLE_STATE_MASK)))
2629 return(KERN_INVALID_ARGUMENT);
2630
2631 mem_entry = (vm_named_entry_t) entry_port->ip_kobject;
2632
2633 named_entry_lock(mem_entry);
2634
2635 if (mem_entry->is_sub_map || mem_entry->is_pager) {
2636 named_entry_unlock(mem_entry);
2637 return KERN_INVALID_ARGUMENT;
2638 }
2639
2640 object = mem_entry->backing.object;
2641 if (object == VM_OBJECT_NULL) {
2642 named_entry_unlock(mem_entry);
2643 return KERN_INVALID_ARGUMENT;
2644 }
2645
2646 vm_object_lock(object);
2647
2648 /* check that named entry covers entire object ? */
2649 if (mem_entry->offset != 0 || object->size != mem_entry->size) {
2650 vm_object_unlock(object);
2651 named_entry_unlock(mem_entry);
2652 return KERN_INVALID_ARGUMENT;
2653 }
2654
2655 named_entry_unlock(mem_entry);
2656
2657 kr = vm_object_purgable_control(object, control, state);
2658
2659 vm_object_unlock(object);
2660
2661 return kr;
2662 }
2663
2664 /*
2665 * mach_memory_entry_port_release:
2666 *
2667 * Release a send right on a named entry port. This is the correct
2668 * way to destroy a named entry. When the last right on the port is
2669 * released, ipc_kobject_destroy() will call mach_destroy_memory_entry().
2670 */
2671 void
2672 mach_memory_entry_port_release(
2673 ipc_port_t port)
2674 {
2675 assert(ip_kotype(port) == IKOT_NAMED_ENTRY);
2676 ipc_port_release_send(port);
2677 }
2678
2679 /*
2680 * mach_destroy_memory_entry:
2681 *
2682 * Drops a reference on a memory entry and destroys the memory entry if
2683 * there are no more references on it.
2684 * NOTE: This routine should not be called to destroy a memory entry from the
2685 * kernel, as it will not release the Mach port associated with the memory
2686 * entry. The proper way to destroy a memory entry in the kernel is to
2687 * call mach_memort_entry_port_release() to release the kernel's send-right on
2688 * the memory entry's port. When the last send right is released, the memory
2689 * entry will be destroyed via ipc_kobject_destroy().
2690 */
2691 void
2692 mach_destroy_memory_entry(
2693 ipc_port_t port)
2694 {
2695 vm_named_entry_t named_entry;
2696 #if MACH_ASSERT
2697 assert(ip_kotype(port) == IKOT_NAMED_ENTRY);
2698 #endif /* MACH_ASSERT */
2699 named_entry = (vm_named_entry_t)port->ip_kobject;
2700 lck_mtx_lock(&(named_entry)->Lock);
2701 named_entry->ref_count -= 1;
2702 if(named_entry->ref_count == 0) {
2703 if (named_entry->is_sub_map) {
2704 vm_map_deallocate(named_entry->backing.map);
2705 } else if (!named_entry->is_pager) {
2706 /* release the memory object we've been pointing to */
2707 vm_object_deallocate(named_entry->backing.object);
2708 } /* else JMM - need to drop reference on pager in that case */
2709
2710 lck_mtx_unlock(&(named_entry)->Lock);
2711
2712 kfree((void *) port->ip_kobject,
2713 sizeof (struct vm_named_entry));
2714 } else
2715 lck_mtx_unlock(&(named_entry)->Lock);
2716 }
2717
2718 /* Allow manipulation of individual page state. This is actually part of */
2719 /* the UPL regimen but takes place on the memory entry rather than on a UPL */
2720
2721 kern_return_t
2722 mach_memory_entry_page_op(
2723 ipc_port_t entry_port,
2724 vm_object_offset_t offset,
2725 int ops,
2726 ppnum_t *phys_entry,
2727 int *flags)
2728 {
2729 vm_named_entry_t mem_entry;
2730 vm_object_t object;
2731 kern_return_t kr;
2732
2733 if (entry_port == IP_NULL ||
2734 ip_kotype(entry_port) != IKOT_NAMED_ENTRY) {
2735 return KERN_INVALID_ARGUMENT;
2736 }
2737
2738 mem_entry = (vm_named_entry_t) entry_port->ip_kobject;
2739
2740 named_entry_lock(mem_entry);
2741
2742 if (mem_entry->is_sub_map || mem_entry->is_pager) {
2743 named_entry_unlock(mem_entry);
2744 return KERN_INVALID_ARGUMENT;
2745 }
2746
2747 object = mem_entry->backing.object;
2748 if (object == VM_OBJECT_NULL) {
2749 named_entry_unlock(mem_entry);
2750 return KERN_INVALID_ARGUMENT;
2751 }
2752
2753 vm_object_reference(object);
2754 named_entry_unlock(mem_entry);
2755
2756 kr = vm_object_page_op(object, offset, ops, phys_entry, flags);
2757
2758 vm_object_deallocate(object);
2759
2760 return kr;
2761 }
2762
2763 /*
2764 * mach_memory_entry_range_op offers performance enhancement over
2765 * mach_memory_entry_page_op for page_op functions which do not require page
2766 * level state to be returned from the call. Page_op was created to provide
2767 * a low-cost alternative to page manipulation via UPLs when only a single
2768 * page was involved. The range_op call establishes the ability in the _op
2769 * family of functions to work on multiple pages where the lack of page level
2770 * state handling allows the caller to avoid the overhead of the upl structures.
2771 */
2772
2773 kern_return_t
2774 mach_memory_entry_range_op(
2775 ipc_port_t entry_port,
2776 vm_object_offset_t offset_beg,
2777 vm_object_offset_t offset_end,
2778 int ops,
2779 int *range)
2780 {
2781 vm_named_entry_t mem_entry;
2782 vm_object_t object;
2783 kern_return_t kr;
2784
2785 if (entry_port == IP_NULL ||
2786 ip_kotype(entry_port) != IKOT_NAMED_ENTRY) {
2787 return KERN_INVALID_ARGUMENT;
2788 }
2789
2790 mem_entry = (vm_named_entry_t) entry_port->ip_kobject;
2791
2792 named_entry_lock(mem_entry);
2793
2794 if (mem_entry->is_sub_map || mem_entry->is_pager) {
2795 named_entry_unlock(mem_entry);
2796 return KERN_INVALID_ARGUMENT;
2797 }
2798
2799 object = mem_entry->backing.object;
2800 if (object == VM_OBJECT_NULL) {
2801 named_entry_unlock(mem_entry);
2802 return KERN_INVALID_ARGUMENT;
2803 }
2804
2805 vm_object_reference(object);
2806 named_entry_unlock(mem_entry);
2807
2808 kr = vm_object_range_op(object,
2809 offset_beg,
2810 offset_end,
2811 ops,
2812 (uint32_t *) range);
2813
2814 vm_object_deallocate(object);
2815
2816 return kr;
2817 }
2818
2819
2820 kern_return_t
2821 set_dp_control_port(
2822 host_priv_t host_priv,
2823 ipc_port_t control_port)
2824 {
2825 if (host_priv == HOST_PRIV_NULL)
2826 return (KERN_INVALID_HOST);
2827
2828 if (IP_VALID(dynamic_pager_control_port))
2829 ipc_port_release_send(dynamic_pager_control_port);
2830
2831 dynamic_pager_control_port = control_port;
2832 return KERN_SUCCESS;
2833 }
2834
2835 kern_return_t
2836 get_dp_control_port(
2837 host_priv_t host_priv,
2838 ipc_port_t *control_port)
2839 {
2840 if (host_priv == HOST_PRIV_NULL)
2841 return (KERN_INVALID_HOST);
2842
2843 *control_port = ipc_port_copy_send(dynamic_pager_control_port);
2844 return KERN_SUCCESS;
2845
2846 }
2847
2848 /* ******* Temporary Internal calls to UPL for BSD ***** */
2849
2850 extern int kernel_upl_map(
2851 vm_map_t map,
2852 upl_t upl,
2853 vm_offset_t *dst_addr);
2854
2855 extern int kernel_upl_unmap(
2856 vm_map_t map,
2857 upl_t upl);
2858
2859 extern int kernel_upl_commit(
2860 upl_t upl,
2861 upl_page_info_t *pl,
2862 mach_msg_type_number_t count);
2863
2864 extern int kernel_upl_commit_range(
2865 upl_t upl,
2866 upl_offset_t offset,
2867 upl_size_t size,
2868 int flags,
2869 upl_page_info_array_t pl,
2870 mach_msg_type_number_t count);
2871
2872 extern int kernel_upl_abort(
2873 upl_t upl,
2874 int abort_type);
2875
2876 extern int kernel_upl_abort_range(
2877 upl_t upl,
2878 upl_offset_t offset,
2879 upl_size_t size,
2880 int abort_flags);
2881
2882
2883 kern_return_t
2884 kernel_upl_map(
2885 vm_map_t map,
2886 upl_t upl,
2887 vm_offset_t *dst_addr)
2888 {
2889 return vm_upl_map(map, upl, dst_addr);
2890 }
2891
2892
2893 kern_return_t
2894 kernel_upl_unmap(
2895 vm_map_t map,
2896 upl_t upl)
2897 {
2898 return vm_upl_unmap(map, upl);
2899 }
2900
2901 kern_return_t
2902 kernel_upl_commit(
2903 upl_t upl,
2904 upl_page_info_t *pl,
2905 mach_msg_type_number_t count)
2906 {
2907 kern_return_t kr;
2908
2909 kr = upl_commit(upl, pl, count);
2910 upl_deallocate(upl);
2911 return kr;
2912 }
2913
2914
2915 kern_return_t
2916 kernel_upl_commit_range(
2917 upl_t upl,
2918 upl_offset_t offset,
2919 upl_size_t size,
2920 int flags,
2921 upl_page_info_array_t pl,
2922 mach_msg_type_number_t count)
2923 {
2924 boolean_t finished = FALSE;
2925 kern_return_t kr;
2926
2927 if (flags & UPL_COMMIT_FREE_ON_EMPTY)
2928 flags |= UPL_COMMIT_NOTIFY_EMPTY;
2929
2930 if (flags & UPL_COMMIT_KERNEL_ONLY_FLAGS) {
2931 return KERN_INVALID_ARGUMENT;
2932 }
2933
2934 kr = upl_commit_range(upl, offset, size, flags, pl, count, &finished);
2935
2936 if ((flags & UPL_COMMIT_NOTIFY_EMPTY) && finished)
2937 upl_deallocate(upl);
2938
2939 return kr;
2940 }
2941
2942 kern_return_t
2943 kernel_upl_abort_range(
2944 upl_t upl,
2945 upl_offset_t offset,
2946 upl_size_t size,
2947 int abort_flags)
2948 {
2949 kern_return_t kr;
2950 boolean_t finished = FALSE;
2951
2952 if (abort_flags & UPL_COMMIT_FREE_ON_EMPTY)
2953 abort_flags |= UPL_COMMIT_NOTIFY_EMPTY;
2954
2955 kr = upl_abort_range(upl, offset, size, abort_flags, &finished);
2956
2957 if ((abort_flags & UPL_COMMIT_FREE_ON_EMPTY) && finished)
2958 upl_deallocate(upl);
2959
2960 return kr;
2961 }
2962
2963 kern_return_t
2964 kernel_upl_abort(
2965 upl_t upl,
2966 int abort_type)
2967 {
2968 kern_return_t kr;
2969
2970 kr = upl_abort(upl, abort_type);
2971 upl_deallocate(upl);
2972 return kr;
2973 }
2974
2975 /*
2976 * Now a kernel-private interface (for BootCache
2977 * use only). Need a cleaner way to create an
2978 * empty vm_map() and return a handle to it.
2979 */
2980
2981 kern_return_t
2982 vm_region_object_create(
2983 __unused vm_map_t target_map,
2984 vm_size_t size,
2985 ipc_port_t *object_handle)
2986 {
2987 vm_named_entry_t user_entry;
2988 ipc_port_t user_handle;
2989
2990 vm_map_t new_map;
2991
2992 if (mach_memory_entry_allocate(&user_entry, &user_handle)
2993 != KERN_SUCCESS) {
2994 return KERN_FAILURE;
2995 }
2996
2997 /* Create a named object based on a submap of specified size */
2998
2999 new_map = vm_map_create(PMAP_NULL, VM_MAP_MIN_ADDRESS,
3000 vm_map_round_page(size), TRUE);
3001
3002 user_entry->backing.map = new_map;
3003 user_entry->internal = TRUE;
3004 user_entry->is_sub_map = TRUE;
3005 user_entry->offset = 0;
3006 user_entry->protection = VM_PROT_ALL;
3007 user_entry->size = size;
3008 assert(user_entry->ref_count == 1);
3009
3010 *object_handle = user_handle;
3011 return KERN_SUCCESS;
3012
3013 }
3014
3015 ppnum_t vm_map_get_phys_page( /* forward */
3016 vm_map_t map,
3017 vm_offset_t offset);
3018
3019 ppnum_t
3020 vm_map_get_phys_page(
3021 vm_map_t map,
3022 vm_offset_t addr)
3023 {
3024 vm_object_offset_t offset;
3025 vm_object_t object;
3026 vm_map_offset_t map_offset;
3027 vm_map_entry_t entry;
3028 ppnum_t phys_page = 0;
3029
3030 map_offset = vm_map_trunc_page(addr);
3031
3032 vm_map_lock(map);
3033 while (vm_map_lookup_entry(map, map_offset, &entry)) {
3034
3035 if (entry->object.vm_object == VM_OBJECT_NULL) {
3036 vm_map_unlock(map);
3037 return (ppnum_t) 0;
3038 }
3039 if (entry->is_sub_map) {
3040 vm_map_t old_map;
3041 vm_map_lock(entry->object.sub_map);
3042 old_map = map;
3043 map = entry->object.sub_map;
3044 map_offset = entry->offset + (map_offset - entry->vme_start);
3045 vm_map_unlock(old_map);
3046 continue;
3047 }
3048 if (entry->object.vm_object->phys_contiguous) {
3049 /* These are not standard pageable memory mappings */
3050 /* If they are not present in the object they will */
3051 /* have to be picked up from the pager through the */
3052 /* fault mechanism. */
3053 if(entry->object.vm_object->shadow_offset == 0) {
3054 /* need to call vm_fault */
3055 vm_map_unlock(map);
3056 vm_fault(map, map_offset, VM_PROT_NONE,
3057 FALSE, THREAD_UNINT, NULL, 0);
3058 vm_map_lock(map);
3059 continue;
3060 }
3061 offset = entry->offset + (map_offset - entry->vme_start);
3062 phys_page = (ppnum_t)
3063 ((entry->object.vm_object->shadow_offset
3064 + offset) >> 12);
3065 break;
3066
3067 }
3068 offset = entry->offset + (map_offset - entry->vme_start);
3069 object = entry->object.vm_object;
3070 vm_object_lock(object);
3071 while (TRUE) {
3072 vm_page_t dst_page = vm_page_lookup(object,offset);
3073 if(dst_page == VM_PAGE_NULL) {
3074 if(object->shadow) {
3075 vm_object_t old_object;
3076 vm_object_lock(object->shadow);
3077 old_object = object;
3078 offset = offset + object->shadow_offset;
3079 object = object->shadow;
3080 vm_object_unlock(old_object);
3081 } else {
3082 vm_object_unlock(object);
3083 break;
3084 }
3085 } else {
3086 phys_page = (ppnum_t)(dst_page->phys_page);
3087 vm_object_unlock(object);
3088 break;
3089 }
3090 }
3091 break;
3092
3093 }
3094
3095 vm_map_unlock(map);
3096 return phys_page;
3097 }
3098
3099
3100
3101 kern_return_t kernel_object_iopl_request( /* forward */
3102 vm_named_entry_t named_entry,
3103 memory_object_offset_t offset,
3104 upl_size_t *upl_size,
3105 upl_t *upl_ptr,
3106 upl_page_info_array_t user_page_list,
3107 unsigned int *page_list_count,
3108 int *flags);
3109
3110 kern_return_t
3111 kernel_object_iopl_request(
3112 vm_named_entry_t named_entry,
3113 memory_object_offset_t offset,
3114 upl_size_t *upl_size,
3115 upl_t *upl_ptr,
3116 upl_page_info_array_t user_page_list,
3117 unsigned int *page_list_count,
3118 int *flags)
3119 {
3120 vm_object_t object;
3121 kern_return_t ret;
3122
3123 int caller_flags;
3124
3125 caller_flags = *flags;
3126
3127 if (caller_flags & ~UPL_VALID_FLAGS) {
3128 /*
3129 * For forward compatibility's sake,
3130 * reject any unknown flag.
3131 */
3132 return KERN_INVALID_VALUE;
3133 }
3134
3135 /* a few checks to make sure user is obeying rules */
3136 if(*upl_size == 0) {
3137 if(offset >= named_entry->size)
3138 return(KERN_INVALID_RIGHT);
3139 *upl_size = (upl_size_t) (named_entry->size - offset);
3140 if (*upl_size != named_entry->size - offset)
3141 return KERN_INVALID_ARGUMENT;
3142 }
3143 if(caller_flags & UPL_COPYOUT_FROM) {
3144 if((named_entry->protection & VM_PROT_READ)
3145 != VM_PROT_READ) {
3146 return(KERN_INVALID_RIGHT);
3147 }
3148 } else {
3149 if((named_entry->protection &
3150 (VM_PROT_READ | VM_PROT_WRITE))
3151 != (VM_PROT_READ | VM_PROT_WRITE)) {
3152 return(KERN_INVALID_RIGHT);
3153 }
3154 }
3155 if(named_entry->size < (offset + *upl_size))
3156 return(KERN_INVALID_ARGUMENT);
3157
3158 /* the callers parameter offset is defined to be the */
3159 /* offset from beginning of named entry offset in object */
3160 offset = offset + named_entry->offset;
3161
3162 if(named_entry->is_sub_map)
3163 return (KERN_INVALID_ARGUMENT);
3164
3165 named_entry_lock(named_entry);
3166
3167 if (named_entry->is_pager) {
3168 object = vm_object_enter(named_entry->backing.pager,
3169 named_entry->offset + named_entry->size,
3170 named_entry->internal,
3171 FALSE,
3172 FALSE);
3173 if (object == VM_OBJECT_NULL) {
3174 named_entry_unlock(named_entry);
3175 return(KERN_INVALID_OBJECT);
3176 }
3177
3178 /* JMM - drop reference on the pager here? */
3179
3180 /* create an extra reference for the object */
3181 vm_object_lock(object);
3182 vm_object_reference_locked(object);
3183 named_entry->backing.object = object;
3184 named_entry->is_pager = FALSE;
3185 named_entry_unlock(named_entry);
3186
3187 /* wait for object (if any) to be ready */
3188 if (!named_entry->internal) {
3189 while (!object->pager_ready) {
3190 vm_object_wait(object,
3191 VM_OBJECT_EVENT_PAGER_READY,
3192 THREAD_UNINT);
3193 vm_object_lock(object);
3194 }
3195 }
3196 vm_object_unlock(object);
3197
3198 } else {
3199 /* This is the case where we are going to operate */
3200 /* an an already known object. If the object is */
3201 /* not ready it is internal. An external */
3202 /* object cannot be mapped until it is ready */
3203 /* we can therefore avoid the ready check */
3204 /* in this case. */
3205 object = named_entry->backing.object;
3206 vm_object_reference(object);
3207 named_entry_unlock(named_entry);
3208 }
3209
3210 if (!object->private) {
3211 if (*upl_size > (MAX_UPL_TRANSFER*PAGE_SIZE))
3212 *upl_size = (MAX_UPL_TRANSFER*PAGE_SIZE);
3213 if (object->phys_contiguous) {
3214 *flags = UPL_PHYS_CONTIG;
3215 } else {
3216 *flags = 0;
3217 }
3218 } else {
3219 *flags = UPL_DEV_MEMORY | UPL_PHYS_CONTIG;
3220 }
3221
3222 ret = vm_object_iopl_request(object,
3223 offset,
3224 *upl_size,
3225 upl_ptr,
3226 user_page_list,
3227 page_list_count,
3228 caller_flags);
3229 vm_object_deallocate(object);
3230 return ret;
3231 }
mirror of XNU