]> git.saurik.com Git - apple/xnu.git/blob - osfmk/vm/vm_user.c
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-1699.22.81.tar.gz
[apple/xnu.git] / osfmk / vm / vm_user.c
1 /*
2 * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56 /*
57 */
58 /*
59 * File: vm/vm_user.c
60 * Author: Avadis Tevanian, Jr., Michael Wayne Young
61 *
62 * User-exported virtual memory functions.
63 */
64
65 /*
66 * There are three implementations of the "XXX_allocate" functionality in
67 * the kernel: mach_vm_allocate (for any task on the platform), vm_allocate
68 * (for a task with the same address space size, especially the current task),
69 * and vm32_vm_allocate (for the specific case of a 32-bit task). vm_allocate
70 * in the kernel should only be used on the kernel_task. vm32_vm_allocate only
71 * makes sense on platforms where a user task can either be 32 or 64, or the kernel
72 * task can be 32 or 64. mach_vm_allocate makes sense everywhere, and is preferred
73 * for new code.
74 *
75 * The entrypoints into the kernel are more complex. All platforms support a
76 * mach_vm_allocate-style API (subsystem 4800) which operates with the largest
77 * size types for the platform. On platforms that only support U32/K32,
78 * subsystem 4800 is all you need. On platforms that support both U32 and U64,
79 * subsystem 3800 is used disambiguate the size of parameters, and they will
80 * always be 32-bit and call into the vm32_vm_allocate APIs. On non-U32/K32 platforms,
81 * the MIG glue should never call into vm_allocate directly, because the calling
82 * task and kernel_task are unlikely to use the same size parameters
83 *
84 * New VM call implementations should be added here and to mach_vm.defs
85 * (subsystem 4800), and use mach_vm_* "wide" types.
86 */
87
88 #include <debug.h>
89
90 #include <vm_cpm.h>
91 #include <mach/boolean.h>
92 #include <mach/kern_return.h>
93 #include <mach/mach_types.h> /* to get vm_address_t */
94 #include <mach/memory_object.h>
95 #include <mach/std_types.h> /* to get pointer_t */
96 #include <mach/upl.h>
97 #include <mach/vm_attributes.h>
98 #include <mach/vm_param.h>
99 #include <mach/vm_statistics.h>
100 #include <mach/mach_syscalls.h>
101
102 #include <mach/host_priv_server.h>
103 #include <mach/mach_vm_server.h>
104 #include <mach/vm_map_server.h>
105
106 #include <kern/host.h>
107 #include <kern/kalloc.h>
108 #include <kern/task.h>
109 #include <kern/misc_protos.h>
110 #include <vm/vm_fault.h>
111 #include <vm/vm_map.h>
112 #include <vm/vm_object.h>
113 #include <vm/vm_page.h>
114 #include <vm/memory_object.h>
115 #include <vm/vm_pageout.h>
116 #include <vm/vm_protos.h>
117
118 vm_size_t upl_offset_to_pagelist = 0;
119
120 #if VM_CPM
121 #include <vm/cpm.h>
122 #endif /* VM_CPM */
123
124 ipc_port_t dynamic_pager_control_port=NULL;
125
126 /*
127 * mach_vm_allocate allocates "zero fill" memory in the specfied
128 * map.
129 */
130 kern_return_t
131 mach_vm_allocate(
132 vm_map_t map,
133 mach_vm_offset_t *addr,
134 mach_vm_size_t size,
135 int flags)
136 {
137 vm_map_offset_t map_addr;
138 vm_map_size_t map_size;
139 kern_return_t result;
140 boolean_t anywhere;
141
142 /* filter out any kernel-only flags */
143 if (flags & ~VM_FLAGS_USER_ALLOCATE)
144 return KERN_INVALID_ARGUMENT;
145
146 if (map == VM_MAP_NULL)
147 return(KERN_INVALID_ARGUMENT);
148 if (size == 0) {
149 *addr = 0;
150 return(KERN_SUCCESS);
151 }
152
153 anywhere = ((VM_FLAGS_ANYWHERE & flags) != 0);
154 if (anywhere) {
155 /*
156 * No specific address requested, so start candidate address
157 * search at the minimum address in the map. However, if that
158 * minimum is 0, bump it up by PAGE_SIZE. We want to limit
159 * allocations of PAGEZERO to explicit requests since its
160 * normal use is to catch dereferences of NULL and many
161 * applications also treat pointers with a value of 0 as
162 * special and suddenly having address 0 contain useable
163 * memory would tend to confuse those applications.
164 */
165 map_addr = vm_map_min(map);
166 if (map_addr == 0)
167 map_addr += PAGE_SIZE;
168 } else
169 map_addr = vm_map_trunc_page(*addr);
170 map_size = vm_map_round_page(size);
171 if (map_size == 0) {
172 return(KERN_INVALID_ARGUMENT);
173 }
174
175 result = vm_map_enter(
176 map,
177 &map_addr,
178 map_size,
179 (vm_map_offset_t)0,
180 flags,
181 VM_OBJECT_NULL,
182 (vm_object_offset_t)0,
183 FALSE,
184 VM_PROT_DEFAULT,
185 VM_PROT_ALL,
186 VM_INHERIT_DEFAULT);
187
188 *addr = map_addr;
189 return(result);
190 }
191
192 /*
193 * vm_allocate
194 * Legacy routine that allocates "zero fill" memory in the specfied
195 * map (which is limited to the same size as the kernel).
196 */
197 kern_return_t
198 vm_allocate(
199 vm_map_t map,
200 vm_offset_t *addr,
201 vm_size_t size,
202 int flags)
203 {
204 vm_map_offset_t map_addr;
205 vm_map_size_t map_size;
206 kern_return_t result;
207 boolean_t anywhere;
208
209 /* filter out any kernel-only flags */
210 if (flags & ~VM_FLAGS_USER_ALLOCATE)
211 return KERN_INVALID_ARGUMENT;
212
213 if (map == VM_MAP_NULL)
214 return(KERN_INVALID_ARGUMENT);
215 if (size == 0) {
216 *addr = 0;
217 return(KERN_SUCCESS);
218 }
219
220 anywhere = ((VM_FLAGS_ANYWHERE & flags) != 0);
221 if (anywhere) {
222 /*
223 * No specific address requested, so start candidate address
224 * search at the minimum address in the map. However, if that
225 * minimum is 0, bump it up by PAGE_SIZE. We want to limit
226 * allocations of PAGEZERO to explicit requests since its
227 * normal use is to catch dereferences of NULL and many
228 * applications also treat pointers with a value of 0 as
229 * special and suddenly having address 0 contain useable
230 * memory would tend to confuse those applications.
231 */
232 map_addr = vm_map_min(map);
233 if (map_addr == 0)
234 map_addr += PAGE_SIZE;
235 } else
236 map_addr = vm_map_trunc_page(*addr);
237 map_size = vm_map_round_page(size);
238 if (map_size == 0) {
239 return(KERN_INVALID_ARGUMENT);
240 }
241
242 result = vm_map_enter(
243 map,
244 &map_addr,
245 map_size,
246 (vm_map_offset_t)0,
247 flags,
248 VM_OBJECT_NULL,
249 (vm_object_offset_t)0,
250 FALSE,
251 VM_PROT_DEFAULT,
252 VM_PROT_ALL,
253 VM_INHERIT_DEFAULT);
254
255 *addr = CAST_DOWN(vm_offset_t, map_addr);
256 return(result);
257 }
258
259 /*
260 * mach_vm_deallocate -
261 * deallocates the specified range of addresses in the
262 * specified address map.
263 */
264 kern_return_t
265 mach_vm_deallocate(
266 vm_map_t map,
267 mach_vm_offset_t start,
268 mach_vm_size_t size)
269 {
270 if ((map == VM_MAP_NULL) || (start + size < start))
271 return(KERN_INVALID_ARGUMENT);
272
273 if (size == (mach_vm_offset_t) 0)
274 return(KERN_SUCCESS);
275
276 return(vm_map_remove(map, vm_map_trunc_page(start),
277 vm_map_round_page(start+size), VM_MAP_NO_FLAGS));
278 }
279
280 /*
281 * vm_deallocate -
282 * deallocates the specified range of addresses in the
283 * specified address map (limited to addresses the same
284 * size as the kernel).
285 */
286 kern_return_t
287 vm_deallocate(
288 register vm_map_t map,
289 vm_offset_t start,
290 vm_size_t size)
291 {
292 if ((map == VM_MAP_NULL) || (start + size < start))
293 return(KERN_INVALID_ARGUMENT);
294
295 if (size == (vm_offset_t) 0)
296 return(KERN_SUCCESS);
297
298 return(vm_map_remove(map, vm_map_trunc_page(start),
299 vm_map_round_page(start+size), VM_MAP_NO_FLAGS));
300 }
301
302 /*
303 * mach_vm_inherit -
304 * Sets the inheritance of the specified range in the
305 * specified map.
306 */
307 kern_return_t
308 mach_vm_inherit(
309 vm_map_t map,
310 mach_vm_offset_t start,
311 mach_vm_size_t size,
312 vm_inherit_t new_inheritance)
313 {
314 if ((map == VM_MAP_NULL) || (start + size < start) ||
315 (new_inheritance > VM_INHERIT_LAST_VALID))
316 return(KERN_INVALID_ARGUMENT);
317
318 if (size == 0)
319 return KERN_SUCCESS;
320
321 return(vm_map_inherit(map,
322 vm_map_trunc_page(start),
323 vm_map_round_page(start+size),
324 new_inheritance));
325 }
326
327 /*
328 * vm_inherit -
329 * Sets the inheritance of the specified range in the
330 * specified map (range limited to addresses
331 */
332 kern_return_t
333 vm_inherit(
334 register vm_map_t map,
335 vm_offset_t start,
336 vm_size_t size,
337 vm_inherit_t new_inheritance)
338 {
339 if ((map == VM_MAP_NULL) || (start + size < start) ||
340 (new_inheritance > VM_INHERIT_LAST_VALID))
341 return(KERN_INVALID_ARGUMENT);
342
343 if (size == 0)
344 return KERN_SUCCESS;
345
346 return(vm_map_inherit(map,
347 vm_map_trunc_page(start),
348 vm_map_round_page(start+size),
349 new_inheritance));
350 }
351
352 /*
353 * mach_vm_protect -
354 * Sets the protection of the specified range in the
355 * specified map.
356 */
357
358 kern_return_t
359 mach_vm_protect(
360 vm_map_t map,
361 mach_vm_offset_t start,
362 mach_vm_size_t size,
363 boolean_t set_maximum,
364 vm_prot_t new_protection)
365 {
366 if ((map == VM_MAP_NULL) || (start + size < start) ||
367 (new_protection & ~(VM_PROT_ALL | VM_PROT_COPY)))
368 return(KERN_INVALID_ARGUMENT);
369
370 if (size == 0)
371 return KERN_SUCCESS;
372
373 return(vm_map_protect(map,
374 vm_map_trunc_page(start),
375 vm_map_round_page(start+size),
376 new_protection,
377 set_maximum));
378 }
379
380 /*
381 * vm_protect -
382 * Sets the protection of the specified range in the
383 * specified map. Addressability of the range limited
384 * to the same size as the kernel.
385 */
386
387 kern_return_t
388 vm_protect(
389 vm_map_t map,
390 vm_offset_t start,
391 vm_size_t size,
392 boolean_t set_maximum,
393 vm_prot_t new_protection)
394 {
395 if ((map == VM_MAP_NULL) || (start + size < start) ||
396 (new_protection & ~(VM_PROT_ALL | VM_PROT_COPY)))
397 return(KERN_INVALID_ARGUMENT);
398
399 if (size == 0)
400 return KERN_SUCCESS;
401
402 return(vm_map_protect(map,
403 vm_map_trunc_page(start),
404 vm_map_round_page(start+size),
405 new_protection,
406 set_maximum));
407 }
408
409 /*
410 * mach_vm_machine_attributes -
411 * Handle machine-specific attributes for a mapping, such
412 * as cachability, migrability, etc.
413 */
414 kern_return_t
415 mach_vm_machine_attribute(
416 vm_map_t map,
417 mach_vm_address_t addr,
418 mach_vm_size_t size,
419 vm_machine_attribute_t attribute,
420 vm_machine_attribute_val_t* value) /* IN/OUT */
421 {
422 if ((map == VM_MAP_NULL) || (addr + size < addr))
423 return(KERN_INVALID_ARGUMENT);
424
425 if (size == 0)
426 return KERN_SUCCESS;
427
428 return vm_map_machine_attribute(map,
429 vm_map_trunc_page(addr),
430 vm_map_round_page(addr+size),
431 attribute,
432 value);
433 }
434
435 /*
436 * vm_machine_attribute -
437 * Handle machine-specific attributes for a mapping, such
438 * as cachability, migrability, etc. Limited addressability
439 * (same range limits as for the native kernel map).
440 */
441 kern_return_t
442 vm_machine_attribute(
443 vm_map_t map,
444 vm_address_t addr,
445 vm_size_t size,
446 vm_machine_attribute_t attribute,
447 vm_machine_attribute_val_t* value) /* IN/OUT */
448 {
449 if ((map == VM_MAP_NULL) || (addr + size < addr))
450 return(KERN_INVALID_ARGUMENT);
451
452 if (size == 0)
453 return KERN_SUCCESS;
454
455 return vm_map_machine_attribute(map,
456 vm_map_trunc_page(addr),
457 vm_map_round_page(addr+size),
458 attribute,
459 value);
460 }
461
462 /*
463 * mach_vm_read -
464 * Read/copy a range from one address space and return it to the caller.
465 *
466 * It is assumed that the address for the returned memory is selected by
467 * the IPC implementation as part of receiving the reply to this call.
468 * If IPC isn't used, the caller must deal with the vm_map_copy_t object
469 * that gets returned.
470 *
471 * JMM - because of mach_msg_type_number_t, this call is limited to a
472 * single 4GB region at this time.
473 *
474 */
475 kern_return_t
476 mach_vm_read(
477 vm_map_t map,
478 mach_vm_address_t addr,
479 mach_vm_size_t size,
480 pointer_t *data,
481 mach_msg_type_number_t *data_size)
482 {
483 kern_return_t error;
484 vm_map_copy_t ipc_address;
485
486 if (map == VM_MAP_NULL)
487 return(KERN_INVALID_ARGUMENT);
488
489 if ((mach_msg_type_number_t) size != size)
490 return KERN_INVALID_ARGUMENT;
491
492 error = vm_map_copyin(map,
493 (vm_map_address_t)addr,
494 (vm_map_size_t)size,
495 FALSE, /* src_destroy */
496 &ipc_address);
497
498 if (KERN_SUCCESS == error) {
499 *data = (pointer_t) ipc_address;
500 *data_size = (mach_msg_type_number_t) size;
501 assert(*data_size == size);
502 }
503 return(error);
504 }
505
506 /*
507 * vm_read -
508 * Read/copy a range from one address space and return it to the caller.
509 * Limited addressability (same range limits as for the native kernel map).
510 *
511 * It is assumed that the address for the returned memory is selected by
512 * the IPC implementation as part of receiving the reply to this call.
513 * If IPC isn't used, the caller must deal with the vm_map_copy_t object
514 * that gets returned.
515 */
516 kern_return_t
517 vm_read(
518 vm_map_t map,
519 vm_address_t addr,
520 vm_size_t size,
521 pointer_t *data,
522 mach_msg_type_number_t *data_size)
523 {
524 kern_return_t error;
525 vm_map_copy_t ipc_address;
526
527 if (map == VM_MAP_NULL)
528 return(KERN_INVALID_ARGUMENT);
529
530 if (size > (unsigned)(mach_msg_type_number_t) -1) {
531 /*
532 * The kernel could handle a 64-bit "size" value, but
533 * it could not return the size of the data in "*data_size"
534 * without overflowing.
535 * Let's reject this "size" as invalid.
536 */
537 return KERN_INVALID_ARGUMENT;
538 }
539
540 error = vm_map_copyin(map,
541 (vm_map_address_t)addr,
542 (vm_map_size_t)size,
543 FALSE, /* src_destroy */
544 &ipc_address);
545
546 if (KERN_SUCCESS == error) {
547 *data = (pointer_t) ipc_address;
548 *data_size = (mach_msg_type_number_t) size;
549 assert(*data_size == size);
550 }
551 return(error);
552 }
553
554 /*
555 * mach_vm_read_list -
556 * Read/copy a list of address ranges from specified map.
557 *
558 * MIG does not know how to deal with a returned array of
559 * vm_map_copy_t structures, so we have to do the copyout
560 * manually here.
561 */
562 kern_return_t
563 mach_vm_read_list(
564 vm_map_t map,
565 mach_vm_read_entry_t data_list,
566 natural_t count)
567 {
568 mach_msg_type_number_t i;
569 kern_return_t error;
570 vm_map_copy_t copy;
571
572 if (map == VM_MAP_NULL ||
573 count > VM_MAP_ENTRY_MAX)
574 return(KERN_INVALID_ARGUMENT);
575
576 error = KERN_SUCCESS;
577 for(i=0; i<count; i++) {
578 vm_map_address_t map_addr;
579 vm_map_size_t map_size;
580
581 map_addr = (vm_map_address_t)(data_list[i].address);
582 map_size = (vm_map_size_t)(data_list[i].size);
583
584 if(map_size != 0) {
585 error = vm_map_copyin(map,
586 map_addr,
587 map_size,
588 FALSE, /* src_destroy */
589 &copy);
590 if (KERN_SUCCESS == error) {
591 error = vm_map_copyout(
592 current_task()->map,
593 &map_addr,
594 copy);
595 if (KERN_SUCCESS == error) {
596 data_list[i].address = map_addr;
597 continue;
598 }
599 vm_map_copy_discard(copy);
600 }
601 }
602 data_list[i].address = (mach_vm_address_t)0;
603 data_list[i].size = (mach_vm_size_t)0;
604 }
605 return(error);
606 }
607
608 /*
609 * vm_read_list -
610 * Read/copy a list of address ranges from specified map.
611 *
612 * MIG does not know how to deal with a returned array of
613 * vm_map_copy_t structures, so we have to do the copyout
614 * manually here.
615 *
616 * The source and destination ranges are limited to those
617 * that can be described with a vm_address_t (i.e. same
618 * size map as the kernel).
619 *
620 * JMM - If the result of the copyout is an address range
621 * that cannot be described with a vm_address_t (i.e. the
622 * caller had a larger address space but used this call
623 * anyway), it will result in a truncated address being
624 * returned (and a likely confused caller).
625 */
626
627 kern_return_t
628 vm_read_list(
629 vm_map_t map,
630 vm_read_entry_t data_list,
631 natural_t count)
632 {
633 mach_msg_type_number_t i;
634 kern_return_t error;
635 vm_map_copy_t copy;
636
637 if (map == VM_MAP_NULL ||
638 count > VM_MAP_ENTRY_MAX)
639 return(KERN_INVALID_ARGUMENT);
640
641 error = KERN_SUCCESS;
642 for(i=0; i<count; i++) {
643 vm_map_address_t map_addr;
644 vm_map_size_t map_size;
645
646 map_addr = (vm_map_address_t)(data_list[i].address);
647 map_size = (vm_map_size_t)(data_list[i].size);
648
649 if(map_size != 0) {
650 error = vm_map_copyin(map,
651 map_addr,
652 map_size,
653 FALSE, /* src_destroy */
654 &copy);
655 if (KERN_SUCCESS == error) {
656 error = vm_map_copyout(current_task()->map,
657 &map_addr,
658 copy);
659 if (KERN_SUCCESS == error) {
660 data_list[i].address =
661 CAST_DOWN(vm_offset_t, map_addr);
662 continue;
663 }
664 vm_map_copy_discard(copy);
665 }
666 }
667 data_list[i].address = (mach_vm_address_t)0;
668 data_list[i].size = (mach_vm_size_t)0;
669 }
670 return(error);
671 }
672
673 /*
674 * mach_vm_read_overwrite -
675 * Overwrite a range of the current map with data from the specified
676 * map/address range.
677 *
678 * In making an assumption that the current thread is local, it is
679 * no longer cluster-safe without a fully supportive local proxy
680 * thread/task (but we don't support cluster's anymore so this is moot).
681 */
682
683 kern_return_t
684 mach_vm_read_overwrite(
685 vm_map_t map,
686 mach_vm_address_t address,
687 mach_vm_size_t size,
688 mach_vm_address_t data,
689 mach_vm_size_t *data_size)
690 {
691 kern_return_t error;
692 vm_map_copy_t copy;
693
694 if (map == VM_MAP_NULL)
695 return(KERN_INVALID_ARGUMENT);
696
697 error = vm_map_copyin(map, (vm_map_address_t)address,
698 (vm_map_size_t)size, FALSE, &copy);
699
700 if (KERN_SUCCESS == error) {
701 error = vm_map_copy_overwrite(current_thread()->map,
702 (vm_map_address_t)data,
703 copy, FALSE);
704 if (KERN_SUCCESS == error) {
705 *data_size = size;
706 return error;
707 }
708 vm_map_copy_discard(copy);
709 }
710 return(error);
711 }
712
713 /*
714 * vm_read_overwrite -
715 * Overwrite a range of the current map with data from the specified
716 * map/address range.
717 *
718 * This routine adds the additional limitation that the source and
719 * destination ranges must be describable with vm_address_t values
720 * (i.e. the same size address spaces as the kernel, or at least the
721 * the ranges are in that first portion of the respective address
722 * spaces).
723 */
724
725 kern_return_t
726 vm_read_overwrite(
727 vm_map_t map,
728 vm_address_t address,
729 vm_size_t size,
730 vm_address_t data,
731 vm_size_t *data_size)
732 {
733 kern_return_t error;
734 vm_map_copy_t copy;
735
736 if (map == VM_MAP_NULL)
737 return(KERN_INVALID_ARGUMENT);
738
739 error = vm_map_copyin(map, (vm_map_address_t)address,
740 (vm_map_size_t)size, FALSE, &copy);
741
742 if (KERN_SUCCESS == error) {
743 error = vm_map_copy_overwrite(current_thread()->map,
744 (vm_map_address_t)data,
745 copy, FALSE);
746 if (KERN_SUCCESS == error) {
747 *data_size = size;
748 return error;
749 }
750 vm_map_copy_discard(copy);
751 }
752 return(error);
753 }
754
755
756 /*
757 * mach_vm_write -
758 * Overwrite the specified address range with the data provided
759 * (from the current map).
760 */
761 kern_return_t
762 mach_vm_write(
763 vm_map_t map,
764 mach_vm_address_t address,
765 pointer_t data,
766 __unused mach_msg_type_number_t size)
767 {
768 if (map == VM_MAP_NULL)
769 return KERN_INVALID_ARGUMENT;
770
771 return vm_map_copy_overwrite(map, (vm_map_address_t)address,
772 (vm_map_copy_t) data, FALSE /* interruptible XXX */);
773 }
774
775 /*
776 * vm_write -
777 * Overwrite the specified address range with the data provided
778 * (from the current map).
779 *
780 * The addressability of the range of addresses to overwrite is
781 * limited bu the use of a vm_address_t (same size as kernel map).
782 * Either the target map is also small, or the range is in the
783 * low addresses within it.
784 */
785 kern_return_t
786 vm_write(
787 vm_map_t map,
788 vm_address_t address,
789 pointer_t data,
790 __unused mach_msg_type_number_t size)
791 {
792 if (map == VM_MAP_NULL)
793 return KERN_INVALID_ARGUMENT;
794
795 return vm_map_copy_overwrite(map, (vm_map_address_t)address,
796 (vm_map_copy_t) data, FALSE /* interruptible XXX */);
797 }
798
799 /*
800 * mach_vm_copy -
801 * Overwrite one range of the specified map with the contents of
802 * another range within that same map (i.e. both address ranges
803 * are "over there").
804 */
805 kern_return_t
806 mach_vm_copy(
807 vm_map_t map,
808 mach_vm_address_t source_address,
809 mach_vm_size_t size,
810 mach_vm_address_t dest_address)
811 {
812 vm_map_copy_t copy;
813 kern_return_t kr;
814
815 if (map == VM_MAP_NULL)
816 return KERN_INVALID_ARGUMENT;
817
818 kr = vm_map_copyin(map, (vm_map_address_t)source_address,
819 (vm_map_size_t)size, FALSE, &copy);
820
821 if (KERN_SUCCESS == kr) {
822 kr = vm_map_copy_overwrite(map,
823 (vm_map_address_t)dest_address,
824 copy, FALSE /* interruptible XXX */);
825
826 if (KERN_SUCCESS != kr)
827 vm_map_copy_discard(copy);
828 }
829 return kr;
830 }
831
832 kern_return_t
833 vm_copy(
834 vm_map_t map,
835 vm_address_t source_address,
836 vm_size_t size,
837 vm_address_t dest_address)
838 {
839 vm_map_copy_t copy;
840 kern_return_t kr;
841
842 if (map == VM_MAP_NULL)
843 return KERN_INVALID_ARGUMENT;
844
845 kr = vm_map_copyin(map, (vm_map_address_t)source_address,
846 (vm_map_size_t)size, FALSE, &copy);
847
848 if (KERN_SUCCESS == kr) {
849 kr = vm_map_copy_overwrite(map,
850 (vm_map_address_t)dest_address,
851 copy, FALSE /* interruptible XXX */);
852
853 if (KERN_SUCCESS != kr)
854 vm_map_copy_discard(copy);
855 }
856 return kr;
857 }
858
859 /*
860 * mach_vm_map -
861 * Map some range of an object into an address space.
862 *
863 * The object can be one of several types of objects:
864 * NULL - anonymous memory
865 * a named entry - a range within another address space
866 * or a range within a memory object
867 * a whole memory object
868 *
869 */
870 kern_return_t
871 mach_vm_map(
872 vm_map_t target_map,
873 mach_vm_offset_t *address,
874 mach_vm_size_t initial_size,
875 mach_vm_offset_t mask,
876 int flags,
877 ipc_port_t port,
878 vm_object_offset_t offset,
879 boolean_t copy,
880 vm_prot_t cur_protection,
881 vm_prot_t max_protection,
882 vm_inherit_t inheritance)
883 {
884 /* filter out any kernel-only flags */
885 if (flags & ~VM_FLAGS_USER_MAP)
886 return KERN_INVALID_ARGUMENT;
887
888 return vm_map_enter_mem_object(target_map,
889 address,
890 initial_size,
891 mask,
892 flags,
893 port,
894 offset,
895 copy,
896 cur_protection,
897 max_protection,
898 inheritance);
899 }
900
901
902 /* legacy interface */
903 kern_return_t
904 vm_map_64(
905 vm_map_t target_map,
906 vm_offset_t *address,
907 vm_size_t size,
908 vm_offset_t mask,
909 int flags,
910 ipc_port_t port,
911 vm_object_offset_t offset,
912 boolean_t copy,
913 vm_prot_t cur_protection,
914 vm_prot_t max_protection,
915 vm_inherit_t inheritance)
916 {
917 mach_vm_address_t map_addr;
918 mach_vm_size_t map_size;
919 mach_vm_offset_t map_mask;
920 kern_return_t kr;
921
922 map_addr = (mach_vm_address_t)*address;
923 map_size = (mach_vm_size_t)size;
924 map_mask = (mach_vm_offset_t)mask;
925
926 kr = mach_vm_map(target_map, &map_addr, map_size, map_mask, flags,
927 port, offset, copy,
928 cur_protection, max_protection, inheritance);
929 *address = CAST_DOWN(vm_offset_t, map_addr);
930 return kr;
931 }
932
933 /* temporary, until world build */
934 kern_return_t
935 vm_map(
936 vm_map_t target_map,
937 vm_offset_t *address,
938 vm_size_t size,
939 vm_offset_t mask,
940 int flags,
941 ipc_port_t port,
942 vm_offset_t offset,
943 boolean_t copy,
944 vm_prot_t cur_protection,
945 vm_prot_t max_protection,
946 vm_inherit_t inheritance)
947 {
948 mach_vm_address_t map_addr;
949 mach_vm_size_t map_size;
950 mach_vm_offset_t map_mask;
951 vm_object_offset_t obj_offset;
952 kern_return_t kr;
953
954 map_addr = (mach_vm_address_t)*address;
955 map_size = (mach_vm_size_t)size;
956 map_mask = (mach_vm_offset_t)mask;
957 obj_offset = (vm_object_offset_t)offset;
958
959 kr = mach_vm_map(target_map, &map_addr, map_size, map_mask, flags,
960 port, obj_offset, copy,
961 cur_protection, max_protection, inheritance);
962 *address = CAST_DOWN(vm_offset_t, map_addr);
963 return kr;
964 }
965
966 /*
967 * mach_vm_remap -
968 * Remap a range of memory from one task into another,
969 * to another address range within the same task, or
970 * over top of itself (with altered permissions and/or
971 * as an in-place copy of itself).
972 */
973
974 kern_return_t
975 mach_vm_remap(
976 vm_map_t target_map,
977 mach_vm_offset_t *address,
978 mach_vm_size_t size,
979 mach_vm_offset_t mask,
980 int flags,
981 vm_map_t src_map,
982 mach_vm_offset_t memory_address,
983 boolean_t copy,
984 vm_prot_t *cur_protection,
985 vm_prot_t *max_protection,
986 vm_inherit_t inheritance)
987 {
988 vm_map_offset_t map_addr;
989 kern_return_t kr;
990
991 if (VM_MAP_NULL == target_map || VM_MAP_NULL == src_map)
992 return KERN_INVALID_ARGUMENT;
993
994 /* filter out any kernel-only flags */
995 if (flags & ~VM_FLAGS_USER_REMAP)
996 return KERN_INVALID_ARGUMENT;
997
998 map_addr = (vm_map_offset_t)*address;
999
1000 kr = vm_map_remap(target_map,
1001 &map_addr,
1002 size,
1003 mask,
1004 flags,
1005 src_map,
1006 memory_address,
1007 copy,
1008 cur_protection,
1009 max_protection,
1010 inheritance);
1011 *address = map_addr;
1012 return kr;
1013 }
1014
1015 /*
1016 * vm_remap -
1017 * Remap a range of memory from one task into another,
1018 * to another address range within the same task, or
1019 * over top of itself (with altered permissions and/or
1020 * as an in-place copy of itself).
1021 *
1022 * The addressability of the source and target address
1023 * range is limited by the size of vm_address_t (in the
1024 * kernel context).
1025 */
1026 kern_return_t
1027 vm_remap(
1028 vm_map_t target_map,
1029 vm_offset_t *address,
1030 vm_size_t size,
1031 vm_offset_t mask,
1032 int flags,
1033 vm_map_t src_map,
1034 vm_offset_t memory_address,
1035 boolean_t copy,
1036 vm_prot_t *cur_protection,
1037 vm_prot_t *max_protection,
1038 vm_inherit_t inheritance)
1039 {
1040 vm_map_offset_t map_addr;
1041 kern_return_t kr;
1042
1043 if (VM_MAP_NULL == target_map || VM_MAP_NULL == src_map)
1044 return KERN_INVALID_ARGUMENT;
1045
1046 /* filter out any kernel-only flags */
1047 if (flags & ~VM_FLAGS_USER_REMAP)
1048 return KERN_INVALID_ARGUMENT;
1049
1050 map_addr = (vm_map_offset_t)*address;
1051
1052 kr = vm_map_remap(target_map,
1053 &map_addr,
1054 size,
1055 mask,
1056 flags,
1057 src_map,
1058 memory_address,
1059 copy,
1060 cur_protection,
1061 max_protection,
1062 inheritance);
1063 *address = CAST_DOWN(vm_offset_t, map_addr);
1064 return kr;
1065 }
1066
1067 /*
1068 * NOTE: these routine (and this file) will no longer require mach_host_server.h
1069 * when mach_vm_wire and vm_wire are changed to use ledgers.
1070 */
1071 #include <mach/mach_host_server.h>
1072 /*
1073 * mach_vm_wire
1074 * Specify that the range of the virtual address space
1075 * of the target task must not cause page faults for
1076 * the indicated accesses.
1077 *
1078 * [ To unwire the pages, specify VM_PROT_NONE. ]
1079 */
1080 kern_return_t
1081 mach_vm_wire(
1082 host_priv_t host_priv,
1083 vm_map_t map,
1084 mach_vm_offset_t start,
1085 mach_vm_size_t size,
1086 vm_prot_t access)
1087 {
1088 kern_return_t rc;
1089
1090 if (host_priv == HOST_PRIV_NULL)
1091 return KERN_INVALID_HOST;
1092
1093 assert(host_priv == &realhost);
1094
1095 if (map == VM_MAP_NULL)
1096 return KERN_INVALID_TASK;
1097
1098 if (access & ~VM_PROT_ALL || (start + size < start))
1099 return KERN_INVALID_ARGUMENT;
1100
1101 if (access != VM_PROT_NONE) {
1102 rc = vm_map_wire(map, vm_map_trunc_page(start),
1103 vm_map_round_page(start+size), access, TRUE);
1104 } else {
1105 rc = vm_map_unwire(map, vm_map_trunc_page(start),
1106 vm_map_round_page(start+size), TRUE);
1107 }
1108 return rc;
1109 }
1110
1111 /*
1112 * vm_wire -
1113 * Specify that the range of the virtual address space
1114 * of the target task must not cause page faults for
1115 * the indicated accesses.
1116 *
1117 * [ To unwire the pages, specify VM_PROT_NONE. ]
1118 */
1119 kern_return_t
1120 vm_wire(
1121 host_priv_t host_priv,
1122 register vm_map_t map,
1123 vm_offset_t start,
1124 vm_size_t size,
1125 vm_prot_t access)
1126 {
1127 kern_return_t rc;
1128
1129 if (host_priv == HOST_PRIV_NULL)
1130 return KERN_INVALID_HOST;
1131
1132 assert(host_priv == &realhost);
1133
1134 if (map == VM_MAP_NULL)
1135 return KERN_INVALID_TASK;
1136
1137 if ((access & ~VM_PROT_ALL) || (start + size < start))
1138 return KERN_INVALID_ARGUMENT;
1139
1140 if (size == 0) {
1141 rc = KERN_SUCCESS;
1142 } else if (access != VM_PROT_NONE) {
1143 rc = vm_map_wire(map, vm_map_trunc_page(start),
1144 vm_map_round_page(start+size), access, TRUE);
1145 } else {
1146 rc = vm_map_unwire(map, vm_map_trunc_page(start),
1147 vm_map_round_page(start+size), TRUE);
1148 }
1149 return rc;
1150 }
1151
1152 /*
1153 * vm_msync
1154 *
1155 * Synchronises the memory range specified with its backing store
1156 * image by either flushing or cleaning the contents to the appropriate
1157 * memory manager.
1158 *
1159 * interpretation of sync_flags
1160 * VM_SYNC_INVALIDATE - discard pages, only return precious
1161 * pages to manager.
1162 *
1163 * VM_SYNC_INVALIDATE & (VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS)
1164 * - discard pages, write dirty or precious
1165 * pages back to memory manager.
1166 *
1167 * VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS
1168 * - write dirty or precious pages back to
1169 * the memory manager.
1170 *
1171 * VM_SYNC_CONTIGUOUS - does everything normally, but if there
1172 * is a hole in the region, and we would
1173 * have returned KERN_SUCCESS, return
1174 * KERN_INVALID_ADDRESS instead.
1175 *
1176 * RETURNS
1177 * KERN_INVALID_TASK Bad task parameter
1178 * KERN_INVALID_ARGUMENT both sync and async were specified.
1179 * KERN_SUCCESS The usual.
1180 * KERN_INVALID_ADDRESS There was a hole in the region.
1181 */
1182
1183 kern_return_t
1184 mach_vm_msync(
1185 vm_map_t map,
1186 mach_vm_address_t address,
1187 mach_vm_size_t size,
1188 vm_sync_t sync_flags)
1189 {
1190
1191 if (map == VM_MAP_NULL)
1192 return(KERN_INVALID_TASK);
1193
1194 return vm_map_msync(map, (vm_map_address_t)address,
1195 (vm_map_size_t)size, sync_flags);
1196 }
1197
1198 /*
1199 * vm_msync
1200 *
1201 * Synchronises the memory range specified with its backing store
1202 * image by either flushing or cleaning the contents to the appropriate
1203 * memory manager.
1204 *
1205 * interpretation of sync_flags
1206 * VM_SYNC_INVALIDATE - discard pages, only return precious
1207 * pages to manager.
1208 *
1209 * VM_SYNC_INVALIDATE & (VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS)
1210 * - discard pages, write dirty or precious
1211 * pages back to memory manager.
1212 *
1213 * VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS
1214 * - write dirty or precious pages back to
1215 * the memory manager.
1216 *
1217 * VM_SYNC_CONTIGUOUS - does everything normally, but if there
1218 * is a hole in the region, and we would
1219 * have returned KERN_SUCCESS, return
1220 * KERN_INVALID_ADDRESS instead.
1221 *
1222 * The addressability of the range is limited to that which can
1223 * be described by a vm_address_t.
1224 *
1225 * RETURNS
1226 * KERN_INVALID_TASK Bad task parameter
1227 * KERN_INVALID_ARGUMENT both sync and async were specified.
1228 * KERN_SUCCESS The usual.
1229 * KERN_INVALID_ADDRESS There was a hole in the region.
1230 */
1231
1232 kern_return_t
1233 vm_msync(
1234 vm_map_t map,
1235 vm_address_t address,
1236 vm_size_t size,
1237 vm_sync_t sync_flags)
1238 {
1239
1240 if (map == VM_MAP_NULL)
1241 return(KERN_INVALID_TASK);
1242
1243 return vm_map_msync(map, (vm_map_address_t)address,
1244 (vm_map_size_t)size, sync_flags);
1245 }
1246
1247
1248 int
1249 vm_toggle_entry_reuse(int toggle, int *old_value)
1250 {
1251 vm_map_t map = current_map();
1252
1253 if(toggle == VM_TOGGLE_GETVALUE && old_value != NULL){
1254 *old_value = map->disable_vmentry_reuse;
1255 } else if(toggle == VM_TOGGLE_SET){
1256 vm_map_lock(map);
1257 map->disable_vmentry_reuse = TRUE;
1258 if (map->first_free == vm_map_to_entry(map)) {
1259 map->highest_entry_end = vm_map_min(map);
1260 } else {
1261 map->highest_entry_end = map->first_free->vme_end;
1262 }
1263 vm_map_unlock(map);
1264 } else if (toggle == VM_TOGGLE_CLEAR){
1265 vm_map_lock(map);
1266 map->disable_vmentry_reuse = FALSE;
1267 vm_map_unlock(map);
1268 } else
1269 return KERN_INVALID_ARGUMENT;
1270
1271 return KERN_SUCCESS;
1272 }
1273
1274 /*
1275 * mach_vm_behavior_set
1276 *
1277 * Sets the paging behavior attribute for the specified range
1278 * in the specified map.
1279 *
1280 * This routine will fail with KERN_INVALID_ADDRESS if any address
1281 * in [start,start+size) is not a valid allocated memory region.
1282 */
1283 kern_return_t
1284 mach_vm_behavior_set(
1285 vm_map_t map,
1286 mach_vm_offset_t start,
1287 mach_vm_size_t size,
1288 vm_behavior_t new_behavior)
1289 {
1290 if ((map == VM_MAP_NULL) || (start + size < start))
1291 return(KERN_INVALID_ARGUMENT);
1292
1293 if (size == 0)
1294 return KERN_SUCCESS;
1295
1296 return(vm_map_behavior_set(map, vm_map_trunc_page(start),
1297 vm_map_round_page(start+size), new_behavior));
1298 }
1299
1300 /*
1301 * vm_behavior_set
1302 *
1303 * Sets the paging behavior attribute for the specified range
1304 * in the specified map.
1305 *
1306 * This routine will fail with KERN_INVALID_ADDRESS if any address
1307 * in [start,start+size) is not a valid allocated memory region.
1308 *
1309 * This routine is potentially limited in addressibility by the
1310 * use of vm_offset_t (if the map provided is larger than the
1311 * kernel's).
1312 */
1313 kern_return_t
1314 vm_behavior_set(
1315 vm_map_t map,
1316 vm_offset_t start,
1317 vm_size_t size,
1318 vm_behavior_t new_behavior)
1319 {
1320 if ((map == VM_MAP_NULL) || (start + size < start))
1321 return(KERN_INVALID_ARGUMENT);
1322
1323 if (size == 0)
1324 return KERN_SUCCESS;
1325
1326 return(vm_map_behavior_set(map, vm_map_trunc_page(start),
1327 vm_map_round_page(start+size), new_behavior));
1328 }
1329
1330 /*
1331 * mach_vm_region:
1332 *
1333 * User call to obtain information about a region in
1334 * a task's address map. Currently, only one flavor is
1335 * supported.
1336 *
1337 * XXX The reserved and behavior fields cannot be filled
1338 * in until the vm merge from the IK is completed, and
1339 * vm_reserve is implemented.
1340 *
1341 * XXX Dependency: syscall_vm_region() also supports only one flavor.
1342 */
1343
1344 kern_return_t
1345 mach_vm_region(
1346 vm_map_t map,
1347 mach_vm_offset_t *address, /* IN/OUT */
1348 mach_vm_size_t *size, /* OUT */
1349 vm_region_flavor_t flavor, /* IN */
1350 vm_region_info_t info, /* OUT */
1351 mach_msg_type_number_t *count, /* IN/OUT */
1352 mach_port_t *object_name) /* OUT */
1353 {
1354 vm_map_offset_t map_addr;
1355 vm_map_size_t map_size;
1356 kern_return_t kr;
1357
1358 if (VM_MAP_NULL == map)
1359 return KERN_INVALID_ARGUMENT;
1360
1361 map_addr = (vm_map_offset_t)*address;
1362 map_size = (vm_map_size_t)*size;
1363
1364 /* legacy conversion */
1365 if (VM_REGION_BASIC_INFO == flavor)
1366 flavor = VM_REGION_BASIC_INFO_64;
1367
1368 kr = vm_map_region(map,
1369 &map_addr, &map_size,
1370 flavor, info, count,
1371 object_name);
1372
1373 *address = map_addr;
1374 *size = map_size;
1375 return kr;
1376 }
1377
1378 /*
1379 * vm_region_64 and vm_region:
1380 *
1381 * User call to obtain information about a region in
1382 * a task's address map. Currently, only one flavor is
1383 * supported.
1384 *
1385 * XXX The reserved and behavior fields cannot be filled
1386 * in until the vm merge from the IK is completed, and
1387 * vm_reserve is implemented.
1388 *
1389 * XXX Dependency: syscall_vm_region() also supports only one flavor.
1390 */
1391
1392 kern_return_t
1393 vm_region_64(
1394 vm_map_t map,
1395 vm_offset_t *address, /* IN/OUT */
1396 vm_size_t *size, /* OUT */
1397 vm_region_flavor_t flavor, /* IN */
1398 vm_region_info_t info, /* OUT */
1399 mach_msg_type_number_t *count, /* IN/OUT */
1400 mach_port_t *object_name) /* OUT */
1401 {
1402 vm_map_offset_t map_addr;
1403 vm_map_size_t map_size;
1404 kern_return_t kr;
1405
1406 if (VM_MAP_NULL == map)
1407 return KERN_INVALID_ARGUMENT;
1408
1409 map_addr = (vm_map_offset_t)*address;
1410 map_size = (vm_map_size_t)*size;
1411
1412 /* legacy conversion */
1413 if (VM_REGION_BASIC_INFO == flavor)
1414 flavor = VM_REGION_BASIC_INFO_64;
1415
1416 kr = vm_map_region(map,
1417 &map_addr, &map_size,
1418 flavor, info, count,
1419 object_name);
1420
1421 *address = CAST_DOWN(vm_offset_t, map_addr);
1422 *size = CAST_DOWN(vm_size_t, map_size);
1423
1424 if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS)
1425 return KERN_INVALID_ADDRESS;
1426 return kr;
1427 }
1428
1429 kern_return_t
1430 vm_region(
1431 vm_map_t map,
1432 vm_address_t *address, /* IN/OUT */
1433 vm_size_t *size, /* OUT */
1434 vm_region_flavor_t flavor, /* IN */
1435 vm_region_info_t info, /* OUT */
1436 mach_msg_type_number_t *count, /* IN/OUT */
1437 mach_port_t *object_name) /* OUT */
1438 {
1439 vm_map_address_t map_addr;
1440 vm_map_size_t map_size;
1441 kern_return_t kr;
1442
1443 if (VM_MAP_NULL == map)
1444 return KERN_INVALID_ARGUMENT;
1445
1446 map_addr = (vm_map_address_t)*address;
1447 map_size = (vm_map_size_t)*size;
1448
1449 kr = vm_map_region(map,
1450 &map_addr, &map_size,
1451 flavor, info, count,
1452 object_name);
1453
1454 *address = CAST_DOWN(vm_address_t, map_addr);
1455 *size = CAST_DOWN(vm_size_t, map_size);
1456
1457 if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS)
1458 return KERN_INVALID_ADDRESS;
1459 return kr;
1460 }
1461
1462 /*
1463 * vm_region_recurse: A form of vm_region which follows the
1464 * submaps in a target map
1465 *
1466 */
1467 kern_return_t
1468 mach_vm_region_recurse(
1469 vm_map_t map,
1470 mach_vm_address_t *address,
1471 mach_vm_size_t *size,
1472 uint32_t *depth,
1473 vm_region_recurse_info_t info,
1474 mach_msg_type_number_t *infoCnt)
1475 {
1476 vm_map_address_t map_addr;
1477 vm_map_size_t map_size;
1478 kern_return_t kr;
1479
1480 if (VM_MAP_NULL == map)
1481 return KERN_INVALID_ARGUMENT;
1482
1483 map_addr = (vm_map_address_t)*address;
1484 map_size = (vm_map_size_t)*size;
1485
1486 kr = vm_map_region_recurse_64(
1487 map,
1488 &map_addr,
1489 &map_size,
1490 depth,
1491 (vm_region_submap_info_64_t)info,
1492 infoCnt);
1493
1494 *address = map_addr;
1495 *size = map_size;
1496 return kr;
1497 }
1498
1499 /*
1500 * vm_region_recurse: A form of vm_region which follows the
1501 * submaps in a target map
1502 *
1503 */
1504 kern_return_t
1505 vm_region_recurse_64(
1506 vm_map_t map,
1507 vm_address_t *address,
1508 vm_size_t *size,
1509 uint32_t *depth,
1510 vm_region_recurse_info_64_t info,
1511 mach_msg_type_number_t *infoCnt)
1512 {
1513 vm_map_address_t map_addr;
1514 vm_map_size_t map_size;
1515 kern_return_t kr;
1516
1517 if (VM_MAP_NULL == map)
1518 return KERN_INVALID_ARGUMENT;
1519
1520 map_addr = (vm_map_address_t)*address;
1521 map_size = (vm_map_size_t)*size;
1522
1523 kr = vm_map_region_recurse_64(
1524 map,
1525 &map_addr,
1526 &map_size,
1527 depth,
1528 (vm_region_submap_info_64_t)info,
1529 infoCnt);
1530
1531 *address = CAST_DOWN(vm_address_t, map_addr);
1532 *size = CAST_DOWN(vm_size_t, map_size);
1533
1534 if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS)
1535 return KERN_INVALID_ADDRESS;
1536 return kr;
1537 }
1538
1539 kern_return_t
1540 vm_region_recurse(
1541 vm_map_t map,
1542 vm_offset_t *address, /* IN/OUT */
1543 vm_size_t *size, /* OUT */
1544 natural_t *depth, /* IN/OUT */
1545 vm_region_recurse_info_t info32, /* IN/OUT */
1546 mach_msg_type_number_t *infoCnt) /* IN/OUT */
1547 {
1548 vm_region_submap_info_data_64_t info64;
1549 vm_region_submap_info_t info;
1550 vm_map_address_t map_addr;
1551 vm_map_size_t map_size;
1552 kern_return_t kr;
1553
1554 if (VM_MAP_NULL == map || *infoCnt < VM_REGION_SUBMAP_INFO_COUNT)
1555 return KERN_INVALID_ARGUMENT;
1556
1557
1558 map_addr = (vm_map_address_t)*address;
1559 map_size = (vm_map_size_t)*size;
1560 info = (vm_region_submap_info_t)info32;
1561 *infoCnt = VM_REGION_SUBMAP_INFO_COUNT_64;
1562
1563 kr = vm_map_region_recurse_64(map, &map_addr,&map_size,
1564 depth, &info64, infoCnt);
1565
1566 info->protection = info64.protection;
1567 info->max_protection = info64.max_protection;
1568 info->inheritance = info64.inheritance;
1569 info->offset = (uint32_t)info64.offset; /* trouble-maker */
1570 info->user_tag = info64.user_tag;
1571 info->pages_resident = info64.pages_resident;
1572 info->pages_shared_now_private = info64.pages_shared_now_private;
1573 info->pages_swapped_out = info64.pages_swapped_out;
1574 info->pages_dirtied = info64.pages_dirtied;
1575 info->ref_count = info64.ref_count;
1576 info->shadow_depth = info64.shadow_depth;
1577 info->external_pager = info64.external_pager;
1578 info->share_mode = info64.share_mode;
1579 info->is_submap = info64.is_submap;
1580 info->behavior = info64.behavior;
1581 info->object_id = info64.object_id;
1582 info->user_wired_count = info64.user_wired_count;
1583
1584 *address = CAST_DOWN(vm_address_t, map_addr);
1585 *size = CAST_DOWN(vm_size_t, map_size);
1586 *infoCnt = VM_REGION_SUBMAP_INFO_COUNT;
1587
1588 if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS)
1589 return KERN_INVALID_ADDRESS;
1590 return kr;
1591 }
1592
1593 kern_return_t
1594 mach_vm_purgable_control(
1595 vm_map_t map,
1596 mach_vm_offset_t address,
1597 vm_purgable_t control,
1598 int *state)
1599 {
1600 if (VM_MAP_NULL == map)
1601 return KERN_INVALID_ARGUMENT;
1602
1603 return vm_map_purgable_control(map,
1604 vm_map_trunc_page(address),
1605 control,
1606 state);
1607 }
1608
1609 kern_return_t
1610 vm_purgable_control(
1611 vm_map_t map,
1612 vm_offset_t address,
1613 vm_purgable_t control,
1614 int *state)
1615 {
1616 if (VM_MAP_NULL == map)
1617 return KERN_INVALID_ARGUMENT;
1618
1619 return vm_map_purgable_control(map,
1620 vm_map_trunc_page(address),
1621 control,
1622 state);
1623 }
1624
1625
1626 /*
1627 * Ordinarily, the right to allocate CPM is restricted
1628 * to privileged applications (those that can gain access
1629 * to the host priv port). Set this variable to zero if
1630 * you want to let any application allocate CPM.
1631 */
1632 unsigned int vm_allocate_cpm_privileged = 0;
1633
1634 /*
1635 * Allocate memory in the specified map, with the caveat that
1636 * the memory is physically contiguous. This call may fail
1637 * if the system can't find sufficient contiguous memory.
1638 * This call may cause or lead to heart-stopping amounts of
1639 * paging activity.
1640 *
1641 * Memory obtained from this call should be freed in the
1642 * normal way, viz., via vm_deallocate.
1643 */
1644 kern_return_t
1645 vm_allocate_cpm(
1646 host_priv_t host_priv,
1647 vm_map_t map,
1648 vm_address_t *addr,
1649 vm_size_t size,
1650 int flags)
1651 {
1652 vm_map_address_t map_addr;
1653 vm_map_size_t map_size;
1654 kern_return_t kr;
1655
1656 if (vm_allocate_cpm_privileged && HOST_PRIV_NULL == host_priv)
1657 return KERN_INVALID_HOST;
1658
1659 if (VM_MAP_NULL == map)
1660 return KERN_INVALID_ARGUMENT;
1661
1662 map_addr = (vm_map_address_t)*addr;
1663 map_size = (vm_map_size_t)size;
1664
1665 kr = vm_map_enter_cpm(map,
1666 &map_addr,
1667 map_size,
1668 flags);
1669
1670 *addr = CAST_DOWN(vm_address_t, map_addr);
1671 return kr;
1672 }
1673
1674
1675 kern_return_t
1676 mach_vm_page_query(
1677 vm_map_t map,
1678 mach_vm_offset_t offset,
1679 int *disposition,
1680 int *ref_count)
1681 {
1682 if (VM_MAP_NULL == map)
1683 return KERN_INVALID_ARGUMENT;
1684
1685 return vm_map_page_query_internal(map,
1686 vm_map_trunc_page(offset),
1687 disposition, ref_count);
1688 }
1689
1690 kern_return_t
1691 vm_map_page_query(
1692 vm_map_t map,
1693 vm_offset_t offset,
1694 int *disposition,
1695 int *ref_count)
1696 {
1697 if (VM_MAP_NULL == map)
1698 return KERN_INVALID_ARGUMENT;
1699
1700 return vm_map_page_query_internal(map,
1701 vm_map_trunc_page(offset),
1702 disposition, ref_count);
1703 }
1704
1705 kern_return_t
1706 mach_vm_page_info(
1707 vm_map_t map,
1708 mach_vm_address_t address,
1709 vm_page_info_flavor_t flavor,
1710 vm_page_info_t info,
1711 mach_msg_type_number_t *count)
1712 {
1713 kern_return_t kr;
1714
1715 if (map == VM_MAP_NULL) {
1716 return KERN_INVALID_ARGUMENT;
1717 }
1718
1719 kr = vm_map_page_info(map, address, flavor, info, count);
1720 return kr;
1721 }
1722
1723 /* map a (whole) upl into an address space */
1724 kern_return_t
1725 vm_upl_map(
1726 vm_map_t map,
1727 upl_t upl,
1728 vm_address_t *dst_addr)
1729 {
1730 vm_map_offset_t map_addr;
1731 kern_return_t kr;
1732
1733 if (VM_MAP_NULL == map)
1734 return KERN_INVALID_ARGUMENT;
1735
1736 kr = vm_map_enter_upl(map, upl, &map_addr);
1737 *dst_addr = CAST_DOWN(vm_address_t, map_addr);
1738 return kr;
1739 }
1740
1741 kern_return_t
1742 vm_upl_unmap(
1743 vm_map_t map,
1744 upl_t upl)
1745 {
1746 if (VM_MAP_NULL == map)
1747 return KERN_INVALID_ARGUMENT;
1748
1749 return (vm_map_remove_upl(map, upl));
1750 }
1751
1752 /* Retrieve a upl for an object underlying an address range in a map */
1753
1754 kern_return_t
1755 vm_map_get_upl(
1756 vm_map_t map,
1757 vm_map_offset_t map_offset,
1758 upl_size_t *upl_size,
1759 upl_t *upl,
1760 upl_page_info_array_t page_list,
1761 unsigned int *count,
1762 int *flags,
1763 int force_data_sync)
1764 {
1765 int map_flags;
1766 kern_return_t kr;
1767
1768 if (VM_MAP_NULL == map)
1769 return KERN_INVALID_ARGUMENT;
1770
1771 map_flags = *flags & ~UPL_NOZEROFILL;
1772 if (force_data_sync)
1773 map_flags |= UPL_FORCE_DATA_SYNC;
1774
1775 kr = vm_map_create_upl(map,
1776 map_offset,
1777 upl_size,
1778 upl,
1779 page_list,
1780 count,
1781 &map_flags);
1782
1783 *flags = (map_flags & ~UPL_FORCE_DATA_SYNC);
1784 return kr;
1785 }
1786
1787 /*
1788 * mach_make_memory_entry_64
1789 *
1790 * Think of it as a two-stage vm_remap() operation. First
1791 * you get a handle. Second, you get map that handle in
1792 * somewhere else. Rather than doing it all at once (and
1793 * without needing access to the other whole map).
1794 */
1795
1796 kern_return_t
1797 mach_make_memory_entry_64(
1798 vm_map_t target_map,
1799 memory_object_size_t *size,
1800 memory_object_offset_t offset,
1801 vm_prot_t permission,
1802 ipc_port_t *object_handle,
1803 ipc_port_t parent_handle)
1804 {
1805 vm_map_version_t version;
1806 vm_named_entry_t parent_entry;
1807 vm_named_entry_t user_entry;
1808 ipc_port_t user_handle;
1809 kern_return_t kr;
1810 vm_map_t real_map;
1811
1812 /* needed for call to vm_map_lookup_locked */
1813 boolean_t wired;
1814 vm_object_offset_t obj_off;
1815 vm_prot_t prot;
1816 struct vm_object_fault_info fault_info;
1817 vm_object_t object;
1818 vm_object_t shadow_object;
1819
1820 /* needed for direct map entry manipulation */
1821 vm_map_entry_t map_entry;
1822 vm_map_entry_t next_entry;
1823 vm_map_t local_map;
1824 vm_map_t original_map = target_map;
1825 vm_map_size_t total_size;
1826 vm_map_size_t map_size;
1827 vm_map_offset_t map_offset;
1828 vm_map_offset_t local_offset;
1829 vm_object_size_t mappable_size;
1830
1831 unsigned int access;
1832 vm_prot_t protections;
1833 vm_prot_t original_protections, mask_protections;
1834 unsigned int wimg_mode;
1835
1836 if (((permission & 0x00FF0000) &
1837 ~(MAP_MEM_ONLY |
1838 MAP_MEM_NAMED_CREATE |
1839 MAP_MEM_PURGABLE |
1840 MAP_MEM_NAMED_REUSE))) {
1841 /*
1842 * Unknown flag: reject for forward compatibility.
1843 */
1844 return KERN_INVALID_VALUE;
1845 }
1846
1847 if (parent_handle != IP_NULL &&
1848 ip_kotype(parent_handle) == IKOT_NAMED_ENTRY) {
1849 parent_entry = (vm_named_entry_t) parent_handle->ip_kobject;
1850 } else {
1851 parent_entry = NULL;
1852 }
1853
1854 original_protections = permission & VM_PROT_ALL;
1855 protections = original_protections;
1856 mask_protections = permission & VM_PROT_IS_MASK;
1857 access = GET_MAP_MEM(permission);
1858
1859 user_handle = IP_NULL;
1860 user_entry = NULL;
1861
1862 map_offset = vm_map_trunc_page(offset);
1863 map_size = vm_map_round_page(*size);
1864
1865 if (permission & MAP_MEM_ONLY) {
1866 boolean_t parent_is_object;
1867
1868 if (parent_entry == NULL) {
1869 return KERN_INVALID_ARGUMENT;
1870 }
1871
1872 parent_is_object = !(parent_entry->is_sub_map || parent_entry->is_pager);
1873 object = parent_entry->backing.object;
1874 if(parent_is_object && object != VM_OBJECT_NULL)
1875 wimg_mode = object->wimg_bits;
1876 else
1877 wimg_mode = VM_WIMG_USE_DEFAULT;
1878 if((access != GET_MAP_MEM(parent_entry->protection)) &&
1879 !(parent_entry->protection & VM_PROT_WRITE)) {
1880 return KERN_INVALID_RIGHT;
1881 }
1882 if(access == MAP_MEM_IO) {
1883 SET_MAP_MEM(access, parent_entry->protection);
1884 wimg_mode = VM_WIMG_IO;
1885 } else if (access == MAP_MEM_COPYBACK) {
1886 SET_MAP_MEM(access, parent_entry->protection);
1887 wimg_mode = VM_WIMG_USE_DEFAULT;
1888 } else if (access == MAP_MEM_WTHRU) {
1889 SET_MAP_MEM(access, parent_entry->protection);
1890 wimg_mode = VM_WIMG_WTHRU;
1891 } else if (access == MAP_MEM_WCOMB) {
1892 SET_MAP_MEM(access, parent_entry->protection);
1893 wimg_mode = VM_WIMG_WCOMB;
1894 }
1895 if (parent_is_object && object &&
1896 (access != MAP_MEM_NOOP) &&
1897 (!(object->nophyscache))) {
1898
1899 if (object->wimg_bits != wimg_mode) {
1900 vm_object_lock(object);
1901 vm_object_change_wimg_mode(object, wimg_mode);
1902 vm_object_unlock(object);
1903 }
1904 }
1905 if (object_handle)
1906 *object_handle = IP_NULL;
1907 return KERN_SUCCESS;
1908 }
1909
1910 if(permission & MAP_MEM_NAMED_CREATE) {
1911 kr = mach_memory_entry_allocate(&user_entry, &user_handle);
1912 if (kr != KERN_SUCCESS) {
1913 return KERN_FAILURE;
1914 }
1915
1916 /*
1917 * Force the creation of the VM object now.
1918 */
1919 if (map_size > (vm_map_size_t) ANON_MAX_SIZE) {
1920 /*
1921 * LP64todo - for now, we can only allocate 4GB-4096
1922 * internal objects because the default pager can't
1923 * page bigger ones. Remove this when it can.
1924 */
1925 kr = KERN_FAILURE;
1926 goto make_mem_done;
1927 }
1928
1929 object = vm_object_allocate(map_size);
1930 assert(object != VM_OBJECT_NULL);
1931
1932 if (permission & MAP_MEM_PURGABLE) {
1933 if (! (permission & VM_PROT_WRITE)) {
1934 /* if we can't write, we can't purge */
1935 vm_object_deallocate(object);
1936 kr = KERN_INVALID_ARGUMENT;
1937 goto make_mem_done;
1938 }
1939 object->purgable = VM_PURGABLE_NONVOLATILE;
1940 }
1941
1942 /*
1943 * The VM object is brand new and nobody else knows about it,
1944 * so we don't need to lock it.
1945 */
1946
1947 wimg_mode = object->wimg_bits;
1948 if (access == MAP_MEM_IO) {
1949 wimg_mode = VM_WIMG_IO;
1950 } else if (access == MAP_MEM_COPYBACK) {
1951 wimg_mode = VM_WIMG_USE_DEFAULT;
1952 } else if (access == MAP_MEM_WTHRU) {
1953 wimg_mode = VM_WIMG_WTHRU;
1954 } else if (access == MAP_MEM_WCOMB) {
1955 wimg_mode = VM_WIMG_WCOMB;
1956 }
1957 if (access != MAP_MEM_NOOP) {
1958 object->wimg_bits = wimg_mode;
1959 }
1960 /* the object has no pages, so no WIMG bits to update here */
1961
1962 /*
1963 * XXX
1964 * We use this path when we want to make sure that
1965 * nobody messes with the object (coalesce, for
1966 * example) before we map it.
1967 * We might want to use these objects for transposition via
1968 * vm_object_transpose() too, so we don't want any copy or
1969 * shadow objects either...
1970 */
1971 object->copy_strategy = MEMORY_OBJECT_COPY_NONE;
1972
1973 user_entry->backing.object = object;
1974 user_entry->internal = TRUE;
1975 user_entry->is_sub_map = FALSE;
1976 user_entry->is_pager = FALSE;
1977 user_entry->offset = 0;
1978 user_entry->protection = protections;
1979 SET_MAP_MEM(access, user_entry->protection);
1980 user_entry->size = map_size;
1981
1982 /* user_object pager and internal fields are not used */
1983 /* when the object field is filled in. */
1984
1985 *size = CAST_DOWN(vm_size_t, map_size);
1986 *object_handle = user_handle;
1987 return KERN_SUCCESS;
1988 }
1989
1990 if (parent_entry == NULL ||
1991 (permission & MAP_MEM_NAMED_REUSE)) {
1992
1993 /* Create a named object based on address range within the task map */
1994 /* Go find the object at given address */
1995
1996 if (target_map == VM_MAP_NULL) {
1997 return KERN_INVALID_TASK;
1998 }
1999
2000 redo_lookup:
2001 protections = original_protections;
2002 vm_map_lock_read(target_map);
2003
2004 /* get the object associated with the target address */
2005 /* note we check the permission of the range against */
2006 /* that requested by the caller */
2007
2008 kr = vm_map_lookup_locked(&target_map, map_offset,
2009 protections | mask_protections,
2010 OBJECT_LOCK_EXCLUSIVE, &version,
2011 &object, &obj_off, &prot, &wired,
2012 &fault_info,
2013 &real_map);
2014 if (kr != KERN_SUCCESS) {
2015 vm_map_unlock_read(target_map);
2016 goto make_mem_done;
2017 }
2018 if (mask_protections) {
2019 /*
2020 * The caller asked us to use the "protections" as
2021 * a mask, so restrict "protections" to what this
2022 * mapping actually allows.
2023 */
2024 protections &= prot;
2025 }
2026 if (((prot & protections) != protections)
2027 || (object == kernel_object)) {
2028 kr = KERN_INVALID_RIGHT;
2029 vm_object_unlock(object);
2030 vm_map_unlock_read(target_map);
2031 if(real_map != target_map)
2032 vm_map_unlock_read(real_map);
2033 if(object == kernel_object) {
2034 printf("Warning: Attempt to create a named"
2035 " entry from the kernel_object\n");
2036 }
2037 goto make_mem_done;
2038 }
2039
2040 /* We have an object, now check to see if this object */
2041 /* is suitable. If not, create a shadow and share that */
2042
2043 /*
2044 * We have to unlock the VM object to avoid deadlocking with
2045 * a VM map lock (the lock ordering is map, the object), if we
2046 * need to modify the VM map to create a shadow object. Since
2047 * we might release the VM map lock below anyway, we have
2048 * to release the VM map lock now.
2049 * XXX FBDP There must be a way to avoid this double lookup...
2050 *
2051 * Take an extra reference on the VM object to make sure it's
2052 * not going to disappear.
2053 */
2054 vm_object_reference_locked(object); /* extra ref to hold obj */
2055 vm_object_unlock(object);
2056
2057 local_map = original_map;
2058 local_offset = map_offset;
2059 if(target_map != local_map) {
2060 vm_map_unlock_read(target_map);
2061 if(real_map != target_map)
2062 vm_map_unlock_read(real_map);
2063 vm_map_lock_read(local_map);
2064 target_map = local_map;
2065 real_map = local_map;
2066 }
2067 while(TRUE) {
2068 if(!vm_map_lookup_entry(local_map,
2069 local_offset, &map_entry)) {
2070 kr = KERN_INVALID_ARGUMENT;
2071 vm_map_unlock_read(target_map);
2072 if(real_map != target_map)
2073 vm_map_unlock_read(real_map);
2074 vm_object_deallocate(object); /* release extra ref */
2075 object = VM_OBJECT_NULL;
2076 goto make_mem_done;
2077 }
2078 if(!(map_entry->is_sub_map)) {
2079 if(map_entry->object.vm_object != object) {
2080 kr = KERN_INVALID_ARGUMENT;
2081 vm_map_unlock_read(target_map);
2082 if(real_map != target_map)
2083 vm_map_unlock_read(real_map);
2084 vm_object_deallocate(object); /* release extra ref */
2085 object = VM_OBJECT_NULL;
2086 goto make_mem_done;
2087 }
2088 break;
2089 } else {
2090 vm_map_t tmap;
2091 tmap = local_map;
2092 local_map = map_entry->object.sub_map;
2093
2094 vm_map_lock_read(local_map);
2095 vm_map_unlock_read(tmap);
2096 target_map = local_map;
2097 real_map = local_map;
2098 local_offset = local_offset - map_entry->vme_start;
2099 local_offset += map_entry->offset;
2100 }
2101 }
2102
2103 /*
2104 * We found the VM map entry, lock the VM object again.
2105 */
2106 vm_object_lock(object);
2107 if(map_entry->wired_count) {
2108 /* JMM - The check below should be reworked instead. */
2109 object->true_share = TRUE;
2110 }
2111 if (mask_protections) {
2112 /*
2113 * The caller asked us to use the "protections" as
2114 * a mask, so restrict "protections" to what this
2115 * mapping actually allows.
2116 */
2117 protections &= map_entry->max_protection;
2118 }
2119 if(((map_entry->max_protection) & protections) != protections) {
2120 kr = KERN_INVALID_RIGHT;
2121 vm_object_unlock(object);
2122 vm_map_unlock_read(target_map);
2123 if(real_map != target_map)
2124 vm_map_unlock_read(real_map);
2125 vm_object_deallocate(object);
2126 object = VM_OBJECT_NULL;
2127 goto make_mem_done;
2128 }
2129
2130 mappable_size = fault_info.hi_offset - obj_off;
2131 total_size = map_entry->vme_end - map_entry->vme_start;
2132 if(map_size > mappable_size) {
2133 /* try to extend mappable size if the entries */
2134 /* following are from the same object and are */
2135 /* compatible */
2136 next_entry = map_entry->vme_next;
2137 /* lets see if the next map entry is still */
2138 /* pointing at this object and is contiguous */
2139 while(map_size > mappable_size) {
2140 if((next_entry->object.vm_object == object) &&
2141 (next_entry->vme_start ==
2142 next_entry->vme_prev->vme_end) &&
2143 (next_entry->offset ==
2144 next_entry->vme_prev->offset +
2145 (next_entry->vme_prev->vme_end -
2146 next_entry->vme_prev->vme_start))) {
2147 if (mask_protections) {
2148 /*
2149 * The caller asked us to use
2150 * the "protections" as a mask,
2151 * so restrict "protections" to
2152 * what this mapping actually
2153 * allows.
2154 */
2155 protections &= next_entry->max_protection;
2156 }
2157 if(((next_entry->max_protection)
2158 & protections) != protections) {
2159 break;
2160 }
2161 if (next_entry->needs_copy !=
2162 map_entry->needs_copy)
2163 break;
2164 mappable_size += next_entry->vme_end
2165 - next_entry->vme_start;
2166 total_size += next_entry->vme_end
2167 - next_entry->vme_start;
2168 next_entry = next_entry->vme_next;
2169 } else {
2170 break;
2171 }
2172
2173 }
2174 }
2175
2176 if(object->internal) {
2177 /* vm_map_lookup_locked will create a shadow if */
2178 /* needs_copy is set but does not check for the */
2179 /* other two conditions shown. It is important to */
2180 /* set up an object which will not be pulled from */
2181 /* under us. */
2182
2183 if ((map_entry->needs_copy || object->shadowed ||
2184 (object->vo_size > total_size))
2185 && !object->true_share) {
2186 /*
2187 * We have to unlock the VM object before
2188 * trying to upgrade the VM map lock, to
2189 * honor lock ordering (map then object).
2190 * Otherwise, we would deadlock if another
2191 * thread holds a read lock on the VM map and
2192 * is trying to acquire the VM object's lock.
2193 * We still hold an extra reference on the
2194 * VM object, guaranteeing that it won't
2195 * disappear.
2196 */
2197 vm_object_unlock(object);
2198
2199 if (vm_map_lock_read_to_write(target_map)) {
2200 /*
2201 * We couldn't upgrade our VM map lock
2202 * from "read" to "write" and we lost
2203 * our "read" lock.
2204 * Start all over again...
2205 */
2206 vm_object_deallocate(object); /* extra ref */
2207 target_map = original_map;
2208 goto redo_lookup;
2209 }
2210 vm_object_lock(object);
2211
2212 /*
2213 * JMM - We need to avoid coming here when the object
2214 * is wired by anybody, not just the current map. Why
2215 * couldn't we use the standard vm_object_copy_quickly()
2216 * approach here?
2217 */
2218
2219 /* create a shadow object */
2220 vm_object_shadow(&map_entry->object.vm_object,
2221 &map_entry->offset, total_size);
2222 shadow_object = map_entry->object.vm_object;
2223 vm_object_unlock(object);
2224
2225 prot = map_entry->protection & ~VM_PROT_WRITE;
2226
2227 if (override_nx(target_map, map_entry->alias) && prot)
2228 prot |= VM_PROT_EXECUTE;
2229
2230 vm_object_pmap_protect(
2231 object, map_entry->offset,
2232 total_size,
2233 ((map_entry->is_shared
2234 || target_map->mapped)
2235 ? PMAP_NULL :
2236 target_map->pmap),
2237 map_entry->vme_start,
2238 prot);
2239 total_size -= (map_entry->vme_end
2240 - map_entry->vme_start);
2241 next_entry = map_entry->vme_next;
2242 map_entry->needs_copy = FALSE;
2243
2244 vm_object_lock(shadow_object);
2245 while (total_size) {
2246 if(next_entry->object.vm_object == object) {
2247 vm_object_reference_locked(shadow_object);
2248 next_entry->object.vm_object
2249 = shadow_object;
2250 vm_object_deallocate(object);
2251 next_entry->offset
2252 = next_entry->vme_prev->offset +
2253 (next_entry->vme_prev->vme_end
2254 - next_entry->vme_prev->vme_start);
2255 next_entry->needs_copy = FALSE;
2256 } else {
2257 panic("mach_make_memory_entry_64:"
2258 " map entries out of sync\n");
2259 }
2260 total_size -=
2261 next_entry->vme_end
2262 - next_entry->vme_start;
2263 next_entry = next_entry->vme_next;
2264 }
2265
2266 /*
2267 * Transfer our extra reference to the
2268 * shadow object.
2269 */
2270 vm_object_reference_locked(shadow_object);
2271 vm_object_deallocate(object); /* extra ref */
2272 object = shadow_object;
2273
2274 obj_off = (local_offset - map_entry->vme_start)
2275 + map_entry->offset;
2276
2277 vm_map_lock_write_to_read(target_map);
2278 }
2279 }
2280
2281 /* note: in the future we can (if necessary) allow for */
2282 /* memory object lists, this will better support */
2283 /* fragmentation, but is it necessary? The user should */
2284 /* be encouraged to create address space oriented */
2285 /* shared objects from CLEAN memory regions which have */
2286 /* a known and defined history. i.e. no inheritence */
2287 /* share, make this call before making the region the */
2288 /* target of ipc's, etc. The code above, protecting */
2289 /* against delayed copy, etc. is mostly defensive. */
2290
2291 wimg_mode = object->wimg_bits;
2292 if(!(object->nophyscache)) {
2293 if(access == MAP_MEM_IO) {
2294 wimg_mode = VM_WIMG_IO;
2295 } else if (access == MAP_MEM_COPYBACK) {
2296 wimg_mode = VM_WIMG_USE_DEFAULT;
2297 } else if (access == MAP_MEM_WTHRU) {
2298 wimg_mode = VM_WIMG_WTHRU;
2299 } else if (access == MAP_MEM_WCOMB) {
2300 wimg_mode = VM_WIMG_WCOMB;
2301 }
2302 }
2303
2304 object->true_share = TRUE;
2305 if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC)
2306 object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
2307
2308 /*
2309 * The memory entry now points to this VM object and we
2310 * need to hold a reference on the VM object. Use the extra
2311 * reference we took earlier to keep the object alive when we
2312 * had to unlock it.
2313 */
2314
2315 vm_map_unlock_read(target_map);
2316 if(real_map != target_map)
2317 vm_map_unlock_read(real_map);
2318
2319 if (object->wimg_bits != wimg_mode)
2320 vm_object_change_wimg_mode(object, wimg_mode);
2321
2322 /* the size of mapped entry that overlaps with our region */
2323 /* which is targeted for share. */
2324 /* (entry_end - entry_start) - */
2325 /* offset of our beg addr within entry */
2326 /* it corresponds to this: */
2327
2328 if(map_size > mappable_size)
2329 map_size = mappable_size;
2330
2331 if (permission & MAP_MEM_NAMED_REUSE) {
2332 /*
2333 * Compare what we got with the "parent_entry".
2334 * If they match, re-use the "parent_entry" instead
2335 * of creating a new one.
2336 */
2337 if (parent_entry != NULL &&
2338 parent_entry->backing.object == object &&
2339 parent_entry->internal == object->internal &&
2340 parent_entry->is_sub_map == FALSE &&
2341 parent_entry->is_pager == FALSE &&
2342 parent_entry->offset == obj_off &&
2343 parent_entry->protection == protections &&
2344 parent_entry->size == map_size) {
2345 /*
2346 * We have a match: re-use "parent_entry".
2347 */
2348 /* release our extra reference on object */
2349 vm_object_unlock(object);
2350 vm_object_deallocate(object);
2351 /* parent_entry->ref_count++; XXX ? */
2352 /* Get an extra send-right on handle */
2353 ipc_port_copy_send(parent_handle);
2354 *object_handle = parent_handle;
2355 return KERN_SUCCESS;
2356 } else {
2357 /*
2358 * No match: we need to create a new entry.
2359 * fall through...
2360 */
2361 }
2362 }
2363
2364 vm_object_unlock(object);
2365 if (mach_memory_entry_allocate(&user_entry, &user_handle)
2366 != KERN_SUCCESS) {
2367 /* release our unused reference on the object */
2368 vm_object_deallocate(object);
2369 return KERN_FAILURE;
2370 }
2371
2372 user_entry->backing.object = object;
2373 user_entry->internal = object->internal;
2374 user_entry->is_sub_map = FALSE;
2375 user_entry->is_pager = FALSE;
2376 user_entry->offset = obj_off;
2377 user_entry->protection = protections;
2378 SET_MAP_MEM(GET_MAP_MEM(permission), user_entry->protection);
2379 user_entry->size = map_size;
2380
2381 /* user_object pager and internal fields are not used */
2382 /* when the object field is filled in. */
2383
2384 *size = CAST_DOWN(vm_size_t, map_size);
2385 *object_handle = user_handle;
2386 return KERN_SUCCESS;
2387
2388 } else {
2389 /* The new object will be base on an existing named object */
2390
2391 if (parent_entry == NULL) {
2392 kr = KERN_INVALID_ARGUMENT;
2393 goto make_mem_done;
2394 }
2395 if((offset + map_size) > parent_entry->size) {
2396 kr = KERN_INVALID_ARGUMENT;
2397 goto make_mem_done;
2398 }
2399
2400 if (mask_protections) {
2401 /*
2402 * The caller asked us to use the "protections" as
2403 * a mask, so restrict "protections" to what this
2404 * mapping actually allows.
2405 */
2406 protections &= parent_entry->protection;
2407 }
2408 if((protections & parent_entry->protection) != protections) {
2409 kr = KERN_PROTECTION_FAILURE;
2410 goto make_mem_done;
2411 }
2412
2413 if (mach_memory_entry_allocate(&user_entry, &user_handle)
2414 != KERN_SUCCESS) {
2415 kr = KERN_FAILURE;
2416 goto make_mem_done;
2417 }
2418
2419 user_entry->size = map_size;
2420 user_entry->offset = parent_entry->offset + map_offset;
2421 user_entry->is_sub_map = parent_entry->is_sub_map;
2422 user_entry->is_pager = parent_entry->is_pager;
2423 user_entry->internal = parent_entry->internal;
2424 user_entry->protection = protections;
2425
2426 if(access != MAP_MEM_NOOP) {
2427 SET_MAP_MEM(access, user_entry->protection);
2428 }
2429
2430 if(parent_entry->is_sub_map) {
2431 user_entry->backing.map = parent_entry->backing.map;
2432 vm_map_lock(user_entry->backing.map);
2433 user_entry->backing.map->ref_count++;
2434 vm_map_unlock(user_entry->backing.map);
2435 }
2436 else if (parent_entry->is_pager) {
2437 user_entry->backing.pager = parent_entry->backing.pager;
2438 /* JMM - don't we need a reference here? */
2439 } else {
2440 object = parent_entry->backing.object;
2441 assert(object != VM_OBJECT_NULL);
2442 user_entry->backing.object = object;
2443 /* we now point to this object, hold on */
2444 vm_object_reference(object);
2445 vm_object_lock(object);
2446 object->true_share = TRUE;
2447 if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC)
2448 object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
2449 vm_object_unlock(object);
2450 }
2451 *size = CAST_DOWN(vm_size_t, map_size);
2452 *object_handle = user_handle;
2453 return KERN_SUCCESS;
2454 }
2455
2456 make_mem_done:
2457 if (user_handle != IP_NULL) {
2458 /*
2459 * Releasing "user_handle" causes the kernel object
2460 * associated with it ("user_entry" here) to also be
2461 * released and freed.
2462 */
2463 mach_memory_entry_port_release(user_handle);
2464 }
2465 return kr;
2466 }
2467
2468 kern_return_t
2469 _mach_make_memory_entry(
2470 vm_map_t target_map,
2471 memory_object_size_t *size,
2472 memory_object_offset_t offset,
2473 vm_prot_t permission,
2474 ipc_port_t *object_handle,
2475 ipc_port_t parent_entry)
2476 {
2477 memory_object_size_t mo_size;
2478 kern_return_t kr;
2479
2480 mo_size = (memory_object_size_t)*size;
2481 kr = mach_make_memory_entry_64(target_map, &mo_size,
2482 (memory_object_offset_t)offset, permission, object_handle,
2483 parent_entry);
2484 *size = mo_size;
2485 return kr;
2486 }
2487
2488 kern_return_t
2489 mach_make_memory_entry(
2490 vm_map_t target_map,
2491 vm_size_t *size,
2492 vm_offset_t offset,
2493 vm_prot_t permission,
2494 ipc_port_t *object_handle,
2495 ipc_port_t parent_entry)
2496 {
2497 memory_object_size_t mo_size;
2498 kern_return_t kr;
2499
2500 mo_size = (memory_object_size_t)*size;
2501 kr = mach_make_memory_entry_64(target_map, &mo_size,
2502 (memory_object_offset_t)offset, permission, object_handle,
2503 parent_entry);
2504 *size = CAST_DOWN(vm_size_t, mo_size);
2505 return kr;
2506 }
2507
2508 /*
2509 * task_wire
2510 *
2511 * Set or clear the map's wiring_required flag. This flag, if set,
2512 * will cause all future virtual memory allocation to allocate
2513 * user wired memory. Unwiring pages wired down as a result of
2514 * this routine is done with the vm_wire interface.
2515 */
2516 kern_return_t
2517 task_wire(
2518 vm_map_t map,
2519 boolean_t must_wire)
2520 {
2521 if (map == VM_MAP_NULL)
2522 return(KERN_INVALID_ARGUMENT);
2523
2524 if (must_wire)
2525 map->wiring_required = TRUE;
2526 else
2527 map->wiring_required = FALSE;
2528
2529 return(KERN_SUCCESS);
2530 }
2531
2532 __private_extern__ kern_return_t
2533 mach_memory_entry_allocate(
2534 vm_named_entry_t *user_entry_p,
2535 ipc_port_t *user_handle_p)
2536 {
2537 vm_named_entry_t user_entry;
2538 ipc_port_t user_handle;
2539 ipc_port_t previous;
2540
2541 user_entry = (vm_named_entry_t) kalloc(sizeof *user_entry);
2542 if (user_entry == NULL)
2543 return KERN_FAILURE;
2544
2545 named_entry_lock_init(user_entry);
2546
2547 user_handle = ipc_port_alloc_kernel();
2548 if (user_handle == IP_NULL) {
2549 kfree(user_entry, sizeof *user_entry);
2550 return KERN_FAILURE;
2551 }
2552 ip_lock(user_handle);
2553
2554 /* make a sonce right */
2555 user_handle->ip_sorights++;
2556 ip_reference(user_handle);
2557
2558 user_handle->ip_destination = IP_NULL;
2559 user_handle->ip_receiver_name = MACH_PORT_NULL;
2560 user_handle->ip_receiver = ipc_space_kernel;
2561
2562 /* make a send right */
2563 user_handle->ip_mscount++;
2564 user_handle->ip_srights++;
2565 ip_reference(user_handle);
2566
2567 ipc_port_nsrequest(user_handle, 1, user_handle, &previous);
2568 /* nsrequest unlocks user_handle */
2569
2570 user_entry->backing.pager = NULL;
2571 user_entry->is_sub_map = FALSE;
2572 user_entry->is_pager = FALSE;
2573 user_entry->internal = FALSE;
2574 user_entry->size = 0;
2575 user_entry->offset = 0;
2576 user_entry->protection = VM_PROT_NONE;
2577 user_entry->ref_count = 1;
2578
2579 ipc_kobject_set(user_handle, (ipc_kobject_t) user_entry,
2580 IKOT_NAMED_ENTRY);
2581
2582 *user_entry_p = user_entry;
2583 *user_handle_p = user_handle;
2584
2585 return KERN_SUCCESS;
2586 }
2587
2588 /*
2589 * mach_memory_object_memory_entry_64
2590 *
2591 * Create a named entry backed by the provided pager.
2592 *
2593 * JMM - we need to hold a reference on the pager -
2594 * and release it when the named entry is destroyed.
2595 */
2596 kern_return_t
2597 mach_memory_object_memory_entry_64(
2598 host_t host,
2599 boolean_t internal,
2600 vm_object_offset_t size,
2601 vm_prot_t permission,
2602 memory_object_t pager,
2603 ipc_port_t *entry_handle)
2604 {
2605 unsigned int access;
2606 vm_named_entry_t user_entry;
2607 ipc_port_t user_handle;
2608
2609 if (host == HOST_NULL)
2610 return(KERN_INVALID_HOST);
2611
2612 if (mach_memory_entry_allocate(&user_entry, &user_handle)
2613 != KERN_SUCCESS) {
2614 return KERN_FAILURE;
2615 }
2616
2617 user_entry->backing.pager = pager;
2618 user_entry->size = size;
2619 user_entry->offset = 0;
2620 user_entry->protection = permission & VM_PROT_ALL;
2621 access = GET_MAP_MEM(permission);
2622 SET_MAP_MEM(access, user_entry->protection);
2623 user_entry->internal = internal;
2624 user_entry->is_sub_map = FALSE;
2625 user_entry->is_pager = TRUE;
2626 assert(user_entry->ref_count == 1);
2627
2628 *entry_handle = user_handle;
2629 return KERN_SUCCESS;
2630 }
2631
2632 kern_return_t
2633 mach_memory_object_memory_entry(
2634 host_t host,
2635 boolean_t internal,
2636 vm_size_t size,
2637 vm_prot_t permission,
2638 memory_object_t pager,
2639 ipc_port_t *entry_handle)
2640 {
2641 return mach_memory_object_memory_entry_64( host, internal,
2642 (vm_object_offset_t)size, permission, pager, entry_handle);
2643 }
2644
2645
2646 kern_return_t
2647 mach_memory_entry_purgable_control(
2648 ipc_port_t entry_port,
2649 vm_purgable_t control,
2650 int *state)
2651 {
2652 kern_return_t kr;
2653 vm_named_entry_t mem_entry;
2654 vm_object_t object;
2655
2656 if (entry_port == IP_NULL ||
2657 ip_kotype(entry_port) != IKOT_NAMED_ENTRY) {
2658 return KERN_INVALID_ARGUMENT;
2659 }
2660 if (control != VM_PURGABLE_SET_STATE &&
2661 control != VM_PURGABLE_GET_STATE)
2662 return(KERN_INVALID_ARGUMENT);
2663
2664 if (control == VM_PURGABLE_SET_STATE &&
2665 (((*state & ~(VM_PURGABLE_ALL_MASKS)) != 0) ||
2666 ((*state & VM_PURGABLE_STATE_MASK) > VM_PURGABLE_STATE_MASK)))
2667 return(KERN_INVALID_ARGUMENT);
2668
2669 mem_entry = (vm_named_entry_t) entry_port->ip_kobject;
2670
2671 named_entry_lock(mem_entry);
2672
2673 if (mem_entry->is_sub_map || mem_entry->is_pager) {
2674 named_entry_unlock(mem_entry);
2675 return KERN_INVALID_ARGUMENT;
2676 }
2677
2678 object = mem_entry->backing.object;
2679 if (object == VM_OBJECT_NULL) {
2680 named_entry_unlock(mem_entry);
2681 return KERN_INVALID_ARGUMENT;
2682 }
2683
2684 vm_object_lock(object);
2685
2686 /* check that named entry covers entire object ? */
2687 if (mem_entry->offset != 0 || object->vo_size != mem_entry->size) {
2688 vm_object_unlock(object);
2689 named_entry_unlock(mem_entry);
2690 return KERN_INVALID_ARGUMENT;
2691 }
2692
2693 named_entry_unlock(mem_entry);
2694
2695 kr = vm_object_purgable_control(object, control, state);
2696
2697 vm_object_unlock(object);
2698
2699 return kr;
2700 }
2701
2702 /*
2703 * mach_memory_entry_port_release:
2704 *
2705 * Release a send right on a named entry port. This is the correct
2706 * way to destroy a named entry. When the last right on the port is
2707 * released, ipc_kobject_destroy() will call mach_destroy_memory_entry().
2708 */
2709 void
2710 mach_memory_entry_port_release(
2711 ipc_port_t port)
2712 {
2713 assert(ip_kotype(port) == IKOT_NAMED_ENTRY);
2714 ipc_port_release_send(port);
2715 }
2716
2717 /*
2718 * mach_destroy_memory_entry:
2719 *
2720 * Drops a reference on a memory entry and destroys the memory entry if
2721 * there are no more references on it.
2722 * NOTE: This routine should not be called to destroy a memory entry from the
2723 * kernel, as it will not release the Mach port associated with the memory
2724 * entry. The proper way to destroy a memory entry in the kernel is to
2725 * call mach_memort_entry_port_release() to release the kernel's send-right on
2726 * the memory entry's port. When the last send right is released, the memory
2727 * entry will be destroyed via ipc_kobject_destroy().
2728 */
2729 void
2730 mach_destroy_memory_entry(
2731 ipc_port_t port)
2732 {
2733 vm_named_entry_t named_entry;
2734 #if MACH_ASSERT
2735 assert(ip_kotype(port) == IKOT_NAMED_ENTRY);
2736 #endif /* MACH_ASSERT */
2737 named_entry = (vm_named_entry_t)port->ip_kobject;
2738 lck_mtx_lock(&(named_entry)->Lock);
2739 named_entry->ref_count -= 1;
2740 if(named_entry->ref_count == 0) {
2741 if (named_entry->is_sub_map) {
2742 vm_map_deallocate(named_entry->backing.map);
2743 } else if (!named_entry->is_pager) {
2744 /* release the memory object we've been pointing to */
2745 vm_object_deallocate(named_entry->backing.object);
2746 } /* else JMM - need to drop reference on pager in that case */
2747
2748 lck_mtx_unlock(&(named_entry)->Lock);
2749
2750 kfree((void *) port->ip_kobject,
2751 sizeof (struct vm_named_entry));
2752 } else
2753 lck_mtx_unlock(&(named_entry)->Lock);
2754 }
2755
2756 /* Allow manipulation of individual page state. This is actually part of */
2757 /* the UPL regimen but takes place on the memory entry rather than on a UPL */
2758
2759 kern_return_t
2760 mach_memory_entry_page_op(
2761 ipc_port_t entry_port,
2762 vm_object_offset_t offset,
2763 int ops,
2764 ppnum_t *phys_entry,
2765 int *flags)
2766 {
2767 vm_named_entry_t mem_entry;
2768 vm_object_t object;
2769 kern_return_t kr;
2770
2771 if (entry_port == IP_NULL ||
2772 ip_kotype(entry_port) != IKOT_NAMED_ENTRY) {
2773 return KERN_INVALID_ARGUMENT;
2774 }
2775
2776 mem_entry = (vm_named_entry_t) entry_port->ip_kobject;
2777
2778 named_entry_lock(mem_entry);
2779
2780 if (mem_entry->is_sub_map || mem_entry->is_pager) {
2781 named_entry_unlock(mem_entry);
2782 return KERN_INVALID_ARGUMENT;
2783 }
2784
2785 object = mem_entry->backing.object;
2786 if (object == VM_OBJECT_NULL) {
2787 named_entry_unlock(mem_entry);
2788 return KERN_INVALID_ARGUMENT;
2789 }
2790
2791 vm_object_reference(object);
2792 named_entry_unlock(mem_entry);
2793
2794 kr = vm_object_page_op(object, offset, ops, phys_entry, flags);
2795
2796 vm_object_deallocate(object);
2797
2798 return kr;
2799 }
2800
2801 /*
2802 * mach_memory_entry_range_op offers performance enhancement over
2803 * mach_memory_entry_page_op for page_op functions which do not require page
2804 * level state to be returned from the call. Page_op was created to provide
2805 * a low-cost alternative to page manipulation via UPLs when only a single
2806 * page was involved. The range_op call establishes the ability in the _op
2807 * family of functions to work on multiple pages where the lack of page level
2808 * state handling allows the caller to avoid the overhead of the upl structures.
2809 */
2810
2811 kern_return_t
2812 mach_memory_entry_range_op(
2813 ipc_port_t entry_port,
2814 vm_object_offset_t offset_beg,
2815 vm_object_offset_t offset_end,
2816 int ops,
2817 int *range)
2818 {
2819 vm_named_entry_t mem_entry;
2820 vm_object_t object;
2821 kern_return_t kr;
2822
2823 if (entry_port == IP_NULL ||
2824 ip_kotype(entry_port) != IKOT_NAMED_ENTRY) {
2825 return KERN_INVALID_ARGUMENT;
2826 }
2827
2828 mem_entry = (vm_named_entry_t) entry_port->ip_kobject;
2829
2830 named_entry_lock(mem_entry);
2831
2832 if (mem_entry->is_sub_map || mem_entry->is_pager) {
2833 named_entry_unlock(mem_entry);
2834 return KERN_INVALID_ARGUMENT;
2835 }
2836
2837 object = mem_entry->backing.object;
2838 if (object == VM_OBJECT_NULL) {
2839 named_entry_unlock(mem_entry);
2840 return KERN_INVALID_ARGUMENT;
2841 }
2842
2843 vm_object_reference(object);
2844 named_entry_unlock(mem_entry);
2845
2846 kr = vm_object_range_op(object,
2847 offset_beg,
2848 offset_end,
2849 ops,
2850 (uint32_t *) range);
2851
2852 vm_object_deallocate(object);
2853
2854 return kr;
2855 }
2856
2857
2858 kern_return_t
2859 set_dp_control_port(
2860 host_priv_t host_priv,
2861 ipc_port_t control_port)
2862 {
2863 if (host_priv == HOST_PRIV_NULL)
2864 return (KERN_INVALID_HOST);
2865
2866 if (IP_VALID(dynamic_pager_control_port))
2867 ipc_port_release_send(dynamic_pager_control_port);
2868
2869 dynamic_pager_control_port = control_port;
2870 return KERN_SUCCESS;
2871 }
2872
2873 kern_return_t
2874 get_dp_control_port(
2875 host_priv_t host_priv,
2876 ipc_port_t *control_port)
2877 {
2878 if (host_priv == HOST_PRIV_NULL)
2879 return (KERN_INVALID_HOST);
2880
2881 *control_port = ipc_port_copy_send(dynamic_pager_control_port);
2882 return KERN_SUCCESS;
2883
2884 }
2885
2886 /* ******* Temporary Internal calls to UPL for BSD ***** */
2887
2888 extern int kernel_upl_map(
2889 vm_map_t map,
2890 upl_t upl,
2891 vm_offset_t *dst_addr);
2892
2893 extern int kernel_upl_unmap(
2894 vm_map_t map,
2895 upl_t upl);
2896
2897 extern int kernel_upl_commit(
2898 upl_t upl,
2899 upl_page_info_t *pl,
2900 mach_msg_type_number_t count);
2901
2902 extern int kernel_upl_commit_range(
2903 upl_t upl,
2904 upl_offset_t offset,
2905 upl_size_t size,
2906 int flags,
2907 upl_page_info_array_t pl,
2908 mach_msg_type_number_t count);
2909
2910 extern int kernel_upl_abort(
2911 upl_t upl,
2912 int abort_type);
2913
2914 extern int kernel_upl_abort_range(
2915 upl_t upl,
2916 upl_offset_t offset,
2917 upl_size_t size,
2918 int abort_flags);
2919
2920
2921 kern_return_t
2922 kernel_upl_map(
2923 vm_map_t map,
2924 upl_t upl,
2925 vm_offset_t *dst_addr)
2926 {
2927 return vm_upl_map(map, upl, dst_addr);
2928 }
2929
2930
2931 kern_return_t
2932 kernel_upl_unmap(
2933 vm_map_t map,
2934 upl_t upl)
2935 {
2936 return vm_upl_unmap(map, upl);
2937 }
2938
2939 kern_return_t
2940 kernel_upl_commit(
2941 upl_t upl,
2942 upl_page_info_t *pl,
2943 mach_msg_type_number_t count)
2944 {
2945 kern_return_t kr;
2946
2947 kr = upl_commit(upl, pl, count);
2948 upl_deallocate(upl);
2949 return kr;
2950 }
2951
2952
2953 kern_return_t
2954 kernel_upl_commit_range(
2955 upl_t upl,
2956 upl_offset_t offset,
2957 upl_size_t size,
2958 int flags,
2959 upl_page_info_array_t pl,
2960 mach_msg_type_number_t count)
2961 {
2962 boolean_t finished = FALSE;
2963 kern_return_t kr;
2964
2965 if (flags & UPL_COMMIT_FREE_ON_EMPTY)
2966 flags |= UPL_COMMIT_NOTIFY_EMPTY;
2967
2968 if (flags & UPL_COMMIT_KERNEL_ONLY_FLAGS) {
2969 return KERN_INVALID_ARGUMENT;
2970 }
2971
2972 kr = upl_commit_range(upl, offset, size, flags, pl, count, &finished);
2973
2974 if ((flags & UPL_COMMIT_NOTIFY_EMPTY) && finished)
2975 upl_deallocate(upl);
2976
2977 return kr;
2978 }
2979
2980 kern_return_t
2981 kernel_upl_abort_range(
2982 upl_t upl,
2983 upl_offset_t offset,
2984 upl_size_t size,
2985 int abort_flags)
2986 {
2987 kern_return_t kr;
2988 boolean_t finished = FALSE;
2989
2990 if (abort_flags & UPL_COMMIT_FREE_ON_EMPTY)
2991 abort_flags |= UPL_COMMIT_NOTIFY_EMPTY;
2992
2993 kr = upl_abort_range(upl, offset, size, abort_flags, &finished);
2994
2995 if ((abort_flags & UPL_COMMIT_FREE_ON_EMPTY) && finished)
2996 upl_deallocate(upl);
2997
2998 return kr;
2999 }
3000
3001 kern_return_t
3002 kernel_upl_abort(
3003 upl_t upl,
3004 int abort_type)
3005 {
3006 kern_return_t kr;
3007
3008 kr = upl_abort(upl, abort_type);
3009 upl_deallocate(upl);
3010 return kr;
3011 }
3012
3013 /*
3014 * Now a kernel-private interface (for BootCache
3015 * use only). Need a cleaner way to create an
3016 * empty vm_map() and return a handle to it.
3017 */
3018
3019 kern_return_t
3020 vm_region_object_create(
3021 __unused vm_map_t target_map,
3022 vm_size_t size,
3023 ipc_port_t *object_handle)
3024 {
3025 vm_named_entry_t user_entry;
3026 ipc_port_t user_handle;
3027
3028 vm_map_t new_map;
3029
3030 if (mach_memory_entry_allocate(&user_entry, &user_handle)
3031 != KERN_SUCCESS) {
3032 return KERN_FAILURE;
3033 }
3034
3035 /* Create a named object based on a submap of specified size */
3036
3037 new_map = vm_map_create(PMAP_NULL, VM_MAP_MIN_ADDRESS,
3038 vm_map_round_page(size), TRUE);
3039
3040 user_entry->backing.map = new_map;
3041 user_entry->internal = TRUE;
3042 user_entry->is_sub_map = TRUE;
3043 user_entry->offset = 0;
3044 user_entry->protection = VM_PROT_ALL;
3045 user_entry->size = size;
3046 assert(user_entry->ref_count == 1);
3047
3048 *object_handle = user_handle;
3049 return KERN_SUCCESS;
3050
3051 }
3052
3053 ppnum_t vm_map_get_phys_page( /* forward */
3054 vm_map_t map,
3055 vm_offset_t offset);
3056
3057 ppnum_t
3058 vm_map_get_phys_page(
3059 vm_map_t map,
3060 vm_offset_t addr)
3061 {
3062 vm_object_offset_t offset;
3063 vm_object_t object;
3064 vm_map_offset_t map_offset;
3065 vm_map_entry_t entry;
3066 ppnum_t phys_page = 0;
3067
3068 map_offset = vm_map_trunc_page(addr);
3069
3070 vm_map_lock(map);
3071 while (vm_map_lookup_entry(map, map_offset, &entry)) {
3072
3073 if (entry->object.vm_object == VM_OBJECT_NULL) {
3074 vm_map_unlock(map);
3075 return (ppnum_t) 0;
3076 }
3077 if (entry->is_sub_map) {
3078 vm_map_t old_map;
3079 vm_map_lock(entry->object.sub_map);
3080 old_map = map;
3081 map = entry->object.sub_map;
3082 map_offset = entry->offset + (map_offset - entry->vme_start);
3083 vm_map_unlock(old_map);
3084 continue;
3085 }
3086 if (entry->object.vm_object->phys_contiguous) {
3087 /* These are not standard pageable memory mappings */
3088 /* If they are not present in the object they will */
3089 /* have to be picked up from the pager through the */
3090 /* fault mechanism. */
3091 if(entry->object.vm_object->vo_shadow_offset == 0) {
3092 /* need to call vm_fault */
3093 vm_map_unlock(map);
3094 vm_fault(map, map_offset, VM_PROT_NONE,
3095 FALSE, THREAD_UNINT, NULL, 0);
3096 vm_map_lock(map);
3097 continue;
3098 }
3099 offset = entry->offset + (map_offset - entry->vme_start);
3100 phys_page = (ppnum_t)
3101 ((entry->object.vm_object->vo_shadow_offset
3102 + offset) >> 12);
3103 break;
3104
3105 }
3106 offset = entry->offset + (map_offset - entry->vme_start);
3107 object = entry->object.vm_object;
3108 vm_object_lock(object);
3109 while (TRUE) {
3110 vm_page_t dst_page = vm_page_lookup(object,offset);
3111 if(dst_page == VM_PAGE_NULL) {
3112 if(object->shadow) {
3113 vm_object_t old_object;
3114 vm_object_lock(object->shadow);
3115 old_object = object;
3116 offset = offset + object->vo_shadow_offset;
3117 object = object->shadow;
3118 vm_object_unlock(old_object);
3119 } else {
3120 vm_object_unlock(object);
3121 break;
3122 }
3123 } else {
3124 phys_page = (ppnum_t)(dst_page->phys_page);
3125 vm_object_unlock(object);
3126 break;
3127 }
3128 }
3129 break;
3130
3131 }
3132
3133 vm_map_unlock(map);
3134 return phys_page;
3135 }
3136
3137
3138
3139 kern_return_t kernel_object_iopl_request( /* forward */
3140 vm_named_entry_t named_entry,
3141 memory_object_offset_t offset,
3142 upl_size_t *upl_size,
3143 upl_t *upl_ptr,
3144 upl_page_info_array_t user_page_list,
3145 unsigned int *page_list_count,
3146 int *flags);
3147
3148 kern_return_t
3149 kernel_object_iopl_request(
3150 vm_named_entry_t named_entry,
3151 memory_object_offset_t offset,
3152 upl_size_t *upl_size,
3153 upl_t *upl_ptr,
3154 upl_page_info_array_t user_page_list,
3155 unsigned int *page_list_count,
3156 int *flags)
3157 {
3158 vm_object_t object;
3159 kern_return_t ret;
3160
3161 int caller_flags;
3162
3163 caller_flags = *flags;
3164
3165 if (caller_flags & ~UPL_VALID_FLAGS) {
3166 /*
3167 * For forward compatibility's sake,
3168 * reject any unknown flag.
3169 */
3170 return KERN_INVALID_VALUE;
3171 }
3172
3173 /* a few checks to make sure user is obeying rules */
3174 if(*upl_size == 0) {
3175 if(offset >= named_entry->size)
3176 return(KERN_INVALID_RIGHT);
3177 *upl_size = (upl_size_t) (named_entry->size - offset);
3178 if (*upl_size != named_entry->size - offset)
3179 return KERN_INVALID_ARGUMENT;
3180 }
3181 if(caller_flags & UPL_COPYOUT_FROM) {
3182 if((named_entry->protection & VM_PROT_READ)
3183 != VM_PROT_READ) {
3184 return(KERN_INVALID_RIGHT);
3185 }
3186 } else {
3187 if((named_entry->protection &
3188 (VM_PROT_READ | VM_PROT_WRITE))
3189 != (VM_PROT_READ | VM_PROT_WRITE)) {
3190 return(KERN_INVALID_RIGHT);
3191 }
3192 }
3193 if(named_entry->size < (offset + *upl_size))
3194 return(KERN_INVALID_ARGUMENT);
3195
3196 /* the callers parameter offset is defined to be the */
3197 /* offset from beginning of named entry offset in object */
3198 offset = offset + named_entry->offset;
3199
3200 if(named_entry->is_sub_map)
3201 return (KERN_INVALID_ARGUMENT);
3202
3203 named_entry_lock(named_entry);
3204
3205 if (named_entry->is_pager) {
3206 object = vm_object_enter(named_entry->backing.pager,
3207 named_entry->offset + named_entry->size,
3208 named_entry->internal,
3209 FALSE,
3210 FALSE);
3211 if (object == VM_OBJECT_NULL) {
3212 named_entry_unlock(named_entry);
3213 return(KERN_INVALID_OBJECT);
3214 }
3215
3216 /* JMM - drop reference on the pager here? */
3217
3218 /* create an extra reference for the object */
3219 vm_object_lock(object);
3220 vm_object_reference_locked(object);
3221 named_entry->backing.object = object;
3222 named_entry->is_pager = FALSE;
3223 named_entry_unlock(named_entry);
3224
3225 /* wait for object (if any) to be ready */
3226 if (!named_entry->internal) {
3227 while (!object->pager_ready) {
3228 vm_object_wait(object,
3229 VM_OBJECT_EVENT_PAGER_READY,
3230 THREAD_UNINT);
3231 vm_object_lock(object);
3232 }
3233 }
3234 vm_object_unlock(object);
3235
3236 } else {
3237 /* This is the case where we are going to operate */
3238 /* an an already known object. If the object is */
3239 /* not ready it is internal. An external */
3240 /* object cannot be mapped until it is ready */
3241 /* we can therefore avoid the ready check */
3242 /* in this case. */
3243 object = named_entry->backing.object;
3244 vm_object_reference(object);
3245 named_entry_unlock(named_entry);
3246 }
3247
3248 if (!object->private) {
3249 if (*upl_size > (MAX_UPL_TRANSFER*PAGE_SIZE))
3250 *upl_size = (MAX_UPL_TRANSFER*PAGE_SIZE);
3251 if (object->phys_contiguous) {
3252 *flags = UPL_PHYS_CONTIG;
3253 } else {
3254 *flags = 0;
3255 }
3256 } else {
3257 *flags = UPL_DEV_MEMORY | UPL_PHYS_CONTIG;
3258 }
3259
3260 ret = vm_object_iopl_request(object,
3261 offset,
3262 *upl_size,
3263 upl_ptr,
3264 user_page_list,
3265 page_list_count,
3266 caller_flags);
3267 vm_object_deallocate(object);
3268 return ret;
3269 }
mirror of XNU