]> git.saurik.com Git - apple/xnu.git/blob - osfmk/vm/vm_user.c
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-6153.121.1.tar.gz
[apple/xnu.git] / osfmk / vm / vm_user.c
1 /*
2 * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56 /*
57 */
58 /*
59 * File: vm/vm_user.c
60 * Author: Avadis Tevanian, Jr., Michael Wayne Young
61 *
62 * User-exported virtual memory functions.
63 */
64
65 /*
66 * There are three implementations of the "XXX_allocate" functionality in
67 * the kernel: mach_vm_allocate (for any task on the platform), vm_allocate
68 * (for a task with the same address space size, especially the current task),
69 * and vm32_vm_allocate (for the specific case of a 32-bit task). vm_allocate
70 * in the kernel should only be used on the kernel_task. vm32_vm_allocate only
71 * makes sense on platforms where a user task can either be 32 or 64, or the kernel
72 * task can be 32 or 64. mach_vm_allocate makes sense everywhere, and is preferred
73 * for new code.
74 *
75 * The entrypoints into the kernel are more complex. All platforms support a
76 * mach_vm_allocate-style API (subsystem 4800) which operates with the largest
77 * size types for the platform. On platforms that only support U32/K32,
78 * subsystem 4800 is all you need. On platforms that support both U32 and U64,
79 * subsystem 3800 is used disambiguate the size of parameters, and they will
80 * always be 32-bit and call into the vm32_vm_allocate APIs. On non-U32/K32 platforms,
81 * the MIG glue should never call into vm_allocate directly, because the calling
82 * task and kernel_task are unlikely to use the same size parameters
83 *
84 * New VM call implementations should be added here and to mach_vm.defs
85 * (subsystem 4800), and use mach_vm_* "wide" types.
86 */
87
88 #include <debug.h>
89
90 #include <vm_cpm.h>
91 #include <mach/boolean.h>
92 #include <mach/kern_return.h>
93 #include <mach/mach_types.h> /* to get vm_address_t */
94 #include <mach/memory_object.h>
95 #include <mach/std_types.h> /* to get pointer_t */
96 #include <mach/upl.h>
97 #include <mach/vm_attributes.h>
98 #include <mach/vm_param.h>
99 #include <mach/vm_statistics.h>
100 #include <mach/mach_syscalls.h>
101 #include <mach/sdt.h>
102
103 #include <mach/host_priv_server.h>
104 #include <mach/mach_vm_server.h>
105 #include <mach/memory_entry_server.h>
106 #include <mach/vm_map_server.h>
107
108 #include <kern/host.h>
109 #include <kern/kalloc.h>
110 #include <kern/task.h>
111 #include <kern/misc_protos.h>
112 #include <vm/vm_fault.h>
113 #include <vm/vm_map.h>
114 #include <vm/vm_object.h>
115 #include <vm/vm_page.h>
116 #include <vm/memory_object.h>
117 #include <vm/vm_pageout.h>
118 #include <vm/vm_protos.h>
119 #include <vm/vm_purgeable_internal.h>
120 #include <vm/vm_init.h>
121
122 #include <san/kasan.h>
123
124 #include <libkern/OSDebug.h>
125 #include <IOKit/IOBSD.h>
126
127 vm_size_t upl_offset_to_pagelist = 0;
128
129 #if VM_CPM
130 #include <vm/cpm.h>
131 #endif /* VM_CPM */
132
133 /*
134 * mach_vm_allocate allocates "zero fill" memory in the specfied
135 * map.
136 */
137 kern_return_t
138 mach_vm_allocate_external(
139 vm_map_t map,
140 mach_vm_offset_t *addr,
141 mach_vm_size_t size,
142 int flags)
143 {
144 vm_tag_t tag;
145
146 VM_GET_FLAGS_ALIAS(flags, tag);
147 return mach_vm_allocate_kernel(map, addr, size, flags, tag);
148 }
149
150 kern_return_t
151 mach_vm_allocate_kernel(
152 vm_map_t map,
153 mach_vm_offset_t *addr,
154 mach_vm_size_t size,
155 int flags,
156 vm_tag_t tag)
157 {
158 vm_map_offset_t map_addr;
159 vm_map_size_t map_size;
160 kern_return_t result;
161 boolean_t anywhere;
162
163 /* filter out any kernel-only flags */
164 if (flags & ~VM_FLAGS_USER_ALLOCATE) {
165 return KERN_INVALID_ARGUMENT;
166 }
167
168 if (map == VM_MAP_NULL) {
169 return KERN_INVALID_ARGUMENT;
170 }
171 if (size == 0) {
172 *addr = 0;
173 return KERN_SUCCESS;
174 }
175
176 anywhere = ((VM_FLAGS_ANYWHERE & flags) != 0);
177 if (anywhere) {
178 /*
179 * No specific address requested, so start candidate address
180 * search at the minimum address in the map. However, if that
181 * minimum is 0, bump it up by PAGE_SIZE. We want to limit
182 * allocations of PAGEZERO to explicit requests since its
183 * normal use is to catch dereferences of NULL and many
184 * applications also treat pointers with a value of 0 as
185 * special and suddenly having address 0 contain useable
186 * memory would tend to confuse those applications.
187 */
188 map_addr = vm_map_min(map);
189 if (map_addr == 0) {
190 map_addr += VM_MAP_PAGE_SIZE(map);
191 }
192 } else {
193 map_addr = vm_map_trunc_page(*addr,
194 VM_MAP_PAGE_MASK(map));
195 }
196 map_size = vm_map_round_page(size,
197 VM_MAP_PAGE_MASK(map));
198 if (map_size == 0) {
199 return KERN_INVALID_ARGUMENT;
200 }
201
202 result = vm_map_enter(
203 map,
204 &map_addr,
205 map_size,
206 (vm_map_offset_t)0,
207 flags,
208 VM_MAP_KERNEL_FLAGS_NONE,
209 tag,
210 VM_OBJECT_NULL,
211 (vm_object_offset_t)0,
212 FALSE,
213 VM_PROT_DEFAULT,
214 VM_PROT_ALL,
215 VM_INHERIT_DEFAULT);
216
217 *addr = map_addr;
218 return result;
219 }
220
221 /*
222 * vm_allocate
223 * Legacy routine that allocates "zero fill" memory in the specfied
224 * map (which is limited to the same size as the kernel).
225 */
226 kern_return_t
227 vm_allocate_external(
228 vm_map_t map,
229 vm_offset_t *addr,
230 vm_size_t size,
231 int flags)
232 {
233 vm_tag_t tag;
234
235 VM_GET_FLAGS_ALIAS(flags, tag);
236 return vm_allocate_kernel(map, addr, size, flags, tag);
237 }
238
239 kern_return_t
240 vm_allocate_kernel(
241 vm_map_t map,
242 vm_offset_t *addr,
243 vm_size_t size,
244 int flags,
245 vm_tag_t tag)
246 {
247 vm_map_offset_t map_addr;
248 vm_map_size_t map_size;
249 kern_return_t result;
250 boolean_t anywhere;
251
252 /* filter out any kernel-only flags */
253 if (flags & ~VM_FLAGS_USER_ALLOCATE) {
254 return KERN_INVALID_ARGUMENT;
255 }
256
257 if (map == VM_MAP_NULL) {
258 return KERN_INVALID_ARGUMENT;
259 }
260 if (size == 0) {
261 *addr = 0;
262 return KERN_SUCCESS;
263 }
264
265 anywhere = ((VM_FLAGS_ANYWHERE & flags) != 0);
266 if (anywhere) {
267 /*
268 * No specific address requested, so start candidate address
269 * search at the minimum address in the map. However, if that
270 * minimum is 0, bump it up by PAGE_SIZE. We want to limit
271 * allocations of PAGEZERO to explicit requests since its
272 * normal use is to catch dereferences of NULL and many
273 * applications also treat pointers with a value of 0 as
274 * special and suddenly having address 0 contain useable
275 * memory would tend to confuse those applications.
276 */
277 map_addr = vm_map_min(map);
278 if (map_addr == 0) {
279 map_addr += VM_MAP_PAGE_SIZE(map);
280 }
281 } else {
282 map_addr = vm_map_trunc_page(*addr,
283 VM_MAP_PAGE_MASK(map));
284 }
285 map_size = vm_map_round_page(size,
286 VM_MAP_PAGE_MASK(map));
287 if (map_size == 0) {
288 return KERN_INVALID_ARGUMENT;
289 }
290
291 result = vm_map_enter(
292 map,
293 &map_addr,
294 map_size,
295 (vm_map_offset_t)0,
296 flags,
297 VM_MAP_KERNEL_FLAGS_NONE,
298 tag,
299 VM_OBJECT_NULL,
300 (vm_object_offset_t)0,
301 FALSE,
302 VM_PROT_DEFAULT,
303 VM_PROT_ALL,
304 VM_INHERIT_DEFAULT);
305
306 #if KASAN
307 if (result == KERN_SUCCESS && map->pmap == kernel_pmap) {
308 kasan_notify_address(map_addr, map_size);
309 }
310 #endif
311
312 *addr = CAST_DOWN(vm_offset_t, map_addr);
313 return result;
314 }
315
316 /*
317 * mach_vm_deallocate -
318 * deallocates the specified range of addresses in the
319 * specified address map.
320 */
321 kern_return_t
322 mach_vm_deallocate(
323 vm_map_t map,
324 mach_vm_offset_t start,
325 mach_vm_size_t size)
326 {
327 if ((map == VM_MAP_NULL) || (start + size < start)) {
328 return KERN_INVALID_ARGUMENT;
329 }
330
331 if (size == (mach_vm_offset_t) 0) {
332 return KERN_SUCCESS;
333 }
334
335 return vm_map_remove(map,
336 vm_map_trunc_page(start,
337 VM_MAP_PAGE_MASK(map)),
338 vm_map_round_page(start + size,
339 VM_MAP_PAGE_MASK(map)),
340 VM_MAP_REMOVE_NO_FLAGS);
341 }
342
343 /*
344 * vm_deallocate -
345 * deallocates the specified range of addresses in the
346 * specified address map (limited to addresses the same
347 * size as the kernel).
348 */
349 kern_return_t
350 vm_deallocate(
351 vm_map_t map,
352 vm_offset_t start,
353 vm_size_t size)
354 {
355 if ((map == VM_MAP_NULL) || (start + size < start)) {
356 return KERN_INVALID_ARGUMENT;
357 }
358
359 if (size == (vm_offset_t) 0) {
360 return KERN_SUCCESS;
361 }
362
363 return vm_map_remove(map,
364 vm_map_trunc_page(start,
365 VM_MAP_PAGE_MASK(map)),
366 vm_map_round_page(start + size,
367 VM_MAP_PAGE_MASK(map)),
368 VM_MAP_REMOVE_NO_FLAGS);
369 }
370
371 /*
372 * mach_vm_inherit -
373 * Sets the inheritance of the specified range in the
374 * specified map.
375 */
376 kern_return_t
377 mach_vm_inherit(
378 vm_map_t map,
379 mach_vm_offset_t start,
380 mach_vm_size_t size,
381 vm_inherit_t new_inheritance)
382 {
383 if ((map == VM_MAP_NULL) || (start + size < start) ||
384 (new_inheritance > VM_INHERIT_LAST_VALID)) {
385 return KERN_INVALID_ARGUMENT;
386 }
387
388 if (size == 0) {
389 return KERN_SUCCESS;
390 }
391
392 return vm_map_inherit(map,
393 vm_map_trunc_page(start,
394 VM_MAP_PAGE_MASK(map)),
395 vm_map_round_page(start + size,
396 VM_MAP_PAGE_MASK(map)),
397 new_inheritance);
398 }
399
400 /*
401 * vm_inherit -
402 * Sets the inheritance of the specified range in the
403 * specified map (range limited to addresses
404 */
405 kern_return_t
406 vm_inherit(
407 vm_map_t map,
408 vm_offset_t start,
409 vm_size_t size,
410 vm_inherit_t new_inheritance)
411 {
412 if ((map == VM_MAP_NULL) || (start + size < start) ||
413 (new_inheritance > VM_INHERIT_LAST_VALID)) {
414 return KERN_INVALID_ARGUMENT;
415 }
416
417 if (size == 0) {
418 return KERN_SUCCESS;
419 }
420
421 return vm_map_inherit(map,
422 vm_map_trunc_page(start,
423 VM_MAP_PAGE_MASK(map)),
424 vm_map_round_page(start + size,
425 VM_MAP_PAGE_MASK(map)),
426 new_inheritance);
427 }
428
429 /*
430 * mach_vm_protect -
431 * Sets the protection of the specified range in the
432 * specified map.
433 */
434
435 kern_return_t
436 mach_vm_protect(
437 vm_map_t map,
438 mach_vm_offset_t start,
439 mach_vm_size_t size,
440 boolean_t set_maximum,
441 vm_prot_t new_protection)
442 {
443 if ((map == VM_MAP_NULL) || (start + size < start) ||
444 (new_protection & ~(VM_PROT_ALL | VM_PROT_COPY))) {
445 return KERN_INVALID_ARGUMENT;
446 }
447
448 if (size == 0) {
449 return KERN_SUCCESS;
450 }
451
452 return vm_map_protect(map,
453 vm_map_trunc_page(start,
454 VM_MAP_PAGE_MASK(map)),
455 vm_map_round_page(start + size,
456 VM_MAP_PAGE_MASK(map)),
457 new_protection,
458 set_maximum);
459 }
460
461 /*
462 * vm_protect -
463 * Sets the protection of the specified range in the
464 * specified map. Addressability of the range limited
465 * to the same size as the kernel.
466 */
467
468 kern_return_t
469 vm_protect(
470 vm_map_t map,
471 vm_offset_t start,
472 vm_size_t size,
473 boolean_t set_maximum,
474 vm_prot_t new_protection)
475 {
476 if ((map == VM_MAP_NULL) || (start + size < start) ||
477 (new_protection & ~(VM_PROT_ALL | VM_PROT_COPY))) {
478 return KERN_INVALID_ARGUMENT;
479 }
480
481 if (size == 0) {
482 return KERN_SUCCESS;
483 }
484
485 return vm_map_protect(map,
486 vm_map_trunc_page(start,
487 VM_MAP_PAGE_MASK(map)),
488 vm_map_round_page(start + size,
489 VM_MAP_PAGE_MASK(map)),
490 new_protection,
491 set_maximum);
492 }
493
494 /*
495 * mach_vm_machine_attributes -
496 * Handle machine-specific attributes for a mapping, such
497 * as cachability, migrability, etc.
498 */
499 kern_return_t
500 mach_vm_machine_attribute(
501 vm_map_t map,
502 mach_vm_address_t addr,
503 mach_vm_size_t size,
504 vm_machine_attribute_t attribute,
505 vm_machine_attribute_val_t* value) /* IN/OUT */
506 {
507 if ((map == VM_MAP_NULL) || (addr + size < addr)) {
508 return KERN_INVALID_ARGUMENT;
509 }
510
511 if (size == 0) {
512 return KERN_SUCCESS;
513 }
514
515 return vm_map_machine_attribute(
516 map,
517 vm_map_trunc_page(addr,
518 VM_MAP_PAGE_MASK(map)),
519 vm_map_round_page(addr + size,
520 VM_MAP_PAGE_MASK(map)),
521 attribute,
522 value);
523 }
524
525 /*
526 * vm_machine_attribute -
527 * Handle machine-specific attributes for a mapping, such
528 * as cachability, migrability, etc. Limited addressability
529 * (same range limits as for the native kernel map).
530 */
531 kern_return_t
532 vm_machine_attribute(
533 vm_map_t map,
534 vm_address_t addr,
535 vm_size_t size,
536 vm_machine_attribute_t attribute,
537 vm_machine_attribute_val_t* value) /* IN/OUT */
538 {
539 if ((map == VM_MAP_NULL) || (addr + size < addr)) {
540 return KERN_INVALID_ARGUMENT;
541 }
542
543 if (size == 0) {
544 return KERN_SUCCESS;
545 }
546
547 return vm_map_machine_attribute(
548 map,
549 vm_map_trunc_page(addr,
550 VM_MAP_PAGE_MASK(map)),
551 vm_map_round_page(addr + size,
552 VM_MAP_PAGE_MASK(map)),
553 attribute,
554 value);
555 }
556
557 /*
558 * mach_vm_read -
559 * Read/copy a range from one address space and return it to the caller.
560 *
561 * It is assumed that the address for the returned memory is selected by
562 * the IPC implementation as part of receiving the reply to this call.
563 * If IPC isn't used, the caller must deal with the vm_map_copy_t object
564 * that gets returned.
565 *
566 * JMM - because of mach_msg_type_number_t, this call is limited to a
567 * single 4GB region at this time.
568 *
569 */
570 kern_return_t
571 mach_vm_read(
572 vm_map_t map,
573 mach_vm_address_t addr,
574 mach_vm_size_t size,
575 pointer_t *data,
576 mach_msg_type_number_t *data_size)
577 {
578 kern_return_t error;
579 vm_map_copy_t ipc_address;
580
581 if (map == VM_MAP_NULL) {
582 return KERN_INVALID_ARGUMENT;
583 }
584
585 if ((mach_msg_type_number_t) size != size) {
586 return KERN_INVALID_ARGUMENT;
587 }
588
589 error = vm_map_copyin(map,
590 (vm_map_address_t)addr,
591 (vm_map_size_t)size,
592 FALSE, /* src_destroy */
593 &ipc_address);
594
595 if (KERN_SUCCESS == error) {
596 *data = (pointer_t) ipc_address;
597 *data_size = (mach_msg_type_number_t) size;
598 assert(*data_size == size);
599 }
600 return error;
601 }
602
603 /*
604 * vm_read -
605 * Read/copy a range from one address space and return it to the caller.
606 * Limited addressability (same range limits as for the native kernel map).
607 *
608 * It is assumed that the address for the returned memory is selected by
609 * the IPC implementation as part of receiving the reply to this call.
610 * If IPC isn't used, the caller must deal with the vm_map_copy_t object
611 * that gets returned.
612 */
613 kern_return_t
614 vm_read(
615 vm_map_t map,
616 vm_address_t addr,
617 vm_size_t size,
618 pointer_t *data,
619 mach_msg_type_number_t *data_size)
620 {
621 kern_return_t error;
622 vm_map_copy_t ipc_address;
623
624 if (map == VM_MAP_NULL) {
625 return KERN_INVALID_ARGUMENT;
626 }
627
628 mach_msg_type_number_t dsize;
629 if (os_convert_overflow(size, &dsize)) {
630 /*
631 * The kernel could handle a 64-bit "size" value, but
632 * it could not return the size of the data in "*data_size"
633 * without overflowing.
634 * Let's reject this "size" as invalid.
635 */
636 return KERN_INVALID_ARGUMENT;
637 }
638
639 error = vm_map_copyin(map,
640 (vm_map_address_t)addr,
641 (vm_map_size_t)size,
642 FALSE, /* src_destroy */
643 &ipc_address);
644
645 if (KERN_SUCCESS == error) {
646 *data = (pointer_t) ipc_address;
647 *data_size = dsize;
648 assert(*data_size == size);
649 }
650 return error;
651 }
652
653 /*
654 * mach_vm_read_list -
655 * Read/copy a list of address ranges from specified map.
656 *
657 * MIG does not know how to deal with a returned array of
658 * vm_map_copy_t structures, so we have to do the copyout
659 * manually here.
660 */
661 kern_return_t
662 mach_vm_read_list(
663 vm_map_t map,
664 mach_vm_read_entry_t data_list,
665 natural_t count)
666 {
667 mach_msg_type_number_t i;
668 kern_return_t error;
669 vm_map_copy_t copy;
670
671 if (map == VM_MAP_NULL ||
672 count > VM_MAP_ENTRY_MAX) {
673 return KERN_INVALID_ARGUMENT;
674 }
675
676 error = KERN_SUCCESS;
677 for (i = 0; i < count; i++) {
678 vm_map_address_t map_addr;
679 vm_map_size_t map_size;
680
681 map_addr = (vm_map_address_t)(data_list[i].address);
682 map_size = (vm_map_size_t)(data_list[i].size);
683
684 if (map_size != 0) {
685 error = vm_map_copyin(map,
686 map_addr,
687 map_size,
688 FALSE, /* src_destroy */
689 &copy);
690 if (KERN_SUCCESS == error) {
691 error = vm_map_copyout(
692 current_task()->map,
693 &map_addr,
694 copy);
695 if (KERN_SUCCESS == error) {
696 data_list[i].address = map_addr;
697 continue;
698 }
699 vm_map_copy_discard(copy);
700 }
701 }
702 data_list[i].address = (mach_vm_address_t)0;
703 data_list[i].size = (mach_vm_size_t)0;
704 }
705 return error;
706 }
707
708 /*
709 * vm_read_list -
710 * Read/copy a list of address ranges from specified map.
711 *
712 * MIG does not know how to deal with a returned array of
713 * vm_map_copy_t structures, so we have to do the copyout
714 * manually here.
715 *
716 * The source and destination ranges are limited to those
717 * that can be described with a vm_address_t (i.e. same
718 * size map as the kernel).
719 *
720 * JMM - If the result of the copyout is an address range
721 * that cannot be described with a vm_address_t (i.e. the
722 * caller had a larger address space but used this call
723 * anyway), it will result in a truncated address being
724 * returned (and a likely confused caller).
725 */
726
727 kern_return_t
728 vm_read_list(
729 vm_map_t map,
730 vm_read_entry_t data_list,
731 natural_t count)
732 {
733 mach_msg_type_number_t i;
734 kern_return_t error;
735 vm_map_copy_t copy;
736
737 if (map == VM_MAP_NULL ||
738 count > VM_MAP_ENTRY_MAX) {
739 return KERN_INVALID_ARGUMENT;
740 }
741
742 error = KERN_SUCCESS;
743 for (i = 0; i < count; i++) {
744 vm_map_address_t map_addr;
745 vm_map_size_t map_size;
746
747 map_addr = (vm_map_address_t)(data_list[i].address);
748 map_size = (vm_map_size_t)(data_list[i].size);
749
750 if (map_size != 0) {
751 error = vm_map_copyin(map,
752 map_addr,
753 map_size,
754 FALSE, /* src_destroy */
755 &copy);
756 if (KERN_SUCCESS == error) {
757 error = vm_map_copyout(current_task()->map,
758 &map_addr,
759 copy);
760 if (KERN_SUCCESS == error) {
761 data_list[i].address =
762 CAST_DOWN(vm_offset_t, map_addr);
763 continue;
764 }
765 vm_map_copy_discard(copy);
766 }
767 }
768 data_list[i].address = (mach_vm_address_t)0;
769 data_list[i].size = (mach_vm_size_t)0;
770 }
771 return error;
772 }
773
774 /*
775 * mach_vm_read_overwrite -
776 * Overwrite a range of the current map with data from the specified
777 * map/address range.
778 *
779 * In making an assumption that the current thread is local, it is
780 * no longer cluster-safe without a fully supportive local proxy
781 * thread/task (but we don't support cluster's anymore so this is moot).
782 */
783
784 kern_return_t
785 mach_vm_read_overwrite(
786 vm_map_t map,
787 mach_vm_address_t address,
788 mach_vm_size_t size,
789 mach_vm_address_t data,
790 mach_vm_size_t *data_size)
791 {
792 kern_return_t error;
793 vm_map_copy_t copy;
794
795 if (map == VM_MAP_NULL) {
796 return KERN_INVALID_ARGUMENT;
797 }
798
799 error = vm_map_copyin(map, (vm_map_address_t)address,
800 (vm_map_size_t)size, FALSE, &copy);
801
802 if (KERN_SUCCESS == error) {
803 error = vm_map_copy_overwrite(current_thread()->map,
804 (vm_map_address_t)data,
805 copy, FALSE);
806 if (KERN_SUCCESS == error) {
807 *data_size = size;
808 return error;
809 }
810 vm_map_copy_discard(copy);
811 }
812 return error;
813 }
814
815 /*
816 * vm_read_overwrite -
817 * Overwrite a range of the current map with data from the specified
818 * map/address range.
819 *
820 * This routine adds the additional limitation that the source and
821 * destination ranges must be describable with vm_address_t values
822 * (i.e. the same size address spaces as the kernel, or at least the
823 * the ranges are in that first portion of the respective address
824 * spaces).
825 */
826
827 kern_return_t
828 vm_read_overwrite(
829 vm_map_t map,
830 vm_address_t address,
831 vm_size_t size,
832 vm_address_t data,
833 vm_size_t *data_size)
834 {
835 kern_return_t error;
836 vm_map_copy_t copy;
837
838 if (map == VM_MAP_NULL) {
839 return KERN_INVALID_ARGUMENT;
840 }
841
842 error = vm_map_copyin(map, (vm_map_address_t)address,
843 (vm_map_size_t)size, FALSE, &copy);
844
845 if (KERN_SUCCESS == error) {
846 error = vm_map_copy_overwrite(current_thread()->map,
847 (vm_map_address_t)data,
848 copy, FALSE);
849 if (KERN_SUCCESS == error) {
850 *data_size = size;
851 return error;
852 }
853 vm_map_copy_discard(copy);
854 }
855 return error;
856 }
857
858
859 /*
860 * mach_vm_write -
861 * Overwrite the specified address range with the data provided
862 * (from the current map).
863 */
864 kern_return_t
865 mach_vm_write(
866 vm_map_t map,
867 mach_vm_address_t address,
868 pointer_t data,
869 __unused mach_msg_type_number_t size)
870 {
871 if (map == VM_MAP_NULL) {
872 return KERN_INVALID_ARGUMENT;
873 }
874
875 return vm_map_copy_overwrite(map, (vm_map_address_t)address,
876 (vm_map_copy_t) data, FALSE /* interruptible XXX */);
877 }
878
879 /*
880 * vm_write -
881 * Overwrite the specified address range with the data provided
882 * (from the current map).
883 *
884 * The addressability of the range of addresses to overwrite is
885 * limited bu the use of a vm_address_t (same size as kernel map).
886 * Either the target map is also small, or the range is in the
887 * low addresses within it.
888 */
889 kern_return_t
890 vm_write(
891 vm_map_t map,
892 vm_address_t address,
893 pointer_t data,
894 __unused mach_msg_type_number_t size)
895 {
896 if (map == VM_MAP_NULL) {
897 return KERN_INVALID_ARGUMENT;
898 }
899
900 return vm_map_copy_overwrite(map, (vm_map_address_t)address,
901 (vm_map_copy_t) data, FALSE /* interruptible XXX */);
902 }
903
904 /*
905 * mach_vm_copy -
906 * Overwrite one range of the specified map with the contents of
907 * another range within that same map (i.e. both address ranges
908 * are "over there").
909 */
910 kern_return_t
911 mach_vm_copy(
912 vm_map_t map,
913 mach_vm_address_t source_address,
914 mach_vm_size_t size,
915 mach_vm_address_t dest_address)
916 {
917 vm_map_copy_t copy;
918 kern_return_t kr;
919
920 if (map == VM_MAP_NULL) {
921 return KERN_INVALID_ARGUMENT;
922 }
923
924 kr = vm_map_copyin(map, (vm_map_address_t)source_address,
925 (vm_map_size_t)size, FALSE, &copy);
926
927 if (KERN_SUCCESS == kr) {
928 kr = vm_map_copy_overwrite(map,
929 (vm_map_address_t)dest_address,
930 copy, FALSE /* interruptible XXX */);
931
932 if (KERN_SUCCESS != kr) {
933 vm_map_copy_discard(copy);
934 }
935 }
936 return kr;
937 }
938
939 kern_return_t
940 vm_copy(
941 vm_map_t map,
942 vm_address_t source_address,
943 vm_size_t size,
944 vm_address_t dest_address)
945 {
946 vm_map_copy_t copy;
947 kern_return_t kr;
948
949 if (map == VM_MAP_NULL) {
950 return KERN_INVALID_ARGUMENT;
951 }
952
953 kr = vm_map_copyin(map, (vm_map_address_t)source_address,
954 (vm_map_size_t)size, FALSE, &copy);
955
956 if (KERN_SUCCESS == kr) {
957 kr = vm_map_copy_overwrite(map,
958 (vm_map_address_t)dest_address,
959 copy, FALSE /* interruptible XXX */);
960
961 if (KERN_SUCCESS != kr) {
962 vm_map_copy_discard(copy);
963 }
964 }
965 return kr;
966 }
967
968 /*
969 * mach_vm_map -
970 * Map some range of an object into an address space.
971 *
972 * The object can be one of several types of objects:
973 * NULL - anonymous memory
974 * a named entry - a range within another address space
975 * or a range within a memory object
976 * a whole memory object
977 *
978 */
979 kern_return_t
980 mach_vm_map_external(
981 vm_map_t target_map,
982 mach_vm_offset_t *address,
983 mach_vm_size_t initial_size,
984 mach_vm_offset_t mask,
985 int flags,
986 ipc_port_t port,
987 vm_object_offset_t offset,
988 boolean_t copy,
989 vm_prot_t cur_protection,
990 vm_prot_t max_protection,
991 vm_inherit_t inheritance)
992 {
993 vm_tag_t tag;
994
995 VM_GET_FLAGS_ALIAS(flags, tag);
996 return mach_vm_map_kernel(target_map, address, initial_size, mask,
997 flags, VM_MAP_KERNEL_FLAGS_NONE, tag,
998 port, offset, copy,
999 cur_protection, max_protection,
1000 inheritance);
1001 }
1002
1003 kern_return_t
1004 mach_vm_map_kernel(
1005 vm_map_t target_map,
1006 mach_vm_offset_t *address,
1007 mach_vm_size_t initial_size,
1008 mach_vm_offset_t mask,
1009 int flags,
1010 vm_map_kernel_flags_t vmk_flags,
1011 vm_tag_t tag,
1012 ipc_port_t port,
1013 vm_object_offset_t offset,
1014 boolean_t copy,
1015 vm_prot_t cur_protection,
1016 vm_prot_t max_protection,
1017 vm_inherit_t inheritance)
1018 {
1019 kern_return_t kr;
1020 vm_map_offset_t vmmaddr;
1021
1022 vmmaddr = (vm_map_offset_t) *address;
1023
1024 /* filter out any kernel-only flags */
1025 if (flags & ~VM_FLAGS_USER_MAP) {
1026 return KERN_INVALID_ARGUMENT;
1027 }
1028
1029 kr = vm_map_enter_mem_object(target_map,
1030 &vmmaddr,
1031 initial_size,
1032 mask,
1033 flags,
1034 vmk_flags,
1035 tag,
1036 port,
1037 offset,
1038 copy,
1039 cur_protection,
1040 max_protection,
1041 inheritance);
1042
1043 #if KASAN
1044 if (kr == KERN_SUCCESS && target_map->pmap == kernel_pmap) {
1045 kasan_notify_address(vmmaddr, initial_size);
1046 }
1047 #endif
1048
1049 *address = vmmaddr;
1050 return kr;
1051 }
1052
1053
1054 /* legacy interface */
1055 kern_return_t
1056 vm_map_64_external(
1057 vm_map_t target_map,
1058 vm_offset_t *address,
1059 vm_size_t size,
1060 vm_offset_t mask,
1061 int flags,
1062 ipc_port_t port,
1063 vm_object_offset_t offset,
1064 boolean_t copy,
1065 vm_prot_t cur_protection,
1066 vm_prot_t max_protection,
1067 vm_inherit_t inheritance)
1068 {
1069 vm_tag_t tag;
1070
1071 VM_GET_FLAGS_ALIAS(flags, tag);
1072 return vm_map_64_kernel(target_map, address, size, mask,
1073 flags, VM_MAP_KERNEL_FLAGS_NONE,
1074 tag, port, offset, copy,
1075 cur_protection, max_protection,
1076 inheritance);
1077 }
1078
1079 kern_return_t
1080 vm_map_64_kernel(
1081 vm_map_t target_map,
1082 vm_offset_t *address,
1083 vm_size_t size,
1084 vm_offset_t mask,
1085 int flags,
1086 vm_map_kernel_flags_t vmk_flags,
1087 vm_tag_t tag,
1088 ipc_port_t port,
1089 vm_object_offset_t offset,
1090 boolean_t copy,
1091 vm_prot_t cur_protection,
1092 vm_prot_t max_protection,
1093 vm_inherit_t inheritance)
1094 {
1095 mach_vm_address_t map_addr;
1096 mach_vm_size_t map_size;
1097 mach_vm_offset_t map_mask;
1098 kern_return_t kr;
1099
1100 map_addr = (mach_vm_address_t)*address;
1101 map_size = (mach_vm_size_t)size;
1102 map_mask = (mach_vm_offset_t)mask;
1103
1104 kr = mach_vm_map_kernel(target_map, &map_addr, map_size, map_mask,
1105 flags, vmk_flags, tag,
1106 port, offset, copy,
1107 cur_protection, max_protection, inheritance);
1108 *address = CAST_DOWN(vm_offset_t, map_addr);
1109 return kr;
1110 }
1111
1112 /* temporary, until world build */
1113 kern_return_t
1114 vm_map_external(
1115 vm_map_t target_map,
1116 vm_offset_t *address,
1117 vm_size_t size,
1118 vm_offset_t mask,
1119 int flags,
1120 ipc_port_t port,
1121 vm_offset_t offset,
1122 boolean_t copy,
1123 vm_prot_t cur_protection,
1124 vm_prot_t max_protection,
1125 vm_inherit_t inheritance)
1126 {
1127 vm_tag_t tag;
1128
1129 VM_GET_FLAGS_ALIAS(flags, tag);
1130 return vm_map_kernel(target_map, address, size, mask,
1131 flags, VM_MAP_KERNEL_FLAGS_NONE, tag,
1132 port, offset, copy,
1133 cur_protection, max_protection, inheritance);
1134 }
1135
1136 kern_return_t
1137 vm_map_kernel(
1138 vm_map_t target_map,
1139 vm_offset_t *address,
1140 vm_size_t size,
1141 vm_offset_t mask,
1142 int flags,
1143 vm_map_kernel_flags_t vmk_flags,
1144 vm_tag_t tag,
1145 ipc_port_t port,
1146 vm_offset_t offset,
1147 boolean_t copy,
1148 vm_prot_t cur_protection,
1149 vm_prot_t max_protection,
1150 vm_inherit_t inheritance)
1151 {
1152 mach_vm_address_t map_addr;
1153 mach_vm_size_t map_size;
1154 mach_vm_offset_t map_mask;
1155 vm_object_offset_t obj_offset;
1156 kern_return_t kr;
1157
1158 map_addr = (mach_vm_address_t)*address;
1159 map_size = (mach_vm_size_t)size;
1160 map_mask = (mach_vm_offset_t)mask;
1161 obj_offset = (vm_object_offset_t)offset;
1162
1163 kr = mach_vm_map_kernel(target_map, &map_addr, map_size, map_mask,
1164 flags, vmk_flags, tag,
1165 port, obj_offset, copy,
1166 cur_protection, max_protection, inheritance);
1167 *address = CAST_DOWN(vm_offset_t, map_addr);
1168 return kr;
1169 }
1170
1171 /*
1172 * mach_vm_remap -
1173 * Remap a range of memory from one task into another,
1174 * to another address range within the same task, or
1175 * over top of itself (with altered permissions and/or
1176 * as an in-place copy of itself).
1177 */
1178 kern_return_t
1179 mach_vm_remap_external(
1180 vm_map_t target_map,
1181 mach_vm_offset_t *address,
1182 mach_vm_size_t size,
1183 mach_vm_offset_t mask,
1184 int flags,
1185 vm_map_t src_map,
1186 mach_vm_offset_t memory_address,
1187 boolean_t copy,
1188 vm_prot_t *cur_protection,
1189 vm_prot_t *max_protection,
1190 vm_inherit_t inheritance)
1191 {
1192 vm_tag_t tag;
1193 VM_GET_FLAGS_ALIAS(flags, tag);
1194
1195 return mach_vm_remap_kernel(target_map, address, size, mask, flags, tag, src_map, memory_address,
1196 copy, cur_protection, max_protection, inheritance);
1197 }
1198
1199 kern_return_t
1200 mach_vm_remap_kernel(
1201 vm_map_t target_map,
1202 mach_vm_offset_t *address,
1203 mach_vm_size_t size,
1204 mach_vm_offset_t mask,
1205 int flags,
1206 vm_tag_t tag,
1207 vm_map_t src_map,
1208 mach_vm_offset_t memory_address,
1209 boolean_t copy,
1210 vm_prot_t *cur_protection,
1211 vm_prot_t *max_protection,
1212 vm_inherit_t inheritance)
1213 {
1214 vm_map_offset_t map_addr;
1215 kern_return_t kr;
1216
1217 if (VM_MAP_NULL == target_map || VM_MAP_NULL == src_map) {
1218 return KERN_INVALID_ARGUMENT;
1219 }
1220
1221 /* filter out any kernel-only flags */
1222 if (flags & ~VM_FLAGS_USER_REMAP) {
1223 return KERN_INVALID_ARGUMENT;
1224 }
1225
1226 map_addr = (vm_map_offset_t)*address;
1227
1228 kr = vm_map_remap(target_map,
1229 &map_addr,
1230 size,
1231 mask,
1232 flags,
1233 VM_MAP_KERNEL_FLAGS_NONE,
1234 tag,
1235 src_map,
1236 memory_address,
1237 copy,
1238 cur_protection,
1239 max_protection,
1240 inheritance);
1241 *address = map_addr;
1242 return kr;
1243 }
1244
1245 /*
1246 * vm_remap -
1247 * Remap a range of memory from one task into another,
1248 * to another address range within the same task, or
1249 * over top of itself (with altered permissions and/or
1250 * as an in-place copy of itself).
1251 *
1252 * The addressability of the source and target address
1253 * range is limited by the size of vm_address_t (in the
1254 * kernel context).
1255 */
1256 kern_return_t
1257 vm_remap_external(
1258 vm_map_t target_map,
1259 vm_offset_t *address,
1260 vm_size_t size,
1261 vm_offset_t mask,
1262 int flags,
1263 vm_map_t src_map,
1264 vm_offset_t memory_address,
1265 boolean_t copy,
1266 vm_prot_t *cur_protection,
1267 vm_prot_t *max_protection,
1268 vm_inherit_t inheritance)
1269 {
1270 vm_tag_t tag;
1271 VM_GET_FLAGS_ALIAS(flags, tag);
1272
1273 return vm_remap_kernel(target_map, address, size, mask, flags, tag, src_map,
1274 memory_address, copy, cur_protection, max_protection, inheritance);
1275 }
1276
1277 kern_return_t
1278 vm_remap_kernel(
1279 vm_map_t target_map,
1280 vm_offset_t *address,
1281 vm_size_t size,
1282 vm_offset_t mask,
1283 int flags,
1284 vm_tag_t tag,
1285 vm_map_t src_map,
1286 vm_offset_t memory_address,
1287 boolean_t copy,
1288 vm_prot_t *cur_protection,
1289 vm_prot_t *max_protection,
1290 vm_inherit_t inheritance)
1291 {
1292 vm_map_offset_t map_addr;
1293 kern_return_t kr;
1294
1295 if (VM_MAP_NULL == target_map || VM_MAP_NULL == src_map) {
1296 return KERN_INVALID_ARGUMENT;
1297 }
1298
1299 /* filter out any kernel-only flags */
1300 if (flags & ~VM_FLAGS_USER_REMAP) {
1301 return KERN_INVALID_ARGUMENT;
1302 }
1303
1304 map_addr = (vm_map_offset_t)*address;
1305
1306 kr = vm_map_remap(target_map,
1307 &map_addr,
1308 size,
1309 mask,
1310 flags,
1311 VM_MAP_KERNEL_FLAGS_NONE,
1312 tag,
1313 src_map,
1314 memory_address,
1315 copy,
1316 cur_protection,
1317 max_protection,
1318 inheritance);
1319 *address = CAST_DOWN(vm_offset_t, map_addr);
1320 return kr;
1321 }
1322
1323 /*
1324 * NOTE: these routine (and this file) will no longer require mach_host_server.h
1325 * when mach_vm_wire and vm_wire are changed to use ledgers.
1326 */
1327 #include <mach/mach_host_server.h>
1328 /*
1329 * mach_vm_wire
1330 * Specify that the range of the virtual address space
1331 * of the target task must not cause page faults for
1332 * the indicated accesses.
1333 *
1334 * [ To unwire the pages, specify VM_PROT_NONE. ]
1335 */
1336 kern_return_t
1337 mach_vm_wire_external(
1338 host_priv_t host_priv,
1339 vm_map_t map,
1340 mach_vm_offset_t start,
1341 mach_vm_size_t size,
1342 vm_prot_t access)
1343 {
1344 return mach_vm_wire_kernel(host_priv, map, start, size, access, VM_KERN_MEMORY_MLOCK);
1345 }
1346
1347 kern_return_t
1348 mach_vm_wire_kernel(
1349 host_priv_t host_priv,
1350 vm_map_t map,
1351 mach_vm_offset_t start,
1352 mach_vm_size_t size,
1353 vm_prot_t access,
1354 vm_tag_t tag)
1355 {
1356 kern_return_t rc;
1357
1358 if (host_priv == HOST_PRIV_NULL) {
1359 return KERN_INVALID_HOST;
1360 }
1361
1362 assert(host_priv == &realhost);
1363
1364 if (map == VM_MAP_NULL) {
1365 return KERN_INVALID_TASK;
1366 }
1367
1368 if (access & ~VM_PROT_ALL || (start + size < start)) {
1369 return KERN_INVALID_ARGUMENT;
1370 }
1371
1372 if (access != VM_PROT_NONE) {
1373 rc = vm_map_wire_kernel(map,
1374 vm_map_trunc_page(start,
1375 VM_MAP_PAGE_MASK(map)),
1376 vm_map_round_page(start + size,
1377 VM_MAP_PAGE_MASK(map)),
1378 access, tag,
1379 TRUE);
1380 } else {
1381 rc = vm_map_unwire(map,
1382 vm_map_trunc_page(start,
1383 VM_MAP_PAGE_MASK(map)),
1384 vm_map_round_page(start + size,
1385 VM_MAP_PAGE_MASK(map)),
1386 TRUE);
1387 }
1388 return rc;
1389 }
1390
1391 /*
1392 * vm_wire -
1393 * Specify that the range of the virtual address space
1394 * of the target task must not cause page faults for
1395 * the indicated accesses.
1396 *
1397 * [ To unwire the pages, specify VM_PROT_NONE. ]
1398 */
1399 kern_return_t
1400 vm_wire(
1401 host_priv_t host_priv,
1402 vm_map_t map,
1403 vm_offset_t start,
1404 vm_size_t size,
1405 vm_prot_t access)
1406 {
1407 kern_return_t rc;
1408
1409 if (host_priv == HOST_PRIV_NULL) {
1410 return KERN_INVALID_HOST;
1411 }
1412
1413 assert(host_priv == &realhost);
1414
1415 if (map == VM_MAP_NULL) {
1416 return KERN_INVALID_TASK;
1417 }
1418
1419 if ((access & ~VM_PROT_ALL) || (start + size < start)) {
1420 return KERN_INVALID_ARGUMENT;
1421 }
1422
1423 if (size == 0) {
1424 rc = KERN_SUCCESS;
1425 } else if (access != VM_PROT_NONE) {
1426 rc = vm_map_wire_kernel(map,
1427 vm_map_trunc_page(start,
1428 VM_MAP_PAGE_MASK(map)),
1429 vm_map_round_page(start + size,
1430 VM_MAP_PAGE_MASK(map)),
1431 access, VM_KERN_MEMORY_OSFMK,
1432 TRUE);
1433 } else {
1434 rc = vm_map_unwire(map,
1435 vm_map_trunc_page(start,
1436 VM_MAP_PAGE_MASK(map)),
1437 vm_map_round_page(start + size,
1438 VM_MAP_PAGE_MASK(map)),
1439 TRUE);
1440 }
1441 return rc;
1442 }
1443
1444 /*
1445 * vm_msync
1446 *
1447 * Synchronises the memory range specified with its backing store
1448 * image by either flushing or cleaning the contents to the appropriate
1449 * memory manager.
1450 *
1451 * interpretation of sync_flags
1452 * VM_SYNC_INVALIDATE - discard pages, only return precious
1453 * pages to manager.
1454 *
1455 * VM_SYNC_INVALIDATE & (VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS)
1456 * - discard pages, write dirty or precious
1457 * pages back to memory manager.
1458 *
1459 * VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS
1460 * - write dirty or precious pages back to
1461 * the memory manager.
1462 *
1463 * VM_SYNC_CONTIGUOUS - does everything normally, but if there
1464 * is a hole in the region, and we would
1465 * have returned KERN_SUCCESS, return
1466 * KERN_INVALID_ADDRESS instead.
1467 *
1468 * RETURNS
1469 * KERN_INVALID_TASK Bad task parameter
1470 * KERN_INVALID_ARGUMENT both sync and async were specified.
1471 * KERN_SUCCESS The usual.
1472 * KERN_INVALID_ADDRESS There was a hole in the region.
1473 */
1474
1475 kern_return_t
1476 mach_vm_msync(
1477 vm_map_t map,
1478 mach_vm_address_t address,
1479 mach_vm_size_t size,
1480 vm_sync_t sync_flags)
1481 {
1482 if (map == VM_MAP_NULL) {
1483 return KERN_INVALID_TASK;
1484 }
1485
1486 return vm_map_msync(map, (vm_map_address_t)address,
1487 (vm_map_size_t)size, sync_flags);
1488 }
1489
1490 /*
1491 * vm_msync
1492 *
1493 * Synchronises the memory range specified with its backing store
1494 * image by either flushing or cleaning the contents to the appropriate
1495 * memory manager.
1496 *
1497 * interpretation of sync_flags
1498 * VM_SYNC_INVALIDATE - discard pages, only return precious
1499 * pages to manager.
1500 *
1501 * VM_SYNC_INVALIDATE & (VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS)
1502 * - discard pages, write dirty or precious
1503 * pages back to memory manager.
1504 *
1505 * VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS
1506 * - write dirty or precious pages back to
1507 * the memory manager.
1508 *
1509 * VM_SYNC_CONTIGUOUS - does everything normally, but if there
1510 * is a hole in the region, and we would
1511 * have returned KERN_SUCCESS, return
1512 * KERN_INVALID_ADDRESS instead.
1513 *
1514 * The addressability of the range is limited to that which can
1515 * be described by a vm_address_t.
1516 *
1517 * RETURNS
1518 * KERN_INVALID_TASK Bad task parameter
1519 * KERN_INVALID_ARGUMENT both sync and async were specified.
1520 * KERN_SUCCESS The usual.
1521 * KERN_INVALID_ADDRESS There was a hole in the region.
1522 */
1523
1524 kern_return_t
1525 vm_msync(
1526 vm_map_t map,
1527 vm_address_t address,
1528 vm_size_t size,
1529 vm_sync_t sync_flags)
1530 {
1531 if (map == VM_MAP_NULL) {
1532 return KERN_INVALID_TASK;
1533 }
1534
1535 return vm_map_msync(map, (vm_map_address_t)address,
1536 (vm_map_size_t)size, sync_flags);
1537 }
1538
1539
1540 int
1541 vm_toggle_entry_reuse(int toggle, int *old_value)
1542 {
1543 vm_map_t map = current_map();
1544
1545 assert(!map->is_nested_map);
1546 if (toggle == VM_TOGGLE_GETVALUE && old_value != NULL) {
1547 *old_value = map->disable_vmentry_reuse;
1548 } else if (toggle == VM_TOGGLE_SET) {
1549 vm_map_entry_t map_to_entry;
1550
1551 vm_map_lock(map);
1552 vm_map_disable_hole_optimization(map);
1553 map->disable_vmentry_reuse = TRUE;
1554 __IGNORE_WCASTALIGN(map_to_entry = vm_map_to_entry(map));
1555 if (map->first_free == map_to_entry) {
1556 map->highest_entry_end = vm_map_min(map);
1557 } else {
1558 map->highest_entry_end = map->first_free->vme_end;
1559 }
1560 vm_map_unlock(map);
1561 } else if (toggle == VM_TOGGLE_CLEAR) {
1562 vm_map_lock(map);
1563 map->disable_vmentry_reuse = FALSE;
1564 vm_map_unlock(map);
1565 } else {
1566 return KERN_INVALID_ARGUMENT;
1567 }
1568
1569 return KERN_SUCCESS;
1570 }
1571
1572 /*
1573 * mach_vm_behavior_set
1574 *
1575 * Sets the paging behavior attribute for the specified range
1576 * in the specified map.
1577 *
1578 * This routine will fail with KERN_INVALID_ADDRESS if any address
1579 * in [start,start+size) is not a valid allocated memory region.
1580 */
1581 kern_return_t
1582 mach_vm_behavior_set(
1583 vm_map_t map,
1584 mach_vm_offset_t start,
1585 mach_vm_size_t size,
1586 vm_behavior_t new_behavior)
1587 {
1588 vm_map_offset_t align_mask;
1589
1590 if ((map == VM_MAP_NULL) || (start + size < start)) {
1591 return KERN_INVALID_ARGUMENT;
1592 }
1593
1594 if (size == 0) {
1595 return KERN_SUCCESS;
1596 }
1597
1598 switch (new_behavior) {
1599 case VM_BEHAVIOR_REUSABLE:
1600 case VM_BEHAVIOR_REUSE:
1601 case VM_BEHAVIOR_CAN_REUSE:
1602 /*
1603 * Align to the hardware page size, to allow
1604 * malloc() to maximize the amount of re-usability,
1605 * even on systems with larger software page size.
1606 */
1607 align_mask = PAGE_MASK;
1608 break;
1609 default:
1610 align_mask = VM_MAP_PAGE_MASK(map);
1611 break;
1612 }
1613
1614 return vm_map_behavior_set(map,
1615 vm_map_trunc_page(start, align_mask),
1616 vm_map_round_page(start + size, align_mask),
1617 new_behavior);
1618 }
1619
1620 /*
1621 * vm_behavior_set
1622 *
1623 * Sets the paging behavior attribute for the specified range
1624 * in the specified map.
1625 *
1626 * This routine will fail with KERN_INVALID_ADDRESS if any address
1627 * in [start,start+size) is not a valid allocated memory region.
1628 *
1629 * This routine is potentially limited in addressibility by the
1630 * use of vm_offset_t (if the map provided is larger than the
1631 * kernel's).
1632 */
1633 kern_return_t
1634 vm_behavior_set(
1635 vm_map_t map,
1636 vm_offset_t start,
1637 vm_size_t size,
1638 vm_behavior_t new_behavior)
1639 {
1640 if (start + size < start) {
1641 return KERN_INVALID_ARGUMENT;
1642 }
1643
1644 return mach_vm_behavior_set(map,
1645 (mach_vm_offset_t) start,
1646 (mach_vm_size_t) size,
1647 new_behavior);
1648 }
1649
1650 /*
1651 * mach_vm_region:
1652 *
1653 * User call to obtain information about a region in
1654 * a task's address map. Currently, only one flavor is
1655 * supported.
1656 *
1657 * XXX The reserved and behavior fields cannot be filled
1658 * in until the vm merge from the IK is completed, and
1659 * vm_reserve is implemented.
1660 *
1661 * XXX Dependency: syscall_vm_region() also supports only one flavor.
1662 */
1663
1664 kern_return_t
1665 mach_vm_region(
1666 vm_map_t map,
1667 mach_vm_offset_t *address, /* IN/OUT */
1668 mach_vm_size_t *size, /* OUT */
1669 vm_region_flavor_t flavor, /* IN */
1670 vm_region_info_t info, /* OUT */
1671 mach_msg_type_number_t *count, /* IN/OUT */
1672 mach_port_t *object_name) /* OUT */
1673 {
1674 vm_map_offset_t map_addr;
1675 vm_map_size_t map_size;
1676 kern_return_t kr;
1677
1678 if (VM_MAP_NULL == map) {
1679 return KERN_INVALID_ARGUMENT;
1680 }
1681
1682 map_addr = (vm_map_offset_t)*address;
1683 map_size = (vm_map_size_t)*size;
1684
1685 /* legacy conversion */
1686 if (VM_REGION_BASIC_INFO == flavor) {
1687 flavor = VM_REGION_BASIC_INFO_64;
1688 }
1689
1690 kr = vm_map_region(map,
1691 &map_addr, &map_size,
1692 flavor, info, count,
1693 object_name);
1694
1695 *address = map_addr;
1696 *size = map_size;
1697 return kr;
1698 }
1699
1700 /*
1701 * vm_region_64 and vm_region:
1702 *
1703 * User call to obtain information about a region in
1704 * a task's address map. Currently, only one flavor is
1705 * supported.
1706 *
1707 * XXX The reserved and behavior fields cannot be filled
1708 * in until the vm merge from the IK is completed, and
1709 * vm_reserve is implemented.
1710 *
1711 * XXX Dependency: syscall_vm_region() also supports only one flavor.
1712 */
1713
1714 kern_return_t
1715 vm_region_64(
1716 vm_map_t map,
1717 vm_offset_t *address, /* IN/OUT */
1718 vm_size_t *size, /* OUT */
1719 vm_region_flavor_t flavor, /* IN */
1720 vm_region_info_t info, /* OUT */
1721 mach_msg_type_number_t *count, /* IN/OUT */
1722 mach_port_t *object_name) /* OUT */
1723 {
1724 vm_map_offset_t map_addr;
1725 vm_map_size_t map_size;
1726 kern_return_t kr;
1727
1728 if (VM_MAP_NULL == map) {
1729 return KERN_INVALID_ARGUMENT;
1730 }
1731
1732 map_addr = (vm_map_offset_t)*address;
1733 map_size = (vm_map_size_t)*size;
1734
1735 /* legacy conversion */
1736 if (VM_REGION_BASIC_INFO == flavor) {
1737 flavor = VM_REGION_BASIC_INFO_64;
1738 }
1739
1740 kr = vm_map_region(map,
1741 &map_addr, &map_size,
1742 flavor, info, count,
1743 object_name);
1744
1745 *address = CAST_DOWN(vm_offset_t, map_addr);
1746 *size = CAST_DOWN(vm_size_t, map_size);
1747
1748 if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS) {
1749 return KERN_INVALID_ADDRESS;
1750 }
1751 return kr;
1752 }
1753
1754 kern_return_t
1755 vm_region(
1756 vm_map_t map,
1757 vm_address_t *address, /* IN/OUT */
1758 vm_size_t *size, /* OUT */
1759 vm_region_flavor_t flavor, /* IN */
1760 vm_region_info_t info, /* OUT */
1761 mach_msg_type_number_t *count, /* IN/OUT */
1762 mach_port_t *object_name) /* OUT */
1763 {
1764 vm_map_address_t map_addr;
1765 vm_map_size_t map_size;
1766 kern_return_t kr;
1767
1768 if (VM_MAP_NULL == map) {
1769 return KERN_INVALID_ARGUMENT;
1770 }
1771
1772 map_addr = (vm_map_address_t)*address;
1773 map_size = (vm_map_size_t)*size;
1774
1775 kr = vm_map_region(map,
1776 &map_addr, &map_size,
1777 flavor, info, count,
1778 object_name);
1779
1780 *address = CAST_DOWN(vm_address_t, map_addr);
1781 *size = CAST_DOWN(vm_size_t, map_size);
1782
1783 if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS) {
1784 return KERN_INVALID_ADDRESS;
1785 }
1786 return kr;
1787 }
1788
1789 /*
1790 * vm_region_recurse: A form of vm_region which follows the
1791 * submaps in a target map
1792 *
1793 */
1794 kern_return_t
1795 mach_vm_region_recurse(
1796 vm_map_t map,
1797 mach_vm_address_t *address,
1798 mach_vm_size_t *size,
1799 uint32_t *depth,
1800 vm_region_recurse_info_t info,
1801 mach_msg_type_number_t *infoCnt)
1802 {
1803 vm_map_address_t map_addr;
1804 vm_map_size_t map_size;
1805 kern_return_t kr;
1806
1807 if (VM_MAP_NULL == map) {
1808 return KERN_INVALID_ARGUMENT;
1809 }
1810
1811 map_addr = (vm_map_address_t)*address;
1812 map_size = (vm_map_size_t)*size;
1813
1814 kr = vm_map_region_recurse_64(
1815 map,
1816 &map_addr,
1817 &map_size,
1818 depth,
1819 (vm_region_submap_info_64_t)info,
1820 infoCnt);
1821
1822 *address = map_addr;
1823 *size = map_size;
1824 return kr;
1825 }
1826
1827 /*
1828 * vm_region_recurse: A form of vm_region which follows the
1829 * submaps in a target map
1830 *
1831 */
1832 kern_return_t
1833 vm_region_recurse_64(
1834 vm_map_t map,
1835 vm_address_t *address,
1836 vm_size_t *size,
1837 uint32_t *depth,
1838 vm_region_recurse_info_64_t info,
1839 mach_msg_type_number_t *infoCnt)
1840 {
1841 vm_map_address_t map_addr;
1842 vm_map_size_t map_size;
1843 kern_return_t kr;
1844
1845 if (VM_MAP_NULL == map) {
1846 return KERN_INVALID_ARGUMENT;
1847 }
1848
1849 map_addr = (vm_map_address_t)*address;
1850 map_size = (vm_map_size_t)*size;
1851
1852 kr = vm_map_region_recurse_64(
1853 map,
1854 &map_addr,
1855 &map_size,
1856 depth,
1857 (vm_region_submap_info_64_t)info,
1858 infoCnt);
1859
1860 *address = CAST_DOWN(vm_address_t, map_addr);
1861 *size = CAST_DOWN(vm_size_t, map_size);
1862
1863 if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS) {
1864 return KERN_INVALID_ADDRESS;
1865 }
1866 return kr;
1867 }
1868
1869 kern_return_t
1870 vm_region_recurse(
1871 vm_map_t map,
1872 vm_offset_t *address, /* IN/OUT */
1873 vm_size_t *size, /* OUT */
1874 natural_t *depth, /* IN/OUT */
1875 vm_region_recurse_info_t info32, /* IN/OUT */
1876 mach_msg_type_number_t *infoCnt) /* IN/OUT */
1877 {
1878 vm_region_submap_info_data_64_t info64;
1879 vm_region_submap_info_t info;
1880 vm_map_address_t map_addr;
1881 vm_map_size_t map_size;
1882 kern_return_t kr;
1883
1884 if (VM_MAP_NULL == map || *infoCnt < VM_REGION_SUBMAP_INFO_COUNT) {
1885 return KERN_INVALID_ARGUMENT;
1886 }
1887
1888
1889 map_addr = (vm_map_address_t)*address;
1890 map_size = (vm_map_size_t)*size;
1891 info = (vm_region_submap_info_t)info32;
1892 *infoCnt = VM_REGION_SUBMAP_INFO_COUNT_64;
1893
1894 kr = vm_map_region_recurse_64(map, &map_addr, &map_size,
1895 depth, &info64, infoCnt);
1896
1897 info->protection = info64.protection;
1898 info->max_protection = info64.max_protection;
1899 info->inheritance = info64.inheritance;
1900 info->offset = (uint32_t)info64.offset; /* trouble-maker */
1901 info->user_tag = info64.user_tag;
1902 info->pages_resident = info64.pages_resident;
1903 info->pages_shared_now_private = info64.pages_shared_now_private;
1904 info->pages_swapped_out = info64.pages_swapped_out;
1905 info->pages_dirtied = info64.pages_dirtied;
1906 info->ref_count = info64.ref_count;
1907 info->shadow_depth = info64.shadow_depth;
1908 info->external_pager = info64.external_pager;
1909 info->share_mode = info64.share_mode;
1910 info->is_submap = info64.is_submap;
1911 info->behavior = info64.behavior;
1912 info->object_id = info64.object_id;
1913 info->user_wired_count = info64.user_wired_count;
1914
1915 *address = CAST_DOWN(vm_address_t, map_addr);
1916 *size = CAST_DOWN(vm_size_t, map_size);
1917 *infoCnt = VM_REGION_SUBMAP_INFO_COUNT;
1918
1919 if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS) {
1920 return KERN_INVALID_ADDRESS;
1921 }
1922 return kr;
1923 }
1924
1925 kern_return_t
1926 mach_vm_purgable_control(
1927 vm_map_t map,
1928 mach_vm_offset_t address,
1929 vm_purgable_t control,
1930 int *state)
1931 {
1932 if (VM_MAP_NULL == map) {
1933 return KERN_INVALID_ARGUMENT;
1934 }
1935
1936 if (control == VM_PURGABLE_SET_STATE_FROM_KERNEL) {
1937 /* not allowed from user-space */
1938 return KERN_INVALID_ARGUMENT;
1939 }
1940
1941 return vm_map_purgable_control(map,
1942 vm_map_trunc_page(address, PAGE_MASK),
1943 control,
1944 state);
1945 }
1946
1947 kern_return_t
1948 vm_purgable_control(
1949 vm_map_t map,
1950 vm_offset_t address,
1951 vm_purgable_t control,
1952 int *state)
1953 {
1954 if (VM_MAP_NULL == map) {
1955 return KERN_INVALID_ARGUMENT;
1956 }
1957
1958 if (control == VM_PURGABLE_SET_STATE_FROM_KERNEL) {
1959 /* not allowed from user-space */
1960 return KERN_INVALID_ARGUMENT;
1961 }
1962
1963 return vm_map_purgable_control(map,
1964 vm_map_trunc_page(address, PAGE_MASK),
1965 control,
1966 state);
1967 }
1968
1969
1970 /*
1971 * Ordinarily, the right to allocate CPM is restricted
1972 * to privileged applications (those that can gain access
1973 * to the host priv port). Set this variable to zero if
1974 * you want to let any application allocate CPM.
1975 */
1976 unsigned int vm_allocate_cpm_privileged = 0;
1977
1978 /*
1979 * Allocate memory in the specified map, with the caveat that
1980 * the memory is physically contiguous. This call may fail
1981 * if the system can't find sufficient contiguous memory.
1982 * This call may cause or lead to heart-stopping amounts of
1983 * paging activity.
1984 *
1985 * Memory obtained from this call should be freed in the
1986 * normal way, viz., via vm_deallocate.
1987 */
1988 kern_return_t
1989 vm_allocate_cpm(
1990 host_priv_t host_priv,
1991 vm_map_t map,
1992 vm_address_t *addr,
1993 vm_size_t size,
1994 int flags)
1995 {
1996 vm_map_address_t map_addr;
1997 vm_map_size_t map_size;
1998 kern_return_t kr;
1999
2000 if (vm_allocate_cpm_privileged && HOST_PRIV_NULL == host_priv) {
2001 return KERN_INVALID_HOST;
2002 }
2003
2004 if (VM_MAP_NULL == map) {
2005 return KERN_INVALID_ARGUMENT;
2006 }
2007
2008 map_addr = (vm_map_address_t)*addr;
2009 map_size = (vm_map_size_t)size;
2010
2011 kr = vm_map_enter_cpm(map,
2012 &map_addr,
2013 map_size,
2014 flags);
2015
2016 *addr = CAST_DOWN(vm_address_t, map_addr);
2017 return kr;
2018 }
2019
2020
2021 kern_return_t
2022 mach_vm_page_query(
2023 vm_map_t map,
2024 mach_vm_offset_t offset,
2025 int *disposition,
2026 int *ref_count)
2027 {
2028 if (VM_MAP_NULL == map) {
2029 return KERN_INVALID_ARGUMENT;
2030 }
2031
2032 return vm_map_page_query_internal(
2033 map,
2034 vm_map_trunc_page(offset, PAGE_MASK),
2035 disposition, ref_count);
2036 }
2037
2038 kern_return_t
2039 vm_map_page_query(
2040 vm_map_t map,
2041 vm_offset_t offset,
2042 int *disposition,
2043 int *ref_count)
2044 {
2045 if (VM_MAP_NULL == map) {
2046 return KERN_INVALID_ARGUMENT;
2047 }
2048
2049 return vm_map_page_query_internal(
2050 map,
2051 vm_map_trunc_page(offset, PAGE_MASK),
2052 disposition, ref_count);
2053 }
2054
2055 kern_return_t
2056 mach_vm_page_range_query(
2057 vm_map_t map,
2058 mach_vm_offset_t address,
2059 mach_vm_size_t size,
2060 mach_vm_address_t dispositions_addr,
2061 mach_vm_size_t *dispositions_count)
2062 {
2063 kern_return_t kr = KERN_SUCCESS;
2064 int num_pages = 0, i = 0;
2065 mach_vm_size_t curr_sz = 0, copy_sz = 0;
2066 mach_vm_size_t disp_buf_req_size = 0, disp_buf_total_size = 0;
2067 mach_msg_type_number_t count = 0;
2068
2069 void *info = NULL;
2070 void *local_disp = NULL;;
2071 vm_map_size_t info_size = 0, local_disp_size = 0;
2072 mach_vm_offset_t start = 0, end = 0;
2073
2074 if (map == VM_MAP_NULL || dispositions_count == NULL) {
2075 return KERN_INVALID_ARGUMENT;
2076 }
2077
2078 disp_buf_req_size = (*dispositions_count * sizeof(int));
2079 start = mach_vm_trunc_page(address);
2080 end = mach_vm_round_page(address + size);
2081
2082 if (end < start) {
2083 return KERN_INVALID_ARGUMENT;
2084 }
2085
2086 if ((end - start) < size) {
2087 /*
2088 * Aligned size is less than unaligned size.
2089 */
2090 return KERN_INVALID_ARGUMENT;
2091 }
2092
2093 if (disp_buf_req_size == 0 || (end == start)) {
2094 return KERN_SUCCESS;
2095 }
2096
2097 /*
2098 * For large requests, we will go through them
2099 * MAX_PAGE_RANGE_QUERY chunk at a time.
2100 */
2101
2102 curr_sz = MIN(end - start, MAX_PAGE_RANGE_QUERY);
2103 num_pages = (int) (curr_sz >> PAGE_SHIFT);
2104
2105 info_size = num_pages * sizeof(vm_page_info_basic_data_t);
2106 info = kalloc(info_size);
2107
2108 if (info == NULL) {
2109 return KERN_RESOURCE_SHORTAGE;
2110 }
2111
2112 local_disp_size = num_pages * sizeof(int);
2113 local_disp = kalloc(local_disp_size);
2114
2115 if (local_disp == NULL) {
2116 kfree(info, info_size);
2117 info = NULL;
2118 return KERN_RESOURCE_SHORTAGE;
2119 }
2120
2121 while (size) {
2122 count = VM_PAGE_INFO_BASIC_COUNT;
2123 kr = vm_map_page_range_info_internal(
2124 map,
2125 start,
2126 mach_vm_round_page(start + curr_sz),
2127 VM_PAGE_INFO_BASIC,
2128 (vm_page_info_t) info,
2129 &count);
2130
2131 assert(kr == KERN_SUCCESS);
2132
2133 for (i = 0; i < num_pages; i++) {
2134 ((int*)local_disp)[i] = ((vm_page_info_basic_t)info)[i].disposition;
2135 }
2136
2137 copy_sz = MIN(disp_buf_req_size, num_pages * sizeof(int) /* an int per page */);
2138 kr = copyout(local_disp, (mach_vm_address_t)dispositions_addr, copy_sz);
2139
2140 start += curr_sz;
2141 disp_buf_req_size -= copy_sz;
2142 disp_buf_total_size += copy_sz;
2143
2144 if (kr != 0) {
2145 break;
2146 }
2147
2148 if ((disp_buf_req_size == 0) || (curr_sz >= size)) {
2149 /*
2150 * We might have inspected the full range OR
2151 * more than it esp. if the user passed in
2152 * non-page aligned start/size and/or if we
2153 * descended into a submap. We are done here.
2154 */
2155
2156 size = 0;
2157 } else {
2158 dispositions_addr += copy_sz;
2159
2160 size -= curr_sz;
2161
2162 curr_sz = MIN(mach_vm_round_page(size), MAX_PAGE_RANGE_QUERY);
2163 num_pages = (int)(curr_sz >> PAGE_SHIFT);
2164 }
2165 }
2166
2167 *dispositions_count = disp_buf_total_size / sizeof(int);
2168
2169 kfree(local_disp, local_disp_size);
2170 local_disp = NULL;
2171
2172 kfree(info, info_size);
2173 info = NULL;
2174
2175 return kr;
2176 }
2177
2178 kern_return_t
2179 mach_vm_page_info(
2180 vm_map_t map,
2181 mach_vm_address_t address,
2182 vm_page_info_flavor_t flavor,
2183 vm_page_info_t info,
2184 mach_msg_type_number_t *count)
2185 {
2186 kern_return_t kr;
2187
2188 if (map == VM_MAP_NULL) {
2189 return KERN_INVALID_ARGUMENT;
2190 }
2191
2192 kr = vm_map_page_info(map, address, flavor, info, count);
2193 return kr;
2194 }
2195
2196 /* map a (whole) upl into an address space */
2197 kern_return_t
2198 vm_upl_map(
2199 vm_map_t map,
2200 upl_t upl,
2201 vm_address_t *dst_addr)
2202 {
2203 vm_map_offset_t map_addr;
2204 kern_return_t kr;
2205
2206 if (VM_MAP_NULL == map) {
2207 return KERN_INVALID_ARGUMENT;
2208 }
2209
2210 kr = vm_map_enter_upl(map, upl, &map_addr);
2211 *dst_addr = CAST_DOWN(vm_address_t, map_addr);
2212 return kr;
2213 }
2214
2215 kern_return_t
2216 vm_upl_unmap(
2217 vm_map_t map,
2218 upl_t upl)
2219 {
2220 if (VM_MAP_NULL == map) {
2221 return KERN_INVALID_ARGUMENT;
2222 }
2223
2224 return vm_map_remove_upl(map, upl);
2225 }
2226
2227 /* Retrieve a upl for an object underlying an address range in a map */
2228
2229 kern_return_t
2230 vm_map_get_upl(
2231 vm_map_t map,
2232 vm_map_offset_t map_offset,
2233 upl_size_t *upl_size,
2234 upl_t *upl,
2235 upl_page_info_array_t page_list,
2236 unsigned int *count,
2237 upl_control_flags_t *flags,
2238 vm_tag_t tag,
2239 int force_data_sync)
2240 {
2241 upl_control_flags_t map_flags;
2242 kern_return_t kr;
2243
2244 if (VM_MAP_NULL == map) {
2245 return KERN_INVALID_ARGUMENT;
2246 }
2247
2248 map_flags = *flags & ~UPL_NOZEROFILL;
2249 if (force_data_sync) {
2250 map_flags |= UPL_FORCE_DATA_SYNC;
2251 }
2252
2253 kr = vm_map_create_upl(map,
2254 map_offset,
2255 upl_size,
2256 upl,
2257 page_list,
2258 count,
2259 &map_flags,
2260 tag);
2261
2262 *flags = (map_flags & ~UPL_FORCE_DATA_SYNC);
2263 return kr;
2264 }
2265
2266 #if CONFIG_EMBEDDED
2267 extern int proc_selfpid(void);
2268 extern char *proc_name_address(void *p);
2269 int cs_executable_mem_entry = 0;
2270 int log_executable_mem_entry = 0;
2271 #endif /* CONFIG_EMBEDDED */
2272
2273 /*
2274 * mach_make_memory_entry_64
2275 *
2276 * Think of it as a two-stage vm_remap() operation. First
2277 * you get a handle. Second, you get map that handle in
2278 * somewhere else. Rather than doing it all at once (and
2279 * without needing access to the other whole map).
2280 */
2281 kern_return_t
2282 mach_make_memory_entry_64(
2283 vm_map_t target_map,
2284 memory_object_size_t *size,
2285 memory_object_offset_t offset,
2286 vm_prot_t permission,
2287 ipc_port_t *object_handle,
2288 ipc_port_t parent_handle)
2289 {
2290 vm_named_entry_kernel_flags_t vmne_kflags;
2291
2292 if ((permission & MAP_MEM_FLAGS_MASK) & ~MAP_MEM_FLAGS_USER) {
2293 /*
2294 * Unknown flag: reject for forward compatibility.
2295 */
2296 return KERN_INVALID_VALUE;
2297 }
2298
2299 vmne_kflags = VM_NAMED_ENTRY_KERNEL_FLAGS_NONE;
2300 if (permission & MAP_MEM_LEDGER_TAGGED) {
2301 vmne_kflags.vmnekf_ledger_tag = VM_LEDGER_TAG_DEFAULT;
2302 }
2303 return mach_make_memory_entry_internal(target_map,
2304 size,
2305 offset,
2306 permission,
2307 vmne_kflags,
2308 object_handle,
2309 parent_handle);
2310 }
2311
2312 kern_return_t
2313 mach_make_memory_entry_internal(
2314 vm_map_t target_map,
2315 memory_object_size_t *size,
2316 memory_object_offset_t offset,
2317 vm_prot_t permission,
2318 vm_named_entry_kernel_flags_t vmne_kflags,
2319 ipc_port_t *object_handle,
2320 ipc_port_t parent_handle)
2321 {
2322 vm_map_version_t version;
2323 vm_named_entry_t parent_entry;
2324 vm_named_entry_t user_entry;
2325 ipc_port_t user_handle;
2326 kern_return_t kr;
2327 vm_map_t real_map;
2328
2329 /* needed for call to vm_map_lookup_locked */
2330 boolean_t wired;
2331 boolean_t iskernel;
2332 vm_object_offset_t obj_off;
2333 vm_prot_t prot;
2334 struct vm_object_fault_info fault_info = {};
2335 vm_object_t object;
2336 vm_object_t shadow_object;
2337
2338 /* needed for direct map entry manipulation */
2339 vm_map_entry_t map_entry;
2340 vm_map_entry_t next_entry;
2341 vm_map_t local_map;
2342 vm_map_t original_map = target_map;
2343 vm_map_size_t total_size, map_size;
2344 vm_map_offset_t map_start, map_end;
2345 vm_map_offset_t local_offset;
2346 vm_object_size_t mappable_size;
2347
2348 /*
2349 * Stash the offset in the page for use by vm_map_enter_mem_object()
2350 * in the VM_FLAGS_RETURN_DATA_ADDR/MAP_MEM_USE_DATA_ADDR case.
2351 */
2352 vm_object_offset_t offset_in_page;
2353
2354 unsigned int access;
2355 vm_prot_t protections;
2356 vm_prot_t original_protections, mask_protections;
2357 unsigned int wimg_mode;
2358
2359 boolean_t force_shadow = FALSE;
2360 boolean_t use_data_addr;
2361 boolean_t use_4K_compat;
2362 #if VM_NAMED_ENTRY_LIST
2363 int alias = -1;
2364 #endif /* VM_NAMED_ENTRY_LIST */
2365
2366 if ((permission & MAP_MEM_FLAGS_MASK) & ~MAP_MEM_FLAGS_ALL) {
2367 /*
2368 * Unknown flag: reject for forward compatibility.
2369 */
2370 return KERN_INVALID_VALUE;
2371 }
2372
2373 if (IP_VALID(parent_handle) &&
2374 ip_kotype(parent_handle) == IKOT_NAMED_ENTRY) {
2375 parent_entry = (vm_named_entry_t) ip_get_kobject(parent_handle);
2376 } else {
2377 parent_entry = NULL;
2378 }
2379
2380 if (parent_entry && parent_entry->is_copy) {
2381 return KERN_INVALID_ARGUMENT;
2382 }
2383
2384 original_protections = permission & VM_PROT_ALL;
2385 protections = original_protections;
2386 mask_protections = permission & VM_PROT_IS_MASK;
2387 access = GET_MAP_MEM(permission);
2388 use_data_addr = ((permission & MAP_MEM_USE_DATA_ADDR) != 0);
2389 use_4K_compat = ((permission & MAP_MEM_4K_DATA_ADDR) != 0);
2390
2391 user_handle = IP_NULL;
2392 user_entry = NULL;
2393
2394 map_start = vm_map_trunc_page(offset, PAGE_MASK);
2395
2396 if (permission & MAP_MEM_ONLY) {
2397 boolean_t parent_is_object;
2398
2399 map_end = vm_map_round_page(offset + *size, PAGE_MASK);
2400 map_size = map_end - map_start;
2401
2402 if (use_data_addr || use_4K_compat || parent_entry == NULL) {
2403 return KERN_INVALID_ARGUMENT;
2404 }
2405
2406 parent_is_object = !parent_entry->is_sub_map;
2407 object = parent_entry->backing.object;
2408 if (parent_is_object && object != VM_OBJECT_NULL) {
2409 wimg_mode = object->wimg_bits;
2410 } else {
2411 wimg_mode = VM_WIMG_USE_DEFAULT;
2412 }
2413 if ((access != GET_MAP_MEM(parent_entry->protection)) &&
2414 !(parent_entry->protection & VM_PROT_WRITE)) {
2415 return KERN_INVALID_RIGHT;
2416 }
2417 vm_prot_to_wimg(access, &wimg_mode);
2418 if (access != MAP_MEM_NOOP) {
2419 SET_MAP_MEM(access, parent_entry->protection);
2420 }
2421 if (parent_is_object && object &&
2422 (access != MAP_MEM_NOOP) &&
2423 (!(object->nophyscache))) {
2424 if (object->wimg_bits != wimg_mode) {
2425 vm_object_lock(object);
2426 vm_object_change_wimg_mode(object, wimg_mode);
2427 vm_object_unlock(object);
2428 }
2429 }
2430 if (object_handle) {
2431 *object_handle = IP_NULL;
2432 }
2433 return KERN_SUCCESS;
2434 } else if (permission & MAP_MEM_NAMED_CREATE) {
2435 int ledger_flags = 0;
2436 task_t owner;
2437
2438 map_end = vm_map_round_page(offset + *size, PAGE_MASK);
2439 map_size = map_end - map_start;
2440
2441 if (use_data_addr || use_4K_compat) {
2442 return KERN_INVALID_ARGUMENT;
2443 }
2444
2445 kr = mach_memory_entry_allocate(&user_entry, &user_handle);
2446 if (kr != KERN_SUCCESS) {
2447 return KERN_FAILURE;
2448 }
2449
2450 /*
2451 * Force the creation of the VM object now.
2452 */
2453 if (map_size > (vm_map_size_t) ANON_MAX_SIZE) {
2454 /*
2455 * LP64todo - for now, we can only allocate 4GB-4096
2456 * internal objects because the default pager can't
2457 * page bigger ones. Remove this when it can.
2458 */
2459 kr = KERN_FAILURE;
2460 goto make_mem_done;
2461 }
2462
2463 object = vm_object_allocate(map_size);
2464 assert(object != VM_OBJECT_NULL);
2465
2466 /*
2467 * XXX
2468 * We use this path when we want to make sure that
2469 * nobody messes with the object (coalesce, for
2470 * example) before we map it.
2471 * We might want to use these objects for transposition via
2472 * vm_object_transpose() too, so we don't want any copy or
2473 * shadow objects either...
2474 */
2475 object->copy_strategy = MEMORY_OBJECT_COPY_NONE;
2476 object->true_share = TRUE;
2477
2478 owner = current_task();
2479 if ((permission & MAP_MEM_PURGABLE) ||
2480 vmne_kflags.vmnekf_ledger_tag) {
2481 assert(object->vo_owner == NULL);
2482 assert(object->resident_page_count == 0);
2483 assert(object->wired_page_count == 0);
2484 assert(owner != TASK_NULL);
2485 if (vmne_kflags.vmnekf_ledger_no_footprint) {
2486 ledger_flags |= VM_LEDGER_FLAG_NO_FOOTPRINT;
2487 object->vo_no_footprint = TRUE;
2488 }
2489 if (permission & MAP_MEM_PURGABLE) {
2490 if (!(permission & VM_PROT_WRITE)) {
2491 /* if we can't write, we can't purge */
2492 vm_object_deallocate(object);
2493 kr = KERN_INVALID_ARGUMENT;
2494 goto make_mem_done;
2495 }
2496 object->purgable = VM_PURGABLE_NONVOLATILE;
2497 if (permission & MAP_MEM_PURGABLE_KERNEL_ONLY) {
2498 object->purgeable_only_by_kernel = TRUE;
2499 }
2500 #if __arm64__
2501 if (owner->task_legacy_footprint) {
2502 /*
2503 * For ios11, we failed to account for
2504 * this memory. Keep doing that for
2505 * legacy apps (built before ios12),
2506 * for backwards compatibility's sake...
2507 */
2508 owner = kernel_task;
2509 }
2510 #endif /* __arm64__ */
2511 vm_object_lock(object);
2512 vm_purgeable_nonvolatile_enqueue(object, owner);
2513 vm_object_unlock(object);
2514 }
2515 }
2516
2517 if (vmne_kflags.vmnekf_ledger_tag) {
2518 /*
2519 * Bill this object to the current task's
2520 * ledgers for the given tag.
2521 */
2522 if (vmne_kflags.vmnekf_ledger_no_footprint) {
2523 ledger_flags |= VM_LEDGER_FLAG_NO_FOOTPRINT;
2524 }
2525 vm_object_lock(object);
2526 object->vo_ledger_tag = vmne_kflags.vmnekf_ledger_tag;
2527 kr = vm_object_ownership_change(
2528 object,
2529 vmne_kflags.vmnekf_ledger_tag,
2530 owner, /* new owner */
2531 ledger_flags,
2532 FALSE); /* task_objq locked? */
2533 vm_object_unlock(object);
2534 if (kr != KERN_SUCCESS) {
2535 vm_object_deallocate(object);
2536 goto make_mem_done;
2537 }
2538 }
2539
2540 #if CONFIG_SECLUDED_MEMORY
2541 if (secluded_for_iokit && /* global boot-arg */
2542 ((permission & MAP_MEM_GRAB_SECLUDED)
2543 #if 11
2544 /* XXX FBDP for my testing only */
2545 || (secluded_for_fbdp && map_size == 97550336)
2546 #endif
2547 )) {
2548 #if 11
2549 if (!(permission & MAP_MEM_GRAB_SECLUDED) &&
2550 secluded_for_fbdp) {
2551 printf("FBDP: object %p size %lld can grab secluded\n", object, (uint64_t) map_size);
2552 }
2553 #endif
2554 object->can_grab_secluded = TRUE;
2555 assert(!object->eligible_for_secluded);
2556 }
2557 #endif /* CONFIG_SECLUDED_MEMORY */
2558
2559 /*
2560 * The VM object is brand new and nobody else knows about it,
2561 * so we don't need to lock it.
2562 */
2563
2564 wimg_mode = object->wimg_bits;
2565 vm_prot_to_wimg(access, &wimg_mode);
2566 if (access != MAP_MEM_NOOP) {
2567 object->wimg_bits = wimg_mode;
2568 }
2569
2570 /* the object has no pages, so no WIMG bits to update here */
2571
2572 user_entry->backing.object = object;
2573 user_entry->internal = TRUE;
2574 user_entry->is_sub_map = FALSE;
2575 user_entry->offset = 0;
2576 user_entry->data_offset = 0;
2577 user_entry->protection = protections;
2578 SET_MAP_MEM(access, user_entry->protection);
2579 user_entry->size = map_size;
2580
2581 /* user_object pager and internal fields are not used */
2582 /* when the object field is filled in. */
2583
2584 *size = CAST_DOWN(vm_size_t, (user_entry->size -
2585 user_entry->data_offset));
2586 *object_handle = user_handle;
2587 return KERN_SUCCESS;
2588 }
2589
2590 if (permission & MAP_MEM_VM_COPY) {
2591 vm_map_copy_t copy;
2592
2593 if (target_map == VM_MAP_NULL) {
2594 return KERN_INVALID_TASK;
2595 }
2596
2597 map_end = vm_map_round_page(offset + *size, PAGE_MASK);
2598 map_size = map_end - map_start;
2599 if (use_data_addr || use_4K_compat) {
2600 offset_in_page = offset - map_start;
2601 if (use_4K_compat) {
2602 offset_in_page &= ~((signed)(0xFFF));
2603 }
2604 } else {
2605 offset_in_page = 0;
2606 }
2607
2608 kr = vm_map_copyin_internal(target_map,
2609 map_start,
2610 map_size,
2611 VM_MAP_COPYIN_ENTRY_LIST,
2612 &copy);
2613 if (kr != KERN_SUCCESS) {
2614 return kr;
2615 }
2616
2617 kr = mach_memory_entry_allocate(&user_entry, &user_handle);
2618 if (kr != KERN_SUCCESS) {
2619 vm_map_copy_discard(copy);
2620 return KERN_FAILURE;
2621 }
2622
2623 user_entry->backing.copy = copy;
2624 user_entry->internal = FALSE;
2625 user_entry->is_sub_map = FALSE;
2626 user_entry->is_copy = TRUE;
2627 user_entry->offset = 0;
2628 user_entry->protection = protections;
2629 user_entry->size = map_size;
2630 user_entry->data_offset = offset_in_page;
2631
2632 *size = CAST_DOWN(vm_size_t, (user_entry->size -
2633 user_entry->data_offset));
2634 *object_handle = user_handle;
2635 return KERN_SUCCESS;
2636 }
2637
2638 if (permission & MAP_MEM_VM_SHARE) {
2639 vm_map_copy_t copy;
2640 vm_prot_t cur_prot, max_prot;
2641
2642 if (target_map == VM_MAP_NULL) {
2643 return KERN_INVALID_TASK;
2644 }
2645
2646 map_end = vm_map_round_page(offset + *size, PAGE_MASK);
2647 map_size = map_end - map_start;
2648 if (use_data_addr || use_4K_compat) {
2649 offset_in_page = offset - map_start;
2650 if (use_4K_compat) {
2651 offset_in_page &= ~((signed)(0xFFF));
2652 }
2653 } else {
2654 offset_in_page = 0;
2655 }
2656
2657 cur_prot = VM_PROT_ALL;
2658 kr = vm_map_copy_extract(target_map,
2659 map_start,
2660 map_size,
2661 &copy,
2662 &cur_prot,
2663 &max_prot);
2664 if (kr != KERN_SUCCESS) {
2665 return kr;
2666 }
2667
2668 if (mask_protections) {
2669 /*
2670 * We just want as much of "original_protections"
2671 * as we can get out of the actual "cur_prot".
2672 */
2673 protections &= cur_prot;
2674 if (protections == VM_PROT_NONE) {
2675 /* no access at all: fail */
2676 vm_map_copy_discard(copy);
2677 return KERN_PROTECTION_FAILURE;
2678 }
2679 } else {
2680 /*
2681 * We want exactly "original_protections"
2682 * out of "cur_prot".
2683 */
2684 if ((cur_prot & protections) != protections) {
2685 vm_map_copy_discard(copy);
2686 return KERN_PROTECTION_FAILURE;
2687 }
2688 }
2689
2690 kr = mach_memory_entry_allocate(&user_entry, &user_handle);
2691 if (kr != KERN_SUCCESS) {
2692 vm_map_copy_discard(copy);
2693 return KERN_FAILURE;
2694 }
2695
2696 user_entry->backing.copy = copy;
2697 user_entry->internal = FALSE;
2698 user_entry->is_sub_map = FALSE;
2699 user_entry->is_copy = TRUE;
2700 user_entry->offset = 0;
2701 user_entry->protection = protections;
2702 user_entry->size = map_size;
2703 user_entry->data_offset = offset_in_page;
2704
2705 *size = CAST_DOWN(vm_size_t, (user_entry->size -
2706 user_entry->data_offset));
2707 *object_handle = user_handle;
2708 return KERN_SUCCESS;
2709 }
2710
2711 if (parent_entry == NULL ||
2712 (permission & MAP_MEM_NAMED_REUSE)) {
2713 map_end = vm_map_round_page(offset + *size, PAGE_MASK);
2714 map_size = map_end - map_start;
2715 if (use_data_addr || use_4K_compat) {
2716 offset_in_page = offset - map_start;
2717 if (use_4K_compat) {
2718 offset_in_page &= ~((signed)(0xFFF));
2719 }
2720 } else {
2721 offset_in_page = 0;
2722 }
2723
2724 /* Create a named object based on address range within the task map */
2725 /* Go find the object at given address */
2726
2727 if (target_map == VM_MAP_NULL) {
2728 return KERN_INVALID_TASK;
2729 }
2730
2731 redo_lookup:
2732 protections = original_protections;
2733 vm_map_lock_read(target_map);
2734
2735 /* get the object associated with the target address */
2736 /* note we check the permission of the range against */
2737 /* that requested by the caller */
2738
2739 kr = vm_map_lookup_locked(&target_map, map_start,
2740 protections | mask_protections,
2741 OBJECT_LOCK_EXCLUSIVE, &version,
2742 &object, &obj_off, &prot, &wired,
2743 &fault_info,
2744 &real_map);
2745 if (kr != KERN_SUCCESS) {
2746 vm_map_unlock_read(target_map);
2747 goto make_mem_done;
2748 }
2749 if (mask_protections) {
2750 /*
2751 * The caller asked us to use the "protections" as
2752 * a mask, so restrict "protections" to what this
2753 * mapping actually allows.
2754 */
2755 protections &= prot;
2756 }
2757 #if CONFIG_EMBEDDED
2758 /*
2759 * Wiring would copy the pages to a shadow object.
2760 * The shadow object would not be code-signed so
2761 * attempting to execute code from these copied pages
2762 * would trigger a code-signing violation.
2763 */
2764 if (prot & VM_PROT_EXECUTE) {
2765 if (log_executable_mem_entry) {
2766 void *bsd_info;
2767 bsd_info = current_task()->bsd_info;
2768 printf("pid %d[%s] making memory entry out of "
2769 "executable range from 0x%llx to 0x%llx:"
2770 "might cause code-signing issues "
2771 "later\n",
2772 proc_selfpid(),
2773 (bsd_info != NULL
2774 ? proc_name_address(bsd_info)
2775 : "?"),
2776 (uint64_t) map_start,
2777 (uint64_t) map_end);
2778 }
2779 DTRACE_VM2(cs_executable_mem_entry,
2780 uint64_t, (uint64_t)map_start,
2781 uint64_t, (uint64_t)map_end);
2782 cs_executable_mem_entry++;
2783
2784 #if 11
2785 /*
2786 * We don't know how the memory entry will be used.
2787 * It might never get wired and might not cause any
2788 * trouble, so let's not reject this request...
2789 */
2790 #else /* 11 */
2791 kr = KERN_PROTECTION_FAILURE;
2792 vm_object_unlock(object);
2793 vm_map_unlock_read(target_map);
2794 if (real_map != target_map) {
2795 vm_map_unlock_read(real_map);
2796 }
2797 goto make_mem_done;
2798 #endif /* 11 */
2799 }
2800 #endif /* CONFIG_EMBEDDED */
2801
2802 if (((prot & protections) != protections)
2803 || (object == kernel_object)) {
2804 kr = KERN_INVALID_RIGHT;
2805 vm_object_unlock(object);
2806 vm_map_unlock_read(target_map);
2807 if (real_map != target_map) {
2808 vm_map_unlock_read(real_map);
2809 }
2810 if (object == kernel_object) {
2811 printf("Warning: Attempt to create a named"
2812 " entry from the kernel_object\n");
2813 }
2814 goto make_mem_done;
2815 }
2816
2817 /* We have an object, now check to see if this object */
2818 /* is suitable. If not, create a shadow and share that */
2819
2820 /*
2821 * We have to unlock the VM object to avoid deadlocking with
2822 * a VM map lock (the lock ordering is map, the object), if we
2823 * need to modify the VM map to create a shadow object. Since
2824 * we might release the VM map lock below anyway, we have
2825 * to release the VM map lock now.
2826 * XXX FBDP There must be a way to avoid this double lookup...
2827 *
2828 * Take an extra reference on the VM object to make sure it's
2829 * not going to disappear.
2830 */
2831 vm_object_reference_locked(object); /* extra ref to hold obj */
2832 vm_object_unlock(object);
2833
2834 local_map = original_map;
2835 local_offset = map_start;
2836 if (target_map != local_map) {
2837 vm_map_unlock_read(target_map);
2838 if (real_map != target_map) {
2839 vm_map_unlock_read(real_map);
2840 }
2841 vm_map_lock_read(local_map);
2842 target_map = local_map;
2843 real_map = local_map;
2844 }
2845 while (TRUE) {
2846 if (!vm_map_lookup_entry(local_map,
2847 local_offset, &map_entry)) {
2848 kr = KERN_INVALID_ARGUMENT;
2849 vm_map_unlock_read(target_map);
2850 if (real_map != target_map) {
2851 vm_map_unlock_read(real_map);
2852 }
2853 vm_object_deallocate(object); /* release extra ref */
2854 object = VM_OBJECT_NULL;
2855 goto make_mem_done;
2856 }
2857 iskernel = (local_map->pmap == kernel_pmap);
2858 if (!(map_entry->is_sub_map)) {
2859 if (VME_OBJECT(map_entry) != object) {
2860 kr = KERN_INVALID_ARGUMENT;
2861 vm_map_unlock_read(target_map);
2862 if (real_map != target_map) {
2863 vm_map_unlock_read(real_map);
2864 }
2865 vm_object_deallocate(object); /* release extra ref */
2866 object = VM_OBJECT_NULL;
2867 goto make_mem_done;
2868 }
2869 break;
2870 } else {
2871 vm_map_t tmap;
2872 tmap = local_map;
2873 local_map = VME_SUBMAP(map_entry);
2874
2875 vm_map_lock_read(local_map);
2876 vm_map_unlock_read(tmap);
2877 target_map = local_map;
2878 real_map = local_map;
2879 local_offset = local_offset - map_entry->vme_start;
2880 local_offset += VME_OFFSET(map_entry);
2881 }
2882 }
2883
2884 #if VM_NAMED_ENTRY_LIST
2885 alias = VME_ALIAS(map_entry);
2886 #endif /* VM_NAMED_ENTRY_LIST */
2887
2888 /*
2889 * We found the VM map entry, lock the VM object again.
2890 */
2891 vm_object_lock(object);
2892 if (map_entry->wired_count) {
2893 /* JMM - The check below should be reworked instead. */
2894 object->true_share = TRUE;
2895 }
2896 if (mask_protections) {
2897 /*
2898 * The caller asked us to use the "protections" as
2899 * a mask, so restrict "protections" to what this
2900 * mapping actually allows.
2901 */
2902 protections &= map_entry->max_protection;
2903 }
2904 if (((map_entry->max_protection) & protections) != protections) {
2905 kr = KERN_INVALID_RIGHT;
2906 vm_object_unlock(object);
2907 vm_map_unlock_read(target_map);
2908 if (real_map != target_map) {
2909 vm_map_unlock_read(real_map);
2910 }
2911 vm_object_deallocate(object);
2912 object = VM_OBJECT_NULL;
2913 goto make_mem_done;
2914 }
2915
2916 mappable_size = fault_info.hi_offset - obj_off;
2917 total_size = map_entry->vme_end - map_entry->vme_start;
2918 if (map_size > mappable_size) {
2919 /* try to extend mappable size if the entries */
2920 /* following are from the same object and are */
2921 /* compatible */
2922 next_entry = map_entry->vme_next;
2923 /* lets see if the next map entry is still */
2924 /* pointing at this object and is contiguous */
2925 while (map_size > mappable_size) {
2926 if ((VME_OBJECT(next_entry) == object) &&
2927 (next_entry->vme_start ==
2928 next_entry->vme_prev->vme_end) &&
2929 (VME_OFFSET(next_entry) ==
2930 (VME_OFFSET(next_entry->vme_prev) +
2931 (next_entry->vme_prev->vme_end -
2932 next_entry->vme_prev->vme_start)))) {
2933 if (mask_protections) {
2934 /*
2935 * The caller asked us to use
2936 * the "protections" as a mask,
2937 * so restrict "protections" to
2938 * what this mapping actually
2939 * allows.
2940 */
2941 protections &= next_entry->max_protection;
2942 }
2943 if ((next_entry->wired_count) &&
2944 (map_entry->wired_count == 0)) {
2945 break;
2946 }
2947 if (((next_entry->max_protection)
2948 & protections) != protections) {
2949 break;
2950 }
2951 if (next_entry->needs_copy !=
2952 map_entry->needs_copy) {
2953 break;
2954 }
2955 mappable_size += next_entry->vme_end
2956 - next_entry->vme_start;
2957 total_size += next_entry->vme_end
2958 - next_entry->vme_start;
2959 next_entry = next_entry->vme_next;
2960 } else {
2961 break;
2962 }
2963 }
2964 }
2965
2966 /* vm_map_entry_should_cow_for_true_share() checks for malloc tags,
2967 * never true in kernel */
2968 if (!iskernel && vm_map_entry_should_cow_for_true_share(map_entry) &&
2969 object->vo_size > map_size &&
2970 map_size != 0) {
2971 /*
2972 * Set up the targeted range for copy-on-write to
2973 * limit the impact of "true_share"/"copy_delay" to
2974 * that range instead of the entire VM object...
2975 */
2976
2977 vm_object_unlock(object);
2978 if (vm_map_lock_read_to_write(target_map)) {
2979 vm_object_deallocate(object);
2980 target_map = original_map;
2981 goto redo_lookup;
2982 }
2983
2984 vm_map_clip_start(target_map,
2985 map_entry,
2986 vm_map_trunc_page(map_start,
2987 VM_MAP_PAGE_MASK(target_map)));
2988 vm_map_clip_end(target_map,
2989 map_entry,
2990 (vm_map_round_page(map_end,
2991 VM_MAP_PAGE_MASK(target_map))));
2992 force_shadow = TRUE;
2993
2994 if ((map_entry->vme_end - offset) < map_size) {
2995 map_size = map_entry->vme_end - map_start;
2996 }
2997 total_size = map_entry->vme_end - map_entry->vme_start;
2998
2999 vm_map_lock_write_to_read(target_map);
3000 vm_object_lock(object);
3001 }
3002
3003 if (object->internal) {
3004 /* vm_map_lookup_locked will create a shadow if */
3005 /* needs_copy is set but does not check for the */
3006 /* other two conditions shown. It is important to */
3007 /* set up an object which will not be pulled from */
3008 /* under us. */
3009
3010 if (force_shadow ||
3011 ((map_entry->needs_copy ||
3012 object->shadowed ||
3013 (object->vo_size > total_size &&
3014 (VME_OFFSET(map_entry) != 0 ||
3015 object->vo_size >
3016 vm_map_round_page(total_size,
3017 VM_MAP_PAGE_MASK(target_map)))))
3018 && !object->true_share
3019 && object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC)) {
3020 /*
3021 * We have to unlock the VM object before
3022 * trying to upgrade the VM map lock, to
3023 * honor lock ordering (map then object).
3024 * Otherwise, we would deadlock if another
3025 * thread holds a read lock on the VM map and
3026 * is trying to acquire the VM object's lock.
3027 * We still hold an extra reference on the
3028 * VM object, guaranteeing that it won't
3029 * disappear.
3030 */
3031 vm_object_unlock(object);
3032
3033 if (vm_map_lock_read_to_write(target_map)) {
3034 /*
3035 * We couldn't upgrade our VM map lock
3036 * from "read" to "write" and we lost
3037 * our "read" lock.
3038 * Start all over again...
3039 */
3040 vm_object_deallocate(object); /* extra ref */
3041 target_map = original_map;
3042 goto redo_lookup;
3043 }
3044 #if 00
3045 vm_object_lock(object);
3046 #endif
3047
3048 /*
3049 * JMM - We need to avoid coming here when the object
3050 * is wired by anybody, not just the current map. Why
3051 * couldn't we use the standard vm_object_copy_quickly()
3052 * approach here?
3053 */
3054
3055 /* create a shadow object */
3056 VME_OBJECT_SHADOW(map_entry, total_size);
3057 shadow_object = VME_OBJECT(map_entry);
3058 #if 00
3059 vm_object_unlock(object);
3060 #endif
3061
3062 prot = map_entry->protection & ~VM_PROT_WRITE;
3063
3064 if (override_nx(target_map,
3065 VME_ALIAS(map_entry))
3066 && prot) {
3067 prot |= VM_PROT_EXECUTE;
3068 }
3069
3070 vm_object_pmap_protect(
3071 object, VME_OFFSET(map_entry),
3072 total_size,
3073 ((map_entry->is_shared
3074 || target_map->mapped_in_other_pmaps)
3075 ? PMAP_NULL :
3076 target_map->pmap),
3077 map_entry->vme_start,
3078 prot);
3079 total_size -= (map_entry->vme_end
3080 - map_entry->vme_start);
3081 next_entry = map_entry->vme_next;
3082 map_entry->needs_copy = FALSE;
3083
3084 vm_object_lock(shadow_object);
3085 while (total_size) {
3086 assert((next_entry->wired_count == 0) ||
3087 (map_entry->wired_count));
3088
3089 if (VME_OBJECT(next_entry) == object) {
3090 vm_object_reference_locked(shadow_object);
3091 VME_OBJECT_SET(next_entry,
3092 shadow_object);
3093 vm_object_deallocate(object);
3094 VME_OFFSET_SET(
3095 next_entry,
3096 (VME_OFFSET(next_entry->vme_prev) +
3097 (next_entry->vme_prev->vme_end
3098 - next_entry->vme_prev->vme_start)));
3099 next_entry->use_pmap = TRUE;
3100 next_entry->needs_copy = FALSE;
3101 } else {
3102 panic("mach_make_memory_entry_64:"
3103 " map entries out of sync\n");
3104 }
3105 total_size -=
3106 next_entry->vme_end
3107 - next_entry->vme_start;
3108 next_entry = next_entry->vme_next;
3109 }
3110
3111 /*
3112 * Transfer our extra reference to the
3113 * shadow object.
3114 */
3115 vm_object_reference_locked(shadow_object);
3116 vm_object_deallocate(object); /* extra ref */
3117 object = shadow_object;
3118
3119 obj_off = ((local_offset - map_entry->vme_start)
3120 + VME_OFFSET(map_entry));
3121
3122 vm_map_lock_write_to_read(target_map);
3123 }
3124 }
3125
3126 /* note: in the future we can (if necessary) allow for */
3127 /* memory object lists, this will better support */
3128 /* fragmentation, but is it necessary? The user should */
3129 /* be encouraged to create address space oriented */
3130 /* shared objects from CLEAN memory regions which have */
3131 /* a known and defined history. i.e. no inheritence */
3132 /* share, make this call before making the region the */
3133 /* target of ipc's, etc. The code above, protecting */
3134 /* against delayed copy, etc. is mostly defensive. */
3135
3136 wimg_mode = object->wimg_bits;
3137 if (!(object->nophyscache)) {
3138 vm_prot_to_wimg(access, &wimg_mode);
3139 }
3140
3141 #if VM_OBJECT_TRACKING_OP_TRUESHARE
3142 if (!object->true_share &&
3143 vm_object_tracking_inited) {
3144 void *bt[VM_OBJECT_TRACKING_BTDEPTH];
3145 int num = 0;
3146
3147 num = OSBacktrace(bt,
3148 VM_OBJECT_TRACKING_BTDEPTH);
3149 btlog_add_entry(vm_object_tracking_btlog,
3150 object,
3151 VM_OBJECT_TRACKING_OP_TRUESHARE,
3152 bt,
3153 num);
3154 }
3155 #endif /* VM_OBJECT_TRACKING_OP_TRUESHARE */
3156
3157 vm_object_lock_assert_exclusive(object);
3158 object->true_share = TRUE;
3159 if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC) {
3160 object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
3161 }
3162
3163 /*
3164 * The memory entry now points to this VM object and we
3165 * need to hold a reference on the VM object. Use the extra
3166 * reference we took earlier to keep the object alive when we
3167 * had to unlock it.
3168 */
3169
3170 vm_map_unlock_read(target_map);
3171 if (real_map != target_map) {
3172 vm_map_unlock_read(real_map);
3173 }
3174
3175 if (object->wimg_bits != wimg_mode) {
3176 vm_object_change_wimg_mode(object, wimg_mode);
3177 }
3178
3179 /* the size of mapped entry that overlaps with our region */
3180 /* which is targeted for share. */
3181 /* (entry_end - entry_start) - */
3182 /* offset of our beg addr within entry */
3183 /* it corresponds to this: */
3184
3185 if (map_size > mappable_size) {
3186 map_size = mappable_size;
3187 }
3188
3189 if (permission & MAP_MEM_NAMED_REUSE) {
3190 /*
3191 * Compare what we got with the "parent_entry".
3192 * If they match, re-use the "parent_entry" instead
3193 * of creating a new one.
3194 */
3195 if (parent_entry != NULL &&
3196 parent_entry->backing.object == object &&
3197 parent_entry->internal == object->internal &&
3198 parent_entry->is_sub_map == FALSE &&
3199 parent_entry->offset == obj_off &&
3200 parent_entry->protection == protections &&
3201 parent_entry->size == map_size &&
3202 ((!(use_data_addr || use_4K_compat) &&
3203 (parent_entry->data_offset == 0)) ||
3204 ((use_data_addr || use_4K_compat) &&
3205 (parent_entry->data_offset == offset_in_page)))) {
3206 /*
3207 * We have a match: re-use "parent_entry".
3208 */
3209 /* release our extra reference on object */
3210 vm_object_unlock(object);
3211 vm_object_deallocate(object);
3212 /* parent_entry->ref_count++; XXX ? */
3213 /* Get an extra send-right on handle */
3214 ipc_port_copy_send(parent_handle);
3215
3216 *size = CAST_DOWN(vm_size_t,
3217 (parent_entry->size -
3218 parent_entry->data_offset));
3219 *object_handle = parent_handle;
3220 return KERN_SUCCESS;
3221 } else {
3222 /*
3223 * No match: we need to create a new entry.
3224 * fall through...
3225 */
3226 }
3227 }
3228
3229 vm_object_unlock(object);
3230 if (mach_memory_entry_allocate(&user_entry, &user_handle)
3231 != KERN_SUCCESS) {
3232 /* release our unused reference on the object */
3233 vm_object_deallocate(object);
3234 return KERN_FAILURE;
3235 }
3236
3237 user_entry->backing.object = object;
3238 user_entry->internal = object->internal;
3239 user_entry->is_sub_map = FALSE;
3240 user_entry->offset = obj_off;
3241 user_entry->data_offset = offset_in_page;
3242 user_entry->protection = protections;
3243 SET_MAP_MEM(GET_MAP_MEM(permission), user_entry->protection);
3244 user_entry->size = map_size;
3245 #if VM_NAMED_ENTRY_LIST
3246 user_entry->named_entry_alias = alias;
3247 #endif /* VM_NAMED_ENTRY_LIST */
3248
3249 /* user_object pager and internal fields are not used */
3250 /* when the object field is filled in. */
3251
3252 *size = CAST_DOWN(vm_size_t, (user_entry->size -
3253 user_entry->data_offset));
3254 *object_handle = user_handle;
3255 return KERN_SUCCESS;
3256 } else {
3257 /* The new object will be base on an existing named object */
3258 if (parent_entry == NULL) {
3259 kr = KERN_INVALID_ARGUMENT;
3260 goto make_mem_done;
3261 }
3262
3263 if (use_data_addr || use_4K_compat) {
3264 /*
3265 * submaps and pagers should only be accessible from within
3266 * the kernel, which shouldn't use the data address flag, so can fail here.
3267 */
3268 if (parent_entry->is_sub_map) {
3269 panic("Shouldn't be using data address with a parent entry that is a submap.");
3270 }
3271 /*
3272 * Account for offset to data in parent entry and
3273 * compute our own offset to data.
3274 */
3275 if ((offset + *size + parent_entry->data_offset) > parent_entry->size) {
3276 kr = KERN_INVALID_ARGUMENT;
3277 goto make_mem_done;
3278 }
3279
3280 map_start = vm_map_trunc_page(offset + parent_entry->data_offset, PAGE_MASK);
3281 offset_in_page = (offset + parent_entry->data_offset) - map_start;
3282 if (use_4K_compat) {
3283 offset_in_page &= ~((signed)(0xFFF));
3284 }
3285 map_end = vm_map_round_page(offset + parent_entry->data_offset + *size, PAGE_MASK);
3286 map_size = map_end - map_start;
3287 } else {
3288 map_end = vm_map_round_page(offset + *size, PAGE_MASK);
3289 map_size = map_end - map_start;
3290 offset_in_page = 0;
3291
3292 if ((offset + map_size) > parent_entry->size) {
3293 kr = KERN_INVALID_ARGUMENT;
3294 goto make_mem_done;
3295 }
3296 }
3297
3298 if (mask_protections) {
3299 /*
3300 * The caller asked us to use the "protections" as
3301 * a mask, so restrict "protections" to what this
3302 * mapping actually allows.
3303 */
3304 protections &= parent_entry->protection;
3305 }
3306 if ((protections & parent_entry->protection) != protections) {
3307 kr = KERN_PROTECTION_FAILURE;
3308 goto make_mem_done;
3309 }
3310
3311 if (mach_memory_entry_allocate(&user_entry, &user_handle)
3312 != KERN_SUCCESS) {
3313 kr = KERN_FAILURE;
3314 goto make_mem_done;
3315 }
3316
3317 user_entry->size = map_size;
3318 user_entry->offset = parent_entry->offset + map_start;
3319 user_entry->data_offset = offset_in_page;
3320 user_entry->is_sub_map = parent_entry->is_sub_map;
3321 user_entry->is_copy = parent_entry->is_copy;
3322 user_entry->internal = parent_entry->internal;
3323 user_entry->protection = protections;
3324
3325 if (access != MAP_MEM_NOOP) {
3326 SET_MAP_MEM(access, user_entry->protection);
3327 }
3328
3329 if (parent_entry->is_sub_map) {
3330 vm_map_t map = parent_entry->backing.map;
3331 user_entry->backing.map = map;
3332 lck_mtx_lock(&map->s_lock);
3333 os_ref_retain_locked(&map->map_refcnt);
3334 lck_mtx_unlock(&map->s_lock);
3335 } else {
3336 object = parent_entry->backing.object;
3337 assert(object != VM_OBJECT_NULL);
3338 user_entry->backing.object = object;
3339 /* we now point to this object, hold on */
3340 vm_object_lock(object);
3341 vm_object_reference_locked(object);
3342 #if VM_OBJECT_TRACKING_OP_TRUESHARE
3343 if (!object->true_share &&
3344 vm_object_tracking_inited) {
3345 void *bt[VM_OBJECT_TRACKING_BTDEPTH];
3346 int num = 0;
3347
3348 num = OSBacktrace(bt,
3349 VM_OBJECT_TRACKING_BTDEPTH);
3350 btlog_add_entry(vm_object_tracking_btlog,
3351 object,
3352 VM_OBJECT_TRACKING_OP_TRUESHARE,
3353 bt,
3354 num);
3355 }
3356 #endif /* VM_OBJECT_TRACKING_OP_TRUESHARE */
3357
3358 object->true_share = TRUE;
3359 if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC) {
3360 object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
3361 }
3362 vm_object_unlock(object);
3363 }
3364 *size = CAST_DOWN(vm_size_t, (user_entry->size -
3365 user_entry->data_offset));
3366 *object_handle = user_handle;
3367 return KERN_SUCCESS;
3368 }
3369
3370 make_mem_done:
3371 if (user_handle != IP_NULL) {
3372 /*
3373 * Releasing "user_handle" causes the kernel object
3374 * associated with it ("user_entry" here) to also be
3375 * released and freed.
3376 */
3377 mach_memory_entry_port_release(user_handle);
3378 }
3379 return kr;
3380 }
3381
3382 kern_return_t
3383 _mach_make_memory_entry(
3384 vm_map_t target_map,
3385 memory_object_size_t *size,
3386 memory_object_offset_t offset,
3387 vm_prot_t permission,
3388 ipc_port_t *object_handle,
3389 ipc_port_t parent_entry)
3390 {
3391 memory_object_size_t mo_size;
3392 kern_return_t kr;
3393
3394 mo_size = (memory_object_size_t)*size;
3395 kr = mach_make_memory_entry_64(target_map, &mo_size,
3396 (memory_object_offset_t)offset, permission, object_handle,
3397 parent_entry);
3398 *size = mo_size;
3399 return kr;
3400 }
3401
3402 kern_return_t
3403 mach_make_memory_entry(
3404 vm_map_t target_map,
3405 vm_size_t *size,
3406 vm_offset_t offset,
3407 vm_prot_t permission,
3408 ipc_port_t *object_handle,
3409 ipc_port_t parent_entry)
3410 {
3411 memory_object_size_t mo_size;
3412 kern_return_t kr;
3413
3414 mo_size = (memory_object_size_t)*size;
3415 kr = mach_make_memory_entry_64(target_map, &mo_size,
3416 (memory_object_offset_t)offset, permission, object_handle,
3417 parent_entry);
3418 *size = CAST_DOWN(vm_size_t, mo_size);
3419 return kr;
3420 }
3421
3422 /*
3423 * task_wire
3424 *
3425 * Set or clear the map's wiring_required flag. This flag, if set,
3426 * will cause all future virtual memory allocation to allocate
3427 * user wired memory. Unwiring pages wired down as a result of
3428 * this routine is done with the vm_wire interface.
3429 */
3430 kern_return_t
3431 task_wire(
3432 vm_map_t map,
3433 boolean_t must_wire)
3434 {
3435 if (map == VM_MAP_NULL) {
3436 return KERN_INVALID_ARGUMENT;
3437 }
3438
3439 vm_map_lock(map);
3440 map->wiring_required = (must_wire == TRUE);
3441 vm_map_unlock(map);
3442
3443 return KERN_SUCCESS;
3444 }
3445
3446 kern_return_t
3447 vm_map_exec_lockdown(
3448 vm_map_t map)
3449 {
3450 if (map == VM_MAP_NULL) {
3451 return KERN_INVALID_ARGUMENT;
3452 }
3453
3454 vm_map_lock(map);
3455 map->map_disallow_new_exec = TRUE;
3456 vm_map_unlock(map);
3457
3458 return KERN_SUCCESS;
3459 }
3460
3461 #if VM_NAMED_ENTRY_LIST
3462 queue_head_t vm_named_entry_list;
3463 int vm_named_entry_count = 0;
3464 lck_mtx_t vm_named_entry_list_lock_data;
3465 lck_mtx_ext_t vm_named_entry_list_lock_data_ext;
3466 #endif /* VM_NAMED_ENTRY_LIST */
3467
3468 void vm_named_entry_init(void);
3469 void
3470 vm_named_entry_init(void)
3471 {
3472 #if VM_NAMED_ENTRY_LIST
3473 queue_init(&vm_named_entry_list);
3474 vm_named_entry_count = 0;
3475 lck_mtx_init_ext(&vm_named_entry_list_lock_data,
3476 &vm_named_entry_list_lock_data_ext,
3477 &vm_object_lck_grp,
3478 &vm_object_lck_attr);
3479 #endif /* VM_NAMED_ENTRY_LIST */
3480 }
3481
3482 __private_extern__ kern_return_t
3483 mach_memory_entry_allocate(
3484 vm_named_entry_t *user_entry_p,
3485 ipc_port_t *user_handle_p)
3486 {
3487 vm_named_entry_t user_entry;
3488 ipc_port_t user_handle;
3489
3490 user_entry = (vm_named_entry_t) kalloc(sizeof *user_entry);
3491 if (user_entry == NULL) {
3492 return KERN_FAILURE;
3493 }
3494 bzero(user_entry, sizeof(*user_entry));
3495
3496 named_entry_lock_init(user_entry);
3497
3498 user_entry->backing.object = NULL;
3499 user_entry->is_sub_map = FALSE;
3500 user_entry->is_copy = FALSE;
3501 user_entry->internal = FALSE;
3502 user_entry->size = 0;
3503 user_entry->offset = 0;
3504 user_entry->data_offset = 0;
3505 user_entry->protection = VM_PROT_NONE;
3506 user_entry->ref_count = 1;
3507
3508 user_handle = ipc_kobject_alloc_port((ipc_kobject_t)user_entry,
3509 IKOT_NAMED_ENTRY,
3510 IPC_KOBJECT_ALLOC_MAKE_SEND | IPC_KOBJECT_ALLOC_NSREQUEST);
3511
3512 *user_entry_p = user_entry;
3513 *user_handle_p = user_handle;
3514
3515 #if VM_NAMED_ENTRY_LIST
3516 /* keep a loose (no reference) pointer to the Mach port, for debugging only */
3517 user_entry->named_entry_port = user_handle;
3518 /* backtrace at allocation time, for debugging only */
3519 OSBacktrace(&user_entry->named_entry_bt[0],
3520 NAMED_ENTRY_BT_DEPTH);
3521
3522 /* add this new named entry to the global list */
3523 lck_mtx_lock_spin(&vm_named_entry_list_lock_data);
3524 queue_enter(&vm_named_entry_list, user_entry,
3525 vm_named_entry_t, named_entry_list);
3526 vm_named_entry_count++;
3527 lck_mtx_unlock(&vm_named_entry_list_lock_data);
3528 #endif /* VM_NAMED_ENTRY_LIST */
3529
3530 return KERN_SUCCESS;
3531 }
3532
3533 /*
3534 * mach_memory_object_memory_entry_64
3535 *
3536 * Create a named entry backed by the provided pager.
3537 *
3538 */
3539 kern_return_t
3540 mach_memory_object_memory_entry_64(
3541 host_t host,
3542 boolean_t internal,
3543 vm_object_offset_t size,
3544 vm_prot_t permission,
3545 memory_object_t pager,
3546 ipc_port_t *entry_handle)
3547 {
3548 unsigned int access;
3549 vm_named_entry_t user_entry;
3550 ipc_port_t user_handle;
3551 vm_object_t object;
3552
3553 if (host == HOST_NULL) {
3554 return KERN_INVALID_HOST;
3555 }
3556
3557 if (pager == MEMORY_OBJECT_NULL && internal) {
3558 object = vm_object_allocate(size);
3559 if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC) {
3560 object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
3561 }
3562 } else {
3563 object = memory_object_to_vm_object(pager);
3564 if (object != VM_OBJECT_NULL) {
3565 vm_object_reference(object);
3566 }
3567 }
3568 if (object == VM_OBJECT_NULL) {
3569 return KERN_INVALID_ARGUMENT;
3570 }
3571
3572 if (mach_memory_entry_allocate(&user_entry, &user_handle)
3573 != KERN_SUCCESS) {
3574 vm_object_deallocate(object);
3575 return KERN_FAILURE;
3576 }
3577
3578 user_entry->size = size;
3579 user_entry->offset = 0;
3580 user_entry->protection = permission & VM_PROT_ALL;
3581 access = GET_MAP_MEM(permission);
3582 SET_MAP_MEM(access, user_entry->protection);
3583 user_entry->is_sub_map = FALSE;
3584 assert(user_entry->ref_count == 1);
3585
3586 user_entry->backing.object = object;
3587 user_entry->internal = object->internal;
3588 assert(object->internal == internal);
3589
3590 *entry_handle = user_handle;
3591 return KERN_SUCCESS;
3592 }
3593
3594 kern_return_t
3595 mach_memory_object_memory_entry(
3596 host_t host,
3597 boolean_t internal,
3598 vm_size_t size,
3599 vm_prot_t permission,
3600 memory_object_t pager,
3601 ipc_port_t *entry_handle)
3602 {
3603 return mach_memory_object_memory_entry_64( host, internal,
3604 (vm_object_offset_t)size, permission, pager, entry_handle);
3605 }
3606
3607
3608 kern_return_t
3609 mach_memory_entry_purgable_control(
3610 ipc_port_t entry_port,
3611 vm_purgable_t control,
3612 int *state)
3613 {
3614 if (control == VM_PURGABLE_SET_STATE_FROM_KERNEL) {
3615 /* not allowed from user-space */
3616 return KERN_INVALID_ARGUMENT;
3617 }
3618
3619 return memory_entry_purgeable_control_internal(entry_port, control, state);
3620 }
3621
3622 kern_return_t
3623 memory_entry_purgeable_control_internal(
3624 ipc_port_t entry_port,
3625 vm_purgable_t control,
3626 int *state)
3627 {
3628 kern_return_t kr;
3629 vm_named_entry_t mem_entry;
3630 vm_object_t object;
3631
3632 if (!IP_VALID(entry_port) ||
3633 ip_kotype(entry_port) != IKOT_NAMED_ENTRY) {
3634 return KERN_INVALID_ARGUMENT;
3635 }
3636 if (control != VM_PURGABLE_SET_STATE &&
3637 control != VM_PURGABLE_GET_STATE &&
3638 control != VM_PURGABLE_SET_STATE_FROM_KERNEL) {
3639 return KERN_INVALID_ARGUMENT;
3640 }
3641
3642 if ((control == VM_PURGABLE_SET_STATE ||
3643 control == VM_PURGABLE_SET_STATE_FROM_KERNEL) &&
3644 (((*state & ~(VM_PURGABLE_ALL_MASKS)) != 0) ||
3645 ((*state & VM_PURGABLE_STATE_MASK) > VM_PURGABLE_STATE_MASK))) {
3646 return KERN_INVALID_ARGUMENT;
3647 }
3648
3649 mem_entry = (vm_named_entry_t) ip_get_kobject(entry_port);
3650
3651 named_entry_lock(mem_entry);
3652
3653 if (mem_entry->is_sub_map ||
3654 mem_entry->is_copy) {
3655 named_entry_unlock(mem_entry);
3656 return KERN_INVALID_ARGUMENT;
3657 }
3658
3659 object = mem_entry->backing.object;
3660 if (object == VM_OBJECT_NULL) {
3661 named_entry_unlock(mem_entry);
3662 return KERN_INVALID_ARGUMENT;
3663 }
3664
3665 vm_object_lock(object);
3666
3667 /* check that named entry covers entire object ? */
3668 if (mem_entry->offset != 0 || object->vo_size != mem_entry->size) {
3669 vm_object_unlock(object);
3670 named_entry_unlock(mem_entry);
3671 return KERN_INVALID_ARGUMENT;
3672 }
3673
3674 named_entry_unlock(mem_entry);
3675
3676 kr = vm_object_purgable_control(object, control, state);
3677
3678 vm_object_unlock(object);
3679
3680 return kr;
3681 }
3682
3683 kern_return_t
3684 mach_memory_entry_access_tracking(
3685 ipc_port_t entry_port,
3686 int *access_tracking,
3687 uint32_t *access_tracking_reads,
3688 uint32_t *access_tracking_writes)
3689 {
3690 return memory_entry_access_tracking_internal(entry_port,
3691 access_tracking,
3692 access_tracking_reads,
3693 access_tracking_writes);
3694 }
3695
3696 kern_return_t
3697 memory_entry_access_tracking_internal(
3698 ipc_port_t entry_port,
3699 int *access_tracking,
3700 uint32_t *access_tracking_reads,
3701 uint32_t *access_tracking_writes)
3702 {
3703 vm_named_entry_t mem_entry;
3704 vm_object_t object;
3705 kern_return_t kr;
3706
3707 if (!IP_VALID(entry_port) ||
3708 ip_kotype(entry_port) != IKOT_NAMED_ENTRY) {
3709 return KERN_INVALID_ARGUMENT;
3710 }
3711
3712 mem_entry = (vm_named_entry_t) ip_get_kobject(entry_port);
3713
3714 named_entry_lock(mem_entry);
3715
3716 if (mem_entry->is_sub_map ||
3717 mem_entry->is_copy) {
3718 named_entry_unlock(mem_entry);
3719 return KERN_INVALID_ARGUMENT;
3720 }
3721
3722 object = mem_entry->backing.object;
3723 if (object == VM_OBJECT_NULL) {
3724 named_entry_unlock(mem_entry);
3725 return KERN_INVALID_ARGUMENT;
3726 }
3727
3728 #if VM_OBJECT_ACCESS_TRACKING
3729 vm_object_access_tracking(object,
3730 access_tracking,
3731 access_tracking_reads,
3732 access_tracking_writes);
3733 kr = KERN_SUCCESS;
3734 #else /* VM_OBJECT_ACCESS_TRACKING */
3735 (void) access_tracking;
3736 (void) access_tracking_reads;
3737 (void) access_tracking_writes;
3738 kr = KERN_NOT_SUPPORTED;
3739 #endif /* VM_OBJECT_ACCESS_TRACKING */
3740
3741 named_entry_unlock(mem_entry);
3742
3743 return kr;
3744 }
3745
3746 kern_return_t
3747 mach_memory_entry_ownership(
3748 ipc_port_t entry_port,
3749 task_t owner,
3750 int ledger_tag,
3751 int ledger_flags)
3752 {
3753 task_t cur_task;
3754 kern_return_t kr;
3755 vm_named_entry_t mem_entry;
3756 vm_object_t object;
3757
3758 cur_task = current_task();
3759 if (cur_task != kernel_task &&
3760 (owner != cur_task ||
3761 (ledger_flags & VM_LEDGER_FLAG_NO_FOOTPRINT) ||
3762 ledger_tag == VM_LEDGER_TAG_NETWORK)) {
3763 /*
3764 * An entitlement is required to:
3765 * + tranfer memory ownership to someone else,
3766 * + request that the memory not count against the footprint,
3767 * + tag as "network" (since that implies "no footprint")
3768 */
3769 if (!cur_task->task_can_transfer_memory_ownership &&
3770 IOTaskHasEntitlement(cur_task,
3771 "com.apple.private.memory.ownership_transfer")) {
3772 cur_task->task_can_transfer_memory_ownership = TRUE;
3773 }
3774 if (!cur_task->task_can_transfer_memory_ownership) {
3775 return KERN_NO_ACCESS;
3776 }
3777 }
3778
3779 if (ledger_flags & ~VM_LEDGER_FLAGS) {
3780 return KERN_INVALID_ARGUMENT;
3781 }
3782 if (ledger_tag <= 0 ||
3783 ledger_tag > VM_LEDGER_TAG_MAX) {
3784 return KERN_INVALID_ARGUMENT;
3785 }
3786
3787 if (!IP_VALID(entry_port) ||
3788 ip_kotype(entry_port) != IKOT_NAMED_ENTRY) {
3789 return KERN_INVALID_ARGUMENT;
3790 }
3791 mem_entry = (vm_named_entry_t) ip_get_kobject(entry_port);
3792
3793 named_entry_lock(mem_entry);
3794
3795 if (mem_entry->is_sub_map ||
3796 mem_entry->is_copy) {
3797 named_entry_unlock(mem_entry);
3798 return KERN_INVALID_ARGUMENT;
3799 }
3800
3801 object = mem_entry->backing.object;
3802 if (object == VM_OBJECT_NULL) {
3803 named_entry_unlock(mem_entry);
3804 return KERN_INVALID_ARGUMENT;
3805 }
3806
3807 vm_object_lock(object);
3808
3809 /* check that named entry covers entire object ? */
3810 if (mem_entry->offset != 0 || object->vo_size != mem_entry->size) {
3811 vm_object_unlock(object);
3812 named_entry_unlock(mem_entry);
3813 return KERN_INVALID_ARGUMENT;
3814 }
3815
3816 named_entry_unlock(mem_entry);
3817
3818 kr = vm_object_ownership_change(object,
3819 ledger_tag,
3820 owner,
3821 ledger_flags,
3822 FALSE); /* task_objq_locked */
3823 vm_object_unlock(object);
3824
3825 return kr;
3826 }
3827
3828 kern_return_t
3829 mach_memory_entry_get_page_counts(
3830 ipc_port_t entry_port,
3831 unsigned int *resident_page_count,
3832 unsigned int *dirty_page_count)
3833 {
3834 kern_return_t kr;
3835 vm_named_entry_t mem_entry;
3836 vm_object_t object;
3837 vm_object_offset_t offset;
3838 vm_object_size_t size;
3839
3840 if (!IP_VALID(entry_port) ||
3841 ip_kotype(entry_port) != IKOT_NAMED_ENTRY) {
3842 return KERN_INVALID_ARGUMENT;
3843 }
3844
3845 mem_entry = (vm_named_entry_t) ip_get_kobject(entry_port);
3846
3847 named_entry_lock(mem_entry);
3848
3849 if (mem_entry->is_sub_map ||
3850 mem_entry->is_copy) {
3851 named_entry_unlock(mem_entry);
3852 return KERN_INVALID_ARGUMENT;
3853 }
3854
3855 object = mem_entry->backing.object;
3856 if (object == VM_OBJECT_NULL) {
3857 named_entry_unlock(mem_entry);
3858 return KERN_INVALID_ARGUMENT;
3859 }
3860
3861 vm_object_lock(object);
3862
3863 offset = mem_entry->offset;
3864 size = mem_entry->size;
3865
3866 named_entry_unlock(mem_entry);
3867
3868 kr = vm_object_get_page_counts(object, offset, size, resident_page_count, dirty_page_count);
3869
3870 vm_object_unlock(object);
3871
3872 return kr;
3873 }
3874
3875 /*
3876 * mach_memory_entry_port_release:
3877 *
3878 * Release a send right on a named entry port. This is the correct
3879 * way to destroy a named entry. When the last right on the port is
3880 * released, ipc_kobject_destroy() will call mach_destroy_memory_entry().
3881 */
3882 void
3883 mach_memory_entry_port_release(
3884 ipc_port_t port)
3885 {
3886 assert(ip_kotype(port) == IKOT_NAMED_ENTRY);
3887 ipc_port_release_send(port);
3888 }
3889
3890 /*
3891 * mach_destroy_memory_entry:
3892 *
3893 * Drops a reference on a memory entry and destroys the memory entry if
3894 * there are no more references on it.
3895 * NOTE: This routine should not be called to destroy a memory entry from the
3896 * kernel, as it will not release the Mach port associated with the memory
3897 * entry. The proper way to destroy a memory entry in the kernel is to
3898 * call mach_memort_entry_port_release() to release the kernel's send-right on
3899 * the memory entry's port. When the last send right is released, the memory
3900 * entry will be destroyed via ipc_kobject_destroy().
3901 */
3902 void
3903 mach_destroy_memory_entry(
3904 ipc_port_t port)
3905 {
3906 vm_named_entry_t named_entry;
3907 #if MACH_ASSERT
3908 assert(ip_kotype(port) == IKOT_NAMED_ENTRY);
3909 #endif /* MACH_ASSERT */
3910 named_entry = (vm_named_entry_t) ip_get_kobject(port);
3911
3912 named_entry_lock(named_entry);
3913 named_entry->ref_count -= 1;
3914
3915 if (named_entry->ref_count == 0) {
3916 if (named_entry->is_sub_map) {
3917 vm_map_deallocate(named_entry->backing.map);
3918 } else if (named_entry->is_copy) {
3919 vm_map_copy_discard(named_entry->backing.copy);
3920 } else {
3921 /* release the VM object we've been pointing to */
3922 vm_object_deallocate(named_entry->backing.object);
3923 }
3924
3925 named_entry_unlock(named_entry);
3926 named_entry_lock_destroy(named_entry);
3927
3928 #if VM_NAMED_ENTRY_LIST
3929 lck_mtx_lock_spin(&vm_named_entry_list_lock_data);
3930 queue_remove(&vm_named_entry_list, named_entry,
3931 vm_named_entry_t, named_entry_list);
3932 assert(vm_named_entry_count > 0);
3933 vm_named_entry_count--;
3934 lck_mtx_unlock(&vm_named_entry_list_lock_data);
3935 #endif /* VM_NAMED_ENTRY_LIST */
3936
3937 kfree(named_entry, sizeof(struct vm_named_entry));
3938 } else {
3939 named_entry_unlock(named_entry);
3940 }
3941 }
3942
3943 /* Allow manipulation of individual page state. This is actually part of */
3944 /* the UPL regimen but takes place on the memory entry rather than on a UPL */
3945
3946 kern_return_t
3947 mach_memory_entry_page_op(
3948 ipc_port_t entry_port,
3949 vm_object_offset_t offset,
3950 int ops,
3951 ppnum_t *phys_entry,
3952 int *flags)
3953 {
3954 vm_named_entry_t mem_entry;
3955 vm_object_t object;
3956 kern_return_t kr;
3957
3958 if (!IP_VALID(entry_port) ||
3959 ip_kotype(entry_port) != IKOT_NAMED_ENTRY) {
3960 return KERN_INVALID_ARGUMENT;
3961 }
3962
3963 mem_entry = (vm_named_entry_t) ip_get_kobject(entry_port);
3964
3965 named_entry_lock(mem_entry);
3966
3967 if (mem_entry->is_sub_map ||
3968 mem_entry->is_copy) {
3969 named_entry_unlock(mem_entry);
3970 return KERN_INVALID_ARGUMENT;
3971 }
3972
3973 object = mem_entry->backing.object;
3974 if (object == VM_OBJECT_NULL) {
3975 named_entry_unlock(mem_entry);
3976 return KERN_INVALID_ARGUMENT;
3977 }
3978
3979 vm_object_reference(object);
3980 named_entry_unlock(mem_entry);
3981
3982 kr = vm_object_page_op(object, offset, ops, phys_entry, flags);
3983
3984 vm_object_deallocate(object);
3985
3986 return kr;
3987 }
3988
3989 /*
3990 * mach_memory_entry_range_op offers performance enhancement over
3991 * mach_memory_entry_page_op for page_op functions which do not require page
3992 * level state to be returned from the call. Page_op was created to provide
3993 * a low-cost alternative to page manipulation via UPLs when only a single
3994 * page was involved. The range_op call establishes the ability in the _op
3995 * family of functions to work on multiple pages where the lack of page level
3996 * state handling allows the caller to avoid the overhead of the upl structures.
3997 */
3998
3999 kern_return_t
4000 mach_memory_entry_range_op(
4001 ipc_port_t entry_port,
4002 vm_object_offset_t offset_beg,
4003 vm_object_offset_t offset_end,
4004 int ops,
4005 int *range)
4006 {
4007 vm_named_entry_t mem_entry;
4008 vm_object_t object;
4009 kern_return_t kr;
4010
4011 if (!IP_VALID(entry_port) ||
4012 ip_kotype(entry_port) != IKOT_NAMED_ENTRY) {
4013 return KERN_INVALID_ARGUMENT;
4014 }
4015
4016 mem_entry = (vm_named_entry_t) ip_get_kobject(entry_port);
4017
4018 named_entry_lock(mem_entry);
4019
4020 if (mem_entry->is_sub_map ||
4021 mem_entry->is_copy) {
4022 named_entry_unlock(mem_entry);
4023 return KERN_INVALID_ARGUMENT;
4024 }
4025
4026 object = mem_entry->backing.object;
4027 if (object == VM_OBJECT_NULL) {
4028 named_entry_unlock(mem_entry);
4029 return KERN_INVALID_ARGUMENT;
4030 }
4031
4032 vm_object_reference(object);
4033 named_entry_unlock(mem_entry);
4034
4035 kr = vm_object_range_op(object,
4036 offset_beg,
4037 offset_end,
4038 ops,
4039 (uint32_t *) range);
4040
4041 vm_object_deallocate(object);
4042
4043 return kr;
4044 }
4045
4046 /* ******* Temporary Internal calls to UPL for BSD ***** */
4047
4048 extern int kernel_upl_map(
4049 vm_map_t map,
4050 upl_t upl,
4051 vm_offset_t *dst_addr);
4052
4053 extern int kernel_upl_unmap(
4054 vm_map_t map,
4055 upl_t upl);
4056
4057 extern int kernel_upl_commit(
4058 upl_t upl,
4059 upl_page_info_t *pl,
4060 mach_msg_type_number_t count);
4061
4062 extern int kernel_upl_commit_range(
4063 upl_t upl,
4064 upl_offset_t offset,
4065 upl_size_t size,
4066 int flags,
4067 upl_page_info_array_t pl,
4068 mach_msg_type_number_t count);
4069
4070 extern int kernel_upl_abort(
4071 upl_t upl,
4072 int abort_type);
4073
4074 extern int kernel_upl_abort_range(
4075 upl_t upl,
4076 upl_offset_t offset,
4077 upl_size_t size,
4078 int abort_flags);
4079
4080
4081 kern_return_t
4082 kernel_upl_map(
4083 vm_map_t map,
4084 upl_t upl,
4085 vm_offset_t *dst_addr)
4086 {
4087 return vm_upl_map(map, upl, dst_addr);
4088 }
4089
4090
4091 kern_return_t
4092 kernel_upl_unmap(
4093 vm_map_t map,
4094 upl_t upl)
4095 {
4096 return vm_upl_unmap(map, upl);
4097 }
4098
4099 kern_return_t
4100 kernel_upl_commit(
4101 upl_t upl,
4102 upl_page_info_t *pl,
4103 mach_msg_type_number_t count)
4104 {
4105 kern_return_t kr;
4106
4107 kr = upl_commit(upl, pl, count);
4108 upl_deallocate(upl);
4109 return kr;
4110 }
4111
4112
4113 kern_return_t
4114 kernel_upl_commit_range(
4115 upl_t upl,
4116 upl_offset_t offset,
4117 upl_size_t size,
4118 int flags,
4119 upl_page_info_array_t pl,
4120 mach_msg_type_number_t count)
4121 {
4122 boolean_t finished = FALSE;
4123 kern_return_t kr;
4124
4125 if (flags & UPL_COMMIT_FREE_ON_EMPTY) {
4126 flags |= UPL_COMMIT_NOTIFY_EMPTY;
4127 }
4128
4129 if (flags & UPL_COMMIT_KERNEL_ONLY_FLAGS) {
4130 return KERN_INVALID_ARGUMENT;
4131 }
4132
4133 kr = upl_commit_range(upl, offset, size, flags, pl, count, &finished);
4134
4135 if ((flags & UPL_COMMIT_NOTIFY_EMPTY) && finished) {
4136 upl_deallocate(upl);
4137 }
4138
4139 return kr;
4140 }
4141
4142 kern_return_t
4143 kernel_upl_abort_range(
4144 upl_t upl,
4145 upl_offset_t offset,
4146 upl_size_t size,
4147 int abort_flags)
4148 {
4149 kern_return_t kr;
4150 boolean_t finished = FALSE;
4151
4152 if (abort_flags & UPL_COMMIT_FREE_ON_EMPTY) {
4153 abort_flags |= UPL_COMMIT_NOTIFY_EMPTY;
4154 }
4155
4156 kr = upl_abort_range(upl, offset, size, abort_flags, &finished);
4157
4158 if ((abort_flags & UPL_COMMIT_FREE_ON_EMPTY) && finished) {
4159 upl_deallocate(upl);
4160 }
4161
4162 return kr;
4163 }
4164
4165 kern_return_t
4166 kernel_upl_abort(
4167 upl_t upl,
4168 int abort_type)
4169 {
4170 kern_return_t kr;
4171
4172 kr = upl_abort(upl, abort_type);
4173 upl_deallocate(upl);
4174 return kr;
4175 }
4176
4177 /*
4178 * Now a kernel-private interface (for BootCache
4179 * use only). Need a cleaner way to create an
4180 * empty vm_map() and return a handle to it.
4181 */
4182
4183 kern_return_t
4184 vm_region_object_create(
4185 __unused vm_map_t target_map,
4186 vm_size_t size,
4187 ipc_port_t *object_handle)
4188 {
4189 vm_named_entry_t user_entry;
4190 ipc_port_t user_handle;
4191
4192 vm_map_t new_map;
4193
4194 if (mach_memory_entry_allocate(&user_entry, &user_handle)
4195 != KERN_SUCCESS) {
4196 return KERN_FAILURE;
4197 }
4198
4199 /* Create a named object based on a submap of specified size */
4200
4201 new_map = vm_map_create(PMAP_NULL, VM_MAP_MIN_ADDRESS,
4202 vm_map_round_page(size,
4203 VM_MAP_PAGE_MASK(target_map)),
4204 TRUE);
4205 vm_map_set_page_shift(new_map, VM_MAP_PAGE_SHIFT(target_map));
4206
4207 user_entry->backing.map = new_map;
4208 user_entry->internal = TRUE;
4209 user_entry->is_sub_map = TRUE;
4210 user_entry->offset = 0;
4211 user_entry->protection = VM_PROT_ALL;
4212 user_entry->size = size;
4213 assert(user_entry->ref_count == 1);
4214
4215 *object_handle = user_handle;
4216 return KERN_SUCCESS;
4217 }
4218
4219 ppnum_t vm_map_get_phys_page( /* forward */
4220 vm_map_t map,
4221 vm_offset_t offset);
4222
4223 ppnum_t
4224 vm_map_get_phys_page(
4225 vm_map_t map,
4226 vm_offset_t addr)
4227 {
4228 vm_object_offset_t offset;
4229 vm_object_t object;
4230 vm_map_offset_t map_offset;
4231 vm_map_entry_t entry;
4232 ppnum_t phys_page = 0;
4233
4234 map_offset = vm_map_trunc_page(addr, PAGE_MASK);
4235
4236 vm_map_lock(map);
4237 while (vm_map_lookup_entry(map, map_offset, &entry)) {
4238 if (VME_OBJECT(entry) == VM_OBJECT_NULL) {
4239 vm_map_unlock(map);
4240 return (ppnum_t) 0;
4241 }
4242 if (entry->is_sub_map) {
4243 vm_map_t old_map;
4244 vm_map_lock(VME_SUBMAP(entry));
4245 old_map = map;
4246 map = VME_SUBMAP(entry);
4247 map_offset = (VME_OFFSET(entry) +
4248 (map_offset - entry->vme_start));
4249 vm_map_unlock(old_map);
4250 continue;
4251 }
4252 if (VME_OBJECT(entry)->phys_contiguous) {
4253 /* These are not standard pageable memory mappings */
4254 /* If they are not present in the object they will */
4255 /* have to be picked up from the pager through the */
4256 /* fault mechanism. */
4257 if (VME_OBJECT(entry)->vo_shadow_offset == 0) {
4258 /* need to call vm_fault */
4259 vm_map_unlock(map);
4260 vm_fault(map, map_offset, VM_PROT_NONE,
4261 FALSE /* change_wiring */, VM_KERN_MEMORY_NONE,
4262 THREAD_UNINT, NULL, 0);
4263 vm_map_lock(map);
4264 continue;
4265 }
4266 offset = (VME_OFFSET(entry) +
4267 (map_offset - entry->vme_start));
4268 phys_page = (ppnum_t)
4269 ((VME_OBJECT(entry)->vo_shadow_offset
4270 + offset) >> PAGE_SHIFT);
4271 break;
4272 }
4273 offset = (VME_OFFSET(entry) + (map_offset - entry->vme_start));
4274 object = VME_OBJECT(entry);
4275 vm_object_lock(object);
4276 while (TRUE) {
4277 vm_page_t dst_page = vm_page_lookup(object, offset);
4278 if (dst_page == VM_PAGE_NULL) {
4279 if (object->shadow) {
4280 vm_object_t old_object;
4281 vm_object_lock(object->shadow);
4282 old_object = object;
4283 offset = offset + object->vo_shadow_offset;
4284 object = object->shadow;
4285 vm_object_unlock(old_object);
4286 } else {
4287 vm_object_unlock(object);
4288 break;
4289 }
4290 } else {
4291 phys_page = (ppnum_t)(VM_PAGE_GET_PHYS_PAGE(dst_page));
4292 vm_object_unlock(object);
4293 break;
4294 }
4295 }
4296 break;
4297 }
4298
4299 vm_map_unlock(map);
4300 return phys_page;
4301 }
4302
4303 #if 0
4304 kern_return_t kernel_object_iopl_request( /* forward */
4305 vm_named_entry_t named_entry,
4306 memory_object_offset_t offset,
4307 upl_size_t *upl_size,
4308 upl_t *upl_ptr,
4309 upl_page_info_array_t user_page_list,
4310 unsigned int *page_list_count,
4311 int *flags);
4312
4313 kern_return_t
4314 kernel_object_iopl_request(
4315 vm_named_entry_t named_entry,
4316 memory_object_offset_t offset,
4317 upl_size_t *upl_size,
4318 upl_t *upl_ptr,
4319 upl_page_info_array_t user_page_list,
4320 unsigned int *page_list_count,
4321 int *flags)
4322 {
4323 vm_object_t object;
4324 kern_return_t ret;
4325
4326 int caller_flags;
4327
4328 caller_flags = *flags;
4329
4330 if (caller_flags & ~UPL_VALID_FLAGS) {
4331 /*
4332 * For forward compatibility's sake,
4333 * reject any unknown flag.
4334 */
4335 return KERN_INVALID_VALUE;
4336 }
4337
4338 /* a few checks to make sure user is obeying rules */
4339 if (*upl_size == 0) {
4340 if (offset >= named_entry->size) {
4341 return KERN_INVALID_RIGHT;
4342 }
4343 *upl_size = (upl_size_t) (named_entry->size - offset);
4344 if (*upl_size != named_entry->size - offset) {
4345 return KERN_INVALID_ARGUMENT;
4346 }
4347 }
4348 if (caller_flags & UPL_COPYOUT_FROM) {
4349 if ((named_entry->protection & VM_PROT_READ)
4350 != VM_PROT_READ) {
4351 return KERN_INVALID_RIGHT;
4352 }
4353 } else {
4354 if ((named_entry->protection &
4355 (VM_PROT_READ | VM_PROT_WRITE))
4356 != (VM_PROT_READ | VM_PROT_WRITE)) {
4357 return KERN_INVALID_RIGHT;
4358 }
4359 }
4360 if (named_entry->size < (offset + *upl_size)) {
4361 return KERN_INVALID_ARGUMENT;
4362 }
4363
4364 /* the callers parameter offset is defined to be the */
4365 /* offset from beginning of named entry offset in object */
4366 offset = offset + named_entry->offset;
4367
4368 if (named_entry->is_sub_map ||
4369 named_entry->is_copy) {
4370 return KERN_INVALID_ARGUMENT;
4371 }
4372
4373 named_entry_lock(named_entry);
4374
4375 /* This is the case where we are going to operate */
4376 /* on an already known object. If the object is */
4377 /* not ready it is internal. An external */
4378 /* object cannot be mapped until it is ready */
4379 /* we can therefore avoid the ready check */
4380 /* in this case. */
4381 object = named_entry->backing.object;
4382 vm_object_reference(object);
4383 named_entry_unlock(named_entry);
4384
4385 if (!object->private) {
4386 if (*upl_size > MAX_UPL_TRANSFER_BYTES) {
4387 *upl_size = MAX_UPL_TRANSFER_BYTES;
4388 }
4389 if (object->phys_contiguous) {
4390 *flags = UPL_PHYS_CONTIG;
4391 } else {
4392 *flags = 0;
4393 }
4394 } else {
4395 *flags = UPL_DEV_MEMORY | UPL_PHYS_CONTIG;
4396 }
4397
4398 ret = vm_object_iopl_request(object,
4399 offset,
4400 *upl_size,
4401 upl_ptr,
4402 user_page_list,
4403 page_list_count,
4404 (upl_control_flags_t)(unsigned int)caller_flags);
4405 vm_object_deallocate(object);
4406 return ret;
4407 }
4408 #endif
4409
4410 /*
4411 * These symbols are looked up at runtime by vmware, VirtualBox,
4412 * despite not being exported in the symbol sets.
4413 */
4414
4415 #if defined(__x86_64__)
4416
4417 kern_return_t
4418 mach_vm_map(
4419 vm_map_t target_map,
4420 mach_vm_offset_t *address,
4421 mach_vm_size_t initial_size,
4422 mach_vm_offset_t mask,
4423 int flags,
4424 ipc_port_t port,
4425 vm_object_offset_t offset,
4426 boolean_t copy,
4427 vm_prot_t cur_protection,
4428 vm_prot_t max_protection,
4429 vm_inherit_t inheritance);
4430
4431 kern_return_t
4432 mach_vm_remap(
4433 vm_map_t target_map,
4434 mach_vm_offset_t *address,
4435 mach_vm_size_t size,
4436 mach_vm_offset_t mask,
4437 int flags,
4438 vm_map_t src_map,
4439 mach_vm_offset_t memory_address,
4440 boolean_t copy,
4441 vm_prot_t *cur_protection,
4442 vm_prot_t *max_protection,
4443 vm_inherit_t inheritance);
4444
4445 kern_return_t
4446 mach_vm_map(
4447 vm_map_t target_map,
4448 mach_vm_offset_t *address,
4449 mach_vm_size_t initial_size,
4450 mach_vm_offset_t mask,
4451 int flags,
4452 ipc_port_t port,
4453 vm_object_offset_t offset,
4454 boolean_t copy,
4455 vm_prot_t cur_protection,
4456 vm_prot_t max_protection,
4457 vm_inherit_t inheritance)
4458 {
4459 return mach_vm_map_external(target_map, address, initial_size, mask, flags, port,
4460 offset, copy, cur_protection, max_protection, inheritance);
4461 }
4462
4463 kern_return_t
4464 mach_vm_remap(
4465 vm_map_t target_map,
4466 mach_vm_offset_t *address,
4467 mach_vm_size_t size,
4468 mach_vm_offset_t mask,
4469 int flags,
4470 vm_map_t src_map,
4471 mach_vm_offset_t memory_address,
4472 boolean_t copy,
4473 vm_prot_t *cur_protection,
4474 vm_prot_t *max_protection,
4475 vm_inherit_t inheritance)
4476 {
4477 return mach_vm_remap_external(target_map, address, size, mask, flags, src_map, memory_address,
4478 copy, cur_protection, max_protection, inheritance);
4479 }
4480
4481 kern_return_t
4482 vm_map(
4483 vm_map_t target_map,
4484 vm_offset_t *address,
4485 vm_size_t size,
4486 vm_offset_t mask,
4487 int flags,
4488 ipc_port_t port,
4489 vm_offset_t offset,
4490 boolean_t copy,
4491 vm_prot_t cur_protection,
4492 vm_prot_t max_protection,
4493 vm_inherit_t inheritance);
4494
4495 kern_return_t
4496 vm_map(
4497 vm_map_t target_map,
4498 vm_offset_t *address,
4499 vm_size_t size,
4500 vm_offset_t mask,
4501 int flags,
4502 ipc_port_t port,
4503 vm_offset_t offset,
4504 boolean_t copy,
4505 vm_prot_t cur_protection,
4506 vm_prot_t max_protection,
4507 vm_inherit_t inheritance)
4508 {
4509 vm_tag_t tag;
4510
4511 VM_GET_FLAGS_ALIAS(flags, tag);
4512 return vm_map_kernel(target_map, address, size, mask,
4513 flags, VM_MAP_KERNEL_FLAGS_NONE, tag,
4514 port, offset, copy,
4515 cur_protection, max_protection, inheritance);
4516 }
4517
4518 #endif /* __x86_64__ */
mirror of XNU