]> git.saurik.com Git - apple/xnu.git/blob - osfmk/vm/vm_user.c
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-4570.31.3.tar.gz
[apple/xnu.git] / osfmk / vm / vm_user.c
1 /*
2 * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56 /*
57 */
58 /*
59 * File: vm/vm_user.c
60 * Author: Avadis Tevanian, Jr., Michael Wayne Young
61 *
62 * User-exported virtual memory functions.
63 */
64
65 /*
66 * There are three implementations of the "XXX_allocate" functionality in
67 * the kernel: mach_vm_allocate (for any task on the platform), vm_allocate
68 * (for a task with the same address space size, especially the current task),
69 * and vm32_vm_allocate (for the specific case of a 32-bit task). vm_allocate
70 * in the kernel should only be used on the kernel_task. vm32_vm_allocate only
71 * makes sense on platforms where a user task can either be 32 or 64, or the kernel
72 * task can be 32 or 64. mach_vm_allocate makes sense everywhere, and is preferred
73 * for new code.
74 *
75 * The entrypoints into the kernel are more complex. All platforms support a
76 * mach_vm_allocate-style API (subsystem 4800) which operates with the largest
77 * size types for the platform. On platforms that only support U32/K32,
78 * subsystem 4800 is all you need. On platforms that support both U32 and U64,
79 * subsystem 3800 is used disambiguate the size of parameters, and they will
80 * always be 32-bit and call into the vm32_vm_allocate APIs. On non-U32/K32 platforms,
81 * the MIG glue should never call into vm_allocate directly, because the calling
82 * task and kernel_task are unlikely to use the same size parameters
83 *
84 * New VM call implementations should be added here and to mach_vm.defs
85 * (subsystem 4800), and use mach_vm_* "wide" types.
86 */
87
88 #include <debug.h>
89
90 #include <vm_cpm.h>
91 #include <mach/boolean.h>
92 #include <mach/kern_return.h>
93 #include <mach/mach_types.h> /* to get vm_address_t */
94 #include <mach/memory_object.h>
95 #include <mach/std_types.h> /* to get pointer_t */
96 #include <mach/upl.h>
97 #include <mach/vm_attributes.h>
98 #include <mach/vm_param.h>
99 #include <mach/vm_statistics.h>
100 #include <mach/mach_syscalls.h>
101 #include <mach/sdt.h>
102
103 #include <mach/host_priv_server.h>
104 #include <mach/mach_vm_server.h>
105 #include <mach/vm_map_server.h>
106
107 #include <kern/host.h>
108 #include <kern/kalloc.h>
109 #include <kern/task.h>
110 #include <kern/misc_protos.h>
111 #include <vm/vm_fault.h>
112 #include <vm/vm_map.h>
113 #include <vm/vm_object.h>
114 #include <vm/vm_page.h>
115 #include <vm/memory_object.h>
116 #include <vm/vm_pageout.h>
117 #include <vm/vm_protos.h>
118 #include <vm/vm_purgeable_internal.h>
119 #include <vm/vm_init.h>
120
121 #include <san/kasan.h>
122
123 vm_size_t upl_offset_to_pagelist = 0;
124
125 #if VM_CPM
126 #include <vm/cpm.h>
127 #endif /* VM_CPM */
128
129 /*
130 * mach_vm_allocate allocates "zero fill" memory in the specfied
131 * map.
132 */
133 kern_return_t
134 mach_vm_allocate_external(
135 vm_map_t map,
136 mach_vm_offset_t *addr,
137 mach_vm_size_t size,
138 int flags)
139 {
140 vm_tag_t tag;
141
142 VM_GET_FLAGS_ALIAS(flags, tag);
143 return (mach_vm_allocate_kernel(map, addr, size, flags, tag));
144 }
145
146 kern_return_t
147 mach_vm_allocate_kernel(
148 vm_map_t map,
149 mach_vm_offset_t *addr,
150 mach_vm_size_t size,
151 int flags,
152 vm_tag_t tag)
153 {
154 vm_map_offset_t map_addr;
155 vm_map_size_t map_size;
156 kern_return_t result;
157 boolean_t anywhere;
158
159 /* filter out any kernel-only flags */
160 if (flags & ~VM_FLAGS_USER_ALLOCATE)
161 return KERN_INVALID_ARGUMENT;
162
163 if (map == VM_MAP_NULL)
164 return(KERN_INVALID_ARGUMENT);
165 if (size == 0) {
166 *addr = 0;
167 return(KERN_SUCCESS);
168 }
169
170 anywhere = ((VM_FLAGS_ANYWHERE & flags) != 0);
171 if (anywhere) {
172 /*
173 * No specific address requested, so start candidate address
174 * search at the minimum address in the map. However, if that
175 * minimum is 0, bump it up by PAGE_SIZE. We want to limit
176 * allocations of PAGEZERO to explicit requests since its
177 * normal use is to catch dereferences of NULL and many
178 * applications also treat pointers with a value of 0 as
179 * special and suddenly having address 0 contain useable
180 * memory would tend to confuse those applications.
181 */
182 map_addr = vm_map_min(map);
183 if (map_addr == 0)
184 map_addr += VM_MAP_PAGE_SIZE(map);
185 } else
186 map_addr = vm_map_trunc_page(*addr,
187 VM_MAP_PAGE_MASK(map));
188 map_size = vm_map_round_page(size,
189 VM_MAP_PAGE_MASK(map));
190 if (map_size == 0) {
191 return(KERN_INVALID_ARGUMENT);
192 }
193
194 result = vm_map_enter(
195 map,
196 &map_addr,
197 map_size,
198 (vm_map_offset_t)0,
199 flags,
200 VM_MAP_KERNEL_FLAGS_NONE,
201 tag,
202 VM_OBJECT_NULL,
203 (vm_object_offset_t)0,
204 FALSE,
205 VM_PROT_DEFAULT,
206 VM_PROT_ALL,
207 VM_INHERIT_DEFAULT);
208
209 *addr = map_addr;
210 return(result);
211 }
212
213 /*
214 * vm_allocate
215 * Legacy routine that allocates "zero fill" memory in the specfied
216 * map (which is limited to the same size as the kernel).
217 */
218 kern_return_t
219 vm_allocate_external(
220 vm_map_t map,
221 vm_offset_t *addr,
222 vm_size_t size,
223 int flags)
224 {
225 vm_tag_t tag;
226
227 VM_GET_FLAGS_ALIAS(flags, tag);
228 return (vm_allocate_kernel(map, addr, size, flags, tag));
229 }
230
231 kern_return_t
232 vm_allocate_kernel(
233 vm_map_t map,
234 vm_offset_t *addr,
235 vm_size_t size,
236 int flags,
237 vm_tag_t tag)
238 {
239 vm_map_offset_t map_addr;
240 vm_map_size_t map_size;
241 kern_return_t result;
242 boolean_t anywhere;
243
244 /* filter out any kernel-only flags */
245 if (flags & ~VM_FLAGS_USER_ALLOCATE)
246 return KERN_INVALID_ARGUMENT;
247
248 if (map == VM_MAP_NULL)
249 return(KERN_INVALID_ARGUMENT);
250 if (size == 0) {
251 *addr = 0;
252 return(KERN_SUCCESS);
253 }
254
255 anywhere = ((VM_FLAGS_ANYWHERE & flags) != 0);
256 if (anywhere) {
257 /*
258 * No specific address requested, so start candidate address
259 * search at the minimum address in the map. However, if that
260 * minimum is 0, bump it up by PAGE_SIZE. We want to limit
261 * allocations of PAGEZERO to explicit requests since its
262 * normal use is to catch dereferences of NULL and many
263 * applications also treat pointers with a value of 0 as
264 * special and suddenly having address 0 contain useable
265 * memory would tend to confuse those applications.
266 */
267 map_addr = vm_map_min(map);
268 if (map_addr == 0)
269 map_addr += VM_MAP_PAGE_SIZE(map);
270 } else
271 map_addr = vm_map_trunc_page(*addr,
272 VM_MAP_PAGE_MASK(map));
273 map_size = vm_map_round_page(size,
274 VM_MAP_PAGE_MASK(map));
275 if (map_size == 0) {
276 return(KERN_INVALID_ARGUMENT);
277 }
278
279 result = vm_map_enter(
280 map,
281 &map_addr,
282 map_size,
283 (vm_map_offset_t)0,
284 flags,
285 VM_MAP_KERNEL_FLAGS_NONE,
286 tag,
287 VM_OBJECT_NULL,
288 (vm_object_offset_t)0,
289 FALSE,
290 VM_PROT_DEFAULT,
291 VM_PROT_ALL,
292 VM_INHERIT_DEFAULT);
293
294 #if KASAN
295 if (result == KERN_SUCCESS && map->pmap == kernel_pmap) {
296 kasan_notify_address(map_addr, map_size);
297 }
298 #endif
299
300 *addr = CAST_DOWN(vm_offset_t, map_addr);
301 return(result);
302 }
303
304 /*
305 * mach_vm_deallocate -
306 * deallocates the specified range of addresses in the
307 * specified address map.
308 */
309 kern_return_t
310 mach_vm_deallocate(
311 vm_map_t map,
312 mach_vm_offset_t start,
313 mach_vm_size_t size)
314 {
315 if ((map == VM_MAP_NULL) || (start + size < start))
316 return(KERN_INVALID_ARGUMENT);
317
318 if (size == (mach_vm_offset_t) 0)
319 return(KERN_SUCCESS);
320
321 return(vm_map_remove(map,
322 vm_map_trunc_page(start,
323 VM_MAP_PAGE_MASK(map)),
324 vm_map_round_page(start+size,
325 VM_MAP_PAGE_MASK(map)),
326 VM_MAP_NO_FLAGS));
327 }
328
329 /*
330 * vm_deallocate -
331 * deallocates the specified range of addresses in the
332 * specified address map (limited to addresses the same
333 * size as the kernel).
334 */
335 kern_return_t
336 vm_deallocate(
337 vm_map_t map,
338 vm_offset_t start,
339 vm_size_t size)
340 {
341 if ((map == VM_MAP_NULL) || (start + size < start))
342 return(KERN_INVALID_ARGUMENT);
343
344 if (size == (vm_offset_t) 0)
345 return(KERN_SUCCESS);
346
347 return(vm_map_remove(map,
348 vm_map_trunc_page(start,
349 VM_MAP_PAGE_MASK(map)),
350 vm_map_round_page(start+size,
351 VM_MAP_PAGE_MASK(map)),
352 VM_MAP_NO_FLAGS));
353 }
354
355 /*
356 * mach_vm_inherit -
357 * Sets the inheritance of the specified range in the
358 * specified map.
359 */
360 kern_return_t
361 mach_vm_inherit(
362 vm_map_t map,
363 mach_vm_offset_t start,
364 mach_vm_size_t size,
365 vm_inherit_t new_inheritance)
366 {
367 if ((map == VM_MAP_NULL) || (start + size < start) ||
368 (new_inheritance > VM_INHERIT_LAST_VALID))
369 return(KERN_INVALID_ARGUMENT);
370
371 if (size == 0)
372 return KERN_SUCCESS;
373
374 return(vm_map_inherit(map,
375 vm_map_trunc_page(start,
376 VM_MAP_PAGE_MASK(map)),
377 vm_map_round_page(start+size,
378 VM_MAP_PAGE_MASK(map)),
379 new_inheritance));
380 }
381
382 /*
383 * vm_inherit -
384 * Sets the inheritance of the specified range in the
385 * specified map (range limited to addresses
386 */
387 kern_return_t
388 vm_inherit(
389 vm_map_t map,
390 vm_offset_t start,
391 vm_size_t size,
392 vm_inherit_t new_inheritance)
393 {
394 if ((map == VM_MAP_NULL) || (start + size < start) ||
395 (new_inheritance > VM_INHERIT_LAST_VALID))
396 return(KERN_INVALID_ARGUMENT);
397
398 if (size == 0)
399 return KERN_SUCCESS;
400
401 return(vm_map_inherit(map,
402 vm_map_trunc_page(start,
403 VM_MAP_PAGE_MASK(map)),
404 vm_map_round_page(start+size,
405 VM_MAP_PAGE_MASK(map)),
406 new_inheritance));
407 }
408
409 /*
410 * mach_vm_protect -
411 * Sets the protection of the specified range in the
412 * specified map.
413 */
414
415 kern_return_t
416 mach_vm_protect(
417 vm_map_t map,
418 mach_vm_offset_t start,
419 mach_vm_size_t size,
420 boolean_t set_maximum,
421 vm_prot_t new_protection)
422 {
423 if ((map == VM_MAP_NULL) || (start + size < start) ||
424 (new_protection & ~(VM_PROT_ALL | VM_PROT_COPY)))
425 return(KERN_INVALID_ARGUMENT);
426
427 if (size == 0)
428 return KERN_SUCCESS;
429
430 return(vm_map_protect(map,
431 vm_map_trunc_page(start,
432 VM_MAP_PAGE_MASK(map)),
433 vm_map_round_page(start+size,
434 VM_MAP_PAGE_MASK(map)),
435 new_protection,
436 set_maximum));
437 }
438
439 /*
440 * vm_protect -
441 * Sets the protection of the specified range in the
442 * specified map. Addressability of the range limited
443 * to the same size as the kernel.
444 */
445
446 kern_return_t
447 vm_protect(
448 vm_map_t map,
449 vm_offset_t start,
450 vm_size_t size,
451 boolean_t set_maximum,
452 vm_prot_t new_protection)
453 {
454 if ((map == VM_MAP_NULL) || (start + size < start) ||
455 (new_protection & ~(VM_PROT_ALL | VM_PROT_COPY)))
456 return(KERN_INVALID_ARGUMENT);
457
458 if (size == 0)
459 return KERN_SUCCESS;
460
461 return(vm_map_protect(map,
462 vm_map_trunc_page(start,
463 VM_MAP_PAGE_MASK(map)),
464 vm_map_round_page(start+size,
465 VM_MAP_PAGE_MASK(map)),
466 new_protection,
467 set_maximum));
468 }
469
470 /*
471 * mach_vm_machine_attributes -
472 * Handle machine-specific attributes for a mapping, such
473 * as cachability, migrability, etc.
474 */
475 kern_return_t
476 mach_vm_machine_attribute(
477 vm_map_t map,
478 mach_vm_address_t addr,
479 mach_vm_size_t size,
480 vm_machine_attribute_t attribute,
481 vm_machine_attribute_val_t* value) /* IN/OUT */
482 {
483 if ((map == VM_MAP_NULL) || (addr + size < addr))
484 return(KERN_INVALID_ARGUMENT);
485
486 if (size == 0)
487 return KERN_SUCCESS;
488
489 return vm_map_machine_attribute(
490 map,
491 vm_map_trunc_page(addr,
492 VM_MAP_PAGE_MASK(map)),
493 vm_map_round_page(addr+size,
494 VM_MAP_PAGE_MASK(map)),
495 attribute,
496 value);
497 }
498
499 /*
500 * vm_machine_attribute -
501 * Handle machine-specific attributes for a mapping, such
502 * as cachability, migrability, etc. Limited addressability
503 * (same range limits as for the native kernel map).
504 */
505 kern_return_t
506 vm_machine_attribute(
507 vm_map_t map,
508 vm_address_t addr,
509 vm_size_t size,
510 vm_machine_attribute_t attribute,
511 vm_machine_attribute_val_t* value) /* IN/OUT */
512 {
513 if ((map == VM_MAP_NULL) || (addr + size < addr))
514 return(KERN_INVALID_ARGUMENT);
515
516 if (size == 0)
517 return KERN_SUCCESS;
518
519 return vm_map_machine_attribute(
520 map,
521 vm_map_trunc_page(addr,
522 VM_MAP_PAGE_MASK(map)),
523 vm_map_round_page(addr+size,
524 VM_MAP_PAGE_MASK(map)),
525 attribute,
526 value);
527 }
528
529 /*
530 * mach_vm_read -
531 * Read/copy a range from one address space and return it to the caller.
532 *
533 * It is assumed that the address for the returned memory is selected by
534 * the IPC implementation as part of receiving the reply to this call.
535 * If IPC isn't used, the caller must deal with the vm_map_copy_t object
536 * that gets returned.
537 *
538 * JMM - because of mach_msg_type_number_t, this call is limited to a
539 * single 4GB region at this time.
540 *
541 */
542 kern_return_t
543 mach_vm_read(
544 vm_map_t map,
545 mach_vm_address_t addr,
546 mach_vm_size_t size,
547 pointer_t *data,
548 mach_msg_type_number_t *data_size)
549 {
550 kern_return_t error;
551 vm_map_copy_t ipc_address;
552
553 if (map == VM_MAP_NULL)
554 return(KERN_INVALID_ARGUMENT);
555
556 if ((mach_msg_type_number_t) size != size)
557 return KERN_INVALID_ARGUMENT;
558
559 error = vm_map_copyin(map,
560 (vm_map_address_t)addr,
561 (vm_map_size_t)size,
562 FALSE, /* src_destroy */
563 &ipc_address);
564
565 if (KERN_SUCCESS == error) {
566 *data = (pointer_t) ipc_address;
567 *data_size = (mach_msg_type_number_t) size;
568 assert(*data_size == size);
569 }
570 return(error);
571 }
572
573 /*
574 * vm_read -
575 * Read/copy a range from one address space and return it to the caller.
576 * Limited addressability (same range limits as for the native kernel map).
577 *
578 * It is assumed that the address for the returned memory is selected by
579 * the IPC implementation as part of receiving the reply to this call.
580 * If IPC isn't used, the caller must deal with the vm_map_copy_t object
581 * that gets returned.
582 */
583 kern_return_t
584 vm_read(
585 vm_map_t map,
586 vm_address_t addr,
587 vm_size_t size,
588 pointer_t *data,
589 mach_msg_type_number_t *data_size)
590 {
591 kern_return_t error;
592 vm_map_copy_t ipc_address;
593
594 if (map == VM_MAP_NULL)
595 return(KERN_INVALID_ARGUMENT);
596
597 if (size > (unsigned)(mach_msg_type_number_t) -1) {
598 /*
599 * The kernel could handle a 64-bit "size" value, but
600 * it could not return the size of the data in "*data_size"
601 * without overflowing.
602 * Let's reject this "size" as invalid.
603 */
604 return KERN_INVALID_ARGUMENT;
605 }
606
607 error = vm_map_copyin(map,
608 (vm_map_address_t)addr,
609 (vm_map_size_t)size,
610 FALSE, /* src_destroy */
611 &ipc_address);
612
613 if (KERN_SUCCESS == error) {
614 *data = (pointer_t) ipc_address;
615 *data_size = (mach_msg_type_number_t) size;
616 assert(*data_size == size);
617 }
618 return(error);
619 }
620
621 /*
622 * mach_vm_read_list -
623 * Read/copy a list of address ranges from specified map.
624 *
625 * MIG does not know how to deal with a returned array of
626 * vm_map_copy_t structures, so we have to do the copyout
627 * manually here.
628 */
629 kern_return_t
630 mach_vm_read_list(
631 vm_map_t map,
632 mach_vm_read_entry_t data_list,
633 natural_t count)
634 {
635 mach_msg_type_number_t i;
636 kern_return_t error;
637 vm_map_copy_t copy;
638
639 if (map == VM_MAP_NULL ||
640 count > VM_MAP_ENTRY_MAX)
641 return(KERN_INVALID_ARGUMENT);
642
643 error = KERN_SUCCESS;
644 for(i=0; i<count; i++) {
645 vm_map_address_t map_addr;
646 vm_map_size_t map_size;
647
648 map_addr = (vm_map_address_t)(data_list[i].address);
649 map_size = (vm_map_size_t)(data_list[i].size);
650
651 if(map_size != 0) {
652 error = vm_map_copyin(map,
653 map_addr,
654 map_size,
655 FALSE, /* src_destroy */
656 &copy);
657 if (KERN_SUCCESS == error) {
658 error = vm_map_copyout(
659 current_task()->map,
660 &map_addr,
661 copy);
662 if (KERN_SUCCESS == error) {
663 data_list[i].address = map_addr;
664 continue;
665 }
666 vm_map_copy_discard(copy);
667 }
668 }
669 data_list[i].address = (mach_vm_address_t)0;
670 data_list[i].size = (mach_vm_size_t)0;
671 }
672 return(error);
673 }
674
675 /*
676 * vm_read_list -
677 * Read/copy a list of address ranges from specified map.
678 *
679 * MIG does not know how to deal with a returned array of
680 * vm_map_copy_t structures, so we have to do the copyout
681 * manually here.
682 *
683 * The source and destination ranges are limited to those
684 * that can be described with a vm_address_t (i.e. same
685 * size map as the kernel).
686 *
687 * JMM - If the result of the copyout is an address range
688 * that cannot be described with a vm_address_t (i.e. the
689 * caller had a larger address space but used this call
690 * anyway), it will result in a truncated address being
691 * returned (and a likely confused caller).
692 */
693
694 kern_return_t
695 vm_read_list(
696 vm_map_t map,
697 vm_read_entry_t data_list,
698 natural_t count)
699 {
700 mach_msg_type_number_t i;
701 kern_return_t error;
702 vm_map_copy_t copy;
703
704 if (map == VM_MAP_NULL ||
705 count > VM_MAP_ENTRY_MAX)
706 return(KERN_INVALID_ARGUMENT);
707
708 error = KERN_SUCCESS;
709 for(i=0; i<count; i++) {
710 vm_map_address_t map_addr;
711 vm_map_size_t map_size;
712
713 map_addr = (vm_map_address_t)(data_list[i].address);
714 map_size = (vm_map_size_t)(data_list[i].size);
715
716 if(map_size != 0) {
717 error = vm_map_copyin(map,
718 map_addr,
719 map_size,
720 FALSE, /* src_destroy */
721 &copy);
722 if (KERN_SUCCESS == error) {
723 error = vm_map_copyout(current_task()->map,
724 &map_addr,
725 copy);
726 if (KERN_SUCCESS == error) {
727 data_list[i].address =
728 CAST_DOWN(vm_offset_t, map_addr);
729 continue;
730 }
731 vm_map_copy_discard(copy);
732 }
733 }
734 data_list[i].address = (mach_vm_address_t)0;
735 data_list[i].size = (mach_vm_size_t)0;
736 }
737 return(error);
738 }
739
740 /*
741 * mach_vm_read_overwrite -
742 * Overwrite a range of the current map with data from the specified
743 * map/address range.
744 *
745 * In making an assumption that the current thread is local, it is
746 * no longer cluster-safe without a fully supportive local proxy
747 * thread/task (but we don't support cluster's anymore so this is moot).
748 */
749
750 kern_return_t
751 mach_vm_read_overwrite(
752 vm_map_t map,
753 mach_vm_address_t address,
754 mach_vm_size_t size,
755 mach_vm_address_t data,
756 mach_vm_size_t *data_size)
757 {
758 kern_return_t error;
759 vm_map_copy_t copy;
760
761 if (map == VM_MAP_NULL)
762 return(KERN_INVALID_ARGUMENT);
763
764 error = vm_map_copyin(map, (vm_map_address_t)address,
765 (vm_map_size_t)size, FALSE, &copy);
766
767 if (KERN_SUCCESS == error) {
768 error = vm_map_copy_overwrite(current_thread()->map,
769 (vm_map_address_t)data,
770 copy, FALSE);
771 if (KERN_SUCCESS == error) {
772 *data_size = size;
773 return error;
774 }
775 vm_map_copy_discard(copy);
776 }
777 return(error);
778 }
779
780 /*
781 * vm_read_overwrite -
782 * Overwrite a range of the current map with data from the specified
783 * map/address range.
784 *
785 * This routine adds the additional limitation that the source and
786 * destination ranges must be describable with vm_address_t values
787 * (i.e. the same size address spaces as the kernel, or at least the
788 * the ranges are in that first portion of the respective address
789 * spaces).
790 */
791
792 kern_return_t
793 vm_read_overwrite(
794 vm_map_t map,
795 vm_address_t address,
796 vm_size_t size,
797 vm_address_t data,
798 vm_size_t *data_size)
799 {
800 kern_return_t error;
801 vm_map_copy_t copy;
802
803 if (map == VM_MAP_NULL)
804 return(KERN_INVALID_ARGUMENT);
805
806 error = vm_map_copyin(map, (vm_map_address_t)address,
807 (vm_map_size_t)size, FALSE, &copy);
808
809 if (KERN_SUCCESS == error) {
810 error = vm_map_copy_overwrite(current_thread()->map,
811 (vm_map_address_t)data,
812 copy, FALSE);
813 if (KERN_SUCCESS == error) {
814 *data_size = size;
815 return error;
816 }
817 vm_map_copy_discard(copy);
818 }
819 return(error);
820 }
821
822
823 /*
824 * mach_vm_write -
825 * Overwrite the specified address range with the data provided
826 * (from the current map).
827 */
828 kern_return_t
829 mach_vm_write(
830 vm_map_t map,
831 mach_vm_address_t address,
832 pointer_t data,
833 __unused mach_msg_type_number_t size)
834 {
835 if (map == VM_MAP_NULL)
836 return KERN_INVALID_ARGUMENT;
837
838 return vm_map_copy_overwrite(map, (vm_map_address_t)address,
839 (vm_map_copy_t) data, FALSE /* interruptible XXX */);
840 }
841
842 /*
843 * vm_write -
844 * Overwrite the specified address range with the data provided
845 * (from the current map).
846 *
847 * The addressability of the range of addresses to overwrite is
848 * limited bu the use of a vm_address_t (same size as kernel map).
849 * Either the target map is also small, or the range is in the
850 * low addresses within it.
851 */
852 kern_return_t
853 vm_write(
854 vm_map_t map,
855 vm_address_t address,
856 pointer_t data,
857 __unused mach_msg_type_number_t size)
858 {
859 if (map == VM_MAP_NULL)
860 return KERN_INVALID_ARGUMENT;
861
862 return vm_map_copy_overwrite(map, (vm_map_address_t)address,
863 (vm_map_copy_t) data, FALSE /* interruptible XXX */);
864 }
865
866 /*
867 * mach_vm_copy -
868 * Overwrite one range of the specified map with the contents of
869 * another range within that same map (i.e. both address ranges
870 * are "over there").
871 */
872 kern_return_t
873 mach_vm_copy(
874 vm_map_t map,
875 mach_vm_address_t source_address,
876 mach_vm_size_t size,
877 mach_vm_address_t dest_address)
878 {
879 vm_map_copy_t copy;
880 kern_return_t kr;
881
882 if (map == VM_MAP_NULL)
883 return KERN_INVALID_ARGUMENT;
884
885 kr = vm_map_copyin(map, (vm_map_address_t)source_address,
886 (vm_map_size_t)size, FALSE, &copy);
887
888 if (KERN_SUCCESS == kr) {
889 kr = vm_map_copy_overwrite(map,
890 (vm_map_address_t)dest_address,
891 copy, FALSE /* interruptible XXX */);
892
893 if (KERN_SUCCESS != kr)
894 vm_map_copy_discard(copy);
895 }
896 return kr;
897 }
898
899 kern_return_t
900 vm_copy(
901 vm_map_t map,
902 vm_address_t source_address,
903 vm_size_t size,
904 vm_address_t dest_address)
905 {
906 vm_map_copy_t copy;
907 kern_return_t kr;
908
909 if (map == VM_MAP_NULL)
910 return KERN_INVALID_ARGUMENT;
911
912 kr = vm_map_copyin(map, (vm_map_address_t)source_address,
913 (vm_map_size_t)size, FALSE, &copy);
914
915 if (KERN_SUCCESS == kr) {
916 kr = vm_map_copy_overwrite(map,
917 (vm_map_address_t)dest_address,
918 copy, FALSE /* interruptible XXX */);
919
920 if (KERN_SUCCESS != kr)
921 vm_map_copy_discard(copy);
922 }
923 return kr;
924 }
925
926 /*
927 * mach_vm_map -
928 * Map some range of an object into an address space.
929 *
930 * The object can be one of several types of objects:
931 * NULL - anonymous memory
932 * a named entry - a range within another address space
933 * or a range within a memory object
934 * a whole memory object
935 *
936 */
937 kern_return_t
938 mach_vm_map_external(
939 vm_map_t target_map,
940 mach_vm_offset_t *address,
941 mach_vm_size_t initial_size,
942 mach_vm_offset_t mask,
943 int flags,
944 ipc_port_t port,
945 vm_object_offset_t offset,
946 boolean_t copy,
947 vm_prot_t cur_protection,
948 vm_prot_t max_protection,
949 vm_inherit_t inheritance)
950 {
951 vm_tag_t tag;
952
953 VM_GET_FLAGS_ALIAS(flags, tag);
954 return (mach_vm_map_kernel(target_map, address, initial_size, mask, flags, tag, port,
955 offset, copy, cur_protection, max_protection, inheritance));
956 }
957
958 kern_return_t
959 mach_vm_map_kernel(
960 vm_map_t target_map,
961 mach_vm_offset_t *address,
962 mach_vm_size_t initial_size,
963 mach_vm_offset_t mask,
964 int flags,
965 vm_tag_t tag,
966 ipc_port_t port,
967 vm_object_offset_t offset,
968 boolean_t copy,
969 vm_prot_t cur_protection,
970 vm_prot_t max_protection,
971 vm_inherit_t inheritance)
972 {
973 kern_return_t kr;
974 vm_map_offset_t vmmaddr;
975
976 vmmaddr = (vm_map_offset_t) *address;
977
978 /* filter out any kernel-only flags */
979 if (flags & ~VM_FLAGS_USER_MAP)
980 return KERN_INVALID_ARGUMENT;
981
982 kr = vm_map_enter_mem_object(target_map,
983 &vmmaddr,
984 initial_size,
985 mask,
986 flags,
987 VM_MAP_KERNEL_FLAGS_NONE,
988 tag,
989 port,
990 offset,
991 copy,
992 cur_protection,
993 max_protection,
994 inheritance);
995
996 #if KASAN
997 if (kr == KERN_SUCCESS && target_map->pmap == kernel_pmap) {
998 kasan_notify_address(vmmaddr, initial_size);
999 }
1000 #endif
1001
1002 *address = vmmaddr;
1003 return kr;
1004 }
1005
1006
1007 /* legacy interface */
1008 kern_return_t
1009 vm_map_64_external(
1010 vm_map_t target_map,
1011 vm_offset_t *address,
1012 vm_size_t size,
1013 vm_offset_t mask,
1014 int flags,
1015 ipc_port_t port,
1016 vm_object_offset_t offset,
1017 boolean_t copy,
1018 vm_prot_t cur_protection,
1019 vm_prot_t max_protection,
1020 vm_inherit_t inheritance)
1021 {
1022 vm_tag_t tag;
1023
1024 VM_GET_FLAGS_ALIAS(flags, tag);
1025 return (vm_map_64_kernel(target_map, address, size, mask, flags, tag, port, offset,
1026 copy, cur_protection, max_protection, inheritance));
1027 }
1028
1029 kern_return_t
1030 vm_map_64_kernel(
1031 vm_map_t target_map,
1032 vm_offset_t *address,
1033 vm_size_t size,
1034 vm_offset_t mask,
1035 int flags,
1036 vm_tag_t tag,
1037 ipc_port_t port,
1038 vm_object_offset_t offset,
1039 boolean_t copy,
1040 vm_prot_t cur_protection,
1041 vm_prot_t max_protection,
1042 vm_inherit_t inheritance)
1043 {
1044 mach_vm_address_t map_addr;
1045 mach_vm_size_t map_size;
1046 mach_vm_offset_t map_mask;
1047 kern_return_t kr;
1048
1049 map_addr = (mach_vm_address_t)*address;
1050 map_size = (mach_vm_size_t)size;
1051 map_mask = (mach_vm_offset_t)mask;
1052
1053 kr = mach_vm_map_kernel(target_map, &map_addr, map_size, map_mask, flags, tag,
1054 port, offset, copy,
1055 cur_protection, max_protection, inheritance);
1056 *address = CAST_DOWN(vm_offset_t, map_addr);
1057 return kr;
1058 }
1059
1060 /* temporary, until world build */
1061 kern_return_t
1062 vm_map_external(
1063 vm_map_t target_map,
1064 vm_offset_t *address,
1065 vm_size_t size,
1066 vm_offset_t mask,
1067 int flags,
1068 ipc_port_t port,
1069 vm_offset_t offset,
1070 boolean_t copy,
1071 vm_prot_t cur_protection,
1072 vm_prot_t max_protection,
1073 vm_inherit_t inheritance)
1074 {
1075 vm_tag_t tag;
1076
1077 VM_GET_FLAGS_ALIAS(flags, tag);
1078 return (vm_map_kernel(target_map, address, size, mask, flags, tag, port, offset, copy, cur_protection, max_protection, inheritance));
1079 }
1080
1081 kern_return_t
1082 vm_map_kernel(
1083 vm_map_t target_map,
1084 vm_offset_t *address,
1085 vm_size_t size,
1086 vm_offset_t mask,
1087 int flags,
1088 vm_tag_t tag,
1089 ipc_port_t port,
1090 vm_offset_t offset,
1091 boolean_t copy,
1092 vm_prot_t cur_protection,
1093 vm_prot_t max_protection,
1094 vm_inherit_t inheritance)
1095 {
1096 mach_vm_address_t map_addr;
1097 mach_vm_size_t map_size;
1098 mach_vm_offset_t map_mask;
1099 vm_object_offset_t obj_offset;
1100 kern_return_t kr;
1101
1102 map_addr = (mach_vm_address_t)*address;
1103 map_size = (mach_vm_size_t)size;
1104 map_mask = (mach_vm_offset_t)mask;
1105 obj_offset = (vm_object_offset_t)offset;
1106
1107 kr = mach_vm_map_kernel(target_map, &map_addr, map_size, map_mask, flags, tag,
1108 port, obj_offset, copy,
1109 cur_protection, max_protection, inheritance);
1110 *address = CAST_DOWN(vm_offset_t, map_addr);
1111 return kr;
1112 }
1113
1114 /*
1115 * mach_vm_remap -
1116 * Remap a range of memory from one task into another,
1117 * to another address range within the same task, or
1118 * over top of itself (with altered permissions and/or
1119 * as an in-place copy of itself).
1120 */
1121 kern_return_t
1122 mach_vm_remap_external(
1123 vm_map_t target_map,
1124 mach_vm_offset_t *address,
1125 mach_vm_size_t size,
1126 mach_vm_offset_t mask,
1127 int flags,
1128 vm_map_t src_map,
1129 mach_vm_offset_t memory_address,
1130 boolean_t copy,
1131 vm_prot_t *cur_protection,
1132 vm_prot_t *max_protection,
1133 vm_inherit_t inheritance)
1134 {
1135 vm_tag_t tag;
1136 VM_GET_FLAGS_ALIAS(flags, tag);
1137
1138 return (mach_vm_remap_kernel(target_map, address, size, mask, flags, tag, src_map, memory_address,
1139 copy, cur_protection, max_protection, inheritance));
1140 }
1141
1142 kern_return_t
1143 mach_vm_remap_kernel(
1144 vm_map_t target_map,
1145 mach_vm_offset_t *address,
1146 mach_vm_size_t size,
1147 mach_vm_offset_t mask,
1148 int flags,
1149 vm_tag_t tag,
1150 vm_map_t src_map,
1151 mach_vm_offset_t memory_address,
1152 boolean_t copy,
1153 vm_prot_t *cur_protection,
1154 vm_prot_t *max_protection,
1155 vm_inherit_t inheritance)
1156 {
1157 vm_map_offset_t map_addr;
1158 kern_return_t kr;
1159
1160 if (VM_MAP_NULL == target_map || VM_MAP_NULL == src_map)
1161 return KERN_INVALID_ARGUMENT;
1162
1163 /* filter out any kernel-only flags */
1164 if (flags & ~VM_FLAGS_USER_REMAP)
1165 return KERN_INVALID_ARGUMENT;
1166
1167 map_addr = (vm_map_offset_t)*address;
1168
1169 kr = vm_map_remap(target_map,
1170 &map_addr,
1171 size,
1172 mask,
1173 flags,
1174 VM_MAP_KERNEL_FLAGS_NONE,
1175 tag,
1176 src_map,
1177 memory_address,
1178 copy,
1179 cur_protection,
1180 max_protection,
1181 inheritance);
1182 *address = map_addr;
1183 return kr;
1184 }
1185
1186 /*
1187 * vm_remap -
1188 * Remap a range of memory from one task into another,
1189 * to another address range within the same task, or
1190 * over top of itself (with altered permissions and/or
1191 * as an in-place copy of itself).
1192 *
1193 * The addressability of the source and target address
1194 * range is limited by the size of vm_address_t (in the
1195 * kernel context).
1196 */
1197 kern_return_t
1198 vm_remap_external(
1199 vm_map_t target_map,
1200 vm_offset_t *address,
1201 vm_size_t size,
1202 vm_offset_t mask,
1203 int flags,
1204 vm_map_t src_map,
1205 vm_offset_t memory_address,
1206 boolean_t copy,
1207 vm_prot_t *cur_protection,
1208 vm_prot_t *max_protection,
1209 vm_inherit_t inheritance)
1210 {
1211 vm_tag_t tag;
1212 VM_GET_FLAGS_ALIAS(flags, tag);
1213
1214 return (vm_remap_kernel(target_map, address, size, mask, flags, tag, src_map,
1215 memory_address, copy, cur_protection, max_protection, inheritance));
1216 }
1217
1218 kern_return_t
1219 vm_remap_kernel(
1220 vm_map_t target_map,
1221 vm_offset_t *address,
1222 vm_size_t size,
1223 vm_offset_t mask,
1224 int flags,
1225 vm_tag_t tag,
1226 vm_map_t src_map,
1227 vm_offset_t memory_address,
1228 boolean_t copy,
1229 vm_prot_t *cur_protection,
1230 vm_prot_t *max_protection,
1231 vm_inherit_t inheritance)
1232 {
1233 vm_map_offset_t map_addr;
1234 kern_return_t kr;
1235
1236 if (VM_MAP_NULL == target_map || VM_MAP_NULL == src_map)
1237 return KERN_INVALID_ARGUMENT;
1238
1239 /* filter out any kernel-only flags */
1240 if (flags & ~VM_FLAGS_USER_REMAP)
1241 return KERN_INVALID_ARGUMENT;
1242
1243 map_addr = (vm_map_offset_t)*address;
1244
1245 kr = vm_map_remap(target_map,
1246 &map_addr,
1247 size,
1248 mask,
1249 flags,
1250 VM_MAP_KERNEL_FLAGS_NONE,
1251 tag,
1252 src_map,
1253 memory_address,
1254 copy,
1255 cur_protection,
1256 max_protection,
1257 inheritance);
1258 *address = CAST_DOWN(vm_offset_t, map_addr);
1259 return kr;
1260 }
1261
1262 /*
1263 * NOTE: these routine (and this file) will no longer require mach_host_server.h
1264 * when mach_vm_wire and vm_wire are changed to use ledgers.
1265 */
1266 #include <mach/mach_host_server.h>
1267 /*
1268 * mach_vm_wire
1269 * Specify that the range of the virtual address space
1270 * of the target task must not cause page faults for
1271 * the indicated accesses.
1272 *
1273 * [ To unwire the pages, specify VM_PROT_NONE. ]
1274 */
1275 kern_return_t
1276 mach_vm_wire_external(
1277 host_priv_t host_priv,
1278 vm_map_t map,
1279 mach_vm_offset_t start,
1280 mach_vm_size_t size,
1281 vm_prot_t access)
1282 {
1283 return (mach_vm_wire_kernel(host_priv, map, start, size, access, VM_KERN_MEMORY_MLOCK));
1284 }
1285
1286 kern_return_t
1287 mach_vm_wire_kernel(
1288 host_priv_t host_priv,
1289 vm_map_t map,
1290 mach_vm_offset_t start,
1291 mach_vm_size_t size,
1292 vm_prot_t access,
1293 vm_tag_t tag)
1294 {
1295 kern_return_t rc;
1296
1297 if (host_priv == HOST_PRIV_NULL)
1298 return KERN_INVALID_HOST;
1299
1300 assert(host_priv == &realhost);
1301
1302 if (map == VM_MAP_NULL)
1303 return KERN_INVALID_TASK;
1304
1305 if (access & ~VM_PROT_ALL || (start + size < start))
1306 return KERN_INVALID_ARGUMENT;
1307
1308 if (access != VM_PROT_NONE) {
1309 rc = vm_map_wire_kernel(map,
1310 vm_map_trunc_page(start,
1311 VM_MAP_PAGE_MASK(map)),
1312 vm_map_round_page(start+size,
1313 VM_MAP_PAGE_MASK(map)),
1314 access, tag,
1315 TRUE);
1316 } else {
1317 rc = vm_map_unwire(map,
1318 vm_map_trunc_page(start,
1319 VM_MAP_PAGE_MASK(map)),
1320 vm_map_round_page(start+size,
1321 VM_MAP_PAGE_MASK(map)),
1322 TRUE);
1323 }
1324 return rc;
1325 }
1326
1327 /*
1328 * vm_wire -
1329 * Specify that the range of the virtual address space
1330 * of the target task must not cause page faults for
1331 * the indicated accesses.
1332 *
1333 * [ To unwire the pages, specify VM_PROT_NONE. ]
1334 */
1335 kern_return_t
1336 vm_wire(
1337 host_priv_t host_priv,
1338 vm_map_t map,
1339 vm_offset_t start,
1340 vm_size_t size,
1341 vm_prot_t access)
1342 {
1343 kern_return_t rc;
1344
1345 if (host_priv == HOST_PRIV_NULL)
1346 return KERN_INVALID_HOST;
1347
1348 assert(host_priv == &realhost);
1349
1350 if (map == VM_MAP_NULL)
1351 return KERN_INVALID_TASK;
1352
1353 if ((access & ~VM_PROT_ALL) || (start + size < start))
1354 return KERN_INVALID_ARGUMENT;
1355
1356 if (size == 0) {
1357 rc = KERN_SUCCESS;
1358 } else if (access != VM_PROT_NONE) {
1359 rc = vm_map_wire_kernel(map,
1360 vm_map_trunc_page(start,
1361 VM_MAP_PAGE_MASK(map)),
1362 vm_map_round_page(start+size,
1363 VM_MAP_PAGE_MASK(map)),
1364 access, VM_KERN_MEMORY_OSFMK,
1365 TRUE);
1366 } else {
1367 rc = vm_map_unwire(map,
1368 vm_map_trunc_page(start,
1369 VM_MAP_PAGE_MASK(map)),
1370 vm_map_round_page(start+size,
1371 VM_MAP_PAGE_MASK(map)),
1372 TRUE);
1373 }
1374 return rc;
1375 }
1376
1377 /*
1378 * vm_msync
1379 *
1380 * Synchronises the memory range specified with its backing store
1381 * image by either flushing or cleaning the contents to the appropriate
1382 * memory manager.
1383 *
1384 * interpretation of sync_flags
1385 * VM_SYNC_INVALIDATE - discard pages, only return precious
1386 * pages to manager.
1387 *
1388 * VM_SYNC_INVALIDATE & (VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS)
1389 * - discard pages, write dirty or precious
1390 * pages back to memory manager.
1391 *
1392 * VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS
1393 * - write dirty or precious pages back to
1394 * the memory manager.
1395 *
1396 * VM_SYNC_CONTIGUOUS - does everything normally, but if there
1397 * is a hole in the region, and we would
1398 * have returned KERN_SUCCESS, return
1399 * KERN_INVALID_ADDRESS instead.
1400 *
1401 * RETURNS
1402 * KERN_INVALID_TASK Bad task parameter
1403 * KERN_INVALID_ARGUMENT both sync and async were specified.
1404 * KERN_SUCCESS The usual.
1405 * KERN_INVALID_ADDRESS There was a hole in the region.
1406 */
1407
1408 kern_return_t
1409 mach_vm_msync(
1410 vm_map_t map,
1411 mach_vm_address_t address,
1412 mach_vm_size_t size,
1413 vm_sync_t sync_flags)
1414 {
1415
1416 if (map == VM_MAP_NULL)
1417 return(KERN_INVALID_TASK);
1418
1419 return vm_map_msync(map, (vm_map_address_t)address,
1420 (vm_map_size_t)size, sync_flags);
1421 }
1422
1423 /*
1424 * vm_msync
1425 *
1426 * Synchronises the memory range specified with its backing store
1427 * image by either flushing or cleaning the contents to the appropriate
1428 * memory manager.
1429 *
1430 * interpretation of sync_flags
1431 * VM_SYNC_INVALIDATE - discard pages, only return precious
1432 * pages to manager.
1433 *
1434 * VM_SYNC_INVALIDATE & (VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS)
1435 * - discard pages, write dirty or precious
1436 * pages back to memory manager.
1437 *
1438 * VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS
1439 * - write dirty or precious pages back to
1440 * the memory manager.
1441 *
1442 * VM_SYNC_CONTIGUOUS - does everything normally, but if there
1443 * is a hole in the region, and we would
1444 * have returned KERN_SUCCESS, return
1445 * KERN_INVALID_ADDRESS instead.
1446 *
1447 * The addressability of the range is limited to that which can
1448 * be described by a vm_address_t.
1449 *
1450 * RETURNS
1451 * KERN_INVALID_TASK Bad task parameter
1452 * KERN_INVALID_ARGUMENT both sync and async were specified.
1453 * KERN_SUCCESS The usual.
1454 * KERN_INVALID_ADDRESS There was a hole in the region.
1455 */
1456
1457 kern_return_t
1458 vm_msync(
1459 vm_map_t map,
1460 vm_address_t address,
1461 vm_size_t size,
1462 vm_sync_t sync_flags)
1463 {
1464
1465 if (map == VM_MAP_NULL)
1466 return(KERN_INVALID_TASK);
1467
1468 return vm_map_msync(map, (vm_map_address_t)address,
1469 (vm_map_size_t)size, sync_flags);
1470 }
1471
1472
1473 int
1474 vm_toggle_entry_reuse(int toggle, int *old_value)
1475 {
1476 vm_map_t map = current_map();
1477
1478 assert(!map->is_nested_map);
1479 if(toggle == VM_TOGGLE_GETVALUE && old_value != NULL){
1480 *old_value = map->disable_vmentry_reuse;
1481 } else if(toggle == VM_TOGGLE_SET){
1482 vm_map_entry_t map_to_entry;
1483
1484 vm_map_lock(map);
1485 vm_map_disable_hole_optimization(map);
1486 map->disable_vmentry_reuse = TRUE;
1487 __IGNORE_WCASTALIGN(map_to_entry = vm_map_to_entry(map));
1488 if (map->first_free == map_to_entry) {
1489 map->highest_entry_end = vm_map_min(map);
1490 } else {
1491 map->highest_entry_end = map->first_free->vme_end;
1492 }
1493 vm_map_unlock(map);
1494 } else if (toggle == VM_TOGGLE_CLEAR){
1495 vm_map_lock(map);
1496 map->disable_vmentry_reuse = FALSE;
1497 vm_map_unlock(map);
1498 } else
1499 return KERN_INVALID_ARGUMENT;
1500
1501 return KERN_SUCCESS;
1502 }
1503
1504 /*
1505 * mach_vm_behavior_set
1506 *
1507 * Sets the paging behavior attribute for the specified range
1508 * in the specified map.
1509 *
1510 * This routine will fail with KERN_INVALID_ADDRESS if any address
1511 * in [start,start+size) is not a valid allocated memory region.
1512 */
1513 kern_return_t
1514 mach_vm_behavior_set(
1515 vm_map_t map,
1516 mach_vm_offset_t start,
1517 mach_vm_size_t size,
1518 vm_behavior_t new_behavior)
1519 {
1520 vm_map_offset_t align_mask;
1521
1522 if ((map == VM_MAP_NULL) || (start + size < start))
1523 return(KERN_INVALID_ARGUMENT);
1524
1525 if (size == 0)
1526 return KERN_SUCCESS;
1527
1528 switch (new_behavior) {
1529 case VM_BEHAVIOR_REUSABLE:
1530 case VM_BEHAVIOR_REUSE:
1531 case VM_BEHAVIOR_CAN_REUSE:
1532 /*
1533 * Align to the hardware page size, to allow
1534 * malloc() to maximize the amount of re-usability,
1535 * even on systems with larger software page size.
1536 */
1537 align_mask = PAGE_MASK;
1538 break;
1539 default:
1540 align_mask = VM_MAP_PAGE_MASK(map);
1541 break;
1542 }
1543
1544 return vm_map_behavior_set(map,
1545 vm_map_trunc_page(start, align_mask),
1546 vm_map_round_page(start+size, align_mask),
1547 new_behavior);
1548 }
1549
1550 /*
1551 * vm_behavior_set
1552 *
1553 * Sets the paging behavior attribute for the specified range
1554 * in the specified map.
1555 *
1556 * This routine will fail with KERN_INVALID_ADDRESS if any address
1557 * in [start,start+size) is not a valid allocated memory region.
1558 *
1559 * This routine is potentially limited in addressibility by the
1560 * use of vm_offset_t (if the map provided is larger than the
1561 * kernel's).
1562 */
1563 kern_return_t
1564 vm_behavior_set(
1565 vm_map_t map,
1566 vm_offset_t start,
1567 vm_size_t size,
1568 vm_behavior_t new_behavior)
1569 {
1570 if (start + size < start)
1571 return KERN_INVALID_ARGUMENT;
1572
1573 return mach_vm_behavior_set(map,
1574 (mach_vm_offset_t) start,
1575 (mach_vm_size_t) size,
1576 new_behavior);
1577 }
1578
1579 /*
1580 * mach_vm_region:
1581 *
1582 * User call to obtain information about a region in
1583 * a task's address map. Currently, only one flavor is
1584 * supported.
1585 *
1586 * XXX The reserved and behavior fields cannot be filled
1587 * in until the vm merge from the IK is completed, and
1588 * vm_reserve is implemented.
1589 *
1590 * XXX Dependency: syscall_vm_region() also supports only one flavor.
1591 */
1592
1593 kern_return_t
1594 mach_vm_region(
1595 vm_map_t map,
1596 mach_vm_offset_t *address, /* IN/OUT */
1597 mach_vm_size_t *size, /* OUT */
1598 vm_region_flavor_t flavor, /* IN */
1599 vm_region_info_t info, /* OUT */
1600 mach_msg_type_number_t *count, /* IN/OUT */
1601 mach_port_t *object_name) /* OUT */
1602 {
1603 vm_map_offset_t map_addr;
1604 vm_map_size_t map_size;
1605 kern_return_t kr;
1606
1607 if (VM_MAP_NULL == map)
1608 return KERN_INVALID_ARGUMENT;
1609
1610 map_addr = (vm_map_offset_t)*address;
1611 map_size = (vm_map_size_t)*size;
1612
1613 /* legacy conversion */
1614 if (VM_REGION_BASIC_INFO == flavor)
1615 flavor = VM_REGION_BASIC_INFO_64;
1616
1617 kr = vm_map_region(map,
1618 &map_addr, &map_size,
1619 flavor, info, count,
1620 object_name);
1621
1622 *address = map_addr;
1623 *size = map_size;
1624 return kr;
1625 }
1626
1627 /*
1628 * vm_region_64 and vm_region:
1629 *
1630 * User call to obtain information about a region in
1631 * a task's address map. Currently, only one flavor is
1632 * supported.
1633 *
1634 * XXX The reserved and behavior fields cannot be filled
1635 * in until the vm merge from the IK is completed, and
1636 * vm_reserve is implemented.
1637 *
1638 * XXX Dependency: syscall_vm_region() also supports only one flavor.
1639 */
1640
1641 kern_return_t
1642 vm_region_64(
1643 vm_map_t map,
1644 vm_offset_t *address, /* IN/OUT */
1645 vm_size_t *size, /* OUT */
1646 vm_region_flavor_t flavor, /* IN */
1647 vm_region_info_t info, /* OUT */
1648 mach_msg_type_number_t *count, /* IN/OUT */
1649 mach_port_t *object_name) /* OUT */
1650 {
1651 vm_map_offset_t map_addr;
1652 vm_map_size_t map_size;
1653 kern_return_t kr;
1654
1655 if (VM_MAP_NULL == map)
1656 return KERN_INVALID_ARGUMENT;
1657
1658 map_addr = (vm_map_offset_t)*address;
1659 map_size = (vm_map_size_t)*size;
1660
1661 /* legacy conversion */
1662 if (VM_REGION_BASIC_INFO == flavor)
1663 flavor = VM_REGION_BASIC_INFO_64;
1664
1665 kr = vm_map_region(map,
1666 &map_addr, &map_size,
1667 flavor, info, count,
1668 object_name);
1669
1670 *address = CAST_DOWN(vm_offset_t, map_addr);
1671 *size = CAST_DOWN(vm_size_t, map_size);
1672
1673 if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS)
1674 return KERN_INVALID_ADDRESS;
1675 return kr;
1676 }
1677
1678 kern_return_t
1679 vm_region(
1680 vm_map_t map,
1681 vm_address_t *address, /* IN/OUT */
1682 vm_size_t *size, /* OUT */
1683 vm_region_flavor_t flavor, /* IN */
1684 vm_region_info_t info, /* OUT */
1685 mach_msg_type_number_t *count, /* IN/OUT */
1686 mach_port_t *object_name) /* OUT */
1687 {
1688 vm_map_address_t map_addr;
1689 vm_map_size_t map_size;
1690 kern_return_t kr;
1691
1692 if (VM_MAP_NULL == map)
1693 return KERN_INVALID_ARGUMENT;
1694
1695 map_addr = (vm_map_address_t)*address;
1696 map_size = (vm_map_size_t)*size;
1697
1698 kr = vm_map_region(map,
1699 &map_addr, &map_size,
1700 flavor, info, count,
1701 object_name);
1702
1703 *address = CAST_DOWN(vm_address_t, map_addr);
1704 *size = CAST_DOWN(vm_size_t, map_size);
1705
1706 if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS)
1707 return KERN_INVALID_ADDRESS;
1708 return kr;
1709 }
1710
1711 /*
1712 * vm_region_recurse: A form of vm_region which follows the
1713 * submaps in a target map
1714 *
1715 */
1716 kern_return_t
1717 mach_vm_region_recurse(
1718 vm_map_t map,
1719 mach_vm_address_t *address,
1720 mach_vm_size_t *size,
1721 uint32_t *depth,
1722 vm_region_recurse_info_t info,
1723 mach_msg_type_number_t *infoCnt)
1724 {
1725 vm_map_address_t map_addr;
1726 vm_map_size_t map_size;
1727 kern_return_t kr;
1728
1729 if (VM_MAP_NULL == map)
1730 return KERN_INVALID_ARGUMENT;
1731
1732 map_addr = (vm_map_address_t)*address;
1733 map_size = (vm_map_size_t)*size;
1734
1735 kr = vm_map_region_recurse_64(
1736 map,
1737 &map_addr,
1738 &map_size,
1739 depth,
1740 (vm_region_submap_info_64_t)info,
1741 infoCnt);
1742
1743 *address = map_addr;
1744 *size = map_size;
1745 return kr;
1746 }
1747
1748 /*
1749 * vm_region_recurse: A form of vm_region which follows the
1750 * submaps in a target map
1751 *
1752 */
1753 kern_return_t
1754 vm_region_recurse_64(
1755 vm_map_t map,
1756 vm_address_t *address,
1757 vm_size_t *size,
1758 uint32_t *depth,
1759 vm_region_recurse_info_64_t info,
1760 mach_msg_type_number_t *infoCnt)
1761 {
1762 vm_map_address_t map_addr;
1763 vm_map_size_t map_size;
1764 kern_return_t kr;
1765
1766 if (VM_MAP_NULL == map)
1767 return KERN_INVALID_ARGUMENT;
1768
1769 map_addr = (vm_map_address_t)*address;
1770 map_size = (vm_map_size_t)*size;
1771
1772 kr = vm_map_region_recurse_64(
1773 map,
1774 &map_addr,
1775 &map_size,
1776 depth,
1777 (vm_region_submap_info_64_t)info,
1778 infoCnt);
1779
1780 *address = CAST_DOWN(vm_address_t, map_addr);
1781 *size = CAST_DOWN(vm_size_t, map_size);
1782
1783 if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS)
1784 return KERN_INVALID_ADDRESS;
1785 return kr;
1786 }
1787
1788 kern_return_t
1789 vm_region_recurse(
1790 vm_map_t map,
1791 vm_offset_t *address, /* IN/OUT */
1792 vm_size_t *size, /* OUT */
1793 natural_t *depth, /* IN/OUT */
1794 vm_region_recurse_info_t info32, /* IN/OUT */
1795 mach_msg_type_number_t *infoCnt) /* IN/OUT */
1796 {
1797 vm_region_submap_info_data_64_t info64;
1798 vm_region_submap_info_t info;
1799 vm_map_address_t map_addr;
1800 vm_map_size_t map_size;
1801 kern_return_t kr;
1802
1803 if (VM_MAP_NULL == map || *infoCnt < VM_REGION_SUBMAP_INFO_COUNT)
1804 return KERN_INVALID_ARGUMENT;
1805
1806
1807 map_addr = (vm_map_address_t)*address;
1808 map_size = (vm_map_size_t)*size;
1809 info = (vm_region_submap_info_t)info32;
1810 *infoCnt = VM_REGION_SUBMAP_INFO_COUNT_64;
1811
1812 kr = vm_map_region_recurse_64(map, &map_addr,&map_size,
1813 depth, &info64, infoCnt);
1814
1815 info->protection = info64.protection;
1816 info->max_protection = info64.max_protection;
1817 info->inheritance = info64.inheritance;
1818 info->offset = (uint32_t)info64.offset; /* trouble-maker */
1819 info->user_tag = info64.user_tag;
1820 info->pages_resident = info64.pages_resident;
1821 info->pages_shared_now_private = info64.pages_shared_now_private;
1822 info->pages_swapped_out = info64.pages_swapped_out;
1823 info->pages_dirtied = info64.pages_dirtied;
1824 info->ref_count = info64.ref_count;
1825 info->shadow_depth = info64.shadow_depth;
1826 info->external_pager = info64.external_pager;
1827 info->share_mode = info64.share_mode;
1828 info->is_submap = info64.is_submap;
1829 info->behavior = info64.behavior;
1830 info->object_id = info64.object_id;
1831 info->user_wired_count = info64.user_wired_count;
1832
1833 *address = CAST_DOWN(vm_address_t, map_addr);
1834 *size = CAST_DOWN(vm_size_t, map_size);
1835 *infoCnt = VM_REGION_SUBMAP_INFO_COUNT;
1836
1837 if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS)
1838 return KERN_INVALID_ADDRESS;
1839 return kr;
1840 }
1841
1842 kern_return_t
1843 mach_vm_purgable_control(
1844 vm_map_t map,
1845 mach_vm_offset_t address,
1846 vm_purgable_t control,
1847 int *state)
1848 {
1849 if (VM_MAP_NULL == map)
1850 return KERN_INVALID_ARGUMENT;
1851
1852 if (control == VM_PURGABLE_SET_STATE_FROM_KERNEL) {
1853 /* not allowed from user-space */
1854 return KERN_INVALID_ARGUMENT;
1855 }
1856
1857 return vm_map_purgable_control(map,
1858 vm_map_trunc_page(address, PAGE_MASK),
1859 control,
1860 state);
1861 }
1862
1863 kern_return_t
1864 vm_purgable_control(
1865 vm_map_t map,
1866 vm_offset_t address,
1867 vm_purgable_t control,
1868 int *state)
1869 {
1870 if (VM_MAP_NULL == map)
1871 return KERN_INVALID_ARGUMENT;
1872
1873 if (control == VM_PURGABLE_SET_STATE_FROM_KERNEL) {
1874 /* not allowed from user-space */
1875 return KERN_INVALID_ARGUMENT;
1876 }
1877
1878 return vm_map_purgable_control(map,
1879 vm_map_trunc_page(address, PAGE_MASK),
1880 control,
1881 state);
1882 }
1883
1884
1885 /*
1886 * Ordinarily, the right to allocate CPM is restricted
1887 * to privileged applications (those that can gain access
1888 * to the host priv port). Set this variable to zero if
1889 * you want to let any application allocate CPM.
1890 */
1891 unsigned int vm_allocate_cpm_privileged = 0;
1892
1893 /*
1894 * Allocate memory in the specified map, with the caveat that
1895 * the memory is physically contiguous. This call may fail
1896 * if the system can't find sufficient contiguous memory.
1897 * This call may cause or lead to heart-stopping amounts of
1898 * paging activity.
1899 *
1900 * Memory obtained from this call should be freed in the
1901 * normal way, viz., via vm_deallocate.
1902 */
1903 kern_return_t
1904 vm_allocate_cpm(
1905 host_priv_t host_priv,
1906 vm_map_t map,
1907 vm_address_t *addr,
1908 vm_size_t size,
1909 int flags)
1910 {
1911 vm_map_address_t map_addr;
1912 vm_map_size_t map_size;
1913 kern_return_t kr;
1914
1915 if (vm_allocate_cpm_privileged && HOST_PRIV_NULL == host_priv)
1916 return KERN_INVALID_HOST;
1917
1918 if (VM_MAP_NULL == map)
1919 return KERN_INVALID_ARGUMENT;
1920
1921 map_addr = (vm_map_address_t)*addr;
1922 map_size = (vm_map_size_t)size;
1923
1924 kr = vm_map_enter_cpm(map,
1925 &map_addr,
1926 map_size,
1927 flags);
1928
1929 *addr = CAST_DOWN(vm_address_t, map_addr);
1930 return kr;
1931 }
1932
1933
1934 kern_return_t
1935 mach_vm_page_query(
1936 vm_map_t map,
1937 mach_vm_offset_t offset,
1938 int *disposition,
1939 int *ref_count)
1940 {
1941 if (VM_MAP_NULL == map)
1942 return KERN_INVALID_ARGUMENT;
1943
1944 return vm_map_page_query_internal(
1945 map,
1946 vm_map_trunc_page(offset, PAGE_MASK),
1947 disposition, ref_count);
1948 }
1949
1950 kern_return_t
1951 vm_map_page_query(
1952 vm_map_t map,
1953 vm_offset_t offset,
1954 int *disposition,
1955 int *ref_count)
1956 {
1957 if (VM_MAP_NULL == map)
1958 return KERN_INVALID_ARGUMENT;
1959
1960 return vm_map_page_query_internal(
1961 map,
1962 vm_map_trunc_page(offset, PAGE_MASK),
1963 disposition, ref_count);
1964 }
1965
1966 kern_return_t
1967 mach_vm_page_range_query(
1968 vm_map_t map,
1969 mach_vm_offset_t address,
1970 mach_vm_size_t size,
1971 mach_vm_address_t dispositions_addr,
1972 mach_vm_size_t *dispositions_count)
1973 {
1974 kern_return_t kr = KERN_SUCCESS;
1975 int num_pages = 0, i = 0;
1976 mach_vm_size_t curr_sz = 0, copy_sz = 0;
1977 mach_vm_size_t disp_buf_req_size = 0, disp_buf_total_size = 0;
1978 mach_msg_type_number_t count = 0;
1979
1980 void *info = NULL;
1981 void *local_disp = NULL;;
1982 vm_map_size_t info_size = 0, local_disp_size = 0;
1983 mach_vm_offset_t start = 0, end = 0;
1984
1985 if (map == VM_MAP_NULL || dispositions_count == NULL) {
1986 return KERN_INVALID_ARGUMENT;
1987 }
1988
1989 disp_buf_req_size = ( *dispositions_count * sizeof(int));
1990 start = mach_vm_trunc_page(address);
1991 end = mach_vm_round_page(address + size);
1992
1993 if (end < start) {
1994 return KERN_INVALID_ARGUMENT;
1995 }
1996
1997 if (disp_buf_req_size == 0 || (end == start)) {
1998 return KERN_SUCCESS;
1999 }
2000
2001 /*
2002 * For large requests, we will go through them
2003 * MAX_PAGE_RANGE_QUERY chunk at a time.
2004 */
2005
2006 curr_sz = MIN(end - start, MAX_PAGE_RANGE_QUERY);
2007 num_pages = (int) (curr_sz >> PAGE_SHIFT);
2008
2009 info_size = num_pages * sizeof(vm_page_info_basic_data_t);
2010 info = kalloc(info_size);
2011
2012 if (info == NULL) {
2013 return KERN_RESOURCE_SHORTAGE;
2014 }
2015
2016 local_disp_size = num_pages * sizeof(int);
2017 local_disp = kalloc(local_disp_size);
2018
2019 if (local_disp == NULL) {
2020
2021 kfree(info, info_size);
2022 info = NULL;
2023 return KERN_RESOURCE_SHORTAGE;
2024 }
2025
2026 while (size) {
2027
2028 count = VM_PAGE_INFO_BASIC_COUNT;
2029 kr = vm_map_page_range_info_internal(
2030 map,
2031 start,
2032 mach_vm_round_page(start + curr_sz),
2033 VM_PAGE_INFO_BASIC,
2034 (vm_page_info_t) info,
2035 &count);
2036
2037 assert(kr == KERN_SUCCESS);
2038
2039 for (i = 0; i < num_pages; i++) {
2040
2041 ((int*)local_disp)[i] = ((vm_page_info_basic_t)info)[i].disposition;
2042 }
2043
2044 copy_sz = MIN(disp_buf_req_size, num_pages * sizeof(int)/* an int per page */);
2045 kr = copyout(local_disp, (mach_vm_address_t)dispositions_addr, copy_sz);
2046
2047 start += curr_sz;
2048 disp_buf_req_size -= copy_sz;
2049 disp_buf_total_size += copy_sz;
2050
2051 if (kr != 0) {
2052 break;
2053 }
2054
2055 if ((disp_buf_req_size == 0) || (curr_sz >= size)) {
2056
2057 /*
2058 * We might have inspected the full range OR
2059 * more than it esp. if the user passed in
2060 * non-page aligned start/size and/or if we
2061 * descended into a submap. We are done here.
2062 */
2063
2064 size = 0;
2065
2066 } else {
2067
2068 dispositions_addr += copy_sz;
2069
2070 size -= curr_sz;
2071
2072 curr_sz = MIN(mach_vm_round_page(size), MAX_PAGE_RANGE_QUERY);
2073 num_pages = (int)(curr_sz >> PAGE_SHIFT);
2074 }
2075 }
2076
2077 *dispositions_count = disp_buf_total_size / sizeof(int);
2078
2079 kfree(local_disp, local_disp_size);
2080 local_disp = NULL;
2081
2082 kfree(info, info_size);
2083 info = NULL;
2084
2085 return kr;
2086 }
2087
2088 kern_return_t
2089 mach_vm_page_info(
2090 vm_map_t map,
2091 mach_vm_address_t address,
2092 vm_page_info_flavor_t flavor,
2093 vm_page_info_t info,
2094 mach_msg_type_number_t *count)
2095 {
2096 kern_return_t kr;
2097
2098 if (map == VM_MAP_NULL) {
2099 return KERN_INVALID_ARGUMENT;
2100 }
2101
2102 kr = vm_map_page_info(map, address, flavor, info, count);
2103 return kr;
2104 }
2105
2106 /* map a (whole) upl into an address space */
2107 kern_return_t
2108 vm_upl_map(
2109 vm_map_t map,
2110 upl_t upl,
2111 vm_address_t *dst_addr)
2112 {
2113 vm_map_offset_t map_addr;
2114 kern_return_t kr;
2115
2116 if (VM_MAP_NULL == map)
2117 return KERN_INVALID_ARGUMENT;
2118
2119 kr = vm_map_enter_upl(map, upl, &map_addr);
2120 *dst_addr = CAST_DOWN(vm_address_t, map_addr);
2121 return kr;
2122 }
2123
2124 kern_return_t
2125 vm_upl_unmap(
2126 vm_map_t map,
2127 upl_t upl)
2128 {
2129 if (VM_MAP_NULL == map)
2130 return KERN_INVALID_ARGUMENT;
2131
2132 return (vm_map_remove_upl(map, upl));
2133 }
2134
2135 /* Retrieve a upl for an object underlying an address range in a map */
2136
2137 kern_return_t
2138 vm_map_get_upl(
2139 vm_map_t map,
2140 vm_map_offset_t map_offset,
2141 upl_size_t *upl_size,
2142 upl_t *upl,
2143 upl_page_info_array_t page_list,
2144 unsigned int *count,
2145 upl_control_flags_t *flags,
2146 vm_tag_t tag,
2147 int force_data_sync)
2148 {
2149 upl_control_flags_t map_flags;
2150 kern_return_t kr;
2151
2152 if (VM_MAP_NULL == map)
2153 return KERN_INVALID_ARGUMENT;
2154
2155 map_flags = *flags & ~UPL_NOZEROFILL;
2156 if (force_data_sync)
2157 map_flags |= UPL_FORCE_DATA_SYNC;
2158
2159 kr = vm_map_create_upl(map,
2160 map_offset,
2161 upl_size,
2162 upl,
2163 page_list,
2164 count,
2165 &map_flags,
2166 tag);
2167
2168 *flags = (map_flags & ~UPL_FORCE_DATA_SYNC);
2169 return kr;
2170 }
2171
2172 #if CONFIG_EMBEDDED
2173 extern int proc_selfpid(void);
2174 extern char *proc_name_address(void *p);
2175 int cs_executable_mem_entry = 0;
2176 int log_executable_mem_entry = 0;
2177 #endif /* CONFIG_EMBEDDED */
2178
2179 /*
2180 * mach_make_memory_entry_64
2181 *
2182 * Think of it as a two-stage vm_remap() operation. First
2183 * you get a handle. Second, you get map that handle in
2184 * somewhere else. Rather than doing it all at once (and
2185 * without needing access to the other whole map).
2186 */
2187 kern_return_t
2188 mach_make_memory_entry_64(
2189 vm_map_t target_map,
2190 memory_object_size_t *size,
2191 memory_object_offset_t offset,
2192 vm_prot_t permission,
2193 ipc_port_t *object_handle,
2194 ipc_port_t parent_handle)
2195 {
2196 vm_map_version_t version;
2197 vm_named_entry_t parent_entry;
2198 vm_named_entry_t user_entry;
2199 ipc_port_t user_handle;
2200 kern_return_t kr;
2201 vm_map_t real_map;
2202
2203 /* needed for call to vm_map_lookup_locked */
2204 boolean_t wired;
2205 boolean_t iskernel;
2206 vm_object_offset_t obj_off;
2207 vm_prot_t prot;
2208 struct vm_object_fault_info fault_info;
2209 vm_object_t object;
2210 vm_object_t shadow_object;
2211
2212 /* needed for direct map entry manipulation */
2213 vm_map_entry_t map_entry;
2214 vm_map_entry_t next_entry;
2215 vm_map_t local_map;
2216 vm_map_t original_map = target_map;
2217 vm_map_size_t total_size, map_size;
2218 vm_map_offset_t map_start, map_end;
2219 vm_map_offset_t local_offset;
2220 vm_object_size_t mappable_size;
2221
2222 /*
2223 * Stash the offset in the page for use by vm_map_enter_mem_object()
2224 * in the VM_FLAGS_RETURN_DATA_ADDR/MAP_MEM_USE_DATA_ADDR case.
2225 */
2226 vm_object_offset_t offset_in_page;
2227
2228 unsigned int access;
2229 vm_prot_t protections;
2230 vm_prot_t original_protections, mask_protections;
2231 unsigned int wimg_mode;
2232
2233 boolean_t force_shadow = FALSE;
2234 boolean_t use_data_addr;
2235 boolean_t use_4K_compat;
2236
2237 if ((permission & MAP_MEM_FLAGS_MASK) & ~MAP_MEM_FLAGS_USER) {
2238 /*
2239 * Unknown flag: reject for forward compatibility.
2240 */
2241 return KERN_INVALID_VALUE;
2242 }
2243
2244 if (parent_handle != IP_NULL &&
2245 ip_kotype(parent_handle) == IKOT_NAMED_ENTRY) {
2246 parent_entry = (vm_named_entry_t) parent_handle->ip_kobject;
2247 } else {
2248 parent_entry = NULL;
2249 }
2250
2251 if (parent_entry && parent_entry->is_copy) {
2252 return KERN_INVALID_ARGUMENT;
2253 }
2254
2255 original_protections = permission & VM_PROT_ALL;
2256 protections = original_protections;
2257 mask_protections = permission & VM_PROT_IS_MASK;
2258 access = GET_MAP_MEM(permission);
2259 use_data_addr = ((permission & MAP_MEM_USE_DATA_ADDR) != 0);
2260 use_4K_compat = ((permission & MAP_MEM_4K_DATA_ADDR) != 0);
2261
2262 user_handle = IP_NULL;
2263 user_entry = NULL;
2264
2265 map_start = vm_map_trunc_page(offset, PAGE_MASK);
2266
2267 if (permission & MAP_MEM_ONLY) {
2268 boolean_t parent_is_object;
2269
2270 map_end = vm_map_round_page(offset + *size, PAGE_MASK);
2271 map_size = map_end - map_start;
2272
2273 if (use_data_addr || use_4K_compat || parent_entry == NULL) {
2274 return KERN_INVALID_ARGUMENT;
2275 }
2276
2277 parent_is_object = !parent_entry->is_sub_map;
2278 object = parent_entry->backing.object;
2279 if(parent_is_object && object != VM_OBJECT_NULL)
2280 wimg_mode = object->wimg_bits;
2281 else
2282 wimg_mode = VM_WIMG_USE_DEFAULT;
2283 if((access != GET_MAP_MEM(parent_entry->protection)) &&
2284 !(parent_entry->protection & VM_PROT_WRITE)) {
2285 return KERN_INVALID_RIGHT;
2286 }
2287 vm_prot_to_wimg(access, &wimg_mode);
2288 if (access != MAP_MEM_NOOP)
2289 SET_MAP_MEM(access, parent_entry->protection);
2290 if (parent_is_object && object &&
2291 (access != MAP_MEM_NOOP) &&
2292 (!(object->nophyscache))) {
2293
2294 if (object->wimg_bits != wimg_mode) {
2295 vm_object_lock(object);
2296 vm_object_change_wimg_mode(object, wimg_mode);
2297 vm_object_unlock(object);
2298 }
2299 }
2300 if (object_handle)
2301 *object_handle = IP_NULL;
2302 return KERN_SUCCESS;
2303 } else if (permission & MAP_MEM_NAMED_CREATE) {
2304 map_end = vm_map_round_page(offset + *size, PAGE_MASK);
2305 map_size = map_end - map_start;
2306
2307 if (use_data_addr || use_4K_compat) {
2308 return KERN_INVALID_ARGUMENT;
2309 }
2310
2311 kr = mach_memory_entry_allocate(&user_entry, &user_handle);
2312 if (kr != KERN_SUCCESS) {
2313 return KERN_FAILURE;
2314 }
2315
2316 /*
2317 * Force the creation of the VM object now.
2318 */
2319 if (map_size > (vm_map_size_t) ANON_MAX_SIZE) {
2320 /*
2321 * LP64todo - for now, we can only allocate 4GB-4096
2322 * internal objects because the default pager can't
2323 * page bigger ones. Remove this when it can.
2324 */
2325 kr = KERN_FAILURE;
2326 goto make_mem_done;
2327 }
2328
2329 object = vm_object_allocate(map_size);
2330 assert(object != VM_OBJECT_NULL);
2331
2332 if (permission & MAP_MEM_PURGABLE) {
2333 if (! (permission & VM_PROT_WRITE)) {
2334 /* if we can't write, we can't purge */
2335 vm_object_deallocate(object);
2336 kr = KERN_INVALID_ARGUMENT;
2337 goto make_mem_done;
2338 }
2339 object->purgable = VM_PURGABLE_NONVOLATILE;
2340 if (permission & MAP_MEM_PURGABLE_KERNEL_ONLY) {
2341 object->purgeable_only_by_kernel = TRUE;
2342 }
2343 assert(object->vo_purgeable_owner == NULL);
2344 assert(object->resident_page_count == 0);
2345 assert(object->wired_page_count == 0);
2346 vm_object_lock(object);
2347 if (object->purgeable_only_by_kernel) {
2348 vm_purgeable_nonvolatile_enqueue(object,
2349 kernel_task);
2350 } else {
2351 vm_purgeable_nonvolatile_enqueue(object,
2352 current_task());
2353 }
2354 vm_object_unlock(object);
2355 }
2356
2357 #if CONFIG_SECLUDED_MEMORY
2358 if (secluded_for_iokit && /* global boot-arg */
2359 ((permission & MAP_MEM_GRAB_SECLUDED)
2360 #if 11
2361 /* XXX FBDP for my testing only */
2362 || (secluded_for_fbdp && map_size == 97550336)
2363 #endif
2364 )) {
2365 #if 11
2366 if (!(permission & MAP_MEM_GRAB_SECLUDED) &&
2367 secluded_for_fbdp) {
2368 printf("FBDP: object %p size %lld can grab secluded\n", object, (uint64_t) map_size);
2369 }
2370 #endif
2371 object->can_grab_secluded = TRUE;
2372 assert(!object->eligible_for_secluded);
2373 }
2374 #endif /* CONFIG_SECLUDED_MEMORY */
2375
2376 /*
2377 * The VM object is brand new and nobody else knows about it,
2378 * so we don't need to lock it.
2379 */
2380
2381 wimg_mode = object->wimg_bits;
2382 vm_prot_to_wimg(access, &wimg_mode);
2383 if (access != MAP_MEM_NOOP) {
2384 object->wimg_bits = wimg_mode;
2385 }
2386
2387 /* the object has no pages, so no WIMG bits to update here */
2388
2389 /*
2390 * XXX
2391 * We use this path when we want to make sure that
2392 * nobody messes with the object (coalesce, for
2393 * example) before we map it.
2394 * We might want to use these objects for transposition via
2395 * vm_object_transpose() too, so we don't want any copy or
2396 * shadow objects either...
2397 */
2398 object->copy_strategy = MEMORY_OBJECT_COPY_NONE;
2399 object->true_share = TRUE;
2400
2401 user_entry->backing.object = object;
2402 user_entry->internal = TRUE;
2403 user_entry->is_sub_map = FALSE;
2404 user_entry->offset = 0;
2405 user_entry->data_offset = 0;
2406 user_entry->protection = protections;
2407 SET_MAP_MEM(access, user_entry->protection);
2408 user_entry->size = map_size;
2409
2410 /* user_object pager and internal fields are not used */
2411 /* when the object field is filled in. */
2412
2413 *size = CAST_DOWN(vm_size_t, (user_entry->size -
2414 user_entry->data_offset));
2415 *object_handle = user_handle;
2416 return KERN_SUCCESS;
2417 }
2418
2419 if (permission & MAP_MEM_VM_COPY) {
2420 vm_map_copy_t copy;
2421
2422 if (target_map == VM_MAP_NULL) {
2423 return KERN_INVALID_TASK;
2424 }
2425
2426 map_end = vm_map_round_page(offset + *size, PAGE_MASK);
2427 map_size = map_end - map_start;
2428 if (use_data_addr || use_4K_compat) {
2429 offset_in_page = offset - map_start;
2430 if (use_4K_compat)
2431 offset_in_page &= ~((signed)(0xFFF));
2432 } else {
2433 offset_in_page = 0;
2434 }
2435
2436 kr = vm_map_copyin_internal(target_map,
2437 map_start,
2438 map_size,
2439 VM_MAP_COPYIN_ENTRY_LIST,
2440 &copy);
2441 if (kr != KERN_SUCCESS) {
2442 return kr;
2443 }
2444
2445 kr = mach_memory_entry_allocate(&user_entry, &user_handle);
2446 if (kr != KERN_SUCCESS) {
2447 vm_map_copy_discard(copy);
2448 return KERN_FAILURE;
2449 }
2450
2451 user_entry->backing.copy = copy;
2452 user_entry->internal = FALSE;
2453 user_entry->is_sub_map = FALSE;
2454 user_entry->is_copy = TRUE;
2455 user_entry->offset = 0;
2456 user_entry->protection = protections;
2457 user_entry->size = map_size;
2458 user_entry->data_offset = offset_in_page;
2459
2460 *size = CAST_DOWN(vm_size_t, (user_entry->size -
2461 user_entry->data_offset));
2462 *object_handle = user_handle;
2463 return KERN_SUCCESS;
2464 }
2465
2466 if (permission & MAP_MEM_VM_SHARE) {
2467 vm_map_copy_t copy;
2468 vm_prot_t cur_prot, max_prot;
2469
2470 if (target_map == VM_MAP_NULL) {
2471 return KERN_INVALID_TASK;
2472 }
2473
2474 map_end = vm_map_round_page(offset + *size, PAGE_MASK);
2475 map_size = map_end - map_start;
2476 if (use_data_addr || use_4K_compat) {
2477 offset_in_page = offset - map_start;
2478 if (use_4K_compat)
2479 offset_in_page &= ~((signed)(0xFFF));
2480 } else {
2481 offset_in_page = 0;
2482 }
2483
2484 cur_prot = VM_PROT_ALL;
2485 kr = vm_map_copy_extract(target_map,
2486 map_start,
2487 map_size,
2488 &copy,
2489 &cur_prot,
2490 &max_prot);
2491 if (kr != KERN_SUCCESS) {
2492 return kr;
2493 }
2494
2495 if (mask_protections) {
2496 /*
2497 * We just want as much of "original_protections"
2498 * as we can get out of the actual "cur_prot".
2499 */
2500 protections &= cur_prot;
2501 if (protections == VM_PROT_NONE) {
2502 /* no access at all: fail */
2503 vm_map_copy_discard(copy);
2504 return KERN_PROTECTION_FAILURE;
2505 }
2506 } else {
2507 /*
2508 * We want exactly "original_protections"
2509 * out of "cur_prot".
2510 */
2511 if ((cur_prot & protections) != protections) {
2512 vm_map_copy_discard(copy);
2513 return KERN_PROTECTION_FAILURE;
2514 }
2515 }
2516
2517 kr = mach_memory_entry_allocate(&user_entry, &user_handle);
2518 if (kr != KERN_SUCCESS) {
2519 vm_map_copy_discard(copy);
2520 return KERN_FAILURE;
2521 }
2522
2523 user_entry->backing.copy = copy;
2524 user_entry->internal = FALSE;
2525 user_entry->is_sub_map = FALSE;
2526 user_entry->is_copy = TRUE;
2527 user_entry->offset = 0;
2528 user_entry->protection = protections;
2529 user_entry->size = map_size;
2530 user_entry->data_offset = offset_in_page;
2531
2532 *size = CAST_DOWN(vm_size_t, (user_entry->size -
2533 user_entry->data_offset));
2534 *object_handle = user_handle;
2535 return KERN_SUCCESS;
2536 }
2537
2538 if (parent_entry == NULL ||
2539 (permission & MAP_MEM_NAMED_REUSE)) {
2540
2541 map_end = vm_map_round_page(offset + *size, PAGE_MASK);
2542 map_size = map_end - map_start;
2543 if (use_data_addr || use_4K_compat) {
2544 offset_in_page = offset - map_start;
2545 if (use_4K_compat)
2546 offset_in_page &= ~((signed)(0xFFF));
2547 } else {
2548 offset_in_page = 0;
2549 }
2550
2551 /* Create a named object based on address range within the task map */
2552 /* Go find the object at given address */
2553
2554 if (target_map == VM_MAP_NULL) {
2555 return KERN_INVALID_TASK;
2556 }
2557
2558 redo_lookup:
2559 protections = original_protections;
2560 vm_map_lock_read(target_map);
2561
2562 /* get the object associated with the target address */
2563 /* note we check the permission of the range against */
2564 /* that requested by the caller */
2565
2566 kr = vm_map_lookup_locked(&target_map, map_start,
2567 protections | mask_protections,
2568 OBJECT_LOCK_EXCLUSIVE, &version,
2569 &object, &obj_off, &prot, &wired,
2570 &fault_info,
2571 &real_map);
2572 if (kr != KERN_SUCCESS) {
2573 vm_map_unlock_read(target_map);
2574 goto make_mem_done;
2575 }
2576 if (mask_protections) {
2577 /*
2578 * The caller asked us to use the "protections" as
2579 * a mask, so restrict "protections" to what this
2580 * mapping actually allows.
2581 */
2582 protections &= prot;
2583 }
2584 #if CONFIG_EMBEDDED
2585 /*
2586 * Wiring would copy the pages to a shadow object.
2587 * The shadow object would not be code-signed so
2588 * attempting to execute code from these copied pages
2589 * would trigger a code-signing violation.
2590 */
2591 if (prot & VM_PROT_EXECUTE) {
2592 if (log_executable_mem_entry) {
2593 void *bsd_info;
2594 bsd_info = current_task()->bsd_info;
2595 printf("pid %d[%s] making memory entry out of "
2596 "executable range from 0x%llx to 0x%llx:"
2597 "might cause code-signing issues "
2598 "later\n",
2599 proc_selfpid(),
2600 (bsd_info != NULL
2601 ? proc_name_address(bsd_info)
2602 : "?"),
2603 (uint64_t) map_start,
2604 (uint64_t) map_end);
2605 }
2606 DTRACE_VM2(cs_executable_mem_entry,
2607 uint64_t, (uint64_t)map_start,
2608 uint64_t, (uint64_t)map_end);
2609 cs_executable_mem_entry++;
2610
2611 #if 11
2612 /*
2613 * We don't know how the memory entry will be used.
2614 * It might never get wired and might not cause any
2615 * trouble, so let's not reject this request...
2616 */
2617 #else /* 11 */
2618 kr = KERN_PROTECTION_FAILURE;
2619 vm_object_unlock(object);
2620 vm_map_unlock_read(target_map);
2621 if(real_map != target_map)
2622 vm_map_unlock_read(real_map);
2623 goto make_mem_done;
2624 #endif /* 11 */
2625
2626 }
2627 #endif /* CONFIG_EMBEDDED */
2628
2629 if (((prot & protections) != protections)
2630 || (object == kernel_object)) {
2631 kr = KERN_INVALID_RIGHT;
2632 vm_object_unlock(object);
2633 vm_map_unlock_read(target_map);
2634 if(real_map != target_map)
2635 vm_map_unlock_read(real_map);
2636 if(object == kernel_object) {
2637 printf("Warning: Attempt to create a named"
2638 " entry from the kernel_object\n");
2639 }
2640 goto make_mem_done;
2641 }
2642
2643 /* We have an object, now check to see if this object */
2644 /* is suitable. If not, create a shadow and share that */
2645
2646 /*
2647 * We have to unlock the VM object to avoid deadlocking with
2648 * a VM map lock (the lock ordering is map, the object), if we
2649 * need to modify the VM map to create a shadow object. Since
2650 * we might release the VM map lock below anyway, we have
2651 * to release the VM map lock now.
2652 * XXX FBDP There must be a way to avoid this double lookup...
2653 *
2654 * Take an extra reference on the VM object to make sure it's
2655 * not going to disappear.
2656 */
2657 vm_object_reference_locked(object); /* extra ref to hold obj */
2658 vm_object_unlock(object);
2659
2660 local_map = original_map;
2661 local_offset = map_start;
2662 if(target_map != local_map) {
2663 vm_map_unlock_read(target_map);
2664 if(real_map != target_map)
2665 vm_map_unlock_read(real_map);
2666 vm_map_lock_read(local_map);
2667 target_map = local_map;
2668 real_map = local_map;
2669 }
2670 while(TRUE) {
2671 if(!vm_map_lookup_entry(local_map,
2672 local_offset, &map_entry)) {
2673 kr = KERN_INVALID_ARGUMENT;
2674 vm_map_unlock_read(target_map);
2675 if(real_map != target_map)
2676 vm_map_unlock_read(real_map);
2677 vm_object_deallocate(object); /* release extra ref */
2678 object = VM_OBJECT_NULL;
2679 goto make_mem_done;
2680 }
2681 iskernel = (local_map->pmap == kernel_pmap);
2682 if(!(map_entry->is_sub_map)) {
2683 if (VME_OBJECT(map_entry) != object) {
2684 kr = KERN_INVALID_ARGUMENT;
2685 vm_map_unlock_read(target_map);
2686 if(real_map != target_map)
2687 vm_map_unlock_read(real_map);
2688 vm_object_deallocate(object); /* release extra ref */
2689 object = VM_OBJECT_NULL;
2690 goto make_mem_done;
2691 }
2692 break;
2693 } else {
2694 vm_map_t tmap;
2695 tmap = local_map;
2696 local_map = VME_SUBMAP(map_entry);
2697
2698 vm_map_lock_read(local_map);
2699 vm_map_unlock_read(tmap);
2700 target_map = local_map;
2701 real_map = local_map;
2702 local_offset = local_offset - map_entry->vme_start;
2703 local_offset += VME_OFFSET(map_entry);
2704 }
2705 }
2706
2707 /*
2708 * We found the VM map entry, lock the VM object again.
2709 */
2710 vm_object_lock(object);
2711 if(map_entry->wired_count) {
2712 /* JMM - The check below should be reworked instead. */
2713 object->true_share = TRUE;
2714 }
2715 if (mask_protections) {
2716 /*
2717 * The caller asked us to use the "protections" as
2718 * a mask, so restrict "protections" to what this
2719 * mapping actually allows.
2720 */
2721 protections &= map_entry->max_protection;
2722 }
2723 if(((map_entry->max_protection) & protections) != protections) {
2724 kr = KERN_INVALID_RIGHT;
2725 vm_object_unlock(object);
2726 vm_map_unlock_read(target_map);
2727 if(real_map != target_map)
2728 vm_map_unlock_read(real_map);
2729 vm_object_deallocate(object);
2730 object = VM_OBJECT_NULL;
2731 goto make_mem_done;
2732 }
2733
2734 mappable_size = fault_info.hi_offset - obj_off;
2735 total_size = map_entry->vme_end - map_entry->vme_start;
2736 if(map_size > mappable_size) {
2737 /* try to extend mappable size if the entries */
2738 /* following are from the same object and are */
2739 /* compatible */
2740 next_entry = map_entry->vme_next;
2741 /* lets see if the next map entry is still */
2742 /* pointing at this object and is contiguous */
2743 while(map_size > mappable_size) {
2744 if ((VME_OBJECT(next_entry) == object) &&
2745 (next_entry->vme_start ==
2746 next_entry->vme_prev->vme_end) &&
2747 (VME_OFFSET(next_entry) ==
2748 (VME_OFFSET(next_entry->vme_prev) +
2749 (next_entry->vme_prev->vme_end -
2750 next_entry->vme_prev->vme_start)))) {
2751 if (mask_protections) {
2752 /*
2753 * The caller asked us to use
2754 * the "protections" as a mask,
2755 * so restrict "protections" to
2756 * what this mapping actually
2757 * allows.
2758 */
2759 protections &= next_entry->max_protection;
2760 }
2761 if ((next_entry->wired_count) &&
2762 (map_entry->wired_count == 0)) {
2763 break;
2764 }
2765 if(((next_entry->max_protection)
2766 & protections) != protections) {
2767 break;
2768 }
2769 if (next_entry->needs_copy !=
2770 map_entry->needs_copy)
2771 break;
2772 mappable_size += next_entry->vme_end
2773 - next_entry->vme_start;
2774 total_size += next_entry->vme_end
2775 - next_entry->vme_start;
2776 next_entry = next_entry->vme_next;
2777 } else {
2778 break;
2779 }
2780
2781 }
2782 }
2783
2784 /* vm_map_entry_should_cow_for_true_share() checks for malloc tags,
2785 * never true in kernel */
2786 if (!iskernel && vm_map_entry_should_cow_for_true_share(map_entry) &&
2787 object->vo_size > map_size &&
2788 map_size != 0) {
2789 /*
2790 * Set up the targeted range for copy-on-write to
2791 * limit the impact of "true_share"/"copy_delay" to
2792 * that range instead of the entire VM object...
2793 */
2794
2795 vm_object_unlock(object);
2796 if (vm_map_lock_read_to_write(target_map)) {
2797 vm_object_deallocate(object);
2798 target_map = original_map;
2799 goto redo_lookup;
2800 }
2801
2802 vm_map_clip_start(target_map,
2803 map_entry,
2804 vm_map_trunc_page(map_start,
2805 VM_MAP_PAGE_MASK(target_map)));
2806 vm_map_clip_end(target_map,
2807 map_entry,
2808 (vm_map_round_page(map_end,
2809 VM_MAP_PAGE_MASK(target_map))));
2810 force_shadow = TRUE;
2811
2812 if ((map_entry->vme_end - offset) < map_size) {
2813 map_size = map_entry->vme_end - map_start;
2814 }
2815 total_size = map_entry->vme_end - map_entry->vme_start;
2816
2817 vm_map_lock_write_to_read(target_map);
2818 vm_object_lock(object);
2819 }
2820
2821 if (object->internal) {
2822 /* vm_map_lookup_locked will create a shadow if */
2823 /* needs_copy is set but does not check for the */
2824 /* other two conditions shown. It is important to */
2825 /* set up an object which will not be pulled from */
2826 /* under us. */
2827
2828 if (force_shadow ||
2829 ((map_entry->needs_copy ||
2830 object->shadowed ||
2831 (object->vo_size > total_size &&
2832 (VME_OFFSET(map_entry) != 0 ||
2833 object->vo_size >
2834 vm_map_round_page(total_size,
2835 VM_MAP_PAGE_MASK(target_map)))))
2836 && !object->true_share)) {
2837 /*
2838 * We have to unlock the VM object before
2839 * trying to upgrade the VM map lock, to
2840 * honor lock ordering (map then object).
2841 * Otherwise, we would deadlock if another
2842 * thread holds a read lock on the VM map and
2843 * is trying to acquire the VM object's lock.
2844 * We still hold an extra reference on the
2845 * VM object, guaranteeing that it won't
2846 * disappear.
2847 */
2848 vm_object_unlock(object);
2849
2850 if (vm_map_lock_read_to_write(target_map)) {
2851 /*
2852 * We couldn't upgrade our VM map lock
2853 * from "read" to "write" and we lost
2854 * our "read" lock.
2855 * Start all over again...
2856 */
2857 vm_object_deallocate(object); /* extra ref */
2858 target_map = original_map;
2859 goto redo_lookup;
2860 }
2861 #if 00
2862 vm_object_lock(object);
2863 #endif
2864
2865 /*
2866 * JMM - We need to avoid coming here when the object
2867 * is wired by anybody, not just the current map. Why
2868 * couldn't we use the standard vm_object_copy_quickly()
2869 * approach here?
2870 */
2871
2872 /* create a shadow object */
2873 VME_OBJECT_SHADOW(map_entry, total_size);
2874 shadow_object = VME_OBJECT(map_entry);
2875 #if 00
2876 vm_object_unlock(object);
2877 #endif
2878
2879 prot = map_entry->protection & ~VM_PROT_WRITE;
2880
2881 if (override_nx(target_map,
2882 VME_ALIAS(map_entry))
2883 && prot)
2884 prot |= VM_PROT_EXECUTE;
2885
2886 vm_object_pmap_protect(
2887 object, VME_OFFSET(map_entry),
2888 total_size,
2889 ((map_entry->is_shared
2890 || target_map->mapped_in_other_pmaps)
2891 ? PMAP_NULL :
2892 target_map->pmap),
2893 map_entry->vme_start,
2894 prot);
2895 total_size -= (map_entry->vme_end
2896 - map_entry->vme_start);
2897 next_entry = map_entry->vme_next;
2898 map_entry->needs_copy = FALSE;
2899
2900 vm_object_lock(shadow_object);
2901 while (total_size) {
2902 assert((next_entry->wired_count == 0) ||
2903 (map_entry->wired_count));
2904
2905 if (VME_OBJECT(next_entry) == object) {
2906 vm_object_reference_locked(shadow_object);
2907 VME_OBJECT_SET(next_entry,
2908 shadow_object);
2909 vm_object_deallocate(object);
2910 VME_OFFSET_SET(
2911 next_entry,
2912 (VME_OFFSET(next_entry->vme_prev) +
2913 (next_entry->vme_prev->vme_end
2914 - next_entry->vme_prev->vme_start)));
2915 next_entry->needs_copy = FALSE;
2916 } else {
2917 panic("mach_make_memory_entry_64:"
2918 " map entries out of sync\n");
2919 }
2920 total_size -=
2921 next_entry->vme_end
2922 - next_entry->vme_start;
2923 next_entry = next_entry->vme_next;
2924 }
2925
2926 /*
2927 * Transfer our extra reference to the
2928 * shadow object.
2929 */
2930 vm_object_reference_locked(shadow_object);
2931 vm_object_deallocate(object); /* extra ref */
2932 object = shadow_object;
2933
2934 obj_off = ((local_offset - map_entry->vme_start)
2935 + VME_OFFSET(map_entry));
2936
2937 vm_map_lock_write_to_read(target_map);
2938 }
2939 }
2940
2941 /* note: in the future we can (if necessary) allow for */
2942 /* memory object lists, this will better support */
2943 /* fragmentation, but is it necessary? The user should */
2944 /* be encouraged to create address space oriented */
2945 /* shared objects from CLEAN memory regions which have */
2946 /* a known and defined history. i.e. no inheritence */
2947 /* share, make this call before making the region the */
2948 /* target of ipc's, etc. The code above, protecting */
2949 /* against delayed copy, etc. is mostly defensive. */
2950
2951 wimg_mode = object->wimg_bits;
2952 if(!(object->nophyscache))
2953 vm_prot_to_wimg(access, &wimg_mode);
2954
2955 #if VM_OBJECT_TRACKING_OP_TRUESHARE
2956 if (!object->true_share &&
2957 vm_object_tracking_inited) {
2958 void *bt[VM_OBJECT_TRACKING_BTDEPTH];
2959 int num = 0;
2960
2961 num = OSBacktrace(bt,
2962 VM_OBJECT_TRACKING_BTDEPTH);
2963 btlog_add_entry(vm_object_tracking_btlog,
2964 object,
2965 VM_OBJECT_TRACKING_OP_TRUESHARE,
2966 bt,
2967 num);
2968 }
2969 #endif /* VM_OBJECT_TRACKING_OP_TRUESHARE */
2970
2971 vm_object_lock_assert_exclusive(object);
2972 object->true_share = TRUE;
2973 if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC)
2974 object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
2975
2976 /*
2977 * The memory entry now points to this VM object and we
2978 * need to hold a reference on the VM object. Use the extra
2979 * reference we took earlier to keep the object alive when we
2980 * had to unlock it.
2981 */
2982
2983 vm_map_unlock_read(target_map);
2984 if(real_map != target_map)
2985 vm_map_unlock_read(real_map);
2986
2987 if (object->wimg_bits != wimg_mode)
2988 vm_object_change_wimg_mode(object, wimg_mode);
2989
2990 /* the size of mapped entry that overlaps with our region */
2991 /* which is targeted for share. */
2992 /* (entry_end - entry_start) - */
2993 /* offset of our beg addr within entry */
2994 /* it corresponds to this: */
2995
2996 if(map_size > mappable_size)
2997 map_size = mappable_size;
2998
2999 if (permission & MAP_MEM_NAMED_REUSE) {
3000 /*
3001 * Compare what we got with the "parent_entry".
3002 * If they match, re-use the "parent_entry" instead
3003 * of creating a new one.
3004 */
3005 if (parent_entry != NULL &&
3006 parent_entry->backing.object == object &&
3007 parent_entry->internal == object->internal &&
3008 parent_entry->is_sub_map == FALSE &&
3009 parent_entry->offset == obj_off &&
3010 parent_entry->protection == protections &&
3011 parent_entry->size == map_size &&
3012 ((!(use_data_addr || use_4K_compat) &&
3013 (parent_entry->data_offset == 0)) ||
3014 ((use_data_addr || use_4K_compat) &&
3015 (parent_entry->data_offset == offset_in_page)))) {
3016 /*
3017 * We have a match: re-use "parent_entry".
3018 */
3019 /* release our extra reference on object */
3020 vm_object_unlock(object);
3021 vm_object_deallocate(object);
3022 /* parent_entry->ref_count++; XXX ? */
3023 /* Get an extra send-right on handle */
3024 ipc_port_copy_send(parent_handle);
3025
3026 *size = CAST_DOWN(vm_size_t,
3027 (parent_entry->size -
3028 parent_entry->data_offset));
3029 *object_handle = parent_handle;
3030 return KERN_SUCCESS;
3031 } else {
3032 /*
3033 * No match: we need to create a new entry.
3034 * fall through...
3035 */
3036 }
3037 }
3038
3039 vm_object_unlock(object);
3040 if (mach_memory_entry_allocate(&user_entry, &user_handle)
3041 != KERN_SUCCESS) {
3042 /* release our unused reference on the object */
3043 vm_object_deallocate(object);
3044 return KERN_FAILURE;
3045 }
3046
3047 user_entry->backing.object = object;
3048 user_entry->internal = object->internal;
3049 user_entry->is_sub_map = FALSE;
3050 user_entry->offset = obj_off;
3051 user_entry->data_offset = offset_in_page;
3052 user_entry->protection = protections;
3053 SET_MAP_MEM(GET_MAP_MEM(permission), user_entry->protection);
3054 user_entry->size = map_size;
3055
3056 /* user_object pager and internal fields are not used */
3057 /* when the object field is filled in. */
3058
3059 *size = CAST_DOWN(vm_size_t, (user_entry->size -
3060 user_entry->data_offset));
3061 *object_handle = user_handle;
3062 return KERN_SUCCESS;
3063
3064 } else {
3065 /* The new object will be base on an existing named object */
3066 if (parent_entry == NULL) {
3067 kr = KERN_INVALID_ARGUMENT;
3068 goto make_mem_done;
3069 }
3070
3071 if (use_data_addr || use_4K_compat) {
3072 /*
3073 * submaps and pagers should only be accessible from within
3074 * the kernel, which shouldn't use the data address flag, so can fail here.
3075 */
3076 if (parent_entry->is_sub_map) {
3077 panic("Shouldn't be using data address with a parent entry that is a submap.");
3078 }
3079 /*
3080 * Account for offset to data in parent entry and
3081 * compute our own offset to data.
3082 */
3083 if((offset + *size + parent_entry->data_offset) > parent_entry->size) {
3084 kr = KERN_INVALID_ARGUMENT;
3085 goto make_mem_done;
3086 }
3087
3088 map_start = vm_map_trunc_page(offset + parent_entry->data_offset, PAGE_MASK);
3089 offset_in_page = (offset + parent_entry->data_offset) - map_start;
3090 if (use_4K_compat)
3091 offset_in_page &= ~((signed)(0xFFF));
3092 map_end = vm_map_round_page(offset + parent_entry->data_offset + *size, PAGE_MASK);
3093 map_size = map_end - map_start;
3094 } else {
3095 map_end = vm_map_round_page(offset + *size, PAGE_MASK);
3096 map_size = map_end - map_start;
3097 offset_in_page = 0;
3098
3099 if((offset + map_size) > parent_entry->size) {
3100 kr = KERN_INVALID_ARGUMENT;
3101 goto make_mem_done;
3102 }
3103 }
3104
3105 if (mask_protections) {
3106 /*
3107 * The caller asked us to use the "protections" as
3108 * a mask, so restrict "protections" to what this
3109 * mapping actually allows.
3110 */
3111 protections &= parent_entry->protection;
3112 }
3113 if((protections & parent_entry->protection) != protections) {
3114 kr = KERN_PROTECTION_FAILURE;
3115 goto make_mem_done;
3116 }
3117
3118 if (mach_memory_entry_allocate(&user_entry, &user_handle)
3119 != KERN_SUCCESS) {
3120 kr = KERN_FAILURE;
3121 goto make_mem_done;
3122 }
3123
3124 user_entry->size = map_size;
3125 user_entry->offset = parent_entry->offset + map_start;
3126 user_entry->data_offset = offset_in_page;
3127 user_entry->is_sub_map = parent_entry->is_sub_map;
3128 user_entry->is_copy = parent_entry->is_copy;
3129 user_entry->internal = parent_entry->internal;
3130 user_entry->protection = protections;
3131
3132 if(access != MAP_MEM_NOOP) {
3133 SET_MAP_MEM(access, user_entry->protection);
3134 }
3135
3136 if(parent_entry->is_sub_map) {
3137 user_entry->backing.map = parent_entry->backing.map;
3138 vm_map_lock(user_entry->backing.map);
3139 user_entry->backing.map->ref_count++;
3140 vm_map_unlock(user_entry->backing.map);
3141 } else {
3142 object = parent_entry->backing.object;
3143 assert(object != VM_OBJECT_NULL);
3144 user_entry->backing.object = object;
3145 /* we now point to this object, hold on */
3146 vm_object_lock(object);
3147 vm_object_reference_locked(object);
3148 #if VM_OBJECT_TRACKING_OP_TRUESHARE
3149 if (!object->true_share &&
3150 vm_object_tracking_inited) {
3151 void *bt[VM_OBJECT_TRACKING_BTDEPTH];
3152 int num = 0;
3153
3154 num = OSBacktrace(bt,
3155 VM_OBJECT_TRACKING_BTDEPTH);
3156 btlog_add_entry(vm_object_tracking_btlog,
3157 object,
3158 VM_OBJECT_TRACKING_OP_TRUESHARE,
3159 bt,
3160 num);
3161 }
3162 #endif /* VM_OBJECT_TRACKING_OP_TRUESHARE */
3163
3164 object->true_share = TRUE;
3165 if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC)
3166 object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
3167 vm_object_unlock(object);
3168 }
3169 *size = CAST_DOWN(vm_size_t, (user_entry->size -
3170 user_entry->data_offset));
3171 *object_handle = user_handle;
3172 return KERN_SUCCESS;
3173 }
3174
3175 make_mem_done:
3176 if (user_handle != IP_NULL) {
3177 /*
3178 * Releasing "user_handle" causes the kernel object
3179 * associated with it ("user_entry" here) to also be
3180 * released and freed.
3181 */
3182 mach_memory_entry_port_release(user_handle);
3183 }
3184 return kr;
3185 }
3186
3187 kern_return_t
3188 _mach_make_memory_entry(
3189 vm_map_t target_map,
3190 memory_object_size_t *size,
3191 memory_object_offset_t offset,
3192 vm_prot_t permission,
3193 ipc_port_t *object_handle,
3194 ipc_port_t parent_entry)
3195 {
3196 memory_object_size_t mo_size;
3197 kern_return_t kr;
3198
3199 mo_size = (memory_object_size_t)*size;
3200 kr = mach_make_memory_entry_64(target_map, &mo_size,
3201 (memory_object_offset_t)offset, permission, object_handle,
3202 parent_entry);
3203 *size = mo_size;
3204 return kr;
3205 }
3206
3207 kern_return_t
3208 mach_make_memory_entry(
3209 vm_map_t target_map,
3210 vm_size_t *size,
3211 vm_offset_t offset,
3212 vm_prot_t permission,
3213 ipc_port_t *object_handle,
3214 ipc_port_t parent_entry)
3215 {
3216 memory_object_size_t mo_size;
3217 kern_return_t kr;
3218
3219 mo_size = (memory_object_size_t)*size;
3220 kr = mach_make_memory_entry_64(target_map, &mo_size,
3221 (memory_object_offset_t)offset, permission, object_handle,
3222 parent_entry);
3223 *size = CAST_DOWN(vm_size_t, mo_size);
3224 return kr;
3225 }
3226
3227 /*
3228 * task_wire
3229 *
3230 * Set or clear the map's wiring_required flag. This flag, if set,
3231 * will cause all future virtual memory allocation to allocate
3232 * user wired memory. Unwiring pages wired down as a result of
3233 * this routine is done with the vm_wire interface.
3234 */
3235 kern_return_t
3236 task_wire(
3237 vm_map_t map,
3238 boolean_t must_wire)
3239 {
3240 if (map == VM_MAP_NULL)
3241 return(KERN_INVALID_ARGUMENT);
3242
3243 if (must_wire)
3244 map->wiring_required = TRUE;
3245 else
3246 map->wiring_required = FALSE;
3247
3248 return(KERN_SUCCESS);
3249 }
3250
3251 __private_extern__ kern_return_t
3252 mach_memory_entry_allocate(
3253 vm_named_entry_t *user_entry_p,
3254 ipc_port_t *user_handle_p)
3255 {
3256 vm_named_entry_t user_entry;
3257 ipc_port_t user_handle;
3258 ipc_port_t previous;
3259
3260 user_entry = (vm_named_entry_t) kalloc(sizeof *user_entry);
3261 if (user_entry == NULL)
3262 return KERN_FAILURE;
3263
3264 named_entry_lock_init(user_entry);
3265
3266 user_handle = ipc_port_alloc_kernel();
3267 if (user_handle == IP_NULL) {
3268 kfree(user_entry, sizeof *user_entry);
3269 return KERN_FAILURE;
3270 }
3271 ip_lock(user_handle);
3272
3273 /* make a sonce right */
3274 user_handle->ip_sorights++;
3275 ip_reference(user_handle);
3276
3277 user_handle->ip_destination = IP_NULL;
3278 user_handle->ip_receiver_name = MACH_PORT_NULL;
3279 user_handle->ip_receiver = ipc_space_kernel;
3280
3281 /* make a send right */
3282 user_handle->ip_mscount++;
3283 user_handle->ip_srights++;
3284 ip_reference(user_handle);
3285
3286 ipc_port_nsrequest(user_handle, 1, user_handle, &previous);
3287 /* nsrequest unlocks user_handle */
3288
3289 user_entry->backing.object = NULL;
3290 user_entry->is_sub_map = FALSE;
3291 user_entry->is_copy = FALSE;
3292 user_entry->internal = FALSE;
3293 user_entry->size = 0;
3294 user_entry->offset = 0;
3295 user_entry->data_offset = 0;
3296 user_entry->protection = VM_PROT_NONE;
3297 user_entry->ref_count = 1;
3298
3299 ipc_kobject_set(user_handle, (ipc_kobject_t) user_entry,
3300 IKOT_NAMED_ENTRY);
3301
3302 *user_entry_p = user_entry;
3303 *user_handle_p = user_handle;
3304
3305 return KERN_SUCCESS;
3306 }
3307
3308 /*
3309 * mach_memory_object_memory_entry_64
3310 *
3311 * Create a named entry backed by the provided pager.
3312 *
3313 */
3314 kern_return_t
3315 mach_memory_object_memory_entry_64(
3316 host_t host,
3317 boolean_t internal,
3318 vm_object_offset_t size,
3319 vm_prot_t permission,
3320 memory_object_t pager,
3321 ipc_port_t *entry_handle)
3322 {
3323 unsigned int access;
3324 vm_named_entry_t user_entry;
3325 ipc_port_t user_handle;
3326 vm_object_t object;
3327
3328 if (host == HOST_NULL)
3329 return(KERN_INVALID_HOST);
3330
3331 if (pager == MEMORY_OBJECT_NULL && internal) {
3332 object = vm_object_allocate(size);
3333 if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC) {
3334 object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
3335 }
3336 } else {
3337 object = memory_object_to_vm_object(pager);
3338 if (object != VM_OBJECT_NULL) {
3339 vm_object_reference(object);
3340 }
3341 }
3342 if (object == VM_OBJECT_NULL) {
3343 return KERN_INVALID_ARGUMENT;
3344 }
3345
3346 if (mach_memory_entry_allocate(&user_entry, &user_handle)
3347 != KERN_SUCCESS) {
3348 vm_object_deallocate(object);
3349 return KERN_FAILURE;
3350 }
3351
3352 user_entry->size = size;
3353 user_entry->offset = 0;
3354 user_entry->protection = permission & VM_PROT_ALL;
3355 access = GET_MAP_MEM(permission);
3356 SET_MAP_MEM(access, user_entry->protection);
3357 user_entry->is_sub_map = FALSE;
3358 assert(user_entry->ref_count == 1);
3359
3360 user_entry->backing.object = object;
3361 user_entry->internal = object->internal;
3362 assert(object->internal == internal);
3363
3364 *entry_handle = user_handle;
3365 return KERN_SUCCESS;
3366 }
3367
3368 kern_return_t
3369 mach_memory_object_memory_entry(
3370 host_t host,
3371 boolean_t internal,
3372 vm_size_t size,
3373 vm_prot_t permission,
3374 memory_object_t pager,
3375 ipc_port_t *entry_handle)
3376 {
3377 return mach_memory_object_memory_entry_64( host, internal,
3378 (vm_object_offset_t)size, permission, pager, entry_handle);
3379 }
3380
3381
3382 kern_return_t
3383 mach_memory_entry_purgable_control(
3384 ipc_port_t entry_port,
3385 vm_purgable_t control,
3386 int *state)
3387 {
3388 if (control == VM_PURGABLE_SET_STATE_FROM_KERNEL) {
3389 /* not allowed from user-space */
3390 return KERN_INVALID_ARGUMENT;
3391 }
3392
3393 return memory_entry_purgeable_control_internal(entry_port, control, state);
3394 }
3395
3396 kern_return_t
3397 memory_entry_purgeable_control_internal(
3398 ipc_port_t entry_port,
3399 vm_purgable_t control,
3400 int *state)
3401 {
3402 kern_return_t kr;
3403 vm_named_entry_t mem_entry;
3404 vm_object_t object;
3405
3406 if (entry_port == IP_NULL ||
3407 ip_kotype(entry_port) != IKOT_NAMED_ENTRY) {
3408 return KERN_INVALID_ARGUMENT;
3409 }
3410 if (control != VM_PURGABLE_SET_STATE &&
3411 control != VM_PURGABLE_GET_STATE &&
3412 control != VM_PURGABLE_SET_STATE_FROM_KERNEL)
3413 return(KERN_INVALID_ARGUMENT);
3414
3415 if ((control == VM_PURGABLE_SET_STATE ||
3416 control == VM_PURGABLE_SET_STATE_FROM_KERNEL) &&
3417 (((*state & ~(VM_PURGABLE_ALL_MASKS)) != 0) ||
3418 ((*state & VM_PURGABLE_STATE_MASK) > VM_PURGABLE_STATE_MASK)))
3419 return(KERN_INVALID_ARGUMENT);
3420
3421 mem_entry = (vm_named_entry_t) entry_port->ip_kobject;
3422
3423 named_entry_lock(mem_entry);
3424
3425 if (mem_entry->is_sub_map ||
3426 mem_entry->is_copy) {
3427 named_entry_unlock(mem_entry);
3428 return KERN_INVALID_ARGUMENT;
3429 }
3430
3431 object = mem_entry->backing.object;
3432 if (object == VM_OBJECT_NULL) {
3433 named_entry_unlock(mem_entry);
3434 return KERN_INVALID_ARGUMENT;
3435 }
3436
3437 vm_object_lock(object);
3438
3439 /* check that named entry covers entire object ? */
3440 if (mem_entry->offset != 0 || object->vo_size != mem_entry->size) {
3441 vm_object_unlock(object);
3442 named_entry_unlock(mem_entry);
3443 return KERN_INVALID_ARGUMENT;
3444 }
3445
3446 named_entry_unlock(mem_entry);
3447
3448 kr = vm_object_purgable_control(object, control, state);
3449
3450 vm_object_unlock(object);
3451
3452 return kr;
3453 }
3454
3455 kern_return_t
3456 mach_memory_entry_get_page_counts(
3457 ipc_port_t entry_port,
3458 unsigned int *resident_page_count,
3459 unsigned int *dirty_page_count)
3460 {
3461 kern_return_t kr;
3462 vm_named_entry_t mem_entry;
3463 vm_object_t object;
3464 vm_object_offset_t offset;
3465 vm_object_size_t size;
3466
3467 if (entry_port == IP_NULL ||
3468 ip_kotype(entry_port) != IKOT_NAMED_ENTRY) {
3469 return KERN_INVALID_ARGUMENT;
3470 }
3471
3472 mem_entry = (vm_named_entry_t) entry_port->ip_kobject;
3473
3474 named_entry_lock(mem_entry);
3475
3476 if (mem_entry->is_sub_map ||
3477 mem_entry->is_copy) {
3478 named_entry_unlock(mem_entry);
3479 return KERN_INVALID_ARGUMENT;
3480 }
3481
3482 object = mem_entry->backing.object;
3483 if (object == VM_OBJECT_NULL) {
3484 named_entry_unlock(mem_entry);
3485 return KERN_INVALID_ARGUMENT;
3486 }
3487
3488 vm_object_lock(object);
3489
3490 offset = mem_entry->offset;
3491 size = mem_entry->size;
3492
3493 named_entry_unlock(mem_entry);
3494
3495 kr = vm_object_get_page_counts(object, offset, size, resident_page_count, dirty_page_count);
3496
3497 vm_object_unlock(object);
3498
3499 return kr;
3500 }
3501
3502 /*
3503 * mach_memory_entry_port_release:
3504 *
3505 * Release a send right on a named entry port. This is the correct
3506 * way to destroy a named entry. When the last right on the port is
3507 * released, ipc_kobject_destroy() will call mach_destroy_memory_entry().
3508 */
3509 void
3510 mach_memory_entry_port_release(
3511 ipc_port_t port)
3512 {
3513 assert(ip_kotype(port) == IKOT_NAMED_ENTRY);
3514 ipc_port_release_send(port);
3515 }
3516
3517 /*
3518 * mach_destroy_memory_entry:
3519 *
3520 * Drops a reference on a memory entry and destroys the memory entry if
3521 * there are no more references on it.
3522 * NOTE: This routine should not be called to destroy a memory entry from the
3523 * kernel, as it will not release the Mach port associated with the memory
3524 * entry. The proper way to destroy a memory entry in the kernel is to
3525 * call mach_memort_entry_port_release() to release the kernel's send-right on
3526 * the memory entry's port. When the last send right is released, the memory
3527 * entry will be destroyed via ipc_kobject_destroy().
3528 */
3529 void
3530 mach_destroy_memory_entry(
3531 ipc_port_t port)
3532 {
3533 vm_named_entry_t named_entry;
3534 #if MACH_ASSERT
3535 assert(ip_kotype(port) == IKOT_NAMED_ENTRY);
3536 #endif /* MACH_ASSERT */
3537 named_entry = (vm_named_entry_t)port->ip_kobject;
3538
3539 named_entry_lock(named_entry);
3540 named_entry->ref_count -= 1;
3541
3542 if(named_entry->ref_count == 0) {
3543 if (named_entry->is_sub_map) {
3544 vm_map_deallocate(named_entry->backing.map);
3545 } else if (named_entry->is_copy) {
3546 vm_map_copy_discard(named_entry->backing.copy);
3547 } else {
3548 /* release the VM object we've been pointing to */
3549 vm_object_deallocate(named_entry->backing.object);
3550 }
3551
3552 named_entry_unlock(named_entry);
3553 named_entry_lock_destroy(named_entry);
3554
3555 kfree((void *) port->ip_kobject,
3556 sizeof (struct vm_named_entry));
3557 } else
3558 named_entry_unlock(named_entry);
3559 }
3560
3561 /* Allow manipulation of individual page state. This is actually part of */
3562 /* the UPL regimen but takes place on the memory entry rather than on a UPL */
3563
3564 kern_return_t
3565 mach_memory_entry_page_op(
3566 ipc_port_t entry_port,
3567 vm_object_offset_t offset,
3568 int ops,
3569 ppnum_t *phys_entry,
3570 int *flags)
3571 {
3572 vm_named_entry_t mem_entry;
3573 vm_object_t object;
3574 kern_return_t kr;
3575
3576 if (entry_port == IP_NULL ||
3577 ip_kotype(entry_port) != IKOT_NAMED_ENTRY) {
3578 return KERN_INVALID_ARGUMENT;
3579 }
3580
3581 mem_entry = (vm_named_entry_t) entry_port->ip_kobject;
3582
3583 named_entry_lock(mem_entry);
3584
3585 if (mem_entry->is_sub_map ||
3586 mem_entry->is_copy) {
3587 named_entry_unlock(mem_entry);
3588 return KERN_INVALID_ARGUMENT;
3589 }
3590
3591 object = mem_entry->backing.object;
3592 if (object == VM_OBJECT_NULL) {
3593 named_entry_unlock(mem_entry);
3594 return KERN_INVALID_ARGUMENT;
3595 }
3596
3597 vm_object_reference(object);
3598 named_entry_unlock(mem_entry);
3599
3600 kr = vm_object_page_op(object, offset, ops, phys_entry, flags);
3601
3602 vm_object_deallocate(object);
3603
3604 return kr;
3605 }
3606
3607 /*
3608 * mach_memory_entry_range_op offers performance enhancement over
3609 * mach_memory_entry_page_op for page_op functions which do not require page
3610 * level state to be returned from the call. Page_op was created to provide
3611 * a low-cost alternative to page manipulation via UPLs when only a single
3612 * page was involved. The range_op call establishes the ability in the _op
3613 * family of functions to work on multiple pages where the lack of page level
3614 * state handling allows the caller to avoid the overhead of the upl structures.
3615 */
3616
3617 kern_return_t
3618 mach_memory_entry_range_op(
3619 ipc_port_t entry_port,
3620 vm_object_offset_t offset_beg,
3621 vm_object_offset_t offset_end,
3622 int ops,
3623 int *range)
3624 {
3625 vm_named_entry_t mem_entry;
3626 vm_object_t object;
3627 kern_return_t kr;
3628
3629 if (entry_port == IP_NULL ||
3630 ip_kotype(entry_port) != IKOT_NAMED_ENTRY) {
3631 return KERN_INVALID_ARGUMENT;
3632 }
3633
3634 mem_entry = (vm_named_entry_t) entry_port->ip_kobject;
3635
3636 named_entry_lock(mem_entry);
3637
3638 if (mem_entry->is_sub_map ||
3639 mem_entry->is_copy) {
3640 named_entry_unlock(mem_entry);
3641 return KERN_INVALID_ARGUMENT;
3642 }
3643
3644 object = mem_entry->backing.object;
3645 if (object == VM_OBJECT_NULL) {
3646 named_entry_unlock(mem_entry);
3647 return KERN_INVALID_ARGUMENT;
3648 }
3649
3650 vm_object_reference(object);
3651 named_entry_unlock(mem_entry);
3652
3653 kr = vm_object_range_op(object,
3654 offset_beg,
3655 offset_end,
3656 ops,
3657 (uint32_t *) range);
3658
3659 vm_object_deallocate(object);
3660
3661 return kr;
3662 }
3663
3664 /* ******* Temporary Internal calls to UPL for BSD ***** */
3665
3666 extern int kernel_upl_map(
3667 vm_map_t map,
3668 upl_t upl,
3669 vm_offset_t *dst_addr);
3670
3671 extern int kernel_upl_unmap(
3672 vm_map_t map,
3673 upl_t upl);
3674
3675 extern int kernel_upl_commit(
3676 upl_t upl,
3677 upl_page_info_t *pl,
3678 mach_msg_type_number_t count);
3679
3680 extern int kernel_upl_commit_range(
3681 upl_t upl,
3682 upl_offset_t offset,
3683 upl_size_t size,
3684 int flags,
3685 upl_page_info_array_t pl,
3686 mach_msg_type_number_t count);
3687
3688 extern int kernel_upl_abort(
3689 upl_t upl,
3690 int abort_type);
3691
3692 extern int kernel_upl_abort_range(
3693 upl_t upl,
3694 upl_offset_t offset,
3695 upl_size_t size,
3696 int abort_flags);
3697
3698
3699 kern_return_t
3700 kernel_upl_map(
3701 vm_map_t map,
3702 upl_t upl,
3703 vm_offset_t *dst_addr)
3704 {
3705 return vm_upl_map(map, upl, dst_addr);
3706 }
3707
3708
3709 kern_return_t
3710 kernel_upl_unmap(
3711 vm_map_t map,
3712 upl_t upl)
3713 {
3714 return vm_upl_unmap(map, upl);
3715 }
3716
3717 kern_return_t
3718 kernel_upl_commit(
3719 upl_t upl,
3720 upl_page_info_t *pl,
3721 mach_msg_type_number_t count)
3722 {
3723 kern_return_t kr;
3724
3725 kr = upl_commit(upl, pl, count);
3726 upl_deallocate(upl);
3727 return kr;
3728 }
3729
3730
3731 kern_return_t
3732 kernel_upl_commit_range(
3733 upl_t upl,
3734 upl_offset_t offset,
3735 upl_size_t size,
3736 int flags,
3737 upl_page_info_array_t pl,
3738 mach_msg_type_number_t count)
3739 {
3740 boolean_t finished = FALSE;
3741 kern_return_t kr;
3742
3743 if (flags & UPL_COMMIT_FREE_ON_EMPTY)
3744 flags |= UPL_COMMIT_NOTIFY_EMPTY;
3745
3746 if (flags & UPL_COMMIT_KERNEL_ONLY_FLAGS) {
3747 return KERN_INVALID_ARGUMENT;
3748 }
3749
3750 kr = upl_commit_range(upl, offset, size, flags, pl, count, &finished);
3751
3752 if ((flags & UPL_COMMIT_NOTIFY_EMPTY) && finished)
3753 upl_deallocate(upl);
3754
3755 return kr;
3756 }
3757
3758 kern_return_t
3759 kernel_upl_abort_range(
3760 upl_t upl,
3761 upl_offset_t offset,
3762 upl_size_t size,
3763 int abort_flags)
3764 {
3765 kern_return_t kr;
3766 boolean_t finished = FALSE;
3767
3768 if (abort_flags & UPL_COMMIT_FREE_ON_EMPTY)
3769 abort_flags |= UPL_COMMIT_NOTIFY_EMPTY;
3770
3771 kr = upl_abort_range(upl, offset, size, abort_flags, &finished);
3772
3773 if ((abort_flags & UPL_COMMIT_FREE_ON_EMPTY) && finished)
3774 upl_deallocate(upl);
3775
3776 return kr;
3777 }
3778
3779 kern_return_t
3780 kernel_upl_abort(
3781 upl_t upl,
3782 int abort_type)
3783 {
3784 kern_return_t kr;
3785
3786 kr = upl_abort(upl, abort_type);
3787 upl_deallocate(upl);
3788 return kr;
3789 }
3790
3791 /*
3792 * Now a kernel-private interface (for BootCache
3793 * use only). Need a cleaner way to create an
3794 * empty vm_map() and return a handle to it.
3795 */
3796
3797 kern_return_t
3798 vm_region_object_create(
3799 __unused vm_map_t target_map,
3800 vm_size_t size,
3801 ipc_port_t *object_handle)
3802 {
3803 vm_named_entry_t user_entry;
3804 ipc_port_t user_handle;
3805
3806 vm_map_t new_map;
3807
3808 if (mach_memory_entry_allocate(&user_entry, &user_handle)
3809 != KERN_SUCCESS) {
3810 return KERN_FAILURE;
3811 }
3812
3813 /* Create a named object based on a submap of specified size */
3814
3815 new_map = vm_map_create(PMAP_NULL, VM_MAP_MIN_ADDRESS,
3816 vm_map_round_page(size,
3817 VM_MAP_PAGE_MASK(target_map)),
3818 TRUE);
3819 vm_map_set_page_shift(new_map, VM_MAP_PAGE_SHIFT(target_map));
3820
3821 user_entry->backing.map = new_map;
3822 user_entry->internal = TRUE;
3823 user_entry->is_sub_map = TRUE;
3824 user_entry->offset = 0;
3825 user_entry->protection = VM_PROT_ALL;
3826 user_entry->size = size;
3827 assert(user_entry->ref_count == 1);
3828
3829 *object_handle = user_handle;
3830 return KERN_SUCCESS;
3831
3832 }
3833
3834 ppnum_t vm_map_get_phys_page( /* forward */
3835 vm_map_t map,
3836 vm_offset_t offset);
3837
3838 ppnum_t
3839 vm_map_get_phys_page(
3840 vm_map_t map,
3841 vm_offset_t addr)
3842 {
3843 vm_object_offset_t offset;
3844 vm_object_t object;
3845 vm_map_offset_t map_offset;
3846 vm_map_entry_t entry;
3847 ppnum_t phys_page = 0;
3848
3849 map_offset = vm_map_trunc_page(addr, PAGE_MASK);
3850
3851 vm_map_lock(map);
3852 while (vm_map_lookup_entry(map, map_offset, &entry)) {
3853
3854 if (VME_OBJECT(entry) == VM_OBJECT_NULL) {
3855 vm_map_unlock(map);
3856 return (ppnum_t) 0;
3857 }
3858 if (entry->is_sub_map) {
3859 vm_map_t old_map;
3860 vm_map_lock(VME_SUBMAP(entry));
3861 old_map = map;
3862 map = VME_SUBMAP(entry);
3863 map_offset = (VME_OFFSET(entry) +
3864 (map_offset - entry->vme_start));
3865 vm_map_unlock(old_map);
3866 continue;
3867 }
3868 if (VME_OBJECT(entry)->phys_contiguous) {
3869 /* These are not standard pageable memory mappings */
3870 /* If they are not present in the object they will */
3871 /* have to be picked up from the pager through the */
3872 /* fault mechanism. */
3873 if (VME_OBJECT(entry)->vo_shadow_offset == 0) {
3874 /* need to call vm_fault */
3875 vm_map_unlock(map);
3876 vm_fault(map, map_offset, VM_PROT_NONE,
3877 FALSE /* change_wiring */, VM_KERN_MEMORY_NONE,
3878 THREAD_UNINT, NULL, 0);
3879 vm_map_lock(map);
3880 continue;
3881 }
3882 offset = (VME_OFFSET(entry) +
3883 (map_offset - entry->vme_start));
3884 phys_page = (ppnum_t)
3885 ((VME_OBJECT(entry)->vo_shadow_offset
3886 + offset) >> PAGE_SHIFT);
3887 break;
3888
3889 }
3890 offset = (VME_OFFSET(entry) + (map_offset - entry->vme_start));
3891 object = VME_OBJECT(entry);
3892 vm_object_lock(object);
3893 while (TRUE) {
3894 vm_page_t dst_page = vm_page_lookup(object,offset);
3895 if(dst_page == VM_PAGE_NULL) {
3896 if(object->shadow) {
3897 vm_object_t old_object;
3898 vm_object_lock(object->shadow);
3899 old_object = object;
3900 offset = offset + object->vo_shadow_offset;
3901 object = object->shadow;
3902 vm_object_unlock(old_object);
3903 } else {
3904 vm_object_unlock(object);
3905 break;
3906 }
3907 } else {
3908 phys_page = (ppnum_t)(VM_PAGE_GET_PHYS_PAGE(dst_page));
3909 vm_object_unlock(object);
3910 break;
3911 }
3912 }
3913 break;
3914
3915 }
3916
3917 vm_map_unlock(map);
3918 return phys_page;
3919 }
3920
3921 #if 0
3922 kern_return_t kernel_object_iopl_request( /* forward */
3923 vm_named_entry_t named_entry,
3924 memory_object_offset_t offset,
3925 upl_size_t *upl_size,
3926 upl_t *upl_ptr,
3927 upl_page_info_array_t user_page_list,
3928 unsigned int *page_list_count,
3929 int *flags);
3930
3931 kern_return_t
3932 kernel_object_iopl_request(
3933 vm_named_entry_t named_entry,
3934 memory_object_offset_t offset,
3935 upl_size_t *upl_size,
3936 upl_t *upl_ptr,
3937 upl_page_info_array_t user_page_list,
3938 unsigned int *page_list_count,
3939 int *flags)
3940 {
3941 vm_object_t object;
3942 kern_return_t ret;
3943
3944 int caller_flags;
3945
3946 caller_flags = *flags;
3947
3948 if (caller_flags & ~UPL_VALID_FLAGS) {
3949 /*
3950 * For forward compatibility's sake,
3951 * reject any unknown flag.
3952 */
3953 return KERN_INVALID_VALUE;
3954 }
3955
3956 /* a few checks to make sure user is obeying rules */
3957 if(*upl_size == 0) {
3958 if(offset >= named_entry->size)
3959 return(KERN_INVALID_RIGHT);
3960 *upl_size = (upl_size_t) (named_entry->size - offset);
3961 if (*upl_size != named_entry->size - offset)
3962 return KERN_INVALID_ARGUMENT;
3963 }
3964 if(caller_flags & UPL_COPYOUT_FROM) {
3965 if((named_entry->protection & VM_PROT_READ)
3966 != VM_PROT_READ) {
3967 return(KERN_INVALID_RIGHT);
3968 }
3969 } else {
3970 if((named_entry->protection &
3971 (VM_PROT_READ | VM_PROT_WRITE))
3972 != (VM_PROT_READ | VM_PROT_WRITE)) {
3973 return(KERN_INVALID_RIGHT);
3974 }
3975 }
3976 if(named_entry->size < (offset + *upl_size))
3977 return(KERN_INVALID_ARGUMENT);
3978
3979 /* the callers parameter offset is defined to be the */
3980 /* offset from beginning of named entry offset in object */
3981 offset = offset + named_entry->offset;
3982
3983 if (named_entry->is_sub_map ||
3984 named_entry->is_copy)
3985 return KERN_INVALID_ARGUMENT;
3986
3987 named_entry_lock(named_entry);
3988
3989 /* This is the case where we are going to operate */
3990 /* on an already known object. If the object is */
3991 /* not ready it is internal. An external */
3992 /* object cannot be mapped until it is ready */
3993 /* we can therefore avoid the ready check */
3994 /* in this case. */
3995 object = named_entry->backing.object;
3996 vm_object_reference(object);
3997 named_entry_unlock(named_entry);
3998
3999 if (!object->private) {
4000 if (*upl_size > MAX_UPL_TRANSFER_BYTES)
4001 *upl_size = MAX_UPL_TRANSFER_BYTES;
4002 if (object->phys_contiguous) {
4003 *flags = UPL_PHYS_CONTIG;
4004 } else {
4005 *flags = 0;
4006 }
4007 } else {
4008 *flags = UPL_DEV_MEMORY | UPL_PHYS_CONTIG;
4009 }
4010
4011 ret = vm_object_iopl_request(object,
4012 offset,
4013 *upl_size,
4014 upl_ptr,
4015 user_page_list,
4016 page_list_count,
4017 (upl_control_flags_t)(unsigned int)caller_flags);
4018 vm_object_deallocate(object);
4019 return ret;
4020 }
4021 #endif
4022
4023 /*
4024 * These symbols are looked up at runtime by vmware, VirtualBox,
4025 * despite not being exported in the symbol sets.
4026 */
4027
4028 #if defined(__x86_64__)
4029
4030 kern_return_t
4031 mach_vm_map(
4032 vm_map_t target_map,
4033 mach_vm_offset_t *address,
4034 mach_vm_size_t initial_size,
4035 mach_vm_offset_t mask,
4036 int flags,
4037 ipc_port_t port,
4038 vm_object_offset_t offset,
4039 boolean_t copy,
4040 vm_prot_t cur_protection,
4041 vm_prot_t max_protection,
4042 vm_inherit_t inheritance);
4043
4044 kern_return_t
4045 mach_vm_remap(
4046 vm_map_t target_map,
4047 mach_vm_offset_t *address,
4048 mach_vm_size_t size,
4049 mach_vm_offset_t mask,
4050 int flags,
4051 vm_map_t src_map,
4052 mach_vm_offset_t memory_address,
4053 boolean_t copy,
4054 vm_prot_t *cur_protection,
4055 vm_prot_t *max_protection,
4056 vm_inherit_t inheritance);
4057
4058 kern_return_t
4059 mach_vm_map(
4060 vm_map_t target_map,
4061 mach_vm_offset_t *address,
4062 mach_vm_size_t initial_size,
4063 mach_vm_offset_t mask,
4064 int flags,
4065 ipc_port_t port,
4066 vm_object_offset_t offset,
4067 boolean_t copy,
4068 vm_prot_t cur_protection,
4069 vm_prot_t max_protection,
4070 vm_inherit_t inheritance)
4071 {
4072 return (mach_vm_map_external(target_map, address, initial_size, mask, flags, port,
4073 offset, copy, cur_protection, max_protection, inheritance));
4074 }
4075
4076 kern_return_t
4077 mach_vm_remap(
4078 vm_map_t target_map,
4079 mach_vm_offset_t *address,
4080 mach_vm_size_t size,
4081 mach_vm_offset_t mask,
4082 int flags,
4083 vm_map_t src_map,
4084 mach_vm_offset_t memory_address,
4085 boolean_t copy,
4086 vm_prot_t *cur_protection,
4087 vm_prot_t *max_protection,
4088 vm_inherit_t inheritance)
4089 {
4090 return (mach_vm_remap_external(target_map, address, size, mask, flags, src_map, memory_address,
4091 copy, cur_protection, max_protection, inheritance));
4092 }
4093
4094 kern_return_t
4095 vm_map(
4096 vm_map_t target_map,
4097 vm_offset_t *address,
4098 vm_size_t size,
4099 vm_offset_t mask,
4100 int flags,
4101 ipc_port_t port,
4102 vm_offset_t offset,
4103 boolean_t copy,
4104 vm_prot_t cur_protection,
4105 vm_prot_t max_protection,
4106 vm_inherit_t inheritance);
4107
4108 kern_return_t
4109 vm_map(
4110 vm_map_t target_map,
4111 vm_offset_t *address,
4112 vm_size_t size,
4113 vm_offset_t mask,
4114 int flags,
4115 ipc_port_t port,
4116 vm_offset_t offset,
4117 boolean_t copy,
4118 vm_prot_t cur_protection,
4119 vm_prot_t max_protection,
4120 vm_inherit_t inheritance)
4121 {
4122 vm_tag_t tag;
4123
4124 VM_GET_FLAGS_ALIAS(flags, tag);
4125 return (vm_map_kernel(target_map, address, size, mask, flags, tag, port, offset, copy, cur_protection, max_protection, inheritance));
4126 }
4127
4128 #endif /* __x86_64__ */
mirror of XNU