]> git.saurik.com Git - apple/xnu.git/blob - osfmk/vm/vm_user.c
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-792.13.8.tar.gz
[apple/xnu.git] / osfmk / vm / vm_user.c
1 /*
2 * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_LICENSE_OSREFERENCE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the
10 * License may not be used to create, or enable the creation or
11 * redistribution of, unlawful or unlicensed copies of an Apple operating
12 * system, or to circumvent, violate, or enable the circumvention or
13 * violation of, any terms of an Apple operating system software license
14 * agreement.
15 *
16 * Please obtain a copy of the License at
17 * http://www.opensource.apple.com/apsl/ and read it before using this
18 * file.
19 *
20 * The Original Code and all software distributed under the License are
21 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
22 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
23 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
24 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
25 * Please see the License for the specific language governing rights and
26 * limitations under the License.
27 *
28 * @APPLE_LICENSE_OSREFERENCE_HEADER_END@
29 */
30 /*
31 * @OSF_COPYRIGHT@
32 */
33 /*
34 * Mach Operating System
35 * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University
36 * All Rights Reserved.
37 *
38 * Permission to use, copy, modify and distribute this software and its
39 * documentation is hereby granted, provided that both the copyright
40 * notice and this permission notice appear in all copies of the
41 * software, derivative works or modified versions, and any portions
42 * thereof, and that both notices appear in supporting documentation.
43 *
44 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
45 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
46 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
47 *
48 * Carnegie Mellon requests users of this software to return to
49 *
50 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
51 * School of Computer Science
52 * Carnegie Mellon University
53 * Pittsburgh PA 15213-3890
54 *
55 * any improvements or extensions that they make and grant Carnegie Mellon
56 * the rights to redistribute these changes.
57 */
58 /*
59 */
60 /*
61 * File: vm/vm_user.c
62 * Author: Avadis Tevanian, Jr., Michael Wayne Young
63 *
64 * User-exported virtual memory functions.
65 */
66
67 #include <debug.h>
68
69 #include <vm_cpm.h>
70 #include <mach/boolean.h>
71 #include <mach/kern_return.h>
72 #include <mach/mach_types.h> /* to get vm_address_t */
73 #include <mach/memory_object.h>
74 #include <mach/std_types.h> /* to get pointer_t */
75 #include <mach/upl.h>
76 #include <mach/vm_attributes.h>
77 #include <mach/vm_param.h>
78 #include <mach/vm_statistics.h>
79 #include <mach/mach_syscalls.h>
80
81 #include <mach/host_priv_server.h>
82 #include <mach/mach_vm_server.h>
83 #include <mach/shared_memory_server.h>
84 #include <mach/vm_map_server.h>
85 #include <vm/vm_shared_memory_server.h>
86
87 #include <kern/host.h>
88 #include <kern/kalloc.h>
89 #include <kern/task.h>
90 #include <kern/misc_protos.h>
91 #include <vm/vm_fault.h>
92 #include <vm/vm_map.h>
93 #include <vm/vm_object.h>
94 #include <vm/vm_page.h>
95 #include <vm/memory_object.h>
96 #include <vm/vm_pageout.h>
97 #include <vm/vm_protos.h>
98
99 vm_size_t upl_offset_to_pagelist = 0;
100
101 #if VM_CPM
102 #include <vm/cpm.h>
103 #endif /* VM_CPM */
104
105 ipc_port_t dynamic_pager_control_port=NULL;
106
107 /*
108 * mach_vm_allocate allocates "zero fill" memory in the specfied
109 * map.
110 */
111 kern_return_t
112 mach_vm_allocate(
113 vm_map_t map,
114 mach_vm_offset_t *addr,
115 mach_vm_size_t size,
116 int flags)
117 {
118 vm_map_offset_t map_addr;
119 vm_map_size_t map_size;
120 kern_return_t result;
121 boolean_t anywhere = ((VM_FLAGS_ANYWHERE & flags) != 0);
122
123 if (map == VM_MAP_NULL)
124 return(KERN_INVALID_ARGUMENT);
125 if (size == 0) {
126 *addr = 0;
127 return(KERN_SUCCESS);
128 }
129
130 if (anywhere) {
131 /*
132 * No specific address requested, so start candidate address
133 * search at the minimum address in the map. However, if that
134 * minimum is 0, bump it up by PAGE_SIZE. We want to limit
135 * allocations of PAGEZERO to explicit requests since its
136 * normal use is to catch dereferences of NULL and many
137 * applications also treat pointers with a value of 0 as
138 * special and suddenly having address 0 contain useable
139 * memory would tend to confuse those applications.
140 */
141 map_addr = vm_map_min(map);
142 if (map_addr == 0)
143 map_addr += PAGE_SIZE;
144 } else
145 map_addr = vm_map_trunc_page(*addr);
146 map_size = vm_map_round_page(size);
147 if (map_size == 0) {
148 return(KERN_INVALID_ARGUMENT);
149 }
150
151 result = vm_map_enter(
152 map,
153 &map_addr,
154 map_size,
155 (vm_map_offset_t)0,
156 flags,
157 VM_OBJECT_NULL,
158 (vm_object_offset_t)0,
159 FALSE,
160 VM_PROT_DEFAULT,
161 VM_PROT_ALL,
162 VM_INHERIT_DEFAULT);
163
164 *addr = map_addr;
165 return(result);
166 }
167
168 /*
169 * vm_allocate
170 * Legacy routine that allocates "zero fill" memory in the specfied
171 * map (which is limited to the same size as the kernel).
172 */
173 kern_return_t
174 vm_allocate(
175 vm_map_t map,
176 vm_offset_t *addr,
177 vm_size_t size,
178 int flags)
179 {
180 vm_map_offset_t map_addr;
181 vm_map_size_t map_size;
182 kern_return_t result;
183 boolean_t anywhere = ((VM_FLAGS_ANYWHERE & flags) != 0);
184
185 if (map == VM_MAP_NULL)
186 return(KERN_INVALID_ARGUMENT);
187 if (size == 0) {
188 *addr = 0;
189 return(KERN_SUCCESS);
190 }
191
192 if (anywhere) {
193 /*
194 * No specific address requested, so start candidate address
195 * search at the minimum address in the map. However, if that
196 * minimum is 0, bump it up by PAGE_SIZE. We want to limit
197 * allocations of PAGEZERO to explicit requests since its
198 * normal use is to catch dereferences of NULL and many
199 * applications also treat pointers with a value of 0 as
200 * special and suddenly having address 0 contain useable
201 * memory would tend to confuse those applications.
202 */
203 map_addr = vm_map_min(map);
204 if (map_addr == 0)
205 map_addr += PAGE_SIZE;
206 } else
207 map_addr = vm_map_trunc_page(*addr);
208 map_size = vm_map_round_page(size);
209 if (map_size == 0) {
210 return(KERN_INVALID_ARGUMENT);
211 }
212
213 result = vm_map_enter(
214 map,
215 &map_addr,
216 map_size,
217 (vm_map_offset_t)0,
218 flags,
219 VM_OBJECT_NULL,
220 (vm_object_offset_t)0,
221 FALSE,
222 VM_PROT_DEFAULT,
223 VM_PROT_ALL,
224 VM_INHERIT_DEFAULT);
225
226 *addr = CAST_DOWN(vm_offset_t, map_addr);
227 return(result);
228 }
229
230 /*
231 * mach_vm_deallocate -
232 * deallocates the specified range of addresses in the
233 * specified address map.
234 */
235 kern_return_t
236 mach_vm_deallocate(
237 vm_map_t map,
238 mach_vm_offset_t start,
239 mach_vm_size_t size)
240 {
241 if ((map == VM_MAP_NULL) || (start + size < start))
242 return(KERN_INVALID_ARGUMENT);
243
244 if (size == (mach_vm_offset_t) 0)
245 return(KERN_SUCCESS);
246
247 return(vm_map_remove(map, vm_map_trunc_page(start),
248 vm_map_round_page(start+size), VM_MAP_NO_FLAGS));
249 }
250
251 /*
252 * vm_deallocate -
253 * deallocates the specified range of addresses in the
254 * specified address map (limited to addresses the same
255 * size as the kernel).
256 */
257 kern_return_t
258 vm_deallocate(
259 register vm_map_t map,
260 vm_offset_t start,
261 vm_size_t size)
262 {
263 if ((map == VM_MAP_NULL) || (start + size < start))
264 return(KERN_INVALID_ARGUMENT);
265
266 if (size == (vm_offset_t) 0)
267 return(KERN_SUCCESS);
268
269 return(vm_map_remove(map, vm_map_trunc_page(start),
270 vm_map_round_page(start+size), VM_MAP_NO_FLAGS));
271 }
272
273 /*
274 * mach_vm_inherit -
275 * Sets the inheritance of the specified range in the
276 * specified map.
277 */
278 kern_return_t
279 mach_vm_inherit(
280 vm_map_t map,
281 mach_vm_offset_t start,
282 mach_vm_size_t size,
283 vm_inherit_t new_inheritance)
284 {
285 if ((map == VM_MAP_NULL) || (start + size < start) ||
286 (new_inheritance > VM_INHERIT_LAST_VALID))
287 return(KERN_INVALID_ARGUMENT);
288
289 if (size == 0)
290 return KERN_SUCCESS;
291
292 return(vm_map_inherit(map,
293 vm_map_trunc_page(start),
294 vm_map_round_page(start+size),
295 new_inheritance));
296 }
297
298 /*
299 * vm_inherit -
300 * Sets the inheritance of the specified range in the
301 * specified map (range limited to addresses
302 */
303 kern_return_t
304 vm_inherit(
305 register vm_map_t map,
306 vm_offset_t start,
307 vm_size_t size,
308 vm_inherit_t new_inheritance)
309 {
310 if ((map == VM_MAP_NULL) || (start + size < start) ||
311 (new_inheritance > VM_INHERIT_LAST_VALID))
312 return(KERN_INVALID_ARGUMENT);
313
314 if (size == 0)
315 return KERN_SUCCESS;
316
317 return(vm_map_inherit(map,
318 vm_map_trunc_page(start),
319 vm_map_round_page(start+size),
320 new_inheritance));
321 }
322
323 /*
324 * mach_vm_protect -
325 * Sets the protection of the specified range in the
326 * specified map.
327 */
328
329 kern_return_t
330 mach_vm_protect(
331 vm_map_t map,
332 mach_vm_offset_t start,
333 mach_vm_size_t size,
334 boolean_t set_maximum,
335 vm_prot_t new_protection)
336 {
337 if ((map == VM_MAP_NULL) || (start + size < start) ||
338 (new_protection & ~(VM_PROT_ALL | VM_PROT_COPY)))
339 return(KERN_INVALID_ARGUMENT);
340
341 if (size == 0)
342 return KERN_SUCCESS;
343
344 return(vm_map_protect(map,
345 vm_map_trunc_page(start),
346 vm_map_round_page(start+size),
347 new_protection,
348 set_maximum));
349 }
350
351 /*
352 * vm_protect -
353 * Sets the protection of the specified range in the
354 * specified map. Addressability of the range limited
355 * to the same size as the kernel.
356 */
357
358 kern_return_t
359 vm_protect(
360 vm_map_t map,
361 vm_offset_t start,
362 vm_size_t size,
363 boolean_t set_maximum,
364 vm_prot_t new_protection)
365 {
366 if ((map == VM_MAP_NULL) || (start + size < start) ||
367 (new_protection & ~(VM_PROT_ALL | VM_PROT_COPY)))
368 return(KERN_INVALID_ARGUMENT);
369
370 if (size == 0)
371 return KERN_SUCCESS;
372
373 return(vm_map_protect(map,
374 vm_map_trunc_page(start),
375 vm_map_round_page(start+size),
376 new_protection,
377 set_maximum));
378 }
379
380 /*
381 * mach_vm_machine_attributes -
382 * Handle machine-specific attributes for a mapping, such
383 * as cachability, migrability, etc.
384 */
385 kern_return_t
386 mach_vm_machine_attribute(
387 vm_map_t map,
388 mach_vm_address_t addr,
389 mach_vm_size_t size,
390 vm_machine_attribute_t attribute,
391 vm_machine_attribute_val_t* value) /* IN/OUT */
392 {
393 if ((map == VM_MAP_NULL) || (addr + size < addr))
394 return(KERN_INVALID_ARGUMENT);
395
396 if (size == 0)
397 return KERN_SUCCESS;
398
399 return vm_map_machine_attribute(map,
400 vm_map_trunc_page(addr),
401 vm_map_round_page(addr+size),
402 attribute,
403 value);
404 }
405
406 /*
407 * vm_machine_attribute -
408 * Handle machine-specific attributes for a mapping, such
409 * as cachability, migrability, etc. Limited addressability
410 * (same range limits as for the native kernel map).
411 */
412 kern_return_t
413 vm_machine_attribute(
414 vm_map_t map,
415 vm_address_t addr,
416 vm_size_t size,
417 vm_machine_attribute_t attribute,
418 vm_machine_attribute_val_t* value) /* IN/OUT */
419 {
420 if ((map == VM_MAP_NULL) || (addr + size < addr))
421 return(KERN_INVALID_ARGUMENT);
422
423 if (size == 0)
424 return KERN_SUCCESS;
425
426 return vm_map_machine_attribute(map,
427 vm_map_trunc_page(addr),
428 vm_map_round_page(addr+size),
429 attribute,
430 value);
431 }
432
433 /*
434 * mach_vm_read -
435 * Read/copy a range from one address space and return it to the caller.
436 *
437 * It is assumed that the address for the returned memory is selected by
438 * the IPC implementation as part of receiving the reply to this call.
439 * If IPC isn't used, the caller must deal with the vm_map_copy_t object
440 * that gets returned.
441 *
442 * JMM - because of mach_msg_type_number_t, this call is limited to a
443 * single 4GB region at this time.
444 *
445 */
446 kern_return_t
447 mach_vm_read(
448 vm_map_t map,
449 mach_vm_address_t addr,
450 mach_vm_size_t size,
451 pointer_t *data,
452 mach_msg_type_number_t *data_size)
453 {
454 kern_return_t error;
455 vm_map_copy_t ipc_address;
456
457 if (map == VM_MAP_NULL)
458 return(KERN_INVALID_ARGUMENT);
459
460
461 error = vm_map_copyin(map,
462 (vm_map_address_t)addr,
463 (vm_map_size_t)size,
464 FALSE, /* src_destroy */
465 &ipc_address);
466
467 if (KERN_SUCCESS == error) {
468 *data = (pointer_t) ipc_address;
469 *data_size = size;
470 }
471 return(error);
472 }
473
474 /*
475 * vm_read -
476 * Read/copy a range from one address space and return it to the caller.
477 * Limited addressability (same range limits as for the native kernel map).
478 *
479 * It is assumed that the address for the returned memory is selected by
480 * the IPC implementation as part of receiving the reply to this call.
481 * If IPC isn't used, the caller must deal with the vm_map_copy_t object
482 * that gets returned.
483 */
484 kern_return_t
485 vm_read(
486 vm_map_t map,
487 vm_address_t addr,
488 vm_size_t size,
489 pointer_t *data,
490 mach_msg_type_number_t *data_size)
491 {
492 kern_return_t error;
493 vm_map_copy_t ipc_address;
494
495 if (map == VM_MAP_NULL)
496 return(KERN_INVALID_ARGUMENT);
497
498 error = vm_map_copyin(map,
499 (vm_map_address_t)addr,
500 (vm_map_size_t)size,
501 FALSE, /* src_destroy */
502 &ipc_address);
503
504 if (KERN_SUCCESS == error) {
505 *data = (pointer_t) ipc_address;
506 *data_size = size;
507 }
508 return(error);
509 }
510
511 /*
512 * mach_vm_read_list -
513 * Read/copy a list of address ranges from specified map.
514 *
515 * MIG does not know how to deal with a returned array of
516 * vm_map_copy_t structures, so we have to do the copyout
517 * manually here.
518 */
519 kern_return_t
520 mach_vm_read_list(
521 vm_map_t map,
522 mach_vm_read_entry_t data_list,
523 natural_t count)
524 {
525 mach_msg_type_number_t i;
526 kern_return_t error;
527 vm_map_copy_t copy;
528
529 if (map == VM_MAP_NULL ||
530 count > VM_MAP_ENTRY_MAX)
531 return(KERN_INVALID_ARGUMENT);
532
533 error = KERN_SUCCESS;
534 for(i=0; i<count; i++) {
535 vm_map_address_t map_addr;
536 vm_map_size_t map_size;
537
538 map_addr = (vm_map_address_t)(data_list[i].address);
539 map_size = (vm_map_size_t)(data_list[i].size);
540
541 if(map_size != 0) {
542 error = vm_map_copyin(map,
543 map_addr,
544 map_size,
545 FALSE, /* src_destroy */
546 &copy);
547 if (KERN_SUCCESS == error) {
548 error = vm_map_copyout(
549 current_task()->map,
550 &map_addr,
551 copy);
552 if (KERN_SUCCESS == error) {
553 data_list[i].address = map_addr;
554 continue;
555 }
556 vm_map_copy_discard(copy);
557 }
558 }
559 data_list[i].address = (mach_vm_address_t)0;
560 data_list[i].size = (mach_vm_size_t)0;
561 }
562 return(error);
563 }
564
565 /*
566 * vm_read_list -
567 * Read/copy a list of address ranges from specified map.
568 *
569 * MIG does not know how to deal with a returned array of
570 * vm_map_copy_t structures, so we have to do the copyout
571 * manually here.
572 *
573 * The source and destination ranges are limited to those
574 * that can be described with a vm_address_t (i.e. same
575 * size map as the kernel).
576 *
577 * JMM - If the result of the copyout is an address range
578 * that cannot be described with a vm_address_t (i.e. the
579 * caller had a larger address space but used this call
580 * anyway), it will result in a truncated address being
581 * returned (and a likely confused caller).
582 */
583
584 kern_return_t
585 vm_read_list(
586 vm_map_t map,
587 vm_read_entry_t data_list,
588 natural_t count)
589 {
590 mach_msg_type_number_t i;
591 kern_return_t error;
592 vm_map_copy_t copy;
593
594 if (map == VM_MAP_NULL ||
595 count > VM_MAP_ENTRY_MAX)
596 return(KERN_INVALID_ARGUMENT);
597
598 error = KERN_SUCCESS;
599 for(i=0; i<count; i++) {
600 vm_map_address_t map_addr;
601 vm_map_size_t map_size;
602
603 map_addr = (vm_map_address_t)(data_list[i].address);
604 map_size = (vm_map_size_t)(data_list[i].size);
605
606 if(map_size != 0) {
607 error = vm_map_copyin(map,
608 map_addr,
609 map_size,
610 FALSE, /* src_destroy */
611 &copy);
612 if (KERN_SUCCESS == error) {
613 error = vm_map_copyout(current_task()->map,
614 &map_addr,
615 copy);
616 if (KERN_SUCCESS == error) {
617 data_list[i].address =
618 CAST_DOWN(vm_offset_t, map_addr);
619 continue;
620 }
621 vm_map_copy_discard(copy);
622 }
623 }
624 data_list[i].address = (mach_vm_address_t)0;
625 data_list[i].size = (mach_vm_size_t)0;
626 }
627 return(error);
628 }
629
630 /*
631 * mach_vm_read_overwrite -
632 * Overwrite a range of the current map with data from the specified
633 * map/address range.
634 *
635 * In making an assumption that the current thread is local, it is
636 * no longer cluster-safe without a fully supportive local proxy
637 * thread/task (but we don't support cluster's anymore so this is moot).
638 */
639
640 kern_return_t
641 mach_vm_read_overwrite(
642 vm_map_t map,
643 mach_vm_address_t address,
644 mach_vm_size_t size,
645 mach_vm_address_t data,
646 mach_vm_size_t *data_size)
647 {
648 kern_return_t error;
649 vm_map_copy_t copy;
650
651 if (map == VM_MAP_NULL)
652 return(KERN_INVALID_ARGUMENT);
653
654 error = vm_map_copyin(map, (vm_map_address_t)address,
655 (vm_map_size_t)size, FALSE, &copy);
656
657 if (KERN_SUCCESS == error) {
658 error = vm_map_copy_overwrite(current_thread()->map,
659 (vm_map_address_t)data,
660 copy, FALSE);
661 if (KERN_SUCCESS == error) {
662 *data_size = size;
663 return error;
664 }
665 vm_map_copy_discard(copy);
666 }
667 return(error);
668 }
669
670 /*
671 * vm_read_overwrite -
672 * Overwrite a range of the current map with data from the specified
673 * map/address range.
674 *
675 * This routine adds the additional limitation that the source and
676 * destination ranges must be describable with vm_address_t values
677 * (i.e. the same size address spaces as the kernel, or at least the
678 * the ranges are in that first portion of the respective address
679 * spaces).
680 */
681
682 kern_return_t
683 vm_read_overwrite(
684 vm_map_t map,
685 vm_address_t address,
686 vm_size_t size,
687 vm_address_t data,
688 vm_size_t *data_size)
689 {
690 kern_return_t error;
691 vm_map_copy_t copy;
692
693 if (map == VM_MAP_NULL)
694 return(KERN_INVALID_ARGUMENT);
695
696 error = vm_map_copyin(map, (vm_map_address_t)address,
697 (vm_map_size_t)size, FALSE, &copy);
698
699 if (KERN_SUCCESS == error) {
700 error = vm_map_copy_overwrite(current_thread()->map,
701 (vm_map_address_t)data,
702 copy, FALSE);
703 if (KERN_SUCCESS == error) {
704 *data_size = size;
705 return error;
706 }
707 vm_map_copy_discard(copy);
708 }
709 return(error);
710 }
711
712
713 /*
714 * mach_vm_write -
715 * Overwrite the specified address range with the data provided
716 * (from the current map).
717 */
718 kern_return_t
719 mach_vm_write(
720 vm_map_t map,
721 mach_vm_address_t address,
722 pointer_t data,
723 __unused mach_msg_type_number_t size)
724 {
725 if (map == VM_MAP_NULL)
726 return KERN_INVALID_ARGUMENT;
727
728 return vm_map_copy_overwrite(map, (vm_map_address_t)address,
729 (vm_map_copy_t) data, FALSE /* interruptible XXX */);
730 }
731
732 /*
733 * vm_write -
734 * Overwrite the specified address range with the data provided
735 * (from the current map).
736 *
737 * The addressability of the range of addresses to overwrite is
738 * limited bu the use of a vm_address_t (same size as kernel map).
739 * Either the target map is also small, or the range is in the
740 * low addresses within it.
741 */
742 kern_return_t
743 vm_write(
744 vm_map_t map,
745 vm_address_t address,
746 pointer_t data,
747 __unused mach_msg_type_number_t size)
748 {
749 if (map == VM_MAP_NULL)
750 return KERN_INVALID_ARGUMENT;
751
752 return vm_map_copy_overwrite(map, (vm_map_address_t)address,
753 (vm_map_copy_t) data, FALSE /* interruptible XXX */);
754 }
755
756 /*
757 * mach_vm_copy -
758 * Overwrite one range of the specified map with the contents of
759 * another range within that same map (i.e. both address ranges
760 * are "over there").
761 */
762 kern_return_t
763 mach_vm_copy(
764 vm_map_t map,
765 mach_vm_address_t source_address,
766 mach_vm_size_t size,
767 mach_vm_address_t dest_address)
768 {
769 vm_map_copy_t copy;
770 kern_return_t kr;
771
772 if (map == VM_MAP_NULL)
773 return KERN_INVALID_ARGUMENT;
774
775 kr = vm_map_copyin(map, (vm_map_address_t)source_address,
776 (vm_map_size_t)size, FALSE, &copy);
777
778 if (KERN_SUCCESS == kr) {
779 kr = vm_map_copy_overwrite(map,
780 (vm_map_address_t)dest_address,
781 copy, FALSE /* interruptible XXX */);
782
783 if (KERN_SUCCESS != kr)
784 vm_map_copy_discard(copy);
785 }
786 return kr;
787 }
788
789 kern_return_t
790 vm_copy(
791 vm_map_t map,
792 vm_address_t source_address,
793 vm_size_t size,
794 vm_address_t dest_address)
795 {
796 vm_map_copy_t copy;
797 kern_return_t kr;
798
799 if (map == VM_MAP_NULL)
800 return KERN_INVALID_ARGUMENT;
801
802 kr = vm_map_copyin(map, (vm_map_address_t)source_address,
803 (vm_map_size_t)size, FALSE, &copy);
804
805 if (KERN_SUCCESS == kr) {
806 kr = vm_map_copy_overwrite(map,
807 (vm_map_address_t)dest_address,
808 copy, FALSE /* interruptible XXX */);
809
810 if (KERN_SUCCESS != kr)
811 vm_map_copy_discard(copy);
812 }
813 return kr;
814 }
815
816 /*
817 * mach_vm_map -
818 * Map some range of an object into an address space.
819 *
820 * The object can be one of several types of objects:
821 * NULL - anonymous memory
822 * a named entry - a range within another address space
823 * or a range within a memory object
824 * a whole memory object
825 *
826 */
827 kern_return_t
828 mach_vm_map(
829 vm_map_t target_map,
830 mach_vm_offset_t *address,
831 mach_vm_size_t initial_size,
832 mach_vm_offset_t mask,
833 int flags,
834 ipc_port_t port,
835 vm_object_offset_t offset,
836 boolean_t copy,
837 vm_prot_t cur_protection,
838 vm_prot_t max_protection,
839 vm_inherit_t inheritance)
840 {
841 vm_map_address_t map_addr;
842 vm_map_size_t map_size;
843 vm_object_t object;
844 vm_object_size_t size;
845 kern_return_t result;
846
847 /*
848 * Check arguments for validity
849 */
850 if ((target_map == VM_MAP_NULL) ||
851 (cur_protection & ~VM_PROT_ALL) ||
852 (max_protection & ~VM_PROT_ALL) ||
853 (inheritance > VM_INHERIT_LAST_VALID) ||
854 initial_size == 0)
855 return(KERN_INVALID_ARGUMENT);
856
857 map_addr = vm_map_trunc_page(*address);
858 map_size = vm_map_round_page(initial_size);
859 size = vm_object_round_page(initial_size);
860
861 /*
862 * Find the vm object (if any) corresponding to this port.
863 */
864 if (!IP_VALID(port)) {
865 object = VM_OBJECT_NULL;
866 offset = 0;
867 copy = FALSE;
868 } else if (ip_kotype(port) == IKOT_NAMED_ENTRY) {
869 vm_named_entry_t named_entry;
870
871 named_entry = (vm_named_entry_t)port->ip_kobject;
872 /* a few checks to make sure user is obeying rules */
873 if(size == 0) {
874 if(offset >= named_entry->size)
875 return(KERN_INVALID_RIGHT);
876 size = named_entry->size - offset;
877 }
878 if((named_entry->protection & max_protection) != max_protection)
879 return(KERN_INVALID_RIGHT);
880 if((named_entry->protection & cur_protection) != cur_protection)
881 return(KERN_INVALID_RIGHT);
882 if(named_entry->size < (offset + size))
883 return(KERN_INVALID_ARGUMENT);
884
885 /* the callers parameter offset is defined to be the */
886 /* offset from beginning of named entry offset in object */
887 offset = offset + named_entry->offset;
888
889 named_entry_lock(named_entry);
890 if(named_entry->is_sub_map) {
891 vm_map_entry_t map_entry;
892
893 named_entry_unlock(named_entry);
894 vm_object_reference(vm_submap_object);
895 if ((result = vm_map_enter(target_map,
896 &map_addr, map_size,
897 (vm_map_offset_t)mask, flags,
898 vm_submap_object, 0,
899 FALSE,
900 cur_protection, max_protection, inheritance
901 )) != KERN_SUCCESS) {
902 vm_object_deallocate(vm_submap_object);
903 } else {
904 char alias;
905
906 VM_GET_FLAGS_ALIAS(flags, alias);
907 if ((alias == VM_MEMORY_SHARED_PMAP) &&
908 !copy) {
909 vm_map_submap(target_map, map_addr,
910 map_addr + map_size,
911 named_entry->backing.map,
912 (vm_map_offset_t)offset, TRUE);
913 } else {
914 vm_map_submap(target_map, map_addr,
915 map_addr + map_size,
916 named_entry->backing.map,
917 (vm_map_offset_t)offset, FALSE);
918 }
919 if(copy) {
920 if(vm_map_lookup_entry(
921 target_map, map_addr, &map_entry)) {
922 map_entry->needs_copy = TRUE;
923 }
924 }
925 *address = map_addr;
926 }
927 return(result);
928
929 } else if (named_entry->is_pager) {
930 unsigned int access;
931 vm_prot_t protections;
932 unsigned int wimg_mode;
933 boolean_t cache_attr;
934
935 protections = named_entry->protection
936 & VM_PROT_ALL;
937 access = GET_MAP_MEM(named_entry->protection);
938
939 object = vm_object_enter(
940 named_entry->backing.pager,
941 named_entry->size,
942 named_entry->internal,
943 FALSE,
944 FALSE);
945 if (object == VM_OBJECT_NULL) {
946 named_entry_unlock(named_entry);
947 return(KERN_INVALID_OBJECT);
948 }
949
950 /* JMM - drop reference on pager here */
951
952 /* create an extra ref for the named entry */
953 vm_object_lock(object);
954 vm_object_reference_locked(object);
955 named_entry->backing.object = object;
956 named_entry->is_pager = FALSE;
957 named_entry_unlock(named_entry);
958
959 wimg_mode = object->wimg_bits;
960 if(access == MAP_MEM_IO) {
961 wimg_mode = VM_WIMG_IO;
962 } else if (access == MAP_MEM_COPYBACK) {
963 wimg_mode = VM_WIMG_USE_DEFAULT;
964 } else if (access == MAP_MEM_WTHRU) {
965 wimg_mode = VM_WIMG_WTHRU;
966 } else if (access == MAP_MEM_WCOMB) {
967 wimg_mode = VM_WIMG_WCOMB;
968 }
969 if ((wimg_mode == VM_WIMG_IO)
970 || (wimg_mode == VM_WIMG_WCOMB))
971 cache_attr = TRUE;
972 else
973 cache_attr = FALSE;
974
975 /* wait for object (if any) to be ready */
976 if (!named_entry->internal) {
977 while (!object->pager_ready) {
978 vm_object_wait(object,
979 VM_OBJECT_EVENT_PAGER_READY,
980 THREAD_UNINT);
981 vm_object_lock(object);
982 }
983 }
984
985 if(object->wimg_bits != wimg_mode) {
986 vm_page_t p;
987
988 vm_object_paging_wait(object, THREAD_UNINT);
989
990 object->wimg_bits = wimg_mode;
991 queue_iterate(&object->memq, p, vm_page_t, listq) {
992 if (!p->fictitious) {
993 pmap_disconnect(p->phys_page);
994 if (cache_attr)
995 pmap_sync_page_attributes_phys(p->phys_page);
996 }
997 }
998 }
999 object->true_share = TRUE;
1000 if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC)
1001 object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
1002 vm_object_unlock(object);
1003 } else {
1004 /* This is the case where we are going to map */
1005 /* an already mapped object. If the object is */
1006 /* not ready it is internal. An external */
1007 /* object cannot be mapped until it is ready */
1008 /* we can therefore avoid the ready check */
1009 /* in this case. */
1010 object = named_entry->backing.object;
1011 assert(object != VM_OBJECT_NULL);
1012 named_entry_unlock(named_entry);
1013 vm_object_reference(object);
1014 }
1015 } else if (ip_kotype(port) == IKOT_MEMORY_OBJECT) {
1016 /*
1017 * JMM - This is temporary until we unify named entries
1018 * and raw memory objects.
1019 *
1020 * Detected fake ip_kotype for a memory object. In
1021 * this case, the port isn't really a port at all, but
1022 * instead is just a raw memory object.
1023 */
1024
1025 if ((object = vm_object_enter((memory_object_t)port,
1026 size, FALSE, FALSE, FALSE))
1027 == VM_OBJECT_NULL)
1028 return(KERN_INVALID_OBJECT);
1029
1030 /* wait for object (if any) to be ready */
1031 if (object != VM_OBJECT_NULL) {
1032 if(object == kernel_object) {
1033 printf("Warning: Attempt to map kernel object"
1034 " by a non-private kernel entity\n");
1035 return(KERN_INVALID_OBJECT);
1036 }
1037 vm_object_lock(object);
1038 while (!object->pager_ready) {
1039 vm_object_wait(object,
1040 VM_OBJECT_EVENT_PAGER_READY,
1041 THREAD_UNINT);
1042 vm_object_lock(object);
1043 }
1044 vm_object_unlock(object);
1045 }
1046 } else {
1047 return (KERN_INVALID_OBJECT);
1048 }
1049
1050 /*
1051 * Perform the copy if requested
1052 */
1053
1054 if (copy) {
1055 vm_object_t new_object;
1056 vm_object_offset_t new_offset;
1057
1058 result = vm_object_copy_strategically(object, offset, size,
1059 &new_object, &new_offset,
1060 &copy);
1061
1062
1063 if (result == KERN_MEMORY_RESTART_COPY) {
1064 boolean_t success;
1065 boolean_t src_needs_copy;
1066
1067 /*
1068 * XXX
1069 * We currently ignore src_needs_copy.
1070 * This really is the issue of how to make
1071 * MEMORY_OBJECT_COPY_SYMMETRIC safe for
1072 * non-kernel users to use. Solution forthcoming.
1073 * In the meantime, since we don't allow non-kernel
1074 * memory managers to specify symmetric copy,
1075 * we won't run into problems here.
1076 */
1077 new_object = object;
1078 new_offset = offset;
1079 success = vm_object_copy_quickly(&new_object,
1080 new_offset, size,
1081 &src_needs_copy,
1082 &copy);
1083 assert(success);
1084 result = KERN_SUCCESS;
1085 }
1086 /*
1087 * Throw away the reference to the
1088 * original object, as it won't be mapped.
1089 */
1090
1091 vm_object_deallocate(object);
1092
1093 if (result != KERN_SUCCESS)
1094 return (result);
1095
1096 object = new_object;
1097 offset = new_offset;
1098 }
1099
1100 if ((result = vm_map_enter(target_map,
1101 &map_addr, map_size,
1102 (vm_map_offset_t)mask,
1103 flags,
1104 object, offset,
1105 copy,
1106 cur_protection, max_protection, inheritance
1107 )) != KERN_SUCCESS)
1108 vm_object_deallocate(object);
1109 *address = map_addr;
1110 return(result);
1111 }
1112
1113
1114 /* legacy interface */
1115 kern_return_t
1116 vm_map_64(
1117 vm_map_t target_map,
1118 vm_offset_t *address,
1119 vm_size_t size,
1120 vm_offset_t mask,
1121 int flags,
1122 ipc_port_t port,
1123 vm_object_offset_t offset,
1124 boolean_t copy,
1125 vm_prot_t cur_protection,
1126 vm_prot_t max_protection,
1127 vm_inherit_t inheritance)
1128 {
1129 mach_vm_address_t map_addr;
1130 mach_vm_size_t map_size;
1131 mach_vm_offset_t map_mask;
1132 kern_return_t kr;
1133
1134 map_addr = (mach_vm_address_t)*address;
1135 map_size = (mach_vm_size_t)size;
1136 map_mask = (mach_vm_offset_t)mask;
1137
1138 kr = mach_vm_map(target_map, &map_addr, map_size, map_mask, flags,
1139 port, offset, copy,
1140 cur_protection, max_protection, inheritance);
1141 *address = CAST_DOWN(vm_address_t, map_addr);
1142 return kr;
1143 }
1144
1145 /* temporary, until world build */
1146 kern_return_t
1147 vm_map(
1148 vm_map_t target_map,
1149 vm_offset_t *address,
1150 vm_size_t size,
1151 vm_offset_t mask,
1152 int flags,
1153 ipc_port_t port,
1154 vm_offset_t offset,
1155 boolean_t copy,
1156 vm_prot_t cur_protection,
1157 vm_prot_t max_protection,
1158 vm_inherit_t inheritance)
1159 {
1160 mach_vm_address_t map_addr;
1161 mach_vm_size_t map_size;
1162 mach_vm_offset_t map_mask;
1163 vm_object_offset_t obj_offset;
1164 kern_return_t kr;
1165
1166 map_addr = (mach_vm_address_t)*address;
1167 map_size = (mach_vm_size_t)size;
1168 map_mask = (mach_vm_offset_t)mask;
1169 obj_offset = (vm_object_offset_t)offset;
1170
1171 kr = mach_vm_map(target_map, &map_addr, map_size, map_mask, flags,
1172 port, obj_offset, copy,
1173 cur_protection, max_protection, inheritance);
1174 *address = CAST_DOWN(vm_address_t, map_addr);
1175 return kr;
1176 }
1177
1178 /*
1179 * mach_vm_remap -
1180 * Remap a range of memory from one task into another,
1181 * to another address range within the same task, or
1182 * over top of itself (with altered permissions and/or
1183 * as an in-place copy of itself).
1184 */
1185
1186 kern_return_t
1187 mach_vm_remap(
1188 vm_map_t target_map,
1189 mach_vm_offset_t *address,
1190 mach_vm_size_t size,
1191 mach_vm_offset_t mask,
1192 boolean_t anywhere,
1193 vm_map_t src_map,
1194 mach_vm_offset_t memory_address,
1195 boolean_t copy,
1196 vm_prot_t *cur_protection,
1197 vm_prot_t *max_protection,
1198 vm_inherit_t inheritance)
1199 {
1200 vm_map_offset_t map_addr;
1201 kern_return_t kr;
1202
1203 if (VM_MAP_NULL == target_map || VM_MAP_NULL == src_map)
1204 return KERN_INVALID_ARGUMENT;
1205
1206 map_addr = (vm_map_offset_t)*address;
1207
1208 kr = vm_map_remap(target_map,
1209 &map_addr,
1210 size,
1211 mask,
1212 anywhere,
1213 src_map,
1214 memory_address,
1215 copy,
1216 cur_protection,
1217 max_protection,
1218 inheritance);
1219 *address = map_addr;
1220 return kr;
1221 }
1222
1223 /*
1224 * vm_remap -
1225 * Remap a range of memory from one task into another,
1226 * to another address range within the same task, or
1227 * over top of itself (with altered permissions and/or
1228 * as an in-place copy of itself).
1229 *
1230 * The addressability of the source and target address
1231 * range is limited by the size of vm_address_t (in the
1232 * kernel context).
1233 */
1234 kern_return_t
1235 vm_remap(
1236 vm_map_t target_map,
1237 vm_offset_t *address,
1238 vm_size_t size,
1239 vm_offset_t mask,
1240 boolean_t anywhere,
1241 vm_map_t src_map,
1242 vm_offset_t memory_address,
1243 boolean_t copy,
1244 vm_prot_t *cur_protection,
1245 vm_prot_t *max_protection,
1246 vm_inherit_t inheritance)
1247 {
1248 vm_map_offset_t map_addr;
1249 kern_return_t kr;
1250
1251 if (VM_MAP_NULL == target_map || VM_MAP_NULL == src_map)
1252 return KERN_INVALID_ARGUMENT;
1253
1254 map_addr = (vm_map_offset_t)*address;
1255
1256 kr = vm_map_remap(target_map,
1257 &map_addr,
1258 size,
1259 mask,
1260 anywhere,
1261 src_map,
1262 memory_address,
1263 copy,
1264 cur_protection,
1265 max_protection,
1266 inheritance);
1267 *address = CAST_DOWN(vm_offset_t, map_addr);
1268 return kr;
1269 }
1270
1271 /*
1272 * NOTE: these routine (and this file) will no longer require mach_host_server.h
1273 * when mach_vm_wire and vm_wire are changed to use ledgers.
1274 */
1275 #include <mach/mach_host_server.h>
1276 /*
1277 * mach_vm_wire
1278 * Specify that the range of the virtual address space
1279 * of the target task must not cause page faults for
1280 * the indicated accesses.
1281 *
1282 * [ To unwire the pages, specify VM_PROT_NONE. ]
1283 */
1284 kern_return_t
1285 mach_vm_wire(
1286 host_priv_t host_priv,
1287 vm_map_t map,
1288 mach_vm_offset_t start,
1289 mach_vm_size_t size,
1290 vm_prot_t access)
1291 {
1292 kern_return_t rc;
1293
1294 if (host_priv == HOST_PRIV_NULL)
1295 return KERN_INVALID_HOST;
1296
1297 assert(host_priv == &realhost);
1298
1299 if (map == VM_MAP_NULL)
1300 return KERN_INVALID_TASK;
1301
1302 if (access & ~VM_PROT_ALL)
1303 return KERN_INVALID_ARGUMENT;
1304
1305 if (access != VM_PROT_NONE) {
1306 rc = vm_map_wire(map, vm_map_trunc_page(start),
1307 vm_map_round_page(start+size), access, TRUE);
1308 } else {
1309 rc = vm_map_unwire(map, vm_map_trunc_page(start),
1310 vm_map_round_page(start+size), TRUE);
1311 }
1312 return rc;
1313 }
1314
1315 /*
1316 * vm_wire -
1317 * Specify that the range of the virtual address space
1318 * of the target task must not cause page faults for
1319 * the indicated accesses.
1320 *
1321 * [ To unwire the pages, specify VM_PROT_NONE. ]
1322 */
1323 kern_return_t
1324 vm_wire(
1325 host_priv_t host_priv,
1326 register vm_map_t map,
1327 vm_offset_t start,
1328 vm_size_t size,
1329 vm_prot_t access)
1330 {
1331 kern_return_t rc;
1332
1333 if (host_priv == HOST_PRIV_NULL)
1334 return KERN_INVALID_HOST;
1335
1336 assert(host_priv == &realhost);
1337
1338 if (map == VM_MAP_NULL)
1339 return KERN_INVALID_TASK;
1340
1341 if ((access & ~VM_PROT_ALL) || (start + size < start))
1342 return KERN_INVALID_ARGUMENT;
1343
1344 if (size == 0) {
1345 rc = KERN_SUCCESS;
1346 } else if (access != VM_PROT_NONE) {
1347 rc = vm_map_wire(map, vm_map_trunc_page(start),
1348 vm_map_round_page(start+size), access, TRUE);
1349 } else {
1350 rc = vm_map_unwire(map, vm_map_trunc_page(start),
1351 vm_map_round_page(start+size), TRUE);
1352 }
1353 return rc;
1354 }
1355
1356 /*
1357 * vm_msync
1358 *
1359 * Synchronises the memory range specified with its backing store
1360 * image by either flushing or cleaning the contents to the appropriate
1361 * memory manager.
1362 *
1363 * interpretation of sync_flags
1364 * VM_SYNC_INVALIDATE - discard pages, only return precious
1365 * pages to manager.
1366 *
1367 * VM_SYNC_INVALIDATE & (VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS)
1368 * - discard pages, write dirty or precious
1369 * pages back to memory manager.
1370 *
1371 * VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS
1372 * - write dirty or precious pages back to
1373 * the memory manager.
1374 *
1375 * VM_SYNC_CONTIGUOUS - does everything normally, but if there
1376 * is a hole in the region, and we would
1377 * have returned KERN_SUCCESS, return
1378 * KERN_INVALID_ADDRESS instead.
1379 *
1380 * RETURNS
1381 * KERN_INVALID_TASK Bad task parameter
1382 * KERN_INVALID_ARGUMENT both sync and async were specified.
1383 * KERN_SUCCESS The usual.
1384 * KERN_INVALID_ADDRESS There was a hole in the region.
1385 */
1386
1387 kern_return_t
1388 mach_vm_msync(
1389 vm_map_t map,
1390 mach_vm_address_t address,
1391 mach_vm_size_t size,
1392 vm_sync_t sync_flags)
1393 {
1394
1395 if (map == VM_MAP_NULL)
1396 return(KERN_INVALID_TASK);
1397
1398 return vm_map_msync(map, (vm_map_address_t)address,
1399 (vm_map_size_t)size, sync_flags);
1400 }
1401
1402 /*
1403 * vm_msync
1404 *
1405 * Synchronises the memory range specified with its backing store
1406 * image by either flushing or cleaning the contents to the appropriate
1407 * memory manager.
1408 *
1409 * interpretation of sync_flags
1410 * VM_SYNC_INVALIDATE - discard pages, only return precious
1411 * pages to manager.
1412 *
1413 * VM_SYNC_INVALIDATE & (VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS)
1414 * - discard pages, write dirty or precious
1415 * pages back to memory manager.
1416 *
1417 * VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS
1418 * - write dirty or precious pages back to
1419 * the memory manager.
1420 *
1421 * VM_SYNC_CONTIGUOUS - does everything normally, but if there
1422 * is a hole in the region, and we would
1423 * have returned KERN_SUCCESS, return
1424 * KERN_INVALID_ADDRESS instead.
1425 *
1426 * The addressability of the range is limited to that which can
1427 * be described by a vm_address_t.
1428 *
1429 * RETURNS
1430 * KERN_INVALID_TASK Bad task parameter
1431 * KERN_INVALID_ARGUMENT both sync and async were specified.
1432 * KERN_SUCCESS The usual.
1433 * KERN_INVALID_ADDRESS There was a hole in the region.
1434 */
1435
1436 kern_return_t
1437 vm_msync(
1438 vm_map_t map,
1439 vm_address_t address,
1440 vm_size_t size,
1441 vm_sync_t sync_flags)
1442 {
1443
1444 if (map == VM_MAP_NULL)
1445 return(KERN_INVALID_TASK);
1446
1447 return vm_map_msync(map, (vm_map_address_t)address,
1448 (vm_map_size_t)size, sync_flags);
1449 }
1450
1451
1452 /*
1453 * mach_vm_behavior_set
1454 *
1455 * Sets the paging behavior attribute for the specified range
1456 * in the specified map.
1457 *
1458 * This routine will fail with KERN_INVALID_ADDRESS if any address
1459 * in [start,start+size) is not a valid allocated memory region.
1460 */
1461 kern_return_t
1462 mach_vm_behavior_set(
1463 vm_map_t map,
1464 mach_vm_offset_t start,
1465 mach_vm_size_t size,
1466 vm_behavior_t new_behavior)
1467 {
1468 if ((map == VM_MAP_NULL) || (start + size < start))
1469 return(KERN_INVALID_ARGUMENT);
1470
1471 if (size == 0)
1472 return KERN_SUCCESS;
1473
1474 return(vm_map_behavior_set(map, vm_map_trunc_page(start),
1475 vm_map_round_page(start+size), new_behavior));
1476 }
1477
1478 /*
1479 * vm_behavior_set
1480 *
1481 * Sets the paging behavior attribute for the specified range
1482 * in the specified map.
1483 *
1484 * This routine will fail with KERN_INVALID_ADDRESS if any address
1485 * in [start,start+size) is not a valid allocated memory region.
1486 *
1487 * This routine is potentially limited in addressibility by the
1488 * use of vm_offset_t (if the map provided is larger than the
1489 * kernel's).
1490 */
1491 kern_return_t
1492 vm_behavior_set(
1493 vm_map_t map,
1494 vm_offset_t start,
1495 vm_size_t size,
1496 vm_behavior_t new_behavior)
1497 {
1498 if ((map == VM_MAP_NULL) || (start + size < start))
1499 return(KERN_INVALID_ARGUMENT);
1500
1501 if (size == 0)
1502 return KERN_SUCCESS;
1503
1504 return(vm_map_behavior_set(map, vm_map_trunc_page(start),
1505 vm_map_round_page(start+size), new_behavior));
1506 }
1507
1508 /*
1509 * mach_vm_region:
1510 *
1511 * User call to obtain information about a region in
1512 * a task's address map. Currently, only one flavor is
1513 * supported.
1514 *
1515 * XXX The reserved and behavior fields cannot be filled
1516 * in until the vm merge from the IK is completed, and
1517 * vm_reserve is implemented.
1518 *
1519 * XXX Dependency: syscall_vm_region() also supports only one flavor.
1520 */
1521
1522 kern_return_t
1523 mach_vm_region(
1524 vm_map_t map,
1525 mach_vm_offset_t *address, /* IN/OUT */
1526 mach_vm_size_t *size, /* OUT */
1527 vm_region_flavor_t flavor, /* IN */
1528 vm_region_info_t info, /* OUT */
1529 mach_msg_type_number_t *count, /* IN/OUT */
1530 mach_port_t *object_name) /* OUT */
1531 {
1532 vm_map_offset_t map_addr;
1533 vm_map_size_t map_size;
1534 kern_return_t kr;
1535
1536 if (VM_MAP_NULL == map)
1537 return KERN_INVALID_ARGUMENT;
1538
1539 map_addr = (vm_map_offset_t)*address;
1540 map_size = (vm_map_size_t)*size;
1541
1542 /* legacy conversion */
1543 if (VM_REGION_BASIC_INFO == flavor)
1544 flavor = VM_REGION_BASIC_INFO_64;
1545
1546 kr = vm_map_region(map,
1547 &map_addr, &map_size,
1548 flavor, info, count,
1549 object_name);
1550
1551 *address = map_addr;
1552 *size = map_size;
1553 return kr;
1554 }
1555
1556 /*
1557 * vm_region_64 and vm_region:
1558 *
1559 * User call to obtain information about a region in
1560 * a task's address map. Currently, only one flavor is
1561 * supported.
1562 *
1563 * XXX The reserved and behavior fields cannot be filled
1564 * in until the vm merge from the IK is completed, and
1565 * vm_reserve is implemented.
1566 *
1567 * XXX Dependency: syscall_vm_region() also supports only one flavor.
1568 */
1569
1570 kern_return_t
1571 vm_region_64(
1572 vm_map_t map,
1573 vm_offset_t *address, /* IN/OUT */
1574 vm_size_t *size, /* OUT */
1575 vm_region_flavor_t flavor, /* IN */
1576 vm_region_info_t info, /* OUT */
1577 mach_msg_type_number_t *count, /* IN/OUT */
1578 mach_port_t *object_name) /* OUT */
1579 {
1580 vm_map_offset_t map_addr;
1581 vm_map_size_t map_size;
1582 kern_return_t kr;
1583
1584 if (VM_MAP_NULL == map)
1585 return KERN_INVALID_ARGUMENT;
1586
1587 map_addr = (vm_map_offset_t)*address;
1588 map_size = (vm_map_size_t)*size;
1589
1590 /* legacy conversion */
1591 if (VM_REGION_BASIC_INFO == flavor)
1592 flavor = VM_REGION_BASIC_INFO_64;
1593
1594 kr = vm_map_region(map,
1595 &map_addr, &map_size,
1596 flavor, info, count,
1597 object_name);
1598
1599 *address = CAST_DOWN(vm_offset_t, map_addr);
1600 *size = CAST_DOWN(vm_size_t, map_size);
1601
1602 if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS)
1603 return KERN_INVALID_ADDRESS;
1604 return kr;
1605 }
1606
1607 kern_return_t
1608 vm_region(
1609 vm_map_t map,
1610 vm_address_t *address, /* IN/OUT */
1611 vm_size_t *size, /* OUT */
1612 vm_region_flavor_t flavor, /* IN */
1613 vm_region_info_t info, /* OUT */
1614 mach_msg_type_number_t *count, /* IN/OUT */
1615 mach_port_t *object_name) /* OUT */
1616 {
1617 vm_map_address_t map_addr;
1618 vm_map_size_t map_size;
1619 kern_return_t kr;
1620
1621 if (VM_MAP_NULL == map)
1622 return KERN_INVALID_ARGUMENT;
1623
1624 map_addr = (vm_map_address_t)*address;
1625 map_size = (vm_map_size_t)*size;
1626
1627 kr = vm_map_region(map,
1628 &map_addr, &map_size,
1629 flavor, info, count,
1630 object_name);
1631
1632 *address = CAST_DOWN(vm_address_t, map_addr);
1633 *size = CAST_DOWN(vm_size_t, map_size);
1634
1635 if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS)
1636 return KERN_INVALID_ADDRESS;
1637 return kr;
1638 }
1639
1640 /*
1641 * vm_region_recurse: A form of vm_region which follows the
1642 * submaps in a target map
1643 *
1644 */
1645 kern_return_t
1646 mach_vm_region_recurse(
1647 vm_map_t map,
1648 mach_vm_address_t *address,
1649 mach_vm_size_t *size,
1650 uint32_t *depth,
1651 vm_region_recurse_info_t info,
1652 mach_msg_type_number_t *infoCnt)
1653 {
1654 vm_map_address_t map_addr;
1655 vm_map_size_t map_size;
1656 kern_return_t kr;
1657
1658 if (VM_MAP_NULL == map)
1659 return KERN_INVALID_ARGUMENT;
1660
1661 map_addr = (vm_map_address_t)*address;
1662 map_size = (vm_map_size_t)*size;
1663
1664 kr = vm_map_region_recurse_64(
1665 map,
1666 &map_addr,
1667 &map_size,
1668 depth,
1669 (vm_region_submap_info_64_t)info,
1670 infoCnt);
1671
1672 *address = map_addr;
1673 *size = map_size;
1674 return kr;
1675 }
1676
1677 /*
1678 * vm_region_recurse: A form of vm_region which follows the
1679 * submaps in a target map
1680 *
1681 */
1682 kern_return_t
1683 vm_region_recurse_64(
1684 vm_map_t map,
1685 vm_address_t *address,
1686 vm_size_t *size,
1687 uint32_t *depth,
1688 vm_region_recurse_info_64_t info,
1689 mach_msg_type_number_t *infoCnt)
1690 {
1691 vm_map_address_t map_addr;
1692 vm_map_size_t map_size;
1693 kern_return_t kr;
1694
1695 if (VM_MAP_NULL == map)
1696 return KERN_INVALID_ARGUMENT;
1697
1698 map_addr = (vm_map_address_t)*address;
1699 map_size = (vm_map_size_t)*size;
1700
1701 kr = vm_map_region_recurse_64(
1702 map,
1703 &map_addr,
1704 &map_size,
1705 depth,
1706 (vm_region_submap_info_64_t)info,
1707 infoCnt);
1708
1709 *address = CAST_DOWN(vm_address_t, map_addr);
1710 *size = CAST_DOWN(vm_size_t, map_size);
1711
1712 if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS)
1713 return KERN_INVALID_ADDRESS;
1714 return kr;
1715 }
1716
1717 kern_return_t
1718 vm_region_recurse(
1719 vm_map_t map,
1720 vm_offset_t *address, /* IN/OUT */
1721 vm_size_t *size, /* OUT */
1722 natural_t *depth, /* IN/OUT */
1723 vm_region_recurse_info_t info32, /* IN/OUT */
1724 mach_msg_type_number_t *infoCnt) /* IN/OUT */
1725 {
1726 vm_region_submap_info_data_64_t info64;
1727 vm_region_submap_info_t info;
1728 vm_map_address_t map_addr;
1729 vm_map_size_t map_size;
1730 kern_return_t kr;
1731
1732 if (VM_MAP_NULL == map || *infoCnt < VM_REGION_SUBMAP_INFO_COUNT)
1733 return KERN_INVALID_ARGUMENT;
1734
1735
1736 map_addr = (vm_map_address_t)*address;
1737 map_size = (vm_map_size_t)*size;
1738 info = (vm_region_submap_info_t)info32;
1739 *infoCnt = VM_REGION_SUBMAP_INFO_COUNT_64;
1740
1741 kr = vm_map_region_recurse_64(map, &map_addr,&map_size,
1742 depth, &info64, infoCnt);
1743
1744 info->protection = info64.protection;
1745 info->max_protection = info64.max_protection;
1746 info->inheritance = info64.inheritance;
1747 info->offset = (uint32_t)info64.offset; /* trouble-maker */
1748 info->user_tag = info64.user_tag;
1749 info->pages_resident = info64.pages_resident;
1750 info->pages_shared_now_private = info64.pages_shared_now_private;
1751 info->pages_swapped_out = info64.pages_swapped_out;
1752 info->pages_dirtied = info64.pages_dirtied;
1753 info->ref_count = info64.ref_count;
1754 info->shadow_depth = info64.shadow_depth;
1755 info->external_pager = info64.external_pager;
1756 info->share_mode = info64.share_mode;
1757 info->is_submap = info64.is_submap;
1758 info->behavior = info64.behavior;
1759 info->object_id = info64.object_id;
1760 info->user_wired_count = info64.user_wired_count;
1761
1762 *address = CAST_DOWN(vm_address_t, map_addr);
1763 *size = CAST_DOWN(vm_size_t, map_size);
1764 *infoCnt = VM_REGION_SUBMAP_INFO_COUNT;
1765
1766 if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS)
1767 return KERN_INVALID_ADDRESS;
1768 return kr;
1769 }
1770
1771 kern_return_t
1772 vm_purgable_control(
1773 vm_map_t map,
1774 vm_offset_t address,
1775 vm_purgable_t control,
1776 int *state)
1777 {
1778 if (VM_MAP_NULL == map)
1779 return KERN_INVALID_ARGUMENT;
1780
1781 return vm_map_purgable_control(map,
1782 vm_map_trunc_page(address),
1783 control,
1784 state);
1785 }
1786
1787
1788 /*
1789 * Ordinarily, the right to allocate CPM is restricted
1790 * to privileged applications (those that can gain access
1791 * to the host priv port). Set this variable to zero if
1792 * you want to let any application allocate CPM.
1793 */
1794 unsigned int vm_allocate_cpm_privileged = 0;
1795
1796 /*
1797 * Allocate memory in the specified map, with the caveat that
1798 * the memory is physically contiguous. This call may fail
1799 * if the system can't find sufficient contiguous memory.
1800 * This call may cause or lead to heart-stopping amounts of
1801 * paging activity.
1802 *
1803 * Memory obtained from this call should be freed in the
1804 * normal way, viz., via vm_deallocate.
1805 */
1806 kern_return_t
1807 vm_allocate_cpm(
1808 host_priv_t host_priv,
1809 vm_map_t map,
1810 vm_address_t *addr,
1811 vm_size_t size,
1812 int flags)
1813 {
1814 vm_map_address_t map_addr;
1815 vm_map_size_t map_size;
1816 kern_return_t kr;
1817
1818 if (vm_allocate_cpm_privileged && HOST_PRIV_NULL == host_priv)
1819 return KERN_INVALID_HOST;
1820
1821 if (VM_MAP_NULL == map)
1822 return KERN_INVALID_ARGUMENT;
1823
1824 map_addr = (vm_map_address_t)*addr;
1825 map_size = (vm_map_size_t)size;
1826
1827 kr = vm_map_enter_cpm(map,
1828 &map_addr,
1829 map_size,
1830 flags);
1831
1832 *addr = CAST_DOWN(vm_address_t, map_addr);
1833 return kr;
1834 }
1835
1836
1837 kern_return_t
1838 mach_vm_page_query(
1839 vm_map_t map,
1840 mach_vm_offset_t offset,
1841 int *disposition,
1842 int *ref_count)
1843 {
1844 if (VM_MAP_NULL == map)
1845 return KERN_INVALID_ARGUMENT;
1846
1847 return vm_map_page_info(map,
1848 vm_map_trunc_page(offset),
1849 disposition, ref_count);
1850 }
1851
1852 kern_return_t
1853 vm_map_page_query(
1854 vm_map_t map,
1855 vm_offset_t offset,
1856 int *disposition,
1857 int *ref_count)
1858 {
1859 if (VM_MAP_NULL == map)
1860 return KERN_INVALID_ARGUMENT;
1861
1862 return vm_map_page_info(map,
1863 vm_map_trunc_page(offset),
1864 disposition, ref_count);
1865 }
1866
1867 /* map a (whole) upl into an address space */
1868 kern_return_t
1869 vm_upl_map(
1870 vm_map_t map,
1871 upl_t upl,
1872 vm_offset_t *dst_addr)
1873 {
1874 vm_map_offset_t map_addr;
1875 kern_return_t kr;
1876
1877 if (VM_MAP_NULL == map)
1878 return KERN_INVALID_ARGUMENT;
1879
1880 kr = vm_map_enter_upl(map, upl, &map_addr);
1881 *dst_addr = CAST_DOWN(vm_offset_t, map_addr);
1882 return kr;
1883 }
1884
1885 kern_return_t
1886 vm_upl_unmap(
1887 vm_map_t map,
1888 upl_t upl)
1889 {
1890 if (VM_MAP_NULL == map)
1891 return KERN_INVALID_ARGUMENT;
1892
1893 return (vm_map_remove_upl(map, upl));
1894 }
1895
1896 /* Retrieve a upl for an object underlying an address range in a map */
1897
1898 kern_return_t
1899 vm_map_get_upl(
1900 vm_map_t map,
1901 vm_map_offset_t map_offset,
1902 upl_size_t *upl_size,
1903 upl_t *upl,
1904 upl_page_info_array_t page_list,
1905 unsigned int *count,
1906 int *flags,
1907 int force_data_sync)
1908 {
1909 int map_flags;
1910 kern_return_t kr;
1911
1912 if (VM_MAP_NULL == map)
1913 return KERN_INVALID_ARGUMENT;
1914
1915 map_flags = *flags & ~UPL_NOZEROFILL;
1916 if (force_data_sync)
1917 map_flags |= UPL_FORCE_DATA_SYNC;
1918
1919 kr = vm_map_create_upl(map,
1920 map_offset,
1921 upl_size,
1922 upl,
1923 page_list,
1924 count,
1925 &map_flags);
1926
1927 *flags = (map_flags & ~UPL_FORCE_DATA_SYNC);
1928 return kr;
1929 }
1930
1931
1932 __private_extern__ kern_return_t
1933 mach_memory_entry_allocate(
1934 vm_named_entry_t *user_entry_p,
1935 ipc_port_t *user_handle_p); /* forward */
1936
1937 /*
1938 * mach_make_memory_entry_64
1939 *
1940 * Think of it as a two-stage vm_remap() operation. First
1941 * you get a handle. Second, you get map that handle in
1942 * somewhere else. Rather than doing it all at once (and
1943 * without needing access to the other whole map).
1944 */
1945
1946 kern_return_t
1947 mach_make_memory_entry_64(
1948 vm_map_t target_map,
1949 memory_object_size_t *size,
1950 memory_object_offset_t offset,
1951 vm_prot_t permission,
1952 ipc_port_t *object_handle,
1953 ipc_port_t parent_handle)
1954 {
1955 vm_map_version_t version;
1956 vm_named_entry_t parent_entry;
1957 vm_named_entry_t user_entry;
1958 ipc_port_t user_handle;
1959 kern_return_t kr;
1960 vm_map_t real_map;
1961
1962 /* needed for call to vm_map_lookup_locked */
1963 boolean_t wired;
1964 vm_object_offset_t obj_off;
1965 vm_prot_t prot;
1966 vm_map_offset_t lo_offset, hi_offset;
1967 vm_behavior_t behavior;
1968 vm_object_t object;
1969 vm_object_t shadow_object;
1970
1971 /* needed for direct map entry manipulation */
1972 vm_map_entry_t map_entry;
1973 vm_map_entry_t next_entry;
1974 vm_map_t local_map;
1975 vm_map_t original_map = target_map;
1976 vm_map_size_t total_size;
1977 vm_map_size_t map_size;
1978 vm_map_offset_t map_offset;
1979 vm_map_offset_t local_offset;
1980 vm_object_size_t mappable_size;
1981
1982 unsigned int access;
1983 vm_prot_t protections;
1984 unsigned int wimg_mode;
1985 boolean_t cache_attr = FALSE;
1986
1987 if (((permission & 0x00FF0000) &
1988 ~(MAP_MEM_ONLY |
1989 MAP_MEM_NAMED_CREATE |
1990 MAP_MEM_PURGABLE |
1991 MAP_MEM_NAMED_REUSE))) {
1992 /*
1993 * Unknown flag: reject for forward compatibility.
1994 */
1995 return KERN_INVALID_VALUE;
1996 }
1997
1998 if (parent_handle != IP_NULL &&
1999 ip_kotype(parent_handle) == IKOT_NAMED_ENTRY) {
2000 parent_entry = (vm_named_entry_t) parent_handle->ip_kobject;
2001 } else {
2002 parent_entry = NULL;
2003 }
2004
2005 protections = permission & VM_PROT_ALL;
2006 access = GET_MAP_MEM(permission);
2007
2008 user_handle = IP_NULL;
2009 user_entry = NULL;
2010
2011 map_offset = vm_map_trunc_page(offset);
2012 map_size = vm_map_round_page(*size);
2013
2014 if (permission & MAP_MEM_ONLY) {
2015 boolean_t parent_is_object;
2016
2017 if (parent_entry == NULL) {
2018 return KERN_INVALID_ARGUMENT;
2019 }
2020
2021 parent_is_object = !(parent_entry->is_sub_map || parent_entry->is_pager);
2022 object = parent_entry->backing.object;
2023 if(parent_is_object && object != VM_OBJECT_NULL)
2024 wimg_mode = object->wimg_bits;
2025 else
2026 wimg_mode = VM_WIMG_DEFAULT;
2027 if((access != GET_MAP_MEM(parent_entry->protection)) &&
2028 !(parent_entry->protection & VM_PROT_WRITE)) {
2029 return KERN_INVALID_RIGHT;
2030 }
2031 if(access == MAP_MEM_IO) {
2032 SET_MAP_MEM(access, parent_entry->protection);
2033 wimg_mode = VM_WIMG_IO;
2034 } else if (access == MAP_MEM_COPYBACK) {
2035 SET_MAP_MEM(access, parent_entry->protection);
2036 wimg_mode = VM_WIMG_DEFAULT;
2037 } else if (access == MAP_MEM_WTHRU) {
2038 SET_MAP_MEM(access, parent_entry->protection);
2039 wimg_mode = VM_WIMG_WTHRU;
2040 } else if (access == MAP_MEM_WCOMB) {
2041 SET_MAP_MEM(access, parent_entry->protection);
2042 wimg_mode = VM_WIMG_WCOMB;
2043 }
2044 if(parent_is_object && object &&
2045 (access != MAP_MEM_NOOP) &&
2046 (!(object->nophyscache))) {
2047 if(object->wimg_bits != wimg_mode) {
2048 vm_page_t p;
2049 if ((wimg_mode == VM_WIMG_IO)
2050 || (wimg_mode == VM_WIMG_WCOMB))
2051 cache_attr = TRUE;
2052 else
2053 cache_attr = FALSE;
2054 vm_object_lock(object);
2055 vm_object_paging_wait(object, THREAD_UNINT);
2056 object->wimg_bits = wimg_mode;
2057 queue_iterate(&object->memq,
2058 p, vm_page_t, listq) {
2059 if (!p->fictitious) {
2060 pmap_disconnect(p->phys_page);
2061 if (cache_attr)
2062 pmap_sync_page_attributes_phys(p->phys_page);
2063 }
2064 }
2065 vm_object_unlock(object);
2066 }
2067 }
2068 if (object_handle)
2069 *object_handle = IP_NULL;
2070 return KERN_SUCCESS;
2071 }
2072
2073 if(permission & MAP_MEM_NAMED_CREATE) {
2074 kr = mach_memory_entry_allocate(&user_entry, &user_handle);
2075 if (kr != KERN_SUCCESS) {
2076 return KERN_FAILURE;
2077 }
2078
2079 /*
2080 * Force the creation of the VM object now.
2081 */
2082 if (map_size > (vm_map_size_t) VM_MAX_ADDRESS) {
2083 /*
2084 * LP64todo - for now, we can only allocate 4GB
2085 * internal objects because the default pager can't
2086 * page bigger ones. Remove this when it can.
2087 */
2088 kr = KERN_FAILURE;
2089 goto make_mem_done;
2090 }
2091
2092 object = vm_object_allocate(map_size);
2093 assert(object != VM_OBJECT_NULL);
2094
2095 if (permission & MAP_MEM_PURGABLE) {
2096 if (! (permission & VM_PROT_WRITE)) {
2097 /* if we can't write, we can't purge */
2098 vm_object_deallocate(object);
2099 kr = KERN_INVALID_ARGUMENT;
2100 goto make_mem_done;
2101 }
2102 object->purgable = VM_OBJECT_PURGABLE_NONVOLATILE;
2103 }
2104
2105 /*
2106 * The VM object is brand new and nobody else knows about it,
2107 * so we don't need to lock it.
2108 */
2109
2110 wimg_mode = object->wimg_bits;
2111 if (access == MAP_MEM_IO) {
2112 wimg_mode = VM_WIMG_IO;
2113 } else if (access == MAP_MEM_COPYBACK) {
2114 wimg_mode = VM_WIMG_DEFAULT;
2115 } else if (access == MAP_MEM_WTHRU) {
2116 wimg_mode = VM_WIMG_WTHRU;
2117 } else if (access == MAP_MEM_WCOMB) {
2118 wimg_mode = VM_WIMG_WCOMB;
2119 }
2120 if (access != MAP_MEM_NOOP) {
2121 object->wimg_bits = wimg_mode;
2122 }
2123 /* the object has no pages, so no WIMG bits to update here */
2124
2125 /*
2126 * XXX
2127 * We use this path when we want to make sure that
2128 * nobody messes with the object (coalesce, for
2129 * example) before we map it.
2130 * We might want to use these objects for transposition via
2131 * vm_object_transpose() too, so we don't want any copy or
2132 * shadow objects either...
2133 */
2134 object->copy_strategy = MEMORY_OBJECT_COPY_NONE;
2135
2136 user_entry->backing.object = object;
2137 user_entry->internal = TRUE;
2138 user_entry->is_sub_map = FALSE;
2139 user_entry->is_pager = FALSE;
2140 user_entry->offset = 0;
2141 user_entry->protection = protections;
2142 SET_MAP_MEM(access, user_entry->protection);
2143 user_entry->size = map_size;
2144
2145 /* user_object pager and internal fields are not used */
2146 /* when the object field is filled in. */
2147
2148 *size = CAST_DOWN(vm_size_t, map_size);
2149 *object_handle = user_handle;
2150 return KERN_SUCCESS;
2151 }
2152
2153 if (parent_entry == NULL ||
2154 (permission & MAP_MEM_NAMED_REUSE)) {
2155
2156 /* Create a named object based on address range within the task map */
2157 /* Go find the object at given address */
2158
2159 redo_lookup:
2160 vm_map_lock_read(target_map);
2161
2162 /* get the object associated with the target address */
2163 /* note we check the permission of the range against */
2164 /* that requested by the caller */
2165
2166 kr = vm_map_lookup_locked(&target_map, map_offset,
2167 protections, &version,
2168 &object, &obj_off, &prot, &wired, &behavior,
2169 &lo_offset, &hi_offset, &real_map);
2170 if (kr != KERN_SUCCESS) {
2171 vm_map_unlock_read(target_map);
2172 goto make_mem_done;
2173 }
2174 if (((prot & protections) != protections)
2175 || (object == kernel_object)) {
2176 kr = KERN_INVALID_RIGHT;
2177 vm_object_unlock(object);
2178 vm_map_unlock_read(target_map);
2179 if(real_map != target_map)
2180 vm_map_unlock_read(real_map);
2181 if(object == kernel_object) {
2182 printf("Warning: Attempt to create a named"
2183 " entry from the kernel_object\n");
2184 }
2185 goto make_mem_done;
2186 }
2187
2188 /* We have an object, now check to see if this object */
2189 /* is suitable. If not, create a shadow and share that */
2190
2191 /*
2192 * We have to unlock the VM object to avoid deadlocking with
2193 * a VM map lock (the lock ordering is map, the object), if we
2194 * need to modify the VM map to create a shadow object. Since
2195 * we might release the VM map lock below anyway, we have
2196 * to release the VM map lock now.
2197 * XXX FBDP There must be a way to avoid this double lookup...
2198 *
2199 * Take an extra reference on the VM object to make sure it's
2200 * not going to disappear.
2201 */
2202 vm_object_reference_locked(object); /* extra ref to hold obj */
2203 vm_object_unlock(object);
2204
2205 local_map = original_map;
2206 local_offset = map_offset;
2207 if(target_map != local_map) {
2208 vm_map_unlock_read(target_map);
2209 if(real_map != target_map)
2210 vm_map_unlock_read(real_map);
2211 vm_map_lock_read(local_map);
2212 target_map = local_map;
2213 real_map = local_map;
2214 }
2215 while(TRUE) {
2216 if(!vm_map_lookup_entry(local_map,
2217 local_offset, &map_entry)) {
2218 kr = KERN_INVALID_ARGUMENT;
2219 vm_map_unlock_read(target_map);
2220 if(real_map != target_map)
2221 vm_map_unlock_read(real_map);
2222 vm_object_deallocate(object); /* release extra ref */
2223 object = VM_OBJECT_NULL;
2224 goto make_mem_done;
2225 }
2226 if(!(map_entry->is_sub_map)) {
2227 if(map_entry->object.vm_object != object) {
2228 kr = KERN_INVALID_ARGUMENT;
2229 vm_map_unlock_read(target_map);
2230 if(real_map != target_map)
2231 vm_map_unlock_read(real_map);
2232 vm_object_deallocate(object); /* release extra ref */
2233 object = VM_OBJECT_NULL;
2234 goto make_mem_done;
2235 }
2236 break;
2237 } else {
2238 vm_map_t tmap;
2239 tmap = local_map;
2240 local_map = map_entry->object.sub_map;
2241
2242 vm_map_lock_read(local_map);
2243 vm_map_unlock_read(tmap);
2244 target_map = local_map;
2245 real_map = local_map;
2246 local_offset = local_offset - map_entry->vme_start;
2247 local_offset += map_entry->offset;
2248 }
2249 }
2250
2251 /*
2252 * We found the VM map entry, lock the VM object again.
2253 */
2254 vm_object_lock(object);
2255 if(map_entry->wired_count) {
2256 /* JMM - The check below should be reworked instead. */
2257 object->true_share = TRUE;
2258 }
2259 if(((map_entry->max_protection) & protections) != protections) {
2260 kr = KERN_INVALID_RIGHT;
2261 vm_object_unlock(object);
2262 vm_map_unlock_read(target_map);
2263 if(real_map != target_map)
2264 vm_map_unlock_read(real_map);
2265 vm_object_deallocate(object);
2266 object = VM_OBJECT_NULL;
2267 goto make_mem_done;
2268 }
2269
2270 mappable_size = hi_offset - obj_off;
2271 total_size = map_entry->vme_end - map_entry->vme_start;
2272 if(map_size > mappable_size) {
2273 /* try to extend mappable size if the entries */
2274 /* following are from the same object and are */
2275 /* compatible */
2276 next_entry = map_entry->vme_next;
2277 /* lets see if the next map entry is still */
2278 /* pointing at this object and is contiguous */
2279 while(map_size > mappable_size) {
2280 if((next_entry->object.vm_object == object) &&
2281 (next_entry->vme_start ==
2282 next_entry->vme_prev->vme_end) &&
2283 (next_entry->offset ==
2284 next_entry->vme_prev->offset +
2285 (next_entry->vme_prev->vme_end -
2286 next_entry->vme_prev->vme_start))) {
2287 if(((next_entry->max_protection)
2288 & protections) != protections) {
2289 break;
2290 }
2291 if (next_entry->needs_copy !=
2292 map_entry->needs_copy)
2293 break;
2294 mappable_size += next_entry->vme_end
2295 - next_entry->vme_start;
2296 total_size += next_entry->vme_end
2297 - next_entry->vme_start;
2298 next_entry = next_entry->vme_next;
2299 } else {
2300 break;
2301 }
2302
2303 }
2304 }
2305
2306 if(object->internal) {
2307 /* vm_map_lookup_locked will create a shadow if */
2308 /* needs_copy is set but does not check for the */
2309 /* other two conditions shown. It is important to */
2310 /* set up an object which will not be pulled from */
2311 /* under us. */
2312
2313 if ((map_entry->needs_copy || object->shadowed ||
2314 (object->size > total_size))
2315 && !object->true_share) {
2316 /*
2317 * We have to unlock the VM object before
2318 * trying to upgrade the VM map lock, to
2319 * honor lock ordering (map then object).
2320 * Otherwise, we would deadlock if another
2321 * thread holds a read lock on the VM map and
2322 * is trying to acquire the VM object's lock.
2323 * We still hold an extra reference on the
2324 * VM object, guaranteeing that it won't
2325 * disappear.
2326 */
2327 vm_object_unlock(object);
2328
2329 if (vm_map_lock_read_to_write(target_map)) {
2330 /*
2331 * We couldn't upgrade our VM map lock
2332 * from "read" to "write" and we lost
2333 * our "read" lock.
2334 * Start all over again...
2335 */
2336 vm_object_deallocate(object); /* extra ref */
2337 target_map = original_map;
2338 goto redo_lookup;
2339 }
2340 vm_object_lock(object);
2341
2342 /*
2343 * JMM - We need to avoid coming here when the object
2344 * is wired by anybody, not just the current map. Why
2345 * couldn't we use the standard vm_object_copy_quickly()
2346 * approach here?
2347 */
2348
2349 /* create a shadow object */
2350 vm_object_shadow(&map_entry->object.vm_object,
2351 &map_entry->offset, total_size);
2352 shadow_object = map_entry->object.vm_object;
2353 vm_object_unlock(object);
2354
2355 prot = map_entry->protection & ~VM_PROT_WRITE;
2356 #ifdef STACK_ONLY_NX
2357 if (map_entry->alias != VM_MEMORY_STACK && prot)
2358 prot |= VM_PROT_EXECUTE;
2359 #endif
2360 vm_object_pmap_protect(
2361 object, map_entry->offset,
2362 total_size,
2363 ((map_entry->is_shared
2364 || target_map->mapped)
2365 ? PMAP_NULL :
2366 target_map->pmap),
2367 map_entry->vme_start,
2368 prot);
2369 total_size -= (map_entry->vme_end
2370 - map_entry->vme_start);
2371 next_entry = map_entry->vme_next;
2372 map_entry->needs_copy = FALSE;
2373 while (total_size) {
2374 if(next_entry->object.vm_object == object) {
2375 shadow_object->ref_count++;
2376 vm_object_res_reference(shadow_object);
2377 next_entry->object.vm_object
2378 = shadow_object;
2379 vm_object_deallocate(object);
2380 next_entry->offset
2381 = next_entry->vme_prev->offset +
2382 (next_entry->vme_prev->vme_end
2383 - next_entry->vme_prev->vme_start);
2384 next_entry->needs_copy = FALSE;
2385 } else {
2386 panic("mach_make_memory_entry_64:"
2387 " map entries out of sync\n");
2388 }
2389 total_size -=
2390 next_entry->vme_end
2391 - next_entry->vme_start;
2392 next_entry = next_entry->vme_next;
2393 }
2394
2395 /*
2396 * Transfer our extra reference to the
2397 * shadow object.
2398 */
2399 vm_object_reference_locked(shadow_object);
2400 vm_object_deallocate(object); /* extra ref */
2401 object = shadow_object;
2402
2403 obj_off = (local_offset - map_entry->vme_start)
2404 + map_entry->offset;
2405
2406 vm_map_lock_write_to_read(target_map);
2407 vm_object_lock(object);
2408
2409 }
2410 }
2411
2412 /* note: in the future we can (if necessary) allow for */
2413 /* memory object lists, this will better support */
2414 /* fragmentation, but is it necessary? The user should */
2415 /* be encouraged to create address space oriented */
2416 /* shared objects from CLEAN memory regions which have */
2417 /* a known and defined history. i.e. no inheritence */
2418 /* share, make this call before making the region the */
2419 /* target of ipc's, etc. The code above, protecting */
2420 /* against delayed copy, etc. is mostly defensive. */
2421
2422 wimg_mode = object->wimg_bits;
2423 if(!(object->nophyscache)) {
2424 if(access == MAP_MEM_IO) {
2425 wimg_mode = VM_WIMG_IO;
2426 } else if (access == MAP_MEM_COPYBACK) {
2427 wimg_mode = VM_WIMG_USE_DEFAULT;
2428 } else if (access == MAP_MEM_WTHRU) {
2429 wimg_mode = VM_WIMG_WTHRU;
2430 } else if (access == MAP_MEM_WCOMB) {
2431 wimg_mode = VM_WIMG_WCOMB;
2432 }
2433 }
2434
2435 object->true_share = TRUE;
2436 if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC)
2437 object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
2438
2439 /*
2440 * The memory entry now points to this VM object and we
2441 * need to hold a reference on the VM object. Use the extra
2442 * reference we took earlier to keep the object alive when we
2443 * had to unlock it.
2444 */
2445
2446 vm_map_unlock_read(target_map);
2447 if(real_map != target_map)
2448 vm_map_unlock_read(real_map);
2449
2450 if(object->wimg_bits != wimg_mode) {
2451 vm_page_t p;
2452
2453 vm_object_paging_wait(object, THREAD_UNINT);
2454
2455 if ((wimg_mode == VM_WIMG_IO)
2456 || (wimg_mode == VM_WIMG_WCOMB))
2457 cache_attr = TRUE;
2458 else
2459 cache_attr = FALSE;
2460
2461 queue_iterate(&object->memq,
2462 p, vm_page_t, listq) {
2463 if (!p->fictitious) {
2464 pmap_disconnect(p->phys_page);
2465 if (cache_attr)
2466 pmap_sync_page_attributes_phys(p->phys_page);
2467 }
2468 }
2469 object->wimg_bits = wimg_mode;
2470 }
2471
2472 /* the size of mapped entry that overlaps with our region */
2473 /* which is targeted for share. */
2474 /* (entry_end - entry_start) - */
2475 /* offset of our beg addr within entry */
2476 /* it corresponds to this: */
2477
2478 if(map_size > mappable_size)
2479 map_size = mappable_size;
2480
2481 if (permission & MAP_MEM_NAMED_REUSE) {
2482 /*
2483 * Compare what we got with the "parent_entry".
2484 * If they match, re-use the "parent_entry" instead
2485 * of creating a new one.
2486 */
2487 if (parent_entry != NULL &&
2488 parent_entry->backing.object == object &&
2489 parent_entry->internal == object->internal &&
2490 parent_entry->is_sub_map == FALSE &&
2491 parent_entry->is_pager == FALSE &&
2492 parent_entry->offset == obj_off &&
2493 parent_entry->protection == protections &&
2494 parent_entry->size == map_size) {
2495 /*
2496 * We have a match: re-use "parent_entry".
2497 */
2498 /* release our extra reference on object */
2499 vm_object_unlock(object);
2500 vm_object_deallocate(object);
2501 /* parent_entry->ref_count++; XXX ? */
2502 /* Get an extra send-right on handle */
2503 ipc_port_copy_send(parent_handle);
2504 *object_handle = parent_handle;
2505 return KERN_SUCCESS;
2506 } else {
2507 /*
2508 * No match: we need to create a new entry.
2509 * fall through...
2510 */
2511 }
2512 }
2513
2514 vm_object_unlock(object);
2515 if (mach_memory_entry_allocate(&user_entry, &user_handle)
2516 != KERN_SUCCESS) {
2517 /* release our unused reference on the object */
2518 vm_object_deallocate(object);
2519 return KERN_FAILURE;
2520 }
2521
2522 user_entry->backing.object = object;
2523 user_entry->internal = object->internal;
2524 user_entry->is_sub_map = FALSE;
2525 user_entry->is_pager = FALSE;
2526 user_entry->offset = obj_off;
2527 user_entry->protection = permission;
2528 user_entry->size = map_size;
2529
2530 /* user_object pager and internal fields are not used */
2531 /* when the object field is filled in. */
2532
2533 *size = CAST_DOWN(vm_size_t, map_size);
2534 *object_handle = user_handle;
2535 return KERN_SUCCESS;
2536
2537 } else {
2538 /* The new object will be base on an existing named object */
2539
2540 if (parent_entry == NULL) {
2541 kr = KERN_INVALID_ARGUMENT;
2542 goto make_mem_done;
2543 }
2544 if((offset + map_size) > parent_entry->size) {
2545 kr = KERN_INVALID_ARGUMENT;
2546 goto make_mem_done;
2547 }
2548
2549 if((protections & parent_entry->protection) != protections) {
2550 kr = KERN_PROTECTION_FAILURE;
2551 goto make_mem_done;
2552 }
2553
2554 if (mach_memory_entry_allocate(&user_entry, &user_handle)
2555 != KERN_SUCCESS) {
2556 kr = KERN_FAILURE;
2557 goto make_mem_done;
2558 }
2559
2560 user_entry->size = map_size;
2561 user_entry->offset = parent_entry->offset + map_offset;
2562 user_entry->is_sub_map = parent_entry->is_sub_map;
2563 user_entry->is_pager = parent_entry->is_pager;
2564 user_entry->internal = parent_entry->internal;
2565 user_entry->protection = protections;
2566
2567 if(access != MAP_MEM_NOOP) {
2568 SET_MAP_MEM(access, user_entry->protection);
2569 }
2570
2571 if(parent_entry->is_sub_map) {
2572 user_entry->backing.map = parent_entry->backing.map;
2573 vm_map_lock(user_entry->backing.map);
2574 user_entry->backing.map->ref_count++;
2575 vm_map_unlock(user_entry->backing.map);
2576 }
2577 else if (parent_entry->is_pager) {
2578 user_entry->backing.pager = parent_entry->backing.pager;
2579 /* JMM - don't we need a reference here? */
2580 } else {
2581 object = parent_entry->backing.object;
2582 assert(object != VM_OBJECT_NULL);
2583 user_entry->backing.object = object;
2584 /* we now point to this object, hold on */
2585 vm_object_reference(object);
2586 vm_object_lock(object);
2587 object->true_share = TRUE;
2588 if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC)
2589 object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
2590 vm_object_unlock(object);
2591 }
2592 *size = CAST_DOWN(vm_size_t, map_size);
2593 *object_handle = user_handle;
2594 return KERN_SUCCESS;
2595 }
2596
2597 make_mem_done:
2598 if (user_handle != IP_NULL) {
2599 ipc_port_dealloc_kernel(user_handle);
2600 }
2601 if (user_entry != NULL) {
2602 kfree(user_entry, sizeof *user_entry);
2603 }
2604 return kr;
2605 }
2606
2607 kern_return_t
2608 _mach_make_memory_entry(
2609 vm_map_t target_map,
2610 memory_object_size_t *size,
2611 memory_object_offset_t offset,
2612 vm_prot_t permission,
2613 ipc_port_t *object_handle,
2614 ipc_port_t parent_entry)
2615 {
2616 memory_object_offset_t mo_size;
2617 kern_return_t kr;
2618
2619 mo_size = (memory_object_offset_t)*size;
2620 kr = mach_make_memory_entry_64(target_map, &mo_size,
2621 (memory_object_offset_t)offset, permission, object_handle,
2622 parent_entry);
2623 *size = mo_size;
2624 return kr;
2625 }
2626
2627 kern_return_t
2628 mach_make_memory_entry(
2629 vm_map_t target_map,
2630 vm_size_t *size,
2631 vm_offset_t offset,
2632 vm_prot_t permission,
2633 ipc_port_t *object_handle,
2634 ipc_port_t parent_entry)
2635 {
2636 memory_object_offset_t mo_size;
2637 kern_return_t kr;
2638
2639 mo_size = (memory_object_offset_t)*size;
2640 kr = mach_make_memory_entry_64(target_map, &mo_size,
2641 (memory_object_offset_t)offset, permission, object_handle,
2642 parent_entry);
2643 *size = CAST_DOWN(vm_size_t, mo_size);
2644 return kr;
2645 }
2646
2647 /*
2648 * task_wire
2649 *
2650 * Set or clear the map's wiring_required flag. This flag, if set,
2651 * will cause all future virtual memory allocation to allocate
2652 * user wired memory. Unwiring pages wired down as a result of
2653 * this routine is done with the vm_wire interface.
2654 */
2655 kern_return_t
2656 task_wire(
2657 vm_map_t map,
2658 boolean_t must_wire)
2659 {
2660 if (map == VM_MAP_NULL)
2661 return(KERN_INVALID_ARGUMENT);
2662
2663 if (must_wire)
2664 map->wiring_required = TRUE;
2665 else
2666 map->wiring_required = FALSE;
2667
2668 return(KERN_SUCCESS);
2669 }
2670
2671 __private_extern__ kern_return_t
2672 mach_memory_entry_allocate(
2673 vm_named_entry_t *user_entry_p,
2674 ipc_port_t *user_handle_p)
2675 {
2676 vm_named_entry_t user_entry;
2677 ipc_port_t user_handle;
2678 ipc_port_t previous;
2679
2680 user_entry = (vm_named_entry_t) kalloc(sizeof *user_entry);
2681 if (user_entry == NULL)
2682 return KERN_FAILURE;
2683
2684 named_entry_lock_init(user_entry);
2685
2686 user_handle = ipc_port_alloc_kernel();
2687 if (user_handle == IP_NULL) {
2688 kfree(user_entry, sizeof *user_entry);
2689 return KERN_FAILURE;
2690 }
2691 ip_lock(user_handle);
2692
2693 /* make a sonce right */
2694 user_handle->ip_sorights++;
2695 ip_reference(user_handle);
2696
2697 user_handle->ip_destination = IP_NULL;
2698 user_handle->ip_receiver_name = MACH_PORT_NULL;
2699 user_handle->ip_receiver = ipc_space_kernel;
2700
2701 /* make a send right */
2702 user_handle->ip_mscount++;
2703 user_handle->ip_srights++;
2704 ip_reference(user_handle);
2705
2706 ipc_port_nsrequest(user_handle, 1, user_handle, &previous);
2707 /* nsrequest unlocks user_handle */
2708
2709 user_entry->backing.pager = NULL;
2710 user_entry->is_sub_map = FALSE;
2711 user_entry->is_pager = FALSE;
2712 user_entry->size = 0;
2713 user_entry->internal = FALSE;
2714 user_entry->ref_count = 1;
2715
2716 ipc_kobject_set(user_handle, (ipc_kobject_t) user_entry,
2717 IKOT_NAMED_ENTRY);
2718
2719 *user_entry_p = user_entry;
2720 *user_handle_p = user_handle;
2721
2722 return KERN_SUCCESS;
2723 }
2724
2725 /*
2726 * mach_memory_object_memory_entry_64
2727 *
2728 * Create a named entry backed by the provided pager.
2729 *
2730 * JMM - we need to hold a reference on the pager -
2731 * and release it when the named entry is destroyed.
2732 */
2733 kern_return_t
2734 mach_memory_object_memory_entry_64(
2735 host_t host,
2736 boolean_t internal,
2737 vm_object_offset_t size,
2738 vm_prot_t permission,
2739 memory_object_t pager,
2740 ipc_port_t *entry_handle)
2741 {
2742 unsigned int access;
2743 vm_named_entry_t user_entry;
2744 ipc_port_t user_handle;
2745
2746 if (host == HOST_NULL)
2747 return(KERN_INVALID_HOST);
2748
2749 if (mach_memory_entry_allocate(&user_entry, &user_handle)
2750 != KERN_SUCCESS) {
2751 return KERN_FAILURE;
2752 }
2753
2754 user_entry->backing.pager = pager;
2755 user_entry->size = size;
2756 user_entry->offset = 0;
2757 user_entry->protection = permission & VM_PROT_ALL;
2758 access = GET_MAP_MEM(permission);
2759 SET_MAP_MEM(access, user_entry->protection);
2760 user_entry->internal = internal;
2761 user_entry->is_sub_map = FALSE;
2762 user_entry->is_pager = TRUE;
2763 assert(user_entry->ref_count == 1);
2764
2765 *entry_handle = user_handle;
2766 return KERN_SUCCESS;
2767 }
2768
2769 kern_return_t
2770 mach_memory_object_memory_entry(
2771 host_t host,
2772 boolean_t internal,
2773 vm_size_t size,
2774 vm_prot_t permission,
2775 memory_object_t pager,
2776 ipc_port_t *entry_handle)
2777 {
2778 return mach_memory_object_memory_entry_64( host, internal,
2779 (vm_object_offset_t)size, permission, pager, entry_handle);
2780 }
2781
2782
2783 kern_return_t
2784 mach_memory_entry_purgable_control(
2785 ipc_port_t entry_port,
2786 vm_purgable_t control,
2787 int *state)
2788 {
2789 kern_return_t kr;
2790 vm_named_entry_t mem_entry;
2791 vm_object_t object;
2792
2793 if (entry_port == IP_NULL ||
2794 ip_kotype(entry_port) != IKOT_NAMED_ENTRY) {
2795 return KERN_INVALID_ARGUMENT;
2796 }
2797
2798 mem_entry = (vm_named_entry_t) entry_port->ip_kobject;
2799
2800 named_entry_lock(mem_entry);
2801
2802 if (mem_entry->is_sub_map || mem_entry->is_pager) {
2803 named_entry_unlock(mem_entry);
2804 return KERN_INVALID_ARGUMENT;
2805 }
2806
2807 object = mem_entry->backing.object;
2808 if (object == VM_OBJECT_NULL) {
2809 named_entry_unlock(mem_entry);
2810 return KERN_INVALID_ARGUMENT;
2811 }
2812
2813 vm_object_lock(object);
2814
2815 /* check that named entry covers entire object ? */
2816 if (mem_entry->offset != 0 || object->size != mem_entry->size) {
2817 vm_object_unlock(object);
2818 named_entry_unlock(mem_entry);
2819 return KERN_INVALID_ARGUMENT;
2820 }
2821
2822 named_entry_unlock(mem_entry);
2823
2824 kr = vm_object_purgable_control(object, control, state);
2825
2826 vm_object_unlock(object);
2827
2828 return kr;
2829 }
2830
2831 /*
2832 * mach_memory_entry_port_release:
2833 *
2834 * Release a send right on a named entry port. This is the correct
2835 * way to destroy a named entry. When the last right on the port is
2836 * released, ipc_kobject_destroy() will call mach_destroy_memory_entry().
2837 */
2838 void
2839 mach_memory_entry_port_release(
2840 ipc_port_t port)
2841 {
2842 assert(ip_kotype(port) == IKOT_NAMED_ENTRY);
2843 ipc_port_release_send(port);
2844 }
2845
2846 /*
2847 * mach_destroy_memory_entry:
2848 *
2849 * Drops a reference on a memory entry and destroys the memory entry if
2850 * there are no more references on it.
2851 * NOTE: This routine should not be called to destroy a memory entry from the
2852 * kernel, as it will not release the Mach port associated with the memory
2853 * entry. The proper way to destroy a memory entry in the kernel is to
2854 * call mach_memort_entry_port_release() to release the kernel's send-right on
2855 * the memory entry's port. When the last send right is released, the memory
2856 * entry will be destroyed via ipc_kobject_destroy().
2857 */
2858 void
2859 mach_destroy_memory_entry(
2860 ipc_port_t port)
2861 {
2862 vm_named_entry_t named_entry;
2863 #if MACH_ASSERT
2864 assert(ip_kotype(port) == IKOT_NAMED_ENTRY);
2865 #endif /* MACH_ASSERT */
2866 named_entry = (vm_named_entry_t)port->ip_kobject;
2867 mutex_lock(&(named_entry)->Lock);
2868 named_entry->ref_count -= 1;
2869 if(named_entry->ref_count == 0) {
2870 if (named_entry->is_sub_map) {
2871 vm_map_deallocate(named_entry->backing.map);
2872 } else if (!named_entry->is_pager) {
2873 /* release the memory object we've been pointing to */
2874 vm_object_deallocate(named_entry->backing.object);
2875 } /* else JMM - need to drop reference on pager in that case */
2876
2877 mutex_unlock(&(named_entry)->Lock);
2878
2879 kfree((void *) port->ip_kobject,
2880 sizeof (struct vm_named_entry));
2881 } else
2882 mutex_unlock(&(named_entry)->Lock);
2883 }
2884
2885 /* Allow manipulation of individual page state. This is actually part of */
2886 /* the UPL regimen but takes place on the memory entry rather than on a UPL */
2887
2888 kern_return_t
2889 mach_memory_entry_page_op(
2890 ipc_port_t entry_port,
2891 vm_object_offset_t offset,
2892 int ops,
2893 ppnum_t *phys_entry,
2894 int *flags)
2895 {
2896 vm_named_entry_t mem_entry;
2897 vm_object_t object;
2898 kern_return_t kr;
2899
2900 if (entry_port == IP_NULL ||
2901 ip_kotype(entry_port) != IKOT_NAMED_ENTRY) {
2902 return KERN_INVALID_ARGUMENT;
2903 }
2904
2905 mem_entry = (vm_named_entry_t) entry_port->ip_kobject;
2906
2907 named_entry_lock(mem_entry);
2908
2909 if (mem_entry->is_sub_map || mem_entry->is_pager) {
2910 named_entry_unlock(mem_entry);
2911 return KERN_INVALID_ARGUMENT;
2912 }
2913
2914 object = mem_entry->backing.object;
2915 if (object == VM_OBJECT_NULL) {
2916 named_entry_unlock(mem_entry);
2917 return KERN_INVALID_ARGUMENT;
2918 }
2919
2920 vm_object_reference(object);
2921 named_entry_unlock(mem_entry);
2922
2923 kr = vm_object_page_op(object, offset, ops, phys_entry, flags);
2924
2925 vm_object_deallocate(object);
2926
2927 return kr;
2928 }
2929
2930 /*
2931 * mach_memory_entry_range_op offers performance enhancement over
2932 * mach_memory_entry_page_op for page_op functions which do not require page
2933 * level state to be returned from the call. Page_op was created to provide
2934 * a low-cost alternative to page manipulation via UPLs when only a single
2935 * page was involved. The range_op call establishes the ability in the _op
2936 * family of functions to work on multiple pages where the lack of page level
2937 * state handling allows the caller to avoid the overhead of the upl structures.
2938 */
2939
2940 kern_return_t
2941 mach_memory_entry_range_op(
2942 ipc_port_t entry_port,
2943 vm_object_offset_t offset_beg,
2944 vm_object_offset_t offset_end,
2945 int ops,
2946 int *range)
2947 {
2948 vm_named_entry_t mem_entry;
2949 vm_object_t object;
2950 kern_return_t kr;
2951
2952 if (entry_port == IP_NULL ||
2953 ip_kotype(entry_port) != IKOT_NAMED_ENTRY) {
2954 return KERN_INVALID_ARGUMENT;
2955 }
2956
2957 mem_entry = (vm_named_entry_t) entry_port->ip_kobject;
2958
2959 named_entry_lock(mem_entry);
2960
2961 if (mem_entry->is_sub_map || mem_entry->is_pager) {
2962 named_entry_unlock(mem_entry);
2963 return KERN_INVALID_ARGUMENT;
2964 }
2965
2966 object = mem_entry->backing.object;
2967 if (object == VM_OBJECT_NULL) {
2968 named_entry_unlock(mem_entry);
2969 return KERN_INVALID_ARGUMENT;
2970 }
2971
2972 vm_object_reference(object);
2973 named_entry_unlock(mem_entry);
2974
2975 kr = vm_object_range_op(object,
2976 offset_beg,
2977 offset_end,
2978 ops,
2979 range);
2980
2981 vm_object_deallocate(object);
2982
2983 return kr;
2984 }
2985
2986
2987 kern_return_t
2988 set_dp_control_port(
2989 host_priv_t host_priv,
2990 ipc_port_t control_port)
2991 {
2992 if (host_priv == HOST_PRIV_NULL)
2993 return (KERN_INVALID_HOST);
2994
2995 if (IP_VALID(dynamic_pager_control_port))
2996 ipc_port_release_send(dynamic_pager_control_port);
2997
2998 dynamic_pager_control_port = control_port;
2999 return KERN_SUCCESS;
3000 }
3001
3002 kern_return_t
3003 get_dp_control_port(
3004 host_priv_t host_priv,
3005 ipc_port_t *control_port)
3006 {
3007 if (host_priv == HOST_PRIV_NULL)
3008 return (KERN_INVALID_HOST);
3009
3010 *control_port = ipc_port_copy_send(dynamic_pager_control_port);
3011 return KERN_SUCCESS;
3012
3013 }
3014
3015 /* ******* Temporary Internal calls to UPL for BSD ***** */
3016
3017 extern int kernel_upl_map(
3018 vm_map_t map,
3019 upl_t upl,
3020 vm_offset_t *dst_addr);
3021
3022 extern int kernel_upl_unmap(
3023 vm_map_t map,
3024 upl_t upl);
3025
3026 extern int kernel_upl_commit(
3027 upl_t upl,
3028 upl_page_info_t *pl,
3029 mach_msg_type_number_t count);
3030
3031 extern int kernel_upl_commit_range(
3032 upl_t upl,
3033 upl_offset_t offset,
3034 upl_size_t size,
3035 int flags,
3036 upl_page_info_array_t pl,
3037 mach_msg_type_number_t count);
3038
3039 extern int kernel_upl_abort(
3040 upl_t upl,
3041 int abort_type);
3042
3043 extern int kernel_upl_abort_range(
3044 upl_t upl,
3045 upl_offset_t offset,
3046 upl_size_t size,
3047 int abort_flags);
3048
3049
3050 kern_return_t
3051 kernel_upl_map(
3052 vm_map_t map,
3053 upl_t upl,
3054 vm_offset_t *dst_addr)
3055 {
3056 return vm_upl_map(map, upl, dst_addr);
3057 }
3058
3059
3060 kern_return_t
3061 kernel_upl_unmap(
3062 vm_map_t map,
3063 upl_t upl)
3064 {
3065 return vm_upl_unmap(map, upl);
3066 }
3067
3068 kern_return_t
3069 kernel_upl_commit(
3070 upl_t upl,
3071 upl_page_info_t *pl,
3072 mach_msg_type_number_t count)
3073 {
3074 kern_return_t kr;
3075
3076 kr = upl_commit(upl, pl, count);
3077 upl_deallocate(upl);
3078 return kr;
3079 }
3080
3081
3082 kern_return_t
3083 kernel_upl_commit_range(
3084 upl_t upl,
3085 upl_offset_t offset,
3086 upl_size_t size,
3087 int flags,
3088 upl_page_info_array_t pl,
3089 mach_msg_type_number_t count)
3090 {
3091 boolean_t finished = FALSE;
3092 kern_return_t kr;
3093
3094 if (flags & UPL_COMMIT_FREE_ON_EMPTY)
3095 flags |= UPL_COMMIT_NOTIFY_EMPTY;
3096
3097 kr = upl_commit_range(upl, offset, size, flags, pl, count, &finished);
3098
3099 if ((flags & UPL_COMMIT_NOTIFY_EMPTY) && finished)
3100 upl_deallocate(upl);
3101
3102 return kr;
3103 }
3104
3105 kern_return_t
3106 kernel_upl_abort_range(
3107 upl_t upl,
3108 upl_offset_t offset,
3109 upl_size_t size,
3110 int abort_flags)
3111 {
3112 kern_return_t kr;
3113 boolean_t finished = FALSE;
3114
3115 if (abort_flags & UPL_COMMIT_FREE_ON_EMPTY)
3116 abort_flags |= UPL_COMMIT_NOTIFY_EMPTY;
3117
3118 kr = upl_abort_range(upl, offset, size, abort_flags, &finished);
3119
3120 if ((abort_flags & UPL_COMMIT_FREE_ON_EMPTY) && finished)
3121 upl_deallocate(upl);
3122
3123 return kr;
3124 }
3125
3126 kern_return_t
3127 kernel_upl_abort(
3128 upl_t upl,
3129 int abort_type)
3130 {
3131 kern_return_t kr;
3132
3133 kr = upl_abort(upl, abort_type);
3134 upl_deallocate(upl);
3135 return kr;
3136 }
3137
3138 /*
3139 * Now a kernel-private interface (for BootCache
3140 * use only). Need a cleaner way to create an
3141 * empty vm_map() and return a handle to it.
3142 */
3143
3144 kern_return_t
3145 vm_region_object_create(
3146 __unused vm_map_t target_map,
3147 vm_size_t size,
3148 ipc_port_t *object_handle)
3149 {
3150 vm_named_entry_t user_entry;
3151 ipc_port_t user_handle;
3152
3153 vm_map_t new_map;
3154
3155 if (mach_memory_entry_allocate(&user_entry, &user_handle)
3156 != KERN_SUCCESS) {
3157 return KERN_FAILURE;
3158 }
3159
3160 /* Create a named object based on a submap of specified size */
3161
3162 new_map = vm_map_create(PMAP_NULL, VM_MAP_MIN_ADDRESS,
3163 vm_map_round_page(size), TRUE);
3164
3165 user_entry->backing.map = new_map;
3166 user_entry->internal = TRUE;
3167 user_entry->is_sub_map = TRUE;
3168 user_entry->offset = 0;
3169 user_entry->protection = VM_PROT_ALL;
3170 user_entry->size = size;
3171 assert(user_entry->ref_count == 1);
3172
3173 *object_handle = user_handle;
3174 return KERN_SUCCESS;
3175
3176 }
3177
3178 ppnum_t vm_map_get_phys_page( /* forward */
3179 vm_map_t map,
3180 vm_offset_t offset);
3181
3182 ppnum_t
3183 vm_map_get_phys_page(
3184 vm_map_t map,
3185 vm_offset_t addr)
3186 {
3187 vm_object_offset_t offset;
3188 vm_object_t object;
3189 vm_map_offset_t map_offset;
3190 vm_map_entry_t entry;
3191 ppnum_t phys_page = 0;
3192
3193 map_offset = vm_map_trunc_page(addr);
3194
3195 vm_map_lock(map);
3196 while (vm_map_lookup_entry(map, map_offset, &entry)) {
3197
3198 if (entry->object.vm_object == VM_OBJECT_NULL) {
3199 vm_map_unlock(map);
3200 return (ppnum_t) 0;
3201 }
3202 if (entry->is_sub_map) {
3203 vm_map_t old_map;
3204 vm_map_lock(entry->object.sub_map);
3205 old_map = map;
3206 map = entry->object.sub_map;
3207 map_offset = entry->offset + (map_offset - entry->vme_start);
3208 vm_map_unlock(old_map);
3209 continue;
3210 }
3211 if (entry->object.vm_object->phys_contiguous) {
3212 /* These are not standard pageable memory mappings */
3213 /* If they are not present in the object they will */
3214 /* have to be picked up from the pager through the */
3215 /* fault mechanism. */
3216 if(entry->object.vm_object->shadow_offset == 0) {
3217 /* need to call vm_fault */
3218 vm_map_unlock(map);
3219 vm_fault(map, map_offset, VM_PROT_NONE,
3220 FALSE, THREAD_UNINT, NULL, 0);
3221 vm_map_lock(map);
3222 continue;
3223 }
3224 offset = entry->offset + (map_offset - entry->vme_start);
3225 phys_page = (ppnum_t)
3226 ((entry->object.vm_object->shadow_offset
3227 + offset) >> 12);
3228 break;
3229
3230 }
3231 offset = entry->offset + (map_offset - entry->vme_start);
3232 object = entry->object.vm_object;
3233 vm_object_lock(object);
3234 while (TRUE) {
3235 vm_page_t dst_page = vm_page_lookup(object,offset);
3236 if(dst_page == VM_PAGE_NULL) {
3237 if(object->shadow) {
3238 vm_object_t old_object;
3239 vm_object_lock(object->shadow);
3240 old_object = object;
3241 offset = offset + object->shadow_offset;
3242 object = object->shadow;
3243 vm_object_unlock(old_object);
3244 } else {
3245 vm_object_unlock(object);
3246 break;
3247 }
3248 } else {
3249 phys_page = (ppnum_t)(dst_page->phys_page);
3250 vm_object_unlock(object);
3251 break;
3252 }
3253 }
3254 break;
3255
3256 }
3257
3258 vm_map_unlock(map);
3259 return phys_page;
3260 }
3261
3262
3263
3264 kern_return_t kernel_object_iopl_request( /* forward */
3265 vm_named_entry_t named_entry,
3266 memory_object_offset_t offset,
3267 vm_size_t *upl_size,
3268 upl_t *upl_ptr,
3269 upl_page_info_array_t user_page_list,
3270 unsigned int *page_list_count,
3271 int *flags);
3272
3273 kern_return_t
3274 kernel_object_iopl_request(
3275 vm_named_entry_t named_entry,
3276 memory_object_offset_t offset,
3277 vm_size_t *upl_size,
3278 upl_t *upl_ptr,
3279 upl_page_info_array_t user_page_list,
3280 unsigned int *page_list_count,
3281 int *flags)
3282 {
3283 vm_object_t object;
3284 kern_return_t ret;
3285
3286 int caller_flags;
3287
3288 caller_flags = *flags;
3289
3290 if (caller_flags & ~UPL_VALID_FLAGS) {
3291 /*
3292 * For forward compatibility's sake,
3293 * reject any unknown flag.
3294 */
3295 return KERN_INVALID_VALUE;
3296 }
3297
3298 /* a few checks to make sure user is obeying rules */
3299 if(*upl_size == 0) {
3300 if(offset >= named_entry->size)
3301 return(KERN_INVALID_RIGHT);
3302 *upl_size = named_entry->size - offset;
3303 }
3304 if(caller_flags & UPL_COPYOUT_FROM) {
3305 if((named_entry->protection & VM_PROT_READ)
3306 != VM_PROT_READ) {
3307 return(KERN_INVALID_RIGHT);
3308 }
3309 } else {
3310 if((named_entry->protection &
3311 (VM_PROT_READ | VM_PROT_WRITE))
3312 != (VM_PROT_READ | VM_PROT_WRITE)) {
3313 return(KERN_INVALID_RIGHT);
3314 }
3315 }
3316 if(named_entry->size < (offset + *upl_size))
3317 return(KERN_INVALID_ARGUMENT);
3318
3319 /* the callers parameter offset is defined to be the */
3320 /* offset from beginning of named entry offset in object */
3321 offset = offset + named_entry->offset;
3322
3323 if(named_entry->is_sub_map)
3324 return (KERN_INVALID_ARGUMENT);
3325
3326 named_entry_lock(named_entry);
3327
3328 if (named_entry->is_pager) {
3329 object = vm_object_enter(named_entry->backing.pager,
3330 named_entry->offset + named_entry->size,
3331 named_entry->internal,
3332 FALSE,
3333 FALSE);
3334 if (object == VM_OBJECT_NULL) {
3335 named_entry_unlock(named_entry);
3336 return(KERN_INVALID_OBJECT);
3337 }
3338
3339 /* JMM - drop reference on the pager here? */
3340
3341 /* create an extra reference for the object */
3342 vm_object_lock(object);
3343 vm_object_reference_locked(object);
3344 named_entry->backing.object = object;
3345 named_entry->is_pager = FALSE;
3346 named_entry_unlock(named_entry);
3347
3348 /* wait for object (if any) to be ready */
3349 if (!named_entry->internal) {
3350 while (!object->pager_ready) {
3351 vm_object_wait(object,
3352 VM_OBJECT_EVENT_PAGER_READY,
3353 THREAD_UNINT);
3354 vm_object_lock(object);
3355 }
3356 }
3357 vm_object_unlock(object);
3358
3359 } else {
3360 /* This is the case where we are going to operate */
3361 /* an an already known object. If the object is */
3362 /* not ready it is internal. An external */
3363 /* object cannot be mapped until it is ready */
3364 /* we can therefore avoid the ready check */
3365 /* in this case. */
3366 object = named_entry->backing.object;
3367 vm_object_reference(object);
3368 named_entry_unlock(named_entry);
3369 }
3370
3371 if (!object->private) {
3372 if (*upl_size > (MAX_UPL_TRANSFER*PAGE_SIZE))
3373 *upl_size = (MAX_UPL_TRANSFER*PAGE_SIZE);
3374 if (object->phys_contiguous) {
3375 *flags = UPL_PHYS_CONTIG;
3376 } else {
3377 *flags = 0;
3378 }
3379 } else {
3380 *flags = UPL_DEV_MEMORY | UPL_PHYS_CONTIG;
3381 }
3382
3383 ret = vm_object_iopl_request(object,
3384 offset,
3385 *upl_size,
3386 upl_ptr,
3387 user_page_list,
3388 page_list_count,
3389 caller_flags);
3390 vm_object_deallocate(object);
3391 return ret;
3392 }
mirror of XNU