]> git.saurik.com Git - apple/xnu.git/blob - osfmk/vm/vm_user.c
projects / apple / xnu.git / blob
? search:


846f1642c012580993a2aadc76f57bd1b1e25b1d
[apple/xnu.git] / osfmk / vm / vm_user.c
1 /*
2 * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_LICENSE_OSREFERENCE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the
10 * License may not be used to create, or enable the creation or
11 * redistribution of, unlawful or unlicensed copies of an Apple operating
12 * system, or to circumvent, violate, or enable the circumvention or
13 * violation of, any terms of an Apple operating system software license
14 * agreement.
15 *
16 * Please obtain a copy of the License at
17 * http://www.opensource.apple.com/apsl/ and read it before using this
18 * file.
19 *
20 * The Original Code and all software distributed under the License are
21 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
22 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
23 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
24 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
25 * Please see the License for the specific language governing rights and
26 * limitations under the License.
27 *
28 * @APPLE_LICENSE_OSREFERENCE_HEADER_END@
29 */
30 /*
31 * @OSF_COPYRIGHT@
32 */
33 /*
34 * Mach Operating System
35 * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University
36 * All Rights Reserved.
37 *
38 * Permission to use, copy, modify and distribute this software and its
39 * documentation is hereby granted, provided that both the copyright
40 * notice and this permission notice appear in all copies of the
41 * software, derivative works or modified versions, and any portions
42 * thereof, and that both notices appear in supporting documentation.
43 *
44 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
45 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
46 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
47 *
48 * Carnegie Mellon requests users of this software to return to
49 *
50 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
51 * School of Computer Science
52 * Carnegie Mellon University
53 * Pittsburgh PA 15213-3890
54 *
55 * any improvements or extensions that they make and grant Carnegie Mellon
56 * the rights to redistribute these changes.
57 */
58 /*
59 */
60 /*
61 * File: vm/vm_user.c
62 * Author: Avadis Tevanian, Jr., Michael Wayne Young
63 *
64 * User-exported virtual memory functions.
65 */
66
67 #include <debug.h>
68
69 #include <vm_cpm.h>
70 #include <mach/boolean.h>
71 #include <mach/kern_return.h>
72 #include <mach/mach_types.h> /* to get vm_address_t */
73 #include <mach/memory_object.h>
74 #include <mach/std_types.h> /* to get pointer_t */
75 #include <mach/upl.h>
76 #include <mach/vm_attributes.h>
77 #include <mach/vm_param.h>
78 #include <mach/vm_statistics.h>
79 #include <mach/mach_syscalls.h>
80
81 #include <mach/host_priv_server.h>
82 #include <mach/mach_vm_server.h>
83 #include <mach/shared_memory_server.h>
84 #include <mach/vm_map_server.h>
85 #include <vm/vm_shared_memory_server.h>
86
87 #include <kern/host.h>
88 #include <kern/kalloc.h>
89 #include <kern/task.h>
90 #include <kern/misc_protos.h>
91 #include <vm/vm_fault.h>
92 #include <vm/vm_map.h>
93 #include <vm/vm_object.h>
94 #include <vm/vm_page.h>
95 #include <vm/memory_object.h>
96 #include <vm/vm_pageout.h>
97 #include <vm/vm_protos.h>
98
99 vm_size_t upl_offset_to_pagelist = 0;
100
101 #if VM_CPM
102 #include <vm/cpm.h>
103 #endif /* VM_CPM */
104
105 ipc_port_t dynamic_pager_control_port=NULL;
106
107 /*
108 * mach_vm_allocate allocates "zero fill" memory in the specfied
109 * map.
110 */
111 kern_return_t
112 mach_vm_allocate(
113 vm_map_t map,
114 mach_vm_offset_t *addr,
115 mach_vm_size_t size,
116 int flags)
117 {
118 vm_map_offset_t map_addr;
119 vm_map_size_t map_size;
120 kern_return_t result;
121 boolean_t anywhere = ((VM_FLAGS_ANYWHERE & flags) != 0);
122
123 if (map == VM_MAP_NULL)
124 return(KERN_INVALID_ARGUMENT);
125 if (size == 0) {
126 *addr = 0;
127 return(KERN_SUCCESS);
128 }
129
130 if (anywhere) {
131 /*
132 * No specific address requested, so start candidate address
133 * search at the minimum address in the map. However, if that
134 * minimum is 0, bump it up by PAGE_SIZE. We want to limit
135 * allocations of PAGEZERO to explicit requests since its
136 * normal use is to catch dereferences of NULL and many
137 * applications also treat pointers with a value of 0 as
138 * special and suddenly having address 0 contain useable
139 * memory would tend to confuse those applications.
140 */
141 map_addr = vm_map_min(map);
142 if (map_addr == 0)
143 map_addr += PAGE_SIZE;
144 } else
145 map_addr = vm_map_trunc_page(*addr);
146 map_size = vm_map_round_page(size);
147 if (map_size == 0) {
148 return(KERN_INVALID_ARGUMENT);
149 }
150
151 result = vm_map_enter(
152 map,
153 &map_addr,
154 map_size,
155 (vm_map_offset_t)0,
156 flags,
157 VM_OBJECT_NULL,
158 (vm_object_offset_t)0,
159 FALSE,
160 VM_PROT_DEFAULT,
161 VM_PROT_ALL,
162 VM_INHERIT_DEFAULT);
163
164 *addr = map_addr;
165 return(result);
166 }
167
168 /*
169 * vm_allocate
170 * Legacy routine that allocates "zero fill" memory in the specfied
171 * map (which is limited to the same size as the kernel).
172 */
173 kern_return_t
174 vm_allocate(
175 vm_map_t map,
176 vm_offset_t *addr,
177 vm_size_t size,
178 int flags)
179 {
180 vm_map_offset_t map_addr;
181 vm_map_size_t map_size;
182 kern_return_t result;
183 boolean_t anywhere = ((VM_FLAGS_ANYWHERE & flags) != 0);
184
185 if (map == VM_MAP_NULL)
186 return(KERN_INVALID_ARGUMENT);
187 if (size == 0) {
188 *addr = 0;
189 return(KERN_SUCCESS);
190 }
191
192 if (anywhere) {
193 /*
194 * No specific address requested, so start candidate address
195 * search at the minimum address in the map. However, if that
196 * minimum is 0, bump it up by PAGE_SIZE. We want to limit
197 * allocations of PAGEZERO to explicit requests since its
198 * normal use is to catch dereferences of NULL and many
199 * applications also treat pointers with a value of 0 as
200 * special and suddenly having address 0 contain useable
201 * memory would tend to confuse those applications.
202 */
203 map_addr = vm_map_min(map);
204 if (map_addr == 0)
205 map_addr += PAGE_SIZE;
206 } else
207 map_addr = vm_map_trunc_page(*addr);
208 map_size = vm_map_round_page(size);
209 if (map_size == 0) {
210 return(KERN_INVALID_ARGUMENT);
211 }
212
213 result = vm_map_enter(
214 map,
215 &map_addr,
216 map_size,
217 (vm_map_offset_t)0,
218 flags,
219 VM_OBJECT_NULL,
220 (vm_object_offset_t)0,
221 FALSE,
222 VM_PROT_DEFAULT,
223 VM_PROT_ALL,
224 VM_INHERIT_DEFAULT);
225
226 *addr = CAST_DOWN(vm_offset_t, map_addr);
227 return(result);
228 }
229
230 /*
231 * mach_vm_deallocate -
232 * deallocates the specified range of addresses in the
233 * specified address map.
234 */
235 kern_return_t
236 mach_vm_deallocate(
237 vm_map_t map,
238 mach_vm_offset_t start,
239 mach_vm_size_t size)
240 {
241 if ((map == VM_MAP_NULL) || (start + size < start))
242 return(KERN_INVALID_ARGUMENT);
243
244 if (size == (mach_vm_offset_t) 0)
245 return(KERN_SUCCESS);
246
247 return(vm_map_remove(map, vm_map_trunc_page(start),
248 vm_map_round_page(start+size), VM_MAP_NO_FLAGS));
249 }
250
251 /*
252 * vm_deallocate -
253 * deallocates the specified range of addresses in the
254 * specified address map (limited to addresses the same
255 * size as the kernel).
256 */
257 kern_return_t
258 vm_deallocate(
259 register vm_map_t map,
260 vm_offset_t start,
261 vm_size_t size)
262 {
263 if ((map == VM_MAP_NULL) || (start + size < start))
264 return(KERN_INVALID_ARGUMENT);
265
266 if (size == (vm_offset_t) 0)
267 return(KERN_SUCCESS);
268
269 return(vm_map_remove(map, vm_map_trunc_page(start),
270 vm_map_round_page(start+size), VM_MAP_NO_FLAGS));
271 }
272
273 /*
274 * mach_vm_inherit -
275 * Sets the inheritance of the specified range in the
276 * specified map.
277 */
278 kern_return_t
279 mach_vm_inherit(
280 vm_map_t map,
281 mach_vm_offset_t start,
282 mach_vm_size_t size,
283 vm_inherit_t new_inheritance)
284 {
285 if ((map == VM_MAP_NULL) || (start + size < start) ||
286 (new_inheritance > VM_INHERIT_LAST_VALID))
287 return(KERN_INVALID_ARGUMENT);
288
289 if (size == 0)
290 return KERN_SUCCESS;
291
292 return(vm_map_inherit(map,
293 vm_map_trunc_page(start),
294 vm_map_round_page(start+size),
295 new_inheritance));
296 }
297
298 /*
299 * vm_inherit -
300 * Sets the inheritance of the specified range in the
301 * specified map (range limited to addresses
302 */
303 kern_return_t
304 vm_inherit(
305 register vm_map_t map,
306 vm_offset_t start,
307 vm_size_t size,
308 vm_inherit_t new_inheritance)
309 {
310 if ((map == VM_MAP_NULL) || (start + size < start) ||
311 (new_inheritance > VM_INHERIT_LAST_VALID))
312 return(KERN_INVALID_ARGUMENT);
313
314 if (size == 0)
315 return KERN_SUCCESS;
316
317 return(vm_map_inherit(map,
318 vm_map_trunc_page(start),
319 vm_map_round_page(start+size),
320 new_inheritance));
321 }
322
323 /*
324 * mach_vm_protect -
325 * Sets the protection of the specified range in the
326 * specified map.
327 */
328
329 kern_return_t
330 mach_vm_protect(
331 vm_map_t map,
332 mach_vm_offset_t start,
333 mach_vm_size_t size,
334 boolean_t set_maximum,
335 vm_prot_t new_protection)
336 {
337 if ((map == VM_MAP_NULL) || (start + size < start) ||
338 (new_protection & ~(VM_PROT_ALL | VM_PROT_COPY)))
339 return(KERN_INVALID_ARGUMENT);
340
341 if (size == 0)
342 return KERN_SUCCESS;
343
344 return(vm_map_protect(map,
345 vm_map_trunc_page(start),
346 vm_map_round_page(start+size),
347 new_protection,
348 set_maximum));
349 }
350
351 /*
352 * vm_protect -
353 * Sets the protection of the specified range in the
354 * specified map. Addressability of the range limited
355 * to the same size as the kernel.
356 */
357
358 kern_return_t
359 vm_protect(
360 vm_map_t map,
361 vm_offset_t start,
362 vm_size_t size,
363 boolean_t set_maximum,
364 vm_prot_t new_protection)
365 {
366 if ((map == VM_MAP_NULL) || (start + size < start) ||
367 (new_protection & ~(VM_PROT_ALL | VM_PROT_COPY)))
368 return(KERN_INVALID_ARGUMENT);
369
370 if (size == 0)
371 return KERN_SUCCESS;
372
373 return(vm_map_protect(map,
374 vm_map_trunc_page(start),
375 vm_map_round_page(start+size),
376 new_protection,
377 set_maximum));
378 }
379
380 /*
381 * mach_vm_machine_attributes -
382 * Handle machine-specific attributes for a mapping, such
383 * as cachability, migrability, etc.
384 */
385 kern_return_t
386 mach_vm_machine_attribute(
387 vm_map_t map,
388 mach_vm_address_t addr,
389 mach_vm_size_t size,
390 vm_machine_attribute_t attribute,
391 vm_machine_attribute_val_t* value) /* IN/OUT */
392 {
393 if ((map == VM_MAP_NULL) || (addr + size < addr))
394 return(KERN_INVALID_ARGUMENT);
395
396 if (size == 0)
397 return KERN_SUCCESS;
398
399 return vm_map_machine_attribute(map,
400 vm_map_trunc_page(addr),
401 vm_map_round_page(addr+size),
402 attribute,
403 value);
404 }
405
406 /*
407 * vm_machine_attribute -
408 * Handle machine-specific attributes for a mapping, such
409 * as cachability, migrability, etc. Limited addressability
410 * (same range limits as for the native kernel map).
411 */
412 kern_return_t
413 vm_machine_attribute(
414 vm_map_t map,
415 vm_address_t addr,
416 vm_size_t size,
417 vm_machine_attribute_t attribute,
418 vm_machine_attribute_val_t* value) /* IN/OUT */
419 {
420 if ((map == VM_MAP_NULL) || (addr + size < addr))
421 return(KERN_INVALID_ARGUMENT);
422
423 if (size == 0)
424 return KERN_SUCCESS;
425
426 return vm_map_machine_attribute(map,
427 vm_map_trunc_page(addr),
428 vm_map_round_page(addr+size),
429 attribute,
430 value);
431 }
432
433 /*
434 * mach_vm_read -
435 * Read/copy a range from one address space and return it to the caller.
436 *
437 * It is assumed that the address for the returned memory is selected by
438 * the IPC implementation as part of receiving the reply to this call.
439 * If IPC isn't used, the caller must deal with the vm_map_copy_t object
440 * that gets returned.
441 *
442 * JMM - because of mach_msg_type_number_t, this call is limited to a
443 * single 4GB region at this time.
444 *
445 */
446 kern_return_t
447 mach_vm_read(
448 vm_map_t map,
449 mach_vm_address_t addr,
450 mach_vm_size_t size,
451 pointer_t *data,
452 mach_msg_type_number_t *data_size)
453 {
454 kern_return_t error;
455 vm_map_copy_t ipc_address;
456
457 if (map == VM_MAP_NULL)
458 return(KERN_INVALID_ARGUMENT);
459
460
461 error = vm_map_copyin(map,
462 (vm_map_address_t)addr,
463 (vm_map_size_t)size,
464 FALSE, /* src_destroy */
465 &ipc_address);
466
467 if (KERN_SUCCESS == error) {
468 *data = (pointer_t) ipc_address;
469 *data_size = size;
470 }
471 return(error);
472 }
473
474 /*
475 * vm_read -
476 * Read/copy a range from one address space and return it to the caller.
477 * Limited addressability (same range limits as for the native kernel map).
478 *
479 * It is assumed that the address for the returned memory is selected by
480 * the IPC implementation as part of receiving the reply to this call.
481 * If IPC isn't used, the caller must deal with the vm_map_copy_t object
482 * that gets returned.
483 */
484 kern_return_t
485 vm_read(
486 vm_map_t map,
487 vm_address_t addr,
488 vm_size_t size,
489 pointer_t *data,
490 mach_msg_type_number_t *data_size)
491 {
492 kern_return_t error;
493 vm_map_copy_t ipc_address;
494
495 if (map == VM_MAP_NULL)
496 return(KERN_INVALID_ARGUMENT);
497
498 error = vm_map_copyin(map,
499 (vm_map_address_t)addr,
500 (vm_map_size_t)size,
501 FALSE, /* src_destroy */
502 &ipc_address);
503
504 if (KERN_SUCCESS == error) {
505 *data = (pointer_t) ipc_address;
506 *data_size = size;
507 }
508 return(error);
509 }
510
511 /*
512 * mach_vm_read_list -
513 * Read/copy a list of address ranges from specified map.
514 *
515 * MIG does not know how to deal with a returned array of
516 * vm_map_copy_t structures, so we have to do the copyout
517 * manually here.
518 */
519 kern_return_t
520 mach_vm_read_list(
521 vm_map_t map,
522 mach_vm_read_entry_t data_list,
523 natural_t count)
524 {
525 mach_msg_type_number_t i;
526 kern_return_t error;
527 vm_map_copy_t copy;
528
529 if (map == VM_MAP_NULL ||
530 count > VM_MAP_ENTRY_MAX)
531 return(KERN_INVALID_ARGUMENT);
532
533 error = KERN_SUCCESS;
534 for(i=0; i<count; i++) {
535 vm_map_address_t map_addr;
536 vm_map_size_t map_size;
537
538 map_addr = (vm_map_address_t)(data_list[i].address);
539 map_size = (vm_map_size_t)(data_list[i].size);
540
541 if(map_size != 0) {
542 error = vm_map_copyin(map,
543 map_addr,
544 map_size,
545 FALSE, /* src_destroy */
546 &copy);
547 if (KERN_SUCCESS == error) {
548 error = vm_map_copyout(
549 current_task()->map,
550 &map_addr,
551 copy);
552 if (KERN_SUCCESS == error) {
553 data_list[i].address = map_addr;
554 continue;
555 }
556 vm_map_copy_discard(copy);
557 }
558 }
559 data_list[i].address = (mach_vm_address_t)0;
560 data_list[i].size = (mach_vm_size_t)0;
561 }
562 return(error);
563 }
564
565 /*
566 * vm_read_list -
567 * Read/copy a list of address ranges from specified map.
568 *
569 * MIG does not know how to deal with a returned array of
570 * vm_map_copy_t structures, so we have to do the copyout
571 * manually here.
572 *
573 * The source and destination ranges are limited to those
574 * that can be described with a vm_address_t (i.e. same
575 * size map as the kernel).
576 *
577 * JMM - If the result of the copyout is an address range
578 * that cannot be described with a vm_address_t (i.e. the
579 * caller had a larger address space but used this call
580 * anyway), it will result in a truncated address being
581 * returned (and a likely confused caller).
582 */
583
584 kern_return_t
585 vm_read_list(
586 vm_map_t map,
587 vm_read_entry_t data_list,
588 natural_t count)
589 {
590 mach_msg_type_number_t i;
591 kern_return_t error;
592 vm_map_copy_t copy;
593
594 if (map == VM_MAP_NULL ||
595 count > VM_MAP_ENTRY_MAX)
596 return(KERN_INVALID_ARGUMENT);
597
598 error = KERN_SUCCESS;
599 for(i=0; i<count; i++) {
600 vm_map_address_t map_addr;
601 vm_map_size_t map_size;
602
603 map_addr = (vm_map_address_t)(data_list[i].address);
604 map_size = (vm_map_size_t)(data_list[i].size);
605
606 if(map_size != 0) {
607 error = vm_map_copyin(map,
608 map_addr,
609 map_size,
610 FALSE, /* src_destroy */
611 &copy);
612 if (KERN_SUCCESS == error) {
613 error = vm_map_copyout(current_task()->map,
614 &map_addr,
615 copy);
616 if (KERN_SUCCESS == error) {
617 data_list[i].address =
618 CAST_DOWN(vm_offset_t, map_addr);
619 continue;
620 }
621 vm_map_copy_discard(copy);
622 }
623 }
624 data_list[i].address = (mach_vm_address_t)0;
625 data_list[i].size = (mach_vm_size_t)0;
626 }
627 return(error);
628 }
629
630 /*
631 * mach_vm_read_overwrite -
632 * Overwrite a range of the current map with data from the specified
633 * map/address range.
634 *
635 * In making an assumption that the current thread is local, it is
636 * no longer cluster-safe without a fully supportive local proxy
637 * thread/task (but we don't support cluster's anymore so this is moot).
638 */
639
640 kern_return_t
641 mach_vm_read_overwrite(
642 vm_map_t map,
643 mach_vm_address_t address,
644 mach_vm_size_t size,
645 mach_vm_address_t data,
646 mach_vm_size_t *data_size)
647 {
648 kern_return_t error;
649 vm_map_copy_t copy;
650
651 if (map == VM_MAP_NULL)
652 return(KERN_INVALID_ARGUMENT);
653
654 error = vm_map_copyin(map, (vm_map_address_t)address,
655 (vm_map_size_t)size, FALSE, &copy);
656
657 if (KERN_SUCCESS == error) {
658 error = vm_map_copy_overwrite(current_thread()->map,
659 (vm_map_address_t)data,
660 copy, FALSE);
661 if (KERN_SUCCESS == error) {
662 *data_size = size;
663 return error;
664 }
665 vm_map_copy_discard(copy);
666 }
667 return(error);
668 }
669
670 /*
671 * vm_read_overwrite -
672 * Overwrite a range of the current map with data from the specified
673 * map/address range.
674 *
675 * This routine adds the additional limitation that the source and
676 * destination ranges must be describable with vm_address_t values
677 * (i.e. the same size address spaces as the kernel, or at least the
678 * the ranges are in that first portion of the respective address
679 * spaces).
680 */
681
682 kern_return_t
683 vm_read_overwrite(
684 vm_map_t map,
685 vm_address_t address,
686 vm_size_t size,
687 vm_address_t data,
688 vm_size_t *data_size)
689 {
690 kern_return_t error;
691 vm_map_copy_t copy;
692
693 if (map == VM_MAP_NULL)
694 return(KERN_INVALID_ARGUMENT);
695
696 error = vm_map_copyin(map, (vm_map_address_t)address,
697 (vm_map_size_t)size, FALSE, &copy);
698
699 if (KERN_SUCCESS == error) {
700 error = vm_map_copy_overwrite(current_thread()->map,
701 (vm_map_address_t)data,
702 copy, FALSE);
703 if (KERN_SUCCESS == error) {
704 *data_size = size;
705 return error;
706 }
707 vm_map_copy_discard(copy);
708 }
709 return(error);
710 }
711
712
713 /*
714 * mach_vm_write -
715 * Overwrite the specified address range with the data provided
716 * (from the current map).
717 */
718 kern_return_t
719 mach_vm_write(
720 vm_map_t map,
721 mach_vm_address_t address,
722 pointer_t data,
723 __unused mach_msg_type_number_t size)
724 {
725 if (map == VM_MAP_NULL)
726 return KERN_INVALID_ARGUMENT;
727
728 return vm_map_copy_overwrite(map, (vm_map_address_t)address,
729 (vm_map_copy_t) data, FALSE /* interruptible XXX */);
730 }
731
732 /*
733 * vm_write -
734 * Overwrite the specified address range with the data provided
735 * (from the current map).
736 *
737 * The addressability of the range of addresses to overwrite is
738 * limited bu the use of a vm_address_t (same size as kernel map).
739 * Either the target map is also small, or the range is in the
740 * low addresses within it.
741 */
742 kern_return_t
743 vm_write(
744 vm_map_t map,
745 vm_address_t address,
746 pointer_t data,
747 __unused mach_msg_type_number_t size)
748 {
749 if (map == VM_MAP_NULL)
750 return KERN_INVALID_ARGUMENT;
751
752 return vm_map_copy_overwrite(map, (vm_map_address_t)address,
753 (vm_map_copy_t) data, FALSE /* interruptible XXX */);
754 }
755
756 /*
757 * mach_vm_copy -
758 * Overwrite one range of the specified map with the contents of
759 * another range within that same map (i.e. both address ranges
760 * are "over there").
761 */
762 kern_return_t
763 mach_vm_copy(
764 vm_map_t map,
765 mach_vm_address_t source_address,
766 mach_vm_size_t size,
767 mach_vm_address_t dest_address)
768 {
769 vm_map_copy_t copy;
770 kern_return_t kr;
771
772 if (map == VM_MAP_NULL)
773 return KERN_INVALID_ARGUMENT;
774
775 kr = vm_map_copyin(map, (vm_map_address_t)source_address,
776 (vm_map_size_t)size, FALSE, &copy);
777
778 if (KERN_SUCCESS == kr) {
779 kr = vm_map_copy_overwrite(map,
780 (vm_map_address_t)dest_address,
781 copy, FALSE /* interruptible XXX */);
782
783 if (KERN_SUCCESS != kr)
784 vm_map_copy_discard(copy);
785 }
786 return kr;
787 }
788
789 kern_return_t
790 vm_copy(
791 vm_map_t map,
792 vm_address_t source_address,
793 vm_size_t size,
794 vm_address_t dest_address)
795 {
796 vm_map_copy_t copy;
797 kern_return_t kr;
798
799 if (map == VM_MAP_NULL)
800 return KERN_INVALID_ARGUMENT;
801
802 kr = vm_map_copyin(map, (vm_map_address_t)source_address,
803 (vm_map_size_t)size, FALSE, &copy);
804
805 if (KERN_SUCCESS == kr) {
806 kr = vm_map_copy_overwrite(map,
807 (vm_map_address_t)dest_address,
808 copy, FALSE /* interruptible XXX */);
809
810 if (KERN_SUCCESS != kr)
811 vm_map_copy_discard(copy);
812 }
813 return kr;
814 }
815
816 /*
817 * mach_vm_map -
818 * Map some range of an object into an address space.
819 *
820 * The object can be one of several types of objects:
821 * NULL - anonymous memory
822 * a named entry - a range within another address space
823 * or a range within a memory object
824 * a whole memory object
825 *
826 */
827 kern_return_t
828 mach_vm_map(
829 vm_map_t target_map,
830 mach_vm_offset_t *address,
831 mach_vm_size_t initial_size,
832 mach_vm_offset_t mask,
833 int flags,
834 ipc_port_t port,
835 vm_object_offset_t offset,
836 boolean_t copy,
837 vm_prot_t cur_protection,
838 vm_prot_t max_protection,
839 vm_inherit_t inheritance)
840 {
841 vm_map_address_t map_addr;
842 vm_map_size_t map_size;
843 vm_object_t object;
844 vm_object_size_t size;
845 kern_return_t result;
846
847 /*
848 * Check arguments for validity
849 */
850 if ((target_map == VM_MAP_NULL) ||
851 (cur_protection & ~VM_PROT_ALL) ||
852 (max_protection & ~VM_PROT_ALL) ||
853 (inheritance > VM_INHERIT_LAST_VALID) ||
854 initial_size == 0)
855 return(KERN_INVALID_ARGUMENT);
856
857 map_addr = vm_map_trunc_page(*address);
858 map_size = vm_map_round_page(initial_size);
859 size = vm_object_round_page(initial_size);
860
861 /*
862 * Find the vm object (if any) corresponding to this port.
863 */
864 if (!IP_VALID(port)) {
865 object = VM_OBJECT_NULL;
866 offset = 0;
867 copy = FALSE;
868 } else if (ip_kotype(port) == IKOT_NAMED_ENTRY) {
869 vm_named_entry_t named_entry;
870
871 named_entry = (vm_named_entry_t)port->ip_kobject;
872 /* a few checks to make sure user is obeying rules */
873 if(size == 0) {
874 if(offset >= named_entry->size)
875 return(KERN_INVALID_RIGHT);
876 size = named_entry->size - offset;
877 }
878 if((named_entry->protection & max_protection) != max_protection)
879 return(KERN_INVALID_RIGHT);
880 if((named_entry->protection & cur_protection) != cur_protection)
881 return(KERN_INVALID_RIGHT);
882 if(named_entry->size < (offset + size))
883 return(KERN_INVALID_ARGUMENT);
884
885 /* the callers parameter offset is defined to be the */
886 /* offset from beginning of named entry offset in object */
887 offset = offset + named_entry->offset;
888
889 named_entry_lock(named_entry);
890 if(named_entry->is_sub_map) {
891 vm_map_entry_t map_entry;
892
893 named_entry_unlock(named_entry);
894 vm_object_reference(vm_submap_object);
895 if ((result = vm_map_enter(target_map,
896 &map_addr, map_size,
897 (vm_map_offset_t)mask, flags,
898 vm_submap_object, 0,
899 FALSE,
900 cur_protection, max_protection, inheritance
901 )) != KERN_SUCCESS) {
902 vm_object_deallocate(vm_submap_object);
903 } else {
904 char alias;
905
906 VM_GET_FLAGS_ALIAS(flags, alias);
907 if ((alias == VM_MEMORY_SHARED_PMAP) &&
908 !copy) {
909 vm_map_submap(target_map, map_addr,
910 map_addr + map_size,
911 named_entry->backing.map,
912 (vm_map_offset_t)offset, TRUE);
913 } else {
914 vm_map_submap(target_map, map_addr,
915 map_addr + map_size,
916 named_entry->backing.map,
917 (vm_map_offset_t)offset, FALSE);
918 }
919 if(copy) {
920 if(vm_map_lookup_entry(
921 target_map, map_addr, &map_entry)) {
922 map_entry->needs_copy = TRUE;
923 }
924 }
925 *address = map_addr;
926 }
927 return(result);
928
929 } else if (named_entry->is_pager) {
930 unsigned int access;
931 vm_prot_t protections;
932 unsigned int wimg_mode;
933 boolean_t cache_attr;
934
935 protections = named_entry->protection
936 & VM_PROT_ALL;
937 access = GET_MAP_MEM(named_entry->protection);
938
939 object = vm_object_enter(
940 named_entry->backing.pager,
941 named_entry->size,
942 named_entry->internal,
943 FALSE,
944 FALSE);
945 if (object == VM_OBJECT_NULL) {
946 named_entry_unlock(named_entry);
947 return(KERN_INVALID_OBJECT);
948 }
949
950 /* JMM - drop reference on pager here */
951
952 /* create an extra ref for the named entry */
953 vm_object_lock(object);
954 vm_object_reference_locked(object);
955 named_entry->backing.object = object;
956 named_entry->is_pager = FALSE;
957 named_entry_unlock(named_entry);
958
959 wimg_mode = object->wimg_bits;
960 if(access == MAP_MEM_IO) {
961 wimg_mode = VM_WIMG_IO;
962 } else if (access == MAP_MEM_COPYBACK) {
963 wimg_mode = VM_WIMG_USE_DEFAULT;
964 } else if (access == MAP_MEM_WTHRU) {
965 wimg_mode = VM_WIMG_WTHRU;
966 } else if (access == MAP_MEM_WCOMB) {
967 wimg_mode = VM_WIMG_WCOMB;
968 }
969 if ((wimg_mode == VM_WIMG_IO)
970 || (wimg_mode == VM_WIMG_WCOMB))
971 cache_attr = TRUE;
972 else
973 cache_attr = FALSE;
974
975 /* wait for object (if any) to be ready */
976 if (!named_entry->internal) {
977 while (!object->pager_ready) {
978 vm_object_wait(object,
979 VM_OBJECT_EVENT_PAGER_READY,
980 THREAD_UNINT);
981 vm_object_lock(object);
982 }
983 }
984
985 if(object->wimg_bits != wimg_mode) {
986 vm_page_t p;
987
988 vm_object_paging_wait(object, THREAD_UNINT);
989
990 object->wimg_bits = wimg_mode;
991 queue_iterate(&object->memq, p, vm_page_t, listq) {
992 if (!p->fictitious) {
993 pmap_disconnect(p->phys_page);
994 if (cache_attr)
995 pmap_sync_page_attributes_phys(p->phys_page);
996 }
997 }
998 }
999 object->true_share = TRUE;
1000 if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC)
1001 object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
1002 vm_object_unlock(object);
1003 } else {
1004 /* This is the case where we are going to map */
1005 /* an already mapped object. If the object is */
1006 /* not ready it is internal. An external */
1007 /* object cannot be mapped until it is ready */
1008 /* we can therefore avoid the ready check */
1009 /* in this case. */
1010 object = named_entry->backing.object;
1011 assert(object != VM_OBJECT_NULL);
1012 named_entry_unlock(named_entry);
1013 vm_object_reference(object);
1014 }
1015 } else if (ip_kotype(port) == IKOT_MEMORY_OBJECT) {
1016 /*
1017 * JMM - This is temporary until we unify named entries
1018 * and raw memory objects.
1019 *
1020 * Detected fake ip_kotype for a memory object. In
1021 * this case, the port isn't really a port at all, but
1022 * instead is just a raw memory object.
1023 */
1024
1025 if ((object = vm_object_enter((memory_object_t)port,
1026 size, FALSE, FALSE, FALSE))
1027 == VM_OBJECT_NULL)
1028 return(KERN_INVALID_OBJECT);
1029
1030 /* wait for object (if any) to be ready */
1031 if (object != VM_OBJECT_NULL) {
1032 if(object == kernel_object) {
1033 printf("Warning: Attempt to map kernel object"
1034 " by a non-private kernel entity\n");
1035 return(KERN_INVALID_OBJECT);
1036 }
1037 vm_object_lock(object);
1038 while (!object->pager_ready) {
1039 vm_object_wait(object,
1040 VM_OBJECT_EVENT_PAGER_READY,
1041 THREAD_UNINT);
1042 vm_object_lock(object);
1043 }
1044 vm_object_unlock(object);
1045 }
1046 } else {
1047 return (KERN_INVALID_OBJECT);
1048 }
1049
1050 /*
1051 * Perform the copy if requested
1052 */
1053
1054 if (copy) {
1055 vm_object_t new_object;
1056 vm_object_offset_t new_offset;
1057
1058 result = vm_object_copy_strategically(object, offset, size,
1059 &new_object, &new_offset,
1060 &copy);
1061
1062
1063 if (result == KERN_MEMORY_RESTART_COPY) {
1064 boolean_t success;
1065 boolean_t src_needs_copy;
1066
1067 /*
1068 * XXX
1069 * We currently ignore src_needs_copy.
1070 * This really is the issue of how to make
1071 * MEMORY_OBJECT_COPY_SYMMETRIC safe for
1072 * non-kernel users to use. Solution forthcoming.
1073 * In the meantime, since we don't allow non-kernel
1074 * memory managers to specify symmetric copy,
1075 * we won't run into problems here.
1076 */
1077 new_object = object;
1078 new_offset = offset;
1079 success = vm_object_copy_quickly(&new_object,
1080 new_offset, size,
1081 &src_needs_copy,
1082 &copy);
1083 assert(success);
1084 result = KERN_SUCCESS;
1085 }
1086 /*
1087 * Throw away the reference to the
1088 * original object, as it won't be mapped.
1089 */
1090
1091 vm_object_deallocate(object);
1092
1093 if (result != KERN_SUCCESS)
1094 return (result);
1095
1096 object = new_object;
1097 offset = new_offset;
1098 }
1099
1100 if ((result = vm_map_enter(target_map,
1101 &map_addr, map_size,
1102 (vm_map_offset_t)mask,
1103 flags,
1104 object, offset,
1105 copy,
1106 cur_protection, max_protection, inheritance
1107 )) != KERN_SUCCESS)
1108 vm_object_deallocate(object);
1109 *address = map_addr;
1110 return(result);
1111 }
1112
1113
1114 /* legacy interface */
1115 kern_return_t
1116 vm_map_64(
1117 vm_map_t target_map,
1118 vm_offset_t *address,
1119 vm_size_t size,
1120 vm_offset_t mask,
1121 int flags,
1122 ipc_port_t port,
1123 vm_object_offset_t offset,
1124 boolean_t copy,
1125 vm_prot_t cur_protection,
1126 vm_prot_t max_protection,
1127 vm_inherit_t inheritance)
1128 {
1129 mach_vm_address_t map_addr;
1130 mach_vm_size_t map_size;
1131 mach_vm_offset_t map_mask;
1132 kern_return_t kr;
1133
1134 map_addr = (mach_vm_address_t)*address;
1135 map_size = (mach_vm_size_t)size;
1136 map_mask = (mach_vm_offset_t)mask;
1137
1138 kr = mach_vm_map(target_map, &map_addr, map_size, map_mask, flags,
1139 port, offset, copy,
1140 cur_protection, max_protection, inheritance);
1141 *address = CAST_DOWN(vm_address_t, map_addr);
1142 return kr;
1143 }
1144
1145 /* temporary, until world build */
1146 kern_return_t
1147 vm_map(
1148 vm_map_t target_map,
1149 vm_offset_t *address,
1150 vm_size_t size,
1151 vm_offset_t mask,
1152 int flags,
1153 ipc_port_t port,
1154 vm_offset_t offset,
1155 boolean_t copy,
1156 vm_prot_t cur_protection,
1157 vm_prot_t max_protection,
1158 vm_inherit_t inheritance)
1159 {
1160 mach_vm_address_t map_addr;
1161 mach_vm_size_t map_size;
1162 mach_vm_offset_t map_mask;
1163 vm_object_offset_t obj_offset;
1164 kern_return_t kr;
1165
1166 map_addr = (mach_vm_address_t)*address;
1167 map_size = (mach_vm_size_t)size;
1168 map_mask = (mach_vm_offset_t)mask;
1169 obj_offset = (vm_object_offset_t)offset;
1170
1171 kr = mach_vm_map(target_map, &map_addr, map_size, map_mask, flags,
1172 port, obj_offset, copy,
1173 cur_protection, max_protection, inheritance);
1174 *address = CAST_DOWN(vm_address_t, map_addr);
1175 return kr;
1176 }
1177
1178 /*
1179 * mach_vm_remap -
1180 * Remap a range of memory from one task into another,
1181 * to another address range within the same task, or
1182 * over top of itself (with altered permissions and/or
1183 * as an in-place copy of itself).
1184 */
1185
1186 kern_return_t
1187 mach_vm_remap(
1188 vm_map_t target_map,
1189 mach_vm_offset_t *address,
1190 mach_vm_size_t size,
1191 mach_vm_offset_t mask,
1192 boolean_t anywhere,
1193 vm_map_t src_map,
1194 mach_vm_offset_t memory_address,
1195 boolean_t copy,
1196 vm_prot_t *cur_protection,
1197 vm_prot_t *max_protection,
1198 vm_inherit_t inheritance)
1199 {
1200 vm_map_offset_t map_addr;
1201 kern_return_t kr;
1202
1203 if (VM_MAP_NULL == target_map || VM_MAP_NULL == src_map)
1204 return KERN_INVALID_ARGUMENT;
1205
1206 map_addr = (vm_map_offset_t)*address;
1207
1208 kr = vm_map_remap(target_map,
1209 &map_addr,
1210 size,
1211 mask,
1212 anywhere,
1213 src_map,
1214 memory_address,
1215 copy,
1216 cur_protection,
1217 max_protection,
1218 inheritance);
1219 *address = map_addr;
1220 return kr;
1221 }
1222
1223 /*
1224 * vm_remap -
1225 * Remap a range of memory from one task into another,
1226 * to another address range within the same task, or
1227 * over top of itself (with altered permissions and/or
1228 * as an in-place copy of itself).
1229 *
1230 * The addressability of the source and target address
1231 * range is limited by the size of vm_address_t (in the
1232 * kernel context).
1233 */
1234 kern_return_t
1235 vm_remap(
1236 vm_map_t target_map,
1237 vm_offset_t *address,
1238 vm_size_t size,
1239 vm_offset_t mask,
1240 boolean_t anywhere,
1241 vm_map_t src_map,
1242 vm_offset_t memory_address,
1243 boolean_t copy,
1244 vm_prot_t *cur_protection,
1245 vm_prot_t *max_protection,
1246 vm_inherit_t inheritance)
1247 {
1248 vm_map_offset_t map_addr;
1249 kern_return_t kr;
1250
1251 if (VM_MAP_NULL == target_map || VM_MAP_NULL == src_map)
1252 return KERN_INVALID_ARGUMENT;
1253
1254 map_addr = (vm_map_offset_t)*address;
1255
1256 kr = vm_map_remap(target_map,
1257 &map_addr,
1258 size,
1259 mask,
1260 anywhere,
1261 src_map,
1262 memory_address,
1263 copy,
1264 cur_protection,
1265 max_protection,
1266 inheritance);
1267 *address = CAST_DOWN(vm_offset_t, map_addr);
1268 return kr;
1269 }
1270
1271 /*
1272 * NOTE: these routine (and this file) will no longer require mach_host_server.h
1273 * when mach_vm_wire and vm_wire are changed to use ledgers.
1274 */
1275 #include <mach/mach_host_server.h>
1276 /*
1277 * mach_vm_wire
1278 * Specify that the range of the virtual address space
1279 * of the target task must not cause page faults for
1280 * the indicated accesses.
1281 *
1282 * [ To unwire the pages, specify VM_PROT_NONE. ]
1283 */
1284 kern_return_t
1285 mach_vm_wire(
1286 host_priv_t host_priv,
1287 vm_map_t map,
1288 mach_vm_offset_t start,
1289 mach_vm_size_t size,
1290 vm_prot_t access)
1291 {
1292 kern_return_t rc;
1293
1294 if (host_priv == HOST_PRIV_NULL)
1295 return KERN_INVALID_HOST;
1296
1297 assert(host_priv == &realhost);
1298
1299 if (map == VM_MAP_NULL)
1300 return KERN_INVALID_TASK;
1301
1302 if (access & ~VM_PROT_ALL)
1303 return KERN_INVALID_ARGUMENT;
1304
1305 if (access != VM_PROT_NONE) {
1306 rc = vm_map_wire(map, vm_map_trunc_page(start),
1307 vm_map_round_page(start+size), access, TRUE);
1308 } else {
1309 rc = vm_map_unwire(map, vm_map_trunc_page(start),
1310 vm_map_round_page(start+size), TRUE);
1311 }
1312 return rc;
1313 }
1314
1315 /*
1316 * vm_wire -
1317 * Specify that the range of the virtual address space
1318 * of the target task must not cause page faults for
1319 * the indicated accesses.
1320 *
1321 * [ To unwire the pages, specify VM_PROT_NONE. ]
1322 */
1323 kern_return_t
1324 vm_wire(
1325 host_priv_t host_priv,
1326 register vm_map_t map,
1327 vm_offset_t start,
1328 vm_size_t size,
1329 vm_prot_t access)
1330 {
1331 kern_return_t rc;
1332
1333 if (host_priv == HOST_PRIV_NULL)
1334 return KERN_INVALID_HOST;
1335
1336 assert(host_priv == &realhost);
1337
1338 if (map == VM_MAP_NULL)
1339 return KERN_INVALID_TASK;
1340
1341 if ((access & ~VM_PROT_ALL) || (start + size < start))
1342 return KERN_INVALID_ARGUMENT;
1343
1344 if (size == 0) {
1345 rc = KERN_SUCCESS;
1346 } else if (access != VM_PROT_NONE) {
1347 rc = vm_map_wire(map, vm_map_trunc_page(start),
1348 vm_map_round_page(start+size), access, TRUE);
1349 } else {
1350 rc = vm_map_unwire(map, vm_map_trunc_page(start),
1351 vm_map_round_page(start+size), TRUE);
1352 }
1353 return rc;
1354 }
1355
1356 /*
1357 * vm_msync
1358 *
1359 * Synchronises the memory range specified with its backing store
1360 * image by either flushing or cleaning the contents to the appropriate
1361 * memory manager.
1362 *
1363 * interpretation of sync_flags
1364 * VM_SYNC_INVALIDATE - discard pages, only return precious
1365 * pages to manager.
1366 *
1367 * VM_SYNC_INVALIDATE & (VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS)
1368 * - discard pages, write dirty or precious
1369 * pages back to memory manager.
1370 *
1371 * VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS
1372 * - write dirty or precious pages back to
1373 * the memory manager.
1374 *
1375 * VM_SYNC_CONTIGUOUS - does everything normally, but if there
1376 * is a hole in the region, and we would
1377 * have returned KERN_SUCCESS, return
1378 * KERN_INVALID_ADDRESS instead.
1379 *
1380 * RETURNS
1381 * KERN_INVALID_TASK Bad task parameter
1382 * KERN_INVALID_ARGUMENT both sync and async were specified.
1383 * KERN_SUCCESS The usual.
1384 * KERN_INVALID_ADDRESS There was a hole in the region.
1385 */
1386
1387 kern_return_t
1388 mach_vm_msync(
1389 vm_map_t map,
1390 mach_vm_address_t address,
1391 mach_vm_size_t size,
1392 vm_sync_t sync_flags)
1393 {
1394
1395 if (map == VM_MAP_NULL)
1396 return(KERN_INVALID_TASK);
1397
1398 return vm_map_msync(map, (vm_map_address_t)address,
1399 (vm_map_size_t)size, sync_flags);
1400 }
1401
1402 /*
1403 * vm_msync
1404 *
1405 * Synchronises the memory range specified with its backing store
1406 * image by either flushing or cleaning the contents to the appropriate
1407 * memory manager.
1408 *
1409 * interpretation of sync_flags
1410 * VM_SYNC_INVALIDATE - discard pages, only return precious
1411 * pages to manager.
1412 *
1413 * VM_SYNC_INVALIDATE & (VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS)
1414 * - discard pages, write dirty or precious
1415 * pages back to memory manager.
1416 *
1417 * VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS
1418 * - write dirty or precious pages back to
1419 * the memory manager.
1420 *
1421 * VM_SYNC_CONTIGUOUS - does everything normally, but if there
1422 * is a hole in the region, and we would
1423 * have returned KERN_SUCCESS, return
1424 * KERN_INVALID_ADDRESS instead.
1425 *
1426 * The addressability of the range is limited to that which can
1427 * be described by a vm_address_t.
1428 *
1429 * RETURNS
1430 * KERN_INVALID_TASK Bad task parameter
1431 * KERN_INVALID_ARGUMENT both sync and async were specified.
1432 * KERN_SUCCESS The usual.
1433 * KERN_INVALID_ADDRESS There was a hole in the region.
1434 */
1435
1436 kern_return_t
1437 vm_msync(
1438 vm_map_t map,
1439 vm_address_t address,
1440 vm_size_t size,
1441 vm_sync_t sync_flags)
1442 {
1443
1444 if (map == VM_MAP_NULL)
1445 return(KERN_INVALID_TASK);
1446
1447 return vm_map_msync(map, (vm_map_address_t)address,
1448 (vm_map_size_t)size, sync_flags);
1449 }
1450
1451
1452 /*
1453 * mach_vm_behavior_set
1454 *
1455 * Sets the paging behavior attribute for the specified range
1456 * in the specified map.
1457 *
1458 * This routine will fail with KERN_INVALID_ADDRESS if any address
1459 * in [start,start+size) is not a valid allocated memory region.
1460 */
1461 kern_return_t
1462 mach_vm_behavior_set(
1463 vm_map_t map,
1464 mach_vm_offset_t start,
1465 mach_vm_size_t size,
1466 vm_behavior_t new_behavior)
1467 {
1468 if ((map == VM_MAP_NULL) || (start + size < start))
1469 return(KERN_INVALID_ARGUMENT);
1470
1471 if (size == 0)
1472 return KERN_SUCCESS;
1473
1474 return(vm_map_behavior_set(map, vm_map_trunc_page(start),
1475 vm_map_round_page(start+size), new_behavior));
1476 }
1477
1478 /*
1479 * vm_behavior_set
1480 *
1481 * Sets the paging behavior attribute for the specified range
1482 * in the specified map.
1483 *
1484 * This routine will fail with KERN_INVALID_ADDRESS if any address
1485 * in [start,start+size) is not a valid allocated memory region.
1486 *
1487 * This routine is potentially limited in addressibility by the
1488 * use of vm_offset_t (if the map provided is larger than the
1489 * kernel's).
1490 */
1491 kern_return_t
1492 vm_behavior_set(
1493 vm_map_t map,
1494 vm_offset_t start,
1495 vm_size_t size,
1496 vm_behavior_t new_behavior)
1497 {
1498 if ((map == VM_MAP_NULL) || (start + size < start))
1499 return(KERN_INVALID_ARGUMENT);
1500
1501 if (size == 0)
1502 return KERN_SUCCESS;
1503
1504 return(vm_map_behavior_set(map, vm_map_trunc_page(start),
1505 vm_map_round_page(start+size), new_behavior));
1506 }
1507
1508 /*
1509 * mach_vm_region:
1510 *
1511 * User call to obtain information about a region in
1512 * a task's address map. Currently, only one flavor is
1513 * supported.
1514 *
1515 * XXX The reserved and behavior fields cannot be filled
1516 * in until the vm merge from the IK is completed, and
1517 * vm_reserve is implemented.
1518 *
1519 * XXX Dependency: syscall_vm_region() also supports only one flavor.
1520 */
1521
1522 kern_return_t
1523 mach_vm_region(
1524 vm_map_t map,
1525 mach_vm_offset_t *address, /* IN/OUT */
1526 mach_vm_size_t *size, /* OUT */
1527 vm_region_flavor_t flavor, /* IN */
1528 vm_region_info_t info, /* OUT */
1529 mach_msg_type_number_t *count, /* IN/OUT */
1530 mach_port_t *object_name) /* OUT */
1531 {
1532 vm_map_offset_t map_addr;
1533 vm_map_size_t map_size;
1534 kern_return_t kr;
1535
1536 if (VM_MAP_NULL == map)
1537 return KERN_INVALID_ARGUMENT;
1538
1539 map_addr = (vm_map_offset_t)*address;
1540 map_size = (vm_map_size_t)*size;
1541
1542 /* legacy conversion */
1543 if (VM_REGION_BASIC_INFO == flavor)
1544 flavor = VM_REGION_BASIC_INFO_64;
1545
1546 kr = vm_map_region(map,
1547 &map_addr, &map_size,
1548 flavor, info, count,
1549 object_name);
1550
1551 *address = map_addr;
1552 *size = map_size;
1553 return kr;
1554 }
1555
1556 /*
1557 * vm_region_64 and vm_region:
1558 *
1559 * User call to obtain information about a region in
1560 * a task's address map. Currently, only one flavor is
1561 * supported.
1562 *
1563 * XXX The reserved and behavior fields cannot be filled
1564 * in until the vm merge from the IK is completed, and
1565 * vm_reserve is implemented.
1566 *
1567 * XXX Dependency: syscall_vm_region() also supports only one flavor.
1568 */
1569
1570 kern_return_t
1571 vm_region_64(
1572 vm_map_t map,
1573 vm_offset_t *address, /* IN/OUT */
1574 vm_size_t *size, /* OUT */
1575 vm_region_flavor_t flavor, /* IN */
1576 vm_region_info_t info, /* OUT */
1577 mach_msg_type_number_t *count, /* IN/OUT */
1578 mach_port_t *object_name) /* OUT */
1579 {
1580 vm_map_offset_t map_addr;
1581 vm_map_size_t map_size;
1582 kern_return_t kr;
1583
1584 if (VM_MAP_NULL == map)
1585 return KERN_INVALID_ARGUMENT;
1586
1587 map_addr = (vm_map_offset_t)*address;
1588 map_size = (vm_map_size_t)*size;
1589
1590 /* legacy conversion */
1591 if (VM_REGION_BASIC_INFO == flavor)
1592 flavor = VM_REGION_BASIC_INFO_64;
1593
1594 kr = vm_map_region(map,
1595 &map_addr, &map_size,
1596 flavor, info, count,
1597 object_name);
1598
1599 *address = CAST_DOWN(vm_offset_t, map_addr);
1600 *size = CAST_DOWN(vm_size_t, map_size);
1601
1602 if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS)
1603 return KERN_INVALID_ADDRESS;
1604 return kr;
1605 }
1606
1607 kern_return_t
1608 vm_region(
1609 vm_map_t map,
1610 vm_address_t *address, /* IN/OUT */
1611 vm_size_t *size, /* OUT */
1612 vm_region_flavor_t flavor, /* IN */
1613 vm_region_info_t info, /* OUT */
1614 mach_msg_type_number_t *count, /* IN/OUT */
1615 mach_port_t *object_name) /* OUT */
1616 {
1617 vm_map_address_t map_addr;
1618 vm_map_size_t map_size;
1619 kern_return_t kr;
1620
1621 if (VM_MAP_NULL == map)
1622 return KERN_INVALID_ARGUMENT;
1623
1624 map_addr = (vm_map_address_t)*address;
1625 map_size = (vm_map_size_t)*size;
1626
1627 kr = vm_map_region(map,
1628 &map_addr, &map_size,
1629 flavor, info, count,
1630 object_name);
1631
1632 *address = CAST_DOWN(vm_address_t, map_addr);
1633 *size = CAST_DOWN(vm_size_t, map_size);
1634
1635 if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS)
1636 return KERN_INVALID_ADDRESS;
1637 return kr;
1638 }
1639
1640 /*
1641 * vm_region_recurse: A form of vm_region which follows the
1642 * submaps in a target map
1643 *
1644 */
1645 kern_return_t
1646 mach_vm_region_recurse(
1647 vm_map_t map,
1648 mach_vm_address_t *address,
1649 mach_vm_size_t *size,
1650 uint32_t *depth,
1651 vm_region_recurse_info_t info,
1652 mach_msg_type_number_t *infoCnt)
1653 {
1654 vm_map_address_t map_addr;
1655 vm_map_size_t map_size;
1656 kern_return_t kr;
1657
1658 if (VM_MAP_NULL == map)
1659 return KERN_INVALID_ARGUMENT;
1660
1661 map_addr = (vm_map_address_t)*address;
1662 map_size = (vm_map_size_t)*size;
1663
1664 kr = vm_map_region_recurse_64(
1665 map,
1666 &map_addr,
1667 &map_size,
1668 depth,
1669 (vm_region_submap_info_64_t)info,
1670 infoCnt);
1671
1672 *address = map_addr;
1673 *size = map_size;
1674 return kr;
1675 }
1676
1677 /*
1678 * vm_region_recurse: A form of vm_region which follows the
1679 * submaps in a target map
1680 *
1681 */
1682 kern_return_t
1683 vm_region_recurse_64(
1684 vm_map_t map,
1685 vm_address_t *address,
1686 vm_size_t *size,
1687 uint32_t *depth,
1688 vm_region_recurse_info_64_t info,
1689 mach_msg_type_number_t *infoCnt)
1690 {
1691 vm_map_address_t map_addr;
1692 vm_map_size_t map_size;
1693 kern_return_t kr;
1694
1695 if (VM_MAP_NULL == map)
1696 return KERN_INVALID_ARGUMENT;
1697
1698 map_addr = (vm_map_address_t)*address;
1699 map_size = (vm_map_size_t)*size;
1700
1701 kr = vm_map_region_recurse_64(
1702 map,
1703 &map_addr,
1704 &map_size,
1705 depth,
1706 (vm_region_submap_info_64_t)info,
1707 infoCnt);
1708
1709 *address = CAST_DOWN(vm_address_t, map_addr);
1710 *size = CAST_DOWN(vm_size_t, map_size);
1711
1712 if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS)
1713 return KERN_INVALID_ADDRESS;
1714 return kr;
1715 }
1716
1717 kern_return_t
1718 vm_region_recurse(
1719 vm_map_t map,
1720 vm_offset_t *address, /* IN/OUT */
1721 vm_size_t *size, /* OUT */
1722 natural_t *depth, /* IN/OUT */
1723 vm_region_recurse_info_t info32, /* IN/OUT */
1724 mach_msg_type_number_t *infoCnt) /* IN/OUT */
1725 {
1726 vm_region_submap_info_data_64_t info64;
1727 vm_region_submap_info_t info;
1728 vm_map_address_t map_addr;
1729 vm_map_size_t map_size;
1730 kern_return_t kr;
1731
1732 if (VM_MAP_NULL == map || *infoCnt < VM_REGION_SUBMAP_INFO_COUNT)
1733 return KERN_INVALID_ARGUMENT;
1734
1735
1736 map_addr = (vm_map_address_t)*address;
1737 map_size = (vm_map_size_t)*size;
1738 info = (vm_region_submap_info_t)info32;
1739 *infoCnt = VM_REGION_SUBMAP_INFO_COUNT_64;
1740
1741 kr = vm_map_region_recurse_64(map, &map_addr,&map_size,
1742 depth, &info64, infoCnt);
1743
1744 info->protection = info64.protection;
1745 info->max_protection = info64.max_protection;
1746 info->inheritance = info64.inheritance;
1747 info->offset = (uint32_t)info64.offset; /* trouble-maker */
1748 info->user_tag = info64.user_tag;
1749 info->pages_resident = info64.pages_resident;
1750 info->pages_shared_now_private = info64.pages_shared_now_private;
1751 info->pages_swapped_out = info64.pages_swapped_out;
1752 info->pages_dirtied = info64.pages_dirtied;
1753 info->ref_count = info64.ref_count;
1754 info->shadow_depth = info64.shadow_depth;
1755 info->external_pager = info64.external_pager;
1756 info->share_mode = info64.share_mode;
1757 info->is_submap = info64.is_submap;
1758 info->behavior = info64.behavior;
1759 info->object_id = info64.object_id;
1760 info->user_wired_count = info64.user_wired_count;
1761
1762 *address = CAST_DOWN(vm_address_t, map_addr);
1763 *size = CAST_DOWN(vm_size_t, map_size);
1764 *infoCnt = VM_REGION_SUBMAP_INFO_COUNT;
1765
1766 if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS)
1767 return KERN_INVALID_ADDRESS;
1768 return kr;
1769 }
1770
1771 kern_return_t
1772 vm_purgable_control(
1773 vm_map_t map,
1774 vm_offset_t address,
1775 vm_purgable_t control,
1776 int *state)
1777 {
1778 if (VM_MAP_NULL == map)
1779 return KERN_INVALID_ARGUMENT;
1780
1781 return vm_map_purgable_control(map,
1782 vm_map_trunc_page(address),
1783 control,
1784 state);
1785 }
1786
1787
1788 /*
1789 * Ordinarily, the right to allocate CPM is restricted
1790 * to privileged applications (those that can gain access
1791 * to the host priv port). Set this variable to zero if
1792 * you want to let any application allocate CPM.
1793 */
1794 unsigned int vm_allocate_cpm_privileged = 0;
1795
1796 /*
1797 * Allocate memory in the specified map, with the caveat that
1798 * the memory is physically contiguous. This call may fail
1799 * if the system can't find sufficient contiguous memory.
1800 * This call may cause or lead to heart-stopping amounts of
1801 * paging activity.
1802 *
1803 * Memory obtained from this call should be freed in the
1804 * normal way, viz., via vm_deallocate.
1805 */
1806 kern_return_t
1807 vm_allocate_cpm(
1808 host_priv_t host_priv,
1809 vm_map_t map,
1810 vm_address_t *addr,
1811 vm_size_t size,
1812 int flags)
1813 {
1814 vm_map_address_t map_addr;
1815 vm_map_size_t map_size;
1816 kern_return_t kr;
1817
1818 if (vm_allocate_cpm_privileged && HOST_PRIV_NULL == host_priv)
1819 return KERN_INVALID_HOST;
1820
1821 if (VM_MAP_NULL == map)
1822 return KERN_INVALID_ARGUMENT;
1823
1824 map_addr = (vm_map_address_t)*addr;
1825 map_size = (vm_map_size_t)size;
1826
1827 kr = vm_map_enter_cpm(map,
1828 &map_addr,
1829 map_size,
1830 flags);
1831
1832 *addr = CAST_DOWN(vm_address_t, map_addr);
1833 return kr;
1834 }
1835
1836
1837 kern_return_t
1838 mach_vm_page_query(
1839 vm_map_t map,
1840 mach_vm_offset_t offset,
1841 int *disposition,
1842 int *ref_count)
1843 {
1844 if (VM_MAP_NULL == map)
1845 return KERN_INVALID_ARGUMENT;
1846
1847 return vm_map_page_info(map,
1848 vm_map_trunc_page(offset),
1849 disposition, ref_count);
1850 }
1851
1852 kern_return_t
1853 vm_map_page_query(
1854 vm_map_t map,
1855 vm_offset_t offset,
1856 int *disposition,
1857 int *ref_count)
1858 {
1859 if (VM_MAP_NULL == map)
1860 return KERN_INVALID_ARGUMENT;
1861
1862 return vm_map_page_info(map,
1863 vm_map_trunc_page(offset),
1864 disposition, ref_count);
1865 }
1866
1867 /* map a (whole) upl into an address space */
1868 kern_return_t
1869 vm_upl_map(
1870 vm_map_t map,
1871 upl_t upl,
1872 vm_offset_t *dst_addr)
1873 {
1874 vm_map_offset_t map_addr;
1875 kern_return_t kr;
1876
1877 if (VM_MAP_NULL == map)
1878 return KERN_INVALID_ARGUMENT;
1879
1880 kr = vm_map_enter_upl(map, upl, &map_addr);
1881 *dst_addr = CAST_DOWN(vm_offset_t, map_addr);
1882 return kr;
1883 }
1884
1885 kern_return_t
1886 vm_upl_unmap(
1887 vm_map_t map,
1888 upl_t upl)
1889 {
1890 if (VM_MAP_NULL == map)
1891 return KERN_INVALID_ARGUMENT;
1892
1893 return (vm_map_remove_upl(map, upl));
1894 }
1895
1896 /* Retrieve a upl for an object underlying an address range in a map */
1897
1898 kern_return_t
1899 vm_map_get_upl(
1900 vm_map_t map,
1901 vm_map_offset_t map_offset,
1902 upl_size_t *upl_size,
1903 upl_t *upl,
1904 upl_page_info_array_t page_list,
1905 unsigned int *count,
1906 int *flags,
1907 int force_data_sync)
1908 {
1909 int map_flags;
1910 kern_return_t kr;
1911
1912 if (VM_MAP_NULL == map)
1913 return KERN_INVALID_ARGUMENT;
1914
1915 map_flags = *flags & ~UPL_NOZEROFILL;
1916 if (force_data_sync)
1917 map_flags |= UPL_FORCE_DATA_SYNC;
1918
1919 kr = vm_map_create_upl(map,
1920 map_offset,
1921 upl_size,
1922 upl,
1923 page_list,
1924 count,
1925 &map_flags);
1926
1927 *flags = (map_flags & ~UPL_FORCE_DATA_SYNC);
1928 return kr;
1929 }
1930
1931
1932 __private_extern__ kern_return_t
1933 mach_memory_entry_allocate(
1934 vm_named_entry_t *user_entry_p,
1935 ipc_port_t *user_handle_p); /* forward */
1936
1937 /*
1938 * mach_make_memory_entry_64
1939 *
1940 * Think of it as a two-stage vm_remap() operation. First
1941 * you get a handle. Second, you get map that handle in
1942 * somewhere else. Rather than doing it all at once (and
1943 * without needing access to the other whole map).
1944 */
1945
1946 kern_return_t
1947 mach_make_memory_entry_64(
1948 vm_map_t target_map,
1949 memory_object_size_t *size,
1950 memory_object_offset_t offset,
1951 vm_prot_t permission,
1952 ipc_port_t *object_handle,
1953 ipc_port_t parent_handle)
1954 {
1955 vm_map_version_t version;
1956 vm_named_entry_t parent_entry;
1957 vm_named_entry_t user_entry;
1958 ipc_port_t user_handle;
1959 kern_return_t kr;
1960 vm_map_t real_map;
1961
1962 /* needed for call to vm_map_lookup_locked */
1963 boolean_t wired;
1964 vm_object_offset_t obj_off;
1965 vm_prot_t prot;
1966 vm_map_offset_t lo_offset, hi_offset;
1967 vm_behavior_t behavior;
1968 vm_object_t object;
1969 vm_object_t shadow_object;
1970
1971 /* needed for direct map entry manipulation */
1972 vm_map_entry_t map_entry;
1973 vm_map_entry_t next_entry;
1974 vm_map_t local_map;
1975 vm_map_t original_map = target_map;
1976 vm_map_size_t total_size;
1977 vm_map_size_t map_size;
1978 vm_map_offset_t map_offset;
1979 vm_map_offset_t local_offset;
1980 vm_object_size_t mappable_size;
1981
1982 unsigned int access;
1983 vm_prot_t protections;
1984 unsigned int wimg_mode;
1985 boolean_t cache_attr = FALSE;
1986
1987 if (((permission & 0x00FF0000) &
1988 ~(MAP_MEM_ONLY |
1989 MAP_MEM_NAMED_CREATE |
1990 MAP_MEM_PURGABLE |
1991 MAP_MEM_NAMED_REUSE))) {
1992 /*
1993 * Unknown flag: reject for forward compatibility.
1994 */
1995 return KERN_INVALID_VALUE;
1996 }
1997
1998 if (parent_handle != IP_NULL &&
1999 ip_kotype(parent_handle) == IKOT_NAMED_ENTRY) {
2000 parent_entry = (vm_named_entry_t) parent_handle->ip_kobject;
2001 } else {
2002 parent_entry = NULL;
2003 }
2004
2005 protections = permission & VM_PROT_ALL;
2006 access = GET_MAP_MEM(permission);
2007
2008 user_handle = IP_NULL;
2009 user_entry = NULL;
2010
2011 map_offset = vm_map_trunc_page(offset);
2012 map_size = vm_map_round_page(*size);
2013
2014 if (permission & MAP_MEM_ONLY) {
2015 boolean_t parent_is_object;
2016
2017 if (parent_entry == NULL) {
2018 return KERN_INVALID_ARGUMENT;
2019 }
2020
2021 parent_is_object = !(parent_entry->is_sub_map || parent_entry->is_pager);
2022 object = parent_entry->backing.object;
2023 if(parent_is_object && object != VM_OBJECT_NULL)
2024 wimg_mode = object->wimg_bits;
2025 else
2026 wimg_mode = VM_WIMG_DEFAULT;
2027 if((access != GET_MAP_MEM(parent_entry->protection)) &&
2028 !(parent_entry->protection & VM_PROT_WRITE)) {
2029 return KERN_INVALID_RIGHT;
2030 }
2031 if(access == MAP_MEM_IO) {
2032 SET_MAP_MEM(access, parent_entry->protection);
2033 wimg_mode = VM_WIMG_IO;
2034 } else if (access == MAP_MEM_COPYBACK) {
2035 SET_MAP_MEM(access, parent_entry->protection);
2036 wimg_mode = VM_WIMG_DEFAULT;
2037 } else if (access == MAP_MEM_WTHRU) {
2038 SET_MAP_MEM(access, parent_entry->protection);
2039 wimg_mode = VM_WIMG_WTHRU;
2040 } else if (access == MAP_MEM_WCOMB) {
2041 SET_MAP_MEM(access, parent_entry->protection);
2042 wimg_mode = VM_WIMG_WCOMB;
2043 }
2044 if(parent_is_object && object &&
2045 (access != MAP_MEM_NOOP) &&
2046 (!(object->nophyscache))) {
2047 if(object->wimg_bits != wimg_mode) {
2048 vm_page_t p;
2049 if ((wimg_mode == VM_WIMG_IO)
2050 || (wimg_mode == VM_WIMG_WCOMB))
2051 cache_attr = TRUE;
2052 else
2053 cache_attr = FALSE;
2054 vm_object_lock(object);
2055 vm_object_paging_wait(object, THREAD_UNINT);
2056 object->wimg_bits = wimg_mode;
2057 queue_iterate(&object->memq,
2058 p, vm_page_t, listq) {
2059 if (!p->fictitious) {
2060 pmap_disconnect(p->phys_page);
2061 if (cache_attr)
2062 pmap_sync_page_attributes_phys(p->phys_page);
2063 }
2064 }
2065 vm_object_unlock(object);
2066 }
2067 }
2068 if (object_handle)
2069 *object_handle = IP_NULL;
2070 return KERN_SUCCESS;
2071 }
2072
2073 if(permission & MAP_MEM_NAMED_CREATE) {
2074 kr = mach_memory_entry_allocate(&user_entry, &user_handle);
2075 if (kr != KERN_SUCCESS) {
2076 return KERN_FAILURE;
2077 }
2078
2079 /*
2080 * Force the creation of the VM object now.
2081 */
2082 if (map_size > (vm_map_size_t) VM_MAX_ADDRESS) {
2083 /*
2084 * LP64todo - for now, we can only allocate 4GB
2085 * internal objects because the default pager can't
2086 * page bigger ones. Remove this when it can.
2087 */
2088 kr = KERN_FAILURE;
2089 goto make_mem_done;
2090 }
2091
2092 object = vm_object_allocate(map_size);
2093 assert(object != VM_OBJECT_NULL);
2094
2095 if (permission & MAP_MEM_PURGABLE) {
2096 if (! (permission & VM_PROT_WRITE)) {
2097 /* if we can't write, we can't purge */
2098 vm_object_deallocate(object);
2099 kr = KERN_INVALID_ARGUMENT;
2100 goto make_mem_done;
2101 }
2102 object->purgable = VM_OBJECT_PURGABLE_NONVOLATILE;
2103 }
2104
2105 /*
2106 * The VM object is brand new and nobody else knows about it,
2107 * so we don't need to lock it.
2108 */
2109
2110 wimg_mode = object->wimg_bits;
2111 if (access == MAP_MEM_IO) {
2112 wimg_mode = VM_WIMG_IO;
2113 } else if (access == MAP_MEM_COPYBACK) {
2114 wimg_mode = VM_WIMG_DEFAULT;
2115 } else if (access == MAP_MEM_WTHRU) {
2116 wimg_mode = VM_WIMG_WTHRU;
2117 } else if (access == MAP_MEM_WCOMB) {
2118 wimg_mode = VM_WIMG_WCOMB;
2119 }
2120 if (access != MAP_MEM_NOOP) {
2121 object->wimg_bits = wimg_mode;
2122 }
2123 /* the object has no pages, so no WIMG bits to update here */
2124
2125 /*
2126 * XXX
2127 * We use this path when we want to make sure that
2128 * nobody messes with the object (coalesce, for
2129 * example) before we map it.
2130 * We might want to use these objects for transposition via
2131 * vm_object_transpose() too, so we don't want any copy or
2132 * shadow objects either...
2133 */
2134 object->copy_strategy = MEMORY_OBJECT_COPY_NONE;
2135
2136 user_entry->backing.object = object;
2137 user_entry->internal = TRUE;
2138 user_entry->is_sub_map = FALSE;
2139 user_entry->is_pager = FALSE;
2140 user_entry->offset = 0;
2141 user_entry->protection = protections;
2142 SET_MAP_MEM(access, user_entry->protection);
2143 user_entry->size = map_size;
2144
2145 /* user_object pager and internal fields are not used */
2146 /* when the object field is filled in. */
2147
2148 *size = CAST_DOWN(vm_size_t, map_size);
2149 *object_handle = user_handle;
2150 return KERN_SUCCESS;
2151 }
2152
2153 if (parent_entry == NULL ||
2154 (permission & MAP_MEM_NAMED_REUSE)) {
2155
2156 /* Create a named object based on address range within the task map */
2157 /* Go find the object at given address */
2158
2159 redo_lookup:
2160 vm_map_lock_read(target_map);
2161
2162 /* get the object associated with the target address */
2163 /* note we check the permission of the range against */
2164 /* that requested by the caller */
2165
2166 kr = vm_map_lookup_locked(&target_map, map_offset,
2167 protections, &version,
2168 &object, &obj_off, &prot, &wired, &behavior,
2169 &lo_offset, &hi_offset, &real_map);
2170 if (kr != KERN_SUCCESS) {
2171 vm_map_unlock_read(target_map);
2172 goto make_mem_done;
2173 }
2174 if (((prot & protections) != protections)
2175 || (object == kernel_object)) {
2176 kr = KERN_INVALID_RIGHT;
2177 vm_object_unlock(object);
2178 vm_map_unlock_read(target_map);
2179 if(real_map != target_map)
2180 vm_map_unlock_read(real_map);
2181 if(object == kernel_object) {
2182 printf("Warning: Attempt to create a named"
2183 " entry from the kernel_object\n");
2184 }
2185 goto make_mem_done;
2186 }
2187
2188 /* We have an object, now check to see if this object */
2189 /* is suitable. If not, create a shadow and share that */
2190
2191 /*
2192 * We have to unlock the VM object to avoid deadlocking with
2193 * a VM map lock (the lock ordering is map, the object), if we
2194 * need to modify the VM map to create a shadow object. Since
2195 * we might release the VM map lock below anyway, we have
2196 * to release the VM map lock now.
2197 * XXX FBDP There must be a way to avoid this double lookup...
2198 *
2199 * Take an extra reference on the VM object to make sure it's
2200 * not going to disappear.
2201 */
2202 vm_object_reference_locked(object); /* extra ref to hold obj */
2203 vm_object_unlock(object);
2204
2205 local_map = original_map;
2206 local_offset = map_offset;
2207 if(target_map != local_map) {
2208 vm_map_unlock_read(target_map);
2209 if(real_map != target_map)
2210 vm_map_unlock_read(real_map);
2211 vm_map_lock_read(local_map);
2212 target_map = local_map;
2213 real_map = local_map;
2214 }
2215 while(TRUE) {
2216 if(!vm_map_lookup_entry(local_map,
2217 local_offset, &map_entry)) {
2218 kr = KERN_INVALID_ARGUMENT;
2219 vm_map_unlock_read(target_map);
2220 if(real_map != target_map)
2221 vm_map_unlock_read(real_map);
2222 vm_object_deallocate(object); /* release extra ref */
2223 object = VM_OBJECT_NULL;
2224 goto make_mem_done;
2225 }
2226 if(!(map_entry->is_sub_map)) {
2227 if(map_entry->object.vm_object != object) {
2228 kr = KERN_INVALID_ARGUMENT;
2229 vm_map_unlock_read(target_map);
2230 if(real_map != target_map)
2231 vm_map_unlock_read(real_map);
2232 vm_object_deallocate(object); /* release extra ref */
2233 object = VM_OBJECT_NULL;
2234 goto make_mem_done;
2235 }
2236 break;
2237 } else {
2238 vm_map_t tmap;
2239 tmap = local_map;
2240 local_map = map_entry->object.sub_map;
2241
2242 vm_map_lock_read(local_map);
2243 vm_map_unlock_read(tmap);
2244 target_map = local_map;
2245 real_map = local_map;
2246 local_offset = local_offset - map_entry->vme_start;
2247 local_offset += map_entry->offset;
2248 }
2249 }
2250
2251 /*
2252 * We found the VM map entry, lock the VM object again.
2253 */
2254 vm_object_lock(object);
2255 if(map_entry->wired_count) {
2256 /* JMM - The check below should be reworked instead. */
2257 object->true_share = TRUE;
2258 }
2259 if(((map_entry->max_protection) & protections) != protections) {
2260 kr = KERN_INVALID_RIGHT;
2261 vm_object_unlock(object);
2262 vm_map_unlock_read(target_map);
2263 if(real_map != target_map)
2264 vm_map_unlock_read(real_map);
2265 vm_object_deallocate(object);
2266 object = VM_OBJECT_NULL;
2267 goto make_mem_done;
2268 }
2269
2270 mappable_size = hi_offset - obj_off;
2271 total_size = map_entry->vme_end - map_entry->vme_start;
2272 if(map_size > mappable_size) {
2273 /* try to extend mappable size if the entries */
2274 /* following are from the same object and are */
2275 /* compatible */
2276 next_entry = map_entry->vme_next;
2277 /* lets see if the next map entry is still */
2278 /* pointing at this object and is contiguous */
2279 while(map_size > mappable_size) {
2280 if((next_entry->object.vm_object == object) &&
2281 (next_entry->vme_start ==
2282 next_entry->vme_prev->vme_end) &&
2283 (next_entry->offset ==
2284 next_entry->vme_prev->offset +
2285 (next_entry->vme_prev->vme_end -
2286 next_entry->vme_prev->vme_start))) {
2287 if(((next_entry->max_protection)
2288 & protections) != protections) {
2289 break;
2290 }
2291 if (next_entry->needs_copy !=
2292 map_entry->needs_copy)
2293 break;
2294 mappable_size += next_entry->vme_end
2295 - next_entry->vme_start;
2296 total_size += next_entry->vme_end
2297 - next_entry->vme_start;
2298 next_entry = next_entry->vme_next;
2299 } else {
2300 break;
2301 }
2302
2303 }
2304 }
2305
2306 if(object->internal) {
2307 /* vm_map_lookup_locked will create a shadow if */
2308 /* needs_copy is set but does not check for the */
2309 /* other two conditions shown. It is important to */
2310 /* set up an object which will not be pulled from */
2311 /* under us. */
2312
2313 if ((map_entry->needs_copy || object->shadowed ||
2314 (object->size > total_size))
2315 && !object->true_share) {
2316 /*
2317 * We have to unlock the VM object before
2318 * trying to upgrade the VM map lock, to
2319 * honor lock ordering (map then object).
2320 * Otherwise, we would deadlock if another
2321 * thread holds a read lock on the VM map and
2322 * is trying to acquire the VM object's lock.
2323 * We still hold an extra reference on the
2324 * VM object, guaranteeing that it won't
2325 * disappear.
2326 */
2327 vm_object_unlock(object);
2328
2329 if (vm_map_lock_read_to_write(target_map)) {
2330 /*
2331 * We couldn't upgrade our VM map lock
2332 * from "read" to "write" and we lost
2333 * our "read" lock.
2334 * Start all over again...
2335 */
2336 vm_object_deallocate(object); /* extra ref */
2337 target_map = original_map;
2338 goto redo_lookup;
2339 }
2340 vm_object_lock(object);
2341
2342 /*
2343 * JMM - We need to avoid coming here when the object
2344 * is wired by anybody, not just the current map. Why
2345 * couldn't we use the standard vm_object_copy_quickly()
2346 * approach here?
2347 */
2348
2349 /* create a shadow object */
2350 vm_object_shadow(&map_entry->object.vm_object,
2351 &map_entry->offset, total_size);
2352 shadow_object = map_entry->object.vm_object;
2353 vm_object_unlock(object);
2354
2355 vm_object_pmap_protect(
2356 object, map_entry->offset,
2357 total_size,
2358 ((map_entry->is_shared
2359 || target_map->mapped)
2360 ? PMAP_NULL :
2361 target_map->pmap),
2362 map_entry->vme_start,
2363 map_entry->protection & ~VM_PROT_WRITE);
2364 total_size -= (map_entry->vme_end
2365 - map_entry->vme_start);
2366 next_entry = map_entry->vme_next;
2367 map_entry->needs_copy = FALSE;
2368 while (total_size) {
2369 if(next_entry->object.vm_object == object) {
2370 shadow_object->ref_count++;
2371 vm_object_res_reference(shadow_object);
2372 next_entry->object.vm_object
2373 = shadow_object;
2374 vm_object_deallocate(object);
2375 next_entry->offset
2376 = next_entry->vme_prev->offset +
2377 (next_entry->vme_prev->vme_end
2378 - next_entry->vme_prev->vme_start);
2379 next_entry->needs_copy = FALSE;
2380 } else {
2381 panic("mach_make_memory_entry_64:"
2382 " map entries out of sync\n");
2383 }
2384 total_size -=
2385 next_entry->vme_end
2386 - next_entry->vme_start;
2387 next_entry = next_entry->vme_next;
2388 }
2389
2390 /*
2391 * Transfer our extra reference to the
2392 * shadow object.
2393 */
2394 vm_object_reference_locked(shadow_object);
2395 vm_object_deallocate(object); /* extra ref */
2396 object = shadow_object;
2397
2398 obj_off = (local_offset - map_entry->vme_start)
2399 + map_entry->offset;
2400
2401 vm_map_lock_write_to_read(target_map);
2402 vm_object_lock(object);
2403
2404 }
2405 }
2406
2407 /* note: in the future we can (if necessary) allow for */
2408 /* memory object lists, this will better support */
2409 /* fragmentation, but is it necessary? The user should */
2410 /* be encouraged to create address space oriented */
2411 /* shared objects from CLEAN memory regions which have */
2412 /* a known and defined history. i.e. no inheritence */
2413 /* share, make this call before making the region the */
2414 /* target of ipc's, etc. The code above, protecting */
2415 /* against delayed copy, etc. is mostly defensive. */
2416
2417 wimg_mode = object->wimg_bits;
2418 if(!(object->nophyscache)) {
2419 if(access == MAP_MEM_IO) {
2420 wimg_mode = VM_WIMG_IO;
2421 } else if (access == MAP_MEM_COPYBACK) {
2422 wimg_mode = VM_WIMG_USE_DEFAULT;
2423 } else if (access == MAP_MEM_WTHRU) {
2424 wimg_mode = VM_WIMG_WTHRU;
2425 } else if (access == MAP_MEM_WCOMB) {
2426 wimg_mode = VM_WIMG_WCOMB;
2427 }
2428 }
2429
2430 object->true_share = TRUE;
2431 if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC)
2432 object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
2433
2434 /*
2435 * The memory entry now points to this VM object and we
2436 * need to hold a reference on the VM object. Use the extra
2437 * reference we took earlier to keep the object alive when we
2438 * had to unlock it.
2439 */
2440
2441 vm_map_unlock_read(target_map);
2442 if(real_map != target_map)
2443 vm_map_unlock_read(real_map);
2444
2445 if(object->wimg_bits != wimg_mode) {
2446 vm_page_t p;
2447
2448 vm_object_paging_wait(object, THREAD_UNINT);
2449
2450 if ((wimg_mode == VM_WIMG_IO)
2451 || (wimg_mode == VM_WIMG_WCOMB))
2452 cache_attr = TRUE;
2453 else
2454 cache_attr = FALSE;
2455
2456 queue_iterate(&object->memq,
2457 p, vm_page_t, listq) {
2458 if (!p->fictitious) {
2459 pmap_disconnect(p->phys_page);
2460 if (cache_attr)
2461 pmap_sync_page_attributes_phys(p->phys_page);
2462 }
2463 }
2464 object->wimg_bits = wimg_mode;
2465 }
2466
2467 /* the size of mapped entry that overlaps with our region */
2468 /* which is targeted for share. */
2469 /* (entry_end - entry_start) - */
2470 /* offset of our beg addr within entry */
2471 /* it corresponds to this: */
2472
2473 if(map_size > mappable_size)
2474 map_size = mappable_size;
2475
2476 if (permission & MAP_MEM_NAMED_REUSE) {
2477 /*
2478 * Compare what we got with the "parent_entry".
2479 * If they match, re-use the "parent_entry" instead
2480 * of creating a new one.
2481 */
2482 if (parent_entry != NULL &&
2483 parent_entry->backing.object == object &&
2484 parent_entry->internal == object->internal &&
2485 parent_entry->is_sub_map == FALSE &&
2486 parent_entry->is_pager == FALSE &&
2487 parent_entry->offset == obj_off &&
2488 parent_entry->protection == protections &&
2489 parent_entry->size == map_size) {
2490 /*
2491 * We have a match: re-use "parent_entry".
2492 */
2493 /* release our extra reference on object */
2494 vm_object_unlock(object);
2495 vm_object_deallocate(object);
2496 /* parent_entry->ref_count++; XXX ? */
2497 /* Get an extra send-right on handle */
2498 ipc_port_copy_send(parent_handle);
2499 *object_handle = parent_handle;
2500 return KERN_SUCCESS;
2501 } else {
2502 /*
2503 * No match: we need to create a new entry.
2504 * fall through...
2505 */
2506 }
2507 }
2508
2509 vm_object_unlock(object);
2510 if (mach_memory_entry_allocate(&user_entry, &user_handle)
2511 != KERN_SUCCESS) {
2512 /* release our unused reference on the object */
2513 vm_object_deallocate(object);
2514 return KERN_FAILURE;
2515 }
2516
2517 user_entry->backing.object = object;
2518 user_entry->internal = object->internal;
2519 user_entry->is_sub_map = FALSE;
2520 user_entry->is_pager = FALSE;
2521 user_entry->offset = obj_off;
2522 user_entry->protection = permission;
2523 user_entry->size = map_size;
2524
2525 /* user_object pager and internal fields are not used */
2526 /* when the object field is filled in. */
2527
2528 *size = CAST_DOWN(vm_size_t, map_size);
2529 *object_handle = user_handle;
2530 return KERN_SUCCESS;
2531
2532 } else {
2533 /* The new object will be base on an existing named object */
2534
2535 if (parent_entry == NULL) {
2536 kr = KERN_INVALID_ARGUMENT;
2537 goto make_mem_done;
2538 }
2539 if((offset + map_size) > parent_entry->size) {
2540 kr = KERN_INVALID_ARGUMENT;
2541 goto make_mem_done;
2542 }
2543
2544 if((protections & parent_entry->protection) != protections) {
2545 kr = KERN_PROTECTION_FAILURE;
2546 goto make_mem_done;
2547 }
2548
2549 if (mach_memory_entry_allocate(&user_entry, &user_handle)
2550 != KERN_SUCCESS) {
2551 kr = KERN_FAILURE;
2552 goto make_mem_done;
2553 }
2554
2555 user_entry->size = map_size;
2556 user_entry->offset = parent_entry->offset + map_offset;
2557 user_entry->is_sub_map = parent_entry->is_sub_map;
2558 user_entry->is_pager = parent_entry->is_pager;
2559 user_entry->internal = parent_entry->internal;
2560 user_entry->protection = protections;
2561
2562 if(access != MAP_MEM_NOOP) {
2563 SET_MAP_MEM(access, user_entry->protection);
2564 }
2565
2566 if(parent_entry->is_sub_map) {
2567 user_entry->backing.map = parent_entry->backing.map;
2568 vm_map_lock(user_entry->backing.map);
2569 user_entry->backing.map->ref_count++;
2570 vm_map_unlock(user_entry->backing.map);
2571 }
2572 else if (parent_entry->is_pager) {
2573 user_entry->backing.pager = parent_entry->backing.pager;
2574 /* JMM - don't we need a reference here? */
2575 } else {
2576 object = parent_entry->backing.object;
2577 assert(object != VM_OBJECT_NULL);
2578 user_entry->backing.object = object;
2579 /* we now point to this object, hold on */
2580 vm_object_reference(object);
2581 vm_object_lock(object);
2582 object->true_share = TRUE;
2583 if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC)
2584 object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
2585 vm_object_unlock(object);
2586 }
2587 *size = CAST_DOWN(vm_size_t, map_size);
2588 *object_handle = user_handle;
2589 return KERN_SUCCESS;
2590 }
2591
2592 make_mem_done:
2593 if (user_handle != IP_NULL) {
2594 ipc_port_dealloc_kernel(user_handle);
2595 }
2596 if (user_entry != NULL) {
2597 kfree(user_entry, sizeof *user_entry);
2598 }
2599 return kr;
2600 }
2601
2602 kern_return_t
2603 _mach_make_memory_entry(
2604 vm_map_t target_map,
2605 memory_object_size_t *size,
2606 memory_object_offset_t offset,
2607 vm_prot_t permission,
2608 ipc_port_t *object_handle,
2609 ipc_port_t parent_entry)
2610 {
2611 memory_object_offset_t mo_size;
2612 kern_return_t kr;
2613
2614 mo_size = (memory_object_offset_t)*size;
2615 kr = mach_make_memory_entry_64(target_map, &mo_size,
2616 (memory_object_offset_t)offset, permission, object_handle,
2617 parent_entry);
2618 *size = mo_size;
2619 return kr;
2620 }
2621
2622 kern_return_t
2623 mach_make_memory_entry(
2624 vm_map_t target_map,
2625 vm_size_t *size,
2626 vm_offset_t offset,
2627 vm_prot_t permission,
2628 ipc_port_t *object_handle,
2629 ipc_port_t parent_entry)
2630 {
2631 memory_object_offset_t mo_size;
2632 kern_return_t kr;
2633
2634 mo_size = (memory_object_offset_t)*size;
2635 kr = mach_make_memory_entry_64(target_map, &mo_size,
2636 (memory_object_offset_t)offset, permission, object_handle,
2637 parent_entry);
2638 *size = CAST_DOWN(vm_size_t, mo_size);
2639 return kr;
2640 }
2641
2642 /*
2643 * task_wire
2644 *
2645 * Set or clear the map's wiring_required flag. This flag, if set,
2646 * will cause all future virtual memory allocation to allocate
2647 * user wired memory. Unwiring pages wired down as a result of
2648 * this routine is done with the vm_wire interface.
2649 */
2650 kern_return_t
2651 task_wire(
2652 vm_map_t map,
2653 boolean_t must_wire)
2654 {
2655 if (map == VM_MAP_NULL)
2656 return(KERN_INVALID_ARGUMENT);
2657
2658 if (must_wire)
2659 map->wiring_required = TRUE;
2660 else
2661 map->wiring_required = FALSE;
2662
2663 return(KERN_SUCCESS);
2664 }
2665
2666 __private_extern__ kern_return_t
2667 mach_memory_entry_allocate(
2668 vm_named_entry_t *user_entry_p,
2669 ipc_port_t *user_handle_p)
2670 {
2671 vm_named_entry_t user_entry;
2672 ipc_port_t user_handle;
2673 ipc_port_t previous;
2674
2675 user_entry = (vm_named_entry_t) kalloc(sizeof *user_entry);
2676 if (user_entry == NULL)
2677 return KERN_FAILURE;
2678
2679 named_entry_lock_init(user_entry);
2680
2681 user_handle = ipc_port_alloc_kernel();
2682 if (user_handle == IP_NULL) {
2683 kfree(user_entry, sizeof *user_entry);
2684 return KERN_FAILURE;
2685 }
2686 ip_lock(user_handle);
2687
2688 /* make a sonce right */
2689 user_handle->ip_sorights++;
2690 ip_reference(user_handle);
2691
2692 user_handle->ip_destination = IP_NULL;
2693 user_handle->ip_receiver_name = MACH_PORT_NULL;
2694 user_handle->ip_receiver = ipc_space_kernel;
2695
2696 /* make a send right */
2697 user_handle->ip_mscount++;
2698 user_handle->ip_srights++;
2699 ip_reference(user_handle);
2700
2701 ipc_port_nsrequest(user_handle, 1, user_handle, &previous);
2702 /* nsrequest unlocks user_handle */
2703
2704 user_entry->backing.pager = NULL;
2705 user_entry->is_sub_map = FALSE;
2706 user_entry->is_pager = FALSE;
2707 user_entry->size = 0;
2708 user_entry->internal = FALSE;
2709 user_entry->ref_count = 1;
2710
2711 ipc_kobject_set(user_handle, (ipc_kobject_t) user_entry,
2712 IKOT_NAMED_ENTRY);
2713
2714 *user_entry_p = user_entry;
2715 *user_handle_p = user_handle;
2716
2717 return KERN_SUCCESS;
2718 }
2719
2720 /*
2721 * mach_memory_object_memory_entry_64
2722 *
2723 * Create a named entry backed by the provided pager.
2724 *
2725 * JMM - we need to hold a reference on the pager -
2726 * and release it when the named entry is destroyed.
2727 */
2728 kern_return_t
2729 mach_memory_object_memory_entry_64(
2730 host_t host,
2731 boolean_t internal,
2732 vm_object_offset_t size,
2733 vm_prot_t permission,
2734 memory_object_t pager,
2735 ipc_port_t *entry_handle)
2736 {
2737 unsigned int access;
2738 vm_named_entry_t user_entry;
2739 ipc_port_t user_handle;
2740
2741 if (host == HOST_NULL)
2742 return(KERN_INVALID_HOST);
2743
2744 if (mach_memory_entry_allocate(&user_entry, &user_handle)
2745 != KERN_SUCCESS) {
2746 return KERN_FAILURE;
2747 }
2748
2749 user_entry->backing.pager = pager;
2750 user_entry->size = size;
2751 user_entry->offset = 0;
2752 user_entry->protection = permission & VM_PROT_ALL;
2753 access = GET_MAP_MEM(permission);
2754 SET_MAP_MEM(access, user_entry->protection);
2755 user_entry->internal = internal;
2756 user_entry->is_sub_map = FALSE;
2757 user_entry->is_pager = TRUE;
2758 assert(user_entry->ref_count == 1);
2759
2760 *entry_handle = user_handle;
2761 return KERN_SUCCESS;
2762 }
2763
2764 kern_return_t
2765 mach_memory_object_memory_entry(
2766 host_t host,
2767 boolean_t internal,
2768 vm_size_t size,
2769 vm_prot_t permission,
2770 memory_object_t pager,
2771 ipc_port_t *entry_handle)
2772 {
2773 return mach_memory_object_memory_entry_64( host, internal,
2774 (vm_object_offset_t)size, permission, pager, entry_handle);
2775 }
2776
2777
2778 kern_return_t
2779 mach_memory_entry_purgable_control(
2780 ipc_port_t entry_port,
2781 vm_purgable_t control,
2782 int *state)
2783 {
2784 kern_return_t kr;
2785 vm_named_entry_t mem_entry;
2786 vm_object_t object;
2787
2788 if (entry_port == IP_NULL ||
2789 ip_kotype(entry_port) != IKOT_NAMED_ENTRY) {
2790 return KERN_INVALID_ARGUMENT;
2791 }
2792
2793 mem_entry = (vm_named_entry_t) entry_port->ip_kobject;
2794
2795 named_entry_lock(mem_entry);
2796
2797 if (mem_entry->is_sub_map || mem_entry->is_pager) {
2798 named_entry_unlock(mem_entry);
2799 return KERN_INVALID_ARGUMENT;
2800 }
2801
2802 object = mem_entry->backing.object;
2803 if (object == VM_OBJECT_NULL) {
2804 named_entry_unlock(mem_entry);
2805 return KERN_INVALID_ARGUMENT;
2806 }
2807
2808 vm_object_lock(object);
2809
2810 /* check that named entry covers entire object ? */
2811 if (mem_entry->offset != 0 || object->size != mem_entry->size) {
2812 vm_object_unlock(object);
2813 named_entry_unlock(mem_entry);
2814 return KERN_INVALID_ARGUMENT;
2815 }
2816
2817 named_entry_unlock(mem_entry);
2818
2819 kr = vm_object_purgable_control(object, control, state);
2820
2821 vm_object_unlock(object);
2822
2823 return kr;
2824 }
2825
2826 /*
2827 * mach_memory_entry_port_release:
2828 *
2829 * Release a send right on a named entry port. This is the correct
2830 * way to destroy a named entry. When the last right on the port is
2831 * released, ipc_kobject_destroy() will call mach_destroy_memory_entry().
2832 */
2833 void
2834 mach_memory_entry_port_release(
2835 ipc_port_t port)
2836 {
2837 assert(ip_kotype(port) == IKOT_NAMED_ENTRY);
2838 ipc_port_release_send(port);
2839 }
2840
2841 /*
2842 * mach_destroy_memory_entry:
2843 *
2844 * Drops a reference on a memory entry and destroys the memory entry if
2845 * there are no more references on it.
2846 * NOTE: This routine should not be called to destroy a memory entry from the
2847 * kernel, as it will not release the Mach port associated with the memory
2848 * entry. The proper way to destroy a memory entry in the kernel is to
2849 * call mach_memort_entry_port_release() to release the kernel's send-right on
2850 * the memory entry's port. When the last send right is released, the memory
2851 * entry will be destroyed via ipc_kobject_destroy().
2852 */
2853 void
2854 mach_destroy_memory_entry(
2855 ipc_port_t port)
2856 {
2857 vm_named_entry_t named_entry;
2858 #if MACH_ASSERT
2859 assert(ip_kotype(port) == IKOT_NAMED_ENTRY);
2860 #endif /* MACH_ASSERT */
2861 named_entry = (vm_named_entry_t)port->ip_kobject;
2862 mutex_lock(&(named_entry)->Lock);
2863 named_entry->ref_count -= 1;
2864 if(named_entry->ref_count == 0) {
2865 if (named_entry->is_sub_map) {
2866 vm_map_deallocate(named_entry->backing.map);
2867 } else if (!named_entry->is_pager) {
2868 /* release the memory object we've been pointing to */
2869 vm_object_deallocate(named_entry->backing.object);
2870 } /* else JMM - need to drop reference on pager in that case */
2871
2872 mutex_unlock(&(named_entry)->Lock);
2873
2874 kfree((void *) port->ip_kobject,
2875 sizeof (struct vm_named_entry));
2876 } else
2877 mutex_unlock(&(named_entry)->Lock);
2878 }
2879
2880
2881
2882 kern_return_t
2883 set_dp_control_port(
2884 host_priv_t host_priv,
2885 ipc_port_t control_port)
2886 {
2887 if (host_priv == HOST_PRIV_NULL)
2888 return (KERN_INVALID_HOST);
2889
2890 if (IP_VALID(dynamic_pager_control_port))
2891 ipc_port_release_send(dynamic_pager_control_port);
2892
2893 dynamic_pager_control_port = control_port;
2894 return KERN_SUCCESS;
2895 }
2896
2897 kern_return_t
2898 get_dp_control_port(
2899 host_priv_t host_priv,
2900 ipc_port_t *control_port)
2901 {
2902 if (host_priv == HOST_PRIV_NULL)
2903 return (KERN_INVALID_HOST);
2904
2905 *control_port = ipc_port_copy_send(dynamic_pager_control_port);
2906 return KERN_SUCCESS;
2907
2908 }
2909
2910 /* ******* Temporary Internal calls to UPL for BSD ***** */
2911
2912 extern int kernel_upl_map(
2913 vm_map_t map,
2914 upl_t upl,
2915 vm_offset_t *dst_addr);
2916
2917 extern int kernel_upl_unmap(
2918 vm_map_t map,
2919 upl_t upl);
2920
2921 extern int kernel_upl_commit(
2922 upl_t upl,
2923 upl_page_info_t *pl,
2924 mach_msg_type_number_t count);
2925
2926 extern int kernel_upl_commit_range(
2927 upl_t upl,
2928 upl_offset_t offset,
2929 upl_size_t size,
2930 int flags,
2931 upl_page_info_array_t pl,
2932 mach_msg_type_number_t count);
2933
2934 extern int kernel_upl_abort(
2935 upl_t upl,
2936 int abort_type);
2937
2938 extern int kernel_upl_abort_range(
2939 upl_t upl,
2940 upl_offset_t offset,
2941 upl_size_t size,
2942 int abort_flags);
2943
2944
2945 kern_return_t
2946 kernel_upl_map(
2947 vm_map_t map,
2948 upl_t upl,
2949 vm_offset_t *dst_addr)
2950 {
2951 return vm_upl_map(map, upl, dst_addr);
2952 }
2953
2954
2955 kern_return_t
2956 kernel_upl_unmap(
2957 vm_map_t map,
2958 upl_t upl)
2959 {
2960 return vm_upl_unmap(map, upl);
2961 }
2962
2963 kern_return_t
2964 kernel_upl_commit(
2965 upl_t upl,
2966 upl_page_info_t *pl,
2967 mach_msg_type_number_t count)
2968 {
2969 kern_return_t kr;
2970
2971 kr = upl_commit(upl, pl, count);
2972 upl_deallocate(upl);
2973 return kr;
2974 }
2975
2976
2977 kern_return_t
2978 kernel_upl_commit_range(
2979 upl_t upl,
2980 upl_offset_t offset,
2981 upl_size_t size,
2982 int flags,
2983 upl_page_info_array_t pl,
2984 mach_msg_type_number_t count)
2985 {
2986 boolean_t finished = FALSE;
2987 kern_return_t kr;
2988
2989 if (flags & UPL_COMMIT_FREE_ON_EMPTY)
2990 flags |= UPL_COMMIT_NOTIFY_EMPTY;
2991
2992 kr = upl_commit_range(upl, offset, size, flags, pl, count, &finished);
2993
2994 if ((flags & UPL_COMMIT_NOTIFY_EMPTY) && finished)
2995 upl_deallocate(upl);
2996
2997 return kr;
2998 }
2999
3000 kern_return_t
3001 kernel_upl_abort_range(
3002 upl_t upl,
3003 upl_offset_t offset,
3004 upl_size_t size,
3005 int abort_flags)
3006 {
3007 kern_return_t kr;
3008 boolean_t finished = FALSE;
3009
3010 if (abort_flags & UPL_COMMIT_FREE_ON_EMPTY)
3011 abort_flags |= UPL_COMMIT_NOTIFY_EMPTY;
3012
3013 kr = upl_abort_range(upl, offset, size, abort_flags, &finished);
3014
3015 if ((abort_flags & UPL_COMMIT_FREE_ON_EMPTY) && finished)
3016 upl_deallocate(upl);
3017
3018 return kr;
3019 }
3020
3021 kern_return_t
3022 kernel_upl_abort(
3023 upl_t upl,
3024 int abort_type)
3025 {
3026 kern_return_t kr;
3027
3028 kr = upl_abort(upl, abort_type);
3029 upl_deallocate(upl);
3030 return kr;
3031 }
3032
3033 /*
3034 * Now a kernel-private interface (for BootCache
3035 * use only). Need a cleaner way to create an
3036 * empty vm_map() and return a handle to it.
3037 */
3038
3039 kern_return_t
3040 vm_region_object_create(
3041 __unused vm_map_t target_map,
3042 vm_size_t size,
3043 ipc_port_t *object_handle)
3044 {
3045 vm_named_entry_t user_entry;
3046 ipc_port_t user_handle;
3047
3048 vm_map_t new_map;
3049
3050 if (mach_memory_entry_allocate(&user_entry, &user_handle)
3051 != KERN_SUCCESS) {
3052 return KERN_FAILURE;
3053 }
3054
3055 /* Create a named object based on a submap of specified size */
3056
3057 new_map = vm_map_create(PMAP_NULL, VM_MAP_MIN_ADDRESS,
3058 vm_map_round_page(size), TRUE);
3059
3060 user_entry->backing.map = new_map;
3061 user_entry->internal = TRUE;
3062 user_entry->is_sub_map = TRUE;
3063 user_entry->offset = 0;
3064 user_entry->protection = VM_PROT_ALL;
3065 user_entry->size = size;
3066 assert(user_entry->ref_count == 1);
3067
3068 *object_handle = user_handle;
3069 return KERN_SUCCESS;
3070
3071 }
3072
3073 ppnum_t vm_map_get_phys_page( /* forward */
3074 vm_map_t map,
3075 vm_offset_t offset);
3076
3077 ppnum_t
3078 vm_map_get_phys_page(
3079 vm_map_t map,
3080 vm_offset_t addr)
3081 {
3082 vm_object_offset_t offset;
3083 vm_object_t object;
3084 vm_map_offset_t map_offset;
3085 vm_map_entry_t entry;
3086 ppnum_t phys_page = 0;
3087
3088 map_offset = vm_map_trunc_page(addr);
3089
3090 vm_map_lock(map);
3091 while (vm_map_lookup_entry(map, map_offset, &entry)) {
3092
3093 if (entry->object.vm_object == VM_OBJECT_NULL) {
3094 vm_map_unlock(map);
3095 return (ppnum_t) 0;
3096 }
3097 if (entry->is_sub_map) {
3098 vm_map_t old_map;
3099 vm_map_lock(entry->object.sub_map);
3100 old_map = map;
3101 map = entry->object.sub_map;
3102 map_offset = entry->offset + (map_offset - entry->vme_start);
3103 vm_map_unlock(old_map);
3104 continue;
3105 }
3106 if (entry->object.vm_object->phys_contiguous) {
3107 /* These are not standard pageable memory mappings */
3108 /* If they are not present in the object they will */
3109 /* have to be picked up from the pager through the */
3110 /* fault mechanism. */
3111 if(entry->object.vm_object->shadow_offset == 0) {
3112 /* need to call vm_fault */
3113 vm_map_unlock(map);
3114 vm_fault(map, map_offset, VM_PROT_NONE,
3115 FALSE, THREAD_UNINT, NULL, 0);
3116 vm_map_lock(map);
3117 continue;
3118 }
3119 offset = entry->offset + (map_offset - entry->vme_start);
3120 phys_page = (ppnum_t)
3121 ((entry->object.vm_object->shadow_offset
3122 + offset) >> 12);
3123 break;
3124
3125 }
3126 offset = entry->offset + (map_offset - entry->vme_start);
3127 object = entry->object.vm_object;
3128 vm_object_lock(object);
3129 while (TRUE) {
3130 vm_page_t dst_page = vm_page_lookup(object,offset);
3131 if(dst_page == VM_PAGE_NULL) {
3132 if(object->shadow) {
3133 vm_object_t old_object;
3134 vm_object_lock(object->shadow);
3135 old_object = object;
3136 offset = offset + object->shadow_offset;
3137 object = object->shadow;
3138 vm_object_unlock(old_object);
3139 } else {
3140 vm_object_unlock(object);
3141 break;
3142 }
3143 } else {
3144 phys_page = (ppnum_t)(dst_page->phys_page);
3145 vm_object_unlock(object);
3146 break;
3147 }
3148 }
3149 break;
3150
3151 }
3152
3153 vm_map_unlock(map);
3154 return phys_page;
3155 }
3156
3157
3158
3159 kern_return_t kernel_object_iopl_request( /* forward */
3160 vm_named_entry_t named_entry,
3161 memory_object_offset_t offset,
3162 vm_size_t *upl_size,
3163 upl_t *upl_ptr,
3164 upl_page_info_array_t user_page_list,
3165 unsigned int *page_list_count,
3166 int *flags);
3167
3168 kern_return_t
3169 kernel_object_iopl_request(
3170 vm_named_entry_t named_entry,
3171 memory_object_offset_t offset,
3172 vm_size_t *upl_size,
3173 upl_t *upl_ptr,
3174 upl_page_info_array_t user_page_list,
3175 unsigned int *page_list_count,
3176 int *flags)
3177 {
3178 vm_object_t object;
3179 kern_return_t ret;
3180
3181 int caller_flags;
3182
3183 caller_flags = *flags;
3184
3185 if (caller_flags & ~UPL_VALID_FLAGS) {
3186 /*
3187 * For forward compatibility's sake,
3188 * reject any unknown flag.
3189 */
3190 return KERN_INVALID_VALUE;
3191 }
3192
3193 /* a few checks to make sure user is obeying rules */
3194 if(*upl_size == 0) {
3195 if(offset >= named_entry->size)
3196 return(KERN_INVALID_RIGHT);
3197 *upl_size = named_entry->size - offset;
3198 }
3199 if(caller_flags & UPL_COPYOUT_FROM) {
3200 if((named_entry->protection & VM_PROT_READ)
3201 != VM_PROT_READ) {
3202 return(KERN_INVALID_RIGHT);
3203 }
3204 } else {
3205 if((named_entry->protection &
3206 (VM_PROT_READ | VM_PROT_WRITE))
3207 != (VM_PROT_READ | VM_PROT_WRITE)) {
3208 return(KERN_INVALID_RIGHT);
3209 }
3210 }
3211 if(named_entry->size < (offset + *upl_size))
3212 return(KERN_INVALID_ARGUMENT);
3213
3214 /* the callers parameter offset is defined to be the */
3215 /* offset from beginning of named entry offset in object */
3216 offset = offset + named_entry->offset;
3217
3218 if(named_entry->is_sub_map)
3219 return (KERN_INVALID_ARGUMENT);
3220
3221 named_entry_lock(named_entry);
3222
3223 if (named_entry->is_pager) {
3224 object = vm_object_enter(named_entry->backing.pager,
3225 named_entry->offset + named_entry->size,
3226 named_entry->internal,
3227 FALSE,
3228 FALSE);
3229 if (object == VM_OBJECT_NULL) {
3230 named_entry_unlock(named_entry);
3231 return(KERN_INVALID_OBJECT);
3232 }
3233
3234 /* JMM - drop reference on the pager here? */
3235
3236 /* create an extra reference for the object */
3237 vm_object_lock(object);
3238 vm_object_reference_locked(object);
3239 named_entry->backing.object = object;
3240 named_entry->is_pager = FALSE;
3241 named_entry_unlock(named_entry);
3242
3243 /* wait for object (if any) to be ready */
3244 if (!named_entry->internal) {
3245 while (!object->pager_ready) {
3246 vm_object_wait(object,
3247 VM_OBJECT_EVENT_PAGER_READY,
3248 THREAD_UNINT);
3249 vm_object_lock(object);
3250 }
3251 }
3252 vm_object_unlock(object);
3253
3254 } else {
3255 /* This is the case where we are going to operate */
3256 /* an an already known object. If the object is */
3257 /* not ready it is internal. An external */
3258 /* object cannot be mapped until it is ready */
3259 /* we can therefore avoid the ready check */
3260 /* in this case. */
3261 object = named_entry->backing.object;
3262 vm_object_reference(object);
3263 named_entry_unlock(named_entry);
3264 }
3265
3266 if (!object->private) {
3267 if (*upl_size > (MAX_UPL_TRANSFER*PAGE_SIZE))
3268 *upl_size = (MAX_UPL_TRANSFER*PAGE_SIZE);
3269 if (object->phys_contiguous) {
3270 *flags = UPL_PHYS_CONTIG;
3271 } else {
3272 *flags = 0;
3273 }
3274 } else {
3275 *flags = UPL_DEV_MEMORY | UPL_PHYS_CONTIG;
3276 }
3277
3278 ret = vm_object_iopl_request(object,
3279 offset,
3280 *upl_size,
3281 upl_ptr,
3282 user_page_list,
3283 page_list_count,
3284 caller_flags);
3285 vm_object_deallocate(object);
3286 return ret;
3287 }
mirror of XNU