]> git.saurik.com Git - apple/xnu.git/blob - osfmk/vm/vm_user.c
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-792.2.4.tar.gz
[apple/xnu.git] / osfmk / vm / vm_user.c
1 /*
2 * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_LICENSE_HEADER_START@
5 *
6 * The contents of this file constitute Original Code as defined in and
7 * are subject to the Apple Public Source License Version 1.1 (the
8 * "License"). You may not use this file except in compliance with the
9 * License. Please obtain a copy of the License at
10 * http://www.apple.com/publicsource and read it before using this file.
11 *
12 * This Original Code and all software distributed under the License are
13 * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
14 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
15 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
17 * License for the specific language governing rights and limitations
18 * under the License.
19 *
20 * @APPLE_LICENSE_HEADER_END@
21 */
22 /*
23 * @OSF_COPYRIGHT@
24 */
25 /*
26 * Mach Operating System
27 * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University
28 * All Rights Reserved.
29 *
30 * Permission to use, copy, modify and distribute this software and its
31 * documentation is hereby granted, provided that both the copyright
32 * notice and this permission notice appear in all copies of the
33 * software, derivative works or modified versions, and any portions
34 * thereof, and that both notices appear in supporting documentation.
35 *
36 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
37 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
38 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
39 *
40 * Carnegie Mellon requests users of this software to return to
41 *
42 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
43 * School of Computer Science
44 * Carnegie Mellon University
45 * Pittsburgh PA 15213-3890
46 *
47 * any improvements or extensions that they make and grant Carnegie Mellon
48 * the rights to redistribute these changes.
49 */
50 /*
51 */
52 /*
53 * File: vm/vm_user.c
54 * Author: Avadis Tevanian, Jr., Michael Wayne Young
55 *
56 * User-exported virtual memory functions.
57 */
58
59 #include <debug.h>
60
61 #include <vm_cpm.h>
62 #include <mach/boolean.h>
63 #include <mach/kern_return.h>
64 #include <mach/mach_types.h> /* to get vm_address_t */
65 #include <mach/memory_object.h>
66 #include <mach/std_types.h> /* to get pointer_t */
67 #include <mach/upl.h>
68 #include <mach/vm_attributes.h>
69 #include <mach/vm_param.h>
70 #include <mach/vm_statistics.h>
71 #include <mach/mach_syscalls.h>
72
73 #include <mach/host_priv_server.h>
74 #include <mach/mach_vm_server.h>
75 #include <mach/shared_memory_server.h>
76 #include <mach/vm_map_server.h>
77 #include <vm/vm_shared_memory_server.h>
78
79 #include <kern/host.h>
80 #include <kern/kalloc.h>
81 #include <kern/task.h>
82 #include <kern/misc_protos.h>
83 #include <vm/vm_fault.h>
84 #include <vm/vm_map.h>
85 #include <vm/vm_object.h>
86 #include <vm/vm_page.h>
87 #include <vm/memory_object.h>
88 #include <vm/vm_pageout.h>
89 #include <vm/vm_protos.h>
90
91 vm_size_t upl_offset_to_pagelist = 0;
92
93 #if VM_CPM
94 #include <vm/cpm.h>
95 #endif /* VM_CPM */
96
97 ipc_port_t dynamic_pager_control_port=NULL;
98
99 /*
100 * mach_vm_allocate allocates "zero fill" memory in the specfied
101 * map.
102 */
103 kern_return_t
104 mach_vm_allocate(
105 vm_map_t map,
106 mach_vm_offset_t *addr,
107 mach_vm_size_t size,
108 int flags)
109 {
110 vm_map_offset_t map_addr;
111 vm_map_size_t map_size;
112 kern_return_t result;
113 boolean_t anywhere = ((VM_FLAGS_ANYWHERE & flags) != 0);
114
115 if (map == VM_MAP_NULL)
116 return(KERN_INVALID_ARGUMENT);
117 if (size == 0) {
118 *addr = 0;
119 return(KERN_SUCCESS);
120 }
121
122 if (anywhere) {
123 /*
124 * No specific address requested, so start candidate address
125 * search at the minimum address in the map. However, if that
126 * minimum is 0, bump it up by PAGE_SIZE. We want to limit
127 * allocations of PAGEZERO to explicit requests since its
128 * normal use is to catch dereferences of NULL and many
129 * applications also treat pointers with a value of 0 as
130 * special and suddenly having address 0 contain useable
131 * memory would tend to confuse those applications.
132 */
133 map_addr = vm_map_min(map);
134 if (map_addr == 0)
135 map_addr += PAGE_SIZE;
136 } else
137 map_addr = vm_map_trunc_page(*addr);
138 map_size = vm_map_round_page(size);
139 if (map_size == 0) {
140 return(KERN_INVALID_ARGUMENT);
141 }
142
143 result = vm_map_enter(
144 map,
145 &map_addr,
146 map_size,
147 (vm_map_offset_t)0,
148 flags,
149 VM_OBJECT_NULL,
150 (vm_object_offset_t)0,
151 FALSE,
152 VM_PROT_DEFAULT,
153 VM_PROT_ALL,
154 VM_INHERIT_DEFAULT);
155
156 *addr = map_addr;
157 return(result);
158 }
159
160 /*
161 * vm_allocate
162 * Legacy routine that allocates "zero fill" memory in the specfied
163 * map (which is limited to the same size as the kernel).
164 */
165 kern_return_t
166 vm_allocate(
167 vm_map_t map,
168 vm_offset_t *addr,
169 vm_size_t size,
170 int flags)
171 {
172 vm_map_offset_t map_addr;
173 vm_map_size_t map_size;
174 kern_return_t result;
175 boolean_t anywhere = ((VM_FLAGS_ANYWHERE & flags) != 0);
176
177 if (map == VM_MAP_NULL)
178 return(KERN_INVALID_ARGUMENT);
179 if (size == 0) {
180 *addr = 0;
181 return(KERN_SUCCESS);
182 }
183
184 if (anywhere) {
185 /*
186 * No specific address requested, so start candidate address
187 * search at the minimum address in the map. However, if that
188 * minimum is 0, bump it up by PAGE_SIZE. We want to limit
189 * allocations of PAGEZERO to explicit requests since its
190 * normal use is to catch dereferences of NULL and many
191 * applications also treat pointers with a value of 0 as
192 * special and suddenly having address 0 contain useable
193 * memory would tend to confuse those applications.
194 */
195 map_addr = vm_map_min(map);
196 if (map_addr == 0)
197 map_addr += PAGE_SIZE;
198 } else
199 map_addr = vm_map_trunc_page(*addr);
200 map_size = vm_map_round_page(size);
201 if (map_size == 0) {
202 return(KERN_INVALID_ARGUMENT);
203 }
204
205 result = vm_map_enter(
206 map,
207 &map_addr,
208 map_size,
209 (vm_map_offset_t)0,
210 flags,
211 VM_OBJECT_NULL,
212 (vm_object_offset_t)0,
213 FALSE,
214 VM_PROT_DEFAULT,
215 VM_PROT_ALL,
216 VM_INHERIT_DEFAULT);
217
218 *addr = CAST_DOWN(vm_offset_t, map_addr);
219 return(result);
220 }
221
222 /*
223 * mach_vm_deallocate -
224 * deallocates the specified range of addresses in the
225 * specified address map.
226 */
227 kern_return_t
228 mach_vm_deallocate(
229 vm_map_t map,
230 mach_vm_offset_t start,
231 mach_vm_size_t size)
232 {
233 if ((map == VM_MAP_NULL) || (start + size < start))
234 return(KERN_INVALID_ARGUMENT);
235
236 if (size == (mach_vm_offset_t) 0)
237 return(KERN_SUCCESS);
238
239 return(vm_map_remove(map, vm_map_trunc_page(start),
240 vm_map_round_page(start+size), VM_MAP_NO_FLAGS));
241 }
242
243 /*
244 * vm_deallocate -
245 * deallocates the specified range of addresses in the
246 * specified address map (limited to addresses the same
247 * size as the kernel).
248 */
249 kern_return_t
250 vm_deallocate(
251 register vm_map_t map,
252 vm_offset_t start,
253 vm_size_t size)
254 {
255 if ((map == VM_MAP_NULL) || (start + size < start))
256 return(KERN_INVALID_ARGUMENT);
257
258 if (size == (vm_offset_t) 0)
259 return(KERN_SUCCESS);
260
261 return(vm_map_remove(map, vm_map_trunc_page(start),
262 vm_map_round_page(start+size), VM_MAP_NO_FLAGS));
263 }
264
265 /*
266 * mach_vm_inherit -
267 * Sets the inheritance of the specified range in the
268 * specified map.
269 */
270 kern_return_t
271 mach_vm_inherit(
272 vm_map_t map,
273 mach_vm_offset_t start,
274 mach_vm_size_t size,
275 vm_inherit_t new_inheritance)
276 {
277 if ((map == VM_MAP_NULL) || (start + size < start) ||
278 (new_inheritance > VM_INHERIT_LAST_VALID))
279 return(KERN_INVALID_ARGUMENT);
280
281 if (size == 0)
282 return KERN_SUCCESS;
283
284 return(vm_map_inherit(map,
285 vm_map_trunc_page(start),
286 vm_map_round_page(start+size),
287 new_inheritance));
288 }
289
290 /*
291 * vm_inherit -
292 * Sets the inheritance of the specified range in the
293 * specified map (range limited to addresses
294 */
295 kern_return_t
296 vm_inherit(
297 register vm_map_t map,
298 vm_offset_t start,
299 vm_size_t size,
300 vm_inherit_t new_inheritance)
301 {
302 if ((map == VM_MAP_NULL) || (start + size < start) ||
303 (new_inheritance > VM_INHERIT_LAST_VALID))
304 return(KERN_INVALID_ARGUMENT);
305
306 if (size == 0)
307 return KERN_SUCCESS;
308
309 return(vm_map_inherit(map,
310 vm_map_trunc_page(start),
311 vm_map_round_page(start+size),
312 new_inheritance));
313 }
314
315 /*
316 * mach_vm_protect -
317 * Sets the protection of the specified range in the
318 * specified map.
319 */
320
321 kern_return_t
322 mach_vm_protect(
323 vm_map_t map,
324 mach_vm_offset_t start,
325 mach_vm_size_t size,
326 boolean_t set_maximum,
327 vm_prot_t new_protection)
328 {
329 if ((map == VM_MAP_NULL) || (start + size < start) ||
330 (new_protection & ~(VM_PROT_ALL | VM_PROT_COPY)))
331 return(KERN_INVALID_ARGUMENT);
332
333 if (size == 0)
334 return KERN_SUCCESS;
335
336 return(vm_map_protect(map,
337 vm_map_trunc_page(start),
338 vm_map_round_page(start+size),
339 new_protection,
340 set_maximum));
341 }
342
343 /*
344 * vm_protect -
345 * Sets the protection of the specified range in the
346 * specified map. Addressability of the range limited
347 * to the same size as the kernel.
348 */
349
350 kern_return_t
351 vm_protect(
352 vm_map_t map,
353 vm_offset_t start,
354 vm_size_t size,
355 boolean_t set_maximum,
356 vm_prot_t new_protection)
357 {
358 if ((map == VM_MAP_NULL) || (start + size < start) ||
359 (new_protection & ~(VM_PROT_ALL | VM_PROT_COPY)))
360 return(KERN_INVALID_ARGUMENT);
361
362 if (size == 0)
363 return KERN_SUCCESS;
364
365 return(vm_map_protect(map,
366 vm_map_trunc_page(start),
367 vm_map_round_page(start+size),
368 new_protection,
369 set_maximum));
370 }
371
372 /*
373 * mach_vm_machine_attributes -
374 * Handle machine-specific attributes for a mapping, such
375 * as cachability, migrability, etc.
376 */
377 kern_return_t
378 mach_vm_machine_attribute(
379 vm_map_t map,
380 mach_vm_address_t addr,
381 mach_vm_size_t size,
382 vm_machine_attribute_t attribute,
383 vm_machine_attribute_val_t* value) /* IN/OUT */
384 {
385 if ((map == VM_MAP_NULL) || (addr + size < addr))
386 return(KERN_INVALID_ARGUMENT);
387
388 if (size == 0)
389 return KERN_SUCCESS;
390
391 return vm_map_machine_attribute(map,
392 vm_map_trunc_page(addr),
393 vm_map_round_page(addr+size),
394 attribute,
395 value);
396 }
397
398 /*
399 * vm_machine_attribute -
400 * Handle machine-specific attributes for a mapping, such
401 * as cachability, migrability, etc. Limited addressability
402 * (same range limits as for the native kernel map).
403 */
404 kern_return_t
405 vm_machine_attribute(
406 vm_map_t map,
407 vm_address_t addr,
408 vm_size_t size,
409 vm_machine_attribute_t attribute,
410 vm_machine_attribute_val_t* value) /* IN/OUT */
411 {
412 if ((map == VM_MAP_NULL) || (addr + size < addr))
413 return(KERN_INVALID_ARGUMENT);
414
415 if (size == 0)
416 return KERN_SUCCESS;
417
418 return vm_map_machine_attribute(map,
419 vm_map_trunc_page(addr),
420 vm_map_round_page(addr+size),
421 attribute,
422 value);
423 }
424
425 /*
426 * mach_vm_read -
427 * Read/copy a range from one address space and return it to the caller.
428 *
429 * It is assumed that the address for the returned memory is selected by
430 * the IPC implementation as part of receiving the reply to this call.
431 * If IPC isn't used, the caller must deal with the vm_map_copy_t object
432 * that gets returned.
433 *
434 * JMM - because of mach_msg_type_number_t, this call is limited to a
435 * single 4GB region at this time.
436 *
437 */
438 kern_return_t
439 mach_vm_read(
440 vm_map_t map,
441 mach_vm_address_t addr,
442 mach_vm_size_t size,
443 pointer_t *data,
444 mach_msg_type_number_t *data_size)
445 {
446 kern_return_t error;
447 vm_map_copy_t ipc_address;
448
449 if (map == VM_MAP_NULL)
450 return(KERN_INVALID_ARGUMENT);
451
452
453 error = vm_map_copyin(map,
454 (vm_map_address_t)addr,
455 (vm_map_size_t)size,
456 FALSE, /* src_destroy */
457 &ipc_address);
458
459 if (KERN_SUCCESS == error) {
460 *data = (pointer_t) ipc_address;
461 *data_size = size;
462 }
463 return(error);
464 }
465
466 /*
467 * vm_read -
468 * Read/copy a range from one address space and return it to the caller.
469 * Limited addressability (same range limits as for the native kernel map).
470 *
471 * It is assumed that the address for the returned memory is selected by
472 * the IPC implementation as part of receiving the reply to this call.
473 * If IPC isn't used, the caller must deal with the vm_map_copy_t object
474 * that gets returned.
475 */
476 kern_return_t
477 vm_read(
478 vm_map_t map,
479 vm_address_t addr,
480 vm_size_t size,
481 pointer_t *data,
482 mach_msg_type_number_t *data_size)
483 {
484 kern_return_t error;
485 vm_map_copy_t ipc_address;
486
487 if (map == VM_MAP_NULL)
488 return(KERN_INVALID_ARGUMENT);
489
490 error = vm_map_copyin(map,
491 (vm_map_address_t)addr,
492 (vm_map_size_t)size,
493 FALSE, /* src_destroy */
494 &ipc_address);
495
496 if (KERN_SUCCESS == error) {
497 *data = (pointer_t) ipc_address;
498 *data_size = size;
499 }
500 return(error);
501 }
502
503 /*
504 * mach_vm_read_list -
505 * Read/copy a list of address ranges from specified map.
506 *
507 * MIG does not know how to deal with a returned array of
508 * vm_map_copy_t structures, so we have to do the copyout
509 * manually here.
510 */
511 kern_return_t
512 mach_vm_read_list(
513 vm_map_t map,
514 mach_vm_read_entry_t data_list,
515 natural_t count)
516 {
517 mach_msg_type_number_t i;
518 kern_return_t error;
519 vm_map_copy_t copy;
520
521 if (map == VM_MAP_NULL)
522 return(KERN_INVALID_ARGUMENT);
523
524 error = KERN_SUCCESS;
525 for(i=0; i<count; i++) {
526 vm_map_address_t map_addr;
527 vm_map_size_t map_size;
528
529 map_addr = (vm_map_address_t)(data_list[i].address);
530 map_size = (vm_map_size_t)(data_list[i].size);
531
532 if(map_size != 0) {
533 error = vm_map_copyin(map,
534 map_addr,
535 map_size,
536 FALSE, /* src_destroy */
537 &copy);
538 if (KERN_SUCCESS == error) {
539 error = vm_map_copyout(
540 current_task()->map,
541 &map_addr,
542 copy);
543 if (KERN_SUCCESS == error) {
544 data_list[i].address = map_addr;
545 continue;
546 }
547 vm_map_copy_discard(copy);
548 }
549 }
550 data_list[i].address = (mach_vm_address_t)0;
551 data_list[i].size = (mach_vm_size_t)0;
552 }
553 return(error);
554 }
555
556 /*
557 * vm_read_list -
558 * Read/copy a list of address ranges from specified map.
559 *
560 * MIG does not know how to deal with a returned array of
561 * vm_map_copy_t structures, so we have to do the copyout
562 * manually here.
563 *
564 * The source and destination ranges are limited to those
565 * that can be described with a vm_address_t (i.e. same
566 * size map as the kernel).
567 *
568 * JMM - If the result of the copyout is an address range
569 * that cannot be described with a vm_address_t (i.e. the
570 * caller had a larger address space but used this call
571 * anyway), it will result in a truncated address being
572 * returned (and a likely confused caller).
573 */
574
575 kern_return_t
576 vm_read_list(
577 vm_map_t map,
578 vm_read_entry_t data_list,
579 natural_t count)
580 {
581 mach_msg_type_number_t i;
582 kern_return_t error;
583 vm_map_copy_t copy;
584
585 if (map == VM_MAP_NULL)
586 return(KERN_INVALID_ARGUMENT);
587
588 error = KERN_SUCCESS;
589 for(i=0; i<count; i++) {
590 vm_map_address_t map_addr;
591 vm_map_size_t map_size;
592
593 map_addr = (vm_map_address_t)(data_list[i].address);
594 map_size = (vm_map_size_t)(data_list[i].size);
595
596 if(map_size != 0) {
597 error = vm_map_copyin(map,
598 map_addr,
599 map_size,
600 FALSE, /* src_destroy */
601 &copy);
602 if (KERN_SUCCESS == error) {
603 error = vm_map_copyout(current_task()->map,
604 &map_addr,
605 copy);
606 if (KERN_SUCCESS == error) {
607 data_list[i].address =
608 CAST_DOWN(vm_offset_t, map_addr);
609 continue;
610 }
611 vm_map_copy_discard(copy);
612 }
613 }
614 data_list[i].address = (mach_vm_address_t)0;
615 data_list[i].size = (mach_vm_size_t)0;
616 }
617 return(error);
618 }
619
620 /*
621 * mach_vm_read_overwrite -
622 * Overwrite a range of the current map with data from the specified
623 * map/address range.
624 *
625 * In making an assumption that the current thread is local, it is
626 * no longer cluster-safe without a fully supportive local proxy
627 * thread/task (but we don't support cluster's anymore so this is moot).
628 */
629
630 kern_return_t
631 mach_vm_read_overwrite(
632 vm_map_t map,
633 mach_vm_address_t address,
634 mach_vm_size_t size,
635 mach_vm_address_t data,
636 mach_vm_size_t *data_size)
637 {
638 kern_return_t error;
639 vm_map_copy_t copy;
640
641 if (map == VM_MAP_NULL)
642 return(KERN_INVALID_ARGUMENT);
643
644 error = vm_map_copyin(map, (vm_map_address_t)address,
645 (vm_map_size_t)size, FALSE, &copy);
646
647 if (KERN_SUCCESS == error) {
648 error = vm_map_copy_overwrite(current_thread()->map,
649 (vm_map_address_t)data,
650 copy, FALSE);
651 if (KERN_SUCCESS == error) {
652 *data_size = size;
653 return error;
654 }
655 vm_map_copy_discard(copy);
656 }
657 return(error);
658 }
659
660 /*
661 * vm_read_overwrite -
662 * Overwrite a range of the current map with data from the specified
663 * map/address range.
664 *
665 * This routine adds the additional limitation that the source and
666 * destination ranges must be describable with vm_address_t values
667 * (i.e. the same size address spaces as the kernel, or at least the
668 * the ranges are in that first portion of the respective address
669 * spaces).
670 */
671
672 kern_return_t
673 vm_read_overwrite(
674 vm_map_t map,
675 vm_address_t address,
676 vm_size_t size,
677 vm_address_t data,
678 vm_size_t *data_size)
679 {
680 kern_return_t error;
681 vm_map_copy_t copy;
682
683 if (map == VM_MAP_NULL)
684 return(KERN_INVALID_ARGUMENT);
685
686 error = vm_map_copyin(map, (vm_map_address_t)address,
687 (vm_map_size_t)size, FALSE, &copy);
688
689 if (KERN_SUCCESS == error) {
690 error = vm_map_copy_overwrite(current_thread()->map,
691 (vm_map_address_t)data,
692 copy, FALSE);
693 if (KERN_SUCCESS == error) {
694 *data_size = size;
695 return error;
696 }
697 vm_map_copy_discard(copy);
698 }
699 return(error);
700 }
701
702
703 /*
704 * mach_vm_write -
705 * Overwrite the specified address range with the data provided
706 * (from the current map).
707 */
708 kern_return_t
709 mach_vm_write(
710 vm_map_t map,
711 mach_vm_address_t address,
712 pointer_t data,
713 __unused mach_msg_type_number_t size)
714 {
715 if (map == VM_MAP_NULL)
716 return KERN_INVALID_ARGUMENT;
717
718 return vm_map_copy_overwrite(map, (vm_map_address_t)address,
719 (vm_map_copy_t) data, FALSE /* interruptible XXX */);
720 }
721
722 /*
723 * vm_write -
724 * Overwrite the specified address range with the data provided
725 * (from the current map).
726 *
727 * The addressability of the range of addresses to overwrite is
728 * limited bu the use of a vm_address_t (same size as kernel map).
729 * Either the target map is also small, or the range is in the
730 * low addresses within it.
731 */
732 kern_return_t
733 vm_write(
734 vm_map_t map,
735 vm_address_t address,
736 pointer_t data,
737 __unused mach_msg_type_number_t size)
738 {
739 if (map == VM_MAP_NULL)
740 return KERN_INVALID_ARGUMENT;
741
742 return vm_map_copy_overwrite(map, (vm_map_address_t)address,
743 (vm_map_copy_t) data, FALSE /* interruptible XXX */);
744 }
745
746 /*
747 * mach_vm_copy -
748 * Overwrite one range of the specified map with the contents of
749 * another range within that same map (i.e. both address ranges
750 * are "over there").
751 */
752 kern_return_t
753 mach_vm_copy(
754 vm_map_t map,
755 mach_vm_address_t source_address,
756 mach_vm_size_t size,
757 mach_vm_address_t dest_address)
758 {
759 vm_map_copy_t copy;
760 kern_return_t kr;
761
762 if (map == VM_MAP_NULL)
763 return KERN_INVALID_ARGUMENT;
764
765 kr = vm_map_copyin(map, (vm_map_address_t)source_address,
766 (vm_map_size_t)size, FALSE, &copy);
767
768 if (KERN_SUCCESS == kr) {
769 kr = vm_map_copy_overwrite(map,
770 (vm_map_address_t)dest_address,
771 copy, FALSE /* interruptible XXX */);
772
773 if (KERN_SUCCESS != kr)
774 vm_map_copy_discard(copy);
775 }
776 return kr;
777 }
778
779 kern_return_t
780 vm_copy(
781 vm_map_t map,
782 vm_address_t source_address,
783 vm_size_t size,
784 vm_address_t dest_address)
785 {
786 vm_map_copy_t copy;
787 kern_return_t kr;
788
789 if (map == VM_MAP_NULL)
790 return KERN_INVALID_ARGUMENT;
791
792 kr = vm_map_copyin(map, (vm_map_address_t)source_address,
793 (vm_map_size_t)size, FALSE, &copy);
794
795 if (KERN_SUCCESS == kr) {
796 kr = vm_map_copy_overwrite(map,
797 (vm_map_address_t)dest_address,
798 copy, FALSE /* interruptible XXX */);
799
800 if (KERN_SUCCESS != kr)
801 vm_map_copy_discard(copy);
802 }
803 return kr;
804 }
805
806 /*
807 * mach_vm_map -
808 * Map some range of an object into an address space.
809 *
810 * The object can be one of several types of objects:
811 * NULL - anonymous memory
812 * a named entry - a range within another address space
813 * or a range within a memory object
814 * a whole memory object
815 *
816 */
817 kern_return_t
818 mach_vm_map(
819 vm_map_t target_map,
820 mach_vm_offset_t *address,
821 mach_vm_size_t initial_size,
822 mach_vm_offset_t mask,
823 int flags,
824 ipc_port_t port,
825 vm_object_offset_t offset,
826 boolean_t copy,
827 vm_prot_t cur_protection,
828 vm_prot_t max_protection,
829 vm_inherit_t inheritance)
830 {
831 vm_map_address_t map_addr;
832 vm_map_size_t map_size;
833 vm_object_t object;
834 vm_object_size_t size;
835 kern_return_t result;
836
837 /*
838 * Check arguments for validity
839 */
840 if ((target_map == VM_MAP_NULL) ||
841 (cur_protection & ~VM_PROT_ALL) ||
842 (max_protection & ~VM_PROT_ALL) ||
843 (inheritance > VM_INHERIT_LAST_VALID) ||
844 initial_size == 0)
845 return(KERN_INVALID_ARGUMENT);
846
847 map_addr = vm_map_trunc_page(*address);
848 map_size = vm_map_round_page(initial_size);
849 size = vm_object_round_page(initial_size);
850
851 /*
852 * Find the vm object (if any) corresponding to this port.
853 */
854 if (!IP_VALID(port)) {
855 object = VM_OBJECT_NULL;
856 offset = 0;
857 copy = FALSE;
858 } else if (ip_kotype(port) == IKOT_NAMED_ENTRY) {
859 vm_named_entry_t named_entry;
860
861 named_entry = (vm_named_entry_t)port->ip_kobject;
862 /* a few checks to make sure user is obeying rules */
863 if(size == 0) {
864 if(offset >= named_entry->size)
865 return(KERN_INVALID_RIGHT);
866 size = named_entry->size - offset;
867 }
868 if((named_entry->protection & max_protection) != max_protection)
869 return(KERN_INVALID_RIGHT);
870 if((named_entry->protection & cur_protection) != cur_protection)
871 return(KERN_INVALID_RIGHT);
872 if(named_entry->size < (offset + size))
873 return(KERN_INVALID_ARGUMENT);
874
875 /* the callers parameter offset is defined to be the */
876 /* offset from beginning of named entry offset in object */
877 offset = offset + named_entry->offset;
878
879 named_entry_lock(named_entry);
880 if(named_entry->is_sub_map) {
881 vm_map_entry_t map_entry;
882
883 named_entry_unlock(named_entry);
884 vm_object_reference(vm_submap_object);
885 if ((result = vm_map_enter(target_map,
886 &map_addr, map_size,
887 (vm_map_offset_t)mask, flags,
888 vm_submap_object, 0,
889 FALSE,
890 cur_protection, max_protection, inheritance
891 )) != KERN_SUCCESS) {
892 vm_object_deallocate(vm_submap_object);
893 } else {
894 char alias;
895
896 VM_GET_FLAGS_ALIAS(flags, alias);
897 if ((alias == VM_MEMORY_SHARED_PMAP) &&
898 !copy) {
899 vm_map_submap(target_map, map_addr,
900 map_addr + map_size,
901 named_entry->backing.map,
902 (vm_map_offset_t)offset, TRUE);
903 } else {
904 vm_map_submap(target_map, map_addr,
905 map_addr + map_size,
906 named_entry->backing.map,
907 (vm_map_offset_t)offset, FALSE);
908 }
909 if(copy) {
910 if(vm_map_lookup_entry(
911 target_map, map_addr, &map_entry)) {
912 map_entry->needs_copy = TRUE;
913 }
914 }
915 *address = map_addr;
916 }
917 return(result);
918
919 } else if (named_entry->is_pager) {
920 unsigned int access;
921 vm_prot_t protections;
922 unsigned int wimg_mode;
923 boolean_t cache_attr;
924
925 protections = named_entry->protection
926 & VM_PROT_ALL;
927 access = GET_MAP_MEM(named_entry->protection);
928
929 object = vm_object_enter(
930 named_entry->backing.pager,
931 named_entry->size,
932 named_entry->internal,
933 FALSE,
934 FALSE);
935 if (object == VM_OBJECT_NULL) {
936 named_entry_unlock(named_entry);
937 return(KERN_INVALID_OBJECT);
938 }
939
940 /* JMM - drop reference on pager here */
941
942 /* create an extra ref for the named entry */
943 vm_object_lock(object);
944 vm_object_reference_locked(object);
945 named_entry->backing.object = object;
946 named_entry->is_pager = FALSE;
947 named_entry_unlock(named_entry);
948
949 wimg_mode = object->wimg_bits;
950 if(access == MAP_MEM_IO) {
951 wimg_mode = VM_WIMG_IO;
952 } else if (access == MAP_MEM_COPYBACK) {
953 wimg_mode = VM_WIMG_USE_DEFAULT;
954 } else if (access == MAP_MEM_WTHRU) {
955 wimg_mode = VM_WIMG_WTHRU;
956 } else if (access == MAP_MEM_WCOMB) {
957 wimg_mode = VM_WIMG_WCOMB;
958 }
959 if ((wimg_mode == VM_WIMG_IO)
960 || (wimg_mode == VM_WIMG_WCOMB))
961 cache_attr = TRUE;
962 else
963 cache_attr = FALSE;
964
965 /* wait for object (if any) to be ready */
966 if (!named_entry->internal) {
967 while (!object->pager_ready) {
968 vm_object_wait(object,
969 VM_OBJECT_EVENT_PAGER_READY,
970 THREAD_UNINT);
971 vm_object_lock(object);
972 }
973 }
974
975 if(object->wimg_bits != wimg_mode) {
976 vm_page_t p;
977
978 vm_object_paging_wait(object, THREAD_UNINT);
979
980 object->wimg_bits = wimg_mode;
981 queue_iterate(&object->memq, p, vm_page_t, listq) {
982 if (!p->fictitious) {
983 pmap_disconnect(p->phys_page);
984 if (cache_attr)
985 pmap_sync_page_attributes_phys(p->phys_page);
986 }
987 }
988 }
989 object->true_share = TRUE;
990 if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC)
991 object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
992 vm_object_unlock(object);
993 } else {
994 /* This is the case where we are going to map */
995 /* an already mapped object. If the object is */
996 /* not ready it is internal. An external */
997 /* object cannot be mapped until it is ready */
998 /* we can therefore avoid the ready check */
999 /* in this case. */
1000 object = named_entry->backing.object;
1001 assert(object != VM_OBJECT_NULL);
1002 named_entry_unlock(named_entry);
1003 vm_object_reference(object);
1004 }
1005 } else if (ip_kotype(port) == IKOT_MEMORY_OBJECT) {
1006 /*
1007 * JMM - This is temporary until we unify named entries
1008 * and raw memory objects.
1009 *
1010 * Detected fake ip_kotype for a memory object. In
1011 * this case, the port isn't really a port at all, but
1012 * instead is just a raw memory object.
1013 */
1014
1015 if ((object = vm_object_enter((memory_object_t)port,
1016 size, FALSE, FALSE, FALSE))
1017 == VM_OBJECT_NULL)
1018 return(KERN_INVALID_OBJECT);
1019
1020 /* wait for object (if any) to be ready */
1021 if (object != VM_OBJECT_NULL) {
1022 if(object == kernel_object) {
1023 printf("Warning: Attempt to map kernel object"
1024 " by a non-private kernel entity\n");
1025 return(KERN_INVALID_OBJECT);
1026 }
1027 vm_object_lock(object);
1028 while (!object->pager_ready) {
1029 vm_object_wait(object,
1030 VM_OBJECT_EVENT_PAGER_READY,
1031 THREAD_UNINT);
1032 vm_object_lock(object);
1033 }
1034 vm_object_unlock(object);
1035 }
1036 } else {
1037 return (KERN_INVALID_OBJECT);
1038 }
1039
1040 /*
1041 * Perform the copy if requested
1042 */
1043
1044 if (copy) {
1045 vm_object_t new_object;
1046 vm_object_offset_t new_offset;
1047
1048 result = vm_object_copy_strategically(object, offset, size,
1049 &new_object, &new_offset,
1050 &copy);
1051
1052
1053 if (result == KERN_MEMORY_RESTART_COPY) {
1054 boolean_t success;
1055 boolean_t src_needs_copy;
1056
1057 /*
1058 * XXX
1059 * We currently ignore src_needs_copy.
1060 * This really is the issue of how to make
1061 * MEMORY_OBJECT_COPY_SYMMETRIC safe for
1062 * non-kernel users to use. Solution forthcoming.
1063 * In the meantime, since we don't allow non-kernel
1064 * memory managers to specify symmetric copy,
1065 * we won't run into problems here.
1066 */
1067 new_object = object;
1068 new_offset = offset;
1069 success = vm_object_copy_quickly(&new_object,
1070 new_offset, size,
1071 &src_needs_copy,
1072 &copy);
1073 assert(success);
1074 result = KERN_SUCCESS;
1075 }
1076 /*
1077 * Throw away the reference to the
1078 * original object, as it won't be mapped.
1079 */
1080
1081 vm_object_deallocate(object);
1082
1083 if (result != KERN_SUCCESS)
1084 return (result);
1085
1086 object = new_object;
1087 offset = new_offset;
1088 }
1089
1090 if ((result = vm_map_enter(target_map,
1091 &map_addr, map_size,
1092 (vm_map_offset_t)mask,
1093 flags,
1094 object, offset,
1095 copy,
1096 cur_protection, max_protection, inheritance
1097 )) != KERN_SUCCESS)
1098 vm_object_deallocate(object);
1099 *address = map_addr;
1100 return(result);
1101 }
1102
1103
1104 /* legacy interface */
1105 kern_return_t
1106 vm_map_64(
1107 vm_map_t target_map,
1108 vm_offset_t *address,
1109 vm_size_t size,
1110 vm_offset_t mask,
1111 int flags,
1112 ipc_port_t port,
1113 vm_object_offset_t offset,
1114 boolean_t copy,
1115 vm_prot_t cur_protection,
1116 vm_prot_t max_protection,
1117 vm_inherit_t inheritance)
1118 {
1119 mach_vm_address_t map_addr;
1120 mach_vm_size_t map_size;
1121 mach_vm_offset_t map_mask;
1122 kern_return_t kr;
1123
1124 map_addr = (mach_vm_address_t)*address;
1125 map_size = (mach_vm_size_t)size;
1126 map_mask = (mach_vm_offset_t)mask;
1127
1128 kr = mach_vm_map(target_map, &map_addr, map_size, map_mask, flags,
1129 port, offset, copy,
1130 cur_protection, max_protection, inheritance);
1131 *address = CAST_DOWN(vm_address_t, map_addr);
1132 return kr;
1133 }
1134
1135 /* temporary, until world build */
1136 kern_return_t
1137 vm_map(
1138 vm_map_t target_map,
1139 vm_offset_t *address,
1140 vm_size_t size,
1141 vm_offset_t mask,
1142 int flags,
1143 ipc_port_t port,
1144 vm_offset_t offset,
1145 boolean_t copy,
1146 vm_prot_t cur_protection,
1147 vm_prot_t max_protection,
1148 vm_inherit_t inheritance)
1149 {
1150 mach_vm_address_t map_addr;
1151 mach_vm_size_t map_size;
1152 mach_vm_offset_t map_mask;
1153 vm_object_offset_t obj_offset;
1154 kern_return_t kr;
1155
1156 map_addr = (mach_vm_address_t)*address;
1157 map_size = (mach_vm_size_t)size;
1158 map_mask = (mach_vm_offset_t)mask;
1159 obj_offset = (vm_object_offset_t)offset;
1160
1161 kr = mach_vm_map(target_map, &map_addr, map_size, map_mask, flags,
1162 port, obj_offset, copy,
1163 cur_protection, max_protection, inheritance);
1164 *address = CAST_DOWN(vm_address_t, map_addr);
1165 return kr;
1166 }
1167
1168 /*
1169 * mach_vm_remap -
1170 * Remap a range of memory from one task into another,
1171 * to another address range within the same task, or
1172 * over top of itself (with altered permissions and/or
1173 * as an in-place copy of itself).
1174 */
1175
1176 kern_return_t
1177 mach_vm_remap(
1178 vm_map_t target_map,
1179 mach_vm_offset_t *address,
1180 mach_vm_size_t size,
1181 mach_vm_offset_t mask,
1182 boolean_t anywhere,
1183 vm_map_t src_map,
1184 mach_vm_offset_t memory_address,
1185 boolean_t copy,
1186 vm_prot_t *cur_protection,
1187 vm_prot_t *max_protection,
1188 vm_inherit_t inheritance)
1189 {
1190 vm_map_offset_t map_addr;
1191 kern_return_t kr;
1192
1193 if (VM_MAP_NULL == target_map || VM_MAP_NULL == src_map)
1194 return KERN_INVALID_ARGUMENT;
1195
1196 map_addr = (vm_map_offset_t)*address;
1197
1198 kr = vm_map_remap(target_map,
1199 &map_addr,
1200 size,
1201 mask,
1202 anywhere,
1203 src_map,
1204 memory_address,
1205 copy,
1206 cur_protection,
1207 max_protection,
1208 inheritance);
1209 *address = map_addr;
1210 return kr;
1211 }
1212
1213 /*
1214 * vm_remap -
1215 * Remap a range of memory from one task into another,
1216 * to another address range within the same task, or
1217 * over top of itself (with altered permissions and/or
1218 * as an in-place copy of itself).
1219 *
1220 * The addressability of the source and target address
1221 * range is limited by the size of vm_address_t (in the
1222 * kernel context).
1223 */
1224 kern_return_t
1225 vm_remap(
1226 vm_map_t target_map,
1227 vm_offset_t *address,
1228 vm_size_t size,
1229 vm_offset_t mask,
1230 boolean_t anywhere,
1231 vm_map_t src_map,
1232 vm_offset_t memory_address,
1233 boolean_t copy,
1234 vm_prot_t *cur_protection,
1235 vm_prot_t *max_protection,
1236 vm_inherit_t inheritance)
1237 {
1238 vm_map_offset_t map_addr;
1239 kern_return_t kr;
1240
1241 if (VM_MAP_NULL == target_map || VM_MAP_NULL == src_map)
1242 return KERN_INVALID_ARGUMENT;
1243
1244 map_addr = (vm_map_offset_t)*address;
1245
1246 kr = vm_map_remap(target_map,
1247 &map_addr,
1248 size,
1249 mask,
1250 anywhere,
1251 src_map,
1252 memory_address,
1253 copy,
1254 cur_protection,
1255 max_protection,
1256 inheritance);
1257 *address = CAST_DOWN(vm_offset_t, map_addr);
1258 return kr;
1259 }
1260
1261 /*
1262 * NOTE: these routine (and this file) will no longer require mach_host_server.h
1263 * when mach_vm_wire and vm_wire are changed to use ledgers.
1264 */
1265 #include <mach/mach_host_server.h>
1266 /*
1267 * mach_vm_wire
1268 * Specify that the range of the virtual address space
1269 * of the target task must not cause page faults for
1270 * the indicated accesses.
1271 *
1272 * [ To unwire the pages, specify VM_PROT_NONE. ]
1273 */
1274 kern_return_t
1275 mach_vm_wire(
1276 host_priv_t host_priv,
1277 vm_map_t map,
1278 mach_vm_offset_t start,
1279 mach_vm_size_t size,
1280 vm_prot_t access)
1281 {
1282 kern_return_t rc;
1283
1284 if (host_priv == HOST_PRIV_NULL)
1285 return KERN_INVALID_HOST;
1286
1287 assert(host_priv == &realhost);
1288
1289 if (map == VM_MAP_NULL)
1290 return KERN_INVALID_TASK;
1291
1292 if (access & ~VM_PROT_ALL)
1293 return KERN_INVALID_ARGUMENT;
1294
1295 if (access != VM_PROT_NONE) {
1296 rc = vm_map_wire(map, vm_map_trunc_page(start),
1297 vm_map_round_page(start+size), access, TRUE);
1298 } else {
1299 rc = vm_map_unwire(map, vm_map_trunc_page(start),
1300 vm_map_round_page(start+size), TRUE);
1301 }
1302 return rc;
1303 }
1304
1305 /*
1306 * vm_wire -
1307 * Specify that the range of the virtual address space
1308 * of the target task must not cause page faults for
1309 * the indicated accesses.
1310 *
1311 * [ To unwire the pages, specify VM_PROT_NONE. ]
1312 */
1313 kern_return_t
1314 vm_wire(
1315 host_priv_t host_priv,
1316 register vm_map_t map,
1317 vm_offset_t start,
1318 vm_size_t size,
1319 vm_prot_t access)
1320 {
1321 kern_return_t rc;
1322
1323 if (host_priv == HOST_PRIV_NULL)
1324 return KERN_INVALID_HOST;
1325
1326 assert(host_priv == &realhost);
1327
1328 if (map == VM_MAP_NULL)
1329 return KERN_INVALID_TASK;
1330
1331 if ((access & ~VM_PROT_ALL) || (start + size < start))
1332 return KERN_INVALID_ARGUMENT;
1333
1334 if (size == 0) {
1335 rc = KERN_SUCCESS;
1336 } else if (access != VM_PROT_NONE) {
1337 rc = vm_map_wire(map, vm_map_trunc_page(start),
1338 vm_map_round_page(start+size), access, TRUE);
1339 } else {
1340 rc = vm_map_unwire(map, vm_map_trunc_page(start),
1341 vm_map_round_page(start+size), TRUE);
1342 }
1343 return rc;
1344 }
1345
1346 /*
1347 * vm_msync
1348 *
1349 * Synchronises the memory range specified with its backing store
1350 * image by either flushing or cleaning the contents to the appropriate
1351 * memory manager.
1352 *
1353 * interpretation of sync_flags
1354 * VM_SYNC_INVALIDATE - discard pages, only return precious
1355 * pages to manager.
1356 *
1357 * VM_SYNC_INVALIDATE & (VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS)
1358 * - discard pages, write dirty or precious
1359 * pages back to memory manager.
1360 *
1361 * VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS
1362 * - write dirty or precious pages back to
1363 * the memory manager.
1364 *
1365 * VM_SYNC_CONTIGUOUS - does everything normally, but if there
1366 * is a hole in the region, and we would
1367 * have returned KERN_SUCCESS, return
1368 * KERN_INVALID_ADDRESS instead.
1369 *
1370 * RETURNS
1371 * KERN_INVALID_TASK Bad task parameter
1372 * KERN_INVALID_ARGUMENT both sync and async were specified.
1373 * KERN_SUCCESS The usual.
1374 * KERN_INVALID_ADDRESS There was a hole in the region.
1375 */
1376
1377 kern_return_t
1378 mach_vm_msync(
1379 vm_map_t map,
1380 mach_vm_address_t address,
1381 mach_vm_size_t size,
1382 vm_sync_t sync_flags)
1383 {
1384
1385 if (map == VM_MAP_NULL)
1386 return(KERN_INVALID_TASK);
1387
1388 return vm_map_msync(map, (vm_map_address_t)address,
1389 (vm_map_size_t)size, sync_flags);
1390 }
1391
1392 /*
1393 * vm_msync
1394 *
1395 * Synchronises the memory range specified with its backing store
1396 * image by either flushing or cleaning the contents to the appropriate
1397 * memory manager.
1398 *
1399 * interpretation of sync_flags
1400 * VM_SYNC_INVALIDATE - discard pages, only return precious
1401 * pages to manager.
1402 *
1403 * VM_SYNC_INVALIDATE & (VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS)
1404 * - discard pages, write dirty or precious
1405 * pages back to memory manager.
1406 *
1407 * VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS
1408 * - write dirty or precious pages back to
1409 * the memory manager.
1410 *
1411 * VM_SYNC_CONTIGUOUS - does everything normally, but if there
1412 * is a hole in the region, and we would
1413 * have returned KERN_SUCCESS, return
1414 * KERN_INVALID_ADDRESS instead.
1415 *
1416 * The addressability of the range is limited to that which can
1417 * be described by a vm_address_t.
1418 *
1419 * RETURNS
1420 * KERN_INVALID_TASK Bad task parameter
1421 * KERN_INVALID_ARGUMENT both sync and async were specified.
1422 * KERN_SUCCESS The usual.
1423 * KERN_INVALID_ADDRESS There was a hole in the region.
1424 */
1425
1426 kern_return_t
1427 vm_msync(
1428 vm_map_t map,
1429 vm_address_t address,
1430 vm_size_t size,
1431 vm_sync_t sync_flags)
1432 {
1433
1434 if (map == VM_MAP_NULL)
1435 return(KERN_INVALID_TASK);
1436
1437 return vm_map_msync(map, (vm_map_address_t)address,
1438 (vm_map_size_t)size, sync_flags);
1439 }
1440
1441
1442 /*
1443 * mach_vm_behavior_set
1444 *
1445 * Sets the paging behavior attribute for the specified range
1446 * in the specified map.
1447 *
1448 * This routine will fail with KERN_INVALID_ADDRESS if any address
1449 * in [start,start+size) is not a valid allocated memory region.
1450 */
1451 kern_return_t
1452 mach_vm_behavior_set(
1453 vm_map_t map,
1454 mach_vm_offset_t start,
1455 mach_vm_size_t size,
1456 vm_behavior_t new_behavior)
1457 {
1458 if ((map == VM_MAP_NULL) || (start + size < start))
1459 return(KERN_INVALID_ARGUMENT);
1460
1461 if (size == 0)
1462 return KERN_SUCCESS;
1463
1464 return(vm_map_behavior_set(map, vm_map_trunc_page(start),
1465 vm_map_round_page(start+size), new_behavior));
1466 }
1467
1468 /*
1469 * vm_behavior_set
1470 *
1471 * Sets the paging behavior attribute for the specified range
1472 * in the specified map.
1473 *
1474 * This routine will fail with KERN_INVALID_ADDRESS if any address
1475 * in [start,start+size) is not a valid allocated memory region.
1476 *
1477 * This routine is potentially limited in addressibility by the
1478 * use of vm_offset_t (if the map provided is larger than the
1479 * kernel's).
1480 */
1481 kern_return_t
1482 vm_behavior_set(
1483 vm_map_t map,
1484 vm_offset_t start,
1485 vm_size_t size,
1486 vm_behavior_t new_behavior)
1487 {
1488 if ((map == VM_MAP_NULL) || (start + size < start))
1489 return(KERN_INVALID_ARGUMENT);
1490
1491 if (size == 0)
1492 return KERN_SUCCESS;
1493
1494 return(vm_map_behavior_set(map, vm_map_trunc_page(start),
1495 vm_map_round_page(start+size), new_behavior));
1496 }
1497
1498 /*
1499 * mach_vm_region:
1500 *
1501 * User call to obtain information about a region in
1502 * a task's address map. Currently, only one flavor is
1503 * supported.
1504 *
1505 * XXX The reserved and behavior fields cannot be filled
1506 * in until the vm merge from the IK is completed, and
1507 * vm_reserve is implemented.
1508 *
1509 * XXX Dependency: syscall_vm_region() also supports only one flavor.
1510 */
1511
1512 kern_return_t
1513 mach_vm_region(
1514 vm_map_t map,
1515 mach_vm_offset_t *address, /* IN/OUT */
1516 mach_vm_size_t *size, /* OUT */
1517 vm_region_flavor_t flavor, /* IN */
1518 vm_region_info_t info, /* OUT */
1519 mach_msg_type_number_t *count, /* IN/OUT */
1520 mach_port_t *object_name) /* OUT */
1521 {
1522 vm_map_offset_t map_addr;
1523 vm_map_size_t map_size;
1524 kern_return_t kr;
1525
1526 if (VM_MAP_NULL == map)
1527 return KERN_INVALID_ARGUMENT;
1528
1529 map_addr = (vm_map_offset_t)*address;
1530 map_size = (vm_map_size_t)*size;
1531
1532 /* legacy conversion */
1533 if (VM_REGION_BASIC_INFO == flavor)
1534 flavor = VM_REGION_BASIC_INFO_64;
1535
1536 kr = vm_map_region(map,
1537 &map_addr, &map_size,
1538 flavor, info, count,
1539 object_name);
1540
1541 *address = map_addr;
1542 *size = map_size;
1543 return kr;
1544 }
1545
1546 /*
1547 * vm_region_64 and vm_region:
1548 *
1549 * User call to obtain information about a region in
1550 * a task's address map. Currently, only one flavor is
1551 * supported.
1552 *
1553 * XXX The reserved and behavior fields cannot be filled
1554 * in until the vm merge from the IK is completed, and
1555 * vm_reserve is implemented.
1556 *
1557 * XXX Dependency: syscall_vm_region() also supports only one flavor.
1558 */
1559
1560 kern_return_t
1561 vm_region_64(
1562 vm_map_t map,
1563 vm_offset_t *address, /* IN/OUT */
1564 vm_size_t *size, /* OUT */
1565 vm_region_flavor_t flavor, /* IN */
1566 vm_region_info_t info, /* OUT */
1567 mach_msg_type_number_t *count, /* IN/OUT */
1568 mach_port_t *object_name) /* OUT */
1569 {
1570 vm_map_offset_t map_addr;
1571 vm_map_size_t map_size;
1572 kern_return_t kr;
1573
1574 if (VM_MAP_NULL == map)
1575 return KERN_INVALID_ARGUMENT;
1576
1577 map_addr = (vm_map_offset_t)*address;
1578 map_size = (vm_map_size_t)*size;
1579
1580 /* legacy conversion */
1581 if (VM_REGION_BASIC_INFO == flavor)
1582 flavor = VM_REGION_BASIC_INFO_64;
1583
1584 kr = vm_map_region(map,
1585 &map_addr, &map_size,
1586 flavor, info, count,
1587 object_name);
1588
1589 *address = CAST_DOWN(vm_offset_t, map_addr);
1590 *size = CAST_DOWN(vm_size_t, map_size);
1591
1592 if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS)
1593 return KERN_INVALID_ADDRESS;
1594 return kr;
1595 }
1596
1597 kern_return_t
1598 vm_region(
1599 vm_map_t map,
1600 vm_address_t *address, /* IN/OUT */
1601 vm_size_t *size, /* OUT */
1602 vm_region_flavor_t flavor, /* IN */
1603 vm_region_info_t info, /* OUT */
1604 mach_msg_type_number_t *count, /* IN/OUT */
1605 mach_port_t *object_name) /* OUT */
1606 {
1607 vm_map_address_t map_addr;
1608 vm_map_size_t map_size;
1609 kern_return_t kr;
1610
1611 if (VM_MAP_NULL == map)
1612 return KERN_INVALID_ARGUMENT;
1613
1614 map_addr = (vm_map_address_t)*address;
1615 map_size = (vm_map_size_t)*size;
1616
1617 kr = vm_map_region(map,
1618 &map_addr, &map_size,
1619 flavor, info, count,
1620 object_name);
1621
1622 *address = CAST_DOWN(vm_address_t, map_addr);
1623 *size = CAST_DOWN(vm_size_t, map_size);
1624
1625 if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS)
1626 return KERN_INVALID_ADDRESS;
1627 return kr;
1628 }
1629
1630 /*
1631 * vm_region_recurse: A form of vm_region which follows the
1632 * submaps in a target map
1633 *
1634 */
1635 kern_return_t
1636 mach_vm_region_recurse(
1637 vm_map_t map,
1638 mach_vm_address_t *address,
1639 mach_vm_size_t *size,
1640 uint32_t *depth,
1641 vm_region_recurse_info_t info,
1642 mach_msg_type_number_t *infoCnt)
1643 {
1644 vm_map_address_t map_addr;
1645 vm_map_size_t map_size;
1646 kern_return_t kr;
1647
1648 if (VM_MAP_NULL == map)
1649 return KERN_INVALID_ARGUMENT;
1650
1651 map_addr = (vm_map_address_t)*address;
1652 map_size = (vm_map_size_t)*size;
1653
1654 kr = vm_map_region_recurse_64(
1655 map,
1656 &map_addr,
1657 &map_size,
1658 depth,
1659 (vm_region_submap_info_64_t)info,
1660 infoCnt);
1661
1662 *address = map_addr;
1663 *size = map_size;
1664 return kr;
1665 }
1666
1667 /*
1668 * vm_region_recurse: A form of vm_region which follows the
1669 * submaps in a target map
1670 *
1671 */
1672 kern_return_t
1673 vm_region_recurse_64(
1674 vm_map_t map,
1675 vm_address_t *address,
1676 vm_size_t *size,
1677 uint32_t *depth,
1678 vm_region_recurse_info_64_t info,
1679 mach_msg_type_number_t *infoCnt)
1680 {
1681 vm_map_address_t map_addr;
1682 vm_map_size_t map_size;
1683 kern_return_t kr;
1684
1685 if (VM_MAP_NULL == map)
1686 return KERN_INVALID_ARGUMENT;
1687
1688 map_addr = (vm_map_address_t)*address;
1689 map_size = (vm_map_size_t)*size;
1690
1691 kr = vm_map_region_recurse_64(
1692 map,
1693 &map_addr,
1694 &map_size,
1695 depth,
1696 (vm_region_submap_info_64_t)info,
1697 infoCnt);
1698
1699 *address = CAST_DOWN(vm_address_t, map_addr);
1700 *size = CAST_DOWN(vm_size_t, map_size);
1701
1702 if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS)
1703 return KERN_INVALID_ADDRESS;
1704 return kr;
1705 }
1706
1707 kern_return_t
1708 vm_region_recurse(
1709 vm_map_t map,
1710 vm_offset_t *address, /* IN/OUT */
1711 vm_size_t *size, /* OUT */
1712 natural_t *depth, /* IN/OUT */
1713 vm_region_recurse_info_t info32, /* IN/OUT */
1714 mach_msg_type_number_t *infoCnt) /* IN/OUT */
1715 {
1716 vm_region_submap_info_data_64_t info64;
1717 vm_region_submap_info_t info;
1718 vm_map_address_t map_addr;
1719 vm_map_size_t map_size;
1720 kern_return_t kr;
1721
1722 if (VM_MAP_NULL == map || *infoCnt < VM_REGION_SUBMAP_INFO_COUNT)
1723 return KERN_INVALID_ARGUMENT;
1724
1725
1726 map_addr = (vm_map_address_t)*address;
1727 map_size = (vm_map_size_t)*size;
1728 info = (vm_region_submap_info_t)info32;
1729 *infoCnt = VM_REGION_SUBMAP_INFO_COUNT_64;
1730
1731 kr = vm_map_region_recurse_64(map, &map_addr,&map_size,
1732 depth, &info64, infoCnt);
1733
1734 info->protection = info64.protection;
1735 info->max_protection = info64.max_protection;
1736 info->inheritance = info64.inheritance;
1737 info->offset = (uint32_t)info64.offset; /* trouble-maker */
1738 info->user_tag = info64.user_tag;
1739 info->pages_resident = info64.pages_resident;
1740 info->pages_shared_now_private = info64.pages_shared_now_private;
1741 info->pages_swapped_out = info64.pages_swapped_out;
1742 info->pages_dirtied = info64.pages_dirtied;
1743 info->ref_count = info64.ref_count;
1744 info->shadow_depth = info64.shadow_depth;
1745 info->external_pager = info64.external_pager;
1746 info->share_mode = info64.share_mode;
1747 info->is_submap = info64.is_submap;
1748 info->behavior = info64.behavior;
1749 info->object_id = info64.object_id;
1750 info->user_wired_count = info64.user_wired_count;
1751
1752 *address = CAST_DOWN(vm_address_t, map_addr);
1753 *size = CAST_DOWN(vm_size_t, map_size);
1754 *infoCnt = VM_REGION_SUBMAP_INFO_COUNT;
1755
1756 if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS)
1757 return KERN_INVALID_ADDRESS;
1758 return kr;
1759 }
1760
1761 kern_return_t
1762 vm_purgable_control(
1763 vm_map_t map,
1764 vm_offset_t address,
1765 vm_purgable_t control,
1766 int *state)
1767 {
1768 if (VM_MAP_NULL == map)
1769 return KERN_INVALID_ARGUMENT;
1770
1771 return vm_map_purgable_control(map,
1772 vm_map_trunc_page(address),
1773 control,
1774 state);
1775 }
1776
1777
1778 /*
1779 * Ordinarily, the right to allocate CPM is restricted
1780 * to privileged applications (those that can gain access
1781 * to the host priv port). Set this variable to zero if
1782 * you want to let any application allocate CPM.
1783 */
1784 unsigned int vm_allocate_cpm_privileged = 0;
1785
1786 /*
1787 * Allocate memory in the specified map, with the caveat that
1788 * the memory is physically contiguous. This call may fail
1789 * if the system can't find sufficient contiguous memory.
1790 * This call may cause or lead to heart-stopping amounts of
1791 * paging activity.
1792 *
1793 * Memory obtained from this call should be freed in the
1794 * normal way, viz., via vm_deallocate.
1795 */
1796 kern_return_t
1797 vm_allocate_cpm(
1798 host_priv_t host_priv,
1799 vm_map_t map,
1800 vm_address_t *addr,
1801 vm_size_t size,
1802 int flags)
1803 {
1804 vm_map_address_t map_addr;
1805 vm_map_size_t map_size;
1806 kern_return_t kr;
1807
1808 if (vm_allocate_cpm_privileged && HOST_PRIV_NULL == host_priv)
1809 return KERN_INVALID_HOST;
1810
1811 if (VM_MAP_NULL == map)
1812 return KERN_INVALID_ARGUMENT;
1813
1814 map_addr = (vm_map_address_t)*addr;
1815 map_size = (vm_map_size_t)size;
1816
1817 kr = vm_map_enter_cpm(map,
1818 &map_addr,
1819 map_size,
1820 flags);
1821
1822 *addr = CAST_DOWN(vm_address_t, map_addr);
1823 return kr;
1824 }
1825
1826
1827 kern_return_t
1828 mach_vm_page_query(
1829 vm_map_t map,
1830 mach_vm_offset_t offset,
1831 int *disposition,
1832 int *ref_count)
1833 {
1834 if (VM_MAP_NULL == map)
1835 return KERN_INVALID_ARGUMENT;
1836
1837 return vm_map_page_info(map,
1838 vm_map_trunc_page(offset),
1839 disposition, ref_count);
1840 }
1841
1842 kern_return_t
1843 vm_map_page_query(
1844 vm_map_t map,
1845 vm_offset_t offset,
1846 int *disposition,
1847 int *ref_count)
1848 {
1849 if (VM_MAP_NULL == map)
1850 return KERN_INVALID_ARGUMENT;
1851
1852 return vm_map_page_info(map,
1853 vm_map_trunc_page(offset),
1854 disposition, ref_count);
1855 }
1856
1857 /* map a (whole) upl into an address space */
1858 kern_return_t
1859 vm_upl_map(
1860 vm_map_t map,
1861 upl_t upl,
1862 vm_offset_t *dst_addr)
1863 {
1864 vm_map_offset_t map_addr;
1865 kern_return_t kr;
1866
1867 if (VM_MAP_NULL == map)
1868 return KERN_INVALID_ARGUMENT;
1869
1870 kr = vm_map_enter_upl(map, upl, &map_addr);
1871 *dst_addr = CAST_DOWN(vm_offset_t, map_addr);
1872 return kr;
1873 }
1874
1875 kern_return_t
1876 vm_upl_unmap(
1877 vm_map_t map,
1878 upl_t upl)
1879 {
1880 if (VM_MAP_NULL == map)
1881 return KERN_INVALID_ARGUMENT;
1882
1883 return (vm_map_remove_upl(map, upl));
1884 }
1885
1886 /* Retrieve a upl for an object underlying an address range in a map */
1887
1888 kern_return_t
1889 vm_map_get_upl(
1890 vm_map_t map,
1891 vm_map_offset_t map_offset,
1892 upl_size_t *upl_size,
1893 upl_t *upl,
1894 upl_page_info_array_t page_list,
1895 unsigned int *count,
1896 int *flags,
1897 int force_data_sync)
1898 {
1899 int map_flags;
1900 kern_return_t kr;
1901
1902 if (VM_MAP_NULL == map)
1903 return KERN_INVALID_ARGUMENT;
1904
1905 map_flags = *flags & ~UPL_NOZEROFILL;
1906 if (force_data_sync)
1907 map_flags |= UPL_FORCE_DATA_SYNC;
1908
1909 kr = vm_map_create_upl(map,
1910 map_offset,
1911 upl_size,
1912 upl,
1913 page_list,
1914 count,
1915 &map_flags);
1916
1917 *flags = (map_flags & ~UPL_FORCE_DATA_SYNC);
1918 return kr;
1919 }
1920
1921
1922 __private_extern__ kern_return_t
1923 mach_memory_entry_allocate(
1924 vm_named_entry_t *user_entry_p,
1925 ipc_port_t *user_handle_p); /* forward */
1926
1927 /*
1928 * mach_make_memory_entry_64
1929 *
1930 * Think of it as a two-stage vm_remap() operation. First
1931 * you get a handle. Second, you get map that handle in
1932 * somewhere else. Rather than doing it all at once (and
1933 * without needing access to the other whole map).
1934 */
1935
1936 kern_return_t
1937 mach_make_memory_entry_64(
1938 vm_map_t target_map,
1939 memory_object_size_t *size,
1940 memory_object_offset_t offset,
1941 vm_prot_t permission,
1942 ipc_port_t *object_handle,
1943 ipc_port_t parent_handle)
1944 {
1945 vm_map_version_t version;
1946 vm_named_entry_t parent_entry;
1947 vm_named_entry_t user_entry;
1948 ipc_port_t user_handle;
1949 kern_return_t kr;
1950 vm_map_t real_map;
1951
1952 /* needed for call to vm_map_lookup_locked */
1953 boolean_t wired;
1954 vm_object_offset_t obj_off;
1955 vm_prot_t prot;
1956 vm_map_offset_t lo_offset, hi_offset;
1957 vm_behavior_t behavior;
1958 vm_object_t object;
1959 vm_object_t shadow_object;
1960
1961 /* needed for direct map entry manipulation */
1962 vm_map_entry_t map_entry;
1963 vm_map_entry_t next_entry;
1964 vm_map_t local_map;
1965 vm_map_t original_map = target_map;
1966 vm_map_size_t total_size;
1967 vm_map_size_t map_size;
1968 vm_map_offset_t map_offset;
1969 vm_map_offset_t local_offset;
1970 vm_object_size_t mappable_size;
1971
1972 unsigned int access;
1973 vm_prot_t protections;
1974 unsigned int wimg_mode;
1975 boolean_t cache_attr = FALSE;
1976
1977 if (((permission & 0x00FF0000) &
1978 ~(MAP_MEM_ONLY |
1979 MAP_MEM_NAMED_CREATE |
1980 MAP_MEM_PURGABLE |
1981 MAP_MEM_NAMED_REUSE))) {
1982 /*
1983 * Unknown flag: reject for forward compatibility.
1984 */
1985 return KERN_INVALID_VALUE;
1986 }
1987
1988 if (parent_handle != IP_NULL &&
1989 ip_kotype(parent_handle) == IKOT_NAMED_ENTRY) {
1990 parent_entry = (vm_named_entry_t) parent_handle->ip_kobject;
1991 } else {
1992 parent_entry = NULL;
1993 }
1994
1995 protections = permission & VM_PROT_ALL;
1996 access = GET_MAP_MEM(permission);
1997
1998 user_handle = IP_NULL;
1999 user_entry = NULL;
2000
2001 map_offset = vm_map_trunc_page(offset);
2002 map_size = vm_map_round_page(*size);
2003
2004 if (permission & MAP_MEM_ONLY) {
2005 boolean_t parent_is_object;
2006
2007 if (parent_entry == NULL) {
2008 return KERN_INVALID_ARGUMENT;
2009 }
2010
2011 parent_is_object = !(parent_entry->is_sub_map || parent_entry->is_pager);
2012 object = parent_entry->backing.object;
2013 if(parent_is_object && object != VM_OBJECT_NULL)
2014 wimg_mode = object->wimg_bits;
2015 else
2016 wimg_mode = VM_WIMG_DEFAULT;
2017 if((access != GET_MAP_MEM(parent_entry->protection)) &&
2018 !(parent_entry->protection & VM_PROT_WRITE)) {
2019 return KERN_INVALID_RIGHT;
2020 }
2021 if(access == MAP_MEM_IO) {
2022 SET_MAP_MEM(access, parent_entry->protection);
2023 wimg_mode = VM_WIMG_IO;
2024 } else if (access == MAP_MEM_COPYBACK) {
2025 SET_MAP_MEM(access, parent_entry->protection);
2026 wimg_mode = VM_WIMG_DEFAULT;
2027 } else if (access == MAP_MEM_WTHRU) {
2028 SET_MAP_MEM(access, parent_entry->protection);
2029 wimg_mode = VM_WIMG_WTHRU;
2030 } else if (access == MAP_MEM_WCOMB) {
2031 SET_MAP_MEM(access, parent_entry->protection);
2032 wimg_mode = VM_WIMG_WCOMB;
2033 }
2034 if(parent_is_object && object &&
2035 (access != MAP_MEM_NOOP) &&
2036 (!(object->nophyscache))) {
2037 if(object->wimg_bits != wimg_mode) {
2038 vm_page_t p;
2039 if ((wimg_mode == VM_WIMG_IO)
2040 || (wimg_mode == VM_WIMG_WCOMB))
2041 cache_attr = TRUE;
2042 else
2043 cache_attr = FALSE;
2044 vm_object_lock(object);
2045 vm_object_paging_wait(object, THREAD_UNINT);
2046 object->wimg_bits = wimg_mode;
2047 queue_iterate(&object->memq,
2048 p, vm_page_t, listq) {
2049 if (!p->fictitious) {
2050 pmap_disconnect(p->phys_page);
2051 if (cache_attr)
2052 pmap_sync_page_attributes_phys(p->phys_page);
2053 }
2054 }
2055 vm_object_unlock(object);
2056 }
2057 }
2058 if (object_handle)
2059 *object_handle = IP_NULL;
2060 return KERN_SUCCESS;
2061 }
2062
2063 if(permission & MAP_MEM_NAMED_CREATE) {
2064 kr = mach_memory_entry_allocate(&user_entry, &user_handle);
2065 if (kr != KERN_SUCCESS) {
2066 return KERN_FAILURE;
2067 }
2068
2069 /*
2070 * Force the creation of the VM object now.
2071 */
2072 if (map_size > (vm_map_size_t) VM_MAX_ADDRESS) {
2073 /*
2074 * LP64todo - for now, we can only allocate 4GB
2075 * internal objects because the default pager can't
2076 * page bigger ones. Remove this when it can.
2077 */
2078 kr = KERN_FAILURE;
2079 goto make_mem_done;
2080 }
2081
2082 object = vm_object_allocate(map_size);
2083 assert(object != VM_OBJECT_NULL);
2084
2085 if (permission & MAP_MEM_PURGABLE) {
2086 if (! (permission & VM_PROT_WRITE)) {
2087 /* if we can't write, we can't purge */
2088 vm_object_deallocate(object);
2089 kr = KERN_INVALID_ARGUMENT;
2090 goto make_mem_done;
2091 }
2092 object->purgable = VM_OBJECT_PURGABLE_NONVOLATILE;
2093 }
2094
2095 /*
2096 * The VM object is brand new and nobody else knows about it,
2097 * so we don't need to lock it.
2098 */
2099
2100 wimg_mode = object->wimg_bits;
2101 if (access == MAP_MEM_IO) {
2102 wimg_mode = VM_WIMG_IO;
2103 } else if (access == MAP_MEM_COPYBACK) {
2104 wimg_mode = VM_WIMG_DEFAULT;
2105 } else if (access == MAP_MEM_WTHRU) {
2106 wimg_mode = VM_WIMG_WTHRU;
2107 } else if (access == MAP_MEM_WCOMB) {
2108 wimg_mode = VM_WIMG_WCOMB;
2109 }
2110 if (access != MAP_MEM_NOOP) {
2111 object->wimg_bits = wimg_mode;
2112 }
2113 /* the object has no pages, so no WIMG bits to update here */
2114
2115 /*
2116 * XXX
2117 * We use this path when we want to make sure that
2118 * nobody messes with the object (coalesce, for
2119 * example) before we map it.
2120 * We might want to use these objects for transposition via
2121 * vm_object_transpose() too, so we don't want any copy or
2122 * shadow objects either...
2123 */
2124 object->copy_strategy = MEMORY_OBJECT_COPY_NONE;
2125
2126 user_entry->backing.object = object;
2127 user_entry->internal = TRUE;
2128 user_entry->is_sub_map = FALSE;
2129 user_entry->is_pager = FALSE;
2130 user_entry->offset = 0;
2131 user_entry->protection = protections;
2132 SET_MAP_MEM(access, user_entry->protection);
2133 user_entry->size = map_size;
2134
2135 /* user_object pager and internal fields are not used */
2136 /* when the object field is filled in. */
2137
2138 *size = CAST_DOWN(vm_size_t, map_size);
2139 *object_handle = user_handle;
2140 return KERN_SUCCESS;
2141 }
2142
2143 if (parent_entry == NULL ||
2144 (permission & MAP_MEM_NAMED_REUSE)) {
2145
2146 /* Create a named object based on address range within the task map */
2147 /* Go find the object at given address */
2148
2149 redo_lookup:
2150 vm_map_lock_read(target_map);
2151
2152 /* get the object associated with the target address */
2153 /* note we check the permission of the range against */
2154 /* that requested by the caller */
2155
2156 kr = vm_map_lookup_locked(&target_map, map_offset,
2157 protections, &version,
2158 &object, &obj_off, &prot, &wired, &behavior,
2159 &lo_offset, &hi_offset, &real_map);
2160 if (kr != KERN_SUCCESS) {
2161 vm_map_unlock_read(target_map);
2162 goto make_mem_done;
2163 }
2164 if (((prot & protections) != protections)
2165 || (object == kernel_object)) {
2166 kr = KERN_INVALID_RIGHT;
2167 vm_object_unlock(object);
2168 vm_map_unlock_read(target_map);
2169 if(real_map != target_map)
2170 vm_map_unlock_read(real_map);
2171 if(object == kernel_object) {
2172 printf("Warning: Attempt to create a named"
2173 " entry from the kernel_object\n");
2174 }
2175 goto make_mem_done;
2176 }
2177
2178 /* We have an object, now check to see if this object */
2179 /* is suitable. If not, create a shadow and share that */
2180
2181 /*
2182 * We have to unlock the VM object to avoid deadlocking with
2183 * a VM map lock (the lock ordering is map, the object), if we
2184 * need to modify the VM map to create a shadow object. Since
2185 * we might release the VM map lock below anyway, we have
2186 * to release the VM map lock now.
2187 * XXX FBDP There must be a way to avoid this double lookup...
2188 *
2189 * Take an extra reference on the VM object to make sure it's
2190 * not going to disappear.
2191 */
2192 vm_object_reference_locked(object); /* extra ref to hold obj */
2193 vm_object_unlock(object);
2194
2195 local_map = original_map;
2196 local_offset = map_offset;
2197 if(target_map != local_map) {
2198 vm_map_unlock_read(target_map);
2199 if(real_map != target_map)
2200 vm_map_unlock_read(real_map);
2201 vm_map_lock_read(local_map);
2202 target_map = local_map;
2203 real_map = local_map;
2204 }
2205 while(TRUE) {
2206 if(!vm_map_lookup_entry(local_map,
2207 local_offset, &map_entry)) {
2208 kr = KERN_INVALID_ARGUMENT;
2209 vm_map_unlock_read(target_map);
2210 if(real_map != target_map)
2211 vm_map_unlock_read(real_map);
2212 vm_object_deallocate(object); /* release extra ref */
2213 object = VM_OBJECT_NULL;
2214 goto make_mem_done;
2215 }
2216 if(!(map_entry->is_sub_map)) {
2217 if(map_entry->object.vm_object != object) {
2218 kr = KERN_INVALID_ARGUMENT;
2219 vm_map_unlock_read(target_map);
2220 if(real_map != target_map)
2221 vm_map_unlock_read(real_map);
2222 vm_object_deallocate(object); /* release extra ref */
2223 object = VM_OBJECT_NULL;
2224 goto make_mem_done;
2225 }
2226 break;
2227 } else {
2228 vm_map_t tmap;
2229 tmap = local_map;
2230 local_map = map_entry->object.sub_map;
2231
2232 vm_map_lock_read(local_map);
2233 vm_map_unlock_read(tmap);
2234 target_map = local_map;
2235 real_map = local_map;
2236 local_offset = local_offset - map_entry->vme_start;
2237 local_offset += map_entry->offset;
2238 }
2239 }
2240
2241 /*
2242 * We found the VM map entry, lock the VM object again.
2243 */
2244 vm_object_lock(object);
2245 if(map_entry->wired_count) {
2246 /* JMM - The check below should be reworked instead. */
2247 object->true_share = TRUE;
2248 }
2249 if(((map_entry->max_protection) & protections) != protections) {
2250 kr = KERN_INVALID_RIGHT;
2251 vm_object_unlock(object);
2252 vm_map_unlock_read(target_map);
2253 if(real_map != target_map)
2254 vm_map_unlock_read(real_map);
2255 vm_object_deallocate(object);
2256 object = VM_OBJECT_NULL;
2257 goto make_mem_done;
2258 }
2259
2260 mappable_size = hi_offset - obj_off;
2261 total_size = map_entry->vme_end - map_entry->vme_start;
2262 if(map_size > mappable_size) {
2263 /* try to extend mappable size if the entries */
2264 /* following are from the same object and are */
2265 /* compatible */
2266 next_entry = map_entry->vme_next;
2267 /* lets see if the next map entry is still */
2268 /* pointing at this object and is contiguous */
2269 while(map_size > mappable_size) {
2270 if((next_entry->object.vm_object == object) &&
2271 (next_entry->vme_start ==
2272 next_entry->vme_prev->vme_end) &&
2273 (next_entry->offset ==
2274 next_entry->vme_prev->offset +
2275 (next_entry->vme_prev->vme_end -
2276 next_entry->vme_prev->vme_start))) {
2277 if(((next_entry->max_protection)
2278 & protections) != protections) {
2279 break;
2280 }
2281 if (next_entry->needs_copy !=
2282 map_entry->needs_copy)
2283 break;
2284 mappable_size += next_entry->vme_end
2285 - next_entry->vme_start;
2286 total_size += next_entry->vme_end
2287 - next_entry->vme_start;
2288 next_entry = next_entry->vme_next;
2289 } else {
2290 break;
2291 }
2292
2293 }
2294 }
2295
2296 if(object->internal) {
2297 /* vm_map_lookup_locked will create a shadow if */
2298 /* needs_copy is set but does not check for the */
2299 /* other two conditions shown. It is important to */
2300 /* set up an object which will not be pulled from */
2301 /* under us. */
2302
2303 if ((map_entry->needs_copy || object->shadowed ||
2304 (object->size > total_size))
2305 && !object->true_share) {
2306 /*
2307 * We have to unlock the VM object before
2308 * trying to upgrade the VM map lock, to
2309 * honor lock ordering (map then object).
2310 * Otherwise, we would deadlock if another
2311 * thread holds a read lock on the VM map and
2312 * is trying to acquire the VM object's lock.
2313 * We still hold an extra reference on the
2314 * VM object, guaranteeing that it won't
2315 * disappear.
2316 */
2317 vm_object_unlock(object);
2318
2319 if (vm_map_lock_read_to_write(target_map)) {
2320 /*
2321 * We couldn't upgrade our VM map lock
2322 * from "read" to "write" and we lost
2323 * our "read" lock.
2324 * Start all over again...
2325 */
2326 vm_object_deallocate(object); /* extra ref */
2327 target_map = original_map;
2328 goto redo_lookup;
2329 }
2330 vm_object_lock(object);
2331
2332 /*
2333 * JMM - We need to avoid coming here when the object
2334 * is wired by anybody, not just the current map. Why
2335 * couldn't we use the standard vm_object_copy_quickly()
2336 * approach here?
2337 */
2338
2339 /* create a shadow object */
2340 vm_object_shadow(&map_entry->object.vm_object,
2341 &map_entry->offset, total_size);
2342 shadow_object = map_entry->object.vm_object;
2343 vm_object_unlock(object);
2344
2345 vm_object_pmap_protect(
2346 object, map_entry->offset,
2347 total_size,
2348 ((map_entry->is_shared
2349 || target_map->mapped)
2350 ? PMAP_NULL :
2351 target_map->pmap),
2352 map_entry->vme_start,
2353 map_entry->protection & ~VM_PROT_WRITE);
2354 total_size -= (map_entry->vme_end
2355 - map_entry->vme_start);
2356 next_entry = map_entry->vme_next;
2357 map_entry->needs_copy = FALSE;
2358 while (total_size) {
2359 if(next_entry->object.vm_object == object) {
2360 shadow_object->ref_count++;
2361 vm_object_res_reference(shadow_object);
2362 next_entry->object.vm_object
2363 = shadow_object;
2364 vm_object_deallocate(object);
2365 next_entry->offset
2366 = next_entry->vme_prev->offset +
2367 (next_entry->vme_prev->vme_end
2368 - next_entry->vme_prev->vme_start);
2369 next_entry->needs_copy = FALSE;
2370 } else {
2371 panic("mach_make_memory_entry_64:"
2372 " map entries out of sync\n");
2373 }
2374 total_size -=
2375 next_entry->vme_end
2376 - next_entry->vme_start;
2377 next_entry = next_entry->vme_next;
2378 }
2379
2380 /*
2381 * Transfer our extra reference to the
2382 * shadow object.
2383 */
2384 vm_object_reference_locked(shadow_object);
2385 vm_object_deallocate(object); /* extra ref */
2386 object = shadow_object;
2387
2388 obj_off = (local_offset - map_entry->vme_start)
2389 + map_entry->offset;
2390
2391 vm_map_lock_write_to_read(target_map);
2392 vm_object_lock(object);
2393
2394 }
2395 }
2396
2397 /* note: in the future we can (if necessary) allow for */
2398 /* memory object lists, this will better support */
2399 /* fragmentation, but is it necessary? The user should */
2400 /* be encouraged to create address space oriented */
2401 /* shared objects from CLEAN memory regions which have */
2402 /* a known and defined history. i.e. no inheritence */
2403 /* share, make this call before making the region the */
2404 /* target of ipc's, etc. The code above, protecting */
2405 /* against delayed copy, etc. is mostly defensive. */
2406
2407 wimg_mode = object->wimg_bits;
2408 if(!(object->nophyscache)) {
2409 if(access == MAP_MEM_IO) {
2410 wimg_mode = VM_WIMG_IO;
2411 } else if (access == MAP_MEM_COPYBACK) {
2412 wimg_mode = VM_WIMG_USE_DEFAULT;
2413 } else if (access == MAP_MEM_WTHRU) {
2414 wimg_mode = VM_WIMG_WTHRU;
2415 } else if (access == MAP_MEM_WCOMB) {
2416 wimg_mode = VM_WIMG_WCOMB;
2417 }
2418 }
2419
2420 object->true_share = TRUE;
2421 if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC)
2422 object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
2423
2424 /*
2425 * The memory entry now points to this VM object and we
2426 * need to hold a reference on the VM object. Use the extra
2427 * reference we took earlier to keep the object alive when we
2428 * had to unlock it.
2429 */
2430
2431 vm_map_unlock_read(target_map);
2432 if(real_map != target_map)
2433 vm_map_unlock_read(real_map);
2434
2435 if(object->wimg_bits != wimg_mode) {
2436 vm_page_t p;
2437
2438 vm_object_paging_wait(object, THREAD_UNINT);
2439
2440 if ((wimg_mode == VM_WIMG_IO)
2441 || (wimg_mode == VM_WIMG_WCOMB))
2442 cache_attr = TRUE;
2443 else
2444 cache_attr = FALSE;
2445
2446 queue_iterate(&object->memq,
2447 p, vm_page_t, listq) {
2448 if (!p->fictitious) {
2449 pmap_disconnect(p->phys_page);
2450 if (cache_attr)
2451 pmap_sync_page_attributes_phys(p->phys_page);
2452 }
2453 }
2454 object->wimg_bits = wimg_mode;
2455 }
2456
2457 /* the size of mapped entry that overlaps with our region */
2458 /* which is targeted for share. */
2459 /* (entry_end - entry_start) - */
2460 /* offset of our beg addr within entry */
2461 /* it corresponds to this: */
2462
2463 if(map_size > mappable_size)
2464 map_size = mappable_size;
2465
2466 if (permission & MAP_MEM_NAMED_REUSE) {
2467 /*
2468 * Compare what we got with the "parent_entry".
2469 * If they match, re-use the "parent_entry" instead
2470 * of creating a new one.
2471 */
2472 if (parent_entry != NULL &&
2473 parent_entry->backing.object == object &&
2474 parent_entry->internal == object->internal &&
2475 parent_entry->is_sub_map == FALSE &&
2476 parent_entry->is_pager == FALSE &&
2477 parent_entry->offset == obj_off &&
2478 parent_entry->protection == protections &&
2479 parent_entry->size == map_size) {
2480 /*
2481 * We have a match: re-use "parent_entry".
2482 */
2483 /* release our extra reference on object */
2484 vm_object_unlock(object);
2485 vm_object_deallocate(object);
2486 /* parent_entry->ref_count++; XXX ? */
2487 /* Get an extra send-right on handle */
2488 ipc_port_copy_send(parent_handle);
2489 *object_handle = parent_handle;
2490 return KERN_SUCCESS;
2491 } else {
2492 /*
2493 * No match: we need to create a new entry.
2494 * fall through...
2495 */
2496 }
2497 }
2498
2499 vm_object_unlock(object);
2500 if (mach_memory_entry_allocate(&user_entry, &user_handle)
2501 != KERN_SUCCESS) {
2502 /* release our unused reference on the object */
2503 vm_object_deallocate(object);
2504 return KERN_FAILURE;
2505 }
2506
2507 user_entry->backing.object = object;
2508 user_entry->internal = object->internal;
2509 user_entry->is_sub_map = FALSE;
2510 user_entry->is_pager = FALSE;
2511 user_entry->offset = obj_off;
2512 user_entry->protection = permission;
2513 user_entry->size = map_size;
2514
2515 /* user_object pager and internal fields are not used */
2516 /* when the object field is filled in. */
2517
2518 *size = CAST_DOWN(vm_size_t, map_size);
2519 *object_handle = user_handle;
2520 return KERN_SUCCESS;
2521
2522 } else {
2523 /* The new object will be base on an existing named object */
2524
2525 if (parent_entry == NULL) {
2526 kr = KERN_INVALID_ARGUMENT;
2527 goto make_mem_done;
2528 }
2529 if((offset + map_size) > parent_entry->size) {
2530 kr = KERN_INVALID_ARGUMENT;
2531 goto make_mem_done;
2532 }
2533
2534 if((protections & parent_entry->protection) != protections) {
2535 kr = KERN_PROTECTION_FAILURE;
2536 goto make_mem_done;
2537 }
2538
2539 if (mach_memory_entry_allocate(&user_entry, &user_handle)
2540 != KERN_SUCCESS) {
2541 kr = KERN_FAILURE;
2542 goto make_mem_done;
2543 }
2544
2545 user_entry->size = map_size;
2546 user_entry->offset = parent_entry->offset + map_offset;
2547 user_entry->is_sub_map = parent_entry->is_sub_map;
2548 user_entry->is_pager = parent_entry->is_pager;
2549 user_entry->internal = parent_entry->internal;
2550 user_entry->protection = protections;
2551
2552 if(access != MAP_MEM_NOOP) {
2553 SET_MAP_MEM(access, user_entry->protection);
2554 }
2555
2556 if(parent_entry->is_sub_map) {
2557 user_entry->backing.map = parent_entry->backing.map;
2558 vm_map_lock(user_entry->backing.map);
2559 user_entry->backing.map->ref_count++;
2560 vm_map_unlock(user_entry->backing.map);
2561 }
2562 else if (parent_entry->is_pager) {
2563 user_entry->backing.pager = parent_entry->backing.pager;
2564 /* JMM - don't we need a reference here? */
2565 } else {
2566 object = parent_entry->backing.object;
2567 assert(object != VM_OBJECT_NULL);
2568 user_entry->backing.object = object;
2569 /* we now point to this object, hold on */
2570 vm_object_reference(object);
2571 vm_object_lock(object);
2572 object->true_share = TRUE;
2573 if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC)
2574 object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
2575 vm_object_unlock(object);
2576 }
2577 *size = CAST_DOWN(vm_size_t, map_size);
2578 *object_handle = user_handle;
2579 return KERN_SUCCESS;
2580 }
2581
2582 make_mem_done:
2583 if (user_handle != IP_NULL) {
2584 ipc_port_dealloc_kernel(user_handle);
2585 }
2586 if (user_entry != NULL) {
2587 kfree(user_entry, sizeof *user_entry);
2588 }
2589 return kr;
2590 }
2591
2592 kern_return_t
2593 _mach_make_memory_entry(
2594 vm_map_t target_map,
2595 memory_object_size_t *size,
2596 memory_object_offset_t offset,
2597 vm_prot_t permission,
2598 ipc_port_t *object_handle,
2599 ipc_port_t parent_entry)
2600 {
2601 memory_object_offset_t mo_size;
2602 kern_return_t kr;
2603
2604 mo_size = (memory_object_offset_t)*size;
2605 kr = mach_make_memory_entry_64(target_map, &mo_size,
2606 (memory_object_offset_t)offset, permission, object_handle,
2607 parent_entry);
2608 *size = mo_size;
2609 return kr;
2610 }
2611
2612 kern_return_t
2613 mach_make_memory_entry(
2614 vm_map_t target_map,
2615 vm_size_t *size,
2616 vm_offset_t offset,
2617 vm_prot_t permission,
2618 ipc_port_t *object_handle,
2619 ipc_port_t parent_entry)
2620 {
2621 memory_object_offset_t mo_size;
2622 kern_return_t kr;
2623
2624 mo_size = (memory_object_offset_t)*size;
2625 kr = mach_make_memory_entry_64(target_map, &mo_size,
2626 (memory_object_offset_t)offset, permission, object_handle,
2627 parent_entry);
2628 *size = CAST_DOWN(vm_size_t, mo_size);
2629 return kr;
2630 }
2631
2632 /*
2633 * task_wire
2634 *
2635 * Set or clear the map's wiring_required flag. This flag, if set,
2636 * will cause all future virtual memory allocation to allocate
2637 * user wired memory. Unwiring pages wired down as a result of
2638 * this routine is done with the vm_wire interface.
2639 */
2640 kern_return_t
2641 task_wire(
2642 vm_map_t map,
2643 boolean_t must_wire)
2644 {
2645 if (map == VM_MAP_NULL)
2646 return(KERN_INVALID_ARGUMENT);
2647
2648 if (must_wire)
2649 map->wiring_required = TRUE;
2650 else
2651 map->wiring_required = FALSE;
2652
2653 return(KERN_SUCCESS);
2654 }
2655
2656 __private_extern__ kern_return_t
2657 mach_memory_entry_allocate(
2658 vm_named_entry_t *user_entry_p,
2659 ipc_port_t *user_handle_p)
2660 {
2661 vm_named_entry_t user_entry;
2662 ipc_port_t user_handle;
2663 ipc_port_t previous;
2664
2665 user_entry = (vm_named_entry_t) kalloc(sizeof *user_entry);
2666 if (user_entry == NULL)
2667 return KERN_FAILURE;
2668
2669 named_entry_lock_init(user_entry);
2670
2671 user_handle = ipc_port_alloc_kernel();
2672 if (user_handle == IP_NULL) {
2673 kfree(user_entry, sizeof *user_entry);
2674 return KERN_FAILURE;
2675 }
2676 ip_lock(user_handle);
2677
2678 /* make a sonce right */
2679 user_handle->ip_sorights++;
2680 ip_reference(user_handle);
2681
2682 user_handle->ip_destination = IP_NULL;
2683 user_handle->ip_receiver_name = MACH_PORT_NULL;
2684 user_handle->ip_receiver = ipc_space_kernel;
2685
2686 /* make a send right */
2687 user_handle->ip_mscount++;
2688 user_handle->ip_srights++;
2689 ip_reference(user_handle);
2690
2691 ipc_port_nsrequest(user_handle, 1, user_handle, &previous);
2692 /* nsrequest unlocks user_handle */
2693
2694 user_entry->backing.pager = NULL;
2695 user_entry->is_sub_map = FALSE;
2696 user_entry->is_pager = FALSE;
2697 user_entry->size = 0;
2698 user_entry->internal = FALSE;
2699 user_entry->ref_count = 1;
2700
2701 ipc_kobject_set(user_handle, (ipc_kobject_t) user_entry,
2702 IKOT_NAMED_ENTRY);
2703
2704 *user_entry_p = user_entry;
2705 *user_handle_p = user_handle;
2706
2707 return KERN_SUCCESS;
2708 }
2709
2710 /*
2711 * mach_memory_object_memory_entry_64
2712 *
2713 * Create a named entry backed by the provided pager.
2714 *
2715 * JMM - we need to hold a reference on the pager -
2716 * and release it when the named entry is destroyed.
2717 */
2718 kern_return_t
2719 mach_memory_object_memory_entry_64(
2720 host_t host,
2721 boolean_t internal,
2722 vm_object_offset_t size,
2723 vm_prot_t permission,
2724 memory_object_t pager,
2725 ipc_port_t *entry_handle)
2726 {
2727 unsigned int access;
2728 vm_named_entry_t user_entry;
2729 ipc_port_t user_handle;
2730
2731 if (host == HOST_NULL)
2732 return(KERN_INVALID_HOST);
2733
2734 if (mach_memory_entry_allocate(&user_entry, &user_handle)
2735 != KERN_SUCCESS) {
2736 return KERN_FAILURE;
2737 }
2738
2739 user_entry->backing.pager = pager;
2740 user_entry->size = size;
2741 user_entry->offset = 0;
2742 user_entry->protection = permission & VM_PROT_ALL;
2743 access = GET_MAP_MEM(permission);
2744 SET_MAP_MEM(access, user_entry->protection);
2745 user_entry->internal = internal;
2746 user_entry->is_sub_map = FALSE;
2747 user_entry->is_pager = TRUE;
2748 assert(user_entry->ref_count == 1);
2749
2750 *entry_handle = user_handle;
2751 return KERN_SUCCESS;
2752 }
2753
2754 kern_return_t
2755 mach_memory_object_memory_entry(
2756 host_t host,
2757 boolean_t internal,
2758 vm_size_t size,
2759 vm_prot_t permission,
2760 memory_object_t pager,
2761 ipc_port_t *entry_handle)
2762 {
2763 return mach_memory_object_memory_entry_64( host, internal,
2764 (vm_object_offset_t)size, permission, pager, entry_handle);
2765 }
2766
2767
2768 kern_return_t
2769 mach_memory_entry_purgable_control(
2770 ipc_port_t entry_port,
2771 vm_purgable_t control,
2772 int *state)
2773 {
2774 kern_return_t kr;
2775 vm_named_entry_t mem_entry;
2776 vm_object_t object;
2777
2778 if (entry_port == IP_NULL ||
2779 ip_kotype(entry_port) != IKOT_NAMED_ENTRY) {
2780 return KERN_INVALID_ARGUMENT;
2781 }
2782
2783 mem_entry = (vm_named_entry_t) entry_port->ip_kobject;
2784
2785 named_entry_lock(mem_entry);
2786
2787 if (mem_entry->is_sub_map || mem_entry->is_pager) {
2788 named_entry_unlock(mem_entry);
2789 return KERN_INVALID_ARGUMENT;
2790 }
2791
2792 object = mem_entry->backing.object;
2793 if (object == VM_OBJECT_NULL) {
2794 named_entry_unlock(mem_entry);
2795 return KERN_INVALID_ARGUMENT;
2796 }
2797
2798 vm_object_lock(object);
2799
2800 /* check that named entry covers entire object ? */
2801 if (mem_entry->offset != 0 || object->size != mem_entry->size) {
2802 vm_object_unlock(object);
2803 named_entry_unlock(mem_entry);
2804 return KERN_INVALID_ARGUMENT;
2805 }
2806
2807 named_entry_unlock(mem_entry);
2808
2809 kr = vm_object_purgable_control(object, control, state);
2810
2811 vm_object_unlock(object);
2812
2813 return kr;
2814 }
2815
2816 /*
2817 * mach_memory_entry_port_release:
2818 *
2819 * Release a send right on a named entry port. This is the correct
2820 * way to destroy a named entry. When the last right on the port is
2821 * released, ipc_kobject_destroy() will call mach_destroy_memory_entry().
2822 */
2823 void
2824 mach_memory_entry_port_release(
2825 ipc_port_t port)
2826 {
2827 assert(ip_kotype(port) == IKOT_NAMED_ENTRY);
2828 ipc_port_release_send(port);
2829 }
2830
2831 /*
2832 * mach_destroy_memory_entry:
2833 *
2834 * Drops a reference on a memory entry and destroys the memory entry if
2835 * there are no more references on it.
2836 * NOTE: This routine should not be called to destroy a memory entry from the
2837 * kernel, as it will not release the Mach port associated with the memory
2838 * entry. The proper way to destroy a memory entry in the kernel is to
2839 * call mach_memort_entry_port_release() to release the kernel's send-right on
2840 * the memory entry's port. When the last send right is released, the memory
2841 * entry will be destroyed via ipc_kobject_destroy().
2842 */
2843 void
2844 mach_destroy_memory_entry(
2845 ipc_port_t port)
2846 {
2847 vm_named_entry_t named_entry;
2848 #if MACH_ASSERT
2849 assert(ip_kotype(port) == IKOT_NAMED_ENTRY);
2850 #endif /* MACH_ASSERT */
2851 named_entry = (vm_named_entry_t)port->ip_kobject;
2852 mutex_lock(&(named_entry)->Lock);
2853 named_entry->ref_count -= 1;
2854 if(named_entry->ref_count == 0) {
2855 if (named_entry->is_sub_map) {
2856 vm_map_deallocate(named_entry->backing.map);
2857 } else if (!named_entry->is_pager) {
2858 /* release the memory object we've been pointing to */
2859 vm_object_deallocate(named_entry->backing.object);
2860 } /* else JMM - need to drop reference on pager in that case */
2861
2862 mutex_unlock(&(named_entry)->Lock);
2863
2864 kfree((void *) port->ip_kobject,
2865 sizeof (struct vm_named_entry));
2866 } else
2867 mutex_unlock(&(named_entry)->Lock);
2868 }
2869
2870
2871
2872 kern_return_t
2873 set_dp_control_port(
2874 host_priv_t host_priv,
2875 ipc_port_t control_port)
2876 {
2877 if (host_priv == HOST_PRIV_NULL)
2878 return (KERN_INVALID_HOST);
2879
2880 if (IP_VALID(dynamic_pager_control_port))
2881 ipc_port_release_send(dynamic_pager_control_port);
2882
2883 dynamic_pager_control_port = control_port;
2884 return KERN_SUCCESS;
2885 }
2886
2887 kern_return_t
2888 get_dp_control_port(
2889 host_priv_t host_priv,
2890 ipc_port_t *control_port)
2891 {
2892 if (host_priv == HOST_PRIV_NULL)
2893 return (KERN_INVALID_HOST);
2894
2895 *control_port = ipc_port_copy_send(dynamic_pager_control_port);
2896 return KERN_SUCCESS;
2897
2898 }
2899
2900 /* ******* Temporary Internal calls to UPL for BSD ***** */
2901
2902 extern int kernel_upl_map(
2903 vm_map_t map,
2904 upl_t upl,
2905 vm_offset_t *dst_addr);
2906
2907 extern int kernel_upl_unmap(
2908 vm_map_t map,
2909 upl_t upl);
2910
2911 extern int kernel_upl_commit(
2912 upl_t upl,
2913 upl_page_info_t *pl,
2914 mach_msg_type_number_t count);
2915
2916 extern int kernel_upl_commit_range(
2917 upl_t upl,
2918 upl_offset_t offset,
2919 upl_size_t size,
2920 int flags,
2921 upl_page_info_array_t pl,
2922 mach_msg_type_number_t count);
2923
2924 extern int kernel_upl_abort(
2925 upl_t upl,
2926 int abort_type);
2927
2928 extern int kernel_upl_abort_range(
2929 upl_t upl,
2930 upl_offset_t offset,
2931 upl_size_t size,
2932 int abort_flags);
2933
2934
2935 kern_return_t
2936 kernel_upl_map(
2937 vm_map_t map,
2938 upl_t upl,
2939 vm_offset_t *dst_addr)
2940 {
2941 return vm_upl_map(map, upl, dst_addr);
2942 }
2943
2944
2945 kern_return_t
2946 kernel_upl_unmap(
2947 vm_map_t map,
2948 upl_t upl)
2949 {
2950 return vm_upl_unmap(map, upl);
2951 }
2952
2953 kern_return_t
2954 kernel_upl_commit(
2955 upl_t upl,
2956 upl_page_info_t *pl,
2957 mach_msg_type_number_t count)
2958 {
2959 kern_return_t kr;
2960
2961 kr = upl_commit(upl, pl, count);
2962 upl_deallocate(upl);
2963 return kr;
2964 }
2965
2966
2967 kern_return_t
2968 kernel_upl_commit_range(
2969 upl_t upl,
2970 upl_offset_t offset,
2971 upl_size_t size,
2972 int flags,
2973 upl_page_info_array_t pl,
2974 mach_msg_type_number_t count)
2975 {
2976 boolean_t finished = FALSE;
2977 kern_return_t kr;
2978
2979 if (flags & UPL_COMMIT_FREE_ON_EMPTY)
2980 flags |= UPL_COMMIT_NOTIFY_EMPTY;
2981
2982 kr = upl_commit_range(upl, offset, size, flags, pl, count, &finished);
2983
2984 if ((flags & UPL_COMMIT_NOTIFY_EMPTY) && finished)
2985 upl_deallocate(upl);
2986
2987 return kr;
2988 }
2989
2990 kern_return_t
2991 kernel_upl_abort_range(
2992 upl_t upl,
2993 upl_offset_t offset,
2994 upl_size_t size,
2995 int abort_flags)
2996 {
2997 kern_return_t kr;
2998 boolean_t finished = FALSE;
2999
3000 if (abort_flags & UPL_COMMIT_FREE_ON_EMPTY)
3001 abort_flags |= UPL_COMMIT_NOTIFY_EMPTY;
3002
3003 kr = upl_abort_range(upl, offset, size, abort_flags, &finished);
3004
3005 if ((abort_flags & UPL_COMMIT_FREE_ON_EMPTY) && finished)
3006 upl_deallocate(upl);
3007
3008 return kr;
3009 }
3010
3011 kern_return_t
3012 kernel_upl_abort(
3013 upl_t upl,
3014 int abort_type)
3015 {
3016 kern_return_t kr;
3017
3018 kr = upl_abort(upl, abort_type);
3019 upl_deallocate(upl);
3020 return kr;
3021 }
3022
3023 /*
3024 * Now a kernel-private interface (for BootCache
3025 * use only). Need a cleaner way to create an
3026 * empty vm_map() and return a handle to it.
3027 */
3028
3029 kern_return_t
3030 vm_region_object_create(
3031 __unused vm_map_t target_map,
3032 vm_size_t size,
3033 ipc_port_t *object_handle)
3034 {
3035 vm_named_entry_t user_entry;
3036 ipc_port_t user_handle;
3037
3038 vm_map_t new_map;
3039
3040 if (mach_memory_entry_allocate(&user_entry, &user_handle)
3041 != KERN_SUCCESS) {
3042 return KERN_FAILURE;
3043 }
3044
3045 /* Create a named object based on a submap of specified size */
3046
3047 new_map = vm_map_create(PMAP_NULL, VM_MAP_MIN_ADDRESS,
3048 vm_map_round_page(size), TRUE);
3049
3050 user_entry->backing.map = new_map;
3051 user_entry->internal = TRUE;
3052 user_entry->is_sub_map = TRUE;
3053 user_entry->offset = 0;
3054 user_entry->protection = VM_PROT_ALL;
3055 user_entry->size = size;
3056 assert(user_entry->ref_count == 1);
3057
3058 *object_handle = user_handle;
3059 return KERN_SUCCESS;
3060
3061 }
3062
3063 ppnum_t vm_map_get_phys_page( /* forward */
3064 vm_map_t map,
3065 vm_offset_t offset);
3066
3067 ppnum_t
3068 vm_map_get_phys_page(
3069 vm_map_t map,
3070 vm_offset_t addr)
3071 {
3072 vm_object_offset_t offset;
3073 vm_object_t object;
3074 vm_map_offset_t map_offset;
3075 vm_map_entry_t entry;
3076 ppnum_t phys_page = 0;
3077
3078 map_offset = vm_map_trunc_page(addr);
3079
3080 vm_map_lock(map);
3081 while (vm_map_lookup_entry(map, map_offset, &entry)) {
3082
3083 if (entry->object.vm_object == VM_OBJECT_NULL) {
3084 vm_map_unlock(map);
3085 return (ppnum_t) 0;
3086 }
3087 if (entry->is_sub_map) {
3088 vm_map_t old_map;
3089 vm_map_lock(entry->object.sub_map);
3090 old_map = map;
3091 map = entry->object.sub_map;
3092 map_offset = entry->offset + (map_offset - entry->vme_start);
3093 vm_map_unlock(old_map);
3094 continue;
3095 }
3096 if (entry->object.vm_object->phys_contiguous) {
3097 /* These are not standard pageable memory mappings */
3098 /* If they are not present in the object they will */
3099 /* have to be picked up from the pager through the */
3100 /* fault mechanism. */
3101 if(entry->object.vm_object->shadow_offset == 0) {
3102 /* need to call vm_fault */
3103 vm_map_unlock(map);
3104 vm_fault(map, map_offset, VM_PROT_NONE,
3105 FALSE, THREAD_UNINT, NULL, 0);
3106 vm_map_lock(map);
3107 continue;
3108 }
3109 offset = entry->offset + (map_offset - entry->vme_start);
3110 phys_page = (ppnum_t)
3111 ((entry->object.vm_object->shadow_offset
3112 + offset) >> 12);
3113 break;
3114
3115 }
3116 offset = entry->offset + (map_offset - entry->vme_start);
3117 object = entry->object.vm_object;
3118 vm_object_lock(object);
3119 while (TRUE) {
3120 vm_page_t dst_page = vm_page_lookup(object,offset);
3121 if(dst_page == VM_PAGE_NULL) {
3122 if(object->shadow) {
3123 vm_object_t old_object;
3124 vm_object_lock(object->shadow);
3125 old_object = object;
3126 offset = offset + object->shadow_offset;
3127 object = object->shadow;
3128 vm_object_unlock(old_object);
3129 } else {
3130 vm_object_unlock(object);
3131 break;
3132 }
3133 } else {
3134 phys_page = (ppnum_t)(dst_page->phys_page);
3135 vm_object_unlock(object);
3136 break;
3137 }
3138 }
3139 break;
3140
3141 }
3142
3143 vm_map_unlock(map);
3144 return phys_page;
3145 }
3146
3147
3148
3149 kern_return_t kernel_object_iopl_request( /* forward */
3150 vm_named_entry_t named_entry,
3151 memory_object_offset_t offset,
3152 vm_size_t *upl_size,
3153 upl_t *upl_ptr,
3154 upl_page_info_array_t user_page_list,
3155 unsigned int *page_list_count,
3156 int *flags);
3157
3158 kern_return_t
3159 kernel_object_iopl_request(
3160 vm_named_entry_t named_entry,
3161 memory_object_offset_t offset,
3162 vm_size_t *upl_size,
3163 upl_t *upl_ptr,
3164 upl_page_info_array_t user_page_list,
3165 unsigned int *page_list_count,
3166 int *flags)
3167 {
3168 vm_object_t object;
3169 kern_return_t ret;
3170
3171 int caller_flags;
3172
3173 caller_flags = *flags;
3174
3175 if (caller_flags & ~UPL_VALID_FLAGS) {
3176 /*
3177 * For forward compatibility's sake,
3178 * reject any unknown flag.
3179 */
3180 return KERN_INVALID_VALUE;
3181 }
3182
3183 /* a few checks to make sure user is obeying rules */
3184 if(*upl_size == 0) {
3185 if(offset >= named_entry->size)
3186 return(KERN_INVALID_RIGHT);
3187 *upl_size = named_entry->size - offset;
3188 }
3189 if(caller_flags & UPL_COPYOUT_FROM) {
3190 if((named_entry->protection & VM_PROT_READ)
3191 != VM_PROT_READ) {
3192 return(KERN_INVALID_RIGHT);
3193 }
3194 } else {
3195 if((named_entry->protection &
3196 (VM_PROT_READ | VM_PROT_WRITE))
3197 != (VM_PROT_READ | VM_PROT_WRITE)) {
3198 return(KERN_INVALID_RIGHT);
3199 }
3200 }
3201 if(named_entry->size < (offset + *upl_size))
3202 return(KERN_INVALID_ARGUMENT);
3203
3204 /* the callers parameter offset is defined to be the */
3205 /* offset from beginning of named entry offset in object */
3206 offset = offset + named_entry->offset;
3207
3208 if(named_entry->is_sub_map)
3209 return (KERN_INVALID_ARGUMENT);
3210
3211 named_entry_lock(named_entry);
3212
3213 if (named_entry->is_pager) {
3214 object = vm_object_enter(named_entry->backing.pager,
3215 named_entry->offset + named_entry->size,
3216 named_entry->internal,
3217 FALSE,
3218 FALSE);
3219 if (object == VM_OBJECT_NULL) {
3220 named_entry_unlock(named_entry);
3221 return(KERN_INVALID_OBJECT);
3222 }
3223
3224 /* JMM - drop reference on the pager here? */
3225
3226 /* create an extra reference for the object */
3227 vm_object_lock(object);
3228 vm_object_reference_locked(object);
3229 named_entry->backing.object = object;
3230 named_entry->is_pager = FALSE;
3231 named_entry_unlock(named_entry);
3232
3233 /* wait for object (if any) to be ready */
3234 if (!named_entry->internal) {
3235 while (!object->pager_ready) {
3236 vm_object_wait(object,
3237 VM_OBJECT_EVENT_PAGER_READY,
3238 THREAD_UNINT);
3239 vm_object_lock(object);
3240 }
3241 }
3242 vm_object_unlock(object);
3243
3244 } else {
3245 /* This is the case where we are going to operate */
3246 /* an an already known object. If the object is */
3247 /* not ready it is internal. An external */
3248 /* object cannot be mapped until it is ready */
3249 /* we can therefore avoid the ready check */
3250 /* in this case. */
3251 object = named_entry->backing.object;
3252 vm_object_reference(object);
3253 named_entry_unlock(named_entry);
3254 }
3255
3256 if (!object->private) {
3257 if (*upl_size > (MAX_UPL_TRANSFER*PAGE_SIZE))
3258 *upl_size = (MAX_UPL_TRANSFER*PAGE_SIZE);
3259 if (object->phys_contiguous) {
3260 *flags = UPL_PHYS_CONTIG;
3261 } else {
3262 *flags = 0;
3263 }
3264 } else {
3265 *flags = UPL_DEV_MEMORY | UPL_PHYS_CONTIG;
3266 }
3267
3268 ret = vm_object_iopl_request(object,
3269 offset,
3270 *upl_size,
3271 upl_ptr,
3272 user_page_list,
3273 page_list_count,
3274 caller_flags);
3275 vm_object_deallocate(object);
3276 return ret;
3277 }
mirror of XNU