]> git.saurik.com Git - apple/xnu.git/blob - osfmk/vm/vm_user.c
projects / apple / xnu.git / blob
? search:


95cd96294759775a556fb719fca3a2f4b16b65de
[apple/xnu.git] / osfmk / vm / vm_user.c
1 /*
2 * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_LICENSE_HEADER_START@
5 *
6 * The contents of this file constitute Original Code as defined in and
7 * are subject to the Apple Public Source License Version 1.1 (the
8 * "License"). You may not use this file except in compliance with the
9 * License. Please obtain a copy of the License at
10 * http://www.apple.com/publicsource and read it before using this file.
11 *
12 * This Original Code and all software distributed under the License are
13 * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
14 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
15 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
17 * License for the specific language governing rights and limitations
18 * under the License.
19 *
20 * @APPLE_LICENSE_HEADER_END@
21 */
22 /*
23 * @OSF_COPYRIGHT@
24 */
25 /*
26 * Mach Operating System
27 * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University
28 * All Rights Reserved.
29 *
30 * Permission to use, copy, modify and distribute this software and its
31 * documentation is hereby granted, provided that both the copyright
32 * notice and this permission notice appear in all copies of the
33 * software, derivative works or modified versions, and any portions
34 * thereof, and that both notices appear in supporting documentation.
35 *
36 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
37 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
38 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
39 *
40 * Carnegie Mellon requests users of this software to return to
41 *
42 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
43 * School of Computer Science
44 * Carnegie Mellon University
45 * Pittsburgh PA 15213-3890
46 *
47 * any improvements or extensions that they make and grant Carnegie Mellon
48 * the rights to redistribute these changes.
49 */
50 /*
51 */
52 /*
53 * File: vm/vm_user.c
54 * Author: Avadis Tevanian, Jr., Michael Wayne Young
55 *
56 * User-exported virtual memory functions.
57 */
58
59 #include <debug.h>
60
61 #include <vm_cpm.h>
62 #include <mach/boolean.h>
63 #include <mach/kern_return.h>
64 #include <mach/mach_types.h> /* to get vm_address_t */
65 #include <mach/memory_object.h>
66 #include <mach/std_types.h> /* to get pointer_t */
67 #include <mach/upl.h>
68 #include <mach/vm_attributes.h>
69 #include <mach/vm_param.h>
70 #include <mach/vm_statistics.h>
71 #include <mach/mach_syscalls.h>
72
73 #include <mach/host_priv_server.h>
74 #include <mach/mach_vm_server.h>
75 #include <mach/shared_memory_server.h>
76 #include <mach/vm_map_server.h>
77 #include <vm/vm_shared_memory_server.h>
78
79 #include <kern/host.h>
80 #include <kern/kalloc.h>
81 #include <kern/task.h>
82 #include <kern/misc_protos.h>
83 #include <vm/vm_fault.h>
84 #include <vm/vm_map.h>
85 #include <vm/vm_object.h>
86 #include <vm/vm_page.h>
87 #include <vm/memory_object.h>
88 #include <vm/vm_pageout.h>
89 #include <vm/vm_protos.h>
90
91 vm_size_t upl_offset_to_pagelist = 0;
92
93 #if VM_CPM
94 #include <vm/cpm.h>
95 #endif /* VM_CPM */
96
97 ipc_port_t dynamic_pager_control_port=NULL;
98
99 /*
100 * mach_vm_allocate allocates "zero fill" memory in the specfied
101 * map.
102 */
103 kern_return_t
104 mach_vm_allocate(
105 vm_map_t map,
106 mach_vm_offset_t *addr,
107 mach_vm_size_t size,
108 int flags)
109 {
110 vm_map_offset_t map_addr;
111 vm_map_size_t map_size;
112 kern_return_t result;
113 boolean_t anywhere = ((VM_FLAGS_ANYWHERE & flags) != 0);
114
115 if (map == VM_MAP_NULL)
116 return(KERN_INVALID_ARGUMENT);
117 if (size == 0) {
118 *addr = 0;
119 return(KERN_SUCCESS);
120 }
121
122 if (anywhere) {
123 /*
124 * No specific address requested, so start candidate address
125 * search at the minimum address in the map. However, if that
126 * minimum is 0, bump it up by PAGE_SIZE. We want to limit
127 * allocations of PAGEZERO to explicit requests since its
128 * normal use is to catch dereferences of NULL and many
129 * applications also treat pointers with a value of 0 as
130 * special and suddenly having address 0 contain useable
131 * memory would tend to confuse those applications.
132 */
133 map_addr = vm_map_min(map);
134 if (map_addr == 0)
135 map_addr += PAGE_SIZE;
136 } else
137 map_addr = vm_map_trunc_page(*addr);
138 map_size = vm_map_round_page(size);
139 if (map_size == 0) {
140 return(KERN_INVALID_ARGUMENT);
141 }
142
143 result = vm_map_enter(
144 map,
145 &map_addr,
146 map_size,
147 (vm_map_offset_t)0,
148 flags,
149 VM_OBJECT_NULL,
150 (vm_object_offset_t)0,
151 FALSE,
152 VM_PROT_DEFAULT,
153 VM_PROT_ALL,
154 VM_INHERIT_DEFAULT);
155
156 *addr = map_addr;
157 return(result);
158 }
159
160 /*
161 * vm_allocate
162 * Legacy routine that allocates "zero fill" memory in the specfied
163 * map (which is limited to the same size as the kernel).
164 */
165 kern_return_t
166 vm_allocate(
167 vm_map_t map,
168 vm_offset_t *addr,
169 vm_size_t size,
170 int flags)
171 {
172 vm_map_offset_t map_addr;
173 vm_map_size_t map_size;
174 kern_return_t result;
175 boolean_t anywhere = ((VM_FLAGS_ANYWHERE & flags) != 0);
176
177 if (map == VM_MAP_NULL)
178 return(KERN_INVALID_ARGUMENT);
179 if (size == 0) {
180 *addr = 0;
181 return(KERN_SUCCESS);
182 }
183
184 if (anywhere) {
185 /*
186 * No specific address requested, so start candidate address
187 * search at the minimum address in the map. However, if that
188 * minimum is 0, bump it up by PAGE_SIZE. We want to limit
189 * allocations of PAGEZERO to explicit requests since its
190 * normal use is to catch dereferences of NULL and many
191 * applications also treat pointers with a value of 0 as
192 * special and suddenly having address 0 contain useable
193 * memory would tend to confuse those applications.
194 */
195 map_addr = vm_map_min(map);
196 if (map_addr == 0)
197 map_addr += PAGE_SIZE;
198 } else
199 map_addr = vm_map_trunc_page(*addr);
200 map_size = vm_map_round_page(size);
201 if (map_size == 0) {
202 return(KERN_INVALID_ARGUMENT);
203 }
204
205 result = vm_map_enter(
206 map,
207 &map_addr,
208 map_size,
209 (vm_map_offset_t)0,
210 flags,
211 VM_OBJECT_NULL,
212 (vm_object_offset_t)0,
213 FALSE,
214 VM_PROT_DEFAULT,
215 VM_PROT_ALL,
216 VM_INHERIT_DEFAULT);
217
218 *addr = CAST_DOWN(vm_offset_t, map_addr);
219 return(result);
220 }
221
222 /*
223 * mach_vm_deallocate -
224 * deallocates the specified range of addresses in the
225 * specified address map.
226 */
227 kern_return_t
228 mach_vm_deallocate(
229 vm_map_t map,
230 mach_vm_offset_t start,
231 mach_vm_size_t size)
232 {
233 if ((map == VM_MAP_NULL) || (start + size < start))
234 return(KERN_INVALID_ARGUMENT);
235
236 if (size == (mach_vm_offset_t) 0)
237 return(KERN_SUCCESS);
238
239 return(vm_map_remove(map, vm_map_trunc_page(start),
240 vm_map_round_page(start+size), VM_MAP_NO_FLAGS));
241 }
242
243 /*
244 * vm_deallocate -
245 * deallocates the specified range of addresses in the
246 * specified address map (limited to addresses the same
247 * size as the kernel).
248 */
249 kern_return_t
250 vm_deallocate(
251 register vm_map_t map,
252 vm_offset_t start,
253 vm_size_t size)
254 {
255 if ((map == VM_MAP_NULL) || (start + size < start))
256 return(KERN_INVALID_ARGUMENT);
257
258 if (size == (vm_offset_t) 0)
259 return(KERN_SUCCESS);
260
261 return(vm_map_remove(map, vm_map_trunc_page(start),
262 vm_map_round_page(start+size), VM_MAP_NO_FLAGS));
263 }
264
265 /*
266 * mach_vm_inherit -
267 * Sets the inheritance of the specified range in the
268 * specified map.
269 */
270 kern_return_t
271 mach_vm_inherit(
272 vm_map_t map,
273 mach_vm_offset_t start,
274 mach_vm_size_t size,
275 vm_inherit_t new_inheritance)
276 {
277 if ((map == VM_MAP_NULL) || (start + size < start) ||
278 (new_inheritance > VM_INHERIT_LAST_VALID))
279 return(KERN_INVALID_ARGUMENT);
280
281 if (size == 0)
282 return KERN_SUCCESS;
283
284 return(vm_map_inherit(map,
285 vm_map_trunc_page(start),
286 vm_map_round_page(start+size),
287 new_inheritance));
288 }
289
290 /*
291 * vm_inherit -
292 * Sets the inheritance of the specified range in the
293 * specified map (range limited to addresses
294 */
295 kern_return_t
296 vm_inherit(
297 register vm_map_t map,
298 vm_offset_t start,
299 vm_size_t size,
300 vm_inherit_t new_inheritance)
301 {
302 if ((map == VM_MAP_NULL) || (start + size < start) ||
303 (new_inheritance > VM_INHERIT_LAST_VALID))
304 return(KERN_INVALID_ARGUMENT);
305
306 if (size == 0)
307 return KERN_SUCCESS;
308
309 return(vm_map_inherit(map,
310 vm_map_trunc_page(start),
311 vm_map_round_page(start+size),
312 new_inheritance));
313 }
314
315 /*
316 * mach_vm_protect -
317 * Sets the protection of the specified range in the
318 * specified map.
319 */
320
321 kern_return_t
322 mach_vm_protect(
323 vm_map_t map,
324 mach_vm_offset_t start,
325 mach_vm_size_t size,
326 boolean_t set_maximum,
327 vm_prot_t new_protection)
328 {
329 if ((map == VM_MAP_NULL) || (start + size < start) ||
330 (new_protection & ~(VM_PROT_ALL | VM_PROT_COPY)))
331 return(KERN_INVALID_ARGUMENT);
332
333 if (size == 0)
334 return KERN_SUCCESS;
335
336 return(vm_map_protect(map,
337 vm_map_trunc_page(start),
338 vm_map_round_page(start+size),
339 new_protection,
340 set_maximum));
341 }
342
343 /*
344 * vm_protect -
345 * Sets the protection of the specified range in the
346 * specified map. Addressability of the range limited
347 * to the same size as the kernel.
348 */
349
350 kern_return_t
351 vm_protect(
352 vm_map_t map,
353 vm_offset_t start,
354 vm_size_t size,
355 boolean_t set_maximum,
356 vm_prot_t new_protection)
357 {
358 if ((map == VM_MAP_NULL) || (start + size < start) ||
359 (new_protection & ~(VM_PROT_ALL | VM_PROT_COPY)))
360 return(KERN_INVALID_ARGUMENT);
361
362 if (size == 0)
363 return KERN_SUCCESS;
364
365 return(vm_map_protect(map,
366 vm_map_trunc_page(start),
367 vm_map_round_page(start+size),
368 new_protection,
369 set_maximum));
370 }
371
372 /*
373 * mach_vm_machine_attributes -
374 * Handle machine-specific attributes for a mapping, such
375 * as cachability, migrability, etc.
376 */
377 kern_return_t
378 mach_vm_machine_attribute(
379 vm_map_t map,
380 mach_vm_address_t addr,
381 mach_vm_size_t size,
382 vm_machine_attribute_t attribute,
383 vm_machine_attribute_val_t* value) /* IN/OUT */
384 {
385 if ((map == VM_MAP_NULL) || (addr + size < addr))
386 return(KERN_INVALID_ARGUMENT);
387
388 if (size == 0)
389 return KERN_SUCCESS;
390
391 return vm_map_machine_attribute(map,
392 vm_map_trunc_page(addr),
393 vm_map_round_page(addr+size),
394 attribute,
395 value);
396 }
397
398 /*
399 * vm_machine_attribute -
400 * Handle machine-specific attributes for a mapping, such
401 * as cachability, migrability, etc. Limited addressability
402 * (same range limits as for the native kernel map).
403 */
404 kern_return_t
405 vm_machine_attribute(
406 vm_map_t map,
407 vm_address_t addr,
408 vm_size_t size,
409 vm_machine_attribute_t attribute,
410 vm_machine_attribute_val_t* value) /* IN/OUT */
411 {
412 if ((map == VM_MAP_NULL) || (addr + size < addr))
413 return(KERN_INVALID_ARGUMENT);
414
415 if (size == 0)
416 return KERN_SUCCESS;
417
418 return vm_map_machine_attribute(map,
419 vm_map_trunc_page(addr),
420 vm_map_round_page(addr+size),
421 attribute,
422 value);
423 }
424
425 /*
426 * mach_vm_read -
427 * Read/copy a range from one address space and return it to the caller.
428 *
429 * It is assumed that the address for the returned memory is selected by
430 * the IPC implementation as part of receiving the reply to this call.
431 * If IPC isn't used, the caller must deal with the vm_map_copy_t object
432 * that gets returned.
433 *
434 * JMM - because of mach_msg_type_number_t, this call is limited to a
435 * single 4GB region at this time.
436 *
437 */
438 kern_return_t
439 mach_vm_read(
440 vm_map_t map,
441 mach_vm_address_t addr,
442 mach_vm_size_t size,
443 pointer_t *data,
444 mach_msg_type_number_t *data_size)
445 {
446 kern_return_t error;
447 vm_map_copy_t ipc_address;
448
449 if (map == VM_MAP_NULL)
450 return(KERN_INVALID_ARGUMENT);
451
452
453 error = vm_map_copyin(map,
454 (vm_map_address_t)addr,
455 (vm_map_size_t)size,
456 FALSE, /* src_destroy */
457 &ipc_address);
458
459 if (KERN_SUCCESS == error) {
460 *data = (pointer_t) ipc_address;
461 *data_size = size;
462 }
463 return(error);
464 }
465
466 /*
467 * vm_read -
468 * Read/copy a range from one address space and return it to the caller.
469 * Limited addressability (same range limits as for the native kernel map).
470 *
471 * It is assumed that the address for the returned memory is selected by
472 * the IPC implementation as part of receiving the reply to this call.
473 * If IPC isn't used, the caller must deal with the vm_map_copy_t object
474 * that gets returned.
475 */
476 kern_return_t
477 vm_read(
478 vm_map_t map,
479 vm_address_t addr,
480 vm_size_t size,
481 pointer_t *data,
482 mach_msg_type_number_t *data_size)
483 {
484 kern_return_t error;
485 vm_map_copy_t ipc_address;
486
487 if (map == VM_MAP_NULL)
488 return(KERN_INVALID_ARGUMENT);
489
490 error = vm_map_copyin(map,
491 (vm_map_address_t)addr,
492 (vm_map_size_t)size,
493 FALSE, /* src_destroy */
494 &ipc_address);
495
496 if (KERN_SUCCESS == error) {
497 *data = (pointer_t) ipc_address;
498 *data_size = size;
499 }
500 return(error);
501 }
502
503 /*
504 * mach_vm_read_list -
505 * Read/copy a list of address ranges from specified map.
506 *
507 * MIG does not know how to deal with a returned array of
508 * vm_map_copy_t structures, so we have to do the copyout
509 * manually here.
510 */
511 kern_return_t
512 mach_vm_read_list(
513 vm_map_t map,
514 mach_vm_read_entry_t data_list,
515 natural_t count)
516 {
517 mach_msg_type_number_t i;
518 kern_return_t error;
519 vm_map_copy_t copy;
520
521 if (map == VM_MAP_NULL ||
522 count > VM_MAP_ENTRY_MAX)
523 return(KERN_INVALID_ARGUMENT);
524
525 error = KERN_SUCCESS;
526 for(i=0; i<count; i++) {
527 vm_map_address_t map_addr;
528 vm_map_size_t map_size;
529
530 map_addr = (vm_map_address_t)(data_list[i].address);
531 map_size = (vm_map_size_t)(data_list[i].size);
532
533 if(map_size != 0) {
534 error = vm_map_copyin(map,
535 map_addr,
536 map_size,
537 FALSE, /* src_destroy */
538 &copy);
539 if (KERN_SUCCESS == error) {
540 error = vm_map_copyout(
541 current_task()->map,
542 &map_addr,
543 copy);
544 if (KERN_SUCCESS == error) {
545 data_list[i].address = map_addr;
546 continue;
547 }
548 vm_map_copy_discard(copy);
549 }
550 }
551 data_list[i].address = (mach_vm_address_t)0;
552 data_list[i].size = (mach_vm_size_t)0;
553 }
554 return(error);
555 }
556
557 /*
558 * vm_read_list -
559 * Read/copy a list of address ranges from specified map.
560 *
561 * MIG does not know how to deal with a returned array of
562 * vm_map_copy_t structures, so we have to do the copyout
563 * manually here.
564 *
565 * The source and destination ranges are limited to those
566 * that can be described with a vm_address_t (i.e. same
567 * size map as the kernel).
568 *
569 * JMM - If the result of the copyout is an address range
570 * that cannot be described with a vm_address_t (i.e. the
571 * caller had a larger address space but used this call
572 * anyway), it will result in a truncated address being
573 * returned (and a likely confused caller).
574 */
575
576 kern_return_t
577 vm_read_list(
578 vm_map_t map,
579 vm_read_entry_t data_list,
580 natural_t count)
581 {
582 mach_msg_type_number_t i;
583 kern_return_t error;
584 vm_map_copy_t copy;
585
586 if (map == VM_MAP_NULL ||
587 count > VM_MAP_ENTRY_MAX)
588 return(KERN_INVALID_ARGUMENT);
589
590 error = KERN_SUCCESS;
591 for(i=0; i<count; i++) {
592 vm_map_address_t map_addr;
593 vm_map_size_t map_size;
594
595 map_addr = (vm_map_address_t)(data_list[i].address);
596 map_size = (vm_map_size_t)(data_list[i].size);
597
598 if(map_size != 0) {
599 error = vm_map_copyin(map,
600 map_addr,
601 map_size,
602 FALSE, /* src_destroy */
603 &copy);
604 if (KERN_SUCCESS == error) {
605 error = vm_map_copyout(current_task()->map,
606 &map_addr,
607 copy);
608 if (KERN_SUCCESS == error) {
609 data_list[i].address =
610 CAST_DOWN(vm_offset_t, map_addr);
611 continue;
612 }
613 vm_map_copy_discard(copy);
614 }
615 }
616 data_list[i].address = (mach_vm_address_t)0;
617 data_list[i].size = (mach_vm_size_t)0;
618 }
619 return(error);
620 }
621
622 /*
623 * mach_vm_read_overwrite -
624 * Overwrite a range of the current map with data from the specified
625 * map/address range.
626 *
627 * In making an assumption that the current thread is local, it is
628 * no longer cluster-safe without a fully supportive local proxy
629 * thread/task (but we don't support cluster's anymore so this is moot).
630 */
631
632 kern_return_t
633 mach_vm_read_overwrite(
634 vm_map_t map,
635 mach_vm_address_t address,
636 mach_vm_size_t size,
637 mach_vm_address_t data,
638 mach_vm_size_t *data_size)
639 {
640 kern_return_t error;
641 vm_map_copy_t copy;
642
643 if (map == VM_MAP_NULL)
644 return(KERN_INVALID_ARGUMENT);
645
646 error = vm_map_copyin(map, (vm_map_address_t)address,
647 (vm_map_size_t)size, FALSE, &copy);
648
649 if (KERN_SUCCESS == error) {
650 error = vm_map_copy_overwrite(current_thread()->map,
651 (vm_map_address_t)data,
652 copy, FALSE);
653 if (KERN_SUCCESS == error) {
654 *data_size = size;
655 return error;
656 }
657 vm_map_copy_discard(copy);
658 }
659 return(error);
660 }
661
662 /*
663 * vm_read_overwrite -
664 * Overwrite a range of the current map with data from the specified
665 * map/address range.
666 *
667 * This routine adds the additional limitation that the source and
668 * destination ranges must be describable with vm_address_t values
669 * (i.e. the same size address spaces as the kernel, or at least the
670 * the ranges are in that first portion of the respective address
671 * spaces).
672 */
673
674 kern_return_t
675 vm_read_overwrite(
676 vm_map_t map,
677 vm_address_t address,
678 vm_size_t size,
679 vm_address_t data,
680 vm_size_t *data_size)
681 {
682 kern_return_t error;
683 vm_map_copy_t copy;
684
685 if (map == VM_MAP_NULL)
686 return(KERN_INVALID_ARGUMENT);
687
688 error = vm_map_copyin(map, (vm_map_address_t)address,
689 (vm_map_size_t)size, FALSE, &copy);
690
691 if (KERN_SUCCESS == error) {
692 error = vm_map_copy_overwrite(current_thread()->map,
693 (vm_map_address_t)data,
694 copy, FALSE);
695 if (KERN_SUCCESS == error) {
696 *data_size = size;
697 return error;
698 }
699 vm_map_copy_discard(copy);
700 }
701 return(error);
702 }
703
704
705 /*
706 * mach_vm_write -
707 * Overwrite the specified address range with the data provided
708 * (from the current map).
709 */
710 kern_return_t
711 mach_vm_write(
712 vm_map_t map,
713 mach_vm_address_t address,
714 pointer_t data,
715 __unused mach_msg_type_number_t size)
716 {
717 if (map == VM_MAP_NULL)
718 return KERN_INVALID_ARGUMENT;
719
720 return vm_map_copy_overwrite(map, (vm_map_address_t)address,
721 (vm_map_copy_t) data, FALSE /* interruptible XXX */);
722 }
723
724 /*
725 * vm_write -
726 * Overwrite the specified address range with the data provided
727 * (from the current map).
728 *
729 * The addressability of the range of addresses to overwrite is
730 * limited bu the use of a vm_address_t (same size as kernel map).
731 * Either the target map is also small, or the range is in the
732 * low addresses within it.
733 */
734 kern_return_t
735 vm_write(
736 vm_map_t map,
737 vm_address_t address,
738 pointer_t data,
739 __unused mach_msg_type_number_t size)
740 {
741 if (map == VM_MAP_NULL)
742 return KERN_INVALID_ARGUMENT;
743
744 return vm_map_copy_overwrite(map, (vm_map_address_t)address,
745 (vm_map_copy_t) data, FALSE /* interruptible XXX */);
746 }
747
748 /*
749 * mach_vm_copy -
750 * Overwrite one range of the specified map with the contents of
751 * another range within that same map (i.e. both address ranges
752 * are "over there").
753 */
754 kern_return_t
755 mach_vm_copy(
756 vm_map_t map,
757 mach_vm_address_t source_address,
758 mach_vm_size_t size,
759 mach_vm_address_t dest_address)
760 {
761 vm_map_copy_t copy;
762 kern_return_t kr;
763
764 if (map == VM_MAP_NULL)
765 return KERN_INVALID_ARGUMENT;
766
767 kr = vm_map_copyin(map, (vm_map_address_t)source_address,
768 (vm_map_size_t)size, FALSE, &copy);
769
770 if (KERN_SUCCESS == kr) {
771 kr = vm_map_copy_overwrite(map,
772 (vm_map_address_t)dest_address,
773 copy, FALSE /* interruptible XXX */);
774
775 if (KERN_SUCCESS != kr)
776 vm_map_copy_discard(copy);
777 }
778 return kr;
779 }
780
781 kern_return_t
782 vm_copy(
783 vm_map_t map,
784 vm_address_t source_address,
785 vm_size_t size,
786 vm_address_t dest_address)
787 {
788 vm_map_copy_t copy;
789 kern_return_t kr;
790
791 if (map == VM_MAP_NULL)
792 return KERN_INVALID_ARGUMENT;
793
794 kr = vm_map_copyin(map, (vm_map_address_t)source_address,
795 (vm_map_size_t)size, FALSE, &copy);
796
797 if (KERN_SUCCESS == kr) {
798 kr = vm_map_copy_overwrite(map,
799 (vm_map_address_t)dest_address,
800 copy, FALSE /* interruptible XXX */);
801
802 if (KERN_SUCCESS != kr)
803 vm_map_copy_discard(copy);
804 }
805 return kr;
806 }
807
808 /*
809 * mach_vm_map -
810 * Map some range of an object into an address space.
811 *
812 * The object can be one of several types of objects:
813 * NULL - anonymous memory
814 * a named entry - a range within another address space
815 * or a range within a memory object
816 * a whole memory object
817 *
818 */
819 kern_return_t
820 mach_vm_map(
821 vm_map_t target_map,
822 mach_vm_offset_t *address,
823 mach_vm_size_t initial_size,
824 mach_vm_offset_t mask,
825 int flags,
826 ipc_port_t port,
827 vm_object_offset_t offset,
828 boolean_t copy,
829 vm_prot_t cur_protection,
830 vm_prot_t max_protection,
831 vm_inherit_t inheritance)
832 {
833 vm_map_address_t map_addr;
834 vm_map_size_t map_size;
835 vm_object_t object;
836 vm_object_size_t size;
837 kern_return_t result;
838
839 /*
840 * Check arguments for validity
841 */
842 if ((target_map == VM_MAP_NULL) ||
843 (cur_protection & ~VM_PROT_ALL) ||
844 (max_protection & ~VM_PROT_ALL) ||
845 (inheritance > VM_INHERIT_LAST_VALID) ||
846 initial_size == 0)
847 return(KERN_INVALID_ARGUMENT);
848
849 map_addr = vm_map_trunc_page(*address);
850 map_size = vm_map_round_page(initial_size);
851 size = vm_object_round_page(initial_size);
852
853 /*
854 * Find the vm object (if any) corresponding to this port.
855 */
856 if (!IP_VALID(port)) {
857 object = VM_OBJECT_NULL;
858 offset = 0;
859 copy = FALSE;
860 } else if (ip_kotype(port) == IKOT_NAMED_ENTRY) {
861 vm_named_entry_t named_entry;
862
863 named_entry = (vm_named_entry_t)port->ip_kobject;
864 /* a few checks to make sure user is obeying rules */
865 if(size == 0) {
866 if(offset >= named_entry->size)
867 return(KERN_INVALID_RIGHT);
868 size = named_entry->size - offset;
869 }
870 if((named_entry->protection & max_protection) != max_protection)
871 return(KERN_INVALID_RIGHT);
872 if((named_entry->protection & cur_protection) != cur_protection)
873 return(KERN_INVALID_RIGHT);
874 if(named_entry->size < (offset + size))
875 return(KERN_INVALID_ARGUMENT);
876
877 /* the callers parameter offset is defined to be the */
878 /* offset from beginning of named entry offset in object */
879 offset = offset + named_entry->offset;
880
881 named_entry_lock(named_entry);
882 if(named_entry->is_sub_map) {
883 vm_map_entry_t map_entry;
884
885 named_entry_unlock(named_entry);
886 vm_object_reference(vm_submap_object);
887 if ((result = vm_map_enter(target_map,
888 &map_addr, map_size,
889 (vm_map_offset_t)mask, flags,
890 vm_submap_object, 0,
891 FALSE,
892 cur_protection, max_protection, inheritance
893 )) != KERN_SUCCESS) {
894 vm_object_deallocate(vm_submap_object);
895 } else {
896 char alias;
897
898 VM_GET_FLAGS_ALIAS(flags, alias);
899 if ((alias == VM_MEMORY_SHARED_PMAP) &&
900 !copy) {
901 vm_map_submap(target_map, map_addr,
902 map_addr + map_size,
903 named_entry->backing.map,
904 (vm_map_offset_t)offset, TRUE);
905 } else {
906 vm_map_submap(target_map, map_addr,
907 map_addr + map_size,
908 named_entry->backing.map,
909 (vm_map_offset_t)offset, FALSE);
910 }
911 if(copy) {
912 if(vm_map_lookup_entry(
913 target_map, map_addr, &map_entry)) {
914 map_entry->needs_copy = TRUE;
915 }
916 }
917 *address = map_addr;
918 }
919 return(result);
920
921 } else if (named_entry->is_pager) {
922 unsigned int access;
923 vm_prot_t protections;
924 unsigned int wimg_mode;
925 boolean_t cache_attr;
926
927 protections = named_entry->protection
928 & VM_PROT_ALL;
929 access = GET_MAP_MEM(named_entry->protection);
930
931 object = vm_object_enter(
932 named_entry->backing.pager,
933 named_entry->size,
934 named_entry->internal,
935 FALSE,
936 FALSE);
937 if (object == VM_OBJECT_NULL) {
938 named_entry_unlock(named_entry);
939 return(KERN_INVALID_OBJECT);
940 }
941
942 /* JMM - drop reference on pager here */
943
944 /* create an extra ref for the named entry */
945 vm_object_lock(object);
946 vm_object_reference_locked(object);
947 named_entry->backing.object = object;
948 named_entry->is_pager = FALSE;
949 named_entry_unlock(named_entry);
950
951 wimg_mode = object->wimg_bits;
952 if(access == MAP_MEM_IO) {
953 wimg_mode = VM_WIMG_IO;
954 } else if (access == MAP_MEM_COPYBACK) {
955 wimg_mode = VM_WIMG_USE_DEFAULT;
956 } else if (access == MAP_MEM_WTHRU) {
957 wimg_mode = VM_WIMG_WTHRU;
958 } else if (access == MAP_MEM_WCOMB) {
959 wimg_mode = VM_WIMG_WCOMB;
960 }
961 if ((wimg_mode == VM_WIMG_IO)
962 || (wimg_mode == VM_WIMG_WCOMB))
963 cache_attr = TRUE;
964 else
965 cache_attr = FALSE;
966
967 /* wait for object (if any) to be ready */
968 if (!named_entry->internal) {
969 while (!object->pager_ready) {
970 vm_object_wait(object,
971 VM_OBJECT_EVENT_PAGER_READY,
972 THREAD_UNINT);
973 vm_object_lock(object);
974 }
975 }
976
977 if(object->wimg_bits != wimg_mode) {
978 vm_page_t p;
979
980 vm_object_paging_wait(object, THREAD_UNINT);
981
982 object->wimg_bits = wimg_mode;
983 queue_iterate(&object->memq, p, vm_page_t, listq) {
984 if (!p->fictitious) {
985 pmap_disconnect(p->phys_page);
986 if (cache_attr)
987 pmap_sync_page_attributes_phys(p->phys_page);
988 }
989 }
990 }
991 object->true_share = TRUE;
992 if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC)
993 object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
994 vm_object_unlock(object);
995 } else {
996 /* This is the case where we are going to map */
997 /* an already mapped object. If the object is */
998 /* not ready it is internal. An external */
999 /* object cannot be mapped until it is ready */
1000 /* we can therefore avoid the ready check */
1001 /* in this case. */
1002 object = named_entry->backing.object;
1003 assert(object != VM_OBJECT_NULL);
1004 named_entry_unlock(named_entry);
1005 vm_object_reference(object);
1006 }
1007 } else if (ip_kotype(port) == IKOT_MEMORY_OBJECT) {
1008 /*
1009 * JMM - This is temporary until we unify named entries
1010 * and raw memory objects.
1011 *
1012 * Detected fake ip_kotype for a memory object. In
1013 * this case, the port isn't really a port at all, but
1014 * instead is just a raw memory object.
1015 */
1016
1017 if ((object = vm_object_enter((memory_object_t)port,
1018 size, FALSE, FALSE, FALSE))
1019 == VM_OBJECT_NULL)
1020 return(KERN_INVALID_OBJECT);
1021
1022 /* wait for object (if any) to be ready */
1023 if (object != VM_OBJECT_NULL) {
1024 if(object == kernel_object) {
1025 printf("Warning: Attempt to map kernel object"
1026 " by a non-private kernel entity\n");
1027 return(KERN_INVALID_OBJECT);
1028 }
1029 vm_object_lock(object);
1030 while (!object->pager_ready) {
1031 vm_object_wait(object,
1032 VM_OBJECT_EVENT_PAGER_READY,
1033 THREAD_UNINT);
1034 vm_object_lock(object);
1035 }
1036 vm_object_unlock(object);
1037 }
1038 } else {
1039 return (KERN_INVALID_OBJECT);
1040 }
1041
1042 /*
1043 * Perform the copy if requested
1044 */
1045
1046 if (copy) {
1047 vm_object_t new_object;
1048 vm_object_offset_t new_offset;
1049
1050 result = vm_object_copy_strategically(object, offset, size,
1051 &new_object, &new_offset,
1052 &copy);
1053
1054
1055 if (result == KERN_MEMORY_RESTART_COPY) {
1056 boolean_t success;
1057 boolean_t src_needs_copy;
1058
1059 /*
1060 * XXX
1061 * We currently ignore src_needs_copy.
1062 * This really is the issue of how to make
1063 * MEMORY_OBJECT_COPY_SYMMETRIC safe for
1064 * non-kernel users to use. Solution forthcoming.
1065 * In the meantime, since we don't allow non-kernel
1066 * memory managers to specify symmetric copy,
1067 * we won't run into problems here.
1068 */
1069 new_object = object;
1070 new_offset = offset;
1071 success = vm_object_copy_quickly(&new_object,
1072 new_offset, size,
1073 &src_needs_copy,
1074 &copy);
1075 assert(success);
1076 result = KERN_SUCCESS;
1077 }
1078 /*
1079 * Throw away the reference to the
1080 * original object, as it won't be mapped.
1081 */
1082
1083 vm_object_deallocate(object);
1084
1085 if (result != KERN_SUCCESS)
1086 return (result);
1087
1088 object = new_object;
1089 offset = new_offset;
1090 }
1091
1092 if ((result = vm_map_enter(target_map,
1093 &map_addr, map_size,
1094 (vm_map_offset_t)mask,
1095 flags,
1096 object, offset,
1097 copy,
1098 cur_protection, max_protection, inheritance
1099 )) != KERN_SUCCESS)
1100 vm_object_deallocate(object);
1101 *address = map_addr;
1102 return(result);
1103 }
1104
1105
1106 /* legacy interface */
1107 kern_return_t
1108 vm_map_64(
1109 vm_map_t target_map,
1110 vm_offset_t *address,
1111 vm_size_t size,
1112 vm_offset_t mask,
1113 int flags,
1114 ipc_port_t port,
1115 vm_object_offset_t offset,
1116 boolean_t copy,
1117 vm_prot_t cur_protection,
1118 vm_prot_t max_protection,
1119 vm_inherit_t inheritance)
1120 {
1121 mach_vm_address_t map_addr;
1122 mach_vm_size_t map_size;
1123 mach_vm_offset_t map_mask;
1124 kern_return_t kr;
1125
1126 map_addr = (mach_vm_address_t)*address;
1127 map_size = (mach_vm_size_t)size;
1128 map_mask = (mach_vm_offset_t)mask;
1129
1130 kr = mach_vm_map(target_map, &map_addr, map_size, map_mask, flags,
1131 port, offset, copy,
1132 cur_protection, max_protection, inheritance);
1133 *address = CAST_DOWN(vm_address_t, map_addr);
1134 return kr;
1135 }
1136
1137 /* temporary, until world build */
1138 kern_return_t
1139 vm_map(
1140 vm_map_t target_map,
1141 vm_offset_t *address,
1142 vm_size_t size,
1143 vm_offset_t mask,
1144 int flags,
1145 ipc_port_t port,
1146 vm_offset_t offset,
1147 boolean_t copy,
1148 vm_prot_t cur_protection,
1149 vm_prot_t max_protection,
1150 vm_inherit_t inheritance)
1151 {
1152 mach_vm_address_t map_addr;
1153 mach_vm_size_t map_size;
1154 mach_vm_offset_t map_mask;
1155 vm_object_offset_t obj_offset;
1156 kern_return_t kr;
1157
1158 map_addr = (mach_vm_address_t)*address;
1159 map_size = (mach_vm_size_t)size;
1160 map_mask = (mach_vm_offset_t)mask;
1161 obj_offset = (vm_object_offset_t)offset;
1162
1163 kr = mach_vm_map(target_map, &map_addr, map_size, map_mask, flags,
1164 port, obj_offset, copy,
1165 cur_protection, max_protection, inheritance);
1166 *address = CAST_DOWN(vm_address_t, map_addr);
1167 return kr;
1168 }
1169
1170 /*
1171 * mach_vm_remap -
1172 * Remap a range of memory from one task into another,
1173 * to another address range within the same task, or
1174 * over top of itself (with altered permissions and/or
1175 * as an in-place copy of itself).
1176 */
1177
1178 kern_return_t
1179 mach_vm_remap(
1180 vm_map_t target_map,
1181 mach_vm_offset_t *address,
1182 mach_vm_size_t size,
1183 mach_vm_offset_t mask,
1184 boolean_t anywhere,
1185 vm_map_t src_map,
1186 mach_vm_offset_t memory_address,
1187 boolean_t copy,
1188 vm_prot_t *cur_protection,
1189 vm_prot_t *max_protection,
1190 vm_inherit_t inheritance)
1191 {
1192 vm_map_offset_t map_addr;
1193 kern_return_t kr;
1194
1195 if (VM_MAP_NULL == target_map || VM_MAP_NULL == src_map)
1196 return KERN_INVALID_ARGUMENT;
1197
1198 map_addr = (vm_map_offset_t)*address;
1199
1200 kr = vm_map_remap(target_map,
1201 &map_addr,
1202 size,
1203 mask,
1204 anywhere,
1205 src_map,
1206 memory_address,
1207 copy,
1208 cur_protection,
1209 max_protection,
1210 inheritance);
1211 *address = map_addr;
1212 return kr;
1213 }
1214
1215 /*
1216 * vm_remap -
1217 * Remap a range of memory from one task into another,
1218 * to another address range within the same task, or
1219 * over top of itself (with altered permissions and/or
1220 * as an in-place copy of itself).
1221 *
1222 * The addressability of the source and target address
1223 * range is limited by the size of vm_address_t (in the
1224 * kernel context).
1225 */
1226 kern_return_t
1227 vm_remap(
1228 vm_map_t target_map,
1229 vm_offset_t *address,
1230 vm_size_t size,
1231 vm_offset_t mask,
1232 boolean_t anywhere,
1233 vm_map_t src_map,
1234 vm_offset_t memory_address,
1235 boolean_t copy,
1236 vm_prot_t *cur_protection,
1237 vm_prot_t *max_protection,
1238 vm_inherit_t inheritance)
1239 {
1240 vm_map_offset_t map_addr;
1241 kern_return_t kr;
1242
1243 if (VM_MAP_NULL == target_map || VM_MAP_NULL == src_map)
1244 return KERN_INVALID_ARGUMENT;
1245
1246 map_addr = (vm_map_offset_t)*address;
1247
1248 kr = vm_map_remap(target_map,
1249 &map_addr,
1250 size,
1251 mask,
1252 anywhere,
1253 src_map,
1254 memory_address,
1255 copy,
1256 cur_protection,
1257 max_protection,
1258 inheritance);
1259 *address = CAST_DOWN(vm_offset_t, map_addr);
1260 return kr;
1261 }
1262
1263 /*
1264 * NOTE: these routine (and this file) will no longer require mach_host_server.h
1265 * when mach_vm_wire and vm_wire are changed to use ledgers.
1266 */
1267 #include <mach/mach_host_server.h>
1268 /*
1269 * mach_vm_wire
1270 * Specify that the range of the virtual address space
1271 * of the target task must not cause page faults for
1272 * the indicated accesses.
1273 *
1274 * [ To unwire the pages, specify VM_PROT_NONE. ]
1275 */
1276 kern_return_t
1277 mach_vm_wire(
1278 host_priv_t host_priv,
1279 vm_map_t map,
1280 mach_vm_offset_t start,
1281 mach_vm_size_t size,
1282 vm_prot_t access)
1283 {
1284 kern_return_t rc;
1285
1286 if (host_priv == HOST_PRIV_NULL)
1287 return KERN_INVALID_HOST;
1288
1289 assert(host_priv == &realhost);
1290
1291 if (map == VM_MAP_NULL)
1292 return KERN_INVALID_TASK;
1293
1294 if (access & ~VM_PROT_ALL)
1295 return KERN_INVALID_ARGUMENT;
1296
1297 if (access != VM_PROT_NONE) {
1298 rc = vm_map_wire(map, vm_map_trunc_page(start),
1299 vm_map_round_page(start+size), access, TRUE);
1300 } else {
1301 rc = vm_map_unwire(map, vm_map_trunc_page(start),
1302 vm_map_round_page(start+size), TRUE);
1303 }
1304 return rc;
1305 }
1306
1307 /*
1308 * vm_wire -
1309 * Specify that the range of the virtual address space
1310 * of the target task must not cause page faults for
1311 * the indicated accesses.
1312 *
1313 * [ To unwire the pages, specify VM_PROT_NONE. ]
1314 */
1315 kern_return_t
1316 vm_wire(
1317 host_priv_t host_priv,
1318 register vm_map_t map,
1319 vm_offset_t start,
1320 vm_size_t size,
1321 vm_prot_t access)
1322 {
1323 kern_return_t rc;
1324
1325 if (host_priv == HOST_PRIV_NULL)
1326 return KERN_INVALID_HOST;
1327
1328 assert(host_priv == &realhost);
1329
1330 if (map == VM_MAP_NULL)
1331 return KERN_INVALID_TASK;
1332
1333 if ((access & ~VM_PROT_ALL) || (start + size < start))
1334 return KERN_INVALID_ARGUMENT;
1335
1336 if (size == 0) {
1337 rc = KERN_SUCCESS;
1338 } else if (access != VM_PROT_NONE) {
1339 rc = vm_map_wire(map, vm_map_trunc_page(start),
1340 vm_map_round_page(start+size), access, TRUE);
1341 } else {
1342 rc = vm_map_unwire(map, vm_map_trunc_page(start),
1343 vm_map_round_page(start+size), TRUE);
1344 }
1345 return rc;
1346 }
1347
1348 /*
1349 * vm_msync
1350 *
1351 * Synchronises the memory range specified with its backing store
1352 * image by either flushing or cleaning the contents to the appropriate
1353 * memory manager.
1354 *
1355 * interpretation of sync_flags
1356 * VM_SYNC_INVALIDATE - discard pages, only return precious
1357 * pages to manager.
1358 *
1359 * VM_SYNC_INVALIDATE & (VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS)
1360 * - discard pages, write dirty or precious
1361 * pages back to memory manager.
1362 *
1363 * VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS
1364 * - write dirty or precious pages back to
1365 * the memory manager.
1366 *
1367 * VM_SYNC_CONTIGUOUS - does everything normally, but if there
1368 * is a hole in the region, and we would
1369 * have returned KERN_SUCCESS, return
1370 * KERN_INVALID_ADDRESS instead.
1371 *
1372 * RETURNS
1373 * KERN_INVALID_TASK Bad task parameter
1374 * KERN_INVALID_ARGUMENT both sync and async were specified.
1375 * KERN_SUCCESS The usual.
1376 * KERN_INVALID_ADDRESS There was a hole in the region.
1377 */
1378
1379 kern_return_t
1380 mach_vm_msync(
1381 vm_map_t map,
1382 mach_vm_address_t address,
1383 mach_vm_size_t size,
1384 vm_sync_t sync_flags)
1385 {
1386
1387 if (map == VM_MAP_NULL)
1388 return(KERN_INVALID_TASK);
1389
1390 return vm_map_msync(map, (vm_map_address_t)address,
1391 (vm_map_size_t)size, sync_flags);
1392 }
1393
1394 /*
1395 * vm_msync
1396 *
1397 * Synchronises the memory range specified with its backing store
1398 * image by either flushing or cleaning the contents to the appropriate
1399 * memory manager.
1400 *
1401 * interpretation of sync_flags
1402 * VM_SYNC_INVALIDATE - discard pages, only return precious
1403 * pages to manager.
1404 *
1405 * VM_SYNC_INVALIDATE & (VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS)
1406 * - discard pages, write dirty or precious
1407 * pages back to memory manager.
1408 *
1409 * VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS
1410 * - write dirty or precious pages back to
1411 * the memory manager.
1412 *
1413 * VM_SYNC_CONTIGUOUS - does everything normally, but if there
1414 * is a hole in the region, and we would
1415 * have returned KERN_SUCCESS, return
1416 * KERN_INVALID_ADDRESS instead.
1417 *
1418 * The addressability of the range is limited to that which can
1419 * be described by a vm_address_t.
1420 *
1421 * RETURNS
1422 * KERN_INVALID_TASK Bad task parameter
1423 * KERN_INVALID_ARGUMENT both sync and async were specified.
1424 * KERN_SUCCESS The usual.
1425 * KERN_INVALID_ADDRESS There was a hole in the region.
1426 */
1427
1428 kern_return_t
1429 vm_msync(
1430 vm_map_t map,
1431 vm_address_t address,
1432 vm_size_t size,
1433 vm_sync_t sync_flags)
1434 {
1435
1436 if (map == VM_MAP_NULL)
1437 return(KERN_INVALID_TASK);
1438
1439 return vm_map_msync(map, (vm_map_address_t)address,
1440 (vm_map_size_t)size, sync_flags);
1441 }
1442
1443
1444 /*
1445 * mach_vm_behavior_set
1446 *
1447 * Sets the paging behavior attribute for the specified range
1448 * in the specified map.
1449 *
1450 * This routine will fail with KERN_INVALID_ADDRESS if any address
1451 * in [start,start+size) is not a valid allocated memory region.
1452 */
1453 kern_return_t
1454 mach_vm_behavior_set(
1455 vm_map_t map,
1456 mach_vm_offset_t start,
1457 mach_vm_size_t size,
1458 vm_behavior_t new_behavior)
1459 {
1460 if ((map == VM_MAP_NULL) || (start + size < start))
1461 return(KERN_INVALID_ARGUMENT);
1462
1463 if (size == 0)
1464 return KERN_SUCCESS;
1465
1466 return(vm_map_behavior_set(map, vm_map_trunc_page(start),
1467 vm_map_round_page(start+size), new_behavior));
1468 }
1469
1470 /*
1471 * vm_behavior_set
1472 *
1473 * Sets the paging behavior attribute for the specified range
1474 * in the specified map.
1475 *
1476 * This routine will fail with KERN_INVALID_ADDRESS if any address
1477 * in [start,start+size) is not a valid allocated memory region.
1478 *
1479 * This routine is potentially limited in addressibility by the
1480 * use of vm_offset_t (if the map provided is larger than the
1481 * kernel's).
1482 */
1483 kern_return_t
1484 vm_behavior_set(
1485 vm_map_t map,
1486 vm_offset_t start,
1487 vm_size_t size,
1488 vm_behavior_t new_behavior)
1489 {
1490 if ((map == VM_MAP_NULL) || (start + size < start))
1491 return(KERN_INVALID_ARGUMENT);
1492
1493 if (size == 0)
1494 return KERN_SUCCESS;
1495
1496 return(vm_map_behavior_set(map, vm_map_trunc_page(start),
1497 vm_map_round_page(start+size), new_behavior));
1498 }
1499
1500 /*
1501 * mach_vm_region:
1502 *
1503 * User call to obtain information about a region in
1504 * a task's address map. Currently, only one flavor is
1505 * supported.
1506 *
1507 * XXX The reserved and behavior fields cannot be filled
1508 * in until the vm merge from the IK is completed, and
1509 * vm_reserve is implemented.
1510 *
1511 * XXX Dependency: syscall_vm_region() also supports only one flavor.
1512 */
1513
1514 kern_return_t
1515 mach_vm_region(
1516 vm_map_t map,
1517 mach_vm_offset_t *address, /* IN/OUT */
1518 mach_vm_size_t *size, /* OUT */
1519 vm_region_flavor_t flavor, /* IN */
1520 vm_region_info_t info, /* OUT */
1521 mach_msg_type_number_t *count, /* IN/OUT */
1522 mach_port_t *object_name) /* OUT */
1523 {
1524 vm_map_offset_t map_addr;
1525 vm_map_size_t map_size;
1526 kern_return_t kr;
1527
1528 if (VM_MAP_NULL == map)
1529 return KERN_INVALID_ARGUMENT;
1530
1531 map_addr = (vm_map_offset_t)*address;
1532 map_size = (vm_map_size_t)*size;
1533
1534 /* legacy conversion */
1535 if (VM_REGION_BASIC_INFO == flavor)
1536 flavor = VM_REGION_BASIC_INFO_64;
1537
1538 kr = vm_map_region(map,
1539 &map_addr, &map_size,
1540 flavor, info, count,
1541 object_name);
1542
1543 *address = map_addr;
1544 *size = map_size;
1545 return kr;
1546 }
1547
1548 /*
1549 * vm_region_64 and vm_region:
1550 *
1551 * User call to obtain information about a region in
1552 * a task's address map. Currently, only one flavor is
1553 * supported.
1554 *
1555 * XXX The reserved and behavior fields cannot be filled
1556 * in until the vm merge from the IK is completed, and
1557 * vm_reserve is implemented.
1558 *
1559 * XXX Dependency: syscall_vm_region() also supports only one flavor.
1560 */
1561
1562 kern_return_t
1563 vm_region_64(
1564 vm_map_t map,
1565 vm_offset_t *address, /* IN/OUT */
1566 vm_size_t *size, /* OUT */
1567 vm_region_flavor_t flavor, /* IN */
1568 vm_region_info_t info, /* OUT */
1569 mach_msg_type_number_t *count, /* IN/OUT */
1570 mach_port_t *object_name) /* OUT */
1571 {
1572 vm_map_offset_t map_addr;
1573 vm_map_size_t map_size;
1574 kern_return_t kr;
1575
1576 if (VM_MAP_NULL == map)
1577 return KERN_INVALID_ARGUMENT;
1578
1579 map_addr = (vm_map_offset_t)*address;
1580 map_size = (vm_map_size_t)*size;
1581
1582 /* legacy conversion */
1583 if (VM_REGION_BASIC_INFO == flavor)
1584 flavor = VM_REGION_BASIC_INFO_64;
1585
1586 kr = vm_map_region(map,
1587 &map_addr, &map_size,
1588 flavor, info, count,
1589 object_name);
1590
1591 *address = CAST_DOWN(vm_offset_t, map_addr);
1592 *size = CAST_DOWN(vm_size_t, map_size);
1593
1594 if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS)
1595 return KERN_INVALID_ADDRESS;
1596 return kr;
1597 }
1598
1599 kern_return_t
1600 vm_region(
1601 vm_map_t map,
1602 vm_address_t *address, /* IN/OUT */
1603 vm_size_t *size, /* OUT */
1604 vm_region_flavor_t flavor, /* IN */
1605 vm_region_info_t info, /* OUT */
1606 mach_msg_type_number_t *count, /* IN/OUT */
1607 mach_port_t *object_name) /* OUT */
1608 {
1609 vm_map_address_t map_addr;
1610 vm_map_size_t map_size;
1611 kern_return_t kr;
1612
1613 if (VM_MAP_NULL == map)
1614 return KERN_INVALID_ARGUMENT;
1615
1616 map_addr = (vm_map_address_t)*address;
1617 map_size = (vm_map_size_t)*size;
1618
1619 kr = vm_map_region(map,
1620 &map_addr, &map_size,
1621 flavor, info, count,
1622 object_name);
1623
1624 *address = CAST_DOWN(vm_address_t, map_addr);
1625 *size = CAST_DOWN(vm_size_t, map_size);
1626
1627 if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS)
1628 return KERN_INVALID_ADDRESS;
1629 return kr;
1630 }
1631
1632 /*
1633 * vm_region_recurse: A form of vm_region which follows the
1634 * submaps in a target map
1635 *
1636 */
1637 kern_return_t
1638 mach_vm_region_recurse(
1639 vm_map_t map,
1640 mach_vm_address_t *address,
1641 mach_vm_size_t *size,
1642 uint32_t *depth,
1643 vm_region_recurse_info_t info,
1644 mach_msg_type_number_t *infoCnt)
1645 {
1646 vm_map_address_t map_addr;
1647 vm_map_size_t map_size;
1648 kern_return_t kr;
1649
1650 if (VM_MAP_NULL == map)
1651 return KERN_INVALID_ARGUMENT;
1652
1653 map_addr = (vm_map_address_t)*address;
1654 map_size = (vm_map_size_t)*size;
1655
1656 kr = vm_map_region_recurse_64(
1657 map,
1658 &map_addr,
1659 &map_size,
1660 depth,
1661 (vm_region_submap_info_64_t)info,
1662 infoCnt);
1663
1664 *address = map_addr;
1665 *size = map_size;
1666 return kr;
1667 }
1668
1669 /*
1670 * vm_region_recurse: A form of vm_region which follows the
1671 * submaps in a target map
1672 *
1673 */
1674 kern_return_t
1675 vm_region_recurse_64(
1676 vm_map_t map,
1677 vm_address_t *address,
1678 vm_size_t *size,
1679 uint32_t *depth,
1680 vm_region_recurse_info_64_t info,
1681 mach_msg_type_number_t *infoCnt)
1682 {
1683 vm_map_address_t map_addr;
1684 vm_map_size_t map_size;
1685 kern_return_t kr;
1686
1687 if (VM_MAP_NULL == map)
1688 return KERN_INVALID_ARGUMENT;
1689
1690 map_addr = (vm_map_address_t)*address;
1691 map_size = (vm_map_size_t)*size;
1692
1693 kr = vm_map_region_recurse_64(
1694 map,
1695 &map_addr,
1696 &map_size,
1697 depth,
1698 (vm_region_submap_info_64_t)info,
1699 infoCnt);
1700
1701 *address = CAST_DOWN(vm_address_t, map_addr);
1702 *size = CAST_DOWN(vm_size_t, map_size);
1703
1704 if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS)
1705 return KERN_INVALID_ADDRESS;
1706 return kr;
1707 }
1708
1709 kern_return_t
1710 vm_region_recurse(
1711 vm_map_t map,
1712 vm_offset_t *address, /* IN/OUT */
1713 vm_size_t *size, /* OUT */
1714 natural_t *depth, /* IN/OUT */
1715 vm_region_recurse_info_t info32, /* IN/OUT */
1716 mach_msg_type_number_t *infoCnt) /* IN/OUT */
1717 {
1718 vm_region_submap_info_data_64_t info64;
1719 vm_region_submap_info_t info;
1720 vm_map_address_t map_addr;
1721 vm_map_size_t map_size;
1722 kern_return_t kr;
1723
1724 if (VM_MAP_NULL == map || *infoCnt < VM_REGION_SUBMAP_INFO_COUNT)
1725 return KERN_INVALID_ARGUMENT;
1726
1727
1728 map_addr = (vm_map_address_t)*address;
1729 map_size = (vm_map_size_t)*size;
1730 info = (vm_region_submap_info_t)info32;
1731 *infoCnt = VM_REGION_SUBMAP_INFO_COUNT_64;
1732
1733 kr = vm_map_region_recurse_64(map, &map_addr,&map_size,
1734 depth, &info64, infoCnt);
1735
1736 info->protection = info64.protection;
1737 info->max_protection = info64.max_protection;
1738 info->inheritance = info64.inheritance;
1739 info->offset = (uint32_t)info64.offset; /* trouble-maker */
1740 info->user_tag = info64.user_tag;
1741 info->pages_resident = info64.pages_resident;
1742 info->pages_shared_now_private = info64.pages_shared_now_private;
1743 info->pages_swapped_out = info64.pages_swapped_out;
1744 info->pages_dirtied = info64.pages_dirtied;
1745 info->ref_count = info64.ref_count;
1746 info->shadow_depth = info64.shadow_depth;
1747 info->external_pager = info64.external_pager;
1748 info->share_mode = info64.share_mode;
1749 info->is_submap = info64.is_submap;
1750 info->behavior = info64.behavior;
1751 info->object_id = info64.object_id;
1752 info->user_wired_count = info64.user_wired_count;
1753
1754 *address = CAST_DOWN(vm_address_t, map_addr);
1755 *size = CAST_DOWN(vm_size_t, map_size);
1756 *infoCnt = VM_REGION_SUBMAP_INFO_COUNT;
1757
1758 if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS)
1759 return KERN_INVALID_ADDRESS;
1760 return kr;
1761 }
1762
1763 kern_return_t
1764 vm_purgable_control(
1765 vm_map_t map,
1766 vm_offset_t address,
1767 vm_purgable_t control,
1768 int *state)
1769 {
1770 if (VM_MAP_NULL == map)
1771 return KERN_INVALID_ARGUMENT;
1772
1773 return vm_map_purgable_control(map,
1774 vm_map_trunc_page(address),
1775 control,
1776 state);
1777 }
1778
1779
1780 /*
1781 * Ordinarily, the right to allocate CPM is restricted
1782 * to privileged applications (those that can gain access
1783 * to the host priv port). Set this variable to zero if
1784 * you want to let any application allocate CPM.
1785 */
1786 unsigned int vm_allocate_cpm_privileged = 0;
1787
1788 /*
1789 * Allocate memory in the specified map, with the caveat that
1790 * the memory is physically contiguous. This call may fail
1791 * if the system can't find sufficient contiguous memory.
1792 * This call may cause or lead to heart-stopping amounts of
1793 * paging activity.
1794 *
1795 * Memory obtained from this call should be freed in the
1796 * normal way, viz., via vm_deallocate.
1797 */
1798 kern_return_t
1799 vm_allocate_cpm(
1800 host_priv_t host_priv,
1801 vm_map_t map,
1802 vm_address_t *addr,
1803 vm_size_t size,
1804 int flags)
1805 {
1806 vm_map_address_t map_addr;
1807 vm_map_size_t map_size;
1808 kern_return_t kr;
1809
1810 if (vm_allocate_cpm_privileged && HOST_PRIV_NULL == host_priv)
1811 return KERN_INVALID_HOST;
1812
1813 if (VM_MAP_NULL == map)
1814 return KERN_INVALID_ARGUMENT;
1815
1816 map_addr = (vm_map_address_t)*addr;
1817 map_size = (vm_map_size_t)size;
1818
1819 kr = vm_map_enter_cpm(map,
1820 &map_addr,
1821 map_size,
1822 flags);
1823
1824 *addr = CAST_DOWN(vm_address_t, map_addr);
1825 return kr;
1826 }
1827
1828
1829 kern_return_t
1830 mach_vm_page_query(
1831 vm_map_t map,
1832 mach_vm_offset_t offset,
1833 int *disposition,
1834 int *ref_count)
1835 {
1836 if (VM_MAP_NULL == map)
1837 return KERN_INVALID_ARGUMENT;
1838
1839 return vm_map_page_info(map,
1840 vm_map_trunc_page(offset),
1841 disposition, ref_count);
1842 }
1843
1844 kern_return_t
1845 vm_map_page_query(
1846 vm_map_t map,
1847 vm_offset_t offset,
1848 int *disposition,
1849 int *ref_count)
1850 {
1851 if (VM_MAP_NULL == map)
1852 return KERN_INVALID_ARGUMENT;
1853
1854 return vm_map_page_info(map,
1855 vm_map_trunc_page(offset),
1856 disposition, ref_count);
1857 }
1858
1859 /* map a (whole) upl into an address space */
1860 kern_return_t
1861 vm_upl_map(
1862 vm_map_t map,
1863 upl_t upl,
1864 vm_offset_t *dst_addr)
1865 {
1866 vm_map_offset_t map_addr;
1867 kern_return_t kr;
1868
1869 if (VM_MAP_NULL == map)
1870 return KERN_INVALID_ARGUMENT;
1871
1872 kr = vm_map_enter_upl(map, upl, &map_addr);
1873 *dst_addr = CAST_DOWN(vm_offset_t, map_addr);
1874 return kr;
1875 }
1876
1877 kern_return_t
1878 vm_upl_unmap(
1879 vm_map_t map,
1880 upl_t upl)
1881 {
1882 if (VM_MAP_NULL == map)
1883 return KERN_INVALID_ARGUMENT;
1884
1885 return (vm_map_remove_upl(map, upl));
1886 }
1887
1888 /* Retrieve a upl for an object underlying an address range in a map */
1889
1890 kern_return_t
1891 vm_map_get_upl(
1892 vm_map_t map,
1893 vm_map_offset_t map_offset,
1894 upl_size_t *upl_size,
1895 upl_t *upl,
1896 upl_page_info_array_t page_list,
1897 unsigned int *count,
1898 int *flags,
1899 int force_data_sync)
1900 {
1901 int map_flags;
1902 kern_return_t kr;
1903
1904 if (VM_MAP_NULL == map)
1905 return KERN_INVALID_ARGUMENT;
1906
1907 map_flags = *flags & ~UPL_NOZEROFILL;
1908 if (force_data_sync)
1909 map_flags |= UPL_FORCE_DATA_SYNC;
1910
1911 kr = vm_map_create_upl(map,
1912 map_offset,
1913 upl_size,
1914 upl,
1915 page_list,
1916 count,
1917 &map_flags);
1918
1919 *flags = (map_flags & ~UPL_FORCE_DATA_SYNC);
1920 return kr;
1921 }
1922
1923
1924 __private_extern__ kern_return_t
1925 mach_memory_entry_allocate(
1926 vm_named_entry_t *user_entry_p,
1927 ipc_port_t *user_handle_p); /* forward */
1928
1929 /*
1930 * mach_make_memory_entry_64
1931 *
1932 * Think of it as a two-stage vm_remap() operation. First
1933 * you get a handle. Second, you get map that handle in
1934 * somewhere else. Rather than doing it all at once (and
1935 * without needing access to the other whole map).
1936 */
1937
1938 kern_return_t
1939 mach_make_memory_entry_64(
1940 vm_map_t target_map,
1941 memory_object_size_t *size,
1942 memory_object_offset_t offset,
1943 vm_prot_t permission,
1944 ipc_port_t *object_handle,
1945 ipc_port_t parent_handle)
1946 {
1947 vm_map_version_t version;
1948 vm_named_entry_t parent_entry;
1949 vm_named_entry_t user_entry;
1950 ipc_port_t user_handle;
1951 kern_return_t kr;
1952 vm_map_t real_map;
1953
1954 /* needed for call to vm_map_lookup_locked */
1955 boolean_t wired;
1956 vm_object_offset_t obj_off;
1957 vm_prot_t prot;
1958 vm_map_offset_t lo_offset, hi_offset;
1959 vm_behavior_t behavior;
1960 vm_object_t object;
1961 vm_object_t shadow_object;
1962
1963 /* needed for direct map entry manipulation */
1964 vm_map_entry_t map_entry;
1965 vm_map_entry_t next_entry;
1966 vm_map_t local_map;
1967 vm_map_t original_map = target_map;
1968 vm_map_size_t total_size;
1969 vm_map_size_t map_size;
1970 vm_map_offset_t map_offset;
1971 vm_map_offset_t local_offset;
1972 vm_object_size_t mappable_size;
1973
1974 unsigned int access;
1975 vm_prot_t protections;
1976 unsigned int wimg_mode;
1977 boolean_t cache_attr = FALSE;
1978
1979 if (((permission & 0x00FF0000) &
1980 ~(MAP_MEM_ONLY |
1981 MAP_MEM_NAMED_CREATE |
1982 MAP_MEM_PURGABLE |
1983 MAP_MEM_NAMED_REUSE))) {
1984 /*
1985 * Unknown flag: reject for forward compatibility.
1986 */
1987 return KERN_INVALID_VALUE;
1988 }
1989
1990 if (parent_handle != IP_NULL &&
1991 ip_kotype(parent_handle) == IKOT_NAMED_ENTRY) {
1992 parent_entry = (vm_named_entry_t) parent_handle->ip_kobject;
1993 } else {
1994 parent_entry = NULL;
1995 }
1996
1997 protections = permission & VM_PROT_ALL;
1998 access = GET_MAP_MEM(permission);
1999
2000 user_handle = IP_NULL;
2001 user_entry = NULL;
2002
2003 map_offset = vm_map_trunc_page(offset);
2004 map_size = vm_map_round_page(*size);
2005
2006 if (permission & MAP_MEM_ONLY) {
2007 boolean_t parent_is_object;
2008
2009 if (parent_entry == NULL) {
2010 return KERN_INVALID_ARGUMENT;
2011 }
2012
2013 parent_is_object = !(parent_entry->is_sub_map || parent_entry->is_pager);
2014 object = parent_entry->backing.object;
2015 if(parent_is_object && object != VM_OBJECT_NULL)
2016 wimg_mode = object->wimg_bits;
2017 else
2018 wimg_mode = VM_WIMG_DEFAULT;
2019 if((access != GET_MAP_MEM(parent_entry->protection)) &&
2020 !(parent_entry->protection & VM_PROT_WRITE)) {
2021 return KERN_INVALID_RIGHT;
2022 }
2023 if(access == MAP_MEM_IO) {
2024 SET_MAP_MEM(access, parent_entry->protection);
2025 wimg_mode = VM_WIMG_IO;
2026 } else if (access == MAP_MEM_COPYBACK) {
2027 SET_MAP_MEM(access, parent_entry->protection);
2028 wimg_mode = VM_WIMG_DEFAULT;
2029 } else if (access == MAP_MEM_WTHRU) {
2030 SET_MAP_MEM(access, parent_entry->protection);
2031 wimg_mode = VM_WIMG_WTHRU;
2032 } else if (access == MAP_MEM_WCOMB) {
2033 SET_MAP_MEM(access, parent_entry->protection);
2034 wimg_mode = VM_WIMG_WCOMB;
2035 }
2036 if(parent_is_object && object &&
2037 (access != MAP_MEM_NOOP) &&
2038 (!(object->nophyscache))) {
2039 if(object->wimg_bits != wimg_mode) {
2040 vm_page_t p;
2041 if ((wimg_mode == VM_WIMG_IO)
2042 || (wimg_mode == VM_WIMG_WCOMB))
2043 cache_attr = TRUE;
2044 else
2045 cache_attr = FALSE;
2046 vm_object_lock(object);
2047 vm_object_paging_wait(object, THREAD_UNINT);
2048 object->wimg_bits = wimg_mode;
2049 queue_iterate(&object->memq,
2050 p, vm_page_t, listq) {
2051 if (!p->fictitious) {
2052 pmap_disconnect(p->phys_page);
2053 if (cache_attr)
2054 pmap_sync_page_attributes_phys(p->phys_page);
2055 }
2056 }
2057 vm_object_unlock(object);
2058 }
2059 }
2060 if (object_handle)
2061 *object_handle = IP_NULL;
2062 return KERN_SUCCESS;
2063 }
2064
2065 if(permission & MAP_MEM_NAMED_CREATE) {
2066 kr = mach_memory_entry_allocate(&user_entry, &user_handle);
2067 if (kr != KERN_SUCCESS) {
2068 return KERN_FAILURE;
2069 }
2070
2071 /*
2072 * Force the creation of the VM object now.
2073 */
2074 if (map_size > (vm_map_size_t) VM_MAX_ADDRESS) {
2075 /*
2076 * LP64todo - for now, we can only allocate 4GB
2077 * internal objects because the default pager can't
2078 * page bigger ones. Remove this when it can.
2079 */
2080 kr = KERN_FAILURE;
2081 goto make_mem_done;
2082 }
2083
2084 object = vm_object_allocate(map_size);
2085 assert(object != VM_OBJECT_NULL);
2086
2087 if (permission & MAP_MEM_PURGABLE) {
2088 if (! (permission & VM_PROT_WRITE)) {
2089 /* if we can't write, we can't purge */
2090 vm_object_deallocate(object);
2091 kr = KERN_INVALID_ARGUMENT;
2092 goto make_mem_done;
2093 }
2094 object->purgable = VM_OBJECT_PURGABLE_NONVOLATILE;
2095 }
2096
2097 /*
2098 * The VM object is brand new and nobody else knows about it,
2099 * so we don't need to lock it.
2100 */
2101
2102 wimg_mode = object->wimg_bits;
2103 if (access == MAP_MEM_IO) {
2104 wimg_mode = VM_WIMG_IO;
2105 } else if (access == MAP_MEM_COPYBACK) {
2106 wimg_mode = VM_WIMG_DEFAULT;
2107 } else if (access == MAP_MEM_WTHRU) {
2108 wimg_mode = VM_WIMG_WTHRU;
2109 } else if (access == MAP_MEM_WCOMB) {
2110 wimg_mode = VM_WIMG_WCOMB;
2111 }
2112 if (access != MAP_MEM_NOOP) {
2113 object->wimg_bits = wimg_mode;
2114 }
2115 /* the object has no pages, so no WIMG bits to update here */
2116
2117 /*
2118 * XXX
2119 * We use this path when we want to make sure that
2120 * nobody messes with the object (coalesce, for
2121 * example) before we map it.
2122 * We might want to use these objects for transposition via
2123 * vm_object_transpose() too, so we don't want any copy or
2124 * shadow objects either...
2125 */
2126 object->copy_strategy = MEMORY_OBJECT_COPY_NONE;
2127
2128 user_entry->backing.object = object;
2129 user_entry->internal = TRUE;
2130 user_entry->is_sub_map = FALSE;
2131 user_entry->is_pager = FALSE;
2132 user_entry->offset = 0;
2133 user_entry->protection = protections;
2134 SET_MAP_MEM(access, user_entry->protection);
2135 user_entry->size = map_size;
2136
2137 /* user_object pager and internal fields are not used */
2138 /* when the object field is filled in. */
2139
2140 *size = CAST_DOWN(vm_size_t, map_size);
2141 *object_handle = user_handle;
2142 return KERN_SUCCESS;
2143 }
2144
2145 if (parent_entry == NULL ||
2146 (permission & MAP_MEM_NAMED_REUSE)) {
2147
2148 /* Create a named object based on address range within the task map */
2149 /* Go find the object at given address */
2150
2151 redo_lookup:
2152 vm_map_lock_read(target_map);
2153
2154 /* get the object associated with the target address */
2155 /* note we check the permission of the range against */
2156 /* that requested by the caller */
2157
2158 kr = vm_map_lookup_locked(&target_map, map_offset,
2159 protections, &version,
2160 &object, &obj_off, &prot, &wired, &behavior,
2161 &lo_offset, &hi_offset, &real_map);
2162 if (kr != KERN_SUCCESS) {
2163 vm_map_unlock_read(target_map);
2164 goto make_mem_done;
2165 }
2166 if (((prot & protections) != protections)
2167 || (object == kernel_object)) {
2168 kr = KERN_INVALID_RIGHT;
2169 vm_object_unlock(object);
2170 vm_map_unlock_read(target_map);
2171 if(real_map != target_map)
2172 vm_map_unlock_read(real_map);
2173 if(object == kernel_object) {
2174 printf("Warning: Attempt to create a named"
2175 " entry from the kernel_object\n");
2176 }
2177 goto make_mem_done;
2178 }
2179
2180 /* We have an object, now check to see if this object */
2181 /* is suitable. If not, create a shadow and share that */
2182
2183 /*
2184 * We have to unlock the VM object to avoid deadlocking with
2185 * a VM map lock (the lock ordering is map, the object), if we
2186 * need to modify the VM map to create a shadow object. Since
2187 * we might release the VM map lock below anyway, we have
2188 * to release the VM map lock now.
2189 * XXX FBDP There must be a way to avoid this double lookup...
2190 *
2191 * Take an extra reference on the VM object to make sure it's
2192 * not going to disappear.
2193 */
2194 vm_object_reference_locked(object); /* extra ref to hold obj */
2195 vm_object_unlock(object);
2196
2197 local_map = original_map;
2198 local_offset = map_offset;
2199 if(target_map != local_map) {
2200 vm_map_unlock_read(target_map);
2201 if(real_map != target_map)
2202 vm_map_unlock_read(real_map);
2203 vm_map_lock_read(local_map);
2204 target_map = local_map;
2205 real_map = local_map;
2206 }
2207 while(TRUE) {
2208 if(!vm_map_lookup_entry(local_map,
2209 local_offset, &map_entry)) {
2210 kr = KERN_INVALID_ARGUMENT;
2211 vm_map_unlock_read(target_map);
2212 if(real_map != target_map)
2213 vm_map_unlock_read(real_map);
2214 vm_object_deallocate(object); /* release extra ref */
2215 object = VM_OBJECT_NULL;
2216 goto make_mem_done;
2217 }
2218 if(!(map_entry->is_sub_map)) {
2219 if(map_entry->object.vm_object != object) {
2220 kr = KERN_INVALID_ARGUMENT;
2221 vm_map_unlock_read(target_map);
2222 if(real_map != target_map)
2223 vm_map_unlock_read(real_map);
2224 vm_object_deallocate(object); /* release extra ref */
2225 object = VM_OBJECT_NULL;
2226 goto make_mem_done;
2227 }
2228 break;
2229 } else {
2230 vm_map_t tmap;
2231 tmap = local_map;
2232 local_map = map_entry->object.sub_map;
2233
2234 vm_map_lock_read(local_map);
2235 vm_map_unlock_read(tmap);
2236 target_map = local_map;
2237 real_map = local_map;
2238 local_offset = local_offset - map_entry->vme_start;
2239 local_offset += map_entry->offset;
2240 }
2241 }
2242
2243 /*
2244 * We found the VM map entry, lock the VM object again.
2245 */
2246 vm_object_lock(object);
2247 if(map_entry->wired_count) {
2248 /* JMM - The check below should be reworked instead. */
2249 object->true_share = TRUE;
2250 }
2251 if(((map_entry->max_protection) & protections) != protections) {
2252 kr = KERN_INVALID_RIGHT;
2253 vm_object_unlock(object);
2254 vm_map_unlock_read(target_map);
2255 if(real_map != target_map)
2256 vm_map_unlock_read(real_map);
2257 vm_object_deallocate(object);
2258 object = VM_OBJECT_NULL;
2259 goto make_mem_done;
2260 }
2261
2262 mappable_size = hi_offset - obj_off;
2263 total_size = map_entry->vme_end - map_entry->vme_start;
2264 if(map_size > mappable_size) {
2265 /* try to extend mappable size if the entries */
2266 /* following are from the same object and are */
2267 /* compatible */
2268 next_entry = map_entry->vme_next;
2269 /* lets see if the next map entry is still */
2270 /* pointing at this object and is contiguous */
2271 while(map_size > mappable_size) {
2272 if((next_entry->object.vm_object == object) &&
2273 (next_entry->vme_start ==
2274 next_entry->vme_prev->vme_end) &&
2275 (next_entry->offset ==
2276 next_entry->vme_prev->offset +
2277 (next_entry->vme_prev->vme_end -
2278 next_entry->vme_prev->vme_start))) {
2279 if(((next_entry->max_protection)
2280 & protections) != protections) {
2281 break;
2282 }
2283 if (next_entry->needs_copy !=
2284 map_entry->needs_copy)
2285 break;
2286 mappable_size += next_entry->vme_end
2287 - next_entry->vme_start;
2288 total_size += next_entry->vme_end
2289 - next_entry->vme_start;
2290 next_entry = next_entry->vme_next;
2291 } else {
2292 break;
2293 }
2294
2295 }
2296 }
2297
2298 if(object->internal) {
2299 /* vm_map_lookup_locked will create a shadow if */
2300 /* needs_copy is set but does not check for the */
2301 /* other two conditions shown. It is important to */
2302 /* set up an object which will not be pulled from */
2303 /* under us. */
2304
2305 if ((map_entry->needs_copy || object->shadowed ||
2306 (object->size > total_size))
2307 && !object->true_share) {
2308 /*
2309 * We have to unlock the VM object before
2310 * trying to upgrade the VM map lock, to
2311 * honor lock ordering (map then object).
2312 * Otherwise, we would deadlock if another
2313 * thread holds a read lock on the VM map and
2314 * is trying to acquire the VM object's lock.
2315 * We still hold an extra reference on the
2316 * VM object, guaranteeing that it won't
2317 * disappear.
2318 */
2319 vm_object_unlock(object);
2320
2321 if (vm_map_lock_read_to_write(target_map)) {
2322 /*
2323 * We couldn't upgrade our VM map lock
2324 * from "read" to "write" and we lost
2325 * our "read" lock.
2326 * Start all over again...
2327 */
2328 vm_object_deallocate(object); /* extra ref */
2329 target_map = original_map;
2330 goto redo_lookup;
2331 }
2332 vm_object_lock(object);
2333
2334 /*
2335 * JMM - We need to avoid coming here when the object
2336 * is wired by anybody, not just the current map. Why
2337 * couldn't we use the standard vm_object_copy_quickly()
2338 * approach here?
2339 */
2340
2341 /* create a shadow object */
2342 vm_object_shadow(&map_entry->object.vm_object,
2343 &map_entry->offset, total_size);
2344 shadow_object = map_entry->object.vm_object;
2345 vm_object_unlock(object);
2346
2347 vm_object_pmap_protect(
2348 object, map_entry->offset,
2349 total_size,
2350 ((map_entry->is_shared
2351 || target_map->mapped)
2352 ? PMAP_NULL :
2353 target_map->pmap),
2354 map_entry->vme_start,
2355 map_entry->protection & ~VM_PROT_WRITE);
2356 total_size -= (map_entry->vme_end
2357 - map_entry->vme_start);
2358 next_entry = map_entry->vme_next;
2359 map_entry->needs_copy = FALSE;
2360 while (total_size) {
2361 if(next_entry->object.vm_object == object) {
2362 shadow_object->ref_count++;
2363 vm_object_res_reference(shadow_object);
2364 next_entry->object.vm_object
2365 = shadow_object;
2366 vm_object_deallocate(object);
2367 next_entry->offset
2368 = next_entry->vme_prev->offset +
2369 (next_entry->vme_prev->vme_end
2370 - next_entry->vme_prev->vme_start);
2371 next_entry->needs_copy = FALSE;
2372 } else {
2373 panic("mach_make_memory_entry_64:"
2374 " map entries out of sync\n");
2375 }
2376 total_size -=
2377 next_entry->vme_end
2378 - next_entry->vme_start;
2379 next_entry = next_entry->vme_next;
2380 }
2381
2382 /*
2383 * Transfer our extra reference to the
2384 * shadow object.
2385 */
2386 vm_object_reference_locked(shadow_object);
2387 vm_object_deallocate(object); /* extra ref */
2388 object = shadow_object;
2389
2390 obj_off = (local_offset - map_entry->vme_start)
2391 + map_entry->offset;
2392
2393 vm_map_lock_write_to_read(target_map);
2394 vm_object_lock(object);
2395
2396 }
2397 }
2398
2399 /* note: in the future we can (if necessary) allow for */
2400 /* memory object lists, this will better support */
2401 /* fragmentation, but is it necessary? The user should */
2402 /* be encouraged to create address space oriented */
2403 /* shared objects from CLEAN memory regions which have */
2404 /* a known and defined history. i.e. no inheritence */
2405 /* share, make this call before making the region the */
2406 /* target of ipc's, etc. The code above, protecting */
2407 /* against delayed copy, etc. is mostly defensive. */
2408
2409 wimg_mode = object->wimg_bits;
2410 if(!(object->nophyscache)) {
2411 if(access == MAP_MEM_IO) {
2412 wimg_mode = VM_WIMG_IO;
2413 } else if (access == MAP_MEM_COPYBACK) {
2414 wimg_mode = VM_WIMG_USE_DEFAULT;
2415 } else if (access == MAP_MEM_WTHRU) {
2416 wimg_mode = VM_WIMG_WTHRU;
2417 } else if (access == MAP_MEM_WCOMB) {
2418 wimg_mode = VM_WIMG_WCOMB;
2419 }
2420 }
2421
2422 object->true_share = TRUE;
2423 if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC)
2424 object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
2425
2426 /*
2427 * The memory entry now points to this VM object and we
2428 * need to hold a reference on the VM object. Use the extra
2429 * reference we took earlier to keep the object alive when we
2430 * had to unlock it.
2431 */
2432
2433 vm_map_unlock_read(target_map);
2434 if(real_map != target_map)
2435 vm_map_unlock_read(real_map);
2436
2437 if(object->wimg_bits != wimg_mode) {
2438 vm_page_t p;
2439
2440 vm_object_paging_wait(object, THREAD_UNINT);
2441
2442 if ((wimg_mode == VM_WIMG_IO)
2443 || (wimg_mode == VM_WIMG_WCOMB))
2444 cache_attr = TRUE;
2445 else
2446 cache_attr = FALSE;
2447
2448 queue_iterate(&object->memq,
2449 p, vm_page_t, listq) {
2450 if (!p->fictitious) {
2451 pmap_disconnect(p->phys_page);
2452 if (cache_attr)
2453 pmap_sync_page_attributes_phys(p->phys_page);
2454 }
2455 }
2456 object->wimg_bits = wimg_mode;
2457 }
2458
2459 /* the size of mapped entry that overlaps with our region */
2460 /* which is targeted for share. */
2461 /* (entry_end - entry_start) - */
2462 /* offset of our beg addr within entry */
2463 /* it corresponds to this: */
2464
2465 if(map_size > mappable_size)
2466 map_size = mappable_size;
2467
2468 if (permission & MAP_MEM_NAMED_REUSE) {
2469 /*
2470 * Compare what we got with the "parent_entry".
2471 * If they match, re-use the "parent_entry" instead
2472 * of creating a new one.
2473 */
2474 if (parent_entry != NULL &&
2475 parent_entry->backing.object == object &&
2476 parent_entry->internal == object->internal &&
2477 parent_entry->is_sub_map == FALSE &&
2478 parent_entry->is_pager == FALSE &&
2479 parent_entry->offset == obj_off &&
2480 parent_entry->protection == protections &&
2481 parent_entry->size == map_size) {
2482 /*
2483 * We have a match: re-use "parent_entry".
2484 */
2485 /* release our extra reference on object */
2486 vm_object_unlock(object);
2487 vm_object_deallocate(object);
2488 /* parent_entry->ref_count++; XXX ? */
2489 /* Get an extra send-right on handle */
2490 ipc_port_copy_send(parent_handle);
2491 *object_handle = parent_handle;
2492 return KERN_SUCCESS;
2493 } else {
2494 /*
2495 * No match: we need to create a new entry.
2496 * fall through...
2497 */
2498 }
2499 }
2500
2501 vm_object_unlock(object);
2502 if (mach_memory_entry_allocate(&user_entry, &user_handle)
2503 != KERN_SUCCESS) {
2504 /* release our unused reference on the object */
2505 vm_object_deallocate(object);
2506 return KERN_FAILURE;
2507 }
2508
2509 user_entry->backing.object = object;
2510 user_entry->internal = object->internal;
2511 user_entry->is_sub_map = FALSE;
2512 user_entry->is_pager = FALSE;
2513 user_entry->offset = obj_off;
2514 user_entry->protection = permission;
2515 user_entry->size = map_size;
2516
2517 /* user_object pager and internal fields are not used */
2518 /* when the object field is filled in. */
2519
2520 *size = CAST_DOWN(vm_size_t, map_size);
2521 *object_handle = user_handle;
2522 return KERN_SUCCESS;
2523
2524 } else {
2525 /* The new object will be base on an existing named object */
2526
2527 if (parent_entry == NULL) {
2528 kr = KERN_INVALID_ARGUMENT;
2529 goto make_mem_done;
2530 }
2531 if((offset + map_size) > parent_entry->size) {
2532 kr = KERN_INVALID_ARGUMENT;
2533 goto make_mem_done;
2534 }
2535
2536 if((protections & parent_entry->protection) != protections) {
2537 kr = KERN_PROTECTION_FAILURE;
2538 goto make_mem_done;
2539 }
2540
2541 if (mach_memory_entry_allocate(&user_entry, &user_handle)
2542 != KERN_SUCCESS) {
2543 kr = KERN_FAILURE;
2544 goto make_mem_done;
2545 }
2546
2547 user_entry->size = map_size;
2548 user_entry->offset = parent_entry->offset + map_offset;
2549 user_entry->is_sub_map = parent_entry->is_sub_map;
2550 user_entry->is_pager = parent_entry->is_pager;
2551 user_entry->internal = parent_entry->internal;
2552 user_entry->protection = protections;
2553
2554 if(access != MAP_MEM_NOOP) {
2555 SET_MAP_MEM(access, user_entry->protection);
2556 }
2557
2558 if(parent_entry->is_sub_map) {
2559 user_entry->backing.map = parent_entry->backing.map;
2560 vm_map_lock(user_entry->backing.map);
2561 user_entry->backing.map->ref_count++;
2562 vm_map_unlock(user_entry->backing.map);
2563 }
2564 else if (parent_entry->is_pager) {
2565 user_entry->backing.pager = parent_entry->backing.pager;
2566 /* JMM - don't we need a reference here? */
2567 } else {
2568 object = parent_entry->backing.object;
2569 assert(object != VM_OBJECT_NULL);
2570 user_entry->backing.object = object;
2571 /* we now point to this object, hold on */
2572 vm_object_reference(object);
2573 vm_object_lock(object);
2574 object->true_share = TRUE;
2575 if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC)
2576 object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
2577 vm_object_unlock(object);
2578 }
2579 *size = CAST_DOWN(vm_size_t, map_size);
2580 *object_handle = user_handle;
2581 return KERN_SUCCESS;
2582 }
2583
2584 make_mem_done:
2585 if (user_handle != IP_NULL) {
2586 ipc_port_dealloc_kernel(user_handle);
2587 }
2588 if (user_entry != NULL) {
2589 kfree(user_entry, sizeof *user_entry);
2590 }
2591 return kr;
2592 }
2593
2594 kern_return_t
2595 _mach_make_memory_entry(
2596 vm_map_t target_map,
2597 memory_object_size_t *size,
2598 memory_object_offset_t offset,
2599 vm_prot_t permission,
2600 ipc_port_t *object_handle,
2601 ipc_port_t parent_entry)
2602 {
2603 memory_object_offset_t mo_size;
2604 kern_return_t kr;
2605
2606 mo_size = (memory_object_offset_t)*size;
2607 kr = mach_make_memory_entry_64(target_map, &mo_size,
2608 (memory_object_offset_t)offset, permission, object_handle,
2609 parent_entry);
2610 *size = mo_size;
2611 return kr;
2612 }
2613
2614 kern_return_t
2615 mach_make_memory_entry(
2616 vm_map_t target_map,
2617 vm_size_t *size,
2618 vm_offset_t offset,
2619 vm_prot_t permission,
2620 ipc_port_t *object_handle,
2621 ipc_port_t parent_entry)
2622 {
2623 memory_object_offset_t mo_size;
2624 kern_return_t kr;
2625
2626 mo_size = (memory_object_offset_t)*size;
2627 kr = mach_make_memory_entry_64(target_map, &mo_size,
2628 (memory_object_offset_t)offset, permission, object_handle,
2629 parent_entry);
2630 *size = CAST_DOWN(vm_size_t, mo_size);
2631 return kr;
2632 }
2633
2634 /*
2635 * task_wire
2636 *
2637 * Set or clear the map's wiring_required flag. This flag, if set,
2638 * will cause all future virtual memory allocation to allocate
2639 * user wired memory. Unwiring pages wired down as a result of
2640 * this routine is done with the vm_wire interface.
2641 */
2642 kern_return_t
2643 task_wire(
2644 vm_map_t map,
2645 boolean_t must_wire)
2646 {
2647 if (map == VM_MAP_NULL)
2648 return(KERN_INVALID_ARGUMENT);
2649
2650 if (must_wire)
2651 map->wiring_required = TRUE;
2652 else
2653 map->wiring_required = FALSE;
2654
2655 return(KERN_SUCCESS);
2656 }
2657
2658 __private_extern__ kern_return_t
2659 mach_memory_entry_allocate(
2660 vm_named_entry_t *user_entry_p,
2661 ipc_port_t *user_handle_p)
2662 {
2663 vm_named_entry_t user_entry;
2664 ipc_port_t user_handle;
2665 ipc_port_t previous;
2666
2667 user_entry = (vm_named_entry_t) kalloc(sizeof *user_entry);
2668 if (user_entry == NULL)
2669 return KERN_FAILURE;
2670
2671 named_entry_lock_init(user_entry);
2672
2673 user_handle = ipc_port_alloc_kernel();
2674 if (user_handle == IP_NULL) {
2675 kfree(user_entry, sizeof *user_entry);
2676 return KERN_FAILURE;
2677 }
2678 ip_lock(user_handle);
2679
2680 /* make a sonce right */
2681 user_handle->ip_sorights++;
2682 ip_reference(user_handle);
2683
2684 user_handle->ip_destination = IP_NULL;
2685 user_handle->ip_receiver_name = MACH_PORT_NULL;
2686 user_handle->ip_receiver = ipc_space_kernel;
2687
2688 /* make a send right */
2689 user_handle->ip_mscount++;
2690 user_handle->ip_srights++;
2691 ip_reference(user_handle);
2692
2693 ipc_port_nsrequest(user_handle, 1, user_handle, &previous);
2694 /* nsrequest unlocks user_handle */
2695
2696 user_entry->backing.pager = NULL;
2697 user_entry->is_sub_map = FALSE;
2698 user_entry->is_pager = FALSE;
2699 user_entry->size = 0;
2700 user_entry->internal = FALSE;
2701 user_entry->ref_count = 1;
2702
2703 ipc_kobject_set(user_handle, (ipc_kobject_t) user_entry,
2704 IKOT_NAMED_ENTRY);
2705
2706 *user_entry_p = user_entry;
2707 *user_handle_p = user_handle;
2708
2709 return KERN_SUCCESS;
2710 }
2711
2712 /*
2713 * mach_memory_object_memory_entry_64
2714 *
2715 * Create a named entry backed by the provided pager.
2716 *
2717 * JMM - we need to hold a reference on the pager -
2718 * and release it when the named entry is destroyed.
2719 */
2720 kern_return_t
2721 mach_memory_object_memory_entry_64(
2722 host_t host,
2723 boolean_t internal,
2724 vm_object_offset_t size,
2725 vm_prot_t permission,
2726 memory_object_t pager,
2727 ipc_port_t *entry_handle)
2728 {
2729 unsigned int access;
2730 vm_named_entry_t user_entry;
2731 ipc_port_t user_handle;
2732
2733 if (host == HOST_NULL)
2734 return(KERN_INVALID_HOST);
2735
2736 if (mach_memory_entry_allocate(&user_entry, &user_handle)
2737 != KERN_SUCCESS) {
2738 return KERN_FAILURE;
2739 }
2740
2741 user_entry->backing.pager = pager;
2742 user_entry->size = size;
2743 user_entry->offset = 0;
2744 user_entry->protection = permission & VM_PROT_ALL;
2745 access = GET_MAP_MEM(permission);
2746 SET_MAP_MEM(access, user_entry->protection);
2747 user_entry->internal = internal;
2748 user_entry->is_sub_map = FALSE;
2749 user_entry->is_pager = TRUE;
2750 assert(user_entry->ref_count == 1);
2751
2752 *entry_handle = user_handle;
2753 return KERN_SUCCESS;
2754 }
2755
2756 kern_return_t
2757 mach_memory_object_memory_entry(
2758 host_t host,
2759 boolean_t internal,
2760 vm_size_t size,
2761 vm_prot_t permission,
2762 memory_object_t pager,
2763 ipc_port_t *entry_handle)
2764 {
2765 return mach_memory_object_memory_entry_64( host, internal,
2766 (vm_object_offset_t)size, permission, pager, entry_handle);
2767 }
2768
2769
2770 kern_return_t
2771 mach_memory_entry_purgable_control(
2772 ipc_port_t entry_port,
2773 vm_purgable_t control,
2774 int *state)
2775 {
2776 kern_return_t kr;
2777 vm_named_entry_t mem_entry;
2778 vm_object_t object;
2779
2780 if (entry_port == IP_NULL ||
2781 ip_kotype(entry_port) != IKOT_NAMED_ENTRY) {
2782 return KERN_INVALID_ARGUMENT;
2783 }
2784
2785 mem_entry = (vm_named_entry_t) entry_port->ip_kobject;
2786
2787 named_entry_lock(mem_entry);
2788
2789 if (mem_entry->is_sub_map || mem_entry->is_pager) {
2790 named_entry_unlock(mem_entry);
2791 return KERN_INVALID_ARGUMENT;
2792 }
2793
2794 object = mem_entry->backing.object;
2795 if (object == VM_OBJECT_NULL) {
2796 named_entry_unlock(mem_entry);
2797 return KERN_INVALID_ARGUMENT;
2798 }
2799
2800 vm_object_lock(object);
2801
2802 /* check that named entry covers entire object ? */
2803 if (mem_entry->offset != 0 || object->size != mem_entry->size) {
2804 vm_object_unlock(object);
2805 named_entry_unlock(mem_entry);
2806 return KERN_INVALID_ARGUMENT;
2807 }
2808
2809 named_entry_unlock(mem_entry);
2810
2811 kr = vm_object_purgable_control(object, control, state);
2812
2813 vm_object_unlock(object);
2814
2815 return kr;
2816 }
2817
2818 /*
2819 * mach_memory_entry_port_release:
2820 *
2821 * Release a send right on a named entry port. This is the correct
2822 * way to destroy a named entry. When the last right on the port is
2823 * released, ipc_kobject_destroy() will call mach_destroy_memory_entry().
2824 */
2825 void
2826 mach_memory_entry_port_release(
2827 ipc_port_t port)
2828 {
2829 assert(ip_kotype(port) == IKOT_NAMED_ENTRY);
2830 ipc_port_release_send(port);
2831 }
2832
2833 /*
2834 * mach_destroy_memory_entry:
2835 *
2836 * Drops a reference on a memory entry and destroys the memory entry if
2837 * there are no more references on it.
2838 * NOTE: This routine should not be called to destroy a memory entry from the
2839 * kernel, as it will not release the Mach port associated with the memory
2840 * entry. The proper way to destroy a memory entry in the kernel is to
2841 * call mach_memort_entry_port_release() to release the kernel's send-right on
2842 * the memory entry's port. When the last send right is released, the memory
2843 * entry will be destroyed via ipc_kobject_destroy().
2844 */
2845 void
2846 mach_destroy_memory_entry(
2847 ipc_port_t port)
2848 {
2849 vm_named_entry_t named_entry;
2850 #if MACH_ASSERT
2851 assert(ip_kotype(port) == IKOT_NAMED_ENTRY);
2852 #endif /* MACH_ASSERT */
2853 named_entry = (vm_named_entry_t)port->ip_kobject;
2854 mutex_lock(&(named_entry)->Lock);
2855 named_entry->ref_count -= 1;
2856 if(named_entry->ref_count == 0) {
2857 if (named_entry->is_sub_map) {
2858 vm_map_deallocate(named_entry->backing.map);
2859 } else if (!named_entry->is_pager) {
2860 /* release the memory object we've been pointing to */
2861 vm_object_deallocate(named_entry->backing.object);
2862 } /* else JMM - need to drop reference on pager in that case */
2863
2864 mutex_unlock(&(named_entry)->Lock);
2865
2866 kfree((void *) port->ip_kobject,
2867 sizeof (struct vm_named_entry));
2868 } else
2869 mutex_unlock(&(named_entry)->Lock);
2870 }
2871
2872
2873
2874 kern_return_t
2875 set_dp_control_port(
2876 host_priv_t host_priv,
2877 ipc_port_t control_port)
2878 {
2879 if (host_priv == HOST_PRIV_NULL)
2880 return (KERN_INVALID_HOST);
2881
2882 if (IP_VALID(dynamic_pager_control_port))
2883 ipc_port_release_send(dynamic_pager_control_port);
2884
2885 dynamic_pager_control_port = control_port;
2886 return KERN_SUCCESS;
2887 }
2888
2889 kern_return_t
2890 get_dp_control_port(
2891 host_priv_t host_priv,
2892 ipc_port_t *control_port)
2893 {
2894 if (host_priv == HOST_PRIV_NULL)
2895 return (KERN_INVALID_HOST);
2896
2897 *control_port = ipc_port_copy_send(dynamic_pager_control_port);
2898 return KERN_SUCCESS;
2899
2900 }
2901
2902 /* ******* Temporary Internal calls to UPL for BSD ***** */
2903
2904 extern int kernel_upl_map(
2905 vm_map_t map,
2906 upl_t upl,
2907 vm_offset_t *dst_addr);
2908
2909 extern int kernel_upl_unmap(
2910 vm_map_t map,
2911 upl_t upl);
2912
2913 extern int kernel_upl_commit(
2914 upl_t upl,
2915 upl_page_info_t *pl,
2916 mach_msg_type_number_t count);
2917
2918 extern int kernel_upl_commit_range(
2919 upl_t upl,
2920 upl_offset_t offset,
2921 upl_size_t size,
2922 int flags,
2923 upl_page_info_array_t pl,
2924 mach_msg_type_number_t count);
2925
2926 extern int kernel_upl_abort(
2927 upl_t upl,
2928 int abort_type);
2929
2930 extern int kernel_upl_abort_range(
2931 upl_t upl,
2932 upl_offset_t offset,
2933 upl_size_t size,
2934 int abort_flags);
2935
2936
2937 kern_return_t
2938 kernel_upl_map(
2939 vm_map_t map,
2940 upl_t upl,
2941 vm_offset_t *dst_addr)
2942 {
2943 return vm_upl_map(map, upl, dst_addr);
2944 }
2945
2946
2947 kern_return_t
2948 kernel_upl_unmap(
2949 vm_map_t map,
2950 upl_t upl)
2951 {
2952 return vm_upl_unmap(map, upl);
2953 }
2954
2955 kern_return_t
2956 kernel_upl_commit(
2957 upl_t upl,
2958 upl_page_info_t *pl,
2959 mach_msg_type_number_t count)
2960 {
2961 kern_return_t kr;
2962
2963 kr = upl_commit(upl, pl, count);
2964 upl_deallocate(upl);
2965 return kr;
2966 }
2967
2968
2969 kern_return_t
2970 kernel_upl_commit_range(
2971 upl_t upl,
2972 upl_offset_t offset,
2973 upl_size_t size,
2974 int flags,
2975 upl_page_info_array_t pl,
2976 mach_msg_type_number_t count)
2977 {
2978 boolean_t finished = FALSE;
2979 kern_return_t kr;
2980
2981 if (flags & UPL_COMMIT_FREE_ON_EMPTY)
2982 flags |= UPL_COMMIT_NOTIFY_EMPTY;
2983
2984 kr = upl_commit_range(upl, offset, size, flags, pl, count, &finished);
2985
2986 if ((flags & UPL_COMMIT_NOTIFY_EMPTY) && finished)
2987 upl_deallocate(upl);
2988
2989 return kr;
2990 }
2991
2992 kern_return_t
2993 kernel_upl_abort_range(
2994 upl_t upl,
2995 upl_offset_t offset,
2996 upl_size_t size,
2997 int abort_flags)
2998 {
2999 kern_return_t kr;
3000 boolean_t finished = FALSE;
3001
3002 if (abort_flags & UPL_COMMIT_FREE_ON_EMPTY)
3003 abort_flags |= UPL_COMMIT_NOTIFY_EMPTY;
3004
3005 kr = upl_abort_range(upl, offset, size, abort_flags, &finished);
3006
3007 if ((abort_flags & UPL_COMMIT_FREE_ON_EMPTY) && finished)
3008 upl_deallocate(upl);
3009
3010 return kr;
3011 }
3012
3013 kern_return_t
3014 kernel_upl_abort(
3015 upl_t upl,
3016 int abort_type)
3017 {
3018 kern_return_t kr;
3019
3020 kr = upl_abort(upl, abort_type);
3021 upl_deallocate(upl);
3022 return kr;
3023 }
3024
3025 /*
3026 * Now a kernel-private interface (for BootCache
3027 * use only). Need a cleaner way to create an
3028 * empty vm_map() and return a handle to it.
3029 */
3030
3031 kern_return_t
3032 vm_region_object_create(
3033 __unused vm_map_t target_map,
3034 vm_size_t size,
3035 ipc_port_t *object_handle)
3036 {
3037 vm_named_entry_t user_entry;
3038 ipc_port_t user_handle;
3039
3040 vm_map_t new_map;
3041
3042 if (mach_memory_entry_allocate(&user_entry, &user_handle)
3043 != KERN_SUCCESS) {
3044 return KERN_FAILURE;
3045 }
3046
3047 /* Create a named object based on a submap of specified size */
3048
3049 new_map = vm_map_create(PMAP_NULL, VM_MAP_MIN_ADDRESS,
3050 vm_map_round_page(size), TRUE);
3051
3052 user_entry->backing.map = new_map;
3053 user_entry->internal = TRUE;
3054 user_entry->is_sub_map = TRUE;
3055 user_entry->offset = 0;
3056 user_entry->protection = VM_PROT_ALL;
3057 user_entry->size = size;
3058 assert(user_entry->ref_count == 1);
3059
3060 *object_handle = user_handle;
3061 return KERN_SUCCESS;
3062
3063 }
3064
3065 ppnum_t vm_map_get_phys_page( /* forward */
3066 vm_map_t map,
3067 vm_offset_t offset);
3068
3069 ppnum_t
3070 vm_map_get_phys_page(
3071 vm_map_t map,
3072 vm_offset_t addr)
3073 {
3074 vm_object_offset_t offset;
3075 vm_object_t object;
3076 vm_map_offset_t map_offset;
3077 vm_map_entry_t entry;
3078 ppnum_t phys_page = 0;
3079
3080 map_offset = vm_map_trunc_page(addr);
3081
3082 vm_map_lock(map);
3083 while (vm_map_lookup_entry(map, map_offset, &entry)) {
3084
3085 if (entry->object.vm_object == VM_OBJECT_NULL) {
3086 vm_map_unlock(map);
3087 return (ppnum_t) 0;
3088 }
3089 if (entry->is_sub_map) {
3090 vm_map_t old_map;
3091 vm_map_lock(entry->object.sub_map);
3092 old_map = map;
3093 map = entry->object.sub_map;
3094 map_offset = entry->offset + (map_offset - entry->vme_start);
3095 vm_map_unlock(old_map);
3096 continue;
3097 }
3098 if (entry->object.vm_object->phys_contiguous) {
3099 /* These are not standard pageable memory mappings */
3100 /* If they are not present in the object they will */
3101 /* have to be picked up from the pager through the */
3102 /* fault mechanism. */
3103 if(entry->object.vm_object->shadow_offset == 0) {
3104 /* need to call vm_fault */
3105 vm_map_unlock(map);
3106 vm_fault(map, map_offset, VM_PROT_NONE,
3107 FALSE, THREAD_UNINT, NULL, 0);
3108 vm_map_lock(map);
3109 continue;
3110 }
3111 offset = entry->offset + (map_offset - entry->vme_start);
3112 phys_page = (ppnum_t)
3113 ((entry->object.vm_object->shadow_offset
3114 + offset) >> 12);
3115 break;
3116
3117 }
3118 offset = entry->offset + (map_offset - entry->vme_start);
3119 object = entry->object.vm_object;
3120 vm_object_lock(object);
3121 while (TRUE) {
3122 vm_page_t dst_page = vm_page_lookup(object,offset);
3123 if(dst_page == VM_PAGE_NULL) {
3124 if(object->shadow) {
3125 vm_object_t old_object;
3126 vm_object_lock(object->shadow);
3127 old_object = object;
3128 offset = offset + object->shadow_offset;
3129 object = object->shadow;
3130 vm_object_unlock(old_object);
3131 } else {
3132 vm_object_unlock(object);
3133 break;
3134 }
3135 } else {
3136 phys_page = (ppnum_t)(dst_page->phys_page);
3137 vm_object_unlock(object);
3138 break;
3139 }
3140 }
3141 break;
3142
3143 }
3144
3145 vm_map_unlock(map);
3146 return phys_page;
3147 }
3148
3149
3150
3151 kern_return_t kernel_object_iopl_request( /* forward */
3152 vm_named_entry_t named_entry,
3153 memory_object_offset_t offset,
3154 vm_size_t *upl_size,
3155 upl_t *upl_ptr,
3156 upl_page_info_array_t user_page_list,
3157 unsigned int *page_list_count,
3158 int *flags);
3159
3160 kern_return_t
3161 kernel_object_iopl_request(
3162 vm_named_entry_t named_entry,
3163 memory_object_offset_t offset,
3164 vm_size_t *upl_size,
3165 upl_t *upl_ptr,
3166 upl_page_info_array_t user_page_list,
3167 unsigned int *page_list_count,
3168 int *flags)
3169 {
3170 vm_object_t object;
3171 kern_return_t ret;
3172
3173 int caller_flags;
3174
3175 caller_flags = *flags;
3176
3177 if (caller_flags & ~UPL_VALID_FLAGS) {
3178 /*
3179 * For forward compatibility's sake,
3180 * reject any unknown flag.
3181 */
3182 return KERN_INVALID_VALUE;
3183 }
3184
3185 /* a few checks to make sure user is obeying rules */
3186 if(*upl_size == 0) {
3187 if(offset >= named_entry->size)
3188 return(KERN_INVALID_RIGHT);
3189 *upl_size = named_entry->size - offset;
3190 }
3191 if(caller_flags & UPL_COPYOUT_FROM) {
3192 if((named_entry->protection & VM_PROT_READ)
3193 != VM_PROT_READ) {
3194 return(KERN_INVALID_RIGHT);
3195 }
3196 } else {
3197 if((named_entry->protection &
3198 (VM_PROT_READ | VM_PROT_WRITE))
3199 != (VM_PROT_READ | VM_PROT_WRITE)) {
3200 return(KERN_INVALID_RIGHT);
3201 }
3202 }
3203 if(named_entry->size < (offset + *upl_size))
3204 return(KERN_INVALID_ARGUMENT);
3205
3206 /* the callers parameter offset is defined to be the */
3207 /* offset from beginning of named entry offset in object */
3208 offset = offset + named_entry->offset;
3209
3210 if(named_entry->is_sub_map)
3211 return (KERN_INVALID_ARGUMENT);
3212
3213 named_entry_lock(named_entry);
3214
3215 if (named_entry->is_pager) {
3216 object = vm_object_enter(named_entry->backing.pager,
3217 named_entry->offset + named_entry->size,
3218 named_entry->internal,
3219 FALSE,
3220 FALSE);
3221 if (object == VM_OBJECT_NULL) {
3222 named_entry_unlock(named_entry);
3223 return(KERN_INVALID_OBJECT);
3224 }
3225
3226 /* JMM - drop reference on the pager here? */
3227
3228 /* create an extra reference for the object */
3229 vm_object_lock(object);
3230 vm_object_reference_locked(object);
3231 named_entry->backing.object = object;
3232 named_entry->is_pager = FALSE;
3233 named_entry_unlock(named_entry);
3234
3235 /* wait for object (if any) to be ready */
3236 if (!named_entry->internal) {
3237 while (!object->pager_ready) {
3238 vm_object_wait(object,
3239 VM_OBJECT_EVENT_PAGER_READY,
3240 THREAD_UNINT);
3241 vm_object_lock(object);
3242 }
3243 }
3244 vm_object_unlock(object);
3245
3246 } else {
3247 /* This is the case where we are going to operate */
3248 /* an an already known object. If the object is */
3249 /* not ready it is internal. An external */
3250 /* object cannot be mapped until it is ready */
3251 /* we can therefore avoid the ready check */
3252 /* in this case. */
3253 object = named_entry->backing.object;
3254 vm_object_reference(object);
3255 named_entry_unlock(named_entry);
3256 }
3257
3258 if (!object->private) {
3259 if (*upl_size > (MAX_UPL_TRANSFER*PAGE_SIZE))
3260 *upl_size = (MAX_UPL_TRANSFER*PAGE_SIZE);
3261 if (object->phys_contiguous) {
3262 *flags = UPL_PHYS_CONTIG;
3263 } else {
3264 *flags = 0;
3265 }
3266 } else {
3267 *flags = UPL_DEV_MEMORY | UPL_PHYS_CONTIG;
3268 }
3269
3270 ret = vm_object_iopl_request(object,
3271 offset,
3272 *upl_size,
3273 upl_ptr,
3274 user_page_list,
3275 page_list_count,
3276 caller_flags);
3277 vm_object_deallocate(object);
3278 return ret;
3279 }
mirror of XNU