]> git.saurik.com Git - apple/xnu.git/blob - osfmk/vm/vm_kern.c
xnu-1504.9.17.tar.gz
[apple/xnu.git] / osfmk / vm / vm_kern.c
1 /*
2 * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56 /*
57 */
58 /*
59 * File: vm/vm_kern.c
60 * Author: Avadis Tevanian, Jr., Michael Wayne Young
61 * Date: 1985
62 *
63 * Kernel memory management.
64 */
65
66 #include <mach/kern_return.h>
67 #include <mach/vm_param.h>
68 #include <kern/assert.h>
69 #include <kern/lock.h>
70 #include <kern/thread.h>
71 #include <vm/vm_kern.h>
72 #include <vm/vm_map.h>
73 #include <vm/vm_object.h>
74 #include <vm/vm_page.h>
75 #include <vm/vm_pageout.h>
76 #include <kern/misc_protos.h>
77 #include <vm/cpm.h>
78
79 #include <string.h>
80
81 #include <libkern/OSDebug.h>
82 #include <sys/kdebug.h>
83
84 /*
85 * Variables exported by this module.
86 */
87
88 vm_map_t kernel_map;
89 vm_map_t kernel_pageable_map;
90
91 extern boolean_t vm_kernel_ready;
92
93 /*
94 * Forward declarations for internal functions.
95 */
96 extern kern_return_t kmem_alloc_pages(
97 register vm_object_t object,
98 register vm_object_offset_t offset,
99 register vm_object_size_t size);
100
101 extern void kmem_remap_pages(
102 register vm_object_t object,
103 register vm_object_offset_t offset,
104 register vm_offset_t start,
105 register vm_offset_t end,
106 vm_prot_t protection);
107
108 kern_return_t
109 kmem_alloc_contig(
110 vm_map_t map,
111 vm_offset_t *addrp,
112 vm_size_t size,
113 vm_offset_t mask,
114 ppnum_t max_pnum,
115 ppnum_t pnum_mask,
116 int flags)
117 {
118 vm_object_t object;
119 vm_object_offset_t offset;
120 vm_map_offset_t map_addr;
121 vm_map_offset_t map_mask;
122 vm_map_size_t map_size, i;
123 vm_map_entry_t entry;
124 vm_page_t m, pages;
125 kern_return_t kr;
126
127 if (map == VM_MAP_NULL || (flags & ~(KMA_KOBJECT | KMA_LOMEM | KMA_NOPAGEWAIT)))
128 return KERN_INVALID_ARGUMENT;
129
130 if (size == 0) {
131 *addrp = 0;
132 return KERN_INVALID_ARGUMENT;
133 }
134
135 map_size = vm_map_round_page(size);
136 map_mask = (vm_map_offset_t)mask;
137
138 /*
139 * Allocate a new object (if necessary) and the reference we
140 * will be donating to the map entry. We must do this before
141 * locking the map, or risk deadlock with the default pager.
142 */
143 if ((flags & KMA_KOBJECT) != 0) {
144 object = kernel_object;
145 vm_object_reference(object);
146 } else {
147 object = vm_object_allocate(map_size);
148 }
149
150 kr = vm_map_find_space(map, &map_addr, map_size, map_mask, 0, &entry);
151 if (KERN_SUCCESS != kr) {
152 vm_object_deallocate(object);
153 return kr;
154 }
155
156 entry->object.vm_object = object;
157 entry->offset = offset = (object == kernel_object) ?
158 map_addr : 0;
159
160 /* Take an extra object ref in case the map entry gets deleted */
161 vm_object_reference(object);
162 vm_map_unlock(map);
163
164 kr = cpm_allocate(CAST_DOWN(vm_size_t, map_size), &pages, max_pnum, pnum_mask, FALSE, flags);
165
166 if (kr != KERN_SUCCESS) {
167 vm_map_remove(map, vm_map_trunc_page(map_addr),
168 vm_map_round_page(map_addr + map_size), 0);
169 vm_object_deallocate(object);
170 *addrp = 0;
171 return kr;
172 }
173
174 vm_object_lock(object);
175 for (i = 0; i < map_size; i += PAGE_SIZE) {
176 m = pages;
177 pages = NEXT_PAGE(m);
178 *(NEXT_PAGE_PTR(m)) = VM_PAGE_NULL;
179 m->busy = FALSE;
180 vm_page_insert(m, object, offset + i);
181 }
182 vm_object_unlock(object);
183
184 if ((kr = vm_map_wire(map, vm_map_trunc_page(map_addr),
185 vm_map_round_page(map_addr + map_size), VM_PROT_DEFAULT, FALSE))
186 != KERN_SUCCESS) {
187 if (object == kernel_object) {
188 vm_object_lock(object);
189 vm_object_page_remove(object, offset, offset + map_size);
190 vm_object_unlock(object);
191 }
192 vm_map_remove(map, vm_map_trunc_page(map_addr),
193 vm_map_round_page(map_addr + map_size), 0);
194 vm_object_deallocate(object);
195 return kr;
196 }
197 vm_object_deallocate(object);
198
199 if (object == kernel_object)
200 vm_map_simplify(map, map_addr);
201
202 *addrp = (vm_offset_t) map_addr;
203 assert((vm_map_offset_t) *addrp == map_addr);
204 return KERN_SUCCESS;
205 }
206
207 /*
208 * Master entry point for allocating kernel memory.
209 * NOTE: this routine is _never_ interrupt safe.
210 *
211 * map : map to allocate into
212 * addrp : pointer to start address of new memory
213 * size : size of memory requested
214 * flags : options
215 * KMA_HERE *addrp is base address, else "anywhere"
216 * KMA_NOPAGEWAIT don't wait for pages if unavailable
217 * KMA_KOBJECT use kernel_object
218 * KMA_LOMEM support for 32 bit devices in a 64 bit world
219 * if set and a lomemory pool is available
220 * grab pages from it... this also implies
221 * KMA_NOPAGEWAIT
222 */
223
224 kern_return_t
225 kernel_memory_allocate(
226 register vm_map_t map,
227 register vm_offset_t *addrp,
228 register vm_size_t size,
229 register vm_offset_t mask,
230 int flags)
231 {
232 vm_object_t object;
233 vm_object_offset_t offset;
234 vm_object_offset_t pg_offset;
235 vm_map_entry_t entry;
236 vm_map_offset_t map_addr, fill_start;
237 vm_map_offset_t map_mask;
238 vm_map_size_t map_size, fill_size;
239 kern_return_t kr;
240 vm_page_t mem;
241 vm_page_t guard_page_list = NULL;
242 vm_page_t wired_page_list = NULL;
243 int guard_page_count = 0;
244 int wired_page_count = 0;
245 int i;
246 int vm_alloc_flags;
247
248 if (! vm_kernel_ready) {
249 panic("kernel_memory_allocate: VM is not ready");
250 }
251
252 if (size == 0) {
253 *addrp = 0;
254 return KERN_INVALID_ARGUMENT;
255 }
256 map_size = vm_map_round_page(size);
257 map_mask = (vm_map_offset_t) mask;
258 vm_alloc_flags = 0;
259
260
261 /*
262 * limit the size of a single extent of wired memory
263 * to try and limit the damage to the system if
264 * too many pages get wired down
265 */
266 if (map_size > (1 << 30)) {
267 return KERN_RESOURCE_SHORTAGE;
268 }
269
270 /*
271 * Guard pages:
272 *
273 * Guard pages are implemented as ficticious pages. By placing guard pages
274 * on either end of a stack, they can help detect cases where a thread walks
275 * off either end of its stack. They are allocated and set up here and attempts
276 * to access those pages are trapped in vm_fault_page().
277 *
278 * The map_size we were passed may include extra space for
279 * guard pages. If those were requested, then back it out of fill_size
280 * since vm_map_find_space() takes just the actual size not including
281 * guard pages. Similarly, fill_start indicates where the actual pages
282 * will begin in the range.
283 */
284
285 fill_start = 0;
286 fill_size = map_size;
287
288 if (flags & KMA_GUARD_FIRST) {
289 vm_alloc_flags |= VM_FLAGS_GUARD_BEFORE;
290 fill_start += PAGE_SIZE_64;
291 fill_size -= PAGE_SIZE_64;
292 if (map_size < fill_start + fill_size) {
293 /* no space for a guard page */
294 *addrp = 0;
295 return KERN_INVALID_ARGUMENT;
296 }
297 guard_page_count++;
298 }
299 if (flags & KMA_GUARD_LAST) {
300 vm_alloc_flags |= VM_FLAGS_GUARD_AFTER;
301 fill_size -= PAGE_SIZE_64;
302 if (map_size <= fill_start + fill_size) {
303 /* no space for a guard page */
304 *addrp = 0;
305 return KERN_INVALID_ARGUMENT;
306 }
307 guard_page_count++;
308 }
309 wired_page_count = (int) (fill_size / PAGE_SIZE_64);
310 assert(wired_page_count * PAGE_SIZE_64 == fill_size);
311
312 for (i = 0; i < guard_page_count; i++) {
313 for (;;) {
314 mem = vm_page_grab_guard();
315
316 if (mem != VM_PAGE_NULL)
317 break;
318 if (flags & KMA_NOPAGEWAIT) {
319 kr = KERN_RESOURCE_SHORTAGE;
320 goto out;
321 }
322 vm_page_more_fictitious();
323 }
324 mem->pageq.next = (queue_entry_t)guard_page_list;
325 guard_page_list = mem;
326 }
327
328 for (i = 0; i < wired_page_count; i++) {
329 uint64_t unavailable;
330
331 for (;;) {
332 if (flags & KMA_LOMEM)
333 mem = vm_page_grablo();
334 else
335 mem = vm_page_grab();
336
337 if (mem != VM_PAGE_NULL)
338 break;
339
340 if (flags & KMA_NOPAGEWAIT) {
341 kr = KERN_RESOURCE_SHORTAGE;
342 goto out;
343 }
344 if ((flags & KMA_LOMEM) && (vm_lopage_needed == TRUE)) {
345 kr = KERN_RESOURCE_SHORTAGE;
346 goto out;
347 }
348 unavailable = (vm_page_wire_count + vm_page_free_target) * PAGE_SIZE;
349
350 if (unavailable > max_mem || map_size > (max_mem - unavailable)) {
351 kr = KERN_RESOURCE_SHORTAGE;
352 goto out;
353 }
354 VM_PAGE_WAIT();
355 }
356 mem->pageq.next = (queue_entry_t)wired_page_list;
357 wired_page_list = mem;
358 }
359
360 /*
361 * Allocate a new object (if necessary). We must do this before
362 * locking the map, or risk deadlock with the default pager.
363 */
364 if ((flags & KMA_KOBJECT) != 0) {
365 object = kernel_object;
366 vm_object_reference(object);
367 } else {
368 object = vm_object_allocate(map_size);
369 }
370
371 kr = vm_map_find_space(map, &map_addr,
372 fill_size, map_mask,
373 vm_alloc_flags, &entry);
374 if (KERN_SUCCESS != kr) {
375 vm_object_deallocate(object);
376 goto out;
377 }
378
379 entry->object.vm_object = object;
380 entry->offset = offset = (object == kernel_object) ?
381 map_addr : 0;
382
383 entry->wired_count++;
384
385 if (flags & KMA_PERMANENT)
386 entry->permanent = TRUE;
387
388 if (object != kernel_object)
389 vm_object_reference(object);
390
391 vm_object_lock(object);
392 vm_map_unlock(map);
393
394 pg_offset = 0;
395
396 if (fill_start) {
397 if (guard_page_list == NULL)
398 panic("kernel_memory_allocate: guard_page_list == NULL");
399
400 mem = guard_page_list;
401 guard_page_list = (vm_page_t)mem->pageq.next;
402 mem->pageq.next = NULL;
403
404 vm_page_insert(mem, object, offset + pg_offset);
405
406 mem->busy = FALSE;
407 pg_offset += PAGE_SIZE_64;
408 }
409 for (pg_offset = fill_start; pg_offset < fill_start + fill_size; pg_offset += PAGE_SIZE_64) {
410 if (wired_page_list == NULL)
411 panic("kernel_memory_allocate: wired_page_list == NULL");
412
413 mem = wired_page_list;
414 wired_page_list = (vm_page_t)mem->pageq.next;
415 mem->pageq.next = NULL;
416 mem->wire_count++;
417
418 vm_page_insert(mem, object, offset + pg_offset);
419
420 mem->busy = FALSE;
421 mem->pmapped = TRUE;
422 mem->wpmapped = TRUE;
423
424 PMAP_ENTER(kernel_pmap, map_addr + pg_offset, mem,
425 VM_PROT_READ | VM_PROT_WRITE, object->wimg_bits & VM_WIMG_MASK, TRUE);
426
427 if (flags & KMA_NOENCRYPT) {
428 bzero(CAST_DOWN(void *, (map_addr + pg_offset)), PAGE_SIZE);
429
430 pmap_set_noencrypt(mem->phys_page);
431 }
432 }
433 if ((fill_start + fill_size) < map_size) {
434 if (guard_page_list == NULL)
435 panic("kernel_memory_allocate: guard_page_list == NULL");
436
437 mem = guard_page_list;
438 guard_page_list = (vm_page_t)mem->pageq.next;
439 mem->pageq.next = NULL;
440
441 vm_page_insert(mem, object, offset + pg_offset);
442
443 mem->busy = FALSE;
444 }
445 if (guard_page_list || wired_page_list)
446 panic("kernel_memory_allocate: non empty list\n");
447
448 vm_page_lockspin_queues();
449 vm_page_wire_count += wired_page_count;
450 vm_page_unlock_queues();
451
452 vm_object_unlock(object);
453
454 /*
455 * now that the pages are wired, we no longer have to fear coalesce
456 */
457 if (object == kernel_object)
458 vm_map_simplify(map, map_addr);
459 else
460 vm_object_deallocate(object);
461
462 /*
463 * Return the memory, not zeroed.
464 */
465 *addrp = CAST_DOWN(vm_offset_t, map_addr);
466 return KERN_SUCCESS;
467
468 out:
469 if (guard_page_list)
470 vm_page_free_list(guard_page_list, FALSE);
471
472 if (wired_page_list)
473 vm_page_free_list(wired_page_list, FALSE);
474
475 return kr;
476 }
477
478 /*
479 * kmem_alloc:
480 *
481 * Allocate wired-down memory in the kernel's address map
482 * or a submap. The memory is not zero-filled.
483 */
484
485 kern_return_t
486 kmem_alloc(
487 vm_map_t map,
488 vm_offset_t *addrp,
489 vm_size_t size)
490 {
491 kern_return_t kr = kernel_memory_allocate(map, addrp, size, 0, 0);
492 TRACE_MACHLEAKS(KMEM_ALLOC_CODE, KMEM_ALLOC_CODE_2, size, *addrp);
493 return kr;
494 }
495
496 /*
497 * kmem_realloc:
498 *
499 * Reallocate wired-down memory in the kernel's address map
500 * or a submap. Newly allocated pages are not zeroed.
501 * This can only be used on regions allocated with kmem_alloc.
502 *
503 * If successful, the pages in the old region are mapped twice.
504 * The old region is unchanged. Use kmem_free to get rid of it.
505 */
506 kern_return_t
507 kmem_realloc(
508 vm_map_t map,
509 vm_offset_t oldaddr,
510 vm_size_t oldsize,
511 vm_offset_t *newaddrp,
512 vm_size_t newsize)
513 {
514 vm_object_t object;
515 vm_object_offset_t offset;
516 vm_map_offset_t oldmapmin;
517 vm_map_offset_t oldmapmax;
518 vm_map_offset_t newmapaddr;
519 vm_map_size_t oldmapsize;
520 vm_map_size_t newmapsize;
521 vm_map_entry_t oldentry;
522 vm_map_entry_t newentry;
523 vm_page_t mem;
524 kern_return_t kr;
525
526 oldmapmin = vm_map_trunc_page(oldaddr);
527 oldmapmax = vm_map_round_page(oldaddr + oldsize);
528 oldmapsize = oldmapmax - oldmapmin;
529 newmapsize = vm_map_round_page(newsize);
530
531
532 /*
533 * Find the VM object backing the old region.
534 */
535
536 vm_map_lock(map);
537
538 if (!vm_map_lookup_entry(map, oldmapmin, &oldentry))
539 panic("kmem_realloc");
540 object = oldentry->object.vm_object;
541
542 /*
543 * Increase the size of the object and
544 * fill in the new region.
545 */
546
547 vm_object_reference(object);
548 /* by grabbing the object lock before unlocking the map */
549 /* we guarantee that we will panic if more than one */
550 /* attempt is made to realloc a kmem_alloc'd area */
551 vm_object_lock(object);
552 vm_map_unlock(map);
553 if (object->size != oldmapsize)
554 panic("kmem_realloc");
555 object->size = newmapsize;
556 vm_object_unlock(object);
557
558 /* allocate the new pages while expanded portion of the */
559 /* object is still not mapped */
560 kmem_alloc_pages(object, vm_object_round_page(oldmapsize),
561 vm_object_round_page(newmapsize-oldmapsize));
562
563 /*
564 * Find space for the new region.
565 */
566
567 kr = vm_map_find_space(map, &newmapaddr, newmapsize,
568 (vm_map_offset_t) 0, 0, &newentry);
569 if (kr != KERN_SUCCESS) {
570 vm_object_lock(object);
571 for(offset = oldmapsize;
572 offset < newmapsize; offset += PAGE_SIZE) {
573 if ((mem = vm_page_lookup(object, offset)) != VM_PAGE_NULL) {
574 VM_PAGE_FREE(mem);
575 }
576 }
577 object->size = oldmapsize;
578 vm_object_unlock(object);
579 vm_object_deallocate(object);
580 return kr;
581 }
582 newentry->object.vm_object = object;
583 newentry->offset = 0;
584 assert (newentry->wired_count == 0);
585
586
587 /* add an extra reference in case we have someone doing an */
588 /* unexpected deallocate */
589 vm_object_reference(object);
590 vm_map_unlock(map);
591
592 kr = vm_map_wire(map, newmapaddr, newmapaddr + newmapsize, VM_PROT_DEFAULT, FALSE);
593 if (KERN_SUCCESS != kr) {
594 vm_map_remove(map, newmapaddr, newmapaddr + newmapsize, 0);
595 vm_object_lock(object);
596 for(offset = oldsize; offset < newmapsize; offset += PAGE_SIZE) {
597 if ((mem = vm_page_lookup(object, offset)) != VM_PAGE_NULL) {
598 VM_PAGE_FREE(mem);
599 }
600 }
601 object->size = oldmapsize;
602 vm_object_unlock(object);
603 vm_object_deallocate(object);
604 return (kr);
605 }
606 vm_object_deallocate(object);
607
608 *newaddrp = CAST_DOWN(vm_offset_t, newmapaddr);
609 return KERN_SUCCESS;
610 }
611
612 /*
613 * kmem_alloc_kobject:
614 *
615 * Allocate wired-down memory in the kernel's address map
616 * or a submap. The memory is not zero-filled.
617 *
618 * The memory is allocated in the kernel_object.
619 * It may not be copied with vm_map_copy, and
620 * it may not be reallocated with kmem_realloc.
621 */
622
623 kern_return_t
624 kmem_alloc_kobject(
625 vm_map_t map,
626 vm_offset_t *addrp,
627 vm_size_t size)
628 {
629 return kernel_memory_allocate(map, addrp, size, 0, KMA_KOBJECT);
630 }
631
632 /*
633 * kmem_alloc_aligned:
634 *
635 * Like kmem_alloc_kobject, except that the memory is aligned.
636 * The size should be a power-of-2.
637 */
638
639 kern_return_t
640 kmem_alloc_aligned(
641 vm_map_t map,
642 vm_offset_t *addrp,
643 vm_size_t size)
644 {
645 if ((size & (size - 1)) != 0)
646 panic("kmem_alloc_aligned: size not aligned");
647 return kernel_memory_allocate(map, addrp, size, size - 1, KMA_KOBJECT);
648 }
649
650 /*
651 * kmem_alloc_pageable:
652 *
653 * Allocate pageable memory in the kernel's address map.
654 */
655
656 kern_return_t
657 kmem_alloc_pageable(
658 vm_map_t map,
659 vm_offset_t *addrp,
660 vm_size_t size)
661 {
662 vm_map_offset_t map_addr;
663 vm_map_size_t map_size;
664 kern_return_t kr;
665
666 #ifndef normal
667 map_addr = (vm_map_min(map)) + 0x1000;
668 #else
669 map_addr = vm_map_min(map);
670 #endif
671 map_size = vm_map_round_page(size);
672
673 kr = vm_map_enter(map, &map_addr, map_size,
674 (vm_map_offset_t) 0, VM_FLAGS_ANYWHERE,
675 VM_OBJECT_NULL, (vm_object_offset_t) 0, FALSE,
676 VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT);
677
678 if (kr != KERN_SUCCESS)
679 return kr;
680
681 *addrp = CAST_DOWN(vm_offset_t, map_addr);
682 return KERN_SUCCESS;
683 }
684
685 /*
686 * kmem_free:
687 *
688 * Release a region of kernel virtual memory allocated
689 * with kmem_alloc, kmem_alloc_kobject, or kmem_alloc_pageable,
690 * and return the physical pages associated with that region.
691 */
692
693 void
694 kmem_free(
695 vm_map_t map,
696 vm_offset_t addr,
697 vm_size_t size)
698 {
699 kern_return_t kr;
700
701 assert(addr >= VM_MIN_KERNEL_AND_KEXT_ADDRESS);
702
703 TRACE_MACHLEAKS(KMEM_FREE_CODE, KMEM_FREE_CODE_2, size, addr);
704
705 if(size == 0) {
706 #if MACH_ASSERT
707 printf("kmem_free called with size==0 for map: %p with addr: 0x%llx\n",map,(uint64_t)addr);
708 #endif
709 return;
710 }
711
712 kr = vm_map_remove(map, vm_map_trunc_page(addr),
713 vm_map_round_page(addr + size),
714 VM_MAP_REMOVE_KUNWIRE);
715 if (kr != KERN_SUCCESS)
716 panic("kmem_free");
717 }
718
719 /*
720 * Allocate new pages in an object.
721 */
722
723 kern_return_t
724 kmem_alloc_pages(
725 register vm_object_t object,
726 register vm_object_offset_t offset,
727 register vm_object_size_t size)
728 {
729 vm_object_size_t alloc_size;
730
731 alloc_size = vm_object_round_page(size);
732 vm_object_lock(object);
733 while (alloc_size) {
734 register vm_page_t mem;
735
736
737 /*
738 * Allocate a page
739 */
740 while (VM_PAGE_NULL ==
741 (mem = vm_page_alloc(object, offset))) {
742 vm_object_unlock(object);
743 VM_PAGE_WAIT();
744 vm_object_lock(object);
745 }
746 mem->busy = FALSE;
747
748 alloc_size -= PAGE_SIZE;
749 offset += PAGE_SIZE;
750 }
751 vm_object_unlock(object);
752 return KERN_SUCCESS;
753 }
754
755 /*
756 * Remap wired pages in an object into a new region.
757 * The object is assumed to be mapped into the kernel map or
758 * a submap.
759 */
760 void
761 kmem_remap_pages(
762 register vm_object_t object,
763 register vm_object_offset_t offset,
764 register vm_offset_t start,
765 register vm_offset_t end,
766 vm_prot_t protection)
767 {
768
769 vm_map_offset_t map_start;
770 vm_map_offset_t map_end;
771
772 /*
773 * Mark the pmap region as not pageable.
774 */
775 map_start = vm_map_trunc_page(start);
776 map_end = vm_map_round_page(end);
777
778 pmap_pageable(kernel_pmap, map_start, map_end, FALSE);
779
780 while (map_start < map_end) {
781 register vm_page_t mem;
782
783 vm_object_lock(object);
784
785 /*
786 * Find a page
787 */
788 if ((mem = vm_page_lookup(object, offset)) == VM_PAGE_NULL)
789 panic("kmem_remap_pages");
790
791 /*
792 * Wire it down (again)
793 */
794 vm_page_lockspin_queues();
795 vm_page_wire(mem);
796 vm_page_unlock_queues();
797 vm_object_unlock(object);
798
799 /*
800 * ENCRYPTED SWAP:
801 * The page is supposed to be wired now, so it
802 * shouldn't be encrypted at this point. It can
803 * safely be entered in the page table.
804 */
805 ASSERT_PAGE_DECRYPTED(mem);
806
807 /*
808 * Enter it in the kernel pmap. The page isn't busy,
809 * but this shouldn't be a problem because it is wired.
810 */
811
812 mem->pmapped = TRUE;
813 mem->wpmapped = TRUE;
814
815 PMAP_ENTER(kernel_pmap, map_start, mem, protection,
816 ((unsigned int)(mem->object->wimg_bits))
817 & VM_WIMG_MASK,
818 TRUE);
819
820 map_start += PAGE_SIZE;
821 offset += PAGE_SIZE;
822 }
823 }
824
825 /*
826 * kmem_suballoc:
827 *
828 * Allocates a map to manage a subrange
829 * of the kernel virtual address space.
830 *
831 * Arguments are as follows:
832 *
833 * parent Map to take range from
834 * addr Address of start of range (IN/OUT)
835 * size Size of range to find
836 * pageable Can region be paged
837 * anywhere Can region be located anywhere in map
838 * new_map Pointer to new submap
839 */
840 kern_return_t
841 kmem_suballoc(
842 vm_map_t parent,
843 vm_offset_t *addr,
844 vm_size_t size,
845 boolean_t pageable,
846 int flags,
847 vm_map_t *new_map)
848 {
849 vm_map_t map;
850 vm_map_offset_t map_addr;
851 vm_map_size_t map_size;
852 kern_return_t kr;
853
854 map_size = vm_map_round_page(size);
855
856 /*
857 * Need reference on submap object because it is internal
858 * to the vm_system. vm_object_enter will never be called
859 * on it (usual source of reference for vm_map_enter).
860 */
861 vm_object_reference(vm_submap_object);
862
863 map_addr = (flags & VM_FLAGS_ANYWHERE) ?
864 vm_map_min(parent) : vm_map_trunc_page(*addr);
865
866 kr = vm_map_enter(parent, &map_addr, map_size,
867 (vm_map_offset_t) 0, flags,
868 vm_submap_object, (vm_object_offset_t) 0, FALSE,
869 VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT);
870 if (kr != KERN_SUCCESS) {
871 vm_object_deallocate(vm_submap_object);
872 return (kr);
873 }
874
875 pmap_reference(vm_map_pmap(parent));
876 map = vm_map_create(vm_map_pmap(parent), map_addr, map_addr + map_size, pageable);
877 if (map == VM_MAP_NULL)
878 panic("kmem_suballoc: vm_map_create failed"); /* "can't happen" */
879
880 kr = vm_map_submap(parent, map_addr, map_addr + map_size, map, map_addr, FALSE);
881 if (kr != KERN_SUCCESS) {
882 /*
883 * See comment preceding vm_map_submap().
884 */
885 vm_map_remove(parent, map_addr, map_addr + map_size, VM_MAP_NO_FLAGS);
886 vm_map_deallocate(map); /* also removes ref to pmap */
887 vm_object_deallocate(vm_submap_object);
888 return (kr);
889 }
890 *addr = CAST_DOWN(vm_offset_t, map_addr);
891 *new_map = map;
892 return (KERN_SUCCESS);
893 }
894
895
896 /*
897 * kmem_init:
898 *
899 * Initialize the kernel's virtual memory map, taking
900 * into account all memory allocated up to this time.
901 */
902 void
903 kmem_init(
904 vm_offset_t start,
905 vm_offset_t end)
906 {
907 vm_map_offset_t map_start;
908 vm_map_offset_t map_end;
909
910 map_start = vm_map_trunc_page(start);
911 map_end = vm_map_round_page(end);
912
913 kernel_map = vm_map_create(pmap_kernel(),VM_MIN_KERNEL_ADDRESS,
914 map_end, FALSE);
915 /*
916 * Reserve virtual memory allocated up to this time.
917 */
918 if (start != VM_MIN_KERNEL_ADDRESS) {
919 vm_map_offset_t map_addr;
920
921 map_addr = VM_MIN_KERNEL_ADDRESS;
922 (void) vm_map_enter(kernel_map,
923 &map_addr,
924 (vm_map_size_t)(map_start - VM_MIN_KERNEL_ADDRESS),
925 (vm_map_offset_t) 0,
926 VM_FLAGS_ANYWHERE | VM_FLAGS_NO_PMAP_CHECK,
927 VM_OBJECT_NULL,
928 (vm_object_offset_t) 0, FALSE,
929 VM_PROT_NONE, VM_PROT_NONE,
930 VM_INHERIT_DEFAULT);
931 }
932 /*
933 * Set the default global user wire limit which limits the amount of
934 * memory that can be locked via mlock(). We set this to the total
935 * amount of memory that are potentially usable by a user app (max_mem)
936 * minus a certain amount. This can be overridden via a sysctl.
937 */
938 vm_global_no_user_wire_amount = MIN(max_mem*20/100,
939 VM_NOT_USER_WIREABLE);
940 vm_global_user_wire_limit = max_mem - vm_global_no_user_wire_amount;
941
942 /* the default per user limit is the same as the global limit */
943 vm_user_wire_limit = vm_global_user_wire_limit;
944 }
945
946
947 /*
948 * Routine: copyinmap
949 * Purpose:
950 * Like copyin, except that fromaddr is an address
951 * in the specified VM map. This implementation
952 * is incomplete; it handles the current user map
953 * and the kernel map/submaps.
954 */
955 kern_return_t
956 copyinmap(
957 vm_map_t map,
958 vm_map_offset_t fromaddr,
959 void *todata,
960 vm_size_t length)
961 {
962 kern_return_t kr = KERN_SUCCESS;
963 vm_map_t oldmap;
964
965 if (vm_map_pmap(map) == pmap_kernel())
966 {
967 /* assume a correct copy */
968 memcpy(todata, CAST_DOWN(void *, fromaddr), length);
969 }
970 else if (current_map() == map)
971 {
972 if (copyin(fromaddr, todata, length) != 0)
973 kr = KERN_INVALID_ADDRESS;
974 }
975 else
976 {
977 vm_map_reference(map);
978 oldmap = vm_map_switch(map);
979 if (copyin(fromaddr, todata, length) != 0)
980 kr = KERN_INVALID_ADDRESS;
981 vm_map_switch(oldmap);
982 vm_map_deallocate(map);
983 }
984 return kr;
985 }
986
987 /*
988 * Routine: copyoutmap
989 * Purpose:
990 * Like copyout, except that toaddr is an address
991 * in the specified VM map. This implementation
992 * is incomplete; it handles the current user map
993 * and the kernel map/submaps.
994 */
995 kern_return_t
996 copyoutmap(
997 vm_map_t map,
998 void *fromdata,
999 vm_map_address_t toaddr,
1000 vm_size_t length)
1001 {
1002 if (vm_map_pmap(map) == pmap_kernel()) {
1003 /* assume a correct copy */
1004 memcpy(CAST_DOWN(void *, toaddr), fromdata, length);
1005 return KERN_SUCCESS;
1006 }
1007
1008 if (current_map() != map)
1009 return KERN_NOT_SUPPORTED;
1010
1011 if (copyout(fromdata, toaddr, length) != 0)
1012 return KERN_INVALID_ADDRESS;
1013
1014 return KERN_SUCCESS;
1015 }
1016
1017
1018 kern_return_t
1019 vm_conflict_check(
1020 vm_map_t map,
1021 vm_map_offset_t off,
1022 vm_map_size_t len,
1023 memory_object_t pager,
1024 vm_object_offset_t file_off)
1025 {
1026 vm_map_entry_t entry;
1027 vm_object_t obj;
1028 vm_object_offset_t obj_off;
1029 vm_map_t base_map;
1030 vm_map_offset_t base_offset;
1031 vm_map_offset_t original_offset;
1032 kern_return_t kr;
1033 vm_map_size_t local_len;
1034
1035 base_map = map;
1036 base_offset = off;
1037 original_offset = off;
1038 kr = KERN_SUCCESS;
1039 vm_map_lock(map);
1040 while(vm_map_lookup_entry(map, off, &entry)) {
1041 local_len = len;
1042
1043 if (entry->object.vm_object == VM_OBJECT_NULL) {
1044 vm_map_unlock(map);
1045 return KERN_SUCCESS;
1046 }
1047 if (entry->is_sub_map) {
1048 vm_map_t old_map;
1049
1050 old_map = map;
1051 vm_map_lock(entry->object.sub_map);
1052 map = entry->object.sub_map;
1053 off = entry->offset + (off - entry->vme_start);
1054 vm_map_unlock(old_map);
1055 continue;
1056 }
1057 obj = entry->object.vm_object;
1058 obj_off = (off - entry->vme_start) + entry->offset;
1059 while(obj->shadow) {
1060 obj_off += obj->shadow_offset;
1061 obj = obj->shadow;
1062 }
1063 if((obj->pager_created) && (obj->pager == pager)) {
1064 if(((obj->paging_offset) + obj_off) == file_off) {
1065 if(off != base_offset) {
1066 vm_map_unlock(map);
1067 return KERN_FAILURE;
1068 }
1069 kr = KERN_ALREADY_WAITING;
1070 } else {
1071 vm_object_offset_t obj_off_aligned;
1072 vm_object_offset_t file_off_aligned;
1073
1074 obj_off_aligned = obj_off & ~PAGE_MASK;
1075 file_off_aligned = file_off & ~PAGE_MASK;
1076
1077 if (file_off_aligned == (obj->paging_offset + obj_off_aligned)) {
1078 /*
1079 * the target map and the file offset start in the same page
1080 * but are not identical...
1081 */
1082 vm_map_unlock(map);
1083 return KERN_FAILURE;
1084 }
1085 if ((file_off < (obj->paging_offset + obj_off_aligned)) &&
1086 ((file_off + len) > (obj->paging_offset + obj_off_aligned))) {
1087 /*
1088 * some portion of the tail of the I/O will fall
1089 * within the encompass of the target map
1090 */
1091 vm_map_unlock(map);
1092 return KERN_FAILURE;
1093 }
1094 if ((file_off_aligned > (obj->paging_offset + obj_off)) &&
1095 (file_off_aligned < (obj->paging_offset + obj_off) + len)) {
1096 /*
1097 * the beginning page of the file offset falls within
1098 * the target map's encompass
1099 */
1100 vm_map_unlock(map);
1101 return KERN_FAILURE;
1102 }
1103 }
1104 } else if(kr != KERN_SUCCESS) {
1105 vm_map_unlock(map);
1106 return KERN_FAILURE;
1107 }
1108
1109 if(len <= ((entry->vme_end - entry->vme_start) -
1110 (off - entry->vme_start))) {
1111 vm_map_unlock(map);
1112 return kr;
1113 } else {
1114 len -= (entry->vme_end - entry->vme_start) -
1115 (off - entry->vme_start);
1116 }
1117 base_offset = base_offset + (local_len - len);
1118 file_off = file_off + (local_len - len);
1119 off = base_offset;
1120 if(map != base_map) {
1121 vm_map_unlock(map);
1122 vm_map_lock(base_map);
1123 map = base_map;
1124 }
1125 }
1126
1127 vm_map_unlock(map);
1128 return kr;
1129 }