X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/fe8ab488e9161c46dd9885d58fc52996dc0249ff..9d749ea394c01276fa19e397e70f46858e849c76:/osfmk/vm/vm_kern.c diff --git a/osfmk/vm/vm_kern.c b/osfmk/vm/vm_kern.c index 5dcffd740..a14a10db9 100644 --- a/osfmk/vm/vm_kern.c +++ b/osfmk/vm/vm_kern.c @@ -71,6 +71,7 @@ #include #include #include +#include #include #include #include @@ -78,8 +79,11 @@ #include #include +#include #include +#include + /* * Variables exported by this module. */ @@ -93,16 +97,9 @@ extern boolean_t vm_kernel_ready; * Forward declarations for internal functions. */ extern kern_return_t kmem_alloc_pages( - register vm_object_t object, - register vm_object_offset_t offset, - register vm_object_size_t size); - -extern void kmem_remap_pages( - register vm_object_t object, - register vm_object_offset_t offset, - register vm_offset_t start, - register vm_offset_t end, - vm_prot_t protection); + vm_object_t object, + vm_object_offset_t offset, + vm_object_size_t size); kern_return_t kmem_alloc_contig( @@ -112,7 +109,8 @@ kmem_alloc_contig( vm_offset_t mask, ppnum_t max_pnum, ppnum_t pnum_mask, - int flags) + int flags, + vm_tag_t tag) { vm_object_t object; vm_object_offset_t offset; @@ -123,6 +121,8 @@ kmem_alloc_contig( vm_page_t m, pages; kern_return_t kr; + assert(VM_KERN_MEMORY_NONE != tag); + if (map == VM_MAP_NULL || (flags & ~(KMA_KOBJECT | KMA_LOMEM | KMA_NOPAGEWAIT))) return KERN_INVALID_ARGUMENT; @@ -148,15 +148,20 @@ kmem_alloc_contig( object = vm_object_allocate(map_size); } - kr = vm_map_find_space(map, &map_addr, map_size, map_mask, 0, &entry); + kr = vm_map_find_space(map, &map_addr, map_size, map_mask, 0, + VM_MAP_KERNEL_FLAGS_NONE, tag, &entry); if (KERN_SUCCESS != kr) { vm_object_deallocate(object); return kr; } - entry->object.vm_object = object; - entry->offset = offset = (object == kernel_object) ? - map_addr : 0; + if (object == kernel_object) { + offset = map_addr; + } else { + offset = 0; + } + VME_OBJECT_SET(entry, object); + VME_OFFSET_SET(entry, offset); /* Take an extra object ref in case the map entry gets deleted */ vm_object_reference(object); @@ -186,13 +191,14 @@ kmem_alloc_contig( } vm_object_unlock(object); - kr = vm_map_wire(map, + kr = vm_map_wire_kernel(map, vm_map_trunc_page(map_addr, VM_MAP_PAGE_MASK(map)), vm_map_round_page(map_addr + map_size, VM_MAP_PAGE_MASK(map)), - VM_PROT_DEFAULT, + VM_PROT_DEFAULT, tag, FALSE); + if (kr != KERN_SUCCESS) { if (object == kernel_object) { vm_object_lock(object); @@ -210,11 +216,13 @@ kmem_alloc_contig( } vm_object_deallocate(object); - if (object == kernel_object) + if (object == kernel_object) { vm_map_simplify(map, map_addr); - + vm_tag_update_size(tag, map_size); + } *addrp = (vm_offset_t) map_addr; assert((vm_map_offset_t) *addrp == map_addr); + return KERN_SUCCESS; } @@ -237,11 +245,12 @@ kmem_alloc_contig( kern_return_t kernel_memory_allocate( - register vm_map_t map, - register vm_offset_t *addrp, - register vm_size_t size, - register vm_offset_t mask, - int flags) + vm_map_t map, + vm_offset_t *addrp, + vm_size_t size, + vm_offset_t mask, + int flags, + vm_tag_t tag) { vm_object_t object; vm_object_offset_t offset; @@ -258,6 +267,7 @@ kernel_memory_allocate( int wired_page_count = 0; int i; int vm_alloc_flags; + vm_map_kernel_flags_t vmk_flags; vm_prot_t kma_prot; if (! vm_kernel_ready) { @@ -267,7 +277,9 @@ kernel_memory_allocate( map_size = vm_map_round_page(size, VM_MAP_PAGE_MASK(map)); map_mask = (vm_map_offset_t) mask; - vm_alloc_flags = 0; + + vm_alloc_flags = 0; //VM_MAKE_TAG(tag); + vmk_flags = VM_MAP_KERNEL_FLAGS_NONE; /* Check for zero allocation size (either directly or via overflow) */ if (map_size == 0) { @@ -279,9 +291,10 @@ kernel_memory_allocate( * limit the size of a single extent of wired memory * to try and limit the damage to the system if * too many pages get wired down - * limit raised to 2GB with 128GB max physical limit + * limit raised to 2GB with 128GB max physical limit, + * but scaled by installed memory above this */ - if (map_size > (1ULL << 31)) { + if ( !(flags & KMA_VAONLY) && map_size > MAX(1ULL<<31, sane_size/64)) { return KERN_RESOURCE_SHORTAGE; } @@ -304,7 +317,7 @@ kernel_memory_allocate( fill_size = map_size; if (flags & KMA_GUARD_FIRST) { - vm_alloc_flags |= VM_FLAGS_GUARD_BEFORE; + vmk_flags.vmkf_guard_before = TRUE; fill_start += PAGE_SIZE_64; fill_size -= PAGE_SIZE_64; if (map_size < fill_start + fill_size) { @@ -315,7 +328,7 @@ kernel_memory_allocate( guard_page_count++; } if (flags & KMA_GUARD_LAST) { - vm_alloc_flags |= VM_FLAGS_GUARD_AFTER; + vmk_flags.vmkf_guard_after = TRUE; fill_size -= PAGE_SIZE_64; if (map_size <= fill_start + fill_size) { /* no space for a guard page */ @@ -339,7 +352,7 @@ kernel_memory_allocate( } vm_page_more_fictitious(); } - mem->pageq.next = (queue_entry_t)guard_page_list; + mem->snext = guard_page_list; guard_page_list = mem; } @@ -372,7 +385,8 @@ kernel_memory_allocate( } VM_PAGE_WAIT(); } - mem->pageq.next = (queue_entry_t)wired_page_list; + if (KMA_ZERO & flags) vm_page_zero_fill(mem); + mem->snext = wired_page_list; wired_page_list = mem; } } @@ -391,17 +405,24 @@ kernel_memory_allocate( object = vm_object_allocate(map_size); } + if (flags & KMA_ATOMIC) + vmk_flags.vmkf_atomic_entry = TRUE; + kr = vm_map_find_space(map, &map_addr, fill_size, map_mask, - vm_alloc_flags, &entry); + vm_alloc_flags, vmk_flags, tag, &entry); if (KERN_SUCCESS != kr) { vm_object_deallocate(object); goto out; } - entry->object.vm_object = object; - entry->offset = offset = (object == kernel_object || object == compressor_object) ? - map_addr : 0; + if (object == kernel_object || object == compressor_object) { + offset = map_addr; + } else { + offset = 0; + } + VME_OBJECT_SET(entry, object); + VME_OFFSET_SET(entry, offset); if (object != compressor_object) entry->wired_count++; @@ -422,8 +443,8 @@ kernel_memory_allocate( panic("kernel_memory_allocate: guard_page_list == NULL"); mem = guard_page_list; - guard_page_list = (vm_page_t)mem->pageq.next; - mem->pageq.next = NULL; + guard_page_list = mem->snext; + mem->snext = NULL; vm_page_insert(mem, object, offset + pg_offset); @@ -433,6 +454,13 @@ kernel_memory_allocate( kma_prot = VM_PROT_READ | VM_PROT_WRITE; +#if KASAN + if (!(flags & KMA_VAONLY)) { + /* for VAONLY mappings we notify in populate only */ + kasan_notify_address(map_addr, size); + } +#endif + if (flags & KMA_VAONLY) { pg_offset = fill_start + fill_size; } else { @@ -441,11 +469,20 @@ kernel_memory_allocate( panic("kernel_memory_allocate: wired_page_list == NULL"); mem = wired_page_list; - wired_page_list = (vm_page_t)mem->pageq.next; - mem->pageq.next = NULL; + wired_page_list = mem->snext; + mem->snext = NULL; + + assert(mem->wire_count == 0); + assert(mem->vm_page_q_state == VM_PAGE_NOT_ON_Q); + + mem->vm_page_q_state = VM_PAGE_IS_WIRED; mem->wire_count++; + if (__improbable(mem->wire_count == 0)) { + panic("kernel_memory_allocate(%p): wire_count overflow", + mem); + } - vm_page_insert(mem, object, offset + pg_offset); + vm_page_insert_wired(mem, object, offset + pg_offset, tag); mem->busy = FALSE; mem->pmapped = TRUE; @@ -459,24 +496,29 @@ kernel_memory_allocate( vm_object_unlock(object); PMAP_ENTER(kernel_pmap, map_addr + pg_offset, mem, - kma_prot, VM_PROT_NONE, ((flags & KMA_KSTACK) ? VM_MEM_STACK : 0), TRUE); + kma_prot, VM_PROT_NONE, ((flags & KMA_KSTACK) ? VM_MEM_STACK : 0), TRUE, + pe_result); vm_object_lock(object); } + + assert(pe_result == KERN_SUCCESS); + if (flags & KMA_NOENCRYPT) { bzero(CAST_DOWN(void *, (map_addr + pg_offset)), PAGE_SIZE); - pmap_set_noencrypt(mem->phys_page); + pmap_set_noencrypt(VM_PAGE_GET_PHYS_PAGE(mem)); } } + if (kernel_object == object) vm_tag_update_size(tag, fill_size); } if ((fill_start + fill_size) < map_size) { if (guard_page_list == NULL) panic("kernel_memory_allocate: guard_page_list == NULL"); mem = guard_page_list; - guard_page_list = (vm_page_t)mem->pageq.next; - mem->pageq.next = NULL; + guard_page_list = mem->snext; + mem->snext = NULL; vm_page_insert(mem, object, offset + pg_offset); @@ -522,7 +564,8 @@ kernel_memory_populate( vm_map_t map, vm_offset_t addr, vm_size_t size, - int flags) + int flags, + vm_tag_t tag) { vm_object_t object; vm_object_offset_t offset, pg_offset; @@ -538,7 +581,9 @@ kernel_memory_populate( if (flags & KMA_COMPRESSOR) { - for (i = 0; i < page_count; i++) { + pg_offset = page_count * PAGE_SIZE_64; + + do { for (;;) { mem = vm_page_grab(); @@ -547,9 +592,20 @@ kernel_memory_populate( VM_PAGE_WAIT(); } - mem->pageq.next = (queue_entry_t) page_list; + if (KMA_ZERO & flags) vm_page_zero_fill(mem); + mem->snext = page_list; page_list = mem; - } + + pg_offset -= PAGE_SIZE_64; + + kr = pmap_enter_options(kernel_pmap, + addr + pg_offset, VM_PAGE_GET_PHYS_PAGE(mem), + VM_PROT_READ | VM_PROT_WRITE, VM_PROT_NONE, 0, TRUE, + PMAP_OPTIONS_INTERNAL, NULL); + assert(kr == KERN_SUCCESS); + + } while (pg_offset); + offset = addr; object = compressor_object; @@ -560,32 +616,26 @@ kernel_memory_populate( pg_offset += PAGE_SIZE_64) { mem = page_list; - page_list = (vm_page_t) mem->pageq.next; - mem->pageq.next = NULL; + page_list = mem->snext; + mem->snext = NULL; vm_page_insert(mem, object, offset + pg_offset); assert(mem->busy); - PMAP_ENTER_OPTIONS(kernel_pmap, addr + pg_offset, mem, - VM_PROT_READ | VM_PROT_WRITE, VM_PROT_NONE, - 0, TRUE, PMAP_OPTIONS_NOWAIT, pe_result); - - if (pe_result == KERN_RESOURCE_SHORTAGE) { - - vm_object_unlock(object); - - PMAP_ENTER(kernel_pmap, addr + pg_offset, mem, - VM_PROT_READ | VM_PROT_WRITE, VM_PROT_NONE, 0, TRUE); - - vm_object_lock(object); - } mem->busy = FALSE; mem->pmapped = TRUE; mem->wpmapped = TRUE; - mem->compressor = TRUE; + mem->vm_page_q_state = VM_PAGE_USED_BY_COMPRESSOR; } vm_object_unlock(object); +#if KASAN + if (map == compressor_map) { + kasan_notify_address_nopoison(addr, size); + } else { + kasan_notify_address(addr, size); + } +#endif return KERN_SUCCESS; } @@ -610,7 +660,8 @@ kernel_memory_populate( } VM_PAGE_WAIT(); } - mem->pageq.next = (queue_entry_t) page_list; + if (KMA_ZERO & flags) vm_page_zero_fill(mem); + mem->snext = page_list; page_list = mem; } if (flags & KMA_KOBJECT) { @@ -640,12 +691,18 @@ kernel_memory_populate( panic("kernel_memory_populate: page_list == NULL"); mem = page_list; - page_list = (vm_page_t) mem->pageq.next; - mem->pageq.next = NULL; + page_list = mem->snext; + mem->snext = NULL; + assert(mem->vm_page_q_state == VM_PAGE_NOT_ON_Q); + mem->vm_page_q_state = VM_PAGE_IS_WIRED; mem->wire_count++; + if (__improbable(mem->wire_count == 0)) { + panic("kernel_memory_populate(%p): wire_count overflow", + mem); + } - vm_page_insert(mem, object, offset + pg_offset); + vm_page_insert_wired(mem, object, offset + pg_offset, tag); mem->busy = FALSE; mem->pmapped = TRUE; @@ -662,21 +719,34 @@ kernel_memory_populate( PMAP_ENTER(kernel_pmap, addr + pg_offset, mem, VM_PROT_READ | VM_PROT_WRITE, VM_PROT_NONE, - ((flags & KMA_KSTACK) ? VM_MEM_STACK : 0), TRUE); + ((flags & KMA_KSTACK) ? VM_MEM_STACK : 0), TRUE, + pe_result); vm_object_lock(object); } + + assert(pe_result == KERN_SUCCESS); + if (flags & KMA_NOENCRYPT) { bzero(CAST_DOWN(void *, (addr + pg_offset)), PAGE_SIZE); - pmap_set_noencrypt(mem->phys_page); + pmap_set_noencrypt(VM_PAGE_GET_PHYS_PAGE(mem)); } } vm_page_lock_queues(); vm_page_wire_count += page_count; vm_page_unlock_queues(); + if (kernel_object == object) vm_tag_update_size(tag, size); + vm_object_unlock(object); +#if KASAN + if (map == compressor_map) { + kasan_notify_address_nopoison(addr, size); + } else { + kasan_notify_address(addr, size); + } +#endif return KERN_SUCCESS; out: @@ -709,7 +779,6 @@ kernel_memory_depopulate( } else if (flags & KMA_KOBJECT) { offset = addr; object = kernel_object; - vm_object_lock(object); } else { offset = 0; @@ -734,8 +803,9 @@ kernel_memory_depopulate( mem = vm_page_lookup(object, offset + pg_offset); assert(mem); - - pmap_disconnect(mem->phys_page); + + if (mem->vm_page_q_state != VM_PAGE_USED_BY_COMPRESSOR) + pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(mem)); mem->busy = TRUE; @@ -743,9 +813,12 @@ kernel_memory_depopulate( vm_page_remove(mem, TRUE); assert(mem->busy); - assert(mem->pageq.next == NULL && - mem->pageq.prev == NULL); - mem->pageq.next = (queue_entry_t)local_freeq; + assert(mem->pageq.next == 0 && mem->pageq.prev == 0); + assert((mem->vm_page_q_state == VM_PAGE_USED_BY_COMPRESSOR) || + (mem->vm_page_q_state == VM_PAGE_NOT_ON_Q)); + + mem->vm_page_q_state = VM_PAGE_NOT_ON_Q; + mem->snext = local_freeq; local_freeq = mem; } vm_object_unlock(object); @@ -762,12 +835,34 @@ kernel_memory_depopulate( */ kern_return_t -kmem_alloc( +kmem_alloc_external( vm_map_t map, vm_offset_t *addrp, vm_size_t size) { - kern_return_t kr = kernel_memory_allocate(map, addrp, size, 0, 0); + return (kmem_alloc(map, addrp, size, vm_tag_bt())); +} + + +kern_return_t +kmem_alloc( + vm_map_t map, + vm_offset_t *addrp, + vm_size_t size, + vm_tag_t tag) +{ + return kmem_alloc_flags(map, addrp, size, tag, 0); +} + +kern_return_t +kmem_alloc_flags( + vm_map_t map, + vm_offset_t *addrp, + vm_size_t size, + vm_tag_t tag, + int flags) +{ + kern_return_t kr = kernel_memory_allocate(map, addrp, size, 0, flags, tag); TRACE_MACHLEAKS(KMEM_ALLOC_CODE, KMEM_ALLOC_CODE_2, size, *addrp); return kr; } @@ -788,7 +883,8 @@ kmem_realloc( vm_offset_t oldaddr, vm_size_t oldsize, vm_offset_t *newaddrp, - vm_size_t newsize) + vm_size_t newsize, + vm_tag_t tag) { vm_object_t object; vm_object_offset_t offset; @@ -809,7 +905,11 @@ kmem_realloc( oldmapsize = oldmapmax - oldmapmin; newmapsize = vm_map_round_page(newsize, VM_MAP_PAGE_MASK(map)); - + if (newmapsize < newsize) { + /* overflow */ + *newaddrp = 0; + return KERN_INVALID_ARGUMENT; + } /* * Find the VM object backing the old region. @@ -819,7 +919,7 @@ kmem_realloc( if (!vm_map_lookup_entry(map, oldmapmin, &oldentry)) panic("kmem_realloc"); - object = oldentry->object.vm_object; + object = VME_OBJECT(oldentry); /* * Increase the size of the object and @@ -847,7 +947,10 @@ kmem_realloc( */ kr = vm_map_find_space(map, &newmapaddr, newmapsize, - (vm_map_offset_t) 0, 0, &newentry); + (vm_map_offset_t) 0, 0, + VM_MAP_KERNEL_FLAGS_NONE, + tag, + &newentry); if (kr != KERN_SUCCESS) { vm_object_lock(object); for(offset = oldmapsize; @@ -861,9 +964,9 @@ kmem_realloc( vm_object_deallocate(object); return kr; } - newentry->object.vm_object = object; - newentry->offset = 0; - assert (newentry->wired_count == 0); + VME_OBJECT_SET(newentry, object); + VME_OFFSET_SET(newentry, 0); + assert(newentry->wired_count == 0); /* add an extra reference in case we have someone doing an */ @@ -871,7 +974,8 @@ kmem_realloc( vm_object_reference(object); vm_map_unlock(map); - kr = vm_map_wire(map, newmapaddr, newmapaddr + newmapsize, VM_PROT_DEFAULT, FALSE); + kr = vm_map_wire_kernel(map, newmapaddr, newmapaddr + newmapsize, + VM_PROT_DEFAULT, tag, FALSE); if (KERN_SUCCESS != kr) { vm_map_remove(map, newmapaddr, newmapaddr + newmapsize, 0); vm_object_lock(object); @@ -887,6 +991,8 @@ kmem_realloc( } vm_object_deallocate(object); + if (kernel_object == object) vm_tag_update_size(tag, newmapsize); + *newaddrp = CAST_DOWN(vm_offset_t, newmapaddr); return KERN_SUCCESS; } @@ -903,12 +1009,22 @@ kmem_realloc( */ kern_return_t -kmem_alloc_kobject( +kmem_alloc_kobject_external( vm_map_t map, vm_offset_t *addrp, vm_size_t size) { - return kernel_memory_allocate(map, addrp, size, 0, KMA_KOBJECT); + return (kmem_alloc_kobject(map, addrp, size, vm_tag_bt())); +} + +kern_return_t +kmem_alloc_kobject( + vm_map_t map, + vm_offset_t *addrp, + vm_size_t size, + vm_tag_t tag) +{ + return kernel_memory_allocate(map, addrp, size, 0, KMA_KOBJECT, tag); } /* @@ -922,11 +1038,12 @@ kern_return_t kmem_alloc_aligned( vm_map_t map, vm_offset_t *addrp, - vm_size_t size) + vm_size_t size, + vm_tag_t tag) { if ((size & (size - 1)) != 0) panic("kmem_alloc_aligned: size not aligned"); - return kernel_memory_allocate(map, addrp, size, size - 1, KMA_KOBJECT); + return kernel_memory_allocate(map, addrp, size, size - 1, KMA_KOBJECT, tag); } /* @@ -936,10 +1053,20 @@ kmem_alloc_aligned( */ kern_return_t -kmem_alloc_pageable( +kmem_alloc_pageable_external( vm_map_t map, vm_offset_t *addrp, vm_size_t size) +{ + return (kmem_alloc_pageable(map, addrp, size, vm_tag_bt())); +} + +kern_return_t +kmem_alloc_pageable( + vm_map_t map, + vm_offset_t *addrp, + vm_size_t size, + vm_tag_t tag) { vm_map_offset_t map_addr; vm_map_size_t map_size; @@ -952,15 +1079,26 @@ kmem_alloc_pageable( #endif map_size = vm_map_round_page(size, VM_MAP_PAGE_MASK(map)); + if (map_size < size) { + /* overflow */ + *addrp = 0; + return KERN_INVALID_ARGUMENT; + } kr = vm_map_enter(map, &map_addr, map_size, - (vm_map_offset_t) 0, VM_FLAGS_ANYWHERE, + (vm_map_offset_t) 0, + VM_FLAGS_ANYWHERE, + VM_MAP_KERNEL_FLAGS_NONE, + tag, VM_OBJECT_NULL, (vm_object_offset_t) 0, FALSE, VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT); if (kr != KERN_SUCCESS) return kr; +#if KASAN + kasan_notify_address(map_addr, map_size); +#endif *addrp = CAST_DOWN(vm_offset_t, map_addr); return KERN_SUCCESS; } @@ -1008,16 +1146,16 @@ kmem_free( kern_return_t kmem_alloc_pages( - register vm_object_t object, - register vm_object_offset_t offset, - register vm_object_size_t size) + vm_object_t object, + vm_object_offset_t offset, + vm_object_size_t size) { vm_object_size_t alloc_size; alloc_size = vm_object_round_page(size); vm_object_lock(object); while (alloc_size) { - register vm_page_t mem; + vm_page_t mem; /* @@ -1038,75 +1176,6 @@ kmem_alloc_pages( return KERN_SUCCESS; } -/* - * Remap wired pages in an object into a new region. - * The object is assumed to be mapped into the kernel map or - * a submap. - */ -void -kmem_remap_pages( - register vm_object_t object, - register vm_object_offset_t offset, - register vm_offset_t start, - register vm_offset_t end, - vm_prot_t protection) -{ - - vm_map_offset_t map_start; - vm_map_offset_t map_end; - - /* - * Mark the pmap region as not pageable. - */ - map_start = vm_map_trunc_page(start, - VM_MAP_PAGE_MASK(kernel_map)); - map_end = vm_map_round_page(end, - VM_MAP_PAGE_MASK(kernel_map)); - - pmap_pageable(kernel_pmap, map_start, map_end, FALSE); - - while (map_start < map_end) { - register vm_page_t mem; - - vm_object_lock(object); - - /* - * Find a page - */ - if ((mem = vm_page_lookup(object, offset)) == VM_PAGE_NULL) - panic("kmem_remap_pages"); - - /* - * Wire it down (again) - */ - vm_page_lockspin_queues(); - vm_page_wire(mem); - vm_page_unlock_queues(); - vm_object_unlock(object); - - /* - * ENCRYPTED SWAP: - * The page is supposed to be wired now, so it - * shouldn't be encrypted at this point. It can - * safely be entered in the page table. - */ - ASSERT_PAGE_DECRYPTED(mem); - - /* - * Enter it in the kernel pmap. The page isn't busy, - * but this shouldn't be a problem because it is wired. - */ - - mem->pmapped = TRUE; - mem->wpmapped = TRUE; - - PMAP_ENTER(kernel_pmap, map_start, mem, protection, VM_PROT_NONE, 0, TRUE); - - map_start += PAGE_SIZE; - offset += PAGE_SIZE; - } -} - /* * kmem_suballoc: * @@ -1129,6 +1198,8 @@ kmem_suballoc( vm_size_t size, boolean_t pageable, int flags, + vm_map_kernel_flags_t vmk_flags, + vm_tag_t tag, vm_map_t *new_map) { vm_map_t map; @@ -1138,6 +1209,11 @@ kmem_suballoc( map_size = vm_map_round_page(size, VM_MAP_PAGE_MASK(parent)); + if (map_size < size) { + /* overflow */ + *addr = 0; + return KERN_INVALID_ARGUMENT; + } /* * Need reference on submap object because it is internal @@ -1152,7 +1228,7 @@ kmem_suballoc( VM_MAP_PAGE_MASK(parent))); kr = vm_map_enter(parent, &map_addr, map_size, - (vm_map_offset_t) 0, flags, + (vm_map_offset_t) 0, flags, vmk_flags, tag, vm_submap_object, (vm_object_offset_t) 0, FALSE, VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT); if (kr != KERN_SUCCESS) { @@ -1195,12 +1271,54 @@ kmem_init( { vm_map_offset_t map_start; vm_map_offset_t map_end; + vm_map_kernel_flags_t vmk_flags; + + vmk_flags = VM_MAP_KERNEL_FLAGS_NONE; + vmk_flags.vmkf_permanent = TRUE; + vmk_flags.vmkf_no_pmap_check = TRUE; map_start = vm_map_trunc_page(start, VM_MAP_PAGE_MASK(kernel_map)); map_end = vm_map_round_page(end, VM_MAP_PAGE_MASK(kernel_map)); +#if defined(__arm__) || defined(__arm64__) + kernel_map = vm_map_create(pmap_kernel(),VM_MIN_KERNEL_AND_KEXT_ADDRESS, + VM_MAX_KERNEL_ADDRESS, FALSE); + /* + * Reserve virtual memory allocated up to this time. + */ + { + unsigned int region_select = 0; + vm_map_offset_t region_start; + vm_map_size_t region_size; + vm_map_offset_t map_addr; + kern_return_t kr; + + while (pmap_virtual_region(region_select, ®ion_start, ®ion_size)) { + + map_addr = region_start; + kr = vm_map_enter(kernel_map, &map_addr, + vm_map_round_page(region_size, + VM_MAP_PAGE_MASK(kernel_map)), + (vm_map_offset_t) 0, + VM_FLAGS_FIXED, + vmk_flags, + VM_KERN_MEMORY_NONE, + VM_OBJECT_NULL, + (vm_object_offset_t) 0, FALSE, VM_PROT_NONE, VM_PROT_NONE, + VM_INHERIT_DEFAULT); + + if (kr != KERN_SUCCESS) { + panic("kmem_init(0x%llx,0x%llx): vm_map_enter(0x%llx,0x%llx) error 0x%x\n", + (uint64_t) start, (uint64_t) end, (uint64_t) region_start, + (uint64_t) region_size, kr); + } + + region_select++; + } + } +#else kernel_map = vm_map_create(pmap_kernel(),VM_MIN_KERNEL_AND_KEXT_ADDRESS, map_end, FALSE); /* @@ -1210,16 +1328,21 @@ kmem_init( vm_map_offset_t map_addr; kern_return_t kr; + vmk_flags = VM_MAP_KERNEL_FLAGS_NONE; + vmk_flags.vmkf_no_pmap_check = TRUE; + map_addr = VM_MIN_KERNEL_AND_KEXT_ADDRESS; kr = vm_map_enter(kernel_map, - &map_addr, - (vm_map_size_t)(map_start - VM_MIN_KERNEL_AND_KEXT_ADDRESS), - (vm_map_offset_t) 0, - VM_FLAGS_FIXED | VM_FLAGS_NO_PMAP_CHECK, - VM_OBJECT_NULL, - (vm_object_offset_t) 0, FALSE, - VM_PROT_NONE, VM_PROT_NONE, - VM_INHERIT_DEFAULT); + &map_addr, + (vm_map_size_t)(map_start - VM_MIN_KERNEL_AND_KEXT_ADDRESS), + (vm_map_offset_t) 0, + VM_FLAGS_FIXED, + vmk_flags, + VM_KERN_MEMORY_NONE, + VM_OBJECT_NULL, + (vm_object_offset_t) 0, FALSE, + VM_PROT_NONE, VM_PROT_NONE, + VM_INHERIT_DEFAULT); if (kr != KERN_SUCCESS) { panic("kmem_init(0x%llx,0x%llx): vm_map_enter(0x%llx,0x%llx) error 0x%x\n", @@ -1229,6 +1352,7 @@ kmem_init( kr); } } +#endif /* * Set the default global user wire limit which limits the amount of @@ -1315,116 +1439,95 @@ copyoutmap( return KERN_SUCCESS; } +/* + * + * The following two functions are to be used when exposing kernel + * addresses to userspace via any of the various debug or info + * facilities that exist. These are basically the same as VM_KERNEL_ADDRPERM() + * and VM_KERNEL_UNSLIDE_OR_PERM() except they use a different random seed and + * are exported to KEXTs. + * + * NOTE: USE THE MACRO VERSIONS OF THESE FUNCTIONS (in vm_param.h) FROM WITHIN THE KERNEL + */ -kern_return_t -vm_conflict_check( - vm_map_t map, - vm_map_offset_t off, - vm_map_size_t len, - memory_object_t pager, - vm_object_offset_t file_off) +static void +vm_kernel_addrhash_internal( + vm_offset_t addr, + vm_offset_t *hash_addr, + uint64_t salt) { - vm_map_entry_t entry; - vm_object_t obj; - vm_object_offset_t obj_off; - vm_map_t base_map; - vm_map_offset_t base_offset; - vm_map_offset_t original_offset; - kern_return_t kr; - vm_map_size_t local_len; + assert(salt != 0); - base_map = map; - base_offset = off; - original_offset = off; - kr = KERN_SUCCESS; - vm_map_lock(map); - while(vm_map_lookup_entry(map, off, &entry)) { - local_len = len; + if (addr == 0) { + *hash_addr = 0; + return; + } - if (entry->object.vm_object == VM_OBJECT_NULL) { - vm_map_unlock(map); - return KERN_SUCCESS; - } - if (entry->is_sub_map) { - vm_map_t old_map; - - old_map = map; - vm_map_lock(entry->object.sub_map); - map = entry->object.sub_map; - off = entry->offset + (off - entry->vme_start); - vm_map_unlock(old_map); - continue; - } - obj = entry->object.vm_object; - obj_off = (off - entry->vme_start) + entry->offset; - while(obj->shadow) { - obj_off += obj->vo_shadow_offset; - obj = obj->shadow; - } - if((obj->pager_created) && (obj->pager == pager)) { - if(((obj->paging_offset) + obj_off) == file_off) { - if(off != base_offset) { - vm_map_unlock(map); - return KERN_FAILURE; - } - kr = KERN_ALREADY_WAITING; - } else { - vm_object_offset_t obj_off_aligned; - vm_object_offset_t file_off_aligned; - - obj_off_aligned = obj_off & ~PAGE_MASK; - file_off_aligned = file_off & ~PAGE_MASK; - - if (file_off_aligned == (obj->paging_offset + obj_off_aligned)) { - /* - * the target map and the file offset start in the same page - * but are not identical... - */ - vm_map_unlock(map); - return KERN_FAILURE; - } - if ((file_off < (obj->paging_offset + obj_off_aligned)) && - ((file_off + len) > (obj->paging_offset + obj_off_aligned))) { - /* - * some portion of the tail of the I/O will fall - * within the encompass of the target map - */ - vm_map_unlock(map); - return KERN_FAILURE; - } - if ((file_off_aligned > (obj->paging_offset + obj_off)) && - (file_off_aligned < (obj->paging_offset + obj_off) + len)) { - /* - * the beginning page of the file offset falls within - * the target map's encompass - */ - vm_map_unlock(map); - return KERN_FAILURE; - } - } - } else if(kr != KERN_SUCCESS) { - vm_map_unlock(map); - return KERN_FAILURE; - } + if (VM_KERNEL_IS_SLID(addr)) { + *hash_addr = VM_KERNEL_UNSLIDE(addr); + return; + } - if(len <= ((entry->vme_end - entry->vme_start) - - (off - entry->vme_start))) { - vm_map_unlock(map); - return kr; - } else { - len -= (entry->vme_end - entry->vme_start) - - (off - entry->vme_start); - } - base_offset = base_offset + (local_len - len); - file_off = file_off + (local_len - len); - off = base_offset; - if(map != base_map) { - vm_map_unlock(map); - vm_map_lock(base_map); - map = base_map; - } + vm_offset_t sha_digest[SHA256_DIGEST_LENGTH/sizeof(vm_offset_t)]; + SHA256_CTX sha_ctx; + + SHA256_Init(&sha_ctx); + SHA256_Update(&sha_ctx, &salt, sizeof(salt)); + SHA256_Update(&sha_ctx, &addr, sizeof(addr)); + SHA256_Final(sha_digest, &sha_ctx); + + *hash_addr = sha_digest[0]; +} + +void +vm_kernel_addrhash_external( + vm_offset_t addr, + vm_offset_t *hash_addr) +{ + return vm_kernel_addrhash_internal(addr, hash_addr, vm_kernel_addrhash_salt_ext); +} + +vm_offset_t +vm_kernel_addrhash(vm_offset_t addr) +{ + vm_offset_t hash_addr; + vm_kernel_addrhash_internal(addr, &hash_addr, vm_kernel_addrhash_salt); + return hash_addr; +} + +void +vm_kernel_addrhide( + vm_offset_t addr, + vm_offset_t *hide_addr) +{ + *hide_addr = VM_KERNEL_ADDRHIDE(addr); +} + +/* + * vm_kernel_addrperm_external: + * vm_kernel_unslide_or_perm_external: + * + * Use these macros when exposing an address to userspace that could come from + * either kernel text/data *or* the heap. + */ +void +vm_kernel_addrperm_external( + vm_offset_t addr, + vm_offset_t *perm_addr) +{ + if (VM_KERNEL_IS_SLID(addr)) { + *perm_addr = VM_KERNEL_UNSLIDE(addr); + } else if (VM_KERNEL_ADDRESS(addr)) { + *perm_addr = addr + vm_kernel_addrperm_ext; + } else { + *perm_addr = addr; } +} - vm_map_unlock(map); - return kr; +void +vm_kernel_unslide_or_perm_external( + vm_offset_t addr, + vm_offset_t *up_addr) +{ + vm_kernel_addrperm_external(addr, up_addr); }