X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/c18c124eaa464aaaa5549e99e5a70fc9cbb50944..ecc0ceb4089d506a0b8d16686a95817b331af9cb:/osfmk/vm/vm_map.c?ds=sidebyside diff --git a/osfmk/vm/vm_map.c b/osfmk/vm/vm_map.c index 58a8c7828..840c4babf 100644 --- a/osfmk/vm/vm_map.c +++ b/osfmk/vm/vm_map.c @@ -109,6 +109,7 @@ #include #include + extern u_int32_t random(void); /* from */ /* Internal prototypes */ @@ -216,7 +217,7 @@ static kern_return_t vm_map_wire_nested( vm_map_t map, vm_map_offset_t start, vm_map_offset_t end, - vm_prot_t access_type, + vm_prot_t caller_prot, boolean_t user_wire, pmap_t map_pmap, vm_map_offset_t pmap_addr, @@ -297,6 +298,12 @@ static kern_return_t vm_map_can_reuse( vm_map_offset_t start, vm_map_offset_t end); +#if MACH_ASSERT +static kern_return_t vm_map_pageout( + vm_map_t map, + vm_map_offset_t start, + vm_map_offset_t end); +#endif /* MACH_ASSERT */ /* * Macros to copy a vm_map_entry. We must be careful to correctly @@ -321,6 +328,8 @@ boolean_t _vmec_reserved = (NEW)->from_reserved_zone; \ (NEW)->used_for_jit = FALSE; \ (NEW)->from_reserved_zone = _vmec_reserved; \ (NEW)->iokit_acct = FALSE; \ + (NEW)->vme_resilient_codesign = FALSE; \ + (NEW)->vme_resilient_media = FALSE; \ MACRO_END #define vm_map_entry_copy_full(NEW,OLD) \ @@ -367,6 +376,8 @@ override_nx(vm_map_t map, uint32_t user_tag) /* map unused on arm */ { int current_abi; + if (map->pmap == kernel_pmap) return FALSE; + /* * Determine if the app is running in 32 or 64 bit mode. */ @@ -436,6 +447,7 @@ static zone_t vm_map_entry_zone; /* zone for vm_map_entry structures */ static zone_t vm_map_entry_reserved_zone; /* zone with reserve for non-blocking * allocations */ static zone_t vm_map_copy_zone; /* zone for vm_map_copy structures */ +zone_t vm_map_holes_zone; /* zone for vm map holes (vm_map_links) structures */ /* @@ -450,6 +462,8 @@ static void *map_data; static vm_size_t map_data_size; static void *kentry_data; static vm_size_t kentry_data_size; +static void *map_holes_data; +static vm_size_t map_holes_data_size; #define NO_COALESCE_LIMIT ((1024 * 128) - 1) @@ -477,7 +491,7 @@ vm_map_set_cache_attr( kr = KERN_INVALID_ARGUMENT; goto done; } - object = map_entry->object.vm_object; + object = VME_OBJECT(map_entry); if (object == VM_OBJECT_NULL) { /* @@ -509,87 +523,178 @@ done: */ kern_return_t vm_map_apple_protected( - vm_map_t map, - vm_map_offset_t start, - vm_map_offset_t end, + vm_map_t map, + vm_map_offset_t start, + vm_map_offset_t end, + vm_object_offset_t crypto_backing_offset, struct pager_crypt_info *crypt_info) { boolean_t map_locked; kern_return_t kr; vm_map_entry_t map_entry; - memory_object_t protected_mem_obj; + struct vm_map_entry tmp_entry; + memory_object_t unprotected_mem_obj; vm_object_t protected_object; vm_map_offset_t map_addr; + vm_map_offset_t start_aligned, end_aligned; + vm_object_offset_t crypto_start, crypto_end; + int vm_flags; - vm_map_lock_read(map); - map_locked = TRUE; - - /* lookup the protected VM object */ - if (!vm_map_lookup_entry(map, - start, - &map_entry) || - map_entry->vme_end < end || - map_entry->is_sub_map || - !(map_entry->protection & VM_PROT_EXECUTE)) { - /* that memory is not properly mapped */ - kr = KERN_INVALID_ARGUMENT; - goto done; - } - protected_object = map_entry->object.vm_object; - if (protected_object == VM_OBJECT_NULL) { - /* there should be a VM object here at this point */ - kr = KERN_INVALID_ARGUMENT; - goto done; - } + map_locked = FALSE; + unprotected_mem_obj = MEMORY_OBJECT_NULL; - /* make sure protected object stays alive while map is unlocked */ - vm_object_reference(protected_object); + start_aligned = vm_map_trunc_page(start, PAGE_MASK_64); + end_aligned = vm_map_round_page(end, PAGE_MASK_64); + start_aligned = vm_map_trunc_page(start_aligned, VM_MAP_PAGE_MASK(map)); + end_aligned = vm_map_round_page(end_aligned, VM_MAP_PAGE_MASK(map)); - vm_map_unlock_read(map); - map_locked = FALSE; + assert(start_aligned == start); + assert(end_aligned == end); - /* - * Lookup (and create if necessary) the protected memory object - * matching that VM object. - * If successful, this also grabs a reference on the memory object, - * to guarantee that it doesn't go away before we get a chance to map - * it. - */ - protected_mem_obj = apple_protect_pager_setup(protected_object, crypt_info); + map_addr = start_aligned; + for (map_addr = start_aligned; + map_addr < end; + map_addr = tmp_entry.vme_end) { + vm_map_lock(map); + map_locked = TRUE; - /* release extra ref on protected object */ - vm_object_deallocate(protected_object); + /* lookup the protected VM object */ + if (!vm_map_lookup_entry(map, + map_addr, + &map_entry) || + map_entry->is_sub_map || + VME_OBJECT(map_entry) == VM_OBJECT_NULL || + !(map_entry->protection & VM_PROT_EXECUTE)) { + /* that memory is not properly mapped */ + kr = KERN_INVALID_ARGUMENT; + goto done; + } - if (protected_mem_obj == NULL) { - kr = KERN_FAILURE; - goto done; - } + /* get the protected object to be decrypted */ + protected_object = VME_OBJECT(map_entry); + if (protected_object == VM_OBJECT_NULL) { + /* there should be a VM object here at this point */ + kr = KERN_INVALID_ARGUMENT; + goto done; + } + /* ensure protected object stays alive while map is unlocked */ + vm_object_reference(protected_object); - /* map this memory object in place of the current one */ - map_addr = start; - kr = vm_map_enter_mem_object(map, - &map_addr, - end - start, - (mach_vm_offset_t) 0, - VM_FLAGS_FIXED | VM_FLAGS_OVERWRITE, - (ipc_port_t) protected_mem_obj, - (map_entry->offset + - (start - map_entry->vme_start)), - TRUE, - map_entry->protection, - map_entry->max_protection, - map_entry->inheritance); - assert(map_addr == start); - /* - * Release the reference obtained by apple_protect_pager_setup(). - * The mapping (if it succeeded) is now holding a reference on the - * memory object. - */ - memory_object_deallocate(protected_mem_obj); + /* limit the map entry to the area we want to cover */ + vm_map_clip_start(map, map_entry, start_aligned); + vm_map_clip_end(map, map_entry, end_aligned); + + tmp_entry = *map_entry; + map_entry = VM_MAP_ENTRY_NULL; /* not valid after unlocking map */ + vm_map_unlock(map); + map_locked = FALSE; + + /* + * This map entry might be only partially encrypted + * (if not fully "page-aligned"). + */ + crypto_start = 0; + crypto_end = tmp_entry.vme_end - tmp_entry.vme_start; + if (tmp_entry.vme_start < start) { + if (tmp_entry.vme_start != start_aligned) { + kr = KERN_INVALID_ADDRESS; + } + crypto_start += (start - tmp_entry.vme_start); + } + if (tmp_entry.vme_end > end) { + if (tmp_entry.vme_end != end_aligned) { + kr = KERN_INVALID_ADDRESS; + } + crypto_end -= (tmp_entry.vme_end - end); + } + + /* + * This "extra backing offset" is needed to get the decryption + * routine to use the right key. It adjusts for the possibly + * relative offset of an interposed "4K" pager... + */ + if (crypto_backing_offset == (vm_object_offset_t) -1) { + crypto_backing_offset = VME_OFFSET(&tmp_entry); + } + + /* + * Lookup (and create if necessary) the protected memory object + * matching that VM object. + * If successful, this also grabs a reference on the memory object, + * to guarantee that it doesn't go away before we get a chance to map + * it. + */ + unprotected_mem_obj = apple_protect_pager_setup( + protected_object, + VME_OFFSET(&tmp_entry), + crypto_backing_offset, + crypt_info, + crypto_start, + crypto_end); + + /* release extra ref on protected object */ + vm_object_deallocate(protected_object); + + if (unprotected_mem_obj == NULL) { + kr = KERN_FAILURE; + goto done; + } + + vm_flags = VM_FLAGS_FIXED | VM_FLAGS_OVERWRITE; + + /* map this memory object in place of the current one */ + map_addr = tmp_entry.vme_start; + kr = vm_map_enter_mem_object(map, + &map_addr, + (tmp_entry.vme_end - + tmp_entry.vme_start), + (mach_vm_offset_t) 0, + vm_flags, + (ipc_port_t) unprotected_mem_obj, + 0, + TRUE, + tmp_entry.protection, + tmp_entry.max_protection, + tmp_entry.inheritance); + assert(kr == KERN_SUCCESS); + assert(map_addr == tmp_entry.vme_start); + +#if VM_MAP_DEBUG_APPLE_PROTECT + printf("APPLE_PROTECT: map %p [0x%llx:0x%llx] pager %p: " + "backing:[object:%p,offset:0x%llx," + "crypto_backing_offset:0x%llx," + "crypto_start:0x%llx,crypto_end:0x%llx]\n", + map, + (uint64_t) map_addr, + (uint64_t) (map_addr + (tmp_entry.vme_end - + tmp_entry.vme_start)), + unprotected_mem_obj, + protected_object, + VME_OFFSET(&tmp_entry), + crypto_backing_offset, + crypto_start, + crypto_end); +#endif /* VM_MAP_DEBUG_APPLE_PROTECT */ + + /* + * Release the reference obtained by + * apple_protect_pager_setup(). + * The mapping (if it succeeded) is now holding a reference on + * the memory object. + */ + memory_object_deallocate(unprotected_mem_obj); + unprotected_mem_obj = MEMORY_OBJECT_NULL; + + /* continue with next map entry */ + crypto_backing_offset += (tmp_entry.vme_end - + tmp_entry.vme_start); + crypto_backing_offset -= crypto_start; + } + kr = KERN_SUCCESS; done: if (map_locked) { - vm_map_unlock_read(map); + vm_map_unlock(map); } return kr; } @@ -654,6 +759,10 @@ vm_map_init( 16*1024, PAGE_SIZE, "VM map copies"); zone_change(vm_map_copy_zone, Z_NOENCRYPT, TRUE); + vm_map_holes_zone = zinit((vm_map_size_t) sizeof(struct vm_map_links), + 16*1024, PAGE_SIZE, "VM map holes"); + zone_change(vm_map_holes_zone, Z_NOENCRYPT, TRUE); + /* * Cram the map and kentry zones with initial data. * Set reserved_zone non-collectible to aid zone_gc(). @@ -668,9 +777,21 @@ vm_map_init( zone_change(vm_map_copy_zone, Z_CALLERACCT, FALSE); /* don't charge caller */ zone_change(vm_map_entry_reserved_zone, Z_GZALLOC_EXEMPT, TRUE); + zone_change(vm_map_holes_zone, Z_COLLECT, TRUE); + zone_change(vm_map_holes_zone, Z_EXPAND, TRUE); + zone_change(vm_map_holes_zone, Z_FOREIGN, TRUE); + zone_change(vm_map_holes_zone, Z_NOCALLOUT, TRUE); + zone_change(vm_map_holes_zone, Z_CALLERACCT, TRUE); + zone_change(vm_map_holes_zone, Z_GZALLOC_EXEMPT, TRUE); + + /* + * Add the stolen memory to zones, adjust zone size and stolen counts. + */ zcram(vm_map_zone, (vm_offset_t)map_data, map_data_size); zcram(vm_map_entry_reserved_zone, (vm_offset_t)kentry_data, kentry_data_size); - + zcram(vm_map_holes_zone, (vm_offset_t)map_holes_data, map_holes_data_size); + VM_PAGE_MOVE_STOLEN(atop_64(map_data_size) + atop_64(kentry_data_size) + atop_64(map_holes_data_size)); + lck_grp_attr_setdefault(&vm_map_lck_grp_attr); lck_grp_init(&vm_map_lck_grp, "vm_map", &vm_map_lck_grp_attr); lck_attr_setdefault(&vm_map_lck_attr); @@ -714,10 +835,52 @@ vm_map_steal_memory( kentry_data_size = kentry_initial_pages * PAGE_SIZE; kentry_data = pmap_steal_memory(kentry_data_size); + + map_holes_data_size = kentry_data_size; + map_holes_data = pmap_steal_memory(map_holes_data_size); } -void vm_kernel_reserved_entry_init(void) { +void +vm_kernel_reserved_entry_init(void) { zone_prio_refill_configure(vm_map_entry_reserved_zone, (6*PAGE_SIZE)/sizeof(struct vm_map_entry)); + zone_prio_refill_configure(vm_map_holes_zone, (6*PAGE_SIZE)/sizeof(struct vm_map_links)); +} + +void +vm_map_disable_hole_optimization(vm_map_t map) +{ + vm_map_entry_t head_entry, hole_entry, next_hole_entry; + + if (map->holelistenabled) { + + head_entry = hole_entry = (vm_map_entry_t) map->holes_list; + + while (hole_entry != NULL) { + + next_hole_entry = hole_entry->vme_next; + + hole_entry->vme_next = NULL; + hole_entry->vme_prev = NULL; + zfree(vm_map_holes_zone, hole_entry); + + if (next_hole_entry == head_entry) { + hole_entry = NULL; + } else { + hole_entry = next_hole_entry; + } + } + + map->holes_list = NULL; + map->holelistenabled = FALSE; + + map->first_free = vm_map_first_entry(map); + SAVE_HINT_HOLE_WRITE(map, NULL); + } +} + +boolean_t +vm_kernel_map_is_kernel(vm_map_t map) { + return (map->pmap == kernel_pmap); } /* @@ -727,6 +890,9 @@ void vm_kernel_reserved_entry_init(void) { * the given physical map structure, and having * the given lower and upper address bounds. */ + +boolean_t vm_map_supports_hole_optimization = TRUE; + vm_map_t vm_map_create( pmap_t pmap, @@ -736,6 +902,7 @@ vm_map_create( { static int color_seed = 0; register vm_map_t result; + struct vm_map_links *hole_entry = NULL; result = (vm_map_t) zalloc(vm_map_zone); if (result == VM_MAP_NULL) @@ -773,6 +940,21 @@ vm_map_create( result->hint = vm_map_to_entry(result); result->color_rr = (color_seed++) & vm_color_mask; result->jit_entry_exists = FALSE; + + if (vm_map_supports_hole_optimization && pmap != kernel_pmap) { + hole_entry = zalloc(vm_map_holes_zone); + + hole_entry->start = min; + hole_entry->end = (max > (vm_map_offset_t)MACH_VM_MAX_ADDRESS) ? max : (vm_map_offset_t)MACH_VM_MAX_ADDRESS; + result->holes_list = result->hole_hint = hole_entry; + hole_entry->prev = hole_entry->next = (vm_map_entry_t) hole_entry; + result->holelistenabled = TRUE; + + } else { + + result->holelistenabled = FALSE; + } + #if CONFIG_FREEZE result->default_freezer_handle = NULL; #endif @@ -970,6 +1152,9 @@ vm_map_destroy( { vm_map_lock(map); + /* final cleanup: no need to unnest shared region */ + flags |= VM_MAP_REMOVE_NO_UNNESTING; + /* clean up regular map entries */ (void) vm_map_delete(map, map->min_offset, map->max_offset, flags, VM_MAP_NULL); @@ -983,6 +1168,7 @@ vm_map_destroy( map->default_freezer_handle = NULL; } #endif + vm_map_disable_hole_optimization(map); vm_map_unlock(map); assert(map->hdr.nentries == 0); @@ -1088,14 +1274,14 @@ void vm_map_swapin (vm_map_t map) entry = vm_map_first_entry(map); while (entry != vm_map_to_entry(map)) { - if (entry->object.vm_object != VM_OBJECT_NULL) { + if (VME_OBJECT(entry) != VM_OBJECT_NULL) { if (entry->is_sub_map) { - vm_map_t lmap = entry->object.sub_map; + vm_map_t lmap = VME_SUBMAP(entry); lck_mtx_lock(&lmap->s_lock); vm_map_res_reference(lmap); lck_mtx_unlock(&lmap->s_lock); } else { - vm_object_t object = entry->object.vm_object; + vm_object_t object = VME_OBEJCT(entry); vm_object_lock(object); /* * This call may iterate through the @@ -1149,14 +1335,14 @@ void vm_map_swapout(vm_map_t map) entry = vm_map_first_entry(map); while (entry != vm_map_to_entry(map)) { - if (entry->object.vm_object != VM_OBJECT_NULL) { + if (VME_OBJECT(entry) != VM_OBJECT_NULL) { if (entry->is_sub_map) { - vm_map_t lmap = entry->object.sub_map; + vm_map_t lmap = VME_SUBMAP(entry); lck_mtx_lock(&lmap->s_lock); vm_map_res_deallocate(lmap); lck_mtx_unlock(&lmap->s_lock); } else { - vm_object_t object = entry->object.vm_object; + vm_object_t object = VME_OBJECT(entry); vm_object_lock(object); /* * This call may take a long time, @@ -1217,9 +1403,10 @@ vm_map_find_space( int flags, vm_map_entry_t *o_entry) /* OUT */ { - register vm_map_entry_t entry, new_entry; + vm_map_entry_t entry, new_entry; register vm_map_offset_t start; register vm_map_offset_t end; + vm_map_entry_t hole_entry; if (size == 0) { *address = 0; @@ -1243,11 +1430,27 @@ vm_map_find_space( if( map->disable_vmentry_reuse == TRUE) { VM_MAP_HIGHEST_ENTRY(map, entry, start); } else { - assert(first_free_is_valid(map)); - if ((entry = map->first_free) == vm_map_to_entry(map)) - start = map->min_offset; - else - start = entry->vme_end; + if (map->holelistenabled) { + hole_entry = (vm_map_entry_t)map->holes_list; + + if (hole_entry == NULL) { + /* + * No more space in the map? + */ + vm_map_entry_dispose(map, new_entry); + vm_map_unlock(map); + return(KERN_NO_SPACE); + } + + entry = hole_entry; + start = entry->vme_start; + } else { + assert(first_free_is_valid(map)); + if ((entry = map->first_free) == vm_map_to_entry(map)) + start = map->min_offset; + else + start = entry->vme_end; + } } /* @@ -1284,28 +1487,53 @@ vm_map_find_space( return(KERN_NO_SPACE); } - /* - * If there are no more entries, we must win. - */ - next = entry->vme_next; - if (next == vm_map_to_entry(map)) - break; - /* - * If there is another entry, it must be - * after the end of the potential new region. - */ + if (map->holelistenabled) { + if (entry->vme_end >= end) + break; + } else { + /* + * If there are no more entries, we must win. + * + * OR + * + * If there is another entry, it must be + * after the end of the potential new region. + */ - if (next->vme_start >= end) - break; + if (next == vm_map_to_entry(map)) + break; + + if (next->vme_start >= end) + break; + } /* * Didn't fit -- move to the next entry. */ entry = next; - start = entry->vme_end; + + if (map->holelistenabled) { + if (entry == (vm_map_entry_t) map->holes_list) { + /* + * Wrapped around + */ + vm_map_entry_dispose(map, new_entry); + vm_map_unlock(map); + return(KERN_NO_SPACE); + } + start = entry->vme_start; + } else { + start = entry->vme_end; + } + } + + if (map->holelistenabled) { + if (vm_map_lookup_entry(map, entry->vme_start, &entry)) { + panic("Found an existing entry (%p) instead of potential hole at address: 0x%llx.\n", entry, (unsigned long long)entry->vme_start); + } } /* @@ -1337,8 +1565,8 @@ vm_map_find_space( new_entry->is_shared = FALSE; new_entry->is_sub_map = FALSE; new_entry->use_pmap = TRUE; - new_entry->object.vm_object = VM_OBJECT_NULL; - new_entry->offset = (vm_object_offset_t) 0; + VME_OBJECT_SET(new_entry, VM_OBJECT_NULL); + VME_OFFSET_SET(new_entry, (vm_object_offset_t) 0); new_entry->needs_copy = FALSE; @@ -1360,13 +1588,15 @@ vm_map_find_space( new_entry->map_aligned = FALSE; } - new_entry->used_for_jit = 0; - - new_entry->alias = 0; + new_entry->used_for_jit = FALSE; new_entry->zero_wired_pages = FALSE; new_entry->iokit_acct = FALSE; + new_entry->vme_resilient_codesign = FALSE; + new_entry->vme_resilient_media = FALSE; - VM_GET_FLAGS_ALIAS(flags, new_entry->alias); + int alias; + VM_GET_FLAGS_ALIAS(flags, alias); + VME_ALIAS_SET(new_entry, alias); /* * Insert the new entry into the list @@ -1601,11 +1831,14 @@ vm_map_enter( boolean_t permanent = ((flags & VM_FLAGS_PERMANENT) != 0); boolean_t entry_for_jit = ((flags & VM_FLAGS_MAP_JIT) != 0); boolean_t iokit_acct = ((flags & VM_FLAGS_IOKIT_ACCT) != 0); + boolean_t resilient_codesign = ((flags & VM_FLAGS_RESILIENT_CODESIGN) != 0); + boolean_t resilient_media = ((flags & VM_FLAGS_RESILIENT_MEDIA) != 0); unsigned int superpage_size = ((flags & VM_FLAGS_SUPERPAGE_MASK) >> VM_FLAGS_SUPERPAGE_SHIFT); - char alias; + vm_tag_t alias, user_alias; vm_map_offset_t effective_min_offset, effective_max_offset; kern_return_t kr; boolean_t clear_map_aligned = FALSE; + vm_map_entry_t hole_entry; if (superpage_size) { switch (superpage_size) { @@ -1634,6 +1867,13 @@ vm_map_enter( + if (resilient_codesign || resilient_media) { + if ((cur_protection & (VM_PROT_WRITE | VM_PROT_EXECUTE)) || + (max_protection & (VM_PROT_WRITE | VM_PROT_EXECUTE))) { + return KERN_PROTECTION_FAILURE; + } + } + if (is_submap) { if (purgable) { /* submaps can not be purgeable */ @@ -1679,6 +1919,11 @@ vm_map_enter( } VM_GET_FLAGS_ALIAS(flags, alias); + if (map->pmap == kernel_pmap) { + user_alias = VM_KERN_MEMORY_NONE; + } else { + user_alias = alias; + } #define RETURN(value) { result = value; goto BailOut; } @@ -1736,6 +1981,7 @@ vm_map_enter( *address + size, map->hdr.entries_pageable); vm_map_set_page_shift(zap_old_map, VM_MAP_PAGE_SHIFT(map)); + vm_map_disable_hole_optimization(zap_old_map); } StartAgain: ; @@ -1780,41 +2026,82 @@ StartAgain: ; if( map->disable_vmentry_reuse == TRUE) { VM_MAP_HIGHEST_ENTRY(map, entry, start); } else { - assert(first_free_is_valid(map)); - entry = map->first_free; + if (map->holelistenabled) { + hole_entry = (vm_map_entry_t)map->holes_list; - if (entry == vm_map_to_entry(map)) { - entry = NULL; + if (hole_entry == NULL) { + /* + * No more space in the map? + */ + result = KERN_NO_SPACE; + goto BailOut; + } else { + + boolean_t found_hole = FALSE; + + do { + if (hole_entry->vme_start >= start) { + start = hole_entry->vme_start; + found_hole = TRUE; + break; + } + + if (hole_entry->vme_end > start) { + found_hole = TRUE; + break; + } + hole_entry = hole_entry->vme_next; + + } while (hole_entry != (vm_map_entry_t) map->holes_list); + + if (found_hole == FALSE) { + result = KERN_NO_SPACE; + goto BailOut; + } + + entry = hole_entry; + + if (start == 0) + start += PAGE_SIZE_64; + } } else { - if (entry->vme_next == vm_map_to_entry(map)){ - /* - * Hole at the end of the map. - */ + assert(first_free_is_valid(map)); + + entry = map->first_free; + + if (entry == vm_map_to_entry(map)) { entry = NULL; - } else { - if (start < (entry->vme_next)->vme_start ) { - start = entry->vme_end; + } else { + if (entry->vme_next == vm_map_to_entry(map)){ + /* + * Hole at the end of the map. + */ + entry = NULL; + } else { + if (start < (entry->vme_next)->vme_start ) { + start = entry->vme_end; + start = vm_map_round_page(start, + VM_MAP_PAGE_MASK(map)); + } else { + /* + * Need to do a lookup. + */ + entry = NULL; + } + } + } + + if (entry == NULL) { + vm_map_entry_t tmp_entry; + if (vm_map_lookup_entry(map, start, &tmp_entry)) { + assert(!entry_for_jit); + start = tmp_entry->vme_end; start = vm_map_round_page(start, VM_MAP_PAGE_MASK(map)); - } else { - /* - * Need to do a lookup. - */ - entry = NULL; } - } - } - - if (entry == NULL) { - vm_map_entry_t tmp_entry; - if (vm_map_lookup_entry(map, start, &tmp_entry)) { - assert(!entry_for_jit); - start = tmp_entry->vme_end; - start = vm_map_round_page(start, - VM_MAP_PAGE_MASK(map)); + entry = tmp_entry; } - entry = tmp_entry; } } @@ -1859,31 +2146,57 @@ StartAgain: ; RETURN(KERN_NO_SPACE); } - /* - * If there are no more entries, we must win. - */ - next = entry->vme_next; - if (next == vm_map_to_entry(map)) - break; - /* - * If there is another entry, it must be - * after the end of the potential new region. - */ + if (map->holelistenabled) { + if (entry->vme_end >= end) + break; + } else { + /* + * If there are no more entries, we must win. + * + * OR + * + * If there is another entry, it must be + * after the end of the potential new region. + */ - if (next->vme_start >= end) - break; + if (next == vm_map_to_entry(map)) + break; + + if (next->vme_start >= end) + break; + } /* * Didn't fit -- move to the next entry. */ entry = next; - start = entry->vme_end; + + if (map->holelistenabled) { + if (entry == (vm_map_entry_t) map->holes_list) { + /* + * Wrapped around + */ + result = KERN_NO_SPACE; + goto BailOut; + } + start = entry->vme_start; + } else { + start = entry->vme_end; + } + start = vm_map_round_page(start, VM_MAP_PAGE_MASK(map)); } + + if (map->holelistenabled) { + if (vm_map_lookup_entry(map, entry->vme_start, &entry)) { + panic("Found an existing entry (%p) instead of potential hole at address: 0x%llx.\n", entry, (unsigned long long)entry->vme_start); + } + } + *address = start; assert(VM_MAP_PAGE_ALIGNED(*address, VM_MAP_PAGE_MASK(map))); @@ -1950,13 +2263,13 @@ StartAgain: ; if (entry == vm_map_to_entry(map) || entry->vme_start != tmp_start || entry->is_sub_map != is_submap || - entry->offset != tmp_offset || + VME_OFFSET(entry) != tmp_offset || entry->needs_copy != needs_copy || entry->protection != cur_protection || entry->max_protection != max_protection || entry->inheritance != inheritance || entry->iokit_acct != iokit_acct || - entry->alias != alias) { + VME_ALIAS(entry) != alias) { /* not the same mapping ! */ RETURN(KERN_NO_SPACE); } @@ -1964,17 +2277,17 @@ StartAgain: ; * Check if the same object is being mapped. */ if (is_submap) { - if (entry->object.sub_map != + if (VME_SUBMAP(entry) != (vm_map_t) object) { /* not the same submap */ RETURN(KERN_NO_SPACE); } } else { - if (entry->object.vm_object != object) { + if (VME_OBJECT(entry) != object) { /* not the same VM object... */ vm_object_t obj2; - obj2 = entry->object.vm_object; + obj2 = VME_OBJECT(entry); if ((obj2 == VM_OBJECT_NULL || obj2->internal) && (object == VM_OBJECT_NULL || @@ -2032,6 +2345,7 @@ StartAgain: ; if (purgable || entry_for_jit) { if (object == VM_OBJECT_NULL) { + object = vm_object_allocate(size); object->copy_strategy = MEMORY_OBJECT_COPY_NONE; object->true_share = TRUE; @@ -2073,7 +2387,8 @@ StartAgain: ; (entry->protection == cur_protection) && (entry->max_protection == max_protection) && (entry->inheritance == inheritance) && - ((alias == VM_MEMORY_REALLOC) || (entry->alias == alias)) && + ((user_alias == VM_MEMORY_REALLOC) || + (VME_ALIAS(entry) == alias)) && (entry->no_cache == no_cache) && (entry->permanent == permanent) && (!entry->superpage_size && !superpage_size) && @@ -2085,16 +2400,18 @@ StartAgain: ; (!entry->zero_wired_pages) && (!entry->used_for_jit && !entry_for_jit) && (entry->iokit_acct == iokit_acct) && + (!entry->vme_resilient_codesign) && + (!entry->vme_resilient_media) && ((entry->vme_end - entry->vme_start) + size <= - (alias == VM_MEMORY_REALLOC ? + (user_alias == VM_MEMORY_REALLOC ? ANON_CHUNK_SIZE : NO_COALESCE_LIMIT)) && (entry->wired_count == 0)) { /* implies user_wired_count == 0 */ - if (vm_object_coalesce(entry->object.vm_object, + if (vm_object_coalesce(VME_OBJECT(entry), VM_OBJECT_NULL, - entry->offset, + VME_OFFSET(entry), (vm_object_offset_t) 0, (vm_map_size_t)(entry->vme_end - entry->vme_start), (vm_map_size_t)(end - entry->vme_end))) { @@ -2108,8 +2425,14 @@ StartAgain: ; assert(entry->vme_start < end); assert(VM_MAP_PAGE_ALIGNED(end, VM_MAP_PAGE_MASK(map))); + if (__improbable(vm_debug_events)) + DTRACE_VM5(map_entry_extend, vm_map_t, map, vm_map_entry_t, entry, vm_address_t, entry->vme_start, vm_address_t, entry->vme_end, vm_address_t, end); entry->vme_end = end; - vm_map_store_update_first_free(map, map->first_free); + if (map->holelistenabled) { + vm_map_store_update_first_free(map, entry, TRUE); + } else { + vm_map_store_update_first_free(map, map->first_free, TRUE); + } new_mapping_established = TRUE; RETURN(KERN_SUCCESS); } @@ -2155,7 +2478,10 @@ StartAgain: ; superpage_size, clear_map_aligned, is_submap); - new_entry->alias = alias; + + assert((object != kernel_object) || (VM_KERN_MEMORY_NONE != alias)); + VME_ALIAS_SET(new_entry, alias); + if (entry_for_jit){ if (!(map->jit_entry_exists)){ new_entry->used_for_jit = TRUE; @@ -2163,6 +2489,18 @@ StartAgain: ; } } + if (resilient_codesign && + ! ((cur_protection | max_protection) & + (VM_PROT_WRITE | VM_PROT_EXECUTE))) { + new_entry->vme_resilient_codesign = TRUE; + } + + if (resilient_media && + ! ((cur_protection | max_protection) & + (VM_PROT_WRITE | VM_PROT_EXECUTE))) { + new_entry->vme_resilient_media = TRUE; + } + assert(!new_entry->iokit_acct); if (!is_submap && object != VM_OBJECT_NULL && @@ -2176,12 +2514,20 @@ StartAgain: ; */ new_entry->use_pmap = FALSE; } else if (!is_submap && - iokit_acct) { + iokit_acct && + object != VM_OBJECT_NULL && + object->internal) { /* alternate accounting */ assert(!new_entry->iokit_acct); assert(new_entry->use_pmap); new_entry->iokit_acct = TRUE; new_entry->use_pmap = FALSE; + DTRACE_VM4( + vm_map_iokit_mapped_region, + vm_map_t, map, + vm_map_offset_t, new_entry->vme_start, + vm_map_offset_t, new_entry->vme_end, + int, VME_ALIAS(new_entry)); vm_map_iokit_mapped_region( map, (new_entry->vme_end - @@ -2201,7 +2547,7 @@ StartAgain: ; assert(!new_entry->iokit_acct); submap = (vm_map_t) object; submap_is_64bit = vm_map_is_64bit(submap); - use_pmap = (alias == VM_MEMORY_SHARED_PMAP); + use_pmap = (user_alias == VM_MEMORY_SHARED_PMAP); #ifndef NO_NESTED_PMAP if (use_pmap && submap->pmap == NULL) { ledger_t ledger = map->pmap->ledger; @@ -2239,12 +2585,15 @@ StartAgain: ; vm_page_t pages, m; vm_object_t sp_object; - entry->offset = 0; + VME_OFFSET_SET(entry, 0); /* allocate one superpage */ kr = cpm_allocate(SUPERPAGE_SIZE, &pages, 0, SUPERPAGE_NBASEPAGES-1, TRUE, 0); if (kr != KERN_SUCCESS) { - new_mapping_established = TRUE; /* will cause deallocation of whole range */ + /* deallocate whole range... */ + new_mapping_established = TRUE; + /* ... but only up to "tmp_end" */ + size -= end - tmp_end; RETURN(kr); } @@ -2252,7 +2601,7 @@ StartAgain: ; sp_object = vm_object_allocate((vm_map_size_t)(entry->vme_end - entry->vme_start)); sp_object->phys_contiguous = TRUE; sp_object->vo_shadow_offset = (vm_object_offset_t)pages->phys_page*PAGE_SIZE; - entry->object.vm_object = sp_object; + VME_OBJECT_SET(entry, sp_object); assert(entry->use_pmap); /* enter the base pages into the object */ @@ -2262,7 +2611,7 @@ StartAgain: ; pmap_zero_page(m->phys_page); pages = NEXT_PAGE(m); *(NEXT_PAGE_PTR(m)) = VM_PAGE_NULL; - vm_page_insert(m, sp_object, offset); + vm_page_insert_wired(m, sp_object, offset, VM_KERN_MEMORY_OSFMK); } vm_object_unlock(sp_object); } @@ -2350,7 +2699,8 @@ BailOut: assert(!keep_map_locked); pmap_empty = FALSE; /* pmap won't be empty */ kr = vm_map_wire(map, start, end, - new_entry->protection, TRUE); + new_entry->protection | VM_PROT_MEMORY_TAG_MAKE(VM_KERN_MEMORY_MLOCK), + TRUE); result = kr; } @@ -2370,6 +2720,8 @@ BailOut: map->hdr.entries_pageable); vm_map_set_page_shift(zap_new_map, VM_MAP_PAGE_SHIFT(map)); + vm_map_disable_hole_optimization(zap_new_map); + if (!map_locked) { vm_map_lock(map); map_locked = TRUE; @@ -2461,6 +2813,7 @@ BailOut: #undef RETURN } + /* * Counters for the prefault optimization. */ @@ -2490,7 +2843,7 @@ vm_map_enter_mem_object_helper( kern_return_t result; boolean_t mask_cur_protection, mask_max_protection; boolean_t try_prefault = (page_list_count != 0); - vm_map_offset_t offset_in_mapping; + vm_map_offset_t offset_in_mapping = 0; mask_cur_protection = cur_protection & VM_PROT_IS_MASK; mask_max_protection = max_protection & VM_PROT_IS_MASK; @@ -2505,13 +2858,16 @@ vm_map_enter_mem_object_helper( (max_protection & ~VM_PROT_ALL) || (inheritance > VM_INHERIT_LAST_VALID) || (try_prefault && (copy || !page_list)) || - initial_size == 0) + initial_size == 0) { return KERN_INVALID_ARGUMENT; + } - map_addr = vm_map_trunc_page(*address, - VM_MAP_PAGE_MASK(target_map)); - map_size = vm_map_round_page(initial_size, - VM_MAP_PAGE_MASK(target_map)); + { + map_addr = vm_map_trunc_page(*address, + VM_MAP_PAGE_MASK(target_map)); + map_size = vm_map_round_page(initial_size, + VM_MAP_PAGE_MASK(target_map)); + } size = vm_object_round_page(initial_size); /* @@ -2526,7 +2882,8 @@ vm_map_enter_mem_object_helper( named_entry = (vm_named_entry_t) port->ip_kobject; - if ((flags & VM_FLAGS_RETURN_DATA_ADDR) != 0) { + if (flags & (VM_FLAGS_RETURN_DATA_ADDR | + VM_FLAGS_RETURN_4K_DATA_ADDR)) { offset += named_entry->data_offset; } @@ -2552,8 +2909,9 @@ vm_map_enter_mem_object_helper( /* overflow */ return KERN_INVALID_ARGUMENT; } - if (named_entry->size < (offset + size)) + if (named_entry->size < (offset + initial_size)) { return KERN_INVALID_ARGUMENT; + } if (named_entry->is_copy) { /* for a vm_map_copy, we can only map it whole */ @@ -2598,7 +2956,8 @@ vm_map_enter_mem_object_helper( if (named_entry->is_sub_map) { vm_map_t submap; - if ((flags & VM_FLAGS_RETURN_DATA_ADDR) != 0) { + if (flags & (VM_FLAGS_RETURN_DATA_ADDR | + VM_FLAGS_RETURN_4K_DATA_ADDR)) { panic("VM_FLAGS_RETURN_DATA_ADDR not expected for submap."); } @@ -2656,7 +3015,8 @@ vm_map_enter_mem_object_helper( protections = named_entry->protection & VM_PROT_ALL; access = GET_MAP_MEM(named_entry->protection); - if ((flags & VM_FLAGS_RETURN_DATA_ADDR) != 0) { + if (flags & (VM_FLAGS_RETURN_DATA_ADDR| + VM_FLAGS_RETURN_4K_DATA_ADDR)) { panic("VM_FLAGS_RETURN_DATA_ADDR not expected for submap."); } @@ -2738,13 +3098,17 @@ vm_map_enter_mem_object_helper( if (flags & ~(VM_FLAGS_FIXED | VM_FLAGS_ANYWHERE | VM_FLAGS_OVERWRITE | + VM_FLAGS_RETURN_4K_DATA_ADDR | VM_FLAGS_RETURN_DATA_ADDR)) { named_entry_unlock(named_entry); return KERN_INVALID_ARGUMENT; } - if ((flags & VM_FLAGS_RETURN_DATA_ADDR) != 0) { + if (flags & (VM_FLAGS_RETURN_DATA_ADDR | + VM_FLAGS_RETURN_4K_DATA_ADDR)) { offset_in_mapping = offset - vm_object_trunc_page(offset); + if (flags & VM_FLAGS_RETURN_4K_DATA_ADDR) + offset_in_mapping &= ~((signed)(0xFFF)); offset = vm_object_trunc_page(offset); map_size = vm_object_round_page(offset + offset_in_mapping + initial_size) - offset; } @@ -2769,6 +3133,7 @@ vm_map_enter_mem_object_helper( mask, flags & (VM_FLAGS_ANYWHERE | VM_FLAGS_OVERWRITE | + VM_FLAGS_RETURN_4K_DATA_ADDR | VM_FLAGS_RETURN_DATA_ADDR), VM_OBJECT_NULL, 0, @@ -2792,7 +3157,7 @@ vm_map_enter_mem_object_helper( vm_map_size_t copy_size; vm_object_offset_t copy_offset; - copy_offset = copy_entry->offset; + copy_offset = VME_OFFSET(copy_entry); copy_size = (copy_entry->vme_end - copy_entry->vme_start); @@ -2809,15 +3174,13 @@ vm_map_enter_mem_object_helper( /* take a reference on the object */ if (copy_entry->is_sub_map) { remap_flags |= VM_FLAGS_SUBMAP; - copy_submap = - copy_entry->object.sub_map; + copy_submap = VME_SUBMAP(copy_entry); vm_map_lock(copy_submap); vm_map_reference(copy_submap); vm_map_unlock(copy_submap); copy_object = (vm_object_t) copy_submap; } else { - copy_object = - copy_entry->object.vm_object; + copy_object = VME_OBJECT(copy_entry); vm_object_reference(copy_object); } @@ -2852,7 +3215,8 @@ vm_map_enter_mem_object_helper( } if (kr == KERN_SUCCESS) { - if ((flags & VM_FLAGS_RETURN_DATA_ADDR) != 0) { + if (flags & (VM_FLAGS_RETURN_DATA_ADDR | + VM_FLAGS_RETURN_4K_DATA_ADDR)) { *address = map_addr + offset_in_mapping; } else { *address = map_addr; @@ -2902,8 +3266,11 @@ vm_map_enter_mem_object_helper( /* object cannot be mapped until it is ready */ /* we can therefore avoid the ready check */ /* in this case. */ - if ((flags & VM_FLAGS_RETURN_DATA_ADDR) != 0) { + if (flags & (VM_FLAGS_RETURN_DATA_ADDR | + VM_FLAGS_RETURN_4K_DATA_ADDR)) { offset_in_mapping = offset - vm_object_trunc_page(offset); + if (flags & VM_FLAGS_RETURN_4K_DATA_ADDR) + offset_in_mapping &= ~((signed)(0xFFF)); offset = vm_object_trunc_page(offset); map_size = vm_object_round_page(offset + offset_in_mapping + initial_size) - offset; } @@ -2922,7 +3289,8 @@ vm_map_enter_mem_object_helper( * this case, the port isn't really a port at all, but * instead is just a raw memory object. */ - if ((flags & VM_FLAGS_RETURN_DATA_ADDR) != 0) { + if (flags & (VM_FLAGS_RETURN_DATA_ADDR | + VM_FLAGS_RETURN_4K_DATA_ADDR)) { panic("VM_FLAGS_RETURN_DATA_ADDR not expected for raw memory object."); } @@ -3006,7 +3374,8 @@ vm_map_enter_mem_object_helper( vm_object_t new_object; vm_object_offset_t new_offset; - result = vm_object_copy_strategically(object, offset, size, + result = vm_object_copy_strategically(object, offset, + map_size, &new_object, &new_offset, ©); @@ -3028,7 +3397,8 @@ vm_map_enter_mem_object_helper( new_object = object; new_offset = offset; success = vm_object_copy_quickly(&new_object, - new_offset, size, + new_offset, + map_size, &src_needs_copy, ©); assert(success); @@ -3041,8 +3411,9 @@ vm_map_enter_mem_object_helper( vm_object_deallocate(object); - if (result != KERN_SUCCESS) + if (result != KERN_SUCCESS) { return result; + } object = new_object; offset = new_offset; @@ -3054,13 +3425,17 @@ vm_map_enter_mem_object_helper( */ if (try_prefault) flags |= VM_FLAGS_KEEP_MAP_LOCKED; - result = vm_map_enter(target_map, - &map_addr, map_size, - (vm_map_offset_t)mask, - flags, - object, offset, - copy, - cur_protection, max_protection, inheritance); + + { + result = vm_map_enter(target_map, + &map_addr, map_size, + (vm_map_offset_t)mask, + flags, + object, offset, + copy, + cur_protection, max_protection, + inheritance); + } if (result != KERN_SUCCESS) vm_object_deallocate(object); @@ -3089,7 +3464,7 @@ vm_map_enter_mem_object_helper( 0, TRUE, PMAP_OPTIONS_NOWAIT, NULL); if (kr != KERN_SUCCESS) { OSIncrementAtomic64(&vm_prefault_nb_bailout); - goto BailOut; + break; } OSIncrementAtomic64(&vm_prefault_nb_pages); } @@ -3097,11 +3472,11 @@ vm_map_enter_mem_object_helper( /* Next virtual address */ va += PAGE_SIZE; } -BailOut: vm_map_unlock(target_map); } - if ((flags & VM_FLAGS_RETURN_DATA_ADDR) != 0) { + if (flags & (VM_FLAGS_RETURN_DATA_ADDR | + VM_FLAGS_RETURN_4K_DATA_ADDR)) { *address = map_addr + offset_in_mapping; } else { *address = map_addr; @@ -3178,14 +3553,17 @@ vm_map_enter_mem_object_control( (cur_protection & ~VM_PROT_ALL) || (max_protection & ~VM_PROT_ALL) || (inheritance > VM_INHERIT_LAST_VALID) || - initial_size == 0) + initial_size == 0) { return KERN_INVALID_ARGUMENT; + } - map_addr = vm_map_trunc_page(*address, - VM_MAP_PAGE_MASK(target_map)); - map_size = vm_map_round_page(initial_size, - VM_MAP_PAGE_MASK(target_map)); - size = vm_object_round_page(initial_size); + { + map_addr = vm_map_trunc_page(*address, + VM_MAP_PAGE_MASK(target_map)); + map_size = vm_map_round_page(initial_size, + VM_MAP_PAGE_MASK(target_map)); + } + size = vm_object_round_page(initial_size); object = memory_object_control_to_vm_object(control); @@ -3275,20 +3653,24 @@ vm_map_enter_mem_object_control( vm_object_deallocate(object); - if (result != KERN_SUCCESS) + if (result != KERN_SUCCESS) { return result; + } object = new_object; offset = new_offset; } - result = vm_map_enter(target_map, - &map_addr, map_size, - (vm_map_offset_t)mask, - flags, - object, offset, - copy, - cur_protection, max_protection, inheritance); + { + result = vm_map_enter(target_map, + &map_addr, map_size, + (vm_map_offset_t)mask, + flags, + object, offset, + copy, + cur_protection, max_protection, + inheritance); + } if (result != KERN_SUCCESS) vm_object_deallocate(object); *address = map_addr; @@ -3330,6 +3712,9 @@ vm_map_enter_cpm( #endif /* MACH_ASSERT */ boolean_t anywhere = ((VM_FLAGS_ANYWHERE & flags) != 0); + vm_tag_t tag; + + VM_GET_FLAGS_ALIAS(flags, tag); if (size == 0) { *addr = 0; @@ -3551,7 +3936,7 @@ vm_map_clip_unnest( vm_map_offset_t old_end_unnest = end_unnest; assert(entry->is_sub_map); - assert(entry->object.sub_map != NULL); + assert(VME_SUBMAP(entry) != NULL); assert(entry->use_pmap); /* @@ -3578,13 +3963,21 @@ vm_map_clip_unnest( _vm_map_clip_start(&map->hdr, entry, start_unnest); - vm_map_store_update_first_free(map, map->first_free); + if (map->holelistenabled) { + vm_map_store_update_first_free(map, NULL, FALSE); + } else { + vm_map_store_update_first_free(map, map->first_free, FALSE); + } } if (entry->vme_end > end_unnest) { _vm_map_clip_end(&map->hdr, entry, end_unnest); - vm_map_store_update_first_free(map, map->first_free); + if (map->holelistenabled) { + vm_map_store_update_first_free(map, NULL, FALSE); + } else { + vm_map_store_update_first_free(map, map->first_free, FALSE); + } } pmap_unnest(map->pmap, @@ -3595,12 +3988,13 @@ vm_map_clip_unnest( vm_map_submap_pmap_clean( map, entry->vme_start, entry->vme_end, - entry->object.sub_map, - entry->offset); + VME_SUBMAP(entry), + VME_OFFSET(entry)); } entry->use_pmap = FALSE; - if (entry->alias == VM_MEMORY_SHARED_PMAP) { - entry->alias = VM_MEMORY_UNSHARED_PMAP; + if ((map->pmap != kernel_pmap) && + (VME_ALIAS(entry) == VM_MEMORY_SHARED_PMAP)) { + VME_ALIAS_SET(entry, VM_MEMORY_UNSHARED_PMAP); } } #endif /* NO_NESTED_PMAP */ @@ -3637,15 +4031,19 @@ vm_map_clip_start( } #endif /* NO_NESTED_PMAP */ if (startaddr > entry->vme_start) { - if (entry->object.vm_object && + if (VME_OBJECT(entry) && !entry->is_sub_map && - entry->object.vm_object->phys_contiguous) { + VME_OBJECT(entry)->phys_contiguous) { pmap_remove(map->pmap, (addr64_t)(entry->vme_start), (addr64_t)(entry->vme_end)); } _vm_map_clip_start(&map->hdr, entry, startaddr); - vm_map_store_update_first_free(map, map->first_free); + if (map->holelistenabled) { + vm_map_store_update_first_free(map, NULL, FALSE); + } else { + vm_map_store_update_first_free(map, map->first_free, FALSE); + } } } @@ -3664,7 +4062,7 @@ static void _vm_map_clip_start( register struct vm_map_header *map_header, register vm_map_entry_t entry, - register vm_map_offset_t start) + register vm_map_offset_t start) { register vm_map_entry_t new_entry; @@ -3686,16 +4084,16 @@ _vm_map_clip_start( new_entry->vme_end = start; assert(new_entry->vme_start < new_entry->vme_end); - entry->offset += (start - entry->vme_start); + VME_OFFSET_SET(entry, VME_OFFSET(entry) + (start - entry->vme_start)); assert(start < entry->vme_end); entry->vme_start = start; _vm_map_store_entry_link(map_header, entry->vme_prev, new_entry); if (entry->is_sub_map) - vm_map_reference(new_entry->object.sub_map); + vm_map_reference(VME_SUBMAP(new_entry)); else - vm_object_reference(new_entry->object.vm_object); + vm_object_reference(VME_OBJECT(new_entry)); } @@ -3738,15 +4136,19 @@ vm_map_clip_end( } #endif /* NO_NESTED_PMAP */ if (endaddr < entry->vme_end) { - if (entry->object.vm_object && + if (VME_OBJECT(entry) && !entry->is_sub_map && - entry->object.vm_object->phys_contiguous) { + VME_OBJECT(entry)->phys_contiguous) { pmap_remove(map->pmap, (addr64_t)(entry->vme_start), (addr64_t)(entry->vme_end)); } _vm_map_clip_end(&map->hdr, entry, endaddr); - vm_map_store_update_first_free(map, map->first_free); + if (map->holelistenabled) { + vm_map_store_update_first_free(map, NULL, FALSE); + } else { + vm_map_store_update_first_free(map, map->first_free, FALSE); + } } } @@ -3784,15 +4186,16 @@ _vm_map_clip_end( assert(entry->vme_start < end); new_entry->vme_start = entry->vme_end = end; - new_entry->offset += (end - entry->vme_start); + VME_OFFSET_SET(new_entry, + VME_OFFSET(new_entry) + (end - entry->vme_start)); assert(new_entry->vme_start < new_entry->vme_end); _vm_map_store_entry_link(map_header, entry, new_entry); if (entry->is_sub_map) - vm_map_reference(new_entry->object.sub_map); + vm_map_reference(VME_SUBMAP(new_entry)); else - vm_object_reference(new_entry->object.vm_object); + vm_object_reference(VME_OBJECT(new_entry)); } @@ -3924,17 +4327,17 @@ vm_map_submap( if ((entry->vme_start == start) && (entry->vme_end == end) && (!entry->is_sub_map) && - ((object = entry->object.vm_object) == vm_submap_object) && + ((object = VME_OBJECT(entry)) == vm_submap_object) && (object->resident_page_count == 0) && (object->copy == VM_OBJECT_NULL) && (object->shadow == VM_OBJECT_NULL) && (!object->pager_created)) { - entry->offset = (vm_object_offset_t)offset; - entry->object.vm_object = VM_OBJECT_NULL; + VME_OFFSET_SET(entry, (vm_object_offset_t)offset); + VME_OBJECT_SET(entry, VM_OBJECT_NULL); vm_object_deallocate(object); entry->is_sub_map = TRUE; entry->use_pmap = FALSE; - entry->object.sub_map = submap; + VME_SUBMAP_SET(entry, submap); vm_map_reference(submap); if (submap->mapped_in_other_pmaps == FALSE && vm_map_pmap(submap) != PMAP_NULL && @@ -3963,7 +4366,7 @@ vm_map_submap( } } result = pmap_nest(map->pmap, - (entry->object.sub_map)->pmap, + (VME_SUBMAP(entry))->pmap, (addr64_t)start, (addr64_t)start, (uint64_t)(end - start)); @@ -3981,6 +4384,7 @@ vm_map_submap( return(result); } + /* * vm_map_protect: * @@ -4110,11 +4514,17 @@ vm_map_protect( /* for loss of shared memory communication in the */ /* target area after taking this step */ - if (current->is_sub_map == FALSE && current->object.vm_object == VM_OBJECT_NULL){ - current->object.vm_object = vm_object_allocate((vm_map_size_t)(current->vme_end - current->vme_start)); - current->offset = 0; + if (current->is_sub_map == FALSE && + VME_OBJECT(current) == VM_OBJECT_NULL) { + VME_OBJECT_SET(current, + vm_object_allocate( + (vm_map_size_t) + (current->vme_end - + current->vme_start))); + VME_OFFSET_SET(current, 0); assert(current->use_pmap); } + assert(current->wired_count == 0); current->needs_copy = TRUE; current->max_protection |= VM_PROT_WRITE; } @@ -4145,11 +4555,12 @@ vm_map_protect( prot = current->protection & ~VM_PROT_WRITE; - if (override_nx(map, current->alias) && prot) + if (override_nx(map, VME_ALIAS(current)) && prot) prot |= VM_PROT_EXECUTE; + if (current->is_sub_map && current->use_pmap) { - pmap_protect(current->object.sub_map->pmap, + pmap_protect(VME_SUBMAP(current)->pmap, current->vme_start, current->vme_end, prot); @@ -4369,13 +4780,14 @@ vm_map_wire_nested( register vm_map_t map, register vm_map_offset_t start, register vm_map_offset_t end, - register vm_prot_t access_type, + register vm_prot_t caller_prot, boolean_t user_wire, pmap_t map_pmap, vm_map_offset_t pmap_addr, ppnum_t *physpage_p) { register vm_map_entry_t entry; + register vm_prot_t access_type; struct vm_map_entry *first_entry, tmp_entry; vm_map_t real_map; register vm_map_offset_t s,e; @@ -4388,6 +4800,8 @@ vm_map_wire_nested( vm_map_size_t size; boolean_t wire_and_extract; + access_type = (caller_prot & VM_PROT_ALL); + wire_and_extract = FALSE; if (physpage_p != NULL) { /* @@ -4536,9 +4950,9 @@ vm_map_wire_nested( vm_map_clip_start(map, entry, s); vm_map_clip_end(map, entry, end); - sub_start = entry->offset; + sub_start = VME_OFFSET(entry); sub_end = entry->vme_end; - sub_end += entry->offset - entry->vme_start; + sub_end += VME_OFFSET(entry) - entry->vme_start; local_end = entry->vme_end; if(map_pmap == NULL) { @@ -4551,7 +4965,7 @@ vm_map_wire_nested( vm_map_t lookup_map; if(entry->use_pmap) { - pmap = entry->object.sub_map->pmap; + pmap = VME_SUBMAP(entry)->pmap; /* ppc implementation requires that */ /* submaps pmap address ranges line */ /* up with parent map */ @@ -4595,6 +5009,7 @@ vm_map_wire_nested( &real_map)) { vm_map_unlock_read(lookup_map); + assert(map_pmap == NULL); vm_map_unwire(map, start, s, user_wire); return(KERN_FAILURE); @@ -4642,9 +5057,9 @@ vm_map_wire_nested( entry->in_transition = TRUE; vm_map_unlock(map); - rc = vm_map_wire_nested(entry->object.sub_map, + rc = vm_map_wire_nested(VME_SUBMAP(entry), sub_start, sub_end, - access_type, + caller_prot, user_wire, pmap, pmap_addr, NULL); vm_map_lock(map); @@ -4735,18 +5150,18 @@ vm_map_wire_nested( == PAGE_SIZE); assert(!entry->needs_copy); assert(!entry->is_sub_map); - assert(entry->object.vm_object); + assert(VME_OBJECT(entry)); if (((entry->vme_end - entry->vme_start) != PAGE_SIZE) || entry->needs_copy || entry->is_sub_map || - entry->object.vm_object == VM_OBJECT_NULL) { + VME_OBJECT(entry) == VM_OBJECT_NULL) { rc = KERN_INVALID_ARGUMENT; goto done; } - object = entry->object.vm_object; - offset = entry->offset; + object = VME_OBJECT(entry); + offset = VME_OFFSET(entry); /* need exclusive lock to update m->dirty */ if (entry->protection & VM_PROT_WRITE) { vm_object_lock(object); @@ -4802,11 +5217,10 @@ vm_map_wire_nested( rc = KERN_INVALID_ARGUMENT; goto done; } - - vm_object_shadow(&entry->object.vm_object, - &entry->offset, size); + + VME_OBJECT_SHADOW(entry, size); entry->needs_copy = FALSE; - } else if (entry->object.vm_object == VM_OBJECT_NULL) { + } else if (VME_OBJECT(entry) == VM_OBJECT_NULL) { if (wire_and_extract) { /* * We're supposed to share with the original @@ -4815,8 +5229,8 @@ vm_map_wire_nested( rc = KERN_INVALID_ARGUMENT; goto done; } - entry->object.vm_object = vm_object_allocate(size); - entry->offset = (vm_object_offset_t)0; + VME_OBJECT_SET(entry, vm_object_allocate(size)); + VME_OFFSET_SET(entry, (vm_object_offset_t)0); assert(entry->use_pmap); } @@ -4880,11 +5294,11 @@ vm_map_wire_nested( if(map_pmap) rc = vm_fault_wire(map, - &tmp_entry, map_pmap, pmap_addr, + &tmp_entry, caller_prot, map_pmap, pmap_addr, physpage_p); else rc = vm_fault_wire(map, - &tmp_entry, map->pmap, + &tmp_entry, caller_prot, map->pmap, tmp_entry.vme_start, physpage_p); @@ -4944,7 +5358,8 @@ done: if (rc != KERN_SUCCESS) { /* undo what has been wired so far */ - vm_map_unwire(map, start, s, user_wire); + vm_map_unwire_nested(map, start, s, user_wire, + map_pmap, pmap_addr); if (physpage_p) { *physpage_p = 0; } @@ -4955,36 +5370,78 @@ done: } kern_return_t -vm_map_wire( +vm_map_wire_external( register vm_map_t map, register vm_map_offset_t start, register vm_map_offset_t end, - register vm_prot_t access_type, + register vm_prot_t caller_prot, boolean_t user_wire) { + kern_return_t kret; + caller_prot &= ~VM_PROT_MEMORY_TAG_MASK; + caller_prot |= VM_PROT_MEMORY_TAG_MAKE(vm_tag_bt()); + kret = vm_map_wire_nested(map, start, end, caller_prot, + user_wire, (pmap_t)NULL, 0, NULL); + return kret; +} + +kern_return_t +vm_map_wire( + register vm_map_t map, + register vm_map_offset_t start, + register vm_map_offset_t end, + register vm_prot_t caller_prot, + boolean_t user_wire) +{ kern_return_t kret; - kret = vm_map_wire_nested(map, start, end, access_type, + kret = vm_map_wire_nested(map, start, end, caller_prot, user_wire, (pmap_t)NULL, 0, NULL); return kret; } kern_return_t -vm_map_wire_and_extract( +vm_map_wire_and_extract_external( vm_map_t map, vm_map_offset_t start, - vm_prot_t access_type, + vm_prot_t caller_prot, boolean_t user_wire, ppnum_t *physpage_p) { + kern_return_t kret; + + caller_prot &= ~VM_PROT_MEMORY_TAG_MASK; + caller_prot |= VM_PROT_MEMORY_TAG_MAKE(vm_tag_bt()); + kret = vm_map_wire_nested(map, + start, + start+VM_MAP_PAGE_SIZE(map), + caller_prot, + user_wire, + (pmap_t)NULL, + 0, + physpage_p); + if (kret != KERN_SUCCESS && + physpage_p != NULL) { + *physpage_p = 0; + } + return kret; +} +kern_return_t +vm_map_wire_and_extract( + vm_map_t map, + vm_map_offset_t start, + vm_prot_t caller_prot, + boolean_t user_wire, + ppnum_t *physpage_p) +{ kern_return_t kret; kret = vm_map_wire_nested(map, start, start+VM_MAP_PAGE_SIZE(map), - access_type, + caller_prot, user_wire, (pmap_t)NULL, 0, @@ -5117,13 +5574,13 @@ vm_map_unwire_nested( vm_map_clip_start(map, entry, start); vm_map_clip_end(map, entry, end); - sub_start = entry->offset; + sub_start = VME_OFFSET(entry); sub_end = entry->vme_end - entry->vme_start; - sub_end += entry->offset; + sub_end += VME_OFFSET(entry); local_end = entry->vme_end; if(map_pmap == NULL) { if(entry->use_pmap) { - pmap = entry->object.sub_map->pmap; + pmap = VME_SUBMAP(entry)->pmap; pmap_addr = sub_start; } else { pmap = map->pmap; @@ -5169,7 +5626,7 @@ vm_map_unwire_nested( * guarantees existance of the entry. */ vm_map_unlock(map); - vm_map_unwire_nested(entry->object.sub_map, + vm_map_unwire_nested(VME_SUBMAP(entry), sub_start, sub_end, user_wire, pmap, pmap_addr); vm_map_lock(map); @@ -5207,7 +5664,7 @@ vm_map_unwire_nested( continue; } else { vm_map_unlock(map); - vm_map_unwire_nested(entry->object.sub_map, + vm_map_unwire_nested(VME_SUBMAP(entry), sub_start, sub_end, user_wire, map_pmap, pmap_addr); vm_map_lock(map); @@ -5382,10 +5839,10 @@ vm_map_entry_delete( if (entry->is_sub_map) { object = NULL; - submap = entry->object.sub_map; + submap = VME_SUBMAP(entry); } else { submap = NULL; - object = entry->object.vm_object; + object = VME_OBJECT(entry); } vm_map_store_entry_unlink(map, entry); @@ -5438,19 +5895,22 @@ vm_map_submap_pmap_clean( sub_map, start, start + remove_size, - entry->object.sub_map, - entry->offset); + VME_SUBMAP(entry), + VME_OFFSET(entry)); } else { if((map->mapped_in_other_pmaps) && (map->ref_count) - && (entry->object.vm_object != NULL)) { - vm_object_pmap_protect( - entry->object.vm_object, - entry->offset+(offset-entry->vme_start), + && (VME_OBJECT(entry) != NULL)) { + vm_object_pmap_protect_options( + VME_OBJECT(entry), + (VME_OFFSET(entry) + + offset - + entry->vme_start), remove_size, PMAP_NULL, entry->vme_start, - VM_PROT_NONE); + VM_PROT_NONE, + PMAP_OPTIONS_REMOVE); } else { pmap_remove(map->pmap, (addr64_t)start, @@ -5472,18 +5932,19 @@ vm_map_submap_pmap_clean( sub_map, (start + entry->vme_start) - offset, ((start + entry->vme_start) - offset) + remove_size, - entry->object.sub_map, - entry->offset); + VME_SUBMAP(entry), + VME_OFFSET(entry)); } else { if((map->mapped_in_other_pmaps) && (map->ref_count) - && (entry->object.vm_object != NULL)) { - vm_object_pmap_protect( - entry->object.vm_object, - entry->offset, + && (VME_OBJECT(entry) != NULL)) { + vm_object_pmap_protect_options( + VME_OBJECT(entry), + VME_OFFSET(entry), remove_size, PMAP_NULL, entry->vme_start, - VM_PROT_NONE); + VM_PROT_NONE, + PMAP_OPTIONS_REMOVE); } else { pmap_remove(map->pmap, (addr64_t)((start + entry->vme_start) @@ -5735,8 +6196,6 @@ vm_map_delete( * may not exist anymore. Look it up again. */ if (!vm_map_lookup_entry(map, s, &first_entry)) { - assert((map != kernel_map) && - (!entry->is_sub_map)); /* * User: use the next entry */ @@ -5843,8 +6302,8 @@ vm_map_delete( vm_map_offset_t pmap_addr; - sub_map = tmp_entry.object.sub_map; - sub_start = tmp_entry.offset; + sub_map = VME_SUBMAP(&tmp_entry); + sub_start = VME_OFFSET(&tmp_entry); sub_end = sub_start + (tmp_entry.vme_end - tmp_entry.vme_start); if (tmp_entry.use_pmap) { @@ -5860,7 +6319,7 @@ vm_map_delete( pmap, pmap_addr); } else { - if (tmp_entry.object.vm_object == kernel_object) { + if (VME_OBJECT(&tmp_entry) == kernel_object) { pmap_protect_options( map->pmap, tmp_entry.vme_start, @@ -5870,7 +6329,7 @@ vm_map_delete( NULL); } vm_fault_unwire(map, &tmp_entry, - tmp_entry.object.vm_object == kernel_object, + VME_OBJECT(&tmp_entry) == kernel_object, map->pmap, tmp_entry.vme_start); } @@ -5935,37 +6394,61 @@ vm_map_delete( } else if (entry->is_sub_map) { if (entry->use_pmap) { #ifndef NO_NESTED_PMAP - pmap_unnest(map->pmap, - (addr64_t)entry->vme_start, - entry->vme_end - entry->vme_start); + int pmap_flags; + + if (flags & VM_MAP_REMOVE_NO_UNNESTING) { + /* + * This is the final cleanup of the + * address space being terminated. + * No new mappings are expected and + * we don't really need to unnest the + * shared region (and lose the "global" + * pmap mappings, if applicable). + * + * Tell the pmap layer that we're + * "clean" wrt nesting. + */ + pmap_flags = PMAP_UNNEST_CLEAN; + } else { + /* + * We're unmapping part of the nested + * shared region, so we can't keep the + * nested pmap. + */ + pmap_flags = 0; + } + pmap_unnest_options( + map->pmap, + (addr64_t)entry->vme_start, + entry->vme_end - entry->vme_start, + pmap_flags); #endif /* NO_NESTED_PMAP */ if ((map->mapped_in_other_pmaps) && (map->ref_count)) { /* clean up parent map/maps */ vm_map_submap_pmap_clean( map, entry->vme_start, entry->vme_end, - entry->object.sub_map, - entry->offset); + VME_SUBMAP(entry), + VME_OFFSET(entry)); } } else { vm_map_submap_pmap_clean( map, entry->vme_start, entry->vme_end, - entry->object.sub_map, - entry->offset); + VME_SUBMAP(entry), + VME_OFFSET(entry)); } - } else if (entry->object.vm_object != kernel_object && - entry->object.vm_object != compressor_object) { - object = entry->object.vm_object; + } else if (VME_OBJECT(entry) != kernel_object && + VME_OBJECT(entry) != compressor_object) { + object = VME_OBJECT(entry); if ((map->mapped_in_other_pmaps) && (map->ref_count)) { vm_object_pmap_protect_options( - object, entry->offset, + object, VME_OFFSET(entry), entry->vme_end - entry->vme_start, PMAP_NULL, entry->vme_start, VM_PROT_NONE, PMAP_OPTIONS_REMOVE); - } else if ((entry->object.vm_object != - VM_OBJECT_NULL) || + } else if ((VME_OBJECT(entry) != VM_OBJECT_NULL) || (map->pmap == kernel_pmap)) { /* Remove translations associated * with this range unless the entry @@ -5987,6 +6470,11 @@ vm_map_delete( if (entry->iokit_acct) { /* alternate accounting */ + DTRACE_VM4(vm_map_iokit_unmapped_region, + vm_map_t, map, + vm_map_offset_t, entry->vme_start, + vm_map_offset_t, entry->vme_end, + int, VME_ALIAS(entry)); vm_map_iokit_unmapped_region(map, (entry->vme_end - entry->vme_start)); @@ -6144,9 +6632,9 @@ vm_map_copy_discard( vm_map_copy_entry_unlink(copy, entry); if (entry->is_sub_map) { - vm_map_deallocate(entry->object.sub_map); + vm_map_deallocate(VME_SUBMAP(entry)); } else { - vm_object_deallocate(entry->object.vm_object); + vm_object_deallocate(VME_OBJECT(entry)); } vm_map_copy_entry_dispose(copy, entry); } @@ -6161,7 +6649,10 @@ vm_map_copy_discard( * allocated by a single call to kalloc(), i.e. the * vm_map_copy_t was not allocated out of the zone. */ - kfree(copy, copy->cpy_kalloc_size); + if (copy->size > msg_ool_size_small || copy->offset) + panic("Invalid vm_map_copy_t sz:%lld, ofst:%lld", + (long long)copy->size, (long long)copy->offset); + kfree(copy, copy->size + cpy_kdata_hdr_sz); return; } zfree(vm_map_copy_zone, copy); @@ -6286,19 +6777,19 @@ start_pass_1: } encountered_sub_map = TRUE; - sub_start = entry->offset; + sub_start = VME_OFFSET(entry); if(entry->vme_end < dst_end) sub_end = entry->vme_end; else sub_end = dst_end; sub_end -= entry->vme_start; - sub_end += entry->offset; + sub_end += VME_OFFSET(entry); local_end = entry->vme_end; vm_map_unlock(dst_map); result = vm_map_overwrite_submap_recurse( - entry->object.sub_map, + VME_SUBMAP(entry), sub_start, sub_end - sub_start); @@ -6356,9 +6847,9 @@ start_pass_1: /* * Check for permanent objects in the destination. */ - if ((entry->object.vm_object != VM_OBJECT_NULL) && - ((!entry->object.vm_object->internal) || - (entry->object.vm_object->true_share))) { + if ((VME_OBJECT(entry) != VM_OBJECT_NULL) && + ((!VME_OBJECT(entry)->internal) || + (VME_OBJECT(entry)->true_share))) { if(encountered_sub_map) { vm_map_unlock(dst_map); return(KERN_FAILURE); @@ -6542,18 +7033,18 @@ start_pass_1: /* there is no need for the follow- */ /* ing check. */ encountered_sub_map = TRUE; - sub_start = entry->offset; + sub_start = VME_OFFSET(entry); if(entry->vme_end < dst_end) sub_end = entry->vme_end; else sub_end = dst_end; sub_end -= entry->vme_start; - sub_end += entry->offset; + sub_end += VME_OFFSET(entry); vm_map_unlock(dst_map); kr = vm_map_overwrite_submap_recurse( - entry->object.sub_map, + VME_SUBMAP(entry), sub_start, sub_end - sub_start); if(kr != KERN_SUCCESS) @@ -6610,9 +7101,9 @@ start_pass_1: /* * Check for permanent objects in the destination. */ - if ((entry->object.vm_object != VM_OBJECT_NULL) && - ((!entry->object.vm_object->internal) || - (entry->object.vm_object->true_share))) { + if ((VME_OBJECT(entry) != VM_OBJECT_NULL) && + ((!VME_OBJECT(entry)->internal) || + (VME_OBJECT(entry)->true_share))) { contains_permanent_objects = TRUE; } @@ -6717,11 +7208,11 @@ start_overwrite: assert(!entry->use_pmap); entry->is_sub_map = FALSE; vm_map_deallocate( - entry->object.sub_map); - entry->object.sub_map = NULL; + VME_SUBMAP(entry)); + VME_SUBMAP_SET(entry, NULL); entry->is_shared = FALSE; entry->needs_copy = FALSE; - entry->offset = 0; + VME_OFFSET_SET(entry, 0); /* * XXX FBDP * We should propagate the protections @@ -6747,14 +7238,14 @@ start_overwrite: entry->vme_start - base_addr; break; } - sub_start = entry->offset; + sub_start = VME_OFFSET(entry); if(entry->vme_end < dst_end) sub_end = entry->vme_end; else sub_end = dst_end; sub_end -= entry->vme_start; - sub_end += entry->offset; + sub_end += VME_OFFSET(entry); local_end = entry->vme_end; vm_map_unlock(dst_map); copy_size = sub_end - sub_start; @@ -6807,22 +7298,22 @@ start_overwrite: if((entry->use_pmap) && (pmap == NULL)) { kr = vm_map_copy_overwrite_nested( - entry->object.sub_map, + VME_SUBMAP(entry), sub_start, copy, interruptible, - entry->object.sub_map->pmap, + VME_SUBMAP(entry)->pmap, TRUE); } else if (pmap != NULL) { kr = vm_map_copy_overwrite_nested( - entry->object.sub_map, + VME_SUBMAP(entry), sub_start, copy, interruptible, pmap, TRUE); } else { kr = vm_map_copy_overwrite_nested( - entry->object.sub_map, + VME_SUBMAP(entry), sub_start, copy, interruptible, @@ -7387,14 +7878,13 @@ vm_map_copy_overwrite_unaligned( vm_map_lock_read(dst_map); goto RetryLookup; } - vm_object_shadow(&entry->object.vm_object, - &entry->offset, - (vm_map_size_t)(entry->vme_end - - entry->vme_start)); + VME_OBJECT_SHADOW(entry, + (vm_map_size_t)(entry->vme_end + - entry->vme_start)); entry->needs_copy = FALSE; vm_map_lock_write_to_read(dst_map); } - dst_object = entry->object.vm_object; + dst_object = VME_OBJECT(entry); /* * unlike with the virtual (aligned) copy we're going * to fault on it therefore we need a target object. @@ -7406,8 +7896,8 @@ vm_map_copy_overwrite_unaligned( } dst_object = vm_object_allocate((vm_map_size_t) entry->vme_end - entry->vme_start); - entry->object.vm_object = dst_object; - entry->offset = 0; + VME_OBJECT(entry) = dst_object; + VME_OFFSET_SET(entry, 0); assert(entry->use_pmap); vm_map_lock_write_to_read(dst_map); } @@ -7417,15 +7907,15 @@ vm_map_copy_overwrite_unaligned( */ vm_object_reference(dst_object); version.main_timestamp = dst_map->timestamp; - entry_offset = entry->offset; + entry_offset = VME_OFFSET(entry); entry_end = entry->vme_end; vm_map_unlock_read(dst_map); /* * Copy as much as possible in one pass */ kr = vm_fault_copy( - copy_entry->object.vm_object, - copy_entry->offset + src_offset, + VME_OBJECT(copy_entry), + VME_OFFSET(copy_entry) + src_offset, ©_size, dst_object, entry_offset + dst_offset, @@ -7457,8 +7947,7 @@ vm_map_copy_overwrite_unaligned( if (discard_on_success) { vm_map_copy_entry_unlink(copy, copy_entry); assert(!copy_entry->is_sub_map); - vm_object_deallocate( - copy_entry->object.vm_object); + vm_object_deallocate(VME_OBJECT(copy_entry)); vm_map_copy_entry_dispose(copy, copy_entry); } @@ -7624,21 +8113,21 @@ vm_map_copy_overwrite_aligned( * installing the source data. */ - object = entry->object.vm_object; + object = VME_OBJECT(entry); if ((!entry->is_shared && ((object == VM_OBJECT_NULL) || (object->internal && !object->true_share))) || entry->needs_copy) { - vm_object_t old_object = entry->object.vm_object; - vm_object_offset_t old_offset = entry->offset; + vm_object_t old_object = VME_OBJECT(entry); + vm_object_offset_t old_offset = VME_OFFSET(entry); vm_object_offset_t offset; /* * Ensure that the source and destination aren't * identical */ - if (old_object == copy_entry->object.vm_object && - old_offset == copy_entry->offset) { + if (old_object == VME_OBJECT(copy_entry) && + old_offset == VME_OFFSET(copy_entry)) { vm_map_copy_entry_unlink(copy, copy_entry); vm_map_copy_entry_dispose(copy, copy_entry); @@ -7652,8 +8141,8 @@ vm_map_copy_overwrite_aligned( #define __TRADEOFF1_OBJ_SIZE (64 * 1024 * 1024) /* 64 MB */ #define __TRADEOFF1_COPY_SIZE (128 * 1024) /* 128 KB */ - if (copy_entry->object.vm_object != VM_OBJECT_NULL && - copy_entry->object.vm_object->vo_size >= __TRADEOFF1_OBJ_SIZE && + if (VME_OBJECT(copy_entry) != VM_OBJECT_NULL && + VME_OBJECT(copy_entry)->vo_size >= __TRADEOFF1_OBJ_SIZE && copy_size <= __TRADEOFF1_COPY_SIZE) { /* * Virtual vs. Physical copy tradeoff #1. @@ -7668,15 +8157,16 @@ vm_map_copy_overwrite_aligned( goto slow_copy; } - if (entry->alias >= VM_MEMORY_MALLOC && - entry->alias <= VM_MEMORY_MALLOC_LARGE_REUSED) { + if ((dst_map->pmap != kernel_pmap) && + (VME_ALIAS(entry) >= VM_MEMORY_MALLOC) && + (VME_ALIAS(entry) <= VM_MEMORY_MALLOC_LARGE_REUSED)) { vm_object_t new_object, new_shadow; /* * We're about to map something over a mapping * established by malloc()... */ - new_object = copy_entry->object.vm_object; + new_object = VME_OBJECT(copy_entry); if (new_object != VM_OBJECT_NULL) { vm_object_lock_shared(new_object); } @@ -7743,23 +8233,22 @@ vm_map_copy_overwrite_aligned( vm_map_submap_pmap_clean( dst_map, entry->vme_start, entry->vme_end, - entry->object.sub_map, - entry->offset); + VME_SUBMAP(entry), + VME_OFFSET(entry)); } } else { vm_map_submap_pmap_clean( dst_map, entry->vme_start, entry->vme_end, - entry->object.sub_map, - entry->offset); + VME_SUBMAP(entry), + VME_OFFSET(entry)); } - vm_map_deallocate( - entry->object.sub_map); + vm_map_deallocate(VME_SUBMAP(entry)); } else { if(dst_map->mapped_in_other_pmaps) { vm_object_pmap_protect_options( - entry->object.vm_object, - entry->offset, + VME_OBJECT(entry), + VME_OFFSET(entry), entry->vme_end - entry->vme_start, PMAP_NULL, @@ -7778,12 +8267,13 @@ vm_map_copy_overwrite_aligned( } entry->is_sub_map = FALSE; - entry->object = copy_entry->object; - object = entry->object.vm_object; + VME_OBJECT_SET(entry, VME_OBJECT(copy_entry)); + object = VME_OBJECT(entry); entry->needs_copy = copy_entry->needs_copy; entry->wired_count = 0; entry->user_wired_count = 0; - offset = entry->offset = copy_entry->offset; + offset = VME_OFFSET(copy_entry); + VME_OFFSET_SET(entry, offset); vm_map_copy_entry_unlink(copy, copy_entry); vm_map_copy_entry_dispose(copy, copy_entry); @@ -7815,15 +8305,14 @@ vm_map_copy_overwrite_aligned( slow_copy: if (entry->needs_copy) { - vm_object_shadow(&entry->object.vm_object, - &entry->offset, - (entry->vme_end - - entry->vme_start)); + VME_OBJECT_SHADOW(entry, + (entry->vme_end - + entry->vme_start)); entry->needs_copy = FALSE; } - dst_object = entry->object.vm_object; - dst_offset = entry->offset; + dst_object = VME_OBJECT(entry); + dst_offset = VME_OFFSET(entry); /* * Take an object reference, and record @@ -7847,8 +8336,8 @@ vm_map_copy_overwrite_aligned( dst_object = vm_object_allocate( entry->vme_end - entry->vme_start); dst_offset = 0; - entry->object.vm_object = dst_object; - entry->offset = dst_offset; + VME_OBJECT_SET(entry, dst_object); + VME_OFFSET_SET(entry, dst_offset); assert(entry->use_pmap); } @@ -7866,8 +8355,8 @@ vm_map_copy_overwrite_aligned( copy_size = size; r = vm_fault_copy( - copy_entry->object.vm_object, - copy_entry->offset, + VME_OBJECT(copy_entry), + VME_OFFSET(copy_entry), ©_size, dst_object, dst_offset, @@ -7896,7 +8385,7 @@ vm_map_copy_overwrite_aligned( vm_map_copy_clip_end(copy, copy_entry, copy_entry->vme_start + copy_size); vm_map_copy_entry_unlink(copy, copy_entry); - vm_object_deallocate(copy_entry->object.vm_object); + vm_object_deallocate(VME_OBJECT(copy_entry)); vm_map_copy_entry_dispose(copy, copy_entry); } @@ -7966,24 +8455,19 @@ vm_map_copyin_kernel_buffer( vm_map_copy_t copy; vm_size_t kalloc_size; - if ((vm_size_t) len != len) { - /* "len" is too big and doesn't fit in a "vm_size_t" */ - return KERN_RESOURCE_SHORTAGE; - } - kalloc_size = (vm_size_t) (sizeof(struct vm_map_copy) + len); - assert((vm_map_size_t) kalloc_size == sizeof (struct vm_map_copy) + len); + if (len > msg_ool_size_small) + return KERN_INVALID_ARGUMENT; - copy = (vm_map_copy_t) kalloc(kalloc_size); - if (copy == VM_MAP_COPY_NULL) { + kalloc_size = (vm_size_t)(cpy_kdata_hdr_sz + len); + + copy = (vm_map_copy_t)kalloc(kalloc_size); + if (copy == VM_MAP_COPY_NULL) return KERN_RESOURCE_SHORTAGE; - } copy->type = VM_MAP_COPY_KERNEL_BUFFER; copy->size = len; copy->offset = 0; - copy->cpy_kdata = (void *) (copy + 1); - copy->cpy_kalloc_size = kalloc_size; - kr = copyinmap(src_map, src_addr, copy->cpy_kdata, (vm_size_t) len); + kr = copyinmap(src_map, src_addr, copy->cpy_kdata, (vm_size_t)len); if (kr != KERN_SUCCESS) { kfree(copy, kalloc_size); return kr; @@ -8026,6 +8510,13 @@ vm_map_copyout_kernel_buffer( kern_return_t kr = KERN_SUCCESS; thread_t thread = current_thread(); + /* + * check for corrupted vm_map_copy structure + */ + if (copy->size > msg_ool_size_small || copy->offset) + panic("Invalid vm_map_copy_t sz:%lld, ofst:%lld", + (long long)copy->size, (long long)copy->offset); + if (!overwrite) { /* @@ -8103,7 +8594,7 @@ vm_map_copyout_kernel_buffer( } else { /* copy was successful, dicard the copy structure */ if (consume_on_success) { - kfree(copy, copy->cpy_kalloc_size); + kfree(copy, copy->size + cpy_kdata_hdr_sz); } } @@ -8158,11 +8649,11 @@ vm_map_copy_remap( /* take an extra reference on the entry's "object" */ if (new_entry->is_sub_map) { assert(!new_entry->use_pmap); /* not nested */ - vm_map_lock(new_entry->object.sub_map); - vm_map_reference(new_entry->object.sub_map); - vm_map_unlock(new_entry->object.sub_map); + vm_map_lock(VME_SUBMAP(new_entry)); + vm_map_reference(VME_SUBMAP(new_entry)); + vm_map_unlock(VME_SUBMAP(new_entry)); } else { - vm_object_reference(new_entry->object.vm_object); + vm_object_reference(VME_OBJECT(new_entry)); } /* insert the new entry in the map */ vm_map_store_entry_link(map, where, new_entry); @@ -8171,6 +8662,39 @@ vm_map_copy_remap( } } + +boolean_t +vm_map_copy_validate_size( + vm_map_t dst_map, + vm_map_copy_t copy, + vm_map_size_t size) +{ + if (copy == VM_MAP_COPY_NULL) + return FALSE; + switch (copy->type) { + case VM_MAP_COPY_OBJECT: + case VM_MAP_COPY_KERNEL_BUFFER: + if (size == copy->size) + return TRUE; + break; + case VM_MAP_COPY_ENTRY_LIST: + /* + * potential page-size rounding prevents us from exactly + * validating this flavor of vm_map_copy, but we can at least + * assert that it's within a range. + */ + if (copy->size >= size && + copy->size <= vm_map_round_page(size, + VM_MAP_PAGE_MASK(dst_map))) + return TRUE; + break; + default: + break; + } + return FALSE; +} + + /* * Routine: vm_map_copyout * @@ -8211,6 +8735,7 @@ vm_map_copyout_internal( vm_object_offset_t vm_copy_start; vm_map_entry_t last; vm_map_entry_t entry; + vm_map_entry_t hole_entry; /* * Check for null copy object. @@ -8281,9 +8806,24 @@ StartAgain: ; VM_MAP_HIGHEST_ENTRY(dst_map, entry, start); last = entry; } else { - assert(first_free_is_valid(dst_map)); - start = ((last = dst_map->first_free) == vm_map_to_entry(dst_map)) ? - vm_map_min(dst_map) : last->vme_end; + if (dst_map->holelistenabled) { + hole_entry = (vm_map_entry_t)dst_map->holes_list; + + if (hole_entry == NULL) { + /* + * No more space in the map? + */ + vm_map_unlock(dst_map); + return(KERN_NO_SPACE); + } + + last = hole_entry; + start = last->vme_start; + } else { + assert(first_free_is_valid(dst_map)); + start = ((last = dst_map->first_free) == vm_map_to_entry(dst_map)) ? + vm_map_min(dst_map) : last->vme_end; + } start = vm_map_round_page(start, VM_MAP_PAGE_MASK(dst_map)); } @@ -8306,16 +8846,51 @@ StartAgain: ; return(KERN_NO_SPACE); } - if ((next == vm_map_to_entry(dst_map)) || - (next->vme_start >= end)) - break; + if (dst_map->holelistenabled) { + if (last->vme_end >= end) + break; + } else { + /* + * If there are no more entries, we must win. + * + * OR + * + * If there is another entry, it must be + * after the end of the potential new region. + */ + + if (next == vm_map_to_entry(dst_map)) + break; + + if (next->vme_start >= end) + break; + } last = next; - start = last->vme_end; + + if (dst_map->holelistenabled) { + if (last == (vm_map_entry_t) dst_map->holes_list) { + /* + * Wrapped around + */ + vm_map_unlock(dst_map); + return(KERN_NO_SPACE); + } + start = last->vme_start; + } else { + start = last->vme_end; + } start = vm_map_round_page(start, VM_MAP_PAGE_MASK(dst_map)); } + if (dst_map->holelistenabled) { + if (vm_map_lookup_entry(dst_map, last->vme_start, &last)) { + panic("Found an existing entry (%p) instead of potential hole at address: 0x%llx.\n", last, (unsigned long long)last->vme_start); + } + } + + adjustment = start - vm_copy_start; if (! consume_on_success) { /* @@ -8423,8 +8998,8 @@ StartAgain: ; vm_prot_t prot; int type_of_fault; - object = entry->object.vm_object; - offset = entry->offset; + object = VME_OBJECT(entry); + offset = VME_OFFSET(entry); va = entry->vme_start; pmap_pageable(dst_map->pmap, @@ -8471,14 +9046,15 @@ StartAgain: ; prot = entry->protection; - if (override_nx(dst_map, entry->alias) && prot) + if (override_nx(dst_map, VME_ALIAS(entry)) && + prot) prot |= VM_PROT_EXECUTE; type_of_fault = DBG_CACHE_HIT_FAULT; vm_fault_enter(m, dst_map->pmap, va, prot, prot, VM_PAGE_WIRED(m), FALSE, FALSE, - FALSE, entry->alias, + FALSE, VME_ALIAS(entry), ((entry->iokit_acct || (!entry->is_sub_map && !entry->use_pmap)) @@ -8593,13 +9169,35 @@ vm_map_copyin_common( __unused boolean_t src_volatile, vm_map_copy_t *copy_result, /* OUT */ boolean_t use_maxprot) +{ + int flags; + + flags = 0; + if (src_destroy) { + flags |= VM_MAP_COPYIN_SRC_DESTROY; + } + if (use_maxprot) { + flags |= VM_MAP_COPYIN_USE_MAXPROT; + } + return vm_map_copyin_internal(src_map, + src_addr, + len, + flags, + copy_result); +} +kern_return_t +vm_map_copyin_internal( + vm_map_t src_map, + vm_map_address_t src_addr, + vm_map_size_t len, + int flags, + vm_map_copy_t *copy_result) /* OUT */ { vm_map_entry_t tmp_entry; /* Result of last map lookup -- * in multi-level lookup, this * entry contains the actual * vm_object/offset. */ - register vm_map_entry_t new_entry = VM_MAP_ENTRY_NULL; /* Map entry for copy */ vm_map_offset_t src_start; /* Start of current entry -- @@ -8612,10 +9210,18 @@ vm_map_copyin_common( boolean_t map_share=FALSE; submap_map_t *parent_maps = NULL; - register vm_map_copy_t copy; /* Resulting copy */ vm_map_address_t copy_addr; vm_map_size_t copy_size; + boolean_t src_destroy; + boolean_t use_maxprot; + + if (flags & ~VM_MAP_COPYIN_ALL_FLAGS) { + return KERN_INVALID_ARGUMENT; + } + + src_destroy = (flags & VM_MAP_COPYIN_SRC_DESTROY) ? TRUE : FALSE; + use_maxprot = (flags & VM_MAP_COPYIN_USE_MAXPROT) ? TRUE : FALSE; /* * Check for copies of zero bytes. @@ -8639,7 +9245,9 @@ vm_map_copyin_common( * setting up VM (and taking C-O-W faults) dominates the copy costs * for small regions. */ - if ((len < msg_ool_size_small) && !use_maxprot) + if ((len < msg_ool_size_small) && + !use_maxprot && + !(flags & VM_MAP_COPYIN_ENTRY_LIST)) return vm_map_copyin_kernel_buffer(src_map, src_addr, len, src_destroy, copy_result); @@ -8785,9 +9393,9 @@ vm_map_copyin_common( ptr->base_len = submap_len; src_start -= tmp_entry->vme_start; - src_start += tmp_entry->offset; + src_start += VME_OFFSET(tmp_entry); src_end = src_start + submap_len; - src_map = tmp_entry->object.sub_map; + src_map = VME_SUBMAP(tmp_entry); vm_map_lock(src_map); /* keep an outstanding reference for all maps in */ /* the parents tree except the base map */ @@ -8803,8 +9411,8 @@ vm_map_copyin_common( } /* we are now in the lowest level submap... */ - if ((tmp_entry->object.vm_object != VM_OBJECT_NULL) && - (tmp_entry->object.vm_object->phys_contiguous)) { + if ((VME_OBJECT(tmp_entry) != VM_OBJECT_NULL) && + (VME_OBJECT(tmp_entry)->phys_contiguous)) { /* This is not, supported for now.In future */ /* we will need to detect the phys_contig */ /* condition and then upgrade copy_slowly */ @@ -8853,8 +9461,8 @@ vm_map_copyin_common( vm_map_clip_end(src_map, src_entry, src_end); src_size = src_entry->vme_end - src_start; - src_object = src_entry->object.vm_object; - src_offset = src_entry->offset; + src_object = VME_OBJECT(src_entry); + src_offset = VME_OFFSET(src_entry); was_wired = (src_entry->wired_count != 0); vm_map_entry_copy(new_entry, src_entry); @@ -8892,12 +9500,12 @@ vm_map_copyin_common( RestartCopy: XPR(XPR_VM_MAP, "vm_map_copyin_common src_obj 0x%x ent 0x%x obj 0x%x was_wired %d\n", - src_object, new_entry, new_entry->object.vm_object, + src_object, new_entry, VME_OBJECT(new_entry), was_wired, 0); if ((src_object == VM_OBJECT_NULL || (!was_wired && !map_share && !tmp_entry->is_shared)) && vm_object_copy_quickly( - &new_entry->object.vm_object, + &VME_OBJECT(new_entry), src_offset, src_size, &src_needs_copy, @@ -8914,7 +9522,8 @@ vm_map_copyin_common( prot = src_entry->protection & ~VM_PROT_WRITE; - if (override_nx(src_map, src_entry->alias) && prot) + if (override_nx(src_map, VME_ALIAS(src_entry)) + && prot) prot |= VM_PROT_EXECUTE; vm_object_pmap_protect( @@ -8927,6 +9536,7 @@ vm_map_copyin_common( src_entry->vme_start, prot); + assert(tmp_entry->wired_count == 0); tmp_entry->needs_copy = TRUE; } @@ -8967,8 +9577,8 @@ vm_map_copyin_common( src_offset, src_size, THREAD_UNINT, - &new_entry->object.vm_object); - new_entry->offset = 0; + &VME_OBJECT(new_entry)); + VME_OFFSET_SET(new_entry, 0); new_entry->needs_copy = FALSE; } @@ -8985,7 +9595,8 @@ vm_map_copyin_common( if (new_object == VM_OBJECT_NULL) goto CopySlowly; - new_entry->object.vm_object = new_object; + VME_OBJECT_SET(new_entry, new_object); + assert(new_entry->wired_count == 0); new_entry->needs_copy = TRUE; assert(!new_entry->iokit_acct); assert(new_object->purgable == VM_PURGABLE_DENY); @@ -8993,12 +9604,17 @@ vm_map_copyin_common( result = KERN_SUCCESS; } else { + vm_object_offset_t new_offset; + new_offset = VME_OFFSET(new_entry); result = vm_object_copy_strategically(src_object, src_offset, src_size, - &new_entry->object.vm_object, - &new_entry->offset, + &VME_OBJECT(new_entry), + &new_offset, &new_entry_needs_copy); + if (new_offset != VME_OFFSET(new_entry)) { + VME_OFFSET_SET(new_entry, new_offset); + } new_entry->needs_copy = new_entry_needs_copy; } @@ -9039,8 +9655,8 @@ vm_map_copyin_common( if (!vm_map_lookup_entry(src_map, src_start, &tmp_entry)) { if (result != KERN_MEMORY_RESTART_COPY) { - vm_object_deallocate(new_entry->object.vm_object); - new_entry->object.vm_object = VM_OBJECT_NULL; + vm_object_deallocate(VME_OBJECT(new_entry)); + VME_OBJECT_SET(new_entry, VM_OBJECT_NULL); assert(!new_entry->iokit_acct); new_entry->use_pmap = TRUE; } @@ -9062,8 +9678,8 @@ vm_map_copyin_common( src_size = new_entry->vme_end - src_start; } - if ((src_entry->object.vm_object != src_object) || - (src_entry->offset != src_offset) ) { + if ((VME_OBJECT(src_entry) != src_object) || + (VME_OFFSET(src_entry) != src_offset) ) { /* * Verification failed. @@ -9073,7 +9689,7 @@ vm_map_copyin_common( VerificationFailed: ; - vm_object_deallocate(new_entry->object.vm_object); + vm_object_deallocate(VME_OBJECT(new_entry)); tmp_entry = src_entry; continue; } @@ -9213,7 +9829,7 @@ vm_map_copyin_common( vm_map_offset_t adjustment; original_start = tmp_entry->vme_start; - original_offset = tmp_entry->offset; + original_offset = VME_OFFSET(tmp_entry); /* map-align the start of the first copy entry... */ adjustment = (tmp_entry->vme_start - @@ -9221,7 +9837,8 @@ vm_map_copyin_common( tmp_entry->vme_start, VM_MAP_PAGE_MASK(src_map))); tmp_entry->vme_start -= adjustment; - tmp_entry->offset -= adjustment; + VME_OFFSET_SET(tmp_entry, + VME_OFFSET(tmp_entry) - adjustment); copy_addr -= adjustment; assert(tmp_entry->vme_start < tmp_entry->vme_end); /* ... adjust for mis-aligned start of copy range */ @@ -9234,7 +9851,9 @@ vm_map_copyin_common( assert(page_aligned(adjustment)); assert(adjustment < VM_MAP_PAGE_SIZE(src_map)); tmp_entry->vme_start += adjustment; - tmp_entry->offset += adjustment; + VME_OFFSET_SET(tmp_entry, + (VME_OFFSET(tmp_entry) + + adjustment)); copy_addr += adjustment; assert(tmp_entry->vme_start < tmp_entry->vme_end); } @@ -9244,7 +9863,7 @@ vm_map_copyin_common( * more than was originally copied... */ assert(tmp_entry->vme_start >= original_start); - assert(tmp_entry->offset >= original_offset); + assert(VME_OFFSET(tmp_entry) >= original_offset); /* * ... and that it did not adjust outside of a * a single 16K page. @@ -9478,7 +10097,7 @@ vm_map_fork_share( * make a new shadow and share it. */ - object = old_entry->object.vm_object; + object = VME_OBJECT(old_entry); if (old_entry->is_sub_map) { assert(old_entry->wired_count == 0); #ifndef NO_NESTED_PMAP @@ -9486,7 +10105,7 @@ vm_map_fork_share( kern_return_t result; result = pmap_nest(new_map->pmap, - (old_entry->object.sub_map)->pmap, + (VME_SUBMAP(old_entry))->pmap, (addr64_t)old_entry->vme_start, (addr64_t)old_entry->vme_start, (uint64_t)(old_entry->vme_end - old_entry->vme_start)); @@ -9497,8 +10116,8 @@ vm_map_fork_share( } else if (object == VM_OBJECT_NULL) { object = vm_object_allocate((vm_map_size_t)(old_entry->vme_end - old_entry->vme_start)); - old_entry->offset = 0; - old_entry->object.vm_object = object; + VME_OFFSET_SET(old_entry, 0); + VME_OBJECT_SET(old_entry, object); old_entry->use_pmap = TRUE; assert(!old_entry->needs_copy); } else if (object->copy_strategy != @@ -9596,10 +10215,9 @@ vm_map_fork_share( * (This is a preemptive version of * case 2.) */ - vm_object_shadow(&old_entry->object.vm_object, - &old_entry->offset, - (vm_map_size_t) (old_entry->vme_end - - old_entry->vme_start)); + VME_OBJECT_SHADOW(old_entry, + (vm_map_size_t) (old_entry->vme_end - + old_entry->vme_start)); /* * If we're making a shadow for other than @@ -9613,13 +10231,13 @@ vm_map_fork_share( prot = old_entry->protection & ~VM_PROT_WRITE; - if (override_nx(old_map, old_entry->alias) && prot) + if (override_nx(old_map, VME_ALIAS(old_entry)) && prot) prot |= VM_PROT_EXECUTE; if (old_map->mapped_in_other_pmaps) { vm_object_pmap_protect( - old_entry->object.vm_object, - old_entry->offset, + VME_OBJECT(old_entry), + VME_OFFSET(old_entry), (old_entry->vme_end - old_entry->vme_start), PMAP_NULL, @@ -9634,7 +10252,7 @@ vm_map_fork_share( } old_entry->needs_copy = FALSE; - object = old_entry->object.vm_object; + object = VME_OBJECT(old_entry); } @@ -9648,9 +10266,9 @@ vm_map_fork_share( */ if(old_entry->is_sub_map) { - vm_map_lock(old_entry->object.sub_map); - vm_map_reference(old_entry->object.sub_map); - vm_map_unlock(old_entry->object.sub_map); + vm_map_lock(VME_SUBMAP(old_entry)); + vm_map_reference(VME_SUBMAP(old_entry)); + vm_map_unlock(VME_SUBMAP(old_entry)); } else { vm_object_lock(object); vm_object_reference_locked(object); @@ -9788,14 +10406,16 @@ vm_map_fork( vm_map_entry_t new_entry; boolean_t src_needs_copy; boolean_t new_entry_needs_copy; + boolean_t pmap_is64bit; - new_pmap = pmap_create(ledger, (vm_map_size_t) 0, + pmap_is64bit = #if defined(__i386__) || defined(__x86_64__) - old_map->pmap->pm_task_map != TASK_MAP_32BIT + old_map->pmap->pm_task_map != TASK_MAP_32BIT; #else #error Unknown architecture. #endif - ); + + new_pmap = pmap_create(ledger, (vm_map_size_t) 0, pmap_is64bit); vm_map_reference_swap(old_map); vm_map_lock(old_map); @@ -9833,8 +10453,8 @@ vm_map_fork( if(old_entry->is_sub_map) break; if ((old_entry->wired_count != 0) || - ((old_entry->object.vm_object != NULL) && - (old_entry->object.vm_object->true_share))) { + ((VME_OBJECT(old_entry) != NULL) && + (VME_OBJECT(old_entry)->true_share))) { goto slow_vm_map_fork_copy; } @@ -9846,8 +10466,8 @@ vm_map_fork( } if (! vm_object_copy_quickly( - &new_entry->object.vm_object, - old_entry->offset, + &VME_OBJECT(new_entry), + VME_OFFSET(old_entry), (old_entry->vme_end - old_entry->vme_start), &src_needs_copy, @@ -9865,12 +10485,13 @@ vm_map_fork( prot = old_entry->protection & ~VM_PROT_WRITE; - if (override_nx(old_map, old_entry->alias) && prot) + if (override_nx(old_map, VME_ALIAS(old_entry)) + && prot) prot |= VM_PROT_EXECUTE; vm_object_pmap_protect( - old_entry->object.vm_object, - old_entry->offset, + VME_OBJECT(old_entry), + VME_OFFSET(old_entry), (old_entry->vme_end - old_entry->vme_start), ((old_entry->is_shared @@ -9880,6 +10501,7 @@ vm_map_fork( old_entry->vme_start, prot); + assert(old_entry->wired_count == 0); old_entry->needs_copy = TRUE; } new_entry->needs_copy = new_entry_needs_copy; @@ -10059,7 +10681,7 @@ submap_recurse: if ((*real_map != map) && (*real_map != cow_sub_map_parent)) vm_map_unlock(*real_map); - *real_map = entry->object.sub_map; + *real_map = VME_SUBMAP(entry); } if(entry->needs_copy && (fault_type & VM_PROT_WRITE)) { @@ -10069,8 +10691,8 @@ submap_recurse: *real_map = map; goto RetryLookup; } - vm_map_lock_read(entry->object.sub_map); - *var_map = entry->object.sub_map; + vm_map_lock_read(VME_SUBMAP(entry)); + *var_map = VME_SUBMAP(entry); cow_sub_map_parent = map; /* reset base to map before cow object */ /* this is the map which will accept */ @@ -10080,15 +10702,15 @@ submap_recurse: cow_parent_vaddr = vaddr; mapped_needs_copy = TRUE; } else { - vm_map_lock_read(entry->object.sub_map); - *var_map = entry->object.sub_map; + vm_map_lock_read(VME_SUBMAP(entry)); + *var_map = VME_SUBMAP(entry); if((cow_sub_map_parent != map) && (*real_map != map)) vm_map_unlock(map); } } else { - vm_map_lock_read(entry->object.sub_map); - *var_map = entry->object.sub_map; + vm_map_lock_read(VME_SUBMAP(entry)); + *var_map = VME_SUBMAP(entry); /* leave map locked if it is a target */ /* cow sub_map above otherwise, just */ /* follow the maps down to the object */ @@ -10101,7 +10723,7 @@ submap_recurse: map = *var_map; /* calculate the offset in the submap for vaddr */ - local_vaddr = (local_vaddr - entry->vme_start) + entry->offset; + local_vaddr = (local_vaddr - entry->vme_start) + VME_OFFSET(entry); RetrySubMap: if(!vm_map_lookup_entry(map, local_vaddr, &submap_entry)) { @@ -10126,13 +10748,13 @@ submap_recurse: /* ultimately be clipped in the top map will only need */ /* to be as big as the portion of the underlying entry */ /* which is mapped */ - start_delta = submap_entry->vme_start > entry->offset ? - submap_entry->vme_start - entry->offset : 0; + start_delta = submap_entry->vme_start > VME_OFFSET(entry) ? + submap_entry->vme_start - VME_OFFSET(entry) : 0; end_delta = - (entry->offset + start_delta + (old_end - old_start)) <= + (VME_OFFSET(entry) + start_delta + (old_end - old_start)) <= submap_entry->vme_end ? - 0 : (entry->offset + + 0 : (VME_OFFSET(entry) + (old_end - old_start)) - submap_entry->vme_end; @@ -10161,15 +10783,15 @@ submap_recurse: } - sub_object = submap_entry->object.vm_object; + sub_object = VME_OBJECT(submap_entry); if (sub_object == VM_OBJECT_NULL) { sub_object = vm_object_allocate( (vm_map_size_t) (submap_entry->vme_end - submap_entry->vme_start)); - submap_entry->object.vm_object = sub_object; - submap_entry->offset = 0; + VME_OBJECT_SET(submap_entry, sub_object); + VME_OFFSET_SET(submap_entry, 0); } local_start = local_vaddr - (cow_parent_vaddr - old_start); @@ -10193,7 +10815,7 @@ submap_recurse: MEMORY_OBJECT_COPY_NONE)) { vm_object_lock(sub_object); vm_object_copy_slowly(sub_object, - submap_entry->offset, + VME_OFFSET(submap_entry), (submap_entry->vme_end - submap_entry->vme_start), FALSE, @@ -10205,16 +10827,19 @@ submap_recurse: copy_object = sub_object; vm_object_reference(copy_object); sub_object->shadowed = TRUE; + assert(submap_entry->wired_count == 0); submap_entry->needs_copy = TRUE; prot = submap_entry->protection & ~VM_PROT_WRITE; - if (override_nx(old_map, submap_entry->alias) && prot) + if (override_nx(old_map, + VME_ALIAS(submap_entry)) + && prot) prot |= VM_PROT_EXECUTE; vm_object_pmap_protect( sub_object, - submap_entry->offset, + VME_OFFSET(submap_entry), submap_entry->vme_end - submap_entry->vme_start, (submap_entry->is_shared @@ -10229,7 +10854,7 @@ submap_recurse: */ copy_offset = (local_vaddr - submap_entry->vme_start + - submap_entry->offset); + VME_OFFSET(submap_entry)); /* This works diffently than the */ /* normal submap case. We go back */ @@ -10290,22 +10915,23 @@ submap_recurse: /* substitute copy object for */ /* shared map entry */ - vm_map_deallocate(entry->object.sub_map); + vm_map_deallocate(VME_SUBMAP(entry)); assert(!entry->iokit_acct); entry->is_sub_map = FALSE; entry->use_pmap = TRUE; - entry->object.vm_object = copy_object; + VME_OBJECT_SET(entry, copy_object); /* propagate the submap entry's protections */ entry->protection |= submap_entry->protection; entry->max_protection |= submap_entry->max_protection; if(copied_slowly) { - entry->offset = local_start - old_start; + VME_OFFSET_SET(entry, local_start - old_start); entry->needs_copy = FALSE; entry->is_shared = FALSE; } else { - entry->offset = copy_offset; + VME_OFFSET_SET(entry, copy_offset); + assert(entry->wired_count == 0); entry->needs_copy = TRUE; if(entry->inheritance == VM_INHERIT_SHARE) entry->inheritance = VM_INHERIT_COPY; @@ -10334,7 +10960,7 @@ submap_recurse: prot = entry->protection; - if (override_nx(old_map, entry->alias) && prot) { + if (override_nx(old_map, VME_ALIAS(entry)) && prot) { /* * HACK -- if not a stack, then allow execution */ @@ -10395,12 +11021,11 @@ submap_recurse: vm_map_lock_read(map); goto RetryLookup; } - vm_object_shadow(&entry->object.vm_object, - &entry->offset, - (vm_map_size_t) (entry->vme_end - - entry->vme_start)); + VME_OBJECT_SHADOW(entry, + (vm_map_size_t) (entry->vme_end - + entry->vme_start)); - entry->object.vm_object->shadowed = TRUE; + VME_OBJECT(entry)->shadowed = TRUE; entry->needs_copy = FALSE; vm_map_lock_write_to_read(map); } @@ -10417,16 +11042,18 @@ submap_recurse: /* * Create an object if necessary. */ - if (entry->object.vm_object == VM_OBJECT_NULL) { + if (VME_OBJECT(entry) == VM_OBJECT_NULL) { if (vm_map_lock_read_to_write(map)) { vm_map_lock_read(map); goto RetryLookup; } - entry->object.vm_object = vm_object_allocate( - (vm_map_size_t)(entry->vme_end - entry->vme_start)); - entry->offset = 0; + VME_OBJECT_SET(entry, + vm_object_allocate( + (vm_map_size_t)(entry->vme_end - + entry->vme_start))); + VME_OFFSET_SET(entry, 0); vm_map_lock_write_to_read(map); } @@ -10436,27 +11063,33 @@ submap_recurse: * return the protection. */ - *offset = (vaddr - entry->vme_start) + entry->offset; - *object = entry->object.vm_object; + *offset = (vaddr - entry->vme_start) + VME_OFFSET(entry); + *object = VME_OBJECT(entry); *out_prot = prot; if (fault_info) { fault_info->interruptible = THREAD_UNINT; /* for now... */ /* ... the caller will change "interruptible" if needed */ fault_info->cluster_size = 0; - fault_info->user_tag = entry->alias; + fault_info->user_tag = VME_ALIAS(entry); fault_info->pmap_options = 0; if (entry->iokit_acct || (!entry->is_sub_map && !entry->use_pmap)) { fault_info->pmap_options |= PMAP_OPTIONS_ALT_ACCT; } fault_info->behavior = entry->behavior; - fault_info->lo_offset = entry->offset; - fault_info->hi_offset = (entry->vme_end - entry->vme_start) + entry->offset; + fault_info->lo_offset = VME_OFFSET(entry); + fault_info->hi_offset = + (entry->vme_end - entry->vme_start) + VME_OFFSET(entry); fault_info->no_cache = entry->no_cache; fault_info->stealth = FALSE; fault_info->io_sync = FALSE; - fault_info->cs_bypass = (entry->used_for_jit)? TRUE : FALSE; + if (entry->used_for_jit || + entry->vme_resilient_codesign) { + fault_info->cs_bypass = TRUE; + } else { + fault_info->cs_bypass = FALSE; + } fault_info->mark_zf_absent = FALSE; fault_info->batch_pmap_op = FALSE; } @@ -10612,6 +11245,11 @@ vm_map_region_recurse_64( user_address = *address; user_max_depth = *nesting_depth; + if (not_in_kdp) { + vm_map_lock_read(map); + } + +recurse_again: curr_entry = NULL; curr_map = map; curr_address = user_address; @@ -10630,10 +11268,6 @@ vm_map_region_recurse_64( next_max_above = (vm_map_offset_t) -1; next_max_below = (vm_map_offset_t) -1; - if (not_in_kdp) { - vm_map_lock_read(curr_map); - } - for (;;) { if (vm_map_lookup_entry(curr_map, curr_address, @@ -10658,6 +11292,7 @@ vm_map_region_recurse_64( } curr_entry = NULL; curr_map = NULL; + curr_skip = 0; curr_offset = 0; curr_depth = 0; curr_max_above = 0; @@ -10668,7 +11303,7 @@ vm_map_region_recurse_64( /* adjust current address and offset */ skip = curr_entry->vme_start - curr_address; curr_address = curr_entry->vme_start; - curr_skip = skip; + curr_skip += skip; curr_offset += skip; curr_max_above -= skip; curr_max_below = 0; @@ -10707,6 +11342,7 @@ vm_map_region_recurse_64( next_depth = curr_depth; next_address = next_entry->vme_start; next_skip = curr_skip; + next_skip += (next_address - curr_address); next_offset = curr_offset; next_offset += (next_address - curr_address); next_max_above = MIN(next_max_above, curr_max_above); @@ -10723,7 +11359,7 @@ vm_map_region_recurse_64( * the rest of that submap is irrelevant to us, since it's not * mapped here. * The relevant portion of the map starts at - * "curr_entry->offset" up to the size of "curr_entry". + * "VME_OFFSET(curr_entry)" up to the size of "curr_entry". */ curr_max_above = MIN(curr_max_above, curr_entry->vme_end - curr_address); @@ -10750,7 +11386,7 @@ vm_map_region_recurse_64( * later. */ if (not_in_kdp) { - vm_map_lock_read(curr_entry->object.sub_map); + vm_map_lock_read(VME_SUBMAP(curr_entry)); } if (curr_map == next_map) { /* keep "next_map" locked in case we need it */ @@ -10763,17 +11399,17 @@ vm_map_region_recurse_64( /* * Adjust the offset. "curr_entry" maps the submap * at relative address "curr_entry->vme_start" in the - * curr_map but skips the first "curr_entry->offset" + * curr_map but skips the first "VME_OFFSET(curr_entry)" * bytes of the submap. * "curr_offset" always represents the offset of a virtual * address in the curr_map relative to the absolute address * space (i.e. the top-level VM map). */ curr_offset += - (curr_entry->offset - curr_entry->vme_start); + (VME_OFFSET(curr_entry) - curr_entry->vme_start); curr_address = user_address + curr_offset; /* switch to the submap */ - curr_map = curr_entry->object.sub_map; + curr_map = VME_SUBMAP(curr_entry); curr_depth++; curr_entry = NULL; } @@ -10793,9 +11429,6 @@ vm_map_region_recurse_64( curr_depth = next_depth; curr_max_above = next_max_above; curr_max_below = next_max_below; - if (curr_map == map) { - user_address = curr_address; - } } else { /* we won't need "next_entry" after all */ if (next_entry != NULL) { @@ -10813,6 +11446,18 @@ vm_map_region_recurse_64( next_max_below = -1; next_max_above = -1; + if (curr_entry->is_sub_map && + curr_depth < user_max_depth) { + /* + * We're not as deep as we could be: we must have + * gone back up after not finding anything mapped + * below the original top-level map entry's. + * Let's move "curr_address" forward and recurse again. + */ + user_address = curr_address; + goto recurse_again; + } + *nesting_depth = curr_depth; *size = curr_max_above + curr_max_below; *address = user_address + curr_skip - curr_max_below; @@ -10823,25 +11468,25 @@ vm_map_region_recurse_64( #define INFO_MAKE_OBJECT_ID(p) ((uint32_t)(uintptr_t)VM_KERNEL_ADDRPERM(p)) if (look_for_pages) { - submap_info->user_tag = curr_entry->alias; - submap_info->offset = curr_entry->offset; + submap_info->user_tag = VME_ALIAS(curr_entry); + submap_info->offset = VME_OFFSET(curr_entry); submap_info->protection = curr_entry->protection; submap_info->inheritance = curr_entry->inheritance; submap_info->max_protection = curr_entry->max_protection; submap_info->behavior = curr_entry->behavior; submap_info->user_wired_count = curr_entry->user_wired_count; submap_info->is_submap = curr_entry->is_sub_map; - submap_info->object_id = INFO_MAKE_OBJECT_ID(curr_entry->object.vm_object); + submap_info->object_id = INFO_MAKE_OBJECT_ID(VME_OBJECT(curr_entry)); } else { - short_info->user_tag = curr_entry->alias; - short_info->offset = curr_entry->offset; + short_info->user_tag = VME_ALIAS(curr_entry); + short_info->offset = VME_OFFSET(curr_entry); short_info->protection = curr_entry->protection; short_info->inheritance = curr_entry->inheritance; short_info->max_protection = curr_entry->max_protection; short_info->behavior = curr_entry->behavior; short_info->user_wired_count = curr_entry->user_wired_count; short_info->is_submap = curr_entry->is_sub_map; - short_info->object_id = INFO_MAKE_OBJECT_ID(curr_entry->object.vm_object); + short_info->object_id = INFO_MAKE_OBJECT_ID(VME_OBJECT(curr_entry)); } extended.pages_resident = 0; @@ -10851,6 +11496,8 @@ vm_map_region_recurse_64( extended.pages_reusable = 0; extended.external_pager = 0; extended.shadow_depth = 0; + extended.share_mode = SM_EMPTY; + extended.ref_count = 0; if (not_in_kdp) { if (!curr_entry->is_sub_map) { @@ -10862,7 +11509,7 @@ vm_map_region_recurse_64( vm_map_region_walk(curr_map, range_start, curr_entry, - (curr_entry->offset + + (VME_OFFSET(curr_entry) + (range_start - curr_entry->vme_start)), range_end - range_start, @@ -10879,8 +11526,7 @@ vm_map_region_recurse_64( } else { extended.share_mode = SM_PRIVATE; } - extended.ref_count = - curr_entry->object.sub_map->ref_count; + extended.ref_count = VME_SUBMAP(curr_entry)->ref_count; } } @@ -10968,7 +11614,7 @@ vm_map_region( start = entry->vme_start; - basic->offset = (uint32_t)entry->offset; + basic->offset = (uint32_t)VME_OFFSET(entry); basic->protection = entry->protection; basic->inheritance = entry->inheritance; basic->max_protection = entry->max_protection; @@ -11013,7 +11659,7 @@ vm_map_region( start = entry->vme_start; - basic->offset = entry->offset; + basic->offset = VME_OFFSET(entry); basic->protection = entry->protection; basic->inheritance = entry->inheritance; basic->max_protection = entry->max_protection; @@ -11061,7 +11707,7 @@ vm_map_region( start = entry->vme_start; extended->protection = entry->protection; - extended->user_tag = entry->alias; + extended->user_tag = VME_ALIAS(entry); extended->pages_resident = 0; extended->pages_swapped_out = 0; extended->pages_shared_now_private = 0; @@ -11077,7 +11723,7 @@ vm_map_region( *count = VM_REGION_EXTENDED_INFO_COUNT; } - vm_map_region_walk(map, start, entry, entry->offset, entry->vme_end - start, extended, TRUE, *count); + vm_map_region_walk(map, start, entry, VME_OFFSET(entry), entry->vme_end - start, extended, TRUE, *count); if (extended->external_pager && extended->ref_count == 2 && extended->share_mode == SM_SHARED) extended->share_mode = SM_PRIVATE; @@ -11144,7 +11790,7 @@ vm_map_region_top_walk( vm_region_top_info_t top) { - if (entry->object.vm_object == 0 || entry->is_sub_map) { + if (VME_OBJECT(entry) == 0 || entry->is_sub_map) { top->share_mode = SM_EMPTY; top->ref_count = 0; top->obj_id = 0; @@ -11158,7 +11804,7 @@ vm_map_region_top_walk( entry_size = (uint32_t) ((entry->vme_end - entry->vme_start) / PAGE_SIZE_64); - obj = entry->object.vm_object; + obj = VME_OBJECT(entry); vm_object_lock(obj); @@ -11239,9 +11885,9 @@ vm_map_region_walk( struct vm_object *shadow_object; int shadow_depth; - if ((entry->object.vm_object == 0) || + if ((VME_OBJECT(entry) == 0) || (entry->is_sub_map) || - (entry->object.vm_object->phys_contiguous && + (VME_OBJECT(entry)->phys_contiguous && !entry->superpage_size)) { extended->share_mode = SM_EMPTY; extended->ref_count = 0; @@ -11259,7 +11905,7 @@ vm_map_region_walk( } { - obj = entry->object.vm_object; + obj = VME_OBJECT(entry); vm_object_lock(obj); @@ -11336,7 +11982,7 @@ vm_map_region_walk( register vm_map_entry_t last; int my_refs; - obj = entry->object.vm_object; + obj = VME_OBJECT(entry); last = vm_map_to_entry(map); my_refs = 0; @@ -11492,7 +12138,7 @@ vm_map_region_count_obj_refs( register vm_object_t chk_obj; register vm_object_t tmp_obj; - if (entry->object.vm_object == 0) + if (VME_OBJECT(entry) == 0) return(0); if (entry->is_sub_map) @@ -11500,7 +12146,7 @@ vm_map_region_count_obj_refs( else { ref_count = 0; - chk_obj = entry->object.vm_object; + chk_obj = VME_OBJECT(entry); vm_object_lock(chk_obj); while (chk_obj) { @@ -11548,10 +12194,10 @@ vm_map_simplify_entry( (prev_entry->vme_end == this_entry->vme_start) && (prev_entry->is_sub_map == this_entry->is_sub_map) && - (prev_entry->object.vm_object == this_entry->object.vm_object) && - ((prev_entry->offset + (prev_entry->vme_end - + (VME_OBJECT(prev_entry) == VME_OBJECT(this_entry)) && + ((VME_OFFSET(prev_entry) + (prev_entry->vme_end - prev_entry->vme_start)) - == this_entry->offset) && + == VME_OFFSET(this_entry)) && (prev_entry->behavior == this_entry->behavior) && (prev_entry->needs_copy == this_entry->needs_copy) && @@ -11559,7 +12205,7 @@ vm_map_simplify_entry( (prev_entry->max_protection == this_entry->max_protection) && (prev_entry->inheritance == this_entry->inheritance) && (prev_entry->use_pmap == this_entry->use_pmap) && - (prev_entry->alias == this_entry->alias) && + (VME_ALIAS(prev_entry) == VME_ALIAS(this_entry)) && (prev_entry->no_cache == this_entry->no_cache) && (prev_entry->permanent == this_entry->permanent) && (prev_entry->map_aligned == this_entry->map_aligned) && @@ -11567,6 +12213,10 @@ vm_map_simplify_entry( (prev_entry->used_for_jit == this_entry->used_for_jit) && /* from_reserved_zone: OK if that field doesn't match */ (prev_entry->iokit_acct == this_entry->iokit_acct) && + (prev_entry->vme_resilient_codesign == + this_entry->vme_resilient_codesign) && + (prev_entry->vme_resilient_media == + this_entry->vme_resilient_media) && (prev_entry->wired_count == this_entry->wired_count) && (prev_entry->user_wired_count == this_entry->user_wired_count) && @@ -11586,11 +12236,16 @@ vm_map_simplify_entry( assert(VM_MAP_PAGE_ALIGNED(prev_entry->vme_start, VM_MAP_PAGE_MASK(map))); this_entry->vme_start = prev_entry->vme_start; - this_entry->offset = prev_entry->offset; + VME_OFFSET_SET(this_entry, VME_OFFSET(prev_entry)); + + if (map->holelistenabled) { + vm_map_store_update_first_free(map, this_entry, TRUE); + } + if (prev_entry->is_sub_map) { - vm_map_deallocate(prev_entry->object.sub_map); + vm_map_deallocate(VME_SUBMAP(prev_entry)); } else { - vm_object_deallocate(prev_entry->object.vm_object); + vm_object_deallocate(VME_OBJECT(prev_entry)); } vm_map_entry_dispose(map, prev_entry); SAVE_HINT_MAP_WRITE(map, this_entry); @@ -11716,15 +12371,15 @@ vm_map_machine_attribute( vm_map_offset_t sub_end; sub_start = (start - entry->vme_start) - + entry->offset; + + VME_OFFSET(entry); sub_end = sub_start + sub_size; vm_map_machine_attribute( - entry->object.sub_map, + VME_SUBMAP(entry), sub_start, sub_end, attribute, value); } else { - if(entry->object.vm_object) { + if (VME_OBJECT(entry)) { vm_page_t m; vm_object_t object; vm_object_t base_object; @@ -11734,9 +12389,9 @@ vm_map_machine_attribute( vm_map_size_t range; range = sub_size; offset = (start - entry->vme_start) - + entry->offset; + + VME_OFFSET(entry); base_offset = offset; - object = entry->object.vm_object; + object = VME_OBJECT(entry); base_object = object; last_object = NULL; @@ -11888,6 +12543,11 @@ vm_map_behavior_set( case VM_BEHAVIOR_CAN_REUSE: return vm_map_can_reuse(map, start, end); +#if MACH_ASSERT + case VM_BEHAVIOR_PAGEOUT: + return vm_map_pageout(map, start, end); +#endif /* MACH_ASSERT */ + default: return(KERN_INVALID_ARGUMENT); } @@ -11964,7 +12624,7 @@ vm_map_willneed( * correspond. After that, the offset will always be zero to * correspond to the beginning of the current vm_map_entry. */ - offset = (start - entry->vme_start) + entry->offset; + offset = (start - entry->vme_start) + VME_OFFSET(entry); /* * Set the length so we don't go beyond the end of the @@ -11985,7 +12645,7 @@ vm_map_willneed( fault_info.cluster_size = (vm_size_t) len; fault_info.lo_offset = offset; fault_info.hi_offset = offset + len; - fault_info.user_tag = entry->alias; + fault_info.user_tag = VME_ALIAS(entry); fault_info.pmap_options = 0; if (entry->iokit_acct || (!entry->is_sub_map && !entry->use_pmap)) { @@ -12083,9 +12743,15 @@ static boolean_t vm_map_entry_is_reusable( vm_map_entry_t entry) { + /* Only user map entries */ + vm_object_t object; - switch (entry->alias) { + if (entry->is_sub_map) { + return FALSE; + } + + switch (VME_ALIAS(entry)) { case VM_MEMORY_MALLOC: case VM_MEMORY_MALLOC_SMALL: case VM_MEMORY_MALLOC_LARGE: @@ -12122,7 +12788,7 @@ vm_map_entry_is_reusable( return FALSE; } - object = entry->object.vm_object; + object = VME_OBJECT(entry); if (object == VM_OBJECT_NULL) { return TRUE; } @@ -12173,6 +12839,7 @@ vm_map_reuse_pages( */ vm_map_lock_read(map); + assert(map->pmap != kernel_pmap); /* protect alias access */ /* * The madvise semantics require that the address range be fully @@ -12211,10 +12878,11 @@ vm_map_reuse_pages( start_offset = 0; } end_offset = MIN(end, entry->vme_end) - entry->vme_start; - start_offset += entry->offset; - end_offset += entry->offset; + start_offset += VME_OFFSET(entry); + end_offset += VME_OFFSET(entry); - object = entry->object.vm_object; + assert(!entry->is_sub_map); + object = VME_OBJECT(entry); if (object != VM_OBJECT_NULL) { vm_object_lock(object); vm_object_reuse_pages(object, start_offset, end_offset, @@ -12222,7 +12890,7 @@ vm_map_reuse_pages( vm_object_unlock(object); } - if (entry->alias == VM_MEMORY_MALLOC_LARGE_REUSABLE) { + if (VME_ALIAS(entry) == VM_MEMORY_MALLOC_LARGE_REUSABLE) { /* * XXX * We do not hold the VM map exclusively here. @@ -12231,7 +12899,7 @@ vm_map_reuse_pages( * one that can be modified while holding the VM map * "shared". */ - entry->alias = VM_MEMORY_MALLOC_LARGE_REUSED; + VME_ALIAS_SET(entry, VM_MEMORY_MALLOC_LARGE_REUSED); } } @@ -12250,6 +12918,7 @@ vm_map_reusable_pages( vm_map_entry_t entry; vm_object_t object; vm_object_offset_t start_offset, end_offset; + vm_map_offset_t pmap_offset; /* * The MADV_REUSABLE operation doesn't require any changes to the @@ -12257,6 +12926,7 @@ vm_map_reusable_pages( */ vm_map_lock_read(map); + assert(map->pmap != kernel_pmap); /* protect alias access */ /* * The madvise semantics require that the address range be fully @@ -12293,14 +12963,17 @@ vm_map_reusable_pages( */ if (entry->vme_start < start) { start_offset = start - entry->vme_start; + pmap_offset = start; } else { start_offset = 0; + pmap_offset = entry->vme_start; } end_offset = MIN(end, entry->vme_end) - entry->vme_start; - start_offset += entry->offset; - end_offset += entry->offset; + start_offset += VME_OFFSET(entry); + end_offset += VME_OFFSET(entry); - object = entry->object.vm_object; + assert(!entry->is_sub_map); + object = VME_OBJECT(entry); if (object == VM_OBJECT_NULL) continue; @@ -12325,14 +12998,16 @@ vm_map_reusable_pages( start_offset, end_offset - start_offset, kill_pages, - TRUE /*reusable_pages*/); + TRUE /*reusable_pages*/, + map->pmap, + pmap_offset); } else { vm_page_stats_reusable.reusable_pages_shared++; } vm_object_unlock(object); - if (entry->alias == VM_MEMORY_MALLOC_LARGE || - entry->alias == VM_MEMORY_MALLOC_LARGE_REUSED) { + if (VME_ALIAS(entry) == VM_MEMORY_MALLOC_LARGE || + VME_ALIAS(entry) == VM_MEMORY_MALLOC_LARGE_REUSED) { /* * XXX * We do not hold the VM map exclusively here. @@ -12341,7 +13016,7 @@ vm_map_reusable_pages( * one that can be modified while holding the VM map * "shared". */ - entry->alias = VM_MEMORY_MALLOC_LARGE_REUSABLE; + VME_ALIAS_SET(entry, VM_MEMORY_MALLOC_LARGE_REUSABLE); } } @@ -12365,6 +13040,7 @@ vm_map_can_reuse( */ vm_map_lock_read(map); + assert(map->pmap != kernel_pmap); /* protect alias access */ /* * The madvise semantics require that the address range be fully @@ -12399,6 +13075,97 @@ vm_map_can_reuse( } +#if MACH_ASSERT +static kern_return_t +vm_map_pageout( + vm_map_t map, + vm_map_offset_t start, + vm_map_offset_t end) +{ + vm_map_entry_t entry; + + /* + * The MADV_PAGEOUT operation doesn't require any changes to the + * vm_map_entry_t's, so the read lock is sufficient. + */ + + vm_map_lock_read(map); + + /* + * The madvise semantics require that the address range be fully + * allocated with no holes. Otherwise, we're required to return + * an error. + */ + + if (!vm_map_range_check(map, start, end, &entry)) { + vm_map_unlock_read(map); + return KERN_INVALID_ADDRESS; + } + + /* + * Examine each vm_map_entry_t in the range. + */ + for (; entry != vm_map_to_entry(map) && entry->vme_start < end; + entry = entry->vme_next) { + vm_object_t object; + + /* + * Sanity check on the VM map entry. + */ + if (entry->is_sub_map) { + vm_map_t submap; + vm_map_offset_t submap_start; + vm_map_offset_t submap_end; + vm_map_entry_t submap_entry; + + submap = VME_SUBMAP(entry); + submap_start = VME_OFFSET(entry); + submap_end = submap_start + (entry->vme_end - + entry->vme_start); + + vm_map_lock_read(submap); + + if (! vm_map_range_check(submap, + submap_start, + submap_end, + &submap_entry)) { + vm_map_unlock_read(submap); + vm_map_unlock_read(map); + return KERN_INVALID_ADDRESS; + } + + object = VME_OBJECT(submap_entry); + if (submap_entry->is_sub_map || + object == VM_OBJECT_NULL || + !object->internal) { + vm_map_unlock_read(submap); + continue; + } + + vm_object_pageout(object); + + vm_map_unlock_read(submap); + submap = VM_MAP_NULL; + submap_entry = VM_MAP_ENTRY_NULL; + continue; + } + + object = VME_OBJECT(entry); + if (entry->is_sub_map || + object == VM_OBJECT_NULL || + !object->internal) { + continue; + } + + vm_object_pageout(object); + } + + vm_map_unlock_read(map); + return KERN_SUCCESS; +} +#endif /* MACH_ASSERT */ + + /* * Routine: vm_map_entry_insert * @@ -12455,8 +13222,8 @@ vm_map_entry_insert( } assert(new_entry->vme_start < new_entry->vme_end); - new_entry->object.vm_object = object; - new_entry->offset = offset; + VME_OBJECT_SET(new_entry, object); + VME_OFFSET_SET(new_entry, offset); new_entry->is_shared = is_shared; new_entry->is_sub_map = is_submap; new_entry->needs_copy = needs_copy; @@ -12481,7 +13248,7 @@ vm_map_entry_insert( */ new_entry->use_pmap = TRUE; } - new_entry->alias = 0; + VME_ALIAS_SET(new_entry, 0); new_entry->zero_wired_pages = FALSE; new_entry->no_cache = no_cache; new_entry->permanent = permanent; @@ -12491,6 +13258,8 @@ vm_map_entry_insert( new_entry->superpage_size = FALSE; new_entry->used_for_jit = FALSE; new_entry->iokit_acct = FALSE; + new_entry->vme_resilient_codesign = FALSE; + new_entry->vme_resilient_media = FALSE; /* * Insert the new entry into the list. @@ -12601,10 +13370,10 @@ vm_map_remap_extract( src_entry->vme_start); if(src_entry->is_sub_map) { - vm_map_reference(src_entry->object.sub_map); + vm_map_reference(VME_SUBMAP(src_entry)); object = VM_OBJECT_NULL; } else { - object = src_entry->object.vm_object; + object = VME_OBJECT(src_entry); if (src_entry->iokit_acct) { /* * This entry uses "IOKit accounting". @@ -12626,8 +13395,8 @@ vm_map_remap_extract( if (object == VM_OBJECT_NULL) { object = vm_object_allocate(entry_size); - src_entry->offset = 0; - src_entry->object.vm_object = object; + VME_OFFSET_SET(src_entry, 0); + VME_OBJECT_SET(src_entry, object); } else if (object->copy_strategy != MEMORY_OBJECT_COPY_SYMMETRIC) { /* @@ -12641,9 +13410,7 @@ vm_map_remap_extract( !src_entry->is_shared && object->vo_size > entry_size)) { - vm_object_shadow(&src_entry->object.vm_object, - &src_entry->offset, - entry_size); + VME_OBJECT_SHADOW(src_entry, entry_size); if (!src_entry->needs_copy && (src_entry->protection & VM_PROT_WRITE)) { @@ -12651,13 +13418,15 @@ vm_map_remap_extract( prot = src_entry->protection & ~VM_PROT_WRITE; - if (override_nx(map, src_entry->alias) && prot) + if (override_nx(map, + VME_ALIAS(src_entry)) + && prot) prot |= VM_PROT_EXECUTE; if(map->mapped_in_other_pmaps) { vm_object_pmap_protect( - src_entry->object.vm_object, - src_entry->offset, + VME_OBJECT(src_entry), + VME_OFFSET(src_entry), entry_size, PMAP_NULL, src_entry->vme_start, @@ -12670,7 +13439,7 @@ vm_map_remap_extract( } } - object = src_entry->object.vm_object; + object = VME_OBJECT(src_entry); src_entry->needs_copy = FALSE; } @@ -12685,7 +13454,8 @@ vm_map_remap_extract( vm_object_unlock(object); } - offset = src_entry->offset + (src_start - src_entry->vme_start); + offset = (VME_OFFSET(src_entry) + + (src_start - src_entry->vme_start)); new_entry = _vm_map_entry_create(map_header, !map_header->entries_pageable); vm_map_entry_copy(new_entry, src_entry); @@ -12700,7 +13470,7 @@ vm_map_remap_extract( new_entry->vme_end = map_address + tmp_size; assert(new_entry->vme_start < new_entry->vme_end); new_entry->inheritance = inheritance; - new_entry->offset = offset; + VME_OFFSET_SET(new_entry, offset); /* * The new region has to be copied now if required. @@ -12722,11 +13492,12 @@ vm_map_remap_extract( } else if (src_entry->is_sub_map) { /* make this a COW sub_map if not already */ + assert(new_entry->wired_count == 0); new_entry->needs_copy = TRUE; object = VM_OBJECT_NULL; } else if (src_entry->wired_count == 0 && - vm_object_copy_quickly(&new_entry->object.vm_object, - new_entry->offset, + vm_object_copy_quickly(&VME_OBJECT(new_entry), + VME_OFFSET(new_entry), (new_entry->vme_end - new_entry->vme_start), &src_needs_copy, @@ -12743,7 +13514,9 @@ vm_map_remap_extract( prot = src_entry->protection & ~VM_PROT_WRITE; - if (override_nx(map, src_entry->alias) && prot) + if (override_nx(map, + VME_ALIAS(src_entry)) + && prot) prot |= VM_PROT_EXECUTE; vm_object_pmap_protect(object, @@ -12755,6 +13528,7 @@ vm_map_remap_extract( src_entry->vme_start, prot); + assert(src_entry->wired_count == 0); src_entry->needs_copy = TRUE; } /* @@ -12785,18 +13559,24 @@ vm_map_remap_extract( offset, entry_size, THREAD_UNINT, - &new_entry->object.vm_object); + &VME_OBJECT(new_entry)); - new_entry->offset = 0; + VME_OFFSET_SET(new_entry, 0); new_entry->needs_copy = FALSE; } else { + vm_object_offset_t new_offset; + + new_offset = VME_OFFSET(new_entry); result = vm_object_copy_strategically( object, offset, entry_size, - &new_entry->object.vm_object, - &new_entry->offset, + &VME_OBJECT(new_entry), + &new_offset, &new_entry_needs_copy); + if (new_offset != VME_OFFSET(new_entry)) { + VME_OFFSET_SET(new_entry, new_offset); + } new_entry->needs_copy = new_entry_needs_copy; } @@ -12825,8 +13605,7 @@ vm_map_remap_extract( * Retry the lookup and verify that the * same object/offset are still present. */ - vm_object_deallocate(new_entry-> - object.vm_object); + vm_object_deallocate(VME_OBJECT(new_entry)); _vm_map_entry_dispose(map_header, new_entry); if (result == KERN_MEMORY_RESTART_COPY) result = KERN_SUCCESS; @@ -12864,9 +13643,9 @@ vm_map_remap_extract( new_entry = src_entry->vme_next; _vm_map_store_entry_unlink(map_header, src_entry); if (src_entry->is_sub_map) { - vm_map_deallocate(src_entry->object.sub_map); + vm_map_deallocate(VME_SUBMAP(src_entry)); } else { - vm_object_deallocate(src_entry->object.vm_object); + vm_object_deallocate(VME_OBJECT(src_entry)); } _vm_map_entry_dispose(map_header, src_entry); } @@ -12974,6 +13753,13 @@ vm_map_remap( new_entry = entry->vme_next; _vm_map_store_entry_unlink(&map_header, entry); if (result == KERN_SUCCESS) { + if (flags & VM_FLAGS_RESILIENT_CODESIGN) { + /* no codesigning -> read-only access */ + assert(!entry->used_for_jit); + entry->max_protection = VM_PROT_READ; + entry->protection = VM_PROT_READ; + entry->vme_resilient_codesign = TRUE; + } entry->vme_start += *address; entry->vme_end += *address; assert(!entry->map_aligned); @@ -12981,14 +13767,19 @@ vm_map_remap( insp_entry = entry; } else { if (!entry->is_sub_map) { - vm_object_deallocate(entry->object.vm_object); + vm_object_deallocate(VME_OBJECT(entry)); } else { - vm_map_deallocate(entry->object.sub_map); + vm_map_deallocate(VME_SUBMAP(entry)); } _vm_map_entry_dispose(&map_header, entry); } } + if (flags & VM_FLAGS_RESILIENT_CODESIGN) { + *cur_protection = VM_PROT_READ; + *max_protection = VM_PROT_READ; + } + if( target_map->disable_vmentry_reuse == TRUE) { if( target_map->highest_entry_end < insp_entry->vme_end ){ target_map->highest_entry_end = insp_entry->vme_end; @@ -13003,7 +13794,8 @@ vm_map_remap( if (result == KERN_SUCCESS && target_map->wiring_required) result = vm_map_wire(target_map, *address, - *address + size, *cur_protection, TRUE); + *address + size, *cur_protection | VM_PROT_MEMORY_TAG_MAKE(VM_KERN_MEMORY_MLOCK), + TRUE); /* * If requested, return the address of the data pointed to by the @@ -13040,6 +13832,7 @@ vm_map_remap_range_allocate( vm_map_offset_t start; vm_map_offset_t end; kern_return_t kr; + vm_map_entry_t hole_entry; StartAgain: ; @@ -13065,15 +13858,51 @@ StartAgain: ; if( map->disable_vmentry_reuse == TRUE) { VM_MAP_HIGHEST_ENTRY(map, entry, start); } else { - assert(first_free_is_valid(map)); - if (start == map->min_offset) { - if ((entry = map->first_free) != vm_map_to_entry(map)) - start = entry->vme_end; + + if (map->holelistenabled) { + hole_entry = (vm_map_entry_t)map->holes_list; + + if (hole_entry == NULL) { + /* + * No more space in the map? + */ + return(KERN_NO_SPACE); + } else { + + boolean_t found_hole = FALSE; + + do { + if (hole_entry->vme_start >= start) { + start = hole_entry->vme_start; + found_hole = TRUE; + break; + } + + if (hole_entry->vme_end > start) { + found_hole = TRUE; + break; + } + hole_entry = hole_entry->vme_next; + + } while (hole_entry != (vm_map_entry_t) map->holes_list); + + if (found_hole == FALSE) { + return (KERN_NO_SPACE); + } + + entry = hole_entry; + } } else { - vm_map_entry_t tmp_entry; - if (vm_map_lookup_entry(map, start, &tmp_entry)) - start = tmp_entry->vme_end; - entry = tmp_entry; + assert(first_free_is_valid(map)); + if (start == map->min_offset) { + if ((entry = map->first_free) != vm_map_to_entry(map)) + start = entry->vme_end; + } else { + vm_map_entry_t tmp_entry; + if (vm_map_lookup_entry(map, start, &tmp_entry)) + start = tmp_entry->vme_end; + entry = tmp_entry; + } } start = vm_map_round_page(start, VM_MAP_PAGE_MASK(map)); @@ -13117,30 +13946,56 @@ StartAgain: ; return(KERN_NO_SPACE); } - /* - * If there are no more entries, we must win. - */ - next = entry->vme_next; - if (next == vm_map_to_entry(map)) - break; - /* - * If there is another entry, it must be - * after the end of the potential new region. - */ + if (map->holelistenabled) { + if (entry->vme_end >= end) + break; + } else { + /* + * If there are no more entries, we must win. + * + * OR + * + * If there is another entry, it must be + * after the end of the potential new region. + */ - if (next->vme_start >= end) - break; + if (next == vm_map_to_entry(map)) + break; + + if (next->vme_start >= end) + break; + } /* * Didn't fit -- move to the next entry. */ entry = next; - start = entry->vme_end; + + if (map->holelistenabled) { + if (entry == (vm_map_entry_t) map->holes_list) { + /* + * Wrapped around + */ + return(KERN_NO_SPACE); + } + start = entry->vme_start; + } else { + start = entry->vme_end; + } + } + + if (map->holelistenabled) { + + if (vm_map_lookup_entry(map, entry->vme_start, &entry)) { + panic("Found an existing entry (%p) instead of potential hole at address: 0x%llx.\n", entry, (unsigned long long)entry->vme_start); + } } + *address = start; + } else { vm_map_entry_t temp_entry; @@ -13187,6 +14042,7 @@ StartAgain: ; return KERN_RESOURCE_SHORTAGE; } vm_map_set_page_shift(zap_map, VM_MAP_PAGE_SHIFT(map)); + vm_map_disable_hole_optimization(zap_map); kr = vm_map_delete(map, start, end, (VM_MAP_REMOVE_SAVE_ENTRIES | @@ -13448,7 +14304,7 @@ vm_map_purgable_control( return(KERN_PROTECTION_FAILURE); } - object = entry->object.vm_object; + object = VME_OBJECT(entry); if (object == VM_OBJECT_NULL || object->purgable == VM_PURGABLE_DENY) { /* @@ -13461,7 +14317,7 @@ vm_map_purgable_control( vm_object_lock(object); #if 00 - if (entry->offset != 0 || + if (VME_OFFSET(entry) != 0 || entry->vme_end - entry->vme_start != object->vo_size) { /* * Can only apply purgable controls to the whole (existing) @@ -13582,12 +14438,12 @@ vm_map_page_info( /* compute offset from this map entry's start */ offset -= map_entry->vme_start; /* compute offset into this map entry's object (or submap) */ - offset += map_entry->offset; + offset += VME_OFFSET(map_entry); if (map_entry->is_sub_map) { vm_map_t sub_map; - sub_map = map_entry->object.sub_map; + sub_map = VME_SUBMAP(map_entry); vm_map_lock_read(sub_map); vm_map_unlock_read(map); @@ -13599,7 +14455,7 @@ vm_map_page_info( break; } - object = map_entry->object.vm_object; + object = VME_OBJECT(map_entry); if (object == VM_OBJECT_NULL) { /* no object -> no page */ vm_map_unlock_read(map); @@ -13812,6 +14668,7 @@ vm_map_msync( boolean_t do_sync_req; boolean_t had_hole = FALSE; memory_object_t pager; + vm_map_offset_t pmap_offset; if ((sync_flags & VM_SYNC_ASYNCHRONOUS) && (sync_flags & VM_SYNC_SYNCHRONOUS)) @@ -13842,9 +14699,7 @@ vm_map_msync( vm_map_lock(map); if (!vm_map_lookup_entry(map, - vm_map_trunc_page( - address, - VM_MAP_PAGE_MASK(map)), + address, &entry)) { vm_map_size_t skip; @@ -13885,6 +14740,7 @@ vm_map_msync( } offset = address - entry->vme_start; + pmap_offset = address; /* * do we have more to flush than is contained in this @@ -13903,8 +14759,8 @@ vm_map_msync( vm_map_t local_map; vm_map_offset_t local_offset; - local_map = entry->object.sub_map; - local_offset = entry->offset; + local_map = VME_SUBMAP(entry); + local_offset = VME_OFFSET(entry); vm_map_unlock(map); if (vm_map_msync( local_map, @@ -13915,7 +14771,7 @@ vm_map_msync( } continue; } - object = entry->object.vm_object; + object = VME_OBJECT(entry); /* * We can't sync this object if the object has not been @@ -13925,7 +14781,7 @@ vm_map_msync( vm_map_unlock(map); continue; } - offset += entry->offset; + offset += VME_OFFSET(entry); vm_object_lock(object); @@ -13940,8 +14796,14 @@ vm_map_msync( kill_pages = -1; } if (kill_pages != -1) - vm_object_deactivate_pages(object, offset, - (vm_object_size_t)flush_size, kill_pages, reusable_pages); + vm_object_deactivate_pages( + object, + offset, + (vm_object_size_t) flush_size, + kill_pages, + reusable_pages, + map->pmap, + pmap_offset); vm_object_unlock(object); vm_map_unlock(map); continue; @@ -14306,7 +15168,7 @@ vm_map_set_64bit(vm_map_t map) } vm_map_offset_t -vm_compute_max_offset(unsigned is64) +vm_compute_max_offset(boolean_t is64) { return (is64 ? (vm_map_offset_t)MACH_VM_MAX_ADDRESS : (vm_map_offset_t)VM_MAX_ADDRESS); } @@ -14399,6 +15261,11 @@ vm_map_raise_min_offset( vm_map_unlock(map); return KERN_INVALID_ADDRESS; } + if (new_min_offset >= map->max_offset) { + /* can't go beyond the end of the address space */ + vm_map_unlock(map); + return KERN_INVALID_ADDRESS; + } first_entry = vm_map_first_entry(map); if (first_entry != vm_map_to_entry(map) && @@ -14413,6 +15280,10 @@ vm_map_raise_min_offset( map->min_offset = new_min_offset; + assert(map->holes_list); + map->holes_list->start = new_min_offset; + assert(new_min_offset < map->holes_list->end); + vm_map_unlock(map); return KERN_SUCCESS; @@ -14500,7 +15371,7 @@ kern_return_t vm_map_sign(vm_map_t map, return(KERN_INVALID_ARGUMENT); } - object = entry->object.vm_object; + object = VME_OBJECT(entry); if (object == VM_OBJECT_NULL) { /* * Object must already be present or we can't sign. @@ -14515,7 +15386,8 @@ kern_return_t vm_map_sign(vm_map_t map, while(start < end) { uint32_t refmod; - m = vm_page_lookup(object, start - entry->vme_start + entry->offset ); + m = vm_page_lookup(object, + start - entry->vme_start + VME_OFFSET(entry)); if (m==VM_PAGE_NULL) { /* shoud we try to fault a page here? we can probably * demand it exists and is locked for this request */ @@ -14580,17 +15452,20 @@ kern_return_t vm_map_partial_reap(vm_map_t map, unsigned int *reclaimed_resident vm_map_set_page_shift(zap_map, VM_MAP_PAGE_SHIFT(map)); + vm_map_disable_hole_optimization(zap_map); for (entry = vm_map_first_entry(map); entry != vm_map_to_entry(map); entry = next_entry) { next_entry = entry->vme_next; - if (entry->object.vm_object && !entry->is_sub_map && (entry->object.vm_object->internal == TRUE) - && (entry->object.vm_object->ref_count == 1)) { + if (VME_OBJECT(entry) && + !entry->is_sub_map && + (VME_OBJECT(entry)->internal == TRUE) && + (VME_OBJECT(entry)->ref_count == 1)) { - *reclaimed_resident += entry->object.vm_object->resident_page_count; - *reclaimed_compressed += vm_compressor_pager_get_count(entry->object.vm_object->pager); + *reclaimed_resident += VME_OBJECT(entry)->resident_page_count; + *reclaimed_compressed += vm_compressor_pager_get_count(VME_OBJECT(entry)->pager); (void)vm_map_delete(map, entry->vme_start, @@ -14638,13 +15513,13 @@ kern_return_t vm_map_freeze_walk( unsigned int purgeable, clean, dirty, wired; boolean_t shared; - if ((entry->object.vm_object == 0) || + if ((VME_OBJECT(entry) == 0) || (entry->is_sub_map) || - (entry->object.vm_object->phys_contiguous)) { + (VME_OBJECT(entry)->phys_contiguous)) { continue; } - default_freezer_pack(&purgeable, &wired, &clean, &dirty, dirty_budget, &shared, entry->object.vm_object, NULL); + default_freezer_pack(&purgeable, &wired, &clean, &dirty, dirty_budget, &shared, VME_OBJECT(entry), NULL); *purgeable_count += purgeable; *wired_count += wired; @@ -14669,6 +15544,10 @@ kern_return_t vm_map_freeze_walk( return KERN_SUCCESS; } +int c_freezer_swapout_count; +int c_freezer_compression_count = 0; +AbsoluteTime c_freezer_last_yield_ts = 0; + kern_return_t vm_map_freeze( vm_map_t map, unsigned int *purgeable_count, @@ -14694,7 +15573,13 @@ kern_return_t vm_map_freeze( if (COMPRESSED_PAGER_IS_ACTIVE || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_ACTIVE) { default_freezer_active = FALSE; + + if (vm_compressor_low_on_space() || vm_swap_low_on_space()) { + kr = KERN_NO_SPACE; + goto done; + } } + assert(default_freezer_active == FALSE); if (default_freezer_active) { if (map->default_freezer_handle == NULL) { @@ -14710,14 +15595,18 @@ kern_return_t vm_map_freeze( goto done; } } - + c_freezer_compression_count = 0; + clock_get_uptime(&c_freezer_last_yield_ts); + for (entry2 = vm_map_first_entry(map); entry2 != vm_map_to_entry(map); entry2 = entry2->vme_next) { - vm_object_t src_object = entry2->object.vm_object; + vm_object_t src_object = VME_OBJECT(entry2); - if (entry2->object.vm_object && !entry2->is_sub_map && !entry2->object.vm_object->phys_contiguous) { + if (VME_OBJECT(entry2) && + !entry2->is_sub_map && + !VME_OBJECT(entry2)->phys_contiguous) { /* If eligible, scan the entry, moving eligible pages over to our parent object */ if (default_freezer_active) { unsigned int purgeable, clean, dirty, wired; @@ -14743,11 +15632,24 @@ kern_return_t vm_map_freeze( *has_shared = TRUE; } } else { - /* - * To the compressor. - */ - if (entry2->object.vm_object->internal == TRUE) { - vm_object_pageout(entry2->object.vm_object); + if (VME_OBJECT(entry2)->internal == TRUE) { + + if (DEFAULT_FREEZER_COMPRESSED_PAGER_IS_SWAPBACKED) { + /* + * Pages belonging to this object could be swapped to disk. + * Make sure it's not a shared object because we could end + * up just bringing it back in again. + */ + if (VME_OBJECT(entry2)->ref_count > 1) { + continue; + } + } + vm_object_compressed_freezer_pageout(VME_OBJECT(entry2)); + } + + if (vm_compressor_low_on_space() || vm_swap_low_on_space()) { + kr = KERN_NO_SPACE; + break; } } } @@ -14761,6 +15663,16 @@ kern_return_t vm_map_freeze( done: vm_map_unlock(map); + if (!default_freezer_active) { + vm_object_compressed_freezer_done(); + } + if (DEFAULT_FREEZER_COMPRESSED_PAGER_IS_SWAPBACKED) { + /* + * reset the counter tracking the # of swapped c_segs + * because we are now done with this freeze session and task. + */ + c_freezer_swapout_count = 0; + } return kr; } @@ -14812,6 +15724,8 @@ out: * - copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC * - !true_share * - vo_size == ANON_CHUNK_SIZE + * + * Only non-kernel map entries. */ boolean_t vm_map_entry_should_cow_for_true_share( @@ -14829,8 +15743,8 @@ vm_map_entry_should_cow_for_true_share( return FALSE; } - if (entry->alias != VM_MEMORY_MALLOC && - entry->alias != VM_MEMORY_MALLOC_SMALL) { + if (VME_ALIAS(entry) != VM_MEMORY_MALLOC && + VME_ALIAS(entry) != VM_MEMORY_MALLOC_SMALL) { /* not a malloc heap or Obj-C Garbage Collector heap */ return FALSE; } @@ -14841,7 +15755,7 @@ vm_map_entry_should_cow_for_true_share( return FALSE; } - object = entry->object.vm_object; + object = VME_OBJECT(entry); if (object == VM_OBJECT_NULL) { /* no object yet... */ @@ -14863,13 +15777,13 @@ vm_map_entry_should_cow_for_true_share( return FALSE; } - if (entry->alias == VM_MEMORY_MALLOC && + if (VME_ALIAS(entry) == VM_MEMORY_MALLOC && object->vo_size != ANON_CHUNK_SIZE) { /* ... not an object created for the ObjC Garbage Collector */ return FALSE; } - if (entry->alias == VM_MEMORY_MALLOC_SMALL && + if (VME_ALIAS(entry) == VM_MEMORY_MALLOC_SMALL && object->vo_size != 2048 * 4096) { /* ... not a "MALLOC_SMALL" heap */ return FALSE; @@ -14900,6 +15814,14 @@ vm_map_trunc_page_mask( return VM_MAP_TRUNC_PAGE(offset, mask); } +boolean_t +vm_map_page_aligned( + vm_map_offset_t offset, + vm_map_offset_t mask) +{ + return ((offset) & mask) == 0; +} + int vm_map_page_shift( vm_map_t map) @@ -14914,7 +15836,7 @@ vm_map_page_size( return VM_MAP_PAGE_SIZE(map); } -int +vm_map_offset_t vm_map_page_mask( vm_map_t map) { @@ -14958,7 +15880,7 @@ vm_map_purge( if (! (entry->protection & VM_PROT_WRITE)) { goto next; } - object = entry->object.vm_object; + object = VME_OBJECT(entry); if (object == VM_OBJECT_NULL) { goto next; } @@ -14968,7 +15890,7 @@ vm_map_purge( vm_object_lock(object); #if 00 - if (entry->offset != 0 || + if (VME_OFFSET(entry) != 0 || (entry->vme_end - entry->vme_start) != object->vo_size) { vm_object_unlock(object); goto next; @@ -15002,11 +15924,15 @@ vm_map_query_volatile( vm_map_t map, mach_vm_size_t *volatile_virtual_size_p, mach_vm_size_t *volatile_resident_size_p, - mach_vm_size_t *volatile_pmap_size_p) + mach_vm_size_t *volatile_compressed_size_p, + mach_vm_size_t *volatile_pmap_size_p, + mach_vm_size_t *volatile_compressed_pmap_size_p) { mach_vm_size_t volatile_virtual_size; mach_vm_size_t volatile_resident_count; + mach_vm_size_t volatile_compressed_count; mach_vm_size_t volatile_pmap_count; + mach_vm_size_t volatile_compressed_pmap_count; mach_vm_size_t resident_count; vm_map_entry_t entry; vm_object_t object; @@ -15015,25 +15941,30 @@ vm_map_query_volatile( volatile_virtual_size = 0; volatile_resident_count = 0; + volatile_compressed_count = 0; volatile_pmap_count = 0; + volatile_compressed_pmap_count = 0; for (entry = vm_map_first_entry(map); entry != vm_map_to_entry(map); entry = entry->vme_next) { + mach_vm_size_t pmap_resident_bytes, pmap_compressed_bytes; + if (entry->is_sub_map) { continue; } if (! (entry->protection & VM_PROT_WRITE)) { continue; } - object = entry->object.vm_object; + object = VME_OBJECT(entry); if (object == VM_OBJECT_NULL) { continue; } - if (object->purgable != VM_PURGABLE_VOLATILE) { + if (object->purgable != VM_PURGABLE_VOLATILE && + object->purgable != VM_PURGABLE_EMPTY) { continue; } - if (entry->offset != 0) { + if (VME_OFFSET(entry)) { /* * If the map entry has been split and the object now * appears several times in the VM map, we don't want @@ -15044,28 +15975,75 @@ vm_map_query_volatile( continue; } resident_count = object->resident_page_count; - if ((entry->offset / PAGE_SIZE) >= resident_count) { + if ((VME_OFFSET(entry) / PAGE_SIZE) >= resident_count) { resident_count = 0; } else { - resident_count -= (entry->offset / PAGE_SIZE); + resident_count -= (VME_OFFSET(entry) / PAGE_SIZE); } volatile_virtual_size += entry->vme_end - entry->vme_start; volatile_resident_count += resident_count; - volatile_pmap_count += pmap_query_resident(map->pmap, - entry->vme_start, - entry->vme_end); + if (object->pager) { + volatile_compressed_count += + vm_compressor_pager_get_count(object->pager); + } + pmap_compressed_bytes = 0; + pmap_resident_bytes = + pmap_query_resident(map->pmap, + entry->vme_start, + entry->vme_end, + &pmap_compressed_bytes); + volatile_pmap_count += (pmap_resident_bytes / PAGE_SIZE); + volatile_compressed_pmap_count += (pmap_compressed_bytes + / PAGE_SIZE); } /* map is still locked on return */ *volatile_virtual_size_p = volatile_virtual_size; *volatile_resident_size_p = volatile_resident_count * PAGE_SIZE; + *volatile_compressed_size_p = volatile_compressed_count * PAGE_SIZE; *volatile_pmap_size_p = volatile_pmap_count * PAGE_SIZE; + *volatile_compressed_pmap_size_p = volatile_compressed_pmap_count * PAGE_SIZE; return KERN_SUCCESS; } +void +vm_map_sizes(vm_map_t map, + vm_map_size_t * psize, + vm_map_size_t * pfree, + vm_map_size_t * plargest_free) +{ + vm_map_entry_t entry; + vm_map_offset_t prev; + vm_map_size_t free, total_free, largest_free; + boolean_t end; + + total_free = largest_free = 0; + + vm_map_lock_read(map); + if (psize) *psize = map->max_offset - map->min_offset; + + prev = map->min_offset; + for (entry = vm_map_first_entry(map);; entry = entry->vme_next) + { + end = (entry == vm_map_to_entry(map)); + + if (end) free = entry->vme_end - prev; + else free = entry->vme_start - prev; + + total_free += free; + if (free > largest_free) largest_free = free; + + if (end) break; + prev = entry->vme_end; + } + vm_map_unlock_read(map); + if (pfree) *pfree = total_free; + if (plargest_free) *plargest_free = largest_free; +} + #if VM_SCAN_FOR_SHADOW_CHAIN int vm_map_shadow_max(vm_map_t map); int vm_map_shadow_max( @@ -15088,7 +16066,7 @@ int vm_map_shadow_max( if (entry->is_sub_map) { continue; } - object = entry->object.vm_object; + object = VME_OBJECT(entry); if (object == NULL) { continue; }