X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/5c9f46613a83ebfc29a5b1f099448259e96a98f0..0a7de7458d150b5d4dffc935ba399be265ef0a1a:/osfmk/vm/vm_user.c diff --git a/osfmk/vm/vm_user.c b/osfmk/vm/vm_user.c index 2f9bdc101..92df95613 100644 --- a/osfmk/vm/vm_user.c +++ b/osfmk/vm/vm_user.c @@ -2,7 +2,7 @@ * Copyright (c) 2000-2007 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * + * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in @@ -11,10 +11,10 @@ * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. - * + * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. - * + * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, @@ -22,34 +22,34 @@ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. - * + * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ /* * @OSF_COPYRIGHT@ */ -/* +/* * Mach Operating System * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University * All Rights Reserved. - * + * * Permission to use, copy, modify and distribute this software and its * documentation is hereby granted, provided that both the copyright * notice and this permission notice appear in all copies of the * software, derivative works or modified versions, and any portions * thereof, and that both notices appear in supporting documentation. - * + * * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. - * + * * Carnegie Mellon requests users of this software to return to - * + * * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU * School of Computer Science * Carnegie Mellon University * Pittsburgh PA 15213-3890 - * + * * any improvements or extensions that they make and grant Carnegie Mellon * the rights to redistribute these changes. */ @@ -58,7 +58,7 @@ /* * File: vm/vm_user.c * Author: Avadis Tevanian, Jr., Michael Wayne Young - * + * * User-exported virtual memory functions. */ @@ -90,9 +90,9 @@ #include #include #include -#include /* to get vm_address_t */ +#include /* to get vm_address_t */ #include -#include /* to get pointer_t */ +#include /* to get pointer_t */ #include #include #include @@ -102,6 +102,7 @@ #include #include +#include #include #include @@ -120,11 +121,13 @@ #include +#include + vm_size_t upl_offset_to_pagelist = 0; -#if VM_CPM +#if VM_CPM #include -#endif /* VM_CPM */ +#endif /* VM_CPM */ /* * mach_vm_allocate allocates "zero fill" memory in the specfied @@ -132,39 +135,41 @@ vm_size_t upl_offset_to_pagelist = 0; */ kern_return_t mach_vm_allocate_external( - vm_map_t map, - mach_vm_offset_t *addr, - mach_vm_size_t size, - int flags) + vm_map_t map, + mach_vm_offset_t *addr, + mach_vm_size_t size, + int flags) { - vm_tag_t tag; + vm_tag_t tag; - VM_GET_FLAGS_ALIAS(flags, tag); - return (mach_vm_allocate_kernel(map, addr, size, flags, tag)); + VM_GET_FLAGS_ALIAS(flags, tag); + return mach_vm_allocate_kernel(map, addr, size, flags, tag); } kern_return_t mach_vm_allocate_kernel( - vm_map_t map, - mach_vm_offset_t *addr, - mach_vm_size_t size, - int flags, + vm_map_t map, + mach_vm_offset_t *addr, + mach_vm_size_t size, + int flags, vm_tag_t tag) { vm_map_offset_t map_addr; - vm_map_size_t map_size; - kern_return_t result; - boolean_t anywhere; + vm_map_size_t map_size; + kern_return_t result; + boolean_t anywhere; /* filter out any kernel-only flags */ - if (flags & ~VM_FLAGS_USER_ALLOCATE) + if (flags & ~VM_FLAGS_USER_ALLOCATE) { return KERN_INVALID_ARGUMENT; + } - if (map == VM_MAP_NULL) - return(KERN_INVALID_ARGUMENT); + if (map == VM_MAP_NULL) { + return KERN_INVALID_ARGUMENT; + } if (size == 0) { *addr = 0; - return(KERN_SUCCESS); + return KERN_SUCCESS; } anywhere = ((VM_FLAGS_ANYWHERE & flags) != 0); @@ -180,76 +185,80 @@ mach_vm_allocate_kernel( * memory would tend to confuse those applications. */ map_addr = vm_map_min(map); - if (map_addr == 0) + if (map_addr == 0) { map_addr += VM_MAP_PAGE_SIZE(map); - } else + } + } else { map_addr = vm_map_trunc_page(*addr, - VM_MAP_PAGE_MASK(map)); + VM_MAP_PAGE_MASK(map)); + } map_size = vm_map_round_page(size, - VM_MAP_PAGE_MASK(map)); + VM_MAP_PAGE_MASK(map)); if (map_size == 0) { - return(KERN_INVALID_ARGUMENT); + return KERN_INVALID_ARGUMENT; } result = vm_map_enter( - map, - &map_addr, - map_size, - (vm_map_offset_t)0, - flags, - VM_MAP_KERNEL_FLAGS_NONE, - tag, - VM_OBJECT_NULL, - (vm_object_offset_t)0, - FALSE, - VM_PROT_DEFAULT, - VM_PROT_ALL, - VM_INHERIT_DEFAULT); + map, + &map_addr, + map_size, + (vm_map_offset_t)0, + flags, + VM_MAP_KERNEL_FLAGS_NONE, + tag, + VM_OBJECT_NULL, + (vm_object_offset_t)0, + FALSE, + VM_PROT_DEFAULT, + VM_PROT_ALL, + VM_INHERIT_DEFAULT); *addr = map_addr; - return(result); + return result; } /* - * vm_allocate + * vm_allocate * Legacy routine that allocates "zero fill" memory in the specfied * map (which is limited to the same size as the kernel). */ kern_return_t vm_allocate_external( - vm_map_t map, - vm_offset_t *addr, - vm_size_t size, - int flags) + vm_map_t map, + vm_offset_t *addr, + vm_size_t size, + int flags) { vm_tag_t tag; - VM_GET_FLAGS_ALIAS(flags, tag); - return (vm_allocate_kernel(map, addr, size, flags, tag)); + VM_GET_FLAGS_ALIAS(flags, tag); + return vm_allocate_kernel(map, addr, size, flags, tag); } kern_return_t vm_allocate_kernel( - vm_map_t map, - vm_offset_t *addr, - vm_size_t size, + vm_map_t map, + vm_offset_t *addr, + vm_size_t size, int flags, vm_tag_t tag) { vm_map_offset_t map_addr; - vm_map_size_t map_size; - kern_return_t result; - boolean_t anywhere; + vm_map_size_t map_size; + kern_return_t result; + boolean_t anywhere; /* filter out any kernel-only flags */ - if (flags & ~VM_FLAGS_USER_ALLOCATE) + if (flags & ~VM_FLAGS_USER_ALLOCATE) { return KERN_INVALID_ARGUMENT; + } - if (map == VM_MAP_NULL) - return(KERN_INVALID_ARGUMENT); + if (map == VM_MAP_NULL) { + return KERN_INVALID_ARGUMENT; + } if (size == 0) { *addr = 0; - return(KERN_SUCCESS); + return KERN_SUCCESS; } anywhere = ((VM_FLAGS_ANYWHERE & flags) != 0); @@ -265,31 +274,33 @@ vm_allocate_kernel( * memory would tend to confuse those applications. */ map_addr = vm_map_min(map); - if (map_addr == 0) + if (map_addr == 0) { map_addr += VM_MAP_PAGE_SIZE(map); - } else + } + } else { map_addr = vm_map_trunc_page(*addr, - VM_MAP_PAGE_MASK(map)); + VM_MAP_PAGE_MASK(map)); + } map_size = vm_map_round_page(size, - VM_MAP_PAGE_MASK(map)); + VM_MAP_PAGE_MASK(map)); if (map_size == 0) { - return(KERN_INVALID_ARGUMENT); + return KERN_INVALID_ARGUMENT; } result = vm_map_enter( - map, - &map_addr, - map_size, - (vm_map_offset_t)0, - flags, - VM_MAP_KERNEL_FLAGS_NONE, - tag, - VM_OBJECT_NULL, - (vm_object_offset_t)0, - FALSE, - VM_PROT_DEFAULT, - VM_PROT_ALL, - VM_INHERIT_DEFAULT); + map, + &map_addr, + map_size, + (vm_map_offset_t)0, + flags, + VM_MAP_KERNEL_FLAGS_NONE, + tag, + VM_OBJECT_NULL, + (vm_object_offset_t)0, + FALSE, + VM_PROT_DEFAULT, + VM_PROT_ALL, + VM_INHERIT_DEFAULT); #if KASAN if (result == KERN_SUCCESS && map->pmap == kernel_pmap) { @@ -298,7 +309,7 @@ vm_allocate_kernel( #endif *addr = CAST_DOWN(vm_offset_t, map_addr); - return(result); + return result; } /* @@ -308,22 +319,24 @@ vm_allocate_kernel( */ kern_return_t mach_vm_deallocate( - vm_map_t map, - mach_vm_offset_t start, - mach_vm_size_t size) + vm_map_t map, + mach_vm_offset_t start, + mach_vm_size_t size) { - if ((map == VM_MAP_NULL) || (start + size < start)) - return(KERN_INVALID_ARGUMENT); - - if (size == (mach_vm_offset_t) 0) - return(KERN_SUCCESS); - - return(vm_map_remove(map, - vm_map_trunc_page(start, - VM_MAP_PAGE_MASK(map)), - vm_map_round_page(start+size, - VM_MAP_PAGE_MASK(map)), - VM_MAP_NO_FLAGS)); + if ((map == VM_MAP_NULL) || (start + size < start)) { + return KERN_INVALID_ARGUMENT; + } + + if (size == (mach_vm_offset_t) 0) { + return KERN_SUCCESS; + } + + return vm_map_remove(map, + vm_map_trunc_page(start, + VM_MAP_PAGE_MASK(map)), + vm_map_round_page(start + size, + VM_MAP_PAGE_MASK(map)), + VM_MAP_REMOVE_NO_FLAGS); } /* @@ -334,22 +347,24 @@ mach_vm_deallocate( */ kern_return_t vm_deallocate( - vm_map_t map, - vm_offset_t start, - vm_size_t size) + vm_map_t map, + vm_offset_t start, + vm_size_t size) { - if ((map == VM_MAP_NULL) || (start + size < start)) - return(KERN_INVALID_ARGUMENT); - - if (size == (vm_offset_t) 0) - return(KERN_SUCCESS); - - return(vm_map_remove(map, - vm_map_trunc_page(start, - VM_MAP_PAGE_MASK(map)), - vm_map_round_page(start+size, - VM_MAP_PAGE_MASK(map)), - VM_MAP_NO_FLAGS)); + if ((map == VM_MAP_NULL) || (start + size < start)) { + return KERN_INVALID_ARGUMENT; + } + + if (size == (vm_offset_t) 0) { + return KERN_SUCCESS; + } + + return vm_map_remove(map, + vm_map_trunc_page(start, + VM_MAP_PAGE_MASK(map)), + vm_map_round_page(start + size, + VM_MAP_PAGE_MASK(map)), + VM_MAP_REMOVE_NO_FLAGS); } /* @@ -359,24 +374,26 @@ vm_deallocate( */ kern_return_t mach_vm_inherit( - vm_map_t map, - mach_vm_offset_t start, - mach_vm_size_t size, - vm_inherit_t new_inheritance) + vm_map_t map, + mach_vm_offset_t start, + mach_vm_size_t size, + vm_inherit_t new_inheritance) { if ((map == VM_MAP_NULL) || (start + size < start) || - (new_inheritance > VM_INHERIT_LAST_VALID)) - return(KERN_INVALID_ARGUMENT); + (new_inheritance > VM_INHERIT_LAST_VALID)) { + return KERN_INVALID_ARGUMENT; + } - if (size == 0) + if (size == 0) { return KERN_SUCCESS; + } - return(vm_map_inherit(map, - vm_map_trunc_page(start, - VM_MAP_PAGE_MASK(map)), - vm_map_round_page(start+size, - VM_MAP_PAGE_MASK(map)), - new_inheritance)); + return vm_map_inherit(map, + vm_map_trunc_page(start, + VM_MAP_PAGE_MASK(map)), + vm_map_round_page(start + size, + VM_MAP_PAGE_MASK(map)), + new_inheritance); } /* @@ -386,24 +403,26 @@ mach_vm_inherit( */ kern_return_t vm_inherit( - vm_map_t map, - vm_offset_t start, - vm_size_t size, - vm_inherit_t new_inheritance) + vm_map_t map, + vm_offset_t start, + vm_size_t size, + vm_inherit_t new_inheritance) { if ((map == VM_MAP_NULL) || (start + size < start) || - (new_inheritance > VM_INHERIT_LAST_VALID)) - return(KERN_INVALID_ARGUMENT); + (new_inheritance > VM_INHERIT_LAST_VALID)) { + return KERN_INVALID_ARGUMENT; + } - if (size == 0) + if (size == 0) { return KERN_SUCCESS; + } - return(vm_map_inherit(map, - vm_map_trunc_page(start, - VM_MAP_PAGE_MASK(map)), - vm_map_round_page(start+size, - VM_MAP_PAGE_MASK(map)), - new_inheritance)); + return vm_map_inherit(map, + vm_map_trunc_page(start, + VM_MAP_PAGE_MASK(map)), + vm_map_round_page(start + size, + VM_MAP_PAGE_MASK(map)), + new_inheritance); } /* @@ -414,26 +433,28 @@ vm_inherit( kern_return_t mach_vm_protect( - vm_map_t map, - mach_vm_offset_t start, - mach_vm_size_t size, - boolean_t set_maximum, - vm_prot_t new_protection) + vm_map_t map, + mach_vm_offset_t start, + mach_vm_size_t size, + boolean_t set_maximum, + vm_prot_t new_protection) { if ((map == VM_MAP_NULL) || (start + size < start) || - (new_protection & ~(VM_PROT_ALL | VM_PROT_COPY))) - return(KERN_INVALID_ARGUMENT); + (new_protection & ~(VM_PROT_ALL | VM_PROT_COPY))) { + return KERN_INVALID_ARGUMENT; + } - if (size == 0) + if (size == 0) { return KERN_SUCCESS; + } - return(vm_map_protect(map, - vm_map_trunc_page(start, - VM_MAP_PAGE_MASK(map)), - vm_map_round_page(start+size, - VM_MAP_PAGE_MASK(map)), - new_protection, - set_maximum)); + return vm_map_protect(map, + vm_map_trunc_page(start, + VM_MAP_PAGE_MASK(map)), + vm_map_round_page(start + size, + VM_MAP_PAGE_MASK(map)), + new_protection, + set_maximum); } /* @@ -445,26 +466,28 @@ mach_vm_protect( kern_return_t vm_protect( - vm_map_t map, - vm_offset_t start, - vm_size_t size, - boolean_t set_maximum, - vm_prot_t new_protection) + vm_map_t map, + vm_offset_t start, + vm_size_t size, + boolean_t set_maximum, + vm_prot_t new_protection) { if ((map == VM_MAP_NULL) || (start + size < start) || - (new_protection & ~(VM_PROT_ALL | VM_PROT_COPY))) - return(KERN_INVALID_ARGUMENT); + (new_protection & ~(VM_PROT_ALL | VM_PROT_COPY))) { + return KERN_INVALID_ARGUMENT; + } - if (size == 0) + if (size == 0) { return KERN_SUCCESS; + } - return(vm_map_protect(map, - vm_map_trunc_page(start, - VM_MAP_PAGE_MASK(map)), - vm_map_round_page(start+size, - VM_MAP_PAGE_MASK(map)), - new_protection, - set_maximum)); + return vm_map_protect(map, + vm_map_trunc_page(start, + VM_MAP_PAGE_MASK(map)), + vm_map_round_page(start + size, + VM_MAP_PAGE_MASK(map)), + new_protection, + set_maximum); } /* @@ -474,24 +497,26 @@ vm_protect( */ kern_return_t mach_vm_machine_attribute( - vm_map_t map, - mach_vm_address_t addr, - mach_vm_size_t size, - vm_machine_attribute_t attribute, - vm_machine_attribute_val_t* value) /* IN/OUT */ + vm_map_t map, + mach_vm_address_t addr, + mach_vm_size_t size, + vm_machine_attribute_t attribute, + vm_machine_attribute_val_t* value) /* IN/OUT */ { - if ((map == VM_MAP_NULL) || (addr + size < addr)) - return(KERN_INVALID_ARGUMENT); + if ((map == VM_MAP_NULL) || (addr + size < addr)) { + return KERN_INVALID_ARGUMENT; + } - if (size == 0) + if (size == 0) { return KERN_SUCCESS; + } return vm_map_machine_attribute( - map, + map, vm_map_trunc_page(addr, - VM_MAP_PAGE_MASK(map)), - vm_map_round_page(addr+size, - VM_MAP_PAGE_MASK(map)), + VM_MAP_PAGE_MASK(map)), + vm_map_round_page(addr + size, + VM_MAP_PAGE_MASK(map)), attribute, value); } @@ -504,24 +529,26 @@ mach_vm_machine_attribute( */ kern_return_t vm_machine_attribute( - vm_map_t map, - vm_address_t addr, - vm_size_t size, - vm_machine_attribute_t attribute, - vm_machine_attribute_val_t* value) /* IN/OUT */ + vm_map_t map, + vm_address_t addr, + vm_size_t size, + vm_machine_attribute_t attribute, + vm_machine_attribute_val_t* value) /* IN/OUT */ { - if ((map == VM_MAP_NULL) || (addr + size < addr)) - return(KERN_INVALID_ARGUMENT); + if ((map == VM_MAP_NULL) || (addr + size < addr)) { + return KERN_INVALID_ARGUMENT; + } - if (size == 0) + if (size == 0) { return KERN_SUCCESS; + } return vm_map_machine_attribute( - map, + map, vm_map_trunc_page(addr, - VM_MAP_PAGE_MASK(map)), - vm_map_round_page(addr+size, - VM_MAP_PAGE_MASK(map)), + VM_MAP_PAGE_MASK(map)), + vm_map_round_page(addr + size, + VM_MAP_PAGE_MASK(map)), attribute, value); } @@ -534,47 +561,49 @@ vm_machine_attribute( * the IPC implementation as part of receiving the reply to this call. * If IPC isn't used, the caller must deal with the vm_map_copy_t object * that gets returned. - * + * * JMM - because of mach_msg_type_number_t, this call is limited to a * single 4GB region at this time. * */ kern_return_t mach_vm_read( - vm_map_t map, - mach_vm_address_t addr, - mach_vm_size_t size, - pointer_t *data, - mach_msg_type_number_t *data_size) + vm_map_t map, + mach_vm_address_t addr, + mach_vm_size_t size, + pointer_t *data, + mach_msg_type_number_t *data_size) { - kern_return_t error; - vm_map_copy_t ipc_address; + kern_return_t error; + vm_map_copy_t ipc_address; - if (map == VM_MAP_NULL) - return(KERN_INVALID_ARGUMENT); + if (map == VM_MAP_NULL) { + return KERN_INVALID_ARGUMENT; + } - if ((mach_msg_type_number_t) size != size) + if ((mach_msg_type_number_t) size != size) { return KERN_INVALID_ARGUMENT; - + } + error = vm_map_copyin(map, - (vm_map_address_t)addr, - (vm_map_size_t)size, - FALSE, /* src_destroy */ - &ipc_address); + (vm_map_address_t)addr, + (vm_map_size_t)size, + FALSE, /* src_destroy */ + &ipc_address); if (KERN_SUCCESS == error) { *data = (pointer_t) ipc_address; *data_size = (mach_msg_type_number_t) size; assert(*data_size == size); } - return(error); + return error; } /* * vm_read - * Read/copy a range from one address space and return it to the caller. * Limited addressability (same range limits as for the native kernel map). - * + * * It is assumed that the address for the returned memory is selected by * the IPC implementation as part of receiving the reply to this call. * If IPC isn't used, the caller must deal with the vm_map_copy_t object @@ -582,19 +611,21 @@ mach_vm_read( */ kern_return_t vm_read( - vm_map_t map, - vm_address_t addr, - vm_size_t size, - pointer_t *data, - mach_msg_type_number_t *data_size) + vm_map_t map, + vm_address_t addr, + vm_size_t size, + pointer_t *data, + mach_msg_type_number_t *data_size) { - kern_return_t error; - vm_map_copy_t ipc_address; + kern_return_t error; + vm_map_copy_t ipc_address; - if (map == VM_MAP_NULL) - return(KERN_INVALID_ARGUMENT); + if (map == VM_MAP_NULL) { + return KERN_INVALID_ARGUMENT; + } - if (size > (unsigned)(mach_msg_type_number_t) -1) { + mach_msg_type_number_t dsize; + if (os_convert_overflow(size, &dsize)) { /* * The kernel could handle a 64-bit "size" value, but * it could not return the size of the data in "*data_size" @@ -605,20 +636,20 @@ vm_read( } error = vm_map_copyin(map, - (vm_map_address_t)addr, - (vm_map_size_t)size, - FALSE, /* src_destroy */ - &ipc_address); + (vm_map_address_t)addr, + (vm_map_size_t)size, + FALSE, /* src_destroy */ + &ipc_address); if (KERN_SUCCESS == error) { *data = (pointer_t) ipc_address; - *data_size = (mach_msg_type_number_t) size; + *data_size = dsize; assert(*data_size == size); } - return(error); + return error; } -/* +/* * mach_vm_read_list - * Read/copy a list of address ranges from specified map. * @@ -628,37 +659,38 @@ vm_read( */ kern_return_t mach_vm_read_list( - vm_map_t map, - mach_vm_read_entry_t data_list, - natural_t count) + vm_map_t map, + mach_vm_read_entry_t data_list, + natural_t count) { - mach_msg_type_number_t i; - kern_return_t error; - vm_map_copy_t copy; + mach_msg_type_number_t i; + kern_return_t error; + vm_map_copy_t copy; if (map == VM_MAP_NULL || - count > VM_MAP_ENTRY_MAX) - return(KERN_INVALID_ARGUMENT); + count > VM_MAP_ENTRY_MAX) { + return KERN_INVALID_ARGUMENT; + } error = KERN_SUCCESS; - for(i=0; imap, - &map_addr, - copy); + current_task()->map, + &map_addr, + copy); if (KERN_SUCCESS == error) { data_list[i].address = map_addr; continue; @@ -669,10 +701,10 @@ mach_vm_read_list( data_list[i].address = (mach_vm_address_t)0; data_list[i].size = (mach_vm_size_t)0; } - return(error); + return error; } -/* +/* * vm_read_list - * Read/copy a list of address ranges from specified map. * @@ -693,39 +725,40 @@ mach_vm_read_list( kern_return_t vm_read_list( - vm_map_t map, - vm_read_entry_t data_list, - natural_t count) + vm_map_t map, + vm_read_entry_t data_list, + natural_t count) { - mach_msg_type_number_t i; - kern_return_t error; - vm_map_copy_t copy; + mach_msg_type_number_t i; + kern_return_t error; + vm_map_copy_t copy; if (map == VM_MAP_NULL || - count > VM_MAP_ENTRY_MAX) - return(KERN_INVALID_ARGUMENT); + count > VM_MAP_ENTRY_MAX) { + return KERN_INVALID_ARGUMENT; + } error = KERN_SUCCESS; - for(i=0; imap, - &map_addr, - copy); + error = vm_map_copyout(current_task()->map, + &map_addr, + copy); if (KERN_SUCCESS == error) { data_list[i].address = - CAST_DOWN(vm_offset_t, map_addr); + CAST_DOWN(vm_offset_t, map_addr); continue; } vm_map_copy_discard(copy); @@ -734,14 +767,14 @@ vm_read_list( data_list[i].address = (mach_vm_address_t)0; data_list[i].size = (mach_vm_size_t)0; } - return(error); + return error; } /* * mach_vm_read_overwrite - * Overwrite a range of the current map with data from the specified * map/address range. - * + * * In making an assumption that the current thread is local, it is * no longer cluster-safe without a fully supportive local proxy * thread/task (but we don't support cluster's anymore so this is moot). @@ -749,39 +782,40 @@ vm_read_list( kern_return_t mach_vm_read_overwrite( - vm_map_t map, - mach_vm_address_t address, - mach_vm_size_t size, - mach_vm_address_t data, - mach_vm_size_t *data_size) + vm_map_t map, + mach_vm_address_t address, + mach_vm_size_t size, + mach_vm_address_t data, + mach_vm_size_t *data_size) { - kern_return_t error; - vm_map_copy_t copy; + kern_return_t error; + vm_map_copy_t copy; - if (map == VM_MAP_NULL) - return(KERN_INVALID_ARGUMENT); + if (map == VM_MAP_NULL) { + return KERN_INVALID_ARGUMENT; + } error = vm_map_copyin(map, (vm_map_address_t)address, - (vm_map_size_t)size, FALSE, ©); + (vm_map_size_t)size, FALSE, ©); if (KERN_SUCCESS == error) { error = vm_map_copy_overwrite(current_thread()->map, - (vm_map_address_t)data, - copy, FALSE); + (vm_map_address_t)data, + copy, FALSE); if (KERN_SUCCESS == error) { *data_size = size; return error; } vm_map_copy_discard(copy); } - return(error); + return error; } /* * vm_read_overwrite - * Overwrite a range of the current map with data from the specified * map/address range. - * + * * This routine adds the additional limitation that the source and * destination ranges must be describable with vm_address_t values * (i.e. the same size address spaces as the kernel, or at least the @@ -791,32 +825,33 @@ mach_vm_read_overwrite( kern_return_t vm_read_overwrite( - vm_map_t map, - vm_address_t address, - vm_size_t size, - vm_address_t data, - vm_size_t *data_size) + vm_map_t map, + vm_address_t address, + vm_size_t size, + vm_address_t data, + vm_size_t *data_size) { - kern_return_t error; - vm_map_copy_t copy; + kern_return_t error; + vm_map_copy_t copy; - if (map == VM_MAP_NULL) - return(KERN_INVALID_ARGUMENT); + if (map == VM_MAP_NULL) { + return KERN_INVALID_ARGUMENT; + } error = vm_map_copyin(map, (vm_map_address_t)address, - (vm_map_size_t)size, FALSE, ©); + (vm_map_size_t)size, FALSE, ©); if (KERN_SUCCESS == error) { error = vm_map_copy_overwrite(current_thread()->map, - (vm_map_address_t)data, - copy, FALSE); + (vm_map_address_t)data, + copy, FALSE); if (KERN_SUCCESS == error) { *data_size = size; return error; } vm_map_copy_discard(copy); } - return(error); + return error; } @@ -827,16 +862,17 @@ vm_read_overwrite( */ kern_return_t mach_vm_write( - vm_map_t map, - mach_vm_address_t address, - pointer_t data, - __unused mach_msg_type_number_t size) + vm_map_t map, + mach_vm_address_t address, + pointer_t data, + __unused mach_msg_type_number_t size) { - if (map == VM_MAP_NULL) + if (map == VM_MAP_NULL) { return KERN_INVALID_ARGUMENT; + } return vm_map_copy_overwrite(map, (vm_map_address_t)address, - (vm_map_copy_t) data, FALSE /* interruptible XXX */); + (vm_map_copy_t) data, FALSE /* interruptible XXX */); } /* @@ -851,16 +887,17 @@ mach_vm_write( */ kern_return_t vm_write( - vm_map_t map, - vm_address_t address, - pointer_t data, - __unused mach_msg_type_number_t size) + vm_map_t map, + vm_address_t address, + pointer_t data, + __unused mach_msg_type_number_t size) { - if (map == VM_MAP_NULL) + if (map == VM_MAP_NULL) { return KERN_INVALID_ARGUMENT; + } return vm_map_copy_overwrite(map, (vm_map_address_t)address, - (vm_map_copy_t) data, FALSE /* interruptible XXX */); + (vm_map_copy_t) data, FALSE /* interruptible XXX */); } /* @@ -871,54 +908,58 @@ vm_write( */ kern_return_t mach_vm_copy( - vm_map_t map, - mach_vm_address_t source_address, - mach_vm_size_t size, - mach_vm_address_t dest_address) + vm_map_t map, + mach_vm_address_t source_address, + mach_vm_size_t size, + mach_vm_address_t dest_address) { vm_map_copy_t copy; kern_return_t kr; - if (map == VM_MAP_NULL) + if (map == VM_MAP_NULL) { return KERN_INVALID_ARGUMENT; + } kr = vm_map_copyin(map, (vm_map_address_t)source_address, - (vm_map_size_t)size, FALSE, ©); + (vm_map_size_t)size, FALSE, ©); if (KERN_SUCCESS == kr) { kr = vm_map_copy_overwrite(map, - (vm_map_address_t)dest_address, - copy, FALSE /* interruptible XXX */); + (vm_map_address_t)dest_address, + copy, FALSE /* interruptible XXX */); - if (KERN_SUCCESS != kr) + if (KERN_SUCCESS != kr) { vm_map_copy_discard(copy); + } } return kr; } kern_return_t vm_copy( - vm_map_t map, - vm_address_t source_address, - vm_size_t size, - vm_address_t dest_address) + vm_map_t map, + vm_address_t source_address, + vm_size_t size, + vm_address_t dest_address) { vm_map_copy_t copy; kern_return_t kr; - if (map == VM_MAP_NULL) + if (map == VM_MAP_NULL) { return KERN_INVALID_ARGUMENT; + } kr = vm_map_copyin(map, (vm_map_address_t)source_address, - (vm_map_size_t)size, FALSE, ©); + (vm_map_size_t)size, FALSE, ©); if (KERN_SUCCESS == kr) { kr = vm_map_copy_overwrite(map, - (vm_map_address_t)dest_address, - copy, FALSE /* interruptible XXX */); + (vm_map_address_t)dest_address, + copy, FALSE /* interruptible XXX */); - if (KERN_SUCCESS != kr) + if (KERN_SUCCESS != kr) { vm_map_copy_discard(copy); + } } return kr; } @@ -936,62 +977,67 @@ vm_copy( */ kern_return_t mach_vm_map_external( - vm_map_t target_map, - mach_vm_offset_t *address, - mach_vm_size_t initial_size, - mach_vm_offset_t mask, - int flags, - ipc_port_t port, - vm_object_offset_t offset, - boolean_t copy, - vm_prot_t cur_protection, - vm_prot_t max_protection, - vm_inherit_t inheritance) + vm_map_t target_map, + mach_vm_offset_t *address, + mach_vm_size_t initial_size, + mach_vm_offset_t mask, + int flags, + ipc_port_t port, + vm_object_offset_t offset, + boolean_t copy, + vm_prot_t cur_protection, + vm_prot_t max_protection, + vm_inherit_t inheritance) { vm_tag_t tag; VM_GET_FLAGS_ALIAS(flags, tag); - return (mach_vm_map_kernel(target_map, address, initial_size, mask, flags, tag, port, - offset, copy, cur_protection, max_protection, inheritance)); + return mach_vm_map_kernel(target_map, address, initial_size, mask, + flags, VM_MAP_KERNEL_FLAGS_NONE, tag, + port, offset, copy, + cur_protection, max_protection, + inheritance); } kern_return_t mach_vm_map_kernel( - vm_map_t target_map, - mach_vm_offset_t *address, - mach_vm_size_t initial_size, - mach_vm_offset_t mask, - int flags, - vm_tag_t tag, - ipc_port_t port, - vm_object_offset_t offset, - boolean_t copy, - vm_prot_t cur_protection, - vm_prot_t max_protection, - vm_inherit_t inheritance) + vm_map_t target_map, + mach_vm_offset_t *address, + mach_vm_size_t initial_size, + mach_vm_offset_t mask, + int flags, + vm_map_kernel_flags_t vmk_flags, + vm_tag_t tag, + ipc_port_t port, + vm_object_offset_t offset, + boolean_t copy, + vm_prot_t cur_protection, + vm_prot_t max_protection, + vm_inherit_t inheritance) { - kern_return_t kr; - vm_map_offset_t vmmaddr; + kern_return_t kr; + vm_map_offset_t vmmaddr; vmmaddr = (vm_map_offset_t) *address; /* filter out any kernel-only flags */ - if (flags & ~VM_FLAGS_USER_MAP) + if (flags & ~VM_FLAGS_USER_MAP) { return KERN_INVALID_ARGUMENT; + } kr = vm_map_enter_mem_object(target_map, - &vmmaddr, - initial_size, - mask, - flags, - VM_MAP_KERNEL_FLAGS_NONE, - tag, - port, - offset, - copy, - cur_protection, - max_protection, - inheritance); + &vmmaddr, + initial_size, + mask, + flags, + vmk_flags, + tag, + port, + offset, + copy, + cur_protection, + max_protection, + inheritance); #if KASAN if (kr == KERN_SUCCESS && target_map->pmap == kernel_pmap) { @@ -1007,39 +1053,43 @@ mach_vm_map_kernel( /* legacy interface */ kern_return_t vm_map_64_external( - vm_map_t target_map, - vm_offset_t *address, - vm_size_t size, - vm_offset_t mask, - int flags, - ipc_port_t port, - vm_object_offset_t offset, - boolean_t copy, - vm_prot_t cur_protection, - vm_prot_t max_protection, - vm_inherit_t inheritance) + vm_map_t target_map, + vm_offset_t *address, + vm_size_t size, + vm_offset_t mask, + int flags, + ipc_port_t port, + vm_object_offset_t offset, + boolean_t copy, + vm_prot_t cur_protection, + vm_prot_t max_protection, + vm_inherit_t inheritance) { vm_tag_t tag; VM_GET_FLAGS_ALIAS(flags, tag); - return (vm_map_64_kernel(target_map, address, size, mask, flags, tag, port, offset, - copy, cur_protection, max_protection, inheritance)); + return vm_map_64_kernel(target_map, address, size, mask, + flags, VM_MAP_KERNEL_FLAGS_NONE, + tag, port, offset, copy, + cur_protection, max_protection, + inheritance); } kern_return_t vm_map_64_kernel( - vm_map_t target_map, - vm_offset_t *address, - vm_size_t size, - vm_offset_t mask, - int flags, - vm_tag_t tag, - ipc_port_t port, - vm_object_offset_t offset, - boolean_t copy, - vm_prot_t cur_protection, - vm_prot_t max_protection, - vm_inherit_t inheritance) + vm_map_t target_map, + vm_offset_t *address, + vm_size_t size, + vm_offset_t mask, + int flags, + vm_map_kernel_flags_t vmk_flags, + vm_tag_t tag, + ipc_port_t port, + vm_object_offset_t offset, + boolean_t copy, + vm_prot_t cur_protection, + vm_prot_t max_protection, + vm_inherit_t inheritance) { mach_vm_address_t map_addr; mach_vm_size_t map_size; @@ -1050,9 +1100,10 @@ vm_map_64_kernel( map_size = (mach_vm_size_t)size; map_mask = (mach_vm_offset_t)mask; - kr = mach_vm_map_kernel(target_map, &map_addr, map_size, map_mask, flags, tag, - port, offset, copy, - cur_protection, max_protection, inheritance); + kr = mach_vm_map_kernel(target_map, &map_addr, map_size, map_mask, + flags, vmk_flags, tag, + port, offset, copy, + cur_protection, max_protection, inheritance); *address = CAST_DOWN(vm_offset_t, map_addr); return kr; } @@ -1060,38 +1111,42 @@ vm_map_64_kernel( /* temporary, until world build */ kern_return_t vm_map_external( - vm_map_t target_map, - vm_offset_t *address, - vm_size_t size, - vm_offset_t mask, - int flags, - ipc_port_t port, - vm_offset_t offset, - boolean_t copy, - vm_prot_t cur_protection, - vm_prot_t max_protection, - vm_inherit_t inheritance) + vm_map_t target_map, + vm_offset_t *address, + vm_size_t size, + vm_offset_t mask, + int flags, + ipc_port_t port, + vm_offset_t offset, + boolean_t copy, + vm_prot_t cur_protection, + vm_prot_t max_protection, + vm_inherit_t inheritance) { vm_tag_t tag; VM_GET_FLAGS_ALIAS(flags, tag); - return (vm_map_kernel(target_map, address, size, mask, flags, tag, port, offset, copy, cur_protection, max_protection, inheritance)); + return vm_map_kernel(target_map, address, size, mask, + flags, VM_MAP_KERNEL_FLAGS_NONE, tag, + port, offset, copy, + cur_protection, max_protection, inheritance); } kern_return_t vm_map_kernel( - vm_map_t target_map, - vm_offset_t *address, - vm_size_t size, - vm_offset_t mask, - int flags, - vm_tag_t tag, - ipc_port_t port, - vm_offset_t offset, - boolean_t copy, - vm_prot_t cur_protection, - vm_prot_t max_protection, - vm_inherit_t inheritance) + vm_map_t target_map, + vm_offset_t *address, + vm_size_t size, + vm_offset_t mask, + int flags, + vm_map_kernel_flags_t vmk_flags, + vm_tag_t tag, + ipc_port_t port, + vm_offset_t offset, + boolean_t copy, + vm_prot_t cur_protection, + vm_prot_t max_protection, + vm_inherit_t inheritance) { mach_vm_address_t map_addr; mach_vm_size_t map_size; @@ -1104,9 +1159,10 @@ vm_map_kernel( map_mask = (mach_vm_offset_t)mask; obj_offset = (vm_object_offset_t)offset; - kr = mach_vm_map_kernel(target_map, &map_addr, map_size, map_mask, flags, tag, - port, obj_offset, copy, - cur_protection, max_protection, inheritance); + kr = mach_vm_map_kernel(target_map, &map_addr, map_size, map_mask, + flags, vmk_flags, tag, + port, obj_offset, copy, + cur_protection, max_protection, inheritance); *address = CAST_DOWN(vm_offset_t, map_addr); return kr; } @@ -1120,65 +1176,67 @@ vm_map_kernel( */ kern_return_t mach_vm_remap_external( - vm_map_t target_map, - mach_vm_offset_t *address, - mach_vm_size_t size, - mach_vm_offset_t mask, - int flags, - vm_map_t src_map, - mach_vm_offset_t memory_address, - boolean_t copy, - vm_prot_t *cur_protection, - vm_prot_t *max_protection, - vm_inherit_t inheritance) + vm_map_t target_map, + mach_vm_offset_t *address, + mach_vm_size_t size, + mach_vm_offset_t mask, + int flags, + vm_map_t src_map, + mach_vm_offset_t memory_address, + boolean_t copy, + vm_prot_t *cur_protection, + vm_prot_t *max_protection, + vm_inherit_t inheritance) { vm_tag_t tag; VM_GET_FLAGS_ALIAS(flags, tag); - return (mach_vm_remap_kernel(target_map, address, size, mask, flags, tag, src_map, memory_address, - copy, cur_protection, max_protection, inheritance)); + return mach_vm_remap_kernel(target_map, address, size, mask, flags, tag, src_map, memory_address, + copy, cur_protection, max_protection, inheritance); } kern_return_t mach_vm_remap_kernel( - vm_map_t target_map, - mach_vm_offset_t *address, - mach_vm_size_t size, - mach_vm_offset_t mask, - int flags, - vm_tag_t tag, - vm_map_t src_map, - mach_vm_offset_t memory_address, - boolean_t copy, - vm_prot_t *cur_protection, - vm_prot_t *max_protection, - vm_inherit_t inheritance) + vm_map_t target_map, + mach_vm_offset_t *address, + mach_vm_size_t size, + mach_vm_offset_t mask, + int flags, + vm_tag_t tag, + vm_map_t src_map, + mach_vm_offset_t memory_address, + boolean_t copy, + vm_prot_t *cur_protection, + vm_prot_t *max_protection, + vm_inherit_t inheritance) { - vm_map_offset_t map_addr; - kern_return_t kr; + vm_map_offset_t map_addr; + kern_return_t kr; - if (VM_MAP_NULL == target_map || VM_MAP_NULL == src_map) + if (VM_MAP_NULL == target_map || VM_MAP_NULL == src_map) { return KERN_INVALID_ARGUMENT; + } /* filter out any kernel-only flags */ - if (flags & ~VM_FLAGS_USER_REMAP) + if (flags & ~VM_FLAGS_USER_REMAP) { return KERN_INVALID_ARGUMENT; + } map_addr = (vm_map_offset_t)*address; kr = vm_map_remap(target_map, - &map_addr, - size, - mask, - flags, - VM_MAP_KERNEL_FLAGS_NONE, - tag, - src_map, - memory_address, - copy, - cur_protection, - max_protection, - inheritance); + &map_addr, + size, + mask, + flags, + VM_MAP_KERNEL_FLAGS_NONE, + tag, + src_map, + memory_address, + copy, + cur_protection, + max_protection, + inheritance); *address = map_addr; return kr; } @@ -1196,65 +1254,67 @@ mach_vm_remap_kernel( */ kern_return_t vm_remap_external( - vm_map_t target_map, - vm_offset_t *address, - vm_size_t size, - vm_offset_t mask, - int flags, - vm_map_t src_map, - vm_offset_t memory_address, - boolean_t copy, - vm_prot_t *cur_protection, - vm_prot_t *max_protection, - vm_inherit_t inheritance) + vm_map_t target_map, + vm_offset_t *address, + vm_size_t size, + vm_offset_t mask, + int flags, + vm_map_t src_map, + vm_offset_t memory_address, + boolean_t copy, + vm_prot_t *cur_protection, + vm_prot_t *max_protection, + vm_inherit_t inheritance) { vm_tag_t tag; VM_GET_FLAGS_ALIAS(flags, tag); - return (vm_remap_kernel(target_map, address, size, mask, flags, tag, src_map, - memory_address, copy, cur_protection, max_protection, inheritance)); + return vm_remap_kernel(target_map, address, size, mask, flags, tag, src_map, + memory_address, copy, cur_protection, max_protection, inheritance); } kern_return_t vm_remap_kernel( - vm_map_t target_map, - vm_offset_t *address, - vm_size_t size, - vm_offset_t mask, - int flags, - vm_tag_t tag, - vm_map_t src_map, - vm_offset_t memory_address, - boolean_t copy, - vm_prot_t *cur_protection, - vm_prot_t *max_protection, - vm_inherit_t inheritance) + vm_map_t target_map, + vm_offset_t *address, + vm_size_t size, + vm_offset_t mask, + int flags, + vm_tag_t tag, + vm_map_t src_map, + vm_offset_t memory_address, + boolean_t copy, + vm_prot_t *cur_protection, + vm_prot_t *max_protection, + vm_inherit_t inheritance) { - vm_map_offset_t map_addr; - kern_return_t kr; + vm_map_offset_t map_addr; + kern_return_t kr; - if (VM_MAP_NULL == target_map || VM_MAP_NULL == src_map) + if (VM_MAP_NULL == target_map || VM_MAP_NULL == src_map) { return KERN_INVALID_ARGUMENT; + } /* filter out any kernel-only flags */ - if (flags & ~VM_FLAGS_USER_REMAP) + if (flags & ~VM_FLAGS_USER_REMAP) { return KERN_INVALID_ARGUMENT; + } map_addr = (vm_map_offset_t)*address; kr = vm_map_remap(target_map, - &map_addr, - size, - mask, - flags, - VM_MAP_KERNEL_FLAGS_NONE, - tag, - src_map, - memory_address, - copy, - cur_protection, - max_protection, - inheritance); + &map_addr, + size, + mask, + flags, + VM_MAP_KERNEL_FLAGS_NONE, + tag, + src_map, + memory_address, + copy, + cur_protection, + max_protection, + inheritance); *address = CAST_DOWN(vm_offset_t, map_addr); return kr; } @@ -1274,52 +1334,55 @@ vm_remap_kernel( */ kern_return_t mach_vm_wire_external( - host_priv_t host_priv, - vm_map_t map, - mach_vm_offset_t start, - mach_vm_size_t size, - vm_prot_t access) + host_priv_t host_priv, + vm_map_t map, + mach_vm_offset_t start, + mach_vm_size_t size, + vm_prot_t access) { - return (mach_vm_wire_kernel(host_priv, map, start, size, access, VM_KERN_MEMORY_MLOCK)); + return mach_vm_wire_kernel(host_priv, map, start, size, access, VM_KERN_MEMORY_MLOCK); } kern_return_t mach_vm_wire_kernel( - host_priv_t host_priv, - vm_map_t map, - mach_vm_offset_t start, - mach_vm_size_t size, - vm_prot_t access, - vm_tag_t tag) + host_priv_t host_priv, + vm_map_t map, + mach_vm_offset_t start, + mach_vm_size_t size, + vm_prot_t access, + vm_tag_t tag) { - kern_return_t rc; + kern_return_t rc; - if (host_priv == HOST_PRIV_NULL) + if (host_priv == HOST_PRIV_NULL) { return KERN_INVALID_HOST; + } assert(host_priv == &realhost); - if (map == VM_MAP_NULL) + if (map == VM_MAP_NULL) { return KERN_INVALID_TASK; + } - if (access & ~VM_PROT_ALL || (start + size < start)) + if (access & ~VM_PROT_ALL || (start + size < start)) { return KERN_INVALID_ARGUMENT; + } if (access != VM_PROT_NONE) { rc = vm_map_wire_kernel(map, - vm_map_trunc_page(start, - VM_MAP_PAGE_MASK(map)), - vm_map_round_page(start+size, - VM_MAP_PAGE_MASK(map)), - access, tag, - TRUE); + vm_map_trunc_page(start, + VM_MAP_PAGE_MASK(map)), + vm_map_round_page(start + size, + VM_MAP_PAGE_MASK(map)), + access, tag, + TRUE); } else { rc = vm_map_unwire(map, - vm_map_trunc_page(start, - VM_MAP_PAGE_MASK(map)), - vm_map_round_page(start+size, - VM_MAP_PAGE_MASK(map)), - TRUE); + vm_map_trunc_page(start, + VM_MAP_PAGE_MASK(map)), + vm_map_round_page(start + size, + VM_MAP_PAGE_MASK(map)), + TRUE); } return rc; } @@ -1334,42 +1397,45 @@ mach_vm_wire_kernel( */ kern_return_t vm_wire( - host_priv_t host_priv, - vm_map_t map, - vm_offset_t start, - vm_size_t size, - vm_prot_t access) + host_priv_t host_priv, + vm_map_t map, + vm_offset_t start, + vm_size_t size, + vm_prot_t access) { - kern_return_t rc; + kern_return_t rc; - if (host_priv == HOST_PRIV_NULL) + if (host_priv == HOST_PRIV_NULL) { return KERN_INVALID_HOST; + } assert(host_priv == &realhost); - if (map == VM_MAP_NULL) + if (map == VM_MAP_NULL) { return KERN_INVALID_TASK; + } - if ((access & ~VM_PROT_ALL) || (start + size < start)) + if ((access & ~VM_PROT_ALL) || (start + size < start)) { return KERN_INVALID_ARGUMENT; + } if (size == 0) { rc = KERN_SUCCESS; } else if (access != VM_PROT_NONE) { rc = vm_map_wire_kernel(map, - vm_map_trunc_page(start, - VM_MAP_PAGE_MASK(map)), - vm_map_round_page(start+size, - VM_MAP_PAGE_MASK(map)), - access, VM_KERN_MEMORY_OSFMK, - TRUE); + vm_map_trunc_page(start, + VM_MAP_PAGE_MASK(map)), + vm_map_round_page(start + size, + VM_MAP_PAGE_MASK(map)), + access, VM_KERN_MEMORY_OSFMK, + TRUE); } else { rc = vm_map_unwire(map, - vm_map_trunc_page(start, - VM_MAP_PAGE_MASK(map)), - vm_map_round_page(start+size, - VM_MAP_PAGE_MASK(map)), - TRUE); + vm_map_trunc_page(start, + VM_MAP_PAGE_MASK(map)), + vm_map_round_page(start + size, + VM_MAP_PAGE_MASK(map)), + TRUE); } return rc; } @@ -1407,19 +1473,19 @@ vm_wire( kern_return_t mach_vm_msync( - vm_map_t map, - mach_vm_address_t address, - mach_vm_size_t size, - vm_sync_t sync_flags) + vm_map_t map, + mach_vm_address_t address, + mach_vm_size_t size, + vm_sync_t sync_flags) { - - if (map == VM_MAP_NULL) - return(KERN_INVALID_TASK); + if (map == VM_MAP_NULL) { + return KERN_INVALID_TASK; + } return vm_map_msync(map, (vm_map_address_t)address, - (vm_map_size_t)size, sync_flags); + (vm_map_size_t)size, sync_flags); } - + /* * vm_msync * @@ -1456,17 +1522,17 @@ mach_vm_msync( kern_return_t vm_msync( - vm_map_t map, - vm_address_t address, - vm_size_t size, - vm_sync_t sync_flags) + vm_map_t map, + vm_address_t address, + vm_size_t size, + vm_sync_t sync_flags) { - - if (map == VM_MAP_NULL) - return(KERN_INVALID_TASK); + if (map == VM_MAP_NULL) { + return KERN_INVALID_TASK; + } return vm_map_msync(map, (vm_map_address_t)address, - (vm_map_size_t)size, sync_flags); + (vm_map_size_t)size, sync_flags); } @@ -1474,11 +1540,11 @@ int vm_toggle_entry_reuse(int toggle, int *old_value) { vm_map_t map = current_map(); - + assert(!map->is_nested_map); - if(toggle == VM_TOGGLE_GETVALUE && old_value != NULL){ + if (toggle == VM_TOGGLE_GETVALUE && old_value != NULL) { *old_value = map->disable_vmentry_reuse; - } else if(toggle == VM_TOGGLE_SET){ + } else if (toggle == VM_TOGGLE_SET) { vm_map_entry_t map_to_entry; vm_map_lock(map); @@ -1491,18 +1557,19 @@ vm_toggle_entry_reuse(int toggle, int *old_value) map->highest_entry_end = map->first_free->vme_end; } vm_map_unlock(map); - } else if (toggle == VM_TOGGLE_CLEAR){ + } else if (toggle == VM_TOGGLE_CLEAR) { vm_map_lock(map); map->disable_vmentry_reuse = FALSE; vm_map_unlock(map); - } else + } else { return KERN_INVALID_ARGUMENT; + } return KERN_SUCCESS; } /* - * mach_vm_behavior_set + * mach_vm_behavior_set * * Sets the paging behavior attribute for the specified range * in the specified map. @@ -1510,20 +1577,22 @@ vm_toggle_entry_reuse(int toggle, int *old_value) * This routine will fail with KERN_INVALID_ADDRESS if any address * in [start,start+size) is not a valid allocated memory region. */ -kern_return_t +kern_return_t mach_vm_behavior_set( - vm_map_t map, - mach_vm_offset_t start, - mach_vm_size_t size, - vm_behavior_t new_behavior) + vm_map_t map, + mach_vm_offset_t start, + mach_vm_size_t size, + vm_behavior_t new_behavior) { - vm_map_offset_t align_mask; + vm_map_offset_t align_mask; - if ((map == VM_MAP_NULL) || (start + size < start)) - return(KERN_INVALID_ARGUMENT); + if ((map == VM_MAP_NULL) || (start + size < start)) { + return KERN_INVALID_ARGUMENT; + } - if (size == 0) + if (size == 0) { return KERN_SUCCESS; + } switch (new_behavior) { case VM_BEHAVIOR_REUSABLE: @@ -1542,13 +1611,13 @@ mach_vm_behavior_set( } return vm_map_behavior_set(map, - vm_map_trunc_page(start, align_mask), - vm_map_round_page(start+size, align_mask), - new_behavior); + vm_map_trunc_page(start, align_mask), + vm_map_round_page(start + size, align_mask), + new_behavior); } /* - * vm_behavior_set + * vm_behavior_set * * Sets the paging behavior attribute for the specified range * in the specified map. @@ -1560,20 +1629,21 @@ mach_vm_behavior_set( * use of vm_offset_t (if the map provided is larger than the * kernel's). */ -kern_return_t +kern_return_t vm_behavior_set( - vm_map_t map, - vm_offset_t start, - vm_size_t size, - vm_behavior_t new_behavior) + vm_map_t map, + vm_offset_t start, + vm_size_t size, + vm_behavior_t new_behavior) { - if (start + size < start) + if (start + size < start) { return KERN_INVALID_ARGUMENT; + } return mach_vm_behavior_set(map, - (mach_vm_offset_t) start, - (mach_vm_size_t) size, - new_behavior); + (mach_vm_offset_t) start, + (mach_vm_size_t) size, + new_behavior); } /* @@ -1592,32 +1662,34 @@ vm_behavior_set( kern_return_t mach_vm_region( - vm_map_t map, - mach_vm_offset_t *address, /* IN/OUT */ - mach_vm_size_t *size, /* OUT */ - vm_region_flavor_t flavor, /* IN */ - vm_region_info_t info, /* OUT */ - mach_msg_type_number_t *count, /* IN/OUT */ - mach_port_t *object_name) /* OUT */ + vm_map_t map, + mach_vm_offset_t *address, /* IN/OUT */ + mach_vm_size_t *size, /* OUT */ + vm_region_flavor_t flavor, /* IN */ + vm_region_info_t info, /* OUT */ + mach_msg_type_number_t *count, /* IN/OUT */ + mach_port_t *object_name) /* OUT */ { - vm_map_offset_t map_addr; - vm_map_size_t map_size; - kern_return_t kr; + vm_map_offset_t map_addr; + vm_map_size_t map_size; + kern_return_t kr; - if (VM_MAP_NULL == map) + if (VM_MAP_NULL == map) { return KERN_INVALID_ARGUMENT; + } map_addr = (vm_map_offset_t)*address; map_size = (vm_map_size_t)*size; /* legacy conversion */ - if (VM_REGION_BASIC_INFO == flavor) + if (VM_REGION_BASIC_INFO == flavor) { flavor = VM_REGION_BASIC_INFO_64; + } kr = vm_map_region(map, - &map_addr, &map_size, - flavor, info, count, - object_name); + &map_addr, &map_size, + flavor, info, count, + object_name); *address = map_addr; *size = map_size; @@ -1640,71 +1712,76 @@ mach_vm_region( kern_return_t vm_region_64( - vm_map_t map, - vm_offset_t *address, /* IN/OUT */ - vm_size_t *size, /* OUT */ - vm_region_flavor_t flavor, /* IN */ - vm_region_info_t info, /* OUT */ - mach_msg_type_number_t *count, /* IN/OUT */ - mach_port_t *object_name) /* OUT */ + vm_map_t map, + vm_offset_t *address, /* IN/OUT */ + vm_size_t *size, /* OUT */ + vm_region_flavor_t flavor, /* IN */ + vm_region_info_t info, /* OUT */ + mach_msg_type_number_t *count, /* IN/OUT */ + mach_port_t *object_name) /* OUT */ { - vm_map_offset_t map_addr; - vm_map_size_t map_size; - kern_return_t kr; + vm_map_offset_t map_addr; + vm_map_size_t map_size; + kern_return_t kr; - if (VM_MAP_NULL == map) + if (VM_MAP_NULL == map) { return KERN_INVALID_ARGUMENT; + } map_addr = (vm_map_offset_t)*address; map_size = (vm_map_size_t)*size; /* legacy conversion */ - if (VM_REGION_BASIC_INFO == flavor) + if (VM_REGION_BASIC_INFO == flavor) { flavor = VM_REGION_BASIC_INFO_64; + } kr = vm_map_region(map, - &map_addr, &map_size, - flavor, info, count, - object_name); + &map_addr, &map_size, + flavor, info, count, + object_name); *address = CAST_DOWN(vm_offset_t, map_addr); *size = CAST_DOWN(vm_size_t, map_size); - if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS) + if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS) { return KERN_INVALID_ADDRESS; + } return kr; } kern_return_t vm_region( - vm_map_t map, - vm_address_t *address, /* IN/OUT */ - vm_size_t *size, /* OUT */ - vm_region_flavor_t flavor, /* IN */ - vm_region_info_t info, /* OUT */ - mach_msg_type_number_t *count, /* IN/OUT */ - mach_port_t *object_name) /* OUT */ + vm_map_t map, + vm_address_t *address, /* IN/OUT */ + vm_size_t *size, /* OUT */ + vm_region_flavor_t flavor, /* IN */ + vm_region_info_t info, /* OUT */ + mach_msg_type_number_t *count, /* IN/OUT */ + mach_port_t *object_name) /* OUT */ { - vm_map_address_t map_addr; - vm_map_size_t map_size; - kern_return_t kr; + vm_map_address_t map_addr; + vm_map_size_t map_size; + kern_return_t kr; - if (VM_MAP_NULL == map) + if (VM_MAP_NULL == map) { return KERN_INVALID_ARGUMENT; + } map_addr = (vm_map_address_t)*address; map_size = (vm_map_size_t)*size; kr = vm_map_region(map, - &map_addr, &map_size, - flavor, info, count, - object_name); + &map_addr, &map_size, + flavor, info, count, + object_name); *address = CAST_DOWN(vm_address_t, map_addr); *size = CAST_DOWN(vm_size_t, map_size); - if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS) + if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS) { return KERN_INVALID_ADDRESS; + } return kr; } @@ -1715,30 +1792,31 @@ vm_region( */ kern_return_t mach_vm_region_recurse( - vm_map_t map, - mach_vm_address_t *address, - mach_vm_size_t *size, - uint32_t *depth, - vm_region_recurse_info_t info, - mach_msg_type_number_t *infoCnt) + vm_map_t map, + mach_vm_address_t *address, + mach_vm_size_t *size, + uint32_t *depth, + vm_region_recurse_info_t info, + mach_msg_type_number_t *infoCnt) { - vm_map_address_t map_addr; - vm_map_size_t map_size; - kern_return_t kr; + vm_map_address_t map_addr; + vm_map_size_t map_size; + kern_return_t kr; - if (VM_MAP_NULL == map) + if (VM_MAP_NULL == map) { return KERN_INVALID_ARGUMENT; + } map_addr = (vm_map_address_t)*address; map_size = (vm_map_size_t)*size; kr = vm_map_region_recurse_64( - map, - &map_addr, - &map_size, - depth, - (vm_region_submap_info_64_t)info, - infoCnt); + map, + &map_addr, + &map_size, + depth, + (vm_region_submap_info_64_t)info, + infoCnt); *address = map_addr; *size = map_size; @@ -1752,102 +1830,107 @@ mach_vm_region_recurse( */ kern_return_t vm_region_recurse_64( - vm_map_t map, - vm_address_t *address, - vm_size_t *size, - uint32_t *depth, - vm_region_recurse_info_64_t info, - mach_msg_type_number_t *infoCnt) + vm_map_t map, + vm_address_t *address, + vm_size_t *size, + uint32_t *depth, + vm_region_recurse_info_64_t info, + mach_msg_type_number_t *infoCnt) { - vm_map_address_t map_addr; - vm_map_size_t map_size; - kern_return_t kr; + vm_map_address_t map_addr; + vm_map_size_t map_size; + kern_return_t kr; - if (VM_MAP_NULL == map) + if (VM_MAP_NULL == map) { return KERN_INVALID_ARGUMENT; + } map_addr = (vm_map_address_t)*address; map_size = (vm_map_size_t)*size; kr = vm_map_region_recurse_64( - map, - &map_addr, - &map_size, - depth, - (vm_region_submap_info_64_t)info, - infoCnt); + map, + &map_addr, + &map_size, + depth, + (vm_region_submap_info_64_t)info, + infoCnt); *address = CAST_DOWN(vm_address_t, map_addr); *size = CAST_DOWN(vm_size_t, map_size); - if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS) + if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS) { return KERN_INVALID_ADDRESS; + } return kr; } kern_return_t vm_region_recurse( - vm_map_t map, - vm_offset_t *address, /* IN/OUT */ - vm_size_t *size, /* OUT */ - natural_t *depth, /* IN/OUT */ - vm_region_recurse_info_t info32, /* IN/OUT */ - mach_msg_type_number_t *infoCnt) /* IN/OUT */ + vm_map_t map, + vm_offset_t *address, /* IN/OUT */ + vm_size_t *size, /* OUT */ + natural_t *depth, /* IN/OUT */ + vm_region_recurse_info_t info32, /* IN/OUT */ + mach_msg_type_number_t *infoCnt) /* IN/OUT */ { vm_region_submap_info_data_64_t info64; vm_region_submap_info_t info; - vm_map_address_t map_addr; - vm_map_size_t map_size; - kern_return_t kr; + vm_map_address_t map_addr; + vm_map_size_t map_size; + kern_return_t kr; - if (VM_MAP_NULL == map || *infoCnt < VM_REGION_SUBMAP_INFO_COUNT) + if (VM_MAP_NULL == map || *infoCnt < VM_REGION_SUBMAP_INFO_COUNT) { return KERN_INVALID_ARGUMENT; + } + - map_addr = (vm_map_address_t)*address; map_size = (vm_map_size_t)*size; info = (vm_region_submap_info_t)info32; *infoCnt = VM_REGION_SUBMAP_INFO_COUNT_64; - kr = vm_map_region_recurse_64(map, &map_addr,&map_size, - depth, &info64, infoCnt); + kr = vm_map_region_recurse_64(map, &map_addr, &map_size, + depth, &info64, infoCnt); info->protection = info64.protection; info->max_protection = info64.max_protection; info->inheritance = info64.inheritance; info->offset = (uint32_t)info64.offset; /* trouble-maker */ - info->user_tag = info64.user_tag; - info->pages_resident = info64.pages_resident; - info->pages_shared_now_private = info64.pages_shared_now_private; - info->pages_swapped_out = info64.pages_swapped_out; - info->pages_dirtied = info64.pages_dirtied; - info->ref_count = info64.ref_count; - info->shadow_depth = info64.shadow_depth; - info->external_pager = info64.external_pager; - info->share_mode = info64.share_mode; + info->user_tag = info64.user_tag; + info->pages_resident = info64.pages_resident; + info->pages_shared_now_private = info64.pages_shared_now_private; + info->pages_swapped_out = info64.pages_swapped_out; + info->pages_dirtied = info64.pages_dirtied; + info->ref_count = info64.ref_count; + info->shadow_depth = info64.shadow_depth; + info->external_pager = info64.external_pager; + info->share_mode = info64.share_mode; info->is_submap = info64.is_submap; info->behavior = info64.behavior; info->object_id = info64.object_id; - info->user_wired_count = info64.user_wired_count; + info->user_wired_count = info64.user_wired_count; *address = CAST_DOWN(vm_address_t, map_addr); *size = CAST_DOWN(vm_size_t, map_size); *infoCnt = VM_REGION_SUBMAP_INFO_COUNT; - if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS) + if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS) { return KERN_INVALID_ADDRESS; + } return kr; } kern_return_t mach_vm_purgable_control( - vm_map_t map, - mach_vm_offset_t address, - vm_purgable_t control, - int *state) + vm_map_t map, + mach_vm_offset_t address, + vm_purgable_t control, + int *state) { - if (VM_MAP_NULL == map) + if (VM_MAP_NULL == map) { return KERN_INVALID_ARGUMENT; + } if (control == VM_PURGABLE_SET_STATE_FROM_KERNEL) { /* not allowed from user-space */ @@ -1855,20 +1938,21 @@ mach_vm_purgable_control( } return vm_map_purgable_control(map, - vm_map_trunc_page(address, PAGE_MASK), - control, - state); + vm_map_trunc_page(address, PAGE_MASK), + control, + state); } kern_return_t vm_purgable_control( - vm_map_t map, - vm_offset_t address, - vm_purgable_t control, - int *state) + vm_map_t map, + vm_offset_t address, + vm_purgable_t control, + int *state) { - if (VM_MAP_NULL == map) + if (VM_MAP_NULL == map) { return KERN_INVALID_ARGUMENT; + } if (control == VM_PURGABLE_SET_STATE_FROM_KERNEL) { /* not allowed from user-space */ @@ -1876,11 +1960,11 @@ vm_purgable_control( } return vm_map_purgable_control(map, - vm_map_trunc_page(address, PAGE_MASK), - control, - state); + vm_map_trunc_page(address, PAGE_MASK), + control, + state); } - + /* * Ordinarily, the right to allocate CPM is restricted @@ -1888,7 +1972,7 @@ vm_purgable_control( * to the host priv port). Set this variable to zero if * you want to let any application allocate CPM. */ -unsigned int vm_allocate_cpm_privileged = 0; +unsigned int vm_allocate_cpm_privileged = 0; /* * Allocate memory in the specified map, with the caveat that @@ -1902,29 +1986,31 @@ unsigned int vm_allocate_cpm_privileged = 0; */ kern_return_t vm_allocate_cpm( - host_priv_t host_priv, - vm_map_t map, - vm_address_t *addr, - vm_size_t size, - int flags) + host_priv_t host_priv, + vm_map_t map, + vm_address_t *addr, + vm_size_t size, + int flags) { - vm_map_address_t map_addr; - vm_map_size_t map_size; - kern_return_t kr; + vm_map_address_t map_addr; + vm_map_size_t map_size; + kern_return_t kr; - if (vm_allocate_cpm_privileged && HOST_PRIV_NULL == host_priv) + if (vm_allocate_cpm_privileged && HOST_PRIV_NULL == host_priv) { return KERN_INVALID_HOST; + } - if (VM_MAP_NULL == map) + if (VM_MAP_NULL == map) { return KERN_INVALID_ARGUMENT; + } map_addr = (vm_map_address_t)*addr; map_size = (vm_map_size_t)size; kr = vm_map_enter_cpm(map, - &map_addr, - map_size, - flags); + &map_addr, + map_size, + flags); *addr = CAST_DOWN(vm_address_t, map_addr); return kr; @@ -1933,13 +2019,14 @@ vm_allocate_cpm( kern_return_t mach_vm_page_query( - vm_map_t map, - mach_vm_offset_t offset, - int *disposition, - int *ref_count) + vm_map_t map, + mach_vm_offset_t offset, + int *disposition, + int *ref_count) { - if (VM_MAP_NULL == map) + if (VM_MAP_NULL == map) { return KERN_INVALID_ARGUMENT; + } return vm_map_page_query_internal( map, @@ -1949,13 +2036,14 @@ mach_vm_page_query( kern_return_t vm_map_page_query( - vm_map_t map, - vm_offset_t offset, - int *disposition, - int *ref_count) + vm_map_t map, + vm_offset_t offset, + int *disposition, + int *ref_count) { - if (VM_MAP_NULL == map) + if (VM_MAP_NULL == map) { return KERN_INVALID_ARGUMENT; + } return vm_map_page_query_internal( map, @@ -1965,28 +2053,28 @@ vm_map_page_query( kern_return_t mach_vm_page_range_query( - vm_map_t map, - mach_vm_offset_t address, - mach_vm_size_t size, - mach_vm_address_t dispositions_addr, - mach_vm_size_t *dispositions_count) + vm_map_t map, + mach_vm_offset_t address, + mach_vm_size_t size, + mach_vm_address_t dispositions_addr, + mach_vm_size_t *dispositions_count) { - kern_return_t kr = KERN_SUCCESS; - int num_pages = 0, i = 0; - mach_vm_size_t curr_sz = 0, copy_sz = 0; - mach_vm_size_t disp_buf_req_size = 0, disp_buf_total_size = 0; - mach_msg_type_number_t count = 0; + kern_return_t kr = KERN_SUCCESS; + int num_pages = 0, i = 0; + mach_vm_size_t curr_sz = 0, copy_sz = 0; + mach_vm_size_t disp_buf_req_size = 0, disp_buf_total_size = 0; + mach_msg_type_number_t count = 0; - void *info = NULL; - void *local_disp = NULL;; - vm_map_size_t info_size = 0, local_disp_size = 0; - mach_vm_offset_t start = 0, end = 0; + void *info = NULL; + void *local_disp = NULL;; + vm_map_size_t info_size = 0, local_disp_size = 0; + mach_vm_offset_t start = 0, end = 0; if (map == VM_MAP_NULL || dispositions_count == NULL) { return KERN_INVALID_ARGUMENT; } - disp_buf_req_size = ( *dispositions_count * sizeof(int)); + disp_buf_req_size = (*dispositions_count * sizeof(int)); start = mach_vm_trunc_page(address); end = mach_vm_round_page(address + size); @@ -1994,6 +2082,13 @@ mach_vm_page_range_query( return KERN_INVALID_ARGUMENT; } + if ((end - start) < size) { + /* + * Aligned size is less than unaligned size. + */ + return KERN_INVALID_ARGUMENT; + } + if (disp_buf_req_size == 0 || (end == start)) { return KERN_SUCCESS; } @@ -2017,31 +2112,28 @@ mach_vm_page_range_query( local_disp = kalloc(local_disp_size); if (local_disp == NULL) { - kfree(info, info_size); info = NULL; return KERN_RESOURCE_SHORTAGE; } while (size) { - count = VM_PAGE_INFO_BASIC_COUNT; kr = vm_map_page_range_info_internal( - map, - start, - mach_vm_round_page(start + curr_sz), - VM_PAGE_INFO_BASIC, - (vm_page_info_t) info, - &count); + map, + start, + mach_vm_round_page(start + curr_sz), + VM_PAGE_INFO_BASIC, + (vm_page_info_t) info, + &count); assert(kr == KERN_SUCCESS); for (i = 0; i < num_pages; i++) { - ((int*)local_disp)[i] = ((vm_page_info_basic_t)info)[i].disposition; } - copy_sz = MIN(disp_buf_req_size, num_pages * sizeof(int)/* an int per page */); + copy_sz = MIN(disp_buf_req_size, num_pages * sizeof(int) /* an int per page */); kr = copyout(local_disp, (mach_vm_address_t)dispositions_addr, copy_sz); start += curr_sz; @@ -2053,7 +2145,6 @@ mach_vm_page_range_query( } if ((disp_buf_req_size == 0) || (curr_sz >= size)) { - /* * We might have inspected the full range OR * more than it esp. if the user passed in @@ -2062,9 +2153,7 @@ mach_vm_page_range_query( */ size = 0; - } else { - dispositions_addr += copy_sz; size -= curr_sz; @@ -2087,13 +2176,13 @@ mach_vm_page_range_query( kern_return_t mach_vm_page_info( - vm_map_t map, - mach_vm_address_t address, - vm_page_info_flavor_t flavor, - vm_page_info_t info, - mach_msg_type_number_t *count) + vm_map_t map, + mach_vm_address_t address, + vm_page_info_flavor_t flavor, + vm_page_info_t info, + mach_msg_type_number_t *count) { - kern_return_t kr; + kern_return_t kr; if (map == VM_MAP_NULL) { return KERN_INVALID_ARGUMENT; @@ -2106,15 +2195,16 @@ mach_vm_page_info( /* map a (whole) upl into an address space */ kern_return_t vm_upl_map( - vm_map_t map, - upl_t upl, - vm_address_t *dst_addr) + vm_map_t map, + upl_t upl, + vm_address_t *dst_addr) { - vm_map_offset_t map_addr; - kern_return_t kr; + vm_map_offset_t map_addr; + kern_return_t kr; - if (VM_MAP_NULL == map) + if (VM_MAP_NULL == map) { return KERN_INVALID_ARGUMENT; + } kr = vm_map_enter_upl(map, upl, &map_addr); *dst_addr = CAST_DOWN(vm_address_t, map_addr); @@ -2123,47 +2213,50 @@ vm_upl_map( kern_return_t vm_upl_unmap( - vm_map_t map, - upl_t upl) + vm_map_t map, + upl_t upl) { - if (VM_MAP_NULL == map) + if (VM_MAP_NULL == map) { return KERN_INVALID_ARGUMENT; + } - return (vm_map_remove_upl(map, upl)); + return vm_map_remove_upl(map, upl); } /* Retrieve a upl for an object underlying an address range in a map */ kern_return_t vm_map_get_upl( - vm_map_t map, - vm_map_offset_t map_offset, - upl_size_t *upl_size, - upl_t *upl, - upl_page_info_array_t page_list, - unsigned int *count, - upl_control_flags_t *flags, - vm_tag_t tag, - int force_data_sync) + vm_map_t map, + vm_map_offset_t map_offset, + upl_size_t *upl_size, + upl_t *upl, + upl_page_info_array_t page_list, + unsigned int *count, + upl_control_flags_t *flags, + vm_tag_t tag, + int force_data_sync) { upl_control_flags_t map_flags; - kern_return_t kr; + kern_return_t kr; - if (VM_MAP_NULL == map) + if (VM_MAP_NULL == map) { return KERN_INVALID_ARGUMENT; + } map_flags = *flags & ~UPL_NOZEROFILL; - if (force_data_sync) + if (force_data_sync) { map_flags |= UPL_FORCE_DATA_SYNC; + } kr = vm_map_create_upl(map, - map_offset, - upl_size, - upl, - page_list, - count, - &map_flags, - tag); + map_offset, + upl_size, + upl, + page_list, + count, + &map_flags, + tag); *flags = (map_flags & ~UPL_FORCE_DATA_SYNC); return kr; @@ -2186,62 +2279,89 @@ int log_executable_mem_entry = 0; */ kern_return_t mach_make_memory_entry_64( - vm_map_t target_map, - memory_object_size_t *size, + vm_map_t target_map, + memory_object_size_t *size, + memory_object_offset_t offset, + vm_prot_t permission, + ipc_port_t *object_handle, + ipc_port_t parent_handle) +{ + if ((permission & MAP_MEM_FLAGS_MASK) & ~MAP_MEM_FLAGS_USER) { + /* + * Unknown flag: reject for forward compatibility. + */ + return KERN_INVALID_VALUE; + } + + return mach_make_memory_entry_internal(target_map, + size, + offset, + permission, + object_handle, + parent_handle); +} + +kern_return_t +mach_make_memory_entry_internal( + vm_map_t target_map, + memory_object_size_t *size, memory_object_offset_t offset, - vm_prot_t permission, - ipc_port_t *object_handle, - ipc_port_t parent_handle) + vm_prot_t permission, + ipc_port_t *object_handle, + ipc_port_t parent_handle) { - vm_map_version_t version; - vm_named_entry_t parent_entry; - vm_named_entry_t user_entry; - ipc_port_t user_handle; - kern_return_t kr; - vm_map_t real_map; + vm_map_version_t version; + vm_named_entry_t parent_entry; + vm_named_entry_t user_entry; + ipc_port_t user_handle; + kern_return_t kr; + vm_map_t real_map; /* needed for call to vm_map_lookup_locked */ - boolean_t wired; - boolean_t iskernel; - vm_object_offset_t obj_off; - vm_prot_t prot; - struct vm_object_fault_info fault_info; - vm_object_t object; - vm_object_t shadow_object; + boolean_t wired; + boolean_t iskernel; + vm_object_offset_t obj_off; + vm_prot_t prot; + struct vm_object_fault_info fault_info = {}; + vm_object_t object; + vm_object_t shadow_object; /* needed for direct map entry manipulation */ - vm_map_entry_t map_entry; - vm_map_entry_t next_entry; - vm_map_t local_map; - vm_map_t original_map = target_map; - vm_map_size_t total_size, map_size; - vm_map_offset_t map_start, map_end; - vm_map_offset_t local_offset; - vm_object_size_t mappable_size; - - /* + vm_map_entry_t map_entry; + vm_map_entry_t next_entry; + vm_map_t local_map; + vm_map_t original_map = target_map; + vm_map_size_t total_size, map_size; + vm_map_offset_t map_start, map_end; + vm_map_offset_t local_offset; + vm_object_size_t mappable_size; + + /* * Stash the offset in the page for use by vm_map_enter_mem_object() * in the VM_FLAGS_RETURN_DATA_ADDR/MAP_MEM_USE_DATA_ADDR case. */ - vm_object_offset_t offset_in_page; + vm_object_offset_t offset_in_page; - unsigned int access; - vm_prot_t protections; - vm_prot_t original_protections, mask_protections; - unsigned int wimg_mode; + unsigned int access; + vm_prot_t protections; + vm_prot_t original_protections, mask_protections; + unsigned int wimg_mode; - boolean_t force_shadow = FALSE; - boolean_t use_data_addr; - boolean_t use_4K_compat; + boolean_t force_shadow = FALSE; + boolean_t use_data_addr; + boolean_t use_4K_compat; +#if VM_NAMED_ENTRY_LIST + int alias = -1; +#endif /* VM_NAMED_ENTRY_LIST */ - if ((permission & MAP_MEM_FLAGS_MASK) & ~MAP_MEM_FLAGS_USER) { + if ((permission & MAP_MEM_FLAGS_MASK) & ~MAP_MEM_FLAGS_ALL) { /* * Unknown flag: reject for forward compatibility. */ return KERN_INVALID_VALUE; } - if (parent_handle != IP_NULL && + if (IP_VALID(parent_handle) && ip_kotype(parent_handle) == IKOT_NAMED_ENTRY) { parent_entry = (vm_named_entry_t) parent_handle->ip_kobject; } else { @@ -2265,40 +2385,42 @@ mach_make_memory_entry_64( map_start = vm_map_trunc_page(offset, PAGE_MASK); if (permission & MAP_MEM_ONLY) { - boolean_t parent_is_object; + boolean_t parent_is_object; map_end = vm_map_round_page(offset + *size, PAGE_MASK); map_size = map_end - map_start; - + if (use_data_addr || use_4K_compat || parent_entry == NULL) { return KERN_INVALID_ARGUMENT; } parent_is_object = !parent_entry->is_sub_map; object = parent_entry->backing.object; - if(parent_is_object && object != VM_OBJECT_NULL) + if (parent_is_object && object != VM_OBJECT_NULL) { wimg_mode = object->wimg_bits; - else + } else { wimg_mode = VM_WIMG_USE_DEFAULT; - if((access != GET_MAP_MEM(parent_entry->protection)) && - !(parent_entry->protection & VM_PROT_WRITE)) { + } + if ((access != GET_MAP_MEM(parent_entry->protection)) && + !(parent_entry->protection & VM_PROT_WRITE)) { return KERN_INVALID_RIGHT; } vm_prot_to_wimg(access, &wimg_mode); - if (access != MAP_MEM_NOOP) + if (access != MAP_MEM_NOOP) { SET_MAP_MEM(access, parent_entry->protection); + } if (parent_is_object && object && - (access != MAP_MEM_NOOP) && - (!(object->nophyscache))) { - + (access != MAP_MEM_NOOP) && + (!(object->nophyscache))) { if (object->wimg_bits != wimg_mode) { vm_object_lock(object); vm_object_change_wimg_mode(object, wimg_mode); vm_object_unlock(object); } } - if (object_handle) + if (object_handle) { *object_handle = IP_NULL; + } return KERN_SUCCESS; } else if (permission & MAP_MEM_NAMED_CREATE) { map_end = vm_map_round_page(offset + *size, PAGE_MASK); @@ -2330,7 +2452,9 @@ mach_make_memory_entry_64( assert(object != VM_OBJECT_NULL); if (permission & MAP_MEM_PURGABLE) { - if (! (permission & VM_PROT_WRITE)) { + task_t owner; + + if (!(permission & VM_PROT_WRITE)) { /* if we can't write, we can't purge */ vm_object_deallocate(object); kr = KERN_INVALID_ARGUMENT; @@ -2340,17 +2464,34 @@ mach_make_memory_entry_64( if (permission & MAP_MEM_PURGABLE_KERNEL_ONLY) { object->purgeable_only_by_kernel = TRUE; } - assert(object->vo_purgeable_owner == NULL); + assert(object->vo_owner == NULL); assert(object->resident_page_count == 0); assert(object->wired_page_count == 0); vm_object_lock(object); - if (object->purgeable_only_by_kernel) { - vm_purgeable_nonvolatile_enqueue(object, - kernel_task); - } else { - vm_purgeable_nonvolatile_enqueue(object, - current_task()); + owner = current_task(); +#if __arm64__ + if (owner->task_legacy_footprint) { + /* + * For ios11, we failed to account for + * this memory. Keep doing that for + * legacy apps (built before ios12), + * for backwards compatibility's sake... + */ + owner = kernel_task; } +#endif /* __arm64__ */ + vm_purgeable_nonvolatile_enqueue(object, owner); + vm_object_unlock(object); + } + + if (permission & MAP_MEM_LEDGER_TAG_NETWORK) { + /* make this object owned by the calling task */ + vm_object_lock(object); + vm_object_ownership_change( + object, + VM_OBJECT_LEDGER_TAG_NETWORK, + current_task(), /* new owner */ + FALSE); /* task_objq locked? */ vm_object_unlock(object); } @@ -2358,10 +2499,10 @@ mach_make_memory_entry_64( if (secluded_for_iokit && /* global boot-arg */ ((permission & MAP_MEM_GRAB_SECLUDED) #if 11 - /* XXX FBDP for my testing only */ - || (secluded_for_fbdp && map_size == 97550336) + /* XXX FBDP for my testing only */ + || (secluded_for_fbdp && map_size == 97550336) #endif - )) { + )) { #if 11 if (!(permission & MAP_MEM_GRAB_SECLUDED) && secluded_for_fbdp) { @@ -2380,9 +2521,9 @@ mach_make_memory_entry_64( wimg_mode = object->wimg_bits; vm_prot_to_wimg(access, &wimg_mode); - if (access != MAP_MEM_NOOP) { - object->wimg_bits = wimg_mode; - } + if (access != MAP_MEM_NOOP) { + object->wimg_bits = wimg_mode; + } /* the object has no pages, so no WIMG bits to update here */ @@ -2411,13 +2552,13 @@ mach_make_memory_entry_64( /* when the object field is filled in. */ *size = CAST_DOWN(vm_size_t, (user_entry->size - - user_entry->data_offset)); + user_entry->data_offset)); *object_handle = user_handle; return KERN_SUCCESS; } if (permission & MAP_MEM_VM_COPY) { - vm_map_copy_t copy; + vm_map_copy_t copy; if (target_map == VM_MAP_NULL) { return KERN_INVALID_TASK; @@ -2427,21 +2568,22 @@ mach_make_memory_entry_64( map_size = map_end - map_start; if (use_data_addr || use_4K_compat) { offset_in_page = offset - map_start; - if (use_4K_compat) + if (use_4K_compat) { offset_in_page &= ~((signed)(0xFFF)); + } } else { offset_in_page = 0; } kr = vm_map_copyin_internal(target_map, - map_start, - map_size, - VM_MAP_COPYIN_ENTRY_LIST, - ©); + map_start, + map_size, + VM_MAP_COPYIN_ENTRY_LIST, + ©); if (kr != KERN_SUCCESS) { return kr; } - + kr = mach_memory_entry_allocate(&user_entry, &user_handle); if (kr != KERN_SUCCESS) { vm_map_copy_discard(copy); @@ -2458,14 +2600,14 @@ mach_make_memory_entry_64( user_entry->data_offset = offset_in_page; *size = CAST_DOWN(vm_size_t, (user_entry->size - - user_entry->data_offset)); + user_entry->data_offset)); *object_handle = user_handle; return KERN_SUCCESS; } if (permission & MAP_MEM_VM_SHARE) { - vm_map_copy_t copy; - vm_prot_t cur_prot, max_prot; + vm_map_copy_t copy; + vm_prot_t cur_prot, max_prot; if (target_map == VM_MAP_NULL) { return KERN_INVALID_TASK; @@ -2475,26 +2617,27 @@ mach_make_memory_entry_64( map_size = map_end - map_start; if (use_data_addr || use_4K_compat) { offset_in_page = offset - map_start; - if (use_4K_compat) + if (use_4K_compat) { offset_in_page &= ~((signed)(0xFFF)); + } } else { offset_in_page = 0; } cur_prot = VM_PROT_ALL; kr = vm_map_copy_extract(target_map, - map_start, - map_size, - ©, - &cur_prot, - &max_prot); + map_start, + map_size, + ©, + &cur_prot, + &max_prot); if (kr != KERN_SUCCESS) { return kr; } if (mask_protections) { /* - * We just want as much of "original_protections" + * We just want as much of "original_protections" * as we can get out of the actual "cur_prot". */ protections &= cur_prot; @@ -2530,20 +2673,20 @@ mach_make_memory_entry_64( user_entry->data_offset = offset_in_page; *size = CAST_DOWN(vm_size_t, (user_entry->size - - user_entry->data_offset)); + user_entry->data_offset)); *object_handle = user_handle; return KERN_SUCCESS; } if (parent_entry == NULL || (permission & MAP_MEM_NAMED_REUSE)) { - map_end = vm_map_round_page(offset + *size, PAGE_MASK); map_size = map_end - map_start; if (use_data_addr || use_4K_compat) { offset_in_page = offset - map_start; - if (use_4K_compat) + if (use_4K_compat) { offset_in_page &= ~((signed)(0xFFF)); + } } else { offset_in_page = 0; } @@ -2563,12 +2706,12 @@ redo_lookup: /* note we check the permission of the range against */ /* that requested by the caller */ - kr = vm_map_lookup_locked(&target_map, map_start, - protections | mask_protections, - OBJECT_LOCK_EXCLUSIVE, &version, - &object, &obj_off, &prot, &wired, - &fault_info, - &real_map); + kr = vm_map_lookup_locked(&target_map, map_start, + protections | mask_protections, + OBJECT_LOCK_EXCLUSIVE, &version, + &object, &obj_off, &prot, &wired, + &fault_info, + &real_map); if (kr != KERN_SUCCESS) { vm_map_unlock_read(target_map); goto make_mem_done; @@ -2593,19 +2736,19 @@ redo_lookup: void *bsd_info; bsd_info = current_task()->bsd_info; printf("pid %d[%s] making memory entry out of " - "executable range from 0x%llx to 0x%llx:" - "might cause code-signing issues " - "later\n", - proc_selfpid(), - (bsd_info != NULL - ? proc_name_address(bsd_info) - : "?"), - (uint64_t) map_start, - (uint64_t) map_end); + "executable range from 0x%llx to 0x%llx:" + "might cause code-signing issues " + "later\n", + proc_selfpid(), + (bsd_info != NULL + ? proc_name_address(bsd_info) + : "?"), + (uint64_t) map_start, + (uint64_t) map_end); } DTRACE_VM2(cs_executable_mem_entry, - uint64_t, (uint64_t)map_start, - uint64_t, (uint64_t)map_end); + uint64_t, (uint64_t)map_start, + uint64_t, (uint64_t)map_end); cs_executable_mem_entry++; #if 11 @@ -2618,24 +2761,25 @@ redo_lookup: kr = KERN_PROTECTION_FAILURE; vm_object_unlock(object); vm_map_unlock_read(target_map); - if(real_map != target_map) + if (real_map != target_map) { vm_map_unlock_read(real_map); + } goto make_mem_done; #endif /* 11 */ - } #endif /* CONFIG_EMBEDDED */ - if (((prot & protections) != protections) + if (((prot & protections) != protections) || (object == kernel_object)) { kr = KERN_INVALID_RIGHT; vm_object_unlock(object); vm_map_unlock_read(target_map); - if(real_map != target_map) + if (real_map != target_map) { vm_map_unlock_read(real_map); - if(object == kernel_object) { + } + if (object == kernel_object) { printf("Warning: Attempt to create a named" - " entry from the kernel_object\n"); + " entry from the kernel_object\n"); } goto make_mem_done; } @@ -2659,59 +2803,66 @@ redo_lookup: local_map = original_map; local_offset = map_start; - if(target_map != local_map) { + if (target_map != local_map) { vm_map_unlock_read(target_map); - if(real_map != target_map) + if (real_map != target_map) { vm_map_unlock_read(real_map); + } vm_map_lock_read(local_map); target_map = local_map; real_map = local_map; } - while(TRUE) { - if(!vm_map_lookup_entry(local_map, - local_offset, &map_entry)) { - kr = KERN_INVALID_ARGUMENT; - vm_map_unlock_read(target_map); - if(real_map != target_map) - vm_map_unlock_read(real_map); - vm_object_deallocate(object); /* release extra ref */ - object = VM_OBJECT_NULL; - goto make_mem_done; - } - iskernel = (local_map->pmap == kernel_pmap); - if(!(map_entry->is_sub_map)) { - if (VME_OBJECT(map_entry) != object) { - kr = KERN_INVALID_ARGUMENT; - vm_map_unlock_read(target_map); - if(real_map != target_map) - vm_map_unlock_read(real_map); - vm_object_deallocate(object); /* release extra ref */ - object = VM_OBJECT_NULL; - goto make_mem_done; - } - break; - } else { - vm_map_t tmap; - tmap = local_map; - local_map = VME_SUBMAP(map_entry); - - vm_map_lock_read(local_map); - vm_map_unlock_read(tmap); - target_map = local_map; - real_map = local_map; - local_offset = local_offset - map_entry->vme_start; - local_offset += VME_OFFSET(map_entry); - } + while (TRUE) { + if (!vm_map_lookup_entry(local_map, + local_offset, &map_entry)) { + kr = KERN_INVALID_ARGUMENT; + vm_map_unlock_read(target_map); + if (real_map != target_map) { + vm_map_unlock_read(real_map); + } + vm_object_deallocate(object); /* release extra ref */ + object = VM_OBJECT_NULL; + goto make_mem_done; + } + iskernel = (local_map->pmap == kernel_pmap); + if (!(map_entry->is_sub_map)) { + if (VME_OBJECT(map_entry) != object) { + kr = KERN_INVALID_ARGUMENT; + vm_map_unlock_read(target_map); + if (real_map != target_map) { + vm_map_unlock_read(real_map); + } + vm_object_deallocate(object); /* release extra ref */ + object = VM_OBJECT_NULL; + goto make_mem_done; + } + break; + } else { + vm_map_t tmap; + tmap = local_map; + local_map = VME_SUBMAP(map_entry); + + vm_map_lock_read(local_map); + vm_map_unlock_read(tmap); + target_map = local_map; + real_map = local_map; + local_offset = local_offset - map_entry->vme_start; + local_offset += VME_OFFSET(map_entry); + } } +#if VM_NAMED_ENTRY_LIST + alias = VME_ALIAS(map_entry); +#endif /* VM_NAMED_ENTRY_LIST */ + /* * We found the VM map entry, lock the VM object again. */ vm_object_lock(object); - if(map_entry->wired_count) { - /* JMM - The check below should be reworked instead. */ - object->true_share = TRUE; - } + if (map_entry->wired_count) { + /* JMM - The check below should be reworked instead. */ + object->true_share = TRUE; + } if (mask_protections) { /* * The caller asked us to use the "protections" as @@ -2720,34 +2871,35 @@ redo_lookup: */ protections &= map_entry->max_protection; } - if(((map_entry->max_protection) & protections) != protections) { - kr = KERN_INVALID_RIGHT; - vm_object_unlock(object); - vm_map_unlock_read(target_map); - if(real_map != target_map) + if (((map_entry->max_protection) & protections) != protections) { + kr = KERN_INVALID_RIGHT; + vm_object_unlock(object); + vm_map_unlock_read(target_map); + if (real_map != target_map) { vm_map_unlock_read(real_map); - vm_object_deallocate(object); - object = VM_OBJECT_NULL; - goto make_mem_done; + } + vm_object_deallocate(object); + object = VM_OBJECT_NULL; + goto make_mem_done; } mappable_size = fault_info.hi_offset - obj_off; total_size = map_entry->vme_end - map_entry->vme_start; - if(map_size > mappable_size) { + if (map_size > mappable_size) { /* try to extend mappable size if the entries */ /* following are from the same object and are */ /* compatible */ next_entry = map_entry->vme_next; /* lets see if the next map entry is still */ /* pointing at this object and is contiguous */ - while(map_size > mappable_size) { + while (map_size > mappable_size) { if ((VME_OBJECT(next_entry) == object) && - (next_entry->vme_start == - next_entry->vme_prev->vme_end) && - (VME_OFFSET(next_entry) == - (VME_OFFSET(next_entry->vme_prev) + - (next_entry->vme_prev->vme_end - - next_entry->vme_prev->vme_start)))) { + (next_entry->vme_start == + next_entry->vme_prev->vme_end) && + (VME_OFFSET(next_entry) == + (VME_OFFSET(next_entry->vme_prev) + + (next_entry->vme_prev->vme_end - + next_entry->vme_prev->vme_start)))) { if (mask_protections) { /* * The caller asked us to use @@ -2762,27 +2914,27 @@ redo_lookup: (map_entry->wired_count == 0)) { break; } - if(((next_entry->max_protection) - & protections) != protections) { - break; + if (((next_entry->max_protection) + & protections) != protections) { + break; } if (next_entry->needs_copy != - map_entry->needs_copy) + map_entry->needs_copy) { break; + } mappable_size += next_entry->vme_end - - next_entry->vme_start; + - next_entry->vme_start; total_size += next_entry->vme_end - - next_entry->vme_start; + - next_entry->vme_start; next_entry = next_entry->vme_next; } else { break; } - } } /* vm_map_entry_should_cow_for_true_share() checks for malloc tags, - * never true in kernel */ + * never true in kernel */ if (!iskernel && vm_map_entry_should_cow_for_true_share(map_entry) && object->vo_size > map_size && map_size != 0) { @@ -2791,7 +2943,7 @@ redo_lookup: * limit the impact of "true_share"/"copy_delay" to * that range instead of the entire VM object... */ - + vm_object_unlock(object); if (vm_map_lock_read_to_write(target_map)) { vm_object_deallocate(object); @@ -2800,13 +2952,13 @@ redo_lookup: } vm_map_clip_start(target_map, - map_entry, - vm_map_trunc_page(map_start, - VM_MAP_PAGE_MASK(target_map))); + map_entry, + vm_map_trunc_page(map_start, + VM_MAP_PAGE_MASK(target_map))); vm_map_clip_end(target_map, - map_entry, - (vm_map_round_page(map_end, - VM_MAP_PAGE_MASK(target_map)))); + map_entry, + (vm_map_round_page(map_end, + VM_MAP_PAGE_MASK(target_map)))); force_shadow = TRUE; if ((map_entry->vme_end - offset) < map_size) { @@ -2819,21 +2971,22 @@ redo_lookup: } if (object->internal) { - /* vm_map_lookup_locked will create a shadow if */ - /* needs_copy is set but does not check for the */ - /* other two conditions shown. It is important to */ + /* vm_map_lookup_locked will create a shadow if */ + /* needs_copy is set but does not check for the */ + /* other two conditions shown. It is important to */ /* set up an object which will not be pulled from */ /* under us. */ - if (force_shadow || - ((map_entry->needs_copy || - object->shadowed || - (object->vo_size > total_size && - (VME_OFFSET(map_entry) != 0 || - object->vo_size > - vm_map_round_page(total_size, - VM_MAP_PAGE_MASK(target_map))))) - && !object->true_share)) { + if (force_shadow || + ((map_entry->needs_copy || + object->shadowed || + (object->vo_size > total_size && + (VME_OFFSET(map_entry) != 0 || + object->vo_size > + vm_map_round_page(total_size, + VM_MAP_PAGE_MASK(target_map))))) + && !object->true_share + && object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC)) { /* * We have to unlock the VM object before * trying to upgrade the VM map lock, to @@ -2847,7 +3000,7 @@ redo_lookup: */ vm_object_unlock(object); - if (vm_map_lock_read_to_write(target_map)) { + if (vm_map_lock_read_to_write(target_map)) { /* * We couldn't upgrade our VM map lock * from "read" to "write" and we lost @@ -2856,20 +3009,20 @@ redo_lookup: */ vm_object_deallocate(object); /* extra ref */ target_map = original_map; - goto redo_lookup; - } + goto redo_lookup; + } #if 00 vm_object_lock(object); #endif - /* + /* * JMM - We need to avoid coming here when the object * is wired by anybody, not just the current map. Why * couldn't we use the standard vm_object_copy_quickly() * approach here? */ - - /* create a shadow object */ + + /* create a shadow object */ VME_OBJECT_SHADOW(map_entry, total_size); shadow_object = VME_OBJECT(map_entry); #if 00 @@ -2879,47 +3032,49 @@ redo_lookup: prot = map_entry->protection & ~VM_PROT_WRITE; if (override_nx(target_map, - VME_ALIAS(map_entry)) - && prot) - prot |= VM_PROT_EXECUTE; + VME_ALIAS(map_entry)) + && prot) { + prot |= VM_PROT_EXECUTE; + } vm_object_pmap_protect( object, VME_OFFSET(map_entry), total_size, - ((map_entry->is_shared - || target_map->mapped_in_other_pmaps) - ? PMAP_NULL : - target_map->pmap), + ((map_entry->is_shared + || target_map->mapped_in_other_pmaps) + ? PMAP_NULL : + target_map->pmap), map_entry->vme_start, prot); - total_size -= (map_entry->vme_end - - map_entry->vme_start); + total_size -= (map_entry->vme_end + - map_entry->vme_start); next_entry = map_entry->vme_next; map_entry->needs_copy = FALSE; vm_object_lock(shadow_object); while (total_size) { - assert((next_entry->wired_count == 0) || - (map_entry->wired_count)); - - if (VME_OBJECT(next_entry) == object) { - vm_object_reference_locked(shadow_object); - VME_OBJECT_SET(next_entry, - shadow_object); - vm_object_deallocate(object); - VME_OFFSET_SET( - next_entry, - (VME_OFFSET(next_entry->vme_prev) + - (next_entry->vme_prev->vme_end - - next_entry->vme_prev->vme_start))); + assert((next_entry->wired_count == 0) || + (map_entry->wired_count)); + + if (VME_OBJECT(next_entry) == object) { + vm_object_reference_locked(shadow_object); + VME_OBJECT_SET(next_entry, + shadow_object); + vm_object_deallocate(object); + VME_OFFSET_SET( + next_entry, + (VME_OFFSET(next_entry->vme_prev) + + (next_entry->vme_prev->vme_end + - next_entry->vme_prev->vme_start))); + next_entry->use_pmap = TRUE; next_entry->needs_copy = FALSE; } else { panic("mach_make_memory_entry_64:" - " map entries out of sync\n"); + " map entries out of sync\n"); } - total_size -= - next_entry->vme_end - - next_entry->vme_start; + total_size -= + next_entry->vme_end + - next_entry->vme_start; next_entry = next_entry->vme_next; } @@ -2932,11 +3087,11 @@ redo_lookup: object = shadow_object; obj_off = ((local_offset - map_entry->vme_start) - + VME_OFFSET(map_entry)); + + VME_OFFSET(map_entry)); vm_map_lock_write_to_read(target_map); - } - } + } + } /* note: in the future we can (if necessary) allow for */ /* memory object lists, this will better support */ @@ -2949,8 +3104,9 @@ redo_lookup: /* against delayed copy, etc. is mostly defensive. */ wimg_mode = object->wimg_bits; - if(!(object->nophyscache)) + if (!(object->nophyscache)) { vm_prot_to_wimg(access, &wimg_mode); + } #if VM_OBJECT_TRACKING_OP_TRUESHARE if (!object->true_share && @@ -2959,19 +3115,20 @@ redo_lookup: int num = 0; num = OSBacktrace(bt, - VM_OBJECT_TRACKING_BTDEPTH); + VM_OBJECT_TRACKING_BTDEPTH); btlog_add_entry(vm_object_tracking_btlog, - object, - VM_OBJECT_TRACKING_OP_TRUESHARE, - bt, - num); + object, + VM_OBJECT_TRACKING_OP_TRUESHARE, + bt, + num); } #endif /* VM_OBJECT_TRACKING_OP_TRUESHARE */ vm_object_lock_assert_exclusive(object); object->true_share = TRUE; - if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC) + if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC) { object->copy_strategy = MEMORY_OBJECT_COPY_DELAY; + } /* * The memory entry now points to this VM object and we @@ -2981,11 +3138,13 @@ redo_lookup: */ vm_map_unlock_read(target_map); - if(real_map != target_map) + if (real_map != target_map) { vm_map_unlock_read(real_map); + } - if (object->wimg_bits != wimg_mode) + if (object->wimg_bits != wimg_mode) { vm_object_change_wimg_mode(object, wimg_mode); + } /* the size of mapped entry that overlaps with our region */ /* which is targeted for share. */ @@ -2993,8 +3152,9 @@ redo_lookup: /* offset of our beg addr within entry */ /* it corresponds to this: */ - if(map_size > mappable_size) + if (map_size > mappable_size) { map_size = mappable_size; + } if (permission & MAP_MEM_NAMED_REUSE) { /* @@ -3010,9 +3170,9 @@ redo_lookup: parent_entry->protection == protections && parent_entry->size == map_size && ((!(use_data_addr || use_4K_compat) && - (parent_entry->data_offset == 0)) || - ((use_data_addr || use_4K_compat) && - (parent_entry->data_offset == offset_in_page)))) { + (parent_entry->data_offset == 0)) || + ((use_data_addr || use_4K_compat) && + (parent_entry->data_offset == offset_in_page)))) { /* * We have a match: re-use "parent_entry". */ @@ -3024,8 +3184,8 @@ redo_lookup: ipc_port_copy_send(parent_handle); *size = CAST_DOWN(vm_size_t, - (parent_entry->size - - parent_entry->data_offset)); + (parent_entry->size - + parent_entry->data_offset)); *object_handle = parent_handle; return KERN_SUCCESS; } else { @@ -3052,15 +3212,17 @@ redo_lookup: user_entry->protection = protections; SET_MAP_MEM(GET_MAP_MEM(permission), user_entry->protection); user_entry->size = map_size; +#if VM_NAMED_ENTRY_LIST + user_entry->named_entry_alias = alias; +#endif /* VM_NAMED_ENTRY_LIST */ /* user_object pager and internal fields are not used */ /* when the object field is filled in. */ *size = CAST_DOWN(vm_size_t, (user_entry->size - - user_entry->data_offset)); + user_entry->data_offset)); *object_handle = user_handle; return KERN_SUCCESS; - } else { /* The new object will be base on an existing named object */ if (parent_entry == NULL) { @@ -3080,15 +3242,16 @@ redo_lookup: * Account for offset to data in parent entry and * compute our own offset to data. */ - if((offset + *size + parent_entry->data_offset) > parent_entry->size) { + if ((offset + *size + parent_entry->data_offset) > parent_entry->size) { kr = KERN_INVALID_ARGUMENT; goto make_mem_done; } map_start = vm_map_trunc_page(offset + parent_entry->data_offset, PAGE_MASK); offset_in_page = (offset + parent_entry->data_offset) - map_start; - if (use_4K_compat) + if (use_4K_compat) { offset_in_page &= ~((signed)(0xFFF)); + } map_end = vm_map_round_page(offset + parent_entry->data_offset + *size, PAGE_MASK); map_size = map_end - map_start; } else { @@ -3096,7 +3259,7 @@ redo_lookup: map_size = map_end - map_start; offset_in_page = 0; - if((offset + map_size) > parent_entry->size) { + if ((offset + map_size) > parent_entry->size) { kr = KERN_INVALID_ARGUMENT; goto make_mem_done; } @@ -3110,7 +3273,7 @@ redo_lookup: */ protections &= parent_entry->protection; } - if((protections & parent_entry->protection) != protections) { + if ((protections & parent_entry->protection) != protections) { kr = KERN_PROTECTION_FAILURE; goto make_mem_done; } @@ -3123,51 +3286,52 @@ redo_lookup: user_entry->size = map_size; user_entry->offset = parent_entry->offset + map_start; - user_entry->data_offset = offset_in_page; + user_entry->data_offset = offset_in_page; user_entry->is_sub_map = parent_entry->is_sub_map; user_entry->is_copy = parent_entry->is_copy; user_entry->internal = parent_entry->internal; user_entry->protection = protections; - if(access != MAP_MEM_NOOP) { - SET_MAP_MEM(access, user_entry->protection); + if (access != MAP_MEM_NOOP) { + SET_MAP_MEM(access, user_entry->protection); } - if(parent_entry->is_sub_map) { - user_entry->backing.map = parent_entry->backing.map; - vm_map_lock(user_entry->backing.map); - user_entry->backing.map->ref_count++; - vm_map_unlock(user_entry->backing.map); + if (parent_entry->is_sub_map) { + user_entry->backing.map = parent_entry->backing.map; + vm_map_lock(user_entry->backing.map); + user_entry->backing.map->map_refcnt++; + vm_map_unlock(user_entry->backing.map); } else { - object = parent_entry->backing.object; - assert(object != VM_OBJECT_NULL); - user_entry->backing.object = object; - /* we now point to this object, hold on */ - vm_object_lock(object); - vm_object_reference_locked(object); + object = parent_entry->backing.object; + assert(object != VM_OBJECT_NULL); + user_entry->backing.object = object; + /* we now point to this object, hold on */ + vm_object_lock(object); + vm_object_reference_locked(object); #if VM_OBJECT_TRACKING_OP_TRUESHARE - if (!object->true_share && - vm_object_tracking_inited) { - void *bt[VM_OBJECT_TRACKING_BTDEPTH]; - int num = 0; - - num = OSBacktrace(bt, - VM_OBJECT_TRACKING_BTDEPTH); - btlog_add_entry(vm_object_tracking_btlog, - object, - VM_OBJECT_TRACKING_OP_TRUESHARE, - bt, - num); - } + if (!object->true_share && + vm_object_tracking_inited) { + void *bt[VM_OBJECT_TRACKING_BTDEPTH]; + int num = 0; + + num = OSBacktrace(bt, + VM_OBJECT_TRACKING_BTDEPTH); + btlog_add_entry(vm_object_tracking_btlog, + object, + VM_OBJECT_TRACKING_OP_TRUESHARE, + bt, + num); + } #endif /* VM_OBJECT_TRACKING_OP_TRUESHARE */ - object->true_share = TRUE; - if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC) - object->copy_strategy = MEMORY_OBJECT_COPY_DELAY; - vm_object_unlock(object); + object->true_share = TRUE; + if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC) { + object->copy_strategy = MEMORY_OBJECT_COPY_DELAY; + } + vm_object_unlock(object); } *size = CAST_DOWN(vm_size_t, (user_entry->size - - user_entry->data_offset)); + user_entry->data_offset)); *object_handle = user_handle; return KERN_SUCCESS; } @@ -3186,40 +3350,40 @@ make_mem_done: kern_return_t _mach_make_memory_entry( - vm_map_t target_map, - memory_object_size_t *size, - memory_object_offset_t offset, - vm_prot_t permission, - ipc_port_t *object_handle, - ipc_port_t parent_entry) + vm_map_t target_map, + memory_object_size_t *size, + memory_object_offset_t offset, + vm_prot_t permission, + ipc_port_t *object_handle, + ipc_port_t parent_entry) { - memory_object_size_t mo_size; - kern_return_t kr; - + memory_object_size_t mo_size; + kern_return_t kr; + mo_size = (memory_object_size_t)*size; - kr = mach_make_memory_entry_64(target_map, &mo_size, - (memory_object_offset_t)offset, permission, object_handle, - parent_entry); + kr = mach_make_memory_entry_64(target_map, &mo_size, + (memory_object_offset_t)offset, permission, object_handle, + parent_entry); *size = mo_size; return kr; } kern_return_t mach_make_memory_entry( - vm_map_t target_map, - vm_size_t *size, - vm_offset_t offset, - vm_prot_t permission, - ipc_port_t *object_handle, - ipc_port_t parent_entry) -{ - memory_object_size_t mo_size; - kern_return_t kr; - + vm_map_t target_map, + vm_size_t *size, + vm_offset_t offset, + vm_prot_t permission, + ipc_port_t *object_handle, + ipc_port_t parent_entry) +{ + memory_object_size_t mo_size; + kern_return_t kr; + mo_size = (memory_object_size_t)*size; - kr = mach_make_memory_entry_64(target_map, &mo_size, - (memory_object_offset_t)offset, permission, object_handle, - parent_entry); + kr = mach_make_memory_entry_64(target_map, &mo_size, + (memory_object_offset_t)offset, permission, object_handle, + parent_entry); *size = CAST_DOWN(vm_size_t, mo_size); return kr; } @@ -3234,32 +3398,70 @@ mach_make_memory_entry( */ kern_return_t task_wire( - vm_map_t map, - boolean_t must_wire) + vm_map_t map, + boolean_t must_wire) { - if (map == VM_MAP_NULL) - return(KERN_INVALID_ARGUMENT); + if (map == VM_MAP_NULL) { + return KERN_INVALID_ARGUMENT; + } + + vm_map_lock(map); + map->wiring_required = (must_wire == TRUE); + vm_map_unlock(map); + + return KERN_SUCCESS; +} + +kern_return_t +vm_map_exec_lockdown( + vm_map_t map) +{ + if (map == VM_MAP_NULL) { + return KERN_INVALID_ARGUMENT; + } + + vm_map_lock(map); + map->map_disallow_new_exec = TRUE; + vm_map_unlock(map); + + return KERN_SUCCESS; +} - if (must_wire) - map->wiring_required = TRUE; - else - map->wiring_required = FALSE; +#if VM_NAMED_ENTRY_LIST +queue_head_t vm_named_entry_list; +int vm_named_entry_count = 0; +lck_mtx_t vm_named_entry_list_lock_data; +lck_mtx_ext_t vm_named_entry_list_lock_data_ext; +#endif /* VM_NAMED_ENTRY_LIST */ - return(KERN_SUCCESS); +void vm_named_entry_init(void); +void +vm_named_entry_init(void) +{ +#if VM_NAMED_ENTRY_LIST + queue_init(&vm_named_entry_list); + vm_named_entry_count = 0; + lck_mtx_init_ext(&vm_named_entry_list_lock_data, + &vm_named_entry_list_lock_data_ext, + &vm_object_lck_grp, + &vm_object_lck_attr); +#endif /* VM_NAMED_ENTRY_LIST */ } __private_extern__ kern_return_t mach_memory_entry_allocate( - vm_named_entry_t *user_entry_p, - ipc_port_t *user_handle_p) + vm_named_entry_t *user_entry_p, + ipc_port_t *user_handle_p) { - vm_named_entry_t user_entry; - ipc_port_t user_handle; - ipc_port_t previous; + vm_named_entry_t user_entry; + ipc_port_t user_handle; + ipc_port_t previous; user_entry = (vm_named_entry_t) kalloc(sizeof *user_entry); - if (user_entry == NULL) + if (user_entry == NULL) { return KERN_FAILURE; + } + bzero(user_entry, sizeof(*user_entry)); named_entry_lock_init(user_entry); @@ -3274,14 +3476,10 @@ mach_memory_entry_allocate( user_handle->ip_sorights++; ip_reference(user_handle); - user_handle->ip_destination = IP_NULL; - user_handle->ip_receiver_name = MACH_PORT_NULL; - user_handle->ip_receiver = ipc_space_kernel; - /* make a send right */ - user_handle->ip_mscount++; - user_handle->ip_srights++; - ip_reference(user_handle); + user_handle->ip_mscount++; + user_handle->ip_srights++; + ip_reference(user_handle); ipc_port_nsrequest(user_handle, 1, user_handle, &previous); /* nsrequest unlocks user_handle */ @@ -3297,11 +3495,26 @@ mach_memory_entry_allocate( user_entry->ref_count = 1; ipc_kobject_set(user_handle, (ipc_kobject_t) user_entry, - IKOT_NAMED_ENTRY); + IKOT_NAMED_ENTRY); *user_entry_p = user_entry; *user_handle_p = user_handle; +#if VM_NAMED_ENTRY_LIST + /* keep a loose (no reference) pointer to the Mach port, for debugging only */ + user_entry->named_entry_port = user_handle; + /* backtrace at allocation time, for debugging only */ + OSBacktrace(&user_entry->named_entry_bt[0], + NAMED_ENTRY_BT_DEPTH); + + /* add this new named entry to the global list */ + lck_mtx_lock_spin(&vm_named_entry_list_lock_data); + queue_enter(&vm_named_entry_list, user_entry, + vm_named_entry_t, named_entry_list); + vm_named_entry_count++; + lck_mtx_unlock(&vm_named_entry_list_lock_data); +#endif /* VM_NAMED_ENTRY_LIST */ + return KERN_SUCCESS; } @@ -3313,20 +3526,21 @@ mach_memory_entry_allocate( */ kern_return_t mach_memory_object_memory_entry_64( - host_t host, - boolean_t internal, - vm_object_offset_t size, - vm_prot_t permission, - memory_object_t pager, - ipc_port_t *entry_handle) + host_t host, + boolean_t internal, + vm_object_offset_t size, + vm_prot_t permission, + memory_object_t pager, + ipc_port_t *entry_handle) { - unsigned int access; - vm_named_entry_t user_entry; - ipc_port_t user_handle; - vm_object_t object; + unsigned int access; + vm_named_entry_t user_entry; + ipc_port_t user_handle; + vm_object_t object; - if (host == HOST_NULL) - return(KERN_INVALID_HOST); + if (host == HOST_NULL) { + return KERN_INVALID_HOST; + } if (pager == MEMORY_OBJECT_NULL && internal) { object = vm_object_allocate(size); @@ -3367,23 +3581,23 @@ mach_memory_object_memory_entry_64( kern_return_t mach_memory_object_memory_entry( - host_t host, - boolean_t internal, - vm_size_t size, - vm_prot_t permission, - memory_object_t pager, - ipc_port_t *entry_handle) + host_t host, + boolean_t internal, + vm_size_t size, + vm_prot_t permission, + memory_object_t pager, + ipc_port_t *entry_handle) { - return mach_memory_object_memory_entry_64( host, internal, - (vm_object_offset_t)size, permission, pager, entry_handle); + return mach_memory_object_memory_entry_64( host, internal, + (vm_object_offset_t)size, permission, pager, entry_handle); } kern_return_t mach_memory_entry_purgable_control( - ipc_port_t entry_port, - vm_purgable_t control, - int *state) + ipc_port_t entry_port, + vm_purgable_t control, + int *state) { if (control == VM_PURGABLE_SET_STATE_FROM_KERNEL) { /* not allowed from user-space */ @@ -3395,28 +3609,30 @@ mach_memory_entry_purgable_control( kern_return_t memory_entry_purgeable_control_internal( - ipc_port_t entry_port, - vm_purgable_t control, - int *state) + ipc_port_t entry_port, + vm_purgable_t control, + int *state) { - kern_return_t kr; - vm_named_entry_t mem_entry; - vm_object_t object; + kern_return_t kr; + vm_named_entry_t mem_entry; + vm_object_t object; - if (entry_port == IP_NULL || + if (!IP_VALID(entry_port) || ip_kotype(entry_port) != IKOT_NAMED_ENTRY) { return KERN_INVALID_ARGUMENT; } if (control != VM_PURGABLE_SET_STATE && control != VM_PURGABLE_GET_STATE && - control != VM_PURGABLE_SET_STATE_FROM_KERNEL) - return(KERN_INVALID_ARGUMENT); + control != VM_PURGABLE_SET_STATE_FROM_KERNEL) { + return KERN_INVALID_ARGUMENT; + } if ((control == VM_PURGABLE_SET_STATE || - control == VM_PURGABLE_SET_STATE_FROM_KERNEL) && + control == VM_PURGABLE_SET_STATE_FROM_KERNEL) && (((*state & ~(VM_PURGABLE_ALL_MASKS)) != 0) || - ((*state & VM_PURGABLE_STATE_MASK) > VM_PURGABLE_STATE_MASK))) - return(KERN_INVALID_ARGUMENT); + ((*state & VM_PURGABLE_STATE_MASK) > VM_PURGABLE_STATE_MASK))) { + return KERN_INVALID_ARGUMENT; + } mem_entry = (vm_named_entry_t) entry_port->ip_kobject; @@ -3452,19 +3668,82 @@ memory_entry_purgeable_control_internal( return kr; } +kern_return_t +mach_memory_entry_access_tracking( + ipc_port_t entry_port, + int *access_tracking, + uint32_t *access_tracking_reads, + uint32_t *access_tracking_writes) +{ + return memory_entry_access_tracking_internal(entry_port, + access_tracking, + access_tracking_reads, + access_tracking_writes); +} + +kern_return_t +memory_entry_access_tracking_internal( + ipc_port_t entry_port, + int *access_tracking, + uint32_t *access_tracking_reads, + uint32_t *access_tracking_writes) +{ + vm_named_entry_t mem_entry; + vm_object_t object; + kern_return_t kr; + + if (!IP_VALID(entry_port) || + ip_kotype(entry_port) != IKOT_NAMED_ENTRY) { + return KERN_INVALID_ARGUMENT; + } + + mem_entry = (vm_named_entry_t) entry_port->ip_kobject; + + named_entry_lock(mem_entry); + + if (mem_entry->is_sub_map || + mem_entry->is_copy) { + named_entry_unlock(mem_entry); + return KERN_INVALID_ARGUMENT; + } + + object = mem_entry->backing.object; + if (object == VM_OBJECT_NULL) { + named_entry_unlock(mem_entry); + return KERN_INVALID_ARGUMENT; + } + +#if VM_OBJECT_ACCESS_TRACKING + vm_object_access_tracking(object, + access_tracking, + access_tracking_reads, + access_tracking_writes); + kr = KERN_SUCCESS; +#else /* VM_OBJECT_ACCESS_TRACKING */ + (void) access_tracking; + (void) access_tracking_reads; + (void) access_tracking_writes; + kr = KERN_NOT_SUPPORTED; +#endif /* VM_OBJECT_ACCESS_TRACKING */ + + named_entry_unlock(mem_entry); + + return kr; +} + kern_return_t mach_memory_entry_get_page_counts( - ipc_port_t entry_port, - unsigned int *resident_page_count, - unsigned int *dirty_page_count) + ipc_port_t entry_port, + unsigned int *resident_page_count, + unsigned int *dirty_page_count) { - kern_return_t kr; - vm_named_entry_t mem_entry; - vm_object_t object; - vm_object_offset_t offset; - vm_object_size_t size; + kern_return_t kr; + vm_named_entry_t mem_entry; + vm_object_t object; + vm_object_offset_t offset; + vm_object_size_t size; - if (entry_port == IP_NULL || + if (!IP_VALID(entry_port) || ip_kotype(entry_port) != IKOT_NAMED_ENTRY) { return KERN_INVALID_ARGUMENT; } @@ -3508,7 +3787,7 @@ mach_memory_entry_get_page_counts( */ void mach_memory_entry_port_release( - ipc_port_t port) + ipc_port_t port) { assert(ip_kotype(port) == IKOT_NAMED_ENTRY); ipc_port_release_send(port); @@ -3528,9 +3807,9 @@ mach_memory_entry_port_release( */ void mach_destroy_memory_entry( - ipc_port_t port) + ipc_port_t port) { - vm_named_entry_t named_entry; + vm_named_entry_t named_entry; #if MACH_ASSERT assert(ip_kotype(port) == IKOT_NAMED_ENTRY); #endif /* MACH_ASSERT */ @@ -3539,7 +3818,7 @@ mach_destroy_memory_entry( named_entry_lock(named_entry); named_entry->ref_count -= 1; - if(named_entry->ref_count == 0) { + if (named_entry->ref_count == 0) { if (named_entry->is_sub_map) { vm_map_deallocate(named_entry->backing.map); } else if (named_entry->is_copy) { @@ -3552,10 +3831,20 @@ mach_destroy_memory_entry( named_entry_unlock(named_entry); named_entry_lock_destroy(named_entry); - kfree((void *) port->ip_kobject, - sizeof (struct vm_named_entry)); - } else +#if VM_NAMED_ENTRY_LIST + lck_mtx_lock_spin(&vm_named_entry_list_lock_data); + queue_remove(&vm_named_entry_list, named_entry, + vm_named_entry_t, named_entry_list); + assert(vm_named_entry_count > 0); + vm_named_entry_count--; + lck_mtx_unlock(&vm_named_entry_list_lock_data); +#endif /* VM_NAMED_ENTRY_LIST */ + + kfree(port->ip_kobject, + sizeof(struct vm_named_entry)); + } else { named_entry_unlock(named_entry); + } } /* Allow manipulation of individual page state. This is actually part of */ @@ -3563,17 +3852,17 @@ mach_destroy_memory_entry( kern_return_t mach_memory_entry_page_op( - ipc_port_t entry_port, - vm_object_offset_t offset, - int ops, - ppnum_t *phys_entry, - int *flags) + ipc_port_t entry_port, + vm_object_offset_t offset, + int ops, + ppnum_t *phys_entry, + int *flags) { - vm_named_entry_t mem_entry; - vm_object_t object; - kern_return_t kr; + vm_named_entry_t mem_entry; + vm_object_t object; + kern_return_t kr; - if (entry_port == IP_NULL || + if (!IP_VALID(entry_port) || ip_kotype(entry_port) != IKOT_NAMED_ENTRY) { return KERN_INVALID_ARGUMENT; } @@ -3599,34 +3888,34 @@ mach_memory_entry_page_op( kr = vm_object_page_op(object, offset, ops, phys_entry, flags); - vm_object_deallocate(object); + vm_object_deallocate(object); return kr; } /* - * mach_memory_entry_range_op offers performance enhancement over - * mach_memory_entry_page_op for page_op functions which do not require page - * level state to be returned from the call. Page_op was created to provide - * a low-cost alternative to page manipulation via UPLs when only a single - * page was involved. The range_op call establishes the ability in the _op + * mach_memory_entry_range_op offers performance enhancement over + * mach_memory_entry_page_op for page_op functions which do not require page + * level state to be returned from the call. Page_op was created to provide + * a low-cost alternative to page manipulation via UPLs when only a single + * page was involved. The range_op call establishes the ability in the _op * family of functions to work on multiple pages where the lack of page level * state handling allows the caller to avoid the overhead of the upl structures. */ kern_return_t mach_memory_entry_range_op( - ipc_port_t entry_port, - vm_object_offset_t offset_beg, - vm_object_offset_t offset_end, + ipc_port_t entry_port, + vm_object_offset_t offset_beg, + vm_object_offset_t offset_end, int ops, int *range) { - vm_named_entry_t mem_entry; - vm_object_t object; - kern_return_t kr; + vm_named_entry_t mem_entry; + vm_object_t object; + kern_return_t kr; - if (entry_port == IP_NULL || + if (!IP_VALID(entry_port) || ip_kotype(entry_port) != IKOT_NAMED_ENTRY) { return KERN_INVALID_ARGUMENT; } @@ -3651,10 +3940,10 @@ mach_memory_entry_range_op( named_entry_unlock(mem_entry); kr = vm_object_range_op(object, - offset_beg, - offset_end, - ops, - (uint32_t *) range); + offset_beg, + offset_end, + ops, + (uint32_t *) range); vm_object_deallocate(object); @@ -3675,15 +3964,15 @@ extern int kernel_upl_unmap( extern int kernel_upl_commit( upl_t upl, upl_page_info_t *pl, - mach_msg_type_number_t count); + mach_msg_type_number_t count); extern int kernel_upl_commit_range( upl_t upl, upl_offset_t offset, - upl_size_t size, - int flags, - upl_page_info_array_t pl, - mach_msg_type_number_t count); + upl_size_t size, + int flags, + upl_page_info_array_t pl, + mach_msg_type_number_t count); extern int kernel_upl_abort( upl_t upl, @@ -3698,9 +3987,9 @@ extern int kernel_upl_abort_range( kern_return_t kernel_upl_map( - vm_map_t map, - upl_t upl, - vm_offset_t *dst_addr) + vm_map_t map, + upl_t upl, + vm_offset_t *dst_addr) { return vm_upl_map(map, upl, dst_addr); } @@ -3708,8 +3997,8 @@ kernel_upl_map( kern_return_t kernel_upl_unmap( - vm_map_t map, - upl_t upl) + vm_map_t map, + upl_t upl) { return vm_upl_unmap(map, upl); } @@ -3720,7 +4009,7 @@ kernel_upl_commit( upl_page_info_t *pl, mach_msg_type_number_t count) { - kern_return_t kr; + kern_return_t kr; kr = upl_commit(upl, pl, count); upl_deallocate(upl); @@ -3730,18 +4019,19 @@ kernel_upl_commit( kern_return_t kernel_upl_commit_range( - upl_t upl, - upl_offset_t offset, - upl_size_t size, - int flags, + upl_t upl, + upl_offset_t offset, + upl_size_t size, + int flags, upl_page_info_array_t pl, mach_msg_type_number_t count) { - boolean_t finished = FALSE; - kern_return_t kr; + boolean_t finished = FALSE; + kern_return_t kr; - if (flags & UPL_COMMIT_FREE_ON_EMPTY) + if (flags & UPL_COMMIT_FREE_ON_EMPTY) { flags |= UPL_COMMIT_NOTIFY_EMPTY; + } if (flags & UPL_COMMIT_KERNEL_ONLY_FLAGS) { return KERN_INVALID_ARGUMENT; @@ -3749,39 +4039,42 @@ kernel_upl_commit_range( kr = upl_commit_range(upl, offset, size, flags, pl, count, &finished); - if ((flags & UPL_COMMIT_NOTIFY_EMPTY) && finished) + if ((flags & UPL_COMMIT_NOTIFY_EMPTY) && finished) { upl_deallocate(upl); + } return kr; } - + kern_return_t kernel_upl_abort_range( - upl_t upl, - upl_offset_t offset, - upl_size_t size, - int abort_flags) + upl_t upl, + upl_offset_t offset, + upl_size_t size, + int abort_flags) { - kern_return_t kr; - boolean_t finished = FALSE; + kern_return_t kr; + boolean_t finished = FALSE; - if (abort_flags & UPL_COMMIT_FREE_ON_EMPTY) + if (abort_flags & UPL_COMMIT_FREE_ON_EMPTY) { abort_flags |= UPL_COMMIT_NOTIFY_EMPTY; + } kr = upl_abort_range(upl, offset, size, abort_flags, &finished); - if ((abort_flags & UPL_COMMIT_FREE_ON_EMPTY) && finished) + if ((abort_flags & UPL_COMMIT_FREE_ON_EMPTY) && finished) { upl_deallocate(upl); + } return kr; } kern_return_t kernel_upl_abort( - upl_t upl, - int abort_type) + upl_t upl, + int abort_type) { - kern_return_t kr; + kern_return_t kr; kr = upl_abort(upl, abort_type); upl_deallocate(upl); @@ -3796,15 +4089,15 @@ kernel_upl_abort( kern_return_t vm_region_object_create( - __unused vm_map_t target_map, - vm_size_t size, - ipc_port_t *object_handle) + __unused vm_map_t target_map, + vm_size_t size, + ipc_port_t *object_handle) { - vm_named_entry_t user_entry; - ipc_port_t user_handle; + vm_named_entry_t user_entry; + ipc_port_t user_handle; + + vm_map_t new_map; - vm_map_t new_map; - if (mach_memory_entry_allocate(&user_entry, &user_handle) != KERN_SUCCESS) { return KERN_FAILURE; @@ -3813,9 +4106,9 @@ vm_region_object_create( /* Create a named object based on a submap of specified size */ new_map = vm_map_create(PMAP_NULL, VM_MAP_MIN_ADDRESS, - vm_map_round_page(size, - VM_MAP_PAGE_MASK(target_map)), - TRUE); + vm_map_round_page(size, + VM_MAP_PAGE_MASK(target_map)), + TRUE); vm_map_set_page_shift(new_map, VM_MAP_PAGE_SHIFT(target_map)); user_entry->backing.map = new_map; @@ -3828,40 +4121,38 @@ vm_region_object_create( *object_handle = user_handle; return KERN_SUCCESS; - } -ppnum_t vm_map_get_phys_page( /* forward */ - vm_map_t map, - vm_offset_t offset); +ppnum_t vm_map_get_phys_page( /* forward */ + vm_map_t map, + vm_offset_t offset); ppnum_t vm_map_get_phys_page( - vm_map_t map, - vm_offset_t addr) + vm_map_t map, + vm_offset_t addr) { - vm_object_offset_t offset; - vm_object_t object; - vm_map_offset_t map_offset; - vm_map_entry_t entry; - ppnum_t phys_page = 0; + vm_object_offset_t offset; + vm_object_t object; + vm_map_offset_t map_offset; + vm_map_entry_t entry; + ppnum_t phys_page = 0; map_offset = vm_map_trunc_page(addr, PAGE_MASK); vm_map_lock(map); while (vm_map_lookup_entry(map, map_offset, &entry)) { - if (VME_OBJECT(entry) == VM_OBJECT_NULL) { vm_map_unlock(map); return (ppnum_t) 0; } if (entry->is_sub_map) { - vm_map_t old_map; + vm_map_t old_map; vm_map_lock(VME_SUBMAP(entry)); old_map = map; map = VME_SUBMAP(entry); map_offset = (VME_OFFSET(entry) + - (map_offset - entry->vme_start)); + (map_offset - entry->vme_start)); vm_map_unlock(old_map); continue; } @@ -3873,27 +4164,26 @@ vm_map_get_phys_page( if (VME_OBJECT(entry)->vo_shadow_offset == 0) { /* need to call vm_fault */ vm_map_unlock(map); - vm_fault(map, map_offset, VM_PROT_NONE, - FALSE /* change_wiring */, VM_KERN_MEMORY_NONE, - THREAD_UNINT, NULL, 0); + vm_fault(map, map_offset, VM_PROT_NONE, + FALSE /* change_wiring */, VM_KERN_MEMORY_NONE, + THREAD_UNINT, NULL, 0); vm_map_lock(map); continue; } offset = (VME_OFFSET(entry) + - (map_offset - entry->vme_start)); + (map_offset - entry->vme_start)); phys_page = (ppnum_t) - ((VME_OBJECT(entry)->vo_shadow_offset - + offset) >> PAGE_SHIFT); + ((VME_OBJECT(entry)->vo_shadow_offset + + offset) >> PAGE_SHIFT); break; - } offset = (VME_OFFSET(entry) + (map_offset - entry->vme_start)); object = VME_OBJECT(entry); vm_object_lock(object); while (TRUE) { - vm_page_t dst_page = vm_page_lookup(object,offset); - if(dst_page == VM_PAGE_NULL) { - if(object->shadow) { + vm_page_t dst_page = vm_page_lookup(object, offset); + if (dst_page == VM_PAGE_NULL) { + if (object->shadow) { vm_object_t old_object; vm_object_lock(object->shadow); old_object = object; @@ -3911,37 +4201,36 @@ vm_map_get_phys_page( } } break; - - } + } vm_map_unlock(map); return phys_page; } #if 0 -kern_return_t kernel_object_iopl_request( /* forward */ - vm_named_entry_t named_entry, - memory_object_offset_t offset, - upl_size_t *upl_size, - upl_t *upl_ptr, - upl_page_info_array_t user_page_list, - unsigned int *page_list_count, - int *flags); +kern_return_t kernel_object_iopl_request( /* forward */ + vm_named_entry_t named_entry, + memory_object_offset_t offset, + upl_size_t *upl_size, + upl_t *upl_ptr, + upl_page_info_array_t user_page_list, + unsigned int *page_list_count, + int *flags); kern_return_t kernel_object_iopl_request( - vm_named_entry_t named_entry, - memory_object_offset_t offset, - upl_size_t *upl_size, - upl_t *upl_ptr, - upl_page_info_array_t user_page_list, - unsigned int *page_list_count, - int *flags) + vm_named_entry_t named_entry, + memory_object_offset_t offset, + upl_size_t *upl_size, + upl_t *upl_ptr, + upl_page_info_array_t user_page_list, + unsigned int *page_list_count, + int *flags) { - vm_object_t object; - kern_return_t ret; + vm_object_t object; + kern_return_t ret; - int caller_flags; + int caller_flags; caller_flags = *flags; @@ -3954,36 +4243,40 @@ kernel_object_iopl_request( } /* a few checks to make sure user is obeying rules */ - if(*upl_size == 0) { - if(offset >= named_entry->size) - return(KERN_INVALID_RIGHT); + if (*upl_size == 0) { + if (offset >= named_entry->size) { + return KERN_INVALID_RIGHT; + } *upl_size = (upl_size_t) (named_entry->size - offset); - if (*upl_size != named_entry->size - offset) + if (*upl_size != named_entry->size - offset) { return KERN_INVALID_ARGUMENT; + } } - if(caller_flags & UPL_COPYOUT_FROM) { - if((named_entry->protection & VM_PROT_READ) - != VM_PROT_READ) { - return(KERN_INVALID_RIGHT); + if (caller_flags & UPL_COPYOUT_FROM) { + if ((named_entry->protection & VM_PROT_READ) + != VM_PROT_READ) { + return KERN_INVALID_RIGHT; } } else { - if((named_entry->protection & - (VM_PROT_READ | VM_PROT_WRITE)) - != (VM_PROT_READ | VM_PROT_WRITE)) { - return(KERN_INVALID_RIGHT); + if ((named_entry->protection & + (VM_PROT_READ | VM_PROT_WRITE)) + != (VM_PROT_READ | VM_PROT_WRITE)) { + return KERN_INVALID_RIGHT; } } - if(named_entry->size < (offset + *upl_size)) - return(KERN_INVALID_ARGUMENT); + if (named_entry->size < (offset + *upl_size)) { + return KERN_INVALID_ARGUMENT; + } /* the callers parameter offset is defined to be the */ /* offset from beginning of named entry offset in object */ offset = offset + named_entry->offset; if (named_entry->is_sub_map || - named_entry->is_copy) + named_entry->is_copy) { return KERN_INVALID_ARGUMENT; - + } + named_entry_lock(named_entry); /* This is the case where we are going to operate */ @@ -3997,8 +4290,9 @@ kernel_object_iopl_request( named_entry_unlock(named_entry); if (!object->private) { - if (*upl_size > MAX_UPL_TRANSFER_BYTES) + if (*upl_size > MAX_UPL_TRANSFER_BYTES) { *upl_size = MAX_UPL_TRANSFER_BYTES; + } if (object->phys_contiguous) { *flags = UPL_PHYS_CONTIG; } else { @@ -4009,12 +4303,12 @@ kernel_object_iopl_request( } ret = vm_object_iopl_request(object, - offset, - *upl_size, - upl_ptr, - user_page_list, - page_list_count, - (upl_control_flags_t)(unsigned int)caller_flags); + offset, + *upl_size, + upl_ptr, + user_page_list, + page_list_count, + (upl_control_flags_t)(unsigned int)caller_flags); vm_object_deallocate(object); return ret; } @@ -4029,100 +4323,103 @@ kernel_object_iopl_request( kern_return_t mach_vm_map( - vm_map_t target_map, - mach_vm_offset_t *address, - mach_vm_size_t initial_size, - mach_vm_offset_t mask, - int flags, - ipc_port_t port, - vm_object_offset_t offset, - boolean_t copy, - vm_prot_t cur_protection, - vm_prot_t max_protection, - vm_inherit_t inheritance); + vm_map_t target_map, + mach_vm_offset_t *address, + mach_vm_size_t initial_size, + mach_vm_offset_t mask, + int flags, + ipc_port_t port, + vm_object_offset_t offset, + boolean_t copy, + vm_prot_t cur_protection, + vm_prot_t max_protection, + vm_inherit_t inheritance); kern_return_t mach_vm_remap( - vm_map_t target_map, - mach_vm_offset_t *address, - mach_vm_size_t size, - mach_vm_offset_t mask, - int flags, - vm_map_t src_map, - mach_vm_offset_t memory_address, - boolean_t copy, - vm_prot_t *cur_protection, - vm_prot_t *max_protection, - vm_inherit_t inheritance); + vm_map_t target_map, + mach_vm_offset_t *address, + mach_vm_size_t size, + mach_vm_offset_t mask, + int flags, + vm_map_t src_map, + mach_vm_offset_t memory_address, + boolean_t copy, + vm_prot_t *cur_protection, + vm_prot_t *max_protection, + vm_inherit_t inheritance); kern_return_t mach_vm_map( - vm_map_t target_map, - mach_vm_offset_t *address, - mach_vm_size_t initial_size, - mach_vm_offset_t mask, - int flags, - ipc_port_t port, - vm_object_offset_t offset, - boolean_t copy, - vm_prot_t cur_protection, - vm_prot_t max_protection, - vm_inherit_t inheritance) + vm_map_t target_map, + mach_vm_offset_t *address, + mach_vm_size_t initial_size, + mach_vm_offset_t mask, + int flags, + ipc_port_t port, + vm_object_offset_t offset, + boolean_t copy, + vm_prot_t cur_protection, + vm_prot_t max_protection, + vm_inherit_t inheritance) { - return (mach_vm_map_external(target_map, address, initial_size, mask, flags, port, - offset, copy, cur_protection, max_protection, inheritance)); + return mach_vm_map_external(target_map, address, initial_size, mask, flags, port, + offset, copy, cur_protection, max_protection, inheritance); } kern_return_t mach_vm_remap( - vm_map_t target_map, - mach_vm_offset_t *address, - mach_vm_size_t size, - mach_vm_offset_t mask, - int flags, - vm_map_t src_map, - mach_vm_offset_t memory_address, - boolean_t copy, - vm_prot_t *cur_protection, - vm_prot_t *max_protection, - vm_inherit_t inheritance) + vm_map_t target_map, + mach_vm_offset_t *address, + mach_vm_size_t size, + mach_vm_offset_t mask, + int flags, + vm_map_t src_map, + mach_vm_offset_t memory_address, + boolean_t copy, + vm_prot_t *cur_protection, + vm_prot_t *max_protection, + vm_inherit_t inheritance) { - return (mach_vm_remap_external(target_map, address, size, mask, flags, src_map, memory_address, - copy, cur_protection, max_protection, inheritance)); + return mach_vm_remap_external(target_map, address, size, mask, flags, src_map, memory_address, + copy, cur_protection, max_protection, inheritance); } kern_return_t vm_map( - vm_map_t target_map, - vm_offset_t *address, - vm_size_t size, - vm_offset_t mask, - int flags, - ipc_port_t port, - vm_offset_t offset, - boolean_t copy, - vm_prot_t cur_protection, - vm_prot_t max_protection, - vm_inherit_t inheritance); + vm_map_t target_map, + vm_offset_t *address, + vm_size_t size, + vm_offset_t mask, + int flags, + ipc_port_t port, + vm_offset_t offset, + boolean_t copy, + vm_prot_t cur_protection, + vm_prot_t max_protection, + vm_inherit_t inheritance); kern_return_t vm_map( - vm_map_t target_map, - vm_offset_t *address, - vm_size_t size, - vm_offset_t mask, - int flags, - ipc_port_t port, - vm_offset_t offset, - boolean_t copy, - vm_prot_t cur_protection, - vm_prot_t max_protection, - vm_inherit_t inheritance) + vm_map_t target_map, + vm_offset_t *address, + vm_size_t size, + vm_offset_t mask, + int flags, + ipc_port_t port, + vm_offset_t offset, + boolean_t copy, + vm_prot_t cur_protection, + vm_prot_t max_protection, + vm_inherit_t inheritance) { vm_tag_t tag; VM_GET_FLAGS_ALIAS(flags, tag); - return (vm_map_kernel(target_map, address, size, mask, flags, tag, port, offset, copy, cur_protection, max_protection, inheritance)); + return vm_map_kernel(target_map, address, size, mask, + flags, VM_MAP_KERNEL_FLAGS_NONE, tag, + port, offset, copy, + cur_protection, max_protection, inheritance); } #endif /* __x86_64__ */