X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/3e170ce000f1506b7b5d2c5c7faec85ceabb573d..HEAD:/osfmk/vm/vm_apple_protect.c diff --git a/osfmk/vm/vm_apple_protect.c b/osfmk/vm/vm_apple_protect.c index 81301fd11..f7fcaceb3 100644 --- a/osfmk/vm/vm_apple_protect.c +++ b/osfmk/vm/vm_apple_protect.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2006 Apple Computer, Inc. All rights reserved. + * Copyright (c) 2006-2020 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * + * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in @@ -11,10 +11,10 @@ * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. - * + * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. - * + * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, @@ -22,7 +22,7 @@ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. - * + * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ @@ -45,27 +45,26 @@ #include #include #include +#include +#include #include #include -#include -#include - #include #include #include #include #include #include +#include - -/* - * APPLE PROTECT MEMORY PAGER +/* + * APPLE PROTECT MEMORY PAGER * * This external memory manager (EMM) handles memory from the encrypted * sections of some executables protected by the DSMOS kernel extension. - * + * * It mostly handles page-in requests (from memory_object_data_request()) by * getting the encrypted data from its backing VM object, itself backed by * the encrypted file, decrypting it and providing it to VM. @@ -83,36 +82,41 @@ void apple_protect_pager_reference(memory_object_t mem_obj); void apple_protect_pager_deallocate(memory_object_t mem_obj); kern_return_t apple_protect_pager_init(memory_object_t mem_obj, - memory_object_control_t control, - memory_object_cluster_size_t pg_size); + memory_object_control_t control, + memory_object_cluster_size_t pg_size); kern_return_t apple_protect_pager_terminate(memory_object_t mem_obj); kern_return_t apple_protect_pager_data_request(memory_object_t mem_obj, - memory_object_offset_t offset, - memory_object_cluster_size_t length, - vm_prot_t protection_required, - memory_object_fault_info_t fault_info); + memory_object_offset_t offset, + memory_object_cluster_size_t length, + vm_prot_t protection_required, + memory_object_fault_info_t fault_info); kern_return_t apple_protect_pager_data_return(memory_object_t mem_obj, - memory_object_offset_t offset, - memory_object_cluster_size_t data_cnt, - memory_object_offset_t *resid_offset, - int *io_error, - boolean_t dirty, - boolean_t kernel_copy, - int upl_flags); + memory_object_offset_t offset, + memory_object_cluster_size_t data_cnt, + memory_object_offset_t *resid_offset, + int *io_error, + boolean_t dirty, + boolean_t kernel_copy, + int upl_flags); kern_return_t apple_protect_pager_data_initialize(memory_object_t mem_obj, - memory_object_offset_t offset, - memory_object_cluster_size_t data_cnt); + memory_object_offset_t offset, + memory_object_cluster_size_t data_cnt); kern_return_t apple_protect_pager_data_unlock(memory_object_t mem_obj, - memory_object_offset_t offset, - memory_object_size_t size, - vm_prot_t desired_access); + memory_object_offset_t offset, + memory_object_size_t size, + vm_prot_t desired_access); kern_return_t apple_protect_pager_synchronize(memory_object_t mem_obj, - memory_object_offset_t offset, - memory_object_size_t length, - vm_sync_t sync_flags); + memory_object_offset_t offset, + memory_object_size_t length, + vm_sync_t sync_flags); kern_return_t apple_protect_pager_map(memory_object_t mem_obj, - vm_prot_t prot); + vm_prot_t prot); kern_return_t apple_protect_pager_last_unmap(memory_object_t mem_obj); +boolean_t apple_protect_pager_backing_object( + memory_object_t mem_obj, + memory_object_offset_t mem_obj_offset, + vm_object_t *backing_object, + vm_object_offset_t *backing_offset); #define CRYPT_INFO_DEBUG 0 void crypt_info_reference(struct pager_crypt_info *crypt_info); @@ -123,19 +127,20 @@ void crypt_info_deallocate(struct pager_crypt_info *crypt_info); * These routines are invoked by VM via the memory_object_*() interfaces. */ const struct memory_object_pager_ops apple_protect_pager_ops = { - apple_protect_pager_reference, - apple_protect_pager_deallocate, - apple_protect_pager_init, - apple_protect_pager_terminate, - apple_protect_pager_data_request, - apple_protect_pager_data_return, - apple_protect_pager_data_initialize, - apple_protect_pager_data_unlock, - apple_protect_pager_synchronize, - apple_protect_pager_map, - apple_protect_pager_last_unmap, - NULL, /* data_reclaim */ - "apple_protect" + .memory_object_reference = apple_protect_pager_reference, + .memory_object_deallocate = apple_protect_pager_deallocate, + .memory_object_init = apple_protect_pager_init, + .memory_object_terminate = apple_protect_pager_terminate, + .memory_object_data_request = apple_protect_pager_data_request, + .memory_object_data_return = apple_protect_pager_data_return, + .memory_object_data_initialize = apple_protect_pager_data_initialize, + .memory_object_data_unlock = apple_protect_pager_data_unlock, + .memory_object_synchronize = apple_protect_pager_synchronize, + .memory_object_map = apple_protect_pager_map, + .memory_object_last_unmap = apple_protect_pager_last_unmap, + .memory_object_data_reclaim = NULL, + .memory_object_backing_object = apple_protect_pager_backing_object, + .memory_object_pager_name = "apple_protect" }; /* @@ -143,49 +148,51 @@ const struct memory_object_pager_ops apple_protect_pager_ops = { * the "apple protect" EMM. */ typedef struct apple_protect_pager { - struct ipc_object_header pager_header; /* fake ip_kotype() */ - memory_object_pager_ops_t pager_ops; /* == &apple_protect_pager_ops */ - queue_chain_t pager_queue; /* next & prev pagers */ - unsigned int ref_count; /* reference count */ - boolean_t is_ready; /* is this pager ready ? */ - boolean_t is_mapped; /* is this mem_obj mapped ? */ - memory_object_control_t pager_control; /* mem object control handle */ - vm_object_t backing_object; /* VM obj w/ encrypted data */ - vm_object_offset_t backing_offset; - vm_object_offset_t crypto_backing_offset; /* for key... */ - vm_object_offset_t crypto_start; - vm_object_offset_t crypto_end; + /* mandatory generic header */ + struct memory_object ap_pgr_hdr; + + /* pager-specific data */ + queue_chain_t pager_queue; /* next & prev pagers */ +#if MEMORY_OBJECT_HAS_REFCOUNT +#define ap_pgr_hdr_ref ap_pgr_hdr.mo_ref +#else + os_ref_atomic_t ap_pgr_hdr_ref; /* reference count */ +#endif + bool is_ready; /* is this pager ready ? */ + bool is_mapped; /* is this mem_obj mapped ? */ + bool is_cached; /* is this pager cached ? */ + vm_object_t backing_object; /* VM obj w/ encrypted data */ + vm_object_offset_t backing_offset; + vm_object_offset_t crypto_backing_offset; /* for key... */ + vm_object_offset_t crypto_start; + vm_object_offset_t crypto_end; struct pager_crypt_info *crypt_info; } *apple_protect_pager_t; -#define APPLE_PROTECT_PAGER_NULL ((apple_protect_pager_t) NULL) -#define pager_ikot pager_header.io_bits +#define APPLE_PROTECT_PAGER_NULL ((apple_protect_pager_t) NULL) /* * List of memory objects managed by this EMM. * The list is protected by the "apple_protect_pager_lock" lock. */ -int apple_protect_pager_count = 0; /* number of pagers */ -int apple_protect_pager_count_mapped = 0; /* number of unmapped pagers */ -queue_head_t apple_protect_pager_queue; -decl_lck_mtx_data(,apple_protect_pager_lock) +unsigned int apple_protect_pager_count = 0; /* number of pagers */ +unsigned int apple_protect_pager_count_mapped = 0; /* number of unmapped pagers */ +queue_head_t apple_protect_pager_queue = QUEUE_HEAD_INITIALIZER(apple_protect_pager_queue); +LCK_GRP_DECLARE(apple_protect_pager_lck_grp, "apple_protect"); +LCK_MTX_DECLARE(apple_protect_pager_lock, &apple_protect_pager_lck_grp); /* * Maximum number of unmapped pagers we're willing to keep around. */ -int apple_protect_pager_cache_limit = 10; +unsigned int apple_protect_pager_cache_limit = 20; /* * Statistics & counters. */ -int apple_protect_pager_count_max = 0; -int apple_protect_pager_count_unmapped_max = 0; -int apple_protect_pager_num_trim_max = 0; -int apple_protect_pager_num_trim_total = 0; - +unsigned int apple_protect_pager_count_max = 0; +unsigned int apple_protect_pager_count_unmapped_max = 0; +unsigned int apple_protect_pager_num_trim_max = 0; +unsigned int apple_protect_pager_num_trim_total = 0; -lck_grp_t apple_protect_pager_lck_grp; -lck_grp_attr_t apple_protect_pager_lck_grp_attr; -lck_attr_t apple_protect_pager_lck_attr; /* internal prototypes */ @@ -195,42 +202,32 @@ apple_protect_pager_t apple_protect_pager_create( vm_object_offset_t crypto_backing_offset, struct pager_crypt_info *crypt_info, vm_object_offset_t crypto_start, - vm_object_offset_t crypto_end); + vm_object_offset_t crypto_end, + boolean_t cache_pager); apple_protect_pager_t apple_protect_pager_lookup(memory_object_t mem_obj); void apple_protect_pager_dequeue(apple_protect_pager_t pager); void apple_protect_pager_deallocate_internal(apple_protect_pager_t pager, - boolean_t locked); + boolean_t locked); void apple_protect_pager_terminate_internal(apple_protect_pager_t pager); void apple_protect_pager_trim(void); #if DEBUG int apple_protect_pagerdebug = 0; -#define PAGER_ALL 0xffffffff -#define PAGER_INIT 0x00000001 -#define PAGER_PAGEIN 0x00000002 - -#define PAGER_DEBUG(LEVEL, A) \ - MACRO_BEGIN \ - if ((apple_protect_pagerdebug & LEVEL)==LEVEL) { \ - printf A; \ - } \ +#define PAGER_ALL 0xffffffff +#define PAGER_INIT 0x00000001 +#define PAGER_PAGEIN 0x00000002 + +#define PAGER_DEBUG(LEVEL, A) \ + MACRO_BEGIN \ + if ((apple_protect_pagerdebug & LEVEL)==LEVEL) { \ + printf A; \ + } \ MACRO_END #else #define PAGER_DEBUG(LEVEL, A) #endif - -void -apple_protect_pager_bootstrap(void) -{ - lck_grp_attr_setdefault(&apple_protect_pager_lck_grp_attr); - lck_grp_init(&apple_protect_pager_lck_grp, "apple_protect", &apple_protect_pager_lck_grp_attr); - lck_attr_setdefault(&apple_protect_pager_lck_attr); - lck_mtx_init(&apple_protect_pager_lock, &apple_protect_pager_lck_grp, &apple_protect_pager_lck_attr); - queue_init(&apple_protect_pager_queue); -} - /* * apple_protect_pager_init() * @@ -238,29 +235,30 @@ apple_protect_pager_bootstrap(void) */ kern_return_t apple_protect_pager_init( - memory_object_t mem_obj, - memory_object_control_t control, + memory_object_t mem_obj, + memory_object_control_t control, #if !DEBUG __unused #endif memory_object_cluster_size_t pg_size) { - apple_protect_pager_t pager; - kern_return_t kr; + apple_protect_pager_t pager; + kern_return_t kr; memory_object_attr_info_data_t attributes; PAGER_DEBUG(PAGER_ALL, - ("apple_protect_pager_init: %p, %p, %x\n", - mem_obj, control, pg_size)); + ("apple_protect_pager_init: %p, %p, %x\n", + mem_obj, control, pg_size)); - if (control == MEMORY_OBJECT_CONTROL_NULL) + if (control == MEMORY_OBJECT_CONTROL_NULL) { return KERN_INVALID_ARGUMENT; + } pager = apple_protect_pager_lookup(mem_obj); memory_object_control_reference(control); - pager->pager_control = control; + pager->ap_pgr_hdr.mo_control = control; attributes.copy_strategy = MEMORY_OBJECT_COPY_DELAY; /* attributes.cluster_size = (1 << (CLUSTER_SHIFT + PAGE_SHIFT));*/ @@ -269,13 +267,20 @@ apple_protect_pager_init( attributes.temporary = TRUE; kr = memory_object_change_attributes( - control, - MEMORY_OBJECT_ATTRIBUTE_INFO, - (memory_object_info_t) &attributes, - MEMORY_OBJECT_ATTR_INFO_COUNT); - if (kr != KERN_SUCCESS) + control, + MEMORY_OBJECT_ATTRIBUTE_INFO, + (memory_object_info_t) &attributes, + MEMORY_OBJECT_ATTR_INFO_COUNT); + if (kr != KERN_SUCCESS) { panic("apple_protect_pager_init: " - "memory_object_change_attributes() failed"); + "memory_object_change_attributes() failed"); + } + +#if CONFIG_SECLUDED_MEMORY + if (secluded_for_filecache) { + memory_object_mark_eligible_for_secluded(control, TRUE); + } +#endif /* CONFIG_SECLUDED_MEMORY */ return KERN_SUCCESS; } @@ -290,14 +295,14 @@ apple_protect_pager_init( */ kern_return_t apple_protect_pager_data_return( - __unused memory_object_t mem_obj, - __unused memory_object_offset_t offset, - __unused memory_object_cluster_size_t data_cnt, - __unused memory_object_offset_t *resid_offset, - __unused int *io_error, - __unused boolean_t dirty, - __unused boolean_t kernel_copy, - __unused int upl_flags) + __unused memory_object_t mem_obj, + __unused memory_object_offset_t offset, + __unused memory_object_cluster_size_t data_cnt, + __unused memory_object_offset_t *resid_offset, + __unused int *io_error, + __unused boolean_t dirty, + __unused boolean_t kernel_copy, + __unused int upl_flags) { panic("apple_protect_pager_data_return: should never get called"); return KERN_FAILURE; @@ -305,9 +310,9 @@ apple_protect_pager_data_return( kern_return_t apple_protect_pager_data_initialize( - __unused memory_object_t mem_obj, - __unused memory_object_offset_t offset, - __unused memory_object_cluster_size_t data_cnt) + __unused memory_object_t mem_obj, + __unused memory_object_offset_t offset, + __unused memory_object_cluster_size_t data_cnt) { panic("apple_protect_pager_data_initialize: should never get called"); return KERN_FAILURE; @@ -315,10 +320,10 @@ apple_protect_pager_data_initialize( kern_return_t apple_protect_pager_data_unlock( - __unused memory_object_t mem_obj, - __unused memory_object_offset_t offset, - __unused memory_object_size_t size, - __unused vm_prot_t desired_access) + __unused memory_object_t mem_obj, + __unused memory_object_offset_t offset, + __unused memory_object_size_t size, + __unused vm_prot_t desired_access) { return KERN_FAILURE; } @@ -329,45 +334,44 @@ apple_protect_pager_data_unlock( * Handles page-in requests from VM. */ int apple_protect_pager_data_request_debug = 0; -kern_return_t +kern_return_t apple_protect_pager_data_request( - memory_object_t mem_obj, - memory_object_offset_t offset, - memory_object_cluster_size_t length, + memory_object_t mem_obj, + memory_object_offset_t offset, + memory_object_cluster_size_t length, #if !DEBUG __unused #endif - vm_prot_t protection_required, + vm_prot_t protection_required, memory_object_fault_info_t mo_fault_info) { - apple_protect_pager_t pager; - memory_object_control_t mo_control; - upl_t upl; - int upl_flags; - upl_size_t upl_size; - upl_page_info_t *upl_pl; - unsigned int pl_count; - vm_object_t src_object, dst_object; - kern_return_t kr, retval; - vm_map_offset_t kernel_mapping; - vm_offset_t src_vaddr, dst_vaddr; - vm_offset_t cur_offset; - vm_offset_t offset_in_page; - kern_return_t error_code; - vm_prot_t prot; - vm_page_t src_page, top_page; - int interruptible; - struct vm_object_fault_info fault_info; - int ret; + apple_protect_pager_t pager; + memory_object_control_t mo_control; + upl_t upl; + int upl_flags; + upl_size_t upl_size; + upl_page_info_t *upl_pl; + unsigned int pl_count; + vm_object_t src_top_object, src_page_object, dst_object; + kern_return_t kr, retval; + vm_offset_t src_vaddr, dst_vaddr; + vm_offset_t cur_offset; + vm_offset_t offset_in_page; + kern_return_t error_code; + vm_prot_t prot; + vm_page_t src_page, top_page; + int interruptible; + struct vm_object_fault_info fault_info; + int ret; PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_data_request: %p, %llx, %x, %x\n", mem_obj, offset, length, protection_required)); retval = KERN_SUCCESS; - src_object = VM_OBJECT_NULL; - kernel_mapping = 0; + src_top_object = VM_OBJECT_NULL; + src_page_object = VM_OBJECT_NULL; upl = NULL; upl_pl = NULL; - fault_info = *((struct vm_object_fault_info *) mo_fault_info); + fault_info = *((struct vm_object_fault_info *)(uintptr_t)mo_fault_info); fault_info.stealth = TRUE; fault_info.io_sync = FALSE; fault_info.mark_zf_absent = FALSE; @@ -376,73 +380,44 @@ apple_protect_pager_data_request( pager = apple_protect_pager_lookup(mem_obj); assert(pager->is_ready); - assert(pager->ref_count > 1); /* pager is alive and mapped */ + assert(os_ref_get_count_raw(&pager->ap_pgr_hdr_ref) > 1); /* pager is alive and mapped */ PAGER_DEBUG(PAGER_PAGEIN, ("apple_protect_pager_data_request: %p, %llx, %x, %x, pager %p\n", mem_obj, offset, length, protection_required, pager)); + fault_info.lo_offset += pager->backing_offset; + fault_info.hi_offset += pager->backing_offset; + /* * Gather in a UPL all the VM pages requested by VM. */ - mo_control = pager->pager_control; + mo_control = pager->ap_pgr_hdr.mo_control; upl_size = length; upl_flags = - UPL_RET_ONLY_ABSENT | - UPL_SET_LITE | - UPL_NO_SYNC | - UPL_CLEAN_IN_PLACE | /* triggers UPL_CLEAR_DIRTY */ - UPL_SET_INTERNAL; + UPL_RET_ONLY_ABSENT | + UPL_SET_LITE | + UPL_NO_SYNC | + UPL_CLEAN_IN_PLACE | /* triggers UPL_CLEAR_DIRTY */ + UPL_SET_INTERNAL; pl_count = 0; kr = memory_object_upl_request(mo_control, - offset, upl_size, - &upl, NULL, NULL, upl_flags); + offset, upl_size, + &upl, NULL, NULL, upl_flags, VM_KERN_MEMORY_SECURITY); if (kr != KERN_SUCCESS) { retval = kr; goto done; } - dst_object = mo_control->moc_object; + dst_object = memory_object_control_to_vm_object(mo_control); assert(dst_object != VM_OBJECT_NULL); - -#if __x86_64__ || __arm__ || __arm64__ - /* we'll use the 1-to-1 mapping of physical memory */ - src_vaddr = 0; - dst_vaddr = 0; -#else /* __x86_64__ || __arm__ || __arm64__ */ - /* - * Reserve 2 virtual pages in the kernel address space to map each - * source and destination physical pages when it's their turn to - * be processed. - */ - vm_map_entry_t map_entry; - - vm_object_reference(kernel_object); /* ref. for mapping */ - kr = vm_map_find_space(kernel_map, - &kernel_mapping, - 2 * PAGE_SIZE_64, - 0, - 0, - &map_entry); - if (kr != KERN_SUCCESS) { - vm_object_deallocate(kernel_object); - retval = kr; - goto done; - } - map_entry->object.vm_object = kernel_object; - map_entry->offset = kernel_mapping; - vm_map_unlock(kernel_map); - src_vaddr = CAST_DOWN(vm_offset_t, kernel_mapping); - dst_vaddr = CAST_DOWN(vm_offset_t, kernel_mapping + PAGE_SIZE_64); -#endif /* __x86_64__ || __arm__ || __arm64__ */ - /* - * We'll map the encrypted data in the kernel address space from the + * We'll map the encrypted data in the kernel address space from the * backing VM object (itself backed by the encrypted file via * the vnode pager). */ - src_object = pager->backing_object; - assert(src_object != VM_OBJECT_NULL); - vm_object_reference(src_object); /* to keep the source object alive */ + src_top_object = pager->backing_object; + assert(src_top_object != VM_OBJECT_NULL); + vm_object_reference(src_top_object); /* keep the source object alive */ /* * Fill in the contents of the pages requested by VM. @@ -450,8 +425,8 @@ apple_protect_pager_data_request( upl_pl = UPL_GET_INTERNAL_PAGE_LIST(upl); pl_count = length / PAGE_SIZE; for (cur_offset = 0; - retval == KERN_SUCCESS && cur_offset < length; - cur_offset += PAGE_SIZE) { + retval == KERN_SUCCESS && cur_offset < length; + cur_offset += PAGE_SIZE) { ppnum_t dst_pnum; if (!upl_page_present(upl_pl, (int)(cur_offset / PAGE_SIZE))) { @@ -462,27 +437,27 @@ apple_protect_pager_data_request( /* * Map the source (encrypted) page in the kernel's * virtual address space. - * We already hold a reference on the src_object. + * We already hold a reference on the src_top_object. */ - retry_src_fault: - vm_object_lock(src_object); - vm_object_paging_begin(src_object); +retry_src_fault: + vm_object_lock(src_top_object); + vm_object_paging_begin(src_top_object); error_code = 0; prot = VM_PROT_READ; src_page = VM_PAGE_NULL; - kr = vm_fault_page(src_object, - pager->backing_offset + offset + cur_offset, - VM_PROT_READ, - FALSE, - FALSE, /* src_page not looked up */ - &prot, - &src_page, - &top_page, - NULL, - &error_code, - FALSE, - FALSE, - &fault_info); + kr = vm_fault_page(src_top_object, + pager->backing_offset + offset + cur_offset, + VM_PROT_READ, + FALSE, + FALSE, /* src_page not looked up */ + &prot, + &src_page, + &top_page, + NULL, + &error_code, + FALSE, + FALSE, + &fault_info); switch (kr) { case VM_FAULT_SUCCESS: break; @@ -492,15 +467,15 @@ apple_protect_pager_data_request( if (vm_page_wait(interruptible)) { goto retry_src_fault; } - /* fall thru */ + OS_FALLTHROUGH; case VM_FAULT_INTERRUPTED: retval = MACH_SEND_INTERRUPTED; goto done; case VM_FAULT_SUCCESS_NO_VM_PAGE: /* success but no VM page: fail */ - vm_object_paging_end(src_object); - vm_object_unlock(src_object); - /*FALLTHROUGH*/ + vm_object_paging_end(src_top_object); + vm_object_unlock(src_top_object); + OS_FALLTHROUGH; case VM_FAULT_MEMORY_ERROR: /* the page is not there ! */ if (error_code) { @@ -511,80 +486,54 @@ apple_protect_pager_data_request( goto done; default: panic("apple_protect_pager_data_request: " - "vm_fault_page() unexpected error 0x%x\n", - kr); + "vm_fault_page() unexpected error 0x%x\n", + kr); } assert(src_page != VM_PAGE_NULL); - assert(src_page->busy); + assert(src_page->vmp_busy); - if (!src_page->active && - !src_page->inactive && - !src_page->throttled) { + if (src_page->vmp_q_state != VM_PAGE_ON_SPECULATIVE_Q) { vm_page_lockspin_queues(); - if (!src_page->active && - !src_page->inactive && - !src_page->throttled) { - vm_page_deactivate(src_page); + + if (src_page->vmp_q_state != VM_PAGE_ON_SPECULATIVE_Q) { + vm_page_speculate(src_page, FALSE); } vm_page_unlock_queues(); } /* - * Establish an explicit mapping of the source - * physical page. - */ -#if __x86_64__ - src_vaddr = (vm_map_offset_t) - PHYSMAP_PTOV((pmap_paddr_t)src_page->phys_page - << PAGE_SHIFT); -#else - pmap_enter(kernel_pmap, - src_vaddr, - src_page->phys_page, - VM_PROT_READ, - VM_PROT_NONE, - 0, - TRUE); -#endif - /* - * Establish an explicit pmap mapping of the destination - * physical page. - * We can't do a regular VM mapping because the VM page - * is "busy". + * Establish pointers to the source + * and destination physical pages. */ dst_pnum = (ppnum_t) - upl_phys_page(upl_pl, (int)(cur_offset / PAGE_SIZE)); + upl_phys_page(upl_pl, (int)(cur_offset / PAGE_SIZE)); assert(dst_pnum != 0); -#if __x86_64__ + + src_vaddr = (vm_map_offset_t) + phystokv((pmap_paddr_t)VM_PAGE_GET_PHYS_PAGE(src_page) + << PAGE_SHIFT); dst_vaddr = (vm_map_offset_t) - PHYSMAP_PTOV((pmap_paddr_t)dst_pnum << PAGE_SHIFT); -#else - pmap_enter(kernel_pmap, - dst_vaddr, - dst_pnum, - VM_PROT_READ | VM_PROT_WRITE, - VM_PROT_NONE, - 0, - TRUE); -#endif + phystokv((pmap_paddr_t)dst_pnum << PAGE_SHIFT); + + src_page_object = VM_PAGE_OBJECT(src_page); /* * Validate the original page... */ - if (src_page->object->code_signed) { + if (src_page_object->code_signed) { vm_page_validate_cs_mapped( - src_page, + src_page, PAGE_SIZE, 0, (const void *) src_vaddr); } /* * ... and transfer the results to the destination page. */ UPL_SET_CS_VALIDATED(upl_pl, cur_offset / PAGE_SIZE, - src_page->cs_validated); + src_page->vmp_cs_validated); UPL_SET_CS_TAINTED(upl_pl, cur_offset / PAGE_SIZE, - src_page->cs_tainted); + src_page->vmp_cs_tainted); UPL_SET_CS_NX(upl_pl, cur_offset / PAGE_SIZE, - src_page->cs_nx); + src_page->vmp_cs_nx); /* * page_decrypt() might access a mapped file, so let's release @@ -593,50 +542,51 @@ apple_protect_pager_data_request( * "paging_in_progress" reference on its object, so it's safe * to unlock the object here. */ - assert(src_page->busy); - assert(src_page->object->paging_in_progress > 0); - vm_object_unlock(src_page->object); + assert(src_page->vmp_busy); + assert(src_page_object->paging_in_progress > 0); + vm_object_unlock(src_page_object); /* * Decrypt the encrypted contents of the source page * into the destination page. */ for (offset_in_page = 0; - offset_in_page < PAGE_SIZE; - offset_in_page += 4096) { + offset_in_page < PAGE_SIZE; + offset_in_page += 4096) { if (offset + cur_offset + offset_in_page < pager->crypto_start || offset + cur_offset + offset_in_page >= pager->crypto_end) { /* not encrypted: just copy */ bcopy((const char *)(src_vaddr + - offset_in_page), - (char *)(dst_vaddr + offset_in_page), - 4096); + offset_in_page), + (char *)(dst_vaddr + offset_in_page), + 4096); + if (apple_protect_pager_data_request_debug) { printf("apple_protect_data_request" - "(%p,0x%llx+0x%llx+0x%04llx): " - "out of crypto range " - "[0x%llx:0x%llx]: " - "COPY [0x%016llx 0x%016llx] " - "code_signed=%d " - "cs_validated=%d " - "cs_tainted=%d " - "cs_nx=%d\n", - pager, - offset, - (uint64_t) cur_offset, - (uint64_t) offset_in_page, - pager->crypto_start, - pager->crypto_end, - *(uint64_t *)(dst_vaddr+ - offset_in_page), - *(uint64_t *)(dst_vaddr+ - offset_in_page+8), - src_page->object->code_signed, - src_page->cs_validated, - src_page->cs_tainted, - src_page->cs_nx); + "(%p,0x%llx+0x%llx+0x%04llx): " + "out of crypto range " + "[0x%llx:0x%llx]: " + "COPY [0x%016llx 0x%016llx] " + "code_signed=%d " + "cs_validated=%d " + "cs_tainted=%d " + "cs_nx=%d\n", + pager, + offset, + (uint64_t) cur_offset, + (uint64_t) offset_in_page, + pager->crypto_start, + pager->crypto_end, + *(uint64_t *)(dst_vaddr + + offset_in_page), + *(uint64_t *)(dst_vaddr + + offset_in_page + 8), + src_page_object->code_signed, + src_page->vmp_cs_validated, + src_page->vmp_cs_tainted, + src_page->vmp_cs_nx); } ret = 0; continue; @@ -645,45 +595,46 @@ apple_protect_pager_data_request( (const void *)(src_vaddr + offset_in_page), (void *)(dst_vaddr + offset_in_page), ((pager->crypto_backing_offset - - pager->crypto_start) + /* XXX ? */ - offset + - cur_offset + - offset_in_page), + pager->crypto_start) + /* XXX ? */ + offset + + cur_offset + + offset_in_page), pager->crypt_info->crypt_ops); + if (apple_protect_pager_data_request_debug) { printf("apple_protect_data_request" - "(%p,0x%llx+0x%llx+0x%04llx): " - "in crypto range [0x%llx:0x%llx]: " - "DECRYPT offset 0x%llx=" - "(0x%llx-0x%llx+0x%llx+0x%llx+0x%04llx)" - "[0x%016llx 0x%016llx] " - "code_signed=%d " - "cs_validated=%d " - "cs_tainted=%d " - "cs_nx=%d " - "ret=0x%x\n", - pager, - offset, - (uint64_t) cur_offset, - (uint64_t) offset_in_page, - pager->crypto_start, pager->crypto_end, - ((pager->crypto_backing_offset - - pager->crypto_start) + - offset + - cur_offset + - offset_in_page), - pager->crypto_backing_offset, - pager->crypto_start, - offset, - (uint64_t) cur_offset, - (uint64_t) offset_in_page, - *(uint64_t *)(dst_vaddr+offset_in_page), - *(uint64_t *)(dst_vaddr+offset_in_page+8), - src_page->object->code_signed, - src_page->cs_validated, - src_page->cs_tainted, - src_page->cs_nx, - ret); + "(%p,0x%llx+0x%llx+0x%04llx): " + "in crypto range [0x%llx:0x%llx]: " + "DECRYPT offset 0x%llx=" + "(0x%llx-0x%llx+0x%llx+0x%llx+0x%04llx)" + "[0x%016llx 0x%016llx] " + "code_signed=%d " + "cs_validated=%d " + "cs_tainted=%d " + "cs_nx=%d " + "ret=0x%x\n", + pager, + offset, + (uint64_t) cur_offset, + (uint64_t) offset_in_page, + pager->crypto_start, pager->crypto_end, + ((pager->crypto_backing_offset - + pager->crypto_start) + + offset + + cur_offset + + offset_in_page), + pager->crypto_backing_offset, + pager->crypto_start, + offset, + (uint64_t) cur_offset, + (uint64_t) offset_in_page, + *(uint64_t *)(dst_vaddr + offset_in_page), + *(uint64_t *)(dst_vaddr + offset_in_page + 8), + src_page_object->code_signed, + src_page->vmp_cs_validated, + src_page->vmp_cs_tainted, + src_page->vmp_cs_nx, + ret); } if (ret) { break; @@ -696,38 +647,25 @@ apple_protect_pager_data_request( retval = KERN_ABORTED; } - assert(src_page->busy); - assert(src_page->object->paging_in_progress > 0); - vm_object_lock(src_page->object); - -#if __x86_64__ || __arm__ || __arm64__ - /* we used the 1-to-1 mapping of physical memory */ - src_vaddr = 0; - dst_vaddr = 0; -#else /* __x86_64__ || __arm__ || __arm64__ */ - /* - * Remove the pmap mapping of the source and destination pages - * in the kernel. - */ - pmap_remove(kernel_pmap, - (addr64_t) kernel_mapping, - (addr64_t) (kernel_mapping + (2 * PAGE_SIZE_64))); -#endif /* __x86_64__ || __arm__ || __arm64__ */ + assert(VM_PAGE_OBJECT(src_page) == src_page_object); + assert(src_page->vmp_busy); + assert(src_page_object->paging_in_progress > 0); + vm_object_lock(src_page_object); /* * Cleanup the result of vm_fault_page() of the source page. */ PAGE_WAKEUP_DONE(src_page); - vm_object_paging_end(src_page->object); - vm_object_unlock(src_page->object); - if (top_page != VM_PAGE_NULL) { - vm_object_t top_object; + src_page = VM_PAGE_NULL; + vm_object_paging_end(src_page_object); + vm_object_unlock(src_page_object); - top_object = top_page->object; - vm_object_lock(top_object); + if (top_page != VM_PAGE_NULL) { + assert(VM_PAGE_OBJECT(top_page) == src_top_object); + vm_object_lock(src_top_object); VM_PAGE_FREE(top_page); - vm_object_paging_end(top_object); - vm_object_unlock(top_object); + vm_object_paging_end(src_top_object); + vm_object_unlock(src_top_object); } } @@ -748,7 +686,7 @@ done: if (retval != KERN_SUCCESS) { upl_abort(upl, 0); if (retval == KERN_ABORTED) { - wait_result_t wait_result; + wait_result_t wait_result; /* * We aborted the fault and did not provide @@ -768,7 +706,7 @@ done: wait_result = assert_wait_timeout( (event_t) apple_protect_pager_data_request, THREAD_UNINT, - 10000, /* 10ms */ + 10000, /* 10ms */ NSEC_PER_USEC); assert(wait_result == THREAD_WAITING); wait_result = thread_block(THREAD_CONTINUE_NULL); @@ -776,30 +714,21 @@ done: } } else { boolean_t empty; - upl_commit_range(upl, 0, upl->size, - UPL_COMMIT_CS_VALIDATED | UPL_COMMIT_WRITTEN_BY_KERNEL, - upl_pl, pl_count, &empty); + assertf(page_aligned(upl->u_offset) && page_aligned(upl->u_size), + "upl %p offset 0x%llx size 0x%x", + upl, upl->u_offset, upl->u_size); + upl_commit_range(upl, 0, upl->u_size, + UPL_COMMIT_CS_VALIDATED | UPL_COMMIT_WRITTEN_BY_KERNEL, + upl_pl, pl_count, &empty); } /* and deallocate the UPL */ upl_deallocate(upl); upl = NULL; } - if (kernel_mapping != 0) { - /* clean up the mapping of the source and destination pages */ - kr = vm_map_remove(kernel_map, - kernel_mapping, - kernel_mapping + (2 * PAGE_SIZE_64), - VM_MAP_NO_FLAGS); - assert(kr == KERN_SUCCESS); - kernel_mapping = 0; - src_vaddr = 0; - dst_vaddr = 0; - } - if (src_object != VM_OBJECT_NULL) { - vm_object_deallocate(src_object); + if (src_top_object != VM_OBJECT_NULL) { + vm_object_deallocate(src_top_object); } - return retval; } @@ -812,15 +741,14 @@ done: */ void apple_protect_pager_reference( - memory_object_t mem_obj) -{ - apple_protect_pager_t pager; + memory_object_t mem_obj) +{ + apple_protect_pager_t pager; pager = apple_protect_pager_lookup(mem_obj); lck_mtx_lock(&apple_protect_pager_lock); - assert(pager->ref_count > 0); - pager->ref_count++; + os_ref_retain_locked_raw(&pager->ap_pgr_hdr_ref, NULL); lck_mtx_unlock(&apple_protect_pager_lock); } @@ -839,12 +767,12 @@ apple_protect_pager_dequeue( assert(!pager->is_mapped); queue_remove(&apple_protect_pager_queue, - pager, - apple_protect_pager_t, - pager_queue); + pager, + apple_protect_pager_t, + pager_queue); pager->pager_queue.next = NULL; pager->pager_queue.prev = NULL; - + apple_protect_pager_count--; } @@ -876,15 +804,15 @@ apple_protect_pager_terminate_internal( /* one less pager using this "pager_crypt_info" */ #if CRYPT_INFO_DEBUG printf("CRYPT_INFO %s: deallocate %p ref %d\n", - __FUNCTION__, - pager->crypt_info, - pager->crypt_info->crypt_refcnt); + __FUNCTION__, + pager->crypt_info, + pager->crypt_info->crypt_refcnt); #endif /* CRYPT_INFO_DEBUG */ crypt_info_deallocate(pager->crypt_info); pager->crypt_info = NULL; /* trigger the destruction of the memory object */ - memory_object_destroy(pager->pager_control, 0); + memory_object_destroy(pager->ap_pgr_hdr.mo_control, 0); } /* @@ -897,18 +825,19 @@ apple_protect_pager_terminate_internal( */ void apple_protect_pager_deallocate_internal( - apple_protect_pager_t pager, - boolean_t locked) + apple_protect_pager_t pager, + boolean_t locked) { - boolean_t needs_trimming; - int count_unmapped; + boolean_t needs_trimming; + unsigned int count_unmapped; + os_ref_count_t ref_count; - if (! locked) { + if (!locked) { lck_mtx_lock(&apple_protect_pager_lock); } - count_unmapped = (apple_protect_pager_count - - apple_protect_pager_count_mapped); + count_unmapped = (apple_protect_pager_count - + apple_protect_pager_count_mapped); if (count_unmapped > apple_protect_pager_cache_limit) { /* we have too many unmapped pagers: trim some */ needs_trimming = TRUE; @@ -917,9 +846,9 @@ apple_protect_pager_deallocate_internal( } /* drop a reference on this pager */ - pager->ref_count--; + ref_count = os_ref_release_locked_raw(&pager->ap_pgr_hdr_ref, NULL); - if (pager->ref_count == 1) { + if (ref_count == 1) { /* * Only the "named" reference is left, which means that * no one is really holding on to this pager anymore. @@ -929,18 +858,18 @@ apple_protect_pager_deallocate_internal( /* the pager is all ours: no need for the lock now */ lck_mtx_unlock(&apple_protect_pager_lock); apple_protect_pager_terminate_internal(pager); - } else if (pager->ref_count == 0) { + } else if (ref_count == 0) { /* * Dropped the existence reference; the memory object has * been terminated. Do some final cleanup and release the * pager structure. */ lck_mtx_unlock(&apple_protect_pager_lock); - if (pager->pager_control != MEMORY_OBJECT_CONTROL_NULL) { - memory_object_control_deallocate(pager->pager_control); - pager->pager_control = MEMORY_OBJECT_CONTROL_NULL; + if (pager->ap_pgr_hdr.mo_control != MEMORY_OBJECT_CONTROL_NULL) { + memory_object_control_deallocate(pager->ap_pgr_hdr.mo_control); + pager->ap_pgr_hdr.mo_control = MEMORY_OBJECT_CONTROL_NULL; } - kfree(pager, sizeof (*pager)); + kfree(pager, sizeof(*pager)); pager = APPLE_PROTECT_PAGER_NULL; } else { /* there are still plenty of references: keep going... */ @@ -961,9 +890,9 @@ apple_protect_pager_deallocate_internal( */ void apple_protect_pager_deallocate( - memory_object_t mem_obj) + memory_object_t mem_obj) { - apple_protect_pager_t pager; + apple_protect_pager_t pager; PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_deallocate: %p\n", mem_obj)); pager = apple_protect_pager_lookup(mem_obj); @@ -978,7 +907,7 @@ apple_protect_pager_terminate( #if !DEBUG __unused #endif - memory_object_t mem_obj) + memory_object_t mem_obj) { PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_terminate: %p\n", mem_obj)); @@ -990,21 +919,13 @@ apple_protect_pager_terminate( */ kern_return_t apple_protect_pager_synchronize( - memory_object_t mem_obj, - memory_object_offset_t offset, - memory_object_size_t length, - __unused vm_sync_t sync_flags) + __unused memory_object_t mem_obj, + __unused memory_object_offset_t offset, + __unused memory_object_size_t length, + __unused vm_sync_t sync_flags) { - apple_protect_pager_t pager; - - PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_synchronize: %p\n", mem_obj)); - - pager = apple_protect_pager_lookup(mem_obj); - - memory_object_synchronize_completed(pager->pager_control, - offset, length); - - return KERN_SUCCESS; + panic("apple_protect_pager_synchronize: memory_object_synchronize no longer supported\n"); + return KERN_FAILURE; } /* @@ -1017,10 +938,10 @@ apple_protect_pager_synchronize( */ kern_return_t apple_protect_pager_map( - memory_object_t mem_obj, - __unused vm_prot_t prot) + memory_object_t mem_obj, + __unused vm_prot_t prot) { - apple_protect_pager_t pager; + apple_protect_pager_t pager; PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_map: %p\n", mem_obj)); @@ -1028,7 +949,7 @@ apple_protect_pager_map( lck_mtx_lock(&apple_protect_pager_lock); assert(pager->is_ready); - assert(pager->ref_count > 0); /* pager is alive */ + assert(os_ref_get_count_raw(&pager->ap_pgr_hdr_ref) > 0); /* pager is alive */ if (pager->is_mapped == FALSE) { /* * First mapping of this pager: take an extra reference @@ -1036,7 +957,7 @@ apple_protect_pager_map( * are removed. */ pager->is_mapped = TRUE; - pager->ref_count++; + os_ref_retain_locked_raw(&pager->ap_pgr_hdr_ref, NULL); apple_protect_pager_count_mapped++; } lck_mtx_unlock(&apple_protect_pager_lock); @@ -1051,13 +972,13 @@ apple_protect_pager_map( */ kern_return_t apple_protect_pager_last_unmap( - memory_object_t mem_obj) + memory_object_t mem_obj) { - apple_protect_pager_t pager; - int count_unmapped; + apple_protect_pager_t pager; + unsigned int count_unmapped; PAGER_DEBUG(PAGER_ALL, - ("apple_protect_pager_last_unmap: %p\n", mem_obj)); + ("apple_protect_pager_last_unmap: %p\n", mem_obj)); pager = apple_protect_pager_lookup(mem_obj); @@ -1069,7 +990,7 @@ apple_protect_pager_last_unmap( */ apple_protect_pager_count_mapped--; count_unmapped = (apple_protect_pager_count - - apple_protect_pager_count_mapped); + apple_protect_pager_count_mapped); if (count_unmapped > apple_protect_pager_count_unmapped_max) { apple_protect_pager_count_unmapped_max = count_unmapped; } @@ -1079,41 +1000,61 @@ apple_protect_pager_last_unmap( } else { lck_mtx_unlock(&apple_protect_pager_lock); } - + return KERN_SUCCESS; } +boolean_t +apple_protect_pager_backing_object( + memory_object_t mem_obj, + memory_object_offset_t offset, + vm_object_t *backing_object, + vm_object_offset_t *backing_offset) +{ + apple_protect_pager_t pager; + + PAGER_DEBUG(PAGER_ALL, + ("apple_protect_pager_backing_object: %p\n", mem_obj)); + + pager = apple_protect_pager_lookup(mem_obj); + + *backing_object = pager->backing_object; + *backing_offset = pager->backing_offset + offset; + + return TRUE; +} /* * */ apple_protect_pager_t apple_protect_pager_lookup( - memory_object_t mem_obj) + memory_object_t mem_obj) { - apple_protect_pager_t pager; + apple_protect_pager_t pager; - pager = (apple_protect_pager_t) mem_obj; - assert(pager->pager_ops == &apple_protect_pager_ops); - assert(pager->ref_count > 0); + assert(mem_obj->mo_pager_ops == &apple_protect_pager_ops); + pager = (apple_protect_pager_t)(uintptr_t) mem_obj; + assert(os_ref_get_count_raw(&pager->ap_pgr_hdr_ref) > 0); return pager; } apple_protect_pager_t apple_protect_pager_create( - vm_object_t backing_object, - vm_object_offset_t backing_offset, - vm_object_offset_t crypto_backing_offset, + vm_object_t backing_object, + vm_object_offset_t backing_offset, + vm_object_offset_t crypto_backing_offset, struct pager_crypt_info *crypt_info, - vm_object_offset_t crypto_start, - vm_object_offset_t crypto_end) + vm_object_offset_t crypto_start, + vm_object_offset_t crypto_end, + boolean_t cache_pager) { - apple_protect_pager_t pager, pager2; - memory_object_control_t control; - kern_return_t kr; - struct pager_crypt_info *old_crypt_info; + apple_protect_pager_t pager, pager2; + memory_object_control_t control; + kern_return_t kr; + struct pager_crypt_info *old_crypt_info; - pager = (apple_protect_pager_t) kalloc(sizeof (*pager)); + pager = (apple_protect_pager_t) kalloc(sizeof(*pager)); if (pager == APPLE_PROTECT_PAGER_NULL) { return APPLE_PROTECT_PAGER_NULL; } @@ -1125,13 +1066,21 @@ apple_protect_pager_create( * we reserve the first word in the object for a fake ip_kotype * setting - that will tell vm_map to use it as a memory object. */ - pager->pager_ops = &apple_protect_pager_ops; - pager->pager_ikot = IKOT_MEMORY_OBJECT; + pager->ap_pgr_hdr.mo_ikot = IKOT_MEMORY_OBJECT; + pager->ap_pgr_hdr.mo_pager_ops = &apple_protect_pager_ops; + pager->ap_pgr_hdr.mo_control = MEMORY_OBJECT_CONTROL_NULL; + pager->is_ready = FALSE;/* not ready until it has a "name" */ - pager->ref_count = 1; /* existence reference (for the cache) */ - pager->ref_count++; /* for the caller */ + /* one reference for the caller */ + os_ref_init_count_raw(&pager->ap_pgr_hdr_ref, NULL, 1); pager->is_mapped = FALSE; - pager->pager_control = MEMORY_OBJECT_CONTROL_NULL; + if (cache_pager) { + /* extra reference for the cache */ + os_ref_retain_locked_raw(&pager->ap_pgr_hdr_ref, NULL); + pager->is_cached = true; + } else { + pager->is_cached = false; + } pager->backing_object = backing_object; pager->backing_offset = backing_offset; pager->crypto_backing_offset = crypto_backing_offset; @@ -1141,14 +1090,14 @@ apple_protect_pager_create( #if CRYPT_INFO_DEBUG printf("CRYPT_INFO %s: crypt_info %p [%p,%p,%p,%d]\n", - __FUNCTION__, - crypt_info, - crypt_info->page_decrypt, - crypt_info->crypt_end, - crypt_info->crypt_ops, - crypt_info->crypt_refcnt); + __FUNCTION__, + crypt_info, + crypt_info->page_decrypt, + crypt_info->crypt_end, + crypt_info->crypt_ops, + crypt_info->crypt_refcnt); #endif /* CRYPT_INFO_DEBUG */ - + vm_object_reference(backing_object); old_crypt_info = NULL; @@ -1156,15 +1105,15 @@ apple_protect_pager_create( lck_mtx_lock(&apple_protect_pager_lock); /* see if anyone raced us to create a pager for the same object */ queue_iterate(&apple_protect_pager_queue, - pager2, - apple_protect_pager_t, - pager_queue) { + pager2, + apple_protect_pager_t, + pager_queue) { if ((pager2->crypt_info->page_decrypt != - crypt_info->page_decrypt) || + crypt_info->page_decrypt) || (pager2->crypt_info->crypt_end != - crypt_info->crypt_end) || + crypt_info->crypt_end) || (pager2->crypt_info->crypt_ops != - crypt_info->crypt_ops)) { + crypt_info->crypt_ops)) { /* crypt_info contents do not match: next pager */ continue; } @@ -1177,16 +1126,16 @@ apple_protect_pager_create( /* ... switch to that pager's crypt_info */ #if CRYPT_INFO_DEBUG printf("CRYPT_INFO %s: reference %p ref %d " - "(create match)\n", - __FUNCTION__, - pager2->crypt_info, - pager2->crypt_info->crypt_refcnt); + "(create match)\n", + __FUNCTION__, + pager2->crypt_info, + pager2->crypt_info->crypt_refcnt); #endif /* CRYPT_INFO_DEBUG */ old_crypt_info = pager2->crypt_info; crypt_info_reference(old_crypt_info); pager->crypt_info = old_crypt_info; } - + if (pager2->backing_object == backing_object && pager2->backing_offset == backing_offset && pager2->crypto_backing_offset == crypto_backing_offset && @@ -1196,21 +1145,21 @@ apple_protect_pager_create( break; } } - if (! queue_end(&apple_protect_pager_queue, - (queue_entry_t) pager2)) { + if (!queue_end(&apple_protect_pager_queue, + (queue_entry_t) pager2)) { /* we lost the race, down with the loser... */ lck_mtx_unlock(&apple_protect_pager_lock); vm_object_deallocate(pager->backing_object); pager->backing_object = VM_OBJECT_NULL; #if CRYPT_INFO_DEBUG printf("CRYPT_INFO %s: %p ref %d (create pager match)\n", - __FUNCTION__, - pager->crypt_info, - pager->crypt_info->crypt_refcnt); + __FUNCTION__, + pager->crypt_info, + pager->crypt_info->crypt_refcnt); #endif /* CRYPT_INFO_DEBUG */ crypt_info_deallocate(pager->crypt_info); pager->crypt_info = NULL; - kfree(pager, sizeof (*pager)); + kfree(pager, sizeof(*pager)); /* ... and go with the winner */ pager = pager2; /* let the winner make sure the pager gets ready */ @@ -1219,9 +1168,9 @@ apple_protect_pager_create( /* enter new pager at the head of our list of pagers */ queue_enter_first(&apple_protect_pager_queue, - pager, - apple_protect_pager_t, - pager_queue); + pager, + apple_protect_pager_t, + pager_queue); apple_protect_pager_count++; if (apple_protect_pager_count > apple_protect_pager_count_max) { apple_protect_pager_count_max = apple_protect_pager_count; @@ -1229,10 +1178,12 @@ apple_protect_pager_create( lck_mtx_unlock(&apple_protect_pager_lock); kr = memory_object_create_named((memory_object_t) pager, - 0, - &control); + 0, + &control); assert(kr == KERN_SUCCESS); + memory_object_mark_trusted(control); + lck_mtx_lock(&apple_protect_pager_lock); /* the new pager is now ready to be used */ pager->is_ready = TRUE; @@ -1246,10 +1197,10 @@ apple_protect_pager_create( /* we re-used an old crypt_info instead of using our new one */ #if CRYPT_INFO_DEBUG printf("CRYPT_INFO %s: deallocate %p ref %d " - "(create used old)\n", - __FUNCTION__, - crypt_info, - crypt_info->crypt_refcnt); + "(create used old)\n", + __FUNCTION__, + crypt_info, + crypt_info->crypt_refcnt); #endif /* CRYPT_INFO_DEBUG */ crypt_info_deallocate(crypt_info); crypt_info = NULL; @@ -1267,24 +1218,25 @@ apple_protect_pager_create( */ memory_object_t apple_protect_pager_setup( - vm_object_t backing_object, - vm_object_offset_t backing_offset, - vm_object_offset_t crypto_backing_offset, + vm_object_t backing_object, + vm_object_offset_t backing_offset, + vm_object_offset_t crypto_backing_offset, struct pager_crypt_info *crypt_info, - vm_object_offset_t crypto_start, - vm_object_offset_t crypto_end) + vm_object_offset_t crypto_start, + vm_object_offset_t crypto_end, + boolean_t cache_pager) { - apple_protect_pager_t pager; - struct pager_crypt_info *old_crypt_info, *new_crypt_info; + apple_protect_pager_t pager; + struct pager_crypt_info *old_crypt_info, *new_crypt_info; #if CRYPT_INFO_DEBUG printf("CRYPT_INFO %s: crypt_info=%p [%p,%p,%p,%d]\n", - __FUNCTION__, - crypt_info, - crypt_info->page_decrypt, - crypt_info->crypt_end, - crypt_info->crypt_ops, - crypt_info->crypt_refcnt); + __FUNCTION__, + crypt_info, + crypt_info->page_decrypt, + crypt_info->crypt_end, + crypt_info->crypt_ops, + crypt_info->crypt_refcnt); #endif /* CRYPT_INFO_DEBUG */ old_crypt_info = NULL; @@ -1292,15 +1244,15 @@ apple_protect_pager_setup( lck_mtx_lock(&apple_protect_pager_lock); queue_iterate(&apple_protect_pager_queue, - pager, - apple_protect_pager_t, - pager_queue) { + pager, + apple_protect_pager_t, + pager_queue) { if ((pager->crypt_info->page_decrypt != - crypt_info->page_decrypt) || + crypt_info->page_decrypt) || (pager->crypt_info->crypt_end != - crypt_info->crypt_end) || + crypt_info->crypt_end) || (pager->crypt_info->crypt_ops != - crypt_info->crypt_ops)) { + crypt_info->crypt_ops)) { /* no match for "crypt_info": next pager */ continue; } @@ -1313,28 +1265,28 @@ apple_protect_pager_setup( old_crypt_info = pager->crypt_info; #if CRYPT_INFO_DEBUG printf("CRYPT_INFO %s: " - "switching crypt_info from %p [%p,%p,%p,%d] " - "to %p [%p,%p,%p,%d] from pager %p\n", - __FUNCTION__, - crypt_info, - crypt_info->page_decrypt, - crypt_info->crypt_end, - crypt_info->crypt_ops, - crypt_info->crypt_refcnt, - old_crypt_info, - old_crypt_info->page_decrypt, - old_crypt_info->crypt_end, - old_crypt_info->crypt_ops, - old_crypt_info->crypt_refcnt, - pager); + "switching crypt_info from %p [%p,%p,%p,%d] " + "to %p [%p,%p,%p,%d] from pager %p\n", + __FUNCTION__, + crypt_info, + crypt_info->page_decrypt, + crypt_info->crypt_end, + crypt_info->crypt_ops, + crypt_info->crypt_refcnt, + old_crypt_info, + old_crypt_info->page_decrypt, + old_crypt_info->crypt_end, + old_crypt_info->crypt_ops, + old_crypt_info->crypt_refcnt, + pager); printf("CRYPT_INFO %s: %p ref %d (setup match)\n", - __FUNCTION__, - pager->crypt_info, - pager->crypt_info->crypt_refcnt); + __FUNCTION__, + pager->crypt_info, + pager->crypt_info->crypt_refcnt); #endif /* CRYPT_INFO_DEBUG */ crypt_info_reference(pager->crypt_info); } - + if (pager->backing_object == backing_object && pager->backing_offset == backing_offset && pager->crypto_backing_offset == crypto_backing_offset && @@ -1345,27 +1297,26 @@ apple_protect_pager_setup( assert(old_crypt_info->crypt_refcnt > 1); #if CRYPT_INFO_DEBUG printf("CRYPT_INFO %s: " - "pager match with %p crypt_info %p\n", - __FUNCTION__, - pager, - pager->crypt_info); + "pager match with %p crypt_info %p\n", + __FUNCTION__, + pager, + pager->crypt_info); printf("CRYPT_INFO %s: deallocate %p ref %d " - "(pager match)\n", - __FUNCTION__, - old_crypt_info, - old_crypt_info->crypt_refcnt); + "(pager match)\n", + __FUNCTION__, + old_crypt_info, + old_crypt_info->crypt_refcnt); #endif /* CRYPT_INFO_DEBUG */ /* release the extra ref on crypt_info we got above */ crypt_info_deallocate(old_crypt_info); assert(old_crypt_info->crypt_refcnt > 0); /* give extra reference on pager to the caller */ - assert(pager->ref_count > 0); - pager->ref_count++; + os_ref_retain_locked_raw(&pager->ap_pgr_hdr_ref, NULL); break; } } if (queue_end(&apple_protect_pager_queue, - (queue_entry_t) pager)) { + (queue_entry_t) pager)) { lck_mtx_unlock(&apple_protect_pager_lock); /* no existing pager for this backing object */ pager = APPLE_PROTECT_PAGER_NULL; @@ -1374,20 +1325,20 @@ apple_protect_pager_setup( new_crypt_info = old_crypt_info; #if CRYPT_INFO_DEBUG printf("CRYPT_INFO %s: " - "will use old_crypt_info %p for new pager\n", - __FUNCTION__, - old_crypt_info); + "will use old_crypt_info %p for new pager\n", + __FUNCTION__, + old_crypt_info); #endif /* CRYPT_INFO_DEBUG */ } else { /* allocate a new crypt_info for new pager */ - new_crypt_info = kalloc(sizeof (*new_crypt_info)); + new_crypt_info = kalloc(sizeof(*new_crypt_info)); *new_crypt_info = *crypt_info; new_crypt_info->crypt_refcnt = 1; #if CRYPT_INFO_DEBUG printf("CRYPT_INFO %s: " - "will use new_crypt_info %p for new pager\n", - __FUNCTION__, - new_crypt_info); + "will use new_crypt_info %p for new pager\n", + __FUNCTION__, + new_crypt_info); #endif /* CRYPT_INFO_DEBUG */ } if (new_crypt_info == NULL) { @@ -1400,7 +1351,8 @@ apple_protect_pager_setup( crypto_backing_offset, new_crypt_info, crypto_start, - crypto_end); + crypto_end, + cache_pager); } if (pager == APPLE_PROTECT_PAGER_NULL) { /* could not create a new pager */ @@ -1408,10 +1360,10 @@ apple_protect_pager_setup( /* release extra reference on old_crypt_info */ #if CRYPT_INFO_DEBUG printf("CRYPT_INFO %s: deallocate %p ref %d " - "(create fail old_crypt_info)\n", - __FUNCTION__, - old_crypt_info, - old_crypt_info->crypt_refcnt); + "(create fail old_crypt_info)\n", + __FUNCTION__, + old_crypt_info, + old_crypt_info->crypt_refcnt); #endif /* CRYPT_INFO_DEBUG */ crypt_info_deallocate(old_crypt_info); old_crypt_info = NULL; @@ -1420,10 +1372,10 @@ apple_protect_pager_setup( assert(new_crypt_info->crypt_refcnt == 1); #if CRYPT_INFO_DEBUG printf("CRYPT_INFO %s: deallocate %p ref %d " - "(create fail new_crypt_info)\n", - __FUNCTION__, - new_crypt_info, - new_crypt_info->crypt_refcnt); + "(create fail new_crypt_info)\n", + __FUNCTION__, + new_crypt_info, + new_crypt_info->crypt_refcnt); #endif /* CRYPT_INFO_DEBUG */ crypt_info_deallocate(new_crypt_info); new_crypt_info = NULL; @@ -1437,22 +1389,22 @@ apple_protect_pager_setup( while (!pager->is_ready) { lck_mtx_sleep(&apple_protect_pager_lock, - LCK_SLEEP_DEFAULT, - &pager->is_ready, - THREAD_UNINT); + LCK_SLEEP_DEFAULT, + &pager->is_ready, + THREAD_UNINT); } lck_mtx_unlock(&apple_protect_pager_lock); return (memory_object_t) pager; -} +} void apple_protect_pager_trim(void) { - apple_protect_pager_t pager, prev_pager; - queue_head_t trim_queue; - int num_trim; - int count_unmapped; + apple_protect_pager_t pager, prev_pager; + queue_head_t trim_queue; + unsigned int num_trim; + unsigned int count_unmapped; lck_mtx_lock(&apple_protect_pager_lock); @@ -1464,15 +1416,16 @@ apple_protect_pager_trim(void) num_trim = 0; for (pager = (apple_protect_pager_t) - queue_last(&apple_protect_pager_queue); - !queue_end(&apple_protect_pager_queue, - (queue_entry_t) pager); - pager = prev_pager) { + queue_last(&apple_protect_pager_queue); + !queue_end(&apple_protect_pager_queue, + (queue_entry_t) pager); + pager = prev_pager) { /* get prev elt before we dequeue */ prev_pager = (apple_protect_pager_t) - queue_prev(&pager->pager_queue); + queue_prev(&pager->pager_queue); - if (pager->ref_count == 2 && + if (pager->is_cached && + os_ref_get_count_raw(&pager->ap_pgr_hdr_ref) == 2 && pager->is_ready && !pager->is_mapped) { /* this pager can be trimmed */ @@ -1481,12 +1434,12 @@ apple_protect_pager_trim(void) apple_protect_pager_dequeue(pager); /* ... and add it to our trim queue */ queue_enter_first(&trim_queue, - pager, - apple_protect_pager_t, - pager_queue); + pager, + apple_protect_pager_t, + pager_queue); count_unmapped = (apple_protect_pager_count - - apple_protect_pager_count_mapped); + apple_protect_pager_count_mapped); if (count_unmapped <= apple_protect_pager_cache_limit) { /* we have enough pagers to trim */ break; @@ -1503,18 +1456,21 @@ apple_protect_pager_trim(void) /* terminate the trimmed pagers */ while (!queue_empty(&trim_queue)) { queue_remove_first(&trim_queue, - pager, - apple_protect_pager_t, - pager_queue); + pager, + apple_protect_pager_t, + pager_queue); + assert(pager->is_cached); + pager->is_cached = false; pager->pager_queue.next = NULL; pager->pager_queue.prev = NULL; - assert(pager->ref_count == 2); /* * We can't call deallocate_internal() because the pager * has already been dequeued, but we still need to remove * a reference. */ - pager->ref_count--; + os_ref_count_t __assert_only count; + count = os_ref_release_locked_raw(&pager->ap_pgr_hdr_ref, NULL); + assert(count == 1); apple_protect_pager_terminate_internal(pager); } } @@ -1527,24 +1483,24 @@ crypt_info_reference( assert(crypt_info->crypt_refcnt != 0); #if CRYPT_INFO_DEBUG printf("CRYPT_INFO %s: %p ref %d -> %d\n", - __FUNCTION__, - crypt_info, - crypt_info->crypt_refcnt, - crypt_info->crypt_refcnt + 1); + __FUNCTION__, + crypt_info, + crypt_info->crypt_refcnt, + crypt_info->crypt_refcnt + 1); #endif /* CRYPT_INFO_DEBUG */ OSAddAtomic(+1, &crypt_info->crypt_refcnt); } void crypt_info_deallocate( - struct pager_crypt_info *crypt_info) + struct pager_crypt_info *crypt_info) { #if CRYPT_INFO_DEBUG printf("CRYPT_INFO %s: %p ref %d -> %d\n", - __FUNCTION__, - crypt_info, - crypt_info->crypt_refcnt, - crypt_info->crypt_refcnt - 1); + __FUNCTION__, + crypt_info, + crypt_info->crypt_refcnt, + crypt_info->crypt_refcnt - 1); #endif /* CRYPT_INFO_DEBUG */ OSAddAtomic(-1, &crypt_info->crypt_refcnt); if (crypt_info->crypt_refcnt == 0) { @@ -1555,10 +1511,10 @@ crypt_info_deallocate( } #if CRYPT_INFO_DEBUG printf("CRYPT_INFO %s: freeing %p\n", - __FUNCTION__, - crypt_info); + __FUNCTION__, + crypt_info); #endif /* CRYPT_INFO_DEBUG */ - kfree(crypt_info, sizeof (*crypt_info)); + kfree(crypt_info, sizeof(*crypt_info)); crypt_info = NULL; } }