X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/15129b1c8dbb3650c63b70adb1cad9af601c6c17..HEAD:/osfmk/vm/vm_apple_protect.c diff --git a/osfmk/vm/vm_apple_protect.c b/osfmk/vm/vm_apple_protect.c index 11ccb6716..f7fcaceb3 100644 --- a/osfmk/vm/vm_apple_protect.c +++ b/osfmk/vm/vm_apple_protect.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2006 Apple Computer, Inc. All rights reserved. + * Copyright (c) 2006-2020 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * + * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in @@ -11,10 +11,10 @@ * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. - * + * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. - * + * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, @@ -22,7 +22,7 @@ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. - * + * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ @@ -45,27 +45,26 @@ #include #include #include +#include +#include #include #include -#include -#include - #include #include #include #include #include #include +#include - -/* - * APPLE PROTECT MEMORY PAGER +/* + * APPLE PROTECT MEMORY PAGER * * This external memory manager (EMM) handles memory from the encrypted * sections of some executables protected by the DSMOS kernel extension. - * + * * It mostly handles page-in requests (from memory_object_data_request()) by * getting the encrypted data from its backing VM object, itself backed by * the encrypted file, decrypting it and providing it to VM. @@ -83,55 +82,65 @@ void apple_protect_pager_reference(memory_object_t mem_obj); void apple_protect_pager_deallocate(memory_object_t mem_obj); kern_return_t apple_protect_pager_init(memory_object_t mem_obj, - memory_object_control_t control, - memory_object_cluster_size_t pg_size); + memory_object_control_t control, + memory_object_cluster_size_t pg_size); kern_return_t apple_protect_pager_terminate(memory_object_t mem_obj); kern_return_t apple_protect_pager_data_request(memory_object_t mem_obj, - memory_object_offset_t offset, - memory_object_cluster_size_t length, - vm_prot_t protection_required, - memory_object_fault_info_t fault_info); + memory_object_offset_t offset, + memory_object_cluster_size_t length, + vm_prot_t protection_required, + memory_object_fault_info_t fault_info); kern_return_t apple_protect_pager_data_return(memory_object_t mem_obj, - memory_object_offset_t offset, - memory_object_cluster_size_t data_cnt, - memory_object_offset_t *resid_offset, - int *io_error, - boolean_t dirty, - boolean_t kernel_copy, - int upl_flags); + memory_object_offset_t offset, + memory_object_cluster_size_t data_cnt, + memory_object_offset_t *resid_offset, + int *io_error, + boolean_t dirty, + boolean_t kernel_copy, + int upl_flags); kern_return_t apple_protect_pager_data_initialize(memory_object_t mem_obj, - memory_object_offset_t offset, - memory_object_cluster_size_t data_cnt); + memory_object_offset_t offset, + memory_object_cluster_size_t data_cnt); kern_return_t apple_protect_pager_data_unlock(memory_object_t mem_obj, - memory_object_offset_t offset, - memory_object_size_t size, - vm_prot_t desired_access); + memory_object_offset_t offset, + memory_object_size_t size, + vm_prot_t desired_access); kern_return_t apple_protect_pager_synchronize(memory_object_t mem_obj, - memory_object_offset_t offset, - memory_object_size_t length, - vm_sync_t sync_flags); + memory_object_offset_t offset, + memory_object_size_t length, + vm_sync_t sync_flags); kern_return_t apple_protect_pager_map(memory_object_t mem_obj, - vm_prot_t prot); + vm_prot_t prot); kern_return_t apple_protect_pager_last_unmap(memory_object_t mem_obj); +boolean_t apple_protect_pager_backing_object( + memory_object_t mem_obj, + memory_object_offset_t mem_obj_offset, + vm_object_t *backing_object, + vm_object_offset_t *backing_offset); + +#define CRYPT_INFO_DEBUG 0 +void crypt_info_reference(struct pager_crypt_info *crypt_info); +void crypt_info_deallocate(struct pager_crypt_info *crypt_info); /* * Vector of VM operations for this EMM. * These routines are invoked by VM via the memory_object_*() interfaces. */ const struct memory_object_pager_ops apple_protect_pager_ops = { - apple_protect_pager_reference, - apple_protect_pager_deallocate, - apple_protect_pager_init, - apple_protect_pager_terminate, - apple_protect_pager_data_request, - apple_protect_pager_data_return, - apple_protect_pager_data_initialize, - apple_protect_pager_data_unlock, - apple_protect_pager_synchronize, - apple_protect_pager_map, - apple_protect_pager_last_unmap, - NULL, /* data_reclaim */ - "apple protect pager" + .memory_object_reference = apple_protect_pager_reference, + .memory_object_deallocate = apple_protect_pager_deallocate, + .memory_object_init = apple_protect_pager_init, + .memory_object_terminate = apple_protect_pager_terminate, + .memory_object_data_request = apple_protect_pager_data_request, + .memory_object_data_return = apple_protect_pager_data_return, + .memory_object_data_initialize = apple_protect_pager_data_initialize, + .memory_object_data_unlock = apple_protect_pager_data_unlock, + .memory_object_synchronize = apple_protect_pager_synchronize, + .memory_object_map = apple_protect_pager_map, + .memory_object_last_unmap = apple_protect_pager_last_unmap, + .memory_object_data_reclaim = NULL, + .memory_object_backing_object = apple_protect_pager_backing_object, + .memory_object_pager_name = "apple_protect" }; /* @@ -139,85 +148,86 @@ const struct memory_object_pager_ops apple_protect_pager_ops = { * the "apple protect" EMM. */ typedef struct apple_protect_pager { - struct ipc_object_header pager_header; /* fake ip_kotype() */ - memory_object_pager_ops_t pager_ops; /* == &apple_protect_pager_ops */ - queue_chain_t pager_queue; /* next & prev pagers */ - unsigned int ref_count; /* reference count */ - boolean_t is_ready; /* is this pager ready ? */ - boolean_t is_mapped; /* is this mem_obj mapped ? */ - memory_object_control_t pager_control; /* mem object control handle */ - vm_object_t backing_object; /* VM obj w/ encrypted data */ - struct pager_crypt_info crypt; + /* mandatory generic header */ + struct memory_object ap_pgr_hdr; + + /* pager-specific data */ + queue_chain_t pager_queue; /* next & prev pagers */ +#if MEMORY_OBJECT_HAS_REFCOUNT +#define ap_pgr_hdr_ref ap_pgr_hdr.mo_ref +#else + os_ref_atomic_t ap_pgr_hdr_ref; /* reference count */ +#endif + bool is_ready; /* is this pager ready ? */ + bool is_mapped; /* is this mem_obj mapped ? */ + bool is_cached; /* is this pager cached ? */ + vm_object_t backing_object; /* VM obj w/ encrypted data */ + vm_object_offset_t backing_offset; + vm_object_offset_t crypto_backing_offset; /* for key... */ + vm_object_offset_t crypto_start; + vm_object_offset_t crypto_end; + struct pager_crypt_info *crypt_info; } *apple_protect_pager_t; -#define APPLE_PROTECT_PAGER_NULL ((apple_protect_pager_t) NULL) -#define pager_ikot pager_header.io_bits +#define APPLE_PROTECT_PAGER_NULL ((apple_protect_pager_t) NULL) /* * List of memory objects managed by this EMM. * The list is protected by the "apple_protect_pager_lock" lock. */ -int apple_protect_pager_count = 0; /* number of pagers */ -int apple_protect_pager_count_mapped = 0; /* number of unmapped pagers */ -queue_head_t apple_protect_pager_queue; -decl_lck_mtx_data(,apple_protect_pager_lock) +unsigned int apple_protect_pager_count = 0; /* number of pagers */ +unsigned int apple_protect_pager_count_mapped = 0; /* number of unmapped pagers */ +queue_head_t apple_protect_pager_queue = QUEUE_HEAD_INITIALIZER(apple_protect_pager_queue); +LCK_GRP_DECLARE(apple_protect_pager_lck_grp, "apple_protect"); +LCK_MTX_DECLARE(apple_protect_pager_lock, &apple_protect_pager_lck_grp); /* * Maximum number of unmapped pagers we're willing to keep around. */ -int apple_protect_pager_cache_limit = 10; +unsigned int apple_protect_pager_cache_limit = 20; /* * Statistics & counters. */ -int apple_protect_pager_count_max = 0; -int apple_protect_pager_count_unmapped_max = 0; -int apple_protect_pager_num_trim_max = 0; -int apple_protect_pager_num_trim_total = 0; +unsigned int apple_protect_pager_count_max = 0; +unsigned int apple_protect_pager_count_unmapped_max = 0; +unsigned int apple_protect_pager_num_trim_max = 0; +unsigned int apple_protect_pager_num_trim_total = 0; -lck_grp_t apple_protect_pager_lck_grp; -lck_grp_attr_t apple_protect_pager_lck_grp_attr; -lck_attr_t apple_protect_pager_lck_attr; - /* internal prototypes */ -apple_protect_pager_t apple_protect_pager_create(vm_object_t backing_object, - struct pager_crypt_info *crypt_info); +apple_protect_pager_t apple_protect_pager_create( + vm_object_t backing_object, + vm_object_offset_t backing_offset, + vm_object_offset_t crypto_backing_offset, + struct pager_crypt_info *crypt_info, + vm_object_offset_t crypto_start, + vm_object_offset_t crypto_end, + boolean_t cache_pager); apple_protect_pager_t apple_protect_pager_lookup(memory_object_t mem_obj); void apple_protect_pager_dequeue(apple_protect_pager_t pager); void apple_protect_pager_deallocate_internal(apple_protect_pager_t pager, - boolean_t locked); + boolean_t locked); void apple_protect_pager_terminate_internal(apple_protect_pager_t pager); void apple_protect_pager_trim(void); #if DEBUG int apple_protect_pagerdebug = 0; -#define PAGER_ALL 0xffffffff -#define PAGER_INIT 0x00000001 -#define PAGER_PAGEIN 0x00000002 - -#define PAGER_DEBUG(LEVEL, A) \ - MACRO_BEGIN \ - if ((apple_protect_pagerdebug & LEVEL)==LEVEL) { \ - printf A; \ - } \ +#define PAGER_ALL 0xffffffff +#define PAGER_INIT 0x00000001 +#define PAGER_PAGEIN 0x00000002 + +#define PAGER_DEBUG(LEVEL, A) \ + MACRO_BEGIN \ + if ((apple_protect_pagerdebug & LEVEL)==LEVEL) { \ + printf A; \ + } \ MACRO_END #else #define PAGER_DEBUG(LEVEL, A) #endif - -void -apple_protect_pager_bootstrap(void) -{ - lck_grp_attr_setdefault(&apple_protect_pager_lck_grp_attr); - lck_grp_init(&apple_protect_pager_lck_grp, "apple_protect", &apple_protect_pager_lck_grp_attr); - lck_attr_setdefault(&apple_protect_pager_lck_attr); - lck_mtx_init(&apple_protect_pager_lock, &apple_protect_pager_lck_grp, &apple_protect_pager_lck_attr); - queue_init(&apple_protect_pager_queue); -} - /* * apple_protect_pager_init() * @@ -225,29 +235,30 @@ apple_protect_pager_bootstrap(void) */ kern_return_t apple_protect_pager_init( - memory_object_t mem_obj, - memory_object_control_t control, + memory_object_t mem_obj, + memory_object_control_t control, #if !DEBUG __unused #endif memory_object_cluster_size_t pg_size) { - apple_protect_pager_t pager; - kern_return_t kr; + apple_protect_pager_t pager; + kern_return_t kr; memory_object_attr_info_data_t attributes; PAGER_DEBUG(PAGER_ALL, - ("apple_protect_pager_init: %p, %p, %x\n", - mem_obj, control, pg_size)); + ("apple_protect_pager_init: %p, %p, %x\n", + mem_obj, control, pg_size)); - if (control == MEMORY_OBJECT_CONTROL_NULL) + if (control == MEMORY_OBJECT_CONTROL_NULL) { return KERN_INVALID_ARGUMENT; + } pager = apple_protect_pager_lookup(mem_obj); memory_object_control_reference(control); - pager->pager_control = control; + pager->ap_pgr_hdr.mo_control = control; attributes.copy_strategy = MEMORY_OBJECT_COPY_DELAY; /* attributes.cluster_size = (1 << (CLUSTER_SHIFT + PAGE_SHIFT));*/ @@ -256,13 +267,20 @@ apple_protect_pager_init( attributes.temporary = TRUE; kr = memory_object_change_attributes( - control, - MEMORY_OBJECT_ATTRIBUTE_INFO, - (memory_object_info_t) &attributes, - MEMORY_OBJECT_ATTR_INFO_COUNT); - if (kr != KERN_SUCCESS) + control, + MEMORY_OBJECT_ATTRIBUTE_INFO, + (memory_object_info_t) &attributes, + MEMORY_OBJECT_ATTR_INFO_COUNT); + if (kr != KERN_SUCCESS) { panic("apple_protect_pager_init: " - "memory_object_change_attributes() failed"); + "memory_object_change_attributes() failed"); + } + +#if CONFIG_SECLUDED_MEMORY + if (secluded_for_filecache) { + memory_object_mark_eligible_for_secluded(control, TRUE); + } +#endif /* CONFIG_SECLUDED_MEMORY */ return KERN_SUCCESS; } @@ -277,14 +295,14 @@ apple_protect_pager_init( */ kern_return_t apple_protect_pager_data_return( - __unused memory_object_t mem_obj, - __unused memory_object_offset_t offset, - __unused memory_object_cluster_size_t data_cnt, - __unused memory_object_offset_t *resid_offset, - __unused int *io_error, - __unused boolean_t dirty, - __unused boolean_t kernel_copy, - __unused int upl_flags) + __unused memory_object_t mem_obj, + __unused memory_object_offset_t offset, + __unused memory_object_cluster_size_t data_cnt, + __unused memory_object_offset_t *resid_offset, + __unused int *io_error, + __unused boolean_t dirty, + __unused boolean_t kernel_copy, + __unused int upl_flags) { panic("apple_protect_pager_data_return: should never get called"); return KERN_FAILURE; @@ -292,9 +310,9 @@ apple_protect_pager_data_return( kern_return_t apple_protect_pager_data_initialize( - __unused memory_object_t mem_obj, - __unused memory_object_offset_t offset, - __unused memory_object_cluster_size_t data_cnt) + __unused memory_object_t mem_obj, + __unused memory_object_offset_t offset, + __unused memory_object_cluster_size_t data_cnt) { panic("apple_protect_pager_data_initialize: should never get called"); return KERN_FAILURE; @@ -302,10 +320,10 @@ apple_protect_pager_data_initialize( kern_return_t apple_protect_pager_data_unlock( - __unused memory_object_t mem_obj, - __unused memory_object_offset_t offset, - __unused memory_object_size_t size, - __unused vm_prot_t desired_access) + __unused memory_object_t mem_obj, + __unused memory_object_offset_t offset, + __unused memory_object_size_t size, + __unused vm_prot_t desired_access) { return KERN_FAILURE; } @@ -315,45 +333,45 @@ apple_protect_pager_data_unlock( * * Handles page-in requests from VM. */ -kern_return_t +int apple_protect_pager_data_request_debug = 0; +kern_return_t apple_protect_pager_data_request( - memory_object_t mem_obj, - memory_object_offset_t offset, - memory_object_cluster_size_t length, + memory_object_t mem_obj, + memory_object_offset_t offset, + memory_object_cluster_size_t length, #if !DEBUG __unused #endif - vm_prot_t protection_required, + vm_prot_t protection_required, memory_object_fault_info_t mo_fault_info) { - apple_protect_pager_t pager; - memory_object_control_t mo_control; - upl_t upl; - int upl_flags; - upl_size_t upl_size; - upl_page_info_t *upl_pl; - unsigned int pl_count; - vm_object_t src_object, dst_object; - kern_return_t kr, retval; - vm_map_offset_t kernel_mapping; - vm_offset_t src_vaddr, dst_vaddr; - vm_offset_t cur_offset; - vm_map_entry_t map_entry; - kern_return_t error_code; - vm_prot_t prot; - vm_page_t src_page, top_page; - int interruptible; - struct vm_object_fault_info fault_info; - int ret; + apple_protect_pager_t pager; + memory_object_control_t mo_control; + upl_t upl; + int upl_flags; + upl_size_t upl_size; + upl_page_info_t *upl_pl; + unsigned int pl_count; + vm_object_t src_top_object, src_page_object, dst_object; + kern_return_t kr, retval; + vm_offset_t src_vaddr, dst_vaddr; + vm_offset_t cur_offset; + vm_offset_t offset_in_page; + kern_return_t error_code; + vm_prot_t prot; + vm_page_t src_page, top_page; + int interruptible; + struct vm_object_fault_info fault_info; + int ret; PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_data_request: %p, %llx, %x, %x\n", mem_obj, offset, length, protection_required)); retval = KERN_SUCCESS; - src_object = VM_OBJECT_NULL; - kernel_mapping = 0; + src_top_object = VM_OBJECT_NULL; + src_page_object = VM_OBJECT_NULL; upl = NULL; upl_pl = NULL; - fault_info = *((struct vm_object_fault_info *) mo_fault_info); + fault_info = *((struct vm_object_fault_info *)(uintptr_t)mo_fault_info); fault_info.stealth = TRUE; fault_info.io_sync = FALSE; fault_info.mark_zf_absent = FALSE; @@ -362,65 +380,44 @@ apple_protect_pager_data_request( pager = apple_protect_pager_lookup(mem_obj); assert(pager->is_ready); - assert(pager->ref_count > 1); /* pager is alive and mapped */ + assert(os_ref_get_count_raw(&pager->ap_pgr_hdr_ref) > 1); /* pager is alive and mapped */ PAGER_DEBUG(PAGER_PAGEIN, ("apple_protect_pager_data_request: %p, %llx, %x, %x, pager %p\n", mem_obj, offset, length, protection_required, pager)); + fault_info.lo_offset += pager->backing_offset; + fault_info.hi_offset += pager->backing_offset; + /* * Gather in a UPL all the VM pages requested by VM. */ - mo_control = pager->pager_control; + mo_control = pager->ap_pgr_hdr.mo_control; upl_size = length; upl_flags = - UPL_RET_ONLY_ABSENT | - UPL_SET_LITE | - UPL_NO_SYNC | - UPL_CLEAN_IN_PLACE | /* triggers UPL_CLEAR_DIRTY */ - UPL_SET_INTERNAL; + UPL_RET_ONLY_ABSENT | + UPL_SET_LITE | + UPL_NO_SYNC | + UPL_CLEAN_IN_PLACE | /* triggers UPL_CLEAR_DIRTY */ + UPL_SET_INTERNAL; pl_count = 0; kr = memory_object_upl_request(mo_control, - offset, upl_size, - &upl, NULL, NULL, upl_flags); + offset, upl_size, + &upl, NULL, NULL, upl_flags, VM_KERN_MEMORY_SECURITY); if (kr != KERN_SUCCESS) { retval = kr; goto done; } - dst_object = mo_control->moc_object; + dst_object = memory_object_control_to_vm_object(mo_control); assert(dst_object != VM_OBJECT_NULL); - /* - * Reserve 2 virtual pages in the kernel address space to map each - * source and destination physical pages when it's their turn to - * be processed. - */ - vm_object_reference(kernel_object); /* ref. for mapping */ - kr = vm_map_find_space(kernel_map, - &kernel_mapping, - 2 * PAGE_SIZE_64, - 0, - 0, - &map_entry); - if (kr != KERN_SUCCESS) { - vm_object_deallocate(kernel_object); - retval = kr; - goto done; - } - map_entry->object.vm_object = kernel_object; - map_entry->offset = kernel_mapping; - vm_map_unlock(kernel_map); - src_vaddr = CAST_DOWN(vm_offset_t, kernel_mapping); - dst_vaddr = CAST_DOWN(vm_offset_t, kernel_mapping + PAGE_SIZE_64); - - /* - * We'll map the encrypted data in the kernel address space from the + * We'll map the encrypted data in the kernel address space from the * backing VM object (itself backed by the encrypted file via * the vnode pager). */ - src_object = pager->backing_object; - assert(src_object != VM_OBJECT_NULL); - vm_object_reference(src_object); /* to keep the source object alive */ + src_top_object = pager->backing_object; + assert(src_top_object != VM_OBJECT_NULL); + vm_object_reference(src_top_object); /* keep the source object alive */ /* * Fill in the contents of the pages requested by VM. @@ -428,8 +425,8 @@ apple_protect_pager_data_request( upl_pl = UPL_GET_INTERNAL_PAGE_LIST(upl); pl_count = length / PAGE_SIZE; for (cur_offset = 0; - retval == KERN_SUCCESS && cur_offset < length; - cur_offset += PAGE_SIZE) { + retval == KERN_SUCCESS && cur_offset < length; + cur_offset += PAGE_SIZE) { ppnum_t dst_pnum; if (!upl_page_present(upl_pl, (int)(cur_offset / PAGE_SIZE))) { @@ -440,27 +437,27 @@ apple_protect_pager_data_request( /* * Map the source (encrypted) page in the kernel's * virtual address space. - * We already hold a reference on the src_object. + * We already hold a reference on the src_top_object. */ - retry_src_fault: - vm_object_lock(src_object); - vm_object_paging_begin(src_object); +retry_src_fault: + vm_object_lock(src_top_object); + vm_object_paging_begin(src_top_object); error_code = 0; prot = VM_PROT_READ; src_page = VM_PAGE_NULL; - kr = vm_fault_page(src_object, - offset + cur_offset, - VM_PROT_READ, - FALSE, - FALSE, /* src_page not looked up */ - &prot, - &src_page, - &top_page, - NULL, - &error_code, - FALSE, - FALSE, - &fault_info); + kr = vm_fault_page(src_top_object, + pager->backing_offset + offset + cur_offset, + VM_PROT_READ, + FALSE, + FALSE, /* src_page not looked up */ + &prot, + &src_page, + &top_page, + NULL, + &error_code, + FALSE, + FALSE, + &fault_info); switch (kr) { case VM_FAULT_SUCCESS: break; @@ -470,15 +467,15 @@ apple_protect_pager_data_request( if (vm_page_wait(interruptible)) { goto retry_src_fault; } - /* fall thru */ + OS_FALLTHROUGH; case VM_FAULT_INTERRUPTED: retval = MACH_SEND_INTERRUPTED; goto done; case VM_FAULT_SUCCESS_NO_VM_PAGE: /* success but no VM page: fail */ - vm_object_paging_end(src_object); - vm_object_unlock(src_object); - /*FALLTHROUGH*/ + vm_object_paging_end(src_top_object); + vm_object_unlock(src_top_object); + OS_FALLTHROUGH; case VM_FAULT_MEMORY_ERROR: /* the page is not there ! */ if (error_code) { @@ -489,105 +486,186 @@ apple_protect_pager_data_request( goto done; default: panic("apple_protect_pager_data_request: " - "vm_fault_page() unexpected error 0x%x\n", - kr); + "vm_fault_page() unexpected error 0x%x\n", + kr); } assert(src_page != VM_PAGE_NULL); - assert(src_page->busy); + assert(src_page->vmp_busy); - if (!src_page->active && - !src_page->inactive && - !src_page->throttled) { + if (src_page->vmp_q_state != VM_PAGE_ON_SPECULATIVE_Q) { vm_page_lockspin_queues(); - if (!src_page->active && - !src_page->inactive && - !src_page->throttled) { - vm_page_deactivate(src_page); + + if (src_page->vmp_q_state != VM_PAGE_ON_SPECULATIVE_Q) { + vm_page_speculate(src_page, FALSE); } vm_page_unlock_queues(); } - - /* - * Establish an explicit mapping of the source - * physical page. - */ - pmap_enter(kernel_pmap, - kernel_mapping, - src_page->phys_page, - VM_PROT_READ, - VM_PROT_NONE, - 0, - TRUE); + /* - * Establish an explicit pmap mapping of the destination - * physical page. - * We can't do a regular VM mapping because the VM page - * is "busy". + * Establish pointers to the source + * and destination physical pages. */ dst_pnum = (ppnum_t) - upl_phys_page(upl_pl, (int)(cur_offset / PAGE_SIZE)); + upl_phys_page(upl_pl, (int)(cur_offset / PAGE_SIZE)); assert(dst_pnum != 0); - pmap_enter(kernel_pmap, - kernel_mapping + PAGE_SIZE_64, - dst_pnum, - VM_PROT_READ | VM_PROT_WRITE, - VM_PROT_NONE, - 0, - TRUE); + + src_vaddr = (vm_map_offset_t) + phystokv((pmap_paddr_t)VM_PAGE_GET_PHYS_PAGE(src_page) + << PAGE_SHIFT); + dst_vaddr = (vm_map_offset_t) + phystokv((pmap_paddr_t)dst_pnum << PAGE_SHIFT); + + src_page_object = VM_PAGE_OBJECT(src_page); + + /* + * Validate the original page... + */ + if (src_page_object->code_signed) { + vm_page_validate_cs_mapped( + src_page, PAGE_SIZE, 0, + (const void *) src_vaddr); + } + /* + * ... and transfer the results to the destination page. + */ + UPL_SET_CS_VALIDATED(upl_pl, cur_offset / PAGE_SIZE, + src_page->vmp_cs_validated); + UPL_SET_CS_TAINTED(upl_pl, cur_offset / PAGE_SIZE, + src_page->vmp_cs_tainted); + UPL_SET_CS_NX(upl_pl, cur_offset / PAGE_SIZE, + src_page->vmp_cs_nx); + + /* + * page_decrypt() might access a mapped file, so let's release + * the object lock for the source page to avoid a potential + * deadlock. The source page is kept busy and we have a + * "paging_in_progress" reference on its object, so it's safe + * to unlock the object here. + */ + assert(src_page->vmp_busy); + assert(src_page_object->paging_in_progress > 0); + vm_object_unlock(src_page_object); /* * Decrypt the encrypted contents of the source page * into the destination page. */ - ret = pager->crypt.page_decrypt((const void *) src_vaddr, - (void *) dst_vaddr, - offset+cur_offset, - pager->crypt.crypt_ops); + for (offset_in_page = 0; + offset_in_page < PAGE_SIZE; + offset_in_page += 4096) { + if (offset + cur_offset + offset_in_page < + pager->crypto_start || + offset + cur_offset + offset_in_page >= + pager->crypto_end) { + /* not encrypted: just copy */ + bcopy((const char *)(src_vaddr + + offset_in_page), + (char *)(dst_vaddr + offset_in_page), + 4096); + + if (apple_protect_pager_data_request_debug) { + printf("apple_protect_data_request" + "(%p,0x%llx+0x%llx+0x%04llx): " + "out of crypto range " + "[0x%llx:0x%llx]: " + "COPY [0x%016llx 0x%016llx] " + "code_signed=%d " + "cs_validated=%d " + "cs_tainted=%d " + "cs_nx=%d\n", + pager, + offset, + (uint64_t) cur_offset, + (uint64_t) offset_in_page, + pager->crypto_start, + pager->crypto_end, + *(uint64_t *)(dst_vaddr + + offset_in_page), + *(uint64_t *)(dst_vaddr + + offset_in_page + 8), + src_page_object->code_signed, + src_page->vmp_cs_validated, + src_page->vmp_cs_tainted, + src_page->vmp_cs_nx); + } + ret = 0; + continue; + } + ret = pager->crypt_info->page_decrypt( + (const void *)(src_vaddr + offset_in_page), + (void *)(dst_vaddr + offset_in_page), + ((pager->crypto_backing_offset - + pager->crypto_start) + /* XXX ? */ + offset + + cur_offset + + offset_in_page), + pager->crypt_info->crypt_ops); + + if (apple_protect_pager_data_request_debug) { + printf("apple_protect_data_request" + "(%p,0x%llx+0x%llx+0x%04llx): " + "in crypto range [0x%llx:0x%llx]: " + "DECRYPT offset 0x%llx=" + "(0x%llx-0x%llx+0x%llx+0x%llx+0x%04llx)" + "[0x%016llx 0x%016llx] " + "code_signed=%d " + "cs_validated=%d " + "cs_tainted=%d " + "cs_nx=%d " + "ret=0x%x\n", + pager, + offset, + (uint64_t) cur_offset, + (uint64_t) offset_in_page, + pager->crypto_start, pager->crypto_end, + ((pager->crypto_backing_offset - + pager->crypto_start) + + offset + + cur_offset + + offset_in_page), + pager->crypto_backing_offset, + pager->crypto_start, + offset, + (uint64_t) cur_offset, + (uint64_t) offset_in_page, + *(uint64_t *)(dst_vaddr + offset_in_page), + *(uint64_t *)(dst_vaddr + offset_in_page + 8), + src_page_object->code_signed, + src_page->vmp_cs_validated, + src_page->vmp_cs_tainted, + src_page->vmp_cs_nx, + ret); + } + if (ret) { + break; + } + } if (ret) { /* * Decryption failed. Abort the fault. */ retval = KERN_ABORTED; - } else { - /* - * Validate the original page... - */ - if (src_page->object->code_signed) { - vm_page_validate_cs_mapped( - src_page, - (const void *) src_vaddr); - } - /* - * ... and transfer the results to the destination page. - */ - UPL_SET_CS_VALIDATED(upl_pl, cur_offset / PAGE_SIZE, - src_page->cs_validated); - UPL_SET_CS_TAINTED(upl_pl, cur_offset / PAGE_SIZE, - src_page->cs_tainted); } - - /* - * Remove the pmap mapping of the source and destination pages - * in the kernel. - */ - pmap_remove(kernel_pmap, - (addr64_t) kernel_mapping, - (addr64_t) (kernel_mapping + (2 * PAGE_SIZE_64))); + + assert(VM_PAGE_OBJECT(src_page) == src_page_object); + assert(src_page->vmp_busy); + assert(src_page_object->paging_in_progress > 0); + vm_object_lock(src_page_object); /* * Cleanup the result of vm_fault_page() of the source page. */ PAGE_WAKEUP_DONE(src_page); - vm_object_paging_end(src_page->object); - vm_object_unlock(src_page->object); - if (top_page != VM_PAGE_NULL) { - vm_object_t top_object; + src_page = VM_PAGE_NULL; + vm_object_paging_end(src_page_object); + vm_object_unlock(src_page_object); - top_object = top_page->object; - vm_object_lock(top_object); + if (top_page != VM_PAGE_NULL) { + assert(VM_PAGE_OBJECT(top_page) == src_top_object); + vm_object_lock(src_top_object); VM_PAGE_FREE(top_page); - vm_object_paging_end(top_object); - vm_object_unlock(top_object); + vm_object_paging_end(src_top_object); + vm_object_unlock(src_top_object); } } @@ -608,7 +686,7 @@ done: if (retval != KERN_SUCCESS) { upl_abort(upl, 0); if (retval == KERN_ABORTED) { - wait_result_t wait_result; + wait_result_t wait_result; /* * We aborted the fault and did not provide @@ -628,7 +706,7 @@ done: wait_result = assert_wait_timeout( (event_t) apple_protect_pager_data_request, THREAD_UNINT, - 10000, /* 10ms */ + 10000, /* 10ms */ NSEC_PER_USEC); assert(wait_result == THREAD_WAITING); wait_result = thread_block(THREAD_CONTINUE_NULL); @@ -636,30 +714,21 @@ done: } } else { boolean_t empty; - upl_commit_range(upl, 0, upl->size, - UPL_COMMIT_CS_VALIDATED | UPL_COMMIT_WRITTEN_BY_KERNEL, - upl_pl, pl_count, &empty); + assertf(page_aligned(upl->u_offset) && page_aligned(upl->u_size), + "upl %p offset 0x%llx size 0x%x", + upl, upl->u_offset, upl->u_size); + upl_commit_range(upl, 0, upl->u_size, + UPL_COMMIT_CS_VALIDATED | UPL_COMMIT_WRITTEN_BY_KERNEL, + upl_pl, pl_count, &empty); } /* and deallocate the UPL */ upl_deallocate(upl); upl = NULL; } - if (kernel_mapping != 0) { - /* clean up the mapping of the source and destination pages */ - kr = vm_map_remove(kernel_map, - kernel_mapping, - kernel_mapping + (2 * PAGE_SIZE_64), - VM_MAP_NO_FLAGS); - assert(kr == KERN_SUCCESS); - kernel_mapping = 0; - src_vaddr = 0; - dst_vaddr = 0; + if (src_top_object != VM_OBJECT_NULL) { + vm_object_deallocate(src_top_object); } - if (src_object != VM_OBJECT_NULL) { - vm_object_deallocate(src_object); - } - return retval; } @@ -672,15 +741,14 @@ done: */ void apple_protect_pager_reference( - memory_object_t mem_obj) -{ - apple_protect_pager_t pager; + memory_object_t mem_obj) +{ + apple_protect_pager_t pager; pager = apple_protect_pager_lookup(mem_obj); lck_mtx_lock(&apple_protect_pager_lock); - assert(pager->ref_count > 0); - pager->ref_count++; + os_ref_retain_locked_raw(&pager->ap_pgr_hdr_ref, NULL); lck_mtx_unlock(&apple_protect_pager_lock); } @@ -699,12 +767,12 @@ apple_protect_pager_dequeue( assert(!pager->is_mapped); queue_remove(&apple_protect_pager_queue, - pager, - apple_protect_pager_t, - pager_queue); + pager, + apple_protect_pager_t, + pager_queue); pager->pager_queue.next = NULL; pager->pager_queue.prev = NULL; - + apple_protect_pager_count--; } @@ -732,13 +800,19 @@ apple_protect_pager_terminate_internal( vm_object_deallocate(pager->backing_object); pager->backing_object = VM_OBJECT_NULL; } - - /* deallocate any crypt module data */ - if(pager->crypt.crypt_end) - pager->crypt.crypt_end(pager->crypt.crypt_ops); + + /* one less pager using this "pager_crypt_info" */ +#if CRYPT_INFO_DEBUG + printf("CRYPT_INFO %s: deallocate %p ref %d\n", + __FUNCTION__, + pager->crypt_info, + pager->crypt_info->crypt_refcnt); +#endif /* CRYPT_INFO_DEBUG */ + crypt_info_deallocate(pager->crypt_info); + pager->crypt_info = NULL; /* trigger the destruction of the memory object */ - memory_object_destroy(pager->pager_control, 0); + memory_object_destroy(pager->ap_pgr_hdr.mo_control, 0); } /* @@ -751,18 +825,19 @@ apple_protect_pager_terminate_internal( */ void apple_protect_pager_deallocate_internal( - apple_protect_pager_t pager, - boolean_t locked) + apple_protect_pager_t pager, + boolean_t locked) { - boolean_t needs_trimming; - int count_unmapped; + boolean_t needs_trimming; + unsigned int count_unmapped; + os_ref_count_t ref_count; - if (! locked) { + if (!locked) { lck_mtx_lock(&apple_protect_pager_lock); } - count_unmapped = (apple_protect_pager_count - - apple_protect_pager_count_mapped); + count_unmapped = (apple_protect_pager_count - + apple_protect_pager_count_mapped); if (count_unmapped > apple_protect_pager_cache_limit) { /* we have too many unmapped pagers: trim some */ needs_trimming = TRUE; @@ -771,9 +846,9 @@ apple_protect_pager_deallocate_internal( } /* drop a reference on this pager */ - pager->ref_count--; + ref_count = os_ref_release_locked_raw(&pager->ap_pgr_hdr_ref, NULL); - if (pager->ref_count == 1) { + if (ref_count == 1) { /* * Only the "named" reference is left, which means that * no one is really holding on to this pager anymore. @@ -783,18 +858,18 @@ apple_protect_pager_deallocate_internal( /* the pager is all ours: no need for the lock now */ lck_mtx_unlock(&apple_protect_pager_lock); apple_protect_pager_terminate_internal(pager); - } else if (pager->ref_count == 0) { + } else if (ref_count == 0) { /* * Dropped the existence reference; the memory object has * been terminated. Do some final cleanup and release the * pager structure. */ lck_mtx_unlock(&apple_protect_pager_lock); - if (pager->pager_control != MEMORY_OBJECT_CONTROL_NULL) { - memory_object_control_deallocate(pager->pager_control); - pager->pager_control = MEMORY_OBJECT_CONTROL_NULL; + if (pager->ap_pgr_hdr.mo_control != MEMORY_OBJECT_CONTROL_NULL) { + memory_object_control_deallocate(pager->ap_pgr_hdr.mo_control); + pager->ap_pgr_hdr.mo_control = MEMORY_OBJECT_CONTROL_NULL; } - kfree(pager, sizeof (*pager)); + kfree(pager, sizeof(*pager)); pager = APPLE_PROTECT_PAGER_NULL; } else { /* there are still plenty of references: keep going... */ @@ -815,9 +890,9 @@ apple_protect_pager_deallocate_internal( */ void apple_protect_pager_deallocate( - memory_object_t mem_obj) + memory_object_t mem_obj) { - apple_protect_pager_t pager; + apple_protect_pager_t pager; PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_deallocate: %p\n", mem_obj)); pager = apple_protect_pager_lookup(mem_obj); @@ -832,7 +907,7 @@ apple_protect_pager_terminate( #if !DEBUG __unused #endif - memory_object_t mem_obj) + memory_object_t mem_obj) { PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_terminate: %p\n", mem_obj)); @@ -844,21 +919,13 @@ apple_protect_pager_terminate( */ kern_return_t apple_protect_pager_synchronize( - memory_object_t mem_obj, - memory_object_offset_t offset, - memory_object_size_t length, - __unused vm_sync_t sync_flags) + __unused memory_object_t mem_obj, + __unused memory_object_offset_t offset, + __unused memory_object_size_t length, + __unused vm_sync_t sync_flags) { - apple_protect_pager_t pager; - - PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_synchronize: %p\n", mem_obj)); - - pager = apple_protect_pager_lookup(mem_obj); - - memory_object_synchronize_completed(pager->pager_control, - offset, length); - - return KERN_SUCCESS; + panic("apple_protect_pager_synchronize: memory_object_synchronize no longer supported\n"); + return KERN_FAILURE; } /* @@ -871,10 +938,10 @@ apple_protect_pager_synchronize( */ kern_return_t apple_protect_pager_map( - memory_object_t mem_obj, - __unused vm_prot_t prot) + memory_object_t mem_obj, + __unused vm_prot_t prot) { - apple_protect_pager_t pager; + apple_protect_pager_t pager; PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_map: %p\n", mem_obj)); @@ -882,7 +949,7 @@ apple_protect_pager_map( lck_mtx_lock(&apple_protect_pager_lock); assert(pager->is_ready); - assert(pager->ref_count > 0); /* pager is alive */ + assert(os_ref_get_count_raw(&pager->ap_pgr_hdr_ref) > 0); /* pager is alive */ if (pager->is_mapped == FALSE) { /* * First mapping of this pager: take an extra reference @@ -890,7 +957,7 @@ apple_protect_pager_map( * are removed. */ pager->is_mapped = TRUE; - pager->ref_count++; + os_ref_retain_locked_raw(&pager->ap_pgr_hdr_ref, NULL); apple_protect_pager_count_mapped++; } lck_mtx_unlock(&apple_protect_pager_lock); @@ -905,13 +972,13 @@ apple_protect_pager_map( */ kern_return_t apple_protect_pager_last_unmap( - memory_object_t mem_obj) + memory_object_t mem_obj) { - apple_protect_pager_t pager; - int count_unmapped; + apple_protect_pager_t pager; + unsigned int count_unmapped; PAGER_DEBUG(PAGER_ALL, - ("apple_protect_pager_last_unmap: %p\n", mem_obj)); + ("apple_protect_pager_last_unmap: %p\n", mem_obj)); pager = apple_protect_pager_lookup(mem_obj); @@ -923,7 +990,7 @@ apple_protect_pager_last_unmap( */ apple_protect_pager_count_mapped--; count_unmapped = (apple_protect_pager_count - - apple_protect_pager_count_mapped); + apple_protect_pager_count_mapped); if (count_unmapped > apple_protect_pager_count_unmapped_max) { apple_protect_pager_count_unmapped_max = count_unmapped; } @@ -933,36 +1000,61 @@ apple_protect_pager_last_unmap( } else { lck_mtx_unlock(&apple_protect_pager_lock); } - + return KERN_SUCCESS; } +boolean_t +apple_protect_pager_backing_object( + memory_object_t mem_obj, + memory_object_offset_t offset, + vm_object_t *backing_object, + vm_object_offset_t *backing_offset) +{ + apple_protect_pager_t pager; + + PAGER_DEBUG(PAGER_ALL, + ("apple_protect_pager_backing_object: %p\n", mem_obj)); + + pager = apple_protect_pager_lookup(mem_obj); + + *backing_object = pager->backing_object; + *backing_offset = pager->backing_offset + offset; + + return TRUE; +} /* * */ apple_protect_pager_t apple_protect_pager_lookup( - memory_object_t mem_obj) + memory_object_t mem_obj) { - apple_protect_pager_t pager; + apple_protect_pager_t pager; - pager = (apple_protect_pager_t) mem_obj; - assert(pager->pager_ops == &apple_protect_pager_ops); - assert(pager->ref_count > 0); + assert(mem_obj->mo_pager_ops == &apple_protect_pager_ops); + pager = (apple_protect_pager_t)(uintptr_t) mem_obj; + assert(os_ref_get_count_raw(&pager->ap_pgr_hdr_ref) > 0); return pager; } apple_protect_pager_t apple_protect_pager_create( - vm_object_t backing_object, - struct pager_crypt_info *crypt_info) + vm_object_t backing_object, + vm_object_offset_t backing_offset, + vm_object_offset_t crypto_backing_offset, + struct pager_crypt_info *crypt_info, + vm_object_offset_t crypto_start, + vm_object_offset_t crypto_end, + boolean_t cache_pager) { - apple_protect_pager_t pager, pager2; - memory_object_control_t control; - kern_return_t kr; + apple_protect_pager_t pager, pager2; + memory_object_control_t control; + kern_return_t kr; + struct pager_crypt_info *old_crypt_info; - pager = (apple_protect_pager_t) kalloc(sizeof (*pager)); + pager = (apple_protect_pager_t) kalloc(sizeof(*pager)); if (pager == APPLE_PROTECT_PAGER_NULL) { return APPLE_PROTECT_PAGER_NULL; } @@ -974,36 +1066,100 @@ apple_protect_pager_create( * we reserve the first word in the object for a fake ip_kotype * setting - that will tell vm_map to use it as a memory object. */ - pager->pager_ops = &apple_protect_pager_ops; - pager->pager_ikot = IKOT_MEMORY_OBJECT; + pager->ap_pgr_hdr.mo_ikot = IKOT_MEMORY_OBJECT; + pager->ap_pgr_hdr.mo_pager_ops = &apple_protect_pager_ops; + pager->ap_pgr_hdr.mo_control = MEMORY_OBJECT_CONTROL_NULL; + pager->is_ready = FALSE;/* not ready until it has a "name" */ - pager->ref_count = 2; /* existence + setup reference */ + /* one reference for the caller */ + os_ref_init_count_raw(&pager->ap_pgr_hdr_ref, NULL, 1); pager->is_mapped = FALSE; - pager->pager_control = MEMORY_OBJECT_CONTROL_NULL; + if (cache_pager) { + /* extra reference for the cache */ + os_ref_retain_locked_raw(&pager->ap_pgr_hdr_ref, NULL); + pager->is_cached = true; + } else { + pager->is_cached = false; + } pager->backing_object = backing_object; - pager->crypt = *crypt_info; - + pager->backing_offset = backing_offset; + pager->crypto_backing_offset = crypto_backing_offset; + pager->crypto_start = crypto_start; + pager->crypto_end = crypto_end; + pager->crypt_info = crypt_info; /* allocated by caller */ + +#if CRYPT_INFO_DEBUG + printf("CRYPT_INFO %s: crypt_info %p [%p,%p,%p,%d]\n", + __FUNCTION__, + crypt_info, + crypt_info->page_decrypt, + crypt_info->crypt_end, + crypt_info->crypt_ops, + crypt_info->crypt_refcnt); +#endif /* CRYPT_INFO_DEBUG */ + vm_object_reference(backing_object); + old_crypt_info = NULL; + lck_mtx_lock(&apple_protect_pager_lock); /* see if anyone raced us to create a pager for the same object */ queue_iterate(&apple_protect_pager_queue, - pager2, - apple_protect_pager_t, - pager_queue) { - if (pager2->backing_object == backing_object) { + pager2, + apple_protect_pager_t, + pager_queue) { + if ((pager2->crypt_info->page_decrypt != + crypt_info->page_decrypt) || + (pager2->crypt_info->crypt_end != + crypt_info->crypt_end) || + (pager2->crypt_info->crypt_ops != + crypt_info->crypt_ops)) { + /* crypt_info contents do not match: next pager */ + continue; + } + + /* found a match for crypt_info ... */ + if (old_crypt_info) { + /* ... already switched to that crypt_info */ + assert(old_crypt_info == pager2->crypt_info); + } else if (pager2->crypt_info != crypt_info) { + /* ... switch to that pager's crypt_info */ +#if CRYPT_INFO_DEBUG + printf("CRYPT_INFO %s: reference %p ref %d " + "(create match)\n", + __FUNCTION__, + pager2->crypt_info, + pager2->crypt_info->crypt_refcnt); +#endif /* CRYPT_INFO_DEBUG */ + old_crypt_info = pager2->crypt_info; + crypt_info_reference(old_crypt_info); + pager->crypt_info = old_crypt_info; + } + + if (pager2->backing_object == backing_object && + pager2->backing_offset == backing_offset && + pager2->crypto_backing_offset == crypto_backing_offset && + pager2->crypto_start == crypto_start && + pager2->crypto_end == crypto_end) { + /* full match: use that pager */ break; } } - if (! queue_end(&apple_protect_pager_queue, - (queue_entry_t) pager2)) { - /* while we hold the lock, transfer our setup ref to winner */ - pager2->ref_count++; + if (!queue_end(&apple_protect_pager_queue, + (queue_entry_t) pager2)) { /* we lost the race, down with the loser... */ lck_mtx_unlock(&apple_protect_pager_lock); vm_object_deallocate(pager->backing_object); pager->backing_object = VM_OBJECT_NULL; - kfree(pager, sizeof (*pager)); +#if CRYPT_INFO_DEBUG + printf("CRYPT_INFO %s: %p ref %d (create pager match)\n", + __FUNCTION__, + pager->crypt_info, + pager->crypt_info->crypt_refcnt); +#endif /* CRYPT_INFO_DEBUG */ + crypt_info_deallocate(pager->crypt_info); + pager->crypt_info = NULL; + kfree(pager, sizeof(*pager)); /* ... and go with the winner */ pager = pager2; /* let the winner make sure the pager gets ready */ @@ -1012,9 +1168,9 @@ apple_protect_pager_create( /* enter new pager at the head of our list of pagers */ queue_enter_first(&apple_protect_pager_queue, - pager, - apple_protect_pager_t, - pager_queue); + pager, + apple_protect_pager_t, + pager_queue); apple_protect_pager_count++; if (apple_protect_pager_count > apple_protect_pager_count_max) { apple_protect_pager_count_max = apple_protect_pager_count; @@ -1022,10 +1178,12 @@ apple_protect_pager_create( lck_mtx_unlock(&apple_protect_pager_lock); kr = memory_object_create_named((memory_object_t) pager, - 0, - &control); + 0, + &control); assert(kr == KERN_SUCCESS); + memory_object_mark_trusted(control); + lck_mtx_lock(&apple_protect_pager_lock); /* the new pager is now ready to be used */ pager->is_ready = TRUE; @@ -1034,6 +1192,20 @@ apple_protect_pager_create( /* wakeup anyone waiting for this pager to be ready */ thread_wakeup(&pager->is_ready); + if (old_crypt_info != NULL && + old_crypt_info != crypt_info) { + /* we re-used an old crypt_info instead of using our new one */ +#if CRYPT_INFO_DEBUG + printf("CRYPT_INFO %s: deallocate %p ref %d " + "(create used old)\n", + __FUNCTION__, + crypt_info, + crypt_info->crypt_refcnt); +#endif /* CRYPT_INFO_DEBUG */ + crypt_info_deallocate(crypt_info); + crypt_info = NULL; + } + return pager; } @@ -1046,64 +1218,193 @@ apple_protect_pager_create( */ memory_object_t apple_protect_pager_setup( - vm_object_t backing_object, - struct pager_crypt_info *crypt_info) + vm_object_t backing_object, + vm_object_offset_t backing_offset, + vm_object_offset_t crypto_backing_offset, + struct pager_crypt_info *crypt_info, + vm_object_offset_t crypto_start, + vm_object_offset_t crypto_end, + boolean_t cache_pager) { - apple_protect_pager_t pager; + apple_protect_pager_t pager; + struct pager_crypt_info *old_crypt_info, *new_crypt_info; + +#if CRYPT_INFO_DEBUG + printf("CRYPT_INFO %s: crypt_info=%p [%p,%p,%p,%d]\n", + __FUNCTION__, + crypt_info, + crypt_info->page_decrypt, + crypt_info->crypt_end, + crypt_info->crypt_ops, + crypt_info->crypt_refcnt); +#endif /* CRYPT_INFO_DEBUG */ + + old_crypt_info = NULL; lck_mtx_lock(&apple_protect_pager_lock); queue_iterate(&apple_protect_pager_queue, - pager, - apple_protect_pager_t, - pager_queue) { - if (pager->backing_object == backing_object) { - /* For the same object we must always use the same protection options */ - if (!((pager->crypt.page_decrypt == crypt_info->page_decrypt) && - (pager->crypt.crypt_ops == crypt_info->crypt_ops) )) { - lck_mtx_unlock(&apple_protect_pager_lock); - return MEMORY_OBJECT_NULL; - } + pager, + apple_protect_pager_t, + pager_queue) { + if ((pager->crypt_info->page_decrypt != + crypt_info->page_decrypt) || + (pager->crypt_info->crypt_end != + crypt_info->crypt_end) || + (pager->crypt_info->crypt_ops != + crypt_info->crypt_ops)) { + /* no match for "crypt_info": next pager */ + continue; + } + /* found a match for crypt_info ... */ + if (old_crypt_info) { + /* ... already switched to that crypt_info */ + assert(old_crypt_info == pager->crypt_info); + } else { + /* ... switch to that pager's crypt_info */ + old_crypt_info = pager->crypt_info; +#if CRYPT_INFO_DEBUG + printf("CRYPT_INFO %s: " + "switching crypt_info from %p [%p,%p,%p,%d] " + "to %p [%p,%p,%p,%d] from pager %p\n", + __FUNCTION__, + crypt_info, + crypt_info->page_decrypt, + crypt_info->crypt_end, + crypt_info->crypt_ops, + crypt_info->crypt_refcnt, + old_crypt_info, + old_crypt_info->page_decrypt, + old_crypt_info->crypt_end, + old_crypt_info->crypt_ops, + old_crypt_info->crypt_refcnt, + pager); + printf("CRYPT_INFO %s: %p ref %d (setup match)\n", + __FUNCTION__, + pager->crypt_info, + pager->crypt_info->crypt_refcnt); +#endif /* CRYPT_INFO_DEBUG */ + crypt_info_reference(pager->crypt_info); + } + + if (pager->backing_object == backing_object && + pager->backing_offset == backing_offset && + pager->crypto_backing_offset == crypto_backing_offset && + pager->crypto_start == crypto_start && + pager->crypto_end == crypto_end) { + /* full match: use that pager! */ + assert(old_crypt_info == pager->crypt_info); + assert(old_crypt_info->crypt_refcnt > 1); +#if CRYPT_INFO_DEBUG + printf("CRYPT_INFO %s: " + "pager match with %p crypt_info %p\n", + __FUNCTION__, + pager, + pager->crypt_info); + printf("CRYPT_INFO %s: deallocate %p ref %d " + "(pager match)\n", + __FUNCTION__, + old_crypt_info, + old_crypt_info->crypt_refcnt); +#endif /* CRYPT_INFO_DEBUG */ + /* release the extra ref on crypt_info we got above */ + crypt_info_deallocate(old_crypt_info); + assert(old_crypt_info->crypt_refcnt > 0); + /* give extra reference on pager to the caller */ + os_ref_retain_locked_raw(&pager->ap_pgr_hdr_ref, NULL); break; } } if (queue_end(&apple_protect_pager_queue, - (queue_entry_t) pager)) { + (queue_entry_t) pager)) { + lck_mtx_unlock(&apple_protect_pager_lock); /* no existing pager for this backing object */ pager = APPLE_PROTECT_PAGER_NULL; - } else { - /* make sure pager doesn't disappear */ - pager->ref_count++; - } - - lck_mtx_unlock(&apple_protect_pager_lock); - - if (pager == APPLE_PROTECT_PAGER_NULL) { - pager = apple_protect_pager_create(backing_object, crypt_info); + if (old_crypt_info) { + /* use this old crypt_info for new pager */ + new_crypt_info = old_crypt_info; +#if CRYPT_INFO_DEBUG + printf("CRYPT_INFO %s: " + "will use old_crypt_info %p for new pager\n", + __FUNCTION__, + old_crypt_info); +#endif /* CRYPT_INFO_DEBUG */ + } else { + /* allocate a new crypt_info for new pager */ + new_crypt_info = kalloc(sizeof(*new_crypt_info)); + *new_crypt_info = *crypt_info; + new_crypt_info->crypt_refcnt = 1; +#if CRYPT_INFO_DEBUG + printf("CRYPT_INFO %s: " + "will use new_crypt_info %p for new pager\n", + __FUNCTION__, + new_crypt_info); +#endif /* CRYPT_INFO_DEBUG */ + } + if (new_crypt_info == NULL) { + /* can't create new pager without a crypt_info */ + } else { + /* create new pager */ + pager = apple_protect_pager_create( + backing_object, + backing_offset, + crypto_backing_offset, + new_crypt_info, + crypto_start, + crypto_end, + cache_pager); + } if (pager == APPLE_PROTECT_PAGER_NULL) { + /* could not create a new pager */ + if (new_crypt_info == old_crypt_info) { + /* release extra reference on old_crypt_info */ +#if CRYPT_INFO_DEBUG + printf("CRYPT_INFO %s: deallocate %p ref %d " + "(create fail old_crypt_info)\n", + __FUNCTION__, + old_crypt_info, + old_crypt_info->crypt_refcnt); +#endif /* CRYPT_INFO_DEBUG */ + crypt_info_deallocate(old_crypt_info); + old_crypt_info = NULL; + } else { + /* release unused new_crypt_info */ + assert(new_crypt_info->crypt_refcnt == 1); +#if CRYPT_INFO_DEBUG + printf("CRYPT_INFO %s: deallocate %p ref %d " + "(create fail new_crypt_info)\n", + __FUNCTION__, + new_crypt_info, + new_crypt_info->crypt_refcnt); +#endif /* CRYPT_INFO_DEBUG */ + crypt_info_deallocate(new_crypt_info); + new_crypt_info = NULL; + } return MEMORY_OBJECT_NULL; } + lck_mtx_lock(&apple_protect_pager_lock); + } else { + assert(old_crypt_info == pager->crypt_info); } - lck_mtx_lock(&apple_protect_pager_lock); while (!pager->is_ready) { lck_mtx_sleep(&apple_protect_pager_lock, - LCK_SLEEP_DEFAULT, - &pager->is_ready, - THREAD_UNINT); + LCK_SLEEP_DEFAULT, + &pager->is_ready, + THREAD_UNINT); } lck_mtx_unlock(&apple_protect_pager_lock); return (memory_object_t) pager; -} +} void apple_protect_pager_trim(void) { - apple_protect_pager_t pager, prev_pager; - queue_head_t trim_queue; - int num_trim; - int count_unmapped; + apple_protect_pager_t pager, prev_pager; + queue_head_t trim_queue; + unsigned int num_trim; + unsigned int count_unmapped; lck_mtx_lock(&apple_protect_pager_lock); @@ -1115,15 +1416,16 @@ apple_protect_pager_trim(void) num_trim = 0; for (pager = (apple_protect_pager_t) - queue_last(&apple_protect_pager_queue); - !queue_end(&apple_protect_pager_queue, - (queue_entry_t) pager); - pager = prev_pager) { + queue_last(&apple_protect_pager_queue); + !queue_end(&apple_protect_pager_queue, + (queue_entry_t) pager); + pager = prev_pager) { /* get prev elt before we dequeue */ prev_pager = (apple_protect_pager_t) - queue_prev(&pager->pager_queue); + queue_prev(&pager->pager_queue); - if (pager->ref_count == 2 && + if (pager->is_cached && + os_ref_get_count_raw(&pager->ap_pgr_hdr_ref) == 2 && pager->is_ready && !pager->is_mapped) { /* this pager can be trimmed */ @@ -1132,12 +1434,12 @@ apple_protect_pager_trim(void) apple_protect_pager_dequeue(pager); /* ... and add it to our trim queue */ queue_enter_first(&trim_queue, - pager, - apple_protect_pager_t, - pager_queue); + pager, + apple_protect_pager_t, + pager_queue); count_unmapped = (apple_protect_pager_count - - apple_protect_pager_count_mapped); + apple_protect_pager_count_mapped); if (count_unmapped <= apple_protect_pager_cache_limit) { /* we have enough pagers to trim */ break; @@ -1154,18 +1456,65 @@ apple_protect_pager_trim(void) /* terminate the trimmed pagers */ while (!queue_empty(&trim_queue)) { queue_remove_first(&trim_queue, - pager, - apple_protect_pager_t, - pager_queue); + pager, + apple_protect_pager_t, + pager_queue); + assert(pager->is_cached); + pager->is_cached = false; pager->pager_queue.next = NULL; pager->pager_queue.prev = NULL; - assert(pager->ref_count == 2); /* * We can't call deallocate_internal() because the pager * has already been dequeued, but we still need to remove * a reference. */ - pager->ref_count--; + os_ref_count_t __assert_only count; + count = os_ref_release_locked_raw(&pager->ap_pgr_hdr_ref, NULL); + assert(count == 1); apple_protect_pager_terminate_internal(pager); } } + + +void +crypt_info_reference( + struct pager_crypt_info *crypt_info) +{ + assert(crypt_info->crypt_refcnt != 0); +#if CRYPT_INFO_DEBUG + printf("CRYPT_INFO %s: %p ref %d -> %d\n", + __FUNCTION__, + crypt_info, + crypt_info->crypt_refcnt, + crypt_info->crypt_refcnt + 1); +#endif /* CRYPT_INFO_DEBUG */ + OSAddAtomic(+1, &crypt_info->crypt_refcnt); +} + +void +crypt_info_deallocate( + struct pager_crypt_info *crypt_info) +{ +#if CRYPT_INFO_DEBUG + printf("CRYPT_INFO %s: %p ref %d -> %d\n", + __FUNCTION__, + crypt_info, + crypt_info->crypt_refcnt, + crypt_info->crypt_refcnt - 1); +#endif /* CRYPT_INFO_DEBUG */ + OSAddAtomic(-1, &crypt_info->crypt_refcnt); + if (crypt_info->crypt_refcnt == 0) { + /* deallocate any crypt module data */ + if (crypt_info->crypt_end) { + crypt_info->crypt_end(crypt_info->crypt_ops); + crypt_info->crypt_end = NULL; + } +#if CRYPT_INFO_DEBUG + printf("CRYPT_INFO %s: freeing %p\n", + __FUNCTION__, + crypt_info); +#endif /* CRYPT_INFO_DEBUG */ + kfree(crypt_info, sizeof(*crypt_info)); + crypt_info = NULL; + } +}