X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/5ba3f43ea354af8ad55bea84372a2bc834d8757c..cb3231590a3c94ab4375e2228bd5e86b0cf1ad7e:/osfmk/vm/vm_apple_protect.c diff --git a/osfmk/vm/vm_apple_protect.c b/osfmk/vm/vm_apple_protect.c index 707c3e695..1b7922574 100644 --- a/osfmk/vm/vm_apple_protect.c +++ b/osfmk/vm/vm_apple_protect.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2006 Apple Computer, Inc. All rights reserved. + * Copyright (c) 2019 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * + * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in @@ -11,10 +11,10 @@ * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. - * + * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. - * + * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, @@ -22,7 +22,7 @@ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. - * + * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ @@ -46,6 +46,7 @@ #include #include #include +#include #include #include @@ -59,12 +60,12 @@ #include -/* - * APPLE PROTECT MEMORY PAGER +/* + * APPLE PROTECT MEMORY PAGER * * This external memory manager (EMM) handles memory from the encrypted * sections of some executables protected by the DSMOS kernel extension. - * + * * It mostly handles page-in requests (from memory_object_data_request()) by * getting the encrypted data from its backing VM object, itself backed by * the encrypted file, decrypting it and providing it to VM. @@ -82,35 +83,35 @@ void apple_protect_pager_reference(memory_object_t mem_obj); void apple_protect_pager_deallocate(memory_object_t mem_obj); kern_return_t apple_protect_pager_init(memory_object_t mem_obj, - memory_object_control_t control, - memory_object_cluster_size_t pg_size); + memory_object_control_t control, + memory_object_cluster_size_t pg_size); kern_return_t apple_protect_pager_terminate(memory_object_t mem_obj); kern_return_t apple_protect_pager_data_request(memory_object_t mem_obj, - memory_object_offset_t offset, - memory_object_cluster_size_t length, - vm_prot_t protection_required, - memory_object_fault_info_t fault_info); + memory_object_offset_t offset, + memory_object_cluster_size_t length, + vm_prot_t protection_required, + memory_object_fault_info_t fault_info); kern_return_t apple_protect_pager_data_return(memory_object_t mem_obj, - memory_object_offset_t offset, - memory_object_cluster_size_t data_cnt, - memory_object_offset_t *resid_offset, - int *io_error, - boolean_t dirty, - boolean_t kernel_copy, - int upl_flags); + memory_object_offset_t offset, + memory_object_cluster_size_t data_cnt, + memory_object_offset_t *resid_offset, + int *io_error, + boolean_t dirty, + boolean_t kernel_copy, + int upl_flags); kern_return_t apple_protect_pager_data_initialize(memory_object_t mem_obj, - memory_object_offset_t offset, - memory_object_cluster_size_t data_cnt); + memory_object_offset_t offset, + memory_object_cluster_size_t data_cnt); kern_return_t apple_protect_pager_data_unlock(memory_object_t mem_obj, - memory_object_offset_t offset, - memory_object_size_t size, - vm_prot_t desired_access); + memory_object_offset_t offset, + memory_object_size_t size, + vm_prot_t desired_access); kern_return_t apple_protect_pager_synchronize(memory_object_t mem_obj, - memory_object_offset_t offset, - memory_object_size_t length, - vm_sync_t sync_flags); + memory_object_offset_t offset, + memory_object_size_t length, + vm_sync_t sync_flags); kern_return_t apple_protect_pager_map(memory_object_t mem_obj, - vm_prot_t prot); + vm_prot_t prot); kern_return_t apple_protect_pager_last_unmap(memory_object_t mem_obj); #define CRYPT_INFO_DEBUG 0 @@ -122,19 +123,19 @@ void crypt_info_deallocate(struct pager_crypt_info *crypt_info); * These routines are invoked by VM via the memory_object_*() interfaces. */ const struct memory_object_pager_ops apple_protect_pager_ops = { - apple_protect_pager_reference, - apple_protect_pager_deallocate, - apple_protect_pager_init, - apple_protect_pager_terminate, - apple_protect_pager_data_request, - apple_protect_pager_data_return, - apple_protect_pager_data_initialize, - apple_protect_pager_data_unlock, - apple_protect_pager_synchronize, - apple_protect_pager_map, - apple_protect_pager_last_unmap, - NULL, /* data_reclaim */ - "apple_protect" + .memory_object_reference = apple_protect_pager_reference, + .memory_object_deallocate = apple_protect_pager_deallocate, + .memory_object_init = apple_protect_pager_init, + .memory_object_terminate = apple_protect_pager_terminate, + .memory_object_data_request = apple_protect_pager_data_request, + .memory_object_data_return = apple_protect_pager_data_return, + .memory_object_data_initialize = apple_protect_pager_data_initialize, + .memory_object_data_unlock = apple_protect_pager_data_unlock, + .memory_object_synchronize = apple_protect_pager_synchronize, + .memory_object_map = apple_protect_pager_map, + .memory_object_last_unmap = apple_protect_pager_last_unmap, + .memory_object_data_reclaim = NULL, + .memory_object_pager_name = "apple_protect" }; /* @@ -146,27 +147,27 @@ typedef struct apple_protect_pager { struct memory_object ap_pgr_hdr; /* pager-specific data */ - queue_chain_t pager_queue; /* next & prev pagers */ - unsigned int ref_count; /* reference count */ - boolean_t is_ready; /* is this pager ready ? */ - boolean_t is_mapped; /* is this mem_obj mapped ? */ - vm_object_t backing_object; /* VM obj w/ encrypted data */ - vm_object_offset_t backing_offset; - vm_object_offset_t crypto_backing_offset; /* for key... */ - vm_object_offset_t crypto_start; - vm_object_offset_t crypto_end; + queue_chain_t pager_queue; /* next & prev pagers */ + struct os_refcnt ref_count; /* reference count */ + boolean_t is_ready; /* is this pager ready ? */ + boolean_t is_mapped; /* is this mem_obj mapped ? */ + vm_object_t backing_object; /* VM obj w/ encrypted data */ + vm_object_offset_t backing_offset; + vm_object_offset_t crypto_backing_offset; /* for key... */ + vm_object_offset_t crypto_start; + vm_object_offset_t crypto_end; struct pager_crypt_info *crypt_info; } *apple_protect_pager_t; -#define APPLE_PROTECT_PAGER_NULL ((apple_protect_pager_t) NULL) +#define APPLE_PROTECT_PAGER_NULL ((apple_protect_pager_t) NULL) /* * List of memory objects managed by this EMM. * The list is protected by the "apple_protect_pager_lock" lock. */ -int apple_protect_pager_count = 0; /* number of pagers */ -int apple_protect_pager_count_mapped = 0; /* number of unmapped pagers */ +int apple_protect_pager_count = 0; /* number of pagers */ +int apple_protect_pager_count_mapped = 0; /* number of unmapped pagers */ queue_head_t apple_protect_pager_queue; -decl_lck_mtx_data(,apple_protect_pager_lock) +decl_lck_mtx_data(, apple_protect_pager_lock); /* * Maximum number of unmapped pagers we're willing to keep around. @@ -182,9 +183,9 @@ int apple_protect_pager_num_trim_max = 0; int apple_protect_pager_num_trim_total = 0; -lck_grp_t apple_protect_pager_lck_grp; -lck_grp_attr_t apple_protect_pager_lck_grp_attr; -lck_attr_t apple_protect_pager_lck_attr; +lck_grp_t apple_protect_pager_lck_grp; +lck_grp_attr_t apple_protect_pager_lck_grp_attr; +lck_attr_t apple_protect_pager_lck_attr; /* internal prototypes */ @@ -198,22 +199,22 @@ apple_protect_pager_t apple_protect_pager_create( apple_protect_pager_t apple_protect_pager_lookup(memory_object_t mem_obj); void apple_protect_pager_dequeue(apple_protect_pager_t pager); void apple_protect_pager_deallocate_internal(apple_protect_pager_t pager, - boolean_t locked); + boolean_t locked); void apple_protect_pager_terminate_internal(apple_protect_pager_t pager); void apple_protect_pager_trim(void); #if DEBUG int apple_protect_pagerdebug = 0; -#define PAGER_ALL 0xffffffff -#define PAGER_INIT 0x00000001 -#define PAGER_PAGEIN 0x00000002 - -#define PAGER_DEBUG(LEVEL, A) \ - MACRO_BEGIN \ - if ((apple_protect_pagerdebug & LEVEL)==LEVEL) { \ - printf A; \ - } \ +#define PAGER_ALL 0xffffffff +#define PAGER_INIT 0x00000001 +#define PAGER_PAGEIN 0x00000002 + +#define PAGER_DEBUG(LEVEL, A) \ + MACRO_BEGIN \ + if ((apple_protect_pagerdebug & LEVEL)==LEVEL) { \ + printf A; \ + } \ MACRO_END #else #define PAGER_DEBUG(LEVEL, A) @@ -237,23 +238,24 @@ apple_protect_pager_bootstrap(void) */ kern_return_t apple_protect_pager_init( - memory_object_t mem_obj, - memory_object_control_t control, + memory_object_t mem_obj, + memory_object_control_t control, #if !DEBUG __unused #endif memory_object_cluster_size_t pg_size) { - apple_protect_pager_t pager; - kern_return_t kr; + apple_protect_pager_t pager; + kern_return_t kr; memory_object_attr_info_data_t attributes; PAGER_DEBUG(PAGER_ALL, - ("apple_protect_pager_init: %p, %p, %x\n", - mem_obj, control, pg_size)); + ("apple_protect_pager_init: %p, %p, %x\n", + mem_obj, control, pg_size)); - if (control == MEMORY_OBJECT_CONTROL_NULL) + if (control == MEMORY_OBJECT_CONTROL_NULL) { return KERN_INVALID_ARGUMENT; + } pager = apple_protect_pager_lookup(mem_obj); @@ -268,13 +270,14 @@ apple_protect_pager_init( attributes.temporary = TRUE; kr = memory_object_change_attributes( - control, - MEMORY_OBJECT_ATTRIBUTE_INFO, - (memory_object_info_t) &attributes, - MEMORY_OBJECT_ATTR_INFO_COUNT); - if (kr != KERN_SUCCESS) + control, + MEMORY_OBJECT_ATTRIBUTE_INFO, + (memory_object_info_t) &attributes, + MEMORY_OBJECT_ATTR_INFO_COUNT); + if (kr != KERN_SUCCESS) { panic("apple_protect_pager_init: " - "memory_object_change_attributes() failed"); + "memory_object_change_attributes() failed"); + } #if CONFIG_SECLUDED_MEMORY if (secluded_for_filecache) { @@ -295,14 +298,14 @@ apple_protect_pager_init( */ kern_return_t apple_protect_pager_data_return( - __unused memory_object_t mem_obj, - __unused memory_object_offset_t offset, - __unused memory_object_cluster_size_t data_cnt, - __unused memory_object_offset_t *resid_offset, - __unused int *io_error, - __unused boolean_t dirty, - __unused boolean_t kernel_copy, - __unused int upl_flags) + __unused memory_object_t mem_obj, + __unused memory_object_offset_t offset, + __unused memory_object_cluster_size_t data_cnt, + __unused memory_object_offset_t *resid_offset, + __unused int *io_error, + __unused boolean_t dirty, + __unused boolean_t kernel_copy, + __unused int upl_flags) { panic("apple_protect_pager_data_return: should never get called"); return KERN_FAILURE; @@ -310,9 +313,9 @@ apple_protect_pager_data_return( kern_return_t apple_protect_pager_data_initialize( - __unused memory_object_t mem_obj, - __unused memory_object_offset_t offset, - __unused memory_object_cluster_size_t data_cnt) + __unused memory_object_t mem_obj, + __unused memory_object_offset_t offset, + __unused memory_object_cluster_size_t data_cnt) { panic("apple_protect_pager_data_initialize: should never get called"); return KERN_FAILURE; @@ -320,10 +323,10 @@ apple_protect_pager_data_initialize( kern_return_t apple_protect_pager_data_unlock( - __unused memory_object_t mem_obj, - __unused memory_object_offset_t offset, - __unused memory_object_size_t size, - __unused vm_prot_t desired_access) + __unused memory_object_t mem_obj, + __unused memory_object_offset_t offset, + __unused memory_object_size_t size, + __unused vm_prot_t desired_access) { return KERN_FAILURE; } @@ -334,46 +337,44 @@ apple_protect_pager_data_unlock( * Handles page-in requests from VM. */ int apple_protect_pager_data_request_debug = 0; -kern_return_t +kern_return_t apple_protect_pager_data_request( - memory_object_t mem_obj, - memory_object_offset_t offset, - memory_object_cluster_size_t length, + memory_object_t mem_obj, + memory_object_offset_t offset, + memory_object_cluster_size_t length, #if !DEBUG __unused #endif - vm_prot_t protection_required, + vm_prot_t protection_required, memory_object_fault_info_t mo_fault_info) { - apple_protect_pager_t pager; - memory_object_control_t mo_control; - upl_t upl; - int upl_flags; - upl_size_t upl_size; - upl_page_info_t *upl_pl; - unsigned int pl_count; - vm_object_t src_top_object, src_page_object, dst_object; - kern_return_t kr, retval; - vm_map_offset_t kernel_mapping; - vm_offset_t src_vaddr, dst_vaddr; - vm_offset_t cur_offset; - vm_offset_t offset_in_page; - kern_return_t error_code; - vm_prot_t prot; - vm_page_t src_page, top_page; - int interruptible; - struct vm_object_fault_info fault_info; - int ret; + apple_protect_pager_t pager; + memory_object_control_t mo_control; + upl_t upl; + int upl_flags; + upl_size_t upl_size; + upl_page_info_t *upl_pl; + unsigned int pl_count; + vm_object_t src_top_object, src_page_object, dst_object; + kern_return_t kr, retval; + vm_offset_t src_vaddr, dst_vaddr; + vm_offset_t cur_offset; + vm_offset_t offset_in_page; + kern_return_t error_code; + vm_prot_t prot; + vm_page_t src_page, top_page; + int interruptible; + struct vm_object_fault_info fault_info; + int ret; PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_data_request: %p, %llx, %x, %x\n", mem_obj, offset, length, protection_required)); retval = KERN_SUCCESS; src_top_object = VM_OBJECT_NULL; src_page_object = VM_OBJECT_NULL; - kernel_mapping = 0; upl = NULL; upl_pl = NULL; - fault_info = *((struct vm_object_fault_info *) mo_fault_info); + fault_info = *((struct vm_object_fault_info *)(uintptr_t)mo_fault_info); fault_info.stealth = TRUE; fault_info.io_sync = FALSE; fault_info.mark_zf_absent = FALSE; @@ -382,10 +383,13 @@ apple_protect_pager_data_request( pager = apple_protect_pager_lookup(mem_obj); assert(pager->is_ready); - assert(pager->ref_count > 1); /* pager is alive and mapped */ + assert(os_ref_get_count(&pager->ref_count) > 1); /* pager is alive and mapped */ PAGER_DEBUG(PAGER_PAGEIN, ("apple_protect_pager_data_request: %p, %llx, %x, %x, pager %p\n", mem_obj, offset, length, protection_required, pager)); + fault_info.lo_offset += pager->backing_offset; + fault_info.hi_offset += pager->backing_offset; + /* * Gather in a UPL all the VM pages requested by VM. */ @@ -393,15 +397,15 @@ apple_protect_pager_data_request( upl_size = length; upl_flags = - UPL_RET_ONLY_ABSENT | - UPL_SET_LITE | - UPL_NO_SYNC | - UPL_CLEAN_IN_PLACE | /* triggers UPL_CLEAR_DIRTY */ - UPL_SET_INTERNAL; + UPL_RET_ONLY_ABSENT | + UPL_SET_LITE | + UPL_NO_SYNC | + UPL_CLEAN_IN_PLACE | /* triggers UPL_CLEAR_DIRTY */ + UPL_SET_INTERNAL; pl_count = 0; kr = memory_object_upl_request(mo_control, - offset, upl_size, - &upl, NULL, NULL, upl_flags, VM_KERN_MEMORY_SECURITY); + offset, upl_size, + &upl, NULL, NULL, upl_flags, VM_KERN_MEMORY_SECURITY); if (kr != KERN_SUCCESS) { retval = kr; goto done; @@ -409,41 +413,8 @@ apple_protect_pager_data_request( dst_object = mo_control->moc_object; assert(dst_object != VM_OBJECT_NULL); - -#if __x86_64__ || __arm__ || __arm64__ - /* we'll use the 1-to-1 mapping of physical memory */ - src_vaddr = 0; - dst_vaddr = 0; -#else /* __x86_64__ || __arm__ || __arm64__ */ /* - * Reserve 2 virtual pages in the kernel address space to map each - * source and destination physical pages when it's their turn to - * be processed. - */ - vm_map_entry_t map_entry; - - vm_object_reference(kernel_object); /* ref. for mapping */ - kr = vm_map_find_space(kernel_map, - &kernel_mapping, - 2 * PAGE_SIZE_64, - 0, - 0, - VM_MAP_KERNEL_FLAGS_NONE, - &map_entry); - if (kr != KERN_SUCCESS) { - vm_object_deallocate(kernel_object); - retval = kr; - goto done; - } - map_entry->object.vm_object = kernel_object; - map_entry->offset = kernel_mapping; - vm_map_unlock(kernel_map); - src_vaddr = CAST_DOWN(vm_offset_t, kernel_mapping); - dst_vaddr = CAST_DOWN(vm_offset_t, kernel_mapping + PAGE_SIZE_64); -#endif /* __x86_64__ || __arm__ || __arm64__ */ - - /* - * We'll map the encrypted data in the kernel address space from the + * We'll map the encrypted data in the kernel address space from the * backing VM object (itself backed by the encrypted file via * the vnode pager). */ @@ -457,8 +428,8 @@ apple_protect_pager_data_request( upl_pl = UPL_GET_INTERNAL_PAGE_LIST(upl); pl_count = length / PAGE_SIZE; for (cur_offset = 0; - retval == KERN_SUCCESS && cur_offset < length; - cur_offset += PAGE_SIZE) { + retval == KERN_SUCCESS && cur_offset < length; + cur_offset += PAGE_SIZE) { ppnum_t dst_pnum; if (!upl_page_present(upl_pl, (int)(cur_offset / PAGE_SIZE))) { @@ -471,25 +442,25 @@ apple_protect_pager_data_request( * virtual address space. * We already hold a reference on the src_top_object. */ - retry_src_fault: +retry_src_fault: vm_object_lock(src_top_object); vm_object_paging_begin(src_top_object); error_code = 0; prot = VM_PROT_READ; src_page = VM_PAGE_NULL; kr = vm_fault_page(src_top_object, - pager->backing_offset + offset + cur_offset, - VM_PROT_READ, - FALSE, - FALSE, /* src_page not looked up */ - &prot, - &src_page, - &top_page, - NULL, - &error_code, - FALSE, - FALSE, - &fault_info); + pager->backing_offset + offset + cur_offset, + VM_PROT_READ, + FALSE, + FALSE, /* src_page not looked up */ + &prot, + &src_page, + &top_page, + NULL, + &error_code, + FALSE, + FALSE, + &fault_info); switch (kr) { case VM_FAULT_SUCCESS: break; @@ -499,7 +470,7 @@ apple_protect_pager_data_request( if (vm_page_wait(interruptible)) { goto retry_src_fault; } - /* fall thru */ + /* fall thru */ case VM_FAULT_INTERRUPTED: retval = MACH_SEND_INTERRUPTED; goto done; @@ -507,7 +478,7 @@ apple_protect_pager_data_request( /* success but no VM page: fail */ vm_object_paging_end(src_top_object); vm_object_unlock(src_top_object); - /*FALLTHROUGH*/ + /*FALLTHROUGH*/ case VM_FAULT_MEMORY_ERROR: /* the page is not there ! */ if (error_code) { @@ -518,71 +489,35 @@ apple_protect_pager_data_request( goto done; default: panic("apple_protect_pager_data_request: " - "vm_fault_page() unexpected error 0x%x\n", - kr); + "vm_fault_page() unexpected error 0x%x\n", + kr); } assert(src_page != VM_PAGE_NULL); - assert(src_page->busy); - - if (( !VM_PAGE_NON_SPECULATIVE_PAGEABLE(src_page))) { + assert(src_page->vmp_busy); + if (src_page->vmp_q_state != VM_PAGE_ON_SPECULATIVE_Q) { vm_page_lockspin_queues(); - if (( !VM_PAGE_NON_SPECULATIVE_PAGEABLE(src_page))) { - vm_page_deactivate(src_page); + if (src_page->vmp_q_state != VM_PAGE_ON_SPECULATIVE_Q) { + vm_page_speculate(src_page, FALSE); } vm_page_unlock_queues(); } /* - * Establish an explicit mapping of the source - * physical page. - */ -#if __x86_64__ - src_vaddr = (vm_map_offset_t) - PHYSMAP_PTOV((pmap_paddr_t)VM_PAGE_GET_PHYS_PAGE(src_page) - << PAGE_SHIFT); -#elif __arm__ || __arm64__ - src_vaddr = (vm_map_offset_t) - phystokv((pmap_paddr_t)VM_PAGE_GET_PHYS_PAGE(src_page) - << PAGE_SHIFT); -#else - kr = pmap_enter(kernel_pmap, - src_vaddr, - VM_PAGE_GET_PHYS_PAGE(src_page), - VM_PROT_READ, - VM_PROT_NONE, - 0, - TRUE); - - assert(kr == KERN_SUCCESS); -#endif - /* - * Establish an explicit pmap mapping of the destination - * physical page. - * We can't do a regular VM mapping because the VM page - * is "busy". + * Establish pointers to the source + * and destination physical pages. */ dst_pnum = (ppnum_t) - upl_phys_page(upl_pl, (int)(cur_offset / PAGE_SIZE)); + upl_phys_page(upl_pl, (int)(cur_offset / PAGE_SIZE)); assert(dst_pnum != 0); -#if __x86_64__ - dst_vaddr = (vm_map_offset_t) - PHYSMAP_PTOV((pmap_paddr_t)dst_pnum << PAGE_SHIFT); -#elif __arm__ || __arm64__ + + src_vaddr = (vm_map_offset_t) + phystokv((pmap_paddr_t)VM_PAGE_GET_PHYS_PAGE(src_page) + << PAGE_SHIFT); dst_vaddr = (vm_map_offset_t) - phystokv((pmap_paddr_t)dst_pnum << PAGE_SHIFT); -#else - kr = pmap_enter(kernel_pmap, - dst_vaddr, - dst_pnum, - VM_PROT_READ | VM_PROT_WRITE, - VM_PROT_NONE, - 0, - TRUE); - - assert(kr == KERN_SUCCESS); -#endif + phystokv((pmap_paddr_t)dst_pnum << PAGE_SHIFT); + src_page_object = VM_PAGE_OBJECT(src_page); /* @@ -597,11 +532,11 @@ apple_protect_pager_data_request( * ... and transfer the results to the destination page. */ UPL_SET_CS_VALIDATED(upl_pl, cur_offset / PAGE_SIZE, - src_page->cs_validated); + src_page->vmp_cs_validated); UPL_SET_CS_TAINTED(upl_pl, cur_offset / PAGE_SIZE, - src_page->cs_tainted); + src_page->vmp_cs_tainted); UPL_SET_CS_NX(upl_pl, cur_offset / PAGE_SIZE, - src_page->cs_nx); + src_page->vmp_cs_nx); /* * page_decrypt() might access a mapped file, so let's release @@ -610,7 +545,7 @@ apple_protect_pager_data_request( * "paging_in_progress" reference on its object, so it's safe * to unlock the object here. */ - assert(src_page->busy); + assert(src_page->vmp_busy); assert(src_page_object->paging_in_progress > 0); vm_object_unlock(src_page_object); @@ -619,41 +554,42 @@ apple_protect_pager_data_request( * into the destination page. */ for (offset_in_page = 0; - offset_in_page < PAGE_SIZE; - offset_in_page += 4096) { + offset_in_page < PAGE_SIZE; + offset_in_page += 4096) { if (offset + cur_offset + offset_in_page < pager->crypto_start || offset + cur_offset + offset_in_page >= pager->crypto_end) { /* not encrypted: just copy */ bcopy((const char *)(src_vaddr + - offset_in_page), - (char *)(dst_vaddr + offset_in_page), - 4096); + offset_in_page), + (char *)(dst_vaddr + offset_in_page), + 4096); + if (apple_protect_pager_data_request_debug) { printf("apple_protect_data_request" - "(%p,0x%llx+0x%llx+0x%04llx): " - "out of crypto range " - "[0x%llx:0x%llx]: " - "COPY [0x%016llx 0x%016llx] " - "code_signed=%d " - "cs_validated=%d " - "cs_tainted=%d " - "cs_nx=%d\n", - pager, - offset, - (uint64_t) cur_offset, - (uint64_t) offset_in_page, - pager->crypto_start, - pager->crypto_end, - *(uint64_t *)(dst_vaddr+ - offset_in_page), - *(uint64_t *)(dst_vaddr+ - offset_in_page+8), - src_page_object->code_signed, - src_page->cs_validated, - src_page->cs_tainted, - src_page->cs_nx); + "(%p,0x%llx+0x%llx+0x%04llx): " + "out of crypto range " + "[0x%llx:0x%llx]: " + "COPY [0x%016llx 0x%016llx] " + "code_signed=%d " + "cs_validated=%d " + "cs_tainted=%d " + "cs_nx=%d\n", + pager, + offset, + (uint64_t) cur_offset, + (uint64_t) offset_in_page, + pager->crypto_start, + pager->crypto_end, + *(uint64_t *)(dst_vaddr + + offset_in_page), + *(uint64_t *)(dst_vaddr + + offset_in_page + 8), + src_page_object->code_signed, + src_page->vmp_cs_validated, + src_page->vmp_cs_tainted, + src_page->vmp_cs_nx); } ret = 0; continue; @@ -662,45 +598,46 @@ apple_protect_pager_data_request( (const void *)(src_vaddr + offset_in_page), (void *)(dst_vaddr + offset_in_page), ((pager->crypto_backing_offset - - pager->crypto_start) + /* XXX ? */ - offset + - cur_offset + - offset_in_page), + pager->crypto_start) + /* XXX ? */ + offset + + cur_offset + + offset_in_page), pager->crypt_info->crypt_ops); + if (apple_protect_pager_data_request_debug) { printf("apple_protect_data_request" - "(%p,0x%llx+0x%llx+0x%04llx): " - "in crypto range [0x%llx:0x%llx]: " - "DECRYPT offset 0x%llx=" - "(0x%llx-0x%llx+0x%llx+0x%llx+0x%04llx)" - "[0x%016llx 0x%016llx] " - "code_signed=%d " - "cs_validated=%d " - "cs_tainted=%d " - "cs_nx=%d " - "ret=0x%x\n", - pager, - offset, - (uint64_t) cur_offset, - (uint64_t) offset_in_page, - pager->crypto_start, pager->crypto_end, - ((pager->crypto_backing_offset - - pager->crypto_start) + - offset + - cur_offset + - offset_in_page), - pager->crypto_backing_offset, - pager->crypto_start, - offset, - (uint64_t) cur_offset, - (uint64_t) offset_in_page, - *(uint64_t *)(dst_vaddr+offset_in_page), - *(uint64_t *)(dst_vaddr+offset_in_page+8), - src_page_object->code_signed, - src_page->cs_validated, - src_page->cs_tainted, - src_page->cs_nx, - ret); + "(%p,0x%llx+0x%llx+0x%04llx): " + "in crypto range [0x%llx:0x%llx]: " + "DECRYPT offset 0x%llx=" + "(0x%llx-0x%llx+0x%llx+0x%llx+0x%04llx)" + "[0x%016llx 0x%016llx] " + "code_signed=%d " + "cs_validated=%d " + "cs_tainted=%d " + "cs_nx=%d " + "ret=0x%x\n", + pager, + offset, + (uint64_t) cur_offset, + (uint64_t) offset_in_page, + pager->crypto_start, pager->crypto_end, + ((pager->crypto_backing_offset - + pager->crypto_start) + + offset + + cur_offset + + offset_in_page), + pager->crypto_backing_offset, + pager->crypto_start, + offset, + (uint64_t) cur_offset, + (uint64_t) offset_in_page, + *(uint64_t *)(dst_vaddr + offset_in_page), + *(uint64_t *)(dst_vaddr + offset_in_page + 8), + src_page_object->code_signed, + src_page->vmp_cs_validated, + src_page->vmp_cs_tainted, + src_page->vmp_cs_nx, + ret); } if (ret) { break; @@ -714,53 +651,18 @@ apple_protect_pager_data_request( } assert(VM_PAGE_OBJECT(src_page) == src_page_object); - assert(src_page->busy); + assert(src_page->vmp_busy); assert(src_page_object->paging_in_progress > 0); vm_object_lock(src_page_object); -#if __x86_64__ || __arm__ || __arm64__ - /* we used the 1-to-1 mapping of physical memory */ - src_vaddr = 0; - dst_vaddr = 0; -#else /* __x86_64__ || __arm__ || __arm64__ */ - /* - * Remove the pmap mapping of the source and destination pages - * in the kernel. - */ - pmap_remove(kernel_pmap, - (addr64_t) kernel_mapping, - (addr64_t) (kernel_mapping + (2 * PAGE_SIZE_64))); -#endif /* __x86_64__ || __arm__ || __arm64__ */ - /* * Cleanup the result of vm_fault_page() of the source page. */ - if (retval == KERN_SUCCESS && - src_page->busy && - !VM_PAGE_WIRED(src_page) && - !src_page->dirty && - !src_page->precious && - !src_page->laundry && - !src_page->cleaning) { - int refmod_state; - - refmod_state = pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(src_page)); - - if (refmod_state & VM_MEM_MODIFIED) { - SET_PAGE_DIRTY(src_page, FALSE); - } - if (!src_page->dirty) { - vm_page_free_unlocked(src_page, TRUE); - src_page = VM_PAGE_NULL; - } else { - PAGE_WAKEUP_DONE(src_page); - } - } else { - PAGE_WAKEUP_DONE(src_page); - } + PAGE_WAKEUP_DONE(src_page); src_page = VM_PAGE_NULL; vm_object_paging_end(src_page_object); vm_object_unlock(src_page_object); + if (top_page != VM_PAGE_NULL) { assert(VM_PAGE_OBJECT(top_page) == src_top_object); vm_object_lock(src_top_object); @@ -787,7 +689,7 @@ done: if (retval != KERN_SUCCESS) { upl_abort(upl, 0); if (retval == KERN_ABORTED) { - wait_result_t wait_result; + wait_result_t wait_result; /* * We aborted the fault and did not provide @@ -807,7 +709,7 @@ done: wait_result = assert_wait_timeout( (event_t) apple_protect_pager_data_request, THREAD_UNINT, - 10000, /* 10ms */ + 10000, /* 10ms */ NSEC_PER_USEC); assert(wait_result == THREAD_WAITING); wait_result = thread_block(THREAD_CONTINUE_NULL); @@ -815,30 +717,18 @@ done: } } else { boolean_t empty; - upl_commit_range(upl, 0, upl->size, - UPL_COMMIT_CS_VALIDATED | UPL_COMMIT_WRITTEN_BY_KERNEL, - upl_pl, pl_count, &empty); + upl_commit_range(upl, 0, upl->size, + UPL_COMMIT_CS_VALIDATED | UPL_COMMIT_WRITTEN_BY_KERNEL, + upl_pl, pl_count, &empty); } /* and deallocate the UPL */ upl_deallocate(upl); upl = NULL; } - if (kernel_mapping != 0) { - /* clean up the mapping of the source and destination pages */ - kr = vm_map_remove(kernel_map, - kernel_mapping, - kernel_mapping + (2 * PAGE_SIZE_64), - VM_MAP_NO_FLAGS); - assert(kr == KERN_SUCCESS); - kernel_mapping = 0; - src_vaddr = 0; - dst_vaddr = 0; - } if (src_top_object != VM_OBJECT_NULL) { vm_object_deallocate(src_top_object); } - return retval; } @@ -851,15 +741,14 @@ done: */ void apple_protect_pager_reference( - memory_object_t mem_obj) -{ - apple_protect_pager_t pager; + memory_object_t mem_obj) +{ + apple_protect_pager_t pager; pager = apple_protect_pager_lookup(mem_obj); lck_mtx_lock(&apple_protect_pager_lock); - assert(pager->ref_count > 0); - pager->ref_count++; + os_ref_retain_locked(&pager->ref_count); lck_mtx_unlock(&apple_protect_pager_lock); } @@ -878,12 +767,12 @@ apple_protect_pager_dequeue( assert(!pager->is_mapped); queue_remove(&apple_protect_pager_queue, - pager, - apple_protect_pager_t, - pager_queue); + pager, + apple_protect_pager_t, + pager_queue); pager->pager_queue.next = NULL; pager->pager_queue.prev = NULL; - + apple_protect_pager_count--; } @@ -915,9 +804,9 @@ apple_protect_pager_terminate_internal( /* one less pager using this "pager_crypt_info" */ #if CRYPT_INFO_DEBUG printf("CRYPT_INFO %s: deallocate %p ref %d\n", - __FUNCTION__, - pager->crypt_info, - pager->crypt_info->crypt_refcnt); + __FUNCTION__, + pager->crypt_info, + pager->crypt_info->crypt_refcnt); #endif /* CRYPT_INFO_DEBUG */ crypt_info_deallocate(pager->crypt_info); pager->crypt_info = NULL; @@ -936,18 +825,18 @@ apple_protect_pager_terminate_internal( */ void apple_protect_pager_deallocate_internal( - apple_protect_pager_t pager, - boolean_t locked) + apple_protect_pager_t pager, + boolean_t locked) { - boolean_t needs_trimming; - int count_unmapped; + boolean_t needs_trimming; + int count_unmapped; - if (! locked) { + if (!locked) { lck_mtx_lock(&apple_protect_pager_lock); } - count_unmapped = (apple_protect_pager_count - - apple_protect_pager_count_mapped); + count_unmapped = (apple_protect_pager_count - + apple_protect_pager_count_mapped); if (count_unmapped > apple_protect_pager_cache_limit) { /* we have too many unmapped pagers: trim some */ needs_trimming = TRUE; @@ -956,9 +845,9 @@ apple_protect_pager_deallocate_internal( } /* drop a reference on this pager */ - pager->ref_count--; + os_ref_count_t ref_count = os_ref_release_locked(&pager->ref_count); - if (pager->ref_count == 1) { + if (ref_count == 1) { /* * Only the "named" reference is left, which means that * no one is really holding on to this pager anymore. @@ -968,7 +857,7 @@ apple_protect_pager_deallocate_internal( /* the pager is all ours: no need for the lock now */ lck_mtx_unlock(&apple_protect_pager_lock); apple_protect_pager_terminate_internal(pager); - } else if (pager->ref_count == 0) { + } else if (ref_count == 0) { /* * Dropped the existence reference; the memory object has * been terminated. Do some final cleanup and release the @@ -979,7 +868,7 @@ apple_protect_pager_deallocate_internal( memory_object_control_deallocate(pager->ap_pgr_hdr.mo_control); pager->ap_pgr_hdr.mo_control = MEMORY_OBJECT_CONTROL_NULL; } - kfree(pager, sizeof (*pager)); + kfree(pager, sizeof(*pager)); pager = APPLE_PROTECT_PAGER_NULL; } else { /* there are still plenty of references: keep going... */ @@ -1000,9 +889,9 @@ apple_protect_pager_deallocate_internal( */ void apple_protect_pager_deallocate( - memory_object_t mem_obj) + memory_object_t mem_obj) { - apple_protect_pager_t pager; + apple_protect_pager_t pager; PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_deallocate: %p\n", mem_obj)); pager = apple_protect_pager_lookup(mem_obj); @@ -1017,7 +906,7 @@ apple_protect_pager_terminate( #if !DEBUG __unused #endif - memory_object_t mem_obj) + memory_object_t mem_obj) { PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_terminate: %p\n", mem_obj)); @@ -1029,10 +918,10 @@ apple_protect_pager_terminate( */ kern_return_t apple_protect_pager_synchronize( - __unused memory_object_t mem_obj, - __unused memory_object_offset_t offset, - __unused memory_object_size_t length, - __unused vm_sync_t sync_flags) + __unused memory_object_t mem_obj, + __unused memory_object_offset_t offset, + __unused memory_object_size_t length, + __unused vm_sync_t sync_flags) { panic("apple_protect_pager_synchronize: memory_object_synchronize no longer supported\n"); return KERN_FAILURE; @@ -1048,10 +937,10 @@ apple_protect_pager_synchronize( */ kern_return_t apple_protect_pager_map( - memory_object_t mem_obj, - __unused vm_prot_t prot) + memory_object_t mem_obj, + __unused vm_prot_t prot) { - apple_protect_pager_t pager; + apple_protect_pager_t pager; PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_map: %p\n", mem_obj)); @@ -1059,7 +948,7 @@ apple_protect_pager_map( lck_mtx_lock(&apple_protect_pager_lock); assert(pager->is_ready); - assert(pager->ref_count > 0); /* pager is alive */ + assert(os_ref_get_count(&pager->ref_count) > 0); /* pager is alive */ if (pager->is_mapped == FALSE) { /* * First mapping of this pager: take an extra reference @@ -1067,7 +956,7 @@ apple_protect_pager_map( * are removed. */ pager->is_mapped = TRUE; - pager->ref_count++; + os_ref_retain_locked(&pager->ref_count); apple_protect_pager_count_mapped++; } lck_mtx_unlock(&apple_protect_pager_lock); @@ -1082,13 +971,13 @@ apple_protect_pager_map( */ kern_return_t apple_protect_pager_last_unmap( - memory_object_t mem_obj) + memory_object_t mem_obj) { - apple_protect_pager_t pager; - int count_unmapped; + apple_protect_pager_t pager; + int count_unmapped; PAGER_DEBUG(PAGER_ALL, - ("apple_protect_pager_last_unmap: %p\n", mem_obj)); + ("apple_protect_pager_last_unmap: %p\n", mem_obj)); pager = apple_protect_pager_lookup(mem_obj); @@ -1100,7 +989,7 @@ apple_protect_pager_last_unmap( */ apple_protect_pager_count_mapped--; count_unmapped = (apple_protect_pager_count - - apple_protect_pager_count_mapped); + apple_protect_pager_count_mapped); if (count_unmapped > apple_protect_pager_count_unmapped_max) { apple_protect_pager_count_unmapped_max = count_unmapped; } @@ -1110,7 +999,7 @@ apple_protect_pager_last_unmap( } else { lck_mtx_unlock(&apple_protect_pager_lock); } - + return KERN_SUCCESS; } @@ -1120,31 +1009,31 @@ apple_protect_pager_last_unmap( */ apple_protect_pager_t apple_protect_pager_lookup( - memory_object_t mem_obj) + memory_object_t mem_obj) { - apple_protect_pager_t pager; + apple_protect_pager_t pager; assert(mem_obj->mo_pager_ops == &apple_protect_pager_ops); - pager = (apple_protect_pager_t) mem_obj; - assert(pager->ref_count > 0); + pager = (apple_protect_pager_t)(uintptr_t) mem_obj; + assert(os_ref_get_count(&pager->ref_count) > 0); return pager; } apple_protect_pager_t apple_protect_pager_create( - vm_object_t backing_object, - vm_object_offset_t backing_offset, - vm_object_offset_t crypto_backing_offset, + vm_object_t backing_object, + vm_object_offset_t backing_offset, + vm_object_offset_t crypto_backing_offset, struct pager_crypt_info *crypt_info, - vm_object_offset_t crypto_start, - vm_object_offset_t crypto_end) + vm_object_offset_t crypto_start, + vm_object_offset_t crypto_end) { - apple_protect_pager_t pager, pager2; - memory_object_control_t control; - kern_return_t kr; - struct pager_crypt_info *old_crypt_info; + apple_protect_pager_t pager, pager2; + memory_object_control_t control; + kern_return_t kr; + struct pager_crypt_info *old_crypt_info; - pager = (apple_protect_pager_t) kalloc(sizeof (*pager)); + pager = (apple_protect_pager_t) kalloc(sizeof(*pager)); if (pager == APPLE_PROTECT_PAGER_NULL) { return APPLE_PROTECT_PAGER_NULL; } @@ -1161,8 +1050,7 @@ apple_protect_pager_create( pager->ap_pgr_hdr.mo_control = MEMORY_OBJECT_CONTROL_NULL; pager->is_ready = FALSE;/* not ready until it has a "name" */ - pager->ref_count = 1; /* existence reference (for the cache) */ - pager->ref_count++; /* for the caller */ + os_ref_init_count(&pager->ref_count, NULL, 2); /* existence reference (for the cache) and another for the caller */ pager->is_mapped = FALSE; pager->backing_object = backing_object; pager->backing_offset = backing_offset; @@ -1173,14 +1061,14 @@ apple_protect_pager_create( #if CRYPT_INFO_DEBUG printf("CRYPT_INFO %s: crypt_info %p [%p,%p,%p,%d]\n", - __FUNCTION__, - crypt_info, - crypt_info->page_decrypt, - crypt_info->crypt_end, - crypt_info->crypt_ops, - crypt_info->crypt_refcnt); + __FUNCTION__, + crypt_info, + crypt_info->page_decrypt, + crypt_info->crypt_end, + crypt_info->crypt_ops, + crypt_info->crypt_refcnt); #endif /* CRYPT_INFO_DEBUG */ - + vm_object_reference(backing_object); old_crypt_info = NULL; @@ -1188,15 +1076,15 @@ apple_protect_pager_create( lck_mtx_lock(&apple_protect_pager_lock); /* see if anyone raced us to create a pager for the same object */ queue_iterate(&apple_protect_pager_queue, - pager2, - apple_protect_pager_t, - pager_queue) { + pager2, + apple_protect_pager_t, + pager_queue) { if ((pager2->crypt_info->page_decrypt != - crypt_info->page_decrypt) || + crypt_info->page_decrypt) || (pager2->crypt_info->crypt_end != - crypt_info->crypt_end) || + crypt_info->crypt_end) || (pager2->crypt_info->crypt_ops != - crypt_info->crypt_ops)) { + crypt_info->crypt_ops)) { /* crypt_info contents do not match: next pager */ continue; } @@ -1209,16 +1097,16 @@ apple_protect_pager_create( /* ... switch to that pager's crypt_info */ #if CRYPT_INFO_DEBUG printf("CRYPT_INFO %s: reference %p ref %d " - "(create match)\n", - __FUNCTION__, - pager2->crypt_info, - pager2->crypt_info->crypt_refcnt); + "(create match)\n", + __FUNCTION__, + pager2->crypt_info, + pager2->crypt_info->crypt_refcnt); #endif /* CRYPT_INFO_DEBUG */ old_crypt_info = pager2->crypt_info; crypt_info_reference(old_crypt_info); pager->crypt_info = old_crypt_info; } - + if (pager2->backing_object == backing_object && pager2->backing_offset == backing_offset && pager2->crypto_backing_offset == crypto_backing_offset && @@ -1228,21 +1116,21 @@ apple_protect_pager_create( break; } } - if (! queue_end(&apple_protect_pager_queue, - (queue_entry_t) pager2)) { + if (!queue_end(&apple_protect_pager_queue, + (queue_entry_t) pager2)) { /* we lost the race, down with the loser... */ lck_mtx_unlock(&apple_protect_pager_lock); vm_object_deallocate(pager->backing_object); pager->backing_object = VM_OBJECT_NULL; #if CRYPT_INFO_DEBUG printf("CRYPT_INFO %s: %p ref %d (create pager match)\n", - __FUNCTION__, - pager->crypt_info, - pager->crypt_info->crypt_refcnt); + __FUNCTION__, + pager->crypt_info, + pager->crypt_info->crypt_refcnt); #endif /* CRYPT_INFO_DEBUG */ crypt_info_deallocate(pager->crypt_info); pager->crypt_info = NULL; - kfree(pager, sizeof (*pager)); + kfree(pager, sizeof(*pager)); /* ... and go with the winner */ pager = pager2; /* let the winner make sure the pager gets ready */ @@ -1251,9 +1139,9 @@ apple_protect_pager_create( /* enter new pager at the head of our list of pagers */ queue_enter_first(&apple_protect_pager_queue, - pager, - apple_protect_pager_t, - pager_queue); + pager, + apple_protect_pager_t, + pager_queue); apple_protect_pager_count++; if (apple_protect_pager_count > apple_protect_pager_count_max) { apple_protect_pager_count_max = apple_protect_pager_count; @@ -1261,10 +1149,12 @@ apple_protect_pager_create( lck_mtx_unlock(&apple_protect_pager_lock); kr = memory_object_create_named((memory_object_t) pager, - 0, - &control); + 0, + &control); assert(kr == KERN_SUCCESS); + memory_object_mark_trusted(control); + lck_mtx_lock(&apple_protect_pager_lock); /* the new pager is now ready to be used */ pager->is_ready = TRUE; @@ -1278,10 +1168,10 @@ apple_protect_pager_create( /* we re-used an old crypt_info instead of using our new one */ #if CRYPT_INFO_DEBUG printf("CRYPT_INFO %s: deallocate %p ref %d " - "(create used old)\n", - __FUNCTION__, - crypt_info, - crypt_info->crypt_refcnt); + "(create used old)\n", + __FUNCTION__, + crypt_info, + crypt_info->crypt_refcnt); #endif /* CRYPT_INFO_DEBUG */ crypt_info_deallocate(crypt_info); crypt_info = NULL; @@ -1299,24 +1189,24 @@ apple_protect_pager_create( */ memory_object_t apple_protect_pager_setup( - vm_object_t backing_object, - vm_object_offset_t backing_offset, - vm_object_offset_t crypto_backing_offset, + vm_object_t backing_object, + vm_object_offset_t backing_offset, + vm_object_offset_t crypto_backing_offset, struct pager_crypt_info *crypt_info, - vm_object_offset_t crypto_start, - vm_object_offset_t crypto_end) + vm_object_offset_t crypto_start, + vm_object_offset_t crypto_end) { - apple_protect_pager_t pager; - struct pager_crypt_info *old_crypt_info, *new_crypt_info; + apple_protect_pager_t pager; + struct pager_crypt_info *old_crypt_info, *new_crypt_info; #if CRYPT_INFO_DEBUG printf("CRYPT_INFO %s: crypt_info=%p [%p,%p,%p,%d]\n", - __FUNCTION__, - crypt_info, - crypt_info->page_decrypt, - crypt_info->crypt_end, - crypt_info->crypt_ops, - crypt_info->crypt_refcnt); + __FUNCTION__, + crypt_info, + crypt_info->page_decrypt, + crypt_info->crypt_end, + crypt_info->crypt_ops, + crypt_info->crypt_refcnt); #endif /* CRYPT_INFO_DEBUG */ old_crypt_info = NULL; @@ -1324,15 +1214,15 @@ apple_protect_pager_setup( lck_mtx_lock(&apple_protect_pager_lock); queue_iterate(&apple_protect_pager_queue, - pager, - apple_protect_pager_t, - pager_queue) { + pager, + apple_protect_pager_t, + pager_queue) { if ((pager->crypt_info->page_decrypt != - crypt_info->page_decrypt) || + crypt_info->page_decrypt) || (pager->crypt_info->crypt_end != - crypt_info->crypt_end) || + crypt_info->crypt_end) || (pager->crypt_info->crypt_ops != - crypt_info->crypt_ops)) { + crypt_info->crypt_ops)) { /* no match for "crypt_info": next pager */ continue; } @@ -1345,28 +1235,28 @@ apple_protect_pager_setup( old_crypt_info = pager->crypt_info; #if CRYPT_INFO_DEBUG printf("CRYPT_INFO %s: " - "switching crypt_info from %p [%p,%p,%p,%d] " - "to %p [%p,%p,%p,%d] from pager %p\n", - __FUNCTION__, - crypt_info, - crypt_info->page_decrypt, - crypt_info->crypt_end, - crypt_info->crypt_ops, - crypt_info->crypt_refcnt, - old_crypt_info, - old_crypt_info->page_decrypt, - old_crypt_info->crypt_end, - old_crypt_info->crypt_ops, - old_crypt_info->crypt_refcnt, - pager); + "switching crypt_info from %p [%p,%p,%p,%d] " + "to %p [%p,%p,%p,%d] from pager %p\n", + __FUNCTION__, + crypt_info, + crypt_info->page_decrypt, + crypt_info->crypt_end, + crypt_info->crypt_ops, + crypt_info->crypt_refcnt, + old_crypt_info, + old_crypt_info->page_decrypt, + old_crypt_info->crypt_end, + old_crypt_info->crypt_ops, + old_crypt_info->crypt_refcnt, + pager); printf("CRYPT_INFO %s: %p ref %d (setup match)\n", - __FUNCTION__, - pager->crypt_info, - pager->crypt_info->crypt_refcnt); + __FUNCTION__, + pager->crypt_info, + pager->crypt_info->crypt_refcnt); #endif /* CRYPT_INFO_DEBUG */ crypt_info_reference(pager->crypt_info); } - + if (pager->backing_object == backing_object && pager->backing_offset == backing_offset && pager->crypto_backing_offset == crypto_backing_offset && @@ -1377,27 +1267,26 @@ apple_protect_pager_setup( assert(old_crypt_info->crypt_refcnt > 1); #if CRYPT_INFO_DEBUG printf("CRYPT_INFO %s: " - "pager match with %p crypt_info %p\n", - __FUNCTION__, - pager, - pager->crypt_info); + "pager match with %p crypt_info %p\n", + __FUNCTION__, + pager, + pager->crypt_info); printf("CRYPT_INFO %s: deallocate %p ref %d " - "(pager match)\n", - __FUNCTION__, - old_crypt_info, - old_crypt_info->crypt_refcnt); + "(pager match)\n", + __FUNCTION__, + old_crypt_info, + old_crypt_info->crypt_refcnt); #endif /* CRYPT_INFO_DEBUG */ /* release the extra ref on crypt_info we got above */ crypt_info_deallocate(old_crypt_info); assert(old_crypt_info->crypt_refcnt > 0); /* give extra reference on pager to the caller */ - assert(pager->ref_count > 0); - pager->ref_count++; + os_ref_retain_locked(&pager->ref_count); break; } } if (queue_end(&apple_protect_pager_queue, - (queue_entry_t) pager)) { + (queue_entry_t) pager)) { lck_mtx_unlock(&apple_protect_pager_lock); /* no existing pager for this backing object */ pager = APPLE_PROTECT_PAGER_NULL; @@ -1406,20 +1295,20 @@ apple_protect_pager_setup( new_crypt_info = old_crypt_info; #if CRYPT_INFO_DEBUG printf("CRYPT_INFO %s: " - "will use old_crypt_info %p for new pager\n", - __FUNCTION__, - old_crypt_info); + "will use old_crypt_info %p for new pager\n", + __FUNCTION__, + old_crypt_info); #endif /* CRYPT_INFO_DEBUG */ } else { /* allocate a new crypt_info for new pager */ - new_crypt_info = kalloc(sizeof (*new_crypt_info)); + new_crypt_info = kalloc(sizeof(*new_crypt_info)); *new_crypt_info = *crypt_info; new_crypt_info->crypt_refcnt = 1; #if CRYPT_INFO_DEBUG printf("CRYPT_INFO %s: " - "will use new_crypt_info %p for new pager\n", - __FUNCTION__, - new_crypt_info); + "will use new_crypt_info %p for new pager\n", + __FUNCTION__, + new_crypt_info); #endif /* CRYPT_INFO_DEBUG */ } if (new_crypt_info == NULL) { @@ -1440,10 +1329,10 @@ apple_protect_pager_setup( /* release extra reference on old_crypt_info */ #if CRYPT_INFO_DEBUG printf("CRYPT_INFO %s: deallocate %p ref %d " - "(create fail old_crypt_info)\n", - __FUNCTION__, - old_crypt_info, - old_crypt_info->crypt_refcnt); + "(create fail old_crypt_info)\n", + __FUNCTION__, + old_crypt_info, + old_crypt_info->crypt_refcnt); #endif /* CRYPT_INFO_DEBUG */ crypt_info_deallocate(old_crypt_info); old_crypt_info = NULL; @@ -1452,10 +1341,10 @@ apple_protect_pager_setup( assert(new_crypt_info->crypt_refcnt == 1); #if CRYPT_INFO_DEBUG printf("CRYPT_INFO %s: deallocate %p ref %d " - "(create fail new_crypt_info)\n", - __FUNCTION__, - new_crypt_info, - new_crypt_info->crypt_refcnt); + "(create fail new_crypt_info)\n", + __FUNCTION__, + new_crypt_info, + new_crypt_info->crypt_refcnt); #endif /* CRYPT_INFO_DEBUG */ crypt_info_deallocate(new_crypt_info); new_crypt_info = NULL; @@ -1469,22 +1358,22 @@ apple_protect_pager_setup( while (!pager->is_ready) { lck_mtx_sleep(&apple_protect_pager_lock, - LCK_SLEEP_DEFAULT, - &pager->is_ready, - THREAD_UNINT); + LCK_SLEEP_DEFAULT, + &pager->is_ready, + THREAD_UNINT); } lck_mtx_unlock(&apple_protect_pager_lock); return (memory_object_t) pager; -} +} void apple_protect_pager_trim(void) { - apple_protect_pager_t pager, prev_pager; - queue_head_t trim_queue; - int num_trim; - int count_unmapped; + apple_protect_pager_t pager, prev_pager; + queue_head_t trim_queue; + int num_trim; + int count_unmapped; lck_mtx_lock(&apple_protect_pager_lock); @@ -1496,15 +1385,15 @@ apple_protect_pager_trim(void) num_trim = 0; for (pager = (apple_protect_pager_t) - queue_last(&apple_protect_pager_queue); - !queue_end(&apple_protect_pager_queue, - (queue_entry_t) pager); - pager = prev_pager) { + queue_last(&apple_protect_pager_queue); + !queue_end(&apple_protect_pager_queue, + (queue_entry_t) pager); + pager = prev_pager) { /* get prev elt before we dequeue */ prev_pager = (apple_protect_pager_t) - queue_prev(&pager->pager_queue); + queue_prev(&pager->pager_queue); - if (pager->ref_count == 2 && + if (os_ref_get_count(&pager->ref_count) == 2 && pager->is_ready && !pager->is_mapped) { /* this pager can be trimmed */ @@ -1513,12 +1402,12 @@ apple_protect_pager_trim(void) apple_protect_pager_dequeue(pager); /* ... and add it to our trim queue */ queue_enter_first(&trim_queue, - pager, - apple_protect_pager_t, - pager_queue); + pager, + apple_protect_pager_t, + pager_queue); count_unmapped = (apple_protect_pager_count - - apple_protect_pager_count_mapped); + apple_protect_pager_count_mapped); if (count_unmapped <= apple_protect_pager_cache_limit) { /* we have enough pagers to trim */ break; @@ -1535,18 +1424,18 @@ apple_protect_pager_trim(void) /* terminate the trimmed pagers */ while (!queue_empty(&trim_queue)) { queue_remove_first(&trim_queue, - pager, - apple_protect_pager_t, - pager_queue); + pager, + apple_protect_pager_t, + pager_queue); pager->pager_queue.next = NULL; pager->pager_queue.prev = NULL; - assert(pager->ref_count == 2); /* * We can't call deallocate_internal() because the pager * has already been dequeued, but we still need to remove * a reference. */ - pager->ref_count--; + os_ref_count_t __assert_only count = os_ref_release_locked(&pager->ref_count); + assert(count == 1); apple_protect_pager_terminate_internal(pager); } } @@ -1559,24 +1448,24 @@ crypt_info_reference( assert(crypt_info->crypt_refcnt != 0); #if CRYPT_INFO_DEBUG printf("CRYPT_INFO %s: %p ref %d -> %d\n", - __FUNCTION__, - crypt_info, - crypt_info->crypt_refcnt, - crypt_info->crypt_refcnt + 1); + __FUNCTION__, + crypt_info, + crypt_info->crypt_refcnt, + crypt_info->crypt_refcnt + 1); #endif /* CRYPT_INFO_DEBUG */ OSAddAtomic(+1, &crypt_info->crypt_refcnt); } void crypt_info_deallocate( - struct pager_crypt_info *crypt_info) + struct pager_crypt_info *crypt_info) { #if CRYPT_INFO_DEBUG printf("CRYPT_INFO %s: %p ref %d -> %d\n", - __FUNCTION__, - crypt_info, - crypt_info->crypt_refcnt, - crypt_info->crypt_refcnt - 1); + __FUNCTION__, + crypt_info, + crypt_info->crypt_refcnt, + crypt_info->crypt_refcnt - 1); #endif /* CRYPT_INFO_DEBUG */ OSAddAtomic(-1, &crypt_info->crypt_refcnt); if (crypt_info->crypt_refcnt == 0) { @@ -1587,10 +1476,10 @@ crypt_info_deallocate( } #if CRYPT_INFO_DEBUG printf("CRYPT_INFO %s: freeing %p\n", - __FUNCTION__, - crypt_info); + __FUNCTION__, + crypt_info); #endif /* CRYPT_INFO_DEBUG */ - kfree(crypt_info, sizeof (*crypt_info)); + kfree(crypt_info, sizeof(*crypt_info)); crypt_info = NULL; } }