X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/d9a64523371fa019c4575bb400cbbc3a50ac9903..HEAD:/osfmk/vm/vm_apple_protect.c

diff --git a/osfmk/vm/vm_apple_protect.c b/osfmk/vm/vm_apple_protect.c
index 508211e68..f7fcaceb3 100644
--- a/osfmk/vm/vm_apple_protect.c
+++ b/osfmk/vm/vm_apple_protect.c
@@ -1,8 +1,8 @@
 /*
- * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2006-2020 Apple Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
- * 
+ *
  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in
@@ -11,10 +11,10 @@
  * unlawful or unlicensed copies of an Apple operating system, or to
  * circumvent, violate, or enable the circumvention or violation of, any
  * terms of an Apple operating system software license agreement.
- * 
+ *
  * Please obtain a copy of the License at
  * http://www.opensource.apple.com/apsl/ and read it before using this file.
- * 
+ *
  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
@@ -22,7 +22,7 @@
  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  * Please see the License for the specific language governing rights and
  * limitations under the License.
- * 
+ *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
@@ -46,6 +46,7 @@
 #include <kern/queue.h>
 #include <kern/thread.h>
 #include <kern/ipc_kobject.h>
+#include <os/refcnt.h>
 
 #include <ipc/ipc_port.h>
 #include <ipc/ipc_space.h>
@@ -58,13 +59,12 @@
 #include <vm/vm_protos.h>
 #include <vm/vm_kern.h>
 
-
-/* 
- * APPLE PROTECT MEMORY PAGER 
+/*
+ * APPLE PROTECT MEMORY PAGER
  *
  * This external memory manager (EMM) handles memory from the encrypted
  * sections of some executables protected by the DSMOS kernel extension.
- * 
+ *
  * It mostly handles page-in requests (from memory_object_data_request()) by
  * getting the encrypted data from its backing VM object, itself backed by
  * the encrypted file, decrypting it and providing it to VM.
@@ -82,36 +82,41 @@
 void apple_protect_pager_reference(memory_object_t mem_obj);
 void apple_protect_pager_deallocate(memory_object_t mem_obj);
 kern_return_t apple_protect_pager_init(memory_object_t mem_obj,
-				       memory_object_control_t control,
-				       memory_object_cluster_size_t pg_size);
+    memory_object_control_t control,
+    memory_object_cluster_size_t pg_size);
 kern_return_t apple_protect_pager_terminate(memory_object_t mem_obj);
 kern_return_t apple_protect_pager_data_request(memory_object_t mem_obj,
-					       memory_object_offset_t offset,
-					       memory_object_cluster_size_t length,
-					       vm_prot_t protection_required,
-					       memory_object_fault_info_t fault_info);
+    memory_object_offset_t offset,
+    memory_object_cluster_size_t length,
+    vm_prot_t protection_required,
+    memory_object_fault_info_t fault_info);
 kern_return_t apple_protect_pager_data_return(memory_object_t mem_obj,
-					      memory_object_offset_t offset,
-					      memory_object_cluster_size_t	data_cnt,
-					      memory_object_offset_t *resid_offset,
-					      int *io_error,
-					      boolean_t dirty,
-					      boolean_t kernel_copy,
-					      int upl_flags);
+    memory_object_offset_t offset,
+    memory_object_cluster_size_t      data_cnt,
+    memory_object_offset_t *resid_offset,
+    int *io_error,
+    boolean_t dirty,
+    boolean_t kernel_copy,
+    int upl_flags);
 kern_return_t apple_protect_pager_data_initialize(memory_object_t mem_obj,
-						  memory_object_offset_t offset,
-						  memory_object_cluster_size_t data_cnt);
+    memory_object_offset_t offset,
+    memory_object_cluster_size_t data_cnt);
 kern_return_t apple_protect_pager_data_unlock(memory_object_t mem_obj,
-					      memory_object_offset_t offset,
-					      memory_object_size_t size,
-					      vm_prot_t desired_access);
+    memory_object_offset_t offset,
+    memory_object_size_t size,
+    vm_prot_t desired_access);
 kern_return_t apple_protect_pager_synchronize(memory_object_t mem_obj,
-					      memory_object_offset_t offset,
-					      memory_object_size_t length,
-					      vm_sync_t sync_flags);
+    memory_object_offset_t offset,
+    memory_object_size_t length,
+    vm_sync_t sync_flags);
 kern_return_t apple_protect_pager_map(memory_object_t mem_obj,
-				      vm_prot_t prot);
+    vm_prot_t prot);
 kern_return_t apple_protect_pager_last_unmap(memory_object_t mem_obj);
+boolean_t apple_protect_pager_backing_object(
+	memory_object_t mem_obj,
+	memory_object_offset_t mem_obj_offset,
+	vm_object_t *backing_object,
+	vm_object_offset_t *backing_offset);
 
 #define CRYPT_INFO_DEBUG 0
 void crypt_info_reference(struct pager_crypt_info *crypt_info);
@@ -122,19 +127,20 @@ void crypt_info_deallocate(struct pager_crypt_info *crypt_info);
  * These routines are invoked by VM via the memory_object_*() interfaces.
  */
 const struct memory_object_pager_ops apple_protect_pager_ops = {
-	apple_protect_pager_reference,
-	apple_protect_pager_deallocate,
-	apple_protect_pager_init,
-	apple_protect_pager_terminate,
-	apple_protect_pager_data_request,
-	apple_protect_pager_data_return,
-	apple_protect_pager_data_initialize,
-	apple_protect_pager_data_unlock,
-	apple_protect_pager_synchronize,
-	apple_protect_pager_map,
-	apple_protect_pager_last_unmap,
-	NULL, /* data_reclaim */
-	"apple_protect"
+	.memory_object_reference = apple_protect_pager_reference,
+	.memory_object_deallocate = apple_protect_pager_deallocate,
+	.memory_object_init = apple_protect_pager_init,
+	.memory_object_terminate = apple_protect_pager_terminate,
+	.memory_object_data_request = apple_protect_pager_data_request,
+	.memory_object_data_return = apple_protect_pager_data_return,
+	.memory_object_data_initialize = apple_protect_pager_data_initialize,
+	.memory_object_data_unlock = apple_protect_pager_data_unlock,
+	.memory_object_synchronize = apple_protect_pager_synchronize,
+	.memory_object_map = apple_protect_pager_map,
+	.memory_object_last_unmap = apple_protect_pager_last_unmap,
+	.memory_object_data_reclaim = NULL,
+	.memory_object_backing_object = apple_protect_pager_backing_object,
+	.memory_object_pager_name = "apple_protect"
 };
 
 /*
@@ -143,49 +149,51 @@ const struct memory_object_pager_ops apple_protect_pager_ops = {
  */
 typedef struct apple_protect_pager {
 	/* mandatory generic header */
-	struct memory_object ap_pgr_hdr;
+	struct memory_object    ap_pgr_hdr;
 
 	/* pager-specific data */
-	queue_chain_t		pager_queue;	/* next & prev pagers */
-	unsigned int		ref_count;	/* reference count */
-	boolean_t		is_ready;	/* is this pager ready ? */
-	boolean_t		is_mapped;	/* is this mem_obj mapped ? */
-	vm_object_t		backing_object; /* VM obj w/ encrypted data */
-	vm_object_offset_t	backing_offset;
-	vm_object_offset_t	crypto_backing_offset; /* for key... */
-	vm_object_offset_t	crypto_start;
-	vm_object_offset_t	crypto_end;
+	queue_chain_t           pager_queue;    /* next & prev pagers */
+#if MEMORY_OBJECT_HAS_REFCOUNT
+#define ap_pgr_hdr_ref          ap_pgr_hdr.mo_ref
+#else
+	os_ref_atomic_t         ap_pgr_hdr_ref;      /* reference count */
+#endif
+	bool                    is_ready;       /* is this pager ready ? */
+	bool                    is_mapped;      /* is this mem_obj mapped ? */
+	bool                    is_cached;      /* is this pager cached ? */
+	vm_object_t             backing_object; /* VM obj w/ encrypted data */
+	vm_object_offset_t      backing_offset;
+	vm_object_offset_t      crypto_backing_offset; /* for key... */
+	vm_object_offset_t      crypto_start;
+	vm_object_offset_t      crypto_end;
 	struct pager_crypt_info *crypt_info;
 } *apple_protect_pager_t;
-#define	APPLE_PROTECT_PAGER_NULL	((apple_protect_pager_t) NULL)
+#define APPLE_PROTECT_PAGER_NULL        ((apple_protect_pager_t) NULL)
 
 /*
  * List of memory objects managed by this EMM.
  * The list is protected by the "apple_protect_pager_lock" lock.
  */
-int apple_protect_pager_count = 0;		/* number of pagers */
-int apple_protect_pager_count_mapped = 0;	/* number of unmapped pagers */
-queue_head_t apple_protect_pager_queue;
-decl_lck_mtx_data(,apple_protect_pager_lock)
+unsigned int apple_protect_pager_count = 0;        /* number of pagers */
+unsigned int apple_protect_pager_count_mapped = 0; /* number of unmapped pagers */
+queue_head_t apple_protect_pager_queue = QUEUE_HEAD_INITIALIZER(apple_protect_pager_queue);
+LCK_GRP_DECLARE(apple_protect_pager_lck_grp, "apple_protect");
+LCK_MTX_DECLARE(apple_protect_pager_lock, &apple_protect_pager_lck_grp);
 
 /*
  * Maximum number of unmapped pagers we're willing to keep around.
  */
-int apple_protect_pager_cache_limit = 20;
+unsigned int apple_protect_pager_cache_limit = 20;
 
 /*
  * Statistics & counters.
  */
-int apple_protect_pager_count_max = 0;
-int apple_protect_pager_count_unmapped_max = 0;
-int apple_protect_pager_num_trim_max = 0;
-int apple_protect_pager_num_trim_total = 0;
+unsigned int apple_protect_pager_count_max = 0;
+unsigned int apple_protect_pager_count_unmapped_max = 0;
+unsigned int apple_protect_pager_num_trim_max = 0;
+unsigned int apple_protect_pager_num_trim_total = 0;
 
 
-lck_grp_t		apple_protect_pager_lck_grp;
-lck_grp_attr_t	apple_protect_pager_lck_grp_attr;
-lck_attr_t		apple_protect_pager_lck_attr;
-
 
 /* internal prototypes */
 apple_protect_pager_t apple_protect_pager_create(
@@ -194,42 +202,32 @@ apple_protect_pager_t apple_protect_pager_create(
 	vm_object_offset_t crypto_backing_offset,
 	struct pager_crypt_info *crypt_info,
 	vm_object_offset_t crypto_start,
-	vm_object_offset_t crypto_end);
+	vm_object_offset_t crypto_end,
+	boolean_t cache_pager);
 apple_protect_pager_t apple_protect_pager_lookup(memory_object_t mem_obj);
 void apple_protect_pager_dequeue(apple_protect_pager_t pager);
 void apple_protect_pager_deallocate_internal(apple_protect_pager_t pager,
-					     boolean_t locked);
+    boolean_t locked);
 void apple_protect_pager_terminate_internal(apple_protect_pager_t pager);
 void apple_protect_pager_trim(void);
 
 
 #if DEBUG
 int apple_protect_pagerdebug = 0;
-#define PAGER_ALL		0xffffffff
-#define	PAGER_INIT		0x00000001
-#define	PAGER_PAGEIN		0x00000002
-
-#define PAGER_DEBUG(LEVEL, A)						\
-	MACRO_BEGIN							\
-	if ((apple_protect_pagerdebug & LEVEL)==LEVEL) {		\
-		printf A;						\
-	}								\
+#define PAGER_ALL               0xffffffff
+#define PAGER_INIT              0x00000001
+#define PAGER_PAGEIN            0x00000002
+
+#define PAGER_DEBUG(LEVEL, A)                                           \
+	MACRO_BEGIN                                                     \
+	if ((apple_protect_pagerdebug & LEVEL)==LEVEL) {                \
+	        printf A;                                               \
+	}                                                               \
 	MACRO_END
 #else
 #define PAGER_DEBUG(LEVEL, A)
 #endif
 
-
-void
-apple_protect_pager_bootstrap(void)
-{
-	lck_grp_attr_setdefault(&apple_protect_pager_lck_grp_attr);
-	lck_grp_init(&apple_protect_pager_lck_grp, "apple_protect", &apple_protect_pager_lck_grp_attr);
-	lck_attr_setdefault(&apple_protect_pager_lck_attr);
-	lck_mtx_init(&apple_protect_pager_lock, &apple_protect_pager_lck_grp, &apple_protect_pager_lck_attr);
-	queue_init(&apple_protect_pager_queue);
-}
-
 /*
  * apple_protect_pager_init()
  *
@@ -237,23 +235,24 @@ apple_protect_pager_bootstrap(void)
  */
 kern_return_t
 apple_protect_pager_init(
-	memory_object_t		mem_obj, 
-	memory_object_control_t	control, 
+	memory_object_t         mem_obj,
+	memory_object_control_t control,
 #if !DEBUG
 	__unused
 #endif
 	memory_object_cluster_size_t pg_size)
 {
-	apple_protect_pager_t	pager;
-	kern_return_t   	kr;
+	apple_protect_pager_t   pager;
+	kern_return_t           kr;
 	memory_object_attr_info_data_t  attributes;
 
 	PAGER_DEBUG(PAGER_ALL,
-		    ("apple_protect_pager_init: %p, %p, %x\n",
-		     mem_obj, control, pg_size));
+	    ("apple_protect_pager_init: %p, %p, %x\n",
+	    mem_obj, control, pg_size));
 
-	if (control == MEMORY_OBJECT_CONTROL_NULL)
+	if (control == MEMORY_OBJECT_CONTROL_NULL) {
 		return KERN_INVALID_ARGUMENT;
+	}
 
 	pager = apple_protect_pager_lookup(mem_obj);
 
@@ -268,13 +267,14 @@ apple_protect_pager_init(
 	attributes.temporary = TRUE;
 
 	kr = memory_object_change_attributes(
-					control,
-					MEMORY_OBJECT_ATTRIBUTE_INFO,
-					(memory_object_info_t) &attributes,
-					MEMORY_OBJECT_ATTR_INFO_COUNT);
-	if (kr != KERN_SUCCESS)
+		control,
+		MEMORY_OBJECT_ATTRIBUTE_INFO,
+		(memory_object_info_t) &attributes,
+		MEMORY_OBJECT_ATTR_INFO_COUNT);
+	if (kr != KERN_SUCCESS) {
 		panic("apple_protect_pager_init: "
-		      "memory_object_change_attributes() failed");
+		    "memory_object_change_attributes() failed");
+	}
 
 #if CONFIG_SECLUDED_MEMORY
 	if (secluded_for_filecache) {
@@ -295,14 +295,14 @@ apple_protect_pager_init(
  */
 kern_return_t
 apple_protect_pager_data_return(
-        __unused memory_object_t	mem_obj,
-        __unused memory_object_offset_t	offset,
-        __unused memory_object_cluster_size_t		data_cnt,
-        __unused memory_object_offset_t	*resid_offset,
-	__unused int			*io_error,
-	__unused boolean_t		dirty,
-	__unused boolean_t		kernel_copy,
-	__unused int			upl_flags)  
+	__unused memory_object_t        mem_obj,
+	__unused memory_object_offset_t offset,
+	__unused memory_object_cluster_size_t           data_cnt,
+	__unused memory_object_offset_t *resid_offset,
+	__unused int                    *io_error,
+	__unused boolean_t              dirty,
+	__unused boolean_t              kernel_copy,
+	__unused int                    upl_flags)
 {
 	panic("apple_protect_pager_data_return: should never get called");
 	return KERN_FAILURE;
@@ -310,9 +310,9 @@ apple_protect_pager_data_return(
 
 kern_return_t
 apple_protect_pager_data_initialize(
-	__unused memory_object_t	mem_obj,
-	__unused memory_object_offset_t	offset,
-	__unused memory_object_cluster_size_t		data_cnt)
+	__unused memory_object_t        mem_obj,
+	__unused memory_object_offset_t offset,
+	__unused memory_object_cluster_size_t           data_cnt)
 {
 	panic("apple_protect_pager_data_initialize: should never get called");
 	return KERN_FAILURE;
@@ -320,10 +320,10 @@ apple_protect_pager_data_initialize(
 
 kern_return_t
 apple_protect_pager_data_unlock(
-	__unused memory_object_t	mem_obj,
-	__unused memory_object_offset_t	offset,
-	__unused memory_object_size_t		size,
-	__unused vm_prot_t		desired_access)
+	__unused memory_object_t        mem_obj,
+	__unused memory_object_offset_t offset,
+	__unused memory_object_size_t           size,
+	__unused vm_prot_t              desired_access)
 {
 	return KERN_FAILURE;
 }
@@ -334,35 +334,35 @@ apple_protect_pager_data_unlock(
  * Handles page-in requests from VM.
  */
 int apple_protect_pager_data_request_debug = 0;
-kern_return_t	
+kern_return_t
 apple_protect_pager_data_request(
-	memory_object_t		mem_obj,
-	memory_object_offset_t	offset,
-	memory_object_cluster_size_t		length,
+	memory_object_t         mem_obj,
+	memory_object_offset_t  offset,
+	memory_object_cluster_size_t            length,
 #if !DEBUG
 	__unused
 #endif
-	vm_prot_t		protection_required,
+	vm_prot_t               protection_required,
 	memory_object_fault_info_t mo_fault_info)
 {
-	apple_protect_pager_t	pager;
-	memory_object_control_t	mo_control;
-	upl_t			upl;
-	int			upl_flags;
-	upl_size_t		upl_size;
-	upl_page_info_t		*upl_pl;
-	unsigned int		pl_count;
-	vm_object_t		src_top_object, src_page_object, dst_object;
-	kern_return_t		kr, retval;
-	vm_offset_t		src_vaddr, dst_vaddr;
-	vm_offset_t		cur_offset;
-	vm_offset_t		offset_in_page;
-	kern_return_t		error_code;
-	vm_prot_t		prot;
-	vm_page_t		src_page, top_page;
-	int			interruptible;
-	struct vm_object_fault_info	fault_info;
-	int			ret;
+	apple_protect_pager_t   pager;
+	memory_object_control_t mo_control;
+	upl_t                   upl;
+	int                     upl_flags;
+	upl_size_t              upl_size;
+	upl_page_info_t         *upl_pl;
+	unsigned int            pl_count;
+	vm_object_t             src_top_object, src_page_object, dst_object;
+	kern_return_t           kr, retval;
+	vm_offset_t             src_vaddr, dst_vaddr;
+	vm_offset_t             cur_offset;
+	vm_offset_t             offset_in_page;
+	kern_return_t           error_code;
+	vm_prot_t               prot;
+	vm_page_t               src_page, top_page;
+	int                     interruptible;
+	struct vm_object_fault_info     fault_info;
+	int                     ret;
 
 	PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_data_request: %p, %llx, %x, %x\n", mem_obj, offset, length, protection_required));
 
@@ -380,7 +380,7 @@ apple_protect_pager_data_request(
 
 	pager = apple_protect_pager_lookup(mem_obj);
 	assert(pager->is_ready);
-	assert(pager->ref_count > 1); /* pager is alive and mapped */
+	assert(os_ref_get_count_raw(&pager->ap_pgr_hdr_ref) > 1); /* pager is alive and mapped */
 
 	PAGER_DEBUG(PAGER_PAGEIN, ("apple_protect_pager_data_request: %p, %llx, %x, %x, pager %p\n", mem_obj, offset, length, protection_required, pager));
 
@@ -394,24 +394,24 @@ apple_protect_pager_data_request(
 
 	upl_size = length;
 	upl_flags =
-		UPL_RET_ONLY_ABSENT |
-		UPL_SET_LITE |
-		UPL_NO_SYNC |
-		UPL_CLEAN_IN_PLACE |	/* triggers UPL_CLEAR_DIRTY */
-		UPL_SET_INTERNAL;
+	    UPL_RET_ONLY_ABSENT |
+	    UPL_SET_LITE |
+	    UPL_NO_SYNC |
+	    UPL_CLEAN_IN_PLACE |        /* triggers UPL_CLEAR_DIRTY */
+	    UPL_SET_INTERNAL;
 	pl_count = 0;
 	kr = memory_object_upl_request(mo_control,
-				       offset, upl_size,
-				       &upl, NULL, NULL, upl_flags, VM_KERN_MEMORY_SECURITY);
+	    offset, upl_size,
+	    &upl, NULL, NULL, upl_flags, VM_KERN_MEMORY_SECURITY);
 	if (kr != KERN_SUCCESS) {
 		retval = kr;
 		goto done;
 	}
-	dst_object = mo_control->moc_object;
+	dst_object = memory_object_control_to_vm_object(mo_control);
 	assert(dst_object != VM_OBJECT_NULL);
 
 	/*
-	 * We'll map the encrypted data in the kernel address space from the 
+	 * We'll map the encrypted data in the kernel address space from the
 	 * backing VM object (itself backed by the encrypted file via
 	 * the vnode pager).
 	 */
@@ -425,8 +425,8 @@ apple_protect_pager_data_request(
 	upl_pl = UPL_GET_INTERNAL_PAGE_LIST(upl);
 	pl_count = length / PAGE_SIZE;
 	for (cur_offset = 0;
-	     retval == KERN_SUCCESS && cur_offset < length;
-	     cur_offset += PAGE_SIZE) {
+	    retval == KERN_SUCCESS && cur_offset < length;
+	    cur_offset += PAGE_SIZE) {
 		ppnum_t dst_pnum;
 
 		if (!upl_page_present(upl_pl, (int)(cur_offset / PAGE_SIZE))) {
@@ -439,25 +439,25 @@ apple_protect_pager_data_request(
 		 * virtual address space.
 		 * We already hold a reference on the src_top_object.
 		 */
-	retry_src_fault:
+retry_src_fault:
 		vm_object_lock(src_top_object);
 		vm_object_paging_begin(src_top_object);
 		error_code = 0;
 		prot = VM_PROT_READ;
 		src_page = VM_PAGE_NULL;
 		kr = vm_fault_page(src_top_object,
-				   pager->backing_offset + offset + cur_offset,
-				   VM_PROT_READ,
-				   FALSE,
-				   FALSE, /* src_page not looked up */
-				   &prot,
-				   &src_page,
-				   &top_page,
-				   NULL,
-				   &error_code,
-				   FALSE,
-				   FALSE,
-				   &fault_info);
+		    pager->backing_offset + offset + cur_offset,
+		    VM_PROT_READ,
+		    FALSE,
+		    FALSE,                /* src_page not looked up */
+		    &prot,
+		    &src_page,
+		    &top_page,
+		    NULL,
+		    &error_code,
+		    FALSE,
+		    FALSE,
+		    &fault_info);
 		switch (kr) {
 		case VM_FAULT_SUCCESS:
 			break;
@@ -467,7 +467,7 @@ apple_protect_pager_data_request(
 			if (vm_page_wait(interruptible)) {
 				goto retry_src_fault;
 			}
-			/* fall thru */
+			OS_FALLTHROUGH;
 		case VM_FAULT_INTERRUPTED:
 			retval = MACH_SEND_INTERRUPTED;
 			goto done;
@@ -475,7 +475,7 @@ apple_protect_pager_data_request(
 			/* success but no VM page: fail */
 			vm_object_paging_end(src_top_object);
 			vm_object_unlock(src_top_object);
-			/*FALLTHROUGH*/
+			OS_FALLTHROUGH;
 		case VM_FAULT_MEMORY_ERROR:
 			/* the page is not there ! */
 			if (error_code) {
@@ -486,18 +486,17 @@ apple_protect_pager_data_request(
 			goto done;
 		default:
 			panic("apple_protect_pager_data_request: "
-			      "vm_fault_page() unexpected error 0x%x\n",
-			      kr);
+			    "vm_fault_page() unexpected error 0x%x\n",
+			    kr);
 		}
 		assert(src_page != VM_PAGE_NULL);
 		assert(src_page->vmp_busy);
 
 		if (src_page->vmp_q_state != VM_PAGE_ON_SPECULATIVE_Q) {
-
 			vm_page_lockspin_queues();
 
 			if (src_page->vmp_q_state != VM_PAGE_ON_SPECULATIVE_Q) {
-			        vm_page_speculate(src_page, FALSE);
+				vm_page_speculate(src_page, FALSE);
 			}
 			vm_page_unlock_queues();
 		}
@@ -507,26 +506,15 @@ apple_protect_pager_data_request(
 		 * and destination physical pages.
 		 */
 		dst_pnum = (ppnum_t)
-		        upl_phys_page(upl_pl, (int)(cur_offset / PAGE_SIZE));
-                assert(dst_pnum != 0);
-#if __x86_64__
-		src_vaddr = (vm_map_offset_t)
-			PHYSMAP_PTOV((pmap_paddr_t)VM_PAGE_GET_PHYS_PAGE(src_page)
-				     << PAGE_SHIFT);
-		dst_vaddr = (vm_map_offset_t)
-			PHYSMAP_PTOV((pmap_paddr_t)dst_pnum << PAGE_SHIFT);
+		    upl_phys_page(upl_pl, (int)(cur_offset / PAGE_SIZE));
+		assert(dst_pnum != 0);
 
-#elif __arm__ || __arm64__
 		src_vaddr = (vm_map_offset_t)
-			phystokv((pmap_paddr_t)VM_PAGE_GET_PHYS_PAGE(src_page)
-				 << PAGE_SHIFT);
+		    phystokv((pmap_paddr_t)VM_PAGE_GET_PHYS_PAGE(src_page)
+		        << PAGE_SHIFT);
 		dst_vaddr = (vm_map_offset_t)
-			phystokv((pmap_paddr_t)dst_pnum << PAGE_SHIFT);
-#else
-#error "vm_paging_map_object: no 1-to-1 kernel mapping of physical memory..."
-		src_vaddr = 0;
-		dst_vaddr = 0;
-#endif
+		    phystokv((pmap_paddr_t)dst_pnum << PAGE_SHIFT);
+
 		src_page_object = VM_PAGE_OBJECT(src_page);
 
 		/*
@@ -534,18 +522,18 @@ apple_protect_pager_data_request(
 		 */
 		if (src_page_object->code_signed) {
 			vm_page_validate_cs_mapped(
-				src_page,
+				src_page, PAGE_SIZE, 0,
 				(const void *) src_vaddr);
 		}
 		/*
 		 * ... and transfer the results to the destination page.
 		 */
 		UPL_SET_CS_VALIDATED(upl_pl, cur_offset / PAGE_SIZE,
-				     src_page->vmp_cs_validated);
+		    src_page->vmp_cs_validated);
 		UPL_SET_CS_TAINTED(upl_pl, cur_offset / PAGE_SIZE,
-				   src_page->vmp_cs_tainted);
+		    src_page->vmp_cs_tainted);
 		UPL_SET_CS_NX(upl_pl, cur_offset / PAGE_SIZE,
-				   src_page->vmp_cs_nx);
+		    src_page->vmp_cs_nx);
 
 		/*
 		 * page_decrypt() might access a mapped file, so let's release
@@ -563,42 +551,42 @@ apple_protect_pager_data_request(
 		 * into the destination page.
 		 */
 		for (offset_in_page = 0;
-		     offset_in_page < PAGE_SIZE;
-		     offset_in_page += 4096) {
+		    offset_in_page < PAGE_SIZE;
+		    offset_in_page += 4096) {
 			if (offset + cur_offset + offset_in_page <
 			    pager->crypto_start ||
 			    offset + cur_offset + offset_in_page >=
 			    pager->crypto_end) {
 				/* not encrypted: just copy */
 				bcopy((const char *)(src_vaddr +
-						     offset_in_page),
-				      (char *)(dst_vaddr + offset_in_page),
-				      4096);
+				    offset_in_page),
+				    (char *)(dst_vaddr + offset_in_page),
+				    4096);
 
 				if (apple_protect_pager_data_request_debug) {
 					printf("apple_protect_data_request"
-					       "(%p,0x%llx+0x%llx+0x%04llx): "
-					       "out of crypto range "
-					       "[0x%llx:0x%llx]: "
-					       "COPY [0x%016llx 0x%016llx] "
-					       "code_signed=%d "
-					       "cs_validated=%d "
-					       "cs_tainted=%d "
-					       "cs_nx=%d\n",
-					       pager,
-					       offset,
-					       (uint64_t) cur_offset,
-					       (uint64_t) offset_in_page,
-					       pager->crypto_start,
-					       pager->crypto_end,
-					       *(uint64_t *)(dst_vaddr+
-							     offset_in_page),
-					       *(uint64_t *)(dst_vaddr+
-							     offset_in_page+8),
-					       src_page_object->code_signed,
-					       src_page->vmp_cs_validated,
-					       src_page->vmp_cs_tainted,
-					       src_page->vmp_cs_nx);
+					    "(%p,0x%llx+0x%llx+0x%04llx): "
+					    "out of crypto range "
+					    "[0x%llx:0x%llx]: "
+					    "COPY [0x%016llx 0x%016llx] "
+					    "code_signed=%d "
+					    "cs_validated=%d "
+					    "cs_tainted=%d "
+					    "cs_nx=%d\n",
+					    pager,
+					    offset,
+					    (uint64_t) cur_offset,
+					    (uint64_t) offset_in_page,
+					    pager->crypto_start,
+					    pager->crypto_end,
+					    *(uint64_t *)(dst_vaddr +
+					    offset_in_page),
+					    *(uint64_t *)(dst_vaddr +
+					    offset_in_page + 8),
+					    src_page_object->code_signed,
+					    src_page->vmp_cs_validated,
+					    src_page->vmp_cs_tainted,
+					    src_page->vmp_cs_nx);
 				}
 				ret = 0;
 				continue;
@@ -607,46 +595,46 @@ apple_protect_pager_data_request(
 				(const void *)(src_vaddr + offset_in_page),
 				(void *)(dst_vaddr + offset_in_page),
 				((pager->crypto_backing_offset -
-				  pager->crypto_start) + /* XXX ? */
-				 offset +
-				 cur_offset +
-				 offset_in_page),
+				pager->crypto_start) +   /* XXX ? */
+				offset +
+				cur_offset +
+				offset_in_page),
 				pager->crypt_info->crypt_ops);
 
 			if (apple_protect_pager_data_request_debug) {
 				printf("apple_protect_data_request"
-				       "(%p,0x%llx+0x%llx+0x%04llx): "
-				       "in crypto range [0x%llx:0x%llx]: "
-				       "DECRYPT offset 0x%llx="
-				       "(0x%llx-0x%llx+0x%llx+0x%llx+0x%04llx)"
-				       "[0x%016llx 0x%016llx] "
-				       "code_signed=%d "
-				       "cs_validated=%d "
-				       "cs_tainted=%d "
-				       "cs_nx=%d "
-				       "ret=0x%x\n",
-				       pager,
-				       offset,
-				       (uint64_t) cur_offset,
-				       (uint64_t) offset_in_page,
-				       pager->crypto_start, pager->crypto_end,
-				       ((pager->crypto_backing_offset -
-					 pager->crypto_start) +
-					offset +
-					cur_offset +
-					offset_in_page),
-				       pager->crypto_backing_offset,
-				       pager->crypto_start,
-				       offset,
-				       (uint64_t) cur_offset,
-				       (uint64_t) offset_in_page,
-				       *(uint64_t *)(dst_vaddr+offset_in_page),
-				       *(uint64_t *)(dst_vaddr+offset_in_page+8),
-				       src_page_object->code_signed,
-				       src_page->vmp_cs_validated,
-				       src_page->vmp_cs_tainted,
-				       src_page->vmp_cs_nx,
-				       ret);
+				    "(%p,0x%llx+0x%llx+0x%04llx): "
+				    "in crypto range [0x%llx:0x%llx]: "
+				    "DECRYPT offset 0x%llx="
+				    "(0x%llx-0x%llx+0x%llx+0x%llx+0x%04llx)"
+				    "[0x%016llx 0x%016llx] "
+				    "code_signed=%d "
+				    "cs_validated=%d "
+				    "cs_tainted=%d "
+				    "cs_nx=%d "
+				    "ret=0x%x\n",
+				    pager,
+				    offset,
+				    (uint64_t) cur_offset,
+				    (uint64_t) offset_in_page,
+				    pager->crypto_start, pager->crypto_end,
+				    ((pager->crypto_backing_offset -
+				    pager->crypto_start) +
+				    offset +
+				    cur_offset +
+				    offset_in_page),
+				    pager->crypto_backing_offset,
+				    pager->crypto_start,
+				    offset,
+				    (uint64_t) cur_offset,
+				    (uint64_t) offset_in_page,
+				    *(uint64_t *)(dst_vaddr + offset_in_page),
+				    *(uint64_t *)(dst_vaddr + offset_in_page + 8),
+				    src_page_object->code_signed,
+				    src_page->vmp_cs_validated,
+				    src_page->vmp_cs_tainted,
+				    src_page->vmp_cs_nx,
+				    ret);
 			}
 			if (ret) {
 				break;
@@ -698,7 +686,7 @@ done:
 		if (retval != KERN_SUCCESS) {
 			upl_abort(upl, 0);
 			if (retval == KERN_ABORTED) {
-				wait_result_t	wait_result;
+				wait_result_t   wait_result;
 
 				/*
 				 * We aborted the fault and did not provide
@@ -718,7 +706,7 @@ done:
 				wait_result = assert_wait_timeout(
 					(event_t) apple_protect_pager_data_request,
 					THREAD_UNINT,
-					10000,	/* 10ms */
+					10000,  /* 10ms */
 					NSEC_PER_USEC);
 				assert(wait_result == THREAD_WAITING);
 				wait_result = thread_block(THREAD_CONTINUE_NULL);
@@ -726,9 +714,12 @@ done:
 			}
 		} else {
 			boolean_t empty;
-			upl_commit_range(upl, 0, upl->size, 
-					 UPL_COMMIT_CS_VALIDATED | UPL_COMMIT_WRITTEN_BY_KERNEL,
-					 upl_pl, pl_count, &empty);
+			assertf(page_aligned(upl->u_offset) && page_aligned(upl->u_size),
+			    "upl %p offset 0x%llx size 0x%x",
+			    upl, upl->u_offset, upl->u_size);
+			upl_commit_range(upl, 0, upl->u_size,
+			    UPL_COMMIT_CS_VALIDATED | UPL_COMMIT_WRITTEN_BY_KERNEL,
+			    upl_pl, pl_count, &empty);
 		}
 
 		/* and deallocate the UPL */
@@ -750,15 +741,14 @@ done:
  */
 void
 apple_protect_pager_reference(
-	memory_object_t		mem_obj)
-{	
-	apple_protect_pager_t	pager;
+	memory_object_t         mem_obj)
+{
+	apple_protect_pager_t   pager;
 
 	pager = apple_protect_pager_lookup(mem_obj);
 
 	lck_mtx_lock(&apple_protect_pager_lock);
-	assert(pager->ref_count > 0);
-	pager->ref_count++;
+	os_ref_retain_locked_raw(&pager->ap_pgr_hdr_ref, NULL);
 	lck_mtx_unlock(&apple_protect_pager_lock);
 }
 
@@ -777,12 +767,12 @@ apple_protect_pager_dequeue(
 	assert(!pager->is_mapped);
 
 	queue_remove(&apple_protect_pager_queue,
-		     pager,
-		     apple_protect_pager_t,
-		     pager_queue);
+	    pager,
+	    apple_protect_pager_t,
+	    pager_queue);
 	pager->pager_queue.next = NULL;
 	pager->pager_queue.prev = NULL;
-	
+
 	apple_protect_pager_count--;
 }
 
@@ -814,9 +804,9 @@ apple_protect_pager_terminate_internal(
 	/* one less pager using this "pager_crypt_info" */
 #if CRYPT_INFO_DEBUG
 	printf("CRYPT_INFO %s: deallocate %p ref %d\n",
-	       __FUNCTION__,
-	       pager->crypt_info,
-	       pager->crypt_info->crypt_refcnt);
+	    __FUNCTION__,
+	    pager->crypt_info,
+	    pager->crypt_info->crypt_refcnt);
 #endif /* CRYPT_INFO_DEBUG */
 	crypt_info_deallocate(pager->crypt_info);
 	pager->crypt_info = NULL;
@@ -835,18 +825,19 @@ apple_protect_pager_terminate_internal(
  */
 void
 apple_protect_pager_deallocate_internal(
-	apple_protect_pager_t	pager,
-	boolean_t		locked)
+	apple_protect_pager_t   pager,
+	boolean_t               locked)
 {
-	boolean_t	needs_trimming;
-	int		count_unmapped;
+	boolean_t       needs_trimming;
+	unsigned int    count_unmapped;
+	os_ref_count_t  ref_count;
 
-	if (! locked) {
+	if (!locked) {
 		lck_mtx_lock(&apple_protect_pager_lock);
 	}
 
-	count_unmapped = (apple_protect_pager_count - 
-			  apple_protect_pager_count_mapped);
+	count_unmapped = (apple_protect_pager_count -
+	    apple_protect_pager_count_mapped);
 	if (count_unmapped > apple_protect_pager_cache_limit) {
 		/* we have too many unmapped pagers:  trim some */
 		needs_trimming = TRUE;
@@ -855,9 +846,9 @@ apple_protect_pager_deallocate_internal(
 	}
 
 	/* drop a reference on this pager */
-	pager->ref_count--;
+	ref_count = os_ref_release_locked_raw(&pager->ap_pgr_hdr_ref, NULL);
 
-	if (pager->ref_count == 1) {
+	if (ref_count == 1) {
 		/*
 		 * Only the "named" reference is left, which means that
 		 * no one is really holding on to this pager anymore.
@@ -867,7 +858,7 @@ apple_protect_pager_deallocate_internal(
 		/* the pager is all ours: no need for the lock now */
 		lck_mtx_unlock(&apple_protect_pager_lock);
 		apple_protect_pager_terminate_internal(pager);
-	} else if (pager->ref_count == 0) {
+	} else if (ref_count == 0) {
 		/*
 		 * Dropped the existence reference;  the memory object has
 		 * been terminated.  Do some final cleanup and release the
@@ -878,7 +869,7 @@ apple_protect_pager_deallocate_internal(
 			memory_object_control_deallocate(pager->ap_pgr_hdr.mo_control);
 			pager->ap_pgr_hdr.mo_control = MEMORY_OBJECT_CONTROL_NULL;
 		}
-		kfree(pager, sizeof (*pager));
+		kfree(pager, sizeof(*pager));
 		pager = APPLE_PROTECT_PAGER_NULL;
 	} else {
 		/* there are still plenty of references:  keep going... */
@@ -899,9 +890,9 @@ apple_protect_pager_deallocate_internal(
  */
 void
 apple_protect_pager_deallocate(
-	memory_object_t		mem_obj)
+	memory_object_t         mem_obj)
 {
-	apple_protect_pager_t	pager;
+	apple_protect_pager_t   pager;
 
 	PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_deallocate: %p\n", mem_obj));
 	pager = apple_protect_pager_lookup(mem_obj);
@@ -916,7 +907,7 @@ apple_protect_pager_terminate(
 #if !DEBUG
 	__unused
 #endif
-	memory_object_t	mem_obj)
+	memory_object_t mem_obj)
 {
 	PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_terminate: %p\n", mem_obj));
 
@@ -928,10 +919,10 @@ apple_protect_pager_terminate(
  */
 kern_return_t
 apple_protect_pager_synchronize(
-	__unused memory_object_t		mem_obj,
-	__unused memory_object_offset_t	offset,
-	__unused memory_object_size_t		length,
-	__unused vm_sync_t		sync_flags)
+	__unused memory_object_t                mem_obj,
+	__unused memory_object_offset_t offset,
+	__unused memory_object_size_t           length,
+	__unused vm_sync_t              sync_flags)
 {
 	panic("apple_protect_pager_synchronize: memory_object_synchronize no longer supported\n");
 	return KERN_FAILURE;
@@ -947,10 +938,10 @@ apple_protect_pager_synchronize(
  */
 kern_return_t
 apple_protect_pager_map(
-	memory_object_t		mem_obj,
-	__unused vm_prot_t	prot)
+	memory_object_t         mem_obj,
+	__unused vm_prot_t      prot)
 {
-	apple_protect_pager_t	pager;
+	apple_protect_pager_t   pager;
 
 	PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_map: %p\n", mem_obj));
 
@@ -958,7 +949,7 @@ apple_protect_pager_map(
 
 	lck_mtx_lock(&apple_protect_pager_lock);
 	assert(pager->is_ready);
-	assert(pager->ref_count > 0); /* pager is alive */
+	assert(os_ref_get_count_raw(&pager->ap_pgr_hdr_ref) > 0); /* pager is alive */
 	if (pager->is_mapped == FALSE) {
 		/*
 		 * First mapping of this pager:  take an extra reference
@@ -966,7 +957,7 @@ apple_protect_pager_map(
 		 * are removed.
 		 */
 		pager->is_mapped = TRUE;
-		pager->ref_count++;
+		os_ref_retain_locked_raw(&pager->ap_pgr_hdr_ref, NULL);
 		apple_protect_pager_count_mapped++;
 	}
 	lck_mtx_unlock(&apple_protect_pager_lock);
@@ -981,13 +972,13 @@ apple_protect_pager_map(
  */
 kern_return_t
 apple_protect_pager_last_unmap(
-	memory_object_t		mem_obj)
+	memory_object_t         mem_obj)
 {
-	apple_protect_pager_t	pager;
-	int			count_unmapped;
+	apple_protect_pager_t   pager;
+	unsigned int            count_unmapped;
 
 	PAGER_DEBUG(PAGER_ALL,
-		    ("apple_protect_pager_last_unmap: %p\n", mem_obj));
+	    ("apple_protect_pager_last_unmap: %p\n", mem_obj));
 
 	pager = apple_protect_pager_lookup(mem_obj);
 
@@ -999,7 +990,7 @@ apple_protect_pager_last_unmap(
 		 */
 		apple_protect_pager_count_mapped--;
 		count_unmapped = (apple_protect_pager_count -
-				  apple_protect_pager_count_mapped);
+		    apple_protect_pager_count_mapped);
 		if (count_unmapped > apple_protect_pager_count_unmapped_max) {
 			apple_protect_pager_count_unmapped_max = count_unmapped;
 		}
@@ -1009,41 +1000,61 @@ apple_protect_pager_last_unmap(
 	} else {
 		lck_mtx_unlock(&apple_protect_pager_lock);
 	}
-	
+
 	return KERN_SUCCESS;
 }
 
+boolean_t
+apple_protect_pager_backing_object(
+	memory_object_t mem_obj,
+	memory_object_offset_t offset,
+	vm_object_t *backing_object,
+	vm_object_offset_t *backing_offset)
+{
+	apple_protect_pager_t   pager;
+
+	PAGER_DEBUG(PAGER_ALL,
+	    ("apple_protect_pager_backing_object: %p\n", mem_obj));
+
+	pager = apple_protect_pager_lookup(mem_obj);
+
+	*backing_object = pager->backing_object;
+	*backing_offset = pager->backing_offset + offset;
+
+	return TRUE;
+}
 
 /*
  *
  */
 apple_protect_pager_t
 apple_protect_pager_lookup(
-	memory_object_t	 mem_obj)
+	memory_object_t  mem_obj)
 {
-	apple_protect_pager_t	pager;
+	apple_protect_pager_t   pager;
 
 	assert(mem_obj->mo_pager_ops == &apple_protect_pager_ops);
 	pager = (apple_protect_pager_t)(uintptr_t) mem_obj;
-	assert(pager->ref_count > 0);
+	assert(os_ref_get_count_raw(&pager->ap_pgr_hdr_ref) > 0);
 	return pager;
 }
 
 apple_protect_pager_t
 apple_protect_pager_create(
-	vm_object_t		backing_object,
-	vm_object_offset_t	backing_offset,
-	vm_object_offset_t	crypto_backing_offset,
+	vm_object_t             backing_object,
+	vm_object_offset_t      backing_offset,
+	vm_object_offset_t      crypto_backing_offset,
 	struct pager_crypt_info *crypt_info,
-	vm_object_offset_t	crypto_start,
-	vm_object_offset_t	crypto_end)
+	vm_object_offset_t      crypto_start,
+	vm_object_offset_t      crypto_end,
+	boolean_t               cache_pager)
 {
-	apple_protect_pager_t	pager, pager2;
-	memory_object_control_t	control;
-	kern_return_t		kr;
-	struct pager_crypt_info	*old_crypt_info;
+	apple_protect_pager_t   pager, pager2;
+	memory_object_control_t control;
+	kern_return_t           kr;
+	struct pager_crypt_info *old_crypt_info;
 
-	pager = (apple_protect_pager_t) kalloc(sizeof (*pager));
+	pager = (apple_protect_pager_t) kalloc(sizeof(*pager));
 	if (pager == APPLE_PROTECT_PAGER_NULL) {
 		return APPLE_PROTECT_PAGER_NULL;
 	}
@@ -1060,9 +1071,16 @@ apple_protect_pager_create(
 	pager->ap_pgr_hdr.mo_control = MEMORY_OBJECT_CONTROL_NULL;
 
 	pager->is_ready = FALSE;/* not ready until it has a "name" */
-	pager->ref_count = 1;	/* existence reference (for the cache) */
-	pager->ref_count++;	/* for the caller */
+	/* one reference for the caller */
+	os_ref_init_count_raw(&pager->ap_pgr_hdr_ref, NULL, 1);
 	pager->is_mapped = FALSE;
+	if (cache_pager) {
+		/* extra reference for the cache */
+		os_ref_retain_locked_raw(&pager->ap_pgr_hdr_ref, NULL);
+		pager->is_cached = true;
+	} else {
+		pager->is_cached = false;
+	}
 	pager->backing_object = backing_object;
 	pager->backing_offset = backing_offset;
 	pager->crypto_backing_offset = crypto_backing_offset;
@@ -1072,14 +1090,14 @@ apple_protect_pager_create(
 
 #if CRYPT_INFO_DEBUG
 	printf("CRYPT_INFO %s: crypt_info %p [%p,%p,%p,%d]\n",
-	       __FUNCTION__,
-	       crypt_info,
-	       crypt_info->page_decrypt,
-	       crypt_info->crypt_end,
-	       crypt_info->crypt_ops,
-	       crypt_info->crypt_refcnt);
+	    __FUNCTION__,
+	    crypt_info,
+	    crypt_info->page_decrypt,
+	    crypt_info->crypt_end,
+	    crypt_info->crypt_ops,
+	    crypt_info->crypt_refcnt);
 #endif /* CRYPT_INFO_DEBUG */
-	
+
 	vm_object_reference(backing_object);
 
 	old_crypt_info = NULL;
@@ -1087,15 +1105,15 @@ apple_protect_pager_create(
 	lck_mtx_lock(&apple_protect_pager_lock);
 	/* see if anyone raced us to create a pager for the same object */
 	queue_iterate(&apple_protect_pager_queue,
-		      pager2,
-		      apple_protect_pager_t,
-		      pager_queue) {
+	    pager2,
+	    apple_protect_pager_t,
+	    pager_queue) {
 		if ((pager2->crypt_info->page_decrypt !=
-		     crypt_info->page_decrypt) ||
+		    crypt_info->page_decrypt) ||
 		    (pager2->crypt_info->crypt_end !=
-		     crypt_info->crypt_end) ||
+		    crypt_info->crypt_end) ||
 		    (pager2->crypt_info->crypt_ops !=
-		     crypt_info->crypt_ops)) {
+		    crypt_info->crypt_ops)) {
 			/* crypt_info contents do not match: next pager */
 			continue;
 		}
@@ -1108,16 +1126,16 @@ apple_protect_pager_create(
 			/* ... switch to that pager's crypt_info */
 #if CRYPT_INFO_DEBUG
 			printf("CRYPT_INFO %s: reference %p ref %d "
-			       "(create match)\n",
-			       __FUNCTION__,
-			       pager2->crypt_info,
-			       pager2->crypt_info->crypt_refcnt);
+			    "(create match)\n",
+			    __FUNCTION__,
+			    pager2->crypt_info,
+			    pager2->crypt_info->crypt_refcnt);
 #endif /* CRYPT_INFO_DEBUG */
 			old_crypt_info = pager2->crypt_info;
 			crypt_info_reference(old_crypt_info);
 			pager->crypt_info = old_crypt_info;
 		}
-		
+
 		if (pager2->backing_object == backing_object &&
 		    pager2->backing_offset == backing_offset &&
 		    pager2->crypto_backing_offset == crypto_backing_offset &&
@@ -1127,21 +1145,21 @@ apple_protect_pager_create(
 			break;
 		}
 	}
-	if (! queue_end(&apple_protect_pager_queue,
-			(queue_entry_t) pager2)) {
+	if (!queue_end(&apple_protect_pager_queue,
+	    (queue_entry_t) pager2)) {
 		/* we lost the race, down with the loser... */
 		lck_mtx_unlock(&apple_protect_pager_lock);
 		vm_object_deallocate(pager->backing_object);
 		pager->backing_object = VM_OBJECT_NULL;
 #if CRYPT_INFO_DEBUG
 		printf("CRYPT_INFO %s: %p ref %d (create pager match)\n",
-		       __FUNCTION__,
-		       pager->crypt_info,
-		       pager->crypt_info->crypt_refcnt);
+		    __FUNCTION__,
+		    pager->crypt_info,
+		    pager->crypt_info->crypt_refcnt);
 #endif /* CRYPT_INFO_DEBUG */
 		crypt_info_deallocate(pager->crypt_info);
 		pager->crypt_info = NULL;
-		kfree(pager, sizeof (*pager));
+		kfree(pager, sizeof(*pager));
 		/* ... and go with the winner */
 		pager = pager2;
 		/* let the winner make sure the pager gets ready */
@@ -1150,9 +1168,9 @@ apple_protect_pager_create(
 
 	/* enter new pager at the head of our list of pagers */
 	queue_enter_first(&apple_protect_pager_queue,
-			  pager,
-			  apple_protect_pager_t,
-			  pager_queue);
+	    pager,
+	    apple_protect_pager_t,
+	    pager_queue);
 	apple_protect_pager_count++;
 	if (apple_protect_pager_count > apple_protect_pager_count_max) {
 		apple_protect_pager_count_max = apple_protect_pager_count;
@@ -1160,10 +1178,12 @@ apple_protect_pager_create(
 	lck_mtx_unlock(&apple_protect_pager_lock);
 
 	kr = memory_object_create_named((memory_object_t) pager,
-					0,
-					&control);
+	    0,
+	    &control);
 	assert(kr == KERN_SUCCESS);
 
+	memory_object_mark_trusted(control);
+
 	lck_mtx_lock(&apple_protect_pager_lock);
 	/* the new pager is now ready to be used */
 	pager->is_ready = TRUE;
@@ -1177,10 +1197,10 @@ apple_protect_pager_create(
 		/* we re-used an old crypt_info instead of using our new one */
 #if CRYPT_INFO_DEBUG
 		printf("CRYPT_INFO %s: deallocate %p ref %d "
-		       "(create used old)\n",
-		       __FUNCTION__,
-		       crypt_info,
-		       crypt_info->crypt_refcnt);
+		    "(create used old)\n",
+		    __FUNCTION__,
+		    crypt_info,
+		    crypt_info->crypt_refcnt);
 #endif /* CRYPT_INFO_DEBUG */
 		crypt_info_deallocate(crypt_info);
 		crypt_info = NULL;
@@ -1198,24 +1218,25 @@ apple_protect_pager_create(
  */
 memory_object_t
 apple_protect_pager_setup(
-	vm_object_t		backing_object,
-	vm_object_offset_t	backing_offset,
-	vm_object_offset_t	crypto_backing_offset,
+	vm_object_t             backing_object,
+	vm_object_offset_t      backing_offset,
+	vm_object_offset_t      crypto_backing_offset,
 	struct pager_crypt_info *crypt_info,
-	vm_object_offset_t	crypto_start,
-	vm_object_offset_t	crypto_end)
+	vm_object_offset_t      crypto_start,
+	vm_object_offset_t      crypto_end,
+	boolean_t               cache_pager)
 {
-	apple_protect_pager_t	pager;
-	struct pager_crypt_info	*old_crypt_info, *new_crypt_info;
+	apple_protect_pager_t   pager;
+	struct pager_crypt_info *old_crypt_info, *new_crypt_info;
 
 #if CRYPT_INFO_DEBUG
 	printf("CRYPT_INFO %s: crypt_info=%p [%p,%p,%p,%d]\n",
-	       __FUNCTION__,
-	       crypt_info,
-	       crypt_info->page_decrypt,
-	       crypt_info->crypt_end,
-	       crypt_info->crypt_ops,
-	       crypt_info->crypt_refcnt);
+	    __FUNCTION__,
+	    crypt_info,
+	    crypt_info->page_decrypt,
+	    crypt_info->crypt_end,
+	    crypt_info->crypt_ops,
+	    crypt_info->crypt_refcnt);
 #endif /* CRYPT_INFO_DEBUG */
 
 	old_crypt_info = NULL;
@@ -1223,15 +1244,15 @@ apple_protect_pager_setup(
 	lck_mtx_lock(&apple_protect_pager_lock);
 
 	queue_iterate(&apple_protect_pager_queue,
-		      pager,
-		      apple_protect_pager_t,
-		      pager_queue) {
+	    pager,
+	    apple_protect_pager_t,
+	    pager_queue) {
 		if ((pager->crypt_info->page_decrypt !=
-		     crypt_info->page_decrypt) ||
+		    crypt_info->page_decrypt) ||
 		    (pager->crypt_info->crypt_end !=
-		     crypt_info->crypt_end) ||
+		    crypt_info->crypt_end) ||
 		    (pager->crypt_info->crypt_ops !=
-		     crypt_info->crypt_ops)) {
+		    crypt_info->crypt_ops)) {
 			/* no match for "crypt_info": next pager */
 			continue;
 		}
@@ -1244,28 +1265,28 @@ apple_protect_pager_setup(
 			old_crypt_info = pager->crypt_info;
 #if CRYPT_INFO_DEBUG
 			printf("CRYPT_INFO %s: "
-			       "switching crypt_info from %p [%p,%p,%p,%d] "
-			       "to %p [%p,%p,%p,%d] from pager %p\n",
-			       __FUNCTION__,
-			       crypt_info,
-			       crypt_info->page_decrypt,
-			       crypt_info->crypt_end,
-			       crypt_info->crypt_ops,
-			       crypt_info->crypt_refcnt,
-			       old_crypt_info,
-			       old_crypt_info->page_decrypt,
-			       old_crypt_info->crypt_end,
-			       old_crypt_info->crypt_ops,
-			       old_crypt_info->crypt_refcnt,
-			       pager);
+			    "switching crypt_info from %p [%p,%p,%p,%d] "
+			    "to %p [%p,%p,%p,%d] from pager %p\n",
+			    __FUNCTION__,
+			    crypt_info,
+			    crypt_info->page_decrypt,
+			    crypt_info->crypt_end,
+			    crypt_info->crypt_ops,
+			    crypt_info->crypt_refcnt,
+			    old_crypt_info,
+			    old_crypt_info->page_decrypt,
+			    old_crypt_info->crypt_end,
+			    old_crypt_info->crypt_ops,
+			    old_crypt_info->crypt_refcnt,
+			    pager);
 			printf("CRYPT_INFO %s: %p ref %d (setup match)\n",
-			       __FUNCTION__,
-			       pager->crypt_info,
-			       pager->crypt_info->crypt_refcnt);
+			    __FUNCTION__,
+			    pager->crypt_info,
+			    pager->crypt_info->crypt_refcnt);
 #endif /* CRYPT_INFO_DEBUG */
 			crypt_info_reference(pager->crypt_info);
 		}
-		
+
 		if (pager->backing_object == backing_object &&
 		    pager->backing_offset == backing_offset &&
 		    pager->crypto_backing_offset == crypto_backing_offset &&
@@ -1276,27 +1297,26 @@ apple_protect_pager_setup(
 			assert(old_crypt_info->crypt_refcnt > 1);
 #if CRYPT_INFO_DEBUG
 			printf("CRYPT_INFO %s: "
-			       "pager match with %p crypt_info %p\n",
-			       __FUNCTION__,
-			       pager,
-			       pager->crypt_info);
+			    "pager match with %p crypt_info %p\n",
+			    __FUNCTION__,
+			    pager,
+			    pager->crypt_info);
 			printf("CRYPT_INFO %s: deallocate %p ref %d "
-			       "(pager match)\n",
-			       __FUNCTION__,
-			       old_crypt_info,
-			       old_crypt_info->crypt_refcnt);
+			    "(pager match)\n",
+			    __FUNCTION__,
+			    old_crypt_info,
+			    old_crypt_info->crypt_refcnt);
 #endif /* CRYPT_INFO_DEBUG */
 			/* release the extra ref on crypt_info we got above */
 			crypt_info_deallocate(old_crypt_info);
 			assert(old_crypt_info->crypt_refcnt > 0);
 			/* give extra reference on pager to the caller */
-			assert(pager->ref_count > 0);
-			pager->ref_count++;
+			os_ref_retain_locked_raw(&pager->ap_pgr_hdr_ref, NULL);
 			break;
 		}
 	}
 	if (queue_end(&apple_protect_pager_queue,
-		      (queue_entry_t) pager)) {
+	    (queue_entry_t) pager)) {
 		lck_mtx_unlock(&apple_protect_pager_lock);
 		/* no existing pager for this backing object */
 		pager = APPLE_PROTECT_PAGER_NULL;
@@ -1305,20 +1325,20 @@ apple_protect_pager_setup(
 			new_crypt_info = old_crypt_info;
 #if CRYPT_INFO_DEBUG
 			printf("CRYPT_INFO %s: "
-			       "will use old_crypt_info %p for new pager\n",
-			       __FUNCTION__,
-			       old_crypt_info);
+			    "will use old_crypt_info %p for new pager\n",
+			    __FUNCTION__,
+			    old_crypt_info);
 #endif /* CRYPT_INFO_DEBUG */
 		} else {
 			/* allocate a new crypt_info for new pager */
-			new_crypt_info = kalloc(sizeof (*new_crypt_info));
+			new_crypt_info = kalloc(sizeof(*new_crypt_info));
 			*new_crypt_info = *crypt_info;
 			new_crypt_info->crypt_refcnt = 1;
 #if CRYPT_INFO_DEBUG
 			printf("CRYPT_INFO %s: "
-			       "will use new_crypt_info %p for new pager\n",
-			       __FUNCTION__,
-			       new_crypt_info);
+			    "will use new_crypt_info %p for new pager\n",
+			    __FUNCTION__,
+			    new_crypt_info);
 #endif /* CRYPT_INFO_DEBUG */
 		}
 		if (new_crypt_info == NULL) {
@@ -1331,7 +1351,8 @@ apple_protect_pager_setup(
 				crypto_backing_offset,
 				new_crypt_info,
 				crypto_start,
-				crypto_end);
+				crypto_end,
+				cache_pager);
 		}
 		if (pager == APPLE_PROTECT_PAGER_NULL) {
 			/* could not create a new pager */
@@ -1339,10 +1360,10 @@ apple_protect_pager_setup(
 				/* release extra reference on old_crypt_info */
 #if CRYPT_INFO_DEBUG
 				printf("CRYPT_INFO %s: deallocate %p ref %d "
-				       "(create fail old_crypt_info)\n",
-				       __FUNCTION__,
-				       old_crypt_info, 
-				       old_crypt_info->crypt_refcnt);
+				    "(create fail old_crypt_info)\n",
+				    __FUNCTION__,
+				    old_crypt_info,
+				    old_crypt_info->crypt_refcnt);
 #endif /* CRYPT_INFO_DEBUG */
 				crypt_info_deallocate(old_crypt_info);
 				old_crypt_info = NULL;
@@ -1351,10 +1372,10 @@ apple_protect_pager_setup(
 				assert(new_crypt_info->crypt_refcnt == 1);
 #if CRYPT_INFO_DEBUG
 				printf("CRYPT_INFO %s: deallocate %p ref %d "
-				       "(create fail new_crypt_info)\n",
-				       __FUNCTION__,
-				       new_crypt_info,
-				       new_crypt_info->crypt_refcnt);
+				    "(create fail new_crypt_info)\n",
+				    __FUNCTION__,
+				    new_crypt_info,
+				    new_crypt_info->crypt_refcnt);
 #endif /* CRYPT_INFO_DEBUG */
 				crypt_info_deallocate(new_crypt_info);
 				new_crypt_info = NULL;
@@ -1368,22 +1389,22 @@ apple_protect_pager_setup(
 
 	while (!pager->is_ready) {
 		lck_mtx_sleep(&apple_protect_pager_lock,
-			LCK_SLEEP_DEFAULT,
-			&pager->is_ready,
-			THREAD_UNINT);
+		    LCK_SLEEP_DEFAULT,
+		    &pager->is_ready,
+		    THREAD_UNINT);
 	}
 	lck_mtx_unlock(&apple_protect_pager_lock);
 
 	return (memory_object_t) pager;
-}	
+}
 
 void
 apple_protect_pager_trim(void)
 {
-	apple_protect_pager_t	pager, prev_pager;
-	queue_head_t		trim_queue;
-	int			num_trim;
-	int			count_unmapped;
+	apple_protect_pager_t   pager, prev_pager;
+	queue_head_t            trim_queue;
+	unsigned int            num_trim;
+	unsigned int            count_unmapped;
 
 	lck_mtx_lock(&apple_protect_pager_lock);
 
@@ -1395,15 +1416,16 @@ apple_protect_pager_trim(void)
 	num_trim = 0;
 
 	for (pager = (apple_protect_pager_t)
-		     queue_last(&apple_protect_pager_queue);
-	     !queue_end(&apple_protect_pager_queue,
-			(queue_entry_t) pager);
-	     pager = prev_pager) {
+	    queue_last(&apple_protect_pager_queue);
+	    !queue_end(&apple_protect_pager_queue,
+	    (queue_entry_t) pager);
+	    pager = prev_pager) {
 		/* get prev elt before we dequeue */
 		prev_pager = (apple_protect_pager_t)
-			queue_prev(&pager->pager_queue);
+		    queue_prev(&pager->pager_queue);
 
-		if (pager->ref_count == 2 &&
+		if (pager->is_cached &&
+		    os_ref_get_count_raw(&pager->ap_pgr_hdr_ref) == 2 &&
 		    pager->is_ready &&
 		    !pager->is_mapped) {
 			/* this pager can be trimmed */
@@ -1412,12 +1434,12 @@ apple_protect_pager_trim(void)
 			apple_protect_pager_dequeue(pager);
 			/* ... and add it to our trim queue */
 			queue_enter_first(&trim_queue,
-					  pager,
-					  apple_protect_pager_t,
-					  pager_queue);
+			    pager,
+			    apple_protect_pager_t,
+			    pager_queue);
 
 			count_unmapped = (apple_protect_pager_count -
-					  apple_protect_pager_count_mapped);
+			    apple_protect_pager_count_mapped);
 			if (count_unmapped <= apple_protect_pager_cache_limit) {
 				/* we have enough pagers to trim */
 				break;
@@ -1434,18 +1456,21 @@ apple_protect_pager_trim(void)
 	/* terminate the trimmed pagers */
 	while (!queue_empty(&trim_queue)) {
 		queue_remove_first(&trim_queue,
-				   pager,
-				   apple_protect_pager_t,
-				   pager_queue);
+		    pager,
+		    apple_protect_pager_t,
+		    pager_queue);
+		assert(pager->is_cached);
+		pager->is_cached = false;
 		pager->pager_queue.next = NULL;
 		pager->pager_queue.prev = NULL;
-		assert(pager->ref_count == 2);
 		/*
 		 * We can't call deallocate_internal() because the pager
 		 * has already been dequeued, but we still need to remove
 		 * a reference.
 		 */
-		pager->ref_count--;
+		os_ref_count_t __assert_only count;
+		count = os_ref_release_locked_raw(&pager->ap_pgr_hdr_ref, NULL);
+		assert(count == 1);
 		apple_protect_pager_terminate_internal(pager);
 	}
 }
@@ -1458,24 +1483,24 @@ crypt_info_reference(
 	assert(crypt_info->crypt_refcnt != 0);
 #if CRYPT_INFO_DEBUG
 	printf("CRYPT_INFO %s: %p ref %d -> %d\n",
-	       __FUNCTION__,
-	       crypt_info,
-	       crypt_info->crypt_refcnt,
-	       crypt_info->crypt_refcnt + 1);
+	    __FUNCTION__,
+	    crypt_info,
+	    crypt_info->crypt_refcnt,
+	    crypt_info->crypt_refcnt + 1);
 #endif /* CRYPT_INFO_DEBUG */
 	OSAddAtomic(+1, &crypt_info->crypt_refcnt);
 }
 
 void
 crypt_info_deallocate(
-	struct pager_crypt_info	*crypt_info)
+	struct pager_crypt_info *crypt_info)
 {
 #if CRYPT_INFO_DEBUG
 	printf("CRYPT_INFO %s: %p ref %d -> %d\n",
-	       __FUNCTION__,
-	       crypt_info,
-	       crypt_info->crypt_refcnt,
-	       crypt_info->crypt_refcnt - 1);
+	    __FUNCTION__,
+	    crypt_info,
+	    crypt_info->crypt_refcnt,
+	    crypt_info->crypt_refcnt - 1);
 #endif /* CRYPT_INFO_DEBUG */
 	OSAddAtomic(-1, &crypt_info->crypt_refcnt);
 	if (crypt_info->crypt_refcnt == 0) {
@@ -1486,10 +1511,10 @@ crypt_info_deallocate(
 		}
 #if CRYPT_INFO_DEBUG
 		printf("CRYPT_INFO %s: freeing %p\n",
-		       __FUNCTION__,
-		       crypt_info);
+		    __FUNCTION__,
+		    crypt_info);
 #endif /* CRYPT_INFO_DEBUG */
-		kfree(crypt_info, sizeof (*crypt_info));
+		kfree(crypt_info, sizeof(*crypt_info));
 		crypt_info = NULL;
 	}
 }