xnu-6153.11.26.tar.gz

[apple/xnu.git] / osfmk / vm / vm_apple_protect.c

diff --git a/osfmk/vm/vm_apple_protect.c b/osfmk/vm/vm_apple_protect.c
index a98fdbb3af325444e14a9e28c0b7b81810f9d19d..1b79225742f5e97d9c86b3ce07a55a380f032d20 100644 (file)
--- a/osfmk/vm/vm_apple_protect.c
+++ b/osfmk/vm/vm_apple_protect.c
@@ -1,8 +1,8 @@
 /*
 /*

- * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2019 Apple Inc. All rights reserved.

  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@

- * 
+ *

  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in
  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in


@@ -11,10 +11,10 @@
  * unlawful or unlicensed copies of an Apple operating system, or to
  * circumvent, violate, or enable the circumvention or violation of, any
  * terms of an Apple operating system software license agreement.
  * unlawful or unlicensed copies of an Apple operating system, or to
  * circumvent, violate, or enable the circumvention or violation of, any
  * terms of an Apple operating system software license agreement.

- * 
+ *

  * Please obtain a copy of the License at
  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  * Please obtain a copy of the License at
  * http://www.opensource.apple.com/apsl/ and read it before using this file.

- * 
+ *

  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,


@@ -22,7 +22,7 @@
  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  * Please see the License for the specific language governing rights and
  * limitations under the License.
  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  * Please see the License for the specific language governing rights and
  * limitations under the License.

- * 
+ *

  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 


@@ -45,27 +45,27 @@
 #include <kern/page_decrypt.h>
 #include <kern/queue.h>
 #include <kern/thread.h>
 #include <kern/page_decrypt.h>
 #include <kern/queue.h>
 #include <kern/thread.h>

+#include <kern/ipc_kobject.h>
+#include <os/refcnt.h>

 
 #include <ipc/ipc_port.h>
 #include <ipc/ipc_space.h>
 
 
 #include <ipc/ipc_port.h>
 #include <ipc/ipc_space.h>
 

-#include <default_pager/default_pager_types.h>
-#include <default_pager/default_pager_object_server.h>
-

 #include <vm/vm_fault.h>
 #include <vm/vm_map.h>
 #include <vm/vm_pageout.h>
 #include <vm/memory_object.h>
 #include <vm/vm_pageout.h>
 #include <vm/vm_protos.h>
 #include <vm/vm_fault.h>
 #include <vm/vm_map.h>
 #include <vm/vm_pageout.h>
 #include <vm/memory_object.h>
 #include <vm/vm_pageout.h>
 #include <vm/vm_protos.h>

+#include <vm/vm_kern.h>

 
 
 
 

-/* 
- * APPLE PROTECT MEMORY PAGER 
+/*
+ * APPLE PROTECT MEMORY PAGER

  *
  * This external memory manager (EMM) handles memory from the encrypted
  * sections of some executables protected by the DSMOS kernel extension.
  *
  * This external memory manager (EMM) handles memory from the encrypted
  * sections of some executables protected by the DSMOS kernel extension.

- * 
+ *

  * It mostly handles page-in requests (from memory_object_data_request()) by
  * getting the encrypted data from its backing VM object, itself backed by
  * the encrypted file, decrypting it and providing it to VM.
  * It mostly handles page-in requests (from memory_object_data_request()) by
  * getting the encrypted data from its backing VM object, itself backed by
  * the encrypted file, decrypting it and providing it to VM.


@@ -83,55 +83,59 @@
 void apple_protect_pager_reference(memory_object_t mem_obj);
 void apple_protect_pager_deallocate(memory_object_t mem_obj);
 kern_return_t apple_protect_pager_init(memory_object_t mem_obj,
 void apple_protect_pager_reference(memory_object_t mem_obj);
 void apple_protect_pager_deallocate(memory_object_t mem_obj);
 kern_return_t apple_protect_pager_init(memory_object_t mem_obj,

-                                      memory_object_control_t control,
-                                      memory_object_cluster_size_t pg_size);
+    memory_object_control_t control,
+    memory_object_cluster_size_t pg_size);

 kern_return_t apple_protect_pager_terminate(memory_object_t mem_obj);
 kern_return_t apple_protect_pager_data_request(memory_object_t mem_obj,
 kern_return_t apple_protect_pager_terminate(memory_object_t mem_obj);
 kern_return_t apple_protect_pager_data_request(memory_object_t mem_obj,

-                                              memory_object_offset_t offset,
-                                              memory_object_cluster_size_t length,
-                                              vm_prot_t protection_required,
-                                              memory_object_fault_info_t fault_info);
+    memory_object_offset_t offset,
+    memory_object_cluster_size_t length,
+    vm_prot_t protection_required,
+    memory_object_fault_info_t fault_info);

 kern_return_t apple_protect_pager_data_return(memory_object_t mem_obj,
 kern_return_t apple_protect_pager_data_return(memory_object_t mem_obj,

-                                             memory_object_offset_t offset,
-                                             memory_object_cluster_size_t      data_cnt,
-                                             memory_object_offset_t *resid_offset,
-                                             int *io_error,
-                                             boolean_t dirty,
-                                             boolean_t kernel_copy,
-                                             int upl_flags);
+    memory_object_offset_t offset,
+    memory_object_cluster_size_t      data_cnt,
+    memory_object_offset_t *resid_offset,
+    int *io_error,
+    boolean_t dirty,
+    boolean_t kernel_copy,
+    int upl_flags);

 kern_return_t apple_protect_pager_data_initialize(memory_object_t mem_obj,
 kern_return_t apple_protect_pager_data_initialize(memory_object_t mem_obj,

-                                                 memory_object_offset_t offset,
-                                                 memory_object_cluster_size_t data_cnt);
+    memory_object_offset_t offset,
+    memory_object_cluster_size_t data_cnt);

 kern_return_t apple_protect_pager_data_unlock(memory_object_t mem_obj,
 kern_return_t apple_protect_pager_data_unlock(memory_object_t mem_obj,

-                                             memory_object_offset_t offset,
-                                             memory_object_size_t size,
-                                             vm_prot_t desired_access);
+    memory_object_offset_t offset,
+    memory_object_size_t size,
+    vm_prot_t desired_access);

 kern_return_t apple_protect_pager_synchronize(memory_object_t mem_obj,
 kern_return_t apple_protect_pager_synchronize(memory_object_t mem_obj,

-                                             memory_object_offset_t offset,
-                                             memory_object_size_t length,
-                                             vm_sync_t sync_flags);
+    memory_object_offset_t offset,
+    memory_object_size_t length,
+    vm_sync_t sync_flags);

 kern_return_t apple_protect_pager_map(memory_object_t mem_obj,
 kern_return_t apple_protect_pager_map(memory_object_t mem_obj,

-                                     vm_prot_t prot);
+    vm_prot_t prot);

 kern_return_t apple_protect_pager_last_unmap(memory_object_t mem_obj);
 
 kern_return_t apple_protect_pager_last_unmap(memory_object_t mem_obj);
 

+#define CRYPT_INFO_DEBUG 0
+void crypt_info_reference(struct pager_crypt_info *crypt_info);
+void crypt_info_deallocate(struct pager_crypt_info *crypt_info);
+

 /*
  * Vector of VM operations for this EMM.
  * These routines are invoked by VM via the memory_object_*() interfaces.
  */
 const struct memory_object_pager_ops apple_protect_pager_ops = {
 /*
  * Vector of VM operations for this EMM.
  * These routines are invoked by VM via the memory_object_*() interfaces.
  */
 const struct memory_object_pager_ops apple_protect_pager_ops = {

-       apple_protect_pager_reference,
-       apple_protect_pager_deallocate,
-       apple_protect_pager_init,
-       apple_protect_pager_terminate,
-       apple_protect_pager_data_request,
-       apple_protect_pager_data_return,
-       apple_protect_pager_data_initialize,
-       apple_protect_pager_data_unlock,
-       apple_protect_pager_synchronize,
-       apple_protect_pager_map,
-       apple_protect_pager_last_unmap,
-       NULL, /* data_reclaim */
-       "apple protect pager"
+       .memory_object_reference = apple_protect_pager_reference,
+       .memory_object_deallocate = apple_protect_pager_deallocate,
+       .memory_object_init = apple_protect_pager_init,
+       .memory_object_terminate = apple_protect_pager_terminate,
+       .memory_object_data_request = apple_protect_pager_data_request,
+       .memory_object_data_return = apple_protect_pager_data_return,
+       .memory_object_data_initialize = apple_protect_pager_data_initialize,
+       .memory_object_data_unlock = apple_protect_pager_data_unlock,
+       .memory_object_synchronize = apple_protect_pager_synchronize,
+       .memory_object_map = apple_protect_pager_map,
+       .memory_object_last_unmap = apple_protect_pager_last_unmap,
+       .memory_object_data_reclaim = NULL,
+       .memory_object_pager_name = "apple_protect"

 };
 
 /*
 };
 
 /*


@@ -139,32 +143,36 @@ const struct memory_object_pager_ops apple_protect_pager_ops = {
  * the "apple protect" EMM.
  */
 typedef struct apple_protect_pager {
  * the "apple protect" EMM.
  */
 typedef struct apple_protect_pager {

-       struct ipc_object_header        pager_header;   /* fake ip_kotype() */
-       memory_object_pager_ops_t pager_ops; /* == &apple_protect_pager_ops */
-       queue_chain_t           pager_queue;    /* next & prev pagers */
-       unsigned int            ref_count;      /* reference count */
-       boolean_t               is_ready;       /* is this pager ready ? */
-       boolean_t               is_mapped;      /* is this mem_obj mapped ? */
-       memory_object_control_t pager_control;  /* mem object control handle */
-       vm_object_t             backing_object; /* VM obj w/ encrypted data */
-       struct pager_crypt_info crypt;
+       /* mandatory generic header */
+       struct memory_object ap_pgr_hdr;
+
+       /* pager-specific data */
+       queue_chain_t           pager_queue;    /* next & prev pagers */
+       struct os_refcnt        ref_count;      /* reference count */
+       boolean_t               is_ready;       /* is this pager ready ? */
+       boolean_t               is_mapped;      /* is this mem_obj mapped ? */
+       vm_object_t             backing_object; /* VM obj w/ encrypted data */
+       vm_object_offset_t      backing_offset;
+       vm_object_offset_t      crypto_backing_offset; /* for key... */
+       vm_object_offset_t      crypto_start;
+       vm_object_offset_t      crypto_end;
+       struct pager_crypt_info *crypt_info;

 } *apple_protect_pager_t;
 } *apple_protect_pager_t;

-#define        APPLE_PROTECT_PAGER_NULL        ((apple_protect_pager_t) NULL)
-#define pager_ikot pager_header.io_bits
+#define APPLE_PROTECT_PAGER_NULL        ((apple_protect_pager_t) NULL)

 
 /*
  * List of memory objects managed by this EMM.
  * The list is protected by the "apple_protect_pager_lock" lock.
  */
 
 /*
  * List of memory objects managed by this EMM.
  * The list is protected by the "apple_protect_pager_lock" lock.
  */

-int apple_protect_pager_count = 0;             /* number of pagers */
-int apple_protect_pager_count_mapped = 0;      /* number of unmapped pagers */
+int apple_protect_pager_count = 0;              /* number of pagers */
+int apple_protect_pager_count_mapped = 0;       /* number of unmapped pagers */

 queue_head_t apple_protect_pager_queue;
 queue_head_t apple_protect_pager_queue;

-decl_lck_mtx_data(,apple_protect_pager_lock)
+decl_lck_mtx_data(, apple_protect_pager_lock);

 
 /*
  * Maximum number of unmapped pagers we're willing to keep around.
  */
 
 /*
  * Maximum number of unmapped pagers we're willing to keep around.
  */

-int apple_protect_pager_cache_limit = 10;
+int apple_protect_pager_cache_limit = 20;

 
 /*
  * Statistics & counters.
 
 /*
  * Statistics & counters.


@@ -175,33 +183,38 @@ int apple_protect_pager_num_trim_max = 0;
 int apple_protect_pager_num_trim_total = 0;
 
 
 int apple_protect_pager_num_trim_total = 0;
 
 

-lck_grp_t              apple_protect_pager_lck_grp;
-lck_grp_attr_t apple_protect_pager_lck_grp_attr;
-lck_attr_t             apple_protect_pager_lck_attr;
+lck_grp_t               apple_protect_pager_lck_grp;
+lck_grp_attr_t  apple_protect_pager_lck_grp_attr;
+lck_attr_t              apple_protect_pager_lck_attr;

 
 
 /* internal prototypes */
 
 
 /* internal prototypes */

-apple_protect_pager_t apple_protect_pager_create(vm_object_t backing_object,
-                                                struct pager_crypt_info *crypt_info);
+apple_protect_pager_t apple_protect_pager_create(
+       vm_object_t backing_object,
+       vm_object_offset_t backing_offset,
+       vm_object_offset_t crypto_backing_offset,
+       struct pager_crypt_info *crypt_info,
+       vm_object_offset_t crypto_start,
+       vm_object_offset_t crypto_end);

 apple_protect_pager_t apple_protect_pager_lookup(memory_object_t mem_obj);
 void apple_protect_pager_dequeue(apple_protect_pager_t pager);
 void apple_protect_pager_deallocate_internal(apple_protect_pager_t pager,
 apple_protect_pager_t apple_protect_pager_lookup(memory_object_t mem_obj);
 void apple_protect_pager_dequeue(apple_protect_pager_t pager);
 void apple_protect_pager_deallocate_internal(apple_protect_pager_t pager,

-                                            boolean_t locked);
+    boolean_t locked);

 void apple_protect_pager_terminate_internal(apple_protect_pager_t pager);
 void apple_protect_pager_trim(void);
 
 
 #if DEBUG
 int apple_protect_pagerdebug = 0;
 void apple_protect_pager_terminate_internal(apple_protect_pager_t pager);
 void apple_protect_pager_trim(void);
 
 
 #if DEBUG
 int apple_protect_pagerdebug = 0;

-#define PAGER_ALL              0xffffffff
-#define        PAGER_INIT              0x00000001
-#define        PAGER_PAGEIN            0x00000002
-
-#define PAGER_DEBUG(LEVEL, A)                                          \
-       MACRO_BEGIN                                                     \
-       if ((apple_protect_pagerdebug & LEVEL)==LEVEL) {                \
-               printf A;                                               \
-       }                                                               \
+#define PAGER_ALL               0xffffffff
+#define PAGER_INIT              0x00000001
+#define PAGER_PAGEIN            0x00000002
+
+#define PAGER_DEBUG(LEVEL, A)                                           \
+       MACRO_BEGIN                                                     \
+       if ((apple_protect_pagerdebug & LEVEL)==LEVEL) {                \
+               printf A;                                               \
+       }                                                               \

        MACRO_END
 #else
 #define PAGER_DEBUG(LEVEL, A)
        MACRO_END
 #else
 #define PAGER_DEBUG(LEVEL, A)


@@ -225,29 +238,30 @@ apple_protect_pager_bootstrap(void)
  */
 kern_return_t
 apple_protect_pager_init(
  */
 kern_return_t
 apple_protect_pager_init(

-       memory_object_t         mem_obj, 
-       memory_object_control_t control, 
+       memory_object_t         mem_obj,
+       memory_object_control_t control,

 #if !DEBUG
        __unused
 #endif
        memory_object_cluster_size_t pg_size)
 {
 #if !DEBUG
        __unused
 #endif
        memory_object_cluster_size_t pg_size)
 {

-       apple_protect_pager_t   pager;
-       kern_return_t           kr;
+       apple_protect_pager_t   pager;
+       kern_return_t           kr;

        memory_object_attr_info_data_t  attributes;
 
        PAGER_DEBUG(PAGER_ALL,
        memory_object_attr_info_data_t  attributes;
 
        PAGER_DEBUG(PAGER_ALL,

-                   ("apple_protect_pager_init: %p, %p, %x\n",
-                    mem_obj, control, pg_size));
+           ("apple_protect_pager_init: %p, %p, %x\n",
+           mem_obj, control, pg_size));

 
 

-       if (control == MEMORY_OBJECT_CONTROL_NULL)
+       if (control == MEMORY_OBJECT_CONTROL_NULL) {

                return KERN_INVALID_ARGUMENT;
                return KERN_INVALID_ARGUMENT;

+       }

 
        pager = apple_protect_pager_lookup(mem_obj);
 
        memory_object_control_reference(control);
 
 
        pager = apple_protect_pager_lookup(mem_obj);
 
        memory_object_control_reference(control);
 

-       pager->pager_control = control;
+       pager->ap_pgr_hdr.mo_control = control;

 
        attributes.copy_strategy = MEMORY_OBJECT_COPY_DELAY;
        /* attributes.cluster_size = (1 << (CLUSTER_SHIFT + PAGE_SHIFT));*/
 
        attributes.copy_strategy = MEMORY_OBJECT_COPY_DELAY;
        /* attributes.cluster_size = (1 << (CLUSTER_SHIFT + PAGE_SHIFT));*/


@@ -256,13 +270,20 @@ apple_protect_pager_init(
        attributes.temporary = TRUE;
 
        kr = memory_object_change_attributes(
        attributes.temporary = TRUE;
 
        kr = memory_object_change_attributes(

-                                       control,
-                                       MEMORY_OBJECT_ATTRIBUTE_INFO,
-                                       (memory_object_info_t) &attributes,
-                                       MEMORY_OBJECT_ATTR_INFO_COUNT);
-       if (kr != KERN_SUCCESS)
+               control,
+               MEMORY_OBJECT_ATTRIBUTE_INFO,
+               (memory_object_info_t) &attributes,
+               MEMORY_OBJECT_ATTR_INFO_COUNT);
+       if (kr != KERN_SUCCESS) {

                panic("apple_protect_pager_init: "
                panic("apple_protect_pager_init: "

-                     "memory_object_change_attributes() failed");
+                   "memory_object_change_attributes() failed");
+       }
+
+#if CONFIG_SECLUDED_MEMORY
+       if (secluded_for_filecache) {
+               memory_object_mark_eligible_for_secluded(control, TRUE);
+       }
+#endif /* CONFIG_SECLUDED_MEMORY */

 
        return KERN_SUCCESS;
 }
 
        return KERN_SUCCESS;
 }


@@ -277,14 +298,14 @@ apple_protect_pager_init(
  */
 kern_return_t
 apple_protect_pager_data_return(
  */
 kern_return_t
 apple_protect_pager_data_return(

-        __unused memory_object_t       mem_obj,
-        __unused memory_object_offset_t        offset,
-        __unused memory_object_cluster_size_t          data_cnt,
-        __unused memory_object_offset_t        *resid_offset,
-       __unused int                    *io_error,
-       __unused boolean_t              dirty,
-       __unused boolean_t              kernel_copy,
-       __unused int                    upl_flags)  
+       __unused memory_object_t        mem_obj,
+       __unused memory_object_offset_t offset,
+       __unused memory_object_cluster_size_t           data_cnt,
+       __unused memory_object_offset_t *resid_offset,
+       __unused int                    *io_error,
+       __unused boolean_t              dirty,
+       __unused boolean_t              kernel_copy,
+       __unused int                    upl_flags)

 {
        panic("apple_protect_pager_data_return: should never get called");
        return KERN_FAILURE;
 {
        panic("apple_protect_pager_data_return: should never get called");
        return KERN_FAILURE;


@@ -292,9 +313,9 @@ apple_protect_pager_data_return(
 
 kern_return_t
 apple_protect_pager_data_initialize(
 
 kern_return_t
 apple_protect_pager_data_initialize(

-       __unused memory_object_t        mem_obj,
-       __unused memory_object_offset_t offset,
-       __unused memory_object_cluster_size_t           data_cnt)
+       __unused memory_object_t        mem_obj,
+       __unused memory_object_offset_t offset,
+       __unused memory_object_cluster_size_t           data_cnt)

 {
        panic("apple_protect_pager_data_initialize: should never get called");
        return KERN_FAILURE;
 {
        panic("apple_protect_pager_data_initialize: should never get called");
        return KERN_FAILURE;


@@ -302,10 +323,10 @@ apple_protect_pager_data_initialize(
 
 kern_return_t
 apple_protect_pager_data_unlock(
 
 kern_return_t
 apple_protect_pager_data_unlock(

-       __unused memory_object_t        mem_obj,
-       __unused memory_object_offset_t offset,
-       __unused memory_object_size_t           size,
-       __unused vm_prot_t              desired_access)
+       __unused memory_object_t        mem_obj,
+       __unused memory_object_offset_t offset,
+       __unused memory_object_size_t           size,
+       __unused vm_prot_t              desired_access)

 {
        return KERN_FAILURE;
 }
 {
        return KERN_FAILURE;
 }


@@ -315,45 +336,45 @@ apple_protect_pager_data_unlock(
  *
  * Handles page-in requests from VM.
  */
  *
  * Handles page-in requests from VM.
  */

-kern_return_t  
+int apple_protect_pager_data_request_debug = 0;
+kern_return_t

 apple_protect_pager_data_request(
 apple_protect_pager_data_request(

-       memory_object_t         mem_obj,
-       memory_object_offset_t  offset,
-       memory_object_cluster_size_t            length,
+       memory_object_t         mem_obj,
+       memory_object_offset_t  offset,
+       memory_object_cluster_size_t            length,

 #if !DEBUG
        __unused
 #endif
 #if !DEBUG
        __unused
 #endif

-       vm_prot_t               protection_required,
+       vm_prot_t               protection_required,

        memory_object_fault_info_t mo_fault_info)
 {
        memory_object_fault_info_t mo_fault_info)
 {

-       apple_protect_pager_t   pager;
-       memory_object_control_t mo_control;
-       upl_t                   upl;
-       int                     upl_flags;
-       upl_size_t              upl_size;
-       upl_page_info_t         *upl_pl;
-       unsigned int            pl_count;
-       vm_object_t             src_object, dst_object;
-       kern_return_t           kr, retval;
-       vm_map_offset_t         kernel_mapping;
-       vm_offset_t             src_vaddr, dst_vaddr;
-       vm_offset_t             cur_offset;
-       vm_map_entry_t          map_entry;
-       kern_return_t           error_code;
-       vm_prot_t               prot;
-       vm_page_t               src_page, top_page;
-       int                     interruptible;
-       struct vm_object_fault_info     fault_info;
-       int                     ret;
+       apple_protect_pager_t   pager;
+       memory_object_control_t mo_control;
+       upl_t                   upl;
+       int                     upl_flags;
+       upl_size_t              upl_size;
+       upl_page_info_t         *upl_pl;
+       unsigned int            pl_count;
+       vm_object_t             src_top_object, src_page_object, dst_object;
+       kern_return_t           kr, retval;
+       vm_offset_t             src_vaddr, dst_vaddr;
+       vm_offset_t             cur_offset;
+       vm_offset_t             offset_in_page;
+       kern_return_t           error_code;
+       vm_prot_t               prot;
+       vm_page_t               src_page, top_page;
+       int                     interruptible;
+       struct vm_object_fault_info     fault_info;
+       int                     ret;

 
        PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_data_request: %p, %llx, %x, %x\n", mem_obj, offset, length, protection_required));
 
        retval = KERN_SUCCESS;
 
        PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_data_request: %p, %llx, %x, %x\n", mem_obj, offset, length, protection_required));
 
        retval = KERN_SUCCESS;

-       src_object = VM_OBJECT_NULL;
-       kernel_mapping = 0;
+       src_top_object = VM_OBJECT_NULL;
+       src_page_object = VM_OBJECT_NULL;

        upl = NULL;
        upl_pl = NULL;
        upl = NULL;
        upl_pl = NULL;

-       fault_info = *((struct vm_object_fault_info *) mo_fault_info);
+       fault_info = *((struct vm_object_fault_info *)(uintptr_t)mo_fault_info);

        fault_info.stealth = TRUE;
        fault_info.io_sync = FALSE;
        fault_info.mark_zf_absent = FALSE;
        fault_info.stealth = TRUE;
        fault_info.io_sync = FALSE;
        fault_info.mark_zf_absent = FALSE;


@@ -362,26 +383,29 @@ apple_protect_pager_data_request(
 
        pager = apple_protect_pager_lookup(mem_obj);
        assert(pager->is_ready);
 
        pager = apple_protect_pager_lookup(mem_obj);
        assert(pager->is_ready);

-       assert(pager->ref_count > 1); /* pager is alive and mapped */
+       assert(os_ref_get_count(&pager->ref_count) > 1); /* pager is alive and mapped */

 
        PAGER_DEBUG(PAGER_PAGEIN, ("apple_protect_pager_data_request: %p, %llx, %x, %x, pager %p\n", mem_obj, offset, length, protection_required, pager));
 
 
        PAGER_DEBUG(PAGER_PAGEIN, ("apple_protect_pager_data_request: %p, %llx, %x, %x, pager %p\n", mem_obj, offset, length, protection_required, pager));
 

+       fault_info.lo_offset += pager->backing_offset;
+       fault_info.hi_offset += pager->backing_offset;
+

        /*
         * Gather in a UPL all the VM pages requested by VM.
         */
        /*
         * Gather in a UPL all the VM pages requested by VM.
         */

-       mo_control = pager->pager_control;
+       mo_control = pager->ap_pgr_hdr.mo_control;

 
        upl_size = length;
        upl_flags =
 
        upl_size = length;
        upl_flags =

-               UPL_RET_ONLY_ABSENT |
-               UPL_SET_LITE |
-               UPL_NO_SYNC |
-               UPL_CLEAN_IN_PLACE |    /* triggers UPL_CLEAR_DIRTY */
-               UPL_SET_INTERNAL;
+           UPL_RET_ONLY_ABSENT |
+           UPL_SET_LITE |
+           UPL_NO_SYNC |
+           UPL_CLEAN_IN_PLACE |        /* triggers UPL_CLEAR_DIRTY */
+           UPL_SET_INTERNAL;

        pl_count = 0;
        kr = memory_object_upl_request(mo_control,
        pl_count = 0;
        kr = memory_object_upl_request(mo_control,

-                                      offset, upl_size,
-                                      &upl, NULL, NULL, upl_flags);
+           offset, upl_size,
+           &upl, NULL, NULL, upl_flags, VM_KERN_MEMORY_SECURITY);

        if (kr != KERN_SUCCESS) {
                retval = kr;
                goto done;
        if (kr != KERN_SUCCESS) {
                retval = kr;
                goto done;


@@ -389,38 +413,14 @@ apple_protect_pager_data_request(
        dst_object = mo_control->moc_object;
        assert(dst_object != VM_OBJECT_NULL);
 
        dst_object = mo_control->moc_object;
        assert(dst_object != VM_OBJECT_NULL);
 

-
-       /*
-        * Reserve 2 virtual pages in the kernel address space to map each
-        * source and destination physical pages when it's their turn to
-        * be processed.
-        */
-       vm_object_reference(kernel_object);     /* ref. for mapping */
-       kr = vm_map_find_space(kernel_map,
-                              &kernel_mapping,
-                              2 * PAGE_SIZE_64,
-                              0,
-                              0,
-                              &map_entry);
-       if (kr != KERN_SUCCESS) {
-               vm_object_deallocate(kernel_object);
-               retval = kr;
-               goto done;
-       }
-       map_entry->object.vm_object = kernel_object;
-       map_entry->offset = kernel_mapping;
-       vm_map_unlock(kernel_map);
-       src_vaddr = CAST_DOWN(vm_offset_t, kernel_mapping);
-       dst_vaddr = CAST_DOWN(vm_offset_t, kernel_mapping + PAGE_SIZE_64);
-

        /*
        /*

-        * We'll map the encrypted data in the kernel address space from the 
+        * We'll map the encrypted data in the kernel address space from the

         * backing VM object (itself backed by the encrypted file via
         * the vnode pager).
         */
         * backing VM object (itself backed by the encrypted file via
         * the vnode pager).
         */

-       src_object = pager->backing_object;
-       assert(src_object != VM_OBJECT_NULL);
-       vm_object_reference(src_object); /* to keep the source object alive */
+       src_top_object = pager->backing_object;
+       assert(src_top_object != VM_OBJECT_NULL);
+       vm_object_reference(src_top_object); /* keep the source object alive */

 
        /*
         * Fill in the contents of the pages requested by VM.
 
        /*
         * Fill in the contents of the pages requested by VM.


@@ -428,8 +428,8 @@ apple_protect_pager_data_request(
        upl_pl = UPL_GET_INTERNAL_PAGE_LIST(upl);
        pl_count = length / PAGE_SIZE;
        for (cur_offset = 0;
        upl_pl = UPL_GET_INTERNAL_PAGE_LIST(upl);
        pl_count = length / PAGE_SIZE;
        for (cur_offset = 0;

-            retval == KERN_SUCCESS && cur_offset < length;
-            cur_offset += PAGE_SIZE) {
+           retval == KERN_SUCCESS && cur_offset < length;
+           cur_offset += PAGE_SIZE) {

                ppnum_t dst_pnum;
 
                if (!upl_page_present(upl_pl, (int)(cur_offset / PAGE_SIZE))) {
                ppnum_t dst_pnum;
 
                if (!upl_page_present(upl_pl, (int)(cur_offset / PAGE_SIZE))) {


@@ -440,25 +440,27 @@ apple_protect_pager_data_request(
                /*
                 * Map the source (encrypted) page in the kernel's
                 * virtual address space.
                /*
                 * Map the source (encrypted) page in the kernel's
                 * virtual address space.

-                * We already hold a reference on the src_object.
+                * We already hold a reference on the src_top_object.

                 */
                 */

-       retry_src_fault:
-               vm_object_lock(src_object);
-               vm_object_paging_begin(src_object);
+retry_src_fault:
+               vm_object_lock(src_top_object);
+               vm_object_paging_begin(src_top_object);

                error_code = 0;
                prot = VM_PROT_READ;
                error_code = 0;
                prot = VM_PROT_READ;

-               kr = vm_fault_page(src_object,
-                                  offset + cur_offset,
-                                  VM_PROT_READ,
-                                  FALSE,
-                                  &prot,
-                                  &src_page,
-                                  &top_page,
-                                  NULL,
-                                  &error_code,
-                                  FALSE,
-                                  FALSE,
-                                  &fault_info);
+               src_page = VM_PAGE_NULL;
+               kr = vm_fault_page(src_top_object,
+                   pager->backing_offset + offset + cur_offset,
+                   VM_PROT_READ,
+                   FALSE,
+                   FALSE,                /* src_page not looked up */
+                   &prot,
+                   &src_page,
+                   &top_page,
+                   NULL,
+                   &error_code,
+                   FALSE,
+                   FALSE,
+                   &fault_info);

                switch (kr) {
                case VM_FAULT_SUCCESS:
                        break;
                switch (kr) {
                case VM_FAULT_SUCCESS:
                        break;


@@ -468,15 +470,15 @@ apple_protect_pager_data_request(
                        if (vm_page_wait(interruptible)) {
                                goto retry_src_fault;
                        }
                        if (vm_page_wait(interruptible)) {
                                goto retry_src_fault;
                        }

-                       /* fall thru */
+               /* fall thru */

                case VM_FAULT_INTERRUPTED:
                        retval = MACH_SEND_INTERRUPTED;
                        goto done;
                case VM_FAULT_SUCCESS_NO_VM_PAGE:
                        /* success but no VM page: fail */
                case VM_FAULT_INTERRUPTED:
                        retval = MACH_SEND_INTERRUPTED;
                        goto done;
                case VM_FAULT_SUCCESS_NO_VM_PAGE:
                        /* success but no VM page: fail */

-                       vm_object_paging_end(src_object);
-                       vm_object_unlock(src_object);
-                       /*FALLTHROUGH*/
+                       vm_object_paging_end(src_top_object);
+                       vm_object_unlock(src_top_object);
+               /*FALLTHROUGH*/

                case VM_FAULT_MEMORY_ERROR:
                        /* the page is not there ! */
                        if (error_code) {
                case VM_FAULT_MEMORY_ERROR:
                        /* the page is not there ! */
                        if (error_code) {


@@ -487,105 +489,186 @@ apple_protect_pager_data_request(
                        goto done;
                default:
                        panic("apple_protect_pager_data_request: "
                        goto done;
                default:
                        panic("apple_protect_pager_data_request: "

-                             "vm_fault_page() unexpected error 0x%x\n",
-                             kr);
+                           "vm_fault_page() unexpected error 0x%x\n",
+                           kr);

                }
                assert(src_page != VM_PAGE_NULL);
                }
                assert(src_page != VM_PAGE_NULL);

-               assert(src_page->busy);
+               assert(src_page->vmp_busy);

 
 

-               if (!src_page->active &&
-                   !src_page->inactive &&
-                   !src_page->throttled) {
+               if (src_page->vmp_q_state != VM_PAGE_ON_SPECULATIVE_Q) {

                        vm_page_lockspin_queues();
                        vm_page_lockspin_queues();

-                       if (!src_page->active &&
-                           !src_page->inactive &&
-                           !src_page->throttled) {
-                               vm_page_deactivate(src_page);
+
+                       if (src_page->vmp_q_state != VM_PAGE_ON_SPECULATIVE_Q) {
+                               vm_page_speculate(src_page, FALSE);

                        }
                        vm_page_unlock_queues();
                }
                        }
                        vm_page_unlock_queues();
                }

-               
-               /*
-                * Establish an explicit mapping of the source
-                * physical page.
-                */
-               pmap_enter(kernel_pmap,
-                          kernel_mapping,
-                          src_page->phys_page,
-                          VM_PROT_READ,
-                          VM_PROT_NONE,
-                          0,
-                          TRUE);
+

                /*
                /*

-                * Establish an explicit pmap mapping of the destination
-                * physical page.
-                * We can't do a regular VM mapping because the VM page
-                * is "busy".
+                * Establish pointers to the source
+                * and destination physical pages.

                 */
                dst_pnum = (ppnum_t)
                 */
                dst_pnum = (ppnum_t)

-                       upl_phys_page(upl_pl, (int)(cur_offset / PAGE_SIZE));
+                   upl_phys_page(upl_pl, (int)(cur_offset / PAGE_SIZE));

                assert(dst_pnum != 0);
                assert(dst_pnum != 0);

-               pmap_enter(kernel_pmap,
-                          kernel_mapping + PAGE_SIZE_64,
-                          dst_pnum,
-                          VM_PROT_READ | VM_PROT_WRITE,
-                          VM_PROT_NONE,
-                          0,
-                          TRUE);
+
+               src_vaddr = (vm_map_offset_t)
+                   phystokv((pmap_paddr_t)VM_PAGE_GET_PHYS_PAGE(src_page)
+                       << PAGE_SHIFT);
+               dst_vaddr = (vm_map_offset_t)
+                   phystokv((pmap_paddr_t)dst_pnum << PAGE_SHIFT);
+
+               src_page_object = VM_PAGE_OBJECT(src_page);
+
+               /*
+                * Validate the original page...
+                */
+               if (src_page_object->code_signed) {
+                       vm_page_validate_cs_mapped(
+                               src_page,
+                               (const void *) src_vaddr);
+               }
+               /*
+                * ... and transfer the results to the destination page.
+                */
+               UPL_SET_CS_VALIDATED(upl_pl, cur_offset / PAGE_SIZE,
+                   src_page->vmp_cs_validated);
+               UPL_SET_CS_TAINTED(upl_pl, cur_offset / PAGE_SIZE,
+                   src_page->vmp_cs_tainted);
+               UPL_SET_CS_NX(upl_pl, cur_offset / PAGE_SIZE,
+                   src_page->vmp_cs_nx);
+
+               /*
+                * page_decrypt() might access a mapped file, so let's release
+                * the object lock for the source page to avoid a potential
+                * deadlock.  The source page is kept busy and we have a
+                * "paging_in_progress" reference on its object, so it's safe
+                * to unlock the object here.
+                */
+               assert(src_page->vmp_busy);
+               assert(src_page_object->paging_in_progress > 0);
+               vm_object_unlock(src_page_object);

 
                /*
                 * Decrypt the encrypted contents of the source page
                 * into the destination page.
                 */
 
                /*
                 * Decrypt the encrypted contents of the source page
                 * into the destination page.
                 */

-               ret = pager->crypt.page_decrypt((const void *) src_vaddr,
-                                               (void *) dst_vaddr,
-                                               offset+cur_offset,
-                                               pager->crypt.crypt_ops);
+               for (offset_in_page = 0;
+                   offset_in_page < PAGE_SIZE;
+                   offset_in_page += 4096) {
+                       if (offset + cur_offset + offset_in_page <
+                           pager->crypto_start ||
+                           offset + cur_offset + offset_in_page >=
+                           pager->crypto_end) {
+                               /* not encrypted: just copy */
+                               bcopy((const char *)(src_vaddr +
+                                   offset_in_page),
+                                   (char *)(dst_vaddr + offset_in_page),
+                                   4096);
+
+                               if (apple_protect_pager_data_request_debug) {
+                                       printf("apple_protect_data_request"
+                                           "(%p,0x%llx+0x%llx+0x%04llx): "
+                                           "out of crypto range "
+                                           "[0x%llx:0x%llx]: "
+                                           "COPY [0x%016llx 0x%016llx] "
+                                           "code_signed=%d "
+                                           "cs_validated=%d "
+                                           "cs_tainted=%d "
+                                           "cs_nx=%d\n",
+                                           pager,
+                                           offset,
+                                           (uint64_t) cur_offset,
+                                           (uint64_t) offset_in_page,
+                                           pager->crypto_start,
+                                           pager->crypto_end,
+                                           *(uint64_t *)(dst_vaddr +
+                                           offset_in_page),
+                                           *(uint64_t *)(dst_vaddr +
+                                           offset_in_page + 8),
+                                           src_page_object->code_signed,
+                                           src_page->vmp_cs_validated,
+                                           src_page->vmp_cs_tainted,
+                                           src_page->vmp_cs_nx);
+                               }
+                               ret = 0;
+                               continue;
+                       }
+                       ret = pager->crypt_info->page_decrypt(
+                               (const void *)(src_vaddr + offset_in_page),
+                               (void *)(dst_vaddr + offset_in_page),
+                               ((pager->crypto_backing_offset -
+                               pager->crypto_start) +   /* XXX ? */
+                               offset +
+                               cur_offset +
+                               offset_in_page),
+                               pager->crypt_info->crypt_ops);
+
+                       if (apple_protect_pager_data_request_debug) {
+                               printf("apple_protect_data_request"
+                                   "(%p,0x%llx+0x%llx+0x%04llx): "
+                                   "in crypto range [0x%llx:0x%llx]: "
+                                   "DECRYPT offset 0x%llx="
+                                   "(0x%llx-0x%llx+0x%llx+0x%llx+0x%04llx)"
+                                   "[0x%016llx 0x%016llx] "
+                                   "code_signed=%d "
+                                   "cs_validated=%d "
+                                   "cs_tainted=%d "
+                                   "cs_nx=%d "
+                                   "ret=0x%x\n",
+                                   pager,
+                                   offset,
+                                   (uint64_t) cur_offset,
+                                   (uint64_t) offset_in_page,
+                                   pager->crypto_start, pager->crypto_end,
+                                   ((pager->crypto_backing_offset -
+                                   pager->crypto_start) +
+                                   offset +
+                                   cur_offset +
+                                   offset_in_page),
+                                   pager->crypto_backing_offset,
+                                   pager->crypto_start,
+                                   offset,
+                                   (uint64_t) cur_offset,
+                                   (uint64_t) offset_in_page,
+                                   *(uint64_t *)(dst_vaddr + offset_in_page),
+                                   *(uint64_t *)(dst_vaddr + offset_in_page + 8),
+                                   src_page_object->code_signed,
+                                   src_page->vmp_cs_validated,
+                                   src_page->vmp_cs_tainted,
+                                   src_page->vmp_cs_nx,
+                                   ret);
+                       }
+                       if (ret) {
+                               break;
+                       }
+               }

                if (ret) {
                        /*
                         * Decryption failed.  Abort the fault.
                         */
                        retval = KERN_ABORTED;
                if (ret) {
                        /*
                         * Decryption failed.  Abort the fault.
                         */
                        retval = KERN_ABORTED;

-               } else {
-                       /*
-                        * Validate the original page...
-                        */
-                       if (src_page->object->code_signed) {
-                               vm_page_validate_cs_mapped(
-                                       src_page,
-                                       (const void *) src_vaddr);
-                       }
-                       /*
-                        * ... and transfer the results to the destination page.
-                        */
-                       UPL_SET_CS_VALIDATED(upl_pl, cur_offset / PAGE_SIZE,
-                                            src_page->cs_validated);
-                       UPL_SET_CS_TAINTED(upl_pl, cur_offset / PAGE_SIZE,
-                                          src_page->cs_tainted);

                }
                }

-               
-               /*
-                * Remove the pmap mapping of the source and destination pages
-                * in the kernel.
-                */
-               pmap_remove(kernel_pmap,
-                           (addr64_t) kernel_mapping,
-                           (addr64_t) (kernel_mapping + (2 * PAGE_SIZE_64)));
+
+               assert(VM_PAGE_OBJECT(src_page) == src_page_object);
+               assert(src_page->vmp_busy);
+               assert(src_page_object->paging_in_progress > 0);
+               vm_object_lock(src_page_object);

 
                /*
                 * Cleanup the result of vm_fault_page() of the source page.
                 */
                PAGE_WAKEUP_DONE(src_page);
 
                /*
                 * Cleanup the result of vm_fault_page() of the source page.
                 */
                PAGE_WAKEUP_DONE(src_page);

-               vm_object_paging_end(src_page->object);
-               vm_object_unlock(src_page->object);
-               if (top_page != VM_PAGE_NULL) {
-                       vm_object_t top_object;
+               src_page = VM_PAGE_NULL;
+               vm_object_paging_end(src_page_object);
+               vm_object_unlock(src_page_object);

 
 

-                       top_object = top_page->object;
-                       vm_object_lock(top_object);
+               if (top_page != VM_PAGE_NULL) {
+                       assert(VM_PAGE_OBJECT(top_page) == src_top_object);
+                       vm_object_lock(src_top_object);

                        VM_PAGE_FREE(top_page);
                        VM_PAGE_FREE(top_page);

-                       vm_object_paging_end(top_object);
-                       vm_object_unlock(top_object);
+                       vm_object_paging_end(src_top_object);
+                       vm_object_unlock(src_top_object);

                }
        }
 
                }
        }
 


@@ -606,7 +689,7 @@ done:
                if (retval != KERN_SUCCESS) {
                        upl_abort(upl, 0);
                        if (retval == KERN_ABORTED) {
                if (retval != KERN_SUCCESS) {
                        upl_abort(upl, 0);
                        if (retval == KERN_ABORTED) {

-                               wait_result_t   wait_result;
+                               wait_result_t   wait_result;

 
                                /*
                                 * We aborted the fault and did not provide
 
                                /*
                                 * We aborted the fault and did not provide


@@ -626,7 +709,7 @@ done:
                                wait_result = assert_wait_timeout(
                                        (event_t) apple_protect_pager_data_request,
                                        THREAD_UNINT,
                                wait_result = assert_wait_timeout(
                                        (event_t) apple_protect_pager_data_request,
                                        THREAD_UNINT,

-                                       10000,  /* 10ms */
+                                       10000,  /* 10ms */

                                        NSEC_PER_USEC);
                                assert(wait_result == THREAD_WAITING);
                                wait_result = thread_block(THREAD_CONTINUE_NULL);
                                        NSEC_PER_USEC);
                                assert(wait_result == THREAD_WAITING);
                                wait_result = thread_block(THREAD_CONTINUE_NULL);


@@ -634,30 +717,18 @@ done:
                        }
                } else {
                        boolean_t empty;
                        }
                } else {
                        boolean_t empty;

-                       upl_commit_range(upl, 0, upl->size, 
-                                        UPL_COMMIT_CS_VALIDATED,
-                                        upl_pl, pl_count, &empty);
+                       upl_commit_range(upl, 0, upl->size,
+                           UPL_COMMIT_CS_VALIDATED | UPL_COMMIT_WRITTEN_BY_KERNEL,
+                           upl_pl, pl_count, &empty);

                }
 
                /* and deallocate the UPL */
                upl_deallocate(upl);
                upl = NULL;
        }
                }
 
                /* and deallocate the UPL */
                upl_deallocate(upl);
                upl = NULL;
        }

-       if (kernel_mapping != 0) {
-               /* clean up the mapping of the source and destination pages */
-               kr = vm_map_remove(kernel_map,
-                                  kernel_mapping,
-                                  kernel_mapping + (2 * PAGE_SIZE_64),
-                                  VM_MAP_NO_FLAGS);
-               assert(kr == KERN_SUCCESS);
-               kernel_mapping = 0;
-               src_vaddr = 0;
-               dst_vaddr = 0;
+       if (src_top_object != VM_OBJECT_NULL) {
+               vm_object_deallocate(src_top_object);

        }
        }

-       if (src_object != VM_OBJECT_NULL) {
-               vm_object_deallocate(src_object);
-       }
-

        return retval;
 }
 
        return retval;
 }
 


@@ -670,15 +741,14 @@ done:
  */
 void
 apple_protect_pager_reference(
  */
 void
 apple_protect_pager_reference(

-       memory_object_t         mem_obj)
-{      
-       apple_protect_pager_t   pager;
+       memory_object_t         mem_obj)
+{
+       apple_protect_pager_t   pager;

 
        pager = apple_protect_pager_lookup(mem_obj);
 
        lck_mtx_lock(&apple_protect_pager_lock);
 
        pager = apple_protect_pager_lookup(mem_obj);
 
        lck_mtx_lock(&apple_protect_pager_lock);

-       assert(pager->ref_count > 0);
-       pager->ref_count++;
+       os_ref_retain_locked(&pager->ref_count);

        lck_mtx_unlock(&apple_protect_pager_lock);
 }
 
        lck_mtx_unlock(&apple_protect_pager_lock);
 }
 


@@ -697,12 +767,12 @@ apple_protect_pager_dequeue(
        assert(!pager->is_mapped);
 
        queue_remove(&apple_protect_pager_queue,
        assert(!pager->is_mapped);
 
        queue_remove(&apple_protect_pager_queue,

-                    pager,
-                    apple_protect_pager_t,
-                    pager_queue);
+           pager,
+           apple_protect_pager_t,
+           pager_queue);

        pager->pager_queue.next = NULL;
        pager->pager_queue.prev = NULL;
        pager->pager_queue.next = NULL;
        pager->pager_queue.prev = NULL;

-       
+

        apple_protect_pager_count--;
 }
 
        apple_protect_pager_count--;
 }
 


@@ -730,13 +800,19 @@ apple_protect_pager_terminate_internal(
                vm_object_deallocate(pager->backing_object);
                pager->backing_object = VM_OBJECT_NULL;
        }
                vm_object_deallocate(pager->backing_object);
                pager->backing_object = VM_OBJECT_NULL;
        }

-       
-       /* deallocate any crypt module data */
-       if(pager->crypt.crypt_end)
-               pager->crypt.crypt_end(pager->crypt.crypt_ops);
+
+       /* one less pager using this "pager_crypt_info" */
+#if CRYPT_INFO_DEBUG
+       printf("CRYPT_INFO %s: deallocate %p ref %d\n",
+           __FUNCTION__,
+           pager->crypt_info,
+           pager->crypt_info->crypt_refcnt);
+#endif /* CRYPT_INFO_DEBUG */
+       crypt_info_deallocate(pager->crypt_info);
+       pager->crypt_info = NULL;

 
        /* trigger the destruction of the memory object */
 
        /* trigger the destruction of the memory object */

-       memory_object_destroy(pager->pager_control, 0);
+       memory_object_destroy(pager->ap_pgr_hdr.mo_control, 0);

 }
 
 /*
 }
 
 /*


@@ -749,18 +825,18 @@ apple_protect_pager_terminate_internal(
  */
 void
 apple_protect_pager_deallocate_internal(
  */
 void
 apple_protect_pager_deallocate_internal(

-       apple_protect_pager_t   pager,
-       boolean_t               locked)
+       apple_protect_pager_t   pager,
+       boolean_t               locked)

 {
 {

-       boolean_t       needs_trimming;
-       int             count_unmapped;
+       boolean_t       needs_trimming;
+       int             count_unmapped;

 
 

-       if (! locked) {
+       if (!locked) {

                lck_mtx_lock(&apple_protect_pager_lock);
        }
 
                lck_mtx_lock(&apple_protect_pager_lock);
        }
 

-       count_unmapped = (apple_protect_pager_count - 
-                         apple_protect_pager_count_mapped);
+       count_unmapped = (apple_protect_pager_count -
+           apple_protect_pager_count_mapped);

        if (count_unmapped > apple_protect_pager_cache_limit) {
                /* we have too many unmapped pagers:  trim some */
                needs_trimming = TRUE;
        if (count_unmapped > apple_protect_pager_cache_limit) {
                /* we have too many unmapped pagers:  trim some */
                needs_trimming = TRUE;


@@ -769,9 +845,9 @@ apple_protect_pager_deallocate_internal(
        }
 
        /* drop a reference on this pager */
        }
 
        /* drop a reference on this pager */

-       pager->ref_count--;
+       os_ref_count_t ref_count = os_ref_release_locked(&pager->ref_count);

 
 

-       if (pager->ref_count == 1) {
+       if (ref_count == 1) {

                /*
                 * Only the "named" reference is left, which means that
                 * no one is really holding on to this pager anymore.
                /*
                 * Only the "named" reference is left, which means that
                 * no one is really holding on to this pager anymore.


@@ -781,18 +857,18 @@ apple_protect_pager_deallocate_internal(
                /* the pager is all ours: no need for the lock now */
                lck_mtx_unlock(&apple_protect_pager_lock);
                apple_protect_pager_terminate_internal(pager);
                /* the pager is all ours: no need for the lock now */
                lck_mtx_unlock(&apple_protect_pager_lock);
                apple_protect_pager_terminate_internal(pager);

-       } else if (pager->ref_count == 0) {
+       } else if (ref_count == 0) {

                /*
                 * Dropped the existence reference;  the memory object has
                 * been terminated.  Do some final cleanup and release the
                 * pager structure.
                 */
                lck_mtx_unlock(&apple_protect_pager_lock);
                /*
                 * Dropped the existence reference;  the memory object has
                 * been terminated.  Do some final cleanup and release the
                 * pager structure.
                 */
                lck_mtx_unlock(&apple_protect_pager_lock);

-               if (pager->pager_control != MEMORY_OBJECT_CONTROL_NULL) {
-                       memory_object_control_deallocate(pager->pager_control);
-                       pager->pager_control = MEMORY_OBJECT_CONTROL_NULL;
+               if (pager->ap_pgr_hdr.mo_control != MEMORY_OBJECT_CONTROL_NULL) {
+                       memory_object_control_deallocate(pager->ap_pgr_hdr.mo_control);
+                       pager->ap_pgr_hdr.mo_control = MEMORY_OBJECT_CONTROL_NULL;

                }
                }

-               kfree(pager, sizeof (*pager));
+               kfree(pager, sizeof(*pager));

                pager = APPLE_PROTECT_PAGER_NULL;
        } else {
                /* there are still plenty of references:  keep going... */
                pager = APPLE_PROTECT_PAGER_NULL;
        } else {
                /* there are still plenty of references:  keep going... */


@@ -813,9 +889,9 @@ apple_protect_pager_deallocate_internal(
  */
 void
 apple_protect_pager_deallocate(
  */
 void
 apple_protect_pager_deallocate(

-       memory_object_t         mem_obj)
+       memory_object_t         mem_obj)

 {
 {

-       apple_protect_pager_t   pager;
+       apple_protect_pager_t   pager;

 
        PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_deallocate: %p\n", mem_obj));
        pager = apple_protect_pager_lookup(mem_obj);
 
        PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_deallocate: %p\n", mem_obj));
        pager = apple_protect_pager_lookup(mem_obj);


@@ -830,7 +906,7 @@ apple_protect_pager_terminate(
 #if !DEBUG
        __unused
 #endif
 #if !DEBUG
        __unused
 #endif

-       memory_object_t mem_obj)
+       memory_object_t mem_obj)

 {
        PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_terminate: %p\n", mem_obj));
 
 {
        PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_terminate: %p\n", mem_obj));
 


@@ -842,21 +918,13 @@ apple_protect_pager_terminate(
  */
 kern_return_t
 apple_protect_pager_synchronize(
  */
 kern_return_t
 apple_protect_pager_synchronize(

-       memory_object_t         mem_obj,
-       memory_object_offset_t  offset,
-       memory_object_size_t            length,
-       __unused vm_sync_t              sync_flags)
+       __unused memory_object_t                mem_obj,
+       __unused memory_object_offset_t offset,
+       __unused memory_object_size_t           length,
+       __unused vm_sync_t              sync_flags)

 {
 {

-       apple_protect_pager_t   pager;
-
-       PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_synchronize: %p\n", mem_obj));
-
-       pager = apple_protect_pager_lookup(mem_obj);
-
-       memory_object_synchronize_completed(pager->pager_control,
-                                           offset, length);
-
-       return KERN_SUCCESS;
+       panic("apple_protect_pager_synchronize: memory_object_synchronize no longer supported\n");
+       return KERN_FAILURE;

 }
 
 /*
 }
 
 /*


@@ -869,10 +937,10 @@ apple_protect_pager_synchronize(
  */
 kern_return_t
 apple_protect_pager_map(
  */
 kern_return_t
 apple_protect_pager_map(

-       memory_object_t         mem_obj,
-       __unused vm_prot_t      prot)
+       memory_object_t         mem_obj,
+       __unused vm_prot_t      prot)

 {
 {

-       apple_protect_pager_t   pager;
+       apple_protect_pager_t   pager;

 
        PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_map: %p\n", mem_obj));
 
 
        PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_map: %p\n", mem_obj));
 


@@ -880,7 +948,7 @@ apple_protect_pager_map(
 
        lck_mtx_lock(&apple_protect_pager_lock);
        assert(pager->is_ready);
 
        lck_mtx_lock(&apple_protect_pager_lock);
        assert(pager->is_ready);

-       assert(pager->ref_count > 0); /* pager is alive */
+       assert(os_ref_get_count(&pager->ref_count) > 0); /* pager is alive */

        if (pager->is_mapped == FALSE) {
                /*
                 * First mapping of this pager:  take an extra reference
        if (pager->is_mapped == FALSE) {
                /*
                 * First mapping of this pager:  take an extra reference


@@ -888,7 +956,7 @@ apple_protect_pager_map(
                 * are removed.
                 */
                pager->is_mapped = TRUE;
                 * are removed.
                 */
                pager->is_mapped = TRUE;

-               pager->ref_count++;
+               os_ref_retain_locked(&pager->ref_count);

                apple_protect_pager_count_mapped++;
        }
        lck_mtx_unlock(&apple_protect_pager_lock);
                apple_protect_pager_count_mapped++;
        }
        lck_mtx_unlock(&apple_protect_pager_lock);


@@ -903,13 +971,13 @@ apple_protect_pager_map(
  */
 kern_return_t
 apple_protect_pager_last_unmap(
  */
 kern_return_t
 apple_protect_pager_last_unmap(

-       memory_object_t         mem_obj)
+       memory_object_t         mem_obj)

 {
 {

-       apple_protect_pager_t   pager;
-       int                     count_unmapped;
+       apple_protect_pager_t   pager;
+       int                     count_unmapped;

 
        PAGER_DEBUG(PAGER_ALL,
 
        PAGER_DEBUG(PAGER_ALL,

-                   ("apple_protect_pager_last_unmap: %p\n", mem_obj));
+           ("apple_protect_pager_last_unmap: %p\n", mem_obj));

 
        pager = apple_protect_pager_lookup(mem_obj);
 
 
        pager = apple_protect_pager_lookup(mem_obj);
 


@@ -921,7 +989,7 @@ apple_protect_pager_last_unmap(
                 */
                apple_protect_pager_count_mapped--;
                count_unmapped = (apple_protect_pager_count -
                 */
                apple_protect_pager_count_mapped--;
                count_unmapped = (apple_protect_pager_count -

-                                 apple_protect_pager_count_mapped);
+                   apple_protect_pager_count_mapped);

                if (count_unmapped > apple_protect_pager_count_unmapped_max) {
                        apple_protect_pager_count_unmapped_max = count_unmapped;
                }
                if (count_unmapped > apple_protect_pager_count_unmapped_max) {
                        apple_protect_pager_count_unmapped_max = count_unmapped;
                }


@@ -931,7 +999,7 @@ apple_protect_pager_last_unmap(
        } else {
                lck_mtx_unlock(&apple_protect_pager_lock);
        }
        } else {
                lck_mtx_unlock(&apple_protect_pager_lock);
        }

-       
+

        return KERN_SUCCESS;
 }
 
        return KERN_SUCCESS;
 }
 


@@ -941,26 +1009,31 @@ apple_protect_pager_last_unmap(
  */
 apple_protect_pager_t
 apple_protect_pager_lookup(
  */
 apple_protect_pager_t
 apple_protect_pager_lookup(

-       memory_object_t  mem_obj)
+       memory_object_t  mem_obj)

 {
 {

-       apple_protect_pager_t   pager;
+       apple_protect_pager_t   pager;

 
 

-       pager = (apple_protect_pager_t) mem_obj;
-       assert(pager->pager_ops == &apple_protect_pager_ops);
-       assert(pager->ref_count > 0);
+       assert(mem_obj->mo_pager_ops == &apple_protect_pager_ops);
+       pager = (apple_protect_pager_t)(uintptr_t) mem_obj;
+       assert(os_ref_get_count(&pager->ref_count) > 0);

        return pager;
 }
 
 apple_protect_pager_t
 apple_protect_pager_create(
        return pager;
 }
 
 apple_protect_pager_t
 apple_protect_pager_create(

-       vm_object_t     backing_object,
-       struct pager_crypt_info *crypt_info)
+       vm_object_t             backing_object,
+       vm_object_offset_t      backing_offset,
+       vm_object_offset_t      crypto_backing_offset,
+       struct pager_crypt_info *crypt_info,
+       vm_object_offset_t      crypto_start,
+       vm_object_offset_t      crypto_end)

 {
 {

-       apple_protect_pager_t   pager, pager2;
-       memory_object_control_t control;
-       kern_return_t           kr;
+       apple_protect_pager_t   pager, pager2;
+       memory_object_control_t control;
+       kern_return_t           kr;
+       struct pager_crypt_info *old_crypt_info;

 
 

-       pager = (apple_protect_pager_t) kalloc(sizeof (*pager));
+       pager = (apple_protect_pager_t) kalloc(sizeof(*pager));

        if (pager == APPLE_PROTECT_PAGER_NULL) {
                return APPLE_PROTECT_PAGER_NULL;
        }
        if (pager == APPLE_PROTECT_PAGER_NULL) {
                return APPLE_PROTECT_PAGER_NULL;
        }


@@ -972,36 +1045,92 @@ apple_protect_pager_create(
         * we reserve the first word in the object for a fake ip_kotype
         * setting - that will tell vm_map to use it as a memory object.
         */
         * we reserve the first word in the object for a fake ip_kotype
         * setting - that will tell vm_map to use it as a memory object.
         */

-       pager->pager_ops = &apple_protect_pager_ops;
-       pager->pager_ikot = IKOT_MEMORY_OBJECT;
+       pager->ap_pgr_hdr.mo_ikot = IKOT_MEMORY_OBJECT;
+       pager->ap_pgr_hdr.mo_pager_ops = &apple_protect_pager_ops;
+       pager->ap_pgr_hdr.mo_control = MEMORY_OBJECT_CONTROL_NULL;
+

        pager->is_ready = FALSE;/* not ready until it has a "name" */
        pager->is_ready = FALSE;/* not ready until it has a "name" */

-       pager->ref_count = 2;   /* existence + setup reference */
+       os_ref_init_count(&pager->ref_count, NULL, 2); /* existence reference (for the cache) and another for the caller */

        pager->is_mapped = FALSE;
        pager->is_mapped = FALSE;

-       pager->pager_control = MEMORY_OBJECT_CONTROL_NULL;

        pager->backing_object = backing_object;
        pager->backing_object = backing_object;

-       pager->crypt = *crypt_info;
-       
+       pager->backing_offset = backing_offset;
+       pager->crypto_backing_offset = crypto_backing_offset;
+       pager->crypto_start = crypto_start;
+       pager->crypto_end = crypto_end;
+       pager->crypt_info = crypt_info; /* allocated by caller */
+
+#if CRYPT_INFO_DEBUG
+       printf("CRYPT_INFO %s: crypt_info %p [%p,%p,%p,%d]\n",
+           __FUNCTION__,
+           crypt_info,
+           crypt_info->page_decrypt,
+           crypt_info->crypt_end,
+           crypt_info->crypt_ops,
+           crypt_info->crypt_refcnt);
+#endif /* CRYPT_INFO_DEBUG */
+

        vm_object_reference(backing_object);
 
        vm_object_reference(backing_object);
 

+       old_crypt_info = NULL;
+

        lck_mtx_lock(&apple_protect_pager_lock);
        /* see if anyone raced us to create a pager for the same object */
        queue_iterate(&apple_protect_pager_queue,
        lck_mtx_lock(&apple_protect_pager_lock);
        /* see if anyone raced us to create a pager for the same object */
        queue_iterate(&apple_protect_pager_queue,

-                     pager2,
-                     apple_protect_pager_t,
-                     pager_queue) {
-               if (pager2->backing_object == backing_object) {
+           pager2,
+           apple_protect_pager_t,
+           pager_queue) {
+               if ((pager2->crypt_info->page_decrypt !=
+                   crypt_info->page_decrypt) ||
+                   (pager2->crypt_info->crypt_end !=
+                   crypt_info->crypt_end) ||
+                   (pager2->crypt_info->crypt_ops !=
+                   crypt_info->crypt_ops)) {
+                       /* crypt_info contents do not match: next pager */
+                       continue;
+               }
+
+               /* found a match for crypt_info ... */
+               if (old_crypt_info) {
+                       /* ... already switched to that crypt_info */
+                       assert(old_crypt_info == pager2->crypt_info);
+               } else if (pager2->crypt_info != crypt_info) {
+                       /* ... switch to that pager's crypt_info */
+#if CRYPT_INFO_DEBUG
+                       printf("CRYPT_INFO %s: reference %p ref %d "
+                           "(create match)\n",
+                           __FUNCTION__,
+                           pager2->crypt_info,
+                           pager2->crypt_info->crypt_refcnt);
+#endif /* CRYPT_INFO_DEBUG */
+                       old_crypt_info = pager2->crypt_info;
+                       crypt_info_reference(old_crypt_info);
+                       pager->crypt_info = old_crypt_info;
+               }
+
+               if (pager2->backing_object == backing_object &&
+                   pager2->backing_offset == backing_offset &&
+                   pager2->crypto_backing_offset == crypto_backing_offset &&
+                   pager2->crypto_start == crypto_start &&
+                   pager2->crypto_end == crypto_end) {
+                       /* full match: use that pager */

                        break;
                }
        }
                        break;
                }
        }

-       if (! queue_end(&apple_protect_pager_queue,
-                       (queue_entry_t) pager2)) {
-               /* while we hold the lock, transfer our setup ref to winner */
-               pager2->ref_count++;
+       if (!queue_end(&apple_protect_pager_queue,
+           (queue_entry_t) pager2)) {

                /* we lost the race, down with the loser... */
                lck_mtx_unlock(&apple_protect_pager_lock);
                vm_object_deallocate(pager->backing_object);
                pager->backing_object = VM_OBJECT_NULL;
                /* we lost the race, down with the loser... */
                lck_mtx_unlock(&apple_protect_pager_lock);
                vm_object_deallocate(pager->backing_object);
                pager->backing_object = VM_OBJECT_NULL;

-               kfree(pager, sizeof (*pager));
+#if CRYPT_INFO_DEBUG
+               printf("CRYPT_INFO %s: %p ref %d (create pager match)\n",
+                   __FUNCTION__,
+                   pager->crypt_info,
+                   pager->crypt_info->crypt_refcnt);
+#endif /* CRYPT_INFO_DEBUG */
+               crypt_info_deallocate(pager->crypt_info);
+               pager->crypt_info = NULL;
+               kfree(pager, sizeof(*pager));

                /* ... and go with the winner */
                pager = pager2;
                /* let the winner make sure the pager gets ready */
                /* ... and go with the winner */
                pager = pager2;
                /* let the winner make sure the pager gets ready */


@@ -1010,9 +1139,9 @@ apple_protect_pager_create(
 
        /* enter new pager at the head of our list of pagers */
        queue_enter_first(&apple_protect_pager_queue,
 
        /* enter new pager at the head of our list of pagers */
        queue_enter_first(&apple_protect_pager_queue,

-                         pager,
-                         apple_protect_pager_t,
-                         pager_queue);
+           pager,
+           apple_protect_pager_t,
+           pager_queue);

        apple_protect_pager_count++;
        if (apple_protect_pager_count > apple_protect_pager_count_max) {
                apple_protect_pager_count_max = apple_protect_pager_count;
        apple_protect_pager_count++;
        if (apple_protect_pager_count > apple_protect_pager_count_max) {
                apple_protect_pager_count_max = apple_protect_pager_count;


@@ -1020,10 +1149,12 @@ apple_protect_pager_create(
        lck_mtx_unlock(&apple_protect_pager_lock);
 
        kr = memory_object_create_named((memory_object_t) pager,
        lck_mtx_unlock(&apple_protect_pager_lock);
 
        kr = memory_object_create_named((memory_object_t) pager,

-                                       0,
-                                       &control);
+           0,
+           &control);

        assert(kr == KERN_SUCCESS);
 
        assert(kr == KERN_SUCCESS);
 

+       memory_object_mark_trusted(control);
+

        lck_mtx_lock(&apple_protect_pager_lock);
        /* the new pager is now ready to be used */
        pager->is_ready = TRUE;
        lck_mtx_lock(&apple_protect_pager_lock);
        /* the new pager is now ready to be used */
        pager->is_ready = TRUE;


@@ -1032,6 +1163,20 @@ apple_protect_pager_create(
        /* wakeup anyone waiting for this pager to be ready */
        thread_wakeup(&pager->is_ready);
 
        /* wakeup anyone waiting for this pager to be ready */
        thread_wakeup(&pager->is_ready);
 

+       if (old_crypt_info != NULL &&
+           old_crypt_info != crypt_info) {
+               /* we re-used an old crypt_info instead of using our new one */
+#if CRYPT_INFO_DEBUG
+               printf("CRYPT_INFO %s: deallocate %p ref %d "
+                   "(create used old)\n",
+                   __FUNCTION__,
+                   crypt_info,
+                   crypt_info->crypt_refcnt);
+#endif /* CRYPT_INFO_DEBUG */
+               crypt_info_deallocate(crypt_info);
+               crypt_info = NULL;
+       }
+

        return pager;
 }
 
        return pager;
 }
 


@@ -1044,64 +1189,191 @@ apple_protect_pager_create(
  */
 memory_object_t
 apple_protect_pager_setup(
  */
 memory_object_t
 apple_protect_pager_setup(

-                         vm_object_t   backing_object,
-                         struct pager_crypt_info *crypt_info)
+       vm_object_t             backing_object,
+       vm_object_offset_t      backing_offset,
+       vm_object_offset_t      crypto_backing_offset,
+       struct pager_crypt_info *crypt_info,
+       vm_object_offset_t      crypto_start,
+       vm_object_offset_t      crypto_end)

 {
 {

-       apple_protect_pager_t   pager;
+       apple_protect_pager_t   pager;
+       struct pager_crypt_info *old_crypt_info, *new_crypt_info;
+
+#if CRYPT_INFO_DEBUG
+       printf("CRYPT_INFO %s: crypt_info=%p [%p,%p,%p,%d]\n",
+           __FUNCTION__,
+           crypt_info,
+           crypt_info->page_decrypt,
+           crypt_info->crypt_end,
+           crypt_info->crypt_ops,
+           crypt_info->crypt_refcnt);
+#endif /* CRYPT_INFO_DEBUG */
+
+       old_crypt_info = NULL;

 
        lck_mtx_lock(&apple_protect_pager_lock);
 
        queue_iterate(&apple_protect_pager_queue,
 
        lck_mtx_lock(&apple_protect_pager_lock);
 
        queue_iterate(&apple_protect_pager_queue,

-                     pager,
-                     apple_protect_pager_t,
-                     pager_queue) {
-               if (pager->backing_object == backing_object) {
-                       /* For the same object we must always use the same protection options */
-                       if (!((pager->crypt.page_decrypt == crypt_info->page_decrypt) &&
-                             (pager->crypt.crypt_ops == crypt_info->crypt_ops) )) {
-                               lck_mtx_unlock(&apple_protect_pager_lock);
-                               return MEMORY_OBJECT_NULL;
-                       }
+           pager,
+           apple_protect_pager_t,
+           pager_queue) {
+               if ((pager->crypt_info->page_decrypt !=
+                   crypt_info->page_decrypt) ||
+                   (pager->crypt_info->crypt_end !=
+                   crypt_info->crypt_end) ||
+                   (pager->crypt_info->crypt_ops !=
+                   crypt_info->crypt_ops)) {
+                       /* no match for "crypt_info": next pager */
+                       continue;
+               }
+               /* found a match for crypt_info ... */
+               if (old_crypt_info) {
+                       /* ... already switched to that crypt_info */
+                       assert(old_crypt_info == pager->crypt_info);
+               } else {
+                       /* ... switch to that pager's crypt_info */
+                       old_crypt_info = pager->crypt_info;
+#if CRYPT_INFO_DEBUG
+                       printf("CRYPT_INFO %s: "
+                           "switching crypt_info from %p [%p,%p,%p,%d] "
+                           "to %p [%p,%p,%p,%d] from pager %p\n",
+                           __FUNCTION__,
+                           crypt_info,
+                           crypt_info->page_decrypt,
+                           crypt_info->crypt_end,
+                           crypt_info->crypt_ops,
+                           crypt_info->crypt_refcnt,
+                           old_crypt_info,
+                           old_crypt_info->page_decrypt,
+                           old_crypt_info->crypt_end,
+                           old_crypt_info->crypt_ops,
+                           old_crypt_info->crypt_refcnt,
+                           pager);
+                       printf("CRYPT_INFO %s: %p ref %d (setup match)\n",
+                           __FUNCTION__,
+                           pager->crypt_info,
+                           pager->crypt_info->crypt_refcnt);
+#endif /* CRYPT_INFO_DEBUG */
+                       crypt_info_reference(pager->crypt_info);
+               }
+
+               if (pager->backing_object == backing_object &&
+                   pager->backing_offset == backing_offset &&
+                   pager->crypto_backing_offset == crypto_backing_offset &&
+                   pager->crypto_start == crypto_start &&
+                   pager->crypto_end == crypto_end) {
+                       /* full match: use that pager! */
+                       assert(old_crypt_info == pager->crypt_info);
+                       assert(old_crypt_info->crypt_refcnt > 1);
+#if CRYPT_INFO_DEBUG
+                       printf("CRYPT_INFO %s: "
+                           "pager match with %p crypt_info %p\n",
+                           __FUNCTION__,
+                           pager,
+                           pager->crypt_info);
+                       printf("CRYPT_INFO %s: deallocate %p ref %d "
+                           "(pager match)\n",
+                           __FUNCTION__,
+                           old_crypt_info,
+                           old_crypt_info->crypt_refcnt);
+#endif /* CRYPT_INFO_DEBUG */
+                       /* release the extra ref on crypt_info we got above */
+                       crypt_info_deallocate(old_crypt_info);
+                       assert(old_crypt_info->crypt_refcnt > 0);
+                       /* give extra reference on pager to the caller */
+                       os_ref_retain_locked(&pager->ref_count);

                        break;
                }
        }
        if (queue_end(&apple_protect_pager_queue,
                        break;
                }
        }
        if (queue_end(&apple_protect_pager_queue,

-                     (queue_entry_t) pager)) {
+           (queue_entry_t) pager)) {
+               lck_mtx_unlock(&apple_protect_pager_lock);

                /* no existing pager for this backing object */
                pager = APPLE_PROTECT_PAGER_NULL;
                /* no existing pager for this backing object */
                pager = APPLE_PROTECT_PAGER_NULL;

-       } else {
-               /* make sure pager doesn't disappear */
-               pager->ref_count++;
-       }
-
-       lck_mtx_unlock(&apple_protect_pager_lock);
-
-       if (pager == APPLE_PROTECT_PAGER_NULL) {
-               pager = apple_protect_pager_create(backing_object, crypt_info);
+               if (old_crypt_info) {
+                       /* use this old crypt_info for new pager */
+                       new_crypt_info = old_crypt_info;
+#if CRYPT_INFO_DEBUG
+                       printf("CRYPT_INFO %s: "
+                           "will use old_crypt_info %p for new pager\n",
+                           __FUNCTION__,
+                           old_crypt_info);
+#endif /* CRYPT_INFO_DEBUG */
+               } else {
+                       /* allocate a new crypt_info for new pager */
+                       new_crypt_info = kalloc(sizeof(*new_crypt_info));
+                       *new_crypt_info = *crypt_info;
+                       new_crypt_info->crypt_refcnt = 1;
+#if CRYPT_INFO_DEBUG
+                       printf("CRYPT_INFO %s: "
+                           "will use new_crypt_info %p for new pager\n",
+                           __FUNCTION__,
+                           new_crypt_info);
+#endif /* CRYPT_INFO_DEBUG */
+               }
+               if (new_crypt_info == NULL) {
+                       /* can't create new pager without a crypt_info */
+               } else {
+                       /* create new pager */
+                       pager = apple_protect_pager_create(
+                               backing_object,
+                               backing_offset,
+                               crypto_backing_offset,
+                               new_crypt_info,
+                               crypto_start,
+                               crypto_end);
+               }

                if (pager == APPLE_PROTECT_PAGER_NULL) {
                if (pager == APPLE_PROTECT_PAGER_NULL) {

+                       /* could not create a new pager */
+                       if (new_crypt_info == old_crypt_info) {
+                               /* release extra reference on old_crypt_info */
+#if CRYPT_INFO_DEBUG
+                               printf("CRYPT_INFO %s: deallocate %p ref %d "
+                                   "(create fail old_crypt_info)\n",
+                                   __FUNCTION__,
+                                   old_crypt_info,
+                                   old_crypt_info->crypt_refcnt);
+#endif /* CRYPT_INFO_DEBUG */
+                               crypt_info_deallocate(old_crypt_info);
+                               old_crypt_info = NULL;
+                       } else {
+                               /* release unused new_crypt_info */
+                               assert(new_crypt_info->crypt_refcnt == 1);
+#if CRYPT_INFO_DEBUG
+                               printf("CRYPT_INFO %s: deallocate %p ref %d "
+                                   "(create fail new_crypt_info)\n",
+                                   __FUNCTION__,
+                                   new_crypt_info,
+                                   new_crypt_info->crypt_refcnt);
+#endif /* CRYPT_INFO_DEBUG */
+                               crypt_info_deallocate(new_crypt_info);
+                               new_crypt_info = NULL;
+                       }

                        return MEMORY_OBJECT_NULL;
                }
                        return MEMORY_OBJECT_NULL;
                }

+               lck_mtx_lock(&apple_protect_pager_lock);
+       } else {
+               assert(old_crypt_info == pager->crypt_info);

        }
 
        }
 

-       lck_mtx_lock(&apple_protect_pager_lock);

        while (!pager->is_ready) {
                lck_mtx_sleep(&apple_protect_pager_lock,
        while (!pager->is_ready) {
                lck_mtx_sleep(&apple_protect_pager_lock,

-                       LCK_SLEEP_DEFAULT,
-                       &pager->is_ready,
-                       THREAD_UNINT);
+                   LCK_SLEEP_DEFAULT,
+                   &pager->is_ready,
+                   THREAD_UNINT);

        }
        lck_mtx_unlock(&apple_protect_pager_lock);
 
        return (memory_object_t) pager;
        }
        lck_mtx_unlock(&apple_protect_pager_lock);
 
        return (memory_object_t) pager;

-}      
+}

 
 void
 apple_protect_pager_trim(void)
 {
 
 void
 apple_protect_pager_trim(void)
 {

-       apple_protect_pager_t   pager, prev_pager;
-       queue_head_t            trim_queue;
-       int                     num_trim;
-       int                     count_unmapped;
+       apple_protect_pager_t   pager, prev_pager;
+       queue_head_t            trim_queue;
+       int                     num_trim;
+       int                     count_unmapped;

 
        lck_mtx_lock(&apple_protect_pager_lock);
 
 
        lck_mtx_lock(&apple_protect_pager_lock);
 


@@ -1113,15 +1385,15 @@ apple_protect_pager_trim(void)
        num_trim = 0;
 
        for (pager = (apple_protect_pager_t)
        num_trim = 0;
 
        for (pager = (apple_protect_pager_t)

-                    queue_last(&apple_protect_pager_queue);
-            !queue_end(&apple_protect_pager_queue,
-                       (queue_entry_t) pager);
-            pager = prev_pager) {
+           queue_last(&apple_protect_pager_queue);
+           !queue_end(&apple_protect_pager_queue,
+           (queue_entry_t) pager);
+           pager = prev_pager) {

                /* get prev elt before we dequeue */
                prev_pager = (apple_protect_pager_t)
                /* get prev elt before we dequeue */
                prev_pager = (apple_protect_pager_t)

-                       queue_prev(&pager->pager_queue);
+                   queue_prev(&pager->pager_queue);

 
 

-               if (pager->ref_count == 2 &&
+               if (os_ref_get_count(&pager->ref_count) == 2 &&

                    pager->is_ready &&
                    !pager->is_mapped) {
                        /* this pager can be trimmed */
                    pager->is_ready &&
                    !pager->is_mapped) {
                        /* this pager can be trimmed */


@@ -1130,12 +1402,12 @@ apple_protect_pager_trim(void)
                        apple_protect_pager_dequeue(pager);
                        /* ... and add it to our trim queue */
                        queue_enter_first(&trim_queue,
                        apple_protect_pager_dequeue(pager);
                        /* ... and add it to our trim queue */
                        queue_enter_first(&trim_queue,

-                                         pager,
-                                         apple_protect_pager_t,
-                                         pager_queue);
+                           pager,
+                           apple_protect_pager_t,
+                           pager_queue);

 
                        count_unmapped = (apple_protect_pager_count -
 
                        count_unmapped = (apple_protect_pager_count -

-                                         apple_protect_pager_count_mapped);
+                           apple_protect_pager_count_mapped);

                        if (count_unmapped <= apple_protect_pager_cache_limit) {
                                /* we have enough pagers to trim */
                                break;
                        if (count_unmapped <= apple_protect_pager_cache_limit) {
                                /* we have enough pagers to trim */
                                break;


@@ -1152,18 +1424,62 @@ apple_protect_pager_trim(void)
        /* terminate the trimmed pagers */
        while (!queue_empty(&trim_queue)) {
                queue_remove_first(&trim_queue,
        /* terminate the trimmed pagers */
        while (!queue_empty(&trim_queue)) {
                queue_remove_first(&trim_queue,

-                                  pager,
-                                  apple_protect_pager_t,
-                                  pager_queue);
+                   pager,
+                   apple_protect_pager_t,
+                   pager_queue);

                pager->pager_queue.next = NULL;
                pager->pager_queue.prev = NULL;
                pager->pager_queue.next = NULL;
                pager->pager_queue.prev = NULL;

-               assert(pager->ref_count == 2);

                /*
                 * We can't call deallocate_internal() because the pager
                 * has already been dequeued, but we still need to remove
                 * a reference.
                 */
                /*
                 * We can't call deallocate_internal() because the pager
                 * has already been dequeued, but we still need to remove
                 * a reference.
                 */

-               pager->ref_count--;
+               os_ref_count_t __assert_only count = os_ref_release_locked(&pager->ref_count);
+               assert(count == 1);

                apple_protect_pager_terminate_internal(pager);
        }
 }
                apple_protect_pager_terminate_internal(pager);
        }
 }

+
+
+void
+crypt_info_reference(
+       struct pager_crypt_info *crypt_info)
+{
+       assert(crypt_info->crypt_refcnt != 0);
+#if CRYPT_INFO_DEBUG
+       printf("CRYPT_INFO %s: %p ref %d -> %d\n",
+           __FUNCTION__,
+           crypt_info,
+           crypt_info->crypt_refcnt,
+           crypt_info->crypt_refcnt + 1);
+#endif /* CRYPT_INFO_DEBUG */
+       OSAddAtomic(+1, &crypt_info->crypt_refcnt);
+}
+
+void
+crypt_info_deallocate(
+       struct pager_crypt_info *crypt_info)
+{
+#if CRYPT_INFO_DEBUG
+       printf("CRYPT_INFO %s: %p ref %d -> %d\n",
+           __FUNCTION__,
+           crypt_info,
+           crypt_info->crypt_refcnt,
+           crypt_info->crypt_refcnt - 1);
+#endif /* CRYPT_INFO_DEBUG */
+       OSAddAtomic(-1, &crypt_info->crypt_refcnt);
+       if (crypt_info->crypt_refcnt == 0) {
+               /* deallocate any crypt module data */
+               if (crypt_info->crypt_end) {
+                       crypt_info->crypt_end(crypt_info->crypt_ops);
+                       crypt_info->crypt_end = NULL;
+               }
+#if CRYPT_INFO_DEBUG
+               printf("CRYPT_INFO %s: freeing %p\n",
+                   __FUNCTION__,
+                   crypt_info);
+#endif /* CRYPT_INFO_DEBUG */
+               kfree(crypt_info, sizeof(*crypt_info));
+               crypt_info = NULL;
+       }
+}