xnu-7195.101.1.tar.gz

[apple/xnu.git] / osfmk / vm / vm_apple_protect.c

diff --git a/osfmk/vm/vm_apple_protect.c b/osfmk/vm/vm_apple_protect.c
index 508211e68b3000b36ad051665cd74a7cf078b7ad..f7fcaceb38c4d418139a7d5883d1654833439f0c 100644 (file)
--- a/osfmk/vm/vm_apple_protect.c
+++ b/osfmk/vm/vm_apple_protect.c
@@ -1,8 +1,8 @@
 /*
 /*

- * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2006-2020 Apple Inc. All rights reserved.

  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@

- * 
+ *

  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in
  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in


@@ -11,10 +11,10 @@
  * unlawful or unlicensed copies of an Apple operating system, or to
  * circumvent, violate, or enable the circumvention or violation of, any
  * terms of an Apple operating system software license agreement.
  * unlawful or unlicensed copies of an Apple operating system, or to
  * circumvent, violate, or enable the circumvention or violation of, any
  * terms of an Apple operating system software license agreement.

- * 
+ *

  * Please obtain a copy of the License at
  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  * Please obtain a copy of the License at
  * http://www.opensource.apple.com/apsl/ and read it before using this file.

- * 
+ *

  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,


@@ -22,7 +22,7 @@
  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  * Please see the License for the specific language governing rights and
  * limitations under the License.
  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  * Please see the License for the specific language governing rights and
  * limitations under the License.

- * 
+ *

  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 


@@ -46,6 +46,7 @@
 #include <kern/queue.h>
 #include <kern/thread.h>
 #include <kern/ipc_kobject.h>
 #include <kern/queue.h>
 #include <kern/thread.h>
 #include <kern/ipc_kobject.h>

+#include <os/refcnt.h>

 
 #include <ipc/ipc_port.h>
 #include <ipc/ipc_space.h>
 
 #include <ipc/ipc_port.h>
 #include <ipc/ipc_space.h>


@@ -58,13 +59,12 @@
 #include <vm/vm_protos.h>
 #include <vm/vm_kern.h>
 
 #include <vm/vm_protos.h>
 #include <vm/vm_kern.h>
 

-
-/* 
- * APPLE PROTECT MEMORY PAGER 
+/*
+ * APPLE PROTECT MEMORY PAGER

  *
  * This external memory manager (EMM) handles memory from the encrypted
  * sections of some executables protected by the DSMOS kernel extension.
  *
  * This external memory manager (EMM) handles memory from the encrypted
  * sections of some executables protected by the DSMOS kernel extension.

- * 
+ *

  * It mostly handles page-in requests (from memory_object_data_request()) by
  * getting the encrypted data from its backing VM object, itself backed by
  * the encrypted file, decrypting it and providing it to VM.
  * It mostly handles page-in requests (from memory_object_data_request()) by
  * getting the encrypted data from its backing VM object, itself backed by
  * the encrypted file, decrypting it and providing it to VM.


@@ -82,36 +82,41 @@
 void apple_protect_pager_reference(memory_object_t mem_obj);
 void apple_protect_pager_deallocate(memory_object_t mem_obj);
 kern_return_t apple_protect_pager_init(memory_object_t mem_obj,
 void apple_protect_pager_reference(memory_object_t mem_obj);
 void apple_protect_pager_deallocate(memory_object_t mem_obj);
 kern_return_t apple_protect_pager_init(memory_object_t mem_obj,

-                                      memory_object_control_t control,
-                                      memory_object_cluster_size_t pg_size);
+    memory_object_control_t control,
+    memory_object_cluster_size_t pg_size);

 kern_return_t apple_protect_pager_terminate(memory_object_t mem_obj);
 kern_return_t apple_protect_pager_data_request(memory_object_t mem_obj,
 kern_return_t apple_protect_pager_terminate(memory_object_t mem_obj);
 kern_return_t apple_protect_pager_data_request(memory_object_t mem_obj,

-                                              memory_object_offset_t offset,
-                                              memory_object_cluster_size_t length,
-                                              vm_prot_t protection_required,
-                                              memory_object_fault_info_t fault_info);
+    memory_object_offset_t offset,
+    memory_object_cluster_size_t length,
+    vm_prot_t protection_required,
+    memory_object_fault_info_t fault_info);

 kern_return_t apple_protect_pager_data_return(memory_object_t mem_obj,
 kern_return_t apple_protect_pager_data_return(memory_object_t mem_obj,

-                                             memory_object_offset_t offset,
-                                             memory_object_cluster_size_t      data_cnt,
-                                             memory_object_offset_t *resid_offset,
-                                             int *io_error,
-                                             boolean_t dirty,
-                                             boolean_t kernel_copy,
-                                             int upl_flags);
+    memory_object_offset_t offset,
+    memory_object_cluster_size_t      data_cnt,
+    memory_object_offset_t *resid_offset,
+    int *io_error,
+    boolean_t dirty,
+    boolean_t kernel_copy,
+    int upl_flags);

 kern_return_t apple_protect_pager_data_initialize(memory_object_t mem_obj,
 kern_return_t apple_protect_pager_data_initialize(memory_object_t mem_obj,

-                                                 memory_object_offset_t offset,
-                                                 memory_object_cluster_size_t data_cnt);
+    memory_object_offset_t offset,
+    memory_object_cluster_size_t data_cnt);

 kern_return_t apple_protect_pager_data_unlock(memory_object_t mem_obj,
 kern_return_t apple_protect_pager_data_unlock(memory_object_t mem_obj,

-                                             memory_object_offset_t offset,
-                                             memory_object_size_t size,
-                                             vm_prot_t desired_access);
+    memory_object_offset_t offset,
+    memory_object_size_t size,
+    vm_prot_t desired_access);

 kern_return_t apple_protect_pager_synchronize(memory_object_t mem_obj,
 kern_return_t apple_protect_pager_synchronize(memory_object_t mem_obj,

-                                             memory_object_offset_t offset,
-                                             memory_object_size_t length,
-                                             vm_sync_t sync_flags);
+    memory_object_offset_t offset,
+    memory_object_size_t length,
+    vm_sync_t sync_flags);

 kern_return_t apple_protect_pager_map(memory_object_t mem_obj,
 kern_return_t apple_protect_pager_map(memory_object_t mem_obj,

-                                     vm_prot_t prot);
+    vm_prot_t prot);

 kern_return_t apple_protect_pager_last_unmap(memory_object_t mem_obj);
 kern_return_t apple_protect_pager_last_unmap(memory_object_t mem_obj);

+boolean_t apple_protect_pager_backing_object(
+       memory_object_t mem_obj,
+       memory_object_offset_t mem_obj_offset,
+       vm_object_t *backing_object,
+       vm_object_offset_t *backing_offset);

 
 #define CRYPT_INFO_DEBUG 0
 void crypt_info_reference(struct pager_crypt_info *crypt_info);
 
 #define CRYPT_INFO_DEBUG 0
 void crypt_info_reference(struct pager_crypt_info *crypt_info);


@@ -122,19 +127,20 @@ void crypt_info_deallocate(struct pager_crypt_info *crypt_info);
  * These routines are invoked by VM via the memory_object_*() interfaces.
  */
 const struct memory_object_pager_ops apple_protect_pager_ops = {
  * These routines are invoked by VM via the memory_object_*() interfaces.
  */
 const struct memory_object_pager_ops apple_protect_pager_ops = {

-       apple_protect_pager_reference,
-       apple_protect_pager_deallocate,
-       apple_protect_pager_init,
-       apple_protect_pager_terminate,
-       apple_protect_pager_data_request,
-       apple_protect_pager_data_return,
-       apple_protect_pager_data_initialize,
-       apple_protect_pager_data_unlock,
-       apple_protect_pager_synchronize,
-       apple_protect_pager_map,
-       apple_protect_pager_last_unmap,
-       NULL, /* data_reclaim */
-       "apple_protect"
+       .memory_object_reference = apple_protect_pager_reference,
+       .memory_object_deallocate = apple_protect_pager_deallocate,
+       .memory_object_init = apple_protect_pager_init,
+       .memory_object_terminate = apple_protect_pager_terminate,
+       .memory_object_data_request = apple_protect_pager_data_request,
+       .memory_object_data_return = apple_protect_pager_data_return,
+       .memory_object_data_initialize = apple_protect_pager_data_initialize,
+       .memory_object_data_unlock = apple_protect_pager_data_unlock,
+       .memory_object_synchronize = apple_protect_pager_synchronize,
+       .memory_object_map = apple_protect_pager_map,
+       .memory_object_last_unmap = apple_protect_pager_last_unmap,
+       .memory_object_data_reclaim = NULL,
+       .memory_object_backing_object = apple_protect_pager_backing_object,
+       .memory_object_pager_name = "apple_protect"

 };
 
 /*
 };
 
 /*


@@ -143,49 +149,51 @@ const struct memory_object_pager_ops apple_protect_pager_ops = {
  */
 typedef struct apple_protect_pager {
        /* mandatory generic header */
  */
 typedef struct apple_protect_pager {
        /* mandatory generic header */

-       struct memory_object ap_pgr_hdr;
+       struct memory_object    ap_pgr_hdr;

 
        /* pager-specific data */
 
        /* pager-specific data */

-       queue_chain_t           pager_queue;    /* next & prev pagers */
-       unsigned int            ref_count;      /* reference count */
-       boolean_t               is_ready;       /* is this pager ready ? */
-       boolean_t               is_mapped;      /* is this mem_obj mapped ? */
-       vm_object_t             backing_object; /* VM obj w/ encrypted data */
-       vm_object_offset_t      backing_offset;
-       vm_object_offset_t      crypto_backing_offset; /* for key... */
-       vm_object_offset_t      crypto_start;
-       vm_object_offset_t      crypto_end;
+       queue_chain_t           pager_queue;    /* next & prev pagers */
+#if MEMORY_OBJECT_HAS_REFCOUNT
+#define ap_pgr_hdr_ref          ap_pgr_hdr.mo_ref
+#else
+       os_ref_atomic_t         ap_pgr_hdr_ref;      /* reference count */
+#endif
+       bool                    is_ready;       /* is this pager ready ? */
+       bool                    is_mapped;      /* is this mem_obj mapped ? */
+       bool                    is_cached;      /* is this pager cached ? */
+       vm_object_t             backing_object; /* VM obj w/ encrypted data */
+       vm_object_offset_t      backing_offset;
+       vm_object_offset_t      crypto_backing_offset; /* for key... */
+       vm_object_offset_t      crypto_start;
+       vm_object_offset_t      crypto_end;

        struct pager_crypt_info *crypt_info;
 } *apple_protect_pager_t;
        struct pager_crypt_info *crypt_info;
 } *apple_protect_pager_t;

-#define        APPLE_PROTECT_PAGER_NULL        ((apple_protect_pager_t) NULL)
+#define APPLE_PROTECT_PAGER_NULL        ((apple_protect_pager_t) NULL)

 
 /*
  * List of memory objects managed by this EMM.
  * The list is protected by the "apple_protect_pager_lock" lock.
  */
 
 /*
  * List of memory objects managed by this EMM.
  * The list is protected by the "apple_protect_pager_lock" lock.
  */

-int apple_protect_pager_count = 0;             /* number of pagers */
-int apple_protect_pager_count_mapped = 0;      /* number of unmapped pagers */
-queue_head_t apple_protect_pager_queue;
-decl_lck_mtx_data(,apple_protect_pager_lock)
+unsigned int apple_protect_pager_count = 0;        /* number of pagers */
+unsigned int apple_protect_pager_count_mapped = 0; /* number of unmapped pagers */
+queue_head_t apple_protect_pager_queue = QUEUE_HEAD_INITIALIZER(apple_protect_pager_queue);
+LCK_GRP_DECLARE(apple_protect_pager_lck_grp, "apple_protect");
+LCK_MTX_DECLARE(apple_protect_pager_lock, &apple_protect_pager_lck_grp);

 
 /*
  * Maximum number of unmapped pagers we're willing to keep around.
  */
 
 /*
  * Maximum number of unmapped pagers we're willing to keep around.
  */

-int apple_protect_pager_cache_limit = 20;
+unsigned int apple_protect_pager_cache_limit = 20;

 
 /*
  * Statistics & counters.
  */
 
 /*
  * Statistics & counters.
  */

-int apple_protect_pager_count_max = 0;
-int apple_protect_pager_count_unmapped_max = 0;
-int apple_protect_pager_num_trim_max = 0;
-int apple_protect_pager_num_trim_total = 0;
+unsigned int apple_protect_pager_count_max = 0;
+unsigned int apple_protect_pager_count_unmapped_max = 0;
+unsigned int apple_protect_pager_num_trim_max = 0;
+unsigned int apple_protect_pager_num_trim_total = 0;

 
 
 
 

-lck_grp_t              apple_protect_pager_lck_grp;
-lck_grp_attr_t apple_protect_pager_lck_grp_attr;
-lck_attr_t             apple_protect_pager_lck_attr;
-

 
 /* internal prototypes */
 apple_protect_pager_t apple_protect_pager_create(
 
 /* internal prototypes */
 apple_protect_pager_t apple_protect_pager_create(


@@ -194,42 +202,32 @@ apple_protect_pager_t apple_protect_pager_create(
        vm_object_offset_t crypto_backing_offset,
        struct pager_crypt_info *crypt_info,
        vm_object_offset_t crypto_start,
        vm_object_offset_t crypto_backing_offset,
        struct pager_crypt_info *crypt_info,
        vm_object_offset_t crypto_start,

-       vm_object_offset_t crypto_end);
+       vm_object_offset_t crypto_end,
+       boolean_t cache_pager);

 apple_protect_pager_t apple_protect_pager_lookup(memory_object_t mem_obj);
 void apple_protect_pager_dequeue(apple_protect_pager_t pager);
 void apple_protect_pager_deallocate_internal(apple_protect_pager_t pager,
 apple_protect_pager_t apple_protect_pager_lookup(memory_object_t mem_obj);
 void apple_protect_pager_dequeue(apple_protect_pager_t pager);
 void apple_protect_pager_deallocate_internal(apple_protect_pager_t pager,

-                                            boolean_t locked);
+    boolean_t locked);

 void apple_protect_pager_terminate_internal(apple_protect_pager_t pager);
 void apple_protect_pager_trim(void);
 
 
 #if DEBUG
 int apple_protect_pagerdebug = 0;
 void apple_protect_pager_terminate_internal(apple_protect_pager_t pager);
 void apple_protect_pager_trim(void);
 
 
 #if DEBUG
 int apple_protect_pagerdebug = 0;

-#define PAGER_ALL              0xffffffff
-#define        PAGER_INIT              0x00000001
-#define        PAGER_PAGEIN            0x00000002
-
-#define PAGER_DEBUG(LEVEL, A)                                          \
-       MACRO_BEGIN                                                     \
-       if ((apple_protect_pagerdebug & LEVEL)==LEVEL) {                \
-               printf A;                                               \
-       }                                                               \
+#define PAGER_ALL               0xffffffff
+#define PAGER_INIT              0x00000001
+#define PAGER_PAGEIN            0x00000002
+
+#define PAGER_DEBUG(LEVEL, A)                                           \
+       MACRO_BEGIN                                                     \
+       if ((apple_protect_pagerdebug & LEVEL)==LEVEL) {                \
+               printf A;                                               \
+       }                                                               \

        MACRO_END
 #else
 #define PAGER_DEBUG(LEVEL, A)
 #endif
 
        MACRO_END
 #else
 #define PAGER_DEBUG(LEVEL, A)
 #endif
 

-
-void
-apple_protect_pager_bootstrap(void)
-{
-       lck_grp_attr_setdefault(&apple_protect_pager_lck_grp_attr);
-       lck_grp_init(&apple_protect_pager_lck_grp, "apple_protect", &apple_protect_pager_lck_grp_attr);
-       lck_attr_setdefault(&apple_protect_pager_lck_attr);
-       lck_mtx_init(&apple_protect_pager_lock, &apple_protect_pager_lck_grp, &apple_protect_pager_lck_attr);
-       queue_init(&apple_protect_pager_queue);
-}
-

 /*
  * apple_protect_pager_init()
  *
 /*
  * apple_protect_pager_init()
  *


@@ -237,23 +235,24 @@ apple_protect_pager_bootstrap(void)
  */
 kern_return_t
 apple_protect_pager_init(
  */
 kern_return_t
 apple_protect_pager_init(

-       memory_object_t         mem_obj, 
-       memory_object_control_t control, 
+       memory_object_t         mem_obj,
+       memory_object_control_t control,

 #if !DEBUG
        __unused
 #endif
        memory_object_cluster_size_t pg_size)
 {
 #if !DEBUG
        __unused
 #endif
        memory_object_cluster_size_t pg_size)
 {

-       apple_protect_pager_t   pager;
-       kern_return_t           kr;
+       apple_protect_pager_t   pager;
+       kern_return_t           kr;

        memory_object_attr_info_data_t  attributes;
 
        PAGER_DEBUG(PAGER_ALL,
        memory_object_attr_info_data_t  attributes;
 
        PAGER_DEBUG(PAGER_ALL,

-                   ("apple_protect_pager_init: %p, %p, %x\n",
-                    mem_obj, control, pg_size));
+           ("apple_protect_pager_init: %p, %p, %x\n",
+           mem_obj, control, pg_size));

 
 

-       if (control == MEMORY_OBJECT_CONTROL_NULL)
+       if (control == MEMORY_OBJECT_CONTROL_NULL) {

                return KERN_INVALID_ARGUMENT;
                return KERN_INVALID_ARGUMENT;

+       }

 
        pager = apple_protect_pager_lookup(mem_obj);
 
 
        pager = apple_protect_pager_lookup(mem_obj);
 


@@ -268,13 +267,14 @@ apple_protect_pager_init(
        attributes.temporary = TRUE;
 
        kr = memory_object_change_attributes(
        attributes.temporary = TRUE;
 
        kr = memory_object_change_attributes(

-                                       control,
-                                       MEMORY_OBJECT_ATTRIBUTE_INFO,
-                                       (memory_object_info_t) &attributes,
-                                       MEMORY_OBJECT_ATTR_INFO_COUNT);
-       if (kr != KERN_SUCCESS)
+               control,
+               MEMORY_OBJECT_ATTRIBUTE_INFO,
+               (memory_object_info_t) &attributes,
+               MEMORY_OBJECT_ATTR_INFO_COUNT);
+       if (kr != KERN_SUCCESS) {

                panic("apple_protect_pager_init: "
                panic("apple_protect_pager_init: "

-                     "memory_object_change_attributes() failed");
+                   "memory_object_change_attributes() failed");
+       }

 
 #if CONFIG_SECLUDED_MEMORY
        if (secluded_for_filecache) {
 
 #if CONFIG_SECLUDED_MEMORY
        if (secluded_for_filecache) {


@@ -295,14 +295,14 @@ apple_protect_pager_init(
  */
 kern_return_t
 apple_protect_pager_data_return(
  */
 kern_return_t
 apple_protect_pager_data_return(

-        __unused memory_object_t       mem_obj,
-        __unused memory_object_offset_t        offset,
-        __unused memory_object_cluster_size_t          data_cnt,
-        __unused memory_object_offset_t        *resid_offset,
-       __unused int                    *io_error,
-       __unused boolean_t              dirty,
-       __unused boolean_t              kernel_copy,
-       __unused int                    upl_flags)  
+       __unused memory_object_t        mem_obj,
+       __unused memory_object_offset_t offset,
+       __unused memory_object_cluster_size_t           data_cnt,
+       __unused memory_object_offset_t *resid_offset,
+       __unused int                    *io_error,
+       __unused boolean_t              dirty,
+       __unused boolean_t              kernel_copy,
+       __unused int                    upl_flags)

 {
        panic("apple_protect_pager_data_return: should never get called");
        return KERN_FAILURE;
 {
        panic("apple_protect_pager_data_return: should never get called");
        return KERN_FAILURE;


@@ -310,9 +310,9 @@ apple_protect_pager_data_return(
 
 kern_return_t
 apple_protect_pager_data_initialize(
 
 kern_return_t
 apple_protect_pager_data_initialize(

-       __unused memory_object_t        mem_obj,
-       __unused memory_object_offset_t offset,
-       __unused memory_object_cluster_size_t           data_cnt)
+       __unused memory_object_t        mem_obj,
+       __unused memory_object_offset_t offset,
+       __unused memory_object_cluster_size_t           data_cnt)

 {
        panic("apple_protect_pager_data_initialize: should never get called");
        return KERN_FAILURE;
 {
        panic("apple_protect_pager_data_initialize: should never get called");
        return KERN_FAILURE;


@@ -320,10 +320,10 @@ apple_protect_pager_data_initialize(
 
 kern_return_t
 apple_protect_pager_data_unlock(
 
 kern_return_t
 apple_protect_pager_data_unlock(

-       __unused memory_object_t        mem_obj,
-       __unused memory_object_offset_t offset,
-       __unused memory_object_size_t           size,
-       __unused vm_prot_t              desired_access)
+       __unused memory_object_t        mem_obj,
+       __unused memory_object_offset_t offset,
+       __unused memory_object_size_t           size,
+       __unused vm_prot_t              desired_access)

 {
        return KERN_FAILURE;
 }
 {
        return KERN_FAILURE;
 }


@@ -334,35 +334,35 @@ apple_protect_pager_data_unlock(
  * Handles page-in requests from VM.
  */
 int apple_protect_pager_data_request_debug = 0;
  * Handles page-in requests from VM.
  */
 int apple_protect_pager_data_request_debug = 0;

-kern_return_t  
+kern_return_t

 apple_protect_pager_data_request(
 apple_protect_pager_data_request(

-       memory_object_t         mem_obj,
-       memory_object_offset_t  offset,
-       memory_object_cluster_size_t            length,
+       memory_object_t         mem_obj,
+       memory_object_offset_t  offset,
+       memory_object_cluster_size_t            length,

 #if !DEBUG
        __unused
 #endif
 #if !DEBUG
        __unused
 #endif

-       vm_prot_t               protection_required,
+       vm_prot_t               protection_required,

        memory_object_fault_info_t mo_fault_info)
 {
        memory_object_fault_info_t mo_fault_info)
 {

-       apple_protect_pager_t   pager;
-       memory_object_control_t mo_control;
-       upl_t                   upl;
-       int                     upl_flags;
-       upl_size_t              upl_size;
-       upl_page_info_t         *upl_pl;
-       unsigned int            pl_count;
-       vm_object_t             src_top_object, src_page_object, dst_object;
-       kern_return_t           kr, retval;
-       vm_offset_t             src_vaddr, dst_vaddr;
-       vm_offset_t             cur_offset;
-       vm_offset_t             offset_in_page;
-       kern_return_t           error_code;
-       vm_prot_t               prot;
-       vm_page_t               src_page, top_page;
-       int                     interruptible;
-       struct vm_object_fault_info     fault_info;
-       int                     ret;
+       apple_protect_pager_t   pager;
+       memory_object_control_t mo_control;
+       upl_t                   upl;
+       int                     upl_flags;
+       upl_size_t              upl_size;
+       upl_page_info_t         *upl_pl;
+       unsigned int            pl_count;
+       vm_object_t             src_top_object, src_page_object, dst_object;
+       kern_return_t           kr, retval;
+       vm_offset_t             src_vaddr, dst_vaddr;
+       vm_offset_t             cur_offset;
+       vm_offset_t             offset_in_page;
+       kern_return_t           error_code;
+       vm_prot_t               prot;
+       vm_page_t               src_page, top_page;
+       int                     interruptible;
+       struct vm_object_fault_info     fault_info;
+       int                     ret;

 
        PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_data_request: %p, %llx, %x, %x\n", mem_obj, offset, length, protection_required));
 
 
        PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_data_request: %p, %llx, %x, %x\n", mem_obj, offset, length, protection_required));
 


@@ -380,7 +380,7 @@ apple_protect_pager_data_request(
 
        pager = apple_protect_pager_lookup(mem_obj);
        assert(pager->is_ready);
 
        pager = apple_protect_pager_lookup(mem_obj);
        assert(pager->is_ready);

-       assert(pager->ref_count > 1); /* pager is alive and mapped */
+       assert(os_ref_get_count_raw(&pager->ap_pgr_hdr_ref) > 1); /* pager is alive and mapped */

 
        PAGER_DEBUG(PAGER_PAGEIN, ("apple_protect_pager_data_request: %p, %llx, %x, %x, pager %p\n", mem_obj, offset, length, protection_required, pager));
 
 
        PAGER_DEBUG(PAGER_PAGEIN, ("apple_protect_pager_data_request: %p, %llx, %x, %x, pager %p\n", mem_obj, offset, length, protection_required, pager));
 


@@ -394,24 +394,24 @@ apple_protect_pager_data_request(
 
        upl_size = length;
        upl_flags =
 
        upl_size = length;
        upl_flags =

-               UPL_RET_ONLY_ABSENT |
-               UPL_SET_LITE |
-               UPL_NO_SYNC |
-               UPL_CLEAN_IN_PLACE |    /* triggers UPL_CLEAR_DIRTY */
-               UPL_SET_INTERNAL;
+           UPL_RET_ONLY_ABSENT |
+           UPL_SET_LITE |
+           UPL_NO_SYNC |
+           UPL_CLEAN_IN_PLACE |        /* triggers UPL_CLEAR_DIRTY */
+           UPL_SET_INTERNAL;

        pl_count = 0;
        kr = memory_object_upl_request(mo_control,
        pl_count = 0;
        kr = memory_object_upl_request(mo_control,

-                                      offset, upl_size,
-                                      &upl, NULL, NULL, upl_flags, VM_KERN_MEMORY_SECURITY);
+           offset, upl_size,
+           &upl, NULL, NULL, upl_flags, VM_KERN_MEMORY_SECURITY);

        if (kr != KERN_SUCCESS) {
                retval = kr;
                goto done;
        }
        if (kr != KERN_SUCCESS) {
                retval = kr;
                goto done;
        }

-       dst_object = mo_control->moc_object;
+       dst_object = memory_object_control_to_vm_object(mo_control);

        assert(dst_object != VM_OBJECT_NULL);
 
        /*
        assert(dst_object != VM_OBJECT_NULL);
 
        /*

-        * We'll map the encrypted data in the kernel address space from the 
+        * We'll map the encrypted data in the kernel address space from the

         * backing VM object (itself backed by the encrypted file via
         * the vnode pager).
         */
         * backing VM object (itself backed by the encrypted file via
         * the vnode pager).
         */


@@ -425,8 +425,8 @@ apple_protect_pager_data_request(
        upl_pl = UPL_GET_INTERNAL_PAGE_LIST(upl);
        pl_count = length / PAGE_SIZE;
        for (cur_offset = 0;
        upl_pl = UPL_GET_INTERNAL_PAGE_LIST(upl);
        pl_count = length / PAGE_SIZE;
        for (cur_offset = 0;

-            retval == KERN_SUCCESS && cur_offset < length;
-            cur_offset += PAGE_SIZE) {
+           retval == KERN_SUCCESS && cur_offset < length;
+           cur_offset += PAGE_SIZE) {

                ppnum_t dst_pnum;
 
                if (!upl_page_present(upl_pl, (int)(cur_offset / PAGE_SIZE))) {
                ppnum_t dst_pnum;
 
                if (!upl_page_present(upl_pl, (int)(cur_offset / PAGE_SIZE))) {


@@ -439,25 +439,25 @@ apple_protect_pager_data_request(
                 * virtual address space.
                 * We already hold a reference on the src_top_object.
                 */
                 * virtual address space.
                 * We already hold a reference on the src_top_object.
                 */

-       retry_src_fault:
+retry_src_fault:

                vm_object_lock(src_top_object);
                vm_object_paging_begin(src_top_object);
                error_code = 0;
                prot = VM_PROT_READ;
                src_page = VM_PAGE_NULL;
                kr = vm_fault_page(src_top_object,
                vm_object_lock(src_top_object);
                vm_object_paging_begin(src_top_object);
                error_code = 0;
                prot = VM_PROT_READ;
                src_page = VM_PAGE_NULL;
                kr = vm_fault_page(src_top_object,

-                                  pager->backing_offset + offset + cur_offset,
-                                  VM_PROT_READ,
-                                  FALSE,
-                                  FALSE, /* src_page not looked up */
-                                  &prot,
-                                  &src_page,
-                                  &top_page,
-                                  NULL,
-                                  &error_code,
-                                  FALSE,
-                                  FALSE,
-                                  &fault_info);
+                   pager->backing_offset + offset + cur_offset,
+                   VM_PROT_READ,
+                   FALSE,
+                   FALSE,                /* src_page not looked up */
+                   &prot,
+                   &src_page,
+                   &top_page,
+                   NULL,
+                   &error_code,
+                   FALSE,
+                   FALSE,
+                   &fault_info);

                switch (kr) {
                case VM_FAULT_SUCCESS:
                        break;
                switch (kr) {
                case VM_FAULT_SUCCESS:
                        break;


@@ -467,7 +467,7 @@ apple_protect_pager_data_request(
                        if (vm_page_wait(interruptible)) {
                                goto retry_src_fault;
                        }
                        if (vm_page_wait(interruptible)) {
                                goto retry_src_fault;
                        }

-                       /* fall thru */
+                       OS_FALLTHROUGH;

                case VM_FAULT_INTERRUPTED:
                        retval = MACH_SEND_INTERRUPTED;
                        goto done;
                case VM_FAULT_INTERRUPTED:
                        retval = MACH_SEND_INTERRUPTED;
                        goto done;


@@ -475,7 +475,7 @@ apple_protect_pager_data_request(
                        /* success but no VM page: fail */
                        vm_object_paging_end(src_top_object);
                        vm_object_unlock(src_top_object);
                        /* success but no VM page: fail */
                        vm_object_paging_end(src_top_object);
                        vm_object_unlock(src_top_object);

-                       /*FALLTHROUGH*/
+                       OS_FALLTHROUGH;

                case VM_FAULT_MEMORY_ERROR:
                        /* the page is not there ! */
                        if (error_code) {
                case VM_FAULT_MEMORY_ERROR:
                        /* the page is not there ! */
                        if (error_code) {


@@ -486,18 +486,17 @@ apple_protect_pager_data_request(
                        goto done;
                default:
                        panic("apple_protect_pager_data_request: "
                        goto done;
                default:
                        panic("apple_protect_pager_data_request: "

-                             "vm_fault_page() unexpected error 0x%x\n",
-                             kr);
+                           "vm_fault_page() unexpected error 0x%x\n",
+                           kr);

                }
                assert(src_page != VM_PAGE_NULL);
                assert(src_page->vmp_busy);
 
                if (src_page->vmp_q_state != VM_PAGE_ON_SPECULATIVE_Q) {
                }
                assert(src_page != VM_PAGE_NULL);
                assert(src_page->vmp_busy);
 
                if (src_page->vmp_q_state != VM_PAGE_ON_SPECULATIVE_Q) {

-

                        vm_page_lockspin_queues();
 
                        if (src_page->vmp_q_state != VM_PAGE_ON_SPECULATIVE_Q) {
                        vm_page_lockspin_queues();
 
                        if (src_page->vmp_q_state != VM_PAGE_ON_SPECULATIVE_Q) {

-                               vm_page_speculate(src_page, FALSE);
+                               vm_page_speculate(src_page, FALSE);

                        }
                        vm_page_unlock_queues();
                }
                        }
                        vm_page_unlock_queues();
                }


@@ -507,26 +506,15 @@ apple_protect_pager_data_request(
                 * and destination physical pages.
                 */
                dst_pnum = (ppnum_t)
                 * and destination physical pages.
                 */
                dst_pnum = (ppnum_t)

-                       upl_phys_page(upl_pl, (int)(cur_offset / PAGE_SIZE));
-                assert(dst_pnum != 0);
-#if __x86_64__
-               src_vaddr = (vm_map_offset_t)
-                       PHYSMAP_PTOV((pmap_paddr_t)VM_PAGE_GET_PHYS_PAGE(src_page)
-                                    << PAGE_SHIFT);
-               dst_vaddr = (vm_map_offset_t)
-                       PHYSMAP_PTOV((pmap_paddr_t)dst_pnum << PAGE_SHIFT);
+                   upl_phys_page(upl_pl, (int)(cur_offset / PAGE_SIZE));
+               assert(dst_pnum != 0);

 
 

-#elif __arm__ || __arm64__

                src_vaddr = (vm_map_offset_t)
                src_vaddr = (vm_map_offset_t)

-                       phystokv((pmap_paddr_t)VM_PAGE_GET_PHYS_PAGE(src_page)
-                                << PAGE_SHIFT);
+                   phystokv((pmap_paddr_t)VM_PAGE_GET_PHYS_PAGE(src_page)
+                       << PAGE_SHIFT);

                dst_vaddr = (vm_map_offset_t)
                dst_vaddr = (vm_map_offset_t)

-                       phystokv((pmap_paddr_t)dst_pnum << PAGE_SHIFT);
-#else
-#error "vm_paging_map_object: no 1-to-1 kernel mapping of physical memory..."
-               src_vaddr = 0;
-               dst_vaddr = 0;
-#endif
+                   phystokv((pmap_paddr_t)dst_pnum << PAGE_SHIFT);
+

                src_page_object = VM_PAGE_OBJECT(src_page);
 
                /*
                src_page_object = VM_PAGE_OBJECT(src_page);
 
                /*


@@ -534,18 +522,18 @@ apple_protect_pager_data_request(
                 */
                if (src_page_object->code_signed) {
                        vm_page_validate_cs_mapped(
                 */
                if (src_page_object->code_signed) {
                        vm_page_validate_cs_mapped(

-                               src_page,
+                               src_page, PAGE_SIZE, 0,

                                (const void *) src_vaddr);
                }
                /*
                 * ... and transfer the results to the destination page.
                 */
                UPL_SET_CS_VALIDATED(upl_pl, cur_offset / PAGE_SIZE,
                                (const void *) src_vaddr);
                }
                /*
                 * ... and transfer the results to the destination page.
                 */
                UPL_SET_CS_VALIDATED(upl_pl, cur_offset / PAGE_SIZE,

-                                    src_page->vmp_cs_validated);
+                   src_page->vmp_cs_validated);

                UPL_SET_CS_TAINTED(upl_pl, cur_offset / PAGE_SIZE,
                UPL_SET_CS_TAINTED(upl_pl, cur_offset / PAGE_SIZE,

-                                  src_page->vmp_cs_tainted);
+                   src_page->vmp_cs_tainted);

                UPL_SET_CS_NX(upl_pl, cur_offset / PAGE_SIZE,
                UPL_SET_CS_NX(upl_pl, cur_offset / PAGE_SIZE,

-                                  src_page->vmp_cs_nx);
+                   src_page->vmp_cs_nx);

 
                /*
                 * page_decrypt() might access a mapped file, so let's release
 
                /*
                 * page_decrypt() might access a mapped file, so let's release


@@ -563,42 +551,42 @@ apple_protect_pager_data_request(
                 * into the destination page.
                 */
                for (offset_in_page = 0;
                 * into the destination page.
                 */
                for (offset_in_page = 0;

-                    offset_in_page < PAGE_SIZE;
-                    offset_in_page += 4096) {
+                   offset_in_page < PAGE_SIZE;
+                   offset_in_page += 4096) {

                        if (offset + cur_offset + offset_in_page <
                            pager->crypto_start ||
                            offset + cur_offset + offset_in_page >=
                            pager->crypto_end) {
                                /* not encrypted: just copy */
                                bcopy((const char *)(src_vaddr +
                        if (offset + cur_offset + offset_in_page <
                            pager->crypto_start ||
                            offset + cur_offset + offset_in_page >=
                            pager->crypto_end) {
                                /* not encrypted: just copy */
                                bcopy((const char *)(src_vaddr +

-                                                    offset_in_page),
-                                     (char *)(dst_vaddr + offset_in_page),
-                                     4096);
+                                   offset_in_page),
+                                   (char *)(dst_vaddr + offset_in_page),
+                                   4096);

 
                                if (apple_protect_pager_data_request_debug) {
                                        printf("apple_protect_data_request"
 
                                if (apple_protect_pager_data_request_debug) {
                                        printf("apple_protect_data_request"

-                                              "(%p,0x%llx+0x%llx+0x%04llx): "
-                                              "out of crypto range "
-                                              "[0x%llx:0x%llx]: "
-                                              "COPY [0x%016llx 0x%016llx] "
-                                              "code_signed=%d "
-                                              "cs_validated=%d "
-                                              "cs_tainted=%d "
-                                              "cs_nx=%d\n",
-                                              pager,
-                                              offset,
-                                              (uint64_t) cur_offset,
-                                              (uint64_t) offset_in_page,
-                                              pager->crypto_start,
-                                              pager->crypto_end,
-                                              *(uint64_t *)(dst_vaddr+
-                                                            offset_in_page),
-                                              *(uint64_t *)(dst_vaddr+
-                                                            offset_in_page+8),
-                                              src_page_object->code_signed,
-                                              src_page->vmp_cs_validated,
-                                              src_page->vmp_cs_tainted,
-                                              src_page->vmp_cs_nx);
+                                           "(%p,0x%llx+0x%llx+0x%04llx): "
+                                           "out of crypto range "
+                                           "[0x%llx:0x%llx]: "
+                                           "COPY [0x%016llx 0x%016llx] "
+                                           "code_signed=%d "
+                                           "cs_validated=%d "
+                                           "cs_tainted=%d "
+                                           "cs_nx=%d\n",
+                                           pager,
+                                           offset,
+                                           (uint64_t) cur_offset,
+                                           (uint64_t) offset_in_page,
+                                           pager->crypto_start,
+                                           pager->crypto_end,
+                                           *(uint64_t *)(dst_vaddr +
+                                           offset_in_page),
+                                           *(uint64_t *)(dst_vaddr +
+                                           offset_in_page + 8),
+                                           src_page_object->code_signed,
+                                           src_page->vmp_cs_validated,
+                                           src_page->vmp_cs_tainted,
+                                           src_page->vmp_cs_nx);

                                }
                                ret = 0;
                                continue;
                                }
                                ret = 0;
                                continue;


@@ -607,46 +595,46 @@ apple_protect_pager_data_request(
                                (const void *)(src_vaddr + offset_in_page),
                                (void *)(dst_vaddr + offset_in_page),
                                ((pager->crypto_backing_offset -
                                (const void *)(src_vaddr + offset_in_page),
                                (void *)(dst_vaddr + offset_in_page),
                                ((pager->crypto_backing_offset -

-                                 pager->crypto_start) + /* XXX ? */
-                                offset +
-                                cur_offset +
-                                offset_in_page),
+                               pager->crypto_start) +   /* XXX ? */
+                               offset +
+                               cur_offset +
+                               offset_in_page),

                                pager->crypt_info->crypt_ops);
 
                        if (apple_protect_pager_data_request_debug) {
                                printf("apple_protect_data_request"
                                pager->crypt_info->crypt_ops);
 
                        if (apple_protect_pager_data_request_debug) {
                                printf("apple_protect_data_request"

-                                      "(%p,0x%llx+0x%llx+0x%04llx): "
-                                      "in crypto range [0x%llx:0x%llx]: "
-                                      "DECRYPT offset 0x%llx="
-                                      "(0x%llx-0x%llx+0x%llx+0x%llx+0x%04llx)"
-                                      "[0x%016llx 0x%016llx] "
-                                      "code_signed=%d "
-                                      "cs_validated=%d "
-                                      "cs_tainted=%d "
-                                      "cs_nx=%d "
-                                      "ret=0x%x\n",
-                                      pager,
-                                      offset,
-                                      (uint64_t) cur_offset,
-                                      (uint64_t) offset_in_page,
-                                      pager->crypto_start, pager->crypto_end,
-                                      ((pager->crypto_backing_offset -
-                                        pager->crypto_start) +
-                                       offset +
-                                       cur_offset +
-                                       offset_in_page),
-                                      pager->crypto_backing_offset,
-                                      pager->crypto_start,
-                                      offset,
-                                      (uint64_t) cur_offset,
-                                      (uint64_t) offset_in_page,
-                                      *(uint64_t *)(dst_vaddr+offset_in_page),
-                                      *(uint64_t *)(dst_vaddr+offset_in_page+8),
-                                      src_page_object->code_signed,
-                                      src_page->vmp_cs_validated,
-                                      src_page->vmp_cs_tainted,
-                                      src_page->vmp_cs_nx,
-                                      ret);
+                                   "(%p,0x%llx+0x%llx+0x%04llx): "
+                                   "in crypto range [0x%llx:0x%llx]: "
+                                   "DECRYPT offset 0x%llx="
+                                   "(0x%llx-0x%llx+0x%llx+0x%llx+0x%04llx)"
+                                   "[0x%016llx 0x%016llx] "
+                                   "code_signed=%d "
+                                   "cs_validated=%d "
+                                   "cs_tainted=%d "
+                                   "cs_nx=%d "
+                                   "ret=0x%x\n",
+                                   pager,
+                                   offset,
+                                   (uint64_t) cur_offset,
+                                   (uint64_t) offset_in_page,
+                                   pager->crypto_start, pager->crypto_end,
+                                   ((pager->crypto_backing_offset -
+                                   pager->crypto_start) +
+                                   offset +
+                                   cur_offset +
+                                   offset_in_page),
+                                   pager->crypto_backing_offset,
+                                   pager->crypto_start,
+                                   offset,
+                                   (uint64_t) cur_offset,
+                                   (uint64_t) offset_in_page,
+                                   *(uint64_t *)(dst_vaddr + offset_in_page),
+                                   *(uint64_t *)(dst_vaddr + offset_in_page + 8),
+                                   src_page_object->code_signed,
+                                   src_page->vmp_cs_validated,
+                                   src_page->vmp_cs_tainted,
+                                   src_page->vmp_cs_nx,
+                                   ret);

                        }
                        if (ret) {
                                break;
                        }
                        if (ret) {
                                break;


@@ -698,7 +686,7 @@ done:
                if (retval != KERN_SUCCESS) {
                        upl_abort(upl, 0);
                        if (retval == KERN_ABORTED) {
                if (retval != KERN_SUCCESS) {
                        upl_abort(upl, 0);
                        if (retval == KERN_ABORTED) {

-                               wait_result_t   wait_result;
+                               wait_result_t   wait_result;

 
                                /*
                                 * We aborted the fault and did not provide
 
                                /*
                                 * We aborted the fault and did not provide


@@ -718,7 +706,7 @@ done:
                                wait_result = assert_wait_timeout(
                                        (event_t) apple_protect_pager_data_request,
                                        THREAD_UNINT,
                                wait_result = assert_wait_timeout(
                                        (event_t) apple_protect_pager_data_request,
                                        THREAD_UNINT,

-                                       10000,  /* 10ms */
+                                       10000,  /* 10ms */

                                        NSEC_PER_USEC);
                                assert(wait_result == THREAD_WAITING);
                                wait_result = thread_block(THREAD_CONTINUE_NULL);
                                        NSEC_PER_USEC);
                                assert(wait_result == THREAD_WAITING);
                                wait_result = thread_block(THREAD_CONTINUE_NULL);


@@ -726,9 +714,12 @@ done:
                        }
                } else {
                        boolean_t empty;
                        }
                } else {
                        boolean_t empty;

-                       upl_commit_range(upl, 0, upl->size, 
-                                        UPL_COMMIT_CS_VALIDATED | UPL_COMMIT_WRITTEN_BY_KERNEL,
-                                        upl_pl, pl_count, &empty);
+                       assertf(page_aligned(upl->u_offset) && page_aligned(upl->u_size),
+                           "upl %p offset 0x%llx size 0x%x",
+                           upl, upl->u_offset, upl->u_size);
+                       upl_commit_range(upl, 0, upl->u_size,
+                           UPL_COMMIT_CS_VALIDATED | UPL_COMMIT_WRITTEN_BY_KERNEL,
+                           upl_pl, pl_count, &empty);

                }
 
                /* and deallocate the UPL */
                }
 
                /* and deallocate the UPL */


@@ -750,15 +741,14 @@ done:
  */
 void
 apple_protect_pager_reference(
  */
 void
 apple_protect_pager_reference(

-       memory_object_t         mem_obj)
-{      
-       apple_protect_pager_t   pager;
+       memory_object_t         mem_obj)
+{
+       apple_protect_pager_t   pager;

 
        pager = apple_protect_pager_lookup(mem_obj);
 
        lck_mtx_lock(&apple_protect_pager_lock);
 
        pager = apple_protect_pager_lookup(mem_obj);
 
        lck_mtx_lock(&apple_protect_pager_lock);

-       assert(pager->ref_count > 0);
-       pager->ref_count++;
+       os_ref_retain_locked_raw(&pager->ap_pgr_hdr_ref, NULL);

        lck_mtx_unlock(&apple_protect_pager_lock);
 }
 
        lck_mtx_unlock(&apple_protect_pager_lock);
 }
 


@@ -777,12 +767,12 @@ apple_protect_pager_dequeue(
        assert(!pager->is_mapped);
 
        queue_remove(&apple_protect_pager_queue,
        assert(!pager->is_mapped);
 
        queue_remove(&apple_protect_pager_queue,

-                    pager,
-                    apple_protect_pager_t,
-                    pager_queue);
+           pager,
+           apple_protect_pager_t,
+           pager_queue);

        pager->pager_queue.next = NULL;
        pager->pager_queue.prev = NULL;
        pager->pager_queue.next = NULL;
        pager->pager_queue.prev = NULL;

-       
+

        apple_protect_pager_count--;
 }
 
        apple_protect_pager_count--;
 }
 


@@ -814,9 +804,9 @@ apple_protect_pager_terminate_internal(
        /* one less pager using this "pager_crypt_info" */
 #if CRYPT_INFO_DEBUG
        printf("CRYPT_INFO %s: deallocate %p ref %d\n",
        /* one less pager using this "pager_crypt_info" */
 #if CRYPT_INFO_DEBUG
        printf("CRYPT_INFO %s: deallocate %p ref %d\n",

-              __FUNCTION__,
-              pager->crypt_info,
-              pager->crypt_info->crypt_refcnt);
+           __FUNCTION__,
+           pager->crypt_info,
+           pager->crypt_info->crypt_refcnt);

 #endif /* CRYPT_INFO_DEBUG */
        crypt_info_deallocate(pager->crypt_info);
        pager->crypt_info = NULL;
 #endif /* CRYPT_INFO_DEBUG */
        crypt_info_deallocate(pager->crypt_info);
        pager->crypt_info = NULL;


@@ -835,18 +825,19 @@ apple_protect_pager_terminate_internal(
  */
 void
 apple_protect_pager_deallocate_internal(
  */
 void
 apple_protect_pager_deallocate_internal(

-       apple_protect_pager_t   pager,
-       boolean_t               locked)
+       apple_protect_pager_t   pager,
+       boolean_t               locked)

 {
 {

-       boolean_t       needs_trimming;
-       int             count_unmapped;
+       boolean_t       needs_trimming;
+       unsigned int    count_unmapped;
+       os_ref_count_t  ref_count;

 
 

-       if (! locked) {
+       if (!locked) {

                lck_mtx_lock(&apple_protect_pager_lock);
        }
 
                lck_mtx_lock(&apple_protect_pager_lock);
        }
 

-       count_unmapped = (apple_protect_pager_count - 
-                         apple_protect_pager_count_mapped);
+       count_unmapped = (apple_protect_pager_count -
+           apple_protect_pager_count_mapped);

        if (count_unmapped > apple_protect_pager_cache_limit) {
                /* we have too many unmapped pagers:  trim some */
                needs_trimming = TRUE;
        if (count_unmapped > apple_protect_pager_cache_limit) {
                /* we have too many unmapped pagers:  trim some */
                needs_trimming = TRUE;


@@ -855,9 +846,9 @@ apple_protect_pager_deallocate_internal(
        }
 
        /* drop a reference on this pager */
        }
 
        /* drop a reference on this pager */

-       pager->ref_count--;
+       ref_count = os_ref_release_locked_raw(&pager->ap_pgr_hdr_ref, NULL);

 
 

-       if (pager->ref_count == 1) {
+       if (ref_count == 1) {

                /*
                 * Only the "named" reference is left, which means that
                 * no one is really holding on to this pager anymore.
                /*
                 * Only the "named" reference is left, which means that
                 * no one is really holding on to this pager anymore.


@@ -867,7 +858,7 @@ apple_protect_pager_deallocate_internal(
                /* the pager is all ours: no need for the lock now */
                lck_mtx_unlock(&apple_protect_pager_lock);
                apple_protect_pager_terminate_internal(pager);
                /* the pager is all ours: no need for the lock now */
                lck_mtx_unlock(&apple_protect_pager_lock);
                apple_protect_pager_terminate_internal(pager);

-       } else if (pager->ref_count == 0) {
+       } else if (ref_count == 0) {

                /*
                 * Dropped the existence reference;  the memory object has
                 * been terminated.  Do some final cleanup and release the
                /*
                 * Dropped the existence reference;  the memory object has
                 * been terminated.  Do some final cleanup and release the


@@ -878,7 +869,7 @@ apple_protect_pager_deallocate_internal(
                        memory_object_control_deallocate(pager->ap_pgr_hdr.mo_control);
                        pager->ap_pgr_hdr.mo_control = MEMORY_OBJECT_CONTROL_NULL;
                }
                        memory_object_control_deallocate(pager->ap_pgr_hdr.mo_control);
                        pager->ap_pgr_hdr.mo_control = MEMORY_OBJECT_CONTROL_NULL;
                }

-               kfree(pager, sizeof (*pager));
+               kfree(pager, sizeof(*pager));

                pager = APPLE_PROTECT_PAGER_NULL;
        } else {
                /* there are still plenty of references:  keep going... */
                pager = APPLE_PROTECT_PAGER_NULL;
        } else {
                /* there are still plenty of references:  keep going... */


@@ -899,9 +890,9 @@ apple_protect_pager_deallocate_internal(
  */
 void
 apple_protect_pager_deallocate(
  */
 void
 apple_protect_pager_deallocate(

-       memory_object_t         mem_obj)
+       memory_object_t         mem_obj)

 {
 {

-       apple_protect_pager_t   pager;
+       apple_protect_pager_t   pager;

 
        PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_deallocate: %p\n", mem_obj));
        pager = apple_protect_pager_lookup(mem_obj);
 
        PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_deallocate: %p\n", mem_obj));
        pager = apple_protect_pager_lookup(mem_obj);


@@ -916,7 +907,7 @@ apple_protect_pager_terminate(
 #if !DEBUG
        __unused
 #endif
 #if !DEBUG
        __unused
 #endif

-       memory_object_t mem_obj)
+       memory_object_t mem_obj)

 {
        PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_terminate: %p\n", mem_obj));
 
 {
        PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_terminate: %p\n", mem_obj));
 


@@ -928,10 +919,10 @@ apple_protect_pager_terminate(
  */
 kern_return_t
 apple_protect_pager_synchronize(
  */
 kern_return_t
 apple_protect_pager_synchronize(

-       __unused memory_object_t                mem_obj,
-       __unused memory_object_offset_t offset,
-       __unused memory_object_size_t           length,
-       __unused vm_sync_t              sync_flags)
+       __unused memory_object_t                mem_obj,
+       __unused memory_object_offset_t offset,
+       __unused memory_object_size_t           length,
+       __unused vm_sync_t              sync_flags)

 {
        panic("apple_protect_pager_synchronize: memory_object_synchronize no longer supported\n");
        return KERN_FAILURE;
 {
        panic("apple_protect_pager_synchronize: memory_object_synchronize no longer supported\n");
        return KERN_FAILURE;


@@ -947,10 +938,10 @@ apple_protect_pager_synchronize(
  */
 kern_return_t
 apple_protect_pager_map(
  */
 kern_return_t
 apple_protect_pager_map(

-       memory_object_t         mem_obj,
-       __unused vm_prot_t      prot)
+       memory_object_t         mem_obj,
+       __unused vm_prot_t      prot)

 {
 {

-       apple_protect_pager_t   pager;
+       apple_protect_pager_t   pager;

 
        PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_map: %p\n", mem_obj));
 
 
        PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_map: %p\n", mem_obj));
 


@@ -958,7 +949,7 @@ apple_protect_pager_map(
 
        lck_mtx_lock(&apple_protect_pager_lock);
        assert(pager->is_ready);
 
        lck_mtx_lock(&apple_protect_pager_lock);
        assert(pager->is_ready);

-       assert(pager->ref_count > 0); /* pager is alive */
+       assert(os_ref_get_count_raw(&pager->ap_pgr_hdr_ref) > 0); /* pager is alive */

        if (pager->is_mapped == FALSE) {
                /*
                 * First mapping of this pager:  take an extra reference
        if (pager->is_mapped == FALSE) {
                /*
                 * First mapping of this pager:  take an extra reference


@@ -966,7 +957,7 @@ apple_protect_pager_map(
                 * are removed.
                 */
                pager->is_mapped = TRUE;
                 * are removed.
                 */
                pager->is_mapped = TRUE;

-               pager->ref_count++;
+               os_ref_retain_locked_raw(&pager->ap_pgr_hdr_ref, NULL);

                apple_protect_pager_count_mapped++;
        }
        lck_mtx_unlock(&apple_protect_pager_lock);
                apple_protect_pager_count_mapped++;
        }
        lck_mtx_unlock(&apple_protect_pager_lock);


@@ -981,13 +972,13 @@ apple_protect_pager_map(
  */
 kern_return_t
 apple_protect_pager_last_unmap(
  */
 kern_return_t
 apple_protect_pager_last_unmap(

-       memory_object_t         mem_obj)
+       memory_object_t         mem_obj)

 {
 {

-       apple_protect_pager_t   pager;
-       int                     count_unmapped;
+       apple_protect_pager_t   pager;
+       unsigned int            count_unmapped;

 
        PAGER_DEBUG(PAGER_ALL,
 
        PAGER_DEBUG(PAGER_ALL,

-                   ("apple_protect_pager_last_unmap: %p\n", mem_obj));
+           ("apple_protect_pager_last_unmap: %p\n", mem_obj));

 
        pager = apple_protect_pager_lookup(mem_obj);
 
 
        pager = apple_protect_pager_lookup(mem_obj);
 


@@ -999,7 +990,7 @@ apple_protect_pager_last_unmap(
                 */
                apple_protect_pager_count_mapped--;
                count_unmapped = (apple_protect_pager_count -
                 */
                apple_protect_pager_count_mapped--;
                count_unmapped = (apple_protect_pager_count -

-                                 apple_protect_pager_count_mapped);
+                   apple_protect_pager_count_mapped);

                if (count_unmapped > apple_protect_pager_count_unmapped_max) {
                        apple_protect_pager_count_unmapped_max = count_unmapped;
                }
                if (count_unmapped > apple_protect_pager_count_unmapped_max) {
                        apple_protect_pager_count_unmapped_max = count_unmapped;
                }


@@ -1009,41 +1000,61 @@ apple_protect_pager_last_unmap(
        } else {
                lck_mtx_unlock(&apple_protect_pager_lock);
        }
        } else {
                lck_mtx_unlock(&apple_protect_pager_lock);
        }

-       
+

        return KERN_SUCCESS;
 }
 
        return KERN_SUCCESS;
 }
 

+boolean_t
+apple_protect_pager_backing_object(
+       memory_object_t mem_obj,
+       memory_object_offset_t offset,
+       vm_object_t *backing_object,
+       vm_object_offset_t *backing_offset)
+{
+       apple_protect_pager_t   pager;
+
+       PAGER_DEBUG(PAGER_ALL,
+           ("apple_protect_pager_backing_object: %p\n", mem_obj));
+
+       pager = apple_protect_pager_lookup(mem_obj);
+
+       *backing_object = pager->backing_object;
+       *backing_offset = pager->backing_offset + offset;
+
+       return TRUE;
+}

 
 /*
  *
  */
 apple_protect_pager_t
 apple_protect_pager_lookup(
 
 /*
  *
  */
 apple_protect_pager_t
 apple_protect_pager_lookup(

-       memory_object_t  mem_obj)
+       memory_object_t  mem_obj)

 {
 {

-       apple_protect_pager_t   pager;
+       apple_protect_pager_t   pager;

 
        assert(mem_obj->mo_pager_ops == &apple_protect_pager_ops);
        pager = (apple_protect_pager_t)(uintptr_t) mem_obj;
 
        assert(mem_obj->mo_pager_ops == &apple_protect_pager_ops);
        pager = (apple_protect_pager_t)(uintptr_t) mem_obj;

-       assert(pager->ref_count > 0);
+       assert(os_ref_get_count_raw(&pager->ap_pgr_hdr_ref) > 0);

        return pager;
 }
 
 apple_protect_pager_t
 apple_protect_pager_create(
        return pager;
 }
 
 apple_protect_pager_t
 apple_protect_pager_create(

-       vm_object_t             backing_object,
-       vm_object_offset_t      backing_offset,
-       vm_object_offset_t      crypto_backing_offset,
+       vm_object_t             backing_object,
+       vm_object_offset_t      backing_offset,
+       vm_object_offset_t      crypto_backing_offset,

        struct pager_crypt_info *crypt_info,
        struct pager_crypt_info *crypt_info,

-       vm_object_offset_t      crypto_start,
-       vm_object_offset_t      crypto_end)
+       vm_object_offset_t      crypto_start,
+       vm_object_offset_t      crypto_end,
+       boolean_t               cache_pager)

 {
 {

-       apple_protect_pager_t   pager, pager2;
-       memory_object_control_t control;
-       kern_return_t           kr;
-       struct pager_crypt_info *old_crypt_info;
+       apple_protect_pager_t   pager, pager2;
+       memory_object_control_t control;
+       kern_return_t           kr;
+       struct pager_crypt_info *old_crypt_info;

 
 

-       pager = (apple_protect_pager_t) kalloc(sizeof (*pager));
+       pager = (apple_protect_pager_t) kalloc(sizeof(*pager));

        if (pager == APPLE_PROTECT_PAGER_NULL) {
                return APPLE_PROTECT_PAGER_NULL;
        }
        if (pager == APPLE_PROTECT_PAGER_NULL) {
                return APPLE_PROTECT_PAGER_NULL;
        }


@@ -1060,9 +1071,16 @@ apple_protect_pager_create(
        pager->ap_pgr_hdr.mo_control = MEMORY_OBJECT_CONTROL_NULL;
 
        pager->is_ready = FALSE;/* not ready until it has a "name" */
        pager->ap_pgr_hdr.mo_control = MEMORY_OBJECT_CONTROL_NULL;
 
        pager->is_ready = FALSE;/* not ready until it has a "name" */

-       pager->ref_count = 1;   /* existence reference (for the cache) */
-       pager->ref_count++;     /* for the caller */
+       /* one reference for the caller */
+       os_ref_init_count_raw(&pager->ap_pgr_hdr_ref, NULL, 1);

        pager->is_mapped = FALSE;
        pager->is_mapped = FALSE;

+       if (cache_pager) {
+               /* extra reference for the cache */
+               os_ref_retain_locked_raw(&pager->ap_pgr_hdr_ref, NULL);
+               pager->is_cached = true;
+       } else {
+               pager->is_cached = false;
+       }

        pager->backing_object = backing_object;
        pager->backing_offset = backing_offset;
        pager->crypto_backing_offset = crypto_backing_offset;
        pager->backing_object = backing_object;
        pager->backing_offset = backing_offset;
        pager->crypto_backing_offset = crypto_backing_offset;


@@ -1072,14 +1090,14 @@ apple_protect_pager_create(
 
 #if CRYPT_INFO_DEBUG
        printf("CRYPT_INFO %s: crypt_info %p [%p,%p,%p,%d]\n",
 
 #if CRYPT_INFO_DEBUG
        printf("CRYPT_INFO %s: crypt_info %p [%p,%p,%p,%d]\n",

-              __FUNCTION__,
-              crypt_info,
-              crypt_info->page_decrypt,
-              crypt_info->crypt_end,
-              crypt_info->crypt_ops,
-              crypt_info->crypt_refcnt);
+           __FUNCTION__,
+           crypt_info,
+           crypt_info->page_decrypt,
+           crypt_info->crypt_end,
+           crypt_info->crypt_ops,
+           crypt_info->crypt_refcnt);

 #endif /* CRYPT_INFO_DEBUG */
 #endif /* CRYPT_INFO_DEBUG */

-       
+

        vm_object_reference(backing_object);
 
        old_crypt_info = NULL;
        vm_object_reference(backing_object);
 
        old_crypt_info = NULL;


@@ -1087,15 +1105,15 @@ apple_protect_pager_create(
        lck_mtx_lock(&apple_protect_pager_lock);
        /* see if anyone raced us to create a pager for the same object */
        queue_iterate(&apple_protect_pager_queue,
        lck_mtx_lock(&apple_protect_pager_lock);
        /* see if anyone raced us to create a pager for the same object */
        queue_iterate(&apple_protect_pager_queue,

-                     pager2,
-                     apple_protect_pager_t,
-                     pager_queue) {
+           pager2,
+           apple_protect_pager_t,
+           pager_queue) {

                if ((pager2->crypt_info->page_decrypt !=
                if ((pager2->crypt_info->page_decrypt !=

-                    crypt_info->page_decrypt) ||
+                   crypt_info->page_decrypt) ||

                    (pager2->crypt_info->crypt_end !=
                    (pager2->crypt_info->crypt_end !=

-                    crypt_info->crypt_end) ||
+                   crypt_info->crypt_end) ||

                    (pager2->crypt_info->crypt_ops !=
                    (pager2->crypt_info->crypt_ops !=

-                    crypt_info->crypt_ops)) {
+                   crypt_info->crypt_ops)) {

                        /* crypt_info contents do not match: next pager */
                        continue;
                }
                        /* crypt_info contents do not match: next pager */
                        continue;
                }


@@ -1108,16 +1126,16 @@ apple_protect_pager_create(
                        /* ... switch to that pager's crypt_info */
 #if CRYPT_INFO_DEBUG
                        printf("CRYPT_INFO %s: reference %p ref %d "
                        /* ... switch to that pager's crypt_info */
 #if CRYPT_INFO_DEBUG
                        printf("CRYPT_INFO %s: reference %p ref %d "

-                              "(create match)\n",
-                              __FUNCTION__,
-                              pager2->crypt_info,
-                              pager2->crypt_info->crypt_refcnt);
+                           "(create match)\n",
+                           __FUNCTION__,
+                           pager2->crypt_info,
+                           pager2->crypt_info->crypt_refcnt);

 #endif /* CRYPT_INFO_DEBUG */
                        old_crypt_info = pager2->crypt_info;
                        crypt_info_reference(old_crypt_info);
                        pager->crypt_info = old_crypt_info;
                }
 #endif /* CRYPT_INFO_DEBUG */
                        old_crypt_info = pager2->crypt_info;
                        crypt_info_reference(old_crypt_info);
                        pager->crypt_info = old_crypt_info;
                }

-               
+

                if (pager2->backing_object == backing_object &&
                    pager2->backing_offset == backing_offset &&
                    pager2->crypto_backing_offset == crypto_backing_offset &&
                if (pager2->backing_object == backing_object &&
                    pager2->backing_offset == backing_offset &&
                    pager2->crypto_backing_offset == crypto_backing_offset &&


@@ -1127,21 +1145,21 @@ apple_protect_pager_create(
                        break;
                }
        }
                        break;
                }
        }

-       if (! queue_end(&apple_protect_pager_queue,
-                       (queue_entry_t) pager2)) {
+       if (!queue_end(&apple_protect_pager_queue,
+           (queue_entry_t) pager2)) {

                /* we lost the race, down with the loser... */
                lck_mtx_unlock(&apple_protect_pager_lock);
                vm_object_deallocate(pager->backing_object);
                pager->backing_object = VM_OBJECT_NULL;
 #if CRYPT_INFO_DEBUG
                printf("CRYPT_INFO %s: %p ref %d (create pager match)\n",
                /* we lost the race, down with the loser... */
                lck_mtx_unlock(&apple_protect_pager_lock);
                vm_object_deallocate(pager->backing_object);
                pager->backing_object = VM_OBJECT_NULL;
 #if CRYPT_INFO_DEBUG
                printf("CRYPT_INFO %s: %p ref %d (create pager match)\n",

-                      __FUNCTION__,
-                      pager->crypt_info,
-                      pager->crypt_info->crypt_refcnt);
+                   __FUNCTION__,
+                   pager->crypt_info,
+                   pager->crypt_info->crypt_refcnt);

 #endif /* CRYPT_INFO_DEBUG */
                crypt_info_deallocate(pager->crypt_info);
                pager->crypt_info = NULL;
 #endif /* CRYPT_INFO_DEBUG */
                crypt_info_deallocate(pager->crypt_info);
                pager->crypt_info = NULL;

-               kfree(pager, sizeof (*pager));
+               kfree(pager, sizeof(*pager));

                /* ... and go with the winner */
                pager = pager2;
                /* let the winner make sure the pager gets ready */
                /* ... and go with the winner */
                pager = pager2;
                /* let the winner make sure the pager gets ready */


@@ -1150,9 +1168,9 @@ apple_protect_pager_create(
 
        /* enter new pager at the head of our list of pagers */
        queue_enter_first(&apple_protect_pager_queue,
 
        /* enter new pager at the head of our list of pagers */
        queue_enter_first(&apple_protect_pager_queue,

-                         pager,
-                         apple_protect_pager_t,
-                         pager_queue);
+           pager,
+           apple_protect_pager_t,
+           pager_queue);

        apple_protect_pager_count++;
        if (apple_protect_pager_count > apple_protect_pager_count_max) {
                apple_protect_pager_count_max = apple_protect_pager_count;
        apple_protect_pager_count++;
        if (apple_protect_pager_count > apple_protect_pager_count_max) {
                apple_protect_pager_count_max = apple_protect_pager_count;


@@ -1160,10 +1178,12 @@ apple_protect_pager_create(
        lck_mtx_unlock(&apple_protect_pager_lock);
 
        kr = memory_object_create_named((memory_object_t) pager,
        lck_mtx_unlock(&apple_protect_pager_lock);
 
        kr = memory_object_create_named((memory_object_t) pager,

-                                       0,
-                                       &control);
+           0,
+           &control);

        assert(kr == KERN_SUCCESS);
 
        assert(kr == KERN_SUCCESS);
 

+       memory_object_mark_trusted(control);
+

        lck_mtx_lock(&apple_protect_pager_lock);
        /* the new pager is now ready to be used */
        pager->is_ready = TRUE;
        lck_mtx_lock(&apple_protect_pager_lock);
        /* the new pager is now ready to be used */
        pager->is_ready = TRUE;


@@ -1177,10 +1197,10 @@ apple_protect_pager_create(
                /* we re-used an old crypt_info instead of using our new one */
 #if CRYPT_INFO_DEBUG
                printf("CRYPT_INFO %s: deallocate %p ref %d "
                /* we re-used an old crypt_info instead of using our new one */
 #if CRYPT_INFO_DEBUG
                printf("CRYPT_INFO %s: deallocate %p ref %d "

-                      "(create used old)\n",
-                      __FUNCTION__,
-                      crypt_info,
-                      crypt_info->crypt_refcnt);
+                   "(create used old)\n",
+                   __FUNCTION__,
+                   crypt_info,
+                   crypt_info->crypt_refcnt);

 #endif /* CRYPT_INFO_DEBUG */
                crypt_info_deallocate(crypt_info);
                crypt_info = NULL;
 #endif /* CRYPT_INFO_DEBUG */
                crypt_info_deallocate(crypt_info);
                crypt_info = NULL;


@@ -1198,24 +1218,25 @@ apple_protect_pager_create(
  */
 memory_object_t
 apple_protect_pager_setup(
  */
 memory_object_t
 apple_protect_pager_setup(

-       vm_object_t             backing_object,
-       vm_object_offset_t      backing_offset,
-       vm_object_offset_t      crypto_backing_offset,
+       vm_object_t             backing_object,
+       vm_object_offset_t      backing_offset,
+       vm_object_offset_t      crypto_backing_offset,

        struct pager_crypt_info *crypt_info,
        struct pager_crypt_info *crypt_info,

-       vm_object_offset_t      crypto_start,
-       vm_object_offset_t      crypto_end)
+       vm_object_offset_t      crypto_start,
+       vm_object_offset_t      crypto_end,
+       boolean_t               cache_pager)

 {
 {

-       apple_protect_pager_t   pager;
-       struct pager_crypt_info *old_crypt_info, *new_crypt_info;
+       apple_protect_pager_t   pager;
+       struct pager_crypt_info *old_crypt_info, *new_crypt_info;

 
 #if CRYPT_INFO_DEBUG
        printf("CRYPT_INFO %s: crypt_info=%p [%p,%p,%p,%d]\n",
 
 #if CRYPT_INFO_DEBUG
        printf("CRYPT_INFO %s: crypt_info=%p [%p,%p,%p,%d]\n",

-              __FUNCTION__,
-              crypt_info,
-              crypt_info->page_decrypt,
-              crypt_info->crypt_end,
-              crypt_info->crypt_ops,
-              crypt_info->crypt_refcnt);
+           __FUNCTION__,
+           crypt_info,
+           crypt_info->page_decrypt,
+           crypt_info->crypt_end,
+           crypt_info->crypt_ops,
+           crypt_info->crypt_refcnt);

 #endif /* CRYPT_INFO_DEBUG */
 
        old_crypt_info = NULL;
 #endif /* CRYPT_INFO_DEBUG */
 
        old_crypt_info = NULL;


@@ -1223,15 +1244,15 @@ apple_protect_pager_setup(
        lck_mtx_lock(&apple_protect_pager_lock);
 
        queue_iterate(&apple_protect_pager_queue,
        lck_mtx_lock(&apple_protect_pager_lock);
 
        queue_iterate(&apple_protect_pager_queue,

-                     pager,
-                     apple_protect_pager_t,
-                     pager_queue) {
+           pager,
+           apple_protect_pager_t,
+           pager_queue) {

                if ((pager->crypt_info->page_decrypt !=
                if ((pager->crypt_info->page_decrypt !=

-                    crypt_info->page_decrypt) ||
+                   crypt_info->page_decrypt) ||

                    (pager->crypt_info->crypt_end !=
                    (pager->crypt_info->crypt_end !=

-                    crypt_info->crypt_end) ||
+                   crypt_info->crypt_end) ||

                    (pager->crypt_info->crypt_ops !=
                    (pager->crypt_info->crypt_ops !=

-                    crypt_info->crypt_ops)) {
+                   crypt_info->crypt_ops)) {

                        /* no match for "crypt_info": next pager */
                        continue;
                }
                        /* no match for "crypt_info": next pager */
                        continue;
                }


@@ -1244,28 +1265,28 @@ apple_protect_pager_setup(
                        old_crypt_info = pager->crypt_info;
 #if CRYPT_INFO_DEBUG
                        printf("CRYPT_INFO %s: "
                        old_crypt_info = pager->crypt_info;
 #if CRYPT_INFO_DEBUG
                        printf("CRYPT_INFO %s: "

-                              "switching crypt_info from %p [%p,%p,%p,%d] "
-                              "to %p [%p,%p,%p,%d] from pager %p\n",
-                              __FUNCTION__,
-                              crypt_info,
-                              crypt_info->page_decrypt,
-                              crypt_info->crypt_end,
-                              crypt_info->crypt_ops,
-                              crypt_info->crypt_refcnt,
-                              old_crypt_info,
-                              old_crypt_info->page_decrypt,
-                              old_crypt_info->crypt_end,
-                              old_crypt_info->crypt_ops,
-                              old_crypt_info->crypt_refcnt,
-                              pager);
+                           "switching crypt_info from %p [%p,%p,%p,%d] "
+                           "to %p [%p,%p,%p,%d] from pager %p\n",
+                           __FUNCTION__,
+                           crypt_info,
+                           crypt_info->page_decrypt,
+                           crypt_info->crypt_end,
+                           crypt_info->crypt_ops,
+                           crypt_info->crypt_refcnt,
+                           old_crypt_info,
+                           old_crypt_info->page_decrypt,
+                           old_crypt_info->crypt_end,
+                           old_crypt_info->crypt_ops,
+                           old_crypt_info->crypt_refcnt,
+                           pager);

                        printf("CRYPT_INFO %s: %p ref %d (setup match)\n",
                        printf("CRYPT_INFO %s: %p ref %d (setup match)\n",

-                              __FUNCTION__,
-                              pager->crypt_info,
-                              pager->crypt_info->crypt_refcnt);
+                           __FUNCTION__,
+                           pager->crypt_info,
+                           pager->crypt_info->crypt_refcnt);

 #endif /* CRYPT_INFO_DEBUG */
                        crypt_info_reference(pager->crypt_info);
                }
 #endif /* CRYPT_INFO_DEBUG */
                        crypt_info_reference(pager->crypt_info);
                }

-               
+

                if (pager->backing_object == backing_object &&
                    pager->backing_offset == backing_offset &&
                    pager->crypto_backing_offset == crypto_backing_offset &&
                if (pager->backing_object == backing_object &&
                    pager->backing_offset == backing_offset &&
                    pager->crypto_backing_offset == crypto_backing_offset &&


@@ -1276,27 +1297,26 @@ apple_protect_pager_setup(
                        assert(old_crypt_info->crypt_refcnt > 1);
 #if CRYPT_INFO_DEBUG
                        printf("CRYPT_INFO %s: "
                        assert(old_crypt_info->crypt_refcnt > 1);
 #if CRYPT_INFO_DEBUG
                        printf("CRYPT_INFO %s: "

-                              "pager match with %p crypt_info %p\n",
-                              __FUNCTION__,
-                              pager,
-                              pager->crypt_info);
+                           "pager match with %p crypt_info %p\n",
+                           __FUNCTION__,
+                           pager,
+                           pager->crypt_info);

                        printf("CRYPT_INFO %s: deallocate %p ref %d "
                        printf("CRYPT_INFO %s: deallocate %p ref %d "

-                              "(pager match)\n",
-                              __FUNCTION__,
-                              old_crypt_info,
-                              old_crypt_info->crypt_refcnt);
+                           "(pager match)\n",
+                           __FUNCTION__,
+                           old_crypt_info,
+                           old_crypt_info->crypt_refcnt);

 #endif /* CRYPT_INFO_DEBUG */
                        /* release the extra ref on crypt_info we got above */
                        crypt_info_deallocate(old_crypt_info);
                        assert(old_crypt_info->crypt_refcnt > 0);
                        /* give extra reference on pager to the caller */
 #endif /* CRYPT_INFO_DEBUG */
                        /* release the extra ref on crypt_info we got above */
                        crypt_info_deallocate(old_crypt_info);
                        assert(old_crypt_info->crypt_refcnt > 0);
                        /* give extra reference on pager to the caller */

-                       assert(pager->ref_count > 0);
-                       pager->ref_count++;
+                       os_ref_retain_locked_raw(&pager->ap_pgr_hdr_ref, NULL);

                        break;
                }
        }
        if (queue_end(&apple_protect_pager_queue,
                        break;
                }
        }
        if (queue_end(&apple_protect_pager_queue,

-                     (queue_entry_t) pager)) {
+           (queue_entry_t) pager)) {

                lck_mtx_unlock(&apple_protect_pager_lock);
                /* no existing pager for this backing object */
                pager = APPLE_PROTECT_PAGER_NULL;
                lck_mtx_unlock(&apple_protect_pager_lock);
                /* no existing pager for this backing object */
                pager = APPLE_PROTECT_PAGER_NULL;


@@ -1305,20 +1325,20 @@ apple_protect_pager_setup(
                        new_crypt_info = old_crypt_info;
 #if CRYPT_INFO_DEBUG
                        printf("CRYPT_INFO %s: "
                        new_crypt_info = old_crypt_info;
 #if CRYPT_INFO_DEBUG
                        printf("CRYPT_INFO %s: "

-                              "will use old_crypt_info %p for new pager\n",
-                              __FUNCTION__,
-                              old_crypt_info);
+                           "will use old_crypt_info %p for new pager\n",
+                           __FUNCTION__,
+                           old_crypt_info);

 #endif /* CRYPT_INFO_DEBUG */
                } else {
                        /* allocate a new crypt_info for new pager */
 #endif /* CRYPT_INFO_DEBUG */
                } else {
                        /* allocate a new crypt_info for new pager */

-                       new_crypt_info = kalloc(sizeof (*new_crypt_info));
+                       new_crypt_info = kalloc(sizeof(*new_crypt_info));

                        *new_crypt_info = *crypt_info;
                        new_crypt_info->crypt_refcnt = 1;
 #if CRYPT_INFO_DEBUG
                        printf("CRYPT_INFO %s: "
                        *new_crypt_info = *crypt_info;
                        new_crypt_info->crypt_refcnt = 1;
 #if CRYPT_INFO_DEBUG
                        printf("CRYPT_INFO %s: "

-                              "will use new_crypt_info %p for new pager\n",
-                              __FUNCTION__,
-                              new_crypt_info);
+                           "will use new_crypt_info %p for new pager\n",
+                           __FUNCTION__,
+                           new_crypt_info);

 #endif /* CRYPT_INFO_DEBUG */
                }
                if (new_crypt_info == NULL) {
 #endif /* CRYPT_INFO_DEBUG */
                }
                if (new_crypt_info == NULL) {


@@ -1331,7 +1351,8 @@ apple_protect_pager_setup(
                                crypto_backing_offset,
                                new_crypt_info,
                                crypto_start,
                                crypto_backing_offset,
                                new_crypt_info,
                                crypto_start,

-                               crypto_end);
+                               crypto_end,
+                               cache_pager);

                }
                if (pager == APPLE_PROTECT_PAGER_NULL) {
                        /* could not create a new pager */
                }
                if (pager == APPLE_PROTECT_PAGER_NULL) {
                        /* could not create a new pager */


@@ -1339,10 +1360,10 @@ apple_protect_pager_setup(
                                /* release extra reference on old_crypt_info */
 #if CRYPT_INFO_DEBUG
                                printf("CRYPT_INFO %s: deallocate %p ref %d "
                                /* release extra reference on old_crypt_info */
 #if CRYPT_INFO_DEBUG
                                printf("CRYPT_INFO %s: deallocate %p ref %d "

-                                      "(create fail old_crypt_info)\n",
-                                      __FUNCTION__,
-                                      old_crypt_info, 
-                                      old_crypt_info->crypt_refcnt);
+                                   "(create fail old_crypt_info)\n",
+                                   __FUNCTION__,
+                                   old_crypt_info,
+                                   old_crypt_info->crypt_refcnt);

 #endif /* CRYPT_INFO_DEBUG */
                                crypt_info_deallocate(old_crypt_info);
                                old_crypt_info = NULL;
 #endif /* CRYPT_INFO_DEBUG */
                                crypt_info_deallocate(old_crypt_info);
                                old_crypt_info = NULL;


@@ -1351,10 +1372,10 @@ apple_protect_pager_setup(
                                assert(new_crypt_info->crypt_refcnt == 1);
 #if CRYPT_INFO_DEBUG
                                printf("CRYPT_INFO %s: deallocate %p ref %d "
                                assert(new_crypt_info->crypt_refcnt == 1);
 #if CRYPT_INFO_DEBUG
                                printf("CRYPT_INFO %s: deallocate %p ref %d "

-                                      "(create fail new_crypt_info)\n",
-                                      __FUNCTION__,
-                                      new_crypt_info,
-                                      new_crypt_info->crypt_refcnt);
+                                   "(create fail new_crypt_info)\n",
+                                   __FUNCTION__,
+                                   new_crypt_info,
+                                   new_crypt_info->crypt_refcnt);

 #endif /* CRYPT_INFO_DEBUG */
                                crypt_info_deallocate(new_crypt_info);
                                new_crypt_info = NULL;
 #endif /* CRYPT_INFO_DEBUG */
                                crypt_info_deallocate(new_crypt_info);
                                new_crypt_info = NULL;


@@ -1368,22 +1389,22 @@ apple_protect_pager_setup(
 
        while (!pager->is_ready) {
                lck_mtx_sleep(&apple_protect_pager_lock,
 
        while (!pager->is_ready) {
                lck_mtx_sleep(&apple_protect_pager_lock,

-                       LCK_SLEEP_DEFAULT,
-                       &pager->is_ready,
-                       THREAD_UNINT);
+                   LCK_SLEEP_DEFAULT,
+                   &pager->is_ready,
+                   THREAD_UNINT);

        }
        lck_mtx_unlock(&apple_protect_pager_lock);
 
        return (memory_object_t) pager;
        }
        lck_mtx_unlock(&apple_protect_pager_lock);
 
        return (memory_object_t) pager;

-}      
+}

 
 void
 apple_protect_pager_trim(void)
 {
 
 void
 apple_protect_pager_trim(void)
 {

-       apple_protect_pager_t   pager, prev_pager;
-       queue_head_t            trim_queue;
-       int                     num_trim;
-       int                     count_unmapped;
+       apple_protect_pager_t   pager, prev_pager;
+       queue_head_t            trim_queue;
+       unsigned int            num_trim;
+       unsigned int            count_unmapped;

 
        lck_mtx_lock(&apple_protect_pager_lock);
 
 
        lck_mtx_lock(&apple_protect_pager_lock);
 


@@ -1395,15 +1416,16 @@ apple_protect_pager_trim(void)
        num_trim = 0;
 
        for (pager = (apple_protect_pager_t)
        num_trim = 0;
 
        for (pager = (apple_protect_pager_t)

-                    queue_last(&apple_protect_pager_queue);
-            !queue_end(&apple_protect_pager_queue,
-                       (queue_entry_t) pager);
-            pager = prev_pager) {
+           queue_last(&apple_protect_pager_queue);
+           !queue_end(&apple_protect_pager_queue,
+           (queue_entry_t) pager);
+           pager = prev_pager) {

                /* get prev elt before we dequeue */
                prev_pager = (apple_protect_pager_t)
                /* get prev elt before we dequeue */
                prev_pager = (apple_protect_pager_t)

-                       queue_prev(&pager->pager_queue);
+                   queue_prev(&pager->pager_queue);

 
 

-               if (pager->ref_count == 2 &&
+               if (pager->is_cached &&
+                   os_ref_get_count_raw(&pager->ap_pgr_hdr_ref) == 2 &&

                    pager->is_ready &&
                    !pager->is_mapped) {
                        /* this pager can be trimmed */
                    pager->is_ready &&
                    !pager->is_mapped) {
                        /* this pager can be trimmed */


@@ -1412,12 +1434,12 @@ apple_protect_pager_trim(void)
                        apple_protect_pager_dequeue(pager);
                        /* ... and add it to our trim queue */
                        queue_enter_first(&trim_queue,
                        apple_protect_pager_dequeue(pager);
                        /* ... and add it to our trim queue */
                        queue_enter_first(&trim_queue,

-                                         pager,
-                                         apple_protect_pager_t,
-                                         pager_queue);
+                           pager,
+                           apple_protect_pager_t,
+                           pager_queue);

 
                        count_unmapped = (apple_protect_pager_count -
 
                        count_unmapped = (apple_protect_pager_count -

-                                         apple_protect_pager_count_mapped);
+                           apple_protect_pager_count_mapped);

                        if (count_unmapped <= apple_protect_pager_cache_limit) {
                                /* we have enough pagers to trim */
                                break;
                        if (count_unmapped <= apple_protect_pager_cache_limit) {
                                /* we have enough pagers to trim */
                                break;


@@ -1434,18 +1456,21 @@ apple_protect_pager_trim(void)
        /* terminate the trimmed pagers */
        while (!queue_empty(&trim_queue)) {
                queue_remove_first(&trim_queue,
        /* terminate the trimmed pagers */
        while (!queue_empty(&trim_queue)) {
                queue_remove_first(&trim_queue,

-                                  pager,
-                                  apple_protect_pager_t,
-                                  pager_queue);
+                   pager,
+                   apple_protect_pager_t,
+                   pager_queue);
+               assert(pager->is_cached);
+               pager->is_cached = false;

                pager->pager_queue.next = NULL;
                pager->pager_queue.prev = NULL;
                pager->pager_queue.next = NULL;
                pager->pager_queue.prev = NULL;

-               assert(pager->ref_count == 2);

                /*
                 * We can't call deallocate_internal() because the pager
                 * has already been dequeued, but we still need to remove
                 * a reference.
                 */
                /*
                 * We can't call deallocate_internal() because the pager
                 * has already been dequeued, but we still need to remove
                 * a reference.
                 */

-               pager->ref_count--;
+               os_ref_count_t __assert_only count;
+               count = os_ref_release_locked_raw(&pager->ap_pgr_hdr_ref, NULL);
+               assert(count == 1);

                apple_protect_pager_terminate_internal(pager);
        }
 }
                apple_protect_pager_terminate_internal(pager);
        }
 }


@@ -1458,24 +1483,24 @@ crypt_info_reference(
        assert(crypt_info->crypt_refcnt != 0);
 #if CRYPT_INFO_DEBUG
        printf("CRYPT_INFO %s: %p ref %d -> %d\n",
        assert(crypt_info->crypt_refcnt != 0);
 #if CRYPT_INFO_DEBUG
        printf("CRYPT_INFO %s: %p ref %d -> %d\n",

-              __FUNCTION__,
-              crypt_info,
-              crypt_info->crypt_refcnt,
-              crypt_info->crypt_refcnt + 1);
+           __FUNCTION__,
+           crypt_info,
+           crypt_info->crypt_refcnt,
+           crypt_info->crypt_refcnt + 1);

 #endif /* CRYPT_INFO_DEBUG */
        OSAddAtomic(+1, &crypt_info->crypt_refcnt);
 }
 
 void
 crypt_info_deallocate(
 #endif /* CRYPT_INFO_DEBUG */
        OSAddAtomic(+1, &crypt_info->crypt_refcnt);
 }
 
 void
 crypt_info_deallocate(

-       struct pager_crypt_info *crypt_info)
+       struct pager_crypt_info *crypt_info)

 {
 #if CRYPT_INFO_DEBUG
        printf("CRYPT_INFO %s: %p ref %d -> %d\n",
 {
 #if CRYPT_INFO_DEBUG
        printf("CRYPT_INFO %s: %p ref %d -> %d\n",

-              __FUNCTION__,
-              crypt_info,
-              crypt_info->crypt_refcnt,
-              crypt_info->crypt_refcnt - 1);
+           __FUNCTION__,
+           crypt_info,
+           crypt_info->crypt_refcnt,
+           crypt_info->crypt_refcnt - 1);

 #endif /* CRYPT_INFO_DEBUG */
        OSAddAtomic(-1, &crypt_info->crypt_refcnt);
        if (crypt_info->crypt_refcnt == 0) {
 #endif /* CRYPT_INFO_DEBUG */
        OSAddAtomic(-1, &crypt_info->crypt_refcnt);
        if (crypt_info->crypt_refcnt == 0) {


@@ -1486,10 +1511,10 @@ crypt_info_deallocate(
                }
 #if CRYPT_INFO_DEBUG
                printf("CRYPT_INFO %s: freeing %p\n",
                }
 #if CRYPT_INFO_DEBUG
                printf("CRYPT_INFO %s: freeing %p\n",

-                      __FUNCTION__,
-                      crypt_info);
+                   __FUNCTION__,
+                   crypt_info);

 #endif /* CRYPT_INFO_DEBUG */
 #endif /* CRYPT_INFO_DEBUG */

-               kfree(crypt_info, sizeof (*crypt_info));
+               kfree(crypt_info, sizeof(*crypt_info));

                crypt_info = NULL;
        }
 }
                crypt_info = NULL;
        }
 }