xnu-7195.50.7.100.1.tar.gz

[apple/xnu.git] / osfmk / vm / vm_compressor_pager.c

diff --git a/osfmk/vm/vm_compressor_pager.c b/osfmk/vm/vm_compressor_pager.c
index 6eda976840f17a04fa91e3dd0d40504da5ec12d2..a989ed0c2af43d9b45d671d5f3cee6f7e3cd93fd 100644 (file)
--- a/osfmk/vm/vm_compressor_pager.c
+++ b/osfmk/vm/vm_compressor_pager.c
@@ -1,5 +1,5 @@
 /*
 /*

- * Copyright (c) 2013 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2019-2020 Apple Inc. All rights reserved.

  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  *
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  *


@@ -63,6 +63,8 @@
 #include <kern/kalloc.h>
 #include <kern/ipc_kobject.h>
 
 #include <kern/kalloc.h>
 #include <kern/ipc_kobject.h>
 

+#include <machine/atomic.h>
+

 #include <mach/memory_object_control.h>
 #include <mach/memory_object_types.h>
 #include <mach/upl.h>
 #include <mach/memory_object_control.h>
 #include <mach/memory_object_types.h>
 #include <mach/upl.h>


@@ -119,19 +121,19 @@ kern_return_t compressor_memory_object_data_reclaim(
        __unused boolean_t              reclaim_backing_store);
 
 const struct memory_object_pager_ops compressor_pager_ops = {
        __unused boolean_t              reclaim_backing_store);
 
 const struct memory_object_pager_ops compressor_pager_ops = {

-       compressor_memory_object_reference,
-       compressor_memory_object_deallocate,
-       compressor_memory_object_init,
-       compressor_memory_object_terminate,
-       compressor_memory_object_data_request,
-       compressor_memory_object_data_return,
-       compressor_memory_object_data_initialize,
-       compressor_memory_object_data_unlock,
-       compressor_memory_object_synchronize,
-       compressor_memory_object_map,
-       compressor_memory_object_last_unmap,
-       compressor_memory_object_data_reclaim,
-       "compressor pager"
+       .memory_object_reference = compressor_memory_object_reference,
+       .memory_object_deallocate = compressor_memory_object_deallocate,
+       .memory_object_init = compressor_memory_object_init,
+       .memory_object_terminate = compressor_memory_object_terminate,
+       .memory_object_data_request = compressor_memory_object_data_request,
+       .memory_object_data_return = compressor_memory_object_data_return,
+       .memory_object_data_initialize = compressor_memory_object_data_initialize,
+       .memory_object_data_unlock = compressor_memory_object_data_unlock,
+       .memory_object_synchronize = compressor_memory_object_synchronize,
+       .memory_object_map = compressor_memory_object_map,
+       .memory_object_last_unmap = compressor_memory_object_last_unmap,
+       .memory_object_data_reclaim = compressor_memory_object_data_reclaim,
+       .memory_object_pager_name = "compressor pager"

 };
 
 /* internal data structures */
 };
 
 /* internal data structures */


@@ -176,16 +178,14 @@ typedef struct compressor_pager {
 
 zone_t compressor_pager_zone;
 
 
 zone_t compressor_pager_zone;
 

-lck_grp_t       compressor_pager_lck_grp;
-lck_grp_attr_t  compressor_pager_lck_grp_attr;
-lck_attr_t      compressor_pager_lck_attr;
+LCK_GRP_DECLARE(compressor_pager_lck_grp, "compressor_pager");

 
 #define compressor_pager_lock(_cpgr_) \
        lck_mtx_lock(&(_cpgr_)->cpgr_lock)
 #define compressor_pager_unlock(_cpgr_) \
        lck_mtx_unlock(&(_cpgr_)->cpgr_lock)
 #define compressor_pager_lock_init(_cpgr_) \
 
 #define compressor_pager_lock(_cpgr_) \
        lck_mtx_lock(&(_cpgr_)->cpgr_lock)
 #define compressor_pager_unlock(_cpgr_) \
        lck_mtx_unlock(&(_cpgr_)->cpgr_lock)
 #define compressor_pager_lock_init(_cpgr_) \

-       lck_mtx_init(&(_cpgr_)->cpgr_lock, &compressor_pager_lck_grp, &compressor_pager_lck_attr)
+       lck_mtx_init(&(_cpgr_)->cpgr_lock, &compressor_pager_lck_grp, LCK_ATTR_NULL)

 #define compressor_pager_lock_destroy(_cpgr_) \
        lck_mtx_destroy(&(_cpgr_)->cpgr_lock, &compressor_pager_lck_grp)
 
 #define compressor_pager_lock_destroy(_cpgr_) \
        lck_mtx_destroy(&(_cpgr_)->cpgr_lock, &compressor_pager_lck_grp)
 


@@ -203,6 +203,35 @@ void compressor_pager_slot_lookup(
        memory_object_offset_t  offset,
        compressor_slot_t       **slot_pp);
 
        memory_object_offset_t  offset,
        compressor_slot_t       **slot_pp);
 

+#if     defined(__LP64__)
+
+/* restricted VA zones for slots */
+
+#define NUM_SLOTS_ZONES         3
+
+static const size_t compressor_slots_zones_sizes[NUM_SLOTS_ZONES] = {
+       16,
+       64,
+       COMPRESSOR_SLOTS_CHUNK_SIZE
+};
+
+static const char * compressor_slots_zones_names[NUM_SLOTS_ZONES] = {
+       "compressor_slots.16",
+       "compressor_slots.64",
+       "compressor_slots.512"
+};
+
+static zone_t
+    compressor_slots_zones[NUM_SLOTS_ZONES];
+
+#endif /* defined(__LP64__) */
+
+static void
+zfree_slot_array(compressor_slot_t *slots, size_t size);
+static compressor_slot_t *
+zalloc_slot_array(size_t size, zalloc_flags_t);
+
+

 kern_return_t
 compressor_memory_object_init(
        memory_object_t         mem_obj,
 kern_return_t
 compressor_memory_object_init(
        memory_object_t         mem_obj,


@@ -372,10 +401,10 @@ compressor_memory_object_deallocate(
                                        0,
                                        NULL);
                                pager->cpgr_slots.cpgr_islots[i] = NULL;
                                        0,
                                        NULL);
                                pager->cpgr_slots.cpgr_islots[i] = NULL;

-                               kfree(chunk, COMPRESSOR_SLOTS_CHUNK_SIZE);
+                               zfree_slot_array(chunk, COMPRESSOR_SLOTS_CHUNK_SIZE);

                        }
                }
                        }
                }

-               kfree(pager->cpgr_slots.cpgr_islots,
+               kheap_free(KHEAP_DEFAULT, pager->cpgr_slots.cpgr_islots,

                    num_chunks * sizeof(pager->cpgr_slots.cpgr_islots[0]));
                pager->cpgr_slots.cpgr_islots = NULL;
        } else if (pager->cpgr_num_slots > 2) {
                    num_chunks * sizeof(pager->cpgr_slots.cpgr_islots[0]));
                pager->cpgr_slots.cpgr_islots = NULL;
        } else if (pager->cpgr_num_slots > 2) {


@@ -387,7 +416,7 @@ compressor_memory_object_deallocate(
                        0,
                        NULL);
                pager->cpgr_slots.cpgr_dslots = NULL;
                        0,
                        NULL);
                pager->cpgr_slots.cpgr_dslots = NULL;

-               kfree(chunk,
+               zfree_slot_array(chunk,

                    (pager->cpgr_num_slots *
                    sizeof(pager->cpgr_slots.cpgr_dslots[0])));
        } else {
                    (pager->cpgr_num_slots *
                    sizeof(pager->cpgr_slots.cpgr_dslots[0])));
        } else {


@@ -555,11 +584,12 @@ compressor_memory_object_create(
 
        num_chunks = (pager->cpgr_num_slots + COMPRESSOR_SLOTS_PER_CHUNK - 1) / COMPRESSOR_SLOTS_PER_CHUNK;
        if (num_chunks > 1) {
 
        num_chunks = (pager->cpgr_num_slots + COMPRESSOR_SLOTS_PER_CHUNK - 1) / COMPRESSOR_SLOTS_PER_CHUNK;
        if (num_chunks > 1) {

-               pager->cpgr_slots.cpgr_islots = kalloc(num_chunks * sizeof(pager->cpgr_slots.cpgr_islots[0]));
-               bzero(pager->cpgr_slots.cpgr_islots, num_chunks * sizeof(pager->cpgr_slots.cpgr_islots[0]));
+               pager->cpgr_slots.cpgr_islots = kheap_alloc(KHEAP_DEFAULT,
+                   num_chunks * sizeof(pager->cpgr_slots.cpgr_islots[0]),
+                   Z_WAITOK | Z_ZERO);

        } else if (pager->cpgr_num_slots > 2) {
        } else if (pager->cpgr_num_slots > 2) {

-               pager->cpgr_slots.cpgr_dslots = kalloc(pager->cpgr_num_slots * sizeof(pager->cpgr_slots.cpgr_dslots[0]));
-               bzero(pager->cpgr_slots.cpgr_dslots, pager->cpgr_num_slots * sizeof(pager->cpgr_slots.cpgr_dslots[0]));
+               pager->cpgr_slots.cpgr_dslots = zalloc_slot_array(pager->cpgr_num_slots *
+                   sizeof(pager->cpgr_slots.cpgr_dslots[0]), Z_WAITOK | Z_ZERO);

        } else {
                pager->cpgr_slots.cpgr_eslots[0] = 0;
                pager->cpgr_slots.cpgr_eslots[1] = 0;
        } else {
                pager->cpgr_slots.cpgr_eslots[0] = 0;
                pager->cpgr_slots.cpgr_eslots[1] = 0;


@@ -647,8 +677,8 @@ compressor_pager_slot_lookup(
                chunk = pager->cpgr_slots.cpgr_islots[chunk_idx];
 
                if (chunk == NULL && do_alloc) {
                chunk = pager->cpgr_slots.cpgr_islots[chunk_idx];
 
                if (chunk == NULL && do_alloc) {

-                       t_chunk = kalloc(COMPRESSOR_SLOTS_CHUNK_SIZE);
-                       bzero(t_chunk, COMPRESSOR_SLOTS_CHUNK_SIZE);
+                       t_chunk = zalloc_slot_array(COMPRESSOR_SLOTS_CHUNK_SIZE,
+                           Z_WAITOK | Z_ZERO);

 
                        compressor_pager_lock(pager);
 
 
                        compressor_pager_lock(pager);
 


@@ -662,7 +692,7 @@ compressor_pager_slot_lookup(
                                 * This memory barrier should take care of this
                                 * according to the platform requirements.
                                 */
                                 * This memory barrier should take care of this
                                 * according to the platform requirements.
                                 */

-                               __c11_atomic_thread_fence(memory_order_release);
+                               os_atomic_thread_fence(release);

 
                                chunk = pager->cpgr_slots.cpgr_islots[chunk_idx] = t_chunk;
                                t_chunk = NULL;
 
                                chunk = pager->cpgr_slots.cpgr_islots[chunk_idx] = t_chunk;
                                t_chunk = NULL;


@@ -670,7 +700,7 @@ compressor_pager_slot_lookup(
                        compressor_pager_unlock(pager);
 
                        if (t_chunk) {
                        compressor_pager_unlock(pager);
 
                        if (t_chunk) {

-                               kfree(t_chunk, COMPRESSOR_SLOTS_CHUNK_SIZE);
+                               zfree_slot_array(t_chunk, COMPRESSOR_SLOTS_CHUNK_SIZE);

                        }
                }
                if (chunk == NULL) {
                        }
                }
                if (chunk == NULL) {


@@ -691,19 +721,68 @@ compressor_pager_slot_lookup(
 void
 vm_compressor_pager_init(void)
 {
 void
 vm_compressor_pager_init(void)
 {

-       lck_grp_attr_setdefault(&compressor_pager_lck_grp_attr);
-       lck_grp_init(&compressor_pager_lck_grp, "compressor_pager", &compressor_pager_lck_grp_attr);
-       lck_attr_setdefault(&compressor_pager_lck_attr);
-
-       compressor_pager_zone = zinit(sizeof(struct compressor_pager),
-           10000 * sizeof(struct compressor_pager),
-           8192, "compressor_pager");
-       zone_change(compressor_pager_zone, Z_CALLERACCT, FALSE);
-       zone_change(compressor_pager_zone, Z_NOENCRYPT, TRUE);
+       /* embedded slot pointers in compressor_pager get packed, so VA restricted */
+       compressor_pager_zone = zone_create_ext("compressor_pager",
+           sizeof(struct compressor_pager), ZC_NOENCRYPT,
+           ZONE_ID_ANY, ^(zone_t z){
+#if defined(__LP64__)
+               zone_set_submap_idx(z, Z_SUBMAP_IDX_VA_RESTRICTED_MAP);
+#else
+               (void)z;
+#endif /* defined(__LP64__) */
+       });
+
+#if defined(__LP64__)
+       for (unsigned int idx = 0; idx < NUM_SLOTS_ZONES; idx++) {
+               compressor_slots_zones[idx] = zone_create_ext(
+                       compressor_slots_zones_names[idx],
+                       compressor_slots_zones_sizes[idx], ZC_NONE,
+                       ZONE_ID_ANY, ^(zone_t z){
+                       zone_set_submap_idx(z, Z_SUBMAP_IDX_VA_RESTRICTED_MAP);
+               });
+       }
+#endif /* defined(__LP64__) */

 
        vm_compressor_init();
 }
 
 
        vm_compressor_init();
 }
 

+static compressor_slot_t *
+zalloc_slot_array(size_t size, zalloc_flags_t flags)
+{
+#if defined(__LP64__)
+       compressor_slot_t *slots = NULL;
+
+       assert(size <= COMPRESSOR_SLOTS_CHUNK_SIZE);
+       for (unsigned int idx = 0; idx < NUM_SLOTS_ZONES; idx++) {
+               if (size > compressor_slots_zones_sizes[idx]) {
+                       continue;
+               }
+               slots = zalloc_flags(compressor_slots_zones[idx], flags);
+               break;
+       }
+       return slots;
+#else  /* defined(__LP64__) */
+       return kheap_alloc(KHEAP_DATA_BUFFERS, size, flags);
+#endif /* !defined(__LP64__) */
+}
+
+static void
+zfree_slot_array(compressor_slot_t *slots, size_t size)
+{
+#if defined(__LP64__)
+       assert(size <= COMPRESSOR_SLOTS_CHUNK_SIZE);
+       for (unsigned int idx = 0; idx < NUM_SLOTS_ZONES; idx++) {
+               if (size > compressor_slots_zones_sizes[idx]) {
+                       continue;
+               }
+               zfree(compressor_slots_zones[idx], slots);
+               break;
+       }
+#else  /* defined(__LP64__) */
+       kheap_free(KHEAP_DATA_BUFFERS, slots, size);
+#endif /* !defined(__LP64__) */
+}
+

 kern_return_t
 vm_compressor_pager_put(
        memory_object_t                 mem_obj,
 kern_return_t
 vm_compressor_pager_put(
        memory_object_t                 mem_obj,


@@ -948,7 +1027,7 @@ vm_compressor_pager_reap_pages(
                                        &failures);
                                if (failures == 0) {
                                        pager->cpgr_slots.cpgr_islots[i] = NULL;
                                        &failures);
                                if (failures == 0) {
                                        pager->cpgr_slots.cpgr_islots[i] = NULL;

-                                       kfree(chunk, COMPRESSOR_SLOTS_CHUNK_SIZE);
+                                       zfree_slot_array(chunk, COMPRESSOR_SLOTS_CHUNK_SIZE);

                                }
                        }
                }
                                }
                        }
                }


@@ -1174,3 +1253,29 @@ vm_compressor_pager_relocate(
        return vm_compressor_relocate(current_chead, slot_p);
 }
 #endif /* CONFIG_FREEZE */
        return vm_compressor_relocate(current_chead, slot_p);
 }
 #endif /* CONFIG_FREEZE */

+
+#if DEVELOPMENT || DEBUG
+
+kern_return_t
+vm_compressor_pager_inject_error(memory_object_t mem_obj,
+    memory_object_offset_t offset)
+{
+       kern_return_t result = KERN_FAILURE;
+       compressor_slot_t *slot_p;
+       compressor_pager_t pager;
+
+       assert(mem_obj);
+
+       compressor_pager_lookup(mem_obj, pager);
+       if (pager != NULL) {
+               compressor_pager_slot_lookup(pager, FALSE, offset, &slot_p);
+               if (slot_p != NULL && *slot_p != 0) {
+                       vm_compressor_inject_error(slot_p);
+                       result = KERN_SUCCESS;
+               }
+       }
+
+       return result;
+}
+
+#endif