xnu-4570.61.1.tar.gz

[apple/xnu.git] / osfmk / vm / vm_compressor_pager.c

diff --git a/osfmk/vm/vm_compressor_pager.c b/osfmk/vm/vm_compressor_pager.c
index f4a1124cec5645ea7580aeca30c62fb8eee51965..ae0195c861f7e4fc9256f5eb13c942be921d832e 100644 (file)
--- a/osfmk/vm/vm_compressor_pager.c
+++ b/osfmk/vm/vm_compressor_pager.c
@@ -61,10 +61,10 @@
 
 #include <kern/host_statistics.h>
 #include <kern/kalloc.h>
 
 #include <kern/host_statistics.h>
 #include <kern/kalloc.h>

+#include <kern/ipc_kobject.h>

 
 #include <mach/memory_object_control.h>
 #include <mach/memory_object_types.h>
 
 #include <mach/memory_object_control.h>
 #include <mach/memory_object_types.h>

-#include <mach/memory_object_server.h>

 #include <mach/upl.h>
 
 #include <vm/memory_object.h>
 #include <mach/upl.h>
 
 #include <vm/memory_object.h>


@@ -149,11 +149,11 @@ struct {
 typedef int compressor_slot_t;
 
 typedef struct compressor_pager {
 typedef int compressor_slot_t;
 
 typedef struct compressor_pager {

-       struct ipc_object_header        cpgr_pager_header; /* fake ip_kotype */
-       memory_object_pager_ops_t       cpgr_pager_ops; /* == &compressor_pager_ops */
-       memory_object_control_t         cpgr_control;
-       lck_mtx_t                       cpgr_lock;
+       /* mandatory generic header */
+       struct memory_object cpgr_hdr;

 
 

+       /* pager-specific data */
+       lck_mtx_t                       cpgr_lock;

        unsigned int                    cpgr_references;
        unsigned int                    cpgr_num_slots;
        unsigned int                    cpgr_num_slots_occupied;
        unsigned int                    cpgr_references;
        unsigned int                    cpgr_num_slots;
        unsigned int                    cpgr_num_slots_occupied;


@@ -218,9 +218,9 @@ compressor_memory_object_init(
        compressor_pager_lookup(mem_obj, pager);
        compressor_pager_lock(pager);
 
        compressor_pager_lookup(mem_obj, pager);
        compressor_pager_lock(pager);
 

-       if (pager->cpgr_control != MEMORY_OBJECT_CONTROL_NULL)
+       if (pager->cpgr_hdr.mo_control != MEMORY_OBJECT_CONTROL_NULL)

                panic("compressor_memory_object_init: bad request");
                panic("compressor_memory_object_init: bad request");

-       pager->cpgr_control = control;
+       pager->cpgr_hdr.mo_control = control;

 
        compressor_pager_unlock(pager);
 
 
        compressor_pager_unlock(pager);
 


@@ -229,18 +229,13 @@ compressor_memory_object_init(
 
 kern_return_t
 compressor_memory_object_synchronize(
 
 kern_return_t
 compressor_memory_object_synchronize(

-       memory_object_t         mem_obj,
-       memory_object_offset_t  offset,
-       memory_object_size_t            length,
+       __unused memory_object_t        mem_obj,
+       __unused memory_object_offset_t offset,
+       __unused memory_object_size_t   length,

        __unused vm_sync_t              flags)
 {
        __unused vm_sync_t              flags)
 {

-       compressor_pager_t      pager;
-
-       compressor_pager_lookup(mem_obj, pager);
-
-       memory_object_synchronize_completed(pager->cpgr_control, offset, length);
-
-       return KERN_SUCCESS;
+       panic("compressor_memory_object_synchronize: memory_object_synchronize no longer supported\n");
+       return KERN_FAILURE;

 }
 
 kern_return_t
 }
 
 kern_return_t


@@ -290,8 +285,8 @@ compressor_memory_object_terminate(
         * to prepare for a new init.
         */
 
         * to prepare for a new init.
         */
 

-       control = pager->cpgr_control;
-       pager->cpgr_control = MEMORY_OBJECT_CONTROL_NULL;
+       control = pager->cpgr_hdr.mo_control;
+       pager->cpgr_hdr.mo_control = MEMORY_OBJECT_CONTROL_NULL;

 
        compressor_pager_unlock(pager);
 
 
        compressor_pager_unlock(pager);
 


@@ -346,7 +341,7 @@ compressor_memory_object_deallocate(
         * We shouldn't get a deallocation call
         * when the kernel has the object cached.
         */
         * We shouldn't get a deallocation call
         * when the kernel has the object cached.
         */

-       if (pager->cpgr_control != MEMORY_OBJECT_CONTROL_NULL)
+       if (pager->cpgr_hdr.mo_control != MEMORY_OBJECT_CONTROL_NULL)

                panic("compressor_memory_object_deallocate(): bad request");
 
        /*
                panic("compressor_memory_object_deallocate(): bad request");
 
        /*


@@ -442,7 +437,7 @@ compressor_memory_object_data_request(
        /* find the compressor slot for that page */
        compressor_pager_slot_lookup(pager, FALSE, offset, &slot_p);
 
        /* find the compressor slot for that page */
        compressor_pager_slot_lookup(pager, FALSE, offset, &slot_p);
 

-       if (offset / PAGE_SIZE > pager->cpgr_num_slots) {
+       if (offset / PAGE_SIZE >= pager->cpgr_num_slots) {

                /* out of range */
                kr = KERN_FAILURE;
        } else if (slot_p == NULL || *slot_p == 0) {
                /* out of range */
                kr = KERN_FAILURE;
        } else if (slot_p == NULL || *slot_p == 0) {


@@ -549,7 +544,6 @@ compressor_memory_object_create(
        }
 
        compressor_pager_lock_init(pager);
        }
 
        compressor_pager_lock_init(pager);

-       pager->cpgr_control = MEMORY_OBJECT_CONTROL_NULL;

        pager->cpgr_references = 1;
        pager->cpgr_num_slots = (uint32_t)(new_size/PAGE_SIZE);
        pager->cpgr_num_slots_occupied = 0;
        pager->cpgr_references = 1;
        pager->cpgr_num_slots = (uint32_t)(new_size/PAGE_SIZE);
        pager->cpgr_num_slots_occupied = 0;


@@ -570,9 +564,9 @@ compressor_memory_object_create(
         * Set up associations between this memory object
         * and this compressor_pager structure
         */
         * Set up associations between this memory object
         * and this compressor_pager structure
         */

-
-       pager->cpgr_pager_ops = &compressor_pager_ops;
-       pager->cpgr_pager_header.io_bits = IKOT_MEMORY_OBJECT;
+       pager->cpgr_hdr.mo_ikot = IKOT_MEMORY_OBJECT;
+       pager->cpgr_hdr.mo_pager_ops = &compressor_pager_ops;
+       pager->cpgr_hdr.mo_control = MEMORY_OBJECT_CONTROL_NULL;

 
        *new_mem_obj = (memory_object_t) pager;
        return KERN_SUCCESS;
 
        *new_mem_obj = (memory_object_t) pager;
        return KERN_SUCCESS;


@@ -633,7 +627,7 @@ compressor_pager_slot_lookup(
                *slot_pp = NULL;
                return;
        }
                *slot_pp = NULL;
                return;
        }

-       if (page_num > pager->cpgr_num_slots) {
+       if (page_num >= pager->cpgr_num_slots) {

                /* out of range */
                *slot_pp = NULL;
                return;
                /* out of range */
                *slot_pp = NULL;
                return;


@@ -651,6 +645,18 @@ compressor_pager_slot_lookup(
                        compressor_pager_lock(pager);
 
                        if ((chunk = pager->cpgr_slots.cpgr_islots[chunk_idx]) == NULL) {
                        compressor_pager_lock(pager);
 
                        if ((chunk = pager->cpgr_slots.cpgr_islots[chunk_idx]) == NULL) {

+
+                               /*
+                                * On some platforms, the memory stores from
+                                * the bzero(t_chunk) above might not have been
+                                * made visible and another thread might see
+                                * the contents of this new chunk before it's
+                                * been fully zero-filled.
+                                * This memory barrier should take care of this
+                                * according to the platform requirements.
+                                */
+                               __c11_atomic_thread_fence(memory_order_release);
+

                                chunk = pager->cpgr_slots.cpgr_islots[chunk_idx] = t_chunk;
                                t_chunk = NULL;
                        }
                                chunk = pager->cpgr_slots.cpgr_islots[chunk_idx] = t_chunk;
                                t_chunk = NULL;
                        }


@@ -776,7 +782,7 @@ vm_compressor_pager_get(
        /* find the compressor slot for that page */
        compressor_pager_slot_lookup(pager, FALSE, offset, &slot_p);
 
        /* find the compressor slot for that page */
        compressor_pager_slot_lookup(pager, FALSE, offset, &slot_p);
 

-       if (offset / PAGE_SIZE > pager->cpgr_num_slots) {
+       if (offset / PAGE_SIZE >= pager->cpgr_num_slots) {

                /* out of range */
                kr = KERN_MEMORY_FAILURE;
        } else if (slot_p == NULL || *slot_p == 0) {
                /* out of range */
                kr = KERN_MEMORY_FAILURE;
        } else if (slot_p == NULL || *slot_p == 0) {


@@ -828,6 +834,8 @@ vm_compressor_pager_state_clr(
        compressor_slot_t       *slot_p;
        unsigned int            num_slots_freed;
        
        compressor_slot_t       *slot_p;
        unsigned int            num_slots_freed;
        

+       assert(VM_CONFIG_COMPRESSOR_IS_PRESENT);
+

        compressor_pager_stats.state_clr++;
 
        if ((uint32_t)(offset/PAGE_SIZE) != (offset/PAGE_SIZE)) {
        compressor_pager_stats.state_clr++;
 
        if ((uint32_t)(offset/PAGE_SIZE) != (offset/PAGE_SIZE)) {


@@ -859,6 +867,8 @@ vm_compressor_pager_state_get(
 {
        compressor_pager_t      pager;
        compressor_slot_t       *slot_p;
 {
        compressor_pager_t      pager;
        compressor_slot_t       *slot_p;

+
+       assert(VM_CONFIG_COMPRESSOR_IS_PRESENT);

        
        compressor_pager_stats.state_get++;
 
        
        compressor_pager_stats.state_get++;
 


@@ -874,7 +884,7 @@ vm_compressor_pager_state_get(
        /* find the compressor slot for that page */
        compressor_pager_slot_lookup(pager, FALSE, offset, &slot_p);
 
        /* find the compressor slot for that page */
        compressor_pager_slot_lookup(pager, FALSE, offset, &slot_p);
 

-       if (offset / PAGE_SIZE > pager->cpgr_num_slots) {
+       if (offset / PAGE_SIZE >= pager->cpgr_num_slots) {

                /* out of range */
                return VM_EXTERNAL_STATE_ABSENT;
        } else if (slot_p == NULL || *slot_p == 0) {
                /* out of range */
                return VM_EXTERNAL_STATE_ABSENT;
        } else if (slot_p == NULL || *slot_p == 0) {


@@ -963,7 +973,7 @@ vm_compressor_pager_transfer(
        /* find the compressor slot for the destination */
        assert((uint32_t) dst_offset == dst_offset);
        compressor_pager_lookup(dst_mem_obj, dst_pager);
        /* find the compressor slot for the destination */
        assert((uint32_t) dst_offset == dst_offset);
        compressor_pager_lookup(dst_mem_obj, dst_pager);

-       assert(dst_offset / PAGE_SIZE <= dst_pager->cpgr_num_slots);
+       assert(dst_offset / PAGE_SIZE < dst_pager->cpgr_num_slots);

        compressor_pager_slot_lookup(dst_pager, TRUE, (uint32_t) dst_offset,
                                     &dst_slot_p);
        assert(dst_slot_p != NULL);
        compressor_pager_slot_lookup(dst_pager, TRUE, (uint32_t) dst_offset,
                                     &dst_slot_p);
        assert(dst_slot_p != NULL);


@@ -972,7 +982,7 @@ vm_compressor_pager_transfer(
        /* find the compressor slot for the source */
        assert((uint32_t) src_offset == src_offset);
        compressor_pager_lookup(src_mem_obj, src_pager);
        /* find the compressor slot for the source */
        assert((uint32_t) src_offset == src_offset);
        compressor_pager_lookup(src_mem_obj, src_pager);

-       assert(src_offset / PAGE_SIZE <= src_pager->cpgr_num_slots);
+       assert(src_offset / PAGE_SIZE < src_pager->cpgr_num_slots);

        compressor_pager_slot_lookup(src_pager, FALSE, (uint32_t) src_offset,
                                     &src_slot_p);
        assert(src_slot_p != NULL);
        compressor_pager_slot_lookup(src_pager, FALSE, (uint32_t) src_offset,
                                     &src_slot_p);
        assert(src_slot_p != NULL);


@@ -1003,7 +1013,7 @@ vm_compressor_pager_next_compressed(
                /* overflow */
                return (memory_object_offset_t) -1;
        }
                /* overflow */
                return (memory_object_offset_t) -1;
        }

-       if (page_num > pager->cpgr_num_slots) {
+       if (page_num >= pager->cpgr_num_slots) {

                /* out of range */
                return (memory_object_offset_t) -1;
        }
                /* out of range */
                return (memory_object_offset_t) -1;
        }


@@ -1052,7 +1062,7 @@ vm_compressor_pager_next_compressed(
                                next_slot = ((chunk_idx *
                                              COMPRESSOR_SLOTS_PER_CHUNK) +
                                             slot_idx);
                                next_slot = ((chunk_idx *
                                              COMPRESSOR_SLOTS_PER_CHUNK) +
                                             slot_idx);

-                               if (next_slot > pager->cpgr_num_slots) {
+                               if (next_slot >= pager->cpgr_num_slots) {

                                        /* went beyond end of object */
                                        return (memory_object_offset_t) -1;
                                }
                                        /* went beyond end of object */
                                        return (memory_object_offset_t) -1;
                                }