xnu-7195.101.1.tar.gz

[apple/xnu.git] / osfmk / vm / vm_shared_region_pager.c

diff --git a/osfmk/vm/vm_shared_region_pager.c b/osfmk/vm/vm_shared_region_pager.c
index 3a1bb8fc94ae342412e4197d5e9f98806f02f246..3af00d6026a28f92c7119889998c92d623c7890d 100644 (file)
--- a/osfmk/vm/vm_shared_region_pager.c
+++ b/osfmk/vm/vm_shared_region_pager.c
@@ -291,11 +291,15 @@ done:
  * the "shared_region" EMM.
  */
 typedef struct shared_region_pager {
-       struct memory_object   srp_header;          /* mandatory generic header */
+       struct memory_object    srp_header;          /* mandatory generic header */
 
        /* pager-specific data */
        queue_chain_t           srp_queue;          /* next & prev pagers */
-       uint32_t                srp_ref_count;      /* active uses */
+#if MEMORY_OBJECT_HAS_REFCOUNT
+#define srp_ref_count           srp_header.mo_ref
+#else
+       os_ref_atomic_t         srp_ref_count;      /* active uses */
+#endif
        bool                    srp_is_mapped;      /* has active mappings */
        bool                    srp_is_ready;       /* is this pager ready? */
        vm_object_t             srp_backing_object; /* VM object for shared cache */
@@ -520,7 +524,7 @@ shared_region_pager_data_request(
 
        pager = shared_region_pager_lookup(mem_obj);
        assert(pager->srp_is_ready);
-       assert(pager->srp_ref_count > 1); /* pager is alive */
+       assert(os_ref_get_count_raw(&pager->srp_ref_count) > 1); /* pager is alive */
        assert(pager->srp_is_mapped); /* pager is mapped */
 
        PAGER_DEBUG(PAGER_PAGEIN, ("shared_region_pager_data_request: %p, %llx, %x, %x, pager %p\n", mem_obj, offset, length, protection_required, pager));
@@ -545,7 +549,7 @@ shared_region_pager_data_request(
                retval = kr;
                goto done;
        }
-       dst_object = mo_control->moc_object;
+       dst_object = memory_object_control_to_vm_object(mo_control);
        assert(dst_object != VM_OBJECT_NULL);
 
        /*
@@ -842,8 +846,7 @@ shared_region_pager_reference(
        pager = shared_region_pager_lookup(mem_obj);
 
        lck_mtx_lock(&shared_region_pager_lock);
-       assert(pager->srp_ref_count > 0);
-       pager->srp_ref_count++;
+       os_ref_retain_locked_raw(&pager->srp_ref_count, NULL);
        lck_mtx_unlock(&shared_region_pager_lock);
 }
 
@@ -890,7 +893,7 @@ shared_region_pager_terminate_internal(
 {
        assert(pager->srp_is_ready);
        assert(!pager->srp_is_mapped);
-       assert(pager->srp_ref_count == 1);
+       assert(os_ref_get_count_raw(&pager->srp_ref_count) == 1);
 
        if (pager->srp_backing_object != VM_OBJECT_NULL) {
                vm_object_deallocate(pager->srp_backing_object);
@@ -914,6 +917,7 @@ shared_region_pager_deallocate_internal(
 {
        boolean_t       needs_trimming;
        int             count_unmapped;
+       os_ref_count_t  ref_count;
 
        if (!locked) {
                lck_mtx_lock(&shared_region_pager_lock);
@@ -924,10 +928,9 @@ shared_region_pager_deallocate_internal(
        needs_trimming = (count_unmapped > shared_region_pager_cache_limit);
 
        /* drop a reference on this pager */
-       assert(pager->srp_ref_count > 0);
-       pager->srp_ref_count--;
+       ref_count = os_ref_release_locked_raw(&pager->srp_ref_count, NULL);
 
-       if (pager->srp_ref_count == 1) {
+       if (ref_count == 1) {
                /*
                 * Only the "named" reference is left, which means that
                 * no one is really holding on to this pager anymore.
@@ -937,7 +940,7 @@ shared_region_pager_deallocate_internal(
                /* the pager is all ours: no need for the lock now */
                lck_mtx_unlock(&shared_region_pager_lock);
                shared_region_pager_terminate_internal(pager);
-       } else if (pager->srp_ref_count == 0) {
+       } else if (ref_count == 0) {
                /*
                 * Dropped the existence reference;  the memory object has
                 * been terminated.  Do some final cleanup and release the
@@ -1052,10 +1055,10 @@ shared_region_pager_map(
 
        lck_mtx_lock(&shared_region_pager_lock);
        assert(pager->srp_is_ready);
-       assert(pager->srp_ref_count > 0); /* pager is alive */
+       assert(os_ref_get_count_raw(&pager->srp_ref_count) > 0); /* pager is alive */
        if (!pager->srp_is_mapped) {
                pager->srp_is_mapped = TRUE;
-               pager->srp_ref_count++;
+               os_ref_retain_locked_raw(&pager->srp_ref_count, NULL);
                shared_region_pager_count_mapped++;
        }
        lck_mtx_unlock(&shared_region_pager_lock);
@@ -1133,7 +1136,7 @@ shared_region_pager_lookup(
 
        assert(mem_obj->mo_pager_ops == &shared_region_pager_ops);
        pager = (shared_region_pager_t)(uintptr_t) mem_obj;
-       assert(pager->srp_ref_count > 0);
+       assert(os_ref_get_count_raw(&pager->srp_ref_count) > 0);
        return pager;
 }
 
@@ -1173,8 +1176,8 @@ shared_region_pager_create(
        pager->srp_header.mo_control = MEMORY_OBJECT_CONTROL_NULL;
 
        pager->srp_is_ready = FALSE;/* not ready until it has a "name" */
-       pager->srp_ref_count = 1;   /* existence reference (for the cache) */
-       pager->srp_ref_count++;     /* for the caller */
+       /* existence reference (for the cache) + 1 for the caller */
+       os_ref_init_count_raw(&pager->srp_ref_count, NULL, 2);
        pager->srp_is_mapped = FALSE;
        pager->srp_backing_object = backing_object;
        pager->srp_backing_offset = backing_offset;
@@ -1318,7 +1321,8 @@ shared_region_pager_match(
                if (memcmp(si->si_slide_info_entry, slide_info->si_slide_info_entry, si->si_slide_info_size) != 0) {
                        continue;
                }
-               ++pager->srp_ref_count; /* the caller expects a reference on this */
+               /* the caller expects a reference on this */
+               os_ref_retain_locked_raw(&pager->srp_ref_count, NULL);
                lck_mtx_unlock(&shared_region_pager_lock);
                return (memory_object_t)pager;
        }
@@ -1366,7 +1370,7 @@ shared_region_pager_trim(void)
                /* get prev elt before we dequeue */
                prev_pager = (shared_region_pager_t)queue_prev(&pager->srp_queue);
 
-               if (pager->srp_ref_count == 2 &&
+               if (os_ref_get_count_raw(&pager->srp_ref_count) == 2 &&
                    pager->srp_is_ready &&
                    !pager->srp_is_mapped) {
                        /* this pager can be trimmed */
@@ -1401,13 +1405,13 @@ shared_region_pager_trim(void)
                    srp_queue);
                pager->srp_queue.next = NULL;
                pager->srp_queue.prev = NULL;
-               assert(pager->srp_ref_count == 2);
+               assert(os_ref_get_count_raw(&pager->srp_ref_count) == 2);
                /*
                 * We can't call deallocate_internal() because the pager
                 * has already been dequeued, but we still need to remove
                 * a reference.
                 */
-               pager->srp_ref_count--;
+               (void)os_ref_release_locked_raw(&pager->srp_ref_count, NULL);
                shared_region_pager_terminate_internal(pager);
        }
 }