xnu-7195.101.1.tar.gz

[apple/xnu.git] / osfmk / vm / vm_fourk_pager.c

diff --git a/osfmk/vm/vm_fourk_pager.c b/osfmk/vm/vm_fourk_pager.c
index 73bfa3a244f40091447777c6788011a2e8a9b2fe..d815d85a38f3700da2b1a9d64884c46bf3f2f586 100644 (file)
--- a/osfmk/vm/vm_fourk_pager.c
+++ b/osfmk/vm/vm_fourk_pager.c
@@ -149,9 +149,13 @@ typedef struct fourk_pager {
 
        /* pager-specific data */
        queue_chain_t           pager_queue;    /* next & prev pagers */
 
        /* pager-specific data */
        queue_chain_t           pager_queue;    /* next & prev pagers */

-       unsigned int            ref_count;      /* reference count */
-       int     is_ready;       /* is this pager ready ? */
-       int     is_mapped;      /* is this mem_obj mapped ? */
+#if MEMORY_OBJECT_HAS_REFCOUNT
+#define fourk_pgr_hdr_ref       fourk_pgr_hdr.mo_ref
+#else
+       os_ref_atomic_t         fourk_pgr_hdr_ref;
+#endif
+       bool    is_ready;       /* is this pager ready ? */
+       bool    is_mapped;      /* is this mem_obj mapped ? */

        struct fourk_pager_backing slots[FOURK_PAGER_SLOTS]; /* backing for each
                                                              *  4K-chunk */
 } *fourk_pager_t;
        struct fourk_pager_backing slots[FOURK_PAGER_SLOTS]; /* backing for each
                                                              *  4K-chunk */
 } *fourk_pager_t;


@@ -322,8 +326,7 @@ fourk_pager_reference(
        pager = fourk_pager_lookup(mem_obj);
 
        lck_mtx_lock(&fourk_pager_lock);
        pager = fourk_pager_lookup(mem_obj);
 
        lck_mtx_lock(&fourk_pager_lock);

-       assert(pager->ref_count > 0);
-       pager->ref_count++;
+       os_ref_retain_locked_raw(&pager->fourk_pgr_hdr_ref, NULL);

        lck_mtx_unlock(&fourk_pager_lock);
 }
 
        lck_mtx_unlock(&fourk_pager_lock);
 }
 


@@ -401,6 +404,7 @@ fourk_pager_deallocate_internal(
 {
        boolean_t       needs_trimming;
        int             count_unmapped;
 {
        boolean_t       needs_trimming;
        int             count_unmapped;

+       os_ref_count_t  ref_count;

 
        if (!locked) {
                lck_mtx_lock(&fourk_pager_lock);
 
        if (!locked) {
                lck_mtx_lock(&fourk_pager_lock);


@@ -416,9 +420,9 @@ fourk_pager_deallocate_internal(
        }
 
        /* drop a reference on this pager */
        }
 
        /* drop a reference on this pager */

-       pager->ref_count--;
+       ref_count = os_ref_release_locked_raw(&pager->fourk_pgr_hdr_ref, NULL);

 
 

-       if (pager->ref_count == 1) {
+       if (ref_count == 1) {

                /*
                 * Only the "named" reference is left, which means that
                 * no one is really holding on to this pager anymore.
                /*
                 * Only the "named" reference is left, which means that
                 * no one is really holding on to this pager anymore.


@@ -428,7 +432,7 @@ fourk_pager_deallocate_internal(
                /* the pager is all ours: no need for the lock now */
                lck_mtx_unlock(&fourk_pager_lock);
                fourk_pager_terminate_internal(pager);
                /* the pager is all ours: no need for the lock now */
                lck_mtx_unlock(&fourk_pager_lock);
                fourk_pager_terminate_internal(pager);

-       } else if (pager->ref_count == 0) {
+       } else if (ref_count == 0) {

                /*
                 * Dropped the existence reference;  the memory object has
                 * been terminated.  Do some final cleanup and release the
                /*
                 * Dropped the existence reference;  the memory object has
                 * been terminated.  Do some final cleanup and release the


@@ -519,7 +523,7 @@ fourk_pager_map(
 
        lck_mtx_lock(&fourk_pager_lock);
        assert(pager->is_ready);
 
        lck_mtx_lock(&fourk_pager_lock);
        assert(pager->is_ready);

-       assert(pager->ref_count > 0); /* pager is alive */
+       assert(os_ref_get_count_raw(&pager->fourk_pgr_hdr_ref) > 0); /* pager is alive */

        if (pager->is_mapped == FALSE) {
                /*
                 * First mapping of this pager:  take an extra reference
        if (pager->is_mapped == FALSE) {
                /*
                 * First mapping of this pager:  take an extra reference


@@ -527,7 +531,7 @@ fourk_pager_map(
                 * are removed.
                 */
                pager->is_mapped = TRUE;
                 * are removed.
                 */
                pager->is_mapped = TRUE;

-               pager->ref_count++;
+               os_ref_retain_locked_raw(&pager->fourk_pgr_hdr_ref, NULL);

                fourk_pager_count_mapped++;
        }
        lck_mtx_unlock(&fourk_pager_lock);
                fourk_pager_count_mapped++;
        }
        lck_mtx_unlock(&fourk_pager_lock);


@@ -586,7 +590,7 @@ fourk_pager_lookup(
 
        assert(mem_obj->mo_pager_ops == &fourk_pager_ops);
        pager = (fourk_pager_t) mem_obj;
 
        assert(mem_obj->mo_pager_ops == &fourk_pager_ops);
        pager = (fourk_pager_t) mem_obj;

-       assert(pager->ref_count > 0);
+       assert(os_ref_get_count_raw(&pager->fourk_pgr_hdr_ref) > 0);

        return pager;
 }
 
        return pager;
 }
 


@@ -616,7 +620,7 @@ fourk_pager_trim(void)
                prev_pager = (fourk_pager_t)
                    queue_prev(&pager->pager_queue);
 
                prev_pager = (fourk_pager_t)
                    queue_prev(&pager->pager_queue);
 

-               if (pager->ref_count == 2 &&
+               if (os_ref_get_count_raw(&pager->fourk_pgr_hdr_ref) == 2 &&

                    pager->is_ready &&
                    !pager->is_mapped) {
                        /* this pager can be trimmed */
                    pager->is_ready &&
                    !pager->is_mapped) {
                        /* this pager can be trimmed */


@@ -652,13 +656,13 @@ fourk_pager_trim(void)
                    pager_queue);
                pager->pager_queue.next = NULL;
                pager->pager_queue.prev = NULL;
                    pager_queue);
                pager->pager_queue.next = NULL;
                pager->pager_queue.prev = NULL;

-               assert(pager->ref_count == 2);
+               assert(os_ref_get_count_raw(&pager->fourk_pgr_hdr_ref) == 2);

                /*
                 * We can't call deallocate_internal() because the pager
                 * has already been dequeued, but we still need to remove
                 * a reference.
                 */
                /*
                 * We can't call deallocate_internal() because the pager
                 * has already been dequeued, but we still need to remove
                 * a reference.
                 */

-               pager->ref_count--;
+               (void)os_ref_release_locked_raw(&pager->fourk_pgr_hdr_ref, NULL);

                fourk_pager_terminate_internal(pager);
        }
 }
                fourk_pager_terminate_internal(pager);
        }
 }


@@ -680,7 +684,7 @@ fourk_pager_to_vm_object(
                return VM_OBJECT_NULL;
        }
 
                return VM_OBJECT_NULL;
        }
 

-       assert(pager->ref_count > 0);
+       assert(os_ref_get_count_raw(&pager->fourk_pgr_hdr_ref) > 0);

        assert(pager->fourk_pgr_hdr.mo_control != MEMORY_OBJECT_CONTROL_NULL);
        object = memory_object_control_to_vm_object(pager->fourk_pgr_hdr.mo_control);
        assert(object != VM_OBJECT_NULL);
        assert(pager->fourk_pgr_hdr.mo_control != MEMORY_OBJECT_CONTROL_NULL);
        object = memory_object_control_to_vm_object(pager->fourk_pgr_hdr.mo_control);
        assert(object != VM_OBJECT_NULL);


@@ -718,8 +722,8 @@ fourk_pager_create(void)
        pager->fourk_pgr_hdr.mo_pager_ops = &fourk_pager_ops;
        pager->fourk_pgr_hdr.mo_control = MEMORY_OBJECT_CONTROL_NULL;
 
        pager->fourk_pgr_hdr.mo_pager_ops = &fourk_pager_ops;
        pager->fourk_pgr_hdr.mo_control = MEMORY_OBJECT_CONTROL_NULL;
 

-       pager->ref_count = 2;   /* existence + setup reference */
-       pager->is_ready = FALSE;/* not ready until it has a "name" */
+       os_ref_init_count_raw(&pager->fourk_pgr_hdr_ref, NULL, 2); /* existence + setup reference */
+       pager->is_ready = FALSE; /* not ready until it has a "name" */

        pager->is_mapped = FALSE;
 
        for (i = 0; i < FOURK_PAGER_SLOTS; i++) {
        pager->is_mapped = FALSE;
 
        for (i = 0; i < FOURK_PAGER_SLOTS; i++) {


@@ -792,7 +796,7 @@ fourk_pager_data_request(
 
        pager = fourk_pager_lookup(mem_obj);
        assert(pager->is_ready);
 
        pager = fourk_pager_lookup(mem_obj);
        assert(pager->is_ready);

-       assert(pager->ref_count > 1); /* pager is alive and mapped */
+       assert(os_ref_get_count_raw(&pager->fourk_pgr_hdr_ref) > 1); /* pager is alive and mapped */

 
        PAGER_DEBUG(PAGER_PAGEIN, ("fourk_pager_data_request: %p, %llx, %x, %x, pager %p\n", mem_obj, offset, length, protection_required, pager));
 
 
        PAGER_DEBUG(PAGER_PAGEIN, ("fourk_pager_data_request: %p, %llx, %x, %x, pager %p\n", mem_obj, offset, length, protection_required, pager));
 


@@ -821,7 +825,7 @@ fourk_pager_data_request(
                retval = kr;
                goto done;
        }
                retval = kr;
                goto done;
        }

-       dst_object = mo_control->moc_object;
+       dst_object = memory_object_control_to_vm_object(mo_control);

        assert(dst_object != VM_OBJECT_NULL);
 
 #if __x86_64__ || __arm__ || __arm64__
        assert(dst_object != VM_OBJECT_NULL);
 
 #if __x86_64__ || __arm__ || __arm64__


@@ -1289,7 +1293,7 @@ fourk_pager_populate(
                return KERN_INVALID_ARGUMENT;
        }
 
                return KERN_INVALID_ARGUMENT;
        }
 

-       assert(pager->ref_count > 0);
+       assert(os_ref_get_count_raw(&pager->fourk_pgr_hdr_ref) > 0);

        assert(pager->fourk_pgr_hdr.mo_control != MEMORY_OBJECT_CONTROL_NULL);
 
        if (index < 0 || index > FOURK_PAGER_SLOTS) {
        assert(pager->fourk_pgr_hdr.mo_control != MEMORY_OBJECT_CONTROL_NULL);
 
        if (index < 0 || index > FOURK_PAGER_SLOTS) {