X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/a991bd8d3e7fe02dbca0644054bab73c5b75324a..HEAD:/osfmk/vm/vm_fourk_pager.c

diff --git a/osfmk/vm/vm_fourk_pager.c b/osfmk/vm/vm_fourk_pager.c
index 73bfa3a24..d815d85a3 100644
--- a/osfmk/vm/vm_fourk_pager.c
+++ b/osfmk/vm/vm_fourk_pager.c
@@ -149,9 +149,13 @@ typedef struct fourk_pager {
 
 	/* pager-specific data */
 	queue_chain_t           pager_queue;    /* next & prev pagers */
-	unsigned int            ref_count;      /* reference count */
-	int     is_ready;       /* is this pager ready ? */
-	int     is_mapped;      /* is this mem_obj mapped ? */
+#if MEMORY_OBJECT_HAS_REFCOUNT
+#define fourk_pgr_hdr_ref       fourk_pgr_hdr.mo_ref
+#else
+	os_ref_atomic_t         fourk_pgr_hdr_ref;
+#endif
+	bool    is_ready;       /* is this pager ready ? */
+	bool    is_mapped;      /* is this mem_obj mapped ? */
 	struct fourk_pager_backing slots[FOURK_PAGER_SLOTS]; /* backing for each
 	                                                      *  4K-chunk */
 } *fourk_pager_t;
@@ -322,8 +326,7 @@ fourk_pager_reference(
 	pager = fourk_pager_lookup(mem_obj);
 
 	lck_mtx_lock(&fourk_pager_lock);
-	assert(pager->ref_count > 0);
-	pager->ref_count++;
+	os_ref_retain_locked_raw(&pager->fourk_pgr_hdr_ref, NULL);
 	lck_mtx_unlock(&fourk_pager_lock);
 }
 
@@ -401,6 +404,7 @@ fourk_pager_deallocate_internal(
 {
 	boolean_t       needs_trimming;
 	int             count_unmapped;
+	os_ref_count_t  ref_count;
 
 	if (!locked) {
 		lck_mtx_lock(&fourk_pager_lock);
@@ -416,9 +420,9 @@ fourk_pager_deallocate_internal(
 	}
 
 	/* drop a reference on this pager */
-	pager->ref_count--;
+	ref_count = os_ref_release_locked_raw(&pager->fourk_pgr_hdr_ref, NULL);
 
-	if (pager->ref_count == 1) {
+	if (ref_count == 1) {
 		/*
 		 * Only the "named" reference is left, which means that
 		 * no one is really holding on to this pager anymore.
@@ -428,7 +432,7 @@ fourk_pager_deallocate_internal(
 		/* the pager is all ours: no need for the lock now */
 		lck_mtx_unlock(&fourk_pager_lock);
 		fourk_pager_terminate_internal(pager);
-	} else if (pager->ref_count == 0) {
+	} else if (ref_count == 0) {
 		/*
 		 * Dropped the existence reference;  the memory object has
 		 * been terminated.  Do some final cleanup and release the
@@ -519,7 +523,7 @@ fourk_pager_map(
 
 	lck_mtx_lock(&fourk_pager_lock);
 	assert(pager->is_ready);
-	assert(pager->ref_count > 0); /* pager is alive */
+	assert(os_ref_get_count_raw(&pager->fourk_pgr_hdr_ref) > 0); /* pager is alive */
 	if (pager->is_mapped == FALSE) {
 		/*
 		 * First mapping of this pager:  take an extra reference
@@ -527,7 +531,7 @@ fourk_pager_map(
 		 * are removed.
 		 */
 		pager->is_mapped = TRUE;
-		pager->ref_count++;
+		os_ref_retain_locked_raw(&pager->fourk_pgr_hdr_ref, NULL);
 		fourk_pager_count_mapped++;
 	}
 	lck_mtx_unlock(&fourk_pager_lock);
@@ -586,7 +590,7 @@ fourk_pager_lookup(
 
 	assert(mem_obj->mo_pager_ops == &fourk_pager_ops);
 	pager = (fourk_pager_t) mem_obj;
-	assert(pager->ref_count > 0);
+	assert(os_ref_get_count_raw(&pager->fourk_pgr_hdr_ref) > 0);
 	return pager;
 }
 
@@ -616,7 +620,7 @@ fourk_pager_trim(void)
 		prev_pager = (fourk_pager_t)
 		    queue_prev(&pager->pager_queue);
 
-		if (pager->ref_count == 2 &&
+		if (os_ref_get_count_raw(&pager->fourk_pgr_hdr_ref) == 2 &&
 		    pager->is_ready &&
 		    !pager->is_mapped) {
 			/* this pager can be trimmed */
@@ -652,13 +656,13 @@ fourk_pager_trim(void)
 		    pager_queue);
 		pager->pager_queue.next = NULL;
 		pager->pager_queue.prev = NULL;
-		assert(pager->ref_count == 2);
+		assert(os_ref_get_count_raw(&pager->fourk_pgr_hdr_ref) == 2);
 		/*
 		 * We can't call deallocate_internal() because the pager
 		 * has already been dequeued, but we still need to remove
 		 * a reference.
 		 */
-		pager->ref_count--;
+		(void)os_ref_release_locked_raw(&pager->fourk_pgr_hdr_ref, NULL);
 		fourk_pager_terminate_internal(pager);
 	}
 }
@@ -680,7 +684,7 @@ fourk_pager_to_vm_object(
 		return VM_OBJECT_NULL;
 	}
 
-	assert(pager->ref_count > 0);
+	assert(os_ref_get_count_raw(&pager->fourk_pgr_hdr_ref) > 0);
 	assert(pager->fourk_pgr_hdr.mo_control != MEMORY_OBJECT_CONTROL_NULL);
 	object = memory_object_control_to_vm_object(pager->fourk_pgr_hdr.mo_control);
 	assert(object != VM_OBJECT_NULL);
@@ -718,8 +722,8 @@ fourk_pager_create(void)
 	pager->fourk_pgr_hdr.mo_pager_ops = &fourk_pager_ops;
 	pager->fourk_pgr_hdr.mo_control = MEMORY_OBJECT_CONTROL_NULL;
 
-	pager->ref_count = 2;   /* existence + setup reference */
-	pager->is_ready = FALSE;/* not ready until it has a "name" */
+	os_ref_init_count_raw(&pager->fourk_pgr_hdr_ref, NULL, 2); /* existence + setup reference */
+	pager->is_ready = FALSE; /* not ready until it has a "name" */
 	pager->is_mapped = FALSE;
 
 	for (i = 0; i < FOURK_PAGER_SLOTS; i++) {
@@ -792,7 +796,7 @@ fourk_pager_data_request(
 
 	pager = fourk_pager_lookup(mem_obj);
 	assert(pager->is_ready);
-	assert(pager->ref_count > 1); /* pager is alive and mapped */
+	assert(os_ref_get_count_raw(&pager->fourk_pgr_hdr_ref) > 1); /* pager is alive and mapped */
 
 	PAGER_DEBUG(PAGER_PAGEIN, ("fourk_pager_data_request: %p, %llx, %x, %x, pager %p\n", mem_obj, offset, length, protection_required, pager));
 
@@ -821,7 +825,7 @@ fourk_pager_data_request(
 		retval = kr;
 		goto done;
 	}
-	dst_object = mo_control->moc_object;
+	dst_object = memory_object_control_to_vm_object(mo_control);
 	assert(dst_object != VM_OBJECT_NULL);
 
 #if __x86_64__ || __arm__ || __arm64__
@@ -1289,7 +1293,7 @@ fourk_pager_populate(
 		return KERN_INVALID_ARGUMENT;
 	}
 
-	assert(pager->ref_count > 0);
+	assert(os_ref_get_count_raw(&pager->fourk_pgr_hdr_ref) > 0);
 	assert(pager->fourk_pgr_hdr.mo_control != MEMORY_OBJECT_CONTROL_NULL);
 
 	if (index < 0 || index > FOURK_PAGER_SLOTS) {