xnu-4570.1.46.tar.gz

[apple/xnu.git] / osfmk / vm / vm_purgeable.c

diff --git a/osfmk/vm/vm_purgeable.c b/osfmk/vm/vm_purgeable.c
index d6229881ba2a45e6e07b7d4b58a826b4821b7f62..a58b75001b99990e5041ce4bffcb9dc022cd416c 100644 (file)
--- a/osfmk/vm/vm_purgeable.c
+++ b/osfmk/vm/vm_purgeable.c
@@ -23,6 +23,7 @@
 
 #include <kern/sched_prim.h>
 #include <kern/ledger.h>
 
 #include <kern/sched_prim.h>
 #include <kern/ledger.h>

+#include <kern/policy_internal.h>

 
 #include <libkern/OSDebug.h>
 
 
 #include <libkern/OSDebug.h>
 


@@ -71,14 +72,6 @@ int purgeable_nonvolatile_count;
 
 decl_lck_mtx_data(,vm_purgeable_queue_lock)
 
 
 decl_lck_mtx_data(,vm_purgeable_queue_lock)
 

-#define TOKEN_ADD              0x40    /* 0x100 */
-#define TOKEN_DELETE           0x41    /* 0x104 */
-#define TOKEN_RIPEN            0x42    /* 0x108 */
-#define OBJECT_ADD             0x48    /* 0x120 */
-#define OBJECT_REMOVE          0x49    /* 0x124 */
-#define OBJECT_PURGE           0x4a    /* 0x128 */
-#define OBJECT_PURGE_ALL       0x4b    /* 0x12c */
-

 static token_idx_t vm_purgeable_token_remove_first(purgeable_q_t queue);
 
 static void vm_purgeable_stats_helper(vm_purgeable_stat_t *stat, purgeable_q_t queue, int group, task_t target_task);
 static token_idx_t vm_purgeable_token_remove_first(purgeable_q_t queue);
 
 static void vm_purgeable_stats_helper(vm_purgeable_stat_t *stat, purgeable_q_t queue, int group, task_t target_task);


@@ -149,9 +142,7 @@ vm_purgeable_token_check_queue(purgeable_q_t queue)
 kern_return_t
 vm_purgeable_token_add(purgeable_q_t queue)
 {
 kern_return_t
 vm_purgeable_token_add(purgeable_q_t queue)
 {

-#if MACH_ASSERT
-       lck_mtx_assert(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
-#endif
+       LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);

        
        /* new token */
        token_idx_t     token;
        
        /* new token */
        token_idx_t     token;


@@ -197,11 +188,11 @@ find_available_token:
                                                      (vm_offset_t) tokens,
                                                      token_q_cur_size,
                                                      (vm_offset_t *) &new_loc,
                                                      (vm_offset_t) tokens,
                                                      token_q_cur_size,
                                                      (vm_offset_t *) &new_loc,

-                                                     alloc_size);
+                                                     alloc_size, VM_KERN_MEMORY_OSFMK);

                        } else {
                                result = kmem_alloc(kernel_map,
                                                    (vm_offset_t *) &new_loc,
                        } else {
                                result = kmem_alloc(kernel_map,
                                                    (vm_offset_t *) &new_loc,

-                                                   alloc_size);
+                                                   alloc_size, VM_KERN_MEMORY_OSFMK);

                        }
                }
                
                        }
                }
                


@@ -307,9 +298,7 @@ find_available_token:
 static token_idx_t 
 vm_purgeable_token_remove_first(purgeable_q_t queue)
 {
 static token_idx_t 
 vm_purgeable_token_remove_first(purgeable_q_t queue)
 {

-#if MACH_ASSERT
-       lck_mtx_assert(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
-#endif
+       LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);

        
        token_idx_t     token;
        token = queue->token_q_head;
        
        token_idx_t     token;
        token = queue->token_q_head;


@@ -365,9 +354,7 @@ vm_purgeable_token_remove_first(purgeable_q_t queue)
 static token_idx_t 
 vm_purgeable_token_remove_last(purgeable_q_t queue)
 {
 static token_idx_t 
 vm_purgeable_token_remove_last(purgeable_q_t queue)
 {

-#if MACH_ASSERT
-       lck_mtx_assert(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
-#endif
+       LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);

        
        token_idx_t     token;
        token = queue->token_q_tail;
        
        token_idx_t     token;
        token = queue->token_q_tail;


@@ -431,9 +418,7 @@ vm_purgeable_token_remove_last(purgeable_q_t queue)
 void
 vm_purgeable_token_delete_first(purgeable_q_t queue)
 {
 void
 vm_purgeable_token_delete_first(purgeable_q_t queue)
 {

-#if MACH_ASSERT
-       lck_mtx_assert(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
-#endif
+       LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);

        token_idx_t     token = vm_purgeable_token_remove_first(queue);
 
        if (token) {
        token_idx_t     token = vm_purgeable_token_remove_first(queue);
 
        if (token) {


@@ -447,9 +432,7 @@ vm_purgeable_token_delete_first(purgeable_q_t queue)
 void
 vm_purgeable_token_delete_last(purgeable_q_t queue)
 {
 void
 vm_purgeable_token_delete_last(purgeable_q_t queue)
 {

-#if MACH_ASSERT
-       lck_mtx_assert(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
-#endif
+       LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);

        token_idx_t     token = vm_purgeable_token_remove_last(queue);
 
        if (token) {
        token_idx_t     token = vm_purgeable_token_remove_last(queue);
 
        if (token) {


@@ -465,9 +448,7 @@ vm_purgeable_token_delete_last(purgeable_q_t queue)
 void
 vm_purgeable_q_advance_all()
 {
 void
 vm_purgeable_q_advance_all()
 {

-#if MACH_ASSERT
-       lck_mtx_assert(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
-#endif
+       LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);

        
        /* check queue counters - if they get really large, scale them back.
         * They tend to get that large when there is no purgeable queue action */
        
        /* check queue counters - if they get really large, scale them back.
         * They tend to get that large when there is no purgeable queue action */


@@ -557,9 +538,7 @@ vm_purgeable_q_advance_all()
 static void
 vm_purgeable_token_remove_ripe(purgeable_q_t queue)
 {
 static void
 vm_purgeable_token_remove_ripe(purgeable_q_t queue)
 {

-#if MACH_ASSERT
-       lck_mtx_assert(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
-#endif
+       LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);

        assert(queue->token_q_head && tokens[queue->token_q_head].count == 0);
        /* return token to free list. advance token list. */
        token_idx_t     new_head = tokens[queue->token_q_head].next;
        assert(queue->token_q_head && tokens[queue->token_q_head].count == 0);
        /* return token to free list. advance token list. */
        token_idx_t     new_head = tokens[queue->token_q_head].next;


@@ -589,9 +568,7 @@ vm_purgeable_token_remove_ripe(purgeable_q_t queue)
 static void
 vm_purgeable_token_choose_and_delete_ripe(purgeable_q_t queue, purgeable_q_t queue2)
 {
 static void
 vm_purgeable_token_choose_and_delete_ripe(purgeable_q_t queue, purgeable_q_t queue2)
 {

-#if MACH_ASSERT
-       lck_mtx_assert(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
-#endif
+       LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);

        assert(queue->token_q_head);
 
        if (tokens[queue->token_q_head].count == 0) {
        assert(queue->token_q_head);
 
        if (tokens[queue->token_q_head].count == 0) {


@@ -688,24 +665,43 @@ vm_purgeable_object_find_and_lock(
        int             best_object_task_importance;
        int             best_object_skipped;
        int             num_objects_skipped;
        int             best_object_task_importance;
        int             best_object_skipped;
        int             num_objects_skipped;

+       int             try_lock_failed = 0;
+       int             try_lock_succeeded = 0;

        task_t          owner;
 
        best_object = VM_OBJECT_NULL;
        best_object_task_importance = INT_MAX;
 
        task_t          owner;
 
        best_object = VM_OBJECT_NULL;
        best_object_task_importance = INT_MAX;
 

-       lck_mtx_assert(&vm_purgeable_queue_lock, LCK_MTX_ASSERT_OWNED);
+       LCK_MTX_ASSERT(&vm_purgeable_queue_lock, LCK_MTX_ASSERT_OWNED);

        /*
         * Usually we would pick the first element from a queue. However, we
         * might not be able to get a lock on it, in which case we try the
         * remaining elements in order.
         */
 
        /*
         * Usually we would pick the first element from a queue. However, we
         * might not be able to get a lock on it, in which case we try the
         * remaining elements in order.
         */
 

-       num_objects_skipped = -1;
+       KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (MACHDBG_CODE(DBG_MACH_VM, OBJECT_PURGE_LOOP) | DBG_FUNC_START),
+                             pick_ripe,
+                             group,
+                             VM_KERNEL_UNSLIDE_OR_PERM(queue),
+                             0,
+                             0);
+
+       num_objects_skipped = 0;

        for (object = (vm_object_t) queue_first(&queue->objq[group]);
             !queue_end(&queue->objq[group], (queue_entry_t) object);
             object = (vm_object_t) queue_next(&object->objq),
                num_objects_skipped++) {
 
        for (object = (vm_object_t) queue_first(&queue->objq[group]);
             !queue_end(&queue->objq[group], (queue_entry_t) object);
             object = (vm_object_t) queue_next(&object->objq),
                num_objects_skipped++) {
 

+               /*
+                * To prevent us looping for an excessively long time, choose
+                * the best object we've seen after looking at PURGEABLE_LOOP_MAX elements.
+                * If we haven't seen an eligible object after PURGEABLE_LOOP_MAX elements,
+                * we keep going until we find the first eligible object.
+                */
+               if ((num_objects_skipped >= PURGEABLE_LOOP_MAX) && (best_object != NULL)) {
+                       break;
+               }
+

                if (pick_ripe &&
                    ! object->purgeable_when_ripe) {
                        /* we want an object that has a ripe token */
                if (pick_ripe &&
                    ! object->purgeable_when_ripe) {
                        /* we want an object that has a ripe token */


@@ -716,11 +712,18 @@ vm_purgeable_object_find_and_lock(
 
                owner = object->vo_purgeable_owner;
                if (owner) {
 
                owner = object->vo_purgeable_owner;
                if (owner) {

+#if CONFIG_EMBEDDED
+#if CONFIG_JETSAM
+                       object_task_importance = proc_get_memstat_priority((struct proc *)get_bsdtask_info(owner), TRUE);
+#endif /* CONFIG_JETSAM */
+#else /* CONFIG_EMBEDDED */

                        object_task_importance = task_importance_estimate(owner);
                        object_task_importance = task_importance_estimate(owner);

+#endif /* CONFIG_EMBEDDED */

                }
 
                if (object_task_importance < best_object_task_importance) {
                        if (vm_object_lock_try(object)) {
                }
 
                if (object_task_importance < best_object_task_importance) {
                        if (vm_object_lock_try(object)) {

+                               try_lock_succeeded++;

                                if (best_object != VM_OBJECT_NULL) {
                                        /* forget about previous best object */
                                        vm_object_unlock(best_object);
                                if (best_object != VM_OBJECT_NULL) {
                                        /* forget about previous best object */
                                        vm_object_unlock(best_object);


@@ -732,9 +735,19 @@ vm_purgeable_object_find_and_lock(
                                        /* can't get any better: stop looking */
                                        break;
                                }
                                        /* can't get any better: stop looking */
                                        break;
                                }

+                       } else {
+                               try_lock_failed++;

                        }
                }
        }
                        }
                }
        }

+
+       KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (MACHDBG_CODE(DBG_MACH_VM, OBJECT_PURGE_LOOP) | DBG_FUNC_END),
+                             num_objects_skipped, /* considered objects */
+                             try_lock_failed,
+                             try_lock_succeeded,
+                             VM_KERNEL_UNSLIDE_OR_PERM(best_object),
+                             ((best_object == NULL) ? 0 : best_object->resident_page_count));
+

        object = best_object;
 
        if (object == VM_OBJECT_NULL) {
        object = best_object;
 
        if (object == VM_OBJECT_NULL) {


@@ -866,9 +879,7 @@ vm_purgeable_object_purge_one(
        boolean_t       forced_purge;
 
        /* Need the page queue lock since we'll be changing the token queue. */
        boolean_t       forced_purge;
 
        /* Need the page queue lock since we'll be changing the token queue. */

-#if MACH_ASSERT
-       lck_mtx_assert(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
-#endif
+       LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);

        lck_mtx_lock(&vm_purgeable_queue_lock);
        
        /* Cycle through all queues */
        lck_mtx_lock(&vm_purgeable_queue_lock);
        
        /* Cycle through all queues */


@@ -973,7 +984,7 @@ purge_now:
        vm_page_lock_queues();
 
        KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, OBJECT_PURGE)),
        vm_page_lock_queues();
 
        KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, OBJECT_PURGE)),

-                             object,   /* purged object */
+                             VM_KERNEL_UNSLIDE_OR_PERM(object),        /* purged object */

                              0,
                              available_for_purge,
                              0,
                              0,
                              available_for_purge,
                              0,


@@ -1104,7 +1115,7 @@ vm_purgeable_object_remove(vm_object_t object)
 void
 vm_purgeable_stats_helper(vm_purgeable_stat_t *stat, purgeable_q_t queue, int group, task_t target_task)
 {
 void
 vm_purgeable_stats_helper(vm_purgeable_stat_t *stat, purgeable_q_t queue, int group, task_t target_task)
 {

-       lck_mtx_assert(&vm_purgeable_queue_lock, LCK_MTX_ASSERT_OWNED);
+       LCK_MTX_ASSERT(&vm_purgeable_queue_lock, LCK_MTX_ASSERT_OWNED);

 
        stat->count = stat->size = 0;
        vm_object_t     object;
 
        stat->count = stat->size = 0;
        vm_object_t     object;


@@ -1144,7 +1155,99 @@ vm_purgeable_stats(vm_purgeable_info_t info, task_t target_task)
        lck_mtx_unlock(&vm_purgeable_queue_lock);
        return;
 }
        lck_mtx_unlock(&vm_purgeable_queue_lock);
        return;
 }

-       
+
+#if DEVELOPMENT || DEBUG
+static void
+vm_purgeable_account_volatile_queue(
+       purgeable_q_t queue,
+       int group,
+       task_t task,
+       pvm_account_info_t acnt_info)
+{
+       vm_object_t object;
+       uint64_t compressed_count;
+
+       for (object = (vm_object_t) queue_first(&queue->objq[group]);
+           !queue_end(&queue->objq[group], (queue_entry_t) object);
+           object = (vm_object_t) queue_next(&object->objq)) {
+               if (object->vo_purgeable_owner == task) {
+                       compressed_count = vm_compressor_pager_get_count(object->pager);
+                       acnt_info->pvm_volatile_compressed_count += compressed_count;
+                       acnt_info->pvm_volatile_count += (object->resident_page_count - object->wired_page_count);
+                       acnt_info->pvm_nonvolatile_count += object->wired_page_count;
+               }
+       }
+
+}
+
+/*
+ * Walks the purgeable object queues and calculates the usage
+ * associated with the objects for the given task.
+ */
+kern_return_t
+vm_purgeable_account(
+       task_t                  task,
+       pvm_account_info_t      acnt_info)
+{
+       queue_head_t    *nonvolatile_q;
+       vm_object_t     object;
+       int             group;
+       int             state;
+       uint64_t        compressed_count;
+       purgeable_q_t   volatile_q;
+
+
+       if ((task == NULL) || (acnt_info == NULL)) {
+               return KERN_INVALID_ARGUMENT;
+       }
+
+       acnt_info->pvm_volatile_count = 0;
+       acnt_info->pvm_volatile_compressed_count = 0;
+       acnt_info->pvm_nonvolatile_count = 0;
+       acnt_info->pvm_nonvolatile_compressed_count = 0;
+
+       lck_mtx_lock(&vm_purgeable_queue_lock);
+
+       nonvolatile_q = &purgeable_nonvolatile_queue;
+       for (object = (vm_object_t) queue_first(nonvolatile_q);
+            !queue_end(nonvolatile_q, (queue_entry_t) object);
+            object = (vm_object_t) queue_next(&object->objq)) {
+               if (object->vo_purgeable_owner == task) {
+                       state = object->purgable;
+                       compressed_count =  vm_compressor_pager_get_count(object->pager);
+                       if (state == VM_PURGABLE_EMPTY) {
+                               acnt_info->pvm_volatile_count += (object->resident_page_count - object->wired_page_count);
+                               acnt_info->pvm_volatile_compressed_count += compressed_count;
+                       } else {
+                               acnt_info->pvm_nonvolatile_count += (object->resident_page_count - object->wired_page_count);
+                               acnt_info->pvm_nonvolatile_compressed_count += compressed_count;
+                       }
+                       acnt_info->pvm_nonvolatile_count += object->wired_page_count;
+               }
+       }
+
+       volatile_q = &purgeable_queues[PURGEABLE_Q_TYPE_OBSOLETE];
+       vm_purgeable_account_volatile_queue(volatile_q, 0, task, acnt_info);
+
+       volatile_q = &purgeable_queues[PURGEABLE_Q_TYPE_FIFO];
+       for (group = 0; group < NUM_VOLATILE_GROUPS; group++) {
+               vm_purgeable_account_volatile_queue(volatile_q, group, task, acnt_info);
+       }
+
+       volatile_q = &purgeable_queues[PURGEABLE_Q_TYPE_LIFO];
+       for (group = 0; group < NUM_VOLATILE_GROUPS; group++) {
+               vm_purgeable_account_volatile_queue(volatile_q, group, task, acnt_info);
+       }
+       lck_mtx_unlock(&vm_purgeable_queue_lock);
+
+       acnt_info->pvm_volatile_count = (acnt_info->pvm_volatile_count * PAGE_SIZE);
+       acnt_info->pvm_volatile_compressed_count = (acnt_info->pvm_volatile_compressed_count * PAGE_SIZE);
+       acnt_info->pvm_nonvolatile_count = (acnt_info->pvm_nonvolatile_count * PAGE_SIZE);
+       acnt_info->pvm_nonvolatile_compressed_count = (acnt_info->pvm_nonvolatile_compressed_count * PAGE_SIZE);
+
+       return KERN_SUCCESS;
+}
+#endif /* DEVELOPMENT || DEBUG */

 
 static void
 vm_purgeable_volatile_queue_disown(
 
 static void
 vm_purgeable_volatile_queue_disown(


@@ -1158,7 +1261,7 @@ vm_purgeable_volatile_queue_disown(
        collisions = 0;
 
 again:
        collisions = 0;
 
 again:

-       lck_mtx_assert(&vm_purgeable_queue_lock, LCK_MTX_ASSERT_OWNED);
+       LCK_MTX_ASSERT(&vm_purgeable_queue_lock, LCK_MTX_ASSERT_OWNED);

 
        for (object = (vm_object_t) queue_first(&queue->objq[group]);
             !queue_end(&queue->objq[group], (queue_entry_t) object);
 
        for (object = (vm_object_t) queue_first(&queue->objq[group]);
             !queue_end(&queue->objq[group], (queue_entry_t) object);


@@ -1440,11 +1543,11 @@ vm_purgeable_nonvolatile_enqueue(
 
        assert(object->purgable == VM_PURGABLE_NONVOLATILE);
        assert(object->vo_purgeable_owner == NULL);
 
        assert(object->purgable == VM_PURGABLE_NONVOLATILE);
        assert(object->vo_purgeable_owner == NULL);

-       assert(owner != NULL);

 
        lck_mtx_lock(&vm_purgeable_queue_lock);
 
 
        lck_mtx_lock(&vm_purgeable_queue_lock);
 

-       if (owner->task_purgeable_disowning) {
+       if (owner != NULL &&
+           owner->task_purgeable_disowning) {

                /* task is exiting and no longer tracking purgeable objects */
                owner = NULL;
        }
                /* task is exiting and no longer tracking purgeable objects */
                owner = NULL;
        }


@@ -1459,7 +1562,6 @@ vm_purgeable_nonvolatile_enqueue(
 #endif /* DEBUG */
 
        page_count = object->resident_page_count;
 #endif /* DEBUG */
 
        page_count = object->resident_page_count;

-       assert(page_count == 0); /* should be a freshly-created object */

        if (owner != NULL && page_count != 0) {
                ledger_credit(owner->ledger,
                              task_ledgers.purgeable_nonvolatile,
        if (owner != NULL && page_count != 0) {
                ledger_credit(owner->ledger,
                              task_ledgers.purgeable_nonvolatile,


@@ -1549,9 +1651,8 @@ vm_purgeable_accounting(
 
        resident_page_count = object->resident_page_count;
        wired_page_count = object->wired_page_count;
 
        resident_page_count = object->resident_page_count;
        wired_page_count = object->wired_page_count;

-       if ((COMPRESSED_PAGER_IS_ACTIVE ||
-            DEFAULT_FREEZER_COMPRESSED_PAGER_IS_ACTIVE) &&
-           object->pager != NULL) {
+       if (VM_CONFIG_COMPRESSOR_IS_PRESENT &&
+            object->pager != NULL) {

                compressed_page_count =
                        vm_compressor_pager_get_count(object->pager);
        } else {
                compressed_page_count =
                        vm_compressor_pager_get_count(object->pager);
        } else {