xnu-4903.270.47.tar.gz

[apple/xnu.git] / osfmk / vm / vm_purgeable.c

diff --git a/osfmk/vm/vm_purgeable.c b/osfmk/vm/vm_purgeable.c
index b1d7aeba506d186dff85d64174822fc19710cb6e..6ebfaf77ac89afc85cef62e3fee9312019de5afb 100644 (file)
--- a/osfmk/vm/vm_purgeable.c
+++ b/osfmk/vm/vm_purgeable.c
@@ -32,7 +32,7 @@
 #include <machine/limits.h>
 
 #include <vm/vm_compressor_pager.h>
 #include <machine/limits.h>
 
 #include <vm/vm_compressor_pager.h>

-#include <vm/vm_kern.h>                                /* kmem_alloc */
+#include <vm/vm_kern.h>                         /* kmem_alloc */

 #include <vm/vm_page.h>
 #include <vm/vm_pageout.h>
 #include <vm/vm_protos.h>
 #include <vm/vm_page.h>
 #include <vm/vm_pageout.h>
 #include <vm/vm_protos.h>


@@ -40,47 +40,56 @@
 
 #include <sys/kdebug.h>
 
 
 #include <sys/kdebug.h>
 

+/*
+ * LOCK ORDERING for task-owned purgeable objects
+ *
+ * Whenever we need to hold multiple locks while adding to, removing from,
+ * or scanning a task's task_objq list of VM objects it owns, locks should
+ * be taken in this order:
+ *
+ * VM object ==> vm_purgeable_queue_lock ==> owner_task->task_objq_lock
+ *
+ * If one needs to acquire the VM object lock after any of the other 2 locks,
+ * one needs to use vm_object_lock_try() and, if that fails, release the
+ * other locks and retake them all in the correct order.
+ */
+

 extern vm_pressure_level_t memorystatus_vm_pressure_level;
 
 struct token {
        token_cnt_t     count;
 extern vm_pressure_level_t memorystatus_vm_pressure_level;
 
 struct token {
        token_cnt_t     count;

-       token_idx_t     prev;
+       token_idx_t     prev;

        token_idx_t     next;
 };
 
        token_idx_t     next;
 };
 

-struct token   *tokens;
-token_idx_t    token_q_max_cnt = 0;
-vm_size_t      token_q_cur_size = 0;
+struct token    *tokens;
+token_idx_t     token_q_max_cnt = 0;
+vm_size_t       token_q_cur_size = 0;

 
 

-token_idx_t     token_free_idx = 0;            /* head of free queue */
-token_idx_t     token_init_idx = 1;            /* token 0 is reserved!! */
-int32_t                token_new_pagecount = 0;        /* count of pages that will
-                                                * be added onto token queue */
+token_idx_t     token_free_idx = 0;             /* head of free queue */
+token_idx_t     token_init_idx = 1;             /* token 0 is reserved!! */
+int32_t         token_new_pagecount = 0;        /* count of pages that will
+                                                 * be added onto token queue */

 
 

-int             available_for_purge = 0;       /* increase when ripe token
-                                                * added, decrease when ripe
-                                                * token removed.
-                                                * protected by page_queue_lock 
-                                                */
+int             available_for_purge = 0;        /* increase when ripe token
+                                                 * added, decrease when ripe
+                                                 * token removed.
+                                                 * protected by page_queue_lock
+                                                 */

 
 

-static int token_q_allocating = 0;             /* flag for singlethreading 
-                                                * allocator */
+static int token_q_allocating = 0;              /* flag for singlethreading
+                                                 * allocator */

 
 struct purgeable_q purgeable_queues[PURGEABLE_Q_TYPE_MAX];
 queue_head_t purgeable_nonvolatile_queue;
 int purgeable_nonvolatile_count;
 
 
 struct purgeable_q purgeable_queues[PURGEABLE_Q_TYPE_MAX];
 queue_head_t purgeable_nonvolatile_queue;
 int purgeable_nonvolatile_count;
 

-decl_lck_mtx_data(,vm_purgeable_queue_lock)
+decl_lck_mtx_data(, vm_purgeable_queue_lock)

 
 static token_idx_t vm_purgeable_token_remove_first(purgeable_q_t queue);
 
 static void vm_purgeable_stats_helper(vm_purgeable_stat_t *stat, purgeable_q_t queue, int group, task_t target_task);
 
 
 static token_idx_t vm_purgeable_token_remove_first(purgeable_q_t queue);
 
 static void vm_purgeable_stats_helper(vm_purgeable_stat_t *stat, purgeable_q_t queue, int group, task_t target_task);
 

-void vm_purgeable_nonvolatile_owner_update(task_t      owner,
-                                          int          delta);
-void vm_purgeable_volatile_owner_update(task_t         owner,
-                                       int             delta);
-

 
 #if MACH_ASSERT
 static void
 
 #if MACH_ASSERT
 static void


@@ -92,7 +101,7 @@ vm_purgeable_token_check_queue(purgeable_q_t queue)
        int             our_inactive_count;
 
 #if DEVELOPMENT
        int             our_inactive_count;
 
 #if DEVELOPMENT

-       static unsigned lightweight_check = 0;
+       static unsigned lightweight_check = 0;

 
        /*
         * Due to performance impact, only perform this check
 
        /*
         * Due to performance impact, only perform this check


@@ -114,20 +123,21 @@ vm_purgeable_token_check_queue(purgeable_q_t queue)
                        }
                        page_cnt += tokens[token].count;
                }
                        }
                        page_cnt += tokens[token].count;
                }

-               if (tokens[token].next == 0)
+               if (tokens[token].next == 0) {

                        assert(queue->token_q_tail == token);
                        assert(queue->token_q_tail == token);

+               }

 
                token_cnt++;
                token = tokens[token].next;
        }
 
 
                token_cnt++;
                token = tokens[token].next;
        }
 

-       if (unripe)
+       if (unripe) {

                assert(queue->token_q_unripe == unripe);
                assert(queue->token_q_unripe == unripe);

+       }

        assert(token_cnt == queue->debug_count_tokens);
        assert(token_cnt == queue->debug_count_tokens);

-       
+

        /* obsolete queue doesn't maintain token counts */
        /* obsolete queue doesn't maintain token counts */

-       if(queue->type != PURGEABLE_Q_TYPE_OBSOLETE)
-       {
+       if (queue->type != PURGEABLE_Q_TYPE_OBSOLETE) {

                our_inactive_count = page_cnt + queue->new_pages + token_new_pagecount;
                assert(our_inactive_count >= 0);
                assert((uint32_t) our_inactive_count == vm_page_inactive_count - vm_page_cleaned_count);
                our_inactive_count = page_cnt + queue->new_pages + token_new_pagecount;
                assert(our_inactive_count >= 0);
                assert((uint32_t) our_inactive_count == vm_page_inactive_count - vm_page_cleaned_count);


@@ -143,94 +153,98 @@ kern_return_t
 vm_purgeable_token_add(purgeable_q_t queue)
 {
        LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
 vm_purgeable_token_add(purgeable_q_t queue)
 {
        LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);

-       
+

        /* new token */
        token_idx_t     token;
        enum purgeable_q_type i;
 
 find_available_token:
 
        /* new token */
        token_idx_t     token;
        enum purgeable_q_type i;
 
 find_available_token:
 

-       if (token_free_idx) {                           /* unused tokens available */
+       if (token_free_idx) {                           /* unused tokens available */

                token = token_free_idx;
                token_free_idx = tokens[token_free_idx].next;
                token = token_free_idx;
                token_free_idx = tokens[token_free_idx].next;

-       } else if (token_init_idx < token_q_max_cnt) {  /* lazy token array init */
+       } else if (token_init_idx < token_q_max_cnt) {  /* lazy token array init */

                token = token_init_idx;
                token_init_idx++;
                token = token_init_idx;
                token_init_idx++;

-       } else {                                        /* allocate more memory */
+       } else {                                        /* allocate more memory */

                /* Wait if another thread is inside the memory alloc section */
                /* Wait if another thread is inside the memory alloc section */

-               while(token_q_allocating) {
+               while (token_q_allocating) {

                        wait_result_t res = lck_mtx_sleep(&vm_page_queue_lock,
                        wait_result_t res = lck_mtx_sleep(&vm_page_queue_lock,

-                                                         LCK_SLEEP_DEFAULT,
-                                                         (event_t)&token_q_allocating,
-                                                         THREAD_UNINT);
-                       if(res != THREAD_AWAKENED) return KERN_ABORTED;
-               };
-               
+                           LCK_SLEEP_DEFAULT,
+                           (event_t)&token_q_allocating,
+                           THREAD_UNINT);
+                       if (res != THREAD_AWAKENED) {
+                               return KERN_ABORTED;
+                       }
+               }
+               ;
+

                /* Check whether memory is still maxed out */
                /* Check whether memory is still maxed out */

-               if(token_init_idx < token_q_max_cnt)
+               if (token_init_idx < token_q_max_cnt) {

                        goto find_available_token;
                        goto find_available_token;

-               
+               }
+

                /* Still no memory. Allocate some. */
                token_q_allocating = 1;
                /* Still no memory. Allocate some. */
                token_q_allocating = 1;

-               
+

                /* Drop page queue lock so we can allocate */
                vm_page_unlock_queues();
                /* Drop page queue lock so we can allocate */
                vm_page_unlock_queues();

-               
+

                struct token *new_loc;
                vm_size_t alloc_size = token_q_cur_size + PAGE_SIZE;
                kern_return_t result;
                struct token *new_loc;
                vm_size_t alloc_size = token_q_cur_size + PAGE_SIZE;
                kern_return_t result;

-               
-               if (alloc_size / sizeof (struct token) > TOKEN_COUNT_MAX) {
+
+               if (alloc_size / sizeof(struct token) > TOKEN_COUNT_MAX) {

                        result = KERN_RESOURCE_SHORTAGE;
                } else {
                        if (token_q_cur_size) {
                                result = kmem_realloc(kernel_map,
                        result = KERN_RESOURCE_SHORTAGE;
                } else {
                        if (token_q_cur_size) {
                                result = kmem_realloc(kernel_map,

-                                                     (vm_offset_t) tokens,
-                                                     token_q_cur_size,
-                                                     (vm_offset_t *) &new_loc,
-                                                     alloc_size, VM_KERN_MEMORY_OSFMK);
+                                   (vm_offset_t) tokens,
+                                   token_q_cur_size,
+                                   (vm_offset_t *) &new_loc,
+                                   alloc_size, VM_KERN_MEMORY_OSFMK);

                        } else {
                                result = kmem_alloc(kernel_map,
                        } else {
                                result = kmem_alloc(kernel_map,

-                                                   (vm_offset_t *) &new_loc,
-                                                   alloc_size, VM_KERN_MEMORY_OSFMK);
+                                   (vm_offset_t *) &new_loc,
+                                   alloc_size, VM_KERN_MEMORY_OSFMK);

                        }
                }
                        }
                }

-               
+

                vm_page_lock_queues();
                vm_page_lock_queues();

-               
+

                if (result) {
                        /* Unblock waiting threads */
                        token_q_allocating = 0;
                        thread_wakeup((event_t)&token_q_allocating);
                        return result;
                }
                if (result) {
                        /* Unblock waiting threads */
                        token_q_allocating = 0;
                        thread_wakeup((event_t)&token_q_allocating);
                        return result;
                }

-               
+

                /* If we get here, we allocated new memory. Update pointers and
                 * dealloc old range */
                /* If we get here, we allocated new memory. Update pointers and
                 * dealloc old range */

-               struct token *old_tokens=tokens;
-               tokens=new_loc;
-               vm_size_t old_token_q_cur_size=token_q_cur_size;
-               token_q_cur_size=alloc_size;
+               struct token *old_tokens = tokens;
+               tokens = new_loc;
+               vm_size_t old_token_q_cur_size = token_q_cur_size;
+               token_q_cur_size = alloc_size;

                token_q_max_cnt = (token_idx_t) (token_q_cur_size /
                token_q_max_cnt = (token_idx_t) (token_q_cur_size /

-                                                sizeof(struct token));
-               assert (token_init_idx < token_q_max_cnt);      /* We must have a free token now */
-               
-               if (old_token_q_cur_size) {     /* clean up old mapping */
+                   sizeof(struct token));
+               assert(token_init_idx < token_q_max_cnt);       /* We must have a free token now */
+
+               if (old_token_q_cur_size) {     /* clean up old mapping */

                        vm_page_unlock_queues();
                        /* kmem_realloc leaves the old region mapped. Get rid of it. */
                        kmem_free(kernel_map, (vm_offset_t)old_tokens, old_token_q_cur_size);
                        vm_page_lock_queues();
                }
                        vm_page_unlock_queues();
                        /* kmem_realloc leaves the old region mapped. Get rid of it. */
                        kmem_free(kernel_map, (vm_offset_t)old_tokens, old_token_q_cur_size);
                        vm_page_lock_queues();
                }

-               
+

                /* Unblock waiting threads */
                token_q_allocating = 0;
                thread_wakeup((event_t)&token_q_allocating);
                /* Unblock waiting threads */
                token_q_allocating = 0;
                thread_wakeup((event_t)&token_q_allocating);

-               
+

                goto find_available_token;
        }
                goto find_available_token;
        }

-       
-       assert (token);
-       
+
+       assert(token);
+

        /*
         * the new pagecount we got need to be applied to all queues except
         * obsolete
        /*
         * the new pagecount we got need to be applied to all queues except
         * obsolete


@@ -245,11 +259,12 @@ find_available_token:
        token_new_pagecount = 0;
 
        /* set token counter value */
        token_new_pagecount = 0;
 
        /* set token counter value */

-       if (queue->type != PURGEABLE_Q_TYPE_OBSOLETE)
+       if (queue->type != PURGEABLE_Q_TYPE_OBSOLETE) {

                tokens[token].count = queue->new_pages;
                tokens[token].count = queue->new_pages;

-       else
-               tokens[token].count = 0;        /* all obsolete items are
-                                                * ripe immediately */
+       } else {
+               tokens[token].count = 0;        /* all obsolete items are
+                                                * ripe immediately */
+       }

        queue->new_pages = 0;
 
        /* put token on token counter list */
        queue->new_pages = 0;
 
        /* put token on token counter list */


@@ -262,13 +277,14 @@ find_available_token:
                tokens[queue->token_q_tail].next = token;
                tokens[token].prev = queue->token_q_tail;
        }
                tokens[queue->token_q_tail].next = token;
                tokens[token].prev = queue->token_q_tail;
        }

-       if (queue->token_q_unripe == 0) {       /* only ripe tokens (token
-                                                * count == 0) in queue */
-               if (tokens[token].count > 0)
-                       queue->token_q_unripe = token;  /* first unripe token */
-               else
-                       available_for_purge++;  /* added a ripe token?
-                                                * increase available count */
+       if (queue->token_q_unripe == 0) {       /* only ripe tokens (token
+                                                * count == 0) in queue */
+               if (tokens[token].count > 0) {
+                       queue->token_q_unripe = token;  /* first unripe token */
+               } else {
+                       available_for_purge++;  /* added a ripe token?
+                                                * increase available count */
+               }

        }
        queue->token_q_tail = token;
 
        }
        queue->token_q_tail = token;
 


@@ -279,12 +295,12 @@ find_available_token:
        vm_purgeable_token_check_queue(&purgeable_queues[PURGEABLE_Q_TYPE_LIFO]);
 
        KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, TOKEN_ADD)),
        vm_purgeable_token_check_queue(&purgeable_queues[PURGEABLE_Q_TYPE_LIFO]);
 
        KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, TOKEN_ADD)),

-                             queue->type,
-                             tokens[token].count,      /* num pages on token
-                                                        * (last token) */
-                             queue->debug_count_tokens,
-                             0,
-                             0);
+           queue->type,
+           tokens[token].count,                        /* num pages on token
+                                                        * (last token) */
+           queue->debug_count_tokens,
+           0,
+           0);

 #endif
 
        return KERN_SUCCESS;
 #endif
 
        return KERN_SUCCESS;


@@ -293,13 +309,13 @@ find_available_token:
 /*
  * Remove first token from queue and return its index. Add its count to the
  * count of the next token.
 /*
  * Remove first token from queue and return its index. Add its count to the
  * count of the next token.

- * Call with page queue locked. 
+ * Call with page queue locked.

  */
  */

-static token_idx_t 
+static token_idx_t

 vm_purgeable_token_remove_first(purgeable_q_t queue)
 {
        LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
 vm_purgeable_token_remove_first(purgeable_q_t queue)
 {
        LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);

-       
+

        token_idx_t     token;
        token = queue->token_q_head;
 
        token_idx_t     token;
        token = queue->token_q_head;
 


@@ -316,8 +332,9 @@ vm_purgeable_token_remove_first(purgeable_q_t queue)
                        assert(available_for_purge >= 0);
                }
 
                        assert(available_for_purge >= 0);
                }
 

-               if (queue->token_q_tail == queue->token_q_head)
+               if (queue->token_q_tail == queue->token_q_head) {

                        assert(tokens[token].next == 0);
                        assert(tokens[token].next == 0);

+               }

 
                queue->token_q_head = tokens[token].next;
                if (queue->token_q_head) {
 
                queue->token_q_head = tokens[token].next;
                if (queue->token_q_head) {


@@ -339,23 +356,23 @@ vm_purgeable_token_remove_first(purgeable_q_t queue)
                vm_purgeable_token_check_queue(queue);
 
                KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, TOKEN_DELETE)),
                vm_purgeable_token_check_queue(queue);
 
                KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, TOKEN_DELETE)),

-                                     queue->type,
-                                     tokens[queue->token_q_head].count,        /* num pages on new
-                                                                                * first token */
-                                     token_new_pagecount,      /* num pages waiting for
-                                                                * next token */
-                                     available_for_purge,
-                                     0);
+                   queue->type,
+                   tokens[queue->token_q_head].count,                          /* num pages on new
+                                                                                * first token */
+                   token_new_pagecount,                        /* num pages waiting for
+                                                                * next token */
+                   available_for_purge,
+                   0);

 #endif
        }
        return token;
 }
 
 #endif
        }
        return token;
 }
 

-static token_idx_t 
+static token_idx_t

 vm_purgeable_token_remove_last(purgeable_q_t queue)
 {
        LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
 vm_purgeable_token_remove_last(purgeable_q_t queue)
 {
        LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);

-       
+

        token_idx_t     token;
        token = queue->token_q_tail;
 
        token_idx_t     token;
        token = queue->token_q_tail;
 


@@ -364,8 +381,9 @@ vm_purgeable_token_remove_last(purgeable_q_t queue)
        if (token) {
                assert(queue->token_q_head);
 
        if (token) {
                assert(queue->token_q_head);
 

-               if (queue->token_q_tail == queue->token_q_head)
+               if (queue->token_q_tail == queue->token_q_head) {

                        assert(tokens[token].next == 0);
                        assert(tokens[token].next == 0);

+               }

 
                if (queue->token_q_unripe == 0) {
                        /* we're removing a ripe token. decrease count */
 
                if (queue->token_q_unripe == 0) {
                        /* we're removing a ripe token. decrease count */


@@ -375,7 +393,7 @@ vm_purgeable_token_remove_last(purgeable_q_t queue)
                        /* we're removing the only unripe token */
                        queue->token_q_unripe = 0;
                }
                        /* we're removing the only unripe token */
                        queue->token_q_unripe = 0;
                }

-                       
+

                if (token == queue->token_q_head) {
                        /* token is the last one in the queue */
                        queue->token_q_head = 0;
                if (token == queue->token_q_head) {
                        /* token is the last one in the queue */
                        queue->token_q_head = 0;


@@ -399,21 +417,21 @@ vm_purgeable_token_remove_last(purgeable_q_t queue)
                vm_purgeable_token_check_queue(queue);
 
                KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, TOKEN_DELETE)),
                vm_purgeable_token_check_queue(queue);
 
                KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, TOKEN_DELETE)),

-                                     queue->type,
-                                     tokens[queue->token_q_head].count,        /* num pages on new
-                                                                                * first token */
-                                     token_new_pagecount,      /* num pages waiting for
-                                                                * next token */
-                                     available_for_purge,
-                                     0);
+                   queue->type,
+                   tokens[queue->token_q_head].count,                          /* num pages on new
+                                                                                * first token */
+                   token_new_pagecount,                        /* num pages waiting for
+                                                                * next token */
+                   available_for_purge,
+                   0);

 #endif
        }
        return token;
 }
 
 #endif
        }
        return token;
 }
 

-/* 
+/*

  * Delete first token from queue. Return token to token queue.
  * Delete first token from queue. Return token to token queue.

- * Call with page queue locked. 
+ * Call with page queue locked.

  */
 void
 vm_purgeable_token_delete_first(purgeable_q_t queue)
  */
 void
 vm_purgeable_token_delete_first(purgeable_q_t queue)


@@ -449,12 +467,11 @@ void
 vm_purgeable_q_advance_all()
 {
        LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
 vm_purgeable_q_advance_all()
 {
        LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);

-       
+

        /* check queue counters - if they get really large, scale them back.
         * They tend to get that large when there is no purgeable queue action */
        int i;
        /* check queue counters - if they get really large, scale them back.
         * They tend to get that large when there is no purgeable queue action */
        int i;

-       if(token_new_pagecount > (TOKEN_NEW_PAGECOUNT_MAX >> 1))        /* a system idling years might get there */
-       {
+       if (token_new_pagecount > (TOKEN_NEW_PAGECOUNT_MAX >> 1)) {      /* a system idling years might get there */

                for (i = PURGEABLE_Q_TYPE_FIFO; i < PURGEABLE_Q_TYPE_MAX; i++) {
                        int64_t pages = purgeable_queues[i].new_pages += token_new_pagecount;
                        assert(pages >= 0);
                for (i = PURGEABLE_Q_TYPE_FIFO; i < PURGEABLE_Q_TYPE_MAX; i++) {
                        int64_t pages = purgeable_queues[i].new_pages += token_new_pagecount;
                        assert(pages >= 0);


@@ -464,7 +481,7 @@ vm_purgeable_q_advance_all()
                }
                token_new_pagecount = 0;
        }
                }
                token_new_pagecount = 0;
        }

-       
+

        /*
         * Decrement token counters. A token counter can be zero, this means the
         * object is ripe to be purged. It is not purged immediately, because that
        /*
         * Decrement token counters. A token counter can be zero, this means the
         * object is ripe to be purged. It is not purged immediately, because that


@@ -479,11 +496,10 @@ vm_purgeable_q_advance_all()
        for (i = PURGEABLE_Q_TYPE_FIFO; i < PURGEABLE_Q_TYPE_MAX; i++) {
                purgeable_q_t queue = &purgeable_queues[i];
                uint32_t num_pages = 1;
        for (i = PURGEABLE_Q_TYPE_FIFO; i < PURGEABLE_Q_TYPE_MAX; i++) {
                purgeable_q_t queue = &purgeable_queues[i];
                uint32_t num_pages = 1;

-               
+

                /* Iterate over tokens as long as there are unripe tokens. */
                while (queue->token_q_unripe) {
                /* Iterate over tokens as long as there are unripe tokens. */
                while (queue->token_q_unripe) {

-                       if (tokens[queue->token_q_unripe].count && num_pages)
-                       {
+                       if (tokens[queue->token_q_unripe].count && num_pages) {

                                tokens[queue->token_q_unripe].count -= 1;
                                num_pages -= 1;
                        }
                                tokens[queue->token_q_unripe].count -= 1;
                                num_pages -= 1;
                        }


@@ -492,18 +508,19 @@ vm_purgeable_q_advance_all()
                                queue->token_q_unripe = tokens[queue->token_q_unripe].next;
                                available_for_purge++;
                                KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, TOKEN_RIPEN)),
                                queue->token_q_unripe = tokens[queue->token_q_unripe].next;
                                available_for_purge++;
                                KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, TOKEN_RIPEN)),

-                                                     queue->type,
-                                                     tokens[queue->token_q_head].count,        /* num pages on new
-                                                                                        * first token */
-                                                     0,
-                                                     available_for_purge,
-                                                     0);
-                               continue;       /* One token ripened. Make sure to
-                                                * check the next. */
+                                   queue->type,
+                                   tokens[queue->token_q_head].count,                          /* num pages on new
+                                                                                                * first token */
+                                   0,
+                                   available_for_purge,
+                                   0);
+                               continue;       /* One token ripened. Make sure to
+                                                * check the next. */
+                       }
+                       if (num_pages == 0) {
+                               break;  /* Current token not ripe and no more pages.
+                                        * Work done. */

                        }
                        }

-                       if (num_pages == 0)
-                               break;  /* Current token not ripe and no more pages.
-                                        * Work done. */

                }
 
                /*
                }
 
                /*


@@ -547,8 +564,9 @@ vm_purgeable_token_remove_ripe(purgeable_q_t queue)
        token_free_idx = queue->token_q_head;
        queue->token_q_head = new_head;
        tokens[new_head].prev = 0;
        token_free_idx = queue->token_q_head;
        queue->token_q_head = new_head;
        tokens[new_head].prev = 0;

-       if (new_head == 0)
+       if (new_head == 0) {

                queue->token_q_tail = 0;
                queue->token_q_tail = 0;

+       }

 
 #if MACH_ASSERT
        queue->debug_count_tokens--;
 
 #if MACH_ASSERT
        queue->debug_count_tokens--;


@@ -586,8 +604,8 @@ vm_purgeable_token_choose_and_delete_ripe(purgeable_q_t queue, purgeable_q_t que
                token_cnt_t     count;
 
                /* remove token from queue1 */
                token_cnt_t     count;
 
                /* remove token from queue1 */

-               assert(queue->token_q_unripe == queue->token_q_head);   /* queue1 had no unripe
-                                                                        * tokens, remember? */
+               assert(queue->token_q_unripe == queue->token_q_head);   /* queue1 had no unripe
+                                                                        * tokens, remember? */

                token = vm_purgeable_token_remove_first(queue);
                assert(token);
 
                token = vm_purgeable_token_remove_first(queue);
                assert(token);
 


@@ -604,11 +622,11 @@ vm_purgeable_token_choose_and_delete_ripe(purgeable_q_t queue, purgeable_q_t que
                }
 
                /* token_to_insert_before is now set correctly */
                }
 
                /* token_to_insert_before is now set correctly */

-       
-               /* should the inserted token become the first unripe token? */
-               if ((token_to_insert_before == queue2->token_q_unripe) || (queue2->token_q_unripe == 0))
-                       queue2->token_q_unripe = token; /* if so, must update unripe pointer */

 
 

+               /* should the inserted token become the first unripe token? */
+               if ((token_to_insert_before == queue2->token_q_unripe) || (queue2->token_q_unripe == 0)) {
+                       queue2->token_q_unripe = token; /* if so, must update unripe pointer */
+               }

                /*
                 * insert token.
                 * if inserting at end, reduce new_pages by that value;
                /*
                 * insert token.
                 * if inserting at end, reduce new_pages by that value;


@@ -656,18 +674,18 @@ vm_purgeable_token_choose_and_delete_ripe(purgeable_q_t queue, purgeable_q_t que
 /* Call with purgeable queue locked. */
 static vm_object_t
 vm_purgeable_object_find_and_lock(
 /* Call with purgeable queue locked. */
 static vm_object_t
 vm_purgeable_object_find_and_lock(

-       purgeable_q_t   queue,
-       int             group,
-       boolean_t       pick_ripe)
+       purgeable_q_t   queue,
+       int             group,
+       boolean_t       pick_ripe)

 {
        vm_object_t     object, best_object;
 {
        vm_object_t     object, best_object;

-       int             object_task_importance;
-       int             best_object_task_importance;
-       int             best_object_skipped;
-       int             num_objects_skipped;
-       int             try_lock_failed = 0;
-       int             try_lock_succeeded = 0;
-       task_t          owner;
+       int             object_task_importance;
+       int             best_object_task_importance;
+       int             best_object_skipped;
+       int             num_objects_skipped;
+       int             try_lock_failed = 0;
+       int             try_lock_succeeded = 0;
+       task_t          owner;

 
        best_object = VM_OBJECT_NULL;
        best_object_task_importance = INT_MAX;
 
        best_object = VM_OBJECT_NULL;
        best_object_task_importance = INT_MAX;


@@ -680,18 +698,17 @@ vm_purgeable_object_find_and_lock(
         */
 
        KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (MACHDBG_CODE(DBG_MACH_VM, OBJECT_PURGE_LOOP) | DBG_FUNC_START),
         */
 
        KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (MACHDBG_CODE(DBG_MACH_VM, OBJECT_PURGE_LOOP) | DBG_FUNC_START),

-                             pick_ripe,
-                             group,
-                             VM_KERNEL_UNSLIDE_OR_PERM(queue),
-                             0,
-                             0);
+           pick_ripe,
+           group,
+           VM_KERNEL_UNSLIDE_OR_PERM(queue),
+           0,
+           0);

 
        num_objects_skipped = 0;
        for (object = (vm_object_t) queue_first(&queue->objq[group]);
 
        num_objects_skipped = 0;
        for (object = (vm_object_t) queue_first(&queue->objq[group]);

-            !queue_end(&queue->objq[group], (queue_entry_t) object);
-            object = (vm_object_t) queue_next(&object->objq),
-               num_objects_skipped++) {
-
+           !queue_end(&queue->objq[group], (queue_entry_t) object);
+           object = (vm_object_t) queue_next(&object->objq),
+           num_objects_skipped++) {

                /*
                 * To prevent us looping for an excessively long time, choose
                 * the best object we've seen after looking at PURGEABLE_LOOP_MAX elements.
                /*
                 * To prevent us looping for an excessively long time, choose
                 * the best object we've seen after looking at PURGEABLE_LOOP_MAX elements.


@@ -703,16 +720,27 @@ vm_purgeable_object_find_and_lock(
                }
 
                if (pick_ripe &&
                }
 
                if (pick_ripe &&

-                   ! object->purgeable_when_ripe) {
+                   !object->purgeable_when_ripe) {

                        /* we want an object that has a ripe token */
                        continue;
                }
 
                object_task_importance = 0;
 
                        /* we want an object that has a ripe token */
                        continue;
                }
 
                object_task_importance = 0;
 

-               owner = object->vo_purgeable_owner;
-               if (owner) {
+               /*
+                * We don't want to use VM_OBJECT_OWNER() here: we want to
+                * distinguish kernel-owned and disowned objects.
+                * Disowned objects have no owner and will have no importance...
+                */
+               owner = object->vo_owner;
+               if (owner != NULL && owner != VM_OBJECT_OWNER_DISOWNED) {
+#if CONFIG_EMBEDDED
+#if CONFIG_JETSAM
+                       object_task_importance = proc_get_memstat_priority((struct proc *)get_bsdtask_info(owner), TRUE);
+#endif /* CONFIG_JETSAM */
+#else /* CONFIG_EMBEDDED */

                        object_task_importance = task_importance_estimate(owner);
                        object_task_importance = task_importance_estimate(owner);

+#endif /* CONFIG_EMBEDDED */

                }
 
                if (object_task_importance < best_object_task_importance) {
                }
 
                if (object_task_importance < best_object_task_importance) {


@@ -736,11 +764,11 @@ vm_purgeable_object_find_and_lock(
        }
 
        KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (MACHDBG_CODE(DBG_MACH_VM, OBJECT_PURGE_LOOP) | DBG_FUNC_END),
        }
 
        KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (MACHDBG_CODE(DBG_MACH_VM, OBJECT_PURGE_LOOP) | DBG_FUNC_END),

-                             num_objects_skipped, /* considered objects */
-                             try_lock_failed,
-                             try_lock_succeeded,
-                             VM_KERNEL_UNSLIDE_OR_PERM(best_object),
-                             ((best_object == NULL) ? 0 : best_object->resident_page_count));
+           num_objects_skipped,                   /* considered objects */
+           try_lock_failed,
+           try_lock_succeeded,
+           VM_KERNEL_UNSLIDE_OR_PERM(best_object),
+           ((best_object == NULL) ? 0 : best_object->resident_page_count));

 
        object = best_object;
 
 
        object = best_object;
 


@@ -754,13 +782,13 @@ vm_purgeable_object_find_and_lock(
        vm_object_lock_assert_exclusive(object);
 
        queue_remove(&queue->objq[group], object,
        vm_object_lock_assert_exclusive(object);
 
        queue_remove(&queue->objq[group], object,

-                    vm_object_t, objq);
+           vm_object_t, objq);

        object->objq.next = NULL;
        object->objq.prev = NULL;
        object->purgeable_queue_type = PURGEABLE_Q_TYPE_MAX;
        object->purgeable_queue_group = 0;
        /* one less volatile object for this object's owner */
        object->objq.next = NULL;
        object->objq.prev = NULL;
        object->purgeable_queue_type = PURGEABLE_Q_TYPE_MAX;
        object->purgeable_queue_group = 0;
        /* one less volatile object for this object's owner */

-       vm_purgeable_volatile_owner_update(object->vo_purgeable_owner, -1);
+       vm_purgeable_volatile_owner_update(VM_OBJECT_OWNER(object), -1);

 
 #if DEBUG
        object->vo_purgeable_volatilizer = NULL;
 
 #if DEBUG
        object->vo_purgeable_volatilizer = NULL;


@@ -768,12 +796,12 @@ vm_purgeable_object_find_and_lock(
 
        /* keep queue of non-volatile objects */
        queue_enter(&purgeable_nonvolatile_queue, object,
 
        /* keep queue of non-volatile objects */
        queue_enter(&purgeable_nonvolatile_queue, object,

-                   vm_object_t, objq);
+           vm_object_t, objq);

        assert(purgeable_nonvolatile_count >= 0);
        purgeable_nonvolatile_count++;
        assert(purgeable_nonvolatile_count > 0);
        /* one more nonvolatile object for this object's owner */
        assert(purgeable_nonvolatile_count >= 0);
        purgeable_nonvolatile_count++;
        assert(purgeable_nonvolatile_count > 0);
        /* one more nonvolatile object for this object's owner */

-       vm_purgeable_nonvolatile_owner_update(object->vo_purgeable_owner, +1);
+       vm_purgeable_nonvolatile_owner_update(VM_OBJECT_OWNER(object), +1);

 
 #if MACH_ASSERT
        queue->debug_count_objects--;
 
 #if MACH_ASSERT
        queue->debug_count_objects--;


@@ -788,8 +816,8 @@ vm_purgeable_object_purge_all(void)
        enum purgeable_q_type i;
        int             group;
        vm_object_t     object;
        enum purgeable_q_type i;
        int             group;
        vm_object_t     object;

-       unsigned int    purged_count;
-       uint32_t        collisions;
+       unsigned int    purged_count;
+       uint32_t        collisions;

 
        purged_count = 0;
        collisions = 0;
 
        purged_count = 0;
        collisions = 0;


@@ -818,7 +846,7 @@ restart:
                                }
 
                                lck_mtx_unlock(&vm_purgeable_queue_lock);
                                }
 
                                lck_mtx_unlock(&vm_purgeable_queue_lock);

-                               
+

                                /* Lock the page queue here so we don't hold it
                                 * over the whole, legthy operation */
                                if (object->purgeable_when_ripe) {
                                /* Lock the page queue here so we don't hold it
                                 * over the whole, legthy operation */
                                if (object->purgeable_when_ripe) {


@@ -826,7 +854,7 @@ restart:
                                        vm_purgeable_token_remove_first(queue);
                                        vm_page_unlock_queues();
                                }
                                        vm_purgeable_token_remove_first(queue);
                                        vm_page_unlock_queues();
                                }

-                               
+

                                (void) vm_object_purge(object, 0);
                                assert(object->purgable == VM_PURGABLE_EMPTY);
                                /* no change in purgeable accounting */
                                (void) vm_object_purge(object, 0);
                                assert(object->purgable == VM_PURGABLE_EMPTY);
                                /* no change in purgeable accounting */


@@ -839,20 +867,20 @@ restart:
                }
        }
        KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, OBJECT_PURGE_ALL)),
                }
        }
        KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, OBJECT_PURGE_ALL)),

-                             purged_count, /* # of purged objects */
-                             0,
-                             available_for_purge,
-                             0,
-                             0);
+           purged_count,                   /* # of purged objects */
+           0,
+           available_for_purge,
+           0,
+           0);

        lck_mtx_unlock(&vm_purgeable_queue_lock);
        return;
 }
 
 boolean_t
 vm_purgeable_object_purge_one_unlocked(
        lck_mtx_unlock(&vm_purgeable_queue_lock);
        return;
 }
 
 boolean_t
 vm_purgeable_object_purge_one_unlocked(

-       int     force_purge_below_group)
+       int     force_purge_below_group)

 {
 {

-       boolean_t       retval;
+       boolean_t       retval;

 
        vm_page_lock_queues();
        retval = vm_purgeable_object_purge_one(force_purge_below_group, 0);
 
        vm_page_lock_queues();
        retval = vm_purgeable_object_purge_one(force_purge_below_group, 0);


@@ -863,19 +891,24 @@ vm_purgeable_object_purge_one_unlocked(
 
 boolean_t
 vm_purgeable_object_purge_one(
 
 boolean_t
 vm_purgeable_object_purge_one(

-       int     force_purge_below_group,
-       int     flags)
+       int     force_purge_below_group,
+       int     flags)

 {
        enum purgeable_q_type i;
        int             group;
        vm_object_t     object = 0;
        purgeable_q_t   queue, queue2;
 {
        enum purgeable_q_type i;
        int             group;
        vm_object_t     object = 0;
        purgeable_q_t   queue, queue2;

-       boolean_t       forced_purge;
+       boolean_t       forced_purge;
+       unsigned int    resident_page_count;
+
+
+       KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, OBJECT_PURGE)) | DBG_FUNC_START,
+           force_purge_below_group, flags, 0, 0, 0);

 
        /* Need the page queue lock since we'll be changing the token queue. */
        LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
        lck_mtx_lock(&vm_purgeable_queue_lock);
 
        /* Need the page queue lock since we'll be changing the token queue. */
        LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
        lck_mtx_lock(&vm_purgeable_queue_lock);

-       
+

        /* Cycle through all queues */
        for (i = PURGEABLE_Q_TYPE_OBSOLETE; i < PURGEABLE_Q_TYPE_MAX; i++) {
                queue = &purgeable_queues[i];
        /* Cycle through all queues */
        for (i = PURGEABLE_Q_TYPE_OBSOLETE; i < PURGEABLE_Q_TYPE_MAX; i++) {
                queue = &purgeable_queues[i];


@@ -929,7 +962,7 @@ vm_purgeable_object_purge_one(
                                /* nothing to purge in this group: next group */
                                continue;
                        }
                                /* nothing to purge in this group: next group */
                                continue;
                        }

-                       if (!queue_empty(&queue->objq[group]) && 
+                       if (!queue_empty(&queue->objq[group]) &&

                            (object = vm_purgeable_object_find_and_lock(queue, group, TRUE))) {
                                lck_mtx_unlock(&vm_purgeable_queue_lock);
                                if (object->purgeable_when_ripe) {
                            (object = vm_purgeable_object_find_and_lock(queue, group, TRUE))) {
                                lck_mtx_unlock(&vm_purgeable_queue_lock);
                                if (object->purgeable_when_ripe) {


@@ -938,14 +971,14 @@ vm_purgeable_object_purge_one(
                                forced_purge = FALSE;
                                goto purge_now;
                        }
                                forced_purge = FALSE;
                                goto purge_now;
                        }

-                       if (i != PURGEABLE_Q_TYPE_OBSOLETE) { 
+                       if (i != PURGEABLE_Q_TYPE_OBSOLETE) {

                                /* This is the token migration case, and it works between
                                 * FIFO and LIFO only */
                                /* This is the token migration case, and it works between
                                 * FIFO and LIFO only */

-                               queue2 = &purgeable_queues[i != PURGEABLE_Q_TYPE_FIFO ? 
-                                                          PURGEABLE_Q_TYPE_FIFO : 
-                                                          PURGEABLE_Q_TYPE_LIFO];
+                               queue2 = &purgeable_queues[i != PURGEABLE_Q_TYPE_FIFO ?
+                                   PURGEABLE_Q_TYPE_FIFO :
+                                   PURGEABLE_Q_TYPE_LIFO];

 
 

-                               if (!queue_empty(&queue2->objq[group]) && 
+                               if (!queue_empty(&queue2->objq[group]) &&

                                    (object = vm_purgeable_object_find_and_lock(queue2, group, TRUE))) {
                                        lck_mtx_unlock(&vm_purgeable_queue_lock);
                                        if (object->purgeable_when_ripe) {
                                    (object = vm_purgeable_object_find_and_lock(queue2, group, TRUE))) {
                                        lck_mtx_unlock(&vm_purgeable_queue_lock);
                                        if (object->purgeable_when_ripe) {


@@ -959,31 +992,38 @@ vm_purgeable_object_purge_one(
                }
        }
        /*
                }
        }
        /*

-         * because we have to do a try_lock on the objects which could fail,
-         * we could end up with no object to purge at this time, even though
-         * we have objects in a purgeable state
-         */
+        * because we have to do a try_lock on the objects which could fail,
+        * we could end up with no object to purge at this time, even though
+        * we have objects in a purgeable state
+        */

        lck_mtx_unlock(&vm_purgeable_queue_lock);
        lck_mtx_unlock(&vm_purgeable_queue_lock);

+
+       KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, OBJECT_PURGE)) | DBG_FUNC_END,
+           0, 0, available_for_purge, 0, 0);
+

        return FALSE;
 
 purge_now:
 
        assert(object);
        vm_page_unlock_queues();  /* Unlock for call to vm_object_purge() */
        return FALSE;
 
 purge_now:
 
        assert(object);
        vm_page_unlock_queues();  /* Unlock for call to vm_object_purge() */

-//     printf("%sPURGING object %p task %p importance %d queue %d group %d force_purge_below_group %d memorystatus_vm_pressure_level %d\n", forced_purge ? "FORCED " : "", object, object->vo_purgeable_owner, task_importance_estimate(object->vo_purgeable_owner), i, group, force_purge_below_group, memorystatus_vm_pressure_level);
+//     printf("%sPURGING object %p task %p importance %d queue %d group %d force_purge_below_group %d memorystatus_vm_pressure_level %d\n", forced_purge ? "FORCED " : "", object, object->vo_owner, task_importance_estimate(object->vo_owner), i, group, force_purge_below_group, memorystatus_vm_pressure_level);
+       resident_page_count = object->resident_page_count;

        (void) vm_object_purge(object, flags);
        assert(object->purgable == VM_PURGABLE_EMPTY);
        /* no change in purgeable accounting */
        vm_object_unlock(object);
        vm_page_lock_queues();
 
        (void) vm_object_purge(object, flags);
        assert(object->purgable == VM_PURGABLE_EMPTY);
        /* no change in purgeable accounting */
        vm_object_unlock(object);
        vm_page_lock_queues();
 

-       KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, OBJECT_PURGE)),
-                             object,   /* purged object */
-                             0,
-                             available_for_purge,
-                             0,
-                             0);
-       
+       vm_pageout_vminfo.vm_pageout_pages_purged += resident_page_count;
+
+       KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, OBJECT_PURGE)) | DBG_FUNC_END,
+           VM_KERNEL_UNSLIDE_OR_PERM(object),                          /* purged object */
+           resident_page_count,
+           available_for_purge,
+           0,
+           0);
+

        return TRUE;
 }
 
        return TRUE;
 }
 


@@ -997,25 +1037,27 @@ vm_purgeable_object_add(vm_object_t object, purgeable_q_t queue, int group)
        assert(object->objq.next != NULL);
        assert(object->objq.prev != NULL);
        queue_remove(&purgeable_nonvolatile_queue, object,
        assert(object->objq.next != NULL);
        assert(object->objq.prev != NULL);
        queue_remove(&purgeable_nonvolatile_queue, object,

-                    vm_object_t, objq);
+           vm_object_t, objq);

        object->objq.next = NULL;
        object->objq.prev = NULL;
        assert(purgeable_nonvolatile_count > 0);
        purgeable_nonvolatile_count--;
        assert(purgeable_nonvolatile_count >= 0);
        /* one less nonvolatile object for this object's owner */
        object->objq.next = NULL;
        object->objq.prev = NULL;
        assert(purgeable_nonvolatile_count > 0);
        purgeable_nonvolatile_count--;
        assert(purgeable_nonvolatile_count >= 0);
        /* one less nonvolatile object for this object's owner */

-       vm_purgeable_nonvolatile_owner_update(object->vo_purgeable_owner, -1);
+       vm_purgeable_nonvolatile_owner_update(VM_OBJECT_OWNER(object), -1);

 
 

-       if (queue->type == PURGEABLE_Q_TYPE_OBSOLETE)
+       if (queue->type == PURGEABLE_Q_TYPE_OBSOLETE) {

                group = 0;
                group = 0;

+       }

 
 

-       if (queue->type != PURGEABLE_Q_TYPE_LIFO)       /* fifo and obsolete are
-                                                        * fifo-queued */
-               queue_enter(&queue->objq[group], object, vm_object_t, objq);    /* last to die */
-       else
-               queue_enter_first(&queue->objq[group], object, vm_object_t, objq);      /* first to die */
+       if (queue->type != PURGEABLE_Q_TYPE_LIFO) {     /* fifo and obsolete are
+                                                        * fifo-queued */
+               queue_enter(&queue->objq[group], object, vm_object_t, objq);    /* last to die */
+       } else {
+               queue_enter_first(&queue->objq[group], object, vm_object_t, objq);      /* first to die */
+       }

        /* one more volatile object for this object's owner */
        /* one more volatile object for this object's owner */

-       vm_purgeable_volatile_owner_update(object->vo_purgeable_owner, +1);
+       vm_purgeable_volatile_owner_update(VM_OBJECT_OWNER(object), +1);

 
        object->purgeable_queue_type = queue->type;
        object->purgeable_queue_group = group;
 
        object->purgeable_queue_type = queue->type;
        object->purgeable_queue_group = group;


@@ -1023,17 +1065,18 @@ vm_purgeable_object_add(vm_object_t object, purgeable_q_t queue, int group)
 #if DEBUG
        assert(object->vo_purgeable_volatilizer == NULL);
        object->vo_purgeable_volatilizer = current_task();
 #if DEBUG
        assert(object->vo_purgeable_volatilizer == NULL);
        object->vo_purgeable_volatilizer = current_task();

-       OSBacktrace(&object->purgeable_volatilizer_bt[0], 16);
+       OSBacktrace(&object->purgeable_volatilizer_bt[0],
+           ARRAY_COUNT(object->purgeable_volatilizer_bt));

 #endif /* DEBUG */
 
 #if MACH_ASSERT
        queue->debug_count_objects++;
        KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, OBJECT_ADD)),
 #endif /* DEBUG */
 
 #if MACH_ASSERT
        queue->debug_count_objects++;
        KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, OBJECT_ADD)),

-                             0,
-                             tokens[queue->token_q_head].count,
-                             queue->type,
-                             group,
-                             0);
+           0,
+           tokens[queue->token_q_head].count,
+           queue->type,
+           group,
+           0);

 #endif
 
        lck_mtx_unlock(&vm_purgeable_queue_lock);
 #endif
 
        lck_mtx_unlock(&vm_purgeable_queue_lock);


@@ -1054,12 +1097,14 @@ vm_purgeable_object_remove(vm_object_t object)
        group = object->purgeable_queue_group;
 
        if (type == PURGEABLE_Q_TYPE_MAX) {
        group = object->purgeable_queue_group;
 
        if (type == PURGEABLE_Q_TYPE_MAX) {

-               if (object->objq.prev || object->objq.next)
+               if (object->objq.prev || object->objq.next) {

                        panic("unmarked object on purgeable q");
                        panic("unmarked object on purgeable q");

+               }

 
                return NULL;
 
                return NULL;

-       } else if (!(object->objq.prev && object->objq.next))
+       } else if (!(object->objq.prev && object->objq.next)) {

                panic("marked object not on purgeable q");
                panic("marked object not on purgeable q");

+       }

 
        lck_mtx_lock(&vm_purgeable_queue_lock);
 
 
        lck_mtx_lock(&vm_purgeable_queue_lock);
 


@@ -1069,31 +1114,29 @@ vm_purgeable_object_remove(vm_object_t object)
        object->objq.next = NULL;
        object->objq.prev = NULL;
        /* one less volatile object for this object's owner */
        object->objq.next = NULL;
        object->objq.prev = NULL;
        /* one less volatile object for this object's owner */

-       vm_purgeable_volatile_owner_update(object->vo_purgeable_owner, -1);
+       vm_purgeable_volatile_owner_update(VM_OBJECT_OWNER(object), -1);

 #if DEBUG
        object->vo_purgeable_volatilizer = NULL;
 #endif /* DEBUG */
        /* keep queue of non-volatile objects */
        if (object->alive && !object->terminating) {
 #if DEBUG
        object->vo_purgeable_volatilizer = NULL;
 #endif /* DEBUG */
        /* keep queue of non-volatile objects */
        if (object->alive && !object->terminating) {

-               task_t  owner;

                queue_enter(&purgeable_nonvolatile_queue, object,
                queue_enter(&purgeable_nonvolatile_queue, object,

-                           vm_object_t, objq);
+                   vm_object_t, objq);

                assert(purgeable_nonvolatile_count >= 0);
                purgeable_nonvolatile_count++;
                assert(purgeable_nonvolatile_count > 0);
                /* one more nonvolatile object for this object's owner */
                assert(purgeable_nonvolatile_count >= 0);
                purgeable_nonvolatile_count++;
                assert(purgeable_nonvolatile_count > 0);
                /* one more nonvolatile object for this object's owner */

-               owner = object->vo_purgeable_owner;
-               vm_purgeable_nonvolatile_owner_update(owner, +1);
+               vm_purgeable_nonvolatile_owner_update(VM_OBJECT_OWNER(object), +1);

        }
 
 #if MACH_ASSERT
        queue->debug_count_objects--;
        KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, OBJECT_REMOVE)),
        }
 
 #if MACH_ASSERT
        queue->debug_count_objects--;
        KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, OBJECT_REMOVE)),

-                             0,
-                             tokens[queue->token_q_head].count,
-                             queue->type,
-                             group,
-                             0);
+           0,
+           tokens[queue->token_q_head].count,
+           queue->type,
+           group,
+           0);

 #endif
 
        lck_mtx_unlock(&vm_purgeable_queue_lock);
 #endif
 
        lck_mtx_unlock(&vm_purgeable_queue_lock);


@@ -1114,12 +1157,12 @@ vm_purgeable_stats_helper(vm_purgeable_stat_t *stat, purgeable_q_t queue, int gr
        stat->count = stat->size = 0;
        vm_object_t     object;
        for (object = (vm_object_t) queue_first(&queue->objq[group]);
        stat->count = stat->size = 0;
        vm_object_t     object;
        for (object = (vm_object_t) queue_first(&queue->objq[group]);

-            !queue_end(&queue->objq[group], (queue_entry_t) object);
-            object = (vm_object_t) queue_next(&object->objq)) {
-                       if (!target_task || object->vo_purgeable_owner == target_task) {
-                               stat->count++;
-                               stat->size += (object->resident_page_count * PAGE_SIZE);
-                       }
+           !queue_end(&queue->objq[group], (queue_entry_t) object);
+           object = (vm_object_t) queue_next(&object->objq)) {
+               if (!target_task || VM_OBJECT_OWNER(object) == target_task) {
+                       stat->count++;
+                       stat->size += (object->resident_page_count * PAGE_SIZE);
+               }

        }
        return;
 }
        }
        return;
 }


@@ -1127,20 +1170,22 @@ vm_purgeable_stats_helper(vm_purgeable_stat_t *stat, purgeable_q_t queue, int gr
 void
 vm_purgeable_stats(vm_purgeable_info_t info, task_t target_task)
 {
 void
 vm_purgeable_stats(vm_purgeable_info_t info, task_t target_task)
 {

-       purgeable_q_t   queue;
+       purgeable_q_t   queue;

        int             group;
 
        lck_mtx_lock(&vm_purgeable_queue_lock);
        int             group;
 
        lck_mtx_lock(&vm_purgeable_queue_lock);

-       
+

        /* Populate fifo_data */
        queue = &purgeable_queues[PURGEABLE_Q_TYPE_FIFO];
        /* Populate fifo_data */
        queue = &purgeable_queues[PURGEABLE_Q_TYPE_FIFO];

-       for (group = 0; group < NUM_VOLATILE_GROUPS; group++)
+       for (group = 0; group < NUM_VOLATILE_GROUPS; group++) {

                vm_purgeable_stats_helper(&(info->fifo_data[group]), queue, group, target_task);
                vm_purgeable_stats_helper(&(info->fifo_data[group]), queue, group, target_task);

-       
+       }
+

        /* Populate lifo_data */
        queue = &purgeable_queues[PURGEABLE_Q_TYPE_LIFO];
        /* Populate lifo_data */
        queue = &purgeable_queues[PURGEABLE_Q_TYPE_LIFO];

-       for (group = 0; group < NUM_VOLATILE_GROUPS; group++)
+       for (group = 0; group < NUM_VOLATILE_GROUPS; group++) {

                vm_purgeable_stats_helper(&(info->lifo_data[group]), queue, group, target_task);
                vm_purgeable_stats_helper(&(info->lifo_data[group]), queue, group, target_task);

+       }

 
        /* Populate obsolete data */
        queue = &purgeable_queues[PURGEABLE_Q_TYPE_OBSOLETE];
 
        /* Populate obsolete data */
        queue = &purgeable_queues[PURGEABLE_Q_TYPE_OBSOLETE];


@@ -1164,14 +1209,13 @@ vm_purgeable_account_volatile_queue(
        for (object = (vm_object_t) queue_first(&queue->objq[group]);
            !queue_end(&queue->objq[group], (queue_entry_t) object);
            object = (vm_object_t) queue_next(&object->objq)) {
        for (object = (vm_object_t) queue_first(&queue->objq[group]);
            !queue_end(&queue->objq[group], (queue_entry_t) object);
            object = (vm_object_t) queue_next(&object->objq)) {

-               if (object->vo_purgeable_owner == task) {
+               if (VM_OBJECT_OWNER(object) == task) {

                        compressed_count = vm_compressor_pager_get_count(object->pager);
                        acnt_info->pvm_volatile_compressed_count += compressed_count;
                        acnt_info->pvm_volatile_count += (object->resident_page_count - object->wired_page_count);
                        acnt_info->pvm_nonvolatile_count += object->wired_page_count;
                }
        }
                        compressed_count = vm_compressor_pager_get_count(object->pager);
                        acnt_info->pvm_volatile_compressed_count += compressed_count;
                        acnt_info->pvm_volatile_count += (object->resident_page_count - object->wired_page_count);
                        acnt_info->pvm_nonvolatile_count += object->wired_page_count;
                }
        }

-

 }
 
 /*
 }
 
 /*


@@ -1180,15 +1224,15 @@ vm_purgeable_account_volatile_queue(
  */
 kern_return_t
 vm_purgeable_account(
  */
 kern_return_t
 vm_purgeable_account(

-       task_t                  task,
-       pvm_account_info_t      acnt_info)
+       task_t                  task,
+       pvm_account_info_t      acnt_info)

 {
 {

-       queue_head_t    *nonvolatile_q;
-       vm_object_t     object;
-       int             group;
-       int             state;
-       uint64_t        compressed_count;
-       purgeable_q_t   volatile_q;
+       queue_head_t    *nonvolatile_q;
+       vm_object_t     object;
+       int             group;
+       int             state;
+       uint64_t        compressed_count;
+       purgeable_q_t   volatile_q;

 
 
        if ((task == NULL) || (acnt_info == NULL)) {
 
 
        if ((task == NULL) || (acnt_info == NULL)) {


@@ -1204,9 +1248,9 @@ vm_purgeable_account(
 
        nonvolatile_q = &purgeable_nonvolatile_queue;
        for (object = (vm_object_t) queue_first(nonvolatile_q);
 
        nonvolatile_q = &purgeable_nonvolatile_queue;
        for (object = (vm_object_t) queue_first(nonvolatile_q);

-            !queue_end(nonvolatile_q, (queue_entry_t) object);
-            object = (vm_object_t) queue_next(&object->objq)) {
-               if (object->vo_purgeable_owner == task) {
+           !queue_end(nonvolatile_q, (queue_entry_t) object);
+           object = (vm_object_t) queue_next(&object->objq)) {
+               if (VM_OBJECT_OWNER(object) == task) {

                        state = object->purgable;
                        compressed_count =  vm_compressor_pager_get_count(object->pager);
                        if (state == VM_PURGABLE_EMPTY) {
                        state = object->purgable;
                        compressed_count =  vm_compressor_pager_get_count(object->pager);
                        if (state == VM_PURGABLE_EMPTY) {


@@ -1243,72 +1287,18 @@ vm_purgeable_account(
 }
 #endif /* DEVELOPMENT || DEBUG */
 
 }
 #endif /* DEVELOPMENT || DEBUG */
 

-static void
-vm_purgeable_volatile_queue_disown(
-       purgeable_q_t   queue,
-       int             group,
-       task_t          task)
-{
-       vm_object_t     object;
-       int             collisions;
-
-       collisions = 0;
-
-again:
-       LCK_MTX_ASSERT(&vm_purgeable_queue_lock, LCK_MTX_ASSERT_OWNED);
-
-       for (object = (vm_object_t) queue_first(&queue->objq[group]);
-            !queue_end(&queue->objq[group], (queue_entry_t) object);
-            object = (vm_object_t) queue_next(&object->objq)) {
-#if MACH_ASSERT
-               /*
-                * Sanity check: let's scan the entire queues to
-                * make sure we don't leave any purgeable objects
-                * pointing back at a dead task.  If the counters
-                * are off, we would fail to assert that they go
-                * back to 0 after disowning is done.
-                */
-#else /* MACH_ASSERT */
-               if (task->task_volatile_objects == 0) {
-                       /* no more volatile objects owned by "task" */
-                       break;
-               }
-#endif /* MACH_ASSERT */
-               if (object->vo_purgeable_owner == task) {
-                       if (! vm_object_lock_try(object)) {
-                               lck_mtx_unlock(&vm_purgeable_queue_lock);
-                               mutex_pause(collisions++);
-                               lck_mtx_lock(&vm_purgeable_queue_lock);
-                               goto again;
-                       }
-                       assert(object->purgable == VM_PURGABLE_VOLATILE);
-                       if (object->vo_purgeable_owner == task) {
-                               vm_purgeable_accounting(object,
-                                                       object->purgable,
-                                                       TRUE); /* disown */
-                               assert(object->vo_purgeable_owner == NULL);
-                       }
-                       vm_object_unlock(object);
-               }
-       }
-}
-

 void
 vm_purgeable_disown(
 void
 vm_purgeable_disown(

-       task_t  task)
+       task_t  task)

 {
 {

-       purgeable_q_t   volatile_q;
-       int             group;
-       queue_head_t    *nonvolatile_q;
-       vm_object_t     object;
-       int             collisions;
+       vm_object_t     next_object;
+       vm_object_t     object;
+       int             collisions;

 
        if (task == NULL) {
                return;
        }
 
 
        if (task == NULL) {
                return;
        }
 

-       task->task_purgeable_disowning = TRUE;
-

        /*
         * Scan the purgeable objects queues for objects owned by "task".
         * This has to be done "atomically" under the "vm_purgeable_queue"
        /*
         * Scan the purgeable objects queues for objects owned by "task".
         * This has to be done "atomically" under the "vm_purgeable_queue"


@@ -1329,73 +1319,58 @@ again:
                assert(task->task_nonvolatile_objects == 0);
                return;
        }
                assert(task->task_nonvolatile_objects == 0);
                return;
        }

+

        lck_mtx_lock(&vm_purgeable_queue_lock);
        lck_mtx_lock(&vm_purgeable_queue_lock);

+       task_objq_lock(task);

 
 

-       nonvolatile_q = &purgeable_nonvolatile_queue;
-       for (object = (vm_object_t) queue_first(nonvolatile_q);
-             !queue_end(nonvolatile_q, (queue_entry_t) object);
-            object = (vm_object_t) queue_next(&object->objq)) {
-#if MACH_ASSERT
-               /*
-                * Sanity check: let's scan the entire queues to
-                * make sure we don't leave any purgeable objects
-                * pointing back at a dead task.  If the counters
-                * are off, we would fail to assert that they go
-                * back to 0 after disowning is done.
-                */
-#else /* MACH_ASSERT */
-               if (task->task_nonvolatile_objects == 0) {
-                       /* no more non-volatile objects owned by "task" */
+       task->task_purgeable_disowning = TRUE;
+
+       for (object = (vm_object_t) queue_first(&task->task_objq);
+           !queue_end(&task->task_objq, (queue_entry_t) object);
+           object = next_object) {
+               if (task->task_nonvolatile_objects == 0 &&
+                   task->task_volatile_objects == 0) {
+                       /* no more purgeable objects owned by "task" */

                        break;
                }
                        break;
                }

-#endif /* MACH_ASSERT */
+
+               next_object = (vm_object_t) queue_next(&object->task_objq);
+               if (object->purgable == VM_PURGABLE_DENY) {
+                       /* not a purgeable object: skip */
+                       continue;
+               }
+

 #if DEBUG
                assert(object->vo_purgeable_volatilizer == NULL);
 #endif /* DEBUG */
 #if DEBUG
                assert(object->vo_purgeable_volatilizer == NULL);
 #endif /* DEBUG */

-               if (object->vo_purgeable_owner == task) {
-                       if (!vm_object_lock_try(object)) {
-                               lck_mtx_unlock(&vm_purgeable_queue_lock);
-                               mutex_pause(collisions++);
-                               goto again;
-                       }
-                       if (object->vo_purgeable_owner == task) {
-                               vm_purgeable_accounting(object,
-                                                       object->purgable,
-                                                       TRUE); /* disown */
-                               assert(object->vo_purgeable_owner == NULL);
-                       }
-                       vm_object_unlock(object);
+               assert(object->vo_owner == task);
+               if (!vm_object_lock_try(object)) {
+                       lck_mtx_unlock(&vm_purgeable_queue_lock);
+                       task_objq_unlock(task);
+                       mutex_pause(collisions++);
+                       goto again;

                }
                }

+               /* transfer ownership to the kernel */
+               assert(VM_OBJECT_OWNER(object) != kernel_task);
+               vm_object_ownership_change(
+                       object,
+                       object->vo_ledger_tag, /* unchanged */
+                       VM_OBJECT_OWNER_DISOWNED, /* new owner */
+                       TRUE);  /* old_owner->task_objq locked */
+               assert(object->vo_owner == VM_OBJECT_OWNER_DISOWNED);
+               vm_object_unlock(object);

        }
 
        }
 

-       lck_mtx_yield(&vm_purgeable_queue_lock);
-
-       /*
-        * Scan volatile queues for objects owned by "task".
-        */
-
-       volatile_q = &purgeable_queues[PURGEABLE_Q_TYPE_OBSOLETE];
-       vm_purgeable_volatile_queue_disown(volatile_q, 0, task);
-       lck_mtx_yield(&vm_purgeable_queue_lock);
-
-       volatile_q = &purgeable_queues[PURGEABLE_Q_TYPE_FIFO];
-       for (group = 0; group < NUM_VOLATILE_GROUPS; group++) {
-               vm_purgeable_volatile_queue_disown(volatile_q, group, task);
-               lck_mtx_yield(&vm_purgeable_queue_lock);
-       }
-       
-       volatile_q = &purgeable_queues[PURGEABLE_Q_TYPE_LIFO];
-       for (group = 0; group < NUM_VOLATILE_GROUPS; group++) {
-               vm_purgeable_volatile_queue_disown(volatile_q, group, task);
-               lck_mtx_yield(&vm_purgeable_queue_lock);
-       }
-
-       if (task->task_volatile_objects != 0 ||
-           task->task_nonvolatile_objects != 0) {
-               /* some purgeable objects sneaked into a queue: find them */
-               lck_mtx_unlock(&vm_purgeable_queue_lock);
-               mutex_pause(collisions++);
-               goto again;
+       if (__improbable(task->task_volatile_objects != 0 ||
+           task->task_nonvolatile_objects != 0)) {
+               panic("%s(%p): volatile=%d nonvolatile=%d q=%p q_first=%p q_last=%p",
+                   __FUNCTION__,
+                   task,
+                   task->task_volatile_objects,
+                   task->task_nonvolatile_objects,
+                   &task->task_objq,
+                   queue_first(&task->task_objq),
+                   queue_last(&task->task_objq));

        }
 
        /* there shouldn't be any purgeable objects owned by task now */
        }
 
        /* there shouldn't be any purgeable objects owned by task now */


@@ -1407,34 +1382,30 @@ again:
        task->task_purgeable_disowned = TRUE;
 
        lck_mtx_unlock(&vm_purgeable_queue_lock);
        task->task_purgeable_disowned = TRUE;
 
        lck_mtx_unlock(&vm_purgeable_queue_lock);

+       task_objq_unlock(task);

 }
 
 
 }
 
 

-#if notyet
-static int
+static uint64_t

 vm_purgeable_queue_purge_task_owned(
 vm_purgeable_queue_purge_task_owned(

-       purgeable_q_t   queue,
-       int             group,
-       task_t          task)
+       purgeable_q_t   queue,
+       int             group,
+       task_t          task)

 {
 {

-       vm_object_t     object;
-       int             num_objects;
-       int             collisions;
-       int             num_objects_purged;
+       vm_object_t     object = VM_OBJECT_NULL;
+       int             collisions = 0;
+       uint64_t        num_pages_purged = 0;

 
 

-       num_objects_purged = 0;
+       num_pages_purged = 0;

        collisions = 0;
 
 look_again:
        lck_mtx_lock(&vm_purgeable_queue_lock);
 
        collisions = 0;
 
 look_again:
        lck_mtx_lock(&vm_purgeable_queue_lock);
 

-       num_objects = 0;

        for (object = (vm_object_t) queue_first(&queue->objq[group]);
        for (object = (vm_object_t) queue_first(&queue->objq[group]);

-            !queue_end(&queue->objq[group], (queue_entry_t) object);
-            object = (vm_object_t) queue_next(&object->objq)) {
-
-               if (object->vo_purgeable_owner != task &&
-                   object->vo_purgeable_owner != NULL) {
+           !queue_end(&queue->objq[group], (queue_entry_t) object);
+           object = (vm_object_t) queue_next(&object->objq)) {
+               if (object->vo_owner != task) {

                        continue;
                }
 
                        continue;
                }
 


@@ -1450,23 +1421,25 @@ look_again:
 
                /* remove object from purgeable queue */
                queue_remove(&queue->objq[group], object,
 
                /* remove object from purgeable queue */
                queue_remove(&queue->objq[group], object,

-                            vm_object_t, objq);
+                   vm_object_t, objq);

                object->objq.next = NULL;
                object->objq.prev = NULL;
                object->objq.next = NULL;
                object->objq.prev = NULL;

+               object->purgeable_queue_type = PURGEABLE_Q_TYPE_MAX;
+               object->purgeable_queue_group = 0;

                /* one less volatile object for this object's owner */
                /* one less volatile object for this object's owner */

-               assert(object->vo_purgeable_owner == task);
+               assert(object->vo_owner == task);

                vm_purgeable_volatile_owner_update(task, -1);
 
 #if DEBUG
                object->vo_purgeable_volatilizer = NULL;
 #endif /* DEBUG */
                queue_enter(&purgeable_nonvolatile_queue, object,
                vm_purgeable_volatile_owner_update(task, -1);
 
 #if DEBUG
                object->vo_purgeable_volatilizer = NULL;
 #endif /* DEBUG */
                queue_enter(&purgeable_nonvolatile_queue, object,

-                           vm_object_t, objq);
+                   vm_object_t, objq);

                assert(purgeable_nonvolatile_count >= 0);
                purgeable_nonvolatile_count++;
                assert(purgeable_nonvolatile_count > 0);
                /* one more nonvolatile object for this object's owner */
                assert(purgeable_nonvolatile_count >= 0);
                purgeable_nonvolatile_count++;
                assert(purgeable_nonvolatile_count > 0);
                /* one more nonvolatile object for this object's owner */

-               assert(object->vo_purgeable_owner == task);
+               assert(object->vo_owner == task);

                vm_purgeable_nonvolatile_owner_update(task, +1);
 
                /* unlock purgeable queues */
                vm_purgeable_nonvolatile_owner_update(task, +1);
 
                /* unlock purgeable queues */


@@ -1480,11 +1453,11 @@ look_again:
                }
 
                /* purge the object */
                }
 
                /* purge the object */

-               (void) vm_object_purge(object, 0);
+               num_pages_purged += vm_object_purge(object, 0);
+

                assert(object->purgable == VM_PURGABLE_EMPTY);
                /* no change for purgeable accounting */
                vm_object_unlock(object);
                assert(object->purgable == VM_PURGABLE_EMPTY);
                /* no change for purgeable accounting */
                vm_object_unlock(object);

-               num_objects_purged++;

 
                /* we unlocked the purgeable queues, so start over */
                goto look_again;
 
                /* we unlocked the purgeable queues, so start over */
                goto look_again;


@@ -1492,90 +1465,80 @@ look_again:
 
        lck_mtx_unlock(&vm_purgeable_queue_lock);
 
 
        lck_mtx_unlock(&vm_purgeable_queue_lock);
 

-       return num_objects_purged;
+       return num_pages_purged;

 }
 
 }
 

-int
+uint64_t

 vm_purgeable_purge_task_owned(
 vm_purgeable_purge_task_owned(

-       task_t  task)
+       task_t  task)

 {
 {

-       purgeable_q_t   queue;
-       int             group;
-       int             num_objects_purged;
+       purgeable_q_t   queue = NULL;
+       int             group = 0;
+       uint64_t        num_pages_purged = 0;

 
 

-       num_objects_purged = 0;
+       num_pages_purged = 0;

 
        queue = &purgeable_queues[PURGEABLE_Q_TYPE_OBSOLETE];
 
        queue = &purgeable_queues[PURGEABLE_Q_TYPE_OBSOLETE];

-       num_objects_purged += vm_purgeable_queue_purge_task_owned(queue,
-                                                                 0,
-                                                                 task);
+       num_pages_purged += vm_purgeable_queue_purge_task_owned(queue,
+           0,
+           task);

 
        queue = &purgeable_queues[PURGEABLE_Q_TYPE_FIFO];
 
        queue = &purgeable_queues[PURGEABLE_Q_TYPE_FIFO];

-       for (group = 0; group < NUM_VOLATILE_GROUPS; group++)
-               num_objects_purged += vm_purgeable_queue_purge_task_owned(queue,
-                                                                         group,
-                                                                         task);
-       
+       for (group = 0; group < NUM_VOLATILE_GROUPS; group++) {
+               num_pages_purged += vm_purgeable_queue_purge_task_owned(queue,
+                   group,
+                   task);
+       }
+

        queue = &purgeable_queues[PURGEABLE_Q_TYPE_LIFO];
        queue = &purgeable_queues[PURGEABLE_Q_TYPE_LIFO];

-       for (group = 0; group < NUM_VOLATILE_GROUPS; group++)
-               num_objects_purged += vm_purgeable_queue_purge_task_owned(queue,
-                                                                         group,
-                                                                         task);
+       for (group = 0; group < NUM_VOLATILE_GROUPS; group++) {
+               num_pages_purged += vm_purgeable_queue_purge_task_owned(queue,
+                   group,
+                   task);
+       }

 
 

-       return num_objects_purged;
+       return num_pages_purged;

 }
 }

-#endif

 
 void
 vm_purgeable_nonvolatile_enqueue(
 
 void
 vm_purgeable_nonvolatile_enqueue(

-       vm_object_t     object,
-       task_t          owner)
+       vm_object_t     object,
+       task_t          owner)

 {
 {

-       int page_count;
-

        vm_object_lock_assert_exclusive(object);
 
        assert(object->purgable == VM_PURGABLE_NONVOLATILE);
        vm_object_lock_assert_exclusive(object);
 
        assert(object->purgable == VM_PURGABLE_NONVOLATILE);

-       assert(object->vo_purgeable_owner == NULL);
+       assert(object->vo_owner == NULL);

 
        lck_mtx_lock(&vm_purgeable_queue_lock);
 
        if (owner != NULL &&
            owner->task_purgeable_disowning) {
                /* task is exiting and no longer tracking purgeable objects */
 
        lck_mtx_lock(&vm_purgeable_queue_lock);
 
        if (owner != NULL &&
            owner->task_purgeable_disowning) {
                /* task is exiting and no longer tracking purgeable objects */

-               owner = NULL;
+               owner = VM_OBJECT_OWNER_DISOWNED;
+       }
+       if (owner == NULL) {
+               owner = kernel_task;

        }
        }

-
-       object->vo_purgeable_owner = owner;

 #if DEBUG
 #if DEBUG

+       OSBacktrace(&object->purgeable_owner_bt[0],
+           ARRAY_COUNT(object->purgeable_owner_bt));

        object->vo_purgeable_volatilizer = NULL;
 #endif /* DEBUG */
 
        object->vo_purgeable_volatilizer = NULL;
 #endif /* DEBUG */
 

-#if DEBUG
-       OSBacktrace(&object->purgeable_owner_bt[0], 16);
-#endif /* DEBUG */
+       vm_object_ownership_change(object,
+           object->vo_ledger_tag,                        /* tag unchanged */
+           owner,
+           FALSE);                             /* task_objq_locked */

 
 

-       page_count = object->resident_page_count;
-       if (owner != NULL && page_count != 0) {
-               ledger_credit(owner->ledger,
-                             task_ledgers.purgeable_nonvolatile,
-                             ptoa(page_count));
-               ledger_credit(owner->ledger,
-                             task_ledgers.phys_footprint,
-                             ptoa(page_count));
-       }
-                     

        assert(object->objq.next == NULL);
        assert(object->objq.prev == NULL);
 
        queue_enter(&purgeable_nonvolatile_queue, object,
        assert(object->objq.next == NULL);
        assert(object->objq.prev == NULL);
 
        queue_enter(&purgeable_nonvolatile_queue, object,

-                   vm_object_t, objq);
+           vm_object_t, objq);

        assert(purgeable_nonvolatile_count >= 0);
        purgeable_nonvolatile_count++;
        assert(purgeable_nonvolatile_count > 0);
        assert(purgeable_nonvolatile_count >= 0);
        purgeable_nonvolatile_count++;
        assert(purgeable_nonvolatile_count > 0);

-       /* one more nonvolatile object for this object's owner */
-       assert(object->vo_purgeable_owner == owner);
-       vm_purgeable_nonvolatile_owner_update(owner, +1);

        lck_mtx_unlock(&vm_purgeable_queue_lock);
 
        vm_object_lock_assert_exclusive(object);
        lck_mtx_unlock(&vm_purgeable_queue_lock);
 
        vm_object_lock_assert_exclusive(object);


@@ -1583,13 +1546,13 @@ vm_purgeable_nonvolatile_enqueue(
 
 void
 vm_purgeable_nonvolatile_dequeue(
 
 void
 vm_purgeable_nonvolatile_dequeue(

-       vm_object_t     object)
+       vm_object_t     object)

 {
 {

-       task_t  owner;
+       task_t  owner;

 
        vm_object_lock_assert_exclusive(object);
 
 
        vm_object_lock_assert_exclusive(object);
 

-       owner = object->vo_purgeable_owner;
+       owner = VM_OBJECT_OWNER(object);

 #if DEBUG
        assert(object->vo_purgeable_volatilizer == NULL);
 #endif /* DEBUG */
 #if DEBUG
        assert(object->vo_purgeable_volatilizer == NULL);
 #endif /* DEBUG */


@@ -1598,16 +1561,21 @@ vm_purgeable_nonvolatile_dequeue(
                 * Update the owner's ledger to stop accounting
                 * for this object.
                 */
                 * Update the owner's ledger to stop accounting
                 * for this object.
                 */

-               vm_purgeable_accounting(object,
-                                       object->purgable,
-                                       TRUE); /* disown */
+               /* transfer ownership to the kernel */
+               assert(VM_OBJECT_OWNER(object) != kernel_task);
+               vm_object_ownership_change(
+                       object,
+                       object->vo_ledger_tag,  /* unchanged */
+                       VM_OBJECT_OWNER_DISOWNED, /* new owner */
+                       FALSE); /* old_owner->task_objq locked */
+               assert(object->vo_owner == VM_OBJECT_OWNER_DISOWNED);

        }
 
        lck_mtx_lock(&vm_purgeable_queue_lock);
        assert(object->objq.next != NULL);
        assert(object->objq.prev != NULL);
        queue_remove(&purgeable_nonvolatile_queue, object,
        }
 
        lck_mtx_lock(&vm_purgeable_queue_lock);
        assert(object->objq.next != NULL);
        assert(object->objq.prev != NULL);
        queue_remove(&purgeable_nonvolatile_queue, object,

-                    vm_object_t, objq);
+           vm_object_t, objq);

        object->objq.next = NULL;
        object->objq.prev = NULL;
        assert(purgeable_nonvolatile_count > 0);
        object->objq.next = NULL;
        object->objq.prev = NULL;
        assert(purgeable_nonvolatile_count > 0);


@@ -1620,35 +1588,41 @@ vm_purgeable_nonvolatile_dequeue(
 
 void
 vm_purgeable_accounting(
 
 void
 vm_purgeable_accounting(

-       vm_object_t     object,
-       vm_purgable_t   old_state,
-       boolean_t       disown)
+       vm_object_t     object,
+       vm_purgable_t   old_state)

 {
 {

-       task_t          owner;
-       int             resident_page_count;
-       int             wired_page_count;
-       int             compressed_page_count;
-       boolean_t       disown_on_the_fly;
+       task_t          owner;
+       int             resident_page_count;
+       int             wired_page_count;
+       int             compressed_page_count;
+       int             ledger_idx_volatile;
+       int             ledger_idx_nonvolatile;
+       int             ledger_idx_volatile_compressed;
+       int             ledger_idx_nonvolatile_compressed;
+       boolean_t       do_footprint;

 
        vm_object_lock_assert_exclusive(object);
 
        vm_object_lock_assert_exclusive(object);

+       assert(object->purgable != VM_PURGABLE_DENY);

 
 

-       owner = object->vo_purgeable_owner;
-       if (owner == NULL)
+       owner = VM_OBJECT_OWNER(object);
+       if (owner == NULL ||
+           object->purgable == VM_PURGABLE_DENY) {

                return;
                return;

-
-       if (!disown && owner->task_purgeable_disowning) {
-               /* task is disowning its purgeable objects: help it */
-               disown_on_the_fly = TRUE;
-       } else {
-               disown_on_the_fly = FALSE;

        }
 
        }
 

+       vm_object_ledger_tag_ledgers(object,
+           &ledger_idx_volatile,
+           &ledger_idx_nonvolatile,
+           &ledger_idx_volatile_compressed,
+           &ledger_idx_nonvolatile_compressed,
+           &do_footprint);
+

        resident_page_count = object->resident_page_count;
        wired_page_count = object->wired_page_count;
        if (VM_CONFIG_COMPRESSOR_IS_PRESENT &&
        resident_page_count = object->resident_page_count;
        wired_page_count = object->wired_page_count;
        if (VM_CONFIG_COMPRESSOR_IS_PRESENT &&

-            object->pager != NULL) {
+           object->pager != NULL) {

                compressed_page_count =
                compressed_page_count =

-                       vm_compressor_pager_get_count(object->pager);
+                   vm_compressor_pager_get_count(object->pager);

        } else {
                compressed_page_count = 0;
        }
        } else {
                compressed_page_count = 0;
        }


@@ -1657,109 +1631,59 @@ vm_purgeable_accounting(
            old_state == VM_PURGABLE_EMPTY) {
                /* less volatile bytes in ledger */
                ledger_debit(owner->ledger,
            old_state == VM_PURGABLE_EMPTY) {
                /* less volatile bytes in ledger */
                ledger_debit(owner->ledger,

-                            task_ledgers.purgeable_volatile,
-                            ptoa(resident_page_count - wired_page_count));
+                   ledger_idx_volatile,
+                   ptoa_64(resident_page_count - wired_page_count));

                /* less compressed volatile bytes in ledger */
                ledger_debit(owner->ledger,
                /* less compressed volatile bytes in ledger */
                ledger_debit(owner->ledger,

-                            task_ledgers.purgeable_volatile_compressed,
-                            ptoa(compressed_page_count));
-
-               if (disown || !object->alive || object->terminating) {
-                       /* wired pages were accounted as "non-volatile"... */
-                       ledger_debit(owner->ledger,
-                                    task_ledgers.purgeable_nonvolatile,
-                                    ptoa(wired_page_count));
-                       /* ... and in phys_footprint */
-                       ledger_debit(owner->ledger,
-                                    task_ledgers.phys_footprint,
-                                    ptoa(wired_page_count));
-
-                       if (!disown_on_the_fly &&
-                           (object->purgeable_queue_type ==
-                            PURGEABLE_Q_TYPE_MAX)) {
-                               /*
-                                * Not on a volatile queue:  must be empty
-                                * or emptying.
-                                */
-                               vm_purgeable_nonvolatile_owner_update(owner,-1);
-                       } else {
-                               /* on a volatile queue */
-                               vm_purgeable_volatile_owner_update(owner, -1);
-                       }
-                       /* no more accounting for this dead object */
-                       object->vo_purgeable_owner = NULL;
-#if DEBUG
-                       object->vo_purgeable_volatilizer = NULL;
-#endif /* DEBUG */
-                       return;
-               }
+                   ledger_idx_volatile_compressed,
+                   ptoa_64(compressed_page_count));

 
                /* more non-volatile bytes in ledger */
                ledger_credit(owner->ledger,
 
                /* more non-volatile bytes in ledger */
                ledger_credit(owner->ledger,

-                             task_ledgers.purgeable_nonvolatile,
-                             ptoa(resident_page_count - wired_page_count));
+                   ledger_idx_nonvolatile,
+                   ptoa_64(resident_page_count - wired_page_count));

                /* more compressed non-volatile bytes in ledger */
                ledger_credit(owner->ledger,
                /* more compressed non-volatile bytes in ledger */
                ledger_credit(owner->ledger,

-                             task_ledgers.purgeable_nonvolatile_compressed,
-                             ptoa(compressed_page_count));
-               /* more footprint */
-               ledger_credit(owner->ledger,
-                             task_ledgers.phys_footprint,
-                             ptoa(resident_page_count
-                                  + compressed_page_count
-                                  - wired_page_count));
-
+                   ledger_idx_nonvolatile_compressed,
+                   ptoa_64(compressed_page_count));
+               if (do_footprint) {
+                       /* more footprint */
+                       ledger_credit(owner->ledger,
+                           task_ledgers.phys_footprint,
+                           ptoa_64(resident_page_count
+                           + compressed_page_count
+                           - wired_page_count));
+               }

        } else if (old_state == VM_PURGABLE_NONVOLATILE) {
        } else if (old_state == VM_PURGABLE_NONVOLATILE) {

-

                /* less non-volatile bytes in ledger */
                ledger_debit(owner->ledger,
                /* less non-volatile bytes in ledger */
                ledger_debit(owner->ledger,

-                            task_ledgers.purgeable_nonvolatile,
-                            ptoa(resident_page_count - wired_page_count));
+                   ledger_idx_nonvolatile,
+                   ptoa_64(resident_page_count - wired_page_count));

                /* less compressed non-volatile bytes in ledger */
                ledger_debit(owner->ledger,
                /* less compressed non-volatile bytes in ledger */
                ledger_debit(owner->ledger,

-                            task_ledgers.purgeable_nonvolatile_compressed,
-                            ptoa(compressed_page_count));
-               /* less footprint */
-               ledger_debit(owner->ledger,
-                            task_ledgers.phys_footprint,
-                            ptoa(resident_page_count
-                                 + compressed_page_count
-                                 - wired_page_count));
-
-               if (disown || !object->alive || object->terminating) {
-                       /* wired pages still accounted as "non-volatile" */
-                       ledger_debit(owner->ledger,
-                                    task_ledgers.purgeable_nonvolatile,
-                                    ptoa(wired_page_count));
+                   ledger_idx_nonvolatile_compressed,
+                   ptoa_64(compressed_page_count));
+               if (do_footprint) {
+                       /* less footprint */

                        ledger_debit(owner->ledger,
                        ledger_debit(owner->ledger,

-                                    task_ledgers.phys_footprint,
-                                    ptoa(wired_page_count));
-
-                       /* one less "non-volatile" object for the owner */
-                       if (!disown_on_the_fly) {
-                               assert(object->purgeable_queue_type ==
-                                      PURGEABLE_Q_TYPE_MAX);
-                       }
-                       vm_purgeable_nonvolatile_owner_update(owner, -1);
-                       /* no more accounting for this dead object */
-                       object->vo_purgeable_owner = NULL;
-#if DEBUG
-                       object->vo_purgeable_volatilizer = NULL;
-#endif /* DEBUG */
-                       return;
+                           task_ledgers.phys_footprint,
+                           ptoa_64(resident_page_count
+                           + compressed_page_count
+                           - wired_page_count));

                }
                }

+

                /* more volatile bytes in ledger */
                ledger_credit(owner->ledger,
                /* more volatile bytes in ledger */
                ledger_credit(owner->ledger,

-                             task_ledgers.purgeable_volatile,
-                             ptoa(resident_page_count - wired_page_count));
+                   ledger_idx_volatile,
+                   ptoa_64(resident_page_count - wired_page_count));

                /* more compressed volatile bytes in ledger */
                ledger_credit(owner->ledger,
                /* more compressed volatile bytes in ledger */
                ledger_credit(owner->ledger,

-                             task_ledgers.purgeable_volatile_compressed,
-                             ptoa(compressed_page_count));
+                   ledger_idx_volatile_compressed,
+                   ptoa_64(compressed_page_count));

        } else {
                panic("vm_purgeable_accounting(%p): "
        } else {
                panic("vm_purgeable_accounting(%p): "

-                     "unexpected old_state=%d\n",
-                     object, old_state);
+                   "unexpected old_state=%d\n",
+                   object, old_state);

        }
 
        vm_object_lock_assert_exclusive(object);
        }
 
        vm_object_lock_assert_exclusive(object);


@@ -1767,8 +1691,8 @@ vm_purgeable_accounting(
 
 void
 vm_purgeable_nonvolatile_owner_update(
 
 void
 vm_purgeable_nonvolatile_owner_update(

-       task_t  owner,
-       int     delta)
+       task_t  owner,
+       int     delta)

 {
        if (owner == NULL || delta == 0) {
                return;
 {
        if (owner == NULL || delta == 0) {
                return;


@@ -1787,8 +1711,8 @@ vm_purgeable_nonvolatile_owner_update(
 
 void
 vm_purgeable_volatile_owner_update(
 
 void
 vm_purgeable_volatile_owner_update(

-       task_t  owner,
-       int     delta)
+       task_t  owner,
+       int     delta)

 {
        if (owner == NULL || delta == 0) {
                return;
 {
        if (owner == NULL || delta == 0) {
                return;


@@ -1806,58 +1730,77 @@ vm_purgeable_volatile_owner_update(
 }
 
 void
 }
 
 void

-vm_purgeable_compressed_update(
-       vm_object_t     object,
-       int             delta)
+vm_object_owner_compressed_update(
+       vm_object_t     object,
+       int             delta)

 {
 {

-       task_t  owner;
+       task_t          owner;
+       int             ledger_idx_volatile;
+       int             ledger_idx_nonvolatile;
+       int             ledger_idx_volatile_compressed;
+       int             ledger_idx_nonvolatile_compressed;
+       boolean_t       do_footprint;

 
        vm_object_lock_assert_exclusive(object);
 
 
        vm_object_lock_assert_exclusive(object);
 

+       owner = VM_OBJECT_OWNER(object);
+

        if (delta == 0 ||
            !object->internal ||
        if (delta == 0 ||
            !object->internal ||

-           object->purgable == VM_PURGABLE_DENY ||
-           object->vo_purgeable_owner == NULL) {
-               /* not an owned purgeable VM object: nothing to update */
+           (object->purgable == VM_PURGABLE_DENY &&
+           !object->vo_ledger_tag) ||
+           owner == NULL) {
+               /* not an owned purgeable (or tagged) VM object: nothing to update */

                return;
        }
                return;
        }

-       
-       owner = object->vo_purgeable_owner;
+
+       vm_object_ledger_tag_ledgers(object,
+           &ledger_idx_volatile,
+           &ledger_idx_nonvolatile,
+           &ledger_idx_volatile_compressed,
+           &ledger_idx_nonvolatile_compressed,
+           &do_footprint);

        switch (object->purgable) {
        case VM_PURGABLE_DENY:
        switch (object->purgable) {
        case VM_PURGABLE_DENY:

-               break;
+               /* not purgeable: must be ledger-tagged */
+               assert(object->vo_ledger_tag != VM_OBJECT_LEDGER_TAG_NONE);
+       /* fallthru */

        case VM_PURGABLE_NONVOLATILE:
                if (delta > 0) {
                        ledger_credit(owner->ledger,
        case VM_PURGABLE_NONVOLATILE:
                if (delta > 0) {
                        ledger_credit(owner->ledger,

-                                     task_ledgers.purgeable_nonvolatile_compressed,
-                                     ptoa(delta));
-                       ledger_credit(owner->ledger,
-                                     task_ledgers.phys_footprint,
-                                     ptoa(delta));
+                           ledger_idx_nonvolatile_compressed,
+                           ptoa_64(delta));
+                       if (do_footprint) {
+                               ledger_credit(owner->ledger,
+                                   task_ledgers.phys_footprint,
+                                   ptoa_64(delta));
+                       }

                } else {
                        ledger_debit(owner->ledger,
                } else {
                        ledger_debit(owner->ledger,

-                                    task_ledgers.purgeable_nonvolatile_compressed,
-                                    ptoa(-delta));
-                       ledger_debit(owner->ledger,
-                                    task_ledgers.phys_footprint,
-                                    ptoa(-delta));
+                           ledger_idx_nonvolatile_compressed,
+                           ptoa_64(-delta));
+                       if (do_footprint) {
+                               ledger_debit(owner->ledger,
+                                   task_ledgers.phys_footprint,
+                                   ptoa_64(-delta));
+                       }

                }
                break;
        case VM_PURGABLE_VOLATILE:
        case VM_PURGABLE_EMPTY:
                if (delta > 0) {
                        ledger_credit(owner->ledger,
                }
                break;
        case VM_PURGABLE_VOLATILE:
        case VM_PURGABLE_EMPTY:
                if (delta > 0) {
                        ledger_credit(owner->ledger,

-                                     task_ledgers.purgeable_volatile_compressed,
-                                     ptoa(delta));
+                           ledger_idx_volatile_compressed,
+                           ptoa_64(delta));

                } else {
                        ledger_debit(owner->ledger,
                } else {
                        ledger_debit(owner->ledger,

-                                    task_ledgers.purgeable_volatile_compressed,
-                                    ptoa(-delta));
+                           ledger_idx_volatile_compressed,
+                           ptoa_64(-delta));

                }
                break;
        default:
                panic("vm_purgeable_compressed_update(): "
                }
                break;
        default:
                panic("vm_purgeable_compressed_update(): "

-                     "unexpected purgable %d for object %p\n",
-                     object->purgable, object);
+                   "unexpected purgable %d for object %p\n",
+                   object->purgable, object);

        }
 }
        }
 }