xnu-7195.81.3.tar.gz

[apple/xnu.git] / osfmk / kern / hibernate.c

diff --git a/osfmk/kern/hibernate.c b/osfmk/kern/hibernate.c
index 27a089239338367b714b15b768cdf2619264958b..71ef32fae90d152b3a292ac3f45864bb12f007dd 100644 (file)
--- a/osfmk/kern/hibernate.c
+++ b/osfmk/kern/hibernate.c
@@ -2,7 +2,7 @@
  * Copyright (c) 2004-2005 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * Copyright (c) 2004-2005 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@

- * 
+ *

  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in
  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in


@@ -11,10 +11,10 @@
  * unlawful or unlicensed copies of an Apple operating system, or to
  * circumvent, violate, or enable the circumvention or violation of, any
  * terms of an Apple operating system software license agreement.
  * unlawful or unlicensed copies of an Apple operating system, or to
  * circumvent, violate, or enable the circumvention or violation of, any
  * terms of an Apple operating system software license agreement.

- * 
+ *

  * Please obtain a copy of the License at
  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  * Please obtain a copy of the License at
  * http://www.opensource.apple.com/apsl/ and read it before using this file.

- * 
+ *

  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,


@@ -22,7 +22,7 @@
  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  * Please see the License for the specific language governing rights and
  * limitations under the License.
  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  * Please see the License for the specific language governing rights and
  * limitations under the License.

- * 
+ *

  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 


@@ -34,521 +34,122 @@
 #include <mach/machine.h>
 #include <mach/processor_info.h>
 #include <mach/mach_types.h>
 #include <mach/machine.h>
 #include <mach/processor_info.h>
 #include <mach/mach_types.h>

-#include <default_pager/default_pager_internal.h>

 #include <IOKit/IOPlatformExpert.h>
 
 #include <IOKit/IOHibernatePrivate.h>
 #include <vm/vm_page.h>
 #include <vm/vm_pageout.h>
 #include <vm/vm_purgeable_internal.h>
 #include <IOKit/IOPlatformExpert.h>
 
 #include <IOKit/IOHibernatePrivate.h>
 #include <vm/vm_page.h>
 #include <vm/vm_pageout.h>
 #include <vm/vm_purgeable_internal.h>

+#include <vm/vm_compressor.h>

 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 

-static vm_page_t hibernate_gobble_queue;
-
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-
-static void
-hibernate_page_list_zero(hibernate_page_list_t *list)
-{
-    uint32_t             bank;
-    hibernate_bitmap_t * bitmap;
-
-    bitmap = &list->bank_bitmap[0];
-    for (bank = 0; bank < list->bank_count; bank++)
-    {
-        uint32_t last_bit;
-
-       bzero((void *) &bitmap->bitmap[0], bitmap->bitmapwords << 2); 
-        // set out-of-bound bits at end of bitmap.
-        last_bit = ((bitmap->last_page - bitmap->first_page + 1) & 31);
-       if (last_bit)
-           bitmap->bitmap[bitmap->bitmapwords - 1] = (0xFFFFFFFF >> last_bit);
-
-       bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
-    }
-}
-
-
-static boolean_t 
-consider_discard(vm_page_t m)
-{
-    vm_object_t object = NULL;
-    int                  refmod_state;
-    boolean_t            discard = FALSE;
-
-    do
-    {
-        if(m->private)
-            panic("consider_discard: private");
-
-        if (!vm_object_lock_try(m->object))
-            break;
-
-        object = m->object;
-
-       if (m->wire_count != 0)
-            break;
-        if (m->precious)
-            break;
-
-        if (m->busy || !object->alive)
-           /*
-            *  Somebody is playing with this page.
-            */
-            break;
-
-        if (m->absent || m->unusual || m->error)
-           /*
-            * If it's unusual in anyway, ignore it
-            */
-            break;
-    
-        if (m->cleaning)
-            break;
-
-       if (m->laundry || m->list_req_pending)
-            break;
-
-        if (!m->dirty)
-        {
-            refmod_state = pmap_get_refmod(m->phys_page);
-        
-            if (refmod_state & VM_MEM_REFERENCED)
-                m->reference = TRUE;
-            if (refmod_state & VM_MEM_MODIFIED)
-                m->dirty = TRUE;
-        }
-   
-        /*
-         * If it's clean or purgeable we can discard the page on wakeup.
-        * JMM - consider purgeable (volatile or empty) objects here as well.
-         */
-        discard = (!m->dirty) 
-                   || (VM_PURGABLE_VOLATILE == object->purgable)
-                   || (VM_PURGABLE_EMPTY    == m->object->purgable);
-    }
-    while (FALSE);
-
-    if (object)
-        vm_object_unlock(object);
-
-    return (discard);
-}
-
-
-static void
-discard_page(vm_page_t m)
-{
-    if (m->absent || m->unusual || m->error)
-       /*
-        * If it's unusual in anyway, ignore
-        */
-        return;
-
-    if (m->pmapped == TRUE) 
-    {
-        __unused int refmod_state = pmap_disconnect(m->phys_page);
-    }
-
-    if (m->laundry)
-        panic("discard_page(%p) laundry", m);
-    if (m->private)
-        panic("discard_page(%p) private", m);
-    if (m->fictitious)
-        panic("discard_page(%p) fictitious", m);
-
-    if (VM_PURGABLE_VOLATILE == m->object->purgable)
-       {
-               assert(m->object->objq.next != NULL && m->object->objq.prev != NULL); /* object should be on a queue */
-               purgeable_q_t old_queue=vm_purgeable_object_remove(m->object);
-               assert(old_queue);
-               /* No need to lock page queue for token delete, hibernate_vm_unlock() 
-                       makes sure these locks are uncontended before sleep */
-               vm_purgeable_token_delete_first(old_queue);
-               m->object->purgable = VM_PURGABLE_EMPTY;
-       }
-       
-    if (m->tabled)
-       vm_page_remove(m);
-
-    vm_page_free(m);
-}
-
-/*
- Bits zero in the bitmaps => needs to be saved. All pages default to be saved,
- pages known to VM to not need saving are subtracted.
- Wired pages to be saved are present in page_list_wired, pageable in page_list.
-*/
+boolean_t       need_to_unlock_decompressor = FALSE;

 
 

-void
-hibernate_page_list_setall(hibernate_page_list_t * page_list,
-                          hibernate_page_list_t * page_list_wired,
-                          uint32_t * pagesOut)
+kern_return_t
+hibernate_alloc_page_lists(
+       hibernate_page_list_t ** page_list_ret,
+       hibernate_page_list_t ** page_list_wired_ret,
+       hibernate_page_list_t ** page_list_pal_ret)

 {
 {

-    uint64_t start, end, nsec;
-    vm_page_t m;
-    uint32_t pages = page_list->page_count;
-    uint32_t count_zf = 0, count_throttled = 0;
-    uint32_t count_inactive = 0, count_active = 0, count_speculative = 0;
-    uint32_t count_wire = pages;
-    uint32_t count_discard_active    = 0;
-    uint32_t count_discard_inactive  = 0;
-    uint32_t count_discard_purgeable = 0;
-    uint32_t count_discard_speculative = 0;
-    uint32_t i;
-    uint32_t             bank;
-    hibernate_bitmap_t * bitmap;
-    hibernate_bitmap_t * bitmap_wired;
-
-
-    HIBLOG("hibernate_page_list_setall start\n");
-
-    clock_get_uptime(&start);
-
-    hibernate_page_list_zero(page_list);
-    hibernate_page_list_zero(page_list_wired);
+       kern_return_t       retval = KERN_SUCCESS;

 
 

-    m = (vm_page_t) hibernate_gobble_queue;
-    while(m)
-    {
-       pages--;
-       count_wire--;
-       hibernate_page_bitset(page_list,       TRUE, m->phys_page);
-       hibernate_page_bitset(page_list_wired, TRUE, m->phys_page);
-       m = (vm_page_t) m->pageq.next;
-    }
+       hibernate_page_list_t * page_list = NULL;
+       hibernate_page_list_t * page_list_wired = NULL;
+       hibernate_page_list_t * page_list_pal = NULL;

 
 

-    for( i = 0; i < vm_colors; i++ )
-    {
-       queue_iterate(&vm_page_queue_free[i],
-                     m,
-                     vm_page_t,
-                     pageq)
-       {
-           pages--;
-           count_wire--;
-           hibernate_page_bitset(page_list,       TRUE, m->phys_page);
-           hibernate_page_bitset(page_list_wired, TRUE, m->phys_page);
+       page_list = hibernate_page_list_allocate(TRUE);
+       if (!page_list) {
+               HIBLOG("%s: failed for page_list\n", __FUNCTION__);
+               retval = KERN_RESOURCE_SHORTAGE;
+               goto done;

        }
        }

-    }
-
-    queue_iterate(&vm_lopage_queue_free,
-                 m,
-                 vm_page_t,
-                 pageq)
-    {
-       pages--;
-       count_wire--;
-       hibernate_page_bitset(page_list,       TRUE, m->phys_page);
-       hibernate_page_bitset(page_list_wired, TRUE, m->phys_page);
-    }
-
-    queue_iterate( &vm_page_queue_throttled,
-                    m,
-                    vm_page_t,
-                    pageq )
-    {
-        if ((kIOHibernateModeDiscardCleanInactive & gIOHibernateMode) 
-         && consider_discard(m))
-        {
-            hibernate_page_bitset(page_list, TRUE, m->phys_page);
-            count_discard_inactive++;
-        }
-        else
-            count_throttled++;
-       count_wire--;
-       hibernate_page_bitset(page_list_wired, TRUE, m->phys_page);
-    }
-
-    queue_iterate( &vm_page_queue_zf,
-                    m,
-                    vm_page_t,
-                   pageq )
-    {
-        if ((kIOHibernateModeDiscardCleanInactive & gIOHibernateMode) 
-         && consider_discard(m))
-        {
-            hibernate_page_bitset(page_list, TRUE, m->phys_page);
-           if (m->dirty)
-               count_discard_purgeable++;
-           else
-               count_discard_inactive++;
-        }
-        else
-            count_zf++;
-       count_wire--;
-       hibernate_page_bitset(page_list_wired, TRUE, m->phys_page);
-    }
-
-    queue_iterate( &vm_page_queue_inactive,
-                    m,
-                    vm_page_t,
-                    pageq )
-    {
-        if ((kIOHibernateModeDiscardCleanInactive & gIOHibernateMode) 
-         && consider_discard(m))
-        {
-            hibernate_page_bitset(page_list, TRUE, m->phys_page);
-           if (m->dirty)
-               count_discard_purgeable++;
-           else
-               count_discard_inactive++;
-        }
-        else
-            count_inactive++;
-       count_wire--;
-       hibernate_page_bitset(page_list_wired, TRUE, m->phys_page);
-    }
-
-    for( i = 0; i <= VM_PAGE_MAX_SPECULATIVE_AGE_Q; i++ )
-    {
-       queue_iterate(&vm_page_queue_speculative[i].age_q,
-                     m,
-                     vm_page_t,
-                     pageq)
-       {
-           if ((kIOHibernateModeDiscardCleanInactive & gIOHibernateMode) 
-            && consider_discard(m))
-           {
-               hibernate_page_bitset(page_list, TRUE, m->phys_page);
-               count_discard_speculative++;
-           }
-           else
-               count_speculative++;
-           count_wire--;
-           hibernate_page_bitset(page_list_wired, TRUE, m->phys_page);
+       page_list_wired = hibernate_page_list_allocate(FALSE);
+       if (!page_list_wired) {
+               kfree(page_list, page_list->list_size);
+               HIBLOG("%s: failed for page_list_wired\n", __FUNCTION__);
+               retval = KERN_RESOURCE_SHORTAGE;
+               goto done;

        }
        }

-    }
-
-    queue_iterate( &vm_page_queue_active,
-                    m,
-                    vm_page_t,
-                    pageq )
-    {
-        if ((kIOHibernateModeDiscardCleanActive & gIOHibernateMode) 
-         && consider_discard(m))
-        {
-            hibernate_page_bitset(page_list, TRUE, m->phys_page);
-           if (m->dirty)
-               count_discard_purgeable++;
-           else
-               count_discard_active++;
-        }
-        else
-            count_active++;
-       count_wire--;
-       hibernate_page_bitset(page_list_wired, TRUE, m->phys_page);
-    }
-
-    // pull wired from hibernate_bitmap
-
-    bitmap = &page_list->bank_bitmap[0];
-    bitmap_wired = &page_list_wired->bank_bitmap[0];
-    for (bank = 0; bank < page_list->bank_count; bank++)
-    {
-       for (i = 0; i < bitmap->bitmapwords; i++)
-           bitmap->bitmap[i] = bitmap->bitmap[i] | ~bitmap_wired->bitmap[i];
-       bitmap       = (hibernate_bitmap_t *) &bitmap->bitmap      [bitmap->bitmapwords];
-       bitmap_wired = (hibernate_bitmap_t *) &bitmap_wired->bitmap[bitmap_wired->bitmapwords];
-    }
-
-    // machine dependent adjustments
-    hibernate_page_list_setall_machine(page_list, page_list_wired, &pages);
-
-    clock_get_uptime(&end);
-    absolutetime_to_nanoseconds(end - start, &nsec);
-    HIBLOG("hibernate_page_list_setall time: %qd ms\n", nsec / 1000000ULL);
-
-    HIBLOG("pages %d, wire %d, act %d, inact %d, spec %d, zf %d, throt %d, could discard act %d inact %d purgeable %d spec %d\n", 
-                pages, count_wire, count_active, count_inactive, count_speculative, count_zf, count_throttled,
-                count_discard_active, count_discard_inactive, count_discard_purgeable, count_discard_speculative);
-
-    *pagesOut = pages - count_discard_active - count_discard_inactive - count_discard_purgeable - count_discard_speculative;
-}
-
-void
-hibernate_page_list_discard(hibernate_page_list_t * page_list)
-{
-    uint64_t  start, end, nsec;
-    vm_page_t m;
-    vm_page_t next;
-    uint32_t  i;
-    uint32_t  count_discard_active    = 0;
-    uint32_t  count_discard_inactive  = 0;
-    uint32_t  count_discard_purgeable = 0;
-    uint32_t  count_discard_speculative = 0;
-
-    clock_get_uptime(&start);
-
-    m = (vm_page_t) queue_first(&vm_page_queue_zf);
-    while (m && !queue_end(&vm_page_queue_zf, (queue_entry_t)m))
-    {
-        next = (vm_page_t) m->pageq.next;
-        if (hibernate_page_bittst(page_list, m->phys_page))
-        {
-           if (m->dirty)
-               count_discard_purgeable++;
-           else
-               count_discard_inactive++;
-            discard_page(m);
-        }
-        m = next;
-    }
-
-    for( i = 0; i <= VM_PAGE_MAX_SPECULATIVE_AGE_Q; i++ )
-    {
-       m = (vm_page_t) queue_first(&vm_page_queue_speculative[i].age_q);
-       while (m && !queue_end(&vm_page_queue_speculative[i].age_q, (queue_entry_t)m))
-       {
-           next = (vm_page_t) m->pageq.next;
-           if (hibernate_page_bittst(page_list, m->phys_page))
-           {
-               count_discard_speculative++;
-               discard_page(m);
-           }
-           m = next;
+       page_list_pal = hibernate_page_list_allocate(FALSE);
+       if (!page_list_pal) {
+               kfree(page_list, page_list->list_size);
+               kfree(page_list_wired, page_list_wired->list_size);
+               HIBLOG("%s: failed for page_list_pal\n", __FUNCTION__);
+               retval = KERN_RESOURCE_SHORTAGE;
+               goto done;

        }
        }

-    }
+       *page_list_ret        = page_list;
+       *page_list_wired_ret  = page_list_wired;
+       *page_list_pal_ret    = page_list_pal;

 
 

-    m = (vm_page_t) queue_first(&vm_page_queue_inactive);
-    while (m && !queue_end(&vm_page_queue_inactive, (queue_entry_t)m))
-    {
-        next = (vm_page_t) m->pageq.next;
-        if (hibernate_page_bittst(page_list, m->phys_page))
-        {
-           if (m->dirty)
-               count_discard_purgeable++;
-           else
-               count_discard_inactive++;
-            discard_page(m);
-        }
-        m = next;
-    }
-
-    m = (vm_page_t) queue_first(&vm_page_queue_active);
-    while (m && !queue_end(&vm_page_queue_active, (queue_entry_t)m))
-    {
-        next = (vm_page_t) m->pageq.next;
-        if (hibernate_page_bittst(page_list, m->phys_page))
-        {
-           if (m->dirty)
-               count_discard_purgeable++;
-           else
-               count_discard_active++;
-            discard_page(m);
-        }
-        m = next;
-    }
-
-    clock_get_uptime(&end);
-    absolutetime_to_nanoseconds(end - start, &nsec);
-    HIBLOG("hibernate_page_list_discard time: %qd ms, discarded act %d inact %d purgeable %d spec %d\n",
-                nsec / 1000000ULL,
-                count_discard_active, count_discard_inactive, count_discard_purgeable, count_discard_speculative);
+done:
+       return retval;

 }
 
 }
 

-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+extern int sync_internal(void);

 
 

-kern_return_t 
+kern_return_t

 hibernate_setup(IOHibernateImageHeader * header,
 hibernate_setup(IOHibernateImageHeader * header,

-                        uint32_t free_page_ratio,
-                        uint32_t free_page_time,
-                       hibernate_page_list_t ** page_list_ret,
-                       hibernate_page_list_t ** page_list_wired_ret,
-                        boolean_t * encryptedswap)
+    boolean_t vmflush,
+    hibernate_page_list_t * page_list __unused,
+    hibernate_page_list_t * page_list_wired __unused,
+    hibernate_page_list_t * page_list_pal __unused)

 {
 {

-    hibernate_page_list_t * page_list = NULL;
-    hibernate_page_list_t * page_list_wired = NULL;
-    vm_page_t              m;
-    uint32_t               i, gobble_count;
-
-    *page_list_ret       = NULL;
-    *page_list_wired_ret = NULL;
+       kern_return_t       retval = KERN_SUCCESS;

 
 

+       hibernate_create_paddr_map();

 
 

-    page_list = hibernate_page_list_allocate();
-    if (!page_list)
-        return (KERN_RESOURCE_SHORTAGE);
-    page_list_wired = hibernate_page_list_allocate();
-    if (!page_list_wired)
-    {
-        kfree(page_list, page_list->list_size);
-        return (KERN_RESOURCE_SHORTAGE);
-    }
+       hibernate_reset_stats();

 
 

-    *encryptedswap = dp_encryption;
+       if (vmflush && VM_CONFIG_COMPRESSOR_IS_PRESENT) {
+               sync_internal();

 
 

-    // pages we could force out to reduce hibernate image size
-    gobble_count = (((uint64_t) page_list->page_count) * ((uint64_t) free_page_ratio)) / 100;
+               vm_decompressor_lock();
+               need_to_unlock_decompressor = TRUE;

 
 

-    // no failures hereafter
-
-    hibernate_processor_setup(header);
+               hibernate_flush_memory();
+       }

 
 

-    HIBLOG("hibernate_alloc_pages flags %08x, gobbling %d pages\n", 
-           header->processorFlags, gobble_count);
+       // no failures hereafter

 
 

-    if (gobble_count)
-    {
-        uint64_t start, end, timeout, nsec;
-        clock_interval_to_deadline(free_page_time, 1000 * 1000 /*ms*/, &timeout);
-        clock_get_uptime(&start);
-    
-        for (i = 0; i < gobble_count; i++)
-        {
-            while (VM_PAGE_NULL == (m = vm_page_grab()))
-            {
-                clock_get_uptime(&end);
-                if (end >= timeout)
-                    break;
-                VM_PAGE_WAIT();
-            }
-            if (!m)
-                break;
-            m->busy = FALSE;
-            vm_page_gobble(m);
-    
-            m->pageq.next = (queue_entry_t) hibernate_gobble_queue;
-            hibernate_gobble_queue = m;
-        }
-    
-        clock_get_uptime(&end);
-        absolutetime_to_nanoseconds(end - start, &nsec);
-        HIBLOG("Gobbled %d pages, time: %qd ms\n", i, nsec / 1000000ULL);
-    }
+       hibernate_processor_setup(header);

 
 

-    *page_list_ret       = page_list;
-    *page_list_wired_ret = page_list_wired;
+       HIBLOG("hibernate_alloc_pages act %d, inact %d, anon %d, throt %d, spec %d, wire %d, wireinit %d\n",
+           vm_page_active_count, vm_page_inactive_count,
+           vm_page_anonymous_count, vm_page_throttled_count, vm_page_speculative_count,
+           vm_page_wire_count, vm_page_wire_count_initial);

 
 

-    return (KERN_SUCCESS);
+       if (retval != KERN_SUCCESS && need_to_unlock_decompressor == TRUE) {
+               need_to_unlock_decompressor = FALSE;
+               vm_decompressor_unlock();
+       }
+       return retval;

 }
 
 }
 

-kern_return_t 
+kern_return_t

 hibernate_teardown(hibernate_page_list_t * page_list,
 hibernate_teardown(hibernate_page_list_t * page_list,

-                    hibernate_page_list_t * page_list_wired)
+    hibernate_page_list_t * page_list_wired,
+    hibernate_page_list_t * page_list_pal)

 {
 {

-    vm_page_t m, next;
-    uint32_t  count = 0;
+       hibernate_free_gobble_pages();

 
 

-    m = (vm_page_t) hibernate_gobble_queue;
-    while(m)
-    {
-        next = (vm_page_t) m->pageq.next;
-        vm_page_free(m);
-        count++;
-        m = next;
-    }
-    hibernate_gobble_queue = VM_PAGE_NULL;
-    
-    if (count)
-        HIBLOG("Freed %d pages\n", count);
-
-    if (page_list)
-        kfree(page_list, page_list->list_size);
-    if (page_list_wired)
-        kfree(page_list_wired, page_list_wired->list_size);
+       if (page_list) {
+               kfree(page_list, page_list->list_size);
+       }
+       if (page_list_wired) {
+               kfree(page_list_wired, page_list_wired->list_size);
+       }
+       if (page_list_pal) {
+               kfree(page_list_pal, page_list_pal->list_size);
+       }

 
 

-    return (KERN_SUCCESS);
+       if (VM_CONFIG_COMPRESSOR_IS_PRESENT) {
+               if (need_to_unlock_decompressor == TRUE) {
+                       need_to_unlock_decompressor = FALSE;
+                       vm_decompressor_unlock();
+               }
+               vm_compressor_delay_trim();
+       }
+       return KERN_SUCCESS;

 }
 }

-