xnu-1228.tar.gz

[apple/xnu.git] / osfmk / kern / hibernate.c

diff --git a/osfmk/kern/hibernate.c b/osfmk/kern/hibernate.c
index 96c0275e1c675512e44a3712a9e19b1d80995b1f..3decaefe2306f6c2f16f80c00052f22ad69991da 100644 (file)
--- a/osfmk/kern/hibernate.c
+++ b/osfmk/kern/hibernate.c
@@ -1,5 +1,5 @@
 /*
 /*

- * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004-2005 Apple Computer, Inc. All rights reserved.

  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 


@@ -36,11 +36,11 @@
 #include <mach/mach_types.h>
 #include <default_pager/default_pager_internal.h>
 #include <IOKit/IOPlatformExpert.h>
 #include <mach/mach_types.h>
 #include <default_pager/default_pager_internal.h>
 #include <IOKit/IOPlatformExpert.h>

-#define KERNEL

 
 #include <IOKit/IOHibernatePrivate.h>
 #include <vm/vm_page.h>
 #include <vm/vm_pageout.h>
 
 #include <IOKit/IOHibernatePrivate.h>
 #include <vm/vm_page.h>
 #include <vm/vm_pageout.h>

+#include <vm/vm_purgeable_internal.h>

 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 


@@ -73,7 +73,7 @@ hibernate_page_list_zero(hibernate_page_list_t *list)
 static boolean_t 
 consider_discard(vm_page_t m)
 {
 static boolean_t 
 consider_discard(vm_page_t m)
 {

-    register vm_object_t object = 0;
+    vm_object_t object = NULL;

     int                  refmod_state;
     boolean_t            discard = FALSE;
 
     int                  refmod_state;
     boolean_t            discard = FALSE;
 


@@ -107,6 +107,9 @@ consider_discard(vm_page_t m)
         if (m->cleaning)
             break;
 
         if (m->cleaning)
             break;
 

+       if (m->laundry || m->list_req_pending)
+            break;
+

         if (!m->dirty)
         {
             refmod_state = pmap_get_refmod(m->phys_page);
         if (!m->dirty)
         {
             refmod_state = pmap_get_refmod(m->phys_page);


@@ -118,9 +121,12 @@ consider_discard(vm_page_t m)
         }
    
         /*
         }
    
         /*

-         * If it's clean we can discard the page on wakeup.
+         * If it's clean or purgeable we can discard the page on wakeup.
+        * JMM - consider purgeable (volatile or empty) objects here as well.

          */
          */

-        discard = !m->dirty;
+        discard = (!m->dirty) 
+                   || (VM_PURGABLE_VOLATILE == object->purgable)
+                   || (VM_PURGABLE_EMPTY    == m->object->purgable);

     }
     while (FALSE);
 
     }
     while (FALSE);
 


@@ -140,18 +146,11 @@ discard_page(vm_page_t m)
         */
         return;
 
         */
         return;
 

-    if (!m->no_isync) 
+    if (m->pmapped == TRUE) 

     {
     {

-        int refmod_state = pmap_disconnect(m->phys_page);
-
-        if (refmod_state & VM_MEM_REFERENCED)
-            m->reference = TRUE;
-        if (refmod_state & VM_MEM_MODIFIED)
-            m->dirty = TRUE;
+        __unused int refmod_state = pmap_disconnect(m->phys_page);

     }
 
     }
 

-    if (m->dirty)
-        panic("discard_page(%p) dirty", m);

     if (m->laundry)
         panic("discard_page(%p) laundry", m);
     if (m->private)
     if (m->laundry)
         panic("discard_page(%p) laundry", m);
     if (m->private)


@@ -159,6 +158,20 @@ discard_page(vm_page_t m)
     if (m->fictitious)
         panic("discard_page(%p) fictitious", m);
 
     if (m->fictitious)
         panic("discard_page(%p) fictitious", m);
 

+    if (VM_PURGABLE_VOLATILE == m->object->purgable)
+       {
+               assert(m->object->objq.next != NULL && m->object->objq.prev != NULL); /* object should be on a queue */
+               purgeable_q_t old_queue=vm_purgeable_object_remove(m->object);
+               assert(old_queue);
+               /* No need to lock page queue for token delete, hibernate_vm_unlock() 
+                       makes sure these locks are uncontended before sleep */
+               vm_purgeable_token_delete_first(old_queue);
+               m->object->purgable = VM_PURGABLE_EMPTY;
+       }
+       
+    if (m->tabled)
+       vm_page_remove(m);
+

     vm_page_free(m);
 }
 
     vm_page_free(m);
 }
 


@@ -167,7 +180,6 @@ discard_page(vm_page_t m)
  pages known to VM to not need saving are subtracted.
  Wired pages to be saved are present in page_list_wired, pageable in page_list.
 */
  pages known to VM to not need saving are subtracted.
  Wired pages to be saved are present in page_list_wired, pageable in page_list.
 */

-extern vm_page_t vm_lopage_queue_free;

 
 void
 hibernate_page_list_setall(hibernate_page_list_t * page_list,
 
 void
 hibernate_page_list_setall(hibernate_page_list_t * page_list,


@@ -177,10 +189,16 @@ hibernate_page_list_setall(hibernate_page_list_t * page_list,
     uint64_t start, end, nsec;
     vm_page_t m;
     uint32_t pages = page_list->page_count;
     uint64_t start, end, nsec;
     vm_page_t m;
     uint32_t pages = page_list->page_count;

-    uint32_t count_zf = 0, count_inactive = 0, count_active = 0;
+    uint32_t count_zf = 0, count_throttled = 0, count_inactive = 0, count_active = 0;

     uint32_t count_wire = pages;
     uint32_t count_wire = pages;

-    uint32_t count_discard_active = 0, count_discard_inactive = 0;
+    uint32_t count_discard_active    = 0;
+    uint32_t count_discard_inactive  = 0;
+    uint32_t count_discard_purgeable = 0;

     uint32_t i;
     uint32_t i;

+    uint32_t             bank;
+    hibernate_bitmap_t * bitmap;
+    hibernate_bitmap_t * bitmap_wired;
+

 
     HIBLOG("hibernate_page_list_setall start\n");
 
 
     HIBLOG("hibernate_page_list_setall start\n");
 


@@ -199,24 +217,46 @@ hibernate_page_list_setall(hibernate_page_list_t * page_list,
        m = (vm_page_t) m->pageq.next;
     }
 
        m = (vm_page_t) m->pageq.next;
     }
 

-    m = (vm_page_t) vm_page_queue_free;
-    while(m)
+    for( i = 0; i < vm_colors; i++ )
+    {
+       queue_iterate(&vm_page_queue_free[i],
+                     m,
+                     vm_page_t,
+                     pageq)
+       {
+           pages--;
+           count_wire--;
+           hibernate_page_bitset(page_list,       TRUE, m->phys_page);
+           hibernate_page_bitset(page_list_wired, TRUE, m->phys_page);
+       }
+    }
+
+    queue_iterate(&vm_lopage_queue_free,
+                 m,
+                 vm_page_t,
+                 pageq)

     {
        pages--;
        count_wire--;
        hibernate_page_bitset(page_list,       TRUE, m->phys_page);
        hibernate_page_bitset(page_list_wired, TRUE, m->phys_page);
     {
        pages--;
        count_wire--;
        hibernate_page_bitset(page_list,       TRUE, m->phys_page);
        hibernate_page_bitset(page_list_wired, TRUE, m->phys_page);

-       m = (vm_page_t) m->pageq.next;

     }
 
     }
 

-    m = (vm_page_t) vm_lopage_queue_free;
-    while(m)
+    queue_iterate( &vm_page_queue_throttled,
+                    m,
+                    vm_page_t,
+                    pageq )

     {
     {

-       pages--;
+        if ((kIOHibernateModeDiscardCleanInactive & gIOHibernateMode) 
+         && consider_discard(m))
+        {
+            hibernate_page_bitset(page_list, TRUE, m->phys_page);
+            count_discard_inactive++;
+        }
+        else
+            count_throttled++;

        count_wire--;
        count_wire--;

-       hibernate_page_bitset(page_list,       TRUE, m->phys_page);

        hibernate_page_bitset(page_list_wired, TRUE, m->phys_page);
        hibernate_page_bitset(page_list_wired, TRUE, m->phys_page);

-       m = (vm_page_t) m->pageq.next;

     }
 
     queue_iterate( &vm_page_queue_zf,
     }
 
     queue_iterate( &vm_page_queue_zf,


@@ -228,7 +268,10 @@ hibernate_page_list_setall(hibernate_page_list_t * page_list,
          && consider_discard(m))
         {
             hibernate_page_bitset(page_list, TRUE, m->phys_page);
          && consider_discard(m))
         {
             hibernate_page_bitset(page_list, TRUE, m->phys_page);

-            count_discard_inactive++;
+           if (m->dirty)
+               count_discard_purgeable++;
+           else
+               count_discard_inactive++;

         }
         else
             count_zf++;
         }
         else
             count_zf++;


@@ -245,7 +288,10 @@ hibernate_page_list_setall(hibernate_page_list_t * page_list,
          && consider_discard(m))
         {
             hibernate_page_bitset(page_list, TRUE, m->phys_page);
          && consider_discard(m))
         {
             hibernate_page_bitset(page_list, TRUE, m->phys_page);

-            count_discard_inactive++;
+           if (m->dirty)
+               count_discard_purgeable++;
+           else
+               count_discard_inactive++;

         }
         else
             count_inactive++;
         }
         else
             count_inactive++;


@@ -262,7 +308,10 @@ hibernate_page_list_setall(hibernate_page_list_t * page_list,
          && consider_discard(m))
         {
             hibernate_page_bitset(page_list, TRUE, m->phys_page);
          && consider_discard(m))
         {
             hibernate_page_bitset(page_list, TRUE, m->phys_page);

-            count_discard_active++;
+           if (m->dirty)
+               count_discard_purgeable++;
+           else
+               count_discard_active++;

         }
         else
             count_active++;
         }
         else
             count_active++;


@@ -272,10 +321,6 @@ hibernate_page_list_setall(hibernate_page_list_t * page_list,
 
     // pull wired from hibernate_bitmap
 
 
     // pull wired from hibernate_bitmap
 

-    uint32_t             bank;
-    hibernate_bitmap_t * bitmap;
-    hibernate_bitmap_t * bitmap_wired;
-

     bitmap = &page_list->bank_bitmap[0];
     bitmap_wired = &page_list_wired->bank_bitmap[0];
     for (bank = 0; bank < page_list->bank_count; bank++)
     bitmap = &page_list->bank_bitmap[0];
     bitmap_wired = &page_list_wired->bank_bitmap[0];
     for (bank = 0; bank < page_list->bank_count; bank++)


@@ -293,11 +338,11 @@ hibernate_page_list_setall(hibernate_page_list_t * page_list,
     absolutetime_to_nanoseconds(end - start, &nsec);
     HIBLOG("hibernate_page_list_setall time: %qd ms\n", nsec / 1000000ULL);
 
     absolutetime_to_nanoseconds(end - start, &nsec);
     HIBLOG("hibernate_page_list_setall time: %qd ms\n", nsec / 1000000ULL);
 

-    HIBLOG("pages %d, wire %d, act %d, inact %d, zf %d, could discard act %d inact %d\n", 
-                pages, count_wire, count_active, count_inactive, count_zf,
-                count_discard_active, count_discard_inactive);
+    HIBLOG("pages %d, wire %d, act %d, inact %d, zf %d, throt %d, could discard act %d inact %d purgeable %d\n", 
+                pages, count_wire, count_active, count_inactive, count_zf, count_throttled,
+                count_discard_active, count_discard_inactive, count_discard_purgeable);

 
 

-    *pagesOut = pages;
+    *pagesOut = pages - count_discard_active - count_discard_inactive - count_discard_purgeable;

 }
 
 void
 }
 
 void


@@ -306,7 +351,9 @@ hibernate_page_list_discard(hibernate_page_list_t * page_list)
     uint64_t  start, end, nsec;
     vm_page_t m;
     vm_page_t next;
     uint64_t  start, end, nsec;
     vm_page_t m;
     vm_page_t next;

-    uint32_t  count_discard_active = 0, count_discard_inactive = 0;
+    uint32_t  count_discard_active    = 0;
+    uint32_t  count_discard_inactive  = 0;
+    uint32_t  count_discard_purgeable = 0;

 
     clock_get_uptime(&start);
 
 
     clock_get_uptime(&start);
 


@@ -316,8 +363,11 @@ hibernate_page_list_discard(hibernate_page_list_t * page_list)
         next = (vm_page_t) m->pageq.next;
         if (hibernate_page_bittst(page_list, m->phys_page))
         {
         next = (vm_page_t) m->pageq.next;
         if (hibernate_page_bittst(page_list, m->phys_page))
         {

+           if (m->dirty)
+               count_discard_purgeable++;
+           else
+               count_discard_inactive++;

             discard_page(m);
             discard_page(m);

-            count_discard_inactive++;

         }
         m = next;
     }
         }
         m = next;
     }


@@ -328,8 +378,11 @@ hibernate_page_list_discard(hibernate_page_list_t * page_list)
         next = (vm_page_t) m->pageq.next;
         if (hibernate_page_bittst(page_list, m->phys_page))
         {
         next = (vm_page_t) m->pageq.next;
         if (hibernate_page_bittst(page_list, m->phys_page))
         {

+           if (m->dirty)
+               count_discard_purgeable++;
+           else
+               count_discard_inactive++;

             discard_page(m);
             discard_page(m);

-            count_discard_inactive++;

         }
         m = next;
     }
         }
         m = next;
     }


@@ -340,17 +393,20 @@ hibernate_page_list_discard(hibernate_page_list_t * page_list)
         next = (vm_page_t) m->pageq.next;
         if (hibernate_page_bittst(page_list, m->phys_page))
         {
         next = (vm_page_t) m->pageq.next;
         if (hibernate_page_bittst(page_list, m->phys_page))
         {

+           if (m->dirty)
+               count_discard_purgeable++;
+           else
+               count_discard_active++;

             discard_page(m);
             discard_page(m);

-            count_discard_active++;

         }
         m = next;
     }
 
     clock_get_uptime(&end);
     absolutetime_to_nanoseconds(end - start, &nsec);
         }
         m = next;
     }
 
     clock_get_uptime(&end);
     absolutetime_to_nanoseconds(end - start, &nsec);

-    HIBLOG("hibernate_page_list_discard time: %qd ms, discarded act %d inact %d\n",
+    HIBLOG("hibernate_page_list_discard time: %qd ms, discarded act %d inact %d purgeable %d\n",

                 nsec / 1000000ULL,
                 nsec / 1000000ULL,

-                count_discard_active, count_discard_inactive);
+                count_discard_active, count_discard_inactive, count_discard_purgeable);

 }
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 }
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */


@@ -391,7 +447,7 @@ hibernate_setup(IOHibernateImageHeader * header,
 
     hibernate_processor_setup(header);
 
 
     hibernate_processor_setup(header);
 

-    HIBLOG("hibernate_alloc_pages flags %08lx, gobbling %d pages\n", 
+    HIBLOG("hibernate_alloc_pages flags %08x, gobbling %d pages\n", 

            header->processorFlags, gobble_count);
 
     if (gobble_count)
            header->processorFlags, gobble_count);
 
     if (gobble_count)