[apple/xnu.git] / osfmk / kern / hibernate.c

/*
 * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_LICENSE_HEADER_START@
 * 
 * The contents of this file constitute Original Code as defined in and
 * are subject to the Apple Public Source License Version 1.1 (the
 * "License").  You may not use this file except in compliance with the
 * License.  Please obtain a copy of the License at
 * http://www.apple.com/publicsource and read it before using this file.
 * 
 * This Original Code and all software distributed under the License are
 * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
 * License for the specific language governing rights and limitations
 * under the License.
 * 
 * @APPLE_LICENSE_HEADER_END@
 */

#include <kern/kalloc.h>
#include <kern/machine.h>
#include <kern/misc_protos.h>
#include <kern/thread.h>
#include <kern/processor.h>
#include <mach/machine.h>
#include <mach/processor_info.h>
#include <mach/mach_types.h>
#include <default_pager/default_pager_internal.h>
#include <IOKit/IOPlatformExpert.h>
#define KERNEL

#include <IOKit/IOHibernatePrivate.h>
#include <vm/vm_page.h>
#include <vm/vm_pageout.h>

/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */

static vm_page_t hibernate_gobble_queue;

/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */

static void
hibernate_page_list_zero(hibernate_page_list_t *list)
{
    uint32_t             bank;
    hibernate_bitmap_t * bitmap;

    bitmap = &list->bank_bitmap[0];
    for (bank = 0; bank < list->bank_count; bank++)
    {
        uint32_t bit, last_bit;
        uint32_t *bitmap_word;

	bzero((void *) &bitmap->bitmap[0], bitmap->bitmapwords << 2); 

        // Set out-of-bound bits at end of bitmap.
        bitmap_word = &bitmap->bitmap[bitmap->bitmapwords - 1];
        last_bit = ((bitmap->last_page - bitmap->first_page) & 31);
        for (bit = 31; bit > last_bit; bit--) {
            *bitmap_word |= (0x80000000 >> bit);
        }

	bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
    }
}


static boolean_t 
consider_discard(vm_page_t m)
{
    register vm_object_t object = 0;
    int                  refmod_state;
    boolean_t            discard = FALSE;

    do
    {
        if(m->private)
            panic("consider_discard: private");

        if (!vm_object_lock_try(m->object))
            break;

        object = m->object;

	if (m->wire_count != 0)
            break;
        if (m->precious)
            break;

        if (m->busy || !object->alive)
           /*
            *	Somebody is playing with this page.
            */
            break;

        if (m->absent || m->unusual || m->error)
           /*
            * If it's unusual in anyway, ignore it
            */
            break;
    
        if (m->cleaning)
            break;

        if (!m->dirty)
        {
            refmod_state = pmap_get_refmod(m->phys_page);
        
            if (refmod_state & VM_MEM_REFERENCED)
                m->reference = TRUE;
            if (refmod_state & VM_MEM_MODIFIED)
                m->dirty = TRUE;
        }
   
        /*
         * If it's clean we can discard the page on wakeup.
         */
        discard = !m->dirty;
    }
    while (FALSE);

    if (object)
        vm_object_unlock(object);

    return (discard);
}


static void
discard_page(vm_page_t m)
{
    if (m->absent || m->unusual || m->error)
       /*
        * If it's unusual in anyway, ignore
        */
        return;

    if (!m->no_isync) 
    {
        int refmod_state = pmap_disconnect(m->phys_page);

        if (refmod_state & VM_MEM_REFERENCED)
            m->reference = TRUE;
        if (refmod_state & VM_MEM_MODIFIED)
            m->dirty = TRUE;
    }

    if (m->dirty)
        panic("discard_page(%p) dirty", m);
    if (m->laundry)
        panic("discard_page(%p) laundry", m);
    if (m->private)
        panic("discard_page(%p) private", m);
    if (m->fictitious)
        panic("discard_page(%p) fictitious", m);

    vm_page_free(m);
}

/*
 Bits zero in the bitmaps => needs to be saved. All pages default to be saved,
 pages known to VM to not need saving are subtracted.
 Wired pages to be saved are present in page_list_wired, pageable in page_list.
*/

void
hibernate_page_list_setall(hibernate_page_list_t * page_list,
			   hibernate_page_list_t * page_list_wired,
			   uint32_t * pagesOut)
{
    uint64_t start, end, nsec;
    vm_page_t m;
    uint32_t pages = page_list->page_count;
    uint32_t count_zf = 0, count_inactive = 0, count_active = 0;
    uint32_t count_wire = pages;
    uint32_t count_discard_active = 0, count_discard_inactive = 0;
    uint32_t i;

    HIBLOG("hibernate_page_list_setall start\n");

    clock_get_uptime(&start);

    hibernate_page_list_zero(page_list);
    hibernate_page_list_zero(page_list_wired);

    m = (vm_page_t) hibernate_gobble_queue;
    while(m)
    {
	pages--;
	count_wire--;
	hibernate_page_bitset(page_list,       TRUE, m->phys_page);
	hibernate_page_bitset(page_list_wired, TRUE, m->phys_page);
	m = (vm_page_t) m->pageq.next;
    }

    m = (vm_page_t) vm_page_queue_free;
    while(m)
    {
	pages--;
	count_wire--;
	hibernate_page_bitset(page_list,       TRUE, m->phys_page);
	hibernate_page_bitset(page_list_wired, TRUE, m->phys_page);
	m = (vm_page_t) m->pageq.next;
    }

    queue_iterate( &vm_page_queue_zf,
                    m,
                    vm_page_t,
                    pageq )
    {
        if ((kIOHibernateModeDiscardCleanInactive & gIOHibernateMode) 
         && consider_discard(m))
        {
            hibernate_page_bitset(page_list, TRUE, m->phys_page);
            count_discard_inactive++;
        }
        else
            count_zf++;
	count_wire--;
	hibernate_page_bitset(page_list_wired, TRUE, m->phys_page);
    }

    queue_iterate( &vm_page_queue_inactive,
                    m,
                    vm_page_t,
                    pageq )
    {
        if ((kIOHibernateModeDiscardCleanInactive & gIOHibernateMode) 
         && consider_discard(m))
        {
            hibernate_page_bitset(page_list, TRUE, m->phys_page);
            count_discard_inactive++;
        }
        else
            count_inactive++;
	count_wire--;
	hibernate_page_bitset(page_list_wired, TRUE, m->phys_page);
    }

    queue_iterate( &vm_page_queue_active,
                    m,
                    vm_page_t,
                    pageq )
    {
        if ((kIOHibernateModeDiscardCleanActive & gIOHibernateMode) 
         && consider_discard(m))
        {
            hibernate_page_bitset(page_list, TRUE, m->phys_page);
            count_discard_active++;
        }
        else
            count_active++;
	count_wire--;
	hibernate_page_bitset(page_list_wired, TRUE, m->phys_page);
    }

    // pull wired from hibernate_bitmap

    uint32_t             bank;
    hibernate_bitmap_t * bitmap;
    hibernate_bitmap_t * bitmap_wired;

    bitmap = &page_list->bank_bitmap[0];
    bitmap_wired = &page_list_wired->bank_bitmap[0];
    for (bank = 0; bank < page_list->bank_count; bank++)
    {
	for (i = 0; i < bitmap->bitmapwords; i++)
	    bitmap->bitmap[i] = bitmap->bitmap[i] | ~bitmap_wired->bitmap[i];
	bitmap       = (hibernate_bitmap_t *) &bitmap->bitmap      [bitmap->bitmapwords];
	bitmap_wired = (hibernate_bitmap_t *) &bitmap_wired->bitmap[bitmap_wired->bitmapwords];
    }

    // machine dependent adjustments
    hibernate_page_list_setall_machine(page_list, page_list_wired, &pages);

    clock_get_uptime(&end);
    absolutetime_to_nanoseconds(end - start, &nsec);
    HIBLOG("hibernate_page_list_setall time: %qd ms\n", nsec / 1000000ULL);

    HIBLOG("pages %d, wire %d, act %d, inact %d, zf %d, could discard act %d inact %d\n", 
                pages, count_wire, count_active, count_inactive, count_zf,
                count_discard_active, count_discard_inactive);

    *pagesOut = pages;
}

void
hibernate_page_list_discard(hibernate_page_list_t * page_list)
{
    uint64_t  start, end, nsec;
    vm_page_t m;
    vm_page_t next;
    uint32_t  count_discard_active = 0, count_discard_inactive = 0;

    clock_get_uptime(&start);

    m = (vm_page_t) queue_first(&vm_page_queue_zf);
    while (m && !queue_end(&vm_page_queue_zf, (queue_entry_t)m))
    {
        next = (vm_page_t) m->pageq.next;
        if (hibernate_page_bittst(page_list, m->phys_page))
        {
            discard_page(m);
            count_discard_inactive++;
        }
        m = next;
    }

    m = (vm_page_t) queue_first(&vm_page_queue_inactive);
    while (m && !queue_end(&vm_page_queue_inactive, (queue_entry_t)m))
    {
        next = (vm_page_t) m->pageq.next;
        if (hibernate_page_bittst(page_list, m->phys_page))
        {
            discard_page(m);
            count_discard_inactive++;
        }
        m = next;
    }

    m = (vm_page_t) queue_first(&vm_page_queue_active);
    while (m && !queue_end(&vm_page_queue_active, (queue_entry_t)m))
    {
        next = (vm_page_t) m->pageq.next;
        if (hibernate_page_bittst(page_list, m->phys_page))
        {
            discard_page(m);
            count_discard_active++;
        }
        m = next;
    }

    clock_get_uptime(&end);
    absolutetime_to_nanoseconds(end - start, &nsec);
    HIBLOG("hibernate_page_list_discard time: %qd ms, discarded act %d inact %d\n",
                nsec / 1000000ULL,
                count_discard_active, count_discard_inactive);
}

/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */

kern_return_t 
hibernate_setup(IOHibernateImageHeader * header,
                        uint32_t free_page_ratio,
                        uint32_t free_page_time,
			hibernate_page_list_t ** page_list_ret,
			hibernate_page_list_t ** page_list_wired_ret,
                        boolean_t * encryptedswap)
{
    hibernate_page_list_t * page_list = NULL;
    hibernate_page_list_t * page_list_wired = NULL;
    vm_page_t		    m;
    uint32_t    	    i, gobble_count;

    *page_list_ret       = NULL;
    *page_list_wired_ret = NULL;


    page_list = hibernate_page_list_allocate();
    if (!page_list)
        return (KERN_RESOURCE_SHORTAGE);
    page_list_wired = hibernate_page_list_allocate();
    if (!page_list_wired)
    {
        kfree(page_list, page_list->list_size);
        return (KERN_RESOURCE_SHORTAGE);
    }

    *encryptedswap = dp_encryption;

    // pages we could force out to reduce hibernate image size
    gobble_count = (((uint64_t) page_list->page_count) * ((uint64_t) free_page_ratio)) / 100;

    // no failures hereafter

    hibernate_processor_setup(header);

    HIBLOG("hibernate_alloc_pages flags %08lx, gobbling %d pages\n", 
	    header->processorFlags, gobble_count);

    if (gobble_count)
    {
        uint64_t start, end, timeout, nsec;
        clock_interval_to_deadline(free_page_time, 1000 * 1000 /*ms*/, &timeout);
        clock_get_uptime(&start);
    
        for (i = 0; i < gobble_count; i++)
        {
            while (VM_PAGE_NULL == (m = vm_page_grab()))
            {
                clock_get_uptime(&end);
                if (end >= timeout)
                    break;
                VM_PAGE_WAIT();
            }
            if (!m)
                break;
            m->busy = FALSE;
            vm_page_gobble(m);
    
            m->pageq.next = (queue_entry_t) hibernate_gobble_queue;
            hibernate_gobble_queue = m;
        }
    
        clock_get_uptime(&end);
        absolutetime_to_nanoseconds(end - start, &nsec);
        HIBLOG("Gobbled %d pages, time: %qd ms\n", i, nsec / 1000000ULL);
    }

    *page_list_ret       = page_list;
    *page_list_wired_ret = page_list_wired;

    return (KERN_SUCCESS);
}

kern_return_t 
hibernate_teardown(hibernate_page_list_t * page_list,
                    hibernate_page_list_t * page_list_wired)
{
    vm_page_t m, next;
    uint32_t  count = 0;

    m = (vm_page_t) hibernate_gobble_queue;
    while(m)
    {
        next = (vm_page_t) m->pageq.next;
        vm_page_free(m);
        count++;
        m = next;
    }
    hibernate_gobble_queue = VM_PAGE_NULL;
    
    if (count)
        HIBLOG("Freed %d pages\n", count);

    if (page_list)
        kfree(page_list, page_list->list_size);
    if (page_list_wired)
        kfree(page_list_wired, page_list_wired->list_size);

    return (KERN_SUCCESS);
}
Commit	Line	Data
3a60a9f5 A	1	/*
	2	* Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
	3	*
	4	* @APPLE_LICENSE_HEADER_START@
	5	*
	6	* The contents of this file constitute Original Code as defined in and
	7	* are subject to the Apple Public Source License Version 1.1 (the
	8	* "License"). You may not use this file except in compliance with the
	9	* License. Please obtain a copy of the License at
	10	* http://www.apple.com/publicsource and read it before using this file.
	11	*
	12	* This Original Code and all software distributed under the License are
	13	* distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	14	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	15	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	16	* FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
	17	* License for the specific language governing rights and limitations
	18	* under the License.
	19	*
	20	* @APPLE_LICENSE_HEADER_END@
	21	*/
	22
	23	#include <kern/kalloc.h>
	24	#include <kern/machine.h>
	25	#include <kern/misc_protos.h>
	26	#include <kern/thread.h>
	27	#include <kern/processor.h>
	28	#include <mach/machine.h>
	29	#include <mach/processor_info.h>
	30	#include <mach/mach_types.h>
	31	#include <default_pager/default_pager_internal.h>
	32	#include <IOKit/IOPlatformExpert.h>
	33	#define KERNEL
	34
	35	#include <IOKit/IOHibernatePrivate.h>
	36	#include <vm/vm_page.h>
	37	#include <vm/vm_pageout.h>
	38
	39	/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
	40
	41	static vm_page_t hibernate_gobble_queue;
	42
	43	/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
	44
	45	static void
	46	hibernate_page_list_zero(hibernate_page_list_t *list)
	47	{
	48	uint32_t bank;
	49	hibernate_bitmap_t * bitmap;
	50
	51	bitmap = &list->bank_bitmap[0];
	52	for (bank = 0; bank < list->bank_count; bank++)
	53	{
	54	uint32_t bit, last_bit;
	55	uint32_t *bitmap_word;
	56
	57	bzero((void *) &bitmap->bitmap[0], bitmap->bitmapwords << 2);
	58
	59	// Set out-of-bound bits at end of bitmap.
	60	bitmap_word = &bitmap->bitmap[bitmap->bitmapwords - 1];
	61	last_bit = ((bitmap->last_page - bitmap->first_page) & 31);
	62	for (bit = 31; bit > last_bit; bit--) {
	63	*bitmap_word \|= (0x80000000 >> bit);
	64	}
65
66	bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
67	}
68	}
69
70
71	static boolean_t
72	consider_discard(vm_page_t m)
73	{
74	register vm_object_t object = 0;
75	int refmod_state;
76	boolean_t discard = FALSE;
77
78	do
79	{
80	if(m->private)
81	panic("consider_discard: private");
82
83	if (!vm_object_lock_try(m->object))
84	break;
85
86	object = m->object;
87
88	if (m->wire_count != 0)
89	break;
90	if (m->precious)
91	break;
92
93	if (m->busy \|\| !object->alive)
94	/*
95	* Somebody is playing with this page.
96	*/
97	break;
98
99	if (m->absent \|\| m->unusual \|\| m->error)
100	/*
101	* If it's unusual in anyway, ignore it
102	*/
103	break;
104
105	if (m->cleaning)
106	break;
107
108	if (!m->dirty)
109	{
110	refmod_state = pmap_get_refmod(m->phys_page);
111
112	if (refmod_state & VM_MEM_REFERENCED)
113	m->reference = TRUE;
114	if (refmod_state & VM_MEM_MODIFIED)
115	m->dirty = TRUE;
116	}
117
118	/*
119	* If it's clean we can discard the page on wakeup.
120	*/
121	discard = !m->dirty;
122	}
123	while (FALSE);
124
125	if (object)
126	vm_object_unlock(object);
127
128	return (discard);
129	}
130
131
132	static void
133	discard_page(vm_page_t m)
134	{
135	if (m->absent \|\| m->unusual \|\| m->error)
136	/*
137	* If it's unusual in anyway, ignore
138	*/
139	return;
140
141	if (!m->no_isync)
142	{
143	int refmod_state = pmap_disconnect(m->phys_page);
144
145	if (refmod_state & VM_MEM_REFERENCED)
146	m->reference = TRUE;
147	if (refmod_state & VM_MEM_MODIFIED)
148	m->dirty = TRUE;
149	}
150
151	if (m->dirty)
152	panic("discard_page(%p) dirty", m);
153	if (m->laundry)
154	panic("discard_page(%p) laundry", m);
155	if (m->private)
156	panic("discard_page(%p) private", m);
157	if (m->fictitious)
158	panic("discard_page(%p) fictitious", m);
159
160	vm_page_free(m);
161	}
162
163	/*
164	Bits zero in the bitmaps => needs to be saved. All pages default to be saved,
165	pages known to VM to not need saving are subtracted.
166	Wired pages to be saved are present in page_list_wired, pageable in page_list.
167	*/
168
169	void
170	hibernate_page_list_setall(hibernate_page_list_t * page_list,
171	hibernate_page_list_t * page_list_wired,
172	uint32_t * pagesOut)
173	{
174	uint64_t start, end, nsec;
175	vm_page_t m;
176	uint32_t pages = page_list->page_count;
177	uint32_t count_zf = 0, count_inactive = 0, count_active = 0;
178	uint32_t count_wire = pages;
179	uint32_t count_discard_active = 0, count_discard_inactive = 0;
180	uint32_t i;
181
182	HIBLOG("hibernate_page_list_setall start\n");
183
184	clock_get_uptime(&start);
185
186	hibernate_page_list_zero(page_list);
187	hibernate_page_list_zero(page_list_wired);
188
189	m = (vm_page_t) hibernate_gobble_queue;
190	while(m)
191	{
192	pages--;
193	count_wire--;
194	hibernate_page_bitset(page_list, TRUE, m->phys_page);
195	hibernate_page_bitset(page_list_wired, TRUE, m->phys_page);
196	m = (vm_page_t) m->pageq.next;
197	}
198
199	m = (vm_page_t) vm_page_queue_free;
200	while(m)
201	{
202	pages--;
203	count_wire--;
204	hibernate_page_bitset(page_list, TRUE, m->phys_page);
205	hibernate_page_bitset(page_list_wired, TRUE, m->phys_page);
206	m = (vm_page_t) m->pageq.next;
207	}
208
209	queue_iterate( &vm_page_queue_zf,
210	m,
211	vm_page_t,
212	pageq )
213	{
214	if ((kIOHibernateModeDiscardCleanInactive & gIOHibernateMode)
215	&& consider_discard(m))
216	{
217	hibernate_page_bitset(page_list, TRUE, m->phys_page);
218	count_discard_inactive++;
219	}
220	else
221	count_zf++;
222	count_wire--;
223	hibernate_page_bitset(page_list_wired, TRUE, m->phys_page);
224	}
225
226	queue_iterate( &vm_page_queue_inactive,
227	m,
228	vm_page_t,
229	pageq )
230	{
231	if ((kIOHibernateModeDiscardCleanInactive & gIOHibernateMode)
232	&& consider_discard(m))
233	{
234	hibernate_page_bitset(page_list, TRUE, m->phys_page);
235	count_discard_inactive++;
236	}
237	else
238	count_inactive++;
239	count_wire--;
240	hibernate_page_bitset(page_list_wired, TRUE, m->phys_page);
241	}
242
243	queue_iterate( &vm_page_queue_active,
244	m,
245	vm_page_t,
246	pageq )
247	{
248	if ((kIOHibernateModeDiscardCleanActive & gIOHibernateMode)
249	&& consider_discard(m))
250	{
251	hibernate_page_bitset(page_list, TRUE, m->phys_page);
252	count_discard_active++;
253	}
254	else
255	count_active++;
256	count_wire--;
257	hibernate_page_bitset(page_list_wired, TRUE, m->phys_page);
258	}
259
260	// pull wired from hibernate_bitmap
261
262	uint32_t bank;
263	hibernate_bitmap_t * bitmap;
264	hibernate_bitmap_t * bitmap_wired;
265
266	bitmap = &page_list->bank_bitmap[0];
267	bitmap_wired = &page_list_wired->bank_bitmap[0];
268	for (bank = 0; bank < page_list->bank_count; bank++)
269	{
270	for (i = 0; i < bitmap->bitmapwords; i++)
271	bitmap->bitmap[i] = bitmap->bitmap[i] \| ~bitmap_wired->bitmap[i];
272	bitmap = (hibernate_bitmap_t *) &bitmap->bitmap [bitmap->bitmapwords];
273	bitmap_wired = (hibernate_bitmap_t *) &bitmap_wired->bitmap[bitmap_wired->bitmapwords];
274	}
275
276	// machine dependent adjustments
277	hibernate_page_list_setall_machine(page_list, page_list_wired, &pages);
278
279	clock_get_uptime(&end);
280	absolutetime_to_nanoseconds(end - start, &nsec);
281	HIBLOG("hibernate_page_list_setall time: %qd ms\n", nsec / 1000000ULL);
282
283	HIBLOG("pages %d, wire %d, act %d, inact %d, zf %d, could discard act %d inact %d\n",
284	pages, count_wire, count_active, count_inactive, count_zf,
285	count_discard_active, count_discard_inactive);
286
287	*pagesOut = pages;
288	}
289
290	void
291	hibernate_page_list_discard(hibernate_page_list_t * page_list)
292	{
293	uint64_t start, end, nsec;
294	vm_page_t m;
295	vm_page_t next;
296	uint32_t count_discard_active = 0, count_discard_inactive = 0;
297
298	clock_get_uptime(&start);
299
300	m = (vm_page_t) queue_first(&vm_page_queue_zf);
301	while (m && !queue_end(&vm_page_queue_zf, (queue_entry_t)m))
302	{
303	next = (vm_page_t) m->pageq.next;
304	if (hibernate_page_bittst(page_list, m->phys_page))
305	{
306	discard_page(m);
307	count_discard_inactive++;
308	}
309	m = next;
310	}
311
312	m = (vm_page_t) queue_first(&vm_page_queue_inactive);
313	while (m && !queue_end(&vm_page_queue_inactive, (queue_entry_t)m))
314	{
315	next = (vm_page_t) m->pageq.next;
316	if (hibernate_page_bittst(page_list, m->phys_page))
317	{
318	discard_page(m);
319	count_discard_inactive++;
320	}
321	m = next;
322	}
323
324	m = (vm_page_t) queue_first(&vm_page_queue_active);
325	while (m && !queue_end(&vm_page_queue_active, (queue_entry_t)m))
326	{
327	next = (vm_page_t) m->pageq.next;
328	if (hibernate_page_bittst(page_list, m->phys_page))
329	{
330	discard_page(m);
331	count_discard_active++;
332	}
333	m = next;
334	}
335
336	clock_get_uptime(&end);
337	absolutetime_to_nanoseconds(end - start, &nsec);
338	HIBLOG("hibernate_page_list_discard time: %qd ms, discarded act %d inact %d\n",
339	nsec / 1000000ULL,
340	count_discard_active, count_discard_inactive);
341	}
342
343	/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
344
345	kern_return_t
346	hibernate_setup(IOHibernateImageHeader * header,
347	uint32_t free_page_ratio,
348	uint32_t free_page_time,
349	hibernate_page_list_t ** page_list_ret,
350	hibernate_page_list_t ** page_list_wired_ret,
351	boolean_t * encryptedswap)
352	{
353	hibernate_page_list_t * page_list = NULL;
354	hibernate_page_list_t * page_list_wired = NULL;
355	vm_page_t m;
356	uint32_t i, gobble_count;
357
358	*page_list_ret = NULL;
359	*page_list_wired_ret = NULL;
360
361
362	page_list = hibernate_page_list_allocate();
363	if (!page_list)
364	return (KERN_RESOURCE_SHORTAGE);
365	page_list_wired = hibernate_page_list_allocate();
366	if (!page_list_wired)
367	{
368	kfree(page_list, page_list->list_size);
369	return (KERN_RESOURCE_SHORTAGE);
370	}
371
372	*encryptedswap = dp_encryption;
373
374	// pages we could force out to reduce hibernate image size
375	gobble_count = (((uint64_t) page_list->page_count) * ((uint64_t) free_page_ratio)) / 100;
376
377	// no failures hereafter
378
379	hibernate_processor_setup(header);
380
381	HIBLOG("hibernate_alloc_pages flags %08lx, gobbling %d pages\n",
382	header->processorFlags, gobble_count);
383
384	if (gobble_count)
385	{
386	uint64_t start, end, timeout, nsec;
387	clock_interval_to_deadline(free_page_time, 1000 * 1000 /ms/, &timeout);
388	clock_get_uptime(&start);
389
390	for (i = 0; i < gobble_count; i++)
391	{
392	while (VM_PAGE_NULL == (m = vm_page_grab()))
393	{
394	clock_get_uptime(&end);
395	if (end >= timeout)
396	break;
397	VM_PAGE_WAIT();
398	}
399	if (!m)
400	break;
401	m->busy = FALSE;
402	vm_page_gobble(m);
403
404	m->pageq.next = (queue_entry_t) hibernate_gobble_queue;
405	hibernate_gobble_queue = m;
406	}
407
408	clock_get_uptime(&end);
409	absolutetime_to_nanoseconds(end - start, &nsec);
410	HIBLOG("Gobbled %d pages, time: %qd ms\n", i, nsec / 1000000ULL);
411	}
412
413	*page_list_ret = page_list;
414	*page_list_wired_ret = page_list_wired;
415
416	return (KERN_SUCCESS);
417	}
418
419	kern_return_t
420	hibernate_teardown(hibernate_page_list_t * page_list,
421	hibernate_page_list_t * page_list_wired)
422	{
423	vm_page_t m, next;
424	uint32_t count = 0;
425
426	m = (vm_page_t) hibernate_gobble_queue;
427	while(m)
428	{
429	next = (vm_page_t) m->pageq.next;
430	vm_page_free(m);
431	count++;
432	m = next;
433	}
434	hibernate_gobble_queue = VM_PAGE_NULL;
435
436	if (count)
437	HIBLOG("Freed %d pages\n", count);
438
439	if (page_list)
440	kfree(page_list, page_list->list_size);
441	if (page_list_wired)
442	kfree(page_list_wired, page_list_wired->list_size);
443
444	return (KERN_SUCCESS);
445	}
446