[apple/xnu.git] / osfmk / vm / default_freezer.c

/*
 * Copyright (c) 2000-2010 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 * 
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */

#if CONFIG_FREEZE

#include "default_freezer.h"

/*
 * Indicates that a page has been faulted back in.
 */
#define FREEZER_OFFSET_ABSENT ((vm_object_offset_t)(-1))

/*
 * Create the mapping table that will
 * tell us the object/offset pair that
 * corresponds to the page being sent
 * out or being brought back in.
 */

void*
default_freezer_mapping_create(vm_object_t object, vm_offset_t offset)
{
	default_freezer_mapping_table_t table;
	
	table = kalloc(sizeof(struct default_freezer_mapping_table));
	if (table) {
		memset(table, 0, sizeof(*table));
	} else {
		panic("Could not allocate mapping table\n");
	}
	
	table->object = object;
	table->offset = offset;
	
	return (void*)table;
}

void
default_freezer_mapping_free(void **table, boolean_t all)
{	
	default_freezer_mapping_table_t freezer_table = *((default_freezer_mapping_table_t *)table);
	assert(freezer_table);
	
	if (all) {
		do { 
			default_freezer_mapping_table_t next = freezer_table->next;
			kfree(freezer_table, sizeof(*freezer_table));
			freezer_table = next;	
		} while (freezer_table);
	} else {
		kfree(freezer_table, sizeof(*freezer_table));
	}
}
 
kern_return_t
default_freezer_mapping_store(
		default_freezer_mapping_table_t *table,
		memory_object_offset_t table_offset,
		memory_object_t memory_object,
		memory_object_offset_t offset)
{
	default_freezer_mapping_table_entry_t entry;
	uint32_t index;
	
	assert(*table);
	
	if ((*table)->index >= MAX_FREEZE_TABLE_ENTRIES) {
		vm_object_t compact_object = (*table)->object;
		default_freezer_mapping_table_t next;
		
		next = default_freezer_mapping_create(compact_object, table_offset);
		if (!next) {
			return KERN_FAILURE;
		}
		
		(*table)->next = next;
		*table = next;
	}

	index = (*table)->index++;
	entry = &(*table)->entry[index];

	entry->memory_object = memory_object;
	entry->offset = offset;
	
	return KERN_SUCCESS;
}

kern_return_t
default_freezer_mapping_update(
		default_freezer_mapping_table_t table, 
		memory_object_t memory_object,
		memory_object_offset_t offset,
		memory_object_offset_t *table_offset, /*OUT: contains the offset into the compact object*/
		boolean_t remove_entry)
{

	kern_return_t kr = KERN_SUCCESS;
	vm_object_offset_t compact_offset;
	default_freezer_mapping_table_entry_t entry;
	uint32_t index = 0;
	
	if (table == NULL){
		return KERN_FAILURE;
	}

	compact_offset = table->offset;

	while (1) {	
		if (index >= table->index) {
			if (table->next) {
				table = table->next;
				index = 0;
			} else {
				/* End of tables and we didn't find our candidate entry */
				kr = KERN_FAILURE;
				break;
			}
		}

		entry = &table->entry[index];

		if (memory_object == entry->memory_object && offset == entry->offset) {
			if (remove_entry == TRUE) {
				/*
				 * Mark the page absent whilst retaining the object 
				 * for cleanup during thaw.
				 */
				entry->offset = FREEZER_OFFSET_ABSENT;
			}
			if (table_offset != NULL) {
				*table_offset = compact_offset;
			}
			break;
		}
	
		index++;
		compact_offset += PAGE_SIZE;
	}
	return kr;
}

/*
 * Create a freezer memory object for this
 * vm object.
 */
void
default_freezer_memory_object_create(
			vm_object_t object,
			vm_object_t compact_object,
			default_freezer_mapping_table_t table)
{

	default_freezer_memory_object_t fo = NULL;
	
	fo = kalloc(sizeof(struct default_freezer_memory_object));

	if (fo) {
		memory_object_control_t control = NULL;

		memset(fo, 0, sizeof(*fo));
		
		control = memory_object_control_allocate(object);
		assert (control != MEMORY_OBJECT_CONTROL_NULL);

		df_memory_object_init((memory_object_t)fo, control, 0);		
		fo->fo_compact_object = compact_object;
		fo->fo_table = table;
		
		object->pager = (memory_object_t)fo;
		object->pager_created = TRUE;
		object->pager_initialized = TRUE;
		object->pager_ready = TRUE;
		object->pager_trusted = TRUE;
		object->pager_control = control;
	} else {
		panic(" Could not allocate freezer object\n");
	}
}

void
default_freezer_pack_page(
		vm_page_t p, 
		vm_object_t compact_object, 
		vm_object_offset_t offset, 
		void **table)
{

	default_freezer_mapping_table_t *freeze_table = (default_freezer_mapping_table_t *)table;
	memory_object_t memory_object = p->object->pager;
	
	if (memory_object == NULL) {
		default_freezer_memory_object_create(p->object, compact_object, *freeze_table);
		memory_object = p->object->pager;
	} else {
		default_freezer_memory_object_t fo = (default_freezer_memory_object_t)memory_object;
		if (fo->fo_compact_object == VM_OBJECT_NULL) {
			fo->fo_compact_object = compact_object;
			fo->fo_table = *freeze_table;
		}
	}
	
	default_freezer_mapping_store(freeze_table, offset, memory_object, p->offset + p->object->paging_offset);

	/* Remove from the original and insert into the compact destination object */
	vm_page_rename(p, compact_object, offset, FALSE);
}

void
default_freezer_unpack(
		vm_object_t object, 
		void **table)
{
	
	vm_page_t p = VM_PAGE_NULL;
	uint32_t index = 0;
	vm_object_t src_object = VM_OBJECT_NULL;
	memory_object_t	src_mem_object = MEMORY_OBJECT_NULL;
	memory_object_offset_t	src_offset = 0;
	vm_object_offset_t	compact_offset = 0;
	default_freezer_memory_object_t	fo = NULL;
	default_freezer_memory_object_t last_memory_object_thawed = NULL;
	default_freezer_mapping_table_t freeze_table = *(default_freezer_mapping_table_t *)table;

	assert(freeze_table);
	
	vm_object_lock(object);
	
	for (index = 0, compact_offset = 0; ; index++, compact_offset += PAGE_SIZE){
		if (index >= freeze_table->index) {
			default_freezer_mapping_table_t table_next;
			
			table_next = freeze_table->next; 
			
			/* Free the tables as we go along */
			default_freezer_mapping_free((void**)&freeze_table, FALSE);
			
			if (table_next == NULL){
				break;
			}
			
			freeze_table = table_next;
			index = 0;
		}

		/* 
		 * Skip slots that represent deallocated memory objects.
		 */
		src_mem_object = freeze_table->entry[index].memory_object;
		if (src_mem_object == MEMORY_OBJECT_NULL)
			continue;

		/* 
		 * Skip slots that represent faulted pages.
		 */
		src_offset = freeze_table->entry[index].offset;
		if (src_offset != FREEZER_OFFSET_ABSENT) {
			
			p = vm_page_lookup(object, compact_offset);
			assert(p);

			fo = (default_freezer_memory_object_t)src_mem_object;
		
			src_object = memory_object_control_to_vm_object(fo->fo_pager_control); 
	
			/* Move back over from the freeze object to the original */
			vm_object_lock(src_object);
			vm_page_rename(p, src_object, src_offset - src_object->paging_offset, FALSE);
			vm_object_unlock(src_object);
		}
		
		if (src_mem_object != ((memory_object_t)last_memory_object_thawed)){
			if (last_memory_object_thawed != NULL){
				last_memory_object_thawed->fo_compact_object = VM_OBJECT_NULL;
				last_memory_object_thawed->fo_table = NULL;
			}
			last_memory_object_thawed = (default_freezer_memory_object_t)src_mem_object;
		}
	}
	
	if (last_memory_object_thawed != NULL){
		last_memory_object_thawed->fo_compact_object = VM_OBJECT_NULL;
		last_memory_object_thawed->fo_table = NULL;
	}
	
	vm_object_unlock(object);
}

vm_object_t
default_freezer_get_compact_vm_object(void** table)
{
	default_freezer_mapping_table_t freeze_table = *((default_freezer_mapping_table_t *)table);
	assert(freeze_table);
	return ((vm_object_t)(freeze_table->object));
}

void
df_memory_object_reference(__unused memory_object_t mem_obj)
{
	
	/* No-op */
}

void
df_memory_object_deallocate(memory_object_t mem_obj)
{

	default_freezer_memory_object_t	fo = (default_freezer_memory_object_t)mem_obj;
	vm_object_t compact_object = fo->fo_compact_object;
	
	assert(fo);
	
	if (compact_object != VM_OBJECT_NULL) {
		
		default_freezer_mapping_table_t fo_table = fo->fo_table;
		default_freezer_mapping_table_entry_t entry;
		boolean_t found = FALSE;
		uint32_t index = 0;
		
		vm_object_lock(compact_object);
	
		/* Remove from table */
		while (1) {	
			if (index >= fo_table->index) {
				if (fo_table->next) {
					fo_table = fo_table->next;
					index = 0;
				} else {
					/* End of tables */
					break;
				}
			}

			entry = &fo_table->entry[index];
			if (mem_obj == entry->memory_object) {
				/* It matches, so clear the entry */
				if (!found) {
					found = TRUE;
				} 
				entry->memory_object = MEMORY_OBJECT_NULL;
				entry->offset = 0;
			} else if (MEMORY_OBJECT_NULL != entry->memory_object) {
				/* We have a different valid object; we're done */
				if (found) {
					break;
				}
			}
		
			index++;
		}
	
		vm_object_unlock(compact_object);
	}
	
	kfree(fo, sizeof(*fo));
}

kern_return_t
df_memory_object_init(
		memory_object_t mem_obj,
		memory_object_control_t control,
		__unused memory_object_cluster_size_t pager_page_size)
{

	default_freezer_memory_object_t	fo = (default_freezer_memory_object_t)mem_obj;
	assert(fo);

	fo->fo_pager_ops = &default_freezer_ops;
	fo->fo_pager_header.io_bits = IKOT_MEMORY_OBJECT;
	fo->fo_pager_control = control;
	
	return KERN_SUCCESS;
}

kern_return_t
df_memory_object_terminate(memory_object_t mem_obj)
{

	default_freezer_memory_object_t	fo = (default_freezer_memory_object_t)mem_obj;
	assert(fo);
	memory_object_control_deallocate(fo->fo_pager_control);
	return KERN_SUCCESS;
}

kern_return_t
df_memory_object_data_request(
		memory_object_t mem_obj, 
		memory_object_offset_t offset,
		memory_object_cluster_size_t length,
		vm_prot_t protection_required,
		memory_object_fault_info_t fault_info)
{

	vm_object_t	src_object = VM_OBJECT_NULL, compact_object = VM_OBJECT_NULL;
	memory_object_offset_t	compact_offset = 0;
	memory_object_t pager = NULL;
	kern_return_t kr = KERN_SUCCESS;

	default_freezer_memory_object_t fo = (default_freezer_memory_object_t)mem_obj;

	src_object = memory_object_control_to_vm_object(fo->fo_pager_control);
	compact_object = fo->fo_compact_object;
	
	if (compact_object != VM_OBJECT_NULL) {
		
		vm_object_lock(compact_object);
	
		kr = default_freezer_mapping_update(fo->fo_table,
							mem_obj,
							offset,
							&compact_offset,
							FALSE);
						
		vm_object_unlock(compact_object);
	} else {
		kr = KERN_FAILURE;
	}
	
	if (length == 0){
		/*Caller is just querying to see if we have the page*/
		return kr;
	}

	if (kr != KERN_SUCCESS){

		unsigned int request_flags;
		upl_t        upl;
		unsigned int page_list_count = 0;

		request_flags = UPL_NO_SYNC | UPL_RET_ONLY_ABSENT | UPL_SET_LITE;
		/*
		 * Should we decide to activate USE_PRECIOUS (from default_pager_internal.h)
		 * here, then the request_flags will need to add these to the ones above:
		 *
		 * request_flags |= UPL_PRECIOUS | UPL_CLEAN_IN_PLACE
		 */
		request_flags |= UPL_REQUEST_SET_DIRTY;

		memory_object_super_upl_request(fo->fo_pager_control,
						(memory_object_offset_t)offset,
						PAGE_SIZE, PAGE_SIZE, 
						&upl, NULL, &page_list_count,
						request_flags);

		upl_abort(upl, UPL_ABORT_UNAVAILABLE);
		upl_deallocate(upl);
		
		return KERN_SUCCESS;
	}

	vm_object_lock(compact_object);

	pager = (memory_object_t)compact_object->pager;

	if (!compact_object->pager_ready || pager == MEMORY_OBJECT_NULL){
		vm_object_unlock(compact_object);
		return KERN_FAILURE;
	}
	
	vm_object_paging_wait(compact_object, THREAD_UNINT);
	vm_object_paging_begin(compact_object);

	compact_object->blocked_access = TRUE;
	vm_object_unlock(compact_object);

	((vm_object_fault_info_t) fault_info)->io_sync = TRUE;

	kr = dp_memory_object_data_request(pager,
					compact_offset,
					length,
					protection_required,
					fault_info);
	if (kr == KERN_SUCCESS){

		vm_page_t src_page = VM_PAGE_NULL, dst_page = VM_PAGE_NULL;

		vm_object_lock(compact_object);

		compact_object->blocked_access = FALSE;
		vm_object_paging_end(compact_object);

		vm_object_lock(src_object);

		if ((src_page = vm_page_lookup(compact_object, compact_offset)) != VM_PAGE_NULL){
			
			dst_page = vm_page_lookup(src_object, offset - src_object->paging_offset);
			
			VM_PAGE_FREE(dst_page);
			vm_page_rename(src_page, src_object, offset - src_object->paging_offset, FALSE);
			
			if (default_freezer_mapping_update(fo->fo_table,
							mem_obj,
							offset,
							NULL,
							TRUE) != KERN_SUCCESS) {
				printf("Page for object: 0x%lx at offset: 0x%lx not found in table\n", (uintptr_t)src_object, (uintptr_t)offset);
			}
			
			PAGE_WAKEUP_DONE(src_page);
		} else {
			printf("%d: default_freezer: compact_object doesn't have the page for object 0x%lx at offset 0x%lx \n", kr, (uintptr_t)compact_object, (uintptr_t)compact_offset);
			kr = KERN_FAILURE;
		}
		vm_object_unlock(src_object);
		vm_object_unlock(compact_object);
	} else {
		panic("%d: default_freezer TOC pointed us to default_pager incorrectly\n", kr);
	}
	return kr;
}

kern_return_t
df_memory_object_data_return(
		__unused memory_object_t		mem_obj,
		__unused memory_object_offset_t	offset,
		__unused memory_object_cluster_size_t			size,
		__unused memory_object_offset_t	*resid_offset,
		__unused int		*io_error,
		__unused boolean_t	dirty,
		__unused boolean_t	kernel_copy,
		__unused int	upl_flags)
{

	panic(" default_freezer: df_memory_object_data_return should not be called\n");
	return KERN_SUCCESS;
}

kern_return_t
df_memory_object_data_initialize(
		__unused memory_object_t mem_obj,
		__unused  memory_object_offset_t offset,
		__unused memory_object_cluster_size_t size)
{
	
	panic(" default_freezer: df_memory_object_data_initialize should not be called\n");
	return KERN_SUCCESS;
}

kern_return_t
df_memory_object_data_unlock(
		__unused memory_object_t mem_obj,
		__unused memory_object_offset_t offset,
		__unused memory_object_size_t length,
		__unused vm_prot_t prot)
{

	panic(" default_freezer: df_memory_object_data_unlock should not be called\n");
	return KERN_FAILURE;
}

kern_return_t
df_memory_object_synchronize(
		__unused memory_object_t mem_obj,
		__unused memory_object_offset_t offset,
		__unused memory_object_size_t length,
		__unused vm_sync_t flags)
{

	panic(" default_freezer: df_memory_object_synchronize should not be called\n");
	return KERN_FAILURE;
}

kern_return_t
df_memory_object_map(
		__unused memory_object_t mem_obj,
		__unused vm_prot_t prot)
{

	panic(" default_freezer: df_memory_object_map should not be called\n");
	return KERN_FAILURE;
}

kern_return_t
df_memory_object_last_unmap(__unused memory_object_t mem_obj)
{

	panic(" default_freezer: df_memory_object_last_unmap should not be called\n");
	return KERN_FAILURE;
}


kern_return_t
df_memory_object_data_reclaim(
		__unused memory_object_t mem_obj,
		__unused boolean_t reclaim_backing_store)
{

	panic("df_memory_object_data_reclaim\n");
	return KERN_SUCCESS;
}
#endif /* CONFIG_FREEZE */
Commit	Line	Data
6d2010ae A	1	/*
	2	* Copyright (c) 2000-2010 Apple Inc. All rights reserved.
	3	*
	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
	5	*
	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
	14	*
	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
	25	*
	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
	27	*/
	28
	29	#if CONFIG_FREEZE
	30
	31	#include "default_freezer.h"
	32
	33	/*
	34	* Indicates that a page has been faulted back in.
	35	*/
	36	#define FREEZER_OFFSET_ABSENT ((vm_object_offset_t)(-1))
	37
	38	/*
	39	* Create the mapping table that will
	40	* tell us the object/offset pair that
	41	* corresponds to the page being sent
	42	* out or being brought back in.
	43	*/
	44
	45	void*
	46	default_freezer_mapping_create(vm_object_t object, vm_offset_t offset)
	47	{
	48	default_freezer_mapping_table_t table;
	49
	50	table = kalloc(sizeof(struct default_freezer_mapping_table));
	51	if (table) {
	52	memset(table, 0, sizeof(*table));
	53	} else {
	54	panic("Could not allocate mapping table\n");
	55	}
	56
	57	table->object = object;
	58	table->offset = offset;
	59
	60	return (void*)table;
	61	}
	62
	63	void
	64	default_freezer_mapping_free(void **table, boolean_t all)
65	{
66	default_freezer_mapping_table_t freezer_table = ((default_freezer_mapping_table_t )table);
67	assert(freezer_table);
68
69	if (all) {
70	do {
71	default_freezer_mapping_table_t next = freezer_table->next;
72	kfree(freezer_table, sizeof(*freezer_table));
73	freezer_table = next;
74	} while (freezer_table);
75	} else {
76	kfree(freezer_table, sizeof(*freezer_table));
77	}
78	}
79
80	kern_return_t
81	default_freezer_mapping_store(
82	default_freezer_mapping_table_t *table,
83	memory_object_offset_t table_offset,
84	memory_object_t memory_object,
85	memory_object_offset_t offset)
86	{
87	default_freezer_mapping_table_entry_t entry;
88	uint32_t index;
89
90	assert(*table);
91
92	if ((*table)->index >= MAX_FREEZE_TABLE_ENTRIES) {
93	vm_object_t compact_object = (*table)->object;
94	default_freezer_mapping_table_t next;
95
96	next = default_freezer_mapping_create(compact_object, table_offset);
97	if (!next) {
98	return KERN_FAILURE;
99	}
100
101	(*table)->next = next;
102	*table = next;
103	}
104
105	index = (*table)->index++;
106	entry = &(*table)->entry[index];
107
108	entry->memory_object = memory_object;
109	entry->offset = offset;
110
111	return KERN_SUCCESS;
112	}
113
114	kern_return_t
115	default_freezer_mapping_update(
116	default_freezer_mapping_table_t table,
117	memory_object_t memory_object,
118	memory_object_offset_t offset,
119	memory_object_offset_t table_offset, /OUT: contains the offset into the compact object*/
120	boolean_t remove_entry)
121	{
122
123	kern_return_t kr = KERN_SUCCESS;
124	vm_object_offset_t compact_offset;
125	default_freezer_mapping_table_entry_t entry;
126	uint32_t index = 0;
127
128	if (table == NULL){
129	return KERN_FAILURE;
130	}
131
132	compact_offset = table->offset;
133
134	while (1) {
135	if (index >= table->index) {
136	if (table->next) {
137	table = table->next;
138	index = 0;
139	} else {
140	/* End of tables and we didn't find our candidate entry */
141	kr = KERN_FAILURE;
142	break;
143	}
144	}
145
146	entry = &table->entry[index];
147
148	if (memory_object == entry->memory_object && offset == entry->offset) {
149	if (remove_entry == TRUE) {
150	/*
151	* Mark the page absent whilst retaining the object
152	* for cleanup during thaw.
153	*/
154	entry->offset = FREEZER_OFFSET_ABSENT;
155	}
156	if (table_offset != NULL) {
157	*table_offset = compact_offset;
158	}
159	break;
160	}
161
162	index++;
163	compact_offset += PAGE_SIZE;
164	}
165	return kr;
166	}
167
168	/*
169	* Create a freezer memory object for this
170	* vm object.
171	*/
172	void
173	default_freezer_memory_object_create(
174	vm_object_t object,
175	vm_object_t compact_object,
176	default_freezer_mapping_table_t table)
177	{
178
179	default_freezer_memory_object_t fo = NULL;
180
181	fo = kalloc(sizeof(struct default_freezer_memory_object));
182
183	if (fo) {
184	memory_object_control_t control = NULL;
185
186	memset(fo, 0, sizeof(*fo));
187
188	control = memory_object_control_allocate(object);
189	assert (control != MEMORY_OBJECT_CONTROL_NULL);
190
191	df_memory_object_init((memory_object_t)fo, control, 0);
192	fo->fo_compact_object = compact_object;
193	fo->fo_table = table;
194
195	object->pager = (memory_object_t)fo;
196	object->pager_created = TRUE;
197	object->pager_initialized = TRUE;
198	object->pager_ready = TRUE;
199	object->pager_trusted = TRUE;
200	object->pager_control = control;
201	} else {
202	panic(" Could not allocate freezer object\n");
203	}
204	}
205
206	void
207	default_freezer_pack_page(
208	vm_page_t p,
209	vm_object_t compact_object,
210	vm_object_offset_t offset,
211	void **table)
212	{
213
214	default_freezer_mapping_table_t freeze_table = (default_freezer_mapping_table_t )table;
215	memory_object_t memory_object = p->object->pager;
216
217	if (memory_object == NULL) {
218	default_freezer_memory_object_create(p->object, compact_object, *freeze_table);
219	memory_object = p->object->pager;
220	} else {
221	default_freezer_memory_object_t fo = (default_freezer_memory_object_t)memory_object;
222	if (fo->fo_compact_object == VM_OBJECT_NULL) {
223	fo->fo_compact_object = compact_object;
224	fo->fo_table = *freeze_table;
225	}
226	}
227
228	default_freezer_mapping_store(freeze_table, offset, memory_object, p->offset + p->object->paging_offset);
229
230	/* Remove from the original and insert into the compact destination object */
231	vm_page_rename(p, compact_object, offset, FALSE);
232	}
233
234	void
235	default_freezer_unpack(
236	vm_object_t object,
237	void **table)
238	{
239
240	vm_page_t p = VM_PAGE_NULL;
241	uint32_t index = 0;
242	vm_object_t src_object = VM_OBJECT_NULL;
243	memory_object_t src_mem_object = MEMORY_OBJECT_NULL;
244	memory_object_offset_t src_offset = 0;
245	vm_object_offset_t compact_offset = 0;
246	default_freezer_memory_object_t fo = NULL;
247	default_freezer_memory_object_t last_memory_object_thawed = NULL;
248	default_freezer_mapping_table_t freeze_table = (default_freezer_mapping_table_t )table;
249
250	assert(freeze_table);
251
252	vm_object_lock(object);
253
254	for (index = 0, compact_offset = 0; ; index++, compact_offset += PAGE_SIZE){
255	if (index >= freeze_table->index) {
256	default_freezer_mapping_table_t table_next;
257
258	table_next = freeze_table->next;
259
260	/* Free the tables as we go along */
261	default_freezer_mapping_free((void**)&freeze_table, FALSE);
262
263	if (table_next == NULL){
264	break;
265	}
266
267	freeze_table = table_next;
268	index = 0;
269	}
270
271	/*
272	* Skip slots that represent deallocated memory objects.
273	*/
274	src_mem_object = freeze_table->entry[index].memory_object;
275	if (src_mem_object == MEMORY_OBJECT_NULL)
276	continue;
277
278	/*
279	* Skip slots that represent faulted pages.
280	*/
281	src_offset = freeze_table->entry[index].offset;
282	if (src_offset != FREEZER_OFFSET_ABSENT) {
283
284	p = vm_page_lookup(object, compact_offset);
285	assert(p);
286
287	fo = (default_freezer_memory_object_t)src_mem_object;
288
289	src_object = memory_object_control_to_vm_object(fo->fo_pager_control);
290
291	/* Move back over from the freeze object to the original */
292	vm_object_lock(src_object);
293	vm_page_rename(p, src_object, src_offset - src_object->paging_offset, FALSE);
294	vm_object_unlock(src_object);
295	}
296
297	if (src_mem_object != ((memory_object_t)last_memory_object_thawed)){
298	if (last_memory_object_thawed != NULL){
299	last_memory_object_thawed->fo_compact_object = VM_OBJECT_NULL;
300	last_memory_object_thawed->fo_table = NULL;
301	}
302	last_memory_object_thawed = (default_freezer_memory_object_t)src_mem_object;
303	}
304	}
305
306	if (last_memory_object_thawed != NULL){
307	last_memory_object_thawed->fo_compact_object = VM_OBJECT_NULL;
308	last_memory_object_thawed->fo_table = NULL;
309	}
310
311	vm_object_unlock(object);
312	}
313
314	vm_object_t
315	default_freezer_get_compact_vm_object(void** table)
316	{
317	default_freezer_mapping_table_t freeze_table = ((default_freezer_mapping_table_t )table);
318	assert(freeze_table);
319	return ((vm_object_t)(freeze_table->object));
320	}
321
322	void
323	df_memory_object_reference(__unused memory_object_t mem_obj)
324	{
325
326	/* No-op */
327	}
328
329	void
330	df_memory_object_deallocate(memory_object_t mem_obj)
331	{
332
333	default_freezer_memory_object_t fo = (default_freezer_memory_object_t)mem_obj;
334	vm_object_t compact_object = fo->fo_compact_object;
335
336	assert(fo);
337
338	if (compact_object != VM_OBJECT_NULL) {
339
340	default_freezer_mapping_table_t fo_table = fo->fo_table;
341	default_freezer_mapping_table_entry_t entry;
342	boolean_t found = FALSE;
343	uint32_t index = 0;
344
345	vm_object_lock(compact_object);
346
347	/* Remove from table */
348	while (1) {
349	if (index >= fo_table->index) {
350	if (fo_table->next) {
351	fo_table = fo_table->next;
352	index = 0;
353	} else {
354	/* End of tables */
355	break;
356	}
357	}
358
359	entry = &fo_table->entry[index];
360	if (mem_obj == entry->memory_object) {
361	/* It matches, so clear the entry */
362	if (!found) {
363	found = TRUE;
364	}
365	entry->memory_object = MEMORY_OBJECT_NULL;
366	entry->offset = 0;
367	} else if (MEMORY_OBJECT_NULL != entry->memory_object) {
368	/* We have a different valid object; we're done */
369	if (found) {
370	break;
371	}
372	}
373
374	index++;
375	}
376
377	vm_object_unlock(compact_object);
378	}
379
380	kfree(fo, sizeof(*fo));
381	}
382
383	kern_return_t
384	df_memory_object_init(
385	memory_object_t mem_obj,
386	memory_object_control_t control,
387	__unused memory_object_cluster_size_t pager_page_size)
388	{
389
390	default_freezer_memory_object_t fo = (default_freezer_memory_object_t)mem_obj;
391	assert(fo);
392
393	fo->fo_pager_ops = &default_freezer_ops;
394	fo->fo_pager_header.io_bits = IKOT_MEMORY_OBJECT;
395	fo->fo_pager_control = control;
396
397	return KERN_SUCCESS;
398	}
399
400	kern_return_t
401	df_memory_object_terminate(memory_object_t mem_obj)
402	{
403
404	default_freezer_memory_object_t fo = (default_freezer_memory_object_t)mem_obj;
405	assert(fo);
406	memory_object_control_deallocate(fo->fo_pager_control);
407	return KERN_SUCCESS;
408	}
409
410	kern_return_t
411	df_memory_object_data_request(
412	memory_object_t mem_obj,
413	memory_object_offset_t offset,
414	memory_object_cluster_size_t length,
415	vm_prot_t protection_required,
416	memory_object_fault_info_t fault_info)
417	{
418
419	vm_object_t src_object = VM_OBJECT_NULL, compact_object = VM_OBJECT_NULL;
420	memory_object_offset_t compact_offset = 0;
421	memory_object_t pager = NULL;
422	kern_return_t kr = KERN_SUCCESS;
423
424	default_freezer_memory_object_t fo = (default_freezer_memory_object_t)mem_obj;
425
426	src_object = memory_object_control_to_vm_object(fo->fo_pager_control);
427	compact_object = fo->fo_compact_object;
428
429	if (compact_object != VM_OBJECT_NULL) {
430
431	vm_object_lock(compact_object);
432
433	kr = default_freezer_mapping_update(fo->fo_table,
434	mem_obj,
435	offset,
436	&compact_offset,
437	FALSE);
438
439	vm_object_unlock(compact_object);
440	} else {
441	kr = KERN_FAILURE;
442	}
443
444	if (length == 0){
445	/Caller is just querying to see if we have the page/
446	return kr;
447	}
448
449	if (kr != KERN_SUCCESS){
450
451	unsigned int request_flags;
452	upl_t upl;
453	unsigned int page_list_count = 0;
454
455	request_flags = UPL_NO_SYNC \| UPL_RET_ONLY_ABSENT \| UPL_SET_LITE;
456	/*
457	* Should we decide to activate USE_PRECIOUS (from default_pager_internal.h)
458	* here, then the request_flags will need to add these to the ones above:
459	*
460	* request_flags \|= UPL_PRECIOUS \| UPL_CLEAN_IN_PLACE
461	*/
462	request_flags \|= UPL_REQUEST_SET_DIRTY;
463
464	memory_object_super_upl_request(fo->fo_pager_control,
465	(memory_object_offset_t)offset,
466	PAGE_SIZE, PAGE_SIZE,
467	&upl, NULL, &page_list_count,
468	request_flags);
469
470	upl_abort(upl, UPL_ABORT_UNAVAILABLE);
471	upl_deallocate(upl);
472
473	return KERN_SUCCESS;
474	}
475
476	vm_object_lock(compact_object);
477
478	pager = (memory_object_t)compact_object->pager;
479
480	if (!compact_object->pager_ready \|\| pager == MEMORY_OBJECT_NULL){
481	vm_object_unlock(compact_object);
482	return KERN_FAILURE;
483	}
484
485	vm_object_paging_wait(compact_object, THREAD_UNINT);
486	vm_object_paging_begin(compact_object);
487
488	compact_object->blocked_access = TRUE;
489	vm_object_unlock(compact_object);
490
491	((vm_object_fault_info_t) fault_info)->io_sync = TRUE;
492
493	kr = dp_memory_object_data_request(pager,
494	compact_offset,
495	length,
496	protection_required,
497	fault_info);
498	if (kr == KERN_SUCCESS){
499
500	vm_page_t src_page = VM_PAGE_NULL, dst_page = VM_PAGE_NULL;
501
502	vm_object_lock(compact_object);
503
504	compact_object->blocked_access = FALSE;
505	vm_object_paging_end(compact_object);
506
507	vm_object_lock(src_object);
508
509	if ((src_page = vm_page_lookup(compact_object, compact_offset)) != VM_PAGE_NULL){
510
511	dst_page = vm_page_lookup(src_object, offset - src_object->paging_offset);
512
513	VM_PAGE_FREE(dst_page);
514	vm_page_rename(src_page, src_object, offset - src_object->paging_offset, FALSE);
515
516	if (default_freezer_mapping_update(fo->fo_table,
517	mem_obj,
518	offset,
519	NULL,
520	TRUE) != KERN_SUCCESS) {
521	printf("Page for object: 0x%lx at offset: 0x%lx not found in table\n", (uintptr_t)src_object, (uintptr_t)offset);
522	}
523
524	PAGE_WAKEUP_DONE(src_page);
525	} else {
526	printf("%d: default_freezer: compact_object doesn't have the page for object 0x%lx at offset 0x%lx \n", kr, (uintptr_t)compact_object, (uintptr_t)compact_offset);
527	kr = KERN_FAILURE;
528	}
529	vm_object_unlock(src_object);
530	vm_object_unlock(compact_object);
531	} else {
532	panic("%d: default_freezer TOC pointed us to default_pager incorrectly\n", kr);
533	}
534	return kr;
535	}
536
537	kern_return_t
538	df_memory_object_data_return(
539	__unused memory_object_t mem_obj,
540	__unused memory_object_offset_t offset,
541	__unused memory_object_cluster_size_t size,
542	__unused memory_object_offset_t *resid_offset,
543	__unused int *io_error,
544	__unused boolean_t dirty,
545	__unused boolean_t kernel_copy,
546	__unused int upl_flags)
547	{
548
549	panic(" default_freezer: df_memory_object_data_return should not be called\n");
550	return KERN_SUCCESS;
551	}
552
553	kern_return_t
554	df_memory_object_data_initialize(
555	__unused memory_object_t mem_obj,
556	__unused memory_object_offset_t offset,
557	__unused memory_object_cluster_size_t size)
558	{
559
560	panic(" default_freezer: df_memory_object_data_initialize should not be called\n");
561	return KERN_SUCCESS;
562	}
563
564	kern_return_t
565	df_memory_object_data_unlock(
566	__unused memory_object_t mem_obj,
567	__unused memory_object_offset_t offset,
568	__unused memory_object_size_t length,
569	__unused vm_prot_t prot)
570	{
571
572	panic(" default_freezer: df_memory_object_data_unlock should not be called\n");
573	return KERN_FAILURE;
574	}
575
576	kern_return_t
577	df_memory_object_synchronize(
578	__unused memory_object_t mem_obj,
579	__unused memory_object_offset_t offset,
580	__unused memory_object_size_t length,
581	__unused vm_sync_t flags)
582	{
583
584	panic(" default_freezer: df_memory_object_synchronize should not be called\n");
585	return KERN_FAILURE;
586	}
587
588	kern_return_t
589	df_memory_object_map(
590	__unused memory_object_t mem_obj,
591	__unused vm_prot_t prot)
592	{
593
594	panic(" default_freezer: df_memory_object_map should not be called\n");
595	return KERN_FAILURE;
596	}
597
598	kern_return_t
599	df_memory_object_last_unmap(__unused memory_object_t mem_obj)
600	{
601
602	panic(" default_freezer: df_memory_object_last_unmap should not be called\n");
603	return KERN_FAILURE;
604	}
605
606
607	kern_return_t
608	df_memory_object_data_reclaim(
609	__unused memory_object_t mem_obj,
610	__unused boolean_t reclaim_backing_store)
611	{
612
613	panic("df_memory_object_data_reclaim\n");
614	return KERN_SUCCESS;
615	}
616	#endif /* CONFIG_FREEZE */