[apple/xnu.git] / osfmk / i386 / read_fault.c

/*
 * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_LICENSE_HEADER_START@
 * 
 * The contents of this file constitute Original Code as defined in and
 * are subject to the Apple Public Source License Version 1.1 (the
 * "License").  You may not use this file except in compliance with the
 * License.  Please obtain a copy of the License at
 * http://www.apple.com/publicsource and read it before using this file.
 * 
 * This Original Code and all software distributed under the License are
 * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
 * License for the specific language governing rights and limitations
 * under the License.
 * 
 * @APPLE_LICENSE_HEADER_END@
 */
/*
 * @OSF_COPYRIGHT@
 */
/* 
 * Mach Operating System
 * Copyright (c) 1991,1990 Carnegie Mellon University
 * All Rights Reserved.
 * 
 * Permission to use, copy, modify and distribute this software and its
 * documentation is hereby granted, provided that both the copyright
 * notice and this permission notice appear in all copies of the
 * software, derivative works or modified versions, and any portions
 * thereof, and that both notices appear in supporting documentation.
 * 
 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
 * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
 * 
 * Carnegie Mellon requests users of this software to return to
 * 
 *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
 *  School of Computer Science
 *  Carnegie Mellon University
 *  Pittsburgh PA 15213-3890
 * 
 * any improvements or extensions that they make and grant Carnegie Mellon
 * the rights to redistribute these changes.
 */
/*
 */

#include <kern/thread.h>
#include <vm/vm_fault.h>
#include <mach/kern_return.h>
#include <mach/vm_behavior.h>
#include <vm/vm_map.h>
#include <vm/vm_object.h>
#include <vm/vm_page.h>
#include <vm/pmap.h>

#include <i386/intel_read_fault.h>

#include <kern/macro_help.h>

/*
 *	Expansion of vm_fault for read fault in kernel mode.
 *	Must enter the mapping as writable, since the i386
 *	(and i860 in i386 compatability mode) ignores write
 *	protection in kernel mode.
 *
 *	Note that this routine can be called for pmap's other
 *	than the kernel_pmap, in which case it just enters
 *	a read-only mapping.  (See e.g. kernel_trap().)
 */
kern_return_t
intel_read_fault(
	vm_map_t	map,
	vm_offset_t	vaddr)
{
	vm_map_version_t	version;	/* Map version for
						   verification */
	vm_object_t		object;		/* Top-level object */
	vm_object_offset_t	offset;		/* Top-level offset */
	vm_prot_t		prot;		/* Protection for mapping */
	vm_behavior_t		behavior;	/* Expected paging behavior */
	vm_object_offset_t	lo_offset, hi_offset;
	vm_page_t		result_page;	/* Result of vm_fault_page */
	vm_page_t		top_page;	/* Placeholder page */
	boolean_t		wired;		/* Is map region wired? */
	kern_return_t		result;
	register vm_page_t	m;
	vm_map_t		pmap_map;
	vm_map_t                original_map = map;
	thread_t                cur_thread;
	boolean_t               funnel_set;
	funnel_t                *curflock;

	cur_thread = current_thread();

	if ((cur_thread->funnel_state & TH_FN_OWNED) == TH_FN_OWNED) {
		funnel_set = TRUE;
		curflock = cur_thread->funnel_lock;
		thread_funnel_set( curflock , FALSE);
	} else {
		funnel_set = FALSE;
	}

    RetryFault:

	map = original_map;

	/*
	 *	Find the backing store object and offset into it
	 *	to begin search.
	 */
	vm_map_lock_read(map);
	result = vm_map_lookup_locked(&map, vaddr, VM_PROT_READ, &version,
				      &object, &offset, &prot, &wired,
				      &behavior, &lo_offset, 
				      &hi_offset, &pmap_map);
	
	vm_map_unlock_read(map);

	if (result != KERN_SUCCESS) {
		if (funnel_set)
			thread_funnel_set( curflock, TRUE);
		return (result);
	}

	if(pmap_map != map) {
		vm_map_reference(pmap_map);
		vm_map_unlock_read(pmap_map);
	}

	/*
	 *	Make a reference to this object to prevent its
	 *	disposal while we are playing with it.
	 */
	assert(object->ref_count > 0);
	object->ref_count++;
	vm_object_res_reference(object);
	vm_object_paging_begin(object);

	result = vm_fault_page(object, offset, VM_PROT_READ, FALSE,
			       THREAD_ABORTSAFE,
			       lo_offset, hi_offset, behavior,
			       &prot, &result_page, &top_page, (int *)0,
			       0, map->no_zero_fill, FALSE, map, vaddr);

	if (result != VM_FAULT_SUCCESS) {
	    vm_object_deallocate(object);
	    if(pmap_map != map) {
			vm_map_deallocate(pmap_map);
	   }

	    switch (result) {
		case VM_FAULT_RETRY:
		    goto RetryFault;
		case VM_FAULT_INTERRUPTED:
			if (funnel_set)
				thread_funnel_set( curflock, TRUE);
		    return (KERN_SUCCESS);
		case VM_FAULT_MEMORY_SHORTAGE:
		    VM_PAGE_WAIT();
		    goto RetryFault;
		case VM_FAULT_FICTITIOUS_SHORTAGE:
		    vm_page_more_fictitious();
		    goto RetryFault;
		case VM_FAULT_MEMORY_ERROR:
		    return (KERN_MEMORY_ERROR);
	    }
	}

	m = result_page;

	/*
	 *	How to clean up the result of vm_fault_page.  This
	 *	happens whether the mapping is entered or not.
	 */

#define UNLOCK_AND_DEALLOCATE				\
	MACRO_BEGIN					\
	vm_fault_cleanup(m->object, top_page);		\
	vm_object_deallocate(object);			\
	MACRO_END

	/*
	 *	What to do with the resulting page from vm_fault_page
	 *	if it doesn't get entered into the physical map:
	 */

#define RELEASE_PAGE(m)					\
	MACRO_BEGIN					\
	PAGE_WAKEUP_DONE(m);				\
	vm_page_lock_queues();				\
	if (!m->active && !m->inactive)			\
		vm_page_activate(m);			\
	vm_page_unlock_queues();			\
	MACRO_END

	/*
	 *	We must verify that the maps have not changed.
	 */
	vm_object_unlock(m->object);

	if ((map != original_map) || !vm_map_verify(map, &version)) {
	    vm_object_t		retry_object;
	    vm_object_offset_t	retry_offset;
	    vm_prot_t		retry_prot;

		if (map != pmap_map) {
			vm_map_deallocate(pmap_map);
		}
	    
		map = original_map;
		vm_map_lock_read(map);

	    result = vm_map_lookup_locked(&map, vaddr, VM_PROT_READ, &version,
				&retry_object, &retry_offset, &retry_prot,
				&wired, &behavior, &lo_offset, 
				&hi_offset, &pmap_map);

	    if (result != KERN_SUCCESS) {
	        vm_map_unlock_read(map);
			vm_object_lock(m->object);
			RELEASE_PAGE(m);
			UNLOCK_AND_DEALLOCATE;
			if (funnel_set)
				thread_funnel_set( curflock, TRUE);
			return (result);
	    }

		if (map != pmap_map) {
			vm_map_reference(pmap_map);
		}

	    vm_object_unlock(retry_object);

	    if (retry_object != object || retry_offset != offset) {
			vm_object_lock(m->object);
			RELEASE_PAGE(m);
	        vm_map_unlock_read(map);
	        if(pmap_map != map) {
		   		vm_map_unlock_read(pmap_map);
		   		vm_map_deallocate(pmap_map);
			}
			UNLOCK_AND_DEALLOCATE;
			goto RetryFault;
	    }
	}

	/*
	 *	Put the page in the physical map.
	 */

	PMAP_ENTER(pmap_map->pmap, vaddr, m, VM_PROT_READ, PMAP_DEFAULT_CACHE, wired);

	if(pmap_map != map) {
		vm_map_unlock_read(pmap_map);
		vm_map_deallocate(pmap_map);
	}
	
	vm_object_lock(m->object);
	vm_page_lock_queues();
	if (!m->active && !m->inactive)
		vm_page_activate(m);
	m->reference = TRUE;
	vm_page_unlock_queues();

	vm_map_verify_done(map, &version);
	PAGE_WAKEUP_DONE(m);

	UNLOCK_AND_DEALLOCATE;

#undef	UNLOCK_AND_DEALLOCATE
#undef	RELEASE_PAGE
	if (funnel_set)
		thread_funnel_set( curflock, TRUE);
	return (KERN_SUCCESS);
}
Commit	Line	Data
1c79356b A	1	/*
	2	* Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
	3	*
	4	* @APPLE_LICENSE_HEADER_START@
	5	*
	6	* The contents of this file constitute Original Code as defined in and
	7	* are subject to the Apple Public Source License Version 1.1 (the
	8	* "License"). You may not use this file except in compliance with the
	9	* License. Please obtain a copy of the License at
	10	* http://www.apple.com/publicsource and read it before using this file.
	11	*
	12	* This Original Code and all software distributed under the License are
	13	* distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	14	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	15	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	16	* FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
	17	* License for the specific language governing rights and limitations
	18	* under the License.
	19	*
	20	* @APPLE_LICENSE_HEADER_END@
	21	*/
	22	/*
	23	* @OSF_COPYRIGHT@
	24	*/
	25	/*
	26	* Mach Operating System
	27	* Copyright (c) 1991,1990 Carnegie Mellon University
	28	* All Rights Reserved.
	29	*
	30	* Permission to use, copy, modify and distribute this software and its
	31	* documentation is hereby granted, provided that both the copyright
	32	* notice and this permission notice appear in all copies of the
	33	* software, derivative works or modified versions, and any portions
	34	* thereof, and that both notices appear in supporting documentation.
	35	*
	36	* CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
	37	* CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
	38	* ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
	39	*
	40	* Carnegie Mellon requests users of this software to return to
	41	*
	42	* Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
	43	* School of Computer Science
	44	* Carnegie Mellon University
	45	* Pittsburgh PA 15213-3890
	46	*
	47	* any improvements or extensions that they make and grant Carnegie Mellon
	48	* the rights to redistribute these changes.
	49	*/
	50	/*
	51	*/
	52
	53	#include <kern/thread.h>
	54	#include <vm/vm_fault.h>
	55	#include <mach/kern_return.h>
	56	#include <mach/vm_behavior.h>
	57	#include <vm/vm_map.h>
	58	#include <vm/vm_object.h>
	59	#include <vm/vm_page.h>
	60	#include <vm/pmap.h>
	61
	62	#include <i386/intel_read_fault.h>
	63
	64	#include <kern/macro_help.h>
65
66	/*
67	* Expansion of vm_fault for read fault in kernel mode.
68	* Must enter the mapping as writable, since the i386
69	* (and i860 in i386 compatability mode) ignores write
70	* protection in kernel mode.
71	*
72	* Note that this routine can be called for pmap's other
73	* than the kernel_pmap, in which case it just enters
74	* a read-only mapping. (See e.g. kernel_trap().)
75	*/
76	kern_return_t
77	intel_read_fault(
78	vm_map_t map,
79	vm_offset_t vaddr)
80	{
81	vm_map_version_t version; /* Map version for
82	verification */
83	vm_object_t object; /* Top-level object */
84	vm_object_offset_t offset; /* Top-level offset */
85	vm_prot_t prot; /* Protection for mapping */
86	vm_behavior_t behavior; /* Expected paging behavior */
87	vm_object_offset_t lo_offset, hi_offset;
88	vm_page_t result_page; /* Result of vm_fault_page */
89	vm_page_t top_page; /* Placeholder page */
90	boolean_t wired; /* Is map region wired? */
91	kern_return_t result;
92	register vm_page_t m;
93	vm_map_t pmap_map;
94	vm_map_t original_map = map;
9bccf70c A	95	thread_t cur_thread;
	96	boolean_t funnel_set;
	97	funnel_t *curflock;
	98
	99	cur_thread = current_thread();
	100
	101	if ((cur_thread->funnel_state & TH_FN_OWNED) == TH_FN_OWNED) {
	102	funnel_set = TRUE;
	103	curflock = cur_thread->funnel_lock;
	104	thread_funnel_set( curflock , FALSE);
	105	} else {
	106	funnel_set = FALSE;
	107	}
1c79356b A	108
	109	RetryFault:
	110
	111	map = original_map;
	112
	113	/*
	114	* Find the backing store object and offset into it
	115	* to begin search.
	116	*/
	117	vm_map_lock_read(map);
	118	result = vm_map_lookup_locked(&map, vaddr, VM_PROT_READ, &version,
	119	&object, &offset, &prot, &wired,
	120	&behavior, &lo_offset,
	121	&hi_offset, &pmap_map);
	122
	123	vm_map_unlock_read(map);
	124
	125	if (result != KERN_SUCCESS) {
9bccf70c A	126	if (funnel_set)
	127	thread_funnel_set( curflock, TRUE);
	128	return (result);
1c79356b A	129	}
	130
	131	if(pmap_map != map) {
	132	vm_map_reference(pmap_map);
	133	vm_map_unlock_read(pmap_map);
	134	}
	135
	136	/*
	137	* Make a reference to this object to prevent its
	138	* disposal while we are playing with it.
	139	*/
	140	assert(object->ref_count > 0);
	141	object->ref_count++;
	142	vm_object_res_reference(object);
	143	vm_object_paging_begin(object);
	144
	145	result = vm_fault_page(object, offset, VM_PROT_READ, FALSE,
	146	THREAD_ABORTSAFE,
	147	lo_offset, hi_offset, behavior,
	148	&prot, &result_page, &top_page, (int *)0,
0b4e3aa0	149	0, map->no_zero_fill, FALSE, map, vaddr);
1c79356b A	150
	151	if (result != VM_FAULT_SUCCESS) {
	152	vm_object_deallocate(object);
	153	if(pmap_map != map) {
	154	vm_map_deallocate(pmap_map);
	155	}
	156
	157	switch (result) {
	158	case VM_FAULT_RETRY:
	159	goto RetryFault;
	160	case VM_FAULT_INTERRUPTED:
9bccf70c A	161	if (funnel_set)
9bccf70c A	162	thread_funnel_set( curflock, TRUE);
1c79356b A	163	return (KERN_SUCCESS);
	164	case VM_FAULT_MEMORY_SHORTAGE:
	165	VM_PAGE_WAIT();
	166	goto RetryFault;
	167	case VM_FAULT_FICTITIOUS_SHORTAGE:
	168	vm_page_more_fictitious();
	169	goto RetryFault;
	170	case VM_FAULT_MEMORY_ERROR:
	171	return (KERN_MEMORY_ERROR);
	172	}
	173	}
	174
	175	m = result_page;
	176
	177	/*
	178	* How to clean up the result of vm_fault_page. This
	179	* happens whether the mapping is entered or not.
	180	*/
	181
	182	#define UNLOCK_AND_DEALLOCATE \
	183	MACRO_BEGIN \
	184	vm_fault_cleanup(m->object, top_page); \
	185	vm_object_deallocate(object); \
	186	MACRO_END
	187
	188	/*
	189	* What to do with the resulting page from vm_fault_page
	190	* if it doesn't get entered into the physical map:
	191	*/
	192
	193	#define RELEASE_PAGE(m) \
	194	MACRO_BEGIN \
	195	PAGE_WAKEUP_DONE(m); \
	196	vm_page_lock_queues(); \
	197	if (!m->active && !m->inactive) \
	198	vm_page_activate(m); \
	199	vm_page_unlock_queues(); \
	200	MACRO_END
	201
	202	/*
	203	* We must verify that the maps have not changed.
	204	*/
	205	vm_object_unlock(m->object);
	206
	207	if ((map != original_map) \|\| !vm_map_verify(map, &version)) {
	208	vm_object_t retry_object;
	209	vm_object_offset_t retry_offset;
	210	vm_prot_t retry_prot;
	211
	212	if (map != pmap_map) {
	213	vm_map_deallocate(pmap_map);
	214	}
	215
	216	map = original_map;
	217	vm_map_lock_read(map);
	218
	219	result = vm_map_lookup_locked(&map, vaddr, VM_PROT_READ, &version,
	220	&retry_object, &retry_offset, &retry_prot,
	221	&wired, &behavior, &lo_offset,
	222	&hi_offset, &pmap_map);
	223
	224	if (result != KERN_SUCCESS) {
	225	vm_map_unlock_read(map);
	226	vm_object_lock(m->object);
227	RELEASE_PAGE(m);
228	UNLOCK_AND_DEALLOCATE;
9bccf70c A	229	if (funnel_set)
9bccf70c A	230	thread_funnel_set( curflock, TRUE);
1c79356b A	231	return (result);
	232	}
	233
	234	if (map != pmap_map) {
	235	vm_map_reference(pmap_map);
	236	}
	237
	238	vm_object_unlock(retry_object);
	239
	240	if (retry_object != object \|\| retry_offset != offset) {
	241	vm_object_lock(m->object);
	242	RELEASE_PAGE(m);
	243	vm_map_unlock_read(map);
	244	if(pmap_map != map) {
	245	vm_map_unlock_read(pmap_map);
	246	vm_map_deallocate(pmap_map);
	247	}
	248	UNLOCK_AND_DEALLOCATE;
	249	goto RetryFault;
	250	}
	251	}
	252
	253	/*
	254	* Put the page in the physical map.
	255	*/
	256
9bccf70c	257	PMAP_ENTER(pmap_map->pmap, vaddr, m, VM_PROT_READ, PMAP_DEFAULT_CACHE, wired);
1c79356b A	258
	259	if(pmap_map != map) {
	260	vm_map_unlock_read(pmap_map);
	261	vm_map_deallocate(pmap_map);
	262	}
	263
	264	vm_object_lock(m->object);
	265	vm_page_lock_queues();
	266	if (!m->active && !m->inactive)
	267	vm_page_activate(m);
	268	m->reference = TRUE;
	269	vm_page_unlock_queues();
	270
	271	vm_map_verify_done(map, &version);
	272	PAGE_WAKEUP_DONE(m);
	273
	274	UNLOCK_AND_DEALLOCATE;
	275
	276	#undef UNLOCK_AND_DEALLOCATE
	277	#undef RELEASE_PAGE
9bccf70c A	278	if (funnel_set)
9bccf70c A	279	thread_funnel_set( curflock, TRUE);
1c79356b A	280	return (KERN_SUCCESS);
	281	}
	282