/*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
*
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
*
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License"). You may not use this file except in compliance with the
- * License. Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
*
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ *
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
*
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
*/
/*
* @OSF_COPYRIGHT@
#define MACH_PORT_FACE mach_port_t
+#if 0
#ifndef USE_PRECIOUS
#define USE_PRECIOUS TRUE
#endif
+#endif
#ifdef USER_PAGER
#define UP(stuff) stuff
#define UP(stuff)
#endif /* USER_PAGER */
-extern int norma_mk; /* is the kernel configured with NORMA ? */
-
#ifndef MACH_KERNEL
extern struct mutex dprintf_lock;
#define PRINTF_LOCK_INIT() mutex_init(&dprintf_lock)
#define dprintf(args) \
do { \
PRINTF_LOCK(); \
- printf("%s[%d]%s: ", my_name, dp_thread_id(), here); \
+ printf("%s[%d]: ", my_name, dp_thread_id()); \
printf args; \
PRINTF_UNLOCK(); \
} while (0)
#else
#define dprintf(args) \
do { \
- printf("%s[KERNEL:]%s: ", my_name, here); \
+ printf("%s[KERNEL]: ", my_name); \
printf args; \
} while (0)
#endif
/*
* Debug.
*/
-extern char my_name[];
+__private_extern__ char my_name[];
#define DEFAULT_PAGER_DEBUG 0
#define DEBUG_BS_EXTERNAL 0x01000000
#define DEBUG_BS_INTERNAL 0x02000000
-#define DEBUG(level, args) \
+#define DP_DEBUG(level, args) \
do { \
if (debug_mask & (level)) \
dprintf(args); \
my_name, dp_thread_id(), here, \
__FILE__, __LINE__, # expr); \
#else
- panic("%s[KERNEL]%s: assertion failed in %s line %d: %s",\
- my_name, here, __FILE__, __LINE__, # expr); \
+ panic("%s[KERNEL]: assertion failed in %s line %d: %s",\
+ my_name, __FILE__, __LINE__, # expr); \
#endif
} while (0)
#else /* DEFAULT_PAGER_DEBUG */
-#define DEBUG(level, args) here[0] = here[0]
-#define ASSERT(clause) here[0] = here[0]
+#define DP_DEBUG(level, args)
+#define ASSERT(clause)
#endif /* DEFAULT_PAGER_DEBUG */
*/
#ifdef MACH_KERNEL
#define vm_page_size page_size
-extern vm_size_t page_size;
#else
extern vm_object_size_t vm_page_size;
#endif
#endif
#define howmany(a,b) (((a) + (b) - 1)/(b))
+extern memory_object_default_t default_pager_object;
+
#ifdef MACH_KERNEL
extern mutex_t dpt_lock; /* Lock for the dpt array */
-extern unsigned int default_pager_internal_count;
+extern int default_pager_internal_count;
extern MACH_PORT_FACE default_pager_host_port;
/* extern task_t default_pager_self; */ /* dont need or want */
extern MACH_PORT_FACE default_pager_internal_set;
extern MACH_PORT_FACE default_pager_external_set;
-extern MACH_PORT_FACE default_pager_default_port;
extern MACH_PORT_FACE default_pager_default_set;
#else
extern mach_port_t default_pager_host_port;
extern task_port_t default_pager_self;
extern mach_port_t default_pager_internal_set;
extern mach_port_t default_pager_external_set;
-extern mach_port_t default_pager_default_port;
extern mach_port_t default_pager_default_set;
#endif
#define BS_STAT(bs, clause) VSTATS_ACTION(&(bs)->bs_lock, (clause))
#ifdef MACH_KERNEL
-#define BS_LOCK_INIT(bs) mutex_init(&(bs)->bs_lock, ETAP_DPAGE_BS)
+#define BS_LOCK_INIT(bs) mutex_init(&(bs)->bs_lock, 0)
#else
#define BS_LOCK_INIT(bs) mutex_init(&(bs)->bs_lock)
#endif
#endif
};
extern struct backing_store_list_head backing_store_list;
+extern int backing_store_release_trigger_disable;
#ifdef MACH_KERNEL
-#define BSL_LOCK_INIT() mutex_init(&backing_store_list.bsl_lock, ETAP_DPAGE_BSL)
+#define BSL_LOCK_INIT() mutex_init(&backing_store_list.bsl_lock, 0)
#else
#define BSL_LOCK_INIT() mutex_init(&backing_store_list.bsl_lock)
#endif
unsigned int ps_ncls; /* Number of clusters in segment */
unsigned int ps_clcount; /* Number of free clusters */
unsigned int ps_pgcount; /* Number of free pages */
- long ps_hint; /* Hint of where to look next. */
+ unsigned long ps_hint; /* Hint of where to look next. */
/* bitmap */
#ifdef MACH_KERNEL
#define PAGING_SEGMENT_NULL ((paging_segment_t) 0)
#ifdef MACH_KERNEL
-#define PS_LOCK_INIT(ps) mutex_init(&(ps)->ps_lock, ETAP_DPAGE_SEGMENT)
+#define PS_LOCK_INIT(ps) mutex_init(&(ps)->ps_lock, 0)
#else
#define PS_LOCK_INIT(ps) mutex_init(&(ps)->ps_lock)
#endif
extern int ps_select_array[DEFAULT_PAGER_BACKING_STORE_MAXPRI+1];
#ifdef MACH_KERNEL
-#define PSL_LOCK_INIT() mutex_init(&paging_segments_lock, ETAP_DPAGE_SEGLIST)
+#define PSL_LOCK_INIT() mutex_init(&paging_segments_lock, 0)
#else
#define PSL_LOCK_INIT() mutex_init(&paging_segments_lock)
#endif
* Exported macros for manipulating the vs_map structure --
* checking status, getting and setting bits.
*/
-#define VSCLSIZE(vs) (1 << (vs)->vs_clshift)
+#define VSCLSIZE(vs) (1UL << (vs)->vs_clshift)
#define VSM_ISCLR(vsm) (((vsm).vsmap_entry == VSM_ENTRY_NULL) && \
((vsm).vsmap_error == 0))
#define VSM_ISERR(vsm) ((vsm).vsmap_error)
(clm)->cl_alloc.clb_map >>= (VSCLSIZE(vs) - (clm)->cl_numpages)
typedef struct vstruct_alias {
- vm_offset_t name;
+ memory_object_pager_ops_t name;
struct vstruct *vs;
} vstruct_alias_t;
-/*
- * VM Object Structure: This is the structure used to manage pagers associated
- * to VM objects.Mapping between pager port and paging object.
- */
-
-typedef struct vstruct {
- queue_chain_t vs_links; /* Link in pager-port list */
#ifdef MACH_KERNEL
- hw_lock_data_t vs_lock; /* Lock for the structure */
+#define DPT_LOCK_INIT(lock) mutex_init(&(lock), 0)
+#define DPT_LOCK(lock) mutex_lock(&(lock))
+#define DPT_UNLOCK(lock) mutex_unlock(&(lock))
+#define DPT_SLEEP(lock, e, i) thread_sleep_mutex(&(lock), (event_t)(e), i)
+#define VS_LOCK_TYPE hw_lock_data_t
+#define VS_LOCK_INIT(vs) hw_lock_init(&(vs)->vs_lock)
+#define VS_TRY_LOCK(vs) (VS_LOCK(vs),TRUE)
+#define VS_LOCK(vs) hw_lock_lock(&(vs)->vs_lock)
+#define VS_UNLOCK(vs) hw_lock_unlock(&(vs)->vs_lock)
+#define VS_MAP_LOCK_TYPE mutex_t
+#define VS_MAP_LOCK_INIT(vs) mutex_init(&(vs)->vs_map_lock, 0)
+#define VS_MAP_LOCK(vs) mutex_lock(&(vs)->vs_map_lock)
+#define VS_MAP_TRY_LOCK(vs) mutex_try(&(vs)->vs_map_lock)
+#define VS_MAP_UNLOCK(vs) mutex_unlock(&(vs)->vs_map_lock)
#else
- struct mutex vs_lock; /* Lock for the structure */
+#define VS_LOCK_TYPE struct mutex
+#define VS_LOCK_INIT(vs) mutex_init(&(vs)->vs_lock, 0)
+#define VS_TRY_LOCK(vs) mutex_try(&(vs)->vs_lock)
+#define VS_LOCK(vs) mutex_lock(&(vs)->vs_lock)
+#define VS_UNLOCK(vs) mutex_unlock(&(vs)->vs_lock)
+#define VS_MAP_LOCK_TYPE struct mutex
+#define VS_MAP_LOCK_INIT(vs) mutex_init(&(vs)->vs_map_lock)
+#define VS_MAP_LOCK(vs) mutex_lock(&(vs)->vs_map_lock)
+#define VS_MAP_TRY_LOCK(vs) mutex_try(&(vs)->vs_map_lock)
+#define VS_MAP_UNLOCK(vs) mutex_unlock(&(vs)->vs_map_lock)
#endif
- MACH_PORT_FACE vs_mem_obj_port; /* Memory object port */
- mach_port_seqno_t vs_next_seqno; /* next sequence number to issue */
- mach_port_seqno_t vs_seqno; /* Pager port sequence number */
- MACH_PORT_FACE vs_control_port;/* Memory object's control port */
- mach_port_urefs_t vs_control_refs; /* Mem object's control port refs */
- MACH_PORT_FACE vs_object_name; /* Name port */
- mach_port_urefs_t vs_name_refs; /* Name port user-refs */
+
+
+/*
+ * VM Object Structure: This is the structure used to manage
+ * default pager object associations with their control counter-
+ * parts (VM objects).
+ *
+ * The start of this structure MUST match a "struct memory_object".
+ */
+typedef struct vstruct {
+ memory_object_pager_ops_t vs_pager_ops; /* == &default_pager_ops */
+ int vs_mem_obj_ikot;/* JMM:fake ip_kotype() */
+ memory_object_control_t vs_control; /* our mem obj control ref */
+ VS_LOCK_TYPE vs_lock; /* data for the lock */
+
+ /* JMM - Could combine these first two in a single pending count now */
+ unsigned int vs_next_seqno; /* next sequence num to issue */
+ unsigned int vs_seqno; /* Pager port sequence number */
+ unsigned int vs_readers; /* Reads in progress */
+ unsigned int vs_writers; /* Writes in progress */
#ifdef MACH_KERNEL
- boolean_t vs_waiting_seqno;/* to wait on seqno */
- boolean_t vs_waiting_read; /* to wait on readers */
- boolean_t vs_waiting_write;/* to wait on writers */
- boolean_t vs_waiting_refs; /* to wait on refs */
- boolean_t vs_waiting_async;/* to wait on async_pending */
+ int
+ /* boolean_t */ vs_waiting_seqno:1, /* to wait on seqno */
+ /* boolean_t */ vs_waiting_read:1, /* waiting on reader? */
+ /* boolean_t */ vs_waiting_write:1, /* waiting on writer? */
+ /* boolean_t */ vs_waiting_async:1, /* waiting on async? */
+ /* boolean_t */ vs_indirect:1, /* map indirect? */
+ /* boolean_t */ vs_xfer_pending:1; /* xfer out of seg? */
#else
- event_t vs_waiting_seqno;/* to wait on seqno */
- event_t vs_waiting_read; /* to wait on readers */
- event_t vs_waiting_write;/* to wait on writers */
- event_t vs_waiting_refs; /* to wait on refs */
- event_t vs_waiting_async;/* to wait on async_pending */
+ event_t vs_waiting_seqno;/* to wait on seqno */
+ event_t vs_waiting_read; /* to wait on readers */
+ event_t vs_waiting_write;/* to wait on writers */
+ event_t vs_waiting_async;/* to wait on async_pending */
+ int vs_indirect:1, /* Is the map indirect ? */
+ vs_xfer_pending:1; /* xfering out of a seg ? */
#endif
- unsigned int vs_readers; /* Reads in progress */
- unsigned int vs_writers; /* Writes in progress */
- unsigned int vs_errors; /* Pageout error count */
+ unsigned int vs_async_pending;/* pending async write count */
+ unsigned int vs_errors; /* Pageout error count */
+ unsigned int vs_references; /* references */
+
+ queue_chain_t vs_links; /* Link in pager-wide list */
- int vs_clshift; /* Bit shift: clusters to pages */
- int vs_size; /* Object size in clusters */
- int vs_indirect:1, /* Is the map indirect ? */
- vs_xfer_pending:1; /* xfering out of a seg ? */
- int vs_async_pending; /* Count of pending async writes */
+ unsigned int vs_clshift; /* Bit shift: clusters->pages */
+ unsigned int vs_size; /* Object size in clusters */
#ifdef MACH_KERNEL
mutex_t vs_map_lock; /* to protect map below */
#else
#define vs_dmap vs_un.vsu_dmap
#define vs_imap vs_un.vsu_imap
-#define MEM_OBJ_CTL(vs) ((vs)->vs_control_port)
#define VSTRUCT_NULL ((vstruct_t) 0)
-#ifdef MACH_KERNEL
-#define DPT_LOCK_INIT(lock) mutex_init(&(lock), ETAP_DPAGE_VSTRUCT)
-#define DPT_LOCK(lock) mutex_lock(&(lock))
-#define DPT_UNLOCK(lock) mutex_unlock(&(lock))
-#define VS_LOCK_INIT(vs) hw_lock_init(&(vs)->vs_lock)
-#define VS_TRY_LOCK(vs) (VS_LOCK(vs),TRUE)
-#define VS_LOCK(vs) hw_lock_lock(&(vs)->vs_lock)
-#define VS_UNLOCK(vs) hw_lock_unlock(&(vs)->vs_lock)
-#else
-#define VS_LOCK_INIT(vs) mutex_init(&(vs)->vs_lock, ETAP_DPAGE_VSTRUCT)
-#define VS_TRY_LOCK(vs) mutex_try_lock(&(vs)->vs_lock)
-#define VS_LOCK(vs) mutex_lock(&(vs)->vs_lock)
-#define VS_UNLOCK(vs) mutex_unlock(&(vs)->vs_lock)
-#endif
-
-#ifdef MACH_KERNEL
-#define VS_MAP_LOCK_INIT(vs) mutex_init(&(vs)->vs_map_lock, ETAP_DPAGE_VSMAP)
-#else
-#define VS_MAP_LOCK_INIT(vs) mutex_init(&(vs)->vs_map_lock)
-#endif
-#define VS_MAP_LOCK(vs) mutex_lock(&(vs)->vs_map_lock)
-#ifndef MACH_KERNEL
-#define VS_MAP_TRY_LOCK(vs) mutex_try_lock(&(vs)->vs_map_lock)
-#else
-#define VS_MAP_TRY_LOCK(vs) mutex_try(&(vs)->vs_map_lock)
-#endif
-#define VS_MAP_UNLOCK(vs) mutex_unlock(&(vs)->vs_map_lock)
+__private_extern__ void vs_async_wait(vstruct_t);
+
+#if PARALLEL
+__private_extern__ void vs_lock(vstruct_t);
+__private_extern__ void vs_unlock(vstruct_t);
+__private_extern__ void vs_start_read(vstruct_t);
+__private_extern__ void vs_finish_read(vstruct_t);
+__private_extern__ void vs_wait_for_readers(vstruct_t);
+__private_extern__ void vs_start_write(vstruct_t);
+__private_extern__ void vs_finish_write(vstruct_t);
+__private_extern__ void vs_wait_for_writers(vstruct_t);
+__private_extern__ void vs_wait_for_sync_writers(vstruct_t);
+#else /* PARALLEL */
+#define vs_lock(vs)
+#define vs_unlock(vs)
+#define vs_start_read(vs)
+#define vs_wait_for_readers(vs)
+#define vs_finish_read(vs)
+#define vs_start_write(vs)
+#define vs_wait_for_writers(vs)
+#define vs_wait_for_sync_writers(vs)
+#define vs_finish_write(vs)
+#endif /* PARALLEL */
/*
* Data structures and variables dealing with asynchronous
struct mutex vsl_lock;
#endif
int vsl_count; /* saves code */
- queue_head_t vsl_leak_queue;
};
-extern struct vstruct_list_head vstruct_list;
+
+__private_extern__ struct vstruct_list_head vstruct_list;
+
+__private_extern__ void vstruct_list_insert(vstruct_t vs);
+__private_extern__ void vstruct_list_delete(vstruct_t vs);
+
#ifdef MACH_KERNEL
-#define VSL_LOCK_INIT() mutex_init(&vstruct_list.vsl_lock, ETAP_DPAGE_VSLIST)
+#define VSL_LOCK_INIT() mutex_init(&vstruct_list.vsl_lock, 0)
#else
#define VSL_LOCK_INIT() mutex_init(&vstruct_list.vsl_lock)
#endif
#define VSL_LOCK() mutex_lock(&vstruct_list.vsl_lock)
#define VSL_LOCK_TRY() mutex_try(&vstruct_list.vsl_lock)
#define VSL_UNLOCK() mutex_unlock(&vstruct_list.vsl_lock)
+#define VSL_SLEEP(e,i) thread_sleep_mutex((e), &vstruct_list.vsl_lock, (i))
+
+#ifdef MACH_KERNEL
+__private_extern__ zone_t vstruct_zone;
+#endif
/*
* Create port alias for vstruct address.
* check.
*/
#ifdef MACH_KERNEL
-#define ISVS 123456
-#define port_is_vs(_port_) \
- ((((struct vstruct_alias *)((_port_)->alias)) != NULL) && \
- (((struct vstruct_alias *)(_port_)->alias)->name==ISVS))
-#define port_to_vs(_port_) \
- ((struct vstruct_alias *)(_port_)->alias)->vs
-#define vs_to_port(_vs_) (_vs_->vs_mem_obj_port)
-#define vs_lookup(_port_, _vs_) \
+
+extern const struct memory_object_pager_ops default_pager_ops;
+
+#define mem_obj_is_vs(_mem_obj_) \
+ (((_mem_obj_) != NULL) && \
+ ((_mem_obj_)->mo_pager_ops == &default_pager_ops))
+#define mem_obj_to_vs(_mem_obj_) \
+ ((vstruct_t)(_mem_obj_))
+#define vs_to_mem_obj(_vs_) ((memory_object_t)(_vs_))
+#define vs_lookup(_mem_obj_, _vs_) \
do { \
- if ((((struct vstruct_alias *)(_port_)->alias) == NULL) || \
- (((struct vstruct_alias *)(_port_)->alias)->name!=ISVS)) \
- panic("bad pager port"); \
- _vs_ = port_to_vs(_port_); \
+ if (!mem_obj_is_vs(_mem_obj_)) \
+ panic("bad dp memory object"); \
+ _vs_ = mem_obj_to_vs(_mem_obj_); \
+ } while (0)
+#define vs_lookup_safe(_mem_obj_, _vs_) \
+ do { \
+ if (!mem_obj_is_vs(_mem_obj_)) \
+ _vs_ = VSTRUCT_NULL; \
+ else \
+ _vs_ = mem_obj_to_vs(_mem_obj_); \
} while (0)
#else
+
#define vs_to_port(_vs_) (((vm_offset_t)(_vs_))+1)
#define port_to_vs(_port_) ((vstruct_t)(((vm_offset_t)(_port_))&~3))
#define port_is_vs(_port_) ((((vm_offset_t)(_port_))&3) == 1)
#define vs_lookup(_port_, _vs_) \
do { \
if (!MACH_PORT_VALID(_port_) || !port_is_vs(_port_) \
- || port_to_vs(_port_)->vs_mem_obj_port != (_port_)) \
+ || port_to_vs(_port_)->vs_mem_obj != (_port_)) \
Panic("bad pager port"); \
_vs_ = port_to_vs(_port_); \
} while (0)
extern boolean_t device_reply_server(mach_msg_header_t *,
mach_msg_header_t *);
#ifdef MACH_KERNEL
-extern void default_pager_no_senders(MACH_PORT_FACE,
- mach_port_seqno_t,
+extern boolean_t default_pager_no_senders(memory_object_t,
mach_port_mscount_t);
#else
extern void default_pager_no_senders(memory_object_t,
mach_port_seqno_t,
mach_port_mscount_t);
#endif
+
extern int local_log2(unsigned int);
extern void bs_initialize(void);
extern void bs_global_info(vm_size_t *,
vm_size_t);
extern kern_return_t vs_cluster_write(vstruct_t,
upl_t,
- vm_offset_t,
- vm_size_t,
+ upl_offset_t,
+ upl_size_t,
boolean_t,
int);
extern vm_offset_t ps_clmap(vstruct_t,
extern boolean_t verbose;
+extern thread_call_t default_pager_backing_store_monitor_callout;
+extern void default_pager_backing_store_monitor(thread_call_param_t, thread_call_param_t);
+
+extern ipc_port_t max_pages_trigger_port;
+extern unsigned int dp_pages_free;
+extern unsigned int maximum_pages_free;
+
+/* Do we know yet if swap files need to be encrypted ? */
+extern boolean_t dp_encryption_inited;
+/* Should we encrypt data before writing to swap ? */
+extern boolean_t dp_encryption;
+
#endif /* _DEFAULT_PAGER_INTERNAL_H_ */
projects / apple / xnu.git / blobdiff