/*
- * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
+ * Copyright (c) 2000-2009 Apple Inc. All rights reserved.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
*
#include <kern/host_statistics.h>
#include <kern/machine.h>
#include <kern/misc_protos.h>
+#include <kern/sched.h>
#include <kern/thread.h>
#include <kern/xpr.h>
#include <kern/kalloc.h>
#include <machine/vm_tuning.h>
+#include <machine/commpage.h>
#if CONFIG_EMBEDDED
#include <sys/kern_memorystatus.h>
* ENCRYPTED SWAP:
*/
#include <../bsd/crypto/aes/aes.h>
+extern u_int32_t random(void); /* from <libkern/libkern.h> */
+#if UPL_DEBUG
+#include <libkern/OSDebug.h>
+#endif
#ifndef VM_PAGEOUT_BURST_ACTIVE_THROTTLE /* maximum iterations of the active queue to move pages to inactive */
-#ifdef CONFIG_EMBEDDED
-#define VM_PAGEOUT_BURST_ACTIVE_THROTTLE 2048
-#else
#define VM_PAGEOUT_BURST_ACTIVE_THROTTLE 100
#endif
-#endif
#ifndef VM_PAGEOUT_BURST_INACTIVE_THROTTLE /* maximum iterations of the inactive queue w/o stealing/cleaning a page */
#ifdef CONFIG_EMBEDDED
#ifndef VM_PAGE_FREE_RESERVED
#define VM_PAGE_FREE_RESERVED(n) \
- ((6 * VM_PAGE_LAUNDRY_MAX) + (n))
+ ((unsigned) (6 * VM_PAGE_LAUNDRY_MAX) + (n))
#endif /* VM_PAGE_FREE_RESERVED */
/*
#define VM_PAGEOUT_INACTIVE_FORCE_RECLAIM 100
-/*
- * must hold the page queues lock to
- * manipulate this structure
- */
-struct vm_pageout_queue {
- queue_head_t pgo_pending; /* laundry pages to be processed by pager's iothread */
- unsigned int pgo_laundry; /* current count of laundry pages on queue or in flight */
- unsigned int pgo_maxlaundry;
-
- unsigned int pgo_idle:1, /* iothread is blocked waiting for work to do */
- pgo_busy:1, /* iothread is currently processing request from pgo_pending */
- pgo_throttled:1,/* vm_pageout_scan thread needs a wakeup when pgo_laundry drops */
- :0;
-};
-
-#define VM_PAGE_Q_THROTTLED(q) \
- ((q)->pgo_laundry >= (q)->pgo_maxlaundry)
-
-
/*
* Exported variable used to broadcast the activation of the pageout scan
* Working Set uses this to throttle its use of pmap removes. In this
static void vm_pageout_iothread_continue(struct vm_pageout_queue *);
static void vm_pageout_iothread_external(void);
static void vm_pageout_iothread_internal(void);
-static void vm_pageout_queue_steal(vm_page_t);
extern void vm_pageout_continue(void);
extern void vm_pageout_scan(void);
*/
unsigned int vm_accellerate_zf_pageout_trigger = 400;
unsigned int zf_queue_min_count = 100;
-unsigned int vm_zf_count = 0;
unsigned int vm_zf_queue_count = 0;
+#if defined(__ppc__) /* On ppc, vm statistics are still 32-bit */
+unsigned int vm_zf_count = 0;
+#else
+uint64_t vm_zf_count __attribute__((aligned(8))) = 0;
+#endif
+
/*
* These variables record the pageout daemon's actions:
* how many pages it looks at and what happens to those pages.
unsigned int vm_pageout_inactive_used = 0; /* debugging */
unsigned int vm_pageout_inactive_clean = 0; /* debugging */
unsigned int vm_pageout_inactive_dirty = 0; /* debugging */
+unsigned int vm_pageout_inactive_deactivated = 0; /* debugging */
+unsigned int vm_pageout_inactive_zf = 0; /* debugging */
unsigned int vm_pageout_dirty_no_pager = 0; /* debugging */
unsigned int vm_pageout_purged_objects = 0; /* debugging */
unsigned int vm_stat_discard = 0; /* debugging */
unsigned int vm_pageout_scan_active_throttled = 0;
unsigned int vm_pageout_scan_inactive_throttled = 0;
unsigned int vm_pageout_scan_throttle = 0; /* debugging */
+unsigned int vm_pageout_scan_throttle_aborted = 0; /* debugging */
unsigned int vm_pageout_scan_burst_throttle = 0; /* debugging */
unsigned int vm_pageout_scan_empty_throttle = 0; /* debugging */
unsigned int vm_pageout_scan_deadlock_detected = 0; /* debugging */
unsigned int vm_pageout_scan_active_throttle_success = 0; /* debugging */
unsigned int vm_pageout_scan_inactive_throttle_success = 0; /* debugging */
+
+unsigned int vm_page_speculative_count_drifts = 0;
+unsigned int vm_page_speculative_count_drift_max = 0;
+
/*
* Backing store throttle when BS is exhausted
*/
unsigned int vm_pageout_out_of_line = 0;
unsigned int vm_pageout_in_place = 0;
+unsigned int vm_page_steal_pageout_page = 0;
+
/*
* ENCRYPTED SWAP:
* counters and statistics...
vm_object_t vm_pageout_scan_wants_object = VM_OBJECT_NULL;
+boolean_t (* volatile consider_buffer_cache_collect)(int) = NULL;
+
+#if DEVELOPMENT || DEBUG
unsigned long vm_cs_validated_resets = 0;
+#endif
/*
* Routine: vm_backing_store_disable
if (m->dirty) {
CLUSTER_STAT(vm_pageout_target_page_dirtied++;)
- vm_page_unwire(m);/* reactivates */
+ vm_page_unwire(m, TRUE); /* reactivates */
VM_STAT_INCR(reactivations);
PAGE_WAKEUP_DONE(m);
} else {
/* alternate request page list, write to page_list */
/* case. Occurs when the original page was wired */
/* at the time of the list request */
- assert(m->wire_count != 0);
- vm_page_unwire(m);/* reactivates */
+ assert(VM_PAGE_WIRED(m));
+ vm_page_unwire(m, TRUE); /* reactivates */
m->overwriting = FALSE;
} else {
/*
/*
* Account for the paging reference taken in vm_paging_object_allocate.
*/
- vm_object_paging_end(shadow_object);
+ vm_object_activity_end(shadow_object);
vm_object_unlock(shadow_object);
assert(object->ref_count == 0);
assert(object->paging_in_progress == 0);
+ assert(object->activity_in_progress == 0);
assert(object->resident_page_count == 0);
return;
}
* necessarily flushed from the VM page cache.
* This is accomplished by cleaning in place.
*
- * The page must not be busy, and the object and page
- * queues must be locked.
- *
+ * The page must not be busy, and new_object
+ * must be locked.
+ *
*/
void
vm_pageclean_setup(
XPR(XPR_VM_PAGEOUT,
"vm_pageclean_setup, obj 0x%X off 0x%X page 0x%X new 0x%X new_off 0x%X\n",
- (integer_t)m->object, m->offset, (integer_t)m,
- (integer_t)new_m, new_offset);
+ m->object, m->offset, m,
+ new_m, new_offset);
pmap_clear_modify(m->phys_page);
new_m->private = TRUE;
new_m->pageout = TRUE;
new_m->phys_page = m->phys_page;
+
+ vm_page_lockspin_queues();
vm_page_wire(new_m);
+ vm_page_unlock_queues();
vm_page_insert(new_m, new_object, new_offset);
assert(!new_m->wanted);
XPR(XPR_VM_PAGEOUT,
"vm_pageout_initialize_page, page 0x%X\n",
- (integer_t)m, 0, 0, 0, 0);
+ m, 0, 0, 0, 0);
assert(m->busy);
/*
/* set the page for future call to vm_fault_list_request */
vm_object_paging_begin(object);
holding_page = NULL;
- vm_page_lock_queues();
+
pmap_clear_modify(m->phys_page);
m->dirty = TRUE;
m->busy = TRUE;
m->list_req_pending = TRUE;
m->cleaning = TRUE;
m->pageout = TRUE;
+
+ vm_page_lockspin_queues();
vm_page_wire(m);
vm_page_unlock_queues();
+
vm_object_unlock(object);
/*
XPR(XPR_VM_PAGEOUT,
"vm_pageout_cluster, object 0x%X offset 0x%X page 0x%X\n",
- (integer_t)object, m->offset, (integer_t)m, 0, 0);
+ object, m->offset, m, 0, 0);
+
+ VM_PAGE_CHECK(m);
/*
* Only a certain kind of page is appreciated here.
*/
- assert(m->busy && (m->dirty || m->precious) && (m->wire_count == 0));
+ assert(m->busy && (m->dirty || m->precious) && (!VM_PAGE_WIRED(m)));
assert(!m->cleaning && !m->pageout && !m->inactive && !m->active);
assert(!m->throttled);
m->list_req_pending = TRUE;
m->cleaning = TRUE;
m->pageout = TRUE;
- m->laundry = TRUE;
if (object->internal == TRUE)
q = &vm_pageout_queue_internal;
else
q = &vm_pageout_queue_external;
+
+ /*
+ * pgo_laundry count is tied to the laundry bit
+ */
+ m->laundry = TRUE;
q->pgo_laundry++;
m->pageout_queue = TRUE;
q->pgo_idle = FALSE;
thread_wakeup((event_t) &q->pgo_pending);
}
+
+ VM_PAGE_CHECK(m);
}
unsigned long vm_pageout_throttle_up_count = 0;
/*
- * A page is back from laundry. See if there are some pages waiting to
+ * A page is back from laundry or we are stealing it back from
+ * the laundering state. See if there are some pages waiting to
* go to laundry and if we can let some of them go now.
*
* Object and page queues must be locked.
{
struct vm_pageout_queue *q;
- vm_pageout_throttle_up_count++;
-
- assert(m->laundry);
assert(m->object != VM_OBJECT_NULL);
assert(m->object != kernel_object);
+ vm_pageout_throttle_up_count++;
+
if (m->object->internal == TRUE)
q = &vm_pageout_queue_internal;
else
q = &vm_pageout_queue_external;
- m->laundry = FALSE;
- q->pgo_laundry--;
+ if (m->pageout_queue == TRUE) {
- if (q->pgo_throttled == TRUE) {
- q->pgo_throttled = FALSE;
- thread_wakeup((event_t) &q->pgo_laundry);
+ queue_remove(&q->pgo_pending, m, vm_page_t, pageq);
+ m->pageout_queue = FALSE;
+
+ m->pageq.next = NULL;
+ m->pageq.prev = NULL;
+
+ vm_object_paging_end(m->object);
+ }
+ if (m->laundry == TRUE) {
+ m->laundry = FALSE;
+ q->pgo_laundry--;
+
+ if (q->pgo_throttled == TRUE) {
+ q->pgo_throttled = FALSE;
+ thread_wakeup((event_t) &q->pgo_laundry);
+ }
+ if (q->pgo_draining == TRUE && q->pgo_laundry == 0) {
+ q->pgo_draining = FALSE;
+ thread_wakeup((event_t) (&q->pgo_laundry+1));
+ }
}
}
mach_timespec_t ts;
};
+
+/*
+ * VM memory pressure monitoring.
+ *
+ * vm_pageout_scan() keeps track of the number of pages it considers and
+ * reclaims, in the currently active vm_pageout_stat[vm_pageout_stat_now].
+ *
+ * compute_memory_pressure() is called every second from compute_averages()
+ * and moves "vm_pageout_stat_now" forward, to start accumulating the number
+ * of recalimed pages in a new vm_pageout_stat[] bucket.
+ *
+ * mach_vm_pressure_monitor() collects past statistics about memory pressure.
+ * The caller provides the number of seconds ("nsecs") worth of statistics
+ * it wants, up to 30 seconds.
+ * It computes the number of pages reclaimed in the past "nsecs" seconds and
+ * also returns the number of pages the system still needs to reclaim at this
+ * moment in time.
+ */
+#define VM_PAGEOUT_STAT_SIZE 31
+struct vm_pageout_stat {
+ unsigned int considered;
+ unsigned int reclaimed;
+} vm_pageout_stats[VM_PAGEOUT_STAT_SIZE] = {{0,0}, };
+unsigned int vm_pageout_stat_now = 0;
+unsigned int vm_memory_pressure = 0;
+
+#define VM_PAGEOUT_STAT_BEFORE(i) \
+ (((i) == 0) ? VM_PAGEOUT_STAT_SIZE - 1 : (i) - 1)
+#define VM_PAGEOUT_STAT_AFTER(i) \
+ (((i) == VM_PAGEOUT_STAT_SIZE - 1) ? 0 : (i) + 1)
+
+/*
+ * Called from compute_averages().
+ */
+void
+compute_memory_pressure(
+ __unused void *arg)
+{
+ unsigned int vm_pageout_next;
+
+ vm_memory_pressure =
+ vm_pageout_stats[VM_PAGEOUT_STAT_BEFORE(vm_pageout_stat_now)].reclaimed;
+
+ commpage_set_memory_pressure( vm_memory_pressure );
+
+ /* move "now" forward */
+ vm_pageout_next = VM_PAGEOUT_STAT_AFTER(vm_pageout_stat_now);
+ vm_pageout_stats[vm_pageout_next].considered = 0;
+ vm_pageout_stats[vm_pageout_next].reclaimed = 0;
+ vm_pageout_stat_now = vm_pageout_next;
+}
+
+unsigned int
+mach_vm_ctl_page_free_wanted(void)
+{
+ unsigned int page_free_target, page_free_count, page_free_wanted;
+
+ page_free_target = vm_page_free_target;
+ page_free_count = vm_page_free_count;
+ if (page_free_target > page_free_count) {
+ page_free_wanted = page_free_target - page_free_count;
+ } else {
+ page_free_wanted = 0;
+ }
+
+ return page_free_wanted;
+}
+
+kern_return_t
+mach_vm_pressure_monitor(
+ boolean_t wait_for_pressure,
+ unsigned int nsecs_monitored,
+ unsigned int *pages_reclaimed_p,
+ unsigned int *pages_wanted_p)
+{
+ wait_result_t wr;
+ unsigned int vm_pageout_then, vm_pageout_now;
+ unsigned int pages_reclaimed;
+
+ /*
+ * We don't take the vm_page_queue_lock here because we don't want
+ * vm_pressure_monitor() to get in the way of the vm_pageout_scan()
+ * thread when it's trying to reclaim memory. We don't need fully
+ * accurate monitoring anyway...
+ */
+
+ if (wait_for_pressure) {
+ /* wait until there's memory pressure */
+ while (vm_page_free_count >= vm_page_free_target) {
+ wr = assert_wait((event_t) &vm_page_free_wanted,
+ THREAD_INTERRUPTIBLE);
+ if (wr == THREAD_WAITING) {
+ wr = thread_block(THREAD_CONTINUE_NULL);
+ }
+ if (wr == THREAD_INTERRUPTED) {
+ return KERN_ABORTED;
+ }
+ if (wr == THREAD_AWAKENED) {
+ /*
+ * The memory pressure might have already
+ * been relieved but let's not block again
+ * and let's report that there was memory
+ * pressure at some point.
+ */
+ break;
+ }
+ }
+ }
+
+ /* provide the number of pages the system wants to reclaim */
+ if (pages_wanted_p != NULL) {
+ *pages_wanted_p = mach_vm_ctl_page_free_wanted();
+ }
+
+ if (pages_reclaimed_p == NULL) {
+ return KERN_SUCCESS;
+ }
+
+ /* provide number of pages reclaimed in the last "nsecs_monitored" */
+ do {
+ vm_pageout_now = vm_pageout_stat_now;
+ pages_reclaimed = 0;
+ for (vm_pageout_then =
+ VM_PAGEOUT_STAT_BEFORE(vm_pageout_now);
+ vm_pageout_then != vm_pageout_now &&
+ nsecs_monitored-- != 0;
+ vm_pageout_then =
+ VM_PAGEOUT_STAT_BEFORE(vm_pageout_then)) {
+ pages_reclaimed += vm_pageout_stats[vm_pageout_then].reclaimed;
+ }
+ } while (vm_pageout_now != vm_pageout_stat_now);
+ *pages_reclaimed_p = pages_reclaimed;
+
+ return KERN_SUCCESS;
+}
+
+/* Page States: Used below to maintain the page state
+ before it's removed from it's Q. This saved state
+ helps us do the right accounting in certain cases
+*/
+
+#define PAGE_STATE_SPECULATIVE 1
+#define PAGE_STATE_THROTTLED 2
+#define PAGE_STATE_ZEROFILL 3
+#define PAGE_STATE_INACTIVE 4
+
+#define VM_PAGEOUT_SCAN_HANDLE_REUSABLE_PAGE(m) \
+ MACRO_BEGIN \
+ /* \
+ * If a "reusable" page somehow made it back into \
+ * the active queue, it's been re-used and is not \
+ * quite re-usable. \
+ * If the VM object was "all_reusable", consider it \
+ * as "all re-used" instead of converting it to \
+ * "partially re-used", which could be expensive. \
+ */ \
+ if ((m)->reusable || \
+ (m)->object->all_reusable) { \
+ vm_object_reuse_pages((m)->object, \
+ (m)->offset, \
+ (m)->offset + PAGE_SIZE_64, \
+ FALSE); \
+ } \
+ MACRO_END
+
void
vm_pageout_scan(void)
{
vm_page_t local_freeq = NULL;
int local_freed = 0;
int delayed_unlock;
- int need_internal_inactive = 0;
int refmod_state = 0;
int vm_pageout_deadlock_target = 0;
struct vm_pageout_queue *iq;
struct vm_pageout_queue *eq;
struct vm_speculative_age_q *sq;
- struct flow_control flow_control;
+ struct flow_control flow_control = { 0, { 0, 0 } };
boolean_t inactive_throttled = FALSE;
boolean_t try_failed;
mach_timespec_t ts;
unsigned int msecs = 0;
vm_object_t object;
vm_object_t last_object_tried;
- int zf_ratio;
- int zf_run_count;
+#if defined(__ppc__) /* On ppc, vm statistics are still 32-bit */
+ unsigned int zf_ratio;
+ unsigned int zf_run_count;
+#else
+ uint64_t zf_ratio;
+ uint64_t zf_run_count;
+#endif
uint32_t catch_up_count = 0;
uint32_t inactive_reclaim_run;
boolean_t forced_reclaim;
+ int page_prev_state = 0;
flow_control.state = FCS_IDLE;
iq = &vm_pageout_queue_internal;
* but at the moment mach vm cannot do this.
*/
{
- uint32_t total = vm_page_active_count + vm_page_inactive_count;
- uint32_t normal = total - vm_zf_count;
-
+#if defined(__ppc__) /* On ppc, vm statistics are still 32-bit */
+ uint32_t total = vm_page_active_count + vm_page_inactive_count;
+ uint32_t normal = total - vm_zf_count;
+#else
+ uint64_t total = vm_page_active_count + vm_page_inactive_count;
+ uint64_t normal = total - vm_zf_count;
+#endif
+
/* zf_ratio is the number of zf pages we victimize per normal page */
if (vm_zf_count < vm_accellerate_zf_pageout_trigger)
* Don't sweep through active queue more than the throttle
* which should be kept relatively low
*/
- active_burst_count = vm_pageout_burst_active_throttle;
+ active_burst_count = MIN(vm_pageout_burst_active_throttle,
+ vm_page_active_count);
/*
* Move pages from active to inactive.
*/
- if (need_internal_inactive == 0 && (vm_page_inactive_count + vm_page_speculative_count) >= vm_page_inactive_target)
+ if ((vm_page_inactive_count + vm_page_speculative_count) >= vm_page_inactive_target)
goto done_moving_active_pages;
- while (!queue_empty(&vm_page_queue_active) &&
- (need_internal_inactive || active_burst_count)) {
+ while (!queue_empty(&vm_page_queue_active) && active_burst_count) {
if (active_burst_count)
active_burst_count--;
m = (vm_page_t) queue_first(&vm_page_queue_active);
/*
* this is the next object we're going to be interested in
- * try to make sure its available after the mutex_yield
+ * try to make sure it's available after the mutex_yield
* returns control
*/
vm_pageout_scan_wants_object = m->object;
goto done_with_activepage;
}
+ /* deal with a rogue "reusable" page */
+ VM_PAGEOUT_SCAN_HANDLE_REUSABLE_PAGE(m);
+
/*
* Deactivate the page while holding the object
* locked, so we know the page is still not busy.
*/
vm_page_deactivate(m);
- if (need_internal_inactive) {
- vm_pageout_scan_active_throttle_success++;
- need_internal_inactive--;
- }
done_with_activepage:
if (delayed_unlock++ > VM_PAGEOUT_DELAYED_UNLOCK_LIMIT || try_failed == TRUE) {
if (object != NULL) {
+ vm_pageout_scan_wants_object = VM_OBJECT_NULL;
vm_object_unlock(object);
object = NULL;
- vm_pageout_scan_wants_object = VM_OBJECT_NULL;
}
if (local_freeq) {
- vm_page_free_list(local_freeq);
+ vm_page_unlock_queues();
+ vm_page_free_list(local_freeq, TRUE);
local_freeq = NULL;
local_freed = 0;
- }
- mutex_yield(&vm_page_queue_lock);
+ vm_page_lock_queues();
+ } else
+ lck_mtx_yield(&vm_page_queue_lock);
delayed_unlock = 1;
vm_pageout_scan_wants_object = VM_OBJECT_NULL;
if (local_freeq) {
- vm_page_free_list(local_freeq);
+ vm_page_unlock_queues();
+ vm_page_free_list(local_freeq, TRUE);
local_freeq = NULL;
local_freed = 0;
+ vm_page_lock_queues();
}
/*
* inactive target still not met... keep going
* until we get the queues balanced
*/
+
+ /*
+ * Recalculate vm_page_inactivate_target.
+ */
+ vm_page_inactive_target = VM_PAGE_INACTIVE_TARGET(vm_page_active_count +
+ vm_page_inactive_count +
+ vm_page_speculative_count);
+
+#ifndef CONFIG_EMBEDDED
+ /*
+ * XXX: if no active pages can be reclaimed, pageout scan can be stuck trying
+ * to balance the queues
+ */
if (((vm_page_inactive_count + vm_page_speculative_count) < vm_page_inactive_target) &&
!queue_empty(&vm_page_queue_active))
continue;
+#endif
- mutex_lock(&vm_page_queue_free_lock);
+ lck_mtx_lock(&vm_page_queue_free_lock);
if ((vm_page_free_count >= vm_page_free_target) &&
(vm_page_free_wanted == 0) && (vm_page_free_wanted_privileged == 0)) {
return;
}
- mutex_unlock(&vm_page_queue_free_lock);
+ lck_mtx_unlock(&vm_page_queue_free_lock);
}
+
/*
- * Before anything, we check if we have any ripe volatile objects around.
- * If so, purge the first and see what it gives us.
+ * Before anything, we check if we have any ripe volatile
+ * objects around. If so, try to purge the first object.
+ * If the purge fails, fall through to reclaim a page instead.
+ * If the purge succeeds, go back to the top and reevalute
+ * the new memory situation.
*/
assert (available_for_purge>=0);
if (available_for_purge)
vm_object_unlock(object);
object = NULL;
}
- vm_purgeable_object_purge_one();
- continue;
+ if(TRUE == vm_purgeable_object_purge_one()) {
+ continue;
+ }
}
if (queue_empty(&sq->age_q) && vm_page_speculative_count) {
struct vm_speculative_age_q *aq;
mach_timespec_t ts_fully_aged;
boolean_t can_steal = FALSE;
+ int num_scanned_queues;
aq = &vm_page_queue_speculative[speculative_steal_index];
- while (queue_empty(&aq->age_q)) {
+ num_scanned_queues = 0;
+ while (queue_empty(&aq->age_q) &&
+ num_scanned_queues++ != VM_PAGE_MAX_SPECULATIVE_AGE_Q) {
speculative_steal_index++;
aq = &vm_page_queue_speculative[speculative_steal_index];
}
+
+ if (num_scanned_queues ==
+ VM_PAGE_MAX_SPECULATIVE_AGE_Q + 1) {
+ /*
+ * XXX We've scanned all the speculative
+ * queues but still haven't found one
+ * that is not empty, even though
+ * vm_page_speculative_count is not 0.
+ */
+ /* report the anomaly... */
+ printf("vm_pageout_scan: "
+ "all speculative queues empty "
+ "but count=%d. Re-adjusting.\n",
+ vm_page_speculative_count);
+ if (vm_page_speculative_count >
+ vm_page_speculative_count_drift_max)
+ vm_page_speculative_count_drift_max = vm_page_speculative_count;
+ vm_page_speculative_count_drifts++;
+#if 6553678
+ Debugger("vm_pageout_scan: no speculative pages");
+#endif
+ /* readjust... */
+ vm_page_speculative_count = 0;
+ /* ... and continue */
+ continue;
+ }
+
if (vm_page_speculative_count > vm_page_speculative_target)
can_steal = TRUE;
else {
ADD_MACH_TIMESPEC(&ts_fully_aged, &aq->age_ts);
- clock_get_system_nanotime(&ts.tv_sec, (unsigned *)&ts.tv_nsec);
+ clock_sec_t sec;
+ clock_nsec_t nsec;
+ clock_get_system_nanotime(&sec, &nsec);
+ ts.tv_sec = (unsigned int) sec;
+ ts.tv_nsec = nsec;
if (CMP_MACH_TIMESPEC(&ts, &ts_fully_aged) >= 0)
can_steal = TRUE;
msecs = vm_pageout_empty_wait;
goto vm_pageout_scan_delay;
- } else if (inactive_burst_count >= vm_pageout_burst_inactive_throttle) {
+ } else if (inactive_burst_count >=
+ MIN(vm_pageout_burst_inactive_throttle,
+ (vm_page_inactive_count +
+ vm_page_speculative_count))) {
vm_pageout_scan_burst_throttle++;
msecs = vm_pageout_burst_wait;
goto vm_pageout_scan_delay;
} else if (VM_PAGE_Q_THROTTLED(iq) && IP_VALID(memory_manager_default)) {
+ clock_sec_t sec;
+ clock_nsec_t nsec;
switch (flow_control.state) {
reset_deadlock_timer:
ts.tv_sec = vm_pageout_deadlock_wait / 1000;
ts.tv_nsec = (vm_pageout_deadlock_wait % 1000) * 1000 * NSEC_PER_USEC;
- clock_get_system_nanotime(&flow_control.ts.tv_sec,
- (unsigned *)&flow_control.ts.tv_nsec);
+ clock_get_system_nanotime(&sec, &nsec);
+ flow_control.ts.tv_sec = (unsigned int) sec;
+ flow_control.ts.tv_nsec = nsec;
ADD_MACH_TIMESPEC(&flow_control.ts, &ts);
flow_control.state = FCS_DELAYED;
break;
case FCS_DELAYED:
- clock_get_system_nanotime(&ts.tv_sec,
- (unsigned *)&ts.tv_nsec);
+ clock_get_system_nanotime(&sec, &nsec);
+ ts.tv_sec = (unsigned int) sec;
+ ts.tv_nsec = nsec;
if (CMP_MACH_TIMESPEC(&ts, &flow_control.ts) >= 0) {
/*
vm_pageout_scan_wants_object = VM_OBJECT_NULL;
if (local_freeq) {
- vm_page_free_list(local_freeq);
+ vm_page_unlock_queues();
+ vm_page_free_list(local_freeq, TRUE);
local_freeq = NULL;
local_freed = 0;
+ vm_page_lock_queues();
+
+ if (flow_control.state == FCS_DELAYED &&
+ !VM_PAGE_Q_THROTTLED(iq)) {
+ flow_control.state = FCS_IDLE;
+ vm_pageout_scan_throttle_aborted++;
+ goto consider_inactive;
+ }
}
#if CONFIG_EMBEDDED
{
percent_avail =
(vm_page_active_count + vm_page_inactive_count +
vm_page_speculative_count + vm_page_free_count +
- vm_page_purgeable_count ) * 100 /
+ (IP_VALID(memory_manager_default)?0:vm_page_purgeable_count) ) * 100 /
atop_64(max_mem);
if (percent_avail >= (kern_memorystatus_level + 5) ||
percent_avail <= (kern_memorystatus_level - 5)) {
}
#endif
assert_wait_timeout((event_t) &iq->pgo_laundry, THREAD_INTERRUPTIBLE, msecs, 1000*NSEC_PER_USEC);
-
counter(c_vm_pageout_scan_block++);
vm_page_unlock_queues();
assert(vm_pageout_scan_wants_object == VM_OBJECT_NULL);
-
+
thread_block(THREAD_CONTINUE_NULL);
vm_page_lock_queues();
while (1) {
m = NULL;
- /*
- * the most eligible pages are ones that were throttled because the
- * pager wasn't ready at the time. If a pager is ready now,
- * see if one of these is useful.
- */
- if (!VM_PAGE_Q_THROTTLED(iq) && !queue_empty(&vm_page_queue_throttled)) {
- m = (vm_page_t) queue_first(&vm_page_queue_throttled);
- break;
+ if (IP_VALID(memory_manager_default)) {
+ assert(vm_page_throttled_count == 0);
+ assert(queue_empty(&vm_page_queue_throttled));
}
/*
- * The second most eligible pages are ones we paged in speculatively,
+ * The most eligible pages are ones we paged in speculatively,
* but which have not yet been touched.
*/
if ( !queue_empty(&sq->age_q) ) {
assert(m->object != kernel_object);
assert(m->phys_page != vm_page_guard_addr);
+ if (!m->speculative) {
+ vm_pageout_stats[vm_pageout_stat_now].considered++;
+ }
+
DTRACE_VM2(scan, int, 1, (uint64_t *), NULL);
/*
* object are fairly typical on the inactive and active queues
*/
if (!vm_object_lock_try_scan(m->object)) {
+ vm_pageout_inactive_nolock++;
+
+ requeue_page:
/*
* Move page to end and continue.
* Don't re-issue ticket
*/
if (m->zero_fill) {
+ if (m->speculative) {
+ panic("vm_pageout_scan(): page %p speculative and zero-fill !?\n", m);
+ }
+ assert(!m->speculative);
queue_remove(&vm_page_queue_zf, m,
vm_page_t, pageq);
queue_enter(&vm_page_queue_zf, m,
vm_page_speculative_count--;
/*
- * move to the tail of the inactive queue
+ * move to the head of the inactive queue
* to get it out of the way... the speculative
* queue is generally too small to depend
* on there being enough pages from other
* objects to make cycling it back on the
* same queue a winning proposition
*/
- queue_enter(&vm_page_queue_inactive, m,
- vm_page_t, pageq);
+ queue_enter_first(&vm_page_queue_inactive, m,
+ vm_page_t, pageq);
m->inactive = TRUE;
vm_page_inactive_count++;
token_new_pagecount++;
vm_page_t, pageq);
m->throttled = FALSE;
vm_page_throttled_count--;
-
+
/*
* not throttled any more, so can stick
* it on the inactive queue.
#if MACH_ASSERT
vm_page_inactive_count--; /* balance for purgeable queue asserts */
#endif
- vm_purgeable_q_advance_all(1);
+ vm_purgeable_q_advance_all();
queue_enter(&vm_page_queue_inactive, m,
vm_page_t, pageq);
pmap_clear_reference(m->phys_page);
m->reference = FALSE;
- vm_pageout_inactive_nolock++;
-
if ( !queue_empty(&sq->age_q) )
m = (vm_page_t) queue_first(&sq->age_q);
else if ( ((zf_run_count < zf_ratio) && vm_zf_queue_count >= zf_queue_min_count) ||
* pulled from the queue and paged out whenever
* one of its logically adjacent fellows is
* targeted.
- *
- * Pages found on the speculative list can never be
- * in this state... they always have a pager associated
- * with them.
*/
- assert(!m->speculative);
-
- if (m->zero_fill) {
- queue_remove(&vm_page_queue_zf, m,
- vm_page_t, pageq);
- queue_enter(&vm_page_queue_zf, m,
- vm_page_t, pageq);
- } else {
- queue_remove(&vm_page_queue_inactive, m,
- vm_page_t, pageq);
-#if MACH_ASSERT
- vm_page_inactive_count--; /* balance for purgeable queue asserts */
-#endif
- vm_purgeable_q_advance_all(1);
-
- queue_enter(&vm_page_queue_inactive, m,
- vm_page_t, pageq);
-#if MACH_ASSERT
- vm_page_inactive_count++; /* balance for purgeable queue asserts */
-#endif
- token_new_pagecount++;
- }
vm_pageout_inactive_avoid++;
-
- goto done_with_inactivepage;
+ goto requeue_page;
}
/*
* Remove the page from its list.
*/
if (m->speculative) {
remque(&m->pageq);
+ page_prev_state = PAGE_STATE_SPECULATIVE;
m->speculative = FALSE;
vm_page_speculative_count--;
} else if (m->throttled) {
queue_remove(&vm_page_queue_throttled, m, vm_page_t, pageq);
+ page_prev_state = PAGE_STATE_THROTTLED;
m->throttled = FALSE;
vm_page_throttled_count--;
} else {
if (m->zero_fill) {
queue_remove(&vm_page_queue_zf, m, vm_page_t, pageq);
+ page_prev_state = PAGE_STATE_ZEROFILL;
vm_zf_queue_count--;
} else {
+ page_prev_state = PAGE_STATE_INACTIVE;
queue_remove(&vm_page_queue_inactive, m, vm_page_t, pageq);
}
m->inactive = FALSE;
if (!m->fictitious)
vm_page_inactive_count--;
- vm_purgeable_q_advance_all(1);
+ vm_purgeable_q_advance_all();
}
- /* If the object is empty, the page must be reclaimed even if dirty or used. */
- /* If the page belongs to a volatile object, we stick it back on. */
- if (object->copy == VM_OBJECT_NULL) {
- if(object->purgable == VM_PURGABLE_EMPTY && !m->cleaning) {
- m->busy = TRUE;
- if (m->pmapped == TRUE) {
- /* unmap the page */
- refmod_state = pmap_disconnect(m->phys_page);
- if (refmod_state & VM_MEM_MODIFIED) {
- m->dirty = TRUE;
- }
- }
- if (m->dirty || m->precious) {
- /* we saved the cost of cleaning this page ! */
- vm_page_purged_count++;
- }
- goto reclaim_page;
- }
- if (object->purgable == VM_PURGABLE_VOLATILE) {
- /* if it's wired, we can't put it on our queue */
- assert(m->wire_count == 0);
- /* just stick it back on! */
- goto reactivate_page;
- }
- }
m->pageq.next = NULL;
m->pageq.prev = NULL;
DTRACE_VM2(dfree, int, 1, (uint64_t *), NULL);
- if (m->object->internal) {
+ if (object->internal) {
DTRACE_VM2(anonfree, int, 1, (uint64_t *), NULL);
} else {
DTRACE_VM2(fsfree, int, 1, (uint64_t *), NULL);
}
+ vm_page_free_prepare_queues(m);
- vm_page_free_prepare(m);
+ /*
+ * remove page from object here since we're already
+ * behind the object lock... defer the rest of the work
+ * we'd normally do in vm_page_free_prepare_object
+ * until 'vm_page_free_list' is called
+ */
+ if (m->tabled)
+ vm_page_remove(m, TRUE);
assert(m->pageq.next == NULL &&
m->pageq.prev == NULL);
inactive_burst_count = 0;
+ if(page_prev_state != PAGE_STATE_SPECULATIVE) {
+ vm_pageout_stats[vm_pageout_stat_now].reclaimed++;
+ page_prev_state = 0;
+ }
+
goto done_with_inactivepage;
}
goto done_with_inactivepage;
}
+ /*
+ * If the object is empty, the page must be reclaimed even
+ * if dirty or used.
+ * If the page belongs to a volatile object, we stick it back
+ * on.
+ */
+ if (object->copy == VM_OBJECT_NULL) {
+ if (object->purgable == VM_PURGABLE_EMPTY) {
+ m->busy = TRUE;
+ if (m->pmapped == TRUE) {
+ /* unmap the page */
+ refmod_state = pmap_disconnect(m->phys_page);
+ if (refmod_state & VM_MEM_MODIFIED) {
+ m->dirty = TRUE;
+ }
+ }
+ if (m->dirty || m->precious) {
+ /* we saved the cost of cleaning this page ! */
+ vm_page_purged_count++;
+ }
+ goto reclaim_page;
+ }
+ if (object->purgable == VM_PURGABLE_VOLATILE) {
+ /* if it's wired, we can't put it on our queue */
+ assert(!VM_PAGE_WIRED(m));
+ /* just stick it back on! */
+ goto reactivate_page;
+ }
+ }
+
/*
* If it's being used, reactivate.
* (Fictitious pages are either busy or absent.)
if (refmod_state & VM_MEM_MODIFIED)
m->dirty = TRUE;
}
+
+ if (m->reference || m->dirty) {
+ /* deal with a rogue "reusable" page */
+ VM_PAGEOUT_SCAN_HANDLE_REUSABLE_PAGE(m);
+ }
+
if (m->reference && !m->no_cache) {
/*
* The page we pulled off the inactive list has
} else if (++inactive_reclaim_run >= VM_PAGEOUT_INACTIVE_FORCE_RECLAIM) {
vm_pageout_inactive_force_reclaim++;
} else {
- /*
- * The page was being used, so put back on active list.
- */
+ uint32_t isinuse;
reactivate_page:
- vm_page_activate(m);
- VM_STAT_INCR(reactivations);
-
+ if ( !object->internal && object->pager != MEMORY_OBJECT_NULL &&
+ vnode_pager_get_isinuse(object->pager, &isinuse) == KERN_SUCCESS && !isinuse) {
+ /*
+ * no explict mappings of this object exist
+ * and it's not open via the filesystem
+ */
+ vm_page_deactivate(m);
+ vm_pageout_inactive_deactivated++;
+ } else {
+ /*
+ * The page was/is being used, so put back on active list.
+ */
+ vm_page_activate(m);
+ VM_STAT_INCR(reactivations);
+ }
vm_pageout_inactive_used++;
inactive_burst_count = 0;
XPR(XPR_VM_PAGEOUT,
"vm_pageout_scan, replace object 0x%X offset 0x%X page 0x%X\n",
- (integer_t)object, (integer_t)m->offset, (integer_t)m, 0,0);
+ object, m->offset, m, 0,0);
/*
* we've got a candidate page to steal...
* it if is, we need to skip over it by moving it back
* to the end of the inactive queue
*/
+
inactive_throttled = FALSE;
if (m->dirty || m->precious) {
if (inactive_throttled == TRUE) {
throttle_inactive:
if (!IP_VALID(memory_manager_default) &&
- object->internal &&
- (object->purgable == VM_PURGABLE_DENY ||
- object->purgable == VM_PURGABLE_NONVOLATILE)) {
+ object->internal && m->dirty &&
+ (object->purgable == VM_PURGABLE_DENY ||
+ object->purgable == VM_PURGABLE_NONVOLATILE ||
+ object->purgable == VM_PURGABLE_VOLATILE)) {
queue_enter(&vm_page_queue_throttled, m,
vm_page_t, pageq);
m->throttled = TRUE;
* If it's clean and not precious, we can free the page.
*/
if (!m->dirty && !m->precious) {
+ if (m->zero_fill)
+ vm_pageout_inactive_zf++;
vm_pageout_inactive_clean++;
+
goto reclaim_page;
}
}
}
+ vm_pageout_stats[vm_pageout_stat_now].reclaimed++;
+
vm_pageout_cluster(m);
+ if (m->zero_fill)
+ vm_pageout_inactive_zf++;
vm_pageout_inactive_dirty++;
inactive_burst_count = 0;
if (delayed_unlock++ > VM_PAGEOUT_DELAYED_UNLOCK_LIMIT || try_failed == TRUE) {
if (object != NULL) {
+ vm_pageout_scan_wants_object = VM_OBJECT_NULL;
vm_object_unlock(object);
object = NULL;
- vm_pageout_scan_wants_object = VM_OBJECT_NULL;
}
if (local_freeq) {
- vm_page_free_list(local_freeq);
+ vm_page_unlock_queues();
+ vm_page_free_list(local_freeq, TRUE);
local_freeq = NULL;
local_freed = 0;
- }
- mutex_yield(&vm_page_queue_lock);
+ vm_page_lock_queues();
+ } else
+ lck_mtx_yield(&vm_page_queue_lock);
delayed_unlock = 1;
}
if (vm_page_free_target < vm_page_free_min + 5)
vm_page_free_target = vm_page_free_min + 5;
+ vm_page_throttle_limit = vm_page_free_target - (vm_page_free_target / 3);
+ vm_page_creation_throttle = vm_page_free_target / 2;
}
/*
assert(vm_page_free_wanted == 0);
assert(vm_page_free_wanted_privileged == 0);
assert_wait((event_t) &vm_page_free_wanted, THREAD_UNINT);
- mutex_unlock(&vm_page_queue_free_lock);
+ lck_mtx_unlock(&vm_page_queue_free_lock);
counter(c_vm_pageout_block++);
thread_block((thread_continue_t)vm_pageout_continue);
}
-/*
- * must be called with the
- * queues and object locks held
- */
-static void
-vm_pageout_queue_steal(vm_page_t m)
-{
- struct vm_pageout_queue *q;
-
- if (m->object->internal == TRUE)
- q = &vm_pageout_queue_internal;
- else
- q = &vm_pageout_queue_external;
-
- m->laundry = FALSE;
- m->pageout_queue = FALSE;
- queue_remove(&q->pgo_pending, m, vm_page_t, pageq);
-
- m->pageq.next = NULL;
- m->pageq.prev = NULL;
-
- vm_object_paging_end(m->object);
-
- q->pgo_laundry--;
-}
-
-
-#ifdef FAKE_DEADLOCK
+#ifdef FAKE_DEADLOCK
#define FAKE_COUNT 5000
{
vm_page_t m = NULL;
vm_object_t object;
- boolean_t need_wakeup;
memory_object_t pager;
thread_t self = current_thread();
q->pgo_busy = TRUE;
queue_remove_first(&q->pgo_pending, m, vm_page_t, pageq);
+ VM_PAGE_CHECK(m);
m->pageout_queue = FALSE;
- vm_page_unlock_queues();
-
m->pageq.next = NULL;
m->pageq.prev = NULL;
+ vm_page_unlock_queues();
+
#ifdef FAKE_DEADLOCK
if (q == &vm_pageout_queue_internal) {
vm_offset_t addr;
* Should only happen if there is no
* default pager.
*/
- m->list_req_pending = FALSE;
- m->cleaning = FALSE;
- m->pageout = FALSE;
-
vm_page_lockspin_queues();
- vm_page_unwire(m);
- vm_pageout_throttle_up(m);
+
+ vm_pageout_queue_steal(m, TRUE);
vm_pageout_dirty_no_pager++;
vm_page_activate(m);
+
vm_page_unlock_queues();
/*
* This pager has been destroyed by either
* memory_object_destroy or vm_object_destroy, and
* so there is nowhere for the page to go.
- * Just free the page... VM_PAGE_FREE takes
- * care of cleaning up all the state...
- * including doing the vm_pageout_throttle_up
*/
+ if (m->pageout) {
+ /*
+ * Just free the page... VM_PAGE_FREE takes
+ * care of cleaning up all the state...
+ * including doing the vm_pageout_throttle_up
+ */
+ VM_PAGE_FREE(m);
+ } else {
+ vm_page_lockspin_queues();
- VM_PAGE_FREE(m);
+ vm_pageout_queue_steal(m, TRUE);
+ vm_page_activate(m);
+
+ vm_page_unlock_queues();
+ /*
+ * And we are done with it.
+ */
+ PAGE_WAKEUP_DONE(m);
+ }
vm_object_paging_end(object);
vm_object_unlock(object);
vm_page_lockspin_queues();
continue;
}
+ VM_PAGE_CHECK(m);
vm_object_unlock(object);
/*
* we expect the paging_in_progress reference to have
}
assert_wait((event_t) q, THREAD_UNINT);
-
if (q->pgo_throttled == TRUE && !VM_PAGE_Q_THROTTLED(q)) {
q->pgo_throttled = FALSE;
- need_wakeup = TRUE;
- } else
- need_wakeup = FALSE;
-
+ thread_wakeup((event_t) &q->pgo_laundry);
+ }
+ if (q->pgo_draining == TRUE && q->pgo_laundry == 0) {
+ q->pgo_draining = FALSE;
+ thread_wakeup((event_t) (&q->pgo_laundry+1));
+ }
q->pgo_busy = FALSE;
q->pgo_idle = TRUE;
vm_page_unlock_queues();
- if (need_wakeup == TRUE)
- thread_wakeup((event_t) &q->pgo_laundry);
-
thread_block_parameter((thread_continue_t)vm_pageout_iothread_continue, (void *) &q->pgo_pending);
/*NOTREACHED*/
}
/*NOTREACHED*/
}
+kern_return_t
+vm_set_buffer_cleanup_callout(boolean_t (*func)(int))
+{
+ if (OSCompareAndSwapPtr(NULL, func, (void * volatile *) &consider_buffer_cache_collect)) {
+ return KERN_SUCCESS;
+ } else {
+ return KERN_FAILURE; /* Already set */
+ }
+}
+
static void
vm_pageout_garbage_collect(int collect)
{
if (collect) {
+ boolean_t buf_large_zfree = FALSE;
stack_collect();
/*
* might return memory to zones.
*/
consider_machine_collect();
- consider_zone_gc();
+ if (consider_buffer_cache_collect != NULL) {
+ buf_large_zfree = (*consider_buffer_cache_collect)(0);
+ }
+ consider_zone_gc(buf_large_zfree);
consider_machine_adjust();
}
vm_pageout_queue_external.pgo_idle = FALSE;
vm_pageout_queue_external.pgo_busy = FALSE;
vm_pageout_queue_external.pgo_throttled = FALSE;
+ vm_pageout_queue_external.pgo_draining = FALSE;
queue_init(&vm_pageout_queue_internal.pgo_pending);
vm_pageout_queue_internal.pgo_maxlaundry = 0;
vm_pageout_queue_internal.pgo_idle = FALSE;
vm_pageout_queue_internal.pgo_busy = FALSE;
vm_pageout_queue_internal.pgo_throttled = FALSE;
+ vm_pageout_queue_internal.pgo_draining = FALSE;
/* internal pageout thread started when default pager registered first time */
return result;
}
-#define UPL_DELAYED_UNLOCK_LIMIT (MAX_UPL_TRANSFER / 2)
+
+/*
+ * when marshalling pages into a UPL and subsequently committing
+ * or aborting them, it is necessary to hold
+ * the vm_page_queue_lock (a hot global lock) for certain operations
+ * on the page... however, the majority of the work can be done
+ * while merely holding the object lock... in fact there are certain
+ * collections of pages that don't require any work brokered by the
+ * vm_page_queue_lock... to mitigate the time spent behind the global
+ * lock, go to a 2 pass algorithm... collect pages up to DELAYED_WORK_LIMIT
+ * while doing all of the work that doesn't require the vm_page_queue_lock...
+ * then call dw_do_work to acquire the vm_page_queue_lock and do the
+ * necessary work for each page... we will grab the busy bit on the page
+ * if it's not already held so that dw_do_work can drop the object lock
+ * if it can't immediately take the vm_page_queue_lock in order to compete
+ * for the locks in the same order that vm_pageout_scan takes them.
+ * the operation names are modeled after the names of the routines that
+ * need to be called in order to make the changes very obvious in the
+ * original loop
+ */
+
+#define DELAYED_WORK_LIMIT 32
+
+#define DW_vm_page_unwire 0x01
+#define DW_vm_page_wire 0x02
+#define DW_vm_page_free 0x04
+#define DW_vm_page_activate 0x08
+#define DW_vm_page_deactivate_internal 0x10
+#define DW_vm_page_speculate 0x20
+#define DW_vm_page_lru 0x40
+#define DW_vm_pageout_throttle_up 0x80
+#define DW_PAGE_WAKEUP 0x100
+#define DW_clear_busy 0x200
+#define DW_clear_reference 0x400
+#define DW_set_reference 0x800
+
+struct dw {
+ vm_page_t dw_m;
+ int dw_mask;
+};
+
+
+static void dw_do_work(vm_object_t object, struct dw *dwp, int dw_count);
+
+
static upl_t
upl_create(int type, int flags, upl_size_t size)
int upl_flags = 0;
int upl_size = sizeof(struct upl);
+ size = round_page_32(size);
+
if (type & UPL_CREATE_LITE) {
- page_field_size = ((size/PAGE_SIZE) + 7) >> 3;
+ page_field_size = (atop(size) + 7) >> 3;
page_field_size = (page_field_size + 3) & 0xFFFFFFFC;
upl_flags |= UPL_LITE;
}
if (type & UPL_CREATE_INTERNAL) {
- upl_size += sizeof(struct upl_page_info) * (size/PAGE_SIZE);
+ upl_size += (int) sizeof(struct upl_page_info) * atop(size);
upl_flags |= UPL_INTERNAL;
}
upl->ref_count = 1;
upl->highest_page = 0;
upl_lock_init(upl);
-#ifdef UPL_DEBUG
+ upl->vector_upl = NULL;
+#if UPL_DEBUG
upl->ubc_alias1 = 0;
upl->ubc_alias2 = 0;
+
+ upl->upl_creator = current_thread();
+ upl->upl_state = 0;
+ upl->upl_commit_index = 0;
+ bzero(&upl->upl_commit_records[0], sizeof(upl->upl_commit_records));
+
+ (void) OSBacktrace(&upl->upl_create_retaddr[0], UPL_DEBUG_STACK_FRAMES);
#endif /* UPL_DEBUG */
+
return(upl);
}
int page_field_size; /* bit field in word size buf */
int size;
-#ifdef UPL_DEBUG
+#if UPL_DEBUG
{
vm_object_t object;
page_field_size = ((size/PAGE_SIZE) + 7) >> 3;
page_field_size = (page_field_size + 3) & 0xFFFFFFFC;
}
+ upl_lock_destroy(upl);
+ upl->vector_upl = (vector_upl_t) 0xfeedbeef;
if (upl->flags & UPL_INTERNAL) {
kfree(upl,
sizeof(struct upl) +
void
upl_deallocate(upl_t upl)
{
- if (--upl->ref_count == 0)
+ if (--upl->ref_count == 0) {
+ if(vector_upl_is_valid(upl))
+ vector_upl_deallocate(upl);
upl_destroy(upl);
+ }
}
-/*
+#if DEVELOPMENT || DEBUG
+/*/*
* Statistics about UPL enforcement of copy-on-write obligations.
*/
unsigned long upl_cow = 0;
unsigned long upl_cow_again = 0;
-unsigned long upl_cow_contiguous = 0;
unsigned long upl_cow_pages = 0;
unsigned long upl_cow_again_pages = 0;
-unsigned long upl_cow_contiguous_pages = 0;
+
+unsigned long iopl_cow = 0;
+unsigned long iopl_cow_pages = 0;
+#endif
/*
* Routine: vm_object_upl_request
int refmod_state = 0;
wpl_array_t lite_list = NULL;
vm_object_t last_copy_object;
- int delayed_unlock = 0;
- int j;
+ struct dw dw_array[DELAYED_WORK_LIMIT];
+ struct dw *dwp;
+ int dw_count;
if (cntrl_flags & ~UPL_VALID_FLAGS) {
/*
panic("vm_object_upl_request: contiguous object specified\n");
- if ((size / PAGE_SIZE) > MAX_UPL_TRANSFER)
- size = MAX_UPL_TRANSFER * PAGE_SIZE;
+ if ((size / PAGE_SIZE) > MAX_UPL_SIZE)
+ size = MAX_UPL_SIZE * PAGE_SIZE;
if ( (cntrl_flags & UPL_SET_INTERNAL) && page_list_count != NULL)
- *page_list_count = MAX_UPL_TRANSFER;
+ *page_list_count = MAX_UPL_SIZE;
if (cntrl_flags & UPL_SET_INTERNAL) {
if (cntrl_flags & UPL_SET_LITE) {
lite_list = (wpl_array_t)
(((uintptr_t)user_page_list) +
((size/PAGE_SIZE) * sizeof(upl_page_info_t)));
+ if (size == 0) {
+ user_page_list = NULL;
+ lite_list = NULL;
+ }
} else {
upl = upl_create(UPL_CREATE_INTERNAL, 0, size);
user_page_list = (upl_page_info_t *) (((uintptr_t)upl) + sizeof(struct upl));
+ if (size == 0) {
+ user_page_list = NULL;
+ }
}
} else {
if (cntrl_flags & UPL_SET_LITE) {
upl = upl_create(UPL_CREATE_EXTERNAL | UPL_CREATE_LITE, 0, size);
lite_list = (wpl_array_t) (((uintptr_t)upl) + sizeof(struct upl));
+ if (size == 0) {
+ lite_list = NULL;
+ }
} else {
upl = upl_create(UPL_CREATE_EXTERNAL, 0, size);
}
upl->flags |= UPL_PAGEOUT;
vm_object_lock(object);
- vm_object_paging_begin(object);
+ vm_object_activity_begin(object);
/*
* we can lock in the paging_offset once paging_in_progress is set
upl->size = size;
upl->offset = offset + object->paging_offset;
-#ifdef UPL_DEBUG
+#if UPL_DEBUG
queue_enter(&object->uplq, upl, upl_t, uplq);
#endif /* UPL_DEBUG */
FALSE, /* should_return */
MEMORY_OBJECT_COPY_SYNC,
VM_PROT_NO_CHANGE);
+#if DEVELOPMENT || DEBUG
upl_cow++;
upl_cow_pages += size >> PAGE_SHIFT;
+#endif
}
/*
* remember which copy object we synchronized with
xfer_size = size;
dst_offset = offset;
+ dwp = &dw_array[0];
+ dw_count = 0;
+
while (xfer_size) {
+ dwp->dw_mask = 0;
+
if ((alias_page == NULL) && !(cntrl_flags & UPL_SET_LITE)) {
- if (delayed_unlock) {
- delayed_unlock = 0;
- vm_page_unlock_queues();
- }
vm_object_unlock(object);
VM_PAGE_GRAB_FICTITIOUS(alias_page);
- goto relock;
- }
- if (delayed_unlock == 0) {
- /*
- * pageout_scan takes the vm_page_lock_queues first
- * then tries for the object lock... to avoid what
- * is effectively a lock inversion, we'll go to the
- * trouble of taking them in that same order... otherwise
- * if this object contains the majority of the pages resident
- * in the UBC (or a small set of large objects actively being
- * worked on contain the majority of the pages), we could
- * cause the pageout_scan thread to 'starve' in its attempt
- * to find pages to move to the free queue, since it has to
- * successfully acquire the object lock of any candidate page
- * before it can steal/clean it.
- */
- vm_object_unlock(object);
-relock:
- for (j = 0; ; j++) {
- vm_page_lock_queues();
-
- if (vm_object_lock_try(object))
- break;
- vm_page_unlock_queues();
- mutex_pause(j);
- }
- delayed_unlock = 1;
+ vm_object_lock(object);
}
if (cntrl_flags & UPL_COPYOUT_FROM) {
upl->flags |= UPL_PAGE_SYNC_DONE;
dst_page->fictitious ||
dst_page->absent ||
dst_page->error ||
- (dst_page->wire_count && !dst_page->pageout && !dst_page->list_req_pending)) {
+ (VM_PAGE_WIRED(dst_page) && !dst_page->pageout && !dst_page->list_req_pending)) {
if (user_page_list)
user_page_list[entry].phys_addr = 0;
- goto delay_unlock_queues;
+ goto try_next_page;
}
/*
* grab this up front...
* way of vm_pageout_scan which would have to
* reactivate it upon tripping over it
*/
- vm_page_activate(dst_page);
- VM_STAT_INCR(reactivations);
+ dwp->dw_mask |= DW_vm_page_activate;
}
if (cntrl_flags & UPL_RET_ONLY_DIRTY) {
/*
* currently on the inactive queue or it meets the page
* ticket (generation count) check
*/
- if ( !(refmod_state & VM_MEM_REFERENCED) &&
+ if ( (cntrl_flags & UPL_CLEAN_IN_PLACE || !(refmod_state & VM_MEM_REFERENCED)) &&
((refmod_state & VM_MEM_MODIFIED) || dst_page->dirty || dst_page->precious) ) {
goto check_busy;
}
if (user_page_list)
user_page_list[entry].phys_addr = 0;
- goto delay_unlock_queues;
+ goto try_next_page;
}
check_busy:
- if (dst_page->busy && (!(dst_page->list_req_pending && dst_page->pageout))) {
+ if (dst_page->busy && (!(dst_page->list_req_pending && (dst_page->pageout || dst_page->cleaning)))) {
if (cntrl_flags & UPL_NOBLOCK) {
if (user_page_list)
user_page_list[entry].phys_addr = 0;
- goto delay_unlock_queues;
+ goto try_next_page;
}
/*
* someone else is playing with the
* page. We will have to wait.
*/
- delayed_unlock = 0;
- vm_page_unlock_queues();
-
PAGE_SLEEP(object, dst_page, THREAD_UNINT);
continue;
/*
* Someone else already cleaning the page?
*/
- if ((dst_page->cleaning || dst_page->absent || dst_page->wire_count != 0) && !dst_page->list_req_pending) {
+ if ((dst_page->cleaning || dst_page->absent || VM_PAGE_WIRED(dst_page)) && !dst_page->list_req_pending) {
if (user_page_list)
user_page_list[entry].phys_addr = 0;
- goto delay_unlock_queues;
+ goto try_next_page;
}
/*
* ENCRYPTED SWAP:
if (! (cntrl_flags & UPL_ENCRYPT) && dst_page->encrypted) {
int was_busy;
- delayed_unlock = 0;
- vm_page_unlock_queues();
/*
* save the current state of busy
* mark page as busy while decrypt
* restore to original busy state
*/
dst_page->busy = was_busy;
+ }
+ if (dst_page->pageout_queue == TRUE) {
- vm_page_lock_queues();
- delayed_unlock = 1;
+ vm_page_lockspin_queues();
+
+#if CONFIG_EMBEDDED
+ if (dst_page->laundry)
+#else
+ if (dst_page->pageout_queue == TRUE)
+#endif
+ {
+ /*
+ * we've buddied up a page for a clustered pageout
+ * that has already been moved to the pageout
+ * queue by pageout_scan... we need to remove
+ * it from the queue and drop the laundry count
+ * on that queue
+ */
+ vm_pageout_throttle_up(dst_page);
+ }
+ vm_page_unlock_queues();
}
- if (dst_page->pageout_queue == TRUE)
- /*
- * we've buddied up a page for a clustered pageout
- * that has already been moved to the pageout
- * queue by pageout_scan... we need to remove
- * it from the queue and drop the laundry count
- * on that queue
- */
- vm_pageout_queue_steal(dst_page);
#if MACH_CLUSTER_STATS
/*
* pageout statistics gathering. count
upl->highest_page = dst_page->phys_page;
if (cntrl_flags & UPL_SET_LITE) {
- int pg_num;
+ unsigned int pg_num;
- pg_num = (dst_offset-offset)/PAGE_SIZE;
+ pg_num = (unsigned int) ((dst_offset-offset)/PAGE_SIZE);
+ assert(pg_num == (dst_offset-offset)/PAGE_SIZE);
lite_list[pg_num>>5] |= 1 << (pg_num & 31);
if (hw_dirty)
* deny access to the target page
* while it is being worked on
*/
- if ((!dst_page->pageout) && (dst_page->wire_count == 0)) {
+ if ((!dst_page->pageout) && ( !VM_PAGE_WIRED(dst_page))) {
dst_page->busy = TRUE;
dst_page->pageout = TRUE;
- vm_page_wire(dst_page);
+
+ dwp->dw_mask |= DW_vm_page_wire;
}
}
} else {
* to see both the *before* and *after* pages.
*/
if (object->copy != VM_OBJECT_NULL) {
- delayed_unlock = 0;
- vm_page_unlock_queues();
-
vm_object_update(
object,
dst_offset,/* current offset */
MEMORY_OBJECT_COPY_SYNC,
VM_PROT_NO_CHANGE);
+#if DEVELOPMENT || DEBUG
upl_cow_again++;
upl_cow_again_pages += xfer_size >> PAGE_SHIFT;
-
- vm_page_lock_queues();
- delayed_unlock = 1;
+#endif
}
/*
* remember the copy object we synced with
dst_page = vm_page_lookup(object, dst_offset);
if (dst_page != VM_PAGE_NULL) {
- if ( !(dst_page->list_req_pending) ) {
- if ((cntrl_flags & UPL_RET_ONLY_ABSENT) && !dst_page->absent) {
- /*
+
+ if ((cntrl_flags & UPL_RET_ONLY_ABSENT)) {
+
+ if ( !(dst_page->absent && dst_page->list_req_pending) ) {
+ /*
* skip over pages already present in the cache
*/
- if (user_page_list)
- user_page_list[entry].phys_addr = 0;
+ if (user_page_list)
+ user_page_list[entry].phys_addr = 0;
- goto delay_unlock_queues;
+ goto try_next_page;
}
+ }
+ if ( !(dst_page->list_req_pending) ) {
+
if (dst_page->cleaning) {
/*
* someone else is writing to the page... wait...
*/
- delayed_unlock = 0;
- vm_page_unlock_queues();
-
PAGE_SLEEP(object, dst_page, THREAD_UNINT);
continue;
*/
dst_page->list_req_pending = FALSE;
- vm_page_free(dst_page);
+ VM_PAGE_FREE(dst_page);
dst_page = NULL;
+
} else if (dst_page->absent) {
/*
* the default_pager case
*/
dst_page->list_req_pending = FALSE;
dst_page->busy = FALSE;
+
+ } else if (dst_page->pageout || dst_page->cleaning) {
+ /*
+ * page was earmarked by vm_pageout_scan
+ * to be cleaned and stolen... we're going
+ * to take it back since we are not attempting
+ * to read that page and we don't want to stall
+ * waiting for it to be cleaned for 2 reasons...
+ * 1 - no use paging it out and back in
+ * 2 - if we stall, we may casue a deadlock in
+ * the FS trying to acquire the its locks
+ * on the VNOP_PAGEOUT path presuming that
+ * those locks are already held on the read
+ * path before trying to create this UPL
+ *
+ * so undo all of the state that vm_pageout_scan
+ * hung on this page
+ */
+ dst_page->busy = FALSE;
+
+ vm_pageout_queue_steal(dst_page, FALSE);
}
}
}
if (user_page_list)
user_page_list[entry].phys_addr = 0;
- goto delay_unlock_queues;
+ goto try_next_page;
}
/*
* need to allocate a page
* then try again for the same
* offset...
*/
- delayed_unlock = 0;
- vm_page_unlock_queues();
-
vm_object_unlock(object);
VM_PAGE_WAIT();
-
- /*
- * pageout_scan takes the vm_page_lock_queues first
- * then tries for the object lock... to avoid what
- * is effectively a lock inversion, we'll go to the
- * trouble of taking them in that same order... otherwise
- * if this object contains the majority of the pages resident
- * in the UBC (or a small set of large objects actively being
- * worked on contain the majority of the pages), we could
- * cause the pageout_scan thread to 'starve' in its attempt
- * to find pages to move to the free queue, since it has to
- * successfully acquire the object lock of any candidate page
- * before it can steal/clean it.
- */
- for (j = 0; ; j++) {
- vm_page_lock_queues();
-
- if (vm_object_lock_try(object))
- break;
- vm_page_unlock_queues();
- mutex_pause(j);
- }
- delayed_unlock = 1;
+ vm_object_lock(object);
continue;
}
- vm_page_insert_internal(dst_page, object, dst_offset, TRUE);
+ vm_page_insert(dst_page, object, dst_offset);
dst_page->absent = TRUE;
dst_page->busy = FALSE;
dst_page->clustered = TRUE;
}
}
+ if (dst_page->fictitious) {
+ panic("need corner case for fictitious page");
+ }
+ if (dst_page->busy) {
+ /*
+ * someone else is playing with the
+ * page. We will have to wait.
+ */
+ PAGE_SLEEP(object, dst_page, THREAD_UNINT);
+
+ continue;
+ }
/*
* ENCRYPTED SWAP:
*/
}
dst_page->overwriting = TRUE;
- if (dst_page->fictitious) {
- panic("need corner case for fictitious page");
- }
- if (dst_page->busy) {
- /*
- * someone else is playing with the
- * page. We will have to wait.
- */
- delayed_unlock = 0;
- vm_page_unlock_queues();
-
- PAGE_SLEEP(object, dst_page, THREAD_UNINT);
-
- continue;
- }
if (dst_page->pmapped) {
if ( !(cntrl_flags & UPL_FILE_IO))
/*
dirty = hw_dirty ? TRUE : dst_page->dirty;
if (cntrl_flags & UPL_SET_LITE) {
- int pg_num;
+ unsigned int pg_num;
- pg_num = (dst_offset-offset)/PAGE_SIZE;
+ pg_num = (unsigned int) ((dst_offset-offset)/PAGE_SIZE);
+ assert(pg_num == (dst_offset-offset)/PAGE_SIZE);
lite_list[pg_num>>5] |= 1 << (pg_num & 31);
if (hw_dirty)
if (!dirty)
dst_page->precious = TRUE;
- if (dst_page->wire_count == 0) {
+ if ( !VM_PAGE_WIRED(dst_page)) {
/*
* deny access to the target page while
* it is being worked on
*/
dst_page->busy = TRUE;
} else
- vm_page_wire(dst_page);
+ dwp->dw_mask |= DW_vm_page_wire;
- if (dst_page->clustered) {
- /*
- * expect the page not to be used
- * since it's coming in as part
- * of a speculative cluster...
- * pages that are 'consumed' will
- * get a hardware reference
- */
- dst_page->reference = FALSE;
- } else {
+ /*
+ * We might be about to satisfy a fault which has been
+ * requested. So no need for the "restart" bit.
+ */
+ dst_page->restart = FALSE;
+ if (!dst_page->absent && !(cntrl_flags & UPL_WILL_MODIFY)) {
/*
* expect the page to be used
*/
- dst_page->reference = TRUE;
+ dwp->dw_mask |= DW_set_reference;
}
dst_page->precious = (cntrl_flags & UPL_PRECIOUS) ? TRUE : FALSE;
}
+ if (dst_page->busy)
+ upl->flags |= UPL_HAS_BUSY;
+
if (dst_page->phys_page > upl->highest_page)
upl->highest_page = dst_page->phys_page;
if (user_page_list) {
user_page_list[entry].phys_addr = dst_page->phys_page;
- user_page_list[entry].dirty = dst_page->dirty;
user_page_list[entry].pageout = dst_page->pageout;
user_page_list[entry].absent = dst_page->absent;
+ user_page_list[entry].dirty = dst_page->dirty;
user_page_list[entry].precious = dst_page->precious;
-
+ user_page_list[entry].device = FALSE;
if (dst_page->clustered == TRUE)
user_page_list[entry].speculative = dst_page->speculative;
else
user_page_list[entry].speculative = FALSE;
+ user_page_list[entry].cs_validated = dst_page->cs_validated;
+ user_page_list[entry].cs_tainted = dst_page->cs_tainted;
}
/*
* if UPL_RET_ONLY_ABSENT is set, then
*/
VM_PAGE_CONSUME_CLUSTERED(dst_page);
}
-delay_unlock_queues:
- if (delayed_unlock++ > UPL_DELAYED_UNLOCK_LIMIT) {
- /*
- * pageout_scan takes the vm_page_lock_queues first
- * then tries for the object lock... to avoid what
- * is effectively a lock inversion, we'll go to the
- * trouble of taking them in that same order... otherwise
- * if this object contains the majority of the pages resident
- * in the UBC (or a small set of large objects actively being
- * worked on contain the majority of the pages), we could
- * cause the pageout_scan thread to 'starve' in its attempt
- * to find pages to move to the free queue, since it has to
- * successfully acquire the object lock of any candidate page
- * before it can steal/clean it.
- */
- vm_object_unlock(object);
- mutex_yield(&vm_page_queue_lock);
+try_next_page:
+ if (dwp->dw_mask) {
+ if (dwp->dw_mask & DW_vm_page_activate)
+ VM_STAT_INCR(reactivations);
- for (j = 0; ; j++) {
- if (vm_object_lock_try(object))
- break;
- vm_page_unlock_queues();
- mutex_pause(j);
- vm_page_lock_queues();
+ if (dst_page->busy == FALSE) {
+ /*
+ * dw_do_work may need to drop the object lock
+ * if it does, we need the pages it's looking at to
+ * be held stable via the busy bit.
+ */
+ dst_page->busy = TRUE;
+ dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP);
+ }
+ dwp->dw_m = dst_page;
+ dwp++;
+ dw_count++;
+
+ if (dw_count >= DELAYED_WORK_LIMIT) {
+ dw_do_work(object, &dw_array[0], dw_count);
+
+ dwp = &dw_array[0];
+ dw_count = 0;
}
- delayed_unlock = 1;
}
-try_next_page:
entry++;
dst_offset += PAGE_SIZE_64;
xfer_size -= PAGE_SIZE;
}
+ if (dw_count)
+ dw_do_work(object, &dw_array[0], dw_count);
+
if (alias_page != NULL) {
- if (delayed_unlock == 0) {
- vm_page_lock_queues();
- delayed_unlock = 1;
- }
- vm_page_free(alias_page);
+ VM_PAGE_FREE(alias_page);
}
- if (delayed_unlock)
- vm_page_unlock_queues();
if (page_list_count != NULL) {
if (upl->flags & UPL_INTERNAL)
else if (*page_list_count > entry)
*page_list_count = entry;
}
+#if UPL_DEBUG
+ upl->upl_state = 1;
+#endif
vm_object_unlock(object);
return KERN_SUCCESS;
upl_page_info_t *user_page_list;
kern_return_t kr;
+ if((cntrl_flags & UPL_VECTOR)==UPL_VECTOR)
+ return KERN_INVALID_ARGUMENT;
+
if (user_page_list_ptr != NULL) {
local_list_count = page_list_count;
user_page_list = *user_page_list_ptr;
unsigned int *page_list_count,
int cntrl_flags)
{
- if (object->paging_offset > offset)
+ if (object->paging_offset > offset || ((cntrl_flags & UPL_VECTOR)==UPL_VECTOR))
return KERN_FAILURE;
assert(object->paging_in_progress);
vm_object_offset_t base_offset;
upl_size_t super_size;
+ vm_object_size_t super_size_64;
base_offset = (offset & ~((vm_object_offset_t) super_cluster - 1));
super_size = (offset + size) > (base_offset + super_cluster) ? super_cluster<<1 : super_cluster;
- super_size = ((base_offset + super_size) > object->size) ? (object->size - base_offset) : super_size;
+ super_size_64 = ((base_offset + super_size) > object->size) ? (object->size - base_offset) : super_size;
+ super_size = (upl_size_t) super_size_64;
+ assert(super_size == super_size_64);
if (offset > (base_offset + super_size)) {
panic("vm_object_super_upl_request: Missed target pageout"
* page to be written out who's offset is beyond the
* object size
*/
- if ((offset + size) > (base_offset + super_size))
- super_size = (offset + size) - base_offset;
+ if ((offset + size) > (base_offset + super_size)) {
+ super_size_64 = (offset + size) - base_offset;
+ super_size = (upl_size_t) super_size_64;
+ assert(super_size == super_size_64);
+ }
offset = base_offset;
size = super_size;
return vm_object_upl_request(object, offset, size, upl, user_page_list, page_list_count, cntrl_flags);
}
-
+
kern_return_t
vm_map_create_upl(
vm_map_t map,
return KERN_INVALID_ARGUMENT;
REDISCOVER_ENTRY:
- vm_map_lock(map);
+ vm_map_lock_read(map);
if (vm_map_lookup_entry(map, offset, &entry)) {
- if ((entry->vme_end - offset) < *upl_size)
- *upl_size = entry->vme_end - offset;
+ if ((entry->vme_end - offset) < *upl_size) {
+ *upl_size = (upl_size_t) (entry->vme_end - offset);
+ assert(*upl_size == entry->vme_end - offset);
+ }
if (caller_flags & UPL_QUERY_OBJECT_TYPE) {
*flags = 0;
- if (entry->object.vm_object != VM_OBJECT_NULL) {
+ if ( !entry->is_sub_map && entry->object.vm_object != VM_OBJECT_NULL) {
if (entry->object.vm_object->private)
*flags = UPL_DEV_MEMORY;
if (entry->object.vm_object->phys_contiguous)
*flags |= UPL_PHYS_CONTIG;
}
- vm_map_unlock(map);
+ vm_map_unlock_read(map);
return KERN_SUCCESS;
}
if (entry->object.vm_object == VM_OBJECT_NULL || !entry->object.vm_object->phys_contiguous) {
- if ((*upl_size/page_size) > MAX_UPL_TRANSFER)
- *upl_size = MAX_UPL_TRANSFER * page_size;
+ if ((*upl_size/PAGE_SIZE) > MAX_UPL_SIZE)
+ *upl_size = MAX_UPL_SIZE * PAGE_SIZE;
}
/*
* Create an object if necessary.
*/
if (entry->object.vm_object == VM_OBJECT_NULL) {
+
+ if (vm_map_lock_read_to_write(map))
+ goto REDISCOVER_ENTRY;
+
entry->object.vm_object = vm_object_allocate((vm_size_t)(entry->vme_end - entry->vme_start));
entry->offset = 0;
+
+ vm_map_lock_write_to_read(map);
}
if (!(caller_flags & UPL_COPYOUT_FROM)) {
if (!(entry->protection & VM_PROT_WRITE)) {
- vm_map_unlock(map);
+ vm_map_unlock_read(map);
return KERN_PROTECTION_FAILURE;
}
if (entry->needs_copy) {
+ /*
+ * Honor copy-on-write for COPY_SYMMETRIC
+ * strategy.
+ */
vm_map_t local_map;
vm_object_t object;
vm_object_offset_t new_offset;
vm_map_t real_map;
local_map = map;
- vm_map_lock_write_to_read(map);
if (vm_map_lookup_locked(&local_map,
offset, VM_PROT_WRITE,
&version, &object,
&new_offset, &prot, &wired,
NULL,
- &real_map)) {
- vm_map_unlock(local_map);
+ &real_map) != KERN_SUCCESS) {
+ vm_map_unlock_read(local_map);
return KERN_FAILURE;
}
if (real_map != map)
vm_map_unlock(real_map);
+ vm_map_unlock_read(local_map);
+
vm_object_unlock(object);
- vm_map_unlock(local_map);
goto REDISCOVER_ENTRY;
}
local_offset = entry->offset;
vm_map_reference(submap);
- vm_map_unlock(map);
+ vm_map_unlock_read(map);
ret = vm_map_create_upl(submap,
local_offset + (offset - local_start),
local_offset = entry->offset;
vm_object_reference(local_object);
- vm_map_unlock(map);
+ vm_map_unlock_read(map);
- if (entry->object.vm_object->shadow && entry->object.vm_object->copy) {
+ if (local_object->shadow && local_object->copy) {
vm_object_lock_request(
local_object->shadow,
(vm_object_offset_t)
local_offset = entry->offset;
vm_object_reference(local_object);
- vm_map_unlock(map);
+ vm_map_unlock_read(map);
vm_object_lock_request(
local_object,
local_start = entry->vme_start;
vm_object_reference(local_object);
- vm_map_unlock(map);
+ vm_map_unlock_read(map);
ret = vm_object_iopl_request(local_object,
(vm_object_offset_t) ((offset - local_start) + local_offset),
return(ret);
}
- vm_map_unlock(map);
+ vm_map_unlock_read(map);
return(KERN_FAILURE);
}
vm_map_enter_upl(
vm_map_t map,
upl_t upl,
- vm_map_offset_t *dst_addr)
+ vm_map_offset_t *dst_addr)
{
vm_map_size_t size;
vm_object_offset_t offset;
vm_map_offset_t addr;
vm_page_t m;
kern_return_t kr;
+ int isVectorUPL = 0, curr_upl=0;
+ upl_t vector_upl = NULL;
+ vm_offset_t vector_upl_dst_addr = 0;
+ vm_map_t vector_upl_submap = NULL;
+ upl_offset_t subupl_offset = 0;
+ upl_size_t subupl_size = 0;
if (upl == UPL_NULL)
return KERN_INVALID_ARGUMENT;
- upl_lock(upl);
+ if((isVectorUPL = vector_upl_is_valid(upl))) {
+ int mapped=0,valid_upls=0;
+ vector_upl = upl;
- /*
- * check to see if already mapped
- */
- if (UPL_PAGE_LIST_MAPPED & upl->flags) {
- upl_unlock(upl);
- return KERN_FAILURE;
+ upl_lock(vector_upl);
+ for(curr_upl=0; curr_upl < MAX_VECTOR_UPL_ELEMENTS; curr_upl++) {
+ upl = vector_upl_subupl_byindex(vector_upl, curr_upl );
+ if(upl == NULL)
+ continue;
+ valid_upls++;
+ if (UPL_PAGE_LIST_MAPPED & upl->flags)
+ mapped++;
+ }
+
+ if(mapped) {
+ if(mapped != valid_upls)
+ panic("Only %d of the %d sub-upls within the Vector UPL are alread mapped\n", mapped, valid_upls);
+ else {
+ upl_unlock(vector_upl);
+ return KERN_FAILURE;
+ }
+ }
+
+ kr = kmem_suballoc(map, &vector_upl_dst_addr, vector_upl->size, FALSE, VM_FLAGS_ANYWHERE, &vector_upl_submap);
+ if( kr != KERN_SUCCESS )
+ panic("Vector UPL submap allocation failed\n");
+ map = vector_upl_submap;
+ vector_upl_set_submap(vector_upl, vector_upl_submap, vector_upl_dst_addr);
+ curr_upl=0;
+ }
+ else
+ upl_lock(upl);
+
+process_upl_to_enter:
+ if(isVectorUPL){
+ if(curr_upl == MAX_VECTOR_UPL_ELEMENTS) {
+ *dst_addr = vector_upl_dst_addr;
+ upl_unlock(vector_upl);
+ return KERN_SUCCESS;
+ }
+ upl = vector_upl_subupl_byindex(vector_upl, curr_upl++ );
+ if(upl == NULL)
+ goto process_upl_to_enter;
+ vector_upl_get_iostate(vector_upl, upl, &subupl_offset, &subupl_size);
+ *dst_addr = (vm_map_offset_t)(vector_upl_dst_addr + (vm_map_offset_t)subupl_offset);
+ } else {
+ /*
+ * check to see if already mapped
+ */
+ if (UPL_PAGE_LIST_MAPPED & upl->flags) {
+ upl_unlock(upl);
+ return KERN_FAILURE;
+ }
}
+ if ((!(upl->flags & UPL_SHADOWED)) &&
+ ((upl->flags & UPL_HAS_BUSY) ||
+ !((upl->flags & (UPL_DEVICE_MEMORY | UPL_IO_WIRE)) || (upl->map_object->phys_contiguous)))) {
- if ((!(upl->flags & UPL_SHADOWED)) && !((upl->flags & (UPL_DEVICE_MEMORY | UPL_IO_WIRE)) ||
- (upl->map_object->phys_contiguous))) {
vm_object_t object;
vm_page_t alias_page;
vm_object_offset_t new_offset;
- int pg_num;
+ unsigned int pg_num;
wpl_array_t lite_list;
if (upl->flags & UPL_INTERNAL) {
upl->flags |= UPL_SHADOWED;
while (size) {
- pg_num = (new_offset)/PAGE_SIZE;
+ pg_num = (unsigned int) (new_offset / PAGE_SIZE);
+ assert(pg_num == new_offset / PAGE_SIZE);
if (lite_list[pg_num>>5] & (1 << (pg_num & 31))) {
}
vm_object_unlock(upl->map_object);
}
- if ((upl->flags & (UPL_DEVICE_MEMORY | UPL_IO_WIRE)) || upl->map_object->phys_contiguous)
- offset = upl->offset - upl->map_object->paging_offset;
- else
+ if (upl->flags & UPL_SHADOWED)
offset = 0;
+ else
+ offset = upl->offset - upl->map_object->paging_offset;
size = upl->size;
vm_object_reference(upl->map_object);
- *dst_addr = 0;
- /*
- * NEED A UPL_MAP ALIAS
- */
- kr = vm_map_enter(map, dst_addr, (vm_map_size_t)size, (vm_map_offset_t) 0,
- VM_FLAGS_ANYWHERE, upl->map_object, offset, FALSE,
- VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT);
+ if(!isVectorUPL) {
+ *dst_addr = 0;
+ /*
+ * NEED A UPL_MAP ALIAS
+ */
+ kr = vm_map_enter(map, dst_addr, (vm_map_size_t)size, (vm_map_offset_t) 0,
+ VM_FLAGS_ANYWHERE, upl->map_object, offset, FALSE,
+ VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT);
- if (kr != KERN_SUCCESS) {
- upl_unlock(upl);
- return(kr);
+ if (kr != KERN_SUCCESS) {
+ upl_unlock(upl);
+ return(kr);
+ }
+ }
+ else {
+ kr = vm_map_enter(map, dst_addr, (vm_map_size_t)size, (vm_map_offset_t) 0,
+ VM_FLAGS_FIXED, upl->map_object, offset, FALSE,
+ VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT);
+ if(kr)
+ panic("vm_map_enter failed for a Vector UPL\n");
}
vm_object_lock(upl->map_object);
cache_attr = ((unsigned int)m->object->wimg_bits) & VM_WIMG_MASK;
m->pmapped = TRUE;
- m->wpmapped = TRUE;
+
+ /* CODE SIGNING ENFORCEMENT: page has been wpmapped,
+ * but only in kernel space. If this was on a user map,
+ * we'd have to set the wpmapped bit. */
+ /* m->wpmapped = TRUE; */
+ assert(map==kernel_map);
PMAP_ENTER(map->pmap, addr, m, VM_PROT_ALL, cache_attr, TRUE);
}
*/
upl->ref_count++;
upl->flags |= UPL_PAGE_LIST_MAPPED;
- upl->kaddr = *dst_addr;
+ upl->kaddr = (vm_offset_t) *dst_addr;
+ assert(upl->kaddr == *dst_addr);
+
+ if(isVectorUPL)
+ goto process_upl_to_enter;
+
upl_unlock(upl);
return KERN_SUCCESS;
{
vm_address_t addr;
upl_size_t size;
+ int isVectorUPL = 0, curr_upl = 0;
+ upl_t vector_upl = NULL;
if (upl == UPL_NULL)
return KERN_INVALID_ARGUMENT;
- upl_lock(upl);
+ if((isVectorUPL = vector_upl_is_valid(upl))) {
+ int unmapped=0, valid_upls=0;
+ vector_upl = upl;
+ upl_lock(vector_upl);
+ for(curr_upl=0; curr_upl < MAX_VECTOR_UPL_ELEMENTS; curr_upl++) {
+ upl = vector_upl_subupl_byindex(vector_upl, curr_upl );
+ if(upl == NULL)
+ continue;
+ valid_upls++;
+ if (!(UPL_PAGE_LIST_MAPPED & upl->flags))
+ unmapped++;
+ }
+
+ if(unmapped) {
+ if(unmapped != valid_upls)
+ panic("%d of the %d sub-upls within the Vector UPL is/are not mapped\n", unmapped, valid_upls);
+ else {
+ upl_unlock(vector_upl);
+ return KERN_FAILURE;
+ }
+ }
+ curr_upl=0;
+ }
+ else
+ upl_lock(upl);
+
+process_upl_to_remove:
+ if(isVectorUPL) {
+ if(curr_upl == MAX_VECTOR_UPL_ELEMENTS) {
+ vm_map_t v_upl_submap;
+ vm_offset_t v_upl_submap_dst_addr;
+ vector_upl_get_submap(vector_upl, &v_upl_submap, &v_upl_submap_dst_addr);
+
+ vm_map_remove(map, v_upl_submap_dst_addr, v_upl_submap_dst_addr + vector_upl->size, VM_MAP_NO_FLAGS);
+ vm_map_deallocate(v_upl_submap);
+ upl_unlock(vector_upl);
+ return KERN_SUCCESS;
+ }
+
+ upl = vector_upl_subupl_byindex(vector_upl, curr_upl++ );
+ if(upl == NULL)
+ goto process_upl_to_remove;
+ }
if (upl->flags & UPL_PAGE_LIST_MAPPED) {
addr = upl->kaddr;
upl->flags &= ~UPL_PAGE_LIST_MAPPED;
upl->kaddr = (vm_offset_t) 0;
- upl_unlock(upl);
-
- vm_map_remove(map,
- vm_map_trunc_page(addr),
- vm_map_round_page(addr + size),
- VM_MAP_NO_FLAGS);
-
- return KERN_SUCCESS;
- }
- upl_unlock(upl);
-
- return KERN_FAILURE;
-}
+
+ if(!isVectorUPL) {
+ upl_unlock(upl);
+
+ vm_map_remove(map,
+ vm_map_trunc_page(addr),
+ vm_map_round_page(addr + size),
+ VM_MAP_NO_FLAGS);
+
+ return KERN_SUCCESS;
+ }
+ else {
+ /*
+ * If it's a Vectored UPL, we'll be removing the entire
+ * submap anyways, so no need to remove individual UPL
+ * element mappings from within the submap
+ */
+ goto process_upl_to_remove;
+ }
+ }
+ upl_unlock(upl);
+
+ return KERN_FAILURE;
+}
+
+static void
+dw_do_work(
+ vm_object_t object,
+ struct dw *dwp,
+ int dw_count)
+{
+ int j;
+ boolean_t held_as_spin = TRUE;
+
+ /*
+ * pageout_scan takes the vm_page_lock_queues first
+ * then tries for the object lock... to avoid what
+ * is effectively a lock inversion, we'll go to the
+ * trouble of taking them in that same order... otherwise
+ * if this object contains the majority of the pages resident
+ * in the UBC (or a small set of large objects actively being
+ * worked on contain the majority of the pages), we could
+ * cause the pageout_scan thread to 'starve' in its attempt
+ * to find pages to move to the free queue, since it has to
+ * successfully acquire the object lock of any candidate page
+ * before it can steal/clean it.
+ */
+ if (!vm_page_trylockspin_queues()) {
+ vm_object_unlock(object);
+
+ vm_page_lockspin_queues();
+
+ for (j = 0; ; j++) {
+ if (!vm_object_lock_avoid(object) &&
+ _vm_object_lock_try(object))
+ break;
+ vm_page_unlock_queues();
+ mutex_pause(j);
+ vm_page_lockspin_queues();
+ }
+ }
+ for (j = 0; j < dw_count; j++, dwp++) {
+
+ if (dwp->dw_mask & DW_vm_pageout_throttle_up)
+ vm_pageout_throttle_up(dwp->dw_m);
+
+ if (dwp->dw_mask & DW_vm_page_wire)
+ vm_page_wire(dwp->dw_m);
+ else if (dwp->dw_mask & DW_vm_page_unwire) {
+ boolean_t queueit;
+
+ queueit = (dwp->dw_mask & DW_vm_page_free) ? FALSE : TRUE;
+
+ vm_page_unwire(dwp->dw_m, queueit);
+ }
+ if (dwp->dw_mask & DW_vm_page_free) {
+ if (held_as_spin == TRUE) {
+ vm_page_lockconvert_queues();
+ held_as_spin = FALSE;
+ }
+ vm_page_free(dwp->dw_m);
+ } else {
+ if (dwp->dw_mask & DW_vm_page_deactivate_internal)
+ vm_page_deactivate_internal(dwp->dw_m, FALSE);
+ else if (dwp->dw_mask & DW_vm_page_activate)
+ vm_page_activate(dwp->dw_m);
+ else if (dwp->dw_mask & DW_vm_page_speculate)
+ vm_page_speculate(dwp->dw_m, TRUE);
+ else if (dwp->dw_mask & DW_vm_page_lru)
+ vm_page_lru(dwp->dw_m);
+
+ if (dwp->dw_mask & DW_set_reference)
+ dwp->dw_m->reference = TRUE;
+ else if (dwp->dw_mask & DW_clear_reference)
+ dwp->dw_m->reference = FALSE;
+
+ if (dwp->dw_mask & DW_clear_busy)
+ dwp->dw_m->busy = FALSE;
+
+ if (dwp->dw_mask & DW_PAGE_WAKEUP)
+ PAGE_WAKEUP(dwp->dw_m);
+ }
+ }
+ vm_page_unlock_queues();
+}
+
+
kern_return_t
upl_commit_range(
mach_msg_type_number_t count,
boolean_t *empty)
{
- upl_size_t xfer_size;
+ upl_size_t xfer_size, subupl_size = size;
vm_object_t shadow_object;
vm_object_t object;
vm_object_offset_t target_offset;
+ upl_offset_t subupl_offset = offset;
int entry;
wpl_array_t lite_list;
int occupied;
- int delayed_unlock = 0;
int clear_refmod = 0;
int pgpgout_count = 0;
- int j;
+ struct dw dw_array[DELAYED_WORK_LIMIT];
+ struct dw *dwp;
+ int dw_count, isVectorUPL = 0;
+ upl_t vector_upl = NULL;
*empty = FALSE;
if (count == 0)
page_list = NULL;
+ if((isVectorUPL = vector_upl_is_valid(upl))) {
+ vector_upl = upl;
+ upl_lock(vector_upl);
+ }
+ else
+ upl_lock(upl);
+
+process_upl_to_commit:
+
+ if(isVectorUPL) {
+ size = subupl_size;
+ offset = subupl_offset;
+ if(size == 0) {
+ upl_unlock(vector_upl);
+ return KERN_SUCCESS;
+ }
+ upl = vector_upl_subupl_byoffset(vector_upl, &offset, &size);
+ if(upl == NULL) {
+ upl_unlock(vector_upl);
+ return KERN_FAILURE;
+ }
+ page_list = UPL_GET_INTERNAL_PAGE_LIST_SIMPLE(upl);
+ subupl_size -= size;
+ subupl_offset += size;
+ }
+
+#if UPL_DEBUG
+ if (upl->upl_commit_index < UPL_DEBUG_COMMIT_RECORDS) {
+ (void) OSBacktrace(&upl->upl_commit_records[upl->upl_commit_index].c_retaddr[0], UPL_DEBUG_STACK_FRAMES);
+
+ upl->upl_commit_records[upl->upl_commit_index].c_beg = offset;
+ upl->upl_commit_records[upl->upl_commit_index].c_end = (offset + size);
+
+ upl->upl_commit_index++;
+ }
+#endif
if (upl->flags & UPL_DEVICE_MEMORY)
xfer_size = 0;
else if ((offset + size) <= upl->size)
xfer_size = size;
- else
+ else {
+ if(!isVectorUPL)
+ upl_unlock(upl);
+ else {
+ upl_unlock(vector_upl);
+ }
return KERN_FAILURE;
-
- upl_lock(upl);
-
- if (upl->flags & UPL_ACCESS_BLOCKED) {
- /*
- * We used this UPL to block access to the pages by marking
- * them "busy". Now we need to clear the "busy" bit to allow
- * access to these pages again.
- */
- flags |= UPL_COMMIT_ALLOW_ACCESS;
}
if (upl->flags & UPL_CLEAR_DIRTY)
flags |= UPL_COMMIT_CLEAR_DIRTY;
entry = offset/PAGE_SIZE;
target_offset = (vm_object_offset_t)offset;
- /*
- * pageout_scan takes the vm_page_lock_queues first
- * then tries for the object lock... to avoid what
- * is effectively a lock inversion, we'll go to the
- * trouble of taking them in that same order... otherwise
- * if this object contains the majority of the pages resident
- * in the UBC (or a small set of large objects actively being
- * worked on contain the majority of the pages), we could
- * cause the pageout_scan thread to 'starve' in its attempt
- * to find pages to move to the free queue, since it has to
- * successfully acquire the object lock of any candidate page
- * before it can steal/clean it.
- */
- for (j = 0; ; j++) {
- vm_page_lock_queues();
+ if (upl->flags & UPL_KERNEL_OBJECT)
+ vm_object_lock_shared(shadow_object);
+ else
+ vm_object_lock(shadow_object);
- if (vm_object_lock_try(shadow_object))
- break;
- vm_page_unlock_queues();
- mutex_pause(j);
+ if (upl->flags & UPL_ACCESS_BLOCKED) {
+ assert(shadow_object->blocked_access);
+ shadow_object->blocked_access = FALSE;
+ vm_object_wakeup(object, VM_OBJECT_EVENT_UNBLOCKED);
+ }
+
+ if (shadow_object->code_signed) {
+ /*
+ * CODE SIGNING:
+ * If the object is code-signed, do not let this UPL tell
+ * us if the pages are valid or not. Let the pages be
+ * validated by VM the normal way (when they get mapped or
+ * copied).
+ */
+ flags &= ~UPL_COMMIT_CS_VALIDATED;
+ }
+ if (! page_list) {
+ /*
+ * No page list to get the code-signing info from !?
+ */
+ flags &= ~UPL_COMMIT_CS_VALIDATED;
}
- delayed_unlock = 1;
+
+ dwp = &dw_array[0];
+ dw_count = 0;
while (xfer_size) {
vm_page_t t, m;
+ dwp->dw_mask = 0;
+ clear_refmod = 0;
+
m = VM_PAGE_NULL;
if (upl->flags & UPL_LITE) {
- int pg_num;
+ unsigned int pg_num;
- pg_num = target_offset/PAGE_SIZE;
+ pg_num = (unsigned int) (target_offset/PAGE_SIZE);
+ assert(pg_num == target_offset/PAGE_SIZE);
if (lite_list[pg_num>>5] & (1 << (pg_num & 31))) {
lite_list[pg_num>>5] &= ~(1 << (pg_num & 31));
- m = vm_page_lookup(shadow_object, target_offset + (upl->offset - shadow_object->paging_offset));
+ if (!(upl->flags & UPL_KERNEL_OBJECT))
+ m = vm_page_lookup(shadow_object, target_offset + (upl->offset - shadow_object->paging_offset));
}
}
if (upl->flags & UPL_SHADOWED) {
t->pageout = FALSE;
- vm_page_free(t);
+ VM_PAGE_FREE(t);
if (m == VM_PAGE_NULL)
m = vm_page_lookup(shadow_object, target_offset + object->shadow_offset);
}
}
- if (m != VM_PAGE_NULL) {
-
- clear_refmod = 0;
-
- if (upl->flags & UPL_IO_WIRE) {
+ if ((upl->flags & UPL_KERNEL_OBJECT) || m == VM_PAGE_NULL)
+ goto commit_next_page;
- vm_page_unwire(m);
-
- if (page_list)
- page_list[entry].phys_addr = 0;
+ if (flags & UPL_COMMIT_CS_VALIDATED) {
+ /*
+ * CODE SIGNING:
+ * Set the code signing bits according to
+ * what the UPL says they should be.
+ */
+ m->cs_validated = page_list[entry].cs_validated;
+ m->cs_tainted = page_list[entry].cs_tainted;
+ }
+ if (upl->flags & UPL_IO_WIRE) {
- if (flags & UPL_COMMIT_SET_DIRTY)
- m->dirty = TRUE;
- else if (flags & UPL_COMMIT_CLEAR_DIRTY) {
- m->dirty = FALSE;
- if (m->cs_validated && !m->cs_tainted) {
- /*
- * CODE SIGNING:
- * This page is no longer dirty
- * but could have been modified,
- * so it will need to be
- * re-validated.
- */
- m->cs_validated = FALSE;
- vm_cs_validated_resets++;
- }
- clear_refmod |= VM_MEM_MODIFIED;
- }
- if (flags & UPL_COMMIT_INACTIVATE)
- vm_page_deactivate(m);
+ if (page_list)
+ page_list[entry].phys_addr = 0;
- if (clear_refmod)
- pmap_clear_refmod(m->phys_page, clear_refmod);
+ if (flags & UPL_COMMIT_SET_DIRTY)
+ m->dirty = TRUE;
+ else if (flags & UPL_COMMIT_CLEAR_DIRTY) {
+ m->dirty = FALSE;
- if (flags & UPL_COMMIT_ALLOW_ACCESS) {
- /*
- * We blocked access to the pages in this UPL.
- * Clear the "busy" bit and wake up any waiter
- * for this page.
- */
- PAGE_WAKEUP_DONE(m);
- }
- goto commit_next_page;
- }
- /*
- * make sure to clear the hardware
- * modify or reference bits before
- * releasing the BUSY bit on this page
- * otherwise we risk losing a legitimate
- * change of state
- */
- if (flags & UPL_COMMIT_CLEAR_DIRTY) {
- m->dirty = FALSE;
- if (m->cs_validated && !m->cs_tainted) {
+ if (! (flags & UPL_COMMIT_CS_VALIDATED) &&
+ m->cs_validated && !m->cs_tainted) {
/*
* CODE SIGNING:
* This page is no longer dirty
* re-validated.
*/
m->cs_validated = FALSE;
+#if DEVELOPMENT || DEBUG
vm_cs_validated_resets++;
+#endif
+ pmap_disconnect(m->phys_page);
}
clear_refmod |= VM_MEM_MODIFIED;
}
- if (clear_refmod)
- pmap_clear_refmod(m->phys_page, clear_refmod);
-
- if (page_list) {
- upl_page_info_t *p;
-
- p = &(page_list[entry]);
-
- if (p->phys_addr && p->pageout && !m->pageout) {
- m->busy = TRUE;
- m->pageout = TRUE;
- vm_page_wire(m);
- } else if (p->phys_addr &&
- !p->pageout && m->pageout &&
- !m->dump_cleaning) {
- m->pageout = FALSE;
+ if (flags & UPL_COMMIT_INACTIVATE) {
+ dwp->dw_mask |= DW_vm_page_deactivate_internal;
+ clear_refmod |= VM_MEM_REFERENCED;
+ }
+ if (upl->flags & UPL_ACCESS_BLOCKED) {
+ /*
+ * We blocked access to the pages in this UPL.
+ * Clear the "busy" bit and wake up any waiter
+ * for this page.
+ */
+ dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP);
+ }
+ if (m->absent) {
+ if (flags & UPL_COMMIT_FREE_ABSENT)
+ dwp->dw_mask |= DW_vm_page_free;
+ else {
m->absent = FALSE;
- m->overwriting = FALSE;
- vm_page_unwire(m);
-
- PAGE_WAKEUP_DONE(m);
+ dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP);
}
- page_list[entry].phys_addr = 0;
- }
- m->dump_cleaning = FALSE;
+ } else
+ dwp->dw_mask |= DW_vm_page_unwire;
- if (m->laundry)
- vm_pageout_throttle_up(m);
+ goto commit_next_page;
+ }
+ /*
+ * make sure to clear the hardware
+ * modify or reference bits before
+ * releasing the BUSY bit on this page
+ * otherwise we risk losing a legitimate
+ * change of state
+ */
+ if (flags & UPL_COMMIT_CLEAR_DIRTY) {
+ m->dirty = FALSE;
- if (m->pageout) {
- m->cleaning = FALSE;
- m->encrypted_cleaning = FALSE;
- m->pageout = FALSE;
-#if MACH_CLUSTER_STATS
- if (m->wanted) vm_pageout_target_collisions++;
+ if (! (flags & UPL_COMMIT_CS_VALIDATED) &&
+ m->cs_validated && !m->cs_tainted) {
+ /*
+ * CODE SIGNING:
+ * This page is no longer dirty
+ * but could have been modified,
+ * so it will need to be
+ * re-validated.
+ */
+ m->cs_validated = FALSE;
+#if DEVELOPMENT || DEBUG
+ vm_cs_validated_resets++;
#endif
- m->dirty = FALSE;
- if (m->cs_validated && !m->cs_tainted) {
- /*
- * CODE SIGNING:
- * This page is no longer dirty
- * but could have been modified,
- * so it will need to be
- * re-validated.
- */
- m->cs_validated = FALSE;
- vm_cs_validated_resets++;
- }
+ pmap_disconnect(m->phys_page);
+ }
+ clear_refmod |= VM_MEM_MODIFIED;
+ }
+ if (page_list) {
+ upl_page_info_t *p;
- if (m->pmapped && (pmap_disconnect(m->phys_page) & VM_MEM_MODIFIED))
- m->dirty = TRUE;
-
- if (m->dirty) {
- /*
- * page was re-dirtied after we started
- * the pageout... reactivate it since
- * we don't know whether the on-disk
- * copy matches what is now in memory
- */
- vm_page_unwire(m);
-
- if (upl->flags & UPL_PAGEOUT) {
- CLUSTER_STAT(vm_pageout_target_page_dirtied++;)
- VM_STAT_INCR(reactivations);
- DTRACE_VM2(pgrec, int, 1, (uint64_t *), NULL);
- }
- PAGE_WAKEUP_DONE(m);
- } else {
- /*
- * page has been successfully cleaned
- * go ahead and free it for other use
- */
+ p = &(page_list[entry]);
- if (m->object->internal) {
- DTRACE_VM2(anonpgout, int, 1, (uint64_t *), NULL);
- } else {
- DTRACE_VM2(fspgout, int, 1, (uint64_t *), NULL);
- }
+ if (p->phys_addr && p->pageout && !m->pageout) {
+ m->busy = TRUE;
+ m->pageout = TRUE;
- vm_page_free(m);
-
- if (upl->flags & UPL_PAGEOUT) {
- CLUSTER_STAT(vm_pageout_target_page_freed++;)
-
- if (page_list[entry].dirty) {
- VM_STAT_INCR(pageouts);
- DTRACE_VM2(pgout, int, 1, (uint64_t *), NULL);
- pgpgout_count++;
- }
- }
- }
- goto commit_next_page;
+ dwp->dw_mask |= DW_vm_page_wire;
+
+ } else if (p->phys_addr &&
+ !p->pageout && m->pageout &&
+ !m->dump_cleaning) {
+ m->pageout = FALSE;
+ m->absent = FALSE;
+ m->overwriting = FALSE;
+
+ dwp->dw_mask |= (DW_vm_page_unwire | DW_clear_busy | DW_PAGE_WAKEUP);
}
-#if MACH_CLUSTER_STATS
- if (m->wpmapped)
- m->dirty = pmap_is_modified(m->phys_page);
+ page_list[entry].phys_addr = 0;
+ }
+ m->dump_cleaning = FALSE;
+
+ if (m->laundry)
+ dwp->dw_mask |= DW_vm_pageout_throttle_up;
- if (m->dirty) vm_pageout_cluster_dirtied++;
- else vm_pageout_cluster_cleaned++;
- if (m->wanted) vm_pageout_cluster_collisions++;
+ if (m->pageout) {
+ m->cleaning = FALSE;
+ m->encrypted_cleaning = FALSE;
+ m->pageout = FALSE;
+#if MACH_CLUSTER_STATS
+ if (m->wanted) vm_pageout_target_collisions++;
#endif
m->dirty = FALSE;
- if (m->cs_validated && !m->cs_tainted) {
+
+ if (! (flags & UPL_COMMIT_CS_VALIDATED) &&
+ m->cs_validated && !m->cs_tainted) {
/*
* CODE SIGNING:
* This page is no longer dirty
* re-validated.
*/
m->cs_validated = FALSE;
+#if DEVELOPMENT || DEBUG
vm_cs_validated_resets++;
+#endif
+ pmap_disconnect(m->phys_page);
}
- if ((m->busy) && (m->cleaning)) {
- /*
- * the request_page_list case
+ if ((flags & UPL_COMMIT_SET_DIRTY) ||
+ (m->pmapped && (pmap_disconnect(m->phys_page) & VM_MEM_MODIFIED)))
+ m->dirty = TRUE;
+
+ if (m->dirty) {
+ /*
+ * page was re-dirtied after we started
+ * the pageout... reactivate it since
+ * we don't know whether the on-disk
+ * copy matches what is now in memory
*/
- m->absent = FALSE;
- m->overwriting = FALSE;
- m->busy = FALSE;
- } else if (m->overwriting) {
- /*
- * alternate request page list, write to
- * page_list case. Occurs when the original
- * page was wired at the time of the list
- * request
+ dwp->dw_mask |= (DW_vm_page_unwire | DW_clear_busy | DW_PAGE_WAKEUP);
+
+ if (upl->flags & UPL_PAGEOUT) {
+ CLUSTER_STAT(vm_pageout_target_page_dirtied++;)
+ VM_STAT_INCR(reactivations);
+ DTRACE_VM2(pgrec, int, 1, (uint64_t *), NULL);
+ }
+ } else {
+ /*
+ * page has been successfully cleaned
+ * go ahead and free it for other use
*/
- assert(m->wire_count != 0);
- vm_page_unwire(m);/* reactivates */
- m->overwriting = FALSE;
+
+ if (m->object->internal) {
+ DTRACE_VM2(anonpgout, int, 1, (uint64_t *), NULL);
+ } else {
+ DTRACE_VM2(fspgout, int, 1, (uint64_t *), NULL);
+ }
+ dwp->dw_mask |= DW_vm_page_free;
+
+ if (upl->flags & UPL_PAGEOUT) {
+ CLUSTER_STAT(vm_pageout_target_page_freed++;)
+
+ if (page_list[entry].dirty) {
+ VM_STAT_INCR(pageouts);
+ DTRACE_VM2(pgout, int, 1, (uint64_t *), NULL);
+ pgpgout_count++;
+ }
+ }
}
- m->cleaning = FALSE;
- m->encrypted_cleaning = FALSE;
+ goto commit_next_page;
+ }
+#if MACH_CLUSTER_STATS
+ if (m->wpmapped)
+ m->dirty = pmap_is_modified(m->phys_page);
+
+ if (m->dirty) vm_pageout_cluster_dirtied++;
+ else vm_pageout_cluster_cleaned++;
+ if (m->wanted) vm_pageout_cluster_collisions++;
+#endif
+ m->dirty = FALSE;
+ if (! (flags & UPL_COMMIT_CS_VALIDATED) &&
+ m->cs_validated && !m->cs_tainted) {
/*
- * It is a part of the semantic of COPYOUT_FROM
- * UPLs that a commit implies cache sync
- * between the vm page and the backing store
- * this can be used to strip the precious bit
- * as well as clean
+ * CODE SIGNING:
+ * This page is no longer dirty
+ * but could have been modified,
+ * so it will need to be
+ * re-validated.
*/
- if (upl->flags & UPL_PAGE_SYNC_DONE)
- m->precious = FALSE;
+ m->cs_validated = FALSE;
+#if DEVELOPMENT || DEBUG
+ vm_cs_validated_resets++;
+#endif
+ pmap_disconnect(m->phys_page);
+ }
- if (flags & UPL_COMMIT_SET_DIRTY)
- m->dirty = TRUE;
+ if ((m->busy) && (m->cleaning)) {
+ /*
+ * the request_page_list case
+ */
+ m->absent = FALSE;
+ m->overwriting = FALSE;
- if ((flags & UPL_COMMIT_INACTIVATE) && !m->clustered && !m->speculative) {
- vm_page_deactivate(m);
- } else if (!m->active && !m->inactive && !m->speculative) {
+ dwp->dw_mask |= DW_clear_busy;
- if (m->clustered)
- vm_page_speculate(m, TRUE);
- else if (m->reference)
- vm_page_activate(m);
- else
- vm_page_deactivate(m);
- }
- if (flags & UPL_COMMIT_ALLOW_ACCESS) {
- /*
- * We blocked access to the pages in this URL.
- * Clear the "busy" bit on this page before we
- * wake up any waiter.
- */
- m->busy = FALSE;
+ } else if (m->overwriting) {
+ /*
+ * alternate request page list, write to
+ * page_list case. Occurs when the original
+ * page was wired at the time of the list
+ * request
+ */
+ assert(VM_PAGE_WIRED(m));
+ m->overwriting = FALSE;
+
+ dwp->dw_mask |= DW_vm_page_unwire; /* reactivates */
+ }
+ m->cleaning = FALSE;
+ m->encrypted_cleaning = FALSE;
+
+ /*
+ * It is a part of the semantic of COPYOUT_FROM
+ * UPLs that a commit implies cache sync
+ * between the vm page and the backing store
+ * this can be used to strip the precious bit
+ * as well as clean
+ */
+ if ((upl->flags & UPL_PAGE_SYNC_DONE) || (flags & UPL_COMMIT_CLEAR_PRECIOUS))
+ m->precious = FALSE;
+
+ if (flags & UPL_COMMIT_SET_DIRTY)
+ m->dirty = TRUE;
+
+ if ((flags & UPL_COMMIT_INACTIVATE) && !m->clustered && !m->speculative) {
+ dwp->dw_mask |= DW_vm_page_deactivate_internal;
+ clear_refmod |= VM_MEM_REFERENCED;
+
+ } else if (!m->active && !m->inactive && !m->speculative) {
+
+ if (m->clustered || (flags & UPL_COMMIT_SPECULATE))
+ dwp->dw_mask |= DW_vm_page_speculate;
+ else if (m->reference)
+ dwp->dw_mask |= DW_vm_page_activate;
+ else {
+ dwp->dw_mask |= DW_vm_page_deactivate_internal;
+ clear_refmod |= VM_MEM_REFERENCED;
}
+ }
+ if (upl->flags & UPL_ACCESS_BLOCKED) {
/*
- * Wakeup any thread waiting for the page to be un-cleaning.
+ * We blocked access to the pages in this URL.
+ * Clear the "busy" bit on this page before we
+ * wake up any waiter.
*/
- PAGE_WAKEUP(m);
+ dwp->dw_mask |= DW_clear_busy;
}
+ /*
+ * Wakeup any thread waiting for the page to be un-cleaning.
+ */
+ dwp->dw_mask |= DW_PAGE_WAKEUP;
+
commit_next_page:
+ if (clear_refmod)
+ pmap_clear_refmod(m->phys_page, clear_refmod);
+
target_offset += PAGE_SIZE_64;
xfer_size -= PAGE_SIZE;
entry++;
- if (delayed_unlock++ > UPL_DELAYED_UNLOCK_LIMIT) {
- /*
- * pageout_scan takes the vm_page_lock_queues first
- * then tries for the object lock... to avoid what
- * is effectively a lock inversion, we'll go to the
- * trouble of taking them in that same order... otherwise
- * if this object contains the majority of the pages resident
- * in the UBC (or a small set of large objects actively being
- * worked on contain the majority of the pages), we could
- * cause the pageout_scan thread to 'starve' in its attempt
- * to find pages to move to the free queue, since it has to
- * successfully acquire the object lock of any candidate page
- * before it can steal/clean it.
- */
- vm_object_unlock(shadow_object);
- mutex_yield(&vm_page_queue_lock);
+ if (dwp->dw_mask) {
+ if (dwp->dw_mask & ~(DW_clear_busy | DW_PAGE_WAKEUP)) {
+ if (m->busy == FALSE) {
+ /*
+ * dw_do_work may need to drop the object lock
+ * if it does, we need the pages it's looking at to
+ * be held stable via the busy bit.
+ */
+ m->busy = TRUE;
+ dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP);
+ }
+ dwp->dw_m = m;
+ dwp++;
+ dw_count++;
- for (j = 0; ; j++) {
- if (vm_object_lock_try(shadow_object))
- break;
- vm_page_unlock_queues();
- mutex_pause(j);
- vm_page_lock_queues();
+ if (dw_count >= DELAYED_WORK_LIMIT) {
+ dw_do_work(shadow_object, &dw_array[0], dw_count);
+
+ dwp = &dw_array[0];
+ dw_count = 0;
+ }
+ } else {
+ if (dwp->dw_mask & DW_clear_busy)
+ m->busy = FALSE;
+
+ if (dwp->dw_mask & DW_PAGE_WAKEUP)
+ PAGE_WAKEUP(m);
}
- delayed_unlock = 1;
}
}
- if (delayed_unlock)
- vm_page_unlock_queues();
+ if (dw_count)
+ dw_do_work(shadow_object, &dw_array[0], dw_count);
occupied = 1;
occupied = 0;
}
if (occupied == 0) {
- if (upl->flags & UPL_COMMIT_NOTIFY_EMPTY)
+ /*
+ * If this UPL element belongs to a Vector UPL and is
+ * empty, then this is the right function to deallocate
+ * it. So go ahead set the *empty variable. The flag
+ * UPL_COMMIT_NOTIFY_EMPTY, from the caller's point of view
+ * should be considered relevant for the Vector UPL and not
+ * the internal UPLs.
+ */
+ if ((upl->flags & UPL_COMMIT_NOTIFY_EMPTY) || isVectorUPL)
*empty = TRUE;
- if (object == shadow_object) {
+ if (object == shadow_object && !(upl->flags & UPL_KERNEL_OBJECT)) {
/*
* this is not a paging object
* so we need to drop the paging reference
* that was taken when we created the UPL
* against this object
*/
- vm_object_paging_end(shadow_object);
+ vm_object_activity_end(shadow_object);
} else {
/*
* we dontated the paging reference to
vm_object_unlock(shadow_object);
if (object != shadow_object)
vm_object_unlock(object);
- upl_unlock(upl);
+
+ if(!isVectorUPL)
+ upl_unlock(upl);
+ else {
+ /*
+ * If we completed our operations on an UPL that is
+ * part of a Vectored UPL and if empty is TRUE, then
+ * we should go ahead and deallocate this UPL element.
+ * Then we check if this was the last of the UPL elements
+ * within that Vectored UPL. If so, set empty to TRUE
+ * so that in ubc_upl_commit_range or ubc_upl_commit, we
+ * can go ahead and deallocate the Vector UPL too.
+ */
+ if(*empty==TRUE) {
+ *empty = vector_upl_set_subupl(vector_upl, upl, 0);
+ upl_deallocate(upl);
+ }
+ goto process_upl_to_commit;
+ }
if (pgpgout_count) {
DTRACE_VM2(pgpgout, int, pgpgout_count, (uint64_t *), NULL);
int error,
boolean_t *empty)
{
- upl_size_t xfer_size;
+ upl_size_t xfer_size, subupl_size = size;
vm_object_t shadow_object;
vm_object_t object;
vm_object_offset_t target_offset;
+ upl_offset_t subupl_offset = offset;
int entry;
wpl_array_t lite_list;
int occupied;
- int delayed_unlock = 0;
- int j;
+ struct dw dw_array[DELAYED_WORK_LIMIT];
+ struct dw *dwp;
+ int dw_count, isVectorUPL = 0;
+ upl_t vector_upl = NULL;
*empty = FALSE;
return KERN_INVALID_ARGUMENT;
if ( (upl->flags & UPL_IO_WIRE) && !(error & UPL_ABORT_DUMP_PAGES) )
- return upl_commit_range(upl, offset, size, 0, NULL, 0, empty);
+ return upl_commit_range(upl, offset, size, UPL_COMMIT_FREE_ABSENT, NULL, 0, empty);
+
+ if((isVectorUPL = vector_upl_is_valid(upl))) {
+ vector_upl = upl;
+ upl_lock(vector_upl);
+ }
+ else
+ upl_lock(upl);
+
+process_upl_to_abort:
+ if(isVectorUPL) {
+ size = subupl_size;
+ offset = subupl_offset;
+ if(size == 0) {
+ upl_unlock(vector_upl);
+ return KERN_SUCCESS;
+ }
+ upl = vector_upl_subupl_byoffset(vector_upl, &offset, &size);
+ if(upl == NULL) {
+ upl_unlock(vector_upl);
+ return KERN_FAILURE;
+ }
+ subupl_size -= size;
+ subupl_offset += size;
+ }
+
+ *empty = FALSE;
+
+#if UPL_DEBUG
+ if (upl->upl_commit_index < UPL_DEBUG_COMMIT_RECORDS) {
+ (void) OSBacktrace(&upl->upl_commit_records[upl->upl_commit_index].c_retaddr[0], UPL_DEBUG_STACK_FRAMES);
+
+ upl->upl_commit_records[upl->upl_commit_index].c_beg = offset;
+ upl->upl_commit_records[upl->upl_commit_index].c_end = (offset + size);
+ upl->upl_commit_records[upl->upl_commit_index].c_aborted = 1;
+ upl->upl_commit_index++;
+ }
+#endif
if (upl->flags & UPL_DEVICE_MEMORY)
xfer_size = 0;
else if ((offset + size) <= upl->size)
xfer_size = size;
- else
- return KERN_FAILURE;
-
- upl_lock(upl);
+ else {
+ if(!isVectorUPL)
+ upl_unlock(upl);
+ else {
+ upl_unlock(vector_upl);
+ }
+ return KERN_FAILURE;
+ }
if (upl->flags & UPL_INTERNAL) {
lite_list = (wpl_array_t)
((((uintptr_t)upl) + sizeof(struct upl))
entry = offset/PAGE_SIZE;
target_offset = (vm_object_offset_t)offset;
- /*
- * pageout_scan takes the vm_page_lock_queues first
- * then tries for the object lock... to avoid what
- * is effectively a lock inversion, we'll go to the
- * trouble of taking them in that same order... otherwise
- * if this object contains the majority of the pages resident
- * in the UBC (or a small set of large objects actively being
- * worked on contain the majority of the pages), we could
- * cause the pageout_scan thread to 'starve' in its attempt
- * to find pages to move to the free queue, since it has to
- * successfully acquire the object lock of any candidate page
- * before it can steal/clean it.
- */
- for (j = 0; ; j++) {
- vm_page_lock_queues();
+ if (upl->flags & UPL_KERNEL_OBJECT)
+ vm_object_lock_shared(shadow_object);
+ else
+ vm_object_lock(shadow_object);
- if (vm_object_lock_try(shadow_object))
- break;
- vm_page_unlock_queues();
- mutex_pause(j);
+ if (upl->flags & UPL_ACCESS_BLOCKED) {
+ assert(shadow_object->blocked_access);
+ shadow_object->blocked_access = FALSE;
+ vm_object_wakeup(object, VM_OBJECT_EVENT_UNBLOCKED);
}
- delayed_unlock = 1;
+
+ dwp = &dw_array[0];
+ dw_count = 0;
+
+ if ((error & UPL_ABORT_DUMP_PAGES) && (upl->flags & UPL_KERNEL_OBJECT))
+ panic("upl_abort_range: kernel_object being DUMPED");
while (xfer_size) {
vm_page_t t, m;
+ dwp->dw_mask = 0;
+
m = VM_PAGE_NULL;
if (upl->flags & UPL_LITE) {
- int pg_num;
- pg_num = target_offset/PAGE_SIZE;
+ unsigned int pg_num;
+
+ pg_num = (unsigned int) (target_offset/PAGE_SIZE);
+ assert(pg_num == target_offset/PAGE_SIZE);
+
if (lite_list[pg_num>>5] & (1 << (pg_num & 31))) {
lite_list[pg_num>>5] &= ~(1 << (pg_num & 31));
- m = vm_page_lookup(shadow_object, target_offset +
- (upl->offset - shadow_object->paging_offset));
+ if ( !(upl->flags & UPL_KERNEL_OBJECT))
+ m = vm_page_lookup(shadow_object, target_offset +
+ (upl->offset - shadow_object->paging_offset));
}
}
if (upl->flags & UPL_SHADOWED) {
if ((t = vm_page_lookup(object, target_offset)) != VM_PAGE_NULL) {
t->pageout = FALSE;
- vm_page_free(t);
+ VM_PAGE_FREE(t);
if (m == VM_PAGE_NULL)
m = vm_page_lookup(shadow_object, target_offset + object->shadow_offset);
}
}
+ if ((upl->flags & UPL_KERNEL_OBJECT))
+ goto abort_next_page;
+
if (m != VM_PAGE_NULL) {
if (m->absent) {
if (error & UPL_ABORT_RESTART) {
m->restart = TRUE;
m->absent = FALSE;
- m->error = TRUE;
m->unusual = TRUE;
must_free = FALSE;
} else if (error & UPL_ABORT_UNAVAILABLE) {
m->cleaning = FALSE;
m->encrypted_cleaning = FALSE;
m->overwriting = FALSE;
- PAGE_WAKEUP_DONE(m);
+
+ dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP);
if (must_free == TRUE)
- vm_page_free(m);
+ dwp->dw_mask |= DW_vm_page_free;
else
- vm_page_activate(m);
+ dwp->dw_mask |= DW_vm_page_activate;
} else {
/*
* Handle the trusted pager throttle.
*/
if (m->laundry)
- vm_pageout_throttle_up(m);
+ dwp->dw_mask |= DW_vm_pageout_throttle_up;
if (m->pageout) {
assert(m->busy);
assert(m->wire_count == 1);
m->pageout = FALSE;
- vm_page_unwire(m);
+
+ dwp->dw_mask |= DW_vm_page_unwire;
}
m->dump_cleaning = FALSE;
m->cleaning = FALSE;
#endif /* MACH_PAGEMAP */
if (error & UPL_ABORT_DUMP_PAGES) {
pmap_disconnect(m->phys_page);
- vm_page_free(m);
+
+ dwp->dw_mask |= DW_vm_page_free;
} else {
if (error & UPL_ABORT_REFERENCE) {
/*
* file I/O, this is done by
* implementing an LRU on the inactive q
*/
- vm_page_lru(m);
+ dwp->dw_mask |= DW_vm_page_lru;
}
- PAGE_WAKEUP_DONE(m);
+ dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP);
}
}
}
- if (delayed_unlock++ > UPL_DELAYED_UNLOCK_LIMIT) {
- /*
- * pageout_scan takes the vm_page_lock_queues first
- * then tries for the object lock... to avoid what
- * is effectively a lock inversion, we'll go to the
- * trouble of taking them in that same order... otherwise
- * if this object contains the majority of the pages resident
- * in the UBC (or a small set of large objects actively being
- * worked on contain the majority of the pages), we could
- * cause the pageout_scan thread to 'starve' in its attempt
- * to find pages to move to the free queue, since it has to
- * successfully acquire the object lock of any candidate page
- * before it can steal/clean it.
- */
- vm_object_unlock(shadow_object);
- mutex_yield(&vm_page_queue_lock);
-
- for (j = 0; ; j++) {
- if (vm_object_lock_try(shadow_object))
- break;
- vm_page_unlock_queues();
- mutex_pause(j);
- vm_page_lock_queues();
- }
- delayed_unlock = 1;
- }
+abort_next_page:
target_offset += PAGE_SIZE_64;
xfer_size -= PAGE_SIZE;
entry++;
+
+ if (dwp->dw_mask) {
+ if (dwp->dw_mask & ~(DW_clear_busy | DW_PAGE_WAKEUP)) {
+ if (m->busy == FALSE) {
+ /*
+ * dw_do_work may need to drop the object lock
+ * if it does, we need the pages it's looking at to
+ * be held stable via the busy bit.
+ */
+ m->busy = TRUE;
+ dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP);
+ }
+ dwp->dw_m = m;
+ dwp++;
+ dw_count++;
+
+ if (dw_count >= DELAYED_WORK_LIMIT) {
+ dw_do_work(shadow_object, &dw_array[0], dw_count);
+
+ dwp = &dw_array[0];
+ dw_count = 0;
+ }
+ } else {
+ if (dwp->dw_mask & DW_clear_busy)
+ m->busy = FALSE;
+
+ if (dwp->dw_mask & DW_PAGE_WAKEUP)
+ PAGE_WAKEUP(m);
+ }
+ }
}
- if (delayed_unlock)
- vm_page_unlock_queues();
+ if (dw_count)
+ dw_do_work(shadow_object, &dw_array[0], dw_count);
occupied = 1;
occupied = 0;
}
if (occupied == 0) {
- if (upl->flags & UPL_COMMIT_NOTIFY_EMPTY)
+ /*
+ * If this UPL element belongs to a Vector UPL and is
+ * empty, then this is the right function to deallocate
+ * it. So go ahead set the *empty variable. The flag
+ * UPL_COMMIT_NOTIFY_EMPTY, from the caller's point of view
+ * should be considered relevant for the Vector UPL and
+ * not the internal UPLs.
+ */
+ if ((upl->flags & UPL_COMMIT_NOTIFY_EMPTY) || isVectorUPL)
*empty = TRUE;
- if (object == shadow_object) {
+ if (object == shadow_object && !(upl->flags & UPL_KERNEL_OBJECT)) {
/*
* this is not a paging object
* so we need to drop the paging reference
* that was taken when we created the UPL
* against this object
*/
- vm_object_paging_end(shadow_object);
+ vm_object_activity_end(shadow_object);
} else {
/*
* we dontated the paging reference to
vm_object_unlock(shadow_object);
if (object != shadow_object)
vm_object_unlock(object);
- upl_unlock(upl);
+
+ if(!isVectorUPL)
+ upl_unlock(upl);
+ else {
+ /*
+ * If we completed our operations on an UPL that is
+ * part of a Vectored UPL and if empty is TRUE, then
+ * we should go ahead and deallocate this UPL element.
+ * Then we check if this was the last of the UPL elements
+ * within that Vectored UPL. If so, set empty to TRUE
+ * so that in ubc_upl_abort_range or ubc_upl_abort, we
+ * can go ahead and deallocate the Vector UPL too.
+ */
+ if(*empty == TRUE) {
+ *empty = vector_upl_set_subupl(vector_upl, upl,0);
+ upl_deallocate(upl);
+ }
+ goto process_upl_to_abort;
+ }
return KERN_SUCCESS;
}
}
+unsigned int vm_object_iopl_request_sleep_for_cleaning = 0;
+
kern_return_t
vm_object_iopl_request(
vm_object_t object,
upl_t upl = NULL;
unsigned int entry;
wpl_array_t lite_list = NULL;
- int delayed_unlock = 0;
int no_zero_fill = FALSE;
u_int32_t psize;
kern_return_t ret;
vm_prot_t prot;
struct vm_object_fault_info fault_info;
-
+ struct dw dw_array[DELAYED_WORK_LIMIT];
+ struct dw *dwp;
+ int dw_count;
+ int dw_index;
if (cntrl_flags & ~UPL_VALID_FLAGS) {
/*
*/
return KERN_INVALID_VALUE;
}
- if (vm_lopage_poolsize == 0)
+ if (vm_lopage_needed == FALSE)
cntrl_flags &= ~UPL_NEED_32BIT_ADDR;
if (cntrl_flags & UPL_NEED_32BIT_ADDR) {
else
prot = VM_PROT_READ | VM_PROT_WRITE;
- if (((size/page_size) > MAX_UPL_TRANSFER) && !object->phys_contiguous)
- size = MAX_UPL_TRANSFER * page_size;
+ if (((size/PAGE_SIZE) > MAX_UPL_SIZE) && !object->phys_contiguous)
+ size = MAX_UPL_SIZE * PAGE_SIZE;
if (cntrl_flags & UPL_SET_INTERNAL) {
if (page_list_count != NULL)
- *page_list_count = MAX_UPL_TRANSFER;
+ *page_list_count = MAX_UPL_SIZE;
}
if (((cntrl_flags & UPL_SET_INTERNAL) && !(object->phys_contiguous)) &&
((page_list_count != NULL) && (*page_list_count != 0) && *page_list_count < (size/page_size)))
user_page_list = (upl_page_info_t *) (((uintptr_t)upl) + sizeof(struct upl));
lite_list = (wpl_array_t) (((uintptr_t)user_page_list) +
((psize / PAGE_SIZE) * sizeof(upl_page_info_t)));
+ if (size == 0) {
+ user_page_list = NULL;
+ lite_list = NULL;
+ }
} else {
upl = upl_create(UPL_CREATE_LITE, UPL_IO_WIRE, psize);
lite_list = (wpl_array_t) (((uintptr_t)upl) + sizeof(struct upl));
+ if (size == 0) {
+ lite_list = NULL;
+ }
}
if (user_page_list)
user_page_list[0].device = FALSE;
upl->map_object = object;
upl->size = size;
- vm_object_lock(object);
- vm_object_paging_begin(object);
+ if (object == kernel_object &&
+ !(cntrl_flags & (UPL_NEED_32BIT_ADDR | UPL_BLOCK_ACCESS))) {
+ upl->flags |= UPL_KERNEL_OBJECT;
+#if UPL_DEBUG
+ vm_object_lock(object);
+#else
+ vm_object_lock_shared(object);
+#endif
+ } else {
+ vm_object_lock(object);
+ vm_object_activity_begin(object);
+ }
/*
* paging in progress also protects the paging_offset
*/
upl->offset = offset + object->paging_offset;
+ if (cntrl_flags & UPL_BLOCK_ACCESS) {
+ /*
+ * The user requested that access to the pages in this URL
+ * be blocked until the UPL is commited or aborted.
+ */
+ upl->flags |= UPL_ACCESS_BLOCKED;
+ }
+
if (object->phys_contiguous) {
-#ifdef UPL_DEBUG
+#if UPL_DEBUG
queue_enter(&object->uplq, upl, upl_t, uplq);
#endif /* UPL_DEBUG */
+ if (upl->flags & UPL_ACCESS_BLOCKED) {
+ assert(!object->blocked_access);
+ object->blocked_access = TRUE;
+ }
+
vm_object_unlock(object);
/*
*/
upl->flags |= UPL_DEVICE_MEMORY;
- upl->highest_page = (offset + object->shadow_offset + size - 1)>>PAGE_SHIFT;
+ upl->highest_page = (ppnum_t) ((offset + object->shadow_offset + size - 1)>>PAGE_SHIFT);
if (user_page_list) {
- user_page_list[0].phys_addr = (offset + object->shadow_offset)>>PAGE_SHIFT;
+ user_page_list[0].phys_addr = (ppnum_t) ((offset + object->shadow_offset)>>PAGE_SHIFT);
user_page_list[0].device = TRUE;
}
if (page_list_count != NULL) {
}
return KERN_SUCCESS;
}
- /*
- * Protect user space from future COW operations
- */
- object->true_share = TRUE;
+ if (object != kernel_object) {
+ /*
+ * Protect user space from future COW operations
+ */
+ object->true_share = TRUE;
- if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC)
- object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
+ if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC)
+ object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
+ }
-#ifdef UPL_DEBUG
+#if UPL_DEBUG
queue_enter(&object->uplq, upl, upl_t, uplq);
#endif /* UPL_DEBUG */
- if (cntrl_flags & UPL_BLOCK_ACCESS) {
+ if (!(cntrl_flags & UPL_COPYOUT_FROM) &&
+ object->copy != VM_OBJECT_NULL) {
/*
- * The user requested that access to the pages in this URL
- * be blocked until the UPL is commited or aborted.
+ * Honor copy-on-write obligations
+ *
+ * The caller is gathering these pages and
+ * might modify their contents. We need to
+ * make sure that the copy object has its own
+ * private copies of these pages before we let
+ * the caller modify them.
+ *
+ * NOTE: someone else could map the original object
+ * after we've done this copy-on-write here, and they
+ * could then see an inconsistent picture of the memory
+ * while it's being modified via the UPL. To prevent this,
+ * we would have to block access to these pages until the
+ * UPL is released. We could use the UPL_BLOCK_ACCESS
+ * code path for that...
*/
- upl->flags |= UPL_ACCESS_BLOCKED;
+ vm_object_update(object,
+ offset,
+ size,
+ NULL,
+ NULL,
+ FALSE, /* should_return */
+ MEMORY_OBJECT_COPY_SYNC,
+ VM_PROT_NO_CHANGE);
+#if DEVELOPMENT || DEBUG
+ iopl_cow++;
+ iopl_cow_pages += size >> PAGE_SHIFT;
+#endif
}
+
+
entry = 0;
xfer_size = size;
fault_info.lo_offset = offset;
fault_info.hi_offset = offset + xfer_size;
fault_info.no_cache = FALSE;
+ fault_info.stealth = FALSE;
+ fault_info.mark_zf_absent = TRUE;
+
+ dwp = &dw_array[0];
+ dw_count = 0;
while (xfer_size) {
vm_fault_return_t result;
- int pg_num;
+ unsigned int pg_num;
+
+ dwp->dw_mask = 0;
dst_page = vm_page_lookup(object, dst_offset);
* If the page is encrypted, we need to decrypt it,
* so force a soft page fault.
*/
- if ((dst_page == VM_PAGE_NULL) || (dst_page->busy) ||
- (dst_page->encrypted) ||
- (dst_page->unusual && (dst_page->error ||
- dst_page->restart ||
- dst_page->absent ||
- dst_page->fictitious))) {
+ if (dst_page == VM_PAGE_NULL ||
+ dst_page->busy ||
+ dst_page->encrypted ||
+ dst_page->error ||
+ dst_page->restart ||
+ dst_page->absent ||
+ dst_page->fictitious) {
+
+ if (object == kernel_object)
+ panic("vm_object_iopl_request: missing/bad page in kernel object\n");
do {
vm_page_t top_page;
kern_return_t error_code;
int interruptible;
- if (delayed_unlock) {
- delayed_unlock = 0;
- vm_page_unlock_queues();
- }
if (cntrl_flags & UPL_SET_INTERRUPTIBLE)
interruptible = THREAD_ABORTSAFE;
else
fault_info.interruptible = interruptible;
fault_info.cluster_size = xfer_size;
+ vm_object_paging_begin(object);
+
result = vm_fault_page(object, dst_offset,
prot | VM_PROT_WRITE, FALSE,
&prot, &dst_page, &top_page,
case VM_FAULT_SUCCESS:
- PAGE_WAKEUP_DONE(dst_page);
+ if ( !dst_page->absent) {
+ PAGE_WAKEUP_DONE(dst_page);
+ } else {
+ /*
+ * we only get back an absent page if we
+ * requested that it not be zero-filled
+ * because we are about to fill it via I/O
+ *
+ * absent pages should be left BUSY
+ * to prevent them from being faulted
+ * into an address space before we've
+ * had a chance to complete the I/O on
+ * them since they may contain info that
+ * shouldn't be seen by the faulting task
+ */
+ }
/*
* Release paging references and
* top-level placeholder page, if any.
vm_object_paging_end(local_object);
}
}
+ vm_object_paging_end(object);
break;
case VM_FAULT_RETRY:
vm_object_lock(object);
- vm_object_paging_begin(object);
break;
case VM_FAULT_FICTITIOUS_SHORTAGE:
vm_page_more_fictitious();
vm_object_lock(object);
- vm_object_paging_begin(object);
break;
case VM_FAULT_MEMORY_SHORTAGE:
if (vm_page_wait(interruptible)) {
vm_object_lock(object);
- vm_object_paging_begin(object);
break;
}
/* fall thru */
case VM_FAULT_INTERRUPTED:
error_code = MACH_SEND_INTERRUPTED;
case VM_FAULT_MEMORY_ERROR:
+ memory_error:
ret = (error_code ? error_code: KERN_MEMORY_ERROR);
vm_object_lock(object);
- vm_object_paging_begin(object);
goto return_err;
+
+ case VM_FAULT_SUCCESS_NO_VM_PAGE:
+ /* success but no page: fail */
+ vm_object_paging_end(object);
+ vm_object_unlock(object);
+ goto memory_error;
+
+ default:
+ panic("vm_object_iopl_request: unexpected error"
+ " 0x%x from vm_fault_page()\n", result);
}
} while (result != VM_FAULT_SUCCESS);
+
}
+ if (upl->flags & UPL_KERNEL_OBJECT)
+ goto record_phys_addr;
+
+ if (dst_page->cleaning) {
+ /*
+ * Someone else is cleaning this page in place.as
+ * In theory, we should be able to proceed and use this
+ * page but they'll probably end up clearing the "busy"
+ * bit on it in upl_commit_range() but they didn't set
+ * it, so they would clear our "busy" bit and open
+ * us to race conditions.
+ * We'd better wait for the cleaning to complete and
+ * then try again.
+ */
+ vm_object_iopl_request_sleep_for_cleaning++;
+ PAGE_SLEEP(object, dst_page, THREAD_UNINT);
+ continue;
+ }
if ( (cntrl_flags & UPL_NEED_32BIT_ADDR) &&
dst_page->phys_page >= (max_valid_dma_address >> PAGE_SHIFT) ) {
vm_page_t low_page;
* we don't know whether that physical address has been
* handed out to some other 64 bit capable DMA device to use
*/
- if (dst_page->wire_count) {
+ if (VM_PAGE_WIRED(dst_page)) {
ret = KERN_PROTECTION_FAILURE;
goto return_err;
}
- if (delayed_unlock) {
- delayed_unlock = 0;
- vm_page_unlock_queues();
- }
low_page = vm_page_grablo();
if (low_page == VM_PAGE_NULL) {
refmod = pmap_disconnect(dst_page->phys_page);
else
refmod = 0;
- vm_page_copy(dst_page, low_page);
+
+ if ( !dst_page->absent)
+ vm_page_copy(dst_page, low_page);
low_page->reference = dst_page->reference;
low_page->dirty = dst_page->dirty;
+ low_page->absent = dst_page->absent;
if (refmod & VM_MEM_REFERENCED)
low_page->reference = TRUE;
if (refmod & VM_MEM_MODIFIED)
low_page->dirty = TRUE;
- vm_page_lock_queues();
vm_page_replace(low_page, object, dst_offset);
- /*
- * keep the queue lock since we're going to
- * need it immediately
- */
- delayed_unlock = 1;
dst_page = low_page;
/*
* BUSY... we don't need a PAGE_WAKEUP_DONE
* here, because we've never dropped the object lock
*/
- dst_page->busy = FALSE;
+ if ( !dst_page->absent)
+ dst_page->busy = FALSE;
}
- if (delayed_unlock == 0)
- vm_page_lock_queues();
-
- vm_page_wire(dst_page);
+ if ( !dst_page->busy)
+ dwp->dw_mask |= DW_vm_page_wire;
if (cntrl_flags & UPL_BLOCK_ACCESS) {
/*
assert(!dst_page->fictitious);
dst_page->busy = TRUE;
}
- pg_num = (dst_offset-offset)/PAGE_SIZE;
- lite_list[pg_num>>5] |= 1 << (pg_num & 31);
-
/*
* expect the page to be used
* page queues lock must be held to set 'reference'
*/
- dst_page->reference = TRUE;
+ dwp->dw_mask |= DW_set_reference;
if (!(cntrl_flags & UPL_COPYOUT_FROM))
dst_page->dirty = TRUE;
+record_phys_addr:
+ if (dst_page->busy)
+ upl->flags |= UPL_HAS_BUSY;
+
+ pg_num = (unsigned int) ((dst_offset-offset)/PAGE_SIZE);
+ assert(pg_num == (dst_offset-offset)/PAGE_SIZE);
+ lite_list[pg_num>>5] |= 1 << (pg_num & 31);
if (dst_page->phys_page > upl->highest_page)
upl->highest_page = dst_page->phys_page;
if (user_page_list) {
user_page_list[entry].phys_addr = dst_page->phys_page;
- user_page_list[entry].dirty = dst_page->dirty;
user_page_list[entry].pageout = dst_page->pageout;
user_page_list[entry].absent = dst_page->absent;
+ user_page_list[entry].dirty = dst_page->dirty;
user_page_list[entry].precious = dst_page->precious;
-
+ user_page_list[entry].device = FALSE;
if (dst_page->clustered == TRUE)
user_page_list[entry].speculative = dst_page->speculative;
else
user_page_list[entry].speculative = FALSE;
+ user_page_list[entry].cs_validated = dst_page->cs_validated;
+ user_page_list[entry].cs_tainted = dst_page->cs_tainted;
}
- /*
- * someone is explicitly grabbing this page...
- * update clustered and speculative state
- *
- */
- VM_PAGE_CONSUME_CLUSTERED(dst_page);
-
- if (delayed_unlock++ > UPL_DELAYED_UNLOCK_LIMIT) {
- mutex_yield(&vm_page_queue_lock);
- delayed_unlock = 1;
+ if (object != kernel_object) {
+ /*
+ * someone is explicitly grabbing this page...
+ * update clustered and speculative state
+ *
+ */
+ VM_PAGE_CONSUME_CLUSTERED(dst_page);
}
entry++;
dst_offset += PAGE_SIZE_64;
xfer_size -= PAGE_SIZE;
+
+ if (dwp->dw_mask) {
+ if (dst_page->busy == FALSE) {
+ /*
+ * dw_do_work may need to drop the object lock
+ * if it does, we need the pages it's looking at to
+ * be held stable via the busy bit.
+ */
+ dst_page->busy = TRUE;
+ dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP);
+ }
+ dwp->dw_m = dst_page;
+ dwp++;
+ dw_count++;
+
+ if (dw_count >= DELAYED_WORK_LIMIT) {
+ dw_do_work(object, &dw_array[0], dw_count);
+
+ dwp = &dw_array[0];
+ dw_count = 0;
+ }
+ }
}
- if (delayed_unlock)
- vm_page_unlock_queues();
+ if (dw_count)
+ dw_do_work(object, &dw_array[0], dw_count);
if (page_list_count != NULL) {
if (upl->flags & UPL_INTERNAL)
*/
vm_object_pmap_protect(object, offset, (vm_object_size_t)size,
PMAP_NULL, 0, VM_PROT_NONE);
+ assert(!object->blocked_access);
+ object->blocked_access = TRUE;
}
return KERN_SUCCESS;
return_err:
- if (delayed_unlock)
- vm_page_unlock_queues();
+ dw_index = 0;
for (; offset < dst_offset; offset += PAGE_SIZE) {
+ boolean_t need_unwire;
+
dst_page = vm_page_lookup(object, offset);
if (dst_page == VM_PAGE_NULL)
- panic("vm_object_iopl_request: Wired pages missing. \n");
+ panic("vm_object_iopl_request: Wired page missing. \n");
- vm_page_lockspin_queues();
- vm_page_unwire(dst_page);
+ /*
+ * if we've already processed this page in an earlier
+ * dw_do_work, we need to undo the wiring... we will
+ * leave the dirty and reference bits on if they
+ * were set, since we don't have a good way of knowing
+ * what the previous state was and we won't get here
+ * under any normal circumstances... we will always
+ * clear BUSY and wakeup any waiters via vm_page_free
+ * or PAGE_WAKEUP_DONE
+ */
+ need_unwire = TRUE;
+
+ if (dw_count) {
+ if (dw_array[dw_index].dw_m == dst_page) {
+ /*
+ * still in the deferred work list
+ * which means we haven't yet called
+ * vm_page_wire on this page
+ */
+ need_unwire = FALSE;
+
+ dw_index++;
+ dw_count--;
+ }
+ }
+ vm_page_lock_queues();
+
+ if (dst_page->absent) {
+ vm_page_free(dst_page);
+
+ need_unwire = FALSE;
+ } else {
+ if (need_unwire == TRUE)
+ vm_page_unwire(dst_page, TRUE);
+
+ PAGE_WAKEUP_DONE(dst_page);
+ }
vm_page_unlock_queues();
- VM_STAT_INCR(reactivations);
+ if (need_unwire == TRUE)
+ VM_STAT_INCR(reactivations);
+ }
+#if UPL_DEBUG
+ upl->upl_state = 2;
+#endif
+ if (! (upl->flags & UPL_KERNEL_OBJECT)) {
+ vm_object_activity_end(object);
}
- vm_object_paging_end(object);
vm_object_unlock(object);
upl_destroy(upl);
boolean_t upls_locked;
vm_object_t object1, object2;
- if (upl1 == UPL_NULL || upl2 == UPL_NULL || upl1 == upl2) {
+ if (upl1 == UPL_NULL || upl2 == UPL_NULL || upl1 == upl2 || ((upl1->flags & UPL_VECTOR)==UPL_VECTOR) || ((upl2->flags & UPL_VECTOR)==UPL_VECTOR)) {
return KERN_INVALID_ARGUMENT;
}
* Make each UPL point to the correct VM object, i.e. the
* object holding the pages that the UPL refers to...
*/
-#ifdef UPL_DEBUG
+#if UPL_DEBUG
queue_remove(&object1->uplq, upl1, upl_t, uplq);
queue_remove(&object2->uplq, upl2, upl_t, uplq);
#endif
upl1->map_object = object2;
upl2->map_object = object1;
-#ifdef UPL_DEBUG
+#if UPL_DEBUG
queue_enter(&object1->uplq, upl2, upl_t, uplq);
queue_enter(&object2->uplq, upl1, upl_t, uplq);
#endif
panic("vm_paging_map_init: kernel_map full\n");
}
map_entry->object.vm_object = kernel_object;
- map_entry->offset =
- page_map_offset - VM_MIN_KERNEL_ADDRESS;
+ map_entry->offset = page_map_offset;
vm_object_reference(kernel_object);
vm_map_unlock(kernel_map);
vm_object_t object,
vm_object_offset_t offset,
vm_map_size_t *size,
+ vm_prot_t protection,
boolean_t can_unlock_object)
{
kern_return_t kr;
vm_object_offset_t object_offset;
int i;
-
+
if (page != VM_PAGE_NULL && *size == PAGE_SIZE) {
assert(page->busy);
/*
PMAP_ENTER(kernel_pmap,
page_map_offset,
page,
- VM_PROT_DEFAULT,
+ protection,
((int) page->object->wimg_bits &
VM_WIMG_MASK),
TRUE);
object,
object_offset,
FALSE,
- VM_PROT_DEFAULT,
+ protection,
VM_PROT_ALL,
VM_INHERIT_NONE);
if (kr != KERN_SUCCESS) {
pmap_sync_page_data_phys(page->phys_page);
}
page->pmapped = TRUE;
- page->wpmapped = TRUE;
cache_attr = ((unsigned int) object->wimg_bits) & VM_WIMG_MASK;
//assert(pmap_verify_free(page->phys_page));
PMAP_ENTER(kernel_pmap,
*address + page_map_offset,
page,
- VM_PROT_DEFAULT,
+ protection,
cache_attr,
TRUE);
}
vm_paging_objects_mapped_slow++;
- vm_paging_pages_mapped_slow += map_size / PAGE_SIZE_64;
+ vm_paging_pages_mapped_slow += (unsigned long) (map_size / PAGE_SIZE_64);
return KERN_SUCCESS;
}
* for next time.
*/
assert(end - start == PAGE_SIZE);
- i = (start - vm_paging_base_address) >> PAGE_SHIFT;
+ i = (int) ((start - vm_paging_base_address) >> PAGE_SHIFT);
+ assert(i >= 0 && i < VM_PAGING_NUM_PAGES);
/* undo the pmap mapping */
pmap_remove(kernel_pmap, start, end);
unsigned char swap_crypt_test_page_decrypt[4096] __attribute__((aligned(4096)));
#endif /* DEBUG */
-extern u_long random(void);
-
/*
* Initialize the encryption context: key and key size.
*/
page->object,
page->offset,
&kernel_mapping_size,
+ VM_PROT_READ | VM_PROT_WRITE,
FALSE);
if (kr != KERN_SUCCESS) {
panic("vm_page_encrypt: "
page->object,
page->offset,
&kernel_mapping_size,
+ VM_PROT_READ | VM_PROT_WRITE,
FALSE);
if (kr != KERN_SUCCESS) {
panic("vm_page_decrypt: "
* and the decryption doesn't count.
*/
page->dirty = FALSE;
- if (page->cs_validated && !page->cs_tainted) {
- /*
- * CODE SIGNING:
- * This page is no longer dirty
- * but could have been modified,
- * so it will need to be
- * re-validated.
- */
- page->cs_validated = FALSE;
- vm_cs_validated_resets++;
- }
+ assert (page->cs_validated == FALSE);
pmap_clear_refmod(page->phys_page, VM_MEM_MODIFIED | VM_MEM_REFERENCED);
-
page->encrypted = FALSE;
/*
vm_object_paging_end(page->object);
}
+#if DEVELOPMENT || DEBUG
unsigned long upl_encrypt_upls = 0;
unsigned long upl_encrypt_pages = 0;
+#endif
/*
* ENCRYPTED SWAP:
upl_offset_t crypt_offset,
upl_size_t crypt_size)
{
- upl_size_t upl_size;
- upl_offset_t upl_offset;
+ upl_size_t upl_size, subupl_size=crypt_size;
+ upl_offset_t offset_in_upl, subupl_offset=crypt_offset;
vm_object_t upl_object;
+ vm_object_offset_t upl_offset;
vm_page_t page;
vm_object_t shadow_object;
vm_object_offset_t shadow_offset;
vm_object_offset_t paging_offset;
vm_object_offset_t base_offset;
+ int isVectorUPL = 0;
+ upl_t vector_upl = NULL;
+
+ if((isVectorUPL = vector_upl_is_valid(upl)))
+ vector_upl = upl;
+
+process_upl_to_encrypt:
+ if(isVectorUPL) {
+ crypt_size = subupl_size;
+ crypt_offset = subupl_offset;
+ upl = vector_upl_subupl_byoffset(vector_upl, &crypt_offset, &crypt_size);
+ if(upl == NULL)
+ panic("upl_encrypt: Accessing a sub-upl that doesn't exist\n");
+ subupl_size -= crypt_size;
+ subupl_offset += crypt_size;
+ }
+#if DEVELOPMENT || DEBUG
upl_encrypt_upls++;
upl_encrypt_pages += crypt_size / PAGE_SIZE;
-
+#endif
upl_object = upl->map_object;
upl_offset = upl->offset;
upl_size = upl->size;
assert(crypt_offset + crypt_size <= upl_size);
- for (upl_offset = 0;
- upl_offset < crypt_size;
- upl_offset += PAGE_SIZE) {
+ for (offset_in_upl = 0;
+ offset_in_upl < crypt_size;
+ offset_in_upl += PAGE_SIZE) {
page = vm_page_lookup(shadow_object,
- base_offset + upl_offset);
+ base_offset + offset_in_upl);
if (page == VM_PAGE_NULL) {
panic("upl_encrypt: "
"no page for (obj=%p,off=%lld+%d)!\n",
shadow_object,
base_offset,
- upl_offset);
+ offset_in_upl);
}
/*
* Disconnect the page from all pmaps, so that nobody can
pmap_disconnect(page->phys_page);
vm_page_encrypt(page, 0);
- if (shadow_object == vm_pageout_scan_wants_object) {
+ if (vm_object_lock_avoid(shadow_object)) {
/*
* Give vm_pageout_scan() a chance to convert more
* pages from "clean-in-place" to "clean-and-free",
* in this cluster.
*/
vm_object_unlock(shadow_object);
+ mutex_pause(2);
vm_object_lock(shadow_object);
}
}
vm_object_paging_end(shadow_object);
vm_object_unlock(shadow_object);
+
+ if(isVectorUPL && subupl_size)
+ goto process_upl_to_encrypt;
}
#else /* CRYPTO */
#endif /* CRYPTO */
+void
+vm_pageout_queue_steal(vm_page_t page, boolean_t queues_locked)
+{
+ boolean_t pageout;
+
+ pageout = page->pageout;
+
+ page->list_req_pending = FALSE;
+ page->cleaning = FALSE;
+ page->pageout = FALSE;
+
+ if (!queues_locked) {
+ vm_page_lockspin_queues();
+ }
+
+ /*
+ * need to drop the laundry count...
+ * we may also need to remove it
+ * from the I/O paging queue...
+ * vm_pageout_throttle_up handles both cases
+ *
+ * the laundry and pageout_queue flags are cleared...
+ */
+ vm_pageout_throttle_up(page);
+
+ if (pageout == TRUE) {
+ /*
+ * toss the wire count we picked up
+ * when we intially set this page up
+ * to be cleaned...
+ */
+ vm_page_unwire(page, TRUE);
+ }
+ vm_page_steal_pageout_page++;
+
+ if (!queues_locked) {
+ vm_page_unlock_queues();
+ }
+}
+
+upl_t
+vector_upl_create(vm_offset_t upl_offset)
+{
+ int vector_upl_size = sizeof(struct _vector_upl);
+ int i=0;
+ upl_t upl;
+ vector_upl_t vector_upl = (vector_upl_t)kalloc(vector_upl_size);
+
+ upl = upl_create(0,UPL_VECTOR,0);
+ upl->vector_upl = vector_upl;
+ upl->offset = upl_offset;
+ vector_upl->size = 0;
+ vector_upl->offset = upl_offset;
+ vector_upl->invalid_upls=0;
+ vector_upl->num_upls=0;
+ vector_upl->pagelist = NULL;
+
+ for(i=0; i < MAX_VECTOR_UPL_ELEMENTS ; i++) {
+ vector_upl->upl_iostates[i].size = 0;
+ vector_upl->upl_iostates[i].offset = 0;
+
+ }
+ return upl;
+}
+
+void
+vector_upl_deallocate(upl_t upl)
+{
+ if(upl) {
+ vector_upl_t vector_upl = upl->vector_upl;
+ if(vector_upl) {
+ if(vector_upl->invalid_upls != vector_upl->num_upls)
+ panic("Deallocating non-empty Vectored UPL\n");
+ kfree(vector_upl->pagelist,(sizeof(struct upl_page_info)*(vector_upl->size/PAGE_SIZE)));
+ vector_upl->invalid_upls=0;
+ vector_upl->num_upls = 0;
+ vector_upl->pagelist = NULL;
+ vector_upl->size = 0;
+ vector_upl->offset = 0;
+ kfree(vector_upl, sizeof(struct _vector_upl));
+ vector_upl = (vector_upl_t)0xdeadbeef;
+ }
+ else
+ panic("vector_upl_deallocate was passed a non-vectored upl\n");
+ }
+ else
+ panic("vector_upl_deallocate was passed a NULL upl\n");
+}
+
+boolean_t
+vector_upl_is_valid(upl_t upl)
+{
+ if(upl && ((upl->flags & UPL_VECTOR)==UPL_VECTOR)) {
+ vector_upl_t vector_upl = upl->vector_upl;
+ if(vector_upl == NULL || vector_upl == (vector_upl_t)0xdeadbeef || vector_upl == (vector_upl_t)0xfeedbeef)
+ return FALSE;
+ else
+ return TRUE;
+ }
+ return FALSE;
+}
+
+boolean_t
+vector_upl_set_subupl(upl_t upl,upl_t subupl, uint32_t io_size)
+{
+ if(vector_upl_is_valid(upl)) {
+ vector_upl_t vector_upl = upl->vector_upl;
+
+ if(vector_upl) {
+ if(subupl) {
+ if(io_size) {
+ if(io_size < PAGE_SIZE)
+ io_size = PAGE_SIZE;
+ subupl->vector_upl = (void*)vector_upl;
+ vector_upl->upl_elems[vector_upl->num_upls++] = subupl;
+ vector_upl->size += io_size;
+ upl->size += io_size;
+ }
+ else {
+ uint32_t i=0,invalid_upls=0;
+ for(i = 0; i < vector_upl->num_upls; i++) {
+ if(vector_upl->upl_elems[i] == subupl)
+ break;
+ }
+ if(i == vector_upl->num_upls)
+ panic("Trying to remove sub-upl when none exists");
+
+ vector_upl->upl_elems[i] = NULL;
+ invalid_upls = hw_atomic_add(&(vector_upl)->invalid_upls, 1);
+ if(invalid_upls == vector_upl->num_upls)
+ return TRUE;
+ else
+ return FALSE;
+ }
+ }
+ else
+ panic("vector_upl_set_subupl was passed a NULL upl element\n");
+ }
+ else
+ panic("vector_upl_set_subupl was passed a non-vectored upl\n");
+ }
+ else
+ panic("vector_upl_set_subupl was passed a NULL upl\n");
+
+ return FALSE;
+}
+
+void
+vector_upl_set_pagelist(upl_t upl)
+{
+ if(vector_upl_is_valid(upl)) {
+ uint32_t i=0;
+ vector_upl_t vector_upl = upl->vector_upl;
+
+ if(vector_upl) {
+ vm_offset_t pagelist_size=0, cur_upl_pagelist_size=0;
+
+ vector_upl->pagelist = (upl_page_info_array_t)kalloc(sizeof(struct upl_page_info)*(vector_upl->size/PAGE_SIZE));
+
+ for(i=0; i < vector_upl->num_upls; i++) {
+ cur_upl_pagelist_size = sizeof(struct upl_page_info) * vector_upl->upl_elems[i]->size/PAGE_SIZE;
+ bcopy(UPL_GET_INTERNAL_PAGE_LIST_SIMPLE(vector_upl->upl_elems[i]), (char*)vector_upl->pagelist + pagelist_size, cur_upl_pagelist_size);
+ pagelist_size += cur_upl_pagelist_size;
+ if(vector_upl->upl_elems[i]->highest_page > upl->highest_page)
+ upl->highest_page = vector_upl->upl_elems[i]->highest_page;
+ }
+ assert( pagelist_size == (sizeof(struct upl_page_info)*(vector_upl->size/PAGE_SIZE)) );
+ }
+ else
+ panic("vector_upl_set_pagelist was passed a non-vectored upl\n");
+ }
+ else
+ panic("vector_upl_set_pagelist was passed a NULL upl\n");
+
+}
+
+upl_t
+vector_upl_subupl_byindex(upl_t upl, uint32_t index)
+{
+ if(vector_upl_is_valid(upl)) {
+ vector_upl_t vector_upl = upl->vector_upl;
+ if(vector_upl) {
+ if(index < vector_upl->num_upls)
+ return vector_upl->upl_elems[index];
+ }
+ else
+ panic("vector_upl_subupl_byindex was passed a non-vectored upl\n");
+ }
+ return NULL;
+}
+
+upl_t
+vector_upl_subupl_byoffset(upl_t upl, upl_offset_t *upl_offset, upl_size_t *upl_size)
+{
+ if(vector_upl_is_valid(upl)) {
+ uint32_t i=0;
+ vector_upl_t vector_upl = upl->vector_upl;
+
+ if(vector_upl) {
+ upl_t subupl = NULL;
+ vector_upl_iostates_t subupl_state;
+
+ for(i=0; i < vector_upl->num_upls; i++) {
+ subupl = vector_upl->upl_elems[i];
+ subupl_state = vector_upl->upl_iostates[i];
+ if( *upl_offset <= (subupl_state.offset + subupl_state.size - 1)) {
+ /* We could have been passed an offset/size pair that belongs
+ * to an UPL element that has already been committed/aborted.
+ * If so, return NULL.
+ */
+ if(subupl == NULL)
+ return NULL;
+ if((subupl_state.offset + subupl_state.size) < (*upl_offset + *upl_size)) {
+ *upl_size = (subupl_state.offset + subupl_state.size) - *upl_offset;
+ if(*upl_size > subupl_state.size)
+ *upl_size = subupl_state.size;
+ }
+ if(*upl_offset >= subupl_state.offset)
+ *upl_offset -= subupl_state.offset;
+ else if(i)
+ panic("Vector UPL offset miscalculation\n");
+ return subupl;
+ }
+ }
+ }
+ else
+ panic("vector_upl_subupl_byoffset was passed a non-vectored UPL\n");
+ }
+ return NULL;
+}
+
+void
+vector_upl_get_submap(upl_t upl, vm_map_t *v_upl_submap, vm_offset_t *submap_dst_addr)
+{
+ *v_upl_submap = NULL;
+
+ if(vector_upl_is_valid(upl)) {
+ vector_upl_t vector_upl = upl->vector_upl;
+ if(vector_upl) {
+ *v_upl_submap = vector_upl->submap;
+ *submap_dst_addr = vector_upl->submap_dst_addr;
+ }
+ else
+ panic("vector_upl_get_submap was passed a non-vectored UPL\n");
+ }
+ else
+ panic("vector_upl_get_submap was passed a null UPL\n");
+}
+
+void
+vector_upl_set_submap(upl_t upl, vm_map_t submap, vm_offset_t submap_dst_addr)
+{
+ if(vector_upl_is_valid(upl)) {
+ vector_upl_t vector_upl = upl->vector_upl;
+ if(vector_upl) {
+ vector_upl->submap = submap;
+ vector_upl->submap_dst_addr = submap_dst_addr;
+ }
+ else
+ panic("vector_upl_get_submap was passed a non-vectored UPL\n");
+ }
+ else
+ panic("vector_upl_get_submap was passed a NULL UPL\n");
+}
+
+void
+vector_upl_set_iostate(upl_t upl, upl_t subupl, upl_offset_t offset, upl_size_t size)
+{
+ if(vector_upl_is_valid(upl)) {
+ uint32_t i = 0;
+ vector_upl_t vector_upl = upl->vector_upl;
+
+ if(vector_upl) {
+ for(i = 0; i < vector_upl->num_upls; i++) {
+ if(vector_upl->upl_elems[i] == subupl)
+ break;
+ }
+
+ if(i == vector_upl->num_upls)
+ panic("setting sub-upl iostate when none exists");
+
+ vector_upl->upl_iostates[i].offset = offset;
+ if(size < PAGE_SIZE)
+ size = PAGE_SIZE;
+ vector_upl->upl_iostates[i].size = size;
+ }
+ else
+ panic("vector_upl_set_iostate was passed a non-vectored UPL\n");
+ }
+ else
+ panic("vector_upl_set_iostate was passed a NULL UPL\n");
+}
+
+void
+vector_upl_get_iostate(upl_t upl, upl_t subupl, upl_offset_t *offset, upl_size_t *size)
+{
+ if(vector_upl_is_valid(upl)) {
+ uint32_t i = 0;
+ vector_upl_t vector_upl = upl->vector_upl;
+
+ if(vector_upl) {
+ for(i = 0; i < vector_upl->num_upls; i++) {
+ if(vector_upl->upl_elems[i] == subupl)
+ break;
+ }
+
+ if(i == vector_upl->num_upls)
+ panic("getting sub-upl iostate when none exists");
+
+ *offset = vector_upl->upl_iostates[i].offset;
+ *size = vector_upl->upl_iostates[i].size;
+ }
+ else
+ panic("vector_upl_get_iostate was passed a non-vectored UPL\n");
+ }
+ else
+ panic("vector_upl_get_iostate was passed a NULL UPL\n");
+}
+
+void
+vector_upl_get_iostate_byindex(upl_t upl, uint32_t index, upl_offset_t *offset, upl_size_t *size)
+{
+ if(vector_upl_is_valid(upl)) {
+ vector_upl_t vector_upl = upl->vector_upl;
+ if(vector_upl) {
+ if(index < vector_upl->num_upls) {
+ *offset = vector_upl->upl_iostates[index].offset;
+ *size = vector_upl->upl_iostates[index].size;
+ }
+ else
+ *offset = *size = 0;
+ }
+ else
+ panic("vector_upl_get_iostate_byindex was passed a non-vectored UPL\n");
+ }
+ else
+ panic("vector_upl_get_iostate_byindex was passed a NULL UPL\n");
+}
+
+upl_page_info_t *
+upl_get_internal_vectorupl_pagelist(upl_t upl)
+{
+ return ((vector_upl_t)(upl->vector_upl))->pagelist;
+}
+
+void *
+upl_get_internal_vectorupl(upl_t upl)
+{
+ return upl->vector_upl;
+}
+
vm_size_t
upl_get_internal_pagelist_offset(void)
{
return upl->highest_page;
}
-#ifdef UPL_DEBUG
-kern_return_t upl_ubc_alias_set(upl_t upl, unsigned int alias1, unsigned int alias2)
+upl_size_t upl_get_size(
+ upl_t upl)
+{
+ return upl->size;
+}
+
+#if UPL_DEBUG
+kern_return_t upl_ubc_alias_set(upl_t upl, uintptr_t alias1, uintptr_t alias2)
{
upl->ubc_alias1 = alias1;
upl->ubc_alias2 = alias2;
return KERN_SUCCESS;
}
-int upl_ubc_alias_get(upl_t upl, unsigned int * al, unsigned int * al2)
+int upl_ubc_alias_get(upl_t upl, uintptr_t * al, uintptr_t * al2)
{
if(al)
*al = upl->ubc_alias1;
projects / apple / xnu.git / blobdiff