-
-void wake_task_swapper(boolean_t now); /* forward */
-
-/*
- * wake_task_swapper: [exported]
- *
- * Wakes up task swapper if now == TRUE or if at least
- * task_swap_cycle_time has elapsed since the last call.
- *
- * NOTE: this function is not multithreaded, so if there is
- * more than one caller, it must be modified.
- */
-void
-wake_task_swapper(boolean_t now)
-{
- /* last_task_swap_cycle may require locking */
- if (now ||
- (sched_tick > (last_task_swap_cycle + task_swap_cycle_time))) {
- last_task_swap_cycle = sched_tick;
- if (task_swap_debug)
- printf("wake_task_swapper: waking swapper\n");
- thread_wakeup((event_t)&swapped_tasks); /* poke swapper */
- }
-}
-
-task_t pick_intask(void); /* forward */
-/*
- * pick_intask:
- * returns a task to be swapped in, or TASK_NULL if nothing suitable is found.
- *
- * current algorithm: Return the task that has been swapped out the
- * longest, as long as it is > min_swap_time. It will be dequeued
- * if actually swapped in.
- *
- * NOTE:**********************************************
- * task->swap_rss (the size when the task was swapped out) could be used to
- * further refine the selection. Another possibility would be to look at
- * the state of the thread(s) to see if the task/threads would run if they
- * were swapped in.
- * ***************************************************
- *
- * Locking: no locks held upon entry and exit.
- */
-task_t
-pick_intask(void)
-{
- register task_t task = TASK_NULL;
-
- task_swapper_lock();
- /* the oldest task is the first one */
- if (!queue_empty(&swapped_tasks)) {
- task = (task_t) queue_first(&swapped_tasks);
- assert(task != TASK_NULL);
- /* Make sure it's been out min_swap_time */
- if ((sched_tick - task->swap_stamp) < min_swap_time)
- task = TASK_NULL;
- }
- task_swapper_unlock();
- return(task);
-#if 0
- /*
- * This code looks at the entire list of swapped tasks, but since
- * it does not yet do anything but look at time swapped, we
- * can simply use the fact that the queue is ordered, and take
- * the first one off the queue.
- */
- task = (task_t)queue_first(&swapped_tasks);
- while (!queue_end(&swapped_tasks, (queue_entry_t)task)) {
- task_lock(task);
- tmp_time = sched_tick - task->swap_stamp;
- if (tmp_time > min_swap_time && tmp_time > time_swapped) {
- target_task = task;
- time_swapped = tmp_time;
- }
- task_unlock(task);
- task = (task_t)queue_next(&task->swapped_tasks);
- }
- task_swapper_unlock();
- return(target_task);
-#endif
-}
-
-task_t pick_outtask(void); /* forward */
-/*
- * pick_outtask:
- * returns a task to be swapped out, with a reference on the task,
- * or NULL if no suitable task is found.
- *
- * current algorithm:
- *
- * Examine all eligible tasks. While looking, use the first thread in
- * each task as an indication of the task's activity. Count up
- * "active" threads (those either runnable or sleeping). If the task
- * is active (by these criteria), swapped in, and resident
- * for at least min_res_time, then select the task with the largest
- * number of pages in memory. If there are less
- * than min_active_tasks active tasks in the system, then don't
- * swap anything out (this avoids swapping out the only running task
- * in the system, for example).
- *
- * NOTE: the task selected will not be removed from the eligible list.
- * This means that it will be selected again if it is not swapped
- * out, where it is removed from the list.
- *
- * Locking: no locks held upon entry and exit. Task_swapout_lock must be
- * taken before task locks.
- *
- * ***************************************************
- * TBD:
- * This algorithm only examines the first thread in the task. Currently, since
- * most swappable tasks in the system are single-threaded, this generalization
- * works reasonably well. However, the algorithm should be changed
- * to consider all threads in the task if more multi-threaded tasks were used.
- * ***************************************************
- */
-
-#ifdef TASK_SW_STATS
-int inactive_task_count = 0;
-int empty_task_count = 0;
-#endif /* TASK_SW_STATS */
-
-task_t
-pick_outtask(void)
-{
- register task_t task;
- register task_t target_task = TASK_NULL;
- unsigned long task_rss;
- unsigned long target_rss = 0;
- boolean_t wired;
- boolean_t active;
- int nactive = 0;
-
- task_swapout_lock();
- if (queue_empty(&eligible_tasks)) {
- /* not likely to happen */
- task_swapout_unlock();
- return(TASK_NULL);
- }
- task = (task_t)queue_first(&eligible_tasks);
- while (!queue_end(&eligible_tasks, (queue_entry_t)task)) {
- int s;
- register thread_act_t thr_act;
- thread_t th;
-
-
- task_lock(task);
- /*
- * Don't swap real-time tasks.
- * XXX Should we enforce that or can we let really critical
- * tasks use task_swappable() to make sure they never end up
- * n the eligible list ?
- */
- if (task->policy & POLICYCLASS_FIXEDPRI) {
- goto tryagain;
- }
- if (!task->active) {
- TASK_STATS_INCR(inactive_task_count);
- goto tryagain;
- }
- if (task->res_act_count == 0) {
- TASK_STATS_INCR(empty_task_count);
- goto tryagain;
- }
- assert(!queue_empty(&task->thr_acts));
- thr_act = (thread_act_t)queue_first(&task->thr_acts);
- active = FALSE;
- th = act_lock_thread(thr_act);
- s = splsched();
- if (th != THREAD_NULL)
- thread_lock(th);
- if ((th == THREAD_NULL) ||
- (th->state == TH_RUN) ||
- (th->state & TH_WAIT)) {
- /*
- * thread is "active": either runnable
- * or sleeping. Count it and examine
- * it further below.
- */
- nactive++;
- active = TRUE;
- }
- if (th != THREAD_NULL)
- thread_unlock(th);
- splx(s);
- act_unlock_thread(thr_act);
- if (active &&
- (task->swap_state == TASK_SW_IN) &&
- ((sched_tick - task->swap_stamp) > min_res_time)) {
- long rescount = pmap_resident_count(task->map->pmap);
- /*
- * thread must be "active", task must be swapped
- * in and resident for at least min_res_time
- */
-#if 0
-/* DEBUG Test round-robin strategy. Picking biggest task could cause extreme
- * unfairness to such large interactive programs as xterm. Instead, pick the
- * first task that has any pages resident:
- */
- if (rescount > 1) {
- task->ref_count++;
- target_task = task;
- task_unlock(task);
- task_swapout_unlock();
- return(target_task);
- }
-#else
- if (rescount > target_rss) {
- /*
- * task is not swapped, and it has the
- * largest rss seen so far.
- */
- task->ref_count++;
- target_rss = rescount;
- assert(target_task != task);
- if (target_task != TASK_NULL)
- task_deallocate(target_task);
- target_task = task;
- }
-#endif
- }
-tryagain:
- task_unlock(task);
- task = (task_t)queue_next(&task->swapped_tasks);
- }
- task_swapout_unlock();
- /* only swap out if there are at least min_active_tasks */
- if (nactive < min_active_tasks) {
- if (target_task != TASK_NULL) {
- task_deallocate(target_task);
- target_task = TASK_NULL;
- }
- }
- return(target_task);
-}
-
-#if TASK_SW_DEBUG
-void print_pid(task_t task, unsigned long n1, unsigned long n2,
- const char *comp, const char *inout); /* forward */
-void
-print_pid(
- task_t task,
- unsigned long n1,
- unsigned long n2,
- const char *comp,
- const char *inout)
-{
- long rescount;
- task_lock(task);
- rescount = pmap_resident_count(task->map->pmap);
- task_unlock(task);
- printf("task_swapper: swapped %s task %x; %d %s %d; res=%d\n",
- inout, task, n1, comp, n2, rescount);
-}
-#endif
-
-/*
- * task_swapper: [exported]
- *
- * Executes as a separate kernel thread.
- */
-#define MAX_LOOP 3
-void
-task_swapper(void)
-{
- task_t outtask, intask;
- int timeout;
- int loopcnt = 0;
- boolean_t start_swapping;
- boolean_t stop_swapping;
- int local_page_free_avg;
- extern int hz;
-
- thread_swappable(current_act(), FALSE);
- stack_privilege(current_thread());
-
- spllo();
-
- for (;;) {
- local_page_free_avg = vm_page_free_avg;
- while (TRUE) {
-#if 0
- if (task_swap_debug)
- printf("task_swapper: top of loop; cnt = %d\n",loopcnt);
-#endif
- intask = pick_intask();
-
- start_swapping = ((vm_pageout_rate_avg > swap_start_pageout_rate) ||
- (vm_grab_rate_avg > max_grab_rate));
- stop_swapping = (vm_pageout_rate_avg < swap_stop_pageout_rate);
-
- /*
- * If a lot of paging is going on, or another task should come
- * in but memory is tight, find something to swap out and start
- * it. Don't swap any task out if task swapping is disabled.
- * vm_page_queue_free_lock protects the vm globals.
- */
- outtask = TASK_NULL;
- if (start_swapping ||
- (!stop_swapping && intask &&
- ((local_page_free_avg / AVE_SCALE) < vm_page_free_target))
- ) {
- if (task_swap_enable &&
- (outtask = pick_outtask()) &&
- (task_swapout(outtask) == KERN_SUCCESS)) {
- unsigned long rss;
-#if TASK_SW_DEBUG
- if (task_swap_debug)
- print_pid(outtask, local_page_free_avg / AVE_SCALE,
- vm_page_free_target, "<",
- "out");
-#endif
- rss = outtask->swap_rss;
- if (outtask->swap_nswap == 1)
- rss /= 2; /* divide by 2 if never out */
- local_page_free_avg += (rss/short_avg_interval) * AVE_SCALE;
- }
- if (outtask != TASK_NULL)
- task_deallocate(outtask);
- }
-
- /*
- * If there is an eligible task to bring in and there are at
- * least vm_page_free_target free pages, swap it in. If task
- * swapping has been disabled, bring the task in anyway.
- */
- if (intask && ((local_page_free_avg / AVE_SCALE) >=
- vm_page_free_target ||
- stop_swapping || !task_swap_enable)) {
- if (task_swapin(intask, FALSE) == KERN_SUCCESS) {
- unsigned long rss;
-#if TASK_SW_DEBUG
- if (task_swap_debug)
- print_pid(intask, local_page_free_avg / AVE_SCALE,
- vm_page_free_target, ">=",
- "in");
-#endif
- rss = intask->swap_rss;
- if (intask->swap_nswap == 1)
- rss /= 2; /* divide by 2 if never out */
- local_page_free_avg -= (rss/short_avg_interval) * AVE_SCALE;
- }
- }
- /*
- * XXX
- * Here we have to decide whether to continue swapping
- * in and/or out before sleeping. The decision should
- * be made based on the previous action (swapin/out) and
- * current system parameters, such as paging rates and
- * demand.
- * The function, compute_vm_averages, which does these
- * calculations, depends on being called every second,
- * so we can't just do the same thing.
- */
- if (++loopcnt < MAX_LOOP)
- continue;
-
- /*
- * Arrange to be awakened if paging is still heavy or there are
- * any tasks partially or completely swapped out. (Otherwise,
- * the wakeup will come from the external trigger(s).)
- */
- timeout = 0;
- if (start_swapping)
- timeout = task_swap_cycle_time;
- else {
- task_swapper_lock();
- if (!queue_empty(&swapped_tasks))
- timeout = min_swap_time;
- task_swapper_unlock();
- }
- assert_wait((event_t)&swapped_tasks, THREAD_UNINT);
- if (timeout) {
- if (task_swap_debug)
- printf("task_swapper: set timeout of %d\n",
- timeout);
- thread_set_timeout(timeout, NSEC_PER_SEC);
- }
- if (task_swap_debug)
- printf("task_swapper: blocking\n");
- thread_block((void (*)(void)) 0);
- if (timeout) {
- thread_cancel_timeout(current_thread());
- }
- /* reset locals */
- loopcnt = 0;
- local_page_free_avg = vm_page_free_avg;
- }
- }
-}
-
-/* from BSD */
-#define ave(smooth, cnt, time) \
- smooth = ((time - 1) * (smooth) + ((cnt) * AVE_SCALE)) / (time)
-
-/*
- * We estimate the system paging load in more than one metric:
- * 1) the total number of calls into the function, vm_page_grab,
- * which allocates all page frames for real pages.
- * 2) the total number of pages paged in and out of paging files.
- * This is a measure of page cleaning and faulting from backing
- * store.
- *
- * When either metric passes a threshold, tasks are swapped out.
- */
-long last_grab_count = 0;
-long last_pageout_count = 0;
-
-/*
- * compute_vm_averages: [exported]
- *
- * This function is to be called once a second to calculate average paging
- * demand and average numbers of free pages for use by the task swapper.
- * Can also be used to wake up task swapper at desired thresholds.
- *
- * NOTE: this function is single-threaded, and requires locking if
- * ever there are multiple callers.
- */
-void
-compute_vm_averages(void)
-{
- extern unsigned long vm_page_grab_count;
- long grab_count, pageout_count;
- int i;
-
- ave(vm_page_free_avg, vm_page_free_count, short_avg_interval);
- ave(vm_page_free_longavg, vm_page_free_count, long_avg_interval);
-
- /*
- * NOTE: the vm_page_grab_count and vm_stat structure are
- * under control of vm_page_queue_free_lock. We're simply reading
- * memory here, and the numbers don't depend on each other, so
- * no lock is taken.
- */
-
- grab_count = vm_page_grab_count;
- pageout_count = 0;
- for (i = 0; i < NCPUS; i++) {
- pageout_count += vm_stat[i].pageouts;
- }
-
- ave(vm_pageout_rate_avg, pageout_count - last_pageout_count,
- short_avg_interval);
- ave(vm_pageout_rate_longavg, pageout_count - last_pageout_count,
- long_avg_interval);
- ave(vm_grab_rate_avg, grab_count - last_grab_count,
- short_avg_interval);
- last_grab_count = grab_count;
- last_pageout_count = pageout_count;
-
- /*
- * Adjust swap_{start,stop}_pageout_rate to the paging rate peak.
- * This is an attempt to find the optimum paging rates at which
- * to trigger task swapping on or off to regulate paging activity,
- * depending on the hardware capacity.
- */
- if (vm_pageout_rate_avg > vm_pageout_rate_peakavg) {
- unsigned int desired_max;
-
- vm_pageout_rate_peakavg = vm_pageout_rate_avg;
- swap_start_pageout_rate =
- vm_pageout_rate_peakavg * swap_pageout_high_water_mark / 100;
- swap_stop_pageout_rate =
- vm_pageout_rate_peakavg * swap_pageout_low_water_mark / 100;
- }
-
-#if TASK_SW_DEBUG
- /*
- * For measurements, allow fixed values.
- */
- if (fixed_swap_start_pageout_rate)
- swap_start_pageout_rate = fixed_swap_start_pageout_rate;
- if (fixed_swap_stop_pageout_rate)
- swap_stop_pageout_rate = fixed_swap_stop_pageout_rate;
-#endif /* TASK_SW_DEBUG */
-
-#if TASK_SW_DEBUG
- if (task_swap_stats)
- printf("vm_avgs: pageout_rate: %d %d (on/off: %d/%d); page_free: %d %d (tgt: %d)\n",
- vm_pageout_rate_avg / AVE_SCALE,
- vm_pageout_rate_longavg / AVE_SCALE,
- swap_start_pageout_rate / AVE_SCALE,
- swap_stop_pageout_rate / AVE_SCALE,
- vm_page_free_avg / AVE_SCALE,
- vm_page_free_longavg / AVE_SCALE,
- vm_page_free_target);
-#endif /* TASK_SW_DEBUG */
-
- if (vm_page_free_avg / AVE_SCALE <= vm_page_free_target) {
- if (task_swap_on) {
- /* The following is a delicate attempt to balance the
- * need for reasonably rapid response to system
- * thrashing, with the equally important desire to
- * prevent the onset of swapping simply because of a
- * short burst of paging activity.
- */
- if ((vm_pageout_rate_longavg > swap_stop_pageout_rate) &&
- (vm_pageout_rate_avg > swap_start_pageout_rate) ||
- (vm_pageout_rate_avg > vm_pageout_rate_peakavg) ||
- (vm_grab_rate_avg > max_grab_rate))
- wake_task_swapper(FALSE);
- }
- } else /* page demand is low; should consider swapin */ {
- if (tasks_swapped_out != 0)
- wake_task_swapper(TRUE);
- }
-}
-
-void
-task_swapout_eligible(task_t task)
-{
-#if TASK_SW_DEBUG
- task_swapper_lock();
- if (task_swap_debug && on_swapped_list(task)) {
- printf("swapout_eligible: task 0x%X on swapped list\n", task);
- Debugger("");
- }
- task_swapper_unlock();
-#endif
- task_swapout_lock();
- task_lock(task);
-#if TASK_SW_DEBUG
- if (task->swap_flags & TASK_SW_ELIGIBLE) {
- printf("swapout_eligible: task 0x%X already eligible\n", task);
- }
-#endif /* TASK_SW_DEBUG */
- if ((task->swap_state == TASK_SW_IN) &&
- ((task->swap_flags & TASK_SW_ELIGIBLE) == 0)) {
- queue_enter(&eligible_tasks,task,task_t,swapped_tasks);
- task->swap_flags |= TASK_SW_ELIGIBLE;
- }
- task_unlock(task);
- task_swapout_unlock();
-}
-
-void
-task_swapout_ineligible(task_t task)
-{
-#if TASK_SW_DEBUG
- task_swapper_lock();
- if (task_swap_debug && on_swapped_list(task)) {
- printf("swapout_ineligible: task 0x%X on swapped list\n", task);
- Debugger("");
- }
- task_swapper_unlock();
-#endif
- task_swapout_lock();
- task_lock(task);
-#if TASK_SW_DEBUG
- if (!(task->swap_flags & TASK_SW_ELIGIBLE))
- printf("swapout_ineligible: task 0x%X already inel.\n", task);
-#endif /* TASK_SW_DEBUG */
- if ((task->swap_state != TASK_SW_IN) &&
- (task->swap_flags & TASK_SW_ELIGIBLE)) {
- queue_remove(&eligible_tasks, task, task_t, swapped_tasks);
- task->swap_flags &= ~TASK_SW_ELIGIBLE;
- }
- task_unlock(task);
- task_swapout_unlock();
-}
-
-int task_swap_ast_aborted = 0;
-
-/*
- * Process an AST_SWAPOUT.
- */
-void
-swapout_ast()
-{
- spl_t s;
- thread_act_t act;
- thread_t thread;
-
- act = current_act();
-
- /*
- * Task is being swapped out. First mark it as suspended
- * and halted, then call thread_swapout_enqueue to put
- * the thread on the queue for task_swap_swapout_threads
- * to swap out the thread.
- */
- /*
- * Don't swap unswappable threads
- */
- thread = act_lock_thread(act);
- s = splsched();
- if (thread)
- thread_lock(thread);
- if ((act->ast & AST_SWAPOUT) == 0) {
- /*
- * Race with task_swapin. Abort swapout.
- */
- task_swap_ast_aborted++; /* not locked XXX */
- if (thread)
- thread_unlock(thread);
- splx(s);
- act_unlock_thread(act);
- } else if (act->swap_state == TH_SW_IN) {
- /*
- * Mark swap_state as TH_SW_TASK_SWAPPING to avoid
- * race with thread swapper, which will only
- * swap thread if swap_state is TH_SW_IN.
- * This way, the thread can only be swapped by
- * the task swapping mechanism.
- */
- act->swap_state |= TH_SW_TASK_SWAPPING;
- /* assert(act->suspend_count == 0); XXX ? */
- if (thread)
- thread_unlock(thread);
- if (act->suspend_count++ == 0) /* inline thread_hold */
- install_special_handler(act);
- /* self->state |= TH_HALTED; */
- thread_ast_clear(act, AST_SWAPOUT);
- /*
- * Initialize the swap_queue fields to allow an extra
- * queue_remove() in task_swapin if we lose the race
- * (task_swapin can be called before we complete
- * thread_swapout_enqueue).
- */
- queue_init((queue_t) &act->swap_queue);
- splx(s);
- act_unlock_thread(act);
- /* this must be called at normal interrupt level */
- thread_swapout_enqueue(act);
- } else {
- /* thread isn't swappable; continue running */
- assert(act->swap_state == TH_SW_UNSWAPPABLE);
- if (thread)
- thread_unlock(thread);
- thread_ast_clear(act, AST_SWAPOUT);
- splx(s);
- act_unlock_thread(act);
- }
-}
-
-#endif /* TASK_SWAPPER */
projects / apple / xnu.git / blobdiff