-/* Number of maximum jetsam threads configured */
-int max_jetsam_threads = JETSAM_THREADS_LIMIT;
-
-/*
- * Global switch for enabling fast jetsam. Fast jetsam is
- * hooked up via the system_override() system call. It has the
- * following effects:
- * - Raise the jetsam threshold ("clear-the-deck")
- * - Enabled parallel jetsam on eligible devices
- */
-int fast_jetsam_enabled = 0;
-
-/* Routine to find the jetsam state structure for the current jetsam thread */
-static inline struct jetsam_thread_state *
-jetsam_current_thread(void)
-{
- for (int thr_id = 0; thr_id < max_jetsam_threads; thr_id++) {
- if (jetsam_threads[thr_id].thread == current_thread()) {
- return &(jetsam_threads[thr_id]);
- }
- }
- panic("jetsam_current_thread() is being called from a non-jetsam thread\n");
- /* Contol should not reach here */
- return NULL;
-}
-
-
-__private_extern__ void
-memorystatus_init(void)
-{
- kern_return_t result;
- int i;
-
-#if CONFIG_FREEZE
- memorystatus_freeze_jetsam_band = JETSAM_PRIORITY_UI_SUPPORT;
- memorystatus_frozen_processes_max = FREEZE_PROCESSES_MAX;
- memorystatus_frozen_shared_mb_max = ((MAX_FROZEN_SHARED_MB_PERCENT * max_task_footprint_mb) / 100); /* 10% of the system wide task limit */
- memorystatus_freeze_shared_mb_per_process_max = (memorystatus_frozen_shared_mb_max / 4);
- memorystatus_freeze_pages_min = FREEZE_PAGES_MIN;
- memorystatus_freeze_pages_max = FREEZE_PAGES_MAX;
- memorystatus_max_frozen_demotions_daily = MAX_FROZEN_PROCESS_DEMOTIONS;
- memorystatus_thaw_count_demotion_threshold = MIN_THAW_DEMOTION_THRESHOLD;
-#endif
-
-#if DEVELOPMENT || DEBUG
- disconnect_page_mappings_lck_grp_attr = lck_grp_attr_alloc_init();
- disconnect_page_mappings_lck_grp = lck_grp_alloc_init("disconnect_page_mappings", disconnect_page_mappings_lck_grp_attr);
-
- lck_mtx_init(&disconnect_page_mappings_mutex, disconnect_page_mappings_lck_grp, NULL);
-
- if (kill_on_no_paging_space == TRUE) {
- max_kill_priority = JETSAM_PRIORITY_MAX;
- }
-#endif
-
-
- /* Init buckets */
- for (i = 0; i < MEMSTAT_BUCKET_COUNT; i++) {
- TAILQ_INIT(&memstat_bucket[i].list);
- memstat_bucket[i].count = 0;
- }
- memorystatus_idle_demotion_call = thread_call_allocate((thread_call_func_t)memorystatus_perform_idle_demotion, NULL);
-
-#if CONFIG_JETSAM
- nanoseconds_to_absolutetime((uint64_t)DEFERRED_IDLE_EXIT_TIME_SECS * NSEC_PER_SEC, &memorystatus_sysprocs_idle_delay_time);
- nanoseconds_to_absolutetime((uint64_t)DEFERRED_IDLE_EXIT_TIME_SECS * NSEC_PER_SEC, &memorystatus_apps_idle_delay_time);
-
- /* Apply overrides */
- PE_get_default("kern.jetsam_delta", &delta_percentage, sizeof(delta_percentage));
- if (delta_percentage == 0) {
- delta_percentage = 5;
- }
- assert(delta_percentage < 100);
- PE_get_default("kern.jetsam_critical_threshold", &critical_threshold_percentage, sizeof(critical_threshold_percentage));
- assert(critical_threshold_percentage < 100);
- PE_get_default("kern.jetsam_idle_offset", &idle_offset_percentage, sizeof(idle_offset_percentage));
- assert(idle_offset_percentage < 100);
- PE_get_default("kern.jetsam_pressure_threshold", &pressure_threshold_percentage, sizeof(pressure_threshold_percentage));
- assert(pressure_threshold_percentage < 100);
- PE_get_default("kern.jetsam_freeze_threshold", &freeze_threshold_percentage, sizeof(freeze_threshold_percentage));
- assert(freeze_threshold_percentage < 100);
-
- if (!PE_parse_boot_argn("jetsam_aging_policy", &jetsam_aging_policy,
- sizeof(jetsam_aging_policy))) {
- if (!PE_get_default("kern.jetsam_aging_policy", &jetsam_aging_policy,
- sizeof(jetsam_aging_policy))) {
- jetsam_aging_policy = kJetsamAgingPolicyLegacy;
- }
- }
-
- if (jetsam_aging_policy > kJetsamAgingPolicyMax) {
- jetsam_aging_policy = kJetsamAgingPolicyLegacy;
- }
-
- switch (jetsam_aging_policy) {
- case kJetsamAgingPolicyNone:
- system_procs_aging_band = JETSAM_PRIORITY_IDLE;
- applications_aging_band = JETSAM_PRIORITY_IDLE;
- break;
-
- case kJetsamAgingPolicyLegacy:
- /*
- * Legacy behavior where some daemons get a 10s protection once
- * AND only before the first clean->dirty->clean transition before
- * going into IDLE band.
- */
- system_procs_aging_band = JETSAM_PRIORITY_AGING_BAND1;
- applications_aging_band = JETSAM_PRIORITY_IDLE;
- break;
-
- case kJetsamAgingPolicySysProcsReclaimedFirst:
- system_procs_aging_band = JETSAM_PRIORITY_AGING_BAND1;
- applications_aging_band = JETSAM_PRIORITY_AGING_BAND2;
- break;
-
- case kJetsamAgingPolicyAppsReclaimedFirst:
- system_procs_aging_band = JETSAM_PRIORITY_AGING_BAND2;
- applications_aging_band = JETSAM_PRIORITY_AGING_BAND1;
- break;
-
- default:
- break;
- }
-
- /*
- * The aging bands cannot overlap with the JETSAM_PRIORITY_ELEVATED_INACTIVE
- * band and must be below it in priority. This is so that we don't have to make
- * our 'aging' code worry about a mix of processes, some of which need to age
- * and some others that need to stay elevated in the jetsam bands.
- */
- assert(JETSAM_PRIORITY_ELEVATED_INACTIVE > system_procs_aging_band);
- assert(JETSAM_PRIORITY_ELEVATED_INACTIVE > applications_aging_band);
-
- /* Take snapshots for idle-exit kills by default? First check the boot-arg... */
- if (!PE_parse_boot_argn("jetsam_idle_snapshot", &memorystatus_idle_snapshot, sizeof(memorystatus_idle_snapshot))) {
- /* ...no boot-arg, so check the device tree */
- PE_get_default("kern.jetsam_idle_snapshot", &memorystatus_idle_snapshot, sizeof(memorystatus_idle_snapshot));
- }
-
- memorystatus_delta = delta_percentage * atop_64(max_mem) / 100;
- memorystatus_available_pages_critical_idle_offset = idle_offset_percentage * atop_64(max_mem) / 100;
- memorystatus_available_pages_critical_base = (critical_threshold_percentage / delta_percentage) * memorystatus_delta;
- memorystatus_policy_more_free_offset_pages = (policy_more_free_offset_percentage / delta_percentage) * memorystatus_delta;
-
- /* Jetsam Loop Detection */
- if (max_mem <= (512 * 1024 * 1024)) {
- /* 512 MB devices */
- memorystatus_jld_eval_period_msecs = 8000; /* 8000 msecs == 8 second window */
- } else {
- /* 1GB and larger devices */
- memorystatus_jld_eval_period_msecs = 6000; /* 6000 msecs == 6 second window */
- }
-
- memorystatus_jld_enabled = TRUE;
-
- /* No contention at this point */
- memorystatus_update_levels_locked(FALSE);
-
-#endif /* CONFIG_JETSAM */
-
- memorystatus_jetsam_snapshot_max = maxproc;
-
- memorystatus_jetsam_snapshot_size = sizeof(memorystatus_jetsam_snapshot_t) +
- (sizeof(memorystatus_jetsam_snapshot_entry_t) * memorystatus_jetsam_snapshot_max);
-
- memorystatus_jetsam_snapshot =
- (memorystatus_jetsam_snapshot_t*)kalloc(memorystatus_jetsam_snapshot_size);
- if (!memorystatus_jetsam_snapshot) {
- panic("Could not allocate memorystatus_jetsam_snapshot");
- }
-
- memorystatus_jetsam_snapshot_copy =
- (memorystatus_jetsam_snapshot_t*)kalloc(memorystatus_jetsam_snapshot_size);
- if (!memorystatus_jetsam_snapshot_copy) {
- panic("Could not allocate memorystatus_jetsam_snapshot_copy");
- }
-
- nanoseconds_to_absolutetime((uint64_t)JETSAM_SNAPSHOT_TIMEOUT_SECS * NSEC_PER_SEC, &memorystatus_jetsam_snapshot_timeout);
-
- memset(&memorystatus_at_boot_snapshot, 0, sizeof(memorystatus_jetsam_snapshot_t));
-
-#if CONFIG_FREEZE
- memorystatus_freeze_threshold = (freeze_threshold_percentage / delta_percentage) * memorystatus_delta;
-#endif
-
- /* Check the boot-arg to see if fast jetsam is allowed */
- if (!PE_parse_boot_argn("fast_jetsam_enabled", &fast_jetsam_enabled, sizeof(fast_jetsam_enabled))) {
- fast_jetsam_enabled = 0;
- }
-
- /* Check the boot-arg to configure the maximum number of jetsam threads */
- if (!PE_parse_boot_argn("max_jetsam_threads", &max_jetsam_threads, sizeof(max_jetsam_threads))) {
- max_jetsam_threads = JETSAM_THREADS_LIMIT;
- }
-
- /* Restrict the maximum number of jetsam threads to JETSAM_THREADS_LIMIT */
- if (max_jetsam_threads > JETSAM_THREADS_LIMIT) {
- max_jetsam_threads = JETSAM_THREADS_LIMIT;
- }
-
- /* For low CPU systems disable fast jetsam mechanism */
- if (vm_pageout_state.vm_restricted_to_single_processor == TRUE) {
- max_jetsam_threads = 1;
- fast_jetsam_enabled = 0;
- }
-
- /* Initialize the jetsam_threads state array */
- jetsam_threads = kalloc(sizeof(struct jetsam_thread_state) * max_jetsam_threads);
-
- /* Initialize all the jetsam threads */
- for (i = 0; i < max_jetsam_threads; i++) {
- result = kernel_thread_start_priority(memorystatus_thread, NULL, 95 /* MAXPRI_KERNEL */, &jetsam_threads[i].thread);
- if (result == KERN_SUCCESS) {
- jetsam_threads[i].inited = FALSE;
- jetsam_threads[i].index = i;
- thread_deallocate(jetsam_threads[i].thread);
- } else {
- panic("Could not create memorystatus_thread %d", i);
- }
- }
-}
-
-/* Centralised for the purposes of allowing panic-on-jetsam */
-extern void
-vm_run_compactor(void);
-
-/*
- * The jetsam no frills kill call
- * Return: 0 on success
- * error code on failure (EINVAL...)
- */
-static int
-jetsam_do_kill(proc_t p, int jetsam_flags, os_reason_t jetsam_reason)
-{
- int error = 0;
- error = exit_with_reason(p, W_EXITCODE(0, SIGKILL), (int *)NULL, FALSE, FALSE, jetsam_flags, jetsam_reason);
- return error;
-}
-
-/*
- * Wrapper for processes exiting with memorystatus details
- */
-static boolean_t
-memorystatus_do_kill(proc_t p, uint32_t cause, os_reason_t jetsam_reason)
-{
- int error = 0;
- __unused pid_t victim_pid = p->p_pid;
-
- KERNEL_DEBUG_CONSTANT((BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_DO_KILL)) | DBG_FUNC_START,
- victim_pid, cause, vm_page_free_count, 0, 0);
-
- DTRACE_MEMORYSTATUS3(memorystatus_do_kill, proc_t, p, os_reason_t, jetsam_reason, uint32_t, cause);
-#if CONFIG_JETSAM && (DEVELOPMENT || DEBUG)
- if (memorystatus_jetsam_panic_debug & (1 << cause)) {
- panic("memorystatus_do_kill(): jetsam debug panic (cause: %d)", cause);
- }
-#else
-#pragma unused(cause)
-#endif
-
- if (p->p_memstat_effectivepriority >= JETSAM_PRIORITY_FOREGROUND) {
- printf("memorystatus: killing process %d [%s] in high band %s (%d) - memorystatus_available_pages: %llu\n", p->p_pid,
- (*p->p_name ? p->p_name : "unknown"),
- memorystatus_priority_band_name(p->p_memstat_effectivepriority), p->p_memstat_effectivepriority,
- (uint64_t)memorystatus_available_pages);
- }
-
- /*
- * The jetsam_reason (os_reason_t) has enough information about the kill cause.
- * We don't really need jetsam_flags anymore, so it's okay that not all possible kill causes have been mapped.
- */
- int jetsam_flags = P_LTERM_JETSAM;
- switch (cause) {
- case kMemorystatusKilledHiwat: jetsam_flags |= P_JETSAM_HIWAT; break;
- case kMemorystatusKilledVnodes: jetsam_flags |= P_JETSAM_VNODE; break;
- case kMemorystatusKilledVMPageShortage: jetsam_flags |= P_JETSAM_VMPAGESHORTAGE; break;
- case kMemorystatusKilledVMCompressorThrashing:
- case kMemorystatusKilledVMCompressorSpaceShortage: jetsam_flags |= P_JETSAM_VMTHRASHING; break;
- case kMemorystatusKilledFCThrashing: jetsam_flags |= P_JETSAM_FCTHRASHING; break;
- case kMemorystatusKilledPerProcessLimit: jetsam_flags |= P_JETSAM_PID; break;
- case kMemorystatusKilledIdleExit: jetsam_flags |= P_JETSAM_IDLEEXIT; break;
- }
- error = jetsam_do_kill(p, jetsam_flags, jetsam_reason);
-
- KERNEL_DEBUG_CONSTANT((BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_DO_KILL)) | DBG_FUNC_END,
- victim_pid, cause, vm_page_free_count, error, 0);
-
- vm_run_compactor();
-
- return error == 0;
-}
-
-/*
- * Node manipulation
- */
-
-static void
-memorystatus_check_levels_locked(void)
-{
-#if CONFIG_JETSAM
- /* Update levels */
- memorystatus_update_levels_locked(TRUE);
-#else /* CONFIG_JETSAM */
- /*
- * Nothing to do here currently since we update
- * memorystatus_available_pages in vm_pressure_response.
- */
-#endif /* CONFIG_JETSAM */
-}
-
-/*
- * Pin a process to a particular jetsam band when it is in the background i.e. not doing active work.
- * For an application: that means no longer in the FG band
- * For a daemon: that means no longer in its 'requested' jetsam priority band
- */
-
-int
-memorystatus_update_inactive_jetsam_priority_band(pid_t pid, uint32_t op_flags, int jetsam_prio, boolean_t effective_now)
-{
- int error = 0;
- boolean_t enable = FALSE;
- proc_t p = NULL;
-
- if (op_flags == MEMORYSTATUS_CMD_ELEVATED_INACTIVEJETSAMPRIORITY_ENABLE) {
- enable = TRUE;
- } else if (op_flags == MEMORYSTATUS_CMD_ELEVATED_INACTIVEJETSAMPRIORITY_DISABLE) {
- enable = FALSE;
- } else {
- return EINVAL;
- }
-
- p = proc_find(pid);
- if (p != NULL) {
- if ((enable && ((p->p_memstat_state & P_MEMSTAT_USE_ELEVATED_INACTIVE_BAND) == P_MEMSTAT_USE_ELEVATED_INACTIVE_BAND)) ||
- (!enable && ((p->p_memstat_state & P_MEMSTAT_USE_ELEVATED_INACTIVE_BAND) == 0))) {
- /*
- * No change in state.
- */
- } else {
- proc_list_lock();
-
- if (enable) {
- p->p_memstat_state |= P_MEMSTAT_USE_ELEVATED_INACTIVE_BAND;
- memorystatus_invalidate_idle_demotion_locked(p, TRUE);
-
- if (effective_now) {
- if (p->p_memstat_effectivepriority < jetsam_prio) {
- if (memorystatus_highwater_enabled) {
- /*
- * Process is about to transition from
- * inactive --> active
- * assign active state
- */
- boolean_t is_fatal;
- boolean_t use_active = TRUE;
- CACHE_ACTIVE_LIMITS_LOCKED(p, is_fatal);
- task_set_phys_footprint_limit_internal(p->task, (p->p_memstat_memlimit > 0) ? p->p_memstat_memlimit : -1, NULL, use_active, is_fatal);
- }
- memorystatus_update_priority_locked(p, jetsam_prio, FALSE, FALSE);
- }
- } else {
- if (isProcessInAgingBands(p)) {
- memorystatus_update_priority_locked(p, JETSAM_PRIORITY_IDLE, FALSE, TRUE);
- }
- }
- } else {
- p->p_memstat_state &= ~P_MEMSTAT_USE_ELEVATED_INACTIVE_BAND;
- memorystatus_invalidate_idle_demotion_locked(p, TRUE);
-
- if (effective_now) {
- if (p->p_memstat_effectivepriority == jetsam_prio) {
- memorystatus_update_priority_locked(p, JETSAM_PRIORITY_IDLE, FALSE, TRUE);
- }
- } else {
- if (isProcessInAgingBands(p)) {
- memorystatus_update_priority_locked(p, JETSAM_PRIORITY_IDLE, FALSE, TRUE);
- }
- }
- }
-
- proc_list_unlock();
- }
- proc_rele(p);
- error = 0;
- } else {
- error = ESRCH;
- }
-
- return error;
-}
-
-static void
-memorystatus_perform_idle_demotion(__unused void *spare1, __unused void *spare2)
-{
- proc_t p;
- uint64_t current_time = 0, idle_delay_time = 0;
- int demote_prio_band = 0;
- memstat_bucket_t *demotion_bucket;
-
- MEMORYSTATUS_DEBUG(1, "memorystatus_perform_idle_demotion()\n");
-
- KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_IDLE_DEMOTE) | DBG_FUNC_START, 0, 0, 0, 0, 0);
-
- current_time = mach_absolute_time();
-
- proc_list_lock();
-
- demote_prio_band = JETSAM_PRIORITY_IDLE + 1;
-
- for (; demote_prio_band < JETSAM_PRIORITY_MAX; demote_prio_band++) {
- if (demote_prio_band != system_procs_aging_band && demote_prio_band != applications_aging_band) {
- continue;
- }
-
- demotion_bucket = &memstat_bucket[demote_prio_band];
- p = TAILQ_FIRST(&demotion_bucket->list);
-
- while (p) {
- MEMORYSTATUS_DEBUG(1, "memorystatus_perform_idle_demotion() found %d\n", p->p_pid);
-
- assert(p->p_memstat_idledeadline);
-
- assert(p->p_memstat_dirty & P_DIRTY_AGING_IN_PROGRESS);
-
- if (current_time >= p->p_memstat_idledeadline) {
- if ((isSysProc(p) &&
- ((p->p_memstat_dirty & (P_DIRTY_IDLE_EXIT_ENABLED | P_DIRTY_IS_DIRTY)) != P_DIRTY_IDLE_EXIT_ENABLED)) || /* system proc marked dirty*/
- task_has_assertions((struct task *)(p->task))) { /* has outstanding assertions which might indicate outstanding work too */
- idle_delay_time = (isSysProc(p)) ? memorystatus_sysprocs_idle_delay_time : memorystatus_apps_idle_delay_time;
-
- p->p_memstat_idledeadline += idle_delay_time;
- p = TAILQ_NEXT(p, p_memstat_list);
- } else {
- proc_t next_proc = NULL;
-
- next_proc = TAILQ_NEXT(p, p_memstat_list);
- memorystatus_invalidate_idle_demotion_locked(p, TRUE);
-
- memorystatus_update_priority_locked(p, JETSAM_PRIORITY_IDLE, false, true);
-
- p = next_proc;
- continue;
- }
- } else {
- // No further candidates
- break;
- }
- }
- }
-
- memorystatus_reschedule_idle_demotion_locked();
-
- proc_list_unlock();
-
- KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_IDLE_DEMOTE) | DBG_FUNC_END, 0, 0, 0, 0, 0);
-}
-
-static void
-memorystatus_schedule_idle_demotion_locked(proc_t p, boolean_t set_state)
-{
- boolean_t present_in_sysprocs_aging_bucket = FALSE;
- boolean_t present_in_apps_aging_bucket = FALSE;
- uint64_t idle_delay_time = 0;
-
- if (jetsam_aging_policy == kJetsamAgingPolicyNone) {
- return;
- }
-
- if (p->p_memstat_state & P_MEMSTAT_USE_ELEVATED_INACTIVE_BAND) {
- /*
- * This process isn't going to be making the trip to the lower bands.
- */
- return;
- }
-
- if (isProcessInAgingBands(p)) {
- if (jetsam_aging_policy != kJetsamAgingPolicyLegacy) {
- assert((p->p_memstat_dirty & P_DIRTY_AGING_IN_PROGRESS) != P_DIRTY_AGING_IN_PROGRESS);
- }
-
- if (isSysProc(p) && system_procs_aging_band) {
- present_in_sysprocs_aging_bucket = TRUE;
- } else if (isApp(p) && applications_aging_band) {
- present_in_apps_aging_bucket = TRUE;
- }
- }
-
- assert(!present_in_sysprocs_aging_bucket);
- assert(!present_in_apps_aging_bucket);
-
- MEMORYSTATUS_DEBUG(1, "memorystatus_schedule_idle_demotion_locked: scheduling demotion to idle band for pid %d (dirty:0x%x, set_state %d, demotions %d).\n",
- p->p_pid, p->p_memstat_dirty, set_state, (memorystatus_scheduled_idle_demotions_sysprocs + memorystatus_scheduled_idle_demotions_apps));
-
- if (isSysProc(p)) {
- assert((p->p_memstat_dirty & P_DIRTY_IDLE_EXIT_ENABLED) == P_DIRTY_IDLE_EXIT_ENABLED);
- }
-
- idle_delay_time = (isSysProc(p)) ? memorystatus_sysprocs_idle_delay_time : memorystatus_apps_idle_delay_time;
-
- if (set_state) {
- p->p_memstat_dirty |= P_DIRTY_AGING_IN_PROGRESS;
- p->p_memstat_idledeadline = mach_absolute_time() + idle_delay_time;
- }
-
- assert(p->p_memstat_idledeadline);
-
- if (isSysProc(p) && present_in_sysprocs_aging_bucket == FALSE) {
- memorystatus_scheduled_idle_demotions_sysprocs++;
- } else if (isApp(p) && present_in_apps_aging_bucket == FALSE) {
- memorystatus_scheduled_idle_demotions_apps++;
- }
-}
-
-static void
-memorystatus_invalidate_idle_demotion_locked(proc_t p, boolean_t clear_state)
-{
- boolean_t present_in_sysprocs_aging_bucket = FALSE;
- boolean_t present_in_apps_aging_bucket = FALSE;
-
- if (!system_procs_aging_band && !applications_aging_band) {
- return;
- }
-
- if ((p->p_memstat_dirty & P_DIRTY_AGING_IN_PROGRESS) == 0) {
- return;
- }
-
- if (isProcessInAgingBands(p)) {
- if (jetsam_aging_policy != kJetsamAgingPolicyLegacy) {
- assert((p->p_memstat_dirty & P_DIRTY_AGING_IN_PROGRESS) == P_DIRTY_AGING_IN_PROGRESS);
- }
-
- if (isSysProc(p) && system_procs_aging_band) {
- assert(p->p_memstat_effectivepriority == system_procs_aging_band);
- assert(p->p_memstat_idledeadline);
- present_in_sysprocs_aging_bucket = TRUE;
- } else if (isApp(p) && applications_aging_band) {
- assert(p->p_memstat_effectivepriority == applications_aging_band);
- assert(p->p_memstat_idledeadline);
- present_in_apps_aging_bucket = TRUE;
- }
- }
-
- MEMORYSTATUS_DEBUG(1, "memorystatus_invalidate_idle_demotion(): invalidating demotion to idle band for pid %d (clear_state %d, demotions %d).\n",
- p->p_pid, clear_state, (memorystatus_scheduled_idle_demotions_sysprocs + memorystatus_scheduled_idle_demotions_apps));
-
-
- if (clear_state) {
- p->p_memstat_idledeadline = 0;
- p->p_memstat_dirty &= ~P_DIRTY_AGING_IN_PROGRESS;
- }
-
- if (isSysProc(p) && present_in_sysprocs_aging_bucket == TRUE) {
- memorystatus_scheduled_idle_demotions_sysprocs--;
- assert(memorystatus_scheduled_idle_demotions_sysprocs >= 0);
- } else if (isApp(p) && present_in_apps_aging_bucket == TRUE) {
- memorystatus_scheduled_idle_demotions_apps--;
- assert(memorystatus_scheduled_idle_demotions_apps >= 0);
- }
-
- assert((memorystatus_scheduled_idle_demotions_sysprocs + memorystatus_scheduled_idle_demotions_apps) >= 0);
-}
-
-static void
-memorystatus_reschedule_idle_demotion_locked(void)
-{
- if (0 == (memorystatus_scheduled_idle_demotions_sysprocs + memorystatus_scheduled_idle_demotions_apps)) {
- if (memstat_idle_demotion_deadline) {
- /* Transitioned 1->0, so cancel next call */
- thread_call_cancel(memorystatus_idle_demotion_call);
- memstat_idle_demotion_deadline = 0;
- }
- } else {
- memstat_bucket_t *demotion_bucket;
- proc_t p = NULL, p1 = NULL, p2 = NULL;
-
- if (system_procs_aging_band) {
- demotion_bucket = &memstat_bucket[system_procs_aging_band];
- p1 = TAILQ_FIRST(&demotion_bucket->list);
-
- p = p1;
- }
-
- if (applications_aging_band) {
- demotion_bucket = &memstat_bucket[applications_aging_band];
- p2 = TAILQ_FIRST(&demotion_bucket->list);
-
- if (p1 && p2) {
- p = (p1->p_memstat_idledeadline > p2->p_memstat_idledeadline) ? p2 : p1;
- } else {
- p = (p1 == NULL) ? p2 : p1;
- }
- }
-
- assert(p);
-
- if (p != NULL) {
- assert(p && p->p_memstat_idledeadline);
- if (memstat_idle_demotion_deadline != p->p_memstat_idledeadline) {
- thread_call_enter_delayed(memorystatus_idle_demotion_call, p->p_memstat_idledeadline);
- memstat_idle_demotion_deadline = p->p_memstat_idledeadline;
- }
- }
- }
-}
-
-/*
- * List manipulation
- */
-
-int
-memorystatus_add(proc_t p, boolean_t locked)
-{
- memstat_bucket_t *bucket;
-
- MEMORYSTATUS_DEBUG(1, "memorystatus_list_add(): adding pid %d with priority %d.\n", p->p_pid, p->p_memstat_effectivepriority);
-
- if (!locked) {
- proc_list_lock();
- }
-
- DTRACE_MEMORYSTATUS2(memorystatus_add, proc_t, p, int32_t, p->p_memstat_effectivepriority);
-
- /* Processes marked internal do not have priority tracked */
- if (p->p_memstat_state & P_MEMSTAT_INTERNAL) {
- goto exit;
- }
-
- bucket = &memstat_bucket[p->p_memstat_effectivepriority];
-
- if (isSysProc(p) && system_procs_aging_band && (p->p_memstat_effectivepriority == system_procs_aging_band)) {
- assert(bucket->count == memorystatus_scheduled_idle_demotions_sysprocs - 1);
- } else if (isApp(p) && applications_aging_band && (p->p_memstat_effectivepriority == applications_aging_band)) {
- assert(bucket->count == memorystatus_scheduled_idle_demotions_apps - 1);
- } else if (p->p_memstat_effectivepriority == JETSAM_PRIORITY_IDLE) {
- /*
- * Entering the idle band.
- * Record idle start time.
- */
- p->p_memstat_idle_start = mach_absolute_time();
- }
-
- TAILQ_INSERT_TAIL(&bucket->list, p, p_memstat_list);
- bucket->count++;
-
- memorystatus_list_count++;
-
- memorystatus_check_levels_locked();
-
-exit:
- if (!locked) {
- proc_list_unlock();
- }
-
- return 0;
-}
-
-/*
- * Description:
- * Moves a process from one jetsam bucket to another.
- * which changes the LRU position of the process.
- *
- * Monitors transition between buckets and if necessary
- * will update cached memory limits accordingly.
- *
- * skip_demotion_check:
- * - if the 'jetsam aging policy' is NOT 'legacy':
- * When this flag is TRUE, it means we are going
- * to age the ripe processes out of the aging bands and into the
- * IDLE band and apply their inactive memory limits.
- *
- * - if the 'jetsam aging policy' is 'legacy':
- * When this flag is TRUE, it might mean the above aging mechanism
- * OR
- * It might be that we have a process that has used up its 'idle deferral'
- * stay that is given to it once per lifetime. And in this case, the process
- * won't be going through any aging codepaths. But we still need to apply
- * the right inactive limits and so we explicitly set this to TRUE if the
- * new priority for the process is the IDLE band.
- */
-void
-memorystatus_update_priority_locked(proc_t p, int priority, boolean_t head_insert, boolean_t skip_demotion_check)
-{
- memstat_bucket_t *old_bucket, *new_bucket;
-
- assert(priority < MEMSTAT_BUCKET_COUNT);
-
- /* Ensure that exit isn't underway, leaving the proc retained but removed from its bucket */
- if ((p->p_listflag & P_LIST_EXITED) != 0) {
- return;
- }
-
- MEMORYSTATUS_DEBUG(1, "memorystatus_update_priority_locked(): setting %s(%d) to priority %d, inserting at %s\n",
- (*p->p_name ? p->p_name : "unknown"), p->p_pid, priority, head_insert ? "head" : "tail");
-
- DTRACE_MEMORYSTATUS3(memorystatus_update_priority, proc_t, p, int32_t, p->p_memstat_effectivepriority, int, priority);
-
-#if DEVELOPMENT || DEBUG
- if (priority == JETSAM_PRIORITY_IDLE && /* if the process is on its way into the IDLE band */
- skip_demotion_check == FALSE && /* and it isn't via the path that will set the INACTIVE memlimits */
- (p->p_memstat_dirty & P_DIRTY_TRACK) && /* and it has 'DIRTY' tracking enabled */
- ((p->p_memstat_memlimit != p->p_memstat_memlimit_inactive) || /* and we notice that the current limit isn't the right value (inactive) */
- ((p->p_memstat_state & P_MEMSTAT_MEMLIMIT_INACTIVE_FATAL) ? (!(p->p_memstat_state & P_MEMSTAT_FATAL_MEMLIMIT)) : (p->p_memstat_state & P_MEMSTAT_FATAL_MEMLIMIT)))) { /* OR type (fatal vs non-fatal) */
- panic("memorystatus_update_priority_locked: on %s with 0x%x, prio: %d and %d\n", p->p_name, p->p_memstat_state, priority, p->p_memstat_memlimit); /* then we must catch this */
- }
-#endif /* DEVELOPMENT || DEBUG */
-
- old_bucket = &memstat_bucket[p->p_memstat_effectivepriority];
-
- if (skip_demotion_check == FALSE) {
- if (isSysProc(p)) {
- /*
- * For system processes, the memorystatus_dirty_* routines take care of adding/removing
- * the processes from the aging bands and balancing the demotion counts.
- * We can, however, override that if the process has an 'elevated inactive jetsam band' attribute.
- */
-
- if (p->p_memstat_state & P_MEMSTAT_USE_ELEVATED_INACTIVE_BAND) {
- /*
- * 2 types of processes can use the non-standard elevated inactive band:
- * - Frozen processes that always land in memorystatus_freeze_jetsam_band
- * OR
- * - processes that specifically opt-in to the elevated inactive support e.g. docked processes.
- */
-#if CONFIG_FREEZE
- if (p->p_memstat_state & P_MEMSTAT_FROZEN) {
- if (priority <= memorystatus_freeze_jetsam_band) {
- priority = memorystatus_freeze_jetsam_band;
- }
- } else
-#endif /* CONFIG_FREEZE */
- {
- if (priority <= JETSAM_PRIORITY_ELEVATED_INACTIVE) {
- priority = JETSAM_PRIORITY_ELEVATED_INACTIVE;
- }
- }
- assert(!(p->p_memstat_dirty & P_DIRTY_AGING_IN_PROGRESS));
- }
- } else if (isApp(p)) {
- /*
- * Check to see if the application is being lowered in jetsam priority. If so, and:
- * - it has an 'elevated inactive jetsam band' attribute, then put it in the appropriate band.
- * - it is a normal application, then let it age in the aging band if that policy is in effect.
- */
-
- if (p->p_memstat_state & P_MEMSTAT_USE_ELEVATED_INACTIVE_BAND) {
-#if CONFIG_FREEZE
- if (p->p_memstat_state & P_MEMSTAT_FROZEN) {
- if (priority <= memorystatus_freeze_jetsam_band) {
- priority = memorystatus_freeze_jetsam_band;
- }
- } else
-#endif /* CONFIG_FREEZE */
- {
- if (priority <= JETSAM_PRIORITY_ELEVATED_INACTIVE) {
- priority = JETSAM_PRIORITY_ELEVATED_INACTIVE;
- }
- }
- } else {
- if (applications_aging_band) {
- if (p->p_memstat_effectivepriority == applications_aging_band) {
- assert(old_bucket->count == (memorystatus_scheduled_idle_demotions_apps + 1));
- }
-
- if ((jetsam_aging_policy != kJetsamAgingPolicyLegacy) && (priority <= applications_aging_band)) {
- assert(!(p->p_memstat_dirty & P_DIRTY_AGING_IN_PROGRESS));
- priority = applications_aging_band;
- memorystatus_schedule_idle_demotion_locked(p, TRUE);
- }
- }
- }
- }
- }
-
- if ((system_procs_aging_band && (priority == system_procs_aging_band)) || (applications_aging_band && (priority == applications_aging_band))) {
- assert(p->p_memstat_dirty & P_DIRTY_AGING_IN_PROGRESS);
- }
-
- TAILQ_REMOVE(&old_bucket->list, p, p_memstat_list);
- old_bucket->count--;
-
- new_bucket = &memstat_bucket[priority];
- if (head_insert) {
- TAILQ_INSERT_HEAD(&new_bucket->list, p, p_memstat_list);
- } else {
- TAILQ_INSERT_TAIL(&new_bucket->list, p, p_memstat_list);
- }
- new_bucket->count++;
-
- if (memorystatus_highwater_enabled) {
- boolean_t is_fatal;
- boolean_t use_active;
-
- /*
- * If cached limit data is updated, then the limits
- * will be enforced by writing to the ledgers.
- */
- boolean_t ledger_update_needed = TRUE;
-
- /*
- * Here, we must update the cached memory limit if the task
- * is transitioning between:
- * active <--> inactive
- * FG <--> BG
- * but:
- * dirty <--> clean is ignored
- *
- * We bypass non-idle processes that have opted into dirty tracking because
- * a move between buckets does not imply a transition between the
- * dirty <--> clean state.
- */
-
- if (p->p_memstat_dirty & P_DIRTY_TRACK) {
- if (skip_demotion_check == TRUE && priority == JETSAM_PRIORITY_IDLE) {
- CACHE_INACTIVE_LIMITS_LOCKED(p, is_fatal);
- use_active = FALSE;
- } else {
- ledger_update_needed = FALSE;
- }
- } else if ((priority >= JETSAM_PRIORITY_FOREGROUND) && (p->p_memstat_effectivepriority < JETSAM_PRIORITY_FOREGROUND)) {
- /*
- * inactive --> active
- * BG --> FG
- * assign active state
- */
- CACHE_ACTIVE_LIMITS_LOCKED(p, is_fatal);
- use_active = TRUE;
- } else if ((priority < JETSAM_PRIORITY_FOREGROUND) && (p->p_memstat_effectivepriority >= JETSAM_PRIORITY_FOREGROUND)) {
- /*
- * active --> inactive
- * FG --> BG
- * assign inactive state
- */
- CACHE_INACTIVE_LIMITS_LOCKED(p, is_fatal);
- use_active = FALSE;
- } else {
- /*
- * The transition between jetsam priority buckets apparently did
- * not affect active/inactive state.
- * This is not unusual... especially during startup when
- * processes are getting established in their respective bands.
- */
- ledger_update_needed = FALSE;
- }
-
- /*
- * Enforce the new limits by writing to the ledger
- */
- if (ledger_update_needed) {
- task_set_phys_footprint_limit_internal(p->task, (p->p_memstat_memlimit > 0) ? p->p_memstat_memlimit : -1, NULL, use_active, is_fatal);
-
- MEMORYSTATUS_DEBUG(3, "memorystatus_update_priority_locked: new limit on pid %d (%dMB %s) priority old --> new (%d --> %d) dirty?=0x%x %s\n",
- p->p_pid, (p->p_memstat_memlimit > 0 ? p->p_memstat_memlimit : -1),
- (p->p_memstat_state & P_MEMSTAT_FATAL_MEMLIMIT ? "F " : "NF"), p->p_memstat_effectivepriority, priority, p->p_memstat_dirty,
- (p->p_memstat_dirty ? ((p->p_memstat_dirty & P_DIRTY) ? "isdirty" : "isclean") : ""));
- }
- }
-
- /*
- * Record idle start or idle delta.
- */
- if (p->p_memstat_effectivepriority == priority) {
- /*
- * This process is not transitioning between
- * jetsam priority buckets. Do nothing.
- */
- } else if (p->p_memstat_effectivepriority == JETSAM_PRIORITY_IDLE) {
- uint64_t now;
- /*
- * Transitioning out of the idle priority bucket.
- * Record idle delta.
- */
- assert(p->p_memstat_idle_start != 0);
- now = mach_absolute_time();
- if (now > p->p_memstat_idle_start) {
- p->p_memstat_idle_delta = now - p->p_memstat_idle_start;
- }
-
- /*
- * About to become active and so memory footprint could change.
- * So mark it eligible for freeze-considerations next time around.
- */
- if (p->p_memstat_state & P_MEMSTAT_FREEZE_IGNORE) {
- p->p_memstat_state &= ~P_MEMSTAT_FREEZE_IGNORE;
- }
- } else if (priority == JETSAM_PRIORITY_IDLE) {
- /*
- * Transitioning into the idle priority bucket.
- * Record idle start.
- */
- p->p_memstat_idle_start = mach_absolute_time();
- }
-
- KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_CHANGE_PRIORITY), p->p_pid, priority, p->p_memstat_effectivepriority, 0, 0);
-
- p->p_memstat_effectivepriority = priority;
-
-#if CONFIG_SECLUDED_MEMORY
- if (secluded_for_apps &&
- task_could_use_secluded_mem(p->task)) {
- task_set_can_use_secluded_mem(
- p->task,
- (priority >= JETSAM_PRIORITY_FOREGROUND));
- }
-#endif /* CONFIG_SECLUDED_MEMORY */
-
- memorystatus_check_levels_locked();
-}
-
-/*
- *
- * Description: Update the jetsam priority and memory limit attributes for a given process.
- *
- * Parameters:
- * p init this process's jetsam information.
- * priority The jetsam priority band
- * user_data user specific data, unused by the kernel
- * effective guards against race if process's update already occurred
- * update_memlimit When true we know this is the init step via the posix_spawn path.
- *
- * memlimit_active Value in megabytes; The monitored footprint level while the
- * process is active. Exceeding it may result in termination
- * based on it's associated fatal flag.
- *
- * memlimit_active_is_fatal When a process is active and exceeds its memory footprint,
- * this describes whether or not it should be immediately fatal.
- *
- * memlimit_inactive Value in megabytes; The monitored footprint level while the
- * process is inactive. Exceeding it may result in termination
- * based on it's associated fatal flag.
- *
- * memlimit_inactive_is_fatal When a process is inactive and exceeds its memory footprint,
- * this describes whether or not it should be immediatly fatal.
- *
- * Returns: 0 Success
- * non-0 Failure
- */
-
-int
-memorystatus_update(proc_t p, int priority, uint64_t user_data, boolean_t effective, boolean_t update_memlimit,
- int32_t memlimit_active, boolean_t memlimit_active_is_fatal,
- int32_t memlimit_inactive, boolean_t memlimit_inactive_is_fatal)
-{
- int ret;
- boolean_t head_insert = false;
-
- MEMORYSTATUS_DEBUG(1, "memorystatus_update: changing (%s) pid %d: priority %d, user_data 0x%llx\n", (*p->p_name ? p->p_name : "unknown"), p->p_pid, priority, user_data);
-
- KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_UPDATE) | DBG_FUNC_START, p->p_pid, priority, user_data, effective, 0);
-
- if (priority == -1) {
- /* Use as shorthand for default priority */
- priority = JETSAM_PRIORITY_DEFAULT;
- } else if ((priority == system_procs_aging_band) || (priority == applications_aging_band)) {
- /* Both the aging bands are reserved for internal use; if requested, adjust to JETSAM_PRIORITY_IDLE. */
- priority = JETSAM_PRIORITY_IDLE;
- } else if (priority == JETSAM_PRIORITY_IDLE_HEAD) {
- /* JETSAM_PRIORITY_IDLE_HEAD inserts at the head of the idle queue */
- priority = JETSAM_PRIORITY_IDLE;
- head_insert = TRUE;
- } else if ((priority < 0) || (priority >= MEMSTAT_BUCKET_COUNT)) {
- /* Sanity check */
- ret = EINVAL;
- goto out;
- }
-
- proc_list_lock();
-
- assert(!(p->p_memstat_state & P_MEMSTAT_INTERNAL));
-
- if (effective && (p->p_memstat_state & P_MEMSTAT_PRIORITYUPDATED)) {
- ret = EALREADY;
- proc_list_unlock();
- MEMORYSTATUS_DEBUG(1, "memorystatus_update: effective change specified for pid %d, but change already occurred.\n", p->p_pid);
- goto out;
- }
-
- if ((p->p_memstat_state & P_MEMSTAT_TERMINATED) || ((p->p_listflag & P_LIST_EXITED) != 0)) {
- /*
- * This could happen when a process calling posix_spawn() is exiting on the jetsam thread.
- */
- ret = EBUSY;
- proc_list_unlock();
- goto out;
- }
-
- p->p_memstat_state |= P_MEMSTAT_PRIORITYUPDATED;
- p->p_memstat_userdata = user_data;
- p->p_memstat_requestedpriority = priority;
-
- if (update_memlimit) {
- boolean_t is_fatal;
- boolean_t use_active;
-
- /*
- * Posix_spawn'd processes come through this path to instantiate ledger limits.
- * Forked processes do not come through this path, so no ledger limits exist.
- * (That's why forked processes can consume unlimited memory.)
- */
-
- MEMORYSTATUS_DEBUG(3, "memorystatus_update(enter): pid %d, priority %d, dirty=0x%x, Active(%dMB %s), Inactive(%dMB, %s)\n",
- p->p_pid, priority, p->p_memstat_dirty,
- memlimit_active, (memlimit_active_is_fatal ? "F " : "NF"),
- memlimit_inactive, (memlimit_inactive_is_fatal ? "F " : "NF"));
-
- if (memlimit_active <= 0) {
- /*
- * This process will have a system_wide task limit when active.
- * System_wide task limit is always fatal.
- * It's quite common to see non-fatal flag passed in here.
- * It's not an error, we just ignore it.
- */
-
- /*
- * For backward compatibility with some unexplained launchd behavior,
- * we allow a zero sized limit. But we still enforce system_wide limit
- * when written to the ledgers.
- */
-
- if (memlimit_active < 0) {
- memlimit_active = -1; /* enforces system_wide task limit */
- }
- memlimit_active_is_fatal = TRUE;
- }
-
- if (memlimit_inactive <= 0) {
- /*
- * This process will have a system_wide task limit when inactive.
- * System_wide task limit is always fatal.
- */
-
- memlimit_inactive = -1;
- memlimit_inactive_is_fatal = TRUE;
- }
-
- /*
- * Initialize the active limit variants for this process.
- */
- SET_ACTIVE_LIMITS_LOCKED(p, memlimit_active, memlimit_active_is_fatal);
-
- /*
- * Initialize the inactive limit variants for this process.
- */
- SET_INACTIVE_LIMITS_LOCKED(p, memlimit_inactive, memlimit_inactive_is_fatal);
-
- /*
- * Initialize the cached limits for target process.
- * When the target process is dirty tracked, it's typically
- * in a clean state. Non dirty tracked processes are
- * typically active (Foreground or above).
- * But just in case, we don't make assumptions...
- */
-
- if (proc_jetsam_state_is_active_locked(p) == TRUE) {
- CACHE_ACTIVE_LIMITS_LOCKED(p, is_fatal);
- use_active = TRUE;
- } else {
- CACHE_INACTIVE_LIMITS_LOCKED(p, is_fatal);
- use_active = FALSE;
- }
-
- /*
- * Enforce the cached limit by writing to the ledger.
- */
- if (memorystatus_highwater_enabled) {
- /* apply now */
- task_set_phys_footprint_limit_internal(p->task, ((p->p_memstat_memlimit > 0) ? p->p_memstat_memlimit : -1), NULL, use_active, is_fatal);
-
- MEMORYSTATUS_DEBUG(3, "memorystatus_update: init: limit on pid %d (%dMB %s) targeting priority(%d) dirty?=0x%x %s\n",
- p->p_pid, (p->p_memstat_memlimit > 0 ? p->p_memstat_memlimit : -1),
- (p->p_memstat_state & P_MEMSTAT_FATAL_MEMLIMIT ? "F " : "NF"), priority, p->p_memstat_dirty,
- (p->p_memstat_dirty ? ((p->p_memstat_dirty & P_DIRTY) ? "isdirty" : "isclean") : ""));
- }
- }
-
- /*
- * We can't add to the aging bands buckets here.
- * But, we could be removing it from those buckets.
- * Check and take appropriate steps if so.
- */
-
- if (isProcessInAgingBands(p)) {
- memorystatus_invalidate_idle_demotion_locked(p, TRUE);
- memorystatus_update_priority_locked(p, JETSAM_PRIORITY_IDLE, FALSE, TRUE);
- } else {
- if (jetsam_aging_policy == kJetsamAgingPolicyLegacy && priority == JETSAM_PRIORITY_IDLE) {
- /*
- * Daemons with 'inactive' limits will go through the dirty tracking codepath.
- * This path deals with apps that may have 'inactive' limits e.g. WebContent processes.
- * If this is the legacy aging policy we explicitly need to apply those limits. If it
- * is any other aging policy, then we don't need to worry because all processes
- * will go through the aging bands and then the demotion thread will take care to
- * move them into the IDLE band and apply the required limits.
- */
- memorystatus_update_priority_locked(p, priority, head_insert, TRUE);
- }
- }
-
- memorystatus_update_priority_locked(p, priority, head_insert, FALSE);
-
- proc_list_unlock();
- ret = 0;
-
-out:
- KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_UPDATE) | DBG_FUNC_END, ret, 0, 0, 0, 0);
-
- return ret;
-}
-
-int
-memorystatus_remove(proc_t p, boolean_t locked)
-{
- int ret;
- memstat_bucket_t *bucket;
- boolean_t reschedule = FALSE;
-
- MEMORYSTATUS_DEBUG(1, "memorystatus_list_remove: removing pid %d\n", p->p_pid);
-
- if (!locked) {
- proc_list_lock();
- }
-
- assert(!(p->p_memstat_state & P_MEMSTAT_INTERNAL));
-
- bucket = &memstat_bucket[p->p_memstat_effectivepriority];
-
- if (isSysProc(p) && system_procs_aging_band && (p->p_memstat_effectivepriority == system_procs_aging_band)) {
- assert(bucket->count == memorystatus_scheduled_idle_demotions_sysprocs);
- reschedule = TRUE;
- } else if (isApp(p) && applications_aging_band && (p->p_memstat_effectivepriority == applications_aging_band)) {
- assert(bucket->count == memorystatus_scheduled_idle_demotions_apps);
- reschedule = TRUE;
- }
-
- /*
- * Record idle delta
- */
-
- if (p->p_memstat_effectivepriority == JETSAM_PRIORITY_IDLE) {
- uint64_t now = mach_absolute_time();
- if (now > p->p_memstat_idle_start) {
- p->p_memstat_idle_delta = now - p->p_memstat_idle_start;
- }
- }
-
- TAILQ_REMOVE(&bucket->list, p, p_memstat_list);
- bucket->count--;
-
- memorystatus_list_count--;
-
- /* If awaiting demotion to the idle band, clean up */
- if (reschedule) {
- memorystatus_invalidate_idle_demotion_locked(p, TRUE);
- memorystatus_reschedule_idle_demotion_locked();
- }
-
- memorystatus_check_levels_locked();
-
-#if CONFIG_FREEZE
- if (p->p_memstat_state & (P_MEMSTAT_FROZEN)) {
- if (p->p_memstat_state & P_MEMSTAT_REFREEZE_ELIGIBLE) {
- p->p_memstat_state &= ~P_MEMSTAT_REFREEZE_ELIGIBLE;
- memorystatus_refreeze_eligible_count--;
- }
-
- memorystatus_frozen_count--;
- memorystatus_frozen_shared_mb -= p->p_memstat_freeze_sharedanon_pages;
- p->p_memstat_freeze_sharedanon_pages = 0;
- }
-
- if (p->p_memstat_state & P_MEMSTAT_SUSPENDED) {
- memorystatus_suspended_count--;
- }
-#endif
-
- if (!locked) {
- proc_list_unlock();
- }
-
- if (p) {
- ret = 0;
- } else {
- ret = ESRCH;
- }
-
- return ret;
-}
-
-/*
- * Validate dirty tracking flags with process state.
- *
- * Return:
- * 0 on success
- * non-0 on failure
- *
- * The proc_list_lock is held by the caller.
- */
-
-static int
-memorystatus_validate_track_flags(struct proc *target_p, uint32_t pcontrol)
-{
- /* See that the process isn't marked for termination */
- if (target_p->p_memstat_dirty & P_DIRTY_TERMINATED) {
- return EBUSY;
- }
-
- /* Idle exit requires that process be tracked */
- if ((pcontrol & PROC_DIRTY_ALLOW_IDLE_EXIT) &&
- !(pcontrol & PROC_DIRTY_TRACK)) {
- return EINVAL;
- }
-
- /* 'Launch in progress' tracking requires that process have enabled dirty tracking too. */
- if ((pcontrol & PROC_DIRTY_LAUNCH_IN_PROGRESS) &&
- !(pcontrol & PROC_DIRTY_TRACK)) {
- return EINVAL;
- }
-
- /* Only one type of DEFER behavior is allowed.*/
- if ((pcontrol & PROC_DIRTY_DEFER) &&
- (pcontrol & PROC_DIRTY_DEFER_ALWAYS)) {
- return EINVAL;
- }
-
- /* Deferral is only relevant if idle exit is specified */
- if (((pcontrol & PROC_DIRTY_DEFER) ||
- (pcontrol & PROC_DIRTY_DEFER_ALWAYS)) &&
- !(pcontrol & PROC_DIRTY_ALLOWS_IDLE_EXIT)) {
- return EINVAL;
- }
-
- return 0;
-}
-
-static void
-memorystatus_update_idle_priority_locked(proc_t p)
-{
- int32_t priority;
-
- MEMORYSTATUS_DEBUG(1, "memorystatus_update_idle_priority_locked(): pid %d dirty 0x%X\n", p->p_pid, p->p_memstat_dirty);
-
- assert(isSysProc(p));
-
- if ((p->p_memstat_dirty & (P_DIRTY_IDLE_EXIT_ENABLED | P_DIRTY_IS_DIRTY)) == P_DIRTY_IDLE_EXIT_ENABLED) {
- priority = (p->p_memstat_dirty & P_DIRTY_AGING_IN_PROGRESS) ? system_procs_aging_band : JETSAM_PRIORITY_IDLE;
- } else {
- priority = p->p_memstat_requestedpriority;
- }
-
- if (priority != p->p_memstat_effectivepriority) {
- if ((jetsam_aging_policy == kJetsamAgingPolicyLegacy) &&
- (priority == JETSAM_PRIORITY_IDLE)) {
- /*
- * This process is on its way into the IDLE band. The system is
- * using 'legacy' jetsam aging policy. That means, this process
- * has already used up its idle-deferral aging time that is given
- * once per its lifetime. So we need to set the INACTIVE limits
- * explicitly because it won't be going through the demotion paths
- * that take care to apply the limits appropriately.
- */
-
- if (p->p_memstat_state & P_MEMSTAT_USE_ELEVATED_INACTIVE_BAND) {
- /*
- * This process has the 'elevated inactive jetsam band' attribute.
- * So, there will be no trip to IDLE after all.
- * Instead, we pin the process in the elevated band,
- * where its ACTIVE limits will apply.
- */
-
- priority = JETSAM_PRIORITY_ELEVATED_INACTIVE;
- }
-
- memorystatus_update_priority_locked(p, priority, false, true);
- } else {
- memorystatus_update_priority_locked(p, priority, false, false);
- }
- }
-}
-
-/*
- * Processes can opt to have their state tracked by the kernel, indicating when they are busy (dirty) or idle
- * (clean). They may also indicate that they support termination when idle, with the result that they are promoted
- * to their desired, higher, jetsam priority when dirty (and are therefore killed later), and demoted to the low
- * priority idle band when clean (and killed earlier, protecting higher priority procesess).
- *
- * If the deferral flag is set, then newly tracked processes will be protected for an initial period (as determined by
- * memorystatus_sysprocs_idle_delay_time); if they go clean during this time, then they will be moved to a deferred-idle band
- * with a slightly higher priority, guarding against immediate termination under memory pressure and being unable to
- * make forward progress. Finally, when the guard expires, they will be moved to the standard, lowest-priority, idle
- * band. The deferral can be cleared early by clearing the appropriate flag.
- *
- * The deferral timer is active only for the duration that the process is marked as guarded and clean; if the process
- * is marked dirty, the timer will be cancelled. Upon being subsequently marked clean, the deferment will either be
- * re-enabled or the guard state cleared, depending on whether the guard deadline has passed.
- */
-
-int
-memorystatus_dirty_track(proc_t p, uint32_t pcontrol)
-{
- unsigned int old_dirty;
- boolean_t reschedule = FALSE;
- boolean_t already_deferred = FALSE;
- boolean_t defer_now = FALSE;
- int ret = 0;
-
- KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_DIRTY_TRACK),
- p->p_pid, p->p_memstat_dirty, pcontrol, 0, 0);
-
- proc_list_lock();
-
- if ((p->p_listflag & P_LIST_EXITED) != 0) {
- /*
- * Process is on its way out.
- */
- ret = EBUSY;
- goto exit;
- }
-
- if (p->p_memstat_state & P_MEMSTAT_INTERNAL) {
- ret = EPERM;
- goto exit;
- }
-
- if ((ret = memorystatus_validate_track_flags(p, pcontrol)) != 0) {
- /* error */
- goto exit;
- }
-
- old_dirty = p->p_memstat_dirty;
-
- /* These bits are cumulative, as per <rdar://problem/11159924> */
- if (pcontrol & PROC_DIRTY_TRACK) {
- p->p_memstat_dirty |= P_DIRTY_TRACK;
- }
-
- if (pcontrol & PROC_DIRTY_ALLOW_IDLE_EXIT) {
- p->p_memstat_dirty |= P_DIRTY_ALLOW_IDLE_EXIT;
- }
-
- if (pcontrol & PROC_DIRTY_LAUNCH_IN_PROGRESS) {
- p->p_memstat_dirty |= P_DIRTY_LAUNCH_IN_PROGRESS;
- }
-
- if (old_dirty & P_DIRTY_AGING_IN_PROGRESS) {
- already_deferred = TRUE;
- }
-
-
- /* This can be set and cleared exactly once. */
- if (pcontrol & (PROC_DIRTY_DEFER | PROC_DIRTY_DEFER_ALWAYS)) {
- if ((pcontrol & (PROC_DIRTY_DEFER)) &&
- !(old_dirty & P_DIRTY_DEFER)) {
- p->p_memstat_dirty |= P_DIRTY_DEFER;
- }
-
- if ((pcontrol & (PROC_DIRTY_DEFER_ALWAYS)) &&
- !(old_dirty & P_DIRTY_DEFER_ALWAYS)) {
- p->p_memstat_dirty |= P_DIRTY_DEFER_ALWAYS;
- }
-
- defer_now = TRUE;
- }
-
- MEMORYSTATUS_DEBUG(1, "memorystatus_on_track_dirty(): set idle-exit %s / defer %s / dirty %s for pid %d\n",
- ((p->p_memstat_dirty & P_DIRTY_IDLE_EXIT_ENABLED) == P_DIRTY_IDLE_EXIT_ENABLED) ? "Y" : "N",
- defer_now ? "Y" : "N",
- p->p_memstat_dirty & P_DIRTY ? "Y" : "N",
- p->p_pid);
-
- /* Kick off or invalidate the idle exit deferment if there's a state transition. */
- if (!(p->p_memstat_dirty & P_DIRTY_IS_DIRTY)) {
- if ((p->p_memstat_dirty & P_DIRTY_IDLE_EXIT_ENABLED) == P_DIRTY_IDLE_EXIT_ENABLED) {
- if (defer_now && !already_deferred) {
- /*
- * Request to defer a clean process that's idle-exit enabled
- * and not already in the jetsam deferred band. Most likely a
- * new launch.
- */
- memorystatus_schedule_idle_demotion_locked(p, TRUE);
- reschedule = TRUE;
- } else if (!defer_now) {
- /*
- * The process isn't asking for the 'aging' facility.
- * Could be that it is:
- */
-
- if (already_deferred) {
- /*
- * already in the aging bands. Traditionally,
- * some processes have tried to use this to
- * opt out of the 'aging' facility.
- */
-
- memorystatus_invalidate_idle_demotion_locked(p, TRUE);
- } else {
- /*
- * agnostic to the 'aging' facility. In that case,
- * we'll go ahead and opt it in because this is likely
- * a new launch (clean process, dirty tracking enabled)
- */
-
- memorystatus_schedule_idle_demotion_locked(p, TRUE);
- }
-
- reschedule = TRUE;
- }
- }
- } else {
- /*
- * We are trying to operate on a dirty process. Dirty processes have to
- * be removed from the deferred band. The question is do we reset the
- * deferred state or not?
- *
- * This could be a legal request like:
- * - this process had opted into the 'aging' band
- * - but it's now dirty and requests to opt out.
- * In this case, we remove the process from the band and reset its
- * state too. It'll opt back in properly when needed.
- *
- * OR, this request could be a user-space bug. E.g.:
- * - this process had opted into the 'aging' band when clean
- * - and, then issues another request to again put it into the band except
- * this time the process is dirty.
- * The process going dirty, as a transition in memorystatus_dirty_set(), will pull the process out of
- * the deferred band with its state intact. So our request below is no-op.
- * But we do it here anyways for coverage.
- *
- * memorystatus_update_idle_priority_locked()
- * single-mindedly treats a dirty process as "cannot be in the aging band".
- */
-
- if (!defer_now && already_deferred) {
- memorystatus_invalidate_idle_demotion_locked(p, TRUE);
- reschedule = TRUE;
- } else {
- boolean_t reset_state = (jetsam_aging_policy != kJetsamAgingPolicyLegacy) ? TRUE : FALSE;
-
- memorystatus_invalidate_idle_demotion_locked(p, reset_state);
- reschedule = TRUE;
- }
- }
-
- memorystatus_update_idle_priority_locked(p);
-
- if (reschedule) {
- memorystatus_reschedule_idle_demotion_locked();
- }
-
- ret = 0;
-
-exit:
- proc_list_unlock();
-
- return ret;
-}
-
-int
-memorystatus_dirty_set(proc_t p, boolean_t self, uint32_t pcontrol)
-{
- int ret;
- boolean_t kill = false;
- boolean_t reschedule = FALSE;
- boolean_t was_dirty = FALSE;
- boolean_t now_dirty = FALSE;
-
- MEMORYSTATUS_DEBUG(1, "memorystatus_dirty_set(): %d %d 0x%x 0x%x\n", self, p->p_pid, pcontrol, p->p_memstat_dirty);
- KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_DIRTY_SET), p->p_pid, self, pcontrol, 0, 0);
-
- proc_list_lock();
-
- if ((p->p_listflag & P_LIST_EXITED) != 0) {
- /*
- * Process is on its way out.
- */
- ret = EBUSY;
- goto exit;
- }
-
- if (p->p_memstat_state & P_MEMSTAT_INTERNAL) {
- ret = EPERM;
- goto exit;
- }
-
- if (p->p_memstat_dirty & P_DIRTY_IS_DIRTY) {
- was_dirty = TRUE;
- }
-
- if (!(p->p_memstat_dirty & P_DIRTY_TRACK)) {
- /* Dirty tracking not enabled */
- ret = EINVAL;
- } else if (pcontrol && (p->p_memstat_dirty & P_DIRTY_TERMINATED)) {
- /*
- * Process is set to be terminated and we're attempting to mark it dirty.
- * Set for termination and marking as clean is OK - see <rdar://problem/10594349>.
- */
- ret = EBUSY;
- } else {
- int flag = (self == TRUE) ? P_DIRTY : P_DIRTY_SHUTDOWN;
- if (pcontrol && !(p->p_memstat_dirty & flag)) {
- /* Mark the process as having been dirtied at some point */
- p->p_memstat_dirty |= (flag | P_DIRTY_MARKED);
- memorystatus_dirty_count++;
- ret = 0;
- } else if ((pcontrol == 0) && (p->p_memstat_dirty & flag)) {
- if ((flag == P_DIRTY_SHUTDOWN) && (!(p->p_memstat_dirty & P_DIRTY))) {
- /* Clearing the dirty shutdown flag, and the process is otherwise clean - kill */
- p->p_memstat_dirty |= P_DIRTY_TERMINATED;
- kill = true;
- } else if ((flag == P_DIRTY) && (p->p_memstat_dirty & P_DIRTY_TERMINATED)) {
- /* Kill previously terminated processes if set clean */
- kill = true;
- }
- p->p_memstat_dirty &= ~flag;
- memorystatus_dirty_count--;
- ret = 0;
- } else {
- /* Already set */
- ret = EALREADY;
- }
- }
-
- if (ret != 0) {
- goto exit;
- }
-
- if (p->p_memstat_dirty & P_DIRTY_IS_DIRTY) {
- now_dirty = TRUE;
- }
-
- if ((was_dirty == TRUE && now_dirty == FALSE) ||
- (was_dirty == FALSE && now_dirty == TRUE)) {
- /* Manage idle exit deferral, if applied */
- if ((p->p_memstat_dirty & P_DIRTY_IDLE_EXIT_ENABLED) == P_DIRTY_IDLE_EXIT_ENABLED) {
- /*
- * Legacy mode: P_DIRTY_AGING_IN_PROGRESS means the process is in the aging band OR it might be heading back
- * there once it's clean again. For the legacy case, this only applies if it has some protection window left.
- * P_DIRTY_DEFER: one-time protection window given at launch
- * P_DIRTY_DEFER_ALWAYS: protection window given for every dirty->clean transition. Like non-legacy mode.
- *
- * Non-Legacy mode: P_DIRTY_AGING_IN_PROGRESS means the process is in the aging band. It will always stop over
- * in that band on it's way to IDLE.
- */
-
- if (p->p_memstat_dirty & P_DIRTY_IS_DIRTY) {
- /*
- * New dirty process i.e. "was_dirty == FALSE && now_dirty == TRUE"
- *
- * The process will move from its aging band to its higher requested
- * jetsam band.
- */
- boolean_t reset_state = (jetsam_aging_policy != kJetsamAgingPolicyLegacy) ? TRUE : FALSE;
-
- memorystatus_invalidate_idle_demotion_locked(p, reset_state);
- reschedule = TRUE;
- } else {
- /*
- * Process is back from "dirty" to "clean".
- */
-
- if (jetsam_aging_policy == kJetsamAgingPolicyLegacy) {
- if (((p->p_memstat_dirty & P_DIRTY_DEFER_ALWAYS) == FALSE) &&
- (mach_absolute_time() >= p->p_memstat_idledeadline)) {
- /*
- * The process' hasn't enrolled in the "always defer after dirty"
- * mode and its deadline has expired. It currently
- * does not reside in any of the aging buckets.
- *
- * It's on its way to the JETSAM_PRIORITY_IDLE
- * bucket via memorystatus_update_idle_priority_locked()
- * below.
- *
- * So all we need to do is reset all the state on the
- * process that's related to the aging bucket i.e.
- * the AGING_IN_PROGRESS flag and the timer deadline.
- */
-
- memorystatus_invalidate_idle_demotion_locked(p, TRUE);
- reschedule = TRUE;
- } else {
- /*
- * Process enrolled in "always stop in deferral band after dirty" OR
- * it still has some protection window left and so
- * we just re-arm the timer without modifying any
- * state on the process iff it still wants into that band.
- */
-
- if (p->p_memstat_dirty & P_DIRTY_DEFER_ALWAYS) {
- memorystatus_schedule_idle_demotion_locked(p, TRUE);
- reschedule = TRUE;
- } else if (p->p_memstat_dirty & P_DIRTY_AGING_IN_PROGRESS) {
- memorystatus_schedule_idle_demotion_locked(p, FALSE);
- reschedule = TRUE;
- }
- }
- } else {
- memorystatus_schedule_idle_demotion_locked(p, TRUE);
- reschedule = TRUE;
- }
- }
- }
-
- memorystatus_update_idle_priority_locked(p);
-
- if (memorystatus_highwater_enabled) {
- boolean_t ledger_update_needed = TRUE;
- boolean_t use_active;
- boolean_t is_fatal;
- /*
- * We are in this path because this process transitioned between
- * dirty <--> clean state. Update the cached memory limits.
- */
-
- if (proc_jetsam_state_is_active_locked(p) == TRUE) {
- /*
- * process is pinned in elevated band
- * or
- * process is dirty
- */
- CACHE_ACTIVE_LIMITS_LOCKED(p, is_fatal);
- use_active = TRUE;
- ledger_update_needed = TRUE;
- } else {
- /*
- * process is clean...but if it has opted into pressured-exit
- * we don't apply the INACTIVE limit till the process has aged
- * out and is entering the IDLE band.
- * See memorystatus_update_priority_locked() for that.
- */
-
- if (p->p_memstat_dirty & P_DIRTY_ALLOW_IDLE_EXIT) {
- ledger_update_needed = FALSE;
- } else {
- CACHE_INACTIVE_LIMITS_LOCKED(p, is_fatal);
- use_active = FALSE;
- ledger_update_needed = TRUE;
- }
- }
-
- /*
- * Enforce the new limits by writing to the ledger.
- *
- * This is a hot path and holding the proc_list_lock while writing to the ledgers,
- * (where the task lock is taken) is bad. So, we temporarily drop the proc_list_lock.
- * We aren't traversing the jetsam bucket list here, so we should be safe.
- * See rdar://21394491.
- */
-
- if (ledger_update_needed && proc_ref_locked(p) == p) {
- int ledger_limit;
- if (p->p_memstat_memlimit > 0) {
- ledger_limit = p->p_memstat_memlimit;
- } else {
- ledger_limit = -1;
- }
- proc_list_unlock();
- task_set_phys_footprint_limit_internal(p->task, ledger_limit, NULL, use_active, is_fatal);
- proc_list_lock();
- proc_rele_locked(p);
-
- MEMORYSTATUS_DEBUG(3, "memorystatus_dirty_set: new limit on pid %d (%dMB %s) priority(%d) dirty?=0x%x %s\n",
- p->p_pid, (p->p_memstat_memlimit > 0 ? p->p_memstat_memlimit : -1),
- (p->p_memstat_state & P_MEMSTAT_FATAL_MEMLIMIT ? "F " : "NF"), p->p_memstat_effectivepriority, p->p_memstat_dirty,
- (p->p_memstat_dirty ? ((p->p_memstat_dirty & P_DIRTY) ? "isdirty" : "isclean") : ""));
- }
- }
-
- /* If the deferral state changed, reschedule the demotion timer */
- if (reschedule) {
- memorystatus_reschedule_idle_demotion_locked();
- }
- }
-
- if (kill) {
- if (proc_ref_locked(p) == p) {
- proc_list_unlock();
- psignal(p, SIGKILL);
- proc_list_lock();
- proc_rele_locked(p);
- }
- }
-
-exit:
- proc_list_unlock();
-
- return ret;
-}
-
-int
-memorystatus_dirty_clear(proc_t p, uint32_t pcontrol)
-{
- int ret = 0;
-
- MEMORYSTATUS_DEBUG(1, "memorystatus_dirty_clear(): %d 0x%x 0x%x\n", p->p_pid, pcontrol, p->p_memstat_dirty);
-
- KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_DIRTY_CLEAR), p->p_pid, pcontrol, 0, 0, 0);
-
- proc_list_lock();
-
- if ((p->p_listflag & P_LIST_EXITED) != 0) {
- /*
- * Process is on its way out.
- */
- ret = EBUSY;
- goto exit;
- }
-
- if (p->p_memstat_state & P_MEMSTAT_INTERNAL) {
- ret = EPERM;
- goto exit;
- }
-
- if (!(p->p_memstat_dirty & P_DIRTY_TRACK)) {
- /* Dirty tracking not enabled */
- ret = EINVAL;
- goto exit;
- }
-
- if (!pcontrol || (pcontrol & (PROC_DIRTY_LAUNCH_IN_PROGRESS | PROC_DIRTY_DEFER | PROC_DIRTY_DEFER_ALWAYS)) == 0) {
- ret = EINVAL;
- goto exit;
- }
-
- if (pcontrol & PROC_DIRTY_LAUNCH_IN_PROGRESS) {
- p->p_memstat_dirty &= ~P_DIRTY_LAUNCH_IN_PROGRESS;
- }
-
- /* This can be set and cleared exactly once. */
- if (pcontrol & (PROC_DIRTY_DEFER | PROC_DIRTY_DEFER_ALWAYS)) {
- if (p->p_memstat_dirty & P_DIRTY_DEFER) {
- p->p_memstat_dirty &= ~(P_DIRTY_DEFER);
- }
-
- if (p->p_memstat_dirty & P_DIRTY_DEFER_ALWAYS) {
- p->p_memstat_dirty &= ~(P_DIRTY_DEFER_ALWAYS);
- }
-
- memorystatus_invalidate_idle_demotion_locked(p, TRUE);
- memorystatus_update_idle_priority_locked(p);
- memorystatus_reschedule_idle_demotion_locked();
- }
-
- ret = 0;
-exit:
- proc_list_unlock();
-
- return ret;
-}
-
-int
-memorystatus_dirty_get(proc_t p)
-{
- int ret = 0;
-
- proc_list_lock();
-
- if (p->p_memstat_dirty & P_DIRTY_TRACK) {
- ret |= PROC_DIRTY_TRACKED;
- if (p->p_memstat_dirty & P_DIRTY_ALLOW_IDLE_EXIT) {
- ret |= PROC_DIRTY_ALLOWS_IDLE_EXIT;
- }
- if (p->p_memstat_dirty & P_DIRTY) {
- ret |= PROC_DIRTY_IS_DIRTY;
- }
- if (p->p_memstat_dirty & P_DIRTY_LAUNCH_IN_PROGRESS) {
- ret |= PROC_DIRTY_LAUNCH_IS_IN_PROGRESS;
- }
- }
-
- proc_list_unlock();
-
- return ret;
-}
-
-int
-memorystatus_on_terminate(proc_t p)
-{
- int sig;
-
- proc_list_lock();
-
- p->p_memstat_dirty |= P_DIRTY_TERMINATED;
-
- if ((p->p_memstat_dirty & (P_DIRTY_TRACK | P_DIRTY_IS_DIRTY)) == P_DIRTY_TRACK) {
- /* Clean; mark as terminated and issue SIGKILL */
- sig = SIGKILL;
- } else {
- /* Dirty, terminated, or state tracking is unsupported; issue SIGTERM to allow cleanup */
- sig = SIGTERM;
- }
-
- proc_list_unlock();
-
- return sig;
-}
-
-void
-memorystatus_on_suspend(proc_t p)
-{
-#if CONFIG_FREEZE
- uint32_t pages;
- memorystatus_get_task_page_counts(p->task, &pages, NULL, NULL);
-#endif
- proc_list_lock();
-#if CONFIG_FREEZE
- memorystatus_suspended_count++;
-#endif
- p->p_memstat_state |= P_MEMSTAT_SUSPENDED;
- proc_list_unlock();
-}
-
-void
-memorystatus_on_resume(proc_t p)
-{
-#if CONFIG_FREEZE
- boolean_t frozen;
- pid_t pid;
-#endif
-
- proc_list_lock();
-
-#if CONFIG_FREEZE
- frozen = (p->p_memstat_state & P_MEMSTAT_FROZEN);
- if (frozen) {
- /*
- * Now that we don't _thaw_ a process completely,
- * resuming it (and having some on-demand swapins)
- * shouldn't preclude it from being counted as frozen.
- *
- * memorystatus_frozen_count--;
- *
- * We preserve the P_MEMSTAT_FROZEN state since the process
- * could have state on disk AND so will deserve some protection
- * in the jetsam bands.
- */
- if ((p->p_memstat_state & P_MEMSTAT_REFREEZE_ELIGIBLE) == 0) {
- p->p_memstat_state |= P_MEMSTAT_REFREEZE_ELIGIBLE;
- memorystatus_refreeze_eligible_count++;
- }
- p->p_memstat_thaw_count++;
-
- memorystatus_thaw_count++;
- }
-
- memorystatus_suspended_count--;
-
- pid = p->p_pid;
-#endif
-
- /*
- * P_MEMSTAT_FROZEN will remain unchanged. This used to be:
- * p->p_memstat_state &= ~(P_MEMSTAT_SUSPENDED | P_MEMSTAT_FROZEN);
- */
- p->p_memstat_state &= ~P_MEMSTAT_SUSPENDED;
-
- proc_list_unlock();
-
-#if CONFIG_FREEZE
- if (frozen) {
- memorystatus_freeze_entry_t data = { pid, FALSE, 0 };
- memorystatus_send_note(kMemorystatusFreezeNote, &data, sizeof(data));
- }
-#endif
-}
-
-void
-memorystatus_on_inactivity(proc_t p)
-{
-#pragma unused(p)
-#if CONFIG_FREEZE
- /* Wake the freeze thread */
- thread_wakeup((event_t)&memorystatus_freeze_wakeup);
-#endif
-}
-
-/*
- * The proc_list_lock is held by the caller.
- */
-static uint32_t
-memorystatus_build_state(proc_t p)
-{
- uint32_t snapshot_state = 0;
-
- /* General */
- if (p->p_memstat_state & P_MEMSTAT_SUSPENDED) {
- snapshot_state |= kMemorystatusSuspended;
- }
- if (p->p_memstat_state & P_MEMSTAT_FROZEN) {
- snapshot_state |= kMemorystatusFrozen;
- }
- if (p->p_memstat_state & P_MEMSTAT_REFREEZE_ELIGIBLE) {
- snapshot_state |= kMemorystatusWasThawed;
- }
-
- /* Tracking */
- if (p->p_memstat_dirty & P_DIRTY_TRACK) {
- snapshot_state |= kMemorystatusTracked;
- }
- if ((p->p_memstat_dirty & P_DIRTY_IDLE_EXIT_ENABLED) == P_DIRTY_IDLE_EXIT_ENABLED) {
- snapshot_state |= kMemorystatusSupportsIdleExit;
- }
- if (p->p_memstat_dirty & P_DIRTY_IS_DIRTY) {
- snapshot_state |= kMemorystatusDirty;
- }
-
- return snapshot_state;
-}
-
-static boolean_t
-kill_idle_exit_proc(void)
-{
- proc_t p, victim_p = PROC_NULL;
- uint64_t current_time;
- boolean_t killed = FALSE;
- unsigned int i = 0;
- os_reason_t jetsam_reason = OS_REASON_NULL;
-
- /* Pick next idle exit victim. */
- current_time = mach_absolute_time();
-
- jetsam_reason = os_reason_create(OS_REASON_JETSAM, JETSAM_REASON_MEMORY_IDLE_EXIT);
- if (jetsam_reason == OS_REASON_NULL) {
- printf("kill_idle_exit_proc: failed to allocate jetsam reason\n");
- }
-
- proc_list_lock();
-
- p = memorystatus_get_first_proc_locked(&i, FALSE);
- while (p) {
- /* No need to look beyond the idle band */
- if (p->p_memstat_effectivepriority != JETSAM_PRIORITY_IDLE) {
- break;
- }
-
- if ((p->p_memstat_dirty & (P_DIRTY_ALLOW_IDLE_EXIT | P_DIRTY_IS_DIRTY | P_DIRTY_TERMINATED)) == (P_DIRTY_ALLOW_IDLE_EXIT)) {
- if (current_time >= p->p_memstat_idledeadline) {
- p->p_memstat_dirty |= P_DIRTY_TERMINATED;
- victim_p = proc_ref_locked(p);
- break;
- }
- }
-
- p = memorystatus_get_next_proc_locked(&i, p, FALSE);
- }
-
- proc_list_unlock();
-
- if (victim_p) {
- printf("memorystatus: killing_idle_process pid %d [%s]\n", victim_p->p_pid, (*victim_p->p_name ? victim_p->p_name : "unknown"));
- killed = memorystatus_do_kill(victim_p, kMemorystatusKilledIdleExit, jetsam_reason);
- proc_rele(victim_p);
- } else {
- os_reason_free(jetsam_reason);
- }
-
- return killed;
-}
-
-static void
-memorystatus_thread_wake(void)
-{
- int thr_id = 0;
- int active_thr = atomic_load(&active_jetsam_threads);
-
- /* Wakeup all the jetsam threads */
- for (thr_id = 0; thr_id < active_thr; thr_id++) {
- thread_wakeup((event_t)&jetsam_threads[thr_id].memorystatus_wakeup);
- }
-}
-
-#if CONFIG_JETSAM
-
-static void
-memorystatus_thread_pool_max()
-{
- /* Increase the jetsam thread pool to max_jetsam_threads */
- int max_threads = max_jetsam_threads;
- printf("Expanding memorystatus pool to %d!\n", max_threads);
- atomic_store(&active_jetsam_threads, max_threads);
-}
-
-static void
-memorystatus_thread_pool_default()
-{
- /* Restore the jetsam thread pool to a single thread */
- printf("Reverting memorystatus pool back to 1\n");
- atomic_store(&active_jetsam_threads, 1);
-}
-
-#endif /* CONFIG_JETSAM */
-
-extern void vm_pressure_response(void);
-
-static int
-memorystatus_thread_block(uint32_t interval_ms, thread_continue_t continuation)
-{
- struct jetsam_thread_state *jetsam_thread = jetsam_current_thread();
-
- if (interval_ms) {
- assert_wait_timeout(&jetsam_thread->memorystatus_wakeup, THREAD_UNINT, interval_ms, NSEC_PER_MSEC);
- } else {
- assert_wait(&jetsam_thread->memorystatus_wakeup, THREAD_UNINT);
- }
-
- return thread_block(continuation);
-}
-
-static boolean_t
-memorystatus_avail_pages_below_pressure(void)
-{
-#if CONFIG_EMBEDDED
-/*
- * Instead of CONFIG_EMBEDDED for these *avail_pages* routines, we should
- * key off of the system having dynamic swap support. With full swap support,
- * the system shouldn't really need to worry about various page thresholds.
- */
- return memorystatus_available_pages <= memorystatus_available_pages_pressure;
-#else /* CONFIG_EMBEDDED */
- return FALSE;
-#endif /* CONFIG_EMBEDDED */
-}
-
-static boolean_t
-memorystatus_avail_pages_below_critical(void)
-{
-#if CONFIG_EMBEDDED
- return memorystatus_available_pages <= memorystatus_available_pages_critical;
-#else /* CONFIG_EMBEDDED */
- return FALSE;
-#endif /* CONFIG_EMBEDDED */
-}
-
-static boolean_t
-memorystatus_post_snapshot(int32_t priority, uint32_t cause)
-{
-#if CONFIG_EMBEDDED
-#pragma unused(cause)
- /*
- * Don't generate logs for steady-state idle-exit kills,
- * unless it is overridden for debug or by the device
- * tree.
- */
-
- return (priority != JETSAM_PRIORITY_IDLE) || memorystatus_idle_snapshot;
-
-#else /* CONFIG_EMBEDDED */
- /*
- * Don't generate logs for steady-state idle-exit kills,
- * unless
- * - it is overridden for debug or by the device
- * tree.
- * OR
- * - the kill causes are important i.e. not kMemorystatusKilledIdleExit
- */
-
- boolean_t snapshot_eligible_kill_cause = (is_reason_thrashing(cause) || is_reason_zone_map_exhaustion(cause));
- return (priority != JETSAM_PRIORITY_IDLE) || memorystatus_idle_snapshot || snapshot_eligible_kill_cause;
-#endif /* CONFIG_EMBEDDED */
-}
-
-static boolean_t
-memorystatus_action_needed(void)
-{
-#if CONFIG_EMBEDDED
- return is_reason_thrashing(kill_under_pressure_cause) ||
- is_reason_zone_map_exhaustion(kill_under_pressure_cause) ||
- memorystatus_available_pages <= memorystatus_available_pages_pressure;
-#else /* CONFIG_EMBEDDED */
- return is_reason_thrashing(kill_under_pressure_cause) ||
- is_reason_zone_map_exhaustion(kill_under_pressure_cause);
-#endif /* CONFIG_EMBEDDED */
-}
-
-#if CONFIG_FREEZE
-extern void vm_swap_consider_defragmenting(int);
-
-/*
- * This routine will _jetsam_ all frozen processes
- * and reclaim the swap space immediately.
- *
- * So freeze has to be DISABLED when we call this routine.
- */
-
-void
-memorystatus_disable_freeze(void)
-{
- memstat_bucket_t *bucket;
- int bucket_count = 0, retries = 0;
- boolean_t retval = FALSE, killed = FALSE;
- uint32_t errors = 0, errors_over_prev_iteration = 0;
- os_reason_t jetsam_reason = 0;
- unsigned int band = 0;
- proc_t p = PROC_NULL, next_p = PROC_NULL;
-
- assert(memorystatus_freeze_enabled == FALSE);
-
- jetsam_reason = os_reason_create(OS_REASON_JETSAM, JETSAM_REASON_MEMORY_DISK_SPACE_SHORTAGE);
- if (jetsam_reason == OS_REASON_NULL) {
- printf("memorystatus_disable_freeze: failed to allocate jetsam reason\n");
- }
-
- /*
- * Let's relocate all frozen processes into band 8. Demoted frozen processes
- * are sitting in band 0 currently and it's possible to have a frozen process
- * in the FG band being actively used. We don't reset its frozen state when
- * it is resumed because it has state on disk.
- *
- * We choose to do this relocation rather than implement a new 'kill frozen'
- * process function for these reasons:
- * - duplication of code: too many kill functions exist and we need to rework them better.
- * - disk-space-shortage kills are rare
- * - not having the 'real' jetsam band at time of the this frozen kill won't preclude us
- * from answering any imp. questions re. jetsam policy/effectiveness.
- *
- * This is essentially what memorystatus_update_inactive_jetsam_priority_band() does while
- * avoiding the application of memory limits.
- */
-
-again:
- proc_list_lock();
-
- band = JETSAM_PRIORITY_IDLE;
- p = PROC_NULL;
- next_p = PROC_NULL;
-
- next_p = memorystatus_get_first_proc_locked(&band, TRUE);
- while (next_p) {
- p = next_p;
- next_p = memorystatus_get_next_proc_locked(&band, p, TRUE);
-
- if (p->p_memstat_effectivepriority > JETSAM_PRIORITY_FOREGROUND) {
- break;
- }
-
- if ((p->p_memstat_state & P_MEMSTAT_FROZEN) == FALSE) {
- continue;
- }
-
- if (p->p_memstat_state & P_MEMSTAT_ERROR) {
- p->p_memstat_state &= ~P_MEMSTAT_ERROR;
- }
-
- if (p->p_memstat_effectivepriority == memorystatus_freeze_jetsam_band) {
- continue;
- }
-
- /*
- * We explicitly add this flag here so the process looks like a normal
- * frozen process i.e. P_MEMSTAT_FROZEN and P_MEMSTAT_USE_ELEVATED_INACTIVE_BAND.
- * We don't bother with assigning the 'active' memory
- * limits at this point because we are going to be killing it soon below.
- */
- p->p_memstat_state |= P_MEMSTAT_USE_ELEVATED_INACTIVE_BAND;
- memorystatus_invalidate_idle_demotion_locked(p, TRUE);
-
- memorystatus_update_priority_locked(p, memorystatus_freeze_jetsam_band, FALSE, TRUE);
- }
-
- bucket = &memstat_bucket[memorystatus_freeze_jetsam_band];
- bucket_count = bucket->count;
- proc_list_unlock();
-
- /*
- * Bucket count is already stale at this point. But, we don't expect
- * freezing to continue since we have already disabled the freeze functionality.
- * However, an existing freeze might be in progress. So we might miss that process
- * in the first go-around. We hope to catch it in the next.
- */
-
- errors_over_prev_iteration = 0;
- while (bucket_count) {
- bucket_count--;
-
- /*
- * memorystatus_kill_elevated_process() drops a reference,
- * so take another one so we can continue to use this exit reason
- * even after it returns.
- */
-
- os_reason_ref(jetsam_reason);
- retval = memorystatus_kill_elevated_process(
- kMemorystatusKilledDiskSpaceShortage,
- jetsam_reason,
- memorystatus_freeze_jetsam_band,
- 0, /* the iteration of aggressive jetsam..ignored here */
- &errors);
-
- if (errors > 0) {
- printf("memorystatus_disable_freeze: memorystatus_kill_elevated_process returned %d error(s)\n", errors);
- errors_over_prev_iteration += errors;
- errors = 0;
- }
-
- if (retval == 0) {
- /*
- * No frozen processes left to kill.
- */
- break;
- }
-
- killed = TRUE;
- }
-
- proc_list_lock();
-
- if (memorystatus_frozen_count) {
- /*
- * A frozen process snuck in and so
- * go back around to kill it. That
- * process may have been resumed and
- * put into the FG band too. So we
- * have to do the relocation again.
- */
- assert(memorystatus_freeze_enabled == FALSE);
-
- retries++;
- if (retries < 3) {
- proc_list_unlock();
- goto again;
- }
-#if DEVELOPMENT || DEBUG
- panic("memorystatus_disable_freeze: Failed to kill all frozen processes, memorystatus_frozen_count = %d, errors = %d",
- memorystatus_frozen_count, errors_over_prev_iteration);
-#endif /* DEVELOPMENT || DEBUG */
- }
- proc_list_unlock();
-
- os_reason_free(jetsam_reason);
-
- if (killed) {
- vm_swap_consider_defragmenting(VM_SWAP_FLAGS_FORCE_DEFRAG | VM_SWAP_FLAGS_FORCE_RECLAIM);
-
- proc_list_lock();
- size_t snapshot_size = sizeof(memorystatus_jetsam_snapshot_t) +
- sizeof(memorystatus_jetsam_snapshot_entry_t) * (memorystatus_jetsam_snapshot_count);
- uint64_t timestamp_now = mach_absolute_time();
- memorystatus_jetsam_snapshot->notification_time = timestamp_now;
- memorystatus_jetsam_snapshot->js_gencount++;
- if (memorystatus_jetsam_snapshot_count > 0 && (memorystatus_jetsam_snapshot_last_timestamp == 0 ||
- timestamp_now > memorystatus_jetsam_snapshot_last_timestamp + memorystatus_jetsam_snapshot_timeout)) {
- proc_list_unlock();
- int ret = memorystatus_send_note(kMemorystatusSnapshotNote, &snapshot_size, sizeof(snapshot_size));
- if (!ret) {
- proc_list_lock();
- memorystatus_jetsam_snapshot_last_timestamp = timestamp_now;
- proc_list_unlock();
- }
- } else {
- proc_list_unlock();
- }
- }
-
- return;
-}
-#endif /* CONFIG_FREEZE */
-
-static boolean_t
-memorystatus_act_on_hiwat_processes(uint32_t *errors, uint32_t *hwm_kill, boolean_t *post_snapshot, __unused boolean_t *is_critical)
-{
- boolean_t purged = FALSE;
- boolean_t killed = memorystatus_kill_hiwat_proc(errors, &purged);
-
- if (killed) {
- *hwm_kill = *hwm_kill + 1;
- *post_snapshot = TRUE;
- return TRUE;
- } else {
- if (purged == FALSE) {
- /* couldn't purge and couldn't kill */
- memorystatus_hwm_candidates = FALSE;
- }
- }
-
-#if CONFIG_JETSAM
- /* No highwater processes to kill. Continue or stop for now? */
- if (!is_reason_thrashing(kill_under_pressure_cause) &&
- !is_reason_zone_map_exhaustion(kill_under_pressure_cause) &&
- (memorystatus_available_pages > memorystatus_available_pages_critical)) {
- /*
- * We are _not_ out of pressure but we are above the critical threshold and there's:
- * - no compressor thrashing
- * - enough zone memory
- * - no more HWM processes left.
- * For now, don't kill any other processes.
- */
-
- if (*hwm_kill == 0) {
- memorystatus_thread_wasted_wakeup++;
- }
-
- *is_critical = FALSE;
-
- return TRUE;
- }
-#endif /* CONFIG_JETSAM */
-
- return FALSE;
-}
-
-static boolean_t
-memorystatus_act_aggressive(uint32_t cause, os_reason_t jetsam_reason, int *jld_idle_kills, boolean_t *corpse_list_purged, boolean_t *post_snapshot)
-{
- if (memorystatus_jld_enabled == TRUE) {
- boolean_t killed;
- uint32_t errors = 0;
-
- /* Jetsam Loop Detection - locals */
- memstat_bucket_t *bucket;
- int jld_bucket_count = 0;
- struct timeval jld_now_tstamp = {0, 0};
- uint64_t jld_now_msecs = 0;
- int elevated_bucket_count = 0;
-
- /* Jetsam Loop Detection - statics */
- static uint64_t jld_timestamp_msecs = 0;
- static int jld_idle_kill_candidates = 0; /* Number of available processes in band 0,1 at start */
- static int jld_eval_aggressive_count = 0; /* Bumps the max priority in aggressive loop */
- static int32_t jld_priority_band_max = JETSAM_PRIORITY_UI_SUPPORT;
- /*
- * Jetsam Loop Detection: attempt to detect
- * rapid daemon relaunches in the lower bands.
- */
-
- microuptime(&jld_now_tstamp);
-
- /*
- * Ignore usecs in this calculation.
- * msecs granularity is close enough.
- */
- jld_now_msecs = (jld_now_tstamp.tv_sec * 1000);
-
- proc_list_lock();
- switch (jetsam_aging_policy) {
- case kJetsamAgingPolicyLegacy:
- bucket = &memstat_bucket[JETSAM_PRIORITY_IDLE];
- jld_bucket_count = bucket->count;
- bucket = &memstat_bucket[JETSAM_PRIORITY_AGING_BAND1];
- jld_bucket_count += bucket->count;
- break;
- case kJetsamAgingPolicySysProcsReclaimedFirst:
- case kJetsamAgingPolicyAppsReclaimedFirst:
- bucket = &memstat_bucket[JETSAM_PRIORITY_IDLE];
- jld_bucket_count = bucket->count;
- bucket = &memstat_bucket[system_procs_aging_band];
- jld_bucket_count += bucket->count;
- bucket = &memstat_bucket[applications_aging_band];
- jld_bucket_count += bucket->count;
- break;
- case kJetsamAgingPolicyNone:
- default:
- bucket = &memstat_bucket[JETSAM_PRIORITY_IDLE];
- jld_bucket_count = bucket->count;
- break;
- }
projects / apple / xnu.git / blobdiff