+ proc_list_lock();
+
+ if (p->p_memstat_memlimit_active > 0) {
+ memlimit_mb_active = p->p_memstat_memlimit_active;
+ } else if (p->p_memstat_memlimit_active == -1) {
+ memlimit_mb_active = max_task_footprint_mb;
+ } else {
+ /*
+ * Nothing to do for '0' which is
+ * a special value only used internally
+ * to test 'no limits'.
+ */
+ proc_list_unlock();
+ return;
+ }
+
+ if (p->p_memstat_memlimit_inactive > 0) {
+ memlimit_mb_inactive = p->p_memstat_memlimit_inactive;
+ } else if (p->p_memstat_memlimit_inactive == -1) {
+ memlimit_mb_inactive = max_task_footprint_mb;
+ } else {
+ /*
+ * Nothing to do for '0' which is
+ * a special value only used internally
+ * to test 'no limits'.
+ */
+ proc_list_unlock();
+ return;
+ }
+
+ if (footprint_increase) {
+ memlimit_mb_active += legacy_footprint_bonus_mb;
+ memlimit_mb_inactive += legacy_footprint_bonus_mb;
+ } else {
+ memlimit_mb_active -= legacy_footprint_bonus_mb;
+ if (memlimit_mb_active == max_task_footprint_mb) {
+ memlimit_mb_active = -1; /* reverting back to default system limit */
+ }
+
+ memlimit_mb_inactive -= legacy_footprint_bonus_mb;
+ if (memlimit_mb_inactive == max_task_footprint_mb) {
+ memlimit_mb_inactive = -1; /* reverting back to default system limit */
+ }
+ }
+ memorystatus_raise_memlimit(p, memlimit_mb_active, memlimit_mb_inactive);
+
+ proc_list_unlock();
+}
+
+void
+memorystatus_act_on_ios13extended_footprint_entitlement(proc_t p)
+{
+ if (max_mem < 1500ULL * 1024 * 1024 ||
+ max_mem > 2ULL * 1024 * 1024 * 1024) {
+ /* ios13extended_footprint is only for 2GB devices */
+ return;
+ }
+ /* limit to "almost 2GB" */
+ proc_list_lock();
+ memorystatus_raise_memlimit(p, 1800, 1800);
+ proc_list_unlock();
+}
+
+void
+memorystatus_act_on_entitled_task_limit(proc_t p)
+{
+ if (memorystatus_entitled_max_task_footprint_mb == 0) {
+ // Entitlement is not supported on this device.
+ return;
+ }
+ proc_list_lock();
+ memorystatus_raise_memlimit(p, memorystatus_entitled_max_task_footprint_mb, memorystatus_entitled_max_task_footprint_mb);
+ proc_list_unlock();
+}
+#endif /* __arm64__ */
+
+SYSCTL_INT(_kern, OID_AUTO, memorystatus_level, CTLFLAG_RD | CTLFLAG_LOCKED, &memorystatus_level, 0, "");
+
+int
+memorystatus_get_level(__unused struct proc *p, struct memorystatus_get_level_args *args, __unused int *ret)
+{
+ user_addr_t level = 0;
+
+ level = args->level;
+
+ if (copyout(&memorystatus_level, level, sizeof(memorystatus_level)) != 0) {
+ return EFAULT;
+ }
+
+ return 0;
+}
+
+static void memorystatus_thread(void *param __unused, wait_result_t wr __unused);
+
+/* Memory Limits */
+
+static boolean_t memorystatus_kill_specific_process(pid_t victim_pid, uint32_t cause, os_reason_t jetsam_reason);
+static boolean_t memorystatus_kill_process_sync(pid_t victim_pid, uint32_t cause, os_reason_t jetsam_reason);
+
+
+static int memorystatus_cmd_set_memlimit_properties(pid_t pid, user_addr_t buffer, size_t buffer_size, __unused int32_t *retval);
+
+static int memorystatus_set_memlimit_properties(pid_t pid, memorystatus_memlimit_properties_t *entry);
+
+static int memorystatus_cmd_get_memlimit_properties(pid_t pid, user_addr_t buffer, size_t buffer_size, __unused int32_t *retval);
+
+static int memorystatus_cmd_get_memlimit_excess_np(pid_t pid, uint32_t flags, user_addr_t buffer, size_t buffer_size, __unused int32_t *retval);
+
+static void memorystatus_get_memlimit_properties_internal(proc_t p, memorystatus_memlimit_properties_t *p_entry);
+static int memorystatus_set_memlimit_properties_internal(proc_t p, memorystatus_memlimit_properties_t *p_entry);
+
+int proc_get_memstat_priority(proc_t, boolean_t);
+
+static boolean_t memorystatus_idle_snapshot = 0;
+
+unsigned int memorystatus_delta = 0;
+
+/* Jetsam Loop Detection */
+static boolean_t memorystatus_jld_enabled = FALSE; /* Enable jetsam loop detection */
+static uint32_t memorystatus_jld_eval_period_msecs = 0; /* Init pass sets this based on device memory size */
+static int memorystatus_jld_eval_aggressive_count = 3; /* Raise the priority max after 'n' aggressive loops */
+static int memorystatus_jld_eval_aggressive_priority_band_max = 15; /* Kill aggressively up through this band */
+
+/*
+ * A FG app can request that the aggressive jetsam mechanism display some leniency in the FG band. This 'lenient' mode is described as:
+ * --- if aggressive jetsam kills an app in the FG band and gets back >=AGGRESSIVE_JETSAM_LENIENT_MODE_THRESHOLD memory, it will stop the aggressive march further into and up the jetsam bands.
+ *
+ * RESTRICTIONS:
+ * - Such a request is respected/acknowledged only once while that 'requesting' app is in the FG band i.e. if aggressive jetsam was
+ * needed and the 'lenient' mode was deployed then that's it for this special mode while the app is in the FG band.
+ *
+ * - If the app is still in the FG band and aggressive jetsam is needed again, there will be no stop-and-check the next time around.
+ *
+ * - Also, the transition of the 'requesting' app away from the FG band will void this special behavior.
+ */
+
+#define AGGRESSIVE_JETSAM_LENIENT_MODE_THRESHOLD 25
+boolean_t memorystatus_aggressive_jetsam_lenient_allowed = FALSE;
+boolean_t memorystatus_aggressive_jetsam_lenient = FALSE;
+
+#if DEVELOPMENT || DEBUG
+/*
+ * Jetsam Loop Detection tunables.
+ */
+
+SYSCTL_UINT(_kern, OID_AUTO, memorystatus_jld_eval_period_msecs, CTLFLAG_RW | CTLFLAG_LOCKED, &memorystatus_jld_eval_period_msecs, 0, "");
+SYSCTL_UINT(_kern, OID_AUTO, memorystatus_jld_eval_aggressive_count, CTLFLAG_RW | CTLFLAG_LOCKED, &memorystatus_jld_eval_aggressive_count, 0, "");
+SYSCTL_UINT(_kern, OID_AUTO, memorystatus_jld_eval_aggressive_priority_band_max, CTLFLAG_RW | CTLFLAG_LOCKED, &memorystatus_jld_eval_aggressive_priority_band_max, 0, "");
+#endif /* DEVELOPMENT || DEBUG */
+
+static uint32_t kill_under_pressure_cause = 0;
+
+/*
+ * snapshot support for memstats collected at boot.
+ */
+static memorystatus_jetsam_snapshot_t memorystatus_at_boot_snapshot;
+
+static void memorystatus_init_jetsam_snapshot_locked(memorystatus_jetsam_snapshot_t *od_snapshot, uint32_t ods_list_count);
+static boolean_t memorystatus_init_jetsam_snapshot_entry_locked(proc_t p, memorystatus_jetsam_snapshot_entry_t *entry, uint64_t gencount);
+static void memorystatus_update_jetsam_snapshot_entry_locked(proc_t p, uint32_t kill_cause, uint64_t killtime);
+
+static void memorystatus_clear_errors(void);
+static void memorystatus_get_task_phys_footprint_page_counts(task_t task,
+ uint64_t *internal_pages, uint64_t *internal_compressed_pages,
+ uint64_t *purgeable_nonvolatile_pages, uint64_t *purgeable_nonvolatile_compressed_pages,
+ uint64_t *alternate_accounting_pages, uint64_t *alternate_accounting_compressed_pages,
+ uint64_t *iokit_mapped_pages, uint64_t *page_table_pages, uint64_t *frozen_to_swap_pages);
+
+static void memorystatus_get_task_memory_region_count(task_t task, uint64_t *count);
+
+static uint32_t memorystatus_build_state(proc_t p);
+//static boolean_t memorystatus_issue_pressure_kevent(boolean_t pressured);
+
+static boolean_t memorystatus_kill_top_process(boolean_t any, boolean_t sort_flag, uint32_t cause, os_reason_t jetsam_reason, int32_t *priority,
+ uint32_t *errors, uint64_t *memory_reclaimed);
+static boolean_t memorystatus_kill_processes_aggressive(uint32_t cause, int aggr_count, int32_t priority_max, uint32_t *errors, uint64_t *memory_reclaimed);
+static boolean_t memorystatus_kill_hiwat_proc(uint32_t *errors, boolean_t *purged, uint64_t *memory_reclaimed);
+
+static boolean_t memorystatus_kill_process_async(pid_t victim_pid, uint32_t cause);
+
+/* Priority Band Sorting Routines */
+static int memorystatus_sort_bucket(unsigned int bucket_index, int sort_order);
+static int memorystatus_sort_by_largest_coalition_locked(unsigned int bucket_index, int coal_sort_order);
+static void memorystatus_sort_by_largest_process_locked(unsigned int bucket_index);
+static int memorystatus_move_list_locked(unsigned int bucket_index, pid_t *pid_list, int list_sz);
+
+/* qsort routines */
+typedef int (*cmpfunc_t)(const void *a, const void *b);
+extern void qsort(void *a, size_t n, size_t es, cmpfunc_t cmp);
+static int memstat_asc_cmp(const void *a, const void *b);
+
+/* VM pressure */
+
+extern unsigned int vm_page_free_count;
+extern unsigned int vm_page_active_count;
+extern unsigned int vm_page_inactive_count;
+extern unsigned int vm_page_throttled_count;
+extern unsigned int vm_page_purgeable_count;
+extern unsigned int vm_page_wire_count;
+extern unsigned int vm_page_speculative_count;
+
+#if CONFIG_JETSAM
+#define MEMORYSTATUS_LOG_AVAILABLE_PAGES memorystatus_available_pages
+#else /* CONFIG_JETSAM */
+#define MEMORYSTATUS_LOG_AVAILABLE_PAGES (vm_page_active_count + vm_page_inactive_count + vm_page_free_count + vm_page_speculative_count)
+#endif /* CONFIG_JETSAM */
+#if CONFIG_SECLUDED_MEMORY
+extern unsigned int vm_page_secluded_count;
+extern unsigned int vm_page_secluded_count_over_target;
+#endif /* CONFIG_SECLUDED_MEMORY */
+
+/* Aggressive jetsam pages threshold for sysproc aging policy */
+unsigned int memorystatus_sysproc_aging_aggr_pages = 0;
+
+#if CONFIG_JETSAM
+unsigned int memorystatus_available_pages = (unsigned int)-1;
+unsigned int memorystatus_available_pages_pressure = 0;
+unsigned int memorystatus_available_pages_critical = 0;
+unsigned int memorystatus_available_pages_critical_base = 0;
+unsigned int memorystatus_available_pages_critical_idle_offset = 0;
+
+#if DEVELOPMENT || DEBUG
+SYSCTL_UINT(_kern, OID_AUTO, memorystatus_available_pages, CTLFLAG_RD | CTLFLAG_LOCKED, &memorystatus_available_pages, 0, "");
+#else
+SYSCTL_UINT(_kern, OID_AUTO, memorystatus_available_pages, CTLFLAG_RD | CTLFLAG_MASKED | CTLFLAG_LOCKED, &memorystatus_available_pages, 0, "");
+#endif /* DEVELOPMENT || DEBUG */
+
+static unsigned int memorystatus_jetsam_policy = kPolicyDefault;
+unsigned int memorystatus_policy_more_free_offset_pages = 0;
+static void memorystatus_update_levels_locked(boolean_t critical_only);
+static unsigned int memorystatus_thread_wasted_wakeup = 0;
+
+/* Callback into vm_compressor.c to signal that thrashing has been mitigated. */
+extern void vm_thrashing_jetsam_done(void);
+static int memorystatus_cmd_set_jetsam_memory_limit(pid_t pid, int32_t high_water_mark, __unused int32_t *retval, boolean_t is_fatal_limit);
+#if DEVELOPMENT || DEBUG
+static inline uint32_t
+roundToNearestMB(uint32_t in)
+{
+ return (in + ((1 << 20) - 1)) >> 20;
+}
+
+static int memorystatus_cmd_increase_jetsam_task_limit(pid_t pid, uint32_t byte_increase);
+#endif
+
+int32_t max_kill_priority = JETSAM_PRIORITY_MAX;
+
+#else /* CONFIG_JETSAM */
+
+uint64_t memorystatus_available_pages = (uint64_t)-1;
+uint64_t memorystatus_available_pages_pressure = (uint64_t)-1;
+uint64_t memorystatus_available_pages_critical = (uint64_t)-1;
+
+int32_t max_kill_priority = JETSAM_PRIORITY_IDLE;
+#endif /* CONFIG_JETSAM */
+
+#if DEVELOPMENT || DEBUG
+
+static LCK_GRP_DECLARE(disconnect_page_mappings_lck_grp, "disconnect_page_mappings");
+static LCK_MTX_DECLARE(disconnect_page_mappings_mutex, &disconnect_page_mappings_lck_grp);
+
+extern bool kill_on_no_paging_space;
+#endif /* DEVELOPMENT || DEBUG */
+
+
+/* Debug */
+
+extern struct knote *vm_find_knote_from_pid(pid_t, struct klist *);
+
+#if DEVELOPMENT || DEBUG
+
+static unsigned int memorystatus_debug_dump_this_bucket = 0;
+
+static void
+memorystatus_debug_dump_bucket_locked(unsigned int bucket_index)
+{
+ proc_t p = NULL;
+ uint64_t bytes = 0;
+ int ledger_limit = 0;
+ unsigned int b = bucket_index;
+ boolean_t traverse_all_buckets = FALSE;
+
+ if (bucket_index >= MEMSTAT_BUCKET_COUNT) {
+ traverse_all_buckets = TRUE;
+ b = 0;
+ } else {
+ traverse_all_buckets = FALSE;
+ b = bucket_index;
+ }
+
+ /*
+ * footprint reported in [pages / MB ]
+ * limits reported as:
+ * L-limit proc's Ledger limit
+ * C-limit proc's Cached limit, should match Ledger
+ * A-limit proc's Active limit
+ * IA-limit proc's Inactive limit
+ * F==Fatal, NF==NonFatal
+ */
+
+ printf("memorystatus_debug_dump ***START*(PAGE_SIZE_64=%llu)**\n", PAGE_SIZE_64);
+ printf("bucket [pid] [pages / MB] [state] [EP / RP / AP] dirty deadline [L-limit / C-limit / A-limit / IA-limit] name\n");
+ p = memorystatus_get_first_proc_locked(&b, traverse_all_buckets);
projects / apple / xnu.git / blobdiff