]> git.saurik.com Git - apple/xnu.git/blob - bsd/kern/kern_memorystatus.c
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-3248.50.21.tar.gz
[apple/xnu.git] / bsd / kern / kern_memorystatus.c
1 /*
2 * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 *
28 */
29
30 #include <kern/sched_prim.h>
31 #include <kern/kalloc.h>
32 #include <kern/assert.h>
33 #include <kern/debug.h>
34 #include <kern/locks.h>
35 #include <kern/task.h>
36 #include <kern/thread.h>
37 #include <kern/host.h>
38 #include <libkern/libkern.h>
39 #include <mach/coalition.h>
40 #include <mach/mach_time.h>
41 #include <mach/task.h>
42 #include <mach/host_priv.h>
43 #include <mach/mach_host.h>
44 #include <pexpert/pexpert.h>
45 #include <sys/coalition.h>
46 #include <sys/kern_event.h>
47 #include <sys/proc.h>
48 #include <sys/proc_info.h>
49 #include <sys/signal.h>
50 #include <sys/signalvar.h>
51 #include <sys/sysctl.h>
52 #include <sys/sysproto.h>
53 #include <sys/wait.h>
54 #include <sys/tree.h>
55 #include <sys/priv.h>
56 #include <vm/vm_pageout.h>
57 #include <vm/vm_protos.h>
58
59 #if CONFIG_FREEZE
60 #include <vm/vm_map.h>
61 #endif /* CONFIG_FREEZE */
62
63 #include <sys/kern_memorystatus.h>
64
65 #if CONFIG_JETSAM
66 /* For logging clarity */
67 static const char *jetsam_kill_cause_name[] = {
68 "" ,
69 "jettisoned" , /* kMemorystatusKilled */
70 "highwater" , /* kMemorystatusKilledHiwat */
71 "vnode-limit" , /* kMemorystatusKilledVnodes */
72 "vm-pageshortage" , /* kMemorystatusKilledVMPageShortage */
73 "vm-thrashing" , /* kMemorystatusKilledVMThrashing */
74 "fc-thrashing" , /* kMemorystatusKilledFCThrashing */
75 "per-process-limit" , /* kMemorystatusKilledPerProcessLimit */
76 "diagnostic" , /* kMemorystatusKilledDiagnostic */
77 "idle-exit" , /* kMemorystatusKilledIdleExit */
78 };
79
80 /* Does cause indicate vm or fc thrashing? */
81 static boolean_t
82 is_thrashing(unsigned cause)
83 {
84 switch (cause) {
85 case kMemorystatusKilledVMThrashing:
86 case kMemorystatusKilledFCThrashing:
87 return TRUE;
88 default:
89 return FALSE;
90 }
91 }
92
93 /* Callback into vm_compressor.c to signal that thrashing has been mitigated. */
94 extern void vm_thrashing_jetsam_done(void);
95 #endif
96
97 /* These are very verbose printfs(), enable with
98 * MEMORYSTATUS_DEBUG_LOG
99 */
100 #if MEMORYSTATUS_DEBUG_LOG
101 #define MEMORYSTATUS_DEBUG(cond, format, ...) \
102 do { \
103 if (cond) { printf(format, ##__VA_ARGS__); } \
104 } while(0)
105 #else
106 #define MEMORYSTATUS_DEBUG(cond, format, ...)
107 #endif
108
109 /*
110 * Active / Inactive limit support
111 * proc list must be locked
112 *
113 * The SET_*** macros are used to initialize a limit
114 * for the first time.
115 *
116 * The CACHE_*** macros are use to cache the limit that will
117 * soon be in effect down in the ledgers.
118 */
119
120 #define SET_ACTIVE_LIMITS_LOCKED(p, limit, is_fatal) \
121 MACRO_BEGIN \
122 (p)->p_memstat_memlimit_active = (limit); \
123 (p)->p_memstat_state &= ~P_MEMSTAT_MEMLIMIT_ACTIVE_EXC_TRIGGERED; \
124 if (is_fatal) { \
125 (p)->p_memstat_state |= P_MEMSTAT_MEMLIMIT_ACTIVE_FATAL; \
126 } else { \
127 (p)->p_memstat_state &= ~P_MEMSTAT_MEMLIMIT_ACTIVE_FATAL; \
128 } \
129 MACRO_END
130
131 #define SET_INACTIVE_LIMITS_LOCKED(p, limit, is_fatal) \
132 MACRO_BEGIN \
133 (p)->p_memstat_memlimit_inactive = (limit); \
134 (p)->p_memstat_state &= ~P_MEMSTAT_MEMLIMIT_INACTIVE_EXC_TRIGGERED; \
135 if (is_fatal) { \
136 (p)->p_memstat_state |= P_MEMSTAT_MEMLIMIT_INACTIVE_FATAL; \
137 } else { \
138 (p)->p_memstat_state &= ~P_MEMSTAT_MEMLIMIT_INACTIVE_FATAL; \
139 } \
140 MACRO_END
141
142 #define CACHE_ACTIVE_LIMITS_LOCKED(p, trigger_exception) \
143 MACRO_BEGIN \
144 (p)->p_memstat_memlimit = (p)->p_memstat_memlimit_active; \
145 if ((p)->p_memstat_state & P_MEMSTAT_MEMLIMIT_ACTIVE_FATAL) { \
146 (p)->p_memstat_state |= P_MEMSTAT_FATAL_MEMLIMIT; \
147 } else { \
148 (p)->p_memstat_state &= ~P_MEMSTAT_FATAL_MEMLIMIT; \
149 } \
150 if ((p)->p_memstat_state & P_MEMSTAT_MEMLIMIT_ACTIVE_EXC_TRIGGERED) { \
151 trigger_exception = FALSE; \
152 } else { \
153 trigger_exception = TRUE; \
154 } \
155 MACRO_END
156
157 #define CACHE_INACTIVE_LIMITS_LOCKED(p, trigger_exception) \
158 MACRO_BEGIN \
159 (p)->p_memstat_memlimit = (p)->p_memstat_memlimit_inactive; \
160 if ((p)->p_memstat_state & P_MEMSTAT_MEMLIMIT_INACTIVE_FATAL) { \
161 (p)->p_memstat_state |= P_MEMSTAT_FATAL_MEMLIMIT; \
162 } else { \
163 (p)->p_memstat_state &= ~P_MEMSTAT_FATAL_MEMLIMIT; \
164 } \
165 if ((p)->p_memstat_state & P_MEMSTAT_MEMLIMIT_INACTIVE_EXC_TRIGGERED) { \
166 trigger_exception = FALSE; \
167 } else { \
168 trigger_exception = TRUE; \
169 } \
170 MACRO_END
171
172
173 /* General tunables */
174
175 unsigned long delta_percentage = 5;
176 unsigned long critical_threshold_percentage = 5;
177 unsigned long idle_offset_percentage = 5;
178 unsigned long pressure_threshold_percentage = 15;
179 unsigned long freeze_threshold_percentage = 50;
180
181 /* General memorystatus stuff */
182
183 struct klist memorystatus_klist;
184 static lck_mtx_t memorystatus_klist_mutex;
185
186 static void memorystatus_klist_lock(void);
187 static void memorystatus_klist_unlock(void);
188
189 static uint64_t memorystatus_idle_delay_time = 0;
190
191 /*
192 * Memorystatus kevents
193 */
194
195 static int filt_memorystatusattach(struct knote *kn);
196 static void filt_memorystatusdetach(struct knote *kn);
197 static int filt_memorystatus(struct knote *kn, long hint);
198
199 struct filterops memorystatus_filtops = {
200 .f_attach = filt_memorystatusattach,
201 .f_detach = filt_memorystatusdetach,
202 .f_event = filt_memorystatus,
203 };
204
205 enum {
206 kMemorystatusNoPressure = 0x1,
207 kMemorystatusPressure = 0x2,
208 kMemorystatusLowSwap = 0x4
209 };
210
211 /* Idle guard handling */
212
213 static int32_t memorystatus_scheduled_idle_demotions = 0;
214
215 static thread_call_t memorystatus_idle_demotion_call;
216
217 static void memorystatus_perform_idle_demotion(__unused void *spare1, __unused void *spare2);
218 static void memorystatus_schedule_idle_demotion_locked(proc_t p, boolean_t set_state);
219 static void memorystatus_invalidate_idle_demotion_locked(proc_t p, boolean_t clean_state);
220 static void memorystatus_reschedule_idle_demotion_locked(void);
221
222 static void memorystatus_update_priority_locked(proc_t p, int priority, boolean_t head_insert);
223
224 boolean_t is_knote_registered_modify_task_pressure_bits(struct knote*, int, task_t, vm_pressure_level_t, vm_pressure_level_t);
225 void memorystatus_send_low_swap_note(void);
226
227 int memorystatus_wakeup = 0;
228
229 unsigned int memorystatus_level = 0;
230 unsigned int memorystatus_early_boot_level = 0;
231
232 static int memorystatus_list_count = 0;
233
234 #define MEMSTAT_BUCKET_COUNT (JETSAM_PRIORITY_MAX + 1)
235
236 typedef struct memstat_bucket {
237 TAILQ_HEAD(, proc) list;
238 int count;
239 } memstat_bucket_t;
240
241 memstat_bucket_t memstat_bucket[MEMSTAT_BUCKET_COUNT];
242
243 uint64_t memstat_idle_demotion_deadline = 0;
244
245 static unsigned int memorystatus_dirty_count = 0;
246
247 #if CONFIG_JETSAM
248 SYSCTL_INT(_kern, OID_AUTO, max_task_pmem, CTLFLAG_RD|CTLFLAG_LOCKED|CTLFLAG_MASKED, &max_task_footprint_mb, 0, "");
249 #endif // CONFIG_JETSAM
250
251
252 int
253 memorystatus_get_level(__unused struct proc *p, struct memorystatus_get_level_args *args, __unused int *ret)
254 {
255 user_addr_t level = 0;
256
257 level = args->level;
258
259 if (copyout(&memorystatus_level, level, sizeof(memorystatus_level)) != 0) {
260 return EFAULT;
261 }
262
263 return 0;
264 }
265
266 static proc_t memorystatus_get_first_proc_locked(unsigned int *bucket_index, boolean_t search);
267 static proc_t memorystatus_get_next_proc_locked(unsigned int *bucket_index, proc_t p, boolean_t search);
268
269 static void memorystatus_thread(void *param __unused, wait_result_t wr __unused);
270
271 /* Jetsam */
272
273 #if CONFIG_JETSAM
274
275 static int memorystatus_cmd_set_jetsam_memory_limit(pid_t pid, int32_t high_water_mark, __unused int32_t *retval, boolean_t is_fatal_limit);
276
277 static int memorystatus_cmd_set_memlimit_properties(pid_t pid, user_addr_t buffer, size_t buffer_size, __unused int32_t *retval);
278
279 static int memorystatus_set_memlimit_properties(pid_t pid, memorystatus_memlimit_properties_t *entry);
280
281 static int memorystatus_cmd_get_memlimit_properties(pid_t pid, user_addr_t buffer, size_t buffer_size, __unused int32_t *retval);
282
283 static boolean_t proc_jetsam_state_is_active_locked(proc_t);
284
285 int proc_get_memstat_priority(proc_t, boolean_t);
286
287 /* Kill processes exceeding their limit either under memory pressure (1), or as soon as possible (0) */
288 #define LEGACY_HIWATER 1
289
290 static boolean_t memorystatus_idle_snapshot = 0;
291
292 static int memorystatus_highwater_enabled = 1; /* Update the cached memlimit data. This should be removed. */
293
294 unsigned int memorystatus_delta = 0;
295
296 static unsigned int memorystatus_available_pages_critical_base = 0;
297 //static unsigned int memorystatus_last_foreground_pressure_pages = (unsigned int)-1;
298 static unsigned int memorystatus_available_pages_critical_idle_offset = 0;
299
300 /* Jetsam Loop Detection */
301 static boolean_t memorystatus_jld_enabled = TRUE; /* Enables jetsam loop detection on all devices */
302 static uint32_t memorystatus_jld_eval_period_msecs = 0; /* Init pass sets this based on device memory size */
303 static int memorystatus_jld_eval_aggressive_count = 3; /* Raise the priority max after 'n' aggressive loops */
304 static int memorystatus_jld_eval_aggressive_priority_band_max = 15; /* Kill aggressively up through this band */
305
306 /*
307 * A FG app can request that the aggressive jetsam mechanism display some leniency in the FG band. This 'lenient' mode is described as:
308 * --- if aggressive jetsam kills an app in the FG band and gets back >=AGGRESSIVE_JETSAM_LENIENT_MODE_THRESHOLD memory, it will stop the aggressive march further into and up the jetsam bands.
309 *
310 * RESTRICTIONS:
311 * - Such a request is respected/acknowledged only once while that 'requesting' app is in the FG band i.e. if aggressive jetsam was
312 * needed and the 'lenient' mode was deployed then that's it for this special mode while the app is in the FG band.
313 *
314 * - If the app is still in the FG band and aggressive jetsam is needed again, there will be no stop-and-check the next time around.
315 *
316 * - Also, the transition of the 'requesting' app away from the FG band will void this special behavior.
317 */
318
319 #define AGGRESSIVE_JETSAM_LENIENT_MODE_THRESHOLD 25
320 boolean_t memorystatus_aggressive_jetsam_lenient_allowed = FALSE;
321 boolean_t memorystatus_aggressive_jetsam_lenient = FALSE;
322
323 #if DEVELOPMENT || DEBUG
324 /*
325 * Jetsam Loop Detection tunables.
326 */
327
328 SYSCTL_UINT(_kern, OID_AUTO, memorystatus_jld_eval_period_msecs, CTLFLAG_RW|CTLFLAG_LOCKED, &memorystatus_jld_eval_period_msecs, 0, "");
329 SYSCTL_UINT(_kern, OID_AUTO, memorystatus_jld_eval_aggressive_count, CTLFLAG_RW|CTLFLAG_LOCKED, &memorystatus_jld_eval_aggressive_count, 0, "");
330 SYSCTL_UINT(_kern, OID_AUTO, memorystatus_jld_eval_aggressive_priority_band_max, CTLFLAG_RW|CTLFLAG_LOCKED, &memorystatus_jld_eval_aggressive_priority_band_max, 0, "");
331 #endif /* DEVELOPMENT || DEBUG */
332
333 #if DEVELOPMENT || DEBUG
334 static unsigned int memorystatus_jetsam_panic_debug = 0;
335
336 static unsigned int memorystatus_jetsam_policy = kPolicyDefault;
337 static unsigned int memorystatus_jetsam_policy_offset_pages_diagnostic = 0;
338 static unsigned int memorystatus_debug_dump_this_bucket = 0;
339 #endif
340
341 static unsigned int memorystatus_thread_wasted_wakeup = 0;
342
343 static uint32_t kill_under_pressure_cause = 0;
344
345 /*
346 * default jetsam snapshot support
347 */
348 static memorystatus_jetsam_snapshot_t *memorystatus_jetsam_snapshot;
349 #define memorystatus_jetsam_snapshot_list memorystatus_jetsam_snapshot->entries
350 static unsigned int memorystatus_jetsam_snapshot_count = 0;
351 static unsigned int memorystatus_jetsam_snapshot_max = 0;
352 static uint64_t memorystatus_jetsam_snapshot_last_timestamp = 0;
353 static uint64_t memorystatus_jetsam_snapshot_timeout = 0;
354 #define JETSAM_SNAPSHOT_TIMEOUT_SECS 30
355
356 /*
357 * snapshot support for memstats collected at boot.
358 */
359 static memorystatus_jetsam_snapshot_t memorystatus_at_boot_snapshot;
360
361 static void memorystatus_clear_errors(void);
362 static void memorystatus_get_task_page_counts(task_t task, uint32_t *footprint, uint32_t *max_footprint, uint32_t *max_footprint_lifetime, uint32_t *purgeable_pages);
363 static uint32_t memorystatus_build_state(proc_t p);
364 static void memorystatus_update_levels_locked(boolean_t critical_only);
365 //static boolean_t memorystatus_issue_pressure_kevent(boolean_t pressured);
366
367 static boolean_t memorystatus_kill_specific_process(pid_t victim_pid, uint32_t cause);
368 static boolean_t memorystatus_kill_top_process(boolean_t any, boolean_t sort_flag, uint32_t cause, int32_t *priority, uint32_t *errors);
369 static boolean_t memorystatus_kill_top_process_aggressive(boolean_t any, uint32_t cause, int aggr_count, int32_t priority_max, uint32_t *errors);
370 #if LEGACY_HIWATER
371 static boolean_t memorystatus_kill_hiwat_proc(uint32_t *errors);
372 #endif
373
374 static boolean_t memorystatus_kill_process_async(pid_t victim_pid, uint32_t cause);
375 static boolean_t memorystatus_kill_process_sync(pid_t victim_pid, uint32_t cause);
376
377 /* Priority Band Sorting Routines */
378 static int memorystatus_sort_bucket(unsigned int bucket_index, int sort_order);
379 static int memorystatus_sort_by_largest_coalition_locked(unsigned int bucket_index, int coal_sort_order);
380 static void memorystatus_sort_by_largest_process_locked(unsigned int bucket_index);
381 static int memorystatus_move_list_locked(unsigned int bucket_index, pid_t *pid_list, int list_sz);
382
383 /* qsort routines */
384 typedef int (*cmpfunc_t)(const void *a, const void *b);
385 extern void qsort(void *a, size_t n, size_t es, cmpfunc_t cmp);
386 static int memstat_asc_cmp(const void *a, const void *b);
387
388 #endif /* CONFIG_JETSAM */
389
390 /* VM pressure */
391
392 extern unsigned int vm_page_free_count;
393 extern unsigned int vm_page_active_count;
394 extern unsigned int vm_page_inactive_count;
395 extern unsigned int vm_page_throttled_count;
396 extern unsigned int vm_page_purgeable_count;
397 extern unsigned int vm_page_wire_count;
398
399 #if VM_PRESSURE_EVENTS
400
401 #include "vm_pressure.h"
402
403 extern boolean_t memorystatus_warn_process(pid_t pid, boolean_t critical);
404
405 vm_pressure_level_t memorystatus_vm_pressure_level = kVMPressureNormal;
406
407 #if CONFIG_MEMORYSTATUS
408 unsigned int memorystatus_available_pages = (unsigned int)-1;
409 unsigned int memorystatus_available_pages_pressure = 0;
410 unsigned int memorystatus_available_pages_critical = 0;
411 unsigned int memorystatus_frozen_count = 0;
412 unsigned int memorystatus_suspended_count = 0;
413
414 /*
415 * We use this flag to signal if we have any HWM offenders
416 * on the system. This way we can reduce the number of wakeups
417 * of the memorystatus_thread when the system is between the
418 * "pressure" and "critical" threshold.
419 *
420 * The (re-)setting of this variable is done without any locks
421 * or synchronization simply because it is not possible (currently)
422 * to keep track of HWM offenders that drop down below their memory
423 * limit and/or exit. So, we choose to burn a couple of wasted wakeups
424 * by allowing the unguarded modification of this variable.
425 */
426 boolean_t memorystatus_hwm_candidates = 0;
427
428 static int memorystatus_send_note(int event_code, void *data, size_t data_length);
429 #endif /* CONFIG_MEMORYSTATUS */
430
431 #endif /* VM_PRESSURE_EVENTS */
432
433 /* Freeze */
434
435 #if CONFIG_FREEZE
436
437 boolean_t memorystatus_freeze_enabled = FALSE;
438 int memorystatus_freeze_wakeup = 0;
439
440 lck_grp_attr_t *freezer_lck_grp_attr;
441 lck_grp_t *freezer_lck_grp;
442 static lck_mtx_t freezer_mutex;
443
444 static inline boolean_t memorystatus_can_freeze_processes(void);
445 static boolean_t memorystatus_can_freeze(boolean_t *memorystatus_freeze_swap_low);
446
447 static void memorystatus_freeze_thread(void *param __unused, wait_result_t wr __unused);
448
449 /* Thresholds */
450 static unsigned int memorystatus_freeze_threshold = 0;
451
452 static unsigned int memorystatus_freeze_pages_min = 0;
453 static unsigned int memorystatus_freeze_pages_max = 0;
454
455 static unsigned int memorystatus_freeze_suspended_threshold = FREEZE_SUSPENDED_THRESHOLD_DEFAULT;
456
457 static unsigned int memorystatus_freeze_daily_mb_max = FREEZE_DAILY_MB_MAX_DEFAULT;
458
459 /* Stats */
460 static uint64_t memorystatus_freeze_count = 0;
461 static uint64_t memorystatus_freeze_pageouts = 0;
462
463 /* Throttling */
464 static throttle_interval_t throttle_intervals[] = {
465 { 60, 8, 0, 0, { 0, 0 }, FALSE }, /* 1 hour intermediate interval, 8x burst */
466 { 24 * 60, 1, 0, 0, { 0, 0 }, FALSE }, /* 24 hour long interval, no burst */
467 };
468
469 static uint64_t memorystatus_freeze_throttle_count = 0;
470
471 static unsigned int memorystatus_suspended_footprint_total = 0;
472
473 extern uint64_t vm_swap_get_free_space(void);
474
475 static boolean_t memorystatus_freeze_update_throttle();
476
477 #endif /* CONFIG_FREEZE */
478
479 /* Debug */
480
481 extern struct knote *vm_find_knote_from_pid(pid_t, struct klist *);
482
483 #if DEVELOPMENT || DEBUG
484
485 #if CONFIG_JETSAM
486
487 static void
488 memorystatus_debug_dump_bucket_locked (unsigned int bucket_index)
489 {
490 proc_t p = NULL;
491 uint32_t pages = 0;
492 uint32_t pages_in_mb = 0;
493 unsigned int b = bucket_index;
494 boolean_t traverse_all_buckets = FALSE;
495
496 if (bucket_index >= MEMSTAT_BUCKET_COUNT) {
497 traverse_all_buckets = TRUE;
498 b = 0;
499 } else {
500 traverse_all_buckets = FALSE;
501 b = bucket_index;
502 }
503
504 /*
505 * Missing from this dump is the value actually
506 * stored in the ledger... also, format could be better.
507 */
508 printf("memorystatus_debug_dump ***START***\n");
509 printf("bucket [pid] [pages/pages-mb] state [EP / RP] dirty deadline [C-limit / A-limit / IA-limit] name\n");
510 p = memorystatus_get_first_proc_locked(&b, traverse_all_buckets);
511 while (p) {
512 memorystatus_get_task_page_counts(p->task, &pages, NULL, NULL, NULL);
513 pages_in_mb = (pages * 4096) /1024 / 1024;
514 printf("%d [%d] [%d/%dMB] 0x%x [%d / %d] 0x%x %lld [%d%s / %d%s / %d%s] %s\n",
515 b, p->p_pid, pages, pages_in_mb,
516 p->p_memstat_state, p->p_memstat_effectivepriority, p->p_memstat_requestedpriority, p->p_memstat_dirty, p->p_memstat_idledeadline,
517 p->p_memstat_memlimit,
518 (p->p_memstat_state & P_MEMSTAT_FATAL_MEMLIMIT ? "F " : "NF"),
519 p->p_memstat_memlimit_active,
520 (p->p_memstat_state & P_MEMSTAT_MEMLIMIT_ACTIVE_FATAL ? "F " : "NF"),
521 p->p_memstat_memlimit_inactive,
522 (p->p_memstat_state & P_MEMSTAT_MEMLIMIT_INACTIVE_FATAL ? "F " : "NF"),
523 (p->p_comm ? p->p_comm : "unknown"));
524 p = memorystatus_get_next_proc_locked(&b, p, traverse_all_buckets);
525 }
526 printf("memorystatus_debug_dump ***END***\n");
527 }
528
529 static int
530 sysctl_memorystatus_debug_dump_bucket SYSCTL_HANDLER_ARGS
531 {
532 #pragma unused(oidp, arg2)
533 int bucket_index = 0;
534 int error;
535 error = SYSCTL_OUT(req, arg1, sizeof(int));
536 if (error || !req->newptr) {
537 return (error);
538 }
539 error = SYSCTL_IN(req, &bucket_index, sizeof(int));
540 if (error || !req->newptr) {
541 return (error);
542 }
543 if (bucket_index >= MEMSTAT_BUCKET_COUNT) {
544 /*
545 * All jetsam buckets will be dumped.
546 */
547 } else {
548 /*
549 * Only a single bucket will be dumped.
550 */
551 }
552
553 proc_list_lock();
554 memorystatus_debug_dump_bucket_locked(bucket_index);
555 proc_list_unlock();
556 memorystatus_debug_dump_this_bucket = bucket_index;
557 return (error);
558 }
559
560 /*
561 * Debug aid to look at jetsam buckets and proc jetsam fields.
562 * Use this sysctl to act on a particular jetsam bucket.
563 * Writing the sysctl triggers the dump.
564 * Usage: sysctl kern.memorystatus_debug_dump_this_bucket=<bucket_index>
565 */
566
567 SYSCTL_PROC(_kern, OID_AUTO, memorystatus_debug_dump_this_bucket, CTLTYPE_INT|CTLFLAG_RW|CTLFLAG_LOCKED, &memorystatus_debug_dump_this_bucket, 0, sysctl_memorystatus_debug_dump_bucket, "I", "");
568
569
570 /* Debug aid to aid determination of limit */
571
572 static int
573 sysctl_memorystatus_highwater_enable SYSCTL_HANDLER_ARGS
574 {
575 #pragma unused(oidp, arg2)
576 proc_t p;
577 unsigned int b = 0;
578 int error, enable = 0;
579
580 error = SYSCTL_OUT(req, arg1, sizeof(int));
581 if (error || !req->newptr) {
582 return (error);
583 }
584
585 error = SYSCTL_IN(req, &enable, sizeof(int));
586 if (error || !req->newptr) {
587 return (error);
588 }
589
590 if (!(enable == 0 || enable == 1)) {
591 return EINVAL;
592 }
593
594 proc_list_lock();
595
596 p = memorystatus_get_first_proc_locked(&b, TRUE);
597 while (p) {
598 boolean_t trigger_exception;
599
600 if (enable) {
601 /*
602 * No need to consider P_MEMSTAT_MEMLIMIT_BACKGROUND anymore.
603 * Background limits are described via the inactive limit slots.
604 */
605
606 if (proc_jetsam_state_is_active_locked(p) == TRUE) {
607 CACHE_ACTIVE_LIMITS_LOCKED(p, trigger_exception);
608 } else {
609 CACHE_INACTIVE_LIMITS_LOCKED(p, trigger_exception);
610 }
611
612 } else {
613 /*
614 * Disabling limits does not touch the stored variants.
615 * Set the cached limit fields to system_wide defaults.
616 */
617 p->p_memstat_memlimit = -1;
618 p->p_memstat_state |= P_MEMSTAT_FATAL_MEMLIMIT;
619 trigger_exception = TRUE;
620 }
621
622 /*
623 * Enforce the cached limit by writing to the ledger.
624 */
625 task_set_phys_footprint_limit_internal(p->task, (p->p_memstat_memlimit > 0) ? p->p_memstat_memlimit: -1, NULL, trigger_exception);
626
627 p = memorystatus_get_next_proc_locked(&b, p, TRUE);
628 }
629
630 memorystatus_highwater_enabled = enable;
631
632 proc_list_unlock();
633
634 return 0;
635
636 }
637
638 SYSCTL_INT(_kern, OID_AUTO, memorystatus_idle_snapshot, CTLFLAG_RW|CTLFLAG_LOCKED, &memorystatus_idle_snapshot, 0, "");
639
640 SYSCTL_PROC(_kern, OID_AUTO, memorystatus_highwater_enabled, CTLTYPE_INT|CTLFLAG_RW|CTLFLAG_LOCKED, &memorystatus_highwater_enabled, 0, sysctl_memorystatus_highwater_enable, "I", "");
641
642 SYSCTL_UINT(_kern, OID_AUTO, memorystatus_available_pages, CTLFLAG_RD|CTLFLAG_LOCKED, &memorystatus_available_pages, 0, "");
643 SYSCTL_UINT(_kern, OID_AUTO, memorystatus_available_pages_critical, CTLFLAG_RD|CTLFLAG_LOCKED, &memorystatus_available_pages_critical, 0, "");
644 SYSCTL_UINT(_kern, OID_AUTO, memorystatus_available_pages_critical_base, CTLFLAG_RW|CTLFLAG_LOCKED, &memorystatus_available_pages_critical_base, 0, "");
645 SYSCTL_UINT(_kern, OID_AUTO, memorystatus_available_pages_critical_idle_offset, CTLFLAG_RW|CTLFLAG_LOCKED, &memorystatus_available_pages_critical_idle_offset, 0, "");
646
647 /* Diagnostic code */
648
649 enum {
650 kJetsamDiagnosticModeNone = 0,
651 kJetsamDiagnosticModeAll = 1,
652 kJetsamDiagnosticModeStopAtFirstActive = 2,
653 kJetsamDiagnosticModeCount
654 } jetsam_diagnostic_mode = kJetsamDiagnosticModeNone;
655
656 static int jetsam_diagnostic_suspended_one_active_proc = 0;
657
658 static int
659 sysctl_jetsam_diagnostic_mode SYSCTL_HANDLER_ARGS
660 {
661 #pragma unused(arg1, arg2)
662
663 const char *diagnosticStrings[] = {
664 "jetsam: diagnostic mode: resetting critical level.",
665 "jetsam: diagnostic mode: will examine all processes",
666 "jetsam: diagnostic mode: will stop at first active process"
667 };
668
669 int error, val = jetsam_diagnostic_mode;
670 boolean_t changed = FALSE;
671
672 error = sysctl_handle_int(oidp, &val, 0, req);
673 if (error || !req->newptr)
674 return (error);
675 if ((val < 0) || (val >= kJetsamDiagnosticModeCount)) {
676 printf("jetsam: diagnostic mode: invalid value - %d\n", val);
677 return EINVAL;
678 }
679
680 proc_list_lock();
681
682 if ((unsigned int) val != jetsam_diagnostic_mode) {
683 jetsam_diagnostic_mode = val;
684
685 memorystatus_jetsam_policy &= ~kPolicyDiagnoseActive;
686
687 switch (jetsam_diagnostic_mode) {
688 case kJetsamDiagnosticModeNone:
689 /* Already cleared */
690 break;
691 case kJetsamDiagnosticModeAll:
692 memorystatus_jetsam_policy |= kPolicyDiagnoseAll;
693 break;
694 case kJetsamDiagnosticModeStopAtFirstActive:
695 memorystatus_jetsam_policy |= kPolicyDiagnoseFirst;
696 break;
697 default:
698 /* Already validated */
699 break;
700 }
701
702 memorystatus_update_levels_locked(FALSE);
703 changed = TRUE;
704 }
705
706 proc_list_unlock();
707
708 if (changed) {
709 printf("%s\n", diagnosticStrings[val]);
710 }
711
712 return (0);
713 }
714
715 SYSCTL_PROC(_debug, OID_AUTO, jetsam_diagnostic_mode, CTLTYPE_INT|CTLFLAG_RW|CTLFLAG_LOCKED|CTLFLAG_ANYBODY,
716 &jetsam_diagnostic_mode, 0, sysctl_jetsam_diagnostic_mode, "I", "Jetsam Diagnostic Mode");
717
718 SYSCTL_UINT(_kern, OID_AUTO, memorystatus_jetsam_policy_offset_pages_diagnostic, CTLFLAG_RW|CTLFLAG_LOCKED, &memorystatus_jetsam_policy_offset_pages_diagnostic, 0, "");
719
720 #if VM_PRESSURE_EVENTS
721
722 SYSCTL_UINT(_kern, OID_AUTO, memorystatus_available_pages_pressure, CTLFLAG_RW|CTLFLAG_LOCKED, &memorystatus_available_pages_pressure, 0, "");
723
724
725 /*
726 * This routine is used for targeted notifications
727 * regardless of system memory pressure.
728 * "memnote" is the current user.
729 */
730
731 static int
732 sysctl_memorystatus_vm_pressure_send SYSCTL_HANDLER_ARGS
733 {
734 #pragma unused(arg1, arg2)
735
736 int error = 0, pid = 0;
737 int ret = 0;
738 struct knote *kn = NULL;
739 boolean_t found_knote = FALSE;
740
741 error = sysctl_handle_int(oidp, &pid, 0, req);
742 if (error || !req->newptr)
743 return (error);
744
745 /*
746 * We inspect 3 lists here for targeted notifications:
747 * - memorystatus_klist
748 * - vm_pressure_klist
749 * - vm_pressure_dormant_klist
750 *
751 * The vm_pressure_* lists are tied to the old VM_PRESSURE
752 * notification mechanism. We intend to stop using that
753 * mechanism and, in turn, get rid of the 2 lists and
754 * vm_dispatch_pressure_note_to_pid() too.
755 */
756
757 memorystatus_klist_lock();
758
759 SLIST_FOREACH(kn, &memorystatus_klist, kn_selnext) {
760 proc_t knote_proc = kn->kn_kq->kq_p;
761 pid_t knote_pid = knote_proc->p_pid;
762
763 if (knote_pid == pid) {
764 /*
765 * Forcibly send this pid a "warning" memory pressure notification.
766 */
767 kn->kn_fflags = NOTE_MEMORYSTATUS_PRESSURE_WARN;
768 found_knote = TRUE;
769 }
770 }
771
772 if (found_knote) {
773 KNOTE(&memorystatus_klist, 0);
774 ret = 0;
775 } else {
776 ret = vm_dispatch_pressure_note_to_pid(pid, FALSE);
777 }
778
779 memorystatus_klist_unlock();
780
781 return ret;
782 }
783
784 SYSCTL_PROC(_kern, OID_AUTO, memorystatus_vm_pressure_send, CTLTYPE_INT|CTLFLAG_WR|CTLFLAG_LOCKED|CTLFLAG_MASKED,
785 0, 0, &sysctl_memorystatus_vm_pressure_send, "I", "");
786
787 #endif /* VM_PRESSURE_EVENTS */
788
789 #endif /* CONFIG_JETSAM */
790
791 #if CONFIG_FREEZE
792
793 SYSCTL_UINT(_kern, OID_AUTO, memorystatus_freeze_daily_mb_max, CTLFLAG_RW|CTLFLAG_LOCKED, &memorystatus_freeze_daily_mb_max, 0, "");
794
795 SYSCTL_UINT(_kern, OID_AUTO, memorystatus_freeze_threshold, CTLFLAG_RW|CTLFLAG_LOCKED, &memorystatus_freeze_threshold, 0, "");
796
797 SYSCTL_UINT(_kern, OID_AUTO, memorystatus_freeze_pages_min, CTLFLAG_RW|CTLFLAG_LOCKED, &memorystatus_freeze_pages_min, 0, "");
798 SYSCTL_UINT(_kern, OID_AUTO, memorystatus_freeze_pages_max, CTLFLAG_RW|CTLFLAG_LOCKED, &memorystatus_freeze_pages_max, 0, "");
799
800 SYSCTL_QUAD(_kern, OID_AUTO, memorystatus_freeze_count, CTLFLAG_RD|CTLFLAG_LOCKED, &memorystatus_freeze_count, "");
801 SYSCTL_QUAD(_kern, OID_AUTO, memorystatus_freeze_pageouts, CTLFLAG_RD|CTLFLAG_LOCKED, &memorystatus_freeze_pageouts, "");
802 SYSCTL_QUAD(_kern, OID_AUTO, memorystatus_freeze_throttle_count, CTLFLAG_RD|CTLFLAG_LOCKED, &memorystatus_freeze_throttle_count, "");
803 SYSCTL_UINT(_kern, OID_AUTO, memorystatus_freeze_min_processes, CTLFLAG_RW|CTLFLAG_LOCKED, &memorystatus_freeze_suspended_threshold, 0, "");
804
805 boolean_t memorystatus_freeze_throttle_enabled = TRUE;
806 SYSCTL_UINT(_kern, OID_AUTO, memorystatus_freeze_throttle_enabled, CTLFLAG_RW|CTLFLAG_LOCKED, &memorystatus_freeze_throttle_enabled, 0, "");
807
808 /*
809 * Manual trigger of freeze and thaw for dev / debug kernels only.
810 */
811 static int
812 sysctl_memorystatus_freeze SYSCTL_HANDLER_ARGS
813 {
814 #pragma unused(arg1, arg2)
815 int error, pid = 0;
816 proc_t p;
817
818 if (memorystatus_freeze_enabled == FALSE) {
819 return ENOTSUP;
820 }
821
822 error = sysctl_handle_int(oidp, &pid, 0, req);
823 if (error || !req->newptr)
824 return (error);
825
826 if (pid == 2) {
827 vm_pageout_anonymous_pages();
828
829 return 0;
830 }
831
832 lck_mtx_lock(&freezer_mutex);
833
834 p = proc_find(pid);
835 if (p != NULL) {
836 uint32_t purgeable, wired, clean, dirty;
837 boolean_t shared;
838 uint32_t max_pages = 0;
839
840 if (DEFAULT_FREEZER_IS_ACTIVE || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_SWAPBACKED) {
841
842 unsigned int avail_swap_space = 0; /* in pages. */
843
844 if (DEFAULT_FREEZER_IS_ACTIVE) {
845 /*
846 * Freezer backed by default pager and swap file(s).
847 */
848 avail_swap_space = default_pager_swap_pages_free();
849 } else {
850 /*
851 * Freezer backed by the compressor and swap file(s)
852 * while will hold compressed data.
853 */
854 avail_swap_space = vm_swap_get_free_space() / PAGE_SIZE_64;
855 }
856
857 max_pages = MIN(avail_swap_space, memorystatus_freeze_pages_max);
858
859 } else {
860 /*
861 * We only have the compressor without any swap.
862 */
863 max_pages = UINT32_MAX - 1;
864 }
865
866 error = task_freeze(p->task, &purgeable, &wired, &clean, &dirty, max_pages, &shared, FALSE);
867 proc_rele(p);
868
869 if (error)
870 error = EIO;
871
872 lck_mtx_unlock(&freezer_mutex);
873 return error;
874 }
875
876 lck_mtx_unlock(&freezer_mutex);
877 return EINVAL;
878 }
879
880 SYSCTL_PROC(_kern, OID_AUTO, memorystatus_freeze, CTLTYPE_INT|CTLFLAG_WR|CTLFLAG_LOCKED|CTLFLAG_MASKED,
881 0, 0, &sysctl_memorystatus_freeze, "I", "");
882
883 static int
884 sysctl_memorystatus_available_pages_thaw SYSCTL_HANDLER_ARGS
885 {
886 #pragma unused(arg1, arg2)
887
888 int error, pid = 0;
889 proc_t p;
890
891 if (memorystatus_freeze_enabled == FALSE) {
892 return ENOTSUP;
893 }
894
895 error = sysctl_handle_int(oidp, &pid, 0, req);
896 if (error || !req->newptr)
897 return (error);
898
899 p = proc_find(pid);
900 if (p != NULL) {
901 error = task_thaw(p->task);
902 proc_rele(p);
903
904 if (error)
905 error = EIO;
906 return error;
907 }
908
909 return EINVAL;
910 }
911
912 SYSCTL_PROC(_kern, OID_AUTO, memorystatus_thaw, CTLTYPE_INT|CTLFLAG_WR|CTLFLAG_LOCKED|CTLFLAG_MASKED,
913 0, 0, &sysctl_memorystatus_available_pages_thaw, "I", "");
914
915 #endif /* CONFIG_FREEZE */
916
917 #endif /* DEVELOPMENT || DEBUG */
918
919 extern kern_return_t kernel_thread_start_priority(thread_continue_t continuation,
920 void *parameter,
921 integer_t priority,
922 thread_t *new_thread);
923
924 #if CONFIG_JETSAM
925 /*
926 * Picks the sorting routine for a given jetsam priority band.
927 *
928 * Input:
929 * bucket_index - jetsam priority band to be sorted.
930 * sort_order - JETSAM_SORT_xxx from kern_memorystatus.h
931 * Currently sort_order is only meaningful when handling
932 * coalitions.
933 *
934 * Return:
935 * 0 on success
936 * non-0 on failure
937 */
938 static int memorystatus_sort_bucket(unsigned int bucket_index, int sort_order)
939 {
940 int coal_sort_order;
941
942 /*
943 * Verify the jetsam priority
944 */
945 if (bucket_index >= MEMSTAT_BUCKET_COUNT) {
946 return(EINVAL);
947 }
948
949 #if DEVELOPMENT || DEBUG
950 if (sort_order == JETSAM_SORT_DEFAULT) {
951 coal_sort_order = COALITION_SORT_DEFAULT;
952 } else {
953 coal_sort_order = sort_order; /* only used for testing scenarios */
954 }
955 #else
956 /* Verify default */
957 if (sort_order == JETSAM_SORT_DEFAULT) {
958 coal_sort_order = COALITION_SORT_DEFAULT;
959 } else {
960 return(EINVAL);
961 }
962 #endif
963
964 proc_list_lock();
965 switch (bucket_index) {
966 case JETSAM_PRIORITY_FOREGROUND:
967 if (memorystatus_sort_by_largest_coalition_locked(bucket_index, coal_sort_order) == 0) {
968 /*
969 * Fall back to per process sorting when zero coalitions are found.
970 */
971 memorystatus_sort_by_largest_process_locked(bucket_index);
972 }
973 break;
974 default:
975 memorystatus_sort_by_largest_process_locked(bucket_index);
976 break;
977 }
978 proc_list_unlock();
979
980 return(0);
981 }
982
983 /*
984 * Sort processes by size for a single jetsam bucket.
985 */
986
987 static void memorystatus_sort_by_largest_process_locked(unsigned int bucket_index)
988 {
989 proc_t p = NULL, insert_after_proc = NULL, max_proc = NULL;
990 proc_t next_p = NULL, prev_max_proc = NULL;
991 uint32_t pages = 0, max_pages = 0;
992 memstat_bucket_t *current_bucket;
993
994 if (bucket_index >= MEMSTAT_BUCKET_COUNT) {
995 return;
996 }
997
998 current_bucket = &memstat_bucket[bucket_index];
999
1000 p = TAILQ_FIRST(&current_bucket->list);
1001
1002 while (p) {
1003 memorystatus_get_task_page_counts(p->task, &pages, NULL, NULL, NULL);
1004 max_pages = pages;
1005 max_proc = p;
1006 prev_max_proc = p;
1007
1008 while ((next_p = TAILQ_NEXT(p, p_memstat_list)) != NULL) {
1009 /* traversing list until we find next largest process */
1010 p=next_p;
1011 memorystatus_get_task_page_counts(p->task, &pages, NULL, NULL, NULL);
1012 if (pages > max_pages) {
1013 max_pages = pages;
1014 max_proc = p;
1015 }
1016 }
1017
1018 if (prev_max_proc != max_proc) {
1019 /* found a larger process, place it in the list */
1020 TAILQ_REMOVE(&current_bucket->list, max_proc, p_memstat_list);
1021 if (insert_after_proc == NULL) {
1022 TAILQ_INSERT_HEAD(&current_bucket->list, max_proc, p_memstat_list);
1023 } else {
1024 TAILQ_INSERT_AFTER(&current_bucket->list, insert_after_proc, max_proc, p_memstat_list);
1025 }
1026 prev_max_proc = max_proc;
1027 }
1028
1029 insert_after_proc = max_proc;
1030
1031 p = TAILQ_NEXT(max_proc, p_memstat_list);
1032 }
1033 }
1034
1035 #endif /* CONFIG_JETSAM */
1036
1037 static proc_t memorystatus_get_first_proc_locked(unsigned int *bucket_index, boolean_t search) {
1038 memstat_bucket_t *current_bucket;
1039 proc_t next_p;
1040
1041 if ((*bucket_index) >= MEMSTAT_BUCKET_COUNT) {
1042 return NULL;
1043 }
1044
1045 current_bucket = &memstat_bucket[*bucket_index];
1046 next_p = TAILQ_FIRST(&current_bucket->list);
1047 if (!next_p && search) {
1048 while (!next_p && (++(*bucket_index) < MEMSTAT_BUCKET_COUNT)) {
1049 current_bucket = &memstat_bucket[*bucket_index];
1050 next_p = TAILQ_FIRST(&current_bucket->list);
1051 }
1052 }
1053
1054 return next_p;
1055 }
1056
1057 static proc_t memorystatus_get_next_proc_locked(unsigned int *bucket_index, proc_t p, boolean_t search) {
1058 memstat_bucket_t *current_bucket;
1059 proc_t next_p;
1060
1061 if (!p || ((*bucket_index) >= MEMSTAT_BUCKET_COUNT)) {
1062 return NULL;
1063 }
1064
1065 next_p = TAILQ_NEXT(p, p_memstat_list);
1066 while (!next_p && search && (++(*bucket_index) < MEMSTAT_BUCKET_COUNT)) {
1067 current_bucket = &memstat_bucket[*bucket_index];
1068 next_p = TAILQ_FIRST(&current_bucket->list);
1069 }
1070
1071 return next_p;
1072 }
1073
1074 __private_extern__ void
1075 memorystatus_init(void)
1076 {
1077 thread_t thread = THREAD_NULL;
1078 kern_return_t result;
1079 int i;
1080
1081 #if CONFIG_FREEZE
1082 memorystatus_freeze_pages_min = FREEZE_PAGES_MIN;
1083 memorystatus_freeze_pages_max = FREEZE_PAGES_MAX;
1084 #endif
1085
1086 nanoseconds_to_absolutetime((uint64_t)DEFERRED_IDLE_EXIT_TIME_SECS * NSEC_PER_SEC, &memorystatus_idle_delay_time);
1087
1088 /* Init buckets */
1089 for (i = 0; i < MEMSTAT_BUCKET_COUNT; i++) {
1090 TAILQ_INIT(&memstat_bucket[i].list);
1091 memstat_bucket[i].count = 0;
1092 }
1093
1094 memorystatus_idle_demotion_call = thread_call_allocate((thread_call_func_t)memorystatus_perform_idle_demotion, NULL);
1095
1096 /* Apply overrides */
1097 PE_get_default("kern.jetsam_delta", &delta_percentage, sizeof(delta_percentage));
1098 assert(delta_percentage < 100);
1099 PE_get_default("kern.jetsam_critical_threshold", &critical_threshold_percentage, sizeof(critical_threshold_percentage));
1100 assert(critical_threshold_percentage < 100);
1101 PE_get_default("kern.jetsam_idle_offset", &idle_offset_percentage, sizeof(idle_offset_percentage));
1102 assert(idle_offset_percentage < 100);
1103 PE_get_default("kern.jetsam_pressure_threshold", &pressure_threshold_percentage, sizeof(pressure_threshold_percentage));
1104 assert(pressure_threshold_percentage < 100);
1105 PE_get_default("kern.jetsam_freeze_threshold", &freeze_threshold_percentage, sizeof(freeze_threshold_percentage));
1106 assert(freeze_threshold_percentage < 100);
1107
1108 #if CONFIG_JETSAM
1109 /* device tree can request to take snapshots for idle-exit kills by default */
1110 PE_get_default("kern.jetsam_idle_snapshot", &memorystatus_idle_snapshot, sizeof(memorystatus_idle_snapshot));
1111
1112 memorystatus_delta = delta_percentage * atop_64(max_mem) / 100;
1113 memorystatus_available_pages_critical_idle_offset = idle_offset_percentage * atop_64(max_mem) / 100;
1114 memorystatus_available_pages_critical_base = (critical_threshold_percentage / delta_percentage) * memorystatus_delta;
1115
1116 memorystatus_jetsam_snapshot_max = maxproc;
1117 memorystatus_jetsam_snapshot =
1118 (memorystatus_jetsam_snapshot_t*)kalloc(sizeof(memorystatus_jetsam_snapshot_t) +
1119 sizeof(memorystatus_jetsam_snapshot_entry_t) * memorystatus_jetsam_snapshot_max);
1120 if (!memorystatus_jetsam_snapshot) {
1121 panic("Could not allocate memorystatus_jetsam_snapshot");
1122 }
1123
1124 nanoseconds_to_absolutetime((uint64_t)JETSAM_SNAPSHOT_TIMEOUT_SECS * NSEC_PER_SEC, &memorystatus_jetsam_snapshot_timeout);
1125
1126 memset(&memorystatus_at_boot_snapshot, 0, sizeof(memorystatus_jetsam_snapshot_t));
1127
1128 /* No contention at this point */
1129 memorystatus_update_levels_locked(FALSE);
1130
1131 /* Jetsam Loop Detection */
1132 if (max_mem <= (512 * 1024 * 1024)) {
1133 /* 512 MB devices */
1134 memorystatus_jld_eval_period_msecs = 8000; /* 8000 msecs == 8 second window */
1135 } else {
1136 /* 1GB and larger devices */
1137 memorystatus_jld_eval_period_msecs = 6000; /* 6000 msecs == 6 second window */
1138 }
1139 #endif
1140
1141 #if CONFIG_FREEZE
1142 memorystatus_freeze_threshold = (freeze_threshold_percentage / delta_percentage) * memorystatus_delta;
1143 #endif
1144
1145 result = kernel_thread_start_priority(memorystatus_thread, NULL, 95 /* MAXPRI_KERNEL */, &thread);
1146 if (result == KERN_SUCCESS) {
1147 thread_deallocate(thread);
1148 } else {
1149 panic("Could not create memorystatus_thread");
1150 }
1151 }
1152
1153 /* Centralised for the purposes of allowing panic-on-jetsam */
1154 extern void
1155 vm_wake_compactor_swapper(void);
1156
1157 /*
1158 * The jetsam no frills kill call
1159 * Return: 0 on success
1160 * error code on failure (EINVAL...)
1161 */
1162 static int
1163 jetsam_do_kill(proc_t p, int jetsam_flags) {
1164 int error = 0;
1165 error = exit1_internal(p, W_EXITCODE(0, SIGKILL), (int *)NULL, FALSE, FALSE, jetsam_flags);
1166 return(error);
1167 }
1168
1169 /*
1170 * Wrapper for processes exiting with memorystatus details
1171 */
1172 static boolean_t
1173 memorystatus_do_kill(proc_t p, uint32_t cause) {
1174
1175 int error = 0;
1176 __unused pid_t victim_pid = p->p_pid;
1177
1178 KERNEL_DEBUG_CONSTANT( (BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_DO_KILL)) | DBG_FUNC_START,
1179 victim_pid, cause, vm_page_free_count, 0, 0);
1180
1181 #if CONFIG_JETSAM && (DEVELOPMENT || DEBUG)
1182 if (memorystatus_jetsam_panic_debug & (1 << cause)) {
1183 panic("memorystatus_do_kill(): jetsam debug panic (cause: %d)", cause);
1184 }
1185 #else
1186 #pragma unused(cause)
1187 #endif
1188 int jetsam_flags = P_LTERM_JETSAM;
1189 switch (cause) {
1190 case kMemorystatusKilledHiwat: jetsam_flags |= P_JETSAM_HIWAT; break;
1191 case kMemorystatusKilledVnodes: jetsam_flags |= P_JETSAM_VNODE; break;
1192 case kMemorystatusKilledVMPageShortage: jetsam_flags |= P_JETSAM_VMPAGESHORTAGE; break;
1193 case kMemorystatusKilledVMThrashing: jetsam_flags |= P_JETSAM_VMTHRASHING; break;
1194 case kMemorystatusKilledFCThrashing: jetsam_flags |= P_JETSAM_FCTHRASHING; break;
1195 case kMemorystatusKilledPerProcessLimit: jetsam_flags |= P_JETSAM_PID; break;
1196 case kMemorystatusKilledIdleExit: jetsam_flags |= P_JETSAM_IDLEEXIT; break;
1197 }
1198 error = jetsam_do_kill(p, jetsam_flags);
1199
1200 KERNEL_DEBUG_CONSTANT( (BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_DO_KILL)) | DBG_FUNC_END,
1201 victim_pid, cause, vm_page_free_count, error, 0);
1202
1203 vm_wake_compactor_swapper();
1204
1205 return (error == 0);
1206 }
1207
1208 /*
1209 * Node manipulation
1210 */
1211
1212 static void
1213 memorystatus_check_levels_locked(void) {
1214 #if CONFIG_JETSAM
1215 /* Update levels */
1216 memorystatus_update_levels_locked(TRUE);
1217 #endif
1218 }
1219
1220 static void
1221 memorystatus_perform_idle_demotion(__unused void *spare1, __unused void *spare2)
1222 {
1223 proc_t p;
1224 uint64_t current_time;
1225 memstat_bucket_t *demotion_bucket;
1226
1227 MEMORYSTATUS_DEBUG(1, "memorystatus_perform_idle_demotion()\n");
1228
1229 KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_IDLE_DEMOTE) | DBG_FUNC_START, 0, 0, 0, 0, 0);
1230
1231 current_time = mach_absolute_time();
1232
1233 proc_list_lock();
1234
1235 demotion_bucket = &memstat_bucket[JETSAM_PRIORITY_IDLE_DEFERRED];
1236 p = TAILQ_FIRST(&demotion_bucket->list);
1237
1238 while (p) {
1239 MEMORYSTATUS_DEBUG(1, "memorystatus_perform_idle_demotion() found %d\n", p->p_pid);
1240
1241 assert(p->p_memstat_idledeadline);
1242 assert(p->p_memstat_dirty & P_DIRTY_DEFER_IN_PROGRESS);
1243 assert((p->p_memstat_dirty & (P_DIRTY_IDLE_EXIT_ENABLED|P_DIRTY_IS_DIRTY)) == P_DIRTY_IDLE_EXIT_ENABLED);
1244
1245 if (current_time >= p->p_memstat_idledeadline) {
1246 #if DEBUG || DEVELOPMENT
1247 if (!(p->p_memstat_dirty & P_DIRTY_MARKED)) {
1248 printf("memorystatus_perform_idle_demotion: moving process %d [%s] to idle band, but never dirtied (0x%x)!\n",
1249 p->p_pid, (p->p_comm ? p->p_comm : "(unknown)"), p->p_memstat_dirty);
1250 }
1251 #endif
1252 memorystatus_invalidate_idle_demotion_locked(p, TRUE);
1253 memorystatus_update_priority_locked(p, JETSAM_PRIORITY_IDLE, false);
1254
1255 // The prior process has moved out of the demotion bucket, so grab the new head and continue
1256 p = TAILQ_FIRST(&demotion_bucket->list);
1257 continue;
1258 }
1259
1260 // No further candidates
1261 break;
1262 }
1263
1264 memorystatus_reschedule_idle_demotion_locked();
1265
1266 proc_list_unlock();
1267
1268 KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_IDLE_DEMOTE) | DBG_FUNC_END, 0, 0, 0, 0, 0);
1269 }
1270
1271 static void
1272 memorystatus_schedule_idle_demotion_locked(proc_t p, boolean_t set_state)
1273 {
1274 boolean_t present_in_deferred_bucket = FALSE;
1275
1276 if (p->p_memstat_effectivepriority == JETSAM_PRIORITY_IDLE_DEFERRED) {
1277 present_in_deferred_bucket = TRUE;
1278 }
1279
1280 MEMORYSTATUS_DEBUG(1, "memorystatus_schedule_idle_demotion_locked: scheduling demotion to idle band for pid %d (dirty:0x%x, set_state %d, demotions %d).\n",
1281 p->p_pid, p->p_memstat_dirty, set_state, memorystatus_scheduled_idle_demotions);
1282
1283 assert((p->p_memstat_dirty & P_DIRTY_IDLE_EXIT_ENABLED) == P_DIRTY_IDLE_EXIT_ENABLED);
1284
1285 if (set_state) {
1286 assert(p->p_memstat_idledeadline == 0);
1287 p->p_memstat_dirty |= P_DIRTY_DEFER_IN_PROGRESS;
1288 p->p_memstat_idledeadline = mach_absolute_time() + memorystatus_idle_delay_time;
1289 }
1290
1291 assert(p->p_memstat_idledeadline);
1292
1293 if (present_in_deferred_bucket == FALSE) {
1294 memorystatus_scheduled_idle_demotions++;
1295 }
1296 }
1297
1298 static void
1299 memorystatus_invalidate_idle_demotion_locked(proc_t p, boolean_t clear_state)
1300 {
1301 boolean_t present_in_deferred_bucket = FALSE;
1302
1303 if (p->p_memstat_effectivepriority == JETSAM_PRIORITY_IDLE_DEFERRED) {
1304 present_in_deferred_bucket = TRUE;
1305 assert(p->p_memstat_idledeadline);
1306 }
1307
1308 MEMORYSTATUS_DEBUG(1, "memorystatus_invalidate_idle_demotion(): invalidating demotion to idle band for pid %d (clear_state %d, demotions %d).\n",
1309 p->p_pid, clear_state, memorystatus_scheduled_idle_demotions);
1310
1311
1312 if (clear_state) {
1313 p->p_memstat_idledeadline = 0;
1314 p->p_memstat_dirty &= ~P_DIRTY_DEFER_IN_PROGRESS;
1315 }
1316
1317 if (present_in_deferred_bucket == TRUE) {
1318 memorystatus_scheduled_idle_demotions--;
1319 }
1320
1321 assert(memorystatus_scheduled_idle_demotions >= 0);
1322 }
1323
1324 static void
1325 memorystatus_reschedule_idle_demotion_locked(void) {
1326 if (0 == memorystatus_scheduled_idle_demotions) {
1327 if (memstat_idle_demotion_deadline) {
1328 /* Transitioned 1->0, so cancel next call */
1329 thread_call_cancel(memorystatus_idle_demotion_call);
1330 memstat_idle_demotion_deadline = 0;
1331 }
1332 } else {
1333 memstat_bucket_t *demotion_bucket;
1334 proc_t p;
1335 demotion_bucket = &memstat_bucket[JETSAM_PRIORITY_IDLE_DEFERRED];
1336 p = TAILQ_FIRST(&demotion_bucket->list);
1337
1338 assert(p && p->p_memstat_idledeadline);
1339
1340 if (memstat_idle_demotion_deadline != p->p_memstat_idledeadline){
1341 thread_call_enter_delayed(memorystatus_idle_demotion_call, p->p_memstat_idledeadline);
1342 memstat_idle_demotion_deadline = p->p_memstat_idledeadline;
1343 }
1344 }
1345 }
1346
1347 /*
1348 * List manipulation
1349 */
1350
1351 int
1352 memorystatus_add(proc_t p, boolean_t locked)
1353 {
1354 memstat_bucket_t *bucket;
1355
1356 MEMORYSTATUS_DEBUG(1, "memorystatus_list_add(): adding pid %d with priority %d.\n", p->p_pid, p->p_memstat_effectivepriority);
1357
1358 if (!locked) {
1359 proc_list_lock();
1360 }
1361
1362 /* Processes marked internal do not have priority tracked */
1363 if (p->p_memstat_state & P_MEMSTAT_INTERNAL) {
1364 goto exit;
1365 }
1366
1367 bucket = &memstat_bucket[p->p_memstat_effectivepriority];
1368
1369 if (p->p_memstat_effectivepriority == JETSAM_PRIORITY_IDLE_DEFERRED) {
1370 assert(bucket->count == memorystatus_scheduled_idle_demotions);
1371 }
1372
1373 TAILQ_INSERT_TAIL(&bucket->list, p, p_memstat_list);
1374 bucket->count++;
1375
1376 memorystatus_list_count++;
1377
1378 memorystatus_check_levels_locked();
1379
1380 exit:
1381 if (!locked) {
1382 proc_list_unlock();
1383 }
1384
1385 return 0;
1386 }
1387
1388 /*
1389 * Description:
1390 * Moves a process from one jetsam bucket to another.
1391 * which changes the LRU position of the process.
1392 *
1393 * Monitors transition between buckets and if necessary
1394 * will update cached memory limits accordingly.
1395 */
1396 static void
1397 memorystatus_update_priority_locked(proc_t p, int priority, boolean_t head_insert)
1398 {
1399 memstat_bucket_t *old_bucket, *new_bucket;
1400
1401 assert(priority < MEMSTAT_BUCKET_COUNT);
1402
1403 /* Ensure that exit isn't underway, leaving the proc retained but removed from its bucket */
1404 if ((p->p_listflag & P_LIST_EXITED) != 0) {
1405 return;
1406 }
1407
1408 MEMORYSTATUS_DEBUG(1, "memorystatus_update_priority_locked(): setting pid %d to priority %d, inserting at %s\n",
1409 p->p_pid, priority, head_insert ? "head" : "tail");
1410
1411 old_bucket = &memstat_bucket[p->p_memstat_effectivepriority];
1412 if (p->p_memstat_effectivepriority == JETSAM_PRIORITY_IDLE_DEFERRED) {
1413 assert(old_bucket->count == (memorystatus_scheduled_idle_demotions + 1));
1414 }
1415
1416 TAILQ_REMOVE(&old_bucket->list, p, p_memstat_list);
1417 old_bucket->count--;
1418
1419 new_bucket = &memstat_bucket[priority];
1420 if (head_insert)
1421 TAILQ_INSERT_HEAD(&new_bucket->list, p, p_memstat_list);
1422 else
1423 TAILQ_INSERT_TAIL(&new_bucket->list, p, p_memstat_list);
1424 new_bucket->count++;
1425
1426 #if CONFIG_JETSAM
1427 if (memorystatus_highwater_enabled) {
1428 boolean_t trigger_exception;
1429
1430 /*
1431 * If cached limit data is updated, then the limits
1432 * will be enforced by writing to the ledgers.
1433 */
1434 boolean_t ledger_update_needed = TRUE;
1435
1436 /*
1437 * No need to consider P_MEMSTAT_MEMLIMIT_BACKGROUND anymore.
1438 * Background limits are described via the inactive limit slots.
1439 *
1440 * Here, we must update the cached memory limit if the task
1441 * is transitioning between:
1442 * active <--> inactive
1443 * FG <--> BG
1444 * but:
1445 * dirty <--> clean is ignored
1446 *
1447 * We bypass processes that have opted into dirty tracking because
1448 * a move between buckets does not imply a transition between the
1449 * dirty <--> clean state.
1450 * Setting limits on processes opted into dirty tracking is handled
1451 * in memorystatus_dirty_set() where the transition is very clear.
1452 */
1453
1454 if (p->p_memstat_dirty & P_DIRTY_TRACK) {
1455
1456 ledger_update_needed = FALSE;
1457
1458 } else if ((priority >= JETSAM_PRIORITY_FOREGROUND) && (p->p_memstat_effectivepriority < JETSAM_PRIORITY_FOREGROUND)) {
1459 /*
1460 * inactive --> active
1461 * BG --> FG
1462 * assign active state
1463 */
1464 CACHE_ACTIVE_LIMITS_LOCKED(p, trigger_exception);
1465
1466 } else if ((priority < JETSAM_PRIORITY_FOREGROUND) && (p->p_memstat_effectivepriority >= JETSAM_PRIORITY_FOREGROUND)) {
1467 /*
1468 * active --> inactive
1469 * FG --> BG
1470 * assign inactive state
1471 */
1472 CACHE_INACTIVE_LIMITS_LOCKED(p, trigger_exception);
1473 } else {
1474 /*
1475 * The transition between jetsam priority buckets apparently did
1476 * not affect active/inactive state.
1477 * This is not unusual... especially during startup when
1478 * processes are getting established in their respective bands.
1479 */
1480 ledger_update_needed = FALSE;
1481 }
1482
1483 /*
1484 * Enforce the new limits by writing to the ledger
1485 */
1486 if (ledger_update_needed) {
1487 task_set_phys_footprint_limit_internal(p->task, (p->p_memstat_memlimit > 0) ? p->p_memstat_memlimit : -1, NULL, trigger_exception);
1488
1489 MEMORYSTATUS_DEBUG(3, "memorystatus_update_priority_locked: new limit on pid %d (%dMB %s) priority old --> new (%d --> %d) dirty?=0x%x %s\n",
1490 p->p_pid, (p->p_memstat_memlimit > 0 ? p->p_memstat_memlimit : -1),
1491 (p->p_memstat_state & P_MEMSTAT_FATAL_MEMLIMIT ? "F " : "NF"), p->p_memstat_effectivepriority, priority, p->p_memstat_dirty,
1492 (p->p_memstat_dirty ? ((p->p_memstat_dirty & P_DIRTY) ? "isdirty" : "isclean") : ""));
1493 }
1494 }
1495
1496 #endif /* CONFIG_JETSAM */
1497
1498 p->p_memstat_effectivepriority = priority;
1499
1500 memorystatus_check_levels_locked();
1501 }
1502
1503 /*
1504 *
1505 * Description: Update the jetsam priority and memory limit attributes for a given process.
1506 *
1507 * Parameters:
1508 * p init this process's jetsam information.
1509 * priority The jetsam priority band
1510 * user_data user specific data, unused by the kernel
1511 * effective guards against race if process's update already occurred
1512 * update_memlimit When true we know this is the init step via the posix_spawn path.
1513 *
1514 * memlimit_active Value in megabytes; The monitored footprint level while the
1515 * process is active. Exceeding it may result in termination
1516 * based on it's associated fatal flag.
1517 *
1518 * memlimit_active_is_fatal When a process is active and exceeds its memory footprint,
1519 * this describes whether or not it should be immediately fatal.
1520 *
1521 * memlimit_inactive Value in megabytes; The monitored footprint level while the
1522 * process is inactive. Exceeding it may result in termination
1523 * based on it's associated fatal flag.
1524 *
1525 * memlimit_inactive_is_fatal When a process is inactive and exceeds its memory footprint,
1526 * this describes whether or not it should be immediatly fatal.
1527 *
1528 * memlimit_background This process has a high-water-mark while in the background.
1529 * No longer meaningful. Background limits are described via
1530 * the inactive slots. Flag is ignored.
1531 *
1532 *
1533 * Returns: 0 Success
1534 * non-0 Failure
1535 */
1536
1537 int
1538 memorystatus_update(proc_t p, int priority, uint64_t user_data, boolean_t effective, boolean_t update_memlimit,
1539 int32_t memlimit_active, boolean_t memlimit_active_is_fatal,
1540 int32_t memlimit_inactive, boolean_t memlimit_inactive_is_fatal,
1541 __unused boolean_t memlimit_background)
1542 {
1543 int ret;
1544 boolean_t head_insert = false;
1545
1546 #if !CONFIG_JETSAM
1547 #pragma unused(update_memlimit, memlimit_active, memlimit_inactive)
1548 #pragma unused(memlimit_active_is_fatal, memlimit_inactive_is_fatal)
1549 #endif /* !CONFIG_JETSAM */
1550
1551 MEMORYSTATUS_DEBUG(1, "memorystatus_update: changing pid %d: priority %d, user_data 0x%llx\n", p->p_pid, priority, user_data);
1552
1553 KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_UPDATE) | DBG_FUNC_START, p->p_pid, priority, user_data, effective, 0);
1554
1555 if (priority == -1) {
1556 /* Use as shorthand for default priority */
1557 priority = JETSAM_PRIORITY_DEFAULT;
1558 } else if (priority == JETSAM_PRIORITY_IDLE_DEFERRED) {
1559 /* JETSAM_PRIORITY_IDLE_DEFERRED is reserved for internal use; if requested, adjust to JETSAM_PRIORITY_IDLE. */
1560 priority = JETSAM_PRIORITY_IDLE;
1561 } else if (priority == JETSAM_PRIORITY_IDLE_HEAD) {
1562 /* JETSAM_PRIORITY_IDLE_HEAD inserts at the head of the idle queue */
1563 priority = JETSAM_PRIORITY_IDLE;
1564 head_insert = TRUE;
1565 } else if ((priority < 0) || (priority >= MEMSTAT_BUCKET_COUNT)) {
1566 /* Sanity check */
1567 ret = EINVAL;
1568 goto out;
1569 }
1570
1571 proc_list_lock();
1572
1573 assert(!(p->p_memstat_state & P_MEMSTAT_INTERNAL));
1574
1575 if (effective && (p->p_memstat_state & P_MEMSTAT_PRIORITYUPDATED)) {
1576 ret = EALREADY;
1577 proc_list_unlock();
1578 MEMORYSTATUS_DEBUG(1, "memorystatus_update: effective change specified for pid %d, but change already occurred.\n", p->p_pid);
1579 goto out;
1580 }
1581
1582 if ((p->p_memstat_state & P_MEMSTAT_TERMINATED) || ((p->p_listflag & P_LIST_EXITED) != 0)) {
1583 /*
1584 * This could happen when a process calling posix_spawn() is exiting on the jetsam thread.
1585 */
1586 ret = EBUSY;
1587 proc_list_unlock();
1588 goto out;
1589 }
1590
1591 p->p_memstat_state |= P_MEMSTAT_PRIORITYUPDATED;
1592 p->p_memstat_userdata = user_data;
1593 p->p_memstat_requestedpriority = priority;
1594
1595 #if CONFIG_JETSAM
1596 if (update_memlimit) {
1597 boolean_t trigger_exception;
1598
1599 /*
1600 * Posix_spawn'd processes come through this path to instantiate ledger limits.
1601 * Forked processes do not come through this path, so no ledger limits exist.
1602 * (That's why forked processes can consume unlimited memory.)
1603 */
1604
1605 MEMORYSTATUS_DEBUG(3, "memorystatus_update(enter): pid %d, priority %d, dirty=0x%x, Active(%dMB %s), Inactive(%dMB, %s)\n",
1606 p->p_pid, priority, p->p_memstat_dirty,
1607 memlimit_active, (memlimit_active_is_fatal ? "F " : "NF"),
1608 memlimit_inactive, (memlimit_inactive_is_fatal ? "F " : "NF"));
1609
1610 if (memlimit_background) {
1611
1612 /*
1613 * With 2-level HWM support, we no longer honor P_MEMSTAT_MEMLIMIT_BACKGROUND.
1614 * Background limits are described via the inactive limit slots.
1615 */
1616
1617 // p->p_memstat_state |= P_MEMSTAT_MEMLIMIT_BACKGROUND;
1618
1619 #if DEVELOPMENT || DEBUG
1620 printf("memorystatus_update: WARNING %s[%d] set unused flag P_MEMSTAT_MEMLIMIT_BACKGROUND [A==%dMB %s] [IA==%dMB %s]\n",
1621 (p->p_comm ? p->p_comm : "unknown"), p->p_pid,
1622 memlimit_active, (memlimit_active_is_fatal ? "F " : "NF"),
1623 memlimit_inactive, (memlimit_inactive_is_fatal ? "F " : "NF"));
1624 #endif /* DEVELOPMENT || DEBUG */
1625 }
1626
1627 if (memlimit_active <= 0) {
1628 /*
1629 * This process will have a system_wide task limit when active.
1630 * System_wide task limit is always fatal.
1631 * It's quite common to see non-fatal flag passed in here.
1632 * It's not an error, we just ignore it.
1633 */
1634
1635 /*
1636 * For backward compatibility with some unexplained launchd behavior,
1637 * we allow a zero sized limit. But we still enforce system_wide limit
1638 * when written to the ledgers.
1639 */
1640
1641 if (memlimit_active < 0) {
1642 memlimit_active = -1; /* enforces system_wide task limit */
1643 }
1644 memlimit_active_is_fatal = TRUE;
1645 }
1646
1647 if (memlimit_inactive <= 0) {
1648 /*
1649 * This process will have a system_wide task limit when inactive.
1650 * System_wide task limit is always fatal.
1651 */
1652
1653 memlimit_inactive = -1;
1654 memlimit_inactive_is_fatal = TRUE;
1655 }
1656
1657 /*
1658 * Initialize the active limit variants for this process.
1659 */
1660 SET_ACTIVE_LIMITS_LOCKED(p, memlimit_active, memlimit_active_is_fatal);
1661
1662 /*
1663 * Initialize the inactive limit variants for this process.
1664 */
1665 SET_INACTIVE_LIMITS_LOCKED(p, memlimit_inactive, memlimit_inactive_is_fatal);
1666
1667 /*
1668 * Initialize the cached limits for target process.
1669 * When the target process is dirty tracked, it's typically
1670 * in a clean state. Non dirty tracked processes are
1671 * typically active (Foreground or above).
1672 * But just in case, we don't make assumptions...
1673 */
1674
1675 if (proc_jetsam_state_is_active_locked(p) == TRUE) {
1676 CACHE_ACTIVE_LIMITS_LOCKED(p, trigger_exception);
1677 } else {
1678 CACHE_INACTIVE_LIMITS_LOCKED(p, trigger_exception);
1679 }
1680
1681 /*
1682 * Enforce the cached limit by writing to the ledger.
1683 */
1684 if (memorystatus_highwater_enabled) {
1685 /* apply now */
1686 assert(trigger_exception == TRUE);
1687 task_set_phys_footprint_limit_internal(p->task, ((p->p_memstat_memlimit > 0) ? p->p_memstat_memlimit : -1), NULL, trigger_exception);
1688
1689 MEMORYSTATUS_DEBUG(3, "memorystatus_update: init: limit on pid %d (%dMB %s) targeting priority(%d) dirty?=0x%x %s\n",
1690 p->p_pid, (p->p_memstat_memlimit > 0 ? p->p_memstat_memlimit : -1),
1691 (p->p_memstat_state & P_MEMSTAT_FATAL_MEMLIMIT ? "F " : "NF"), priority, p->p_memstat_dirty,
1692 (p->p_memstat_dirty ? ((p->p_memstat_dirty & P_DIRTY) ? "isdirty" : "isclean") : ""));
1693 }
1694 }
1695 #endif /* CONFIG_JETSAM */
1696
1697 /*
1698 * We can't add to the JETSAM_PRIORITY_IDLE_DEFERRED bucket here.
1699 * But, we could be removing it from the bucket.
1700 * Check and take appropriate steps if so.
1701 */
1702
1703 if (p->p_memstat_effectivepriority == JETSAM_PRIORITY_IDLE_DEFERRED) {
1704
1705 memorystatus_invalidate_idle_demotion_locked(p, TRUE);
1706 }
1707
1708 memorystatus_update_priority_locked(p, priority, head_insert);
1709
1710 proc_list_unlock();
1711 ret = 0;
1712
1713 out:
1714 KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_UPDATE) | DBG_FUNC_END, ret, 0, 0, 0, 0);
1715
1716 return ret;
1717 }
1718
1719 int
1720 memorystatus_remove(proc_t p, boolean_t locked)
1721 {
1722 int ret;
1723 memstat_bucket_t *bucket;
1724
1725 MEMORYSTATUS_DEBUG(1, "memorystatus_list_remove: removing pid %d\n", p->p_pid);
1726
1727 if (!locked) {
1728 proc_list_lock();
1729 }
1730
1731 assert(!(p->p_memstat_state & P_MEMSTAT_INTERNAL));
1732
1733 bucket = &memstat_bucket[p->p_memstat_effectivepriority];
1734 if (p->p_memstat_effectivepriority == JETSAM_PRIORITY_IDLE_DEFERRED) {
1735 assert(bucket->count == memorystatus_scheduled_idle_demotions);
1736 }
1737
1738 TAILQ_REMOVE(&bucket->list, p, p_memstat_list);
1739 bucket->count--;
1740
1741 memorystatus_list_count--;
1742
1743 /* If awaiting demotion to the idle band, clean up */
1744 if (p->p_memstat_effectivepriority == JETSAM_PRIORITY_IDLE_DEFERRED) {
1745 memorystatus_invalidate_idle_demotion_locked(p, TRUE);
1746 memorystatus_reschedule_idle_demotion_locked();
1747 }
1748
1749 memorystatus_check_levels_locked();
1750
1751 #if CONFIG_FREEZE
1752 if (p->p_memstat_state & (P_MEMSTAT_FROZEN)) {
1753 memorystatus_frozen_count--;
1754 }
1755
1756 if (p->p_memstat_state & P_MEMSTAT_SUSPENDED) {
1757 memorystatus_suspended_footprint_total -= p->p_memstat_suspendedfootprint;
1758 memorystatus_suspended_count--;
1759 }
1760 #endif
1761
1762 if (!locked) {
1763 proc_list_unlock();
1764 }
1765
1766 if (p) {
1767 ret = 0;
1768 } else {
1769 ret = ESRCH;
1770 }
1771
1772 return ret;
1773 }
1774
1775 /*
1776 * Validate dirty tracking flags with process state.
1777 *
1778 * Return:
1779 * 0 on success
1780 * non-0 on failure
1781 */
1782
1783 static int
1784 memorystatus_validate_track_flags(struct proc *target_p, uint32_t pcontrol) {
1785 /* See that the process isn't marked for termination */
1786 if (target_p->p_memstat_dirty & P_DIRTY_TERMINATED) {
1787 return EBUSY;
1788 }
1789
1790 /* Idle exit requires that process be tracked */
1791 if ((pcontrol & PROC_DIRTY_ALLOW_IDLE_EXIT) &&
1792 !(pcontrol & PROC_DIRTY_TRACK)) {
1793 return EINVAL;
1794 }
1795
1796 /* 'Launch in progress' tracking requires that process have enabled dirty tracking too. */
1797 if ((pcontrol & PROC_DIRTY_LAUNCH_IN_PROGRESS) &&
1798 !(pcontrol & PROC_DIRTY_TRACK)) {
1799 return EINVAL;
1800 }
1801
1802 /* Deferral is only relevant if idle exit is specified */
1803 if ((pcontrol & PROC_DIRTY_DEFER) &&
1804 !(pcontrol & PROC_DIRTY_ALLOWS_IDLE_EXIT)) {
1805 return EINVAL;
1806 }
1807
1808 return(0);
1809 }
1810
1811 static void
1812 memorystatus_update_idle_priority_locked(proc_t p) {
1813 int32_t priority;
1814
1815 MEMORYSTATUS_DEBUG(1, "memorystatus_update_idle_priority_locked(): pid %d dirty 0x%X\n", p->p_pid, p->p_memstat_dirty);
1816
1817 if ((p->p_memstat_dirty & (P_DIRTY_IDLE_EXIT_ENABLED|P_DIRTY_IS_DIRTY)) == P_DIRTY_IDLE_EXIT_ENABLED) {
1818 priority = (p->p_memstat_dirty & P_DIRTY_DEFER_IN_PROGRESS) ? JETSAM_PRIORITY_IDLE_DEFERRED : JETSAM_PRIORITY_IDLE;
1819 } else {
1820 priority = p->p_memstat_requestedpriority;
1821 }
1822
1823 if (priority != p->p_memstat_effectivepriority) {
1824 memorystatus_update_priority_locked(p, priority, false);
1825 }
1826 }
1827
1828 /*
1829 * Processes can opt to have their state tracked by the kernel, indicating when they are busy (dirty) or idle
1830 * (clean). They may also indicate that they support termination when idle, with the result that they are promoted
1831 * to their desired, higher, jetsam priority when dirty (and are therefore killed later), and demoted to the low
1832 * priority idle band when clean (and killed earlier, protecting higher priority procesess).
1833 *
1834 * If the deferral flag is set, then newly tracked processes will be protected for an initial period (as determined by
1835 * memorystatus_idle_delay_time); if they go clean during this time, then they will be moved to a deferred-idle band
1836 * with a slightly higher priority, guarding against immediate termination under memory pressure and being unable to
1837 * make forward progress. Finally, when the guard expires, they will be moved to the standard, lowest-priority, idle
1838 * band. The deferral can be cleared early by clearing the appropriate flag.
1839 *
1840 * The deferral timer is active only for the duration that the process is marked as guarded and clean; if the process
1841 * is marked dirty, the timer will be cancelled. Upon being subsequently marked clean, the deferment will either be
1842 * re-enabled or the guard state cleared, depending on whether the guard deadline has passed.
1843 */
1844
1845 int
1846 memorystatus_dirty_track(proc_t p, uint32_t pcontrol) {
1847 unsigned int old_dirty;
1848 boolean_t reschedule = FALSE;
1849 boolean_t already_deferred = FALSE;
1850 boolean_t defer_now = FALSE;
1851 int ret = 0;
1852
1853 KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_DIRTY_TRACK),
1854 p->p_pid, p->p_memstat_dirty, pcontrol, 0, 0);
1855
1856 proc_list_lock();
1857
1858 if ((p->p_listflag & P_LIST_EXITED) != 0) {
1859 /*
1860 * Process is on its way out.
1861 */
1862 ret = EBUSY;
1863 goto exit;
1864 }
1865
1866 if (p->p_memstat_state & P_MEMSTAT_INTERNAL) {
1867 ret = EPERM;
1868 goto exit;
1869 }
1870
1871 if ((ret = memorystatus_validate_track_flags(p, pcontrol)) != 0) {
1872 /* error */
1873 goto exit;
1874 }
1875
1876 old_dirty = p->p_memstat_dirty;
1877
1878 /* These bits are cumulative, as per <rdar://problem/11159924> */
1879 if (pcontrol & PROC_DIRTY_TRACK) {
1880 p->p_memstat_dirty |= P_DIRTY_TRACK;
1881 }
1882
1883 if (pcontrol & PROC_DIRTY_ALLOW_IDLE_EXIT) {
1884 p->p_memstat_dirty |= P_DIRTY_ALLOW_IDLE_EXIT;
1885 }
1886
1887 if (pcontrol & PROC_DIRTY_LAUNCH_IN_PROGRESS) {
1888 p->p_memstat_dirty |= P_DIRTY_LAUNCH_IN_PROGRESS;
1889 }
1890
1891 if (old_dirty & P_DIRTY_DEFER_IN_PROGRESS) {
1892 already_deferred = TRUE;
1893 }
1894
1895 /* This can be set and cleared exactly once. */
1896 if (pcontrol & PROC_DIRTY_DEFER) {
1897
1898 if ( !(old_dirty & P_DIRTY_DEFER)) {
1899 p->p_memstat_dirty |= P_DIRTY_DEFER;
1900 }
1901
1902 defer_now = TRUE;
1903 }
1904
1905 MEMORYSTATUS_DEBUG(1, "memorystatus_on_track_dirty(): set idle-exit %s / defer %s / dirty %s for pid %d\n",
1906 ((p->p_memstat_dirty & P_DIRTY_IDLE_EXIT_ENABLED) == P_DIRTY_IDLE_EXIT_ENABLED) ? "Y" : "N",
1907 defer_now ? "Y" : "N",
1908 p->p_memstat_dirty & P_DIRTY ? "Y" : "N",
1909 p->p_pid);
1910
1911 /* Kick off or invalidate the idle exit deferment if there's a state transition. */
1912 if (!(p->p_memstat_dirty & P_DIRTY_IS_DIRTY)) {
1913 if (((p->p_memstat_dirty & P_DIRTY_IDLE_EXIT_ENABLED) == P_DIRTY_IDLE_EXIT_ENABLED) &&
1914 defer_now && !already_deferred) {
1915
1916 /*
1917 * Request to defer a clean process that's idle-exit enabled
1918 * and not already in the jetsam deferred band.
1919 */
1920 memorystatus_schedule_idle_demotion_locked(p, TRUE);
1921 reschedule = TRUE;
1922
1923 } else if (!defer_now && already_deferred) {
1924
1925 /*
1926 * Either the process is no longer idle-exit enabled OR
1927 * there's a request to cancel a currently active deferral.
1928 */
1929 memorystatus_invalidate_idle_demotion_locked(p, TRUE);
1930 reschedule = TRUE;
1931 }
1932 } else {
1933
1934 /*
1935 * We are trying to operate on a dirty process. Dirty processes have to
1936 * be removed from the deferred band. The question is do we reset the
1937 * deferred state or not?
1938 *
1939 * This could be a legal request like:
1940 * - this process had opted into the JETSAM_DEFERRED band
1941 * - but it's now dirty and requests to opt out.
1942 * In this case, we remove the process from the band and reset its
1943 * state too. It'll opt back in properly when needed.
1944 *
1945 * OR, this request could be a user-space bug. E.g.:
1946 * - this process had opted into the JETSAM_DEFERRED band when clean
1947 * - and, then issues another request to again put it into the band except
1948 * this time the process is dirty.
1949 * The process going dirty, as a transition in memorystatus_dirty_set(), will pull the process out of
1950 * the deferred band with its state intact. So our request below is no-op.
1951 * But we do it here anyways for coverage.
1952 *
1953 * memorystatus_update_idle_priority_locked()
1954 * single-mindedly treats a dirty process as "cannot be in the deferred band".
1955 */
1956
1957 if (!defer_now && already_deferred) {
1958 memorystatus_invalidate_idle_demotion_locked(p, TRUE);
1959 reschedule = TRUE;
1960 } else {
1961 memorystatus_invalidate_idle_demotion_locked(p, FALSE);
1962 reschedule = TRUE;
1963 }
1964 }
1965
1966 memorystatus_update_idle_priority_locked(p);
1967
1968 if (reschedule) {
1969 memorystatus_reschedule_idle_demotion_locked();
1970 }
1971
1972 ret = 0;
1973
1974 exit:
1975 proc_list_unlock();
1976
1977 return ret;
1978 }
1979
1980 int
1981 memorystatus_dirty_set(proc_t p, boolean_t self, uint32_t pcontrol) {
1982 int ret;
1983 boolean_t kill = false;
1984 boolean_t reschedule = FALSE;
1985 boolean_t was_dirty = FALSE;
1986 boolean_t now_dirty = FALSE;
1987
1988 MEMORYSTATUS_DEBUG(1, "memorystatus_dirty_set(): %d %d 0x%x 0x%x\n", self, p->p_pid, pcontrol, p->p_memstat_dirty);
1989
1990 KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_DIRTY_SET), p->p_pid, self, pcontrol, 0, 0);
1991
1992 proc_list_lock();
1993
1994 if ((p->p_listflag & P_LIST_EXITED) != 0) {
1995 /*
1996 * Process is on its way out.
1997 */
1998 ret = EBUSY;
1999 goto exit;
2000 }
2001
2002 if (p->p_memstat_state & P_MEMSTAT_INTERNAL) {
2003 ret = EPERM;
2004 goto exit;
2005 }
2006
2007 if (p->p_memstat_dirty & P_DIRTY_IS_DIRTY)
2008 was_dirty = TRUE;
2009
2010 if (!(p->p_memstat_dirty & P_DIRTY_TRACK)) {
2011 /* Dirty tracking not enabled */
2012 ret = EINVAL;
2013 } else if (pcontrol && (p->p_memstat_dirty & P_DIRTY_TERMINATED)) {
2014 /*
2015 * Process is set to be terminated and we're attempting to mark it dirty.
2016 * Set for termination and marking as clean is OK - see <rdar://problem/10594349>.
2017 */
2018 ret = EBUSY;
2019 } else {
2020 int flag = (self == TRUE) ? P_DIRTY : P_DIRTY_SHUTDOWN;
2021 if (pcontrol && !(p->p_memstat_dirty & flag)) {
2022 /* Mark the process as having been dirtied at some point */
2023 p->p_memstat_dirty |= (flag | P_DIRTY_MARKED);
2024 memorystatus_dirty_count++;
2025 ret = 0;
2026 } else if ((pcontrol == 0) && (p->p_memstat_dirty & flag)) {
2027 if ((flag == P_DIRTY_SHUTDOWN) && (!(p->p_memstat_dirty & P_DIRTY))) {
2028 /* Clearing the dirty shutdown flag, and the process is otherwise clean - kill */
2029 p->p_memstat_dirty |= P_DIRTY_TERMINATED;
2030 kill = true;
2031 } else if ((flag == P_DIRTY) && (p->p_memstat_dirty & P_DIRTY_TERMINATED)) {
2032 /* Kill previously terminated processes if set clean */
2033 kill = true;
2034 }
2035 p->p_memstat_dirty &= ~flag;
2036 memorystatus_dirty_count--;
2037 ret = 0;
2038 } else {
2039 /* Already set */
2040 ret = EALREADY;
2041 }
2042 }
2043
2044 if (ret != 0) {
2045 goto exit;
2046 }
2047
2048 if (p->p_memstat_dirty & P_DIRTY_IS_DIRTY)
2049 now_dirty = TRUE;
2050
2051 if ((was_dirty == TRUE && now_dirty == FALSE) ||
2052 (was_dirty == FALSE && now_dirty == TRUE)) {
2053
2054 /* Manage idle exit deferral, if applied */
2055 if ((p->p_memstat_dirty & (P_DIRTY_IDLE_EXIT_ENABLED|P_DIRTY_DEFER_IN_PROGRESS)) ==
2056 (P_DIRTY_IDLE_EXIT_ENABLED|P_DIRTY_DEFER_IN_PROGRESS)) {
2057
2058 /*
2059 * P_DIRTY_DEFER_IN_PROGRESS means the process is in the deferred band OR it might be heading back
2060 * there once it's clean again and has some protection window left.
2061 */
2062
2063 if (p->p_memstat_dirty & P_DIRTY_IS_DIRTY) {
2064 /*
2065 * New dirty process i.e. "was_dirty == FALSE && now_dirty == TRUE"
2066 *
2067 * The process will move from the deferred band to its higher requested
2068 * jetsam band. But we don't clear its state i.e. we want to remember that
2069 * this process was part of the "deferred" band and will return to it.
2070 *
2071 * This way, we don't let it age beyond the protection
2072 * window when it returns to "clean". All the while giving
2073 * it a chance to perform its work while "dirty".
2074 *
2075 */
2076 memorystatus_invalidate_idle_demotion_locked(p, FALSE);
2077 reschedule = TRUE;
2078 } else {
2079
2080 /*
2081 * Process is back from "dirty" to "clean".
2082 *
2083 * Is its timer up OR does it still have some protection
2084 * window left?
2085 */
2086
2087 if (mach_absolute_time() >= p->p_memstat_idledeadline) {
2088 /*
2089 * The process' deadline has expired. It currently
2090 * does not reside in the DEFERRED bucket.
2091 *
2092 * It's on its way to the JETSAM_PRIORITY_IDLE
2093 * bucket via memorystatus_update_idle_priority_locked()
2094 * below.
2095
2096 * So all we need to do is reset all the state on the
2097 * process that's related to the DEFERRED bucket i.e.
2098 * the DIRTY_DEFER_IN_PROGRESS flag and the timer deadline.
2099 *
2100 */
2101
2102 memorystatus_invalidate_idle_demotion_locked(p, TRUE);
2103 reschedule = TRUE;
2104 } else {
2105 /*
2106 * It still has some protection window left and so
2107 * we just re-arm the timer without modifying any
2108 * state on the process.
2109 */
2110 memorystatus_schedule_idle_demotion_locked(p, FALSE);
2111 reschedule = TRUE;
2112 }
2113 }
2114 }
2115
2116 memorystatus_update_idle_priority_locked(p);
2117
2118 #if CONFIG_JETSAM
2119 if (memorystatus_highwater_enabled) {
2120 boolean_t trigger_exception;
2121 /*
2122 * We are in this path because this process transitioned between
2123 * dirty <--> clean state. Update the cached memory limits.
2124 */
2125
2126 if (proc_jetsam_state_is_active_locked(p) == TRUE) {
2127 /*
2128 * process is dirty
2129 */
2130 CACHE_ACTIVE_LIMITS_LOCKED(p, trigger_exception);
2131 } else {
2132 /*
2133 * process is clean
2134 */
2135 CACHE_INACTIVE_LIMITS_LOCKED(p, trigger_exception);
2136 }
2137
2138 /*
2139 * Enforce the new limits by writing to the ledger.
2140 *
2141 * This is a hot path and holding the proc_list_lock while writing to the ledgers,
2142 * (where the task lock is taken) is bad. So, we temporarily drop the proc_list_lock.
2143 * We aren't traversing the jetsam bucket list here, so we should be safe.
2144 * See rdar://21394491.
2145 */
2146
2147 if (proc_ref_locked(p) == p) {
2148 int ledger_limit;
2149 if (p->p_memstat_memlimit > 0) {
2150 ledger_limit = p->p_memstat_memlimit;
2151 } else {
2152 ledger_limit = -1;
2153 }
2154 proc_list_unlock();
2155 task_set_phys_footprint_limit_internal(p->task, ledger_limit, NULL, trigger_exception);
2156 proc_list_lock();
2157 proc_rele_locked(p);
2158
2159 MEMORYSTATUS_DEBUG(3, "memorystatus_dirty_set: new limit on pid %d (%dMB %s) priority(%d) dirty?=0x%x %s\n",
2160 p->p_pid, (p->p_memstat_memlimit > 0 ? p->p_memstat_memlimit : -1),
2161 (p->p_memstat_state & P_MEMSTAT_FATAL_MEMLIMIT ? "F " : "NF"), p->p_memstat_effectivepriority, p->p_memstat_dirty,
2162 (p->p_memstat_dirty ? ((p->p_memstat_dirty & P_DIRTY) ? "isdirty" : "isclean") : ""));
2163 }
2164
2165 }
2166 #endif /* CONFIG_JETSAM */
2167
2168 /* If the deferral state changed, reschedule the demotion timer */
2169 if (reschedule) {
2170 memorystatus_reschedule_idle_demotion_locked();
2171 }
2172 }
2173
2174 if (kill) {
2175 if (proc_ref_locked(p) == p) {
2176 proc_list_unlock();
2177 psignal(p, SIGKILL);
2178 proc_list_lock();
2179 proc_rele_locked(p);
2180 }
2181 }
2182
2183 exit:
2184 proc_list_unlock();
2185
2186 return ret;
2187 }
2188
2189 int
2190 memorystatus_dirty_clear(proc_t p, uint32_t pcontrol) {
2191
2192 int ret = 0;
2193
2194 MEMORYSTATUS_DEBUG(1, "memorystatus_dirty_clear(): %d 0x%x 0x%x\n", p->p_pid, pcontrol, p->p_memstat_dirty);
2195
2196 KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_DIRTY_CLEAR), p->p_pid, pcontrol, 0, 0, 0);
2197
2198 proc_list_lock();
2199
2200 if ((p->p_listflag & P_LIST_EXITED) != 0) {
2201 /*
2202 * Process is on its way out.
2203 */
2204 ret = EBUSY;
2205 goto exit;
2206 }
2207
2208 if (p->p_memstat_state & P_MEMSTAT_INTERNAL) {
2209 ret = EPERM;
2210 goto exit;
2211 }
2212
2213 if (!(p->p_memstat_dirty & P_DIRTY_TRACK)) {
2214 /* Dirty tracking not enabled */
2215 ret = EINVAL;
2216 goto exit;
2217 }
2218
2219 if (!pcontrol || (pcontrol & (PROC_DIRTY_LAUNCH_IN_PROGRESS | PROC_DIRTY_DEFER)) == 0) {
2220 ret = EINVAL;
2221 goto exit;
2222 }
2223
2224 if (pcontrol & PROC_DIRTY_LAUNCH_IN_PROGRESS) {
2225 p->p_memstat_dirty &= ~P_DIRTY_LAUNCH_IN_PROGRESS;
2226 }
2227
2228 /* This can be set and cleared exactly once. */
2229 if (pcontrol & PROC_DIRTY_DEFER) {
2230
2231 if (p->p_memstat_dirty & P_DIRTY_DEFER) {
2232
2233 p->p_memstat_dirty &= ~P_DIRTY_DEFER;
2234
2235 memorystatus_invalidate_idle_demotion_locked(p, TRUE);
2236 memorystatus_update_idle_priority_locked(p);
2237 memorystatus_reschedule_idle_demotion_locked();
2238 }
2239 }
2240
2241 ret = 0;
2242 exit:
2243 proc_list_unlock();
2244
2245 return ret;
2246 }
2247
2248 int
2249 memorystatus_dirty_get(proc_t p) {
2250 int ret = 0;
2251
2252 proc_list_lock();
2253
2254 if (p->p_memstat_dirty & P_DIRTY_TRACK) {
2255 ret |= PROC_DIRTY_TRACKED;
2256 if (p->p_memstat_dirty & P_DIRTY_ALLOW_IDLE_EXIT) {
2257 ret |= PROC_DIRTY_ALLOWS_IDLE_EXIT;
2258 }
2259 if (p->p_memstat_dirty & P_DIRTY) {
2260 ret |= PROC_DIRTY_IS_DIRTY;
2261 }
2262 if (p->p_memstat_dirty & P_DIRTY_LAUNCH_IN_PROGRESS) {
2263 ret |= PROC_DIRTY_LAUNCH_IS_IN_PROGRESS;
2264 }
2265 }
2266
2267 proc_list_unlock();
2268
2269 return ret;
2270 }
2271
2272 int
2273 memorystatus_on_terminate(proc_t p) {
2274 int sig;
2275
2276 proc_list_lock();
2277
2278 p->p_memstat_dirty |= P_DIRTY_TERMINATED;
2279
2280 if ((p->p_memstat_dirty & (P_DIRTY_TRACK|P_DIRTY_IS_DIRTY)) == P_DIRTY_TRACK) {
2281 /* Clean; mark as terminated and issue SIGKILL */
2282 sig = SIGKILL;
2283 } else {
2284 /* Dirty, terminated, or state tracking is unsupported; issue SIGTERM to allow cleanup */
2285 sig = SIGTERM;
2286 }
2287
2288 proc_list_unlock();
2289
2290 return sig;
2291 }
2292
2293 void
2294 memorystatus_on_suspend(proc_t p)
2295 {
2296 #if CONFIG_FREEZE
2297 uint32_t pages;
2298 memorystatus_get_task_page_counts(p->task, &pages, NULL, NULL, NULL);
2299 #endif
2300 proc_list_lock();
2301 #if CONFIG_FREEZE
2302 p->p_memstat_suspendedfootprint = pages;
2303 memorystatus_suspended_footprint_total += pages;
2304 memorystatus_suspended_count++;
2305 #endif
2306 p->p_memstat_state |= P_MEMSTAT_SUSPENDED;
2307 proc_list_unlock();
2308 }
2309
2310 void
2311 memorystatus_on_resume(proc_t p)
2312 {
2313 #if CONFIG_FREEZE
2314 boolean_t frozen;
2315 pid_t pid;
2316 #endif
2317
2318 proc_list_lock();
2319
2320 #if CONFIG_FREEZE
2321 frozen = (p->p_memstat_state & P_MEMSTAT_FROZEN);
2322 if (frozen) {
2323 memorystatus_frozen_count--;
2324 p->p_memstat_state |= P_MEMSTAT_PRIOR_THAW;
2325 }
2326
2327 memorystatus_suspended_footprint_total -= p->p_memstat_suspendedfootprint;
2328 memorystatus_suspended_count--;
2329
2330 pid = p->p_pid;
2331 #endif
2332
2333 p->p_memstat_state &= ~(P_MEMSTAT_SUSPENDED | P_MEMSTAT_FROZEN);
2334
2335 proc_list_unlock();
2336
2337 #if CONFIG_FREEZE
2338 if (frozen) {
2339 memorystatus_freeze_entry_t data = { pid, FALSE, 0 };
2340 memorystatus_send_note(kMemorystatusFreezeNote, &data, sizeof(data));
2341 }
2342 #endif
2343 }
2344
2345 void
2346 memorystatus_on_inactivity(proc_t p)
2347 {
2348 #pragma unused(p)
2349 #if CONFIG_FREEZE
2350 /* Wake the freeze thread */
2351 thread_wakeup((event_t)&memorystatus_freeze_wakeup);
2352 #endif
2353 }
2354
2355 static uint32_t
2356 memorystatus_build_state(proc_t p) {
2357 uint32_t snapshot_state = 0;
2358
2359 /* General */
2360 if (p->p_memstat_state & P_MEMSTAT_SUSPENDED) {
2361 snapshot_state |= kMemorystatusSuspended;
2362 }
2363 if (p->p_memstat_state & P_MEMSTAT_FROZEN) {
2364 snapshot_state |= kMemorystatusFrozen;
2365 }
2366 if (p->p_memstat_state & P_MEMSTAT_PRIOR_THAW) {
2367 snapshot_state |= kMemorystatusWasThawed;
2368 }
2369
2370 /* Tracking */
2371 if (p->p_memstat_dirty & P_DIRTY_TRACK) {
2372 snapshot_state |= kMemorystatusTracked;
2373 }
2374 if ((p->p_memstat_dirty & P_DIRTY_IDLE_EXIT_ENABLED) == P_DIRTY_IDLE_EXIT_ENABLED) {
2375 snapshot_state |= kMemorystatusSupportsIdleExit;
2376 }
2377 if (p->p_memstat_dirty & P_DIRTY_IS_DIRTY) {
2378 snapshot_state |= kMemorystatusDirty;
2379 }
2380
2381 return snapshot_state;
2382 }
2383
2384 #if !CONFIG_JETSAM
2385
2386 static boolean_t
2387 kill_idle_exit_proc(void)
2388 {
2389 proc_t p, victim_p = PROC_NULL;
2390 uint64_t current_time;
2391 boolean_t killed = FALSE;
2392 unsigned int i = 0;
2393
2394 /* Pick next idle exit victim. */
2395 current_time = mach_absolute_time();
2396
2397 proc_list_lock();
2398
2399 p = memorystatus_get_first_proc_locked(&i, FALSE);
2400 while (p) {
2401 /* No need to look beyond the idle band */
2402 if (p->p_memstat_effectivepriority != JETSAM_PRIORITY_IDLE) {
2403 break;
2404 }
2405
2406 if ((p->p_memstat_dirty & (P_DIRTY_ALLOW_IDLE_EXIT|P_DIRTY_IS_DIRTY|P_DIRTY_TERMINATED)) == (P_DIRTY_ALLOW_IDLE_EXIT)) {
2407 if (current_time >= p->p_memstat_idledeadline) {
2408 p->p_memstat_dirty |= P_DIRTY_TERMINATED;
2409 victim_p = proc_ref_locked(p);
2410 break;
2411 }
2412 }
2413
2414 p = memorystatus_get_next_proc_locked(&i, p, FALSE);
2415 }
2416
2417 proc_list_unlock();
2418
2419 if (victim_p) {
2420 printf("memorystatus_thread: idle exiting pid %d [%s]\n", victim_p->p_pid, (victim_p->p_comm ? victim_p->p_comm : "(unknown)"));
2421 killed = memorystatus_do_kill(victim_p, kMemorystatusKilledIdleExit);
2422 proc_rele(victim_p);
2423 }
2424
2425 return killed;
2426 }
2427 #endif
2428
2429 #if CONFIG_JETSAM
2430 static void
2431 memorystatus_thread_wake(void) {
2432 thread_wakeup((event_t)&memorystatus_wakeup);
2433 }
2434 #endif /* CONFIG_JETSAM */
2435
2436 extern void vm_pressure_response(void);
2437
2438 static int
2439 memorystatus_thread_block(uint32_t interval_ms, thread_continue_t continuation)
2440 {
2441 if (interval_ms) {
2442 assert_wait_timeout(&memorystatus_wakeup, THREAD_UNINT, interval_ms, 1000 * NSEC_PER_USEC);
2443 } else {
2444 assert_wait(&memorystatus_wakeup, THREAD_UNINT);
2445 }
2446
2447 return thread_block(continuation);
2448 }
2449
2450 static void
2451 memorystatus_thread(void *param __unused, wait_result_t wr __unused)
2452 {
2453 static boolean_t is_vm_privileged = FALSE;
2454
2455 #if CONFIG_JETSAM
2456 boolean_t post_snapshot = FALSE;
2457 uint32_t errors = 0;
2458 uint32_t hwm_kill = 0;
2459 boolean_t sort_flag = TRUE;
2460
2461 /* Jetsam Loop Detection - locals */
2462 memstat_bucket_t *bucket;
2463 int jld_bucket_count = 0;
2464 struct timeval jld_now_tstamp = {0,0};
2465 uint64_t jld_now_msecs = 0;
2466
2467 /* Jetsam Loop Detection - statics */
2468 static uint64_t jld_timestamp_msecs = 0;
2469 static int jld_idle_kill_candidates = 0; /* Number of available processes in band 0,1 at start */
2470 static int jld_idle_kills = 0; /* Number of procs killed during eval period */
2471 static int jld_eval_aggressive_count = 0; /* Bumps the max priority in aggressive loop */
2472 static int32_t jld_priority_band_max = JETSAM_PRIORITY_UI_SUPPORT;
2473 #endif
2474
2475 if (is_vm_privileged == FALSE) {
2476 /*
2477 * It's the first time the thread has run, so just mark the thread as privileged and block.
2478 * This avoids a spurious pass with unset variables, as set out in <rdar://problem/9609402>.
2479 */
2480 thread_wire(host_priv_self(), current_thread(), TRUE);
2481 is_vm_privileged = TRUE;
2482
2483 if (vm_restricted_to_single_processor == TRUE)
2484 thread_vm_bind_group_add();
2485
2486 memorystatus_thread_block(0, memorystatus_thread);
2487 }
2488
2489 #if CONFIG_JETSAM
2490
2491 KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_SCAN) | DBG_FUNC_START,
2492 memorystatus_available_pages, memorystatus_jld_enabled, memorystatus_jld_eval_period_msecs, memorystatus_jld_eval_aggressive_count,0);
2493
2494 /*
2495 * Jetsam aware version.
2496 *
2497 * The VM pressure notification thread is working it's way through clients in parallel.
2498 *
2499 * So, while the pressure notification thread is targeting processes in order of
2500 * increasing jetsam priority, we can hopefully reduce / stop it's work by killing
2501 * any processes that have exceeded their highwater mark.
2502 *
2503 * If we run out of HWM processes and our available pages drops below the critical threshold, then,
2504 * we target the least recently used process in order of increasing jetsam priority (exception: the FG band).
2505 */
2506 while (is_thrashing(kill_under_pressure_cause) ||
2507 memorystatus_available_pages <= memorystatus_available_pages_pressure) {
2508 boolean_t killed;
2509 int32_t priority;
2510 uint32_t cause;
2511
2512 if (kill_under_pressure_cause) {
2513 cause = kill_under_pressure_cause;
2514 } else {
2515 cause = kMemorystatusKilledVMPageShortage;
2516 }
2517
2518 #if LEGACY_HIWATER
2519 /* Highwater */
2520 killed = memorystatus_kill_hiwat_proc(&errors);
2521 if (killed) {
2522 hwm_kill++;
2523 post_snapshot = TRUE;
2524 goto done;
2525 } else {
2526 memorystatus_hwm_candidates = FALSE;
2527 }
2528
2529 /* No highwater processes to kill. Continue or stop for now? */
2530 if (!is_thrashing(kill_under_pressure_cause) &&
2531 (memorystatus_available_pages > memorystatus_available_pages_critical)) {
2532 /*
2533 * We are _not_ out of pressure but we are above the critical threshold and there's:
2534 * - no compressor thrashing
2535 * - no more HWM processes left.
2536 * For now, don't kill any other processes.
2537 */
2538
2539 if (hwm_kill == 0) {
2540 memorystatus_thread_wasted_wakeup++;
2541 }
2542
2543 break;
2544 }
2545 #endif
2546 if (memorystatus_jld_enabled == TRUE) {
2547
2548 /*
2549 * Jetsam Loop Detection: attempt to detect
2550 * rapid daemon relaunches in the lower bands.
2551 */
2552
2553 microuptime(&jld_now_tstamp);
2554
2555 /*
2556 * Ignore usecs in this calculation.
2557 * msecs granularity is close enough.
2558 */
2559 jld_now_msecs = (jld_now_tstamp.tv_sec * 1000);
2560
2561 proc_list_lock();
2562 bucket = &memstat_bucket[JETSAM_PRIORITY_IDLE];
2563 jld_bucket_count = bucket->count;
2564 bucket = &memstat_bucket[JETSAM_PRIORITY_IDLE_DEFERRED];
2565 jld_bucket_count += bucket->count;
2566 proc_list_unlock();
2567
2568 /*
2569 * memorystatus_jld_eval_period_msecs is a tunable
2570 * memorystatus_jld_eval_aggressive_count is a tunable
2571 * memorystatus_jld_eval_aggressive_priority_band_max is a tunable
2572 */
2573 if ( (jld_bucket_count == 0) ||
2574 (jld_now_msecs > (jld_timestamp_msecs + memorystatus_jld_eval_period_msecs))) {
2575
2576 /*
2577 * Refresh evaluation parameters
2578 */
2579 jld_timestamp_msecs = jld_now_msecs;
2580 jld_idle_kill_candidates = jld_bucket_count;
2581 jld_idle_kills = 0;
2582 jld_eval_aggressive_count = 0;
2583 jld_priority_band_max = JETSAM_PRIORITY_UI_SUPPORT;
2584 }
2585
2586 if (jld_idle_kills > jld_idle_kill_candidates) {
2587 jld_eval_aggressive_count++;
2588 if (jld_eval_aggressive_count > memorystatus_jld_eval_aggressive_count) {
2589 /*
2590 * Bump up the jetsam priority limit (eg: the bucket index)
2591 * Enforce bucket index sanity.
2592 */
2593 if ((memorystatus_jld_eval_aggressive_priority_band_max < 0) ||
2594 (memorystatus_jld_eval_aggressive_priority_band_max >= MEMSTAT_BUCKET_COUNT)) {
2595 /*
2596 * Do nothing. Stick with the default level.
2597 */
2598 } else {
2599 jld_priority_band_max = memorystatus_jld_eval_aggressive_priority_band_max;
2600 }
2601 }
2602
2603 killed = memorystatus_kill_top_process_aggressive(
2604 TRUE,
2605 kMemorystatusKilledVMThrashing,
2606 jld_eval_aggressive_count,
2607 jld_priority_band_max,
2608 &errors);
2609
2610
2611 if (killed) {
2612 /* Always generate logs after aggressive kill */
2613 post_snapshot = TRUE;
2614 goto done;
2615 }
2616 }
2617 }
2618
2619 /* LRU */
2620 killed = memorystatus_kill_top_process(TRUE, sort_flag, cause, &priority, &errors);
2621 sort_flag = FALSE;
2622
2623 if (killed) {
2624 /*
2625 * Don't generate logs for steady-state idle-exit kills,
2626 * unless it is overridden for debug or by the device
2627 * tree.
2628 */
2629 if ((priority != JETSAM_PRIORITY_IDLE) || memorystatus_idle_snapshot) {
2630 post_snapshot = TRUE;
2631 }
2632
2633 /* Jetsam Loop Detection */
2634 if (memorystatus_jld_enabled == TRUE) {
2635 if ((priority == JETSAM_PRIORITY_IDLE) || (priority == JETSAM_PRIORITY_IDLE_DEFERRED)) {
2636 jld_idle_kills++;
2637 } else {
2638 /*
2639 * We've reached into bands beyond idle deferred.
2640 * We make no attempt to monitor them
2641 */
2642 }
2643 }
2644 goto done;
2645 }
2646
2647 if (memorystatus_available_pages <= memorystatus_available_pages_critical) {
2648 /* Under pressure and unable to kill a process - panic */
2649 panic("memorystatus_jetsam_thread: no victim! available pages:%d\n", memorystatus_available_pages);
2650 }
2651
2652 done:
2653
2654 /*
2655 * We do not want to over-kill when thrashing has been detected.
2656 * To avoid that, we reset the flag here and notify the
2657 * compressor.
2658 */
2659 if (is_thrashing(kill_under_pressure_cause)) {
2660 kill_under_pressure_cause = 0;
2661 vm_thrashing_jetsam_done();
2662 }
2663 }
2664
2665 kill_under_pressure_cause = 0;
2666
2667 if (errors) {
2668 memorystatus_clear_errors();
2669 }
2670
2671 #if VM_PRESSURE_EVENTS
2672 /*
2673 * LD: We used to target the foreground process first and foremost here.
2674 * Now, we target all processes, starting from the non-suspended, background
2675 * processes first. We will target foreground too.
2676 *
2677 * memorystatus_update_vm_pressure(TRUE);
2678 */
2679 //vm_pressure_response();
2680 #endif
2681
2682 if (post_snapshot) {
2683 size_t snapshot_size = sizeof(memorystatus_jetsam_snapshot_t) +
2684 sizeof(memorystatus_jetsam_snapshot_entry_t) * (memorystatus_jetsam_snapshot_count);
2685 uint64_t timestamp_now = mach_absolute_time();
2686 memorystatus_jetsam_snapshot->notification_time = timestamp_now;
2687 if (memorystatus_jetsam_snapshot_last_timestamp == 0 ||
2688 timestamp_now > memorystatus_jetsam_snapshot_last_timestamp + memorystatus_jetsam_snapshot_timeout) {
2689 int ret = memorystatus_send_note(kMemorystatusSnapshotNote, &snapshot_size, sizeof(snapshot_size));
2690 if (!ret) {
2691 proc_list_lock();
2692 memorystatus_jetsam_snapshot_last_timestamp = timestamp_now;
2693 proc_list_unlock();
2694 }
2695 }
2696 }
2697
2698 KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_SCAN) | DBG_FUNC_END,
2699 memorystatus_available_pages, 0, 0, 0, 0);
2700
2701 #else /* CONFIG_JETSAM */
2702
2703 /*
2704 * Jetsam not enabled
2705 */
2706
2707 #endif /* CONFIG_JETSAM */
2708
2709 memorystatus_thread_block(0, memorystatus_thread);
2710 }
2711
2712 #if !CONFIG_JETSAM
2713 /*
2714 * Returns TRUE:
2715 * when an idle-exitable proc was killed
2716 * Returns FALSE:
2717 * when there are no more idle-exitable procs found
2718 * when the attempt to kill an idle-exitable proc failed
2719 */
2720 boolean_t memorystatus_idle_exit_from_VM(void) {
2721 return(kill_idle_exit_proc());
2722 }
2723 #endif /* !CONFIG_JETSAM */
2724
2725 #if CONFIG_JETSAM
2726
2727 /*
2728 * Callback invoked when allowable physical memory footprint exceeded
2729 * (dirty pages + IOKit mappings)
2730 *
2731 * This is invoked for both advisory, non-fatal per-task high watermarks,
2732 * as well as the fatal task memory limits.
2733 */
2734 void
2735 memorystatus_on_ledger_footprint_exceeded(boolean_t warning, const int max_footprint_mb)
2736 {
2737 boolean_t is_active;
2738 boolean_t is_fatal;
2739
2740 proc_t p = current_proc();
2741
2742 proc_list_lock();
2743
2744 is_active = proc_jetsam_state_is_active_locked(p);
2745 is_fatal = (p->p_memstat_state & P_MEMSTAT_FATAL_MEMLIMIT);
2746
2747 if (warning == FALSE) {
2748 /*
2749 * We only want the EXC_RESOURCE to trigger once per lifetime
2750 * of the active/inactive limit state. So, here, we detect the
2751 * active/inactive state of the process and mark the
2752 * state as exception has been triggered.
2753 */
2754 if (is_active == TRUE) {
2755 /*
2756 * turn off exceptions for active state
2757 */
2758 p->p_memstat_state |= P_MEMSTAT_MEMLIMIT_ACTIVE_EXC_TRIGGERED;
2759 } else {
2760 /*
2761 * turn off exceptions for inactive state
2762 */
2763 p->p_memstat_state |= P_MEMSTAT_MEMLIMIT_INACTIVE_EXC_TRIGGERED;
2764 }
2765
2766 /*
2767 * Soft memory limit is a non-fatal high-water-mark
2768 * Hard memory limit is a fatal custom-task-limit or system-wide per-task memory limit.
2769 */
2770 printf("process %d (%s) exceeded physical memory footprint, the %s%sMemoryLimit of %d MB\n",
2771 p->p_pid, p->p_comm, (is_active ? "Active" : "Inactive"),
2772 (is_fatal ? "Hard" : "Soft"), max_footprint_mb);
2773
2774 }
2775
2776 proc_list_unlock();
2777
2778 #if VM_PRESSURE_EVENTS
2779 if (warning == TRUE) {
2780 if (memorystatus_warn_process(p->p_pid, TRUE /* critical? */) != TRUE) {
2781 /* Print warning, since it's possible that task has not registered for pressure notifications */
2782 printf("task_exceeded_footprint: failed to warn the current task (exiting, or no handler registered?).\n");
2783 }
2784 return;
2785 }
2786 #endif /* VM_PRESSURE_EVENTS */
2787
2788 if (is_fatal) {
2789 /*
2790 * If this process has no high watermark or has a fatal task limit, then we have been invoked because the task
2791 * has violated either the system-wide per-task memory limit OR its own task limit.
2792 */
2793 if (memorystatus_kill_process_sync(p->p_pid, kMemorystatusKilledPerProcessLimit) != TRUE) {
2794 printf("task_exceeded_footprint: failed to kill the current task (exiting?).\n");
2795 }
2796 } else {
2797 /*
2798 * HWM offender exists. Done without locks or synchronization.
2799 * See comment near its declaration for more details.
2800 */
2801 memorystatus_hwm_candidates = TRUE;
2802 }
2803 }
2804
2805 /*
2806 * Toggle the P_MEMSTAT_TERMINATED state.
2807 * Takes the proc_list_lock.
2808 */
2809 void
2810 proc_memstat_terminated(proc_t p, boolean_t set)
2811 {
2812 #if DEVELOPMENT || DEBUG
2813 if (p) {
2814 proc_list_lock();
2815 if (set == TRUE) {
2816 p->p_memstat_state |= P_MEMSTAT_TERMINATED;
2817 } else {
2818 p->p_memstat_state &= ~P_MEMSTAT_TERMINATED;
2819 }
2820 proc_list_unlock();
2821 }
2822 #else
2823 #pragma unused(p, set)
2824 /*
2825 * do nothing
2826 */
2827 #endif /* DEVELOPMENT || DEBUG */
2828 return;
2829 }
2830
2831 /*
2832 * This is invoked when cpulimits have been exceeded while in fatal mode.
2833 * The jetsam_flags do not apply as those are for memory related kills.
2834 * We call this routine so that the offending process is killed with
2835 * a non-zero exit status.
2836 */
2837 void
2838 jetsam_on_ledger_cpulimit_exceeded(void)
2839 {
2840 int retval = 0;
2841 int jetsam_flags = 0; /* make it obvious */
2842 proc_t p = current_proc();
2843
2844 printf("task_exceeded_cpulimit: killing pid %d [%s]\n",
2845 p->p_pid, (p->p_comm ? p->p_comm : "(unknown)"));
2846
2847 retval = jetsam_do_kill(p, jetsam_flags);
2848
2849 if (retval) {
2850 printf("task_exceeded_cpulimit: failed to kill current task (exiting?).\n");
2851 }
2852 }
2853
2854 static void
2855 memorystatus_get_task_page_counts(task_t task, uint32_t *footprint, uint32_t *max_footprint, uint32_t *max_footprint_lifetime, uint32_t *purgeable_pages)
2856 {
2857 assert(task);
2858 assert(footprint);
2859
2860 *footprint = (uint32_t)(get_task_phys_footprint(task) / PAGE_SIZE_64);
2861 if (max_footprint) {
2862 *max_footprint = (uint32_t)(get_task_phys_footprint_max(task) / PAGE_SIZE_64);
2863 }
2864 if (max_footprint_lifetime) {
2865 *max_footprint_lifetime = (uint32_t)(get_task_resident_max(task) / PAGE_SIZE_64);
2866 }
2867 if (purgeable_pages) {
2868 *purgeable_pages = (uint32_t)(get_task_purgeable_size(task) / PAGE_SIZE_64);
2869 }
2870 }
2871
2872 static void
2873 memorystatus_update_jetsam_snapshot_entry_locked(proc_t p, uint32_t kill_cause)
2874 {
2875 unsigned int i;
2876
2877 for (i = 0; i < memorystatus_jetsam_snapshot_count; i++) {
2878 if (memorystatus_jetsam_snapshot_list[i].pid == p->p_pid) {
2879 /* Update if the priority has changed since the snapshot was taken */
2880 if (memorystatus_jetsam_snapshot_list[i].priority != p->p_memstat_effectivepriority) {
2881 memorystatus_jetsam_snapshot_list[i].priority = p->p_memstat_effectivepriority;
2882 strlcpy(memorystatus_jetsam_snapshot_list[i].name, p->p_comm, MAXCOMLEN+1);
2883 memorystatus_jetsam_snapshot_list[i].state = memorystatus_build_state(p);
2884 memorystatus_jetsam_snapshot_list[i].user_data = p->p_memstat_userdata;
2885 memorystatus_jetsam_snapshot_list[i].fds = p->p_fd->fd_nfiles;
2886 }
2887 memorystatus_jetsam_snapshot_list[i].killed = kill_cause;
2888 return;
2889 }
2890 }
2891 }
2892
2893 void memorystatus_pages_update(unsigned int pages_avail)
2894 {
2895 memorystatus_available_pages = pages_avail;
2896
2897 #if VM_PRESSURE_EVENTS
2898 /*
2899 * Since memorystatus_available_pages changes, we should
2900 * re-evaluate the pressure levels on the system and
2901 * check if we need to wake the pressure thread.
2902 * We also update memorystatus_level in that routine.
2903 */
2904 vm_pressure_response();
2905
2906 if (memorystatus_available_pages <= memorystatus_available_pages_pressure) {
2907
2908 if (memorystatus_hwm_candidates || (memorystatus_available_pages <= memorystatus_available_pages_critical)) {
2909 memorystatus_thread_wake();
2910 }
2911 }
2912 #else /* VM_PRESSURE_EVENTS */
2913
2914 boolean_t critical, delta;
2915
2916 if (!memorystatus_delta) {
2917 return;
2918 }
2919
2920 critical = (pages_avail < memorystatus_available_pages_critical) ? TRUE : FALSE;
2921 delta = ((pages_avail >= (memorystatus_available_pages + memorystatus_delta))
2922 || (memorystatus_available_pages >= (pages_avail + memorystatus_delta))) ? TRUE : FALSE;
2923
2924 if (critical || delta) {
2925 memorystatus_level = memorystatus_available_pages * 100 / atop_64(max_mem);
2926 memorystatus_thread_wake();
2927 }
2928 #endif /* VM_PRESSURE_EVENTS */
2929 }
2930
2931 static boolean_t
2932 memorystatus_init_jetsam_snapshot_entry_locked(proc_t p, memorystatus_jetsam_snapshot_entry_t *entry)
2933 {
2934 clock_sec_t tv_sec;
2935 clock_usec_t tv_usec;
2936
2937 memset(entry, 0, sizeof(memorystatus_jetsam_snapshot_entry_t));
2938
2939 entry->pid = p->p_pid;
2940 strlcpy(&entry->name[0], p->p_comm, MAXCOMLEN+1);
2941 entry->priority = p->p_memstat_effectivepriority;
2942 memorystatus_get_task_page_counts(p->task, &entry->pages, &entry->max_pages, &entry->max_pages_lifetime, &entry->purgeable_pages);
2943 entry->state = memorystatus_build_state(p);
2944 entry->user_data = p->p_memstat_userdata;
2945 memcpy(&entry->uuid[0], &p->p_uuid[0], sizeof(p->p_uuid));
2946 entry->fds = p->p_fd->fd_nfiles;
2947
2948 absolutetime_to_microtime(get_task_cpu_time(p->task), &tv_sec, &tv_usec);
2949 entry->cpu_time.tv_sec = tv_sec;
2950 entry->cpu_time.tv_usec = tv_usec;
2951
2952 return TRUE;
2953 }
2954
2955 static void
2956 memorystatus_init_snapshot_vmstats(memorystatus_jetsam_snapshot_t *snapshot)
2957 {
2958 kern_return_t kr = KERN_SUCCESS;
2959 mach_msg_type_number_t count = HOST_VM_INFO64_COUNT;
2960 vm_statistics64_data_t vm_stat;
2961
2962 if ((kr = host_statistics64(host_self(), HOST_VM_INFO64, (host_info64_t)&vm_stat, &count) != KERN_SUCCESS)) {
2963 printf("memorystatus_init_jetsam_snapshot_stats: host_statistics64 failed with %d\n", kr);
2964 memset(&snapshot->stats, 0, sizeof(snapshot->stats));
2965 } else {
2966 snapshot->stats.free_pages = vm_stat.free_count;
2967 snapshot->stats.active_pages = vm_stat.active_count;
2968 snapshot->stats.inactive_pages = vm_stat.inactive_count;
2969 snapshot->stats.throttled_pages = vm_stat.throttled_count;
2970 snapshot->stats.purgeable_pages = vm_stat.purgeable_count;
2971 snapshot->stats.wired_pages = vm_stat.wire_count;
2972
2973 snapshot->stats.speculative_pages = vm_stat.speculative_count;
2974 snapshot->stats.filebacked_pages = vm_stat.external_page_count;
2975 snapshot->stats.anonymous_pages = vm_stat.internal_page_count;
2976 snapshot->stats.compressions = vm_stat.compressions;
2977 snapshot->stats.decompressions = vm_stat.decompressions;
2978 snapshot->stats.compressor_pages = vm_stat.compressor_page_count;
2979 snapshot->stats.total_uncompressed_pages_in_compressor = vm_stat.total_uncompressed_pages_in_compressor;
2980 }
2981 }
2982
2983 /*
2984 * Collect vm statistics at boot.
2985 * Called only once (see kern_exec.c)
2986 * Data can be consumed at any time.
2987 */
2988 void
2989 memorystatus_init_at_boot_snapshot() {
2990 memorystatus_init_snapshot_vmstats(&memorystatus_at_boot_snapshot);
2991 memorystatus_at_boot_snapshot.entry_count = 0;
2992 memorystatus_at_boot_snapshot.notification_time = 0; /* updated when consumed */
2993 memorystatus_at_boot_snapshot.snapshot_time = mach_absolute_time();
2994 }
2995
2996 static void
2997 memorystatus_init_jetsam_snapshot_locked(memorystatus_jetsam_snapshot_t *od_snapshot, uint32_t ods_list_count )
2998 {
2999 proc_t p, next_p;
3000 unsigned int b = 0, i = 0;
3001
3002 memorystatus_jetsam_snapshot_t *snapshot = NULL;
3003 memorystatus_jetsam_snapshot_entry_t *snapshot_list = NULL;
3004 unsigned int snapshot_max = 0;
3005
3006 if (od_snapshot) {
3007 /*
3008 * This is an on_demand snapshot
3009 */
3010 snapshot = od_snapshot;
3011 snapshot_list = od_snapshot->entries;
3012 snapshot_max = ods_list_count;
3013 } else {
3014 /*
3015 * This is a jetsam event snapshot
3016 */
3017 snapshot = memorystatus_jetsam_snapshot;
3018 snapshot_list = memorystatus_jetsam_snapshot->entries;
3019 snapshot_max = memorystatus_jetsam_snapshot_max;
3020 }
3021
3022 memorystatus_init_snapshot_vmstats(snapshot);
3023
3024 next_p = memorystatus_get_first_proc_locked(&b, TRUE);
3025 while (next_p) {
3026 p = next_p;
3027 next_p = memorystatus_get_next_proc_locked(&b, p, TRUE);
3028
3029 if (FALSE == memorystatus_init_jetsam_snapshot_entry_locked(p, &snapshot_list[i])) {
3030 continue;
3031 }
3032
3033 MEMORYSTATUS_DEBUG(0, "jetsam snapshot pid %d, uuid = %02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x\n",
3034 p->p_pid,
3035 p->p_uuid[0], p->p_uuid[1], p->p_uuid[2], p->p_uuid[3], p->p_uuid[4], p->p_uuid[5], p->p_uuid[6], p->p_uuid[7],
3036 p->p_uuid[8], p->p_uuid[9], p->p_uuid[10], p->p_uuid[11], p->p_uuid[12], p->p_uuid[13], p->p_uuid[14], p->p_uuid[15]);
3037
3038 if (++i == snapshot_max) {
3039 break;
3040 }
3041 }
3042
3043 snapshot->snapshot_time = mach_absolute_time();
3044 snapshot->entry_count = i;
3045
3046 if (!od_snapshot) {
3047 /* update the system buffer count */
3048 memorystatus_jetsam_snapshot_count = i;
3049 }
3050 }
3051
3052 #if DEVELOPMENT || DEBUG
3053
3054 static int
3055 memorystatus_cmd_set_panic_bits(user_addr_t buffer, uint32_t buffer_size) {
3056 int ret;
3057 memorystatus_jetsam_panic_options_t debug;
3058
3059 if (buffer_size != sizeof(memorystatus_jetsam_panic_options_t)) {
3060 return EINVAL;
3061 }
3062
3063 ret = copyin(buffer, &debug, buffer_size);
3064 if (ret) {
3065 return ret;
3066 }
3067
3068 /* Panic bits match kMemorystatusKilled* enum */
3069 memorystatus_jetsam_panic_debug = (memorystatus_jetsam_panic_debug & ~debug.mask) | (debug.data & debug.mask);
3070
3071 /* Copyout new value */
3072 debug.data = memorystatus_jetsam_panic_debug;
3073 ret = copyout(&debug, buffer, sizeof(memorystatus_jetsam_panic_options_t));
3074
3075 return ret;
3076 }
3077
3078 /*
3079 * Triggers a sort_order on a specified jetsam priority band.
3080 * This is for testing only, used to force a path through the sort
3081 * function.
3082 */
3083 static int
3084 memorystatus_cmd_test_jetsam_sort(int priority, int sort_order) {
3085
3086 int error = 0;
3087
3088 unsigned int bucket_index = 0;
3089
3090 if (priority == -1) {
3091 /* Use as shorthand for default priority */
3092 bucket_index = JETSAM_PRIORITY_DEFAULT;
3093 } else {
3094 bucket_index = (unsigned int)priority;
3095 }
3096
3097 error = memorystatus_sort_bucket(bucket_index, sort_order);
3098
3099 return (error);
3100 }
3101
3102 #endif
3103
3104 /*
3105 * Jetsam a specific process.
3106 */
3107 static boolean_t
3108 memorystatus_kill_specific_process(pid_t victim_pid, uint32_t cause) {
3109 boolean_t killed;
3110 proc_t p;
3111
3112 /* TODO - add a victim queue and push this into the main jetsam thread */
3113 p = proc_find(victim_pid);
3114 if (!p) {
3115 return FALSE;
3116 }
3117
3118 proc_list_lock();
3119
3120 if (memorystatus_jetsam_snapshot_count == 0) {
3121 memorystatus_init_jetsam_snapshot_locked(NULL,0);
3122 }
3123
3124 memorystatus_update_jetsam_snapshot_entry_locked(p, cause);
3125 proc_list_unlock();
3126
3127 printf("memorystatus: specifically killing pid %d [%s] (%s %d) - memorystatus_available_pages: %d\n",
3128 victim_pid, (p->p_comm ? p->p_comm : "(unknown)"),
3129 jetsam_kill_cause_name[cause], p->p_memstat_effectivepriority, memorystatus_available_pages);
3130
3131
3132 killed = memorystatus_do_kill(p, cause);
3133 proc_rele(p);
3134
3135 return killed;
3136 }
3137
3138 /*
3139 * Jetsam the first process in the queue.
3140 */
3141 static boolean_t
3142 memorystatus_kill_top_process(boolean_t any, boolean_t sort_flag, uint32_t cause, int32_t *priority, uint32_t *errors)
3143 {
3144 pid_t aPid;
3145 proc_t p = PROC_NULL, next_p = PROC_NULL;
3146 boolean_t new_snapshot = FALSE, killed = FALSE;
3147 int kill_count = 0;
3148 unsigned int i = 0;
3149 uint32_t aPid_ep;
3150
3151 #ifndef CONFIG_FREEZE
3152 #pragma unused(any)
3153 #endif
3154
3155 KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_JETSAM) | DBG_FUNC_START,
3156 memorystatus_available_pages, 0, 0, 0, 0);
3157
3158
3159 if (sort_flag == TRUE) {
3160 (void)memorystatus_sort_bucket(JETSAM_PRIORITY_FOREGROUND, JETSAM_SORT_DEFAULT);
3161 }
3162
3163 proc_list_lock();
3164
3165 next_p = memorystatus_get_first_proc_locked(&i, TRUE);
3166 while (next_p) {
3167 #if DEVELOPMENT || DEBUG
3168 int activeProcess;
3169 int procSuspendedForDiagnosis;
3170 #endif /* DEVELOPMENT || DEBUG */
3171
3172 p = next_p;
3173 next_p = memorystatus_get_next_proc_locked(&i, p, TRUE);
3174
3175 #if DEVELOPMENT || DEBUG
3176 activeProcess = p->p_memstat_state & P_MEMSTAT_FOREGROUND;
3177 procSuspendedForDiagnosis = p->p_memstat_state & P_MEMSTAT_DIAG_SUSPENDED;
3178 #endif /* DEVELOPMENT || DEBUG */
3179
3180 aPid = p->p_pid;
3181 aPid_ep = p->p_memstat_effectivepriority;
3182
3183 if (p->p_memstat_state & (P_MEMSTAT_ERROR | P_MEMSTAT_TERMINATED)) {
3184 continue;
3185 }
3186
3187 #if DEVELOPMENT || DEBUG
3188 if ((memorystatus_jetsam_policy & kPolicyDiagnoseActive) && procSuspendedForDiagnosis) {
3189 printf("jetsam: continuing after ignoring proc suspended already for diagnosis - %d\n", aPid);
3190 continue;
3191 }
3192 #endif /* DEVELOPMENT || DEBUG */
3193
3194 if (cause == kMemorystatusKilledVnodes)
3195 {
3196 /*
3197 * If the system runs out of vnodes, we systematically jetsam
3198 * processes in hopes of stumbling onto a vnode gain that helps
3199 * the system recover. The process that happens to trigger
3200 * this path has no known relationship to the vnode consumption.
3201 * We attempt to safeguard that process e.g: do not jetsam it.
3202 */
3203
3204 if (p == current_proc()) {
3205 /* do not jetsam the current process */
3206 continue;
3207 }
3208 }
3209
3210 #if CONFIG_FREEZE
3211 boolean_t skip;
3212 boolean_t reclaim_proc = !(p->p_memstat_state & (P_MEMSTAT_LOCKED | P_MEMSTAT_NORECLAIM));
3213 if (any || reclaim_proc) {
3214 skip = FALSE;
3215 } else {
3216 skip = TRUE;
3217 }
3218
3219 if (skip) {
3220 continue;
3221 } else
3222 #endif
3223 {
3224 /*
3225 * Capture a snapshot if none exists and:
3226 * - priority was not requested (this is something other than an ambient kill)
3227 * - the priority was requested *and* the targeted process is not at idle priority
3228 */
3229 if ((memorystatus_jetsam_snapshot_count == 0) &&
3230 (memorystatus_idle_snapshot || ((!priority) || (priority && (*priority != JETSAM_PRIORITY_IDLE))))) {
3231 memorystatus_init_jetsam_snapshot_locked(NULL,0);
3232 new_snapshot = TRUE;
3233 }
3234
3235 /*
3236 * Mark as terminated so that if exit1() indicates success, but the process (for example)
3237 * is blocked in task_exception_notify(), it'll be skipped if encountered again - see
3238 * <rdar://problem/13553476>. This is cheaper than examining P_LEXIT, which requires the
3239 * acquisition of the proc lock.
3240 */
3241 p->p_memstat_state |= P_MEMSTAT_TERMINATED;
3242
3243 #if DEVELOPMENT || DEBUG
3244 if ((memorystatus_jetsam_policy & kPolicyDiagnoseActive) && activeProcess) {
3245 MEMORYSTATUS_DEBUG(1, "jetsam: suspending pid %d [%s] (active) for diagnosis - memory_status_level: %d\n",
3246 aPid, (p->p_comm ? p->p_comm: "(unknown)"), memorystatus_level);
3247 memorystatus_update_jetsam_snapshot_entry_locked(p, kMemorystatusKilledDiagnostic);
3248 p->p_memstat_state |= P_MEMSTAT_DIAG_SUSPENDED;
3249 if (memorystatus_jetsam_policy & kPolicyDiagnoseFirst) {
3250 jetsam_diagnostic_suspended_one_active_proc = 1;
3251 printf("jetsam: returning after suspending first active proc - %d\n", aPid);
3252 }
3253
3254 p = proc_ref_locked(p);
3255 proc_list_unlock();
3256 if (p) {
3257 task_suspend(p->task);
3258 if (priority) {
3259 *priority = aPid_ep;
3260 }
3261 proc_rele(p);
3262 killed = TRUE;
3263 }
3264
3265 goto exit;
3266 } else
3267 #endif /* DEVELOPMENT || DEBUG */
3268 {
3269 /* Shift queue, update stats */
3270 memorystatus_update_jetsam_snapshot_entry_locked(p, cause);
3271
3272 if (proc_ref_locked(p) == p) {
3273 proc_list_unlock();
3274 printf("memorystatus: %s %d [%s] (%s %d) - memorystatus_available_pages: %d\n",
3275 ((aPid_ep == JETSAM_PRIORITY_IDLE) ?
3276 "idle exiting pid" : "jetsam killing pid"),
3277 aPid, (p->p_comm ? p->p_comm : "(unknown)"),
3278 jetsam_kill_cause_name[cause], aPid_ep, memorystatus_available_pages);
3279
3280 killed = memorystatus_do_kill(p, cause);
3281
3282 /* Success? */
3283 if (killed) {
3284 if (priority) {
3285 *priority = aPid_ep;
3286 }
3287 proc_rele(p);
3288 kill_count++;
3289 goto exit;
3290 }
3291
3292 /*
3293 * Failure - first unwind the state,
3294 * then fall through to restart the search.
3295 */
3296 proc_list_lock();
3297 proc_rele_locked(p);
3298 p->p_memstat_state &= ~P_MEMSTAT_TERMINATED;
3299 p->p_memstat_state |= P_MEMSTAT_ERROR;
3300 *errors += 1;
3301 }
3302
3303 /*
3304 * Failure - restart the search.
3305 *
3306 * We might have raced with "p" exiting on another core, resulting in no
3307 * ref on "p". Or, we may have failed to kill "p".
3308 *
3309 * Either way, we fall thru to here, leaving the proc in the
3310 * P_MEMSTAT_TERMINATED state.
3311 *
3312 * And, we hold the the proc_list_lock at this point.
3313 */
3314
3315 i = 0;
3316 next_p = memorystatus_get_first_proc_locked(&i, TRUE);
3317 }
3318 }
3319 }
3320
3321 proc_list_unlock();
3322
3323 exit:
3324 /* Clear snapshot if freshly captured and no target was found */
3325 if (new_snapshot && !killed) {
3326 memorystatus_jetsam_snapshot->entry_count = memorystatus_jetsam_snapshot_count = 0;
3327 }
3328
3329 KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_JETSAM) | DBG_FUNC_END,
3330 memorystatus_available_pages, killed ? aPid : 0, kill_count, 0, 0);
3331
3332 return killed;
3333 }
3334
3335 /*
3336 * Jetsam aggressively
3337 */
3338 static boolean_t
3339 memorystatus_kill_top_process_aggressive(boolean_t any, uint32_t cause, int aggr_count, int32_t priority_max,
3340 uint32_t *errors)
3341 {
3342 pid_t aPid;
3343 proc_t p = PROC_NULL, next_p = PROC_NULL;
3344 boolean_t new_snapshot = FALSE, killed = FALSE;
3345 int kill_count = 0;
3346 unsigned int i = 0;
3347 int32_t aPid_ep = 0;
3348 unsigned int memorystatus_level_snapshot = 0;
3349
3350 #pragma unused(any)
3351
3352 KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_JETSAM) | DBG_FUNC_START,
3353 memorystatus_available_pages, priority_max, 0, 0, 0);
3354
3355 memorystatus_sort_bucket(JETSAM_PRIORITY_FOREGROUND, JETSAM_SORT_DEFAULT);
3356
3357 proc_list_lock();
3358
3359 next_p = memorystatus_get_first_proc_locked(&i, TRUE);
3360 while (next_p) {
3361 #if DEVELOPMENT || DEBUG
3362 int activeProcess;
3363 int procSuspendedForDiagnosis;
3364 #endif /* DEVELOPMENT || DEBUG */
3365
3366 if ((unsigned int)(next_p->p_memstat_effectivepriority) != i) {
3367
3368 /*
3369 * We have raced with next_p running on another core, as it has
3370 * moved to a different jetsam priority band. This means we have
3371 * lost our place in line while traversing the jetsam list. We
3372 * attempt to recover by rewinding to the beginning of the band
3373 * we were already traversing. By doing this, we do not guarantee
3374 * that no process escapes this aggressive march, but we can make
3375 * skipping an entire range of processes less likely. (PR-21069019)
3376 */
3377
3378 MEMORYSTATUS_DEBUG(1, "memorystatus: aggressive%d: rewinding %s moved from band %d --> %d\n",
3379 aggr_count, next_p->p_comm, i, next_p->p_memstat_effectivepriority);
3380
3381 next_p = memorystatus_get_first_proc_locked(&i, TRUE);
3382 continue;
3383 }
3384
3385 p = next_p;
3386 next_p = memorystatus_get_next_proc_locked(&i, p, TRUE);
3387
3388 if (p->p_memstat_effectivepriority > priority_max) {
3389 /*
3390 * Bail out of this killing spree if we have
3391 * reached beyond the priority_max jetsam band.
3392 * That is, we kill up to and through the
3393 * priority_max jetsam band.
3394 */
3395 proc_list_unlock();
3396 goto exit;
3397 }
3398
3399 #if DEVELOPMENT || DEBUG
3400 activeProcess = p->p_memstat_state & P_MEMSTAT_FOREGROUND;
3401 procSuspendedForDiagnosis = p->p_memstat_state & P_MEMSTAT_DIAG_SUSPENDED;
3402 #endif /* DEVELOPMENT || DEBUG */
3403
3404 aPid = p->p_pid;
3405 aPid_ep = p->p_memstat_effectivepriority;
3406
3407 if (p->p_memstat_state & (P_MEMSTAT_ERROR | P_MEMSTAT_TERMINATED)) {
3408 continue;
3409 }
3410
3411 #if DEVELOPMENT || DEBUG
3412 if ((memorystatus_jetsam_policy & kPolicyDiagnoseActive) && procSuspendedForDiagnosis) {
3413 printf("jetsam: continuing after ignoring proc suspended already for diagnosis - %d\n", aPid);
3414 continue;
3415 }
3416 #endif /* DEVELOPMENT || DEBUG */
3417
3418 /*
3419 * Capture a snapshot if none exists.
3420 */
3421 if (memorystatus_jetsam_snapshot_count == 0) {
3422 memorystatus_init_jetsam_snapshot_locked(NULL,0);
3423 new_snapshot = TRUE;
3424 }
3425
3426 /*
3427 * Mark as terminated so that if exit1() indicates success, but the process (for example)
3428 * is blocked in task_exception_notify(), it'll be skipped if encountered again - see
3429 * <rdar://problem/13553476>. This is cheaper than examining P_LEXIT, which requires the
3430 * acquisition of the proc lock.
3431 */
3432 p->p_memstat_state |= P_MEMSTAT_TERMINATED;
3433
3434 /* Shift queue, update stats */
3435 memorystatus_update_jetsam_snapshot_entry_locked(p, cause);
3436
3437 /*
3438 * In order to kill the target process, we will drop the proc_list_lock.
3439 * To guaranteee that p and next_p don't disappear out from under the lock,
3440 * we must take a ref on both.
3441 * If we cannot get a reference, then it's likely we've raced with
3442 * that process exiting on another core.
3443 */
3444 if (proc_ref_locked(p) == p) {
3445 if (next_p) {
3446 while (next_p && (proc_ref_locked(next_p) != next_p)) {
3447 proc_t temp_p;
3448
3449 /*
3450 * We must have raced with next_p exiting on another core.
3451 * Recover by getting the next eligible process in the band.
3452 */
3453
3454 MEMORYSTATUS_DEBUG(1, "memorystatus: aggressive%d: skipping %d [%s] (exiting?)\n",
3455 aggr_count, next_p->p_pid, (next_p->p_comm ? next_p->p_comm : "(unknown)"));
3456
3457 temp_p = next_p;
3458 next_p = memorystatus_get_next_proc_locked(&i, temp_p, TRUE);
3459 }
3460 }
3461 proc_list_unlock();
3462
3463 printf("memorystatus: aggressive%d: %s %d [%s] (%s %d) - memorystatus_available_pages: %d\n",
3464 aggr_count,
3465 ((aPid_ep == JETSAM_PRIORITY_IDLE) ? "idle exiting pid" : "jetsam killing pid"),
3466 aPid, (p->p_comm ? p->p_comm : "(unknown)"),
3467 jetsam_kill_cause_name[cause], aPid_ep, memorystatus_available_pages);
3468
3469 memorystatus_level_snapshot = memorystatus_level;
3470
3471 killed = memorystatus_do_kill(p, cause);
3472
3473 /* Success? */
3474 if (killed) {
3475 proc_rele(p);
3476 kill_count++;
3477 p = NULL;
3478 killed = FALSE;
3479
3480 /*
3481 * Continue the killing spree.
3482 */
3483 proc_list_lock();
3484 if (next_p) {
3485 proc_rele_locked(next_p);
3486 }
3487
3488 if (aPid_ep == JETSAM_PRIORITY_FOREGROUND && memorystatus_aggressive_jetsam_lenient == TRUE) {
3489 if (memorystatus_level > memorystatus_level_snapshot && ((memorystatus_level - memorystatus_level_snapshot) >= AGGRESSIVE_JETSAM_LENIENT_MODE_THRESHOLD)) {
3490 #if DEVELOPMENT || DEBUG
3491 printf("Disabling Lenient mode after one-time deployment.\n");
3492 #endif /* DEVELOPMENT || DEBUG */
3493 memorystatus_aggressive_jetsam_lenient = FALSE;
3494 break;
3495 }
3496 }
3497
3498 continue;
3499 }
3500
3501 /*
3502 * Failure - first unwind the state,
3503 * then fall through to restart the search.
3504 */
3505 proc_list_lock();
3506 proc_rele_locked(p);
3507 if (next_p) {
3508 proc_rele_locked(next_p);
3509 }
3510 p->p_memstat_state &= ~P_MEMSTAT_TERMINATED;
3511 p->p_memstat_state |= P_MEMSTAT_ERROR;
3512 *errors += 1;
3513 p = NULL;
3514 }
3515
3516 /*
3517 * Failure - restart the search at the beginning of
3518 * the band we were already traversing.
3519 *
3520 * We might have raced with "p" exiting on another core, resulting in no
3521 * ref on "p". Or, we may have failed to kill "p".
3522 *
3523 * Either way, we fall thru to here, leaving the proc in the
3524 * P_MEMSTAT_TERMINATED or P_MEMSTAT_ERROR state.
3525 *
3526 * And, we hold the the proc_list_lock at this point.
3527 */
3528
3529 next_p = memorystatus_get_first_proc_locked(&i, TRUE);
3530 }
3531
3532 proc_list_unlock();
3533
3534 exit:
3535 /* Clear snapshot if freshly captured and no target was found */
3536 if (new_snapshot && (kill_count == 0)) {
3537 memorystatus_jetsam_snapshot->entry_count = memorystatus_jetsam_snapshot_count = 0;
3538 }
3539
3540 KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_JETSAM) | DBG_FUNC_END,
3541 memorystatus_available_pages, killed ? aPid : 0, kill_count, 0, 0);
3542
3543 if (kill_count > 0) {
3544 return(TRUE);
3545 }
3546 else {
3547 return(FALSE);
3548 }
3549 }
3550
3551 #if LEGACY_HIWATER
3552
3553 static boolean_t
3554 memorystatus_kill_hiwat_proc(uint32_t *errors)
3555 {
3556 pid_t aPid = 0;
3557 proc_t p = PROC_NULL, next_p = PROC_NULL;
3558 boolean_t new_snapshot = FALSE, killed = FALSE;
3559 int kill_count = 0;
3560 unsigned int i = 0;
3561 uint32_t aPid_ep;
3562
3563 KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_JETSAM_HIWAT) | DBG_FUNC_START,
3564 memorystatus_available_pages, 0, 0, 0, 0);
3565
3566 proc_list_lock();
3567
3568 next_p = memorystatus_get_first_proc_locked(&i, TRUE);
3569 while (next_p) {
3570 uint32_t footprint;
3571 boolean_t skip;
3572
3573 p = next_p;
3574 next_p = memorystatus_get_next_proc_locked(&i, p, TRUE);
3575
3576 aPid = p->p_pid;
3577 aPid_ep = p->p_memstat_effectivepriority;
3578
3579 if (p->p_memstat_state & (P_MEMSTAT_ERROR | P_MEMSTAT_TERMINATED)) {
3580 continue;
3581 }
3582
3583 /* skip if no limit set */
3584 if (p->p_memstat_memlimit <= 0) {
3585 continue;
3586 }
3587
3588 #if 0
3589 /*
3590 * No need to consider P_MEMSTAT_MEMLIMIT_BACKGROUND anymore.
3591 * Background limits are described via the inactive limit slots.
3592 * Their fatal/non-fatal setting will drive whether or not to be
3593 * considered in this kill path.
3594 */
3595
3596 /* skip if a currently inapplicable limit is encountered */
3597 if ((p->p_memstat_state & P_MEMSTAT_MEMLIMIT_BACKGROUND) && (p->p_memstat_effectivepriority >= JETSAM_PRIORITY_FOREGROUND)) {
3598 continue;
3599 }
3600 #endif
3601
3602 footprint = (uint32_t)(get_task_phys_footprint(p->task) / (1024 * 1024));
3603 skip = (((int32_t)footprint) <= p->p_memstat_memlimit);
3604
3605 #if DEVELOPMENT || DEBUG
3606 if (!skip && (memorystatus_jetsam_policy & kPolicyDiagnoseActive)) {
3607 if (p->p_memstat_state & P_MEMSTAT_DIAG_SUSPENDED) {
3608 continue;
3609 }
3610 }
3611 #endif /* DEVELOPMENT || DEBUG */
3612
3613 #if CONFIG_FREEZE
3614 if (!skip) {
3615 if (p->p_memstat_state & P_MEMSTAT_LOCKED) {
3616 skip = TRUE;
3617 } else {
3618 skip = FALSE;
3619 }
3620 }
3621 #endif
3622
3623 if (skip) {
3624 continue;
3625 } else {
3626 MEMORYSTATUS_DEBUG(1, "jetsam: %s pid %d [%s] - %d Mb > 1 (%d Mb)\n",
3627 (memorystatus_jetsam_policy & kPolicyDiagnoseActive) ? "suspending": "killing", aPid, p->p_comm, footprint, p->p_memstat_memlimit);
3628
3629 if (memorystatus_jetsam_snapshot_count == 0) {
3630 memorystatus_init_jetsam_snapshot_locked(NULL,0);
3631 new_snapshot = TRUE;
3632 }
3633
3634 p->p_memstat_state |= P_MEMSTAT_TERMINATED;
3635
3636 #if DEVELOPMENT || DEBUG
3637 if (memorystatus_jetsam_policy & kPolicyDiagnoseActive) {
3638 MEMORYSTATUS_DEBUG(1, "jetsam: pid %d suspended for diagnosis - memorystatus_available_pages: %d\n", aPid, memorystatus_available_pages);
3639 memorystatus_update_jetsam_snapshot_entry_locked(p, kMemorystatusKilledDiagnostic);
3640 p->p_memstat_state |= P_MEMSTAT_DIAG_SUSPENDED;
3641
3642 p = proc_ref_locked(p);
3643 proc_list_unlock();
3644 if (p) {
3645 task_suspend(p->task);
3646 proc_rele(p);
3647 killed = TRUE;
3648 }
3649
3650 goto exit;
3651 } else
3652 #endif /* DEVELOPMENT || DEBUG */
3653 {
3654 memorystatus_update_jetsam_snapshot_entry_locked(p, kMemorystatusKilledHiwat);
3655
3656 if (proc_ref_locked(p) == p) {
3657 proc_list_unlock();
3658
3659 printf("memorystatus: jetsam killing pid %d [%s] (highwater %d) - memorystatus_available_pages: %d\n",
3660 aPid, (p->p_comm ? p->p_comm : "(unknown)"), aPid_ep, memorystatus_available_pages);
3661
3662 killed = memorystatus_do_kill(p, kMemorystatusKilledHiwat);
3663
3664 /* Success? */
3665 if (killed) {
3666 proc_rele(p);
3667 kill_count++;
3668 goto exit;
3669 }
3670
3671 /*
3672 * Failure - first unwind the state,
3673 * then fall through to restart the search.
3674 */
3675 proc_list_lock();
3676 proc_rele_locked(p);
3677 p->p_memstat_state &= ~P_MEMSTAT_TERMINATED;
3678 p->p_memstat_state |= P_MEMSTAT_ERROR;
3679 *errors += 1;
3680 }
3681
3682 /*
3683 * Failure - restart the search.
3684 *
3685 * We might have raced with "p" exiting on another core, resulting in no
3686 * ref on "p". Or, we may have failed to kill "p".
3687 *
3688 * Either way, we fall thru to here, leaving the proc in the
3689 * P_MEMSTAT_TERMINATED state.
3690 *
3691 * And, we hold the the proc_list_lock at this point.
3692 */
3693
3694 i = 0;
3695 next_p = memorystatus_get_first_proc_locked(&i, TRUE);
3696 }
3697 }
3698 }
3699
3700 proc_list_unlock();
3701
3702 exit:
3703 /* Clear snapshot if freshly captured and no target was found */
3704 if (new_snapshot && !killed) {
3705 memorystatus_jetsam_snapshot->entry_count = memorystatus_jetsam_snapshot_count = 0;
3706 }
3707
3708 KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_JETSAM_HIWAT) | DBG_FUNC_END,
3709 memorystatus_available_pages, killed ? aPid : 0, kill_count, 0, 0);
3710
3711 return killed;
3712 }
3713
3714 #endif /* LEGACY_HIWATER */
3715
3716 static boolean_t
3717 memorystatus_kill_process_async(pid_t victim_pid, uint32_t cause) {
3718 /* TODO: allow a general async path */
3719 if ((victim_pid != -1) || (cause != kMemorystatusKilledVMPageShortage && cause != kMemorystatusKilledVMThrashing &&
3720 cause != kMemorystatusKilledFCThrashing)) {
3721 return FALSE;
3722 }
3723
3724 kill_under_pressure_cause = cause;
3725 memorystatus_thread_wake();
3726 return TRUE;
3727 }
3728
3729 static boolean_t
3730 memorystatus_kill_process_sync(pid_t victim_pid, uint32_t cause) {
3731 boolean_t res;
3732 uint32_t errors = 0;
3733
3734 if (victim_pid == -1) {
3735 /* No pid, so kill first process */
3736 res = memorystatus_kill_top_process(TRUE, TRUE, cause, NULL, &errors);
3737 } else {
3738 res = memorystatus_kill_specific_process(victim_pid, cause);
3739 }
3740
3741 if (errors) {
3742 memorystatus_clear_errors();
3743 }
3744
3745 if (res == TRUE) {
3746 /* Fire off snapshot notification */
3747 size_t snapshot_size = sizeof(memorystatus_jetsam_snapshot_t) +
3748 sizeof(memorystatus_jetsam_snapshot_entry_t) * memorystatus_jetsam_snapshot_count;
3749 uint64_t timestamp_now = mach_absolute_time();
3750 memorystatus_jetsam_snapshot->notification_time = timestamp_now;
3751 if (memorystatus_jetsam_snapshot_last_timestamp == 0 ||
3752 timestamp_now > memorystatus_jetsam_snapshot_last_timestamp + memorystatus_jetsam_snapshot_timeout) {
3753 int ret = memorystatus_send_note(kMemorystatusSnapshotNote, &snapshot_size, sizeof(snapshot_size));
3754 if (!ret) {
3755 proc_list_lock();
3756 memorystatus_jetsam_snapshot_last_timestamp = timestamp_now;
3757 proc_list_unlock();
3758 }
3759 }
3760 }
3761
3762 return res;
3763 }
3764
3765 boolean_t
3766 memorystatus_kill_on_VM_page_shortage(boolean_t async) {
3767 if (async) {
3768 return memorystatus_kill_process_async(-1, kMemorystatusKilledVMPageShortage);
3769 } else {
3770 return memorystatus_kill_process_sync(-1, kMemorystatusKilledVMPageShortage);
3771 }
3772 }
3773
3774 boolean_t
3775 memorystatus_kill_on_VM_thrashing(boolean_t async) {
3776 if (async) {
3777 return memorystatus_kill_process_async(-1, kMemorystatusKilledVMThrashing);
3778 } else {
3779 return memorystatus_kill_process_sync(-1, kMemorystatusKilledVMThrashing);
3780 }
3781 }
3782
3783 boolean_t
3784 memorystatus_kill_on_FC_thrashing(boolean_t async) {
3785 if (async) {
3786 return memorystatus_kill_process_async(-1, kMemorystatusKilledFCThrashing);
3787 } else {
3788 return memorystatus_kill_process_sync(-1, kMemorystatusKilledFCThrashing);
3789 }
3790 }
3791
3792 boolean_t
3793 memorystatus_kill_on_vnode_limit(void) {
3794 return memorystatus_kill_process_sync(-1, kMemorystatusKilledVnodes);
3795 }
3796
3797 #endif /* CONFIG_JETSAM */
3798
3799 #if CONFIG_FREEZE
3800
3801 __private_extern__ void
3802 memorystatus_freeze_init(void)
3803 {
3804 kern_return_t result;
3805 thread_t thread;
3806
3807 freezer_lck_grp_attr = lck_grp_attr_alloc_init();
3808 freezer_lck_grp = lck_grp_alloc_init("freezer", freezer_lck_grp_attr);
3809
3810 lck_mtx_init(&freezer_mutex, freezer_lck_grp, NULL);
3811
3812 result = kernel_thread_start(memorystatus_freeze_thread, NULL, &thread);
3813 if (result == KERN_SUCCESS) {
3814 thread_deallocate(thread);
3815 } else {
3816 panic("Could not create memorystatus_freeze_thread");
3817 }
3818 }
3819
3820 /*
3821 * Synchronously freeze the passed proc. Called with a reference to the proc held.
3822 *
3823 * Returns EINVAL or the value returned by task_freeze().
3824 */
3825 int
3826 memorystatus_freeze_process_sync(proc_t p)
3827 {
3828 int ret = EINVAL;
3829 pid_t aPid = 0;
3830 boolean_t memorystatus_freeze_swap_low = FALSE;
3831
3832 KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_FREEZE) | DBG_FUNC_START,
3833 memorystatus_available_pages, 0, 0, 0, 0);
3834
3835 lck_mtx_lock(&freezer_mutex);
3836
3837 if (p == NULL) {
3838 goto exit;
3839 }
3840
3841 if (memorystatus_freeze_enabled == FALSE) {
3842 goto exit;
3843 }
3844
3845 if (!memorystatus_can_freeze(&memorystatus_freeze_swap_low)) {
3846 goto exit;
3847 }
3848
3849 if (memorystatus_freeze_update_throttle()) {
3850 printf("memorystatus_freeze_process_sync: in throttle, ignorning freeze\n");
3851 memorystatus_freeze_throttle_count++;
3852 goto exit;
3853 }
3854
3855 proc_list_lock();
3856
3857 if (p != NULL) {
3858 uint32_t purgeable, wired, clean, dirty, state;
3859 uint32_t max_pages, pages, i;
3860 boolean_t shared;
3861
3862 aPid = p->p_pid;
3863 state = p->p_memstat_state;
3864
3865 /* Ensure the process is eligible for freezing */
3866 if ((state & (P_MEMSTAT_TERMINATED | P_MEMSTAT_LOCKED | P_MEMSTAT_FROZEN)) || !(state & P_MEMSTAT_SUSPENDED)) {
3867 proc_list_unlock();
3868 goto exit;
3869 }
3870
3871 /* Only freeze processes meeting our minimum resident page criteria */
3872 memorystatus_get_task_page_counts(p->task, &pages, NULL, NULL, NULL);
3873 if (pages < memorystatus_freeze_pages_min) {
3874 proc_list_unlock();
3875 goto exit;
3876 }
3877
3878 if (DEFAULT_FREEZER_IS_ACTIVE || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_SWAPBACKED) {
3879
3880 unsigned int avail_swap_space = 0; /* in pages. */
3881
3882 if (DEFAULT_FREEZER_IS_ACTIVE) {
3883 /*
3884 * Freezer backed by default pager and swap file(s).
3885 */
3886 avail_swap_space = default_pager_swap_pages_free();
3887 } else {
3888 /*
3889 * Freezer backed by the compressor and swap file(s)
3890 * while will hold compressed data.
3891 */
3892 avail_swap_space = vm_swap_get_free_space() / PAGE_SIZE_64;
3893 }
3894
3895 max_pages = MIN(avail_swap_space, memorystatus_freeze_pages_max);
3896
3897 if (max_pages < memorystatus_freeze_pages_min) {
3898 proc_list_unlock();
3899 goto exit;
3900 }
3901 } else {
3902 /*
3903 * We only have the compressor without any swap.
3904 */
3905 max_pages = UINT32_MAX - 1;
3906 }
3907
3908 /* Mark as locked temporarily to avoid kill */
3909 p->p_memstat_state |= P_MEMSTAT_LOCKED;
3910 proc_list_unlock();
3911
3912 ret = task_freeze(p->task, &purgeable, &wired, &clean, &dirty, max_pages, &shared, FALSE);
3913
3914 MEMORYSTATUS_DEBUG(1, "memorystatus_freeze_process_sync: task_freeze %s for pid %d [%s] - "
3915 "memorystatus_pages: %d, purgeable: %d, wired: %d, clean: %d, dirty: %d, shared %d, free swap: %d\n",
3916 (ret == KERN_SUCCESS) ? "SUCCEEDED" : "FAILED", aPid, (p->p_comm ? p->p_comm : "(unknown)"),
3917 memorystatus_available_pages, purgeable, wired, clean, dirty, shared, default_pager_swap_pages_free());
3918
3919 proc_list_lock();
3920 p->p_memstat_state &= ~P_MEMSTAT_LOCKED;
3921
3922 if (ret == KERN_SUCCESS) {
3923 memorystatus_freeze_entry_t data = { aPid, TRUE, dirty };
3924
3925 memorystatus_frozen_count++;
3926
3927 p->p_memstat_state |= (P_MEMSTAT_FROZEN | (shared ? 0: P_MEMSTAT_NORECLAIM));
3928
3929 if (DEFAULT_FREEZER_IS_ACTIVE || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_SWAPBACKED) {
3930 /* Update stats */
3931 for (i = 0; i < sizeof(throttle_intervals) / sizeof(struct throttle_interval_t); i++) {
3932 throttle_intervals[i].pageouts += dirty;
3933 }
3934 }
3935
3936 memorystatus_freeze_pageouts += dirty;
3937 memorystatus_freeze_count++;
3938
3939 proc_list_unlock();
3940
3941 memorystatus_send_note(kMemorystatusFreezeNote, &data, sizeof(data));
3942 } else {
3943 proc_list_unlock();
3944 }
3945 }
3946
3947 exit:
3948 lck_mtx_unlock(&freezer_mutex);
3949 KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_FREEZE) | DBG_FUNC_END,
3950 memorystatus_available_pages, aPid, 0, 0, 0);
3951
3952 return ret;
3953 }
3954
3955 static int
3956 memorystatus_freeze_top_process(boolean_t *memorystatus_freeze_swap_low)
3957 {
3958 pid_t aPid = 0;
3959 int ret = -1;
3960 proc_t p = PROC_NULL, next_p = PROC_NULL;
3961 unsigned int i = 0;
3962
3963 KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_FREEZE) | DBG_FUNC_START,
3964 memorystatus_available_pages, 0, 0, 0, 0);
3965
3966 proc_list_lock();
3967
3968 next_p = memorystatus_get_first_proc_locked(&i, TRUE);
3969 while (next_p) {
3970 kern_return_t kr;
3971 uint32_t purgeable, wired, clean, dirty;
3972 boolean_t shared;
3973 uint32_t pages;
3974 uint32_t max_pages = 0;
3975 uint32_t state;
3976
3977 p = next_p;
3978 next_p = memorystatus_get_next_proc_locked(&i, p, TRUE);
3979
3980 aPid = p->p_pid;
3981 state = p->p_memstat_state;
3982
3983 /* Ensure the process is eligible for freezing */
3984 if ((state & (P_MEMSTAT_TERMINATED | P_MEMSTAT_LOCKED | P_MEMSTAT_FROZEN)) || !(state & P_MEMSTAT_SUSPENDED)) {
3985 continue; // with lock held
3986 }
3987
3988 /* Only freeze processes meeting our minimum resident page criteria */
3989 memorystatus_get_task_page_counts(p->task, &pages, NULL, NULL, NULL);
3990 if (pages < memorystatus_freeze_pages_min) {
3991 continue; // with lock held
3992 }
3993
3994 if (DEFAULT_FREEZER_IS_ACTIVE || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_SWAPBACKED) {
3995
3996 /* Ensure there's enough free space to freeze this process. */
3997
3998 unsigned int avail_swap_space = 0; /* in pages. */
3999
4000 if (DEFAULT_FREEZER_IS_ACTIVE) {
4001 /*
4002 * Freezer backed by default pager and swap file(s).
4003 */
4004 avail_swap_space = default_pager_swap_pages_free();
4005 } else {
4006 /*
4007 * Freezer backed by the compressor and swap file(s)
4008 * while will hold compressed data.
4009 */
4010 avail_swap_space = vm_swap_get_free_space() / PAGE_SIZE_64;
4011 }
4012
4013 max_pages = MIN(avail_swap_space, memorystatus_freeze_pages_max);
4014
4015 if (max_pages < memorystatus_freeze_pages_min) {
4016 *memorystatus_freeze_swap_low = TRUE;
4017 proc_list_unlock();
4018 goto exit;
4019 }
4020 } else {
4021 /*
4022 * We only have the compressor pool.
4023 */
4024 max_pages = UINT32_MAX - 1;
4025 }
4026
4027 /* Mark as locked temporarily to avoid kill */
4028 p->p_memstat_state |= P_MEMSTAT_LOCKED;
4029
4030 p = proc_ref_locked(p);
4031 proc_list_unlock();
4032 if (!p) {
4033 goto exit;
4034 }
4035
4036 kr = task_freeze(p->task, &purgeable, &wired, &clean, &dirty, max_pages, &shared, FALSE);
4037
4038 MEMORYSTATUS_DEBUG(1, "memorystatus_freeze_top_process: task_freeze %s for pid %d [%s] - "
4039 "memorystatus_pages: %d, purgeable: %d, wired: %d, clean: %d, dirty: %d, shared %d, free swap: %d\n",
4040 (kr == KERN_SUCCESS) ? "SUCCEEDED" : "FAILED", aPid, (p->p_comm ? p->p_comm : "(unknown)"),
4041 memorystatus_available_pages, purgeable, wired, clean, dirty, shared, default_pager_swap_pages_free());
4042
4043 proc_list_lock();
4044 p->p_memstat_state &= ~P_MEMSTAT_LOCKED;
4045
4046 /* Success? */
4047 if (KERN_SUCCESS == kr) {
4048 memorystatus_freeze_entry_t data = { aPid, TRUE, dirty };
4049
4050 memorystatus_frozen_count++;
4051
4052 p->p_memstat_state |= (P_MEMSTAT_FROZEN | (shared ? 0: P_MEMSTAT_NORECLAIM));
4053
4054 if (DEFAULT_FREEZER_IS_ACTIVE || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_SWAPBACKED) {
4055 /* Update stats */
4056 for (i = 0; i < sizeof(throttle_intervals) / sizeof(struct throttle_interval_t); i++) {
4057 throttle_intervals[i].pageouts += dirty;
4058 }
4059 }
4060
4061 memorystatus_freeze_pageouts += dirty;
4062 memorystatus_freeze_count++;
4063
4064 proc_list_unlock();
4065
4066 memorystatus_send_note(kMemorystatusFreezeNote, &data, sizeof(data));
4067
4068 /* Return KERN_SUCESS */
4069 ret = kr;
4070
4071 } else {
4072 proc_list_unlock();
4073 }
4074
4075 proc_rele(p);
4076 goto exit;
4077 }
4078
4079 proc_list_unlock();
4080
4081 exit:
4082 KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_FREEZE) | DBG_FUNC_END,
4083 memorystatus_available_pages, aPid, 0, 0, 0);
4084
4085 return ret;
4086 }
4087
4088 static inline boolean_t
4089 memorystatus_can_freeze_processes(void)
4090 {
4091 boolean_t ret;
4092
4093 proc_list_lock();
4094
4095 if (memorystatus_suspended_count) {
4096 uint32_t average_resident_pages, estimated_processes;
4097
4098 /* Estimate the number of suspended processes we can fit */
4099 average_resident_pages = memorystatus_suspended_footprint_total / memorystatus_suspended_count;
4100 estimated_processes = memorystatus_suspended_count +
4101 ((memorystatus_available_pages - memorystatus_available_pages_critical) / average_resident_pages);
4102
4103 /* If it's predicted that no freeze will occur, lower the threshold temporarily */
4104 if (estimated_processes <= FREEZE_SUSPENDED_THRESHOLD_DEFAULT) {
4105 memorystatus_freeze_suspended_threshold = FREEZE_SUSPENDED_THRESHOLD_LOW;
4106 } else {
4107 memorystatus_freeze_suspended_threshold = FREEZE_SUSPENDED_THRESHOLD_DEFAULT;
4108 }
4109
4110 MEMORYSTATUS_DEBUG(1, "memorystatus_can_freeze_processes: %d suspended processes, %d average resident pages / process, %d suspended processes estimated\n",
4111 memorystatus_suspended_count, average_resident_pages, estimated_processes);
4112
4113 if ((memorystatus_suspended_count - memorystatus_frozen_count) > memorystatus_freeze_suspended_threshold) {
4114 ret = TRUE;
4115 } else {
4116 ret = FALSE;
4117 }
4118 } else {
4119 ret = FALSE;
4120 }
4121
4122 proc_list_unlock();
4123
4124 return ret;
4125 }
4126
4127 static boolean_t
4128 memorystatus_can_freeze(boolean_t *memorystatus_freeze_swap_low)
4129 {
4130 boolean_t can_freeze = TRUE;
4131
4132 /* Only freeze if we're sufficiently low on memory; this holds off freeze right
4133 after boot, and is generally is a no-op once we've reached steady state. */
4134 if (memorystatus_available_pages > memorystatus_freeze_threshold) {
4135 return FALSE;
4136 }
4137
4138 /* Check minimum suspended process threshold. */
4139 if (!memorystatus_can_freeze_processes()) {
4140 return FALSE;
4141 }
4142
4143 if (COMPRESSED_PAGER_IS_SWAPLESS || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_SWAPLESS) {
4144 /*
4145 * In-core compressor used for freezing WITHOUT on-disk swap support.
4146 */
4147
4148 if (vm_compressor_low_on_space()) {
4149 if (*memorystatus_freeze_swap_low) {
4150 *memorystatus_freeze_swap_low = TRUE;
4151 }
4152
4153 can_freeze = FALSE;
4154
4155 } else {
4156 if (*memorystatus_freeze_swap_low) {
4157 *memorystatus_freeze_swap_low = FALSE;
4158 }
4159
4160 can_freeze = TRUE;
4161 }
4162 } else {
4163 /*
4164 * Freezing WITH on-disk swap support.
4165 */
4166
4167 if (DEFAULT_FREEZER_COMPRESSED_PAGER_IS_SWAPBACKED) {
4168 /*
4169 * In-core compressor fronts the swap.
4170 */
4171 if (vm_swap_low_on_space()) {
4172 if (*memorystatus_freeze_swap_low) {
4173 *memorystatus_freeze_swap_low = TRUE;
4174 }
4175
4176 can_freeze = FALSE;
4177 }
4178
4179 } else if (DEFAULT_FREEZER_IS_ACTIVE) {
4180 /*
4181 * Legacy freeze mode with no compressor support.
4182 */
4183 if (default_pager_swap_pages_free() < memorystatus_freeze_pages_min) {
4184 if (*memorystatus_freeze_swap_low) {
4185 *memorystatus_freeze_swap_low = TRUE;
4186 }
4187
4188 can_freeze = FALSE;
4189 }
4190 } else {
4191 panic("Not a valid freeze configuration.\n");
4192 }
4193 }
4194
4195 return can_freeze;
4196 }
4197
4198 static void
4199 memorystatus_freeze_update_throttle_interval(mach_timespec_t *ts, struct throttle_interval_t *interval)
4200 {
4201 unsigned int freeze_daily_pageouts_max = memorystatus_freeze_daily_mb_max * (1024 * 1024 / PAGE_SIZE);
4202 if (CMP_MACH_TIMESPEC(ts, &interval->ts) >= 0) {
4203 if (!interval->max_pageouts) {
4204 interval->max_pageouts = (interval->burst_multiple * (((uint64_t)interval->mins * freeze_daily_pageouts_max) / (24 * 60)));
4205 } else {
4206 printf("memorystatus_freeze_update_throttle_interval: %d minute throttle timeout, resetting\n", interval->mins);
4207 }
4208 interval->ts.tv_sec = interval->mins * 60;
4209 interval->ts.tv_nsec = 0;
4210 ADD_MACH_TIMESPEC(&interval->ts, ts);
4211 /* Since we update the throttle stats pre-freeze, adjust for overshoot here */
4212 if (interval->pageouts > interval->max_pageouts) {
4213 interval->pageouts -= interval->max_pageouts;
4214 } else {
4215 interval->pageouts = 0;
4216 }
4217 interval->throttle = FALSE;
4218 } else if (!interval->throttle && interval->pageouts >= interval->max_pageouts) {
4219 printf("memorystatus_freeze_update_throttle_interval: %d minute pageout limit exceeded; enabling throttle\n", interval->mins);
4220 interval->throttle = TRUE;
4221 }
4222
4223 MEMORYSTATUS_DEBUG(1, "memorystatus_freeze_update_throttle_interval: throttle updated - %d frozen (%d max) within %dm; %dm remaining; throttle %s\n",
4224 interval->pageouts, interval->max_pageouts, interval->mins, (interval->ts.tv_sec - ts->tv_sec) / 60,
4225 interval->throttle ? "on" : "off");
4226 }
4227
4228 static boolean_t
4229 memorystatus_freeze_update_throttle(void)
4230 {
4231 clock_sec_t sec;
4232 clock_nsec_t nsec;
4233 mach_timespec_t ts;
4234 uint32_t i;
4235 boolean_t throttled = FALSE;
4236
4237 #if DEVELOPMENT || DEBUG
4238 if (!memorystatus_freeze_throttle_enabled)
4239 return FALSE;
4240 #endif
4241
4242 clock_get_system_nanotime(&sec, &nsec);
4243 ts.tv_sec = sec;
4244 ts.tv_nsec = nsec;
4245
4246 /* Check freeze pageouts over multiple intervals and throttle if we've exceeded our budget.
4247 *
4248 * This ensures that periods of inactivity can't be used as 'credit' towards freeze if the device has
4249 * remained dormant for a long period. We do, however, allow increased thresholds for shorter intervals in
4250 * order to allow for bursts of activity.
4251 */
4252 for (i = 0; i < sizeof(throttle_intervals) / sizeof(struct throttle_interval_t); i++) {
4253 memorystatus_freeze_update_throttle_interval(&ts, &throttle_intervals[i]);
4254 if (throttle_intervals[i].throttle == TRUE)
4255 throttled = TRUE;
4256 }
4257
4258 return throttled;
4259 }
4260
4261 static void
4262 memorystatus_freeze_thread(void *param __unused, wait_result_t wr __unused)
4263 {
4264 static boolean_t memorystatus_freeze_swap_low = FALSE;
4265
4266 lck_mtx_lock(&freezer_mutex);
4267 if (memorystatus_freeze_enabled) {
4268 if (memorystatus_can_freeze(&memorystatus_freeze_swap_low)) {
4269 /* Only freeze if we've not exceeded our pageout budgets.*/
4270 if (!memorystatus_freeze_update_throttle()) {
4271 memorystatus_freeze_top_process(&memorystatus_freeze_swap_low);
4272 } else {
4273 printf("memorystatus_freeze_thread: in throttle, ignoring freeze\n");
4274 memorystatus_freeze_throttle_count++; /* Throttled, update stats */
4275 }
4276 }
4277 }
4278 lck_mtx_unlock(&freezer_mutex);
4279
4280 assert_wait((event_t) &memorystatus_freeze_wakeup, THREAD_UNINT);
4281 thread_block((thread_continue_t) memorystatus_freeze_thread);
4282 }
4283
4284 #endif /* CONFIG_FREEZE */
4285
4286 #if VM_PRESSURE_EVENTS
4287
4288 #if CONFIG_MEMORYSTATUS
4289
4290 static int
4291 memorystatus_send_note(int event_code, void *data, size_t data_length) {
4292 int ret;
4293 struct kev_msg ev_msg;
4294
4295 ev_msg.vendor_code = KEV_VENDOR_APPLE;
4296 ev_msg.kev_class = KEV_SYSTEM_CLASS;
4297 ev_msg.kev_subclass = KEV_MEMORYSTATUS_SUBCLASS;
4298
4299 ev_msg.event_code = event_code;
4300
4301 ev_msg.dv[0].data_length = data_length;
4302 ev_msg.dv[0].data_ptr = data;
4303 ev_msg.dv[1].data_length = 0;
4304
4305 ret = kev_post_msg(&ev_msg);
4306 if (ret) {
4307 printf("%s: kev_post_msg() failed, err %d\n", __func__, ret);
4308 }
4309
4310 return ret;
4311 }
4312
4313 boolean_t
4314 memorystatus_warn_process(pid_t pid, boolean_t critical) {
4315
4316 boolean_t ret = FALSE;
4317 boolean_t found_knote = FALSE;
4318 struct knote *kn = NULL;
4319
4320 /*
4321 * See comment in sysctl_memorystatus_vm_pressure_send.
4322 */
4323
4324 memorystatus_klist_lock();
4325
4326 SLIST_FOREACH(kn, &memorystatus_klist, kn_selnext) {
4327 proc_t knote_proc = kn->kn_kq->kq_p;
4328 pid_t knote_pid = knote_proc->p_pid;
4329
4330 if (knote_pid == pid) {
4331 /*
4332 * By setting the "fflags" here, we are forcing
4333 * a process to deal with the case where it's
4334 * bumping up into its memory limits. If we don't
4335 * do this here, we will end up depending on the
4336 * system pressure snapshot evaluation in
4337 * filt_memorystatus().
4338 */
4339
4340 if (critical) {
4341 if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PRESSURE_CRITICAL) {
4342 kn->kn_fflags = NOTE_MEMORYSTATUS_PRESSURE_CRITICAL;
4343 } else if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PRESSURE_WARN) {
4344 kn->kn_fflags = NOTE_MEMORYSTATUS_PRESSURE_WARN;
4345 }
4346 } else {
4347 if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PRESSURE_WARN) {
4348 kn->kn_fflags = NOTE_MEMORYSTATUS_PRESSURE_WARN;
4349 }
4350 }
4351
4352 found_knote = TRUE;
4353 }
4354 }
4355
4356 if (found_knote) {
4357 KNOTE(&memorystatus_klist, 0);
4358 ret = TRUE;
4359 } else {
4360 if (vm_dispatch_pressure_note_to_pid(pid, FALSE) == 0) {
4361 ret = TRUE;
4362 }
4363 }
4364
4365 memorystatus_klist_unlock();
4366
4367 return ret;
4368 }
4369
4370 /*
4371 * Can only be set by the current task on itself.
4372 */
4373 int
4374 memorystatus_low_mem_privileged_listener(uint32_t op_flags)
4375 {
4376 boolean_t set_privilege = FALSE;
4377 /*
4378 * Need an entitlement check here?
4379 */
4380 if (op_flags == MEMORYSTATUS_CMD_PRIVILEGED_LISTENER_ENABLE) {
4381 set_privilege = TRUE;
4382 } else if (op_flags == MEMORYSTATUS_CMD_PRIVILEGED_LISTENER_DISABLE) {
4383 set_privilege = FALSE;
4384 } else {
4385 return EINVAL;
4386 }
4387
4388 return (task_low_mem_privileged_listener(current_task(), set_privilege, NULL));
4389 }
4390
4391 int
4392 memorystatus_send_pressure_note(pid_t pid) {
4393 MEMORYSTATUS_DEBUG(1, "memorystatus_send_pressure_note(): pid %d\n", pid);
4394 return memorystatus_send_note(kMemorystatusPressureNote, &pid, sizeof(pid));
4395 }
4396
4397 void
4398 memorystatus_send_low_swap_note(void) {
4399
4400 struct knote *kn = NULL;
4401
4402 memorystatus_klist_lock();
4403 SLIST_FOREACH(kn, &memorystatus_klist, kn_selnext) {
4404 /* We call is_knote_registered_modify_task_pressure_bits to check if the sfflags for the
4405 * current note contain NOTE_MEMORYSTATUS_LOW_SWAP. Once we find one note in the memorystatus_klist
4406 * that has the NOTE_MEMORYSTATUS_LOW_SWAP flags in its sfflags set, we call KNOTE with
4407 * kMemoryStatusLowSwap as the hint to process and update all knotes on the memorystatus_klist accordingly. */
4408 if (is_knote_registered_modify_task_pressure_bits(kn, NOTE_MEMORYSTATUS_LOW_SWAP, NULL, 0, 0) == TRUE) {
4409 KNOTE(&memorystatus_klist, kMemorystatusLowSwap);
4410 break;
4411 }
4412 }
4413
4414 memorystatus_klist_unlock();
4415 }
4416
4417 boolean_t
4418 memorystatus_bg_pressure_eligible(proc_t p) {
4419 boolean_t eligible = FALSE;
4420
4421 proc_list_lock();
4422
4423 MEMORYSTATUS_DEBUG(1, "memorystatus_bg_pressure_eligible: pid %d, state 0x%x\n", p->p_pid, p->p_memstat_state);
4424
4425 /* Foreground processes have already been dealt with at this point, so just test for eligibility */
4426 if (!(p->p_memstat_state & (P_MEMSTAT_TERMINATED | P_MEMSTAT_LOCKED | P_MEMSTAT_SUSPENDED | P_MEMSTAT_FROZEN))) {
4427 eligible = TRUE;
4428 }
4429
4430 proc_list_unlock();
4431
4432 return eligible;
4433 }
4434
4435 boolean_t
4436 memorystatus_is_foreground_locked(proc_t p) {
4437 return ((p->p_memstat_effectivepriority == JETSAM_PRIORITY_FOREGROUND) ||
4438 (p->p_memstat_effectivepriority == JETSAM_PRIORITY_FOREGROUND_SUPPORT));
4439 }
4440 #endif /* CONFIG_MEMORYSTATUS */
4441
4442 /*
4443 * Trigger levels to test the mechanism.
4444 * Can be used via a sysctl.
4445 */
4446 #define TEST_LOW_MEMORY_TRIGGER_ONE 1
4447 #define TEST_LOW_MEMORY_TRIGGER_ALL 2
4448 #define TEST_PURGEABLE_TRIGGER_ONE 3
4449 #define TEST_PURGEABLE_TRIGGER_ALL 4
4450 #define TEST_LOW_MEMORY_PURGEABLE_TRIGGER_ONE 5
4451 #define TEST_LOW_MEMORY_PURGEABLE_TRIGGER_ALL 6
4452
4453 boolean_t memorystatus_manual_testing_on = FALSE;
4454 vm_pressure_level_t memorystatus_manual_testing_level = kVMPressureNormal;
4455
4456 extern struct knote *
4457 vm_pressure_select_optimal_candidate_to_notify(struct klist *, int, boolean_t);
4458
4459 extern
4460 kern_return_t vm_pressure_notification_without_levels(boolean_t);
4461
4462 extern void vm_pressure_klist_lock(void);
4463 extern void vm_pressure_klist_unlock(void);
4464
4465 extern void vm_reset_active_list(void);
4466
4467 extern void delay(int);
4468
4469 #define INTER_NOTIFICATION_DELAY (250000) /* .25 second */
4470
4471 void memorystatus_on_pageout_scan_end(void) {
4472 /* No-op */
4473 }
4474
4475 /*
4476 * kn_max - knote
4477 *
4478 * knote_pressure_level - to check if the knote is registered for this notification level.
4479 *
4480 * task - task whose bits we'll be modifying
4481 *
4482 * pressure_level_to_clear - if the task has been notified of this past level, clear that notification bit so that if/when we revert to that level, the task will be notified again.
4483 *
4484 * pressure_level_to_set - the task is about to be notified of this new level. Update the task's bit notification information appropriately.
4485 *
4486 */
4487
4488 boolean_t
4489 is_knote_registered_modify_task_pressure_bits(struct knote *kn_max, int knote_pressure_level, task_t task, vm_pressure_level_t pressure_level_to_clear, vm_pressure_level_t pressure_level_to_set)
4490 {
4491 if (kn_max->kn_sfflags & knote_pressure_level) {
4492
4493 if (task_has_been_notified(task, pressure_level_to_clear) == TRUE) {
4494
4495 task_clear_has_been_notified(task, pressure_level_to_clear);
4496 }
4497
4498 task_mark_has_been_notified(task, pressure_level_to_set);
4499 return TRUE;
4500 }
4501
4502 return FALSE;
4503 }
4504
4505 extern kern_return_t vm_pressure_notify_dispatch_vm_clients(boolean_t target_foreground_process);
4506
4507 #define VM_PRESSURE_DECREASED_SMOOTHING_PERIOD 5000 /* milliseconds */
4508
4509 kern_return_t
4510 memorystatus_update_vm_pressure(boolean_t target_foreground_process)
4511 {
4512 struct knote *kn_max = NULL;
4513 struct knote *kn_cur = NULL, *kn_temp = NULL; /* for safe list traversal */
4514 pid_t target_pid = -1;
4515 struct klist dispatch_klist = { NULL };
4516 proc_t target_proc = PROC_NULL;
4517 struct task *task = NULL;
4518 boolean_t found_candidate = FALSE;
4519
4520 static vm_pressure_level_t level_snapshot = kVMPressureNormal;
4521 static vm_pressure_level_t prev_level_snapshot = kVMPressureNormal;
4522 boolean_t smoothing_window_started = FALSE;
4523 struct timeval smoothing_window_start_tstamp = {0, 0};
4524 struct timeval curr_tstamp = {0, 0};
4525 int elapsed_msecs = 0;
4526
4527 #if !CONFIG_JETSAM
4528 #define MAX_IDLE_KILLS 100 /* limit the number of idle kills allowed */
4529
4530 int idle_kill_counter = 0;
4531
4532 /*
4533 * On desktop we take this opportunity to free up memory pressure
4534 * by immediately killing idle exitable processes. We use a delay
4535 * to avoid overkill. And we impose a max counter as a fail safe
4536 * in case daemons re-launch too fast.
4537 */
4538 while ((memorystatus_vm_pressure_level != kVMPressureNormal) && (idle_kill_counter < MAX_IDLE_KILLS)) {
4539 if (memorystatus_idle_exit_from_VM() == FALSE) {
4540 /* No idle exitable processes left to kill */
4541 break;
4542 }
4543 idle_kill_counter++;
4544
4545 if (memorystatus_manual_testing_on == TRUE) {
4546 /*
4547 * Skip the delay when testing
4548 * the pressure notification scheme.
4549 */
4550 } else {
4551 delay(1000000); /* 1 second */
4552 }
4553 }
4554 #endif /* !CONFIG_JETSAM */
4555
4556 while (1) {
4557
4558 /*
4559 * There is a race window here. But it's not clear
4560 * how much we benefit from having extra synchronization.
4561 */
4562 level_snapshot = memorystatus_vm_pressure_level;
4563
4564 if (prev_level_snapshot > level_snapshot) {
4565 /*
4566 * Pressure decreased? Let's take a little breather
4567 * and see if this condition stays.
4568 */
4569 if (smoothing_window_started == FALSE) {
4570
4571 smoothing_window_started = TRUE;
4572 microuptime(&smoothing_window_start_tstamp);
4573 }
4574
4575 microuptime(&curr_tstamp);
4576 timevalsub(&curr_tstamp, &smoothing_window_start_tstamp);
4577 elapsed_msecs = curr_tstamp.tv_sec * 1000 + curr_tstamp.tv_usec / 1000;
4578
4579 if (elapsed_msecs < VM_PRESSURE_DECREASED_SMOOTHING_PERIOD) {
4580
4581 delay(INTER_NOTIFICATION_DELAY);
4582 continue;
4583 }
4584 }
4585
4586 prev_level_snapshot = level_snapshot;
4587 smoothing_window_started = FALSE;
4588
4589 memorystatus_klist_lock();
4590 kn_max = vm_pressure_select_optimal_candidate_to_notify(&memorystatus_klist, level_snapshot, target_foreground_process);
4591
4592 if (kn_max == NULL) {
4593 memorystatus_klist_unlock();
4594
4595 /*
4596 * No more level-based clients to notify.
4597 * Try the non-level based notification clients.
4598 *
4599 * However, these non-level clients don't understand
4600 * the "return-to-normal" notification.
4601 *
4602 * So don't consider them for those notifications. Just
4603 * return instead.
4604 *
4605 */
4606
4607 if (level_snapshot != kVMPressureNormal) {
4608 goto try_dispatch_vm_clients;
4609 } else {
4610 return KERN_FAILURE;
4611 }
4612 }
4613
4614 target_proc = kn_max->kn_kq->kq_p;
4615
4616 proc_list_lock();
4617 if (target_proc != proc_ref_locked(target_proc)) {
4618 target_proc = PROC_NULL;
4619 proc_list_unlock();
4620 memorystatus_klist_unlock();
4621 continue;
4622 }
4623 proc_list_unlock();
4624
4625 target_pid = target_proc->p_pid;
4626
4627 task = (struct task *)(target_proc->task);
4628
4629 if (level_snapshot != kVMPressureNormal) {
4630
4631 if (level_snapshot == kVMPressureWarning || level_snapshot == kVMPressureUrgent) {
4632
4633 if (is_knote_registered_modify_task_pressure_bits(kn_max, NOTE_MEMORYSTATUS_PRESSURE_WARN, task, kVMPressureCritical, kVMPressureWarning) == TRUE) {
4634 found_candidate = TRUE;
4635 }
4636 } else {
4637 if (level_snapshot == kVMPressureCritical) {
4638
4639 if (is_knote_registered_modify_task_pressure_bits(kn_max, NOTE_MEMORYSTATUS_PRESSURE_CRITICAL, task, kVMPressureWarning, kVMPressureCritical) == TRUE) {
4640 found_candidate = TRUE;
4641 }
4642 }
4643 }
4644 } else {
4645 if (kn_max->kn_sfflags & NOTE_MEMORYSTATUS_PRESSURE_NORMAL) {
4646
4647 task_clear_has_been_notified(task, kVMPressureWarning);
4648 task_clear_has_been_notified(task, kVMPressureCritical);
4649
4650 found_candidate = TRUE;
4651 }
4652 }
4653
4654 if (found_candidate == FALSE) {
4655 proc_rele(target_proc);
4656 memorystatus_klist_unlock();
4657 continue;
4658 }
4659
4660 SLIST_FOREACH_SAFE(kn_cur, &memorystatus_klist, kn_selnext, kn_temp) {
4661 proc_t knote_proc = kn_cur->kn_kq->kq_p;
4662 pid_t knote_pid = knote_proc->p_pid;
4663 if (knote_pid == target_pid) {
4664 KNOTE_DETACH(&memorystatus_klist, kn_cur);
4665 KNOTE_ATTACH(&dispatch_klist, kn_cur);
4666 }
4667 }
4668
4669 KNOTE(&dispatch_klist, (level_snapshot != kVMPressureNormal) ? kMemorystatusPressure : kMemorystatusNoPressure);
4670
4671 SLIST_FOREACH_SAFE(kn_cur, &dispatch_klist, kn_selnext, kn_temp) {
4672 KNOTE_DETACH(&dispatch_klist, kn_cur);
4673 KNOTE_ATTACH(&memorystatus_klist, kn_cur);
4674 }
4675
4676 memorystatus_klist_unlock();
4677
4678 microuptime(&target_proc->vm_pressure_last_notify_tstamp);
4679 proc_rele(target_proc);
4680
4681 if (memorystatus_manual_testing_on == TRUE && target_foreground_process == TRUE) {
4682 break;
4683 }
4684
4685 try_dispatch_vm_clients:
4686 if (kn_max == NULL && level_snapshot != kVMPressureNormal) {
4687 /*
4688 * We will exit this loop when we are done with
4689 * notification clients (level and non-level based).
4690 */
4691 if ((vm_pressure_notify_dispatch_vm_clients(target_foreground_process) == KERN_FAILURE) && (kn_max == NULL)) {
4692 /*
4693 * kn_max == NULL i.e. we didn't find any eligible clients for the level-based notifications
4694 * AND
4695 * we have failed to find any eligible clients for the non-level based notifications too.
4696 * So, we are done.
4697 */
4698
4699 return KERN_FAILURE;
4700 }
4701 }
4702
4703 /*
4704 * LD: This block of code below used to be invoked in the older memory notification scheme on embedded everytime
4705 * a process was sent a memory pressure notification. The "memorystatus_klist" list was used to hold these
4706 * privileged listeners. But now we have moved to the newer scheme and are trying to move away from the extra
4707 * notifications. So the code is here in case we break compat. and need to send out notifications to the privileged
4708 * apps.
4709 */
4710 #if 0
4711 #endif /* 0 */
4712
4713 if (memorystatus_manual_testing_on == TRUE) {
4714 /*
4715 * Testing out the pressure notification scheme.
4716 * No need for delays etc.
4717 */
4718 } else {
4719
4720 uint32_t sleep_interval = INTER_NOTIFICATION_DELAY;
4721 #if CONFIG_JETSAM
4722 unsigned int page_delta = 0;
4723 unsigned int skip_delay_page_threshold = 0;
4724
4725 assert(memorystatus_available_pages_pressure >= memorystatus_available_pages_critical_base);
4726
4727 page_delta = (memorystatus_available_pages_pressure - memorystatus_available_pages_critical_base) / 2;
4728 skip_delay_page_threshold = memorystatus_available_pages_pressure - page_delta;
4729
4730 if (memorystatus_available_pages <= skip_delay_page_threshold) {
4731 /*
4732 * We are nearing the critcal mark fast and can't afford to wait between
4733 * notifications.
4734 */
4735 sleep_interval = 0;
4736 }
4737 #endif /* CONFIG_JETSAM */
4738
4739 if (sleep_interval) {
4740 delay(sleep_interval);
4741 }
4742 }
4743 }
4744
4745 return KERN_SUCCESS;
4746 }
4747
4748 vm_pressure_level_t
4749 convert_internal_pressure_level_to_dispatch_level(vm_pressure_level_t);
4750
4751 vm_pressure_level_t
4752 convert_internal_pressure_level_to_dispatch_level(vm_pressure_level_t internal_pressure_level)
4753 {
4754 vm_pressure_level_t dispatch_level = NOTE_MEMORYSTATUS_PRESSURE_NORMAL;
4755
4756 switch (internal_pressure_level) {
4757
4758 case kVMPressureNormal:
4759 {
4760 dispatch_level = NOTE_MEMORYSTATUS_PRESSURE_NORMAL;
4761 break;
4762 }
4763
4764 case kVMPressureWarning:
4765 case kVMPressureUrgent:
4766 {
4767 dispatch_level = NOTE_MEMORYSTATUS_PRESSURE_WARN;
4768 break;
4769 }
4770
4771 case kVMPressureCritical:
4772 {
4773 dispatch_level = NOTE_MEMORYSTATUS_PRESSURE_CRITICAL;
4774 break;
4775 }
4776
4777 default:
4778 break;
4779 }
4780
4781 return dispatch_level;
4782 }
4783
4784 static int
4785 sysctl_memorystatus_vm_pressure_level SYSCTL_HANDLER_ARGS
4786 {
4787 #pragma unused(arg1, arg2, oidp)
4788 vm_pressure_level_t dispatch_level = convert_internal_pressure_level_to_dispatch_level(memorystatus_vm_pressure_level);
4789
4790 return SYSCTL_OUT(req, &dispatch_level, sizeof(dispatch_level));
4791 }
4792
4793 #if DEBUG || DEVELOPMENT
4794
4795 SYSCTL_PROC(_kern, OID_AUTO, memorystatus_vm_pressure_level, CTLTYPE_INT|CTLFLAG_RD|CTLFLAG_LOCKED,
4796 0, 0, &sysctl_memorystatus_vm_pressure_level, "I", "");
4797
4798 #else /* DEBUG || DEVELOPMENT */
4799
4800 SYSCTL_PROC(_kern, OID_AUTO, memorystatus_vm_pressure_level, CTLTYPE_INT|CTLFLAG_RD|CTLFLAG_LOCKED|CTLFLAG_MASKED,
4801 0, 0, &sysctl_memorystatus_vm_pressure_level, "I", "");
4802
4803 #endif /* DEBUG || DEVELOPMENT */
4804
4805 extern int memorystatus_purge_on_warning;
4806 extern int memorystatus_purge_on_critical;
4807
4808 static int
4809 sysctl_memorypressure_manual_trigger SYSCTL_HANDLER_ARGS
4810 {
4811 #pragma unused(arg1, arg2)
4812
4813 int level = 0;
4814 int error = 0;
4815 int pressure_level = 0;
4816 int trigger_request = 0;
4817 int force_purge;
4818
4819 error = sysctl_handle_int(oidp, &level, 0, req);
4820 if (error || !req->newptr) {
4821 return (error);
4822 }
4823
4824 memorystatus_manual_testing_on = TRUE;
4825
4826 trigger_request = (level >> 16) & 0xFFFF;
4827 pressure_level = (level & 0xFFFF);
4828
4829 if (trigger_request < TEST_LOW_MEMORY_TRIGGER_ONE ||
4830 trigger_request > TEST_LOW_MEMORY_PURGEABLE_TRIGGER_ALL) {
4831 return EINVAL;
4832 }
4833 switch (pressure_level) {
4834 case NOTE_MEMORYSTATUS_PRESSURE_NORMAL:
4835 case NOTE_MEMORYSTATUS_PRESSURE_WARN:
4836 case NOTE_MEMORYSTATUS_PRESSURE_CRITICAL:
4837 break;
4838 default:
4839 return EINVAL;
4840 }
4841
4842 /*
4843 * The pressure level is being set from user-space.
4844 * And user-space uses the constants in sys/event.h
4845 * So we translate those events to our internal levels here.
4846 */
4847 if (pressure_level == NOTE_MEMORYSTATUS_PRESSURE_NORMAL) {
4848
4849 memorystatus_manual_testing_level = kVMPressureNormal;
4850 force_purge = 0;
4851
4852 } else if (pressure_level == NOTE_MEMORYSTATUS_PRESSURE_WARN) {
4853
4854 memorystatus_manual_testing_level = kVMPressureWarning;
4855 force_purge = memorystatus_purge_on_warning;
4856
4857 } else if (pressure_level == NOTE_MEMORYSTATUS_PRESSURE_CRITICAL) {
4858
4859 memorystatus_manual_testing_level = kVMPressureCritical;
4860 force_purge = memorystatus_purge_on_critical;
4861 }
4862
4863 memorystatus_vm_pressure_level = memorystatus_manual_testing_level;
4864
4865 /* purge according to the new pressure level */
4866 switch (trigger_request) {
4867 case TEST_PURGEABLE_TRIGGER_ONE:
4868 case TEST_LOW_MEMORY_PURGEABLE_TRIGGER_ONE:
4869 if (force_purge == 0) {
4870 /* no purging requested */
4871 break;
4872 }
4873 vm_purgeable_object_purge_one_unlocked(force_purge);
4874 break;
4875 case TEST_PURGEABLE_TRIGGER_ALL:
4876 case TEST_LOW_MEMORY_PURGEABLE_TRIGGER_ALL:
4877 if (force_purge == 0) {
4878 /* no purging requested */
4879 break;
4880 }
4881 while (vm_purgeable_object_purge_one_unlocked(force_purge));
4882 break;
4883 }
4884
4885 if ((trigger_request == TEST_LOW_MEMORY_TRIGGER_ONE) ||
4886 (trigger_request == TEST_LOW_MEMORY_PURGEABLE_TRIGGER_ONE)) {
4887
4888 memorystatus_update_vm_pressure(TRUE);
4889 }
4890
4891 if ((trigger_request == TEST_LOW_MEMORY_TRIGGER_ALL) ||
4892 (trigger_request == TEST_LOW_MEMORY_PURGEABLE_TRIGGER_ALL)) {
4893
4894 while (memorystatus_update_vm_pressure(FALSE) == KERN_SUCCESS) {
4895 continue;
4896 }
4897 }
4898
4899 if (pressure_level == NOTE_MEMORYSTATUS_PRESSURE_NORMAL) {
4900 memorystatus_manual_testing_on = FALSE;
4901
4902 vm_pressure_klist_lock();
4903 vm_reset_active_list();
4904 vm_pressure_klist_unlock();
4905 } else {
4906
4907 vm_pressure_klist_lock();
4908 vm_pressure_notification_without_levels(FALSE);
4909 vm_pressure_klist_unlock();
4910 }
4911
4912 return 0;
4913 }
4914
4915 SYSCTL_PROC(_kern, OID_AUTO, memorypressure_manual_trigger, CTLTYPE_INT|CTLFLAG_WR|CTLFLAG_LOCKED|CTLFLAG_MASKED,
4916 0, 0, &sysctl_memorypressure_manual_trigger, "I", "");
4917
4918
4919 extern int memorystatus_purge_on_warning;
4920 extern int memorystatus_purge_on_urgent;
4921 extern int memorystatus_purge_on_critical;
4922
4923 SYSCTL_INT(_kern, OID_AUTO, memorystatus_purge_on_warning, CTLFLAG_RW|CTLFLAG_LOCKED, &memorystatus_purge_on_warning, 0, "");
4924 SYSCTL_INT(_kern, OID_AUTO, memorystatus_purge_on_urgent, CTLTYPE_INT|CTLFLAG_RW|CTLFLAG_LOCKED, &memorystatus_purge_on_urgent, 0, "");
4925 SYSCTL_INT(_kern, OID_AUTO, memorystatus_purge_on_critical, CTLTYPE_INT|CTLFLAG_RW|CTLFLAG_LOCKED, &memorystatus_purge_on_critical, 0, "");
4926
4927
4928 #endif /* VM_PRESSURE_EVENTS */
4929
4930 /* Return both allocated and actual size, since there's a race between allocation and list compilation */
4931 static int
4932 memorystatus_get_priority_list(memorystatus_priority_entry_t **list_ptr, size_t *buffer_size, size_t *list_size, boolean_t size_only)
4933 {
4934 uint32_t list_count, i = 0;
4935 memorystatus_priority_entry_t *list_entry;
4936 proc_t p;
4937
4938 list_count = memorystatus_list_count;
4939 *list_size = sizeof(memorystatus_priority_entry_t) * list_count;
4940
4941 /* Just a size check? */
4942 if (size_only) {
4943 return 0;
4944 }
4945
4946 /* Otherwise, validate the size of the buffer */
4947 if (*buffer_size < *list_size) {
4948 return EINVAL;
4949 }
4950
4951 *list_ptr = (memorystatus_priority_entry_t*)kalloc(*list_size);
4952 if (!list_ptr) {
4953 return ENOMEM;
4954 }
4955
4956 memset(*list_ptr, 0, *list_size);
4957
4958 *buffer_size = *list_size;
4959 *list_size = 0;
4960
4961 list_entry = *list_ptr;
4962
4963 proc_list_lock();
4964
4965 p = memorystatus_get_first_proc_locked(&i, TRUE);
4966 while (p && (*list_size < *buffer_size)) {
4967 list_entry->pid = p->p_pid;
4968 list_entry->priority = p->p_memstat_effectivepriority;
4969 list_entry->user_data = p->p_memstat_userdata;
4970 #if LEGACY_HIWATER
4971
4972 /*
4973 * No need to consider P_MEMSTAT_MEMLIMIT_BACKGROUND anymore.
4974 * Background limits are described via the inactive limit slots.
4975 * So, here, the cached limit should always be valid.
4976 */
4977
4978 if (p->p_memstat_memlimit <= 0) {
4979 task_get_phys_footprint_limit(p->task, &list_entry->limit);
4980 } else {
4981 list_entry->limit = p->p_memstat_memlimit;
4982 }
4983 #else
4984 task_get_phys_footprint_limit(p->task, &list_entry->limit);
4985 #endif
4986 list_entry->state = memorystatus_build_state(p);
4987 list_entry++;
4988
4989 *list_size += sizeof(memorystatus_priority_entry_t);
4990
4991 p = memorystatus_get_next_proc_locked(&i, p, TRUE);
4992 }
4993
4994 proc_list_unlock();
4995
4996 MEMORYSTATUS_DEBUG(1, "memorystatus_get_priority_list: returning %lu for size\n", (unsigned long)*list_size);
4997
4998 return 0;
4999 }
5000
5001 static int
5002 memorystatus_cmd_get_priority_list(user_addr_t buffer, size_t buffer_size, int32_t *retval) {
5003 int error = EINVAL;
5004 boolean_t size_only;
5005 memorystatus_priority_entry_t *list = NULL;
5006 size_t list_size;
5007
5008 size_only = ((buffer == USER_ADDR_NULL) ? TRUE: FALSE);
5009
5010 error = memorystatus_get_priority_list(&list, &buffer_size, &list_size, size_only);
5011 if (error) {
5012 goto out;
5013 }
5014
5015 if (!size_only) {
5016 error = copyout(list, buffer, list_size);
5017 }
5018
5019 if (error == 0) {
5020 *retval = list_size;
5021 }
5022 out:
5023
5024 if (list) {
5025 kfree(list, buffer_size);
5026 }
5027
5028 return error;
5029 }
5030
5031 #if CONFIG_JETSAM
5032
5033 static void
5034 memorystatus_clear_errors(void)
5035 {
5036 proc_t p;
5037 unsigned int i = 0;
5038
5039 KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_CLEAR_ERRORS) | DBG_FUNC_START, 0, 0, 0, 0, 0);
5040
5041 proc_list_lock();
5042
5043 p = memorystatus_get_first_proc_locked(&i, TRUE);
5044 while (p) {
5045 if (p->p_memstat_state & P_MEMSTAT_ERROR) {
5046 p->p_memstat_state &= ~P_MEMSTAT_ERROR;
5047 }
5048 p = memorystatus_get_next_proc_locked(&i, p, TRUE);
5049 }
5050
5051 proc_list_unlock();
5052
5053 KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_CLEAR_ERRORS) | DBG_FUNC_END, 0, 0, 0, 0, 0);
5054 }
5055
5056 static void
5057 memorystatus_update_levels_locked(boolean_t critical_only) {
5058
5059 memorystatus_available_pages_critical = memorystatus_available_pages_critical_base;
5060
5061 /*
5062 * If there's an entry in the first bucket, we have idle processes.
5063 */
5064 memstat_bucket_t *first_bucket = &memstat_bucket[JETSAM_PRIORITY_IDLE];
5065 if (first_bucket->count) {
5066 memorystatus_available_pages_critical += memorystatus_available_pages_critical_idle_offset;
5067
5068 if (memorystatus_available_pages_critical > memorystatus_available_pages_pressure ) {
5069 /*
5070 * The critical threshold must never exceed the pressure threshold
5071 */
5072 memorystatus_available_pages_critical = memorystatus_available_pages_pressure;
5073 }
5074 }
5075
5076 #if DEBUG || DEVELOPMENT
5077 if (memorystatus_jetsam_policy & kPolicyDiagnoseActive) {
5078 memorystatus_available_pages_critical += memorystatus_jetsam_policy_offset_pages_diagnostic;
5079
5080 if (memorystatus_available_pages_critical > memorystatus_available_pages_pressure ) {
5081 /*
5082 * The critical threshold must never exceed the pressure threshold
5083 */
5084 memorystatus_available_pages_critical = memorystatus_available_pages_pressure;
5085 }
5086 }
5087 #endif
5088
5089 if (critical_only) {
5090 return;
5091 }
5092
5093 #if VM_PRESSURE_EVENTS
5094 memorystatus_available_pages_pressure = (pressure_threshold_percentage / delta_percentage) * memorystatus_delta;
5095 #if DEBUG || DEVELOPMENT
5096 if (memorystatus_jetsam_policy & kPolicyDiagnoseActive) {
5097 memorystatus_available_pages_pressure += memorystatus_jetsam_policy_offset_pages_diagnostic;
5098 }
5099 #endif
5100 #endif
5101 }
5102
5103 /*
5104 * Get the at_boot snapshot
5105 */
5106 static int
5107 memorystatus_get_at_boot_snapshot(memorystatus_jetsam_snapshot_t **snapshot, size_t *snapshot_size, boolean_t size_only) {
5108 size_t input_size = *snapshot_size;
5109
5110 /*
5111 * The at_boot snapshot has no entry list.
5112 */
5113 *snapshot_size = sizeof(memorystatus_jetsam_snapshot_t);
5114
5115 if (size_only) {
5116 return 0;
5117 }
5118
5119 /*
5120 * Validate the size of the snapshot buffer
5121 */
5122 if (input_size < *snapshot_size) {
5123 return EINVAL;
5124 }
5125
5126 /*
5127 * Update the notification_time only
5128 */
5129 memorystatus_at_boot_snapshot.notification_time = mach_absolute_time();
5130 *snapshot = &memorystatus_at_boot_snapshot;
5131
5132 MEMORYSTATUS_DEBUG(7, "memorystatus_get_at_boot_snapshot: returned inputsize (%ld), snapshot_size(%ld), listcount(%d)\n",
5133 (long)input_size, (long)*snapshot_size, 0);
5134 return 0;
5135 }
5136
5137 static int
5138 memorystatus_get_on_demand_snapshot(memorystatus_jetsam_snapshot_t **snapshot, size_t *snapshot_size, boolean_t size_only) {
5139 size_t input_size = *snapshot_size;
5140 uint32_t ods_list_count = memorystatus_list_count;
5141 memorystatus_jetsam_snapshot_t *ods = NULL; /* The on_demand snapshot buffer */
5142
5143 *snapshot_size = sizeof(memorystatus_jetsam_snapshot_t) + (sizeof(memorystatus_jetsam_snapshot_entry_t) * (ods_list_count));
5144
5145 if (size_only) {
5146 return 0;
5147 }
5148
5149 /*
5150 * Validate the size of the snapshot buffer.
5151 * This is inherently racey. May want to revisit
5152 * this error condition and trim the output when
5153 * it doesn't fit.
5154 */
5155 if (input_size < *snapshot_size) {
5156 return EINVAL;
5157 }
5158
5159 /*
5160 * Allocate and initialize a snapshot buffer.
5161 */
5162 ods = (memorystatus_jetsam_snapshot_t *)kalloc(*snapshot_size);
5163 if (!ods) {
5164 return (ENOMEM);
5165 }
5166
5167 memset(ods, 0, *snapshot_size);
5168
5169 proc_list_lock();
5170 memorystatus_init_jetsam_snapshot_locked(ods, ods_list_count);
5171 proc_list_unlock();
5172
5173 /*
5174 * Return the kernel allocated, on_demand buffer.
5175 * The caller of this routine will copy the data out
5176 * to user space and then free the kernel allocated
5177 * buffer.
5178 */
5179 *snapshot = ods;
5180
5181 MEMORYSTATUS_DEBUG(7, "memorystatus_get_on_demand_snapshot: returned inputsize (%ld), snapshot_size(%ld), listcount(%ld)\n",
5182 (long)input_size, (long)*snapshot_size, (long)ods_list_count);
5183
5184 return 0;
5185 }
5186
5187 static int
5188 memorystatus_get_jetsam_snapshot(memorystatus_jetsam_snapshot_t **snapshot, size_t *snapshot_size, boolean_t size_only) {
5189 size_t input_size = *snapshot_size;
5190
5191 if (memorystatus_jetsam_snapshot_count > 0) {
5192 *snapshot_size = sizeof(memorystatus_jetsam_snapshot_t) + (sizeof(memorystatus_jetsam_snapshot_entry_t) * (memorystatus_jetsam_snapshot_count));
5193 } else {
5194 *snapshot_size = 0;
5195 }
5196
5197 if (size_only) {
5198 return 0;
5199 }
5200
5201 if (input_size < *snapshot_size) {
5202 return EINVAL;
5203 }
5204
5205 *snapshot = memorystatus_jetsam_snapshot;
5206
5207 MEMORYSTATUS_DEBUG(7, "memorystatus_get_jetsam_snapshot: returned inputsize (%ld), snapshot_size(%ld), listcount(%ld)\n",
5208 (long)input_size, (long)*snapshot_size, (long)memorystatus_jetsam_snapshot_count);
5209
5210 return 0;
5211 }
5212
5213
5214 static int
5215 memorystatus_cmd_get_jetsam_snapshot(int32_t flags, user_addr_t buffer, size_t buffer_size, int32_t *retval) {
5216 int error = EINVAL;
5217 boolean_t size_only;
5218 boolean_t is_default_snapshot = FALSE;
5219 boolean_t is_on_demand_snapshot = FALSE;
5220 boolean_t is_at_boot_snapshot = FALSE;
5221 memorystatus_jetsam_snapshot_t *snapshot;
5222
5223 size_only = ((buffer == USER_ADDR_NULL) ? TRUE : FALSE);
5224
5225 if (flags == 0) {
5226 /* Default */
5227 is_default_snapshot = TRUE;
5228 error = memorystatus_get_jetsam_snapshot(&snapshot, &buffer_size, size_only);
5229 } else {
5230 if (flags & ~(MEMORYSTATUS_SNAPSHOT_ON_DEMAND | MEMORYSTATUS_SNAPSHOT_AT_BOOT)) {
5231 /*
5232 * Unsupported bit set in flag.
5233 */
5234 return EINVAL;
5235 }
5236
5237 if ((flags & (MEMORYSTATUS_SNAPSHOT_ON_DEMAND | MEMORYSTATUS_SNAPSHOT_AT_BOOT)) ==
5238 (MEMORYSTATUS_SNAPSHOT_ON_DEMAND | MEMORYSTATUS_SNAPSHOT_AT_BOOT)) {
5239 /*
5240 * Can't have both set at the same time.
5241 */
5242 return EINVAL;
5243 }
5244
5245 if (flags & MEMORYSTATUS_SNAPSHOT_ON_DEMAND) {
5246 is_on_demand_snapshot = TRUE;
5247 /*
5248 * When not requesting the size only, the following call will allocate
5249 * an on_demand snapshot buffer, which is freed below.
5250 */
5251 error = memorystatus_get_on_demand_snapshot(&snapshot, &buffer_size, size_only);
5252
5253 } else if (flags & MEMORYSTATUS_SNAPSHOT_AT_BOOT) {
5254 is_at_boot_snapshot = TRUE;
5255 error = memorystatus_get_at_boot_snapshot(&snapshot, &buffer_size, size_only);
5256 } else {
5257 /*
5258 * Invalid flag setting.
5259 */
5260 return EINVAL;
5261 }
5262 }
5263
5264 if (error) {
5265 goto out;
5266 }
5267
5268 /*
5269 * Copy the data out to user space and clear the snapshot buffer.
5270 * If working with the jetsam snapshot,
5271 * clearing the buffer means, reset the count.
5272 * If working with an on_demand snapshot
5273 * clearing the buffer means, free it.
5274 * If working with the at_boot snapshot
5275 * there is nothing to clear or update.
5276 */
5277 if (!size_only) {
5278 if ((error = copyout(snapshot, buffer, buffer_size)) == 0) {
5279 if (is_default_snapshot) {
5280 /*
5281 * The jetsam snapshot is never freed, its count is simply reset.
5282 */
5283 snapshot->entry_count = memorystatus_jetsam_snapshot_count = 0;
5284
5285 proc_list_lock();
5286 memorystatus_jetsam_snapshot_last_timestamp = 0;
5287 proc_list_unlock();
5288 }
5289 }
5290
5291 if (is_on_demand_snapshot) {
5292 /*
5293 * The on_demand snapshot is always freed,
5294 * even if the copyout failed.
5295 */
5296 if(snapshot) {
5297 kfree(snapshot, buffer_size);
5298 }
5299 }
5300 }
5301
5302 if (error == 0) {
5303 *retval = buffer_size;
5304 }
5305 out:
5306 return error;
5307 }
5308
5309 /*
5310 * Routine: memorystatus_cmd_grp_set_properties
5311 * Purpose: Update properties for a group of processes.
5312 *
5313 * Supported Properties:
5314 * [priority]
5315 * Move each process out of its effective priority
5316 * band and into a new priority band.
5317 * Maintains relative order from lowest to highest priority.
5318 * In single band, maintains relative order from head to tail.
5319 *
5320 * eg: before [effectivepriority | pid]
5321 * [18 | p101 ]
5322 * [17 | p55, p67, p19 ]
5323 * [12 | p103 p10 ]
5324 * [ 7 | p25 ]
5325 * [ 0 | p71, p82, ]
5326 *
5327 * after [ new band | pid]
5328 * [ xxx | p71, p82, p25, p103, p10, p55, p67, p19, p101]
5329 *
5330 * Returns: 0 on success, else non-zero.
5331 *
5332 * Caveat: We know there is a race window regarding recycled pids.
5333 * A process could be killed before the kernel can act on it here.
5334 * If a pid cannot be found in any of the jetsam priority bands,
5335 * then we simply ignore it. No harm.
5336 * But, if the pid has been recycled then it could be an issue.
5337 * In that scenario, we might move an unsuspecting process to the new
5338 * priority band. It's not clear how the kernel can safeguard
5339 * against this, but it would be an extremely rare case anyway.
5340 * The caller of this api might avoid such race conditions by
5341 * ensuring that the processes passed in the pid list are suspended.
5342 */
5343
5344
5345 /* This internal structure can expand when we add support for more properties */
5346 typedef struct memorystatus_internal_properties
5347 {
5348 proc_t proc;
5349 int32_t priority; /* see memorytstatus_priority_entry_t : priority */
5350 } memorystatus_internal_properties_t;
5351
5352
5353 static int
5354 memorystatus_cmd_grp_set_properties(int32_t flags, user_addr_t buffer, size_t buffer_size, __unused int32_t *retval) {
5355
5356 #pragma unused (flags)
5357
5358 /*
5359 * We only handle setting priority
5360 * per process
5361 */
5362
5363 int error = 0;
5364 memorystatus_priority_entry_t *entries = NULL;
5365 uint32_t entry_count = 0;
5366
5367 /* This will be the ordered proc list */
5368 memorystatus_internal_properties_t *table = NULL;
5369 size_t table_size = 0;
5370 uint32_t table_count = 0;
5371
5372 uint32_t i = 0;
5373 uint32_t bucket_index = 0;
5374 boolean_t head_insert;
5375 int32_t new_priority;
5376
5377 proc_t p;
5378
5379 /* Verify inputs */
5380 if ((buffer == USER_ADDR_NULL) || (buffer_size == 0) || ((buffer_size % sizeof(memorystatus_priority_entry_t)) != 0)) {
5381 error = EINVAL;
5382 goto out;
5383 }
5384
5385 entry_count = (buffer_size / sizeof(memorystatus_priority_entry_t));
5386 if ((entries = (memorystatus_priority_entry_t *)kalloc(buffer_size)) == NULL) {
5387 error = ENOMEM;
5388 goto out;
5389 }
5390
5391 KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_GRP_SET_PROP) | DBG_FUNC_START, entry_count, 0, 0, 0, 0);
5392
5393 if ((error = copyin(buffer, entries, buffer_size)) != 0) {
5394 goto out;
5395 }
5396
5397 /* Verify sanity of input priorities */
5398 for (i=0; i < entry_count; i++) {
5399 if (entries[i].priority == -1) {
5400 /* Use as shorthand for default priority */
5401 entries[i].priority = JETSAM_PRIORITY_DEFAULT;
5402 } else if (entries[i].priority == JETSAM_PRIORITY_IDLE_DEFERRED) {
5403 /* JETSAM_PRIORITY_IDLE_DEFERRED is reserved for internal use;
5404 * if requested, adjust to JETSAM_PRIORITY_IDLE. */
5405 entries[i].priority = JETSAM_PRIORITY_IDLE;
5406 } else if (entries[i].priority == JETSAM_PRIORITY_IDLE_HEAD) {
5407 /* JETSAM_PRIORITY_IDLE_HEAD inserts at the head of the idle
5408 * queue */
5409 /* Deal with this later */
5410 } else if ((entries[i].priority < 0) || (entries[i].priority >= MEMSTAT_BUCKET_COUNT)) {
5411 /* Sanity check */
5412 error = EINVAL;
5413 goto out;
5414 }
5415 }
5416
5417 table_size = sizeof(memorystatus_internal_properties_t) * entry_count;
5418 if ( (table = (memorystatus_internal_properties_t *)kalloc(table_size)) == NULL) {
5419 error = ENOMEM;
5420 goto out;
5421 }
5422 memset(table, 0, table_size);
5423
5424
5425 /*
5426 * For each jetsam bucket entry, spin through the input property list.
5427 * When a matching pid is found, populate an adjacent table with the
5428 * appropriate proc pointer and new property values.
5429 * This traversal automatically preserves order from lowest
5430 * to highest priority.
5431 */
5432
5433 bucket_index=0;
5434
5435 proc_list_lock();
5436
5437 /* Create the ordered table */
5438 p = memorystatus_get_first_proc_locked(&bucket_index, TRUE);
5439 while (p && (table_count < entry_count)) {
5440 for (i=0; i < entry_count; i++ ) {
5441 if (p->p_pid == entries[i].pid) {
5442 /* Build the table data */
5443 table[table_count].proc = p;
5444 table[table_count].priority = entries[i].priority;
5445 table_count++;
5446 break;
5447 }
5448 }
5449 p = memorystatus_get_next_proc_locked(&bucket_index, p, TRUE);
5450 }
5451
5452 /* We now have ordered list of procs ready to move */
5453 for (i=0; i < table_count; i++) {
5454 p = table[i].proc;
5455 assert(p != NULL);
5456
5457 /* Allow head inserts -- but relative order is now */
5458 if (table[i].priority == JETSAM_PRIORITY_IDLE_HEAD) {
5459 new_priority = JETSAM_PRIORITY_IDLE;
5460 head_insert = true;
5461 } else {
5462 new_priority = table[i].priority;
5463 head_insert = false;
5464 }
5465
5466 /* Not allowed */
5467 if (p->p_memstat_state & P_MEMSTAT_INTERNAL) {
5468 continue;
5469 }
5470
5471 /*
5472 * Take appropriate steps if moving proc out of the
5473 * JETSAM_PRIORITY_IDLE_DEFERRED band.
5474 */
5475 if (p->p_memstat_effectivepriority == JETSAM_PRIORITY_IDLE_DEFERRED) {
5476 memorystatus_invalidate_idle_demotion_locked(p, TRUE);
5477 }
5478
5479 memorystatus_update_priority_locked(p, new_priority, head_insert);
5480 }
5481
5482 proc_list_unlock();
5483
5484 /*
5485 * if (table_count != entry_count)
5486 * then some pids were not found in a jetsam band.
5487 * harmless but interesting...
5488 */
5489 KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_GRP_SET_PROP) | DBG_FUNC_END, entry_count, table_count, 0, 0, 0);
5490
5491 out:
5492 if (entries)
5493 kfree(entries, buffer_size);
5494 if (table)
5495 kfree(table, table_size);
5496
5497 return (error);
5498 }
5499
5500
5501 /*
5502 * This routine is used to update a process's jetsam priority position and stored user_data.
5503 * It is not used for the setting of memory limits, which is why the last 6 args to the
5504 * memorystatus_update() call are 0 or FALSE.
5505 */
5506
5507 static int
5508 memorystatus_cmd_set_priority_properties(pid_t pid, user_addr_t buffer, size_t buffer_size, __unused int32_t *retval) {
5509 int error = 0;
5510 memorystatus_priority_properties_t mpp_entry;
5511
5512 /* Validate inputs */
5513 if ((pid == 0) || (buffer == USER_ADDR_NULL) || (buffer_size != sizeof(memorystatus_priority_properties_t))) {
5514 return EINVAL;
5515 }
5516
5517 error = copyin(buffer, &mpp_entry, buffer_size);
5518
5519 if (error == 0) {
5520 proc_t p;
5521
5522 p = proc_find(pid);
5523 if (!p) {
5524 return ESRCH;
5525 }
5526
5527 if (p->p_memstat_state & P_MEMSTAT_INTERNAL) {
5528 proc_rele(p);
5529 return EPERM;
5530 }
5531
5532 error = memorystatus_update(p, mpp_entry.priority, mpp_entry.user_data, FALSE, FALSE, 0, 0, FALSE, FALSE, FALSE);
5533 proc_rele(p);
5534 }
5535
5536 return(error);
5537 }
5538
5539 static int
5540 memorystatus_cmd_set_memlimit_properties(pid_t pid, user_addr_t buffer, size_t buffer_size, __unused int32_t *retval) {
5541 int error = 0;
5542 memorystatus_memlimit_properties_t mmp_entry;
5543
5544 /* Validate inputs */
5545 if ((pid == 0) || (buffer == USER_ADDR_NULL) || (buffer_size != sizeof(memorystatus_memlimit_properties_t))) {
5546 return EINVAL;
5547 }
5548
5549 error = copyin(buffer, &mmp_entry, buffer_size);
5550
5551 if (error == 0) {
5552 error = memorystatus_set_memlimit_properties(pid, &mmp_entry);
5553 }
5554
5555 return(error);
5556 }
5557
5558 /*
5559 * When getting the memlimit settings, we can't simply call task_get_phys_footprint_limit().
5560 * That gets the proc's cached memlimit and there is no guarantee that the active/inactive
5561 * limits will be the same in the no-limit case. Instead we convert limits <= 0 using
5562 * task_convert_phys_footprint_limit(). It computes the same limit value that would be written
5563 * to the task's ledgers via task_set_phys_footprint_limit().
5564 */
5565 static int
5566 memorystatus_cmd_get_memlimit_properties(pid_t pid, user_addr_t buffer, size_t buffer_size, __unused int32_t *retval) {
5567 int error = 0;
5568 memorystatus_memlimit_properties_t mmp_entry;
5569
5570 /* Validate inputs */
5571 if ((pid == 0) || (buffer == USER_ADDR_NULL) || (buffer_size != sizeof(memorystatus_memlimit_properties_t))) {
5572 return EINVAL;
5573 }
5574
5575 memset (&mmp_entry, 0, sizeof(memorystatus_memlimit_properties_t));
5576
5577 proc_t p = proc_find(pid);
5578 if (!p) {
5579 return ESRCH;
5580 }
5581
5582 /*
5583 * Get the active limit and attributes.
5584 * No locks taken since we hold a reference to the proc.
5585 */
5586
5587 if (p->p_memstat_memlimit_active > 0 ) {
5588 mmp_entry.memlimit_active = p->p_memstat_memlimit_active;
5589 } else {
5590 task_convert_phys_footprint_limit(-1, &mmp_entry.memlimit_active);
5591 }
5592
5593 if (p->p_memstat_state & P_MEMSTAT_MEMLIMIT_ACTIVE_FATAL) {
5594 mmp_entry.memlimit_active_attr |= MEMORYSTATUS_MEMLIMIT_ATTR_FATAL;
5595 }
5596
5597 /*
5598 * Get the inactive limit and attributes
5599 */
5600 if (p->p_memstat_memlimit_inactive <= 0) {
5601 task_convert_phys_footprint_limit(-1, &mmp_entry.memlimit_inactive);
5602 } else {
5603 mmp_entry.memlimit_inactive = p->p_memstat_memlimit_inactive;
5604 }
5605 if (p->p_memstat_state & P_MEMSTAT_MEMLIMIT_INACTIVE_FATAL) {
5606 mmp_entry.memlimit_inactive_attr |= MEMORYSTATUS_MEMLIMIT_ATTR_FATAL;
5607 }
5608 proc_rele(p);
5609
5610 error = copyout(&mmp_entry, buffer, buffer_size);
5611
5612 return(error);
5613 }
5614
5615
5616 static int
5617 memorystatus_cmd_get_pressure_status(int32_t *retval) {
5618 int error;
5619
5620 /* Need privilege for check */
5621 error = priv_check_cred(kauth_cred_get(), PRIV_VM_PRESSURE, 0);
5622 if (error) {
5623 return (error);
5624 }
5625
5626 /* Inherently racy, so it's not worth taking a lock here */
5627 *retval = (kVMPressureNormal != memorystatus_vm_pressure_level) ? 1 : 0;
5628
5629 return error;
5630 }
5631
5632 int
5633 memorystatus_get_pressure_status_kdp() {
5634 return (kVMPressureNormal != memorystatus_vm_pressure_level) ? 1 : 0;
5635 }
5636
5637 /*
5638 * Every process, including a P_MEMSTAT_INTERNAL process (currently only pid 1), is allowed to set a HWM.
5639 *
5640 * This call is inflexible -- it does not distinguish between active/inactive, fatal/non-fatal
5641 * So, with 2-level HWM preserving previous behavior will map as follows.
5642 * - treat the limit passed in as both an active and inactive limit.
5643 * - treat the is_fatal_limit flag as though it applies to both active and inactive limits.
5644 *
5645 * When invoked via MEMORYSTATUS_CMD_SET_JETSAM_HIGH_WATER_MARK
5646 * - the is_fatal_limit is FALSE, meaning the active and inactive limits are non-fatal/soft
5647 * - so mapping is (active/non-fatal, inactive/non-fatal)
5648 *
5649 * When invoked via MEMORYSTATUS_CMD_SET_JETSAM_TASK_LIMIT
5650 * - the is_fatal_limit is TRUE, meaning the process's active and inactive limits are fatal/hard
5651 * - so mapping is (active/fatal, inactive/fatal)
5652 */
5653
5654 static int
5655 memorystatus_cmd_set_jetsam_memory_limit(pid_t pid, int32_t high_water_mark, __unused int32_t *retval, boolean_t is_fatal_limit) {
5656 int error = 0;
5657 memorystatus_memlimit_properties_t entry;
5658
5659 entry.memlimit_active = high_water_mark;
5660 entry.memlimit_active_attr = 0;
5661 entry.memlimit_inactive = high_water_mark;
5662 entry.memlimit_inactive_attr = 0;
5663
5664 if (is_fatal_limit == TRUE) {
5665 entry.memlimit_active_attr |= MEMORYSTATUS_MEMLIMIT_ATTR_FATAL;
5666 entry.memlimit_inactive_attr |= MEMORYSTATUS_MEMLIMIT_ATTR_FATAL;
5667 }
5668
5669 error = memorystatus_set_memlimit_properties(pid, &entry);
5670 return (error);
5671 }
5672
5673 static int
5674 memorystatus_set_memlimit_properties(pid_t pid, memorystatus_memlimit_properties_t *entry) {
5675
5676 int32_t memlimit_active;
5677 boolean_t memlimit_active_is_fatal;
5678 int32_t memlimit_inactive;
5679 boolean_t memlimit_inactive_is_fatal;
5680 uint32_t valid_attrs = 0;
5681 int error = 0;
5682
5683 proc_t p = proc_find(pid);
5684 if (!p) {
5685 return ESRCH;
5686 }
5687
5688 /*
5689 * Check for valid attribute flags.
5690 */
5691 valid_attrs |= (MEMORYSTATUS_MEMLIMIT_ATTR_FATAL);
5692 if ((entry->memlimit_active_attr & (~valid_attrs)) != 0) {
5693 proc_rele(p);
5694 return EINVAL;
5695 }
5696 if ((entry->memlimit_inactive_attr & (~valid_attrs)) != 0) {
5697 proc_rele(p);
5698 return EINVAL;
5699 }
5700
5701 /*
5702 * Setup the active memlimit properties
5703 */
5704 memlimit_active = entry->memlimit_active;
5705 if (entry->memlimit_active_attr & MEMORYSTATUS_MEMLIMIT_ATTR_FATAL) {
5706 memlimit_active_is_fatal = TRUE;
5707 } else {
5708 memlimit_active_is_fatal = FALSE;
5709 }
5710
5711 /*
5712 * Setup the inactive memlimit properties
5713 */
5714 memlimit_inactive = entry->memlimit_inactive;
5715 if (entry->memlimit_inactive_attr & MEMORYSTATUS_MEMLIMIT_ATTR_FATAL) {
5716 memlimit_inactive_is_fatal = TRUE;
5717 } else {
5718 memlimit_inactive_is_fatal = FALSE;
5719 }
5720
5721 /*
5722 * Setting a limit of <= 0 implies that the process has no
5723 * high-water-mark and has no per-task-limit. That means
5724 * the system_wide task limit is in place, which by the way,
5725 * is always fatal.
5726 */
5727
5728 if (memlimit_active <= 0) {
5729 /*
5730 * Enforce the fatal system_wide task limit while process is active.
5731 */
5732 memlimit_active = -1;
5733 memlimit_active_is_fatal = TRUE;
5734 }
5735
5736 if (memlimit_inactive <= 0) {
5737 /*
5738 * Enforce the fatal system_wide task limit while process is inactive.
5739 */
5740 memlimit_inactive = -1;
5741 memlimit_inactive_is_fatal = TRUE;
5742 }
5743
5744 proc_list_lock();
5745
5746 /*
5747 * Store the active limit variants in the proc.
5748 */
5749 SET_ACTIVE_LIMITS_LOCKED(p, memlimit_active, memlimit_active_is_fatal);
5750
5751 /*
5752 * Store the inactive limit variants in the proc.
5753 */
5754 SET_INACTIVE_LIMITS_LOCKED(p, memlimit_inactive, memlimit_inactive_is_fatal);
5755
5756 /*
5757 * Enforce appropriate limit variant by updating the cached values
5758 * and writing the ledger.
5759 * Limit choice is based on process active/inactive state.
5760 */
5761
5762 if (memorystatus_highwater_enabled) {
5763 boolean_t trigger_exception;
5764 /*
5765 * No need to consider P_MEMSTAT_MEMLIMIT_BACKGROUND anymore.
5766 * Background limits are described via the inactive limit slots.
5767 */
5768
5769 if (proc_jetsam_state_is_active_locked(p) == TRUE) {
5770 CACHE_ACTIVE_LIMITS_LOCKED(p, trigger_exception);
5771 } else {
5772 CACHE_INACTIVE_LIMITS_LOCKED(p, trigger_exception);
5773 }
5774
5775 /* Enforce the limit by writing to the ledgers */
5776 assert(trigger_exception == TRUE);
5777 error = (task_set_phys_footprint_limit_internal(p->task, ((p->p_memstat_memlimit > 0) ? p->p_memstat_memlimit : -1), NULL, trigger_exception) == 0) ? 0 : EINVAL;
5778
5779 MEMORYSTATUS_DEBUG(3, "memorystatus_set_memlimit_properties: new limit on pid %d (%dMB %s) current priority (%d) dirty_state?=0x%x %s\n",
5780 p->p_pid, (p->p_memstat_memlimit > 0 ? p->p_memstat_memlimit : -1),
5781 (p->p_memstat_state & P_MEMSTAT_FATAL_MEMLIMIT ? "F " : "NF"), p->p_memstat_effectivepriority, p->p_memstat_dirty,
5782 (p->p_memstat_dirty ? ((p->p_memstat_dirty & P_DIRTY) ? "isdirty" : "isclean") : ""));
5783 }
5784
5785 proc_list_unlock();
5786 proc_rele(p);
5787
5788 return error;
5789 }
5790
5791 /*
5792 * Returns the jetsam priority (effective or requested) of the process
5793 * associated with this task.
5794 */
5795 int
5796 proc_get_memstat_priority(proc_t p, boolean_t effective_priority)
5797 {
5798 if (p) {
5799 if (effective_priority) {
5800 return p->p_memstat_effectivepriority;
5801 } else {
5802 return p->p_memstat_requestedpriority;
5803 }
5804 }
5805 return 0;
5806 }
5807
5808 /*
5809 * Description:
5810 * Evaluates active vs. inactive process state.
5811 * Processes that opt into dirty tracking are evaluated
5812 * based on clean vs dirty state.
5813 * dirty ==> active
5814 * clean ==> inactive
5815 *
5816 * Process that do not opt into dirty tracking are
5817 * evalulated based on priority level.
5818 * Foreground or above ==> active
5819 * Below Foreground ==> inactive
5820 *
5821 * Return: TRUE if active
5822 * False if inactive
5823 */
5824
5825 static boolean_t
5826 proc_jetsam_state_is_active_locked(proc_t p) {
5827
5828 if (p->p_memstat_dirty & P_DIRTY_TRACK) {
5829 /*
5830 * process has opted into dirty tracking
5831 * active state is based on dirty vs. clean
5832 */
5833 if (p->p_memstat_dirty & P_DIRTY_IS_DIRTY) {
5834 /*
5835 * process is dirty
5836 * implies active state
5837 */
5838 return TRUE;
5839 } else {
5840 /*
5841 * process is clean
5842 * implies inactive state
5843 */
5844 return FALSE;
5845 }
5846 } else if (p->p_memstat_effectivepriority >= JETSAM_PRIORITY_FOREGROUND) {
5847 /*
5848 * process is Foreground or higher
5849 * implies active state
5850 */
5851 return TRUE;
5852 } else {
5853 /*
5854 * process found below Foreground
5855 * implies inactive state
5856 */
5857 return FALSE;
5858 }
5859 }
5860
5861 #endif /* CONFIG_JETSAM */
5862
5863 int
5864 memorystatus_control(struct proc *p __unused, struct memorystatus_control_args *args, int *ret) {
5865 int error = EINVAL;
5866
5867 #if !CONFIG_JETSAM
5868 #pragma unused(ret)
5869 #endif
5870
5871 /* Root only for now */
5872 if (!kauth_cred_issuser(kauth_cred_get())) {
5873 error = EPERM;
5874 goto out;
5875 }
5876
5877 /* Sanity check */
5878 if (args->buffersize > MEMORYSTATUS_BUFFERSIZE_MAX) {
5879 error = EINVAL;
5880 goto out;
5881 }
5882
5883 switch (args->command) {
5884 case MEMORYSTATUS_CMD_GET_PRIORITY_LIST:
5885 error = memorystatus_cmd_get_priority_list(args->buffer, args->buffersize, ret);
5886 break;
5887 #if CONFIG_JETSAM
5888 case MEMORYSTATUS_CMD_SET_PRIORITY_PROPERTIES:
5889 error = memorystatus_cmd_set_priority_properties(args->pid, args->buffer, args->buffersize, ret);
5890 break;
5891 case MEMORYSTATUS_CMD_SET_MEMLIMIT_PROPERTIES:
5892 error = memorystatus_cmd_set_memlimit_properties(args->pid, args->buffer, args->buffersize, ret);
5893 break;
5894 case MEMORYSTATUS_CMD_GET_MEMLIMIT_PROPERTIES:
5895 error = memorystatus_cmd_get_memlimit_properties(args->pid, args->buffer, args->buffersize, ret);
5896 break;
5897 case MEMORYSTATUS_CMD_GRP_SET_PROPERTIES:
5898 error = memorystatus_cmd_grp_set_properties((int32_t)args->flags, args->buffer, args->buffersize, ret);
5899 break;
5900 case MEMORYSTATUS_CMD_GET_JETSAM_SNAPSHOT:
5901 error = memorystatus_cmd_get_jetsam_snapshot((int32_t)args->flags, args->buffer, args->buffersize, ret);
5902 break;
5903 case MEMORYSTATUS_CMD_GET_PRESSURE_STATUS:
5904 error = memorystatus_cmd_get_pressure_status(ret);
5905 break;
5906 case MEMORYSTATUS_CMD_SET_JETSAM_HIGH_WATER_MARK:
5907 /*
5908 * This call does not distinguish between active and inactive limits.
5909 * Default behavior in 2-level HWM world is to set both.
5910 * Non-fatal limit is also assumed for both.
5911 */
5912 error = memorystatus_cmd_set_jetsam_memory_limit(args->pid, (int32_t)args->flags, ret, FALSE);
5913 break;
5914 case MEMORYSTATUS_CMD_SET_JETSAM_TASK_LIMIT:
5915 /*
5916 * This call does not distinguish between active and inactive limits.
5917 * Default behavior in 2-level HWM world is to set both.
5918 * Fatal limit is also assumed for both.
5919 */
5920 error = memorystatus_cmd_set_jetsam_memory_limit(args->pid, (int32_t)args->flags, ret, TRUE);
5921 break;
5922 /* Test commands */
5923 #if DEVELOPMENT || DEBUG
5924 case MEMORYSTATUS_CMD_TEST_JETSAM:
5925 error = memorystatus_kill_process_sync(args->pid, kMemorystatusKilled) ? 0 : EINVAL;
5926 break;
5927 case MEMORYSTATUS_CMD_TEST_JETSAM_SORT:
5928 error = memorystatus_cmd_test_jetsam_sort(args->pid, (int32_t)args->flags);
5929 break;
5930 case MEMORYSTATUS_CMD_SET_JETSAM_PANIC_BITS:
5931 error = memorystatus_cmd_set_panic_bits(args->buffer, args->buffersize);
5932 break;
5933 #endif /* DEVELOPMENT || DEBUG */
5934 case MEMORYSTATUS_CMD_AGGRESSIVE_JETSAM_LENIENT_MODE_ENABLE:
5935 if (memorystatus_aggressive_jetsam_lenient_allowed == FALSE) {
5936 #if DEVELOPMENT || DEBUG
5937 printf("Enabling Lenient Mode\n");
5938 #endif /* DEVELOPMENT || DEBUG */
5939
5940 memorystatus_aggressive_jetsam_lenient_allowed = TRUE;
5941 memorystatus_aggressive_jetsam_lenient = TRUE;
5942 }
5943 break;
5944 case MEMORYSTATUS_CMD_AGGRESSIVE_JETSAM_LENIENT_MODE_DISABLE:
5945 #if DEVELOPMENT || DEBUG
5946 printf("Disabling Lenient mode\n");
5947 #endif /* DEVELOPMENT || DEBUG */
5948 memorystatus_aggressive_jetsam_lenient_allowed = FALSE;
5949 memorystatus_aggressive_jetsam_lenient = FALSE;
5950 break;
5951 #endif /* CONFIG_JETSAM */
5952 case MEMORYSTATUS_CMD_PRIVILEGED_LISTENER_ENABLE:
5953 case MEMORYSTATUS_CMD_PRIVILEGED_LISTENER_DISABLE:
5954 error = memorystatus_low_mem_privileged_listener(args->command);
5955 break;
5956 default:
5957 break;
5958 }
5959
5960 out:
5961 return error;
5962 }
5963
5964
5965 static int
5966 filt_memorystatusattach(struct knote *kn)
5967 {
5968 kn->kn_flags |= EV_CLEAR;
5969 return memorystatus_knote_register(kn);
5970 }
5971
5972 static void
5973 filt_memorystatusdetach(struct knote *kn)
5974 {
5975 memorystatus_knote_unregister(kn);
5976 }
5977
5978 static int
5979 filt_memorystatus(struct knote *kn __unused, long hint)
5980 {
5981 if (hint) {
5982 switch (hint) {
5983 case kMemorystatusNoPressure:
5984 if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PRESSURE_NORMAL) {
5985 kn->kn_fflags = NOTE_MEMORYSTATUS_PRESSURE_NORMAL;
5986 }
5987 break;
5988 case kMemorystatusPressure:
5989 if (memorystatus_vm_pressure_level == kVMPressureWarning || memorystatus_vm_pressure_level == kVMPressureUrgent) {
5990 if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PRESSURE_WARN) {
5991 kn->kn_fflags = NOTE_MEMORYSTATUS_PRESSURE_WARN;
5992 }
5993 } else if (memorystatus_vm_pressure_level == kVMPressureCritical) {
5994
5995 if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PRESSURE_CRITICAL) {
5996 kn->kn_fflags = NOTE_MEMORYSTATUS_PRESSURE_CRITICAL;
5997 }
5998 }
5999 break;
6000 case kMemorystatusLowSwap:
6001 if (kn->kn_sfflags & NOTE_MEMORYSTATUS_LOW_SWAP) {
6002 kn->kn_fflags = NOTE_MEMORYSTATUS_LOW_SWAP;
6003 }
6004 break;
6005 default:
6006 break;
6007 }
6008 }
6009
6010 return (kn->kn_fflags != 0);
6011 }
6012
6013 static void
6014 memorystatus_klist_lock(void) {
6015 lck_mtx_lock(&memorystatus_klist_mutex);
6016 }
6017
6018 static void
6019 memorystatus_klist_unlock(void) {
6020 lck_mtx_unlock(&memorystatus_klist_mutex);
6021 }
6022
6023 void
6024 memorystatus_kevent_init(lck_grp_t *grp, lck_attr_t *attr) {
6025 lck_mtx_init(&memorystatus_klist_mutex, grp, attr);
6026 klist_init(&memorystatus_klist);
6027 }
6028
6029 int
6030 memorystatus_knote_register(struct knote *kn) {
6031 int error = 0;
6032
6033 memorystatus_klist_lock();
6034
6035 if (kn->kn_sfflags & (NOTE_MEMORYSTATUS_PRESSURE_NORMAL | NOTE_MEMORYSTATUS_PRESSURE_WARN | NOTE_MEMORYSTATUS_PRESSURE_CRITICAL | NOTE_MEMORYSTATUS_LOW_SWAP)) {
6036
6037 KNOTE_ATTACH(&memorystatus_klist, kn);
6038
6039 } else {
6040 error = ENOTSUP;
6041 }
6042
6043 memorystatus_klist_unlock();
6044
6045 return error;
6046 }
6047
6048 void
6049 memorystatus_knote_unregister(struct knote *kn __unused) {
6050 memorystatus_klist_lock();
6051 KNOTE_DETACH(&memorystatus_klist, kn);
6052 memorystatus_klist_unlock();
6053 }
6054
6055
6056 #if 0
6057 #if CONFIG_JETSAM && VM_PRESSURE_EVENTS
6058 static boolean_t
6059 memorystatus_issue_pressure_kevent(boolean_t pressured) {
6060 memorystatus_klist_lock();
6061 KNOTE(&memorystatus_klist, pressured ? kMemorystatusPressure : kMemorystatusNoPressure);
6062 memorystatus_klist_unlock();
6063 return TRUE;
6064 }
6065 #endif /* CONFIG_JETSAM && VM_PRESSURE_EVENTS */
6066 #endif /* 0 */
6067
6068 #if CONFIG_JETSAM
6069 /* Coalition support */
6070
6071 /* sorting info for a particular priority bucket */
6072 typedef struct memstat_sort_info {
6073 coalition_t msi_coal;
6074 uint64_t msi_page_count;
6075 pid_t msi_pid;
6076 int msi_ntasks;
6077 } memstat_sort_info_t;
6078
6079 /*
6080 * qsort from smallest page count to largest page count
6081 *
6082 * return < 0 for a < b
6083 * 0 for a == b
6084 * > 0 for a > b
6085 */
6086 static int memstat_asc_cmp(const void *a, const void *b)
6087 {
6088 const memstat_sort_info_t *msA = (const memstat_sort_info_t *)a;
6089 const memstat_sort_info_t *msB = (const memstat_sort_info_t *)b;
6090
6091 return (int)((uint64_t)msA->msi_page_count - (uint64_t)msB->msi_page_count);
6092 }
6093
6094 /*
6095 * Return the number of pids rearranged during this sort.
6096 */
6097 static int
6098 memorystatus_sort_by_largest_coalition_locked(unsigned int bucket_index, int coal_sort_order)
6099 {
6100 #define MAX_SORT_PIDS 80
6101 #define MAX_COAL_LEADERS 10
6102
6103 unsigned int b = bucket_index;
6104 int nleaders = 0;
6105 int ntasks = 0;
6106 proc_t p = NULL;
6107 coalition_t coal = COALITION_NULL;
6108 int pids_moved = 0;
6109 int total_pids_moved = 0;
6110 int i;
6111
6112 /*
6113 * The system is typically under memory pressure when in this
6114 * path, hence, we want to avoid dynamic memory allocation.
6115 */
6116 memstat_sort_info_t leaders[MAX_COAL_LEADERS];
6117 pid_t pid_list[MAX_SORT_PIDS];
6118
6119 if (bucket_index >= MEMSTAT_BUCKET_COUNT) {
6120 return(0);
6121 }
6122
6123 /*
6124 * Clear the array that holds coalition leader information
6125 */
6126 for (i=0; i < MAX_COAL_LEADERS; i++) {
6127 leaders[i].msi_coal = COALITION_NULL;
6128 leaders[i].msi_page_count = 0; /* will hold total coalition page count */
6129 leaders[i].msi_pid = 0; /* will hold coalition leader pid */
6130 leaders[i].msi_ntasks = 0; /* will hold the number of tasks in a coalition */
6131 }
6132
6133 p = memorystatus_get_first_proc_locked(&b, FALSE);
6134 while (p) {
6135 if (coalition_is_leader(p->task, COALITION_TYPE_JETSAM, &coal)) {
6136 if (nleaders < MAX_COAL_LEADERS) {
6137 int coal_ntasks = 0;
6138 uint64_t coal_page_count = coalition_get_page_count(coal, &coal_ntasks);
6139 leaders[nleaders].msi_coal = coal;
6140 leaders[nleaders].msi_page_count = coal_page_count;
6141 leaders[nleaders].msi_pid = p->p_pid; /* the coalition leader */
6142 leaders[nleaders].msi_ntasks = coal_ntasks;
6143 nleaders++;
6144 } else {
6145 /*
6146 * We've hit MAX_COAL_LEADERS meaning we can handle no more coalitions.
6147 * Abandoned coalitions will linger at the tail of the priority band
6148 * when this sort session ends.
6149 * TODO: should this be an assert?
6150 */
6151 printf("%s: WARNING: more than %d leaders in priority band [%d]\n",
6152 __FUNCTION__, MAX_COAL_LEADERS, bucket_index);
6153 break;
6154 }
6155 }
6156 p=memorystatus_get_next_proc_locked(&b, p, FALSE);
6157 }
6158
6159 if (nleaders == 0) {
6160 /* Nothing to sort */
6161 return(0);
6162 }
6163
6164 /*
6165 * Sort the coalition leader array, from smallest coalition page count
6166 * to largest coalition page count. When inserted in the priority bucket,
6167 * smallest coalition is handled first, resulting in the last to be jetsammed.
6168 */
6169 if (nleaders > 1) {
6170 qsort(leaders, nleaders, sizeof(memstat_sort_info_t), memstat_asc_cmp);
6171 }
6172
6173 #if 0
6174 for (i = 0; i < nleaders; i++) {
6175 printf("%s: coal_leader[%d of %d] pid[%d] pages[%llu] ntasks[%d]\n",
6176 __FUNCTION__, i, nleaders, leaders[i].msi_pid, leaders[i].msi_page_count,
6177 leaders[i].msi_ntasks);
6178 }
6179 #endif
6180
6181 /*
6182 * During coalition sorting, processes in a priority band are rearranged
6183 * by being re-inserted at the head of the queue. So, when handling a
6184 * list, the first process that gets moved to the head of the queue,
6185 * ultimately gets pushed toward the queue tail, and hence, jetsams last.
6186 *
6187 * So, for example, the coalition leader is expected to jetsam last,
6188 * after its coalition members. Therefore, the coalition leader is
6189 * inserted at the head of the queue first.
6190 *
6191 * After processing a coalition, the jetsam order is as follows:
6192 * undefs(jetsam first), extensions, xpc services, leader(jetsam last)
6193 */
6194
6195 /*
6196 * Coalition members are rearranged in the priority bucket here,
6197 * based on their coalition role.
6198 */
6199 total_pids_moved = 0;
6200 for (i=0; i < nleaders; i++) {
6201
6202 /* a bit of bookkeeping */
6203 pids_moved = 0;
6204
6205 /* Coalition leaders are jetsammed last, so move into place first */
6206 pid_list[0] = leaders[i].msi_pid;
6207 pids_moved += memorystatus_move_list_locked(bucket_index, pid_list, 1);
6208
6209 /* xpc services should jetsam after extensions */
6210 ntasks = coalition_get_pid_list (leaders[i].msi_coal, COALITION_ROLEMASK_XPC,
6211 coal_sort_order, pid_list, MAX_SORT_PIDS);
6212
6213 if (ntasks > 0) {
6214 pids_moved += memorystatus_move_list_locked(bucket_index, pid_list,
6215 (ntasks <= MAX_SORT_PIDS ? ntasks : MAX_SORT_PIDS));
6216 }
6217
6218 /* extensions should jetsam after unmarked processes */
6219 ntasks = coalition_get_pid_list (leaders[i].msi_coal, COALITION_ROLEMASK_EXT,
6220 coal_sort_order, pid_list, MAX_SORT_PIDS);
6221
6222 if (ntasks > 0) {
6223 pids_moved += memorystatus_move_list_locked(bucket_index, pid_list,
6224 (ntasks <= MAX_SORT_PIDS ? ntasks : MAX_SORT_PIDS));
6225 }
6226
6227 /* undefined coalition members should be the first to jetsam */
6228 ntasks = coalition_get_pid_list (leaders[i].msi_coal, COALITION_ROLEMASK_UNDEF,
6229 coal_sort_order, pid_list, MAX_SORT_PIDS);
6230
6231 if (ntasks > 0) {
6232 pids_moved += memorystatus_move_list_locked(bucket_index, pid_list,
6233 (ntasks <= MAX_SORT_PIDS ? ntasks : MAX_SORT_PIDS));
6234 }
6235
6236 #if 0
6237 if (pids_moved == leaders[i].msi_ntasks) {
6238 /*
6239 * All the pids in the coalition were found in this band.
6240 */
6241 printf("%s: pids_moved[%d] equal total coalition ntasks[%d] \n", __FUNCTION__,
6242 pids_moved, leaders[i].msi_ntasks);
6243 } else if (pids_moved > leaders[i].msi_ntasks) {
6244 /*
6245 * Apparently new coalition members showed up during the sort?
6246 */
6247 printf("%s: pids_moved[%d] were greater than expected coalition ntasks[%d] \n", __FUNCTION__,
6248 pids_moved, leaders[i].msi_ntasks);
6249 } else {
6250 /*
6251 * Apparently not all the pids in the coalition were found in this band?
6252 */
6253 printf("%s: pids_moved[%d] were less than expected coalition ntasks[%d] \n", __FUNCTION__,
6254 pids_moved, leaders[i].msi_ntasks);
6255 }
6256 #endif
6257
6258 total_pids_moved += pids_moved;
6259
6260 } /* end for */
6261
6262 return(total_pids_moved);
6263 }
6264
6265
6266 /*
6267 * Traverse a list of pids, searching for each within the priority band provided.
6268 * If pid is found, move it to the front of the priority band.
6269 * Never searches outside the priority band provided.
6270 *
6271 * Input:
6272 * bucket_index - jetsam priority band.
6273 * pid_list - pointer to a list of pids.
6274 * list_sz - number of pids in the list.
6275 *
6276 * Pid list ordering is important in that,
6277 * pid_list[n] is expected to jetsam ahead of pid_list[n+1].
6278 * The sort_order is set by the coalition default.
6279 *
6280 * Return:
6281 * the number of pids found and hence moved within the priority band.
6282 */
6283 static int
6284 memorystatus_move_list_locked(unsigned int bucket_index, pid_t *pid_list, int list_sz)
6285 {
6286 memstat_bucket_t *current_bucket;
6287 int i;
6288 int found_pids = 0;
6289
6290 if ((pid_list == NULL) || (list_sz <= 0)) {
6291 return(0);
6292 }
6293
6294 if (bucket_index >= MEMSTAT_BUCKET_COUNT) {
6295 return(0);
6296 }
6297
6298 current_bucket = &memstat_bucket[bucket_index];
6299 for (i=0; i < list_sz; i++) {
6300 unsigned int b = bucket_index;
6301 proc_t p = NULL;
6302 proc_t aProc = NULL;
6303 pid_t aPid;
6304 int list_index;
6305
6306 list_index = ((list_sz - 1) - i);
6307 aPid = pid_list[list_index];
6308
6309 /* never search beyond bucket_index provided */
6310 p = memorystatus_get_first_proc_locked(&b, FALSE);
6311 while (p) {
6312 if (p->p_pid == aPid) {
6313 aProc = p;
6314 break;
6315 }
6316 p = memorystatus_get_next_proc_locked(&b, p, FALSE);
6317 }
6318
6319 if (aProc == NULL) {
6320 /* pid not found in this band, just skip it */
6321 continue;
6322 } else {
6323 TAILQ_REMOVE(&current_bucket->list, aProc, p_memstat_list);
6324 TAILQ_INSERT_HEAD(&current_bucket->list, aProc, p_memstat_list);
6325 found_pids++;
6326 }
6327 }
6328 return(found_pids);
6329 }
6330 #endif /* CONFIG_JETSAM */
mirror of XNU