]> git.saurik.com Git - apple/xnu.git/blob - bsd/kern/kern_memorystatus.c
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-4570.71.2.tar.gz
[apple/xnu.git] / bsd / kern / kern_memorystatus.c
1 /*
2 * Copyright (c) 2006-2018 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 *
28 */
29
30 #include <kern/sched_prim.h>
31 #include <kern/kalloc.h>
32 #include <kern/assert.h>
33 #include <kern/debug.h>
34 #include <kern/locks.h>
35 #include <kern/task.h>
36 #include <kern/thread.h>
37 #include <kern/host.h>
38 #include <kern/policy_internal.h>
39 #include <kern/thread_group.h>
40
41 #include <IOKit/IOBSD.h>
42
43 #include <libkern/libkern.h>
44 #include <mach/coalition.h>
45 #include <mach/mach_time.h>
46 #include <mach/task.h>
47 #include <mach/host_priv.h>
48 #include <mach/mach_host.h>
49 #include <os/log.h>
50 #include <pexpert/pexpert.h>
51 #include <sys/coalition.h>
52 #include <sys/kern_event.h>
53 #include <sys/proc.h>
54 #include <sys/proc_info.h>
55 #include <sys/reason.h>
56 #include <sys/signal.h>
57 #include <sys/signalvar.h>
58 #include <sys/sysctl.h>
59 #include <sys/sysproto.h>
60 #include <sys/wait.h>
61 #include <sys/tree.h>
62 #include <sys/priv.h>
63 #include <vm/vm_pageout.h>
64 #include <vm/vm_protos.h>
65
66 #if CONFIG_FREEZE
67 #include <vm/vm_map.h>
68 #endif /* CONFIG_FREEZE */
69
70 #include <sys/kern_memorystatus.h>
71
72 #include <mach/machine/sdt.h>
73 #include <libkern/section_keywords.h>
74
75 /* For logging clarity */
76 static const char *memorystatus_kill_cause_name[] = {
77 "" ,
78 "jettisoned" , /* kMemorystatusKilled */
79 "highwater" , /* kMemorystatusKilledHiwat */
80 "vnode-limit" , /* kMemorystatusKilledVnodes */
81 "vm-pageshortage" , /* kMemorystatusKilledVMPageShortage */
82 "vm-thrashing" , /* kMemorystatusKilledVMThrashing */
83 "fc-thrashing" , /* kMemorystatusKilledFCThrashing */
84 "per-process-limit" , /* kMemorystatusKilledPerProcessLimit */
85 "diagnostic" , /* kMemorystatusKilledDiagnostic */
86 "idle-exit" , /* kMemorystatusKilledIdleExit */
87 "zone-map-exhaustion" , /* kMemorystatusKilledZoneMapExhaustion */
88 };
89
90 static const char *
91 memorystatus_priority_band_name(int32_t priority)
92 {
93 switch (priority) {
94 case JETSAM_PRIORITY_FOREGROUND:
95 return "FOREGROUND";
96 case JETSAM_PRIORITY_AUDIO_AND_ACCESSORY:
97 return "AUDIO_AND_ACCESSORY";
98 case JETSAM_PRIORITY_CONDUCTOR:
99 return "CONDUCTOR";
100 case JETSAM_PRIORITY_HOME:
101 return "HOME";
102 case JETSAM_PRIORITY_EXECUTIVE:
103 return "EXECUTIVE";
104 case JETSAM_PRIORITY_IMPORTANT:
105 return "IMPORTANT";
106 case JETSAM_PRIORITY_CRITICAL:
107 return "CRITICAL";
108 }
109
110 return ("?");
111 }
112
113 /* Does cause indicate vm or fc thrashing? */
114 static boolean_t
115 is_reason_thrashing(unsigned cause)
116 {
117 switch (cause) {
118 case kMemorystatusKilledVMThrashing:
119 case kMemorystatusKilledFCThrashing:
120 return TRUE;
121 default:
122 return FALSE;
123 }
124 }
125
126 /* Is the zone map almost full? */
127 static boolean_t
128 is_reason_zone_map_exhaustion(unsigned cause)
129 {
130 if (cause == kMemorystatusKilledZoneMapExhaustion)
131 return TRUE;
132 return FALSE;
133 }
134
135 /*
136 * Returns the current zone map size and capacity to include in the jetsam snapshot.
137 * Defined in zalloc.c
138 */
139 extern void get_zone_map_size(uint64_t *current_size, uint64_t *capacity);
140
141 /*
142 * Returns the name of the largest zone and its size to include in the jetsam snapshot.
143 * Defined in zalloc.c
144 */
145 extern void get_largest_zone_info(char *zone_name, size_t zone_name_len, uint64_t *zone_size);
146
147 /* These are very verbose printfs(), enable with
148 * MEMORYSTATUS_DEBUG_LOG
149 */
150 #if MEMORYSTATUS_DEBUG_LOG
151 #define MEMORYSTATUS_DEBUG(cond, format, ...) \
152 do { \
153 if (cond) { printf(format, ##__VA_ARGS__); } \
154 } while(0)
155 #else
156 #define MEMORYSTATUS_DEBUG(cond, format, ...)
157 #endif
158
159 /*
160 * Active / Inactive limit support
161 * proc list must be locked
162 *
163 * The SET_*** macros are used to initialize a limit
164 * for the first time.
165 *
166 * The CACHE_*** macros are use to cache the limit that will
167 * soon be in effect down in the ledgers.
168 */
169
170 #define SET_ACTIVE_LIMITS_LOCKED(p, limit, is_fatal) \
171 MACRO_BEGIN \
172 (p)->p_memstat_memlimit_active = (limit); \
173 if (is_fatal) { \
174 (p)->p_memstat_state |= P_MEMSTAT_MEMLIMIT_ACTIVE_FATAL; \
175 } else { \
176 (p)->p_memstat_state &= ~P_MEMSTAT_MEMLIMIT_ACTIVE_FATAL; \
177 } \
178 MACRO_END
179
180 #define SET_INACTIVE_LIMITS_LOCKED(p, limit, is_fatal) \
181 MACRO_BEGIN \
182 (p)->p_memstat_memlimit_inactive = (limit); \
183 if (is_fatal) { \
184 (p)->p_memstat_state |= P_MEMSTAT_MEMLIMIT_INACTIVE_FATAL; \
185 } else { \
186 (p)->p_memstat_state &= ~P_MEMSTAT_MEMLIMIT_INACTIVE_FATAL; \
187 } \
188 MACRO_END
189
190 #define CACHE_ACTIVE_LIMITS_LOCKED(p, is_fatal) \
191 MACRO_BEGIN \
192 (p)->p_memstat_memlimit = (p)->p_memstat_memlimit_active; \
193 if ((p)->p_memstat_state & P_MEMSTAT_MEMLIMIT_ACTIVE_FATAL) { \
194 (p)->p_memstat_state |= P_MEMSTAT_FATAL_MEMLIMIT; \
195 is_fatal = TRUE; \
196 } else { \
197 (p)->p_memstat_state &= ~P_MEMSTAT_FATAL_MEMLIMIT; \
198 is_fatal = FALSE; \
199 } \
200 MACRO_END
201
202 #define CACHE_INACTIVE_LIMITS_LOCKED(p, is_fatal) \
203 MACRO_BEGIN \
204 (p)->p_memstat_memlimit = (p)->p_memstat_memlimit_inactive; \
205 if ((p)->p_memstat_state & P_MEMSTAT_MEMLIMIT_INACTIVE_FATAL) { \
206 (p)->p_memstat_state |= P_MEMSTAT_FATAL_MEMLIMIT; \
207 is_fatal = TRUE; \
208 } else { \
209 (p)->p_memstat_state &= ~P_MEMSTAT_FATAL_MEMLIMIT; \
210 is_fatal = FALSE; \
211 } \
212 MACRO_END
213
214
215 /* General tunables */
216
217 unsigned long delta_percentage = 5;
218 unsigned long critical_threshold_percentage = 5;
219 unsigned long idle_offset_percentage = 5;
220 unsigned long pressure_threshold_percentage = 15;
221 unsigned long freeze_threshold_percentage = 50;
222 unsigned long policy_more_free_offset_percentage = 5;
223
224 /* General memorystatus stuff */
225
226 struct klist memorystatus_klist;
227 static lck_mtx_t memorystatus_klist_mutex;
228
229 static void memorystatus_klist_lock(void);
230 static void memorystatus_klist_unlock(void);
231
232 static uint64_t memorystatus_sysprocs_idle_delay_time = 0;
233 static uint64_t memorystatus_apps_idle_delay_time = 0;
234
235 /*
236 * Memorystatus kevents
237 */
238
239 static int filt_memorystatusattach(struct knote *kn, struct kevent_internal_s *kev);
240 static void filt_memorystatusdetach(struct knote *kn);
241 static int filt_memorystatus(struct knote *kn, long hint);
242 static int filt_memorystatustouch(struct knote *kn, struct kevent_internal_s *kev);
243 static int filt_memorystatusprocess(struct knote *kn, struct filt_process_s *data, struct kevent_internal_s *kev);
244
245 SECURITY_READ_ONLY_EARLY(struct filterops) memorystatus_filtops = {
246 .f_attach = filt_memorystatusattach,
247 .f_detach = filt_memorystatusdetach,
248 .f_event = filt_memorystatus,
249 .f_touch = filt_memorystatustouch,
250 .f_process = filt_memorystatusprocess,
251 };
252
253 enum {
254 kMemorystatusNoPressure = 0x1,
255 kMemorystatusPressure = 0x2,
256 kMemorystatusLowSwap = 0x4,
257 kMemorystatusProcLimitWarn = 0x8,
258 kMemorystatusProcLimitCritical = 0x10
259 };
260
261 /* Idle guard handling */
262
263 static int32_t memorystatus_scheduled_idle_demotions_sysprocs = 0;
264 static int32_t memorystatus_scheduled_idle_demotions_apps = 0;
265
266 static thread_call_t memorystatus_idle_demotion_call;
267
268 static void memorystatus_perform_idle_demotion(__unused void *spare1, __unused void *spare2);
269 static void memorystatus_schedule_idle_demotion_locked(proc_t p, boolean_t set_state);
270 static void memorystatus_invalidate_idle_demotion_locked(proc_t p, boolean_t clean_state);
271 static void memorystatus_reschedule_idle_demotion_locked(void);
272
273 static void memorystatus_update_priority_locked(proc_t p, int priority, boolean_t head_insert, boolean_t skip_demotion_check);
274
275 int memorystatus_update_priority_for_appnap(proc_t p, boolean_t is_appnap);
276
277 vm_pressure_level_t convert_internal_pressure_level_to_dispatch_level(vm_pressure_level_t);
278
279 boolean_t is_knote_registered_modify_task_pressure_bits(struct knote*, int, task_t, vm_pressure_level_t, vm_pressure_level_t);
280 void memorystatus_klist_reset_all_for_level(vm_pressure_level_t pressure_level_to_clear);
281 void memorystatus_send_low_swap_note(void);
282
283 int memorystatus_wakeup = 0;
284
285 unsigned int memorystatus_level = 0;
286
287 static int memorystatus_list_count = 0;
288
289 #define MEMSTAT_BUCKET_COUNT (JETSAM_PRIORITY_MAX + 1)
290
291 typedef struct memstat_bucket {
292 TAILQ_HEAD(, proc) list;
293 int count;
294 } memstat_bucket_t;
295
296 memstat_bucket_t memstat_bucket[MEMSTAT_BUCKET_COUNT];
297
298 int memorystatus_get_proccnt_upto_priority(int32_t max_bucket_index);
299
300 uint64_t memstat_idle_demotion_deadline = 0;
301
302 int system_procs_aging_band = JETSAM_PRIORITY_AGING_BAND1;
303 int applications_aging_band = JETSAM_PRIORITY_IDLE;
304
305 #define isProcessInAgingBands(p) ((isSysProc(p) && system_procs_aging_band && (p->p_memstat_effectivepriority == system_procs_aging_band)) || (isApp(p) && applications_aging_band && (p->p_memstat_effectivepriority == applications_aging_band)))
306 #define isApp(p) (! (p->p_memstat_dirty & P_DIRTY_TRACK))
307 #define isSysProc(p) ((p->p_memstat_dirty & P_DIRTY_TRACK))
308
309 #define kJetsamAgingPolicyNone (0)
310 #define kJetsamAgingPolicyLegacy (1)
311 #define kJetsamAgingPolicySysProcsReclaimedFirst (2)
312 #define kJetsamAgingPolicyAppsReclaimedFirst (3)
313 #define kJetsamAgingPolicyMax kJetsamAgingPolicyAppsReclaimedFirst
314
315 unsigned int jetsam_aging_policy = kJetsamAgingPolicyLegacy;
316
317 extern int corpse_for_fatal_memkill;
318 extern unsigned long total_corpses_count(void) __attribute__((pure));
319 extern void task_purge_all_corpses(void);
320 extern uint64_t vm_purgeable_purge_task_owned(task_t task);
321 boolean_t memorystatus_allowed_vm_map_fork(task_t);
322 #if DEVELOPMENT || DEBUG
323 void memorystatus_abort_vm_map_fork(task_t);
324 #endif
325
326 #if 0
327
328 /* Keeping around for future use if we need a utility that can do this OR an app that needs a dynamic adjustment. */
329
330 static int
331 sysctl_set_jetsam_aging_policy SYSCTL_HANDLER_ARGS
332 {
333 #pragma unused(oidp, arg1, arg2)
334
335 int error = 0, val = 0;
336 memstat_bucket_t *old_bucket = 0;
337 int old_system_procs_aging_band = 0, new_system_procs_aging_band = 0;
338 int old_applications_aging_band = 0, new_applications_aging_band = 0;
339 proc_t p = NULL, next_proc = NULL;
340
341
342 error = sysctl_io_number(req, jetsam_aging_policy, sizeof(int), &val, NULL);
343 if (error || !req->newptr) {
344 return (error);
345 }
346
347 if ((val < 0) || (val > kJetsamAgingPolicyMax)) {
348 printf("jetsam: ordering policy sysctl has invalid value - %d\n", val);
349 return EINVAL;
350 }
351
352 /*
353 * We need to synchronize with any potential adding/removal from aging bands
354 * that might be in progress currently. We use the proc_list_lock() just for
355 * consistency with all the routines dealing with 'aging' processes. We need
356 * a lighterweight lock.
357 */
358 proc_list_lock();
359
360 old_system_procs_aging_band = system_procs_aging_band;
361 old_applications_aging_band = applications_aging_band;
362
363 switch (val) {
364
365 case kJetsamAgingPolicyNone:
366 new_system_procs_aging_band = JETSAM_PRIORITY_IDLE;
367 new_applications_aging_band = JETSAM_PRIORITY_IDLE;
368 break;
369
370 case kJetsamAgingPolicyLegacy:
371 /*
372 * Legacy behavior where some daemons get a 10s protection once and only before the first clean->dirty->clean transition before going into IDLE band.
373 */
374 new_system_procs_aging_band = JETSAM_PRIORITY_AGING_BAND1;
375 new_applications_aging_band = JETSAM_PRIORITY_IDLE;
376 break;
377
378 case kJetsamAgingPolicySysProcsReclaimedFirst:
379 new_system_procs_aging_band = JETSAM_PRIORITY_AGING_BAND1;
380 new_applications_aging_band = JETSAM_PRIORITY_AGING_BAND2;
381 break;
382
383 case kJetsamAgingPolicyAppsReclaimedFirst:
384 new_system_procs_aging_band = JETSAM_PRIORITY_AGING_BAND2;
385 new_applications_aging_band = JETSAM_PRIORITY_AGING_BAND1;
386 break;
387
388 default:
389 break;
390 }
391
392 if (old_system_procs_aging_band && (old_system_procs_aging_band != new_system_procs_aging_band)) {
393
394 old_bucket = &memstat_bucket[old_system_procs_aging_band];
395 p = TAILQ_FIRST(&old_bucket->list);
396
397 while (p) {
398
399 next_proc = TAILQ_NEXT(p, p_memstat_list);
400
401 if (isSysProc(p)) {
402 if (new_system_procs_aging_band == JETSAM_PRIORITY_IDLE) {
403 memorystatus_invalidate_idle_demotion_locked(p, TRUE);
404 }
405
406 memorystatus_update_priority_locked(p, new_system_procs_aging_band, false, true);
407 }
408
409 p = next_proc;
410 continue;
411 }
412 }
413
414 if (old_applications_aging_band && (old_applications_aging_band != new_applications_aging_band)) {
415
416 old_bucket = &memstat_bucket[old_applications_aging_band];
417 p = TAILQ_FIRST(&old_bucket->list);
418
419 while (p) {
420
421 next_proc = TAILQ_NEXT(p, p_memstat_list);
422
423 if (isApp(p)) {
424 if (new_applications_aging_band == JETSAM_PRIORITY_IDLE) {
425 memorystatus_invalidate_idle_demotion_locked(p, TRUE);
426 }
427
428 memorystatus_update_priority_locked(p, new_applications_aging_band, false, true);
429 }
430
431 p = next_proc;
432 continue;
433 }
434 }
435
436 jetsam_aging_policy = val;
437 system_procs_aging_band = new_system_procs_aging_band;
438 applications_aging_band = new_applications_aging_band;
439
440 proc_list_unlock();
441
442 return (0);
443 }
444
445 SYSCTL_PROC(_kern, OID_AUTO, set_jetsam_aging_policy, CTLTYPE_INT|CTLFLAG_RW,
446 0, 0, sysctl_set_jetsam_aging_policy, "I", "Jetsam Aging Policy");
447 #endif /*0*/
448
449 static int
450 sysctl_jetsam_set_sysprocs_idle_delay_time SYSCTL_HANDLER_ARGS
451 {
452 #pragma unused(oidp, arg1, arg2)
453
454 int error = 0, val = 0, old_time_in_secs = 0;
455 uint64_t old_time_in_ns = 0;
456
457 absolutetime_to_nanoseconds(memorystatus_sysprocs_idle_delay_time, &old_time_in_ns);
458 old_time_in_secs = old_time_in_ns / NSEC_PER_SEC;
459
460 error = sysctl_io_number(req, old_time_in_secs, sizeof(int), &val, NULL);
461 if (error || !req->newptr) {
462 return (error);
463 }
464
465 if ((val < 0) || (val > INT32_MAX)) {
466 printf("jetsam: new idle delay interval has invalid value.\n");
467 return EINVAL;
468 }
469
470 nanoseconds_to_absolutetime((uint64_t)val * NSEC_PER_SEC, &memorystatus_sysprocs_idle_delay_time);
471
472 return(0);
473 }
474
475 SYSCTL_PROC(_kern, OID_AUTO, memorystatus_sysprocs_idle_delay_time, CTLTYPE_INT|CTLFLAG_RW,
476 0, 0, sysctl_jetsam_set_sysprocs_idle_delay_time, "I", "Aging window for system processes");
477
478
479 static int
480 sysctl_jetsam_set_apps_idle_delay_time SYSCTL_HANDLER_ARGS
481 {
482 #pragma unused(oidp, arg1, arg2)
483
484 int error = 0, val = 0, old_time_in_secs = 0;
485 uint64_t old_time_in_ns = 0;
486
487 absolutetime_to_nanoseconds(memorystatus_apps_idle_delay_time, &old_time_in_ns);
488 old_time_in_secs = old_time_in_ns / NSEC_PER_SEC;
489
490 error = sysctl_io_number(req, old_time_in_secs, sizeof(int), &val, NULL);
491 if (error || !req->newptr) {
492 return (error);
493 }
494
495 if ((val < 0) || (val > INT32_MAX)) {
496 printf("jetsam: new idle delay interval has invalid value.\n");
497 return EINVAL;
498 }
499
500 nanoseconds_to_absolutetime((uint64_t)val * NSEC_PER_SEC, &memorystatus_apps_idle_delay_time);
501
502 return(0);
503 }
504
505 SYSCTL_PROC(_kern, OID_AUTO, memorystatus_apps_idle_delay_time, CTLTYPE_INT|CTLFLAG_RW,
506 0, 0, sysctl_jetsam_set_apps_idle_delay_time, "I", "Aging window for applications");
507
508 SYSCTL_INT(_kern, OID_AUTO, jetsam_aging_policy, CTLTYPE_INT|CTLFLAG_RD, &jetsam_aging_policy, 0, "");
509
510 static unsigned int memorystatus_dirty_count = 0;
511
512 SYSCTL_INT(_kern, OID_AUTO, max_task_pmem, CTLFLAG_RD|CTLFLAG_LOCKED|CTLFLAG_MASKED, &max_task_footprint_mb, 0, "");
513
514 #if CONFIG_EMBEDDED
515
516 SYSCTL_INT(_kern, OID_AUTO, memorystatus_level, CTLFLAG_RD|CTLFLAG_LOCKED, &memorystatus_level, 0, "");
517
518 #endif /* CONFIG_EMBEDDED */
519
520 int
521 memorystatus_get_level(__unused struct proc *p, struct memorystatus_get_level_args *args, __unused int *ret)
522 {
523 user_addr_t level = 0;
524
525 level = args->level;
526
527 if (copyout(&memorystatus_level, level, sizeof(memorystatus_level)) != 0) {
528 return EFAULT;
529 }
530
531 return 0;
532 }
533
534 static proc_t memorystatus_get_first_proc_locked(unsigned int *bucket_index, boolean_t search);
535 static proc_t memorystatus_get_next_proc_locked(unsigned int *bucket_index, proc_t p, boolean_t search);
536
537 static void memorystatus_thread(void *param __unused, wait_result_t wr __unused);
538
539 /* Memory Limits */
540
541 static int memorystatus_highwater_enabled = 1; /* Update the cached memlimit data. */
542
543 static boolean_t proc_jetsam_state_is_active_locked(proc_t);
544 static boolean_t memorystatus_kill_specific_process(pid_t victim_pid, uint32_t cause, os_reason_t jetsam_reason);
545 static boolean_t memorystatus_kill_process_sync(pid_t victim_pid, uint32_t cause, os_reason_t jetsam_reason);
546
547
548 static int memorystatus_cmd_set_memlimit_properties(pid_t pid, user_addr_t buffer, size_t buffer_size, __unused int32_t *retval);
549
550 static int memorystatus_set_memlimit_properties(pid_t pid, memorystatus_memlimit_properties_t *entry);
551
552 static int memorystatus_cmd_get_memlimit_properties(pid_t pid, user_addr_t buffer, size_t buffer_size, __unused int32_t *retval);
553
554 static int memorystatus_cmd_get_memlimit_excess_np(pid_t pid, uint32_t flags, user_addr_t buffer, size_t buffer_size, __unused int32_t *retval);
555
556 int proc_get_memstat_priority(proc_t, boolean_t);
557
558 static boolean_t memorystatus_idle_snapshot = 0;
559
560 unsigned int memorystatus_delta = 0;
561
562 /* Jetsam Loop Detection */
563 static boolean_t memorystatus_jld_enabled = FALSE; /* Enable jetsam loop detection */
564 static uint32_t memorystatus_jld_eval_period_msecs = 0; /* Init pass sets this based on device memory size */
565 static int memorystatus_jld_eval_aggressive_count = 3; /* Raise the priority max after 'n' aggressive loops */
566 static int memorystatus_jld_eval_aggressive_priority_band_max = 15; /* Kill aggressively up through this band */
567
568 /*
569 * A FG app can request that the aggressive jetsam mechanism display some leniency in the FG band. This 'lenient' mode is described as:
570 * --- if aggressive jetsam kills an app in the FG band and gets back >=AGGRESSIVE_JETSAM_LENIENT_MODE_THRESHOLD memory, it will stop the aggressive march further into and up the jetsam bands.
571 *
572 * RESTRICTIONS:
573 * - Such a request is respected/acknowledged only once while that 'requesting' app is in the FG band i.e. if aggressive jetsam was
574 * needed and the 'lenient' mode was deployed then that's it for this special mode while the app is in the FG band.
575 *
576 * - If the app is still in the FG band and aggressive jetsam is needed again, there will be no stop-and-check the next time around.
577 *
578 * - Also, the transition of the 'requesting' app away from the FG band will void this special behavior.
579 */
580
581 #define AGGRESSIVE_JETSAM_LENIENT_MODE_THRESHOLD 25
582 boolean_t memorystatus_aggressive_jetsam_lenient_allowed = FALSE;
583 boolean_t memorystatus_aggressive_jetsam_lenient = FALSE;
584
585 #if DEVELOPMENT || DEBUG
586 /*
587 * Jetsam Loop Detection tunables.
588 */
589
590 SYSCTL_UINT(_kern, OID_AUTO, memorystatus_jld_eval_period_msecs, CTLFLAG_RW|CTLFLAG_LOCKED, &memorystatus_jld_eval_period_msecs, 0, "");
591 SYSCTL_UINT(_kern, OID_AUTO, memorystatus_jld_eval_aggressive_count, CTLFLAG_RW|CTLFLAG_LOCKED, &memorystatus_jld_eval_aggressive_count, 0, "");
592 SYSCTL_UINT(_kern, OID_AUTO, memorystatus_jld_eval_aggressive_priority_band_max, CTLFLAG_RW|CTLFLAG_LOCKED, &memorystatus_jld_eval_aggressive_priority_band_max, 0, "");
593 #endif /* DEVELOPMENT || DEBUG */
594
595 static uint32_t kill_under_pressure_cause = 0;
596
597 /*
598 * default jetsam snapshot support
599 */
600 static memorystatus_jetsam_snapshot_t *memorystatus_jetsam_snapshot;
601 #define memorystatus_jetsam_snapshot_list memorystatus_jetsam_snapshot->entries
602 static unsigned int memorystatus_jetsam_snapshot_count = 0;
603 static unsigned int memorystatus_jetsam_snapshot_max = 0;
604 static uint64_t memorystatus_jetsam_snapshot_last_timestamp = 0;
605 static uint64_t memorystatus_jetsam_snapshot_timeout = 0;
606 #define JETSAM_SNAPSHOT_TIMEOUT_SECS 30
607
608 /*
609 * snapshot support for memstats collected at boot.
610 */
611 static memorystatus_jetsam_snapshot_t memorystatus_at_boot_snapshot;
612
613 static void memorystatus_init_jetsam_snapshot_locked(memorystatus_jetsam_snapshot_t *od_snapshot, uint32_t ods_list_count);
614 static boolean_t memorystatus_init_jetsam_snapshot_entry_locked(proc_t p, memorystatus_jetsam_snapshot_entry_t *entry, uint64_t gencount);
615 static void memorystatus_update_jetsam_snapshot_entry_locked(proc_t p, uint32_t kill_cause, uint64_t killtime);
616
617 static void memorystatus_clear_errors(void);
618 static void memorystatus_get_task_page_counts(task_t task, uint32_t *footprint, uint32_t *max_footprint, uint32_t *max_footprint_lifetime, uint32_t *purgeable_pages);
619 static void memorystatus_get_task_phys_footprint_page_counts(task_t task,
620 uint64_t *internal_pages, uint64_t *internal_compressed_pages,
621 uint64_t *purgeable_nonvolatile_pages, uint64_t *purgeable_nonvolatile_compressed_pages,
622 uint64_t *alternate_accounting_pages, uint64_t *alternate_accounting_compressed_pages,
623 uint64_t *iokit_mapped_pages, uint64_t *page_table_pages);
624
625 static void memorystatus_get_task_memory_region_count(task_t task, uint64_t *count);
626
627 static uint32_t memorystatus_build_state(proc_t p);
628 //static boolean_t memorystatus_issue_pressure_kevent(boolean_t pressured);
629
630 static boolean_t memorystatus_kill_top_process(boolean_t any, boolean_t sort_flag, uint32_t cause, os_reason_t jetsam_reason, int32_t *priority, uint32_t *errors);
631 static boolean_t memorystatus_kill_top_process_aggressive(uint32_t cause, int aggr_count, int32_t priority_max, uint32_t *errors);
632 static boolean_t memorystatus_kill_elevated_process(uint32_t cause, os_reason_t jetsam_reason, int aggr_count, uint32_t *errors);
633 static boolean_t memorystatus_kill_hiwat_proc(uint32_t *errors, boolean_t *purged);
634
635 static boolean_t memorystatus_kill_process_async(pid_t victim_pid, uint32_t cause);
636
637 /* Priority Band Sorting Routines */
638 static int memorystatus_sort_bucket(unsigned int bucket_index, int sort_order);
639 static int memorystatus_sort_by_largest_coalition_locked(unsigned int bucket_index, int coal_sort_order);
640 static void memorystatus_sort_by_largest_process_locked(unsigned int bucket_index);
641 static int memorystatus_move_list_locked(unsigned int bucket_index, pid_t *pid_list, int list_sz);
642
643 /* qsort routines */
644 typedef int (*cmpfunc_t)(const void *a, const void *b);
645 extern void qsort(void *a, size_t n, size_t es, cmpfunc_t cmp);
646 static int memstat_asc_cmp(const void *a, const void *b);
647
648 /* VM pressure */
649
650 extern unsigned int vm_page_free_count;
651 extern unsigned int vm_page_active_count;
652 extern unsigned int vm_page_inactive_count;
653 extern unsigned int vm_page_throttled_count;
654 extern unsigned int vm_page_purgeable_count;
655 extern unsigned int vm_page_wire_count;
656 #if CONFIG_SECLUDED_MEMORY
657 extern unsigned int vm_page_secluded_count;
658 #endif /* CONFIG_SECLUDED_MEMORY */
659
660 #if CONFIG_JETSAM
661 unsigned int memorystatus_available_pages = (unsigned int)-1;
662 unsigned int memorystatus_available_pages_pressure = 0;
663 unsigned int memorystatus_available_pages_critical = 0;
664 static unsigned int memorystatus_available_pages_critical_base = 0;
665 static unsigned int memorystatus_available_pages_critical_idle_offset = 0;
666
667 #if DEVELOPMENT || DEBUG
668 SYSCTL_UINT(_kern, OID_AUTO, memorystatus_available_pages, CTLFLAG_RD | CTLFLAG_LOCKED, &memorystatus_available_pages, 0, "");
669 #else
670 SYSCTL_UINT(_kern, OID_AUTO, memorystatus_available_pages, CTLFLAG_RD | CTLFLAG_MASKED | CTLFLAG_LOCKED, &memorystatus_available_pages, 0, "");
671 #endif /* DEVELOPMENT || DEBUG */
672
673 static unsigned int memorystatus_jetsam_policy = kPolicyDefault;
674 unsigned int memorystatus_policy_more_free_offset_pages = 0;
675 static void memorystatus_update_levels_locked(boolean_t critical_only);
676 static unsigned int memorystatus_thread_wasted_wakeup = 0;
677
678 /* Callback into vm_compressor.c to signal that thrashing has been mitigated. */
679 extern void vm_thrashing_jetsam_done(void);
680 static int memorystatus_cmd_set_jetsam_memory_limit(pid_t pid, int32_t high_water_mark, __unused int32_t *retval, boolean_t is_fatal_limit);
681
682 int32_t max_kill_priority = JETSAM_PRIORITY_MAX;
683
684 #else /* CONFIG_JETSAM */
685
686 uint64_t memorystatus_available_pages = (uint64_t)-1;
687 uint64_t memorystatus_available_pages_pressure = (uint64_t)-1;
688 uint64_t memorystatus_available_pages_critical = (uint64_t)-1;
689
690 int32_t max_kill_priority = JETSAM_PRIORITY_IDLE;
691 #endif /* CONFIG_JETSAM */
692
693 unsigned int memorystatus_frozen_count = 0;
694 unsigned int memorystatus_suspended_count = 0;
695
696 #if VM_PRESSURE_EVENTS
697
698 boolean_t memorystatus_warn_process(pid_t pid, __unused boolean_t is_active, __unused boolean_t is_fatal, boolean_t exceeded);
699
700 vm_pressure_level_t memorystatus_vm_pressure_level = kVMPressureNormal;
701
702 /*
703 * We use this flag to signal if we have any HWM offenders
704 * on the system. This way we can reduce the number of wakeups
705 * of the memorystatus_thread when the system is between the
706 * "pressure" and "critical" threshold.
707 *
708 * The (re-)setting of this variable is done without any locks
709 * or synchronization simply because it is not possible (currently)
710 * to keep track of HWM offenders that drop down below their memory
711 * limit and/or exit. So, we choose to burn a couple of wasted wakeups
712 * by allowing the unguarded modification of this variable.
713 */
714 boolean_t memorystatus_hwm_candidates = 0;
715
716 static int memorystatus_send_note(int event_code, void *data, size_t data_length);
717
718 #endif /* VM_PRESSURE_EVENTS */
719
720
721 #if DEVELOPMENT || DEBUG
722
723 lck_grp_attr_t *disconnect_page_mappings_lck_grp_attr;
724 lck_grp_t *disconnect_page_mappings_lck_grp;
725 static lck_mtx_t disconnect_page_mappings_mutex;
726
727 extern boolean_t kill_on_no_paging_space;
728 #endif /* DEVELOPMENT || DEBUG */
729
730
731 /* Freeze */
732
733 #if CONFIG_FREEZE
734
735 boolean_t memorystatus_freeze_enabled = FALSE;
736 int memorystatus_freeze_wakeup = 0;
737
738 lck_grp_attr_t *freezer_lck_grp_attr;
739 lck_grp_t *freezer_lck_grp;
740 static lck_mtx_t freezer_mutex;
741
742 static inline boolean_t memorystatus_can_freeze_processes(void);
743 static boolean_t memorystatus_can_freeze(boolean_t *memorystatus_freeze_swap_low);
744
745 static void memorystatus_freeze_thread(void *param __unused, wait_result_t wr __unused);
746
747 /* Thresholds */
748 static unsigned int memorystatus_freeze_threshold = 0;
749
750 static unsigned int memorystatus_freeze_pages_min = 0;
751 static unsigned int memorystatus_freeze_pages_max = 0;
752
753 static unsigned int memorystatus_freeze_suspended_threshold = FREEZE_SUSPENDED_THRESHOLD_DEFAULT;
754
755 static unsigned int memorystatus_freeze_daily_mb_max = FREEZE_DAILY_MB_MAX_DEFAULT;
756
757 /* Stats */
758 static uint64_t memorystatus_freeze_count = 0;
759 static uint64_t memorystatus_freeze_pageouts = 0;
760
761 /* Throttling */
762 static throttle_interval_t throttle_intervals[] = {
763 { 60, 8, 0, 0, { 0, 0 }, FALSE }, /* 1 hour intermediate interval, 8x burst */
764 { 24 * 60, 1, 0, 0, { 0, 0 }, FALSE }, /* 24 hour long interval, no burst */
765 };
766
767 static uint64_t memorystatus_freeze_throttle_count = 0;
768
769 static unsigned int memorystatus_suspended_footprint_total = 0; /* pages */
770
771 extern uint64_t vm_swap_get_free_space(void);
772
773 static boolean_t memorystatus_freeze_update_throttle(void);
774
775 #endif /* CONFIG_FREEZE */
776
777 /* Debug */
778
779 extern struct knote *vm_find_knote_from_pid(pid_t, struct klist *);
780
781 #if DEVELOPMENT || DEBUG
782
783 static unsigned int memorystatus_debug_dump_this_bucket = 0;
784
785 static void
786 memorystatus_debug_dump_bucket_locked (unsigned int bucket_index)
787 {
788 proc_t p = NULL;
789 uint64_t bytes = 0;
790 int ledger_limit = 0;
791 unsigned int b = bucket_index;
792 boolean_t traverse_all_buckets = FALSE;
793
794 if (bucket_index >= MEMSTAT_BUCKET_COUNT) {
795 traverse_all_buckets = TRUE;
796 b = 0;
797 } else {
798 traverse_all_buckets = FALSE;
799 b = bucket_index;
800 }
801
802 /*
803 * footprint reported in [pages / MB ]
804 * limits reported as:
805 * L-limit proc's Ledger limit
806 * C-limit proc's Cached limit, should match Ledger
807 * A-limit proc's Active limit
808 * IA-limit proc's Inactive limit
809 * F==Fatal, NF==NonFatal
810 */
811
812 printf("memorystatus_debug_dump ***START*(PAGE_SIZE_64=%llu)**\n", PAGE_SIZE_64);
813 printf("bucket [pid] [pages / MB] [state] [EP / RP] dirty deadline [L-limit / C-limit / A-limit / IA-limit] name\n");
814 p = memorystatus_get_first_proc_locked(&b, traverse_all_buckets);
815 while (p) {
816 bytes = get_task_phys_footprint(p->task);
817 task_get_phys_footprint_limit(p->task, &ledger_limit);
818 printf("%2d [%5d] [%5lld /%3lldMB] 0x%-8x [%2d / %2d] 0x%-3x %10lld [%3d / %3d%s / %3d%s / %3d%s] %s\n",
819 b, p->p_pid,
820 (bytes / PAGE_SIZE_64), /* task's footprint converted from bytes to pages */
821 (bytes / (1024ULL * 1024ULL)), /* task's footprint converted from bytes to MB */
822 p->p_memstat_state, p->p_memstat_effectivepriority, p->p_memstat_requestedpriority, p->p_memstat_dirty, p->p_memstat_idledeadline,
823 ledger_limit,
824 p->p_memstat_memlimit,
825 (p->p_memstat_state & P_MEMSTAT_FATAL_MEMLIMIT ? "F " : "NF"),
826 p->p_memstat_memlimit_active,
827 (p->p_memstat_state & P_MEMSTAT_MEMLIMIT_ACTIVE_FATAL ? "F " : "NF"),
828 p->p_memstat_memlimit_inactive,
829 (p->p_memstat_state & P_MEMSTAT_MEMLIMIT_INACTIVE_FATAL ? "F " : "NF"),
830 (*p->p_name ? p->p_name : "unknown"));
831 p = memorystatus_get_next_proc_locked(&b, p, traverse_all_buckets);
832 }
833 printf("memorystatus_debug_dump ***END***\n");
834 }
835
836 static int
837 sysctl_memorystatus_debug_dump_bucket SYSCTL_HANDLER_ARGS
838 {
839 #pragma unused(oidp, arg2)
840 int bucket_index = 0;
841 int error;
842 error = SYSCTL_OUT(req, arg1, sizeof(int));
843 if (error || !req->newptr) {
844 return (error);
845 }
846 error = SYSCTL_IN(req, &bucket_index, sizeof(int));
847 if (error || !req->newptr) {
848 return (error);
849 }
850 if (bucket_index >= MEMSTAT_BUCKET_COUNT) {
851 /*
852 * All jetsam buckets will be dumped.
853 */
854 } else {
855 /*
856 * Only a single bucket will be dumped.
857 */
858 }
859
860 proc_list_lock();
861 memorystatus_debug_dump_bucket_locked(bucket_index);
862 proc_list_unlock();
863 memorystatus_debug_dump_this_bucket = bucket_index;
864 return (error);
865 }
866
867 /*
868 * Debug aid to look at jetsam buckets and proc jetsam fields.
869 * Use this sysctl to act on a particular jetsam bucket.
870 * Writing the sysctl triggers the dump.
871 * Usage: sysctl kern.memorystatus_debug_dump_this_bucket=<bucket_index>
872 */
873
874 SYSCTL_PROC(_kern, OID_AUTO, memorystatus_debug_dump_this_bucket, CTLTYPE_INT|CTLFLAG_RW|CTLFLAG_LOCKED, &memorystatus_debug_dump_this_bucket, 0, sysctl_memorystatus_debug_dump_bucket, "I", "");
875
876
877 /* Debug aid to aid determination of limit */
878
879 static int
880 sysctl_memorystatus_highwater_enable SYSCTL_HANDLER_ARGS
881 {
882 #pragma unused(oidp, arg2)
883 proc_t p;
884 unsigned int b = 0;
885 int error, enable = 0;
886 boolean_t use_active; /* use the active limit and active limit attributes */
887 boolean_t is_fatal;
888
889 error = SYSCTL_OUT(req, arg1, sizeof(int));
890 if (error || !req->newptr) {
891 return (error);
892 }
893
894 error = SYSCTL_IN(req, &enable, sizeof(int));
895 if (error || !req->newptr) {
896 return (error);
897 }
898
899 if (!(enable == 0 || enable == 1)) {
900 return EINVAL;
901 }
902
903 proc_list_lock();
904
905 p = memorystatus_get_first_proc_locked(&b, TRUE);
906 while (p) {
907 use_active = proc_jetsam_state_is_active_locked(p);
908
909 if (enable) {
910
911 if (use_active == TRUE) {
912 CACHE_ACTIVE_LIMITS_LOCKED(p, is_fatal);
913 } else {
914 CACHE_INACTIVE_LIMITS_LOCKED(p, is_fatal);
915 }
916
917 } else {
918 /*
919 * Disabling limits does not touch the stored variants.
920 * Set the cached limit fields to system_wide defaults.
921 */
922 p->p_memstat_memlimit = -1;
923 p->p_memstat_state |= P_MEMSTAT_FATAL_MEMLIMIT;
924 is_fatal = TRUE;
925 }
926
927 /*
928 * Enforce the cached limit by writing to the ledger.
929 */
930 task_set_phys_footprint_limit_internal(p->task, (p->p_memstat_memlimit > 0) ? p->p_memstat_memlimit: -1, NULL, use_active, is_fatal);
931
932 p = memorystatus_get_next_proc_locked(&b, p, TRUE);
933 }
934
935 memorystatus_highwater_enabled = enable;
936
937 proc_list_unlock();
938
939 return 0;
940
941 }
942
943 SYSCTL_PROC(_kern, OID_AUTO, memorystatus_highwater_enabled, CTLTYPE_INT|CTLFLAG_RW|CTLFLAG_LOCKED, &memorystatus_highwater_enabled, 0, sysctl_memorystatus_highwater_enable, "I", "");
944
945 #if VM_PRESSURE_EVENTS
946
947 /*
948 * This routine is used for targeted notifications regardless of system memory pressure
949 * and regardless of whether or not the process has already been notified.
950 * It bypasses and has no effect on the only-one-notification per soft-limit policy.
951 *
952 * "memnote" is the current user.
953 */
954
955 static int
956 sysctl_memorystatus_vm_pressure_send SYSCTL_HANDLER_ARGS
957 {
958 #pragma unused(arg1, arg2)
959
960 int error = 0, pid = 0;
961 struct knote *kn = NULL;
962 boolean_t found_knote = FALSE;
963 int fflags = 0; /* filter flags for EVFILT_MEMORYSTATUS */
964 uint64_t value = 0;
965
966 error = sysctl_handle_quad(oidp, &value, 0, req);
967 if (error || !req->newptr)
968 return (error);
969
970 /*
971 * Find the pid in the low 32 bits of value passed in.
972 */
973 pid = (int)(value & 0xFFFFFFFF);
974
975 /*
976 * Find notification in the high 32 bits of the value passed in.
977 */
978 fflags = (int)((value >> 32) & 0xFFFFFFFF);
979
980 /*
981 * For backwards compatibility, when no notification is
982 * passed in, default to the NOTE_MEMORYSTATUS_PRESSURE_WARN
983 */
984 if (fflags == 0) {
985 fflags = NOTE_MEMORYSTATUS_PRESSURE_WARN;
986 // printf("memorystatus_vm_pressure_send: using default notification [0x%x]\n", fflags);
987 }
988
989 /*
990 * See event.h ... fflags for EVFILT_MEMORYSTATUS
991 */
992 if (!((fflags == NOTE_MEMORYSTATUS_PRESSURE_NORMAL)||
993 (fflags == NOTE_MEMORYSTATUS_PRESSURE_WARN) ||
994 (fflags == NOTE_MEMORYSTATUS_PRESSURE_CRITICAL) ||
995 (fflags == NOTE_MEMORYSTATUS_LOW_SWAP) ||
996 (fflags == NOTE_MEMORYSTATUS_PROC_LIMIT_WARN) ||
997 (fflags == NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL) ||
998 (((fflags & NOTE_MEMORYSTATUS_MSL_STATUS) != 0 &&
999 ((fflags & ~NOTE_MEMORYSTATUS_MSL_STATUS) == 0))))) {
1000
1001 printf("memorystatus_vm_pressure_send: notification [0x%x] not supported \n", fflags);
1002 error = 1;
1003 return (error);
1004 }
1005
1006 /*
1007 * Forcibly send pid a memorystatus notification.
1008 */
1009
1010 memorystatus_klist_lock();
1011
1012 SLIST_FOREACH(kn, &memorystatus_klist, kn_selnext) {
1013 proc_t knote_proc = knote_get_kq(kn)->kq_p;
1014 pid_t knote_pid = knote_proc->p_pid;
1015
1016 if (knote_pid == pid) {
1017 /*
1018 * Forcibly send this pid a memorystatus notification.
1019 */
1020 kn->kn_fflags = fflags;
1021 found_knote = TRUE;
1022 }
1023 }
1024
1025 if (found_knote) {
1026 KNOTE(&memorystatus_klist, 0);
1027 printf("memorystatus_vm_pressure_send: (value 0x%llx) notification [0x%x] sent to process [%d] \n", value, fflags, pid);
1028 error = 0;
1029 } else {
1030 printf("memorystatus_vm_pressure_send: (value 0x%llx) notification [0x%x] not sent to process [%d] (none registered?)\n", value, fflags, pid);
1031 error = 1;
1032 }
1033
1034 memorystatus_klist_unlock();
1035
1036 return (error);
1037 }
1038
1039 SYSCTL_PROC(_kern, OID_AUTO, memorystatus_vm_pressure_send, CTLTYPE_QUAD|CTLFLAG_WR|CTLFLAG_LOCKED|CTLFLAG_MASKED,
1040 0, 0, &sysctl_memorystatus_vm_pressure_send, "Q", "");
1041
1042 #endif /* VM_PRESSURE_EVENTS */
1043
1044 SYSCTL_INT(_kern, OID_AUTO, memorystatus_idle_snapshot, CTLFLAG_RW|CTLFLAG_LOCKED, &memorystatus_idle_snapshot, 0, "");
1045
1046 #if CONFIG_JETSAM
1047 SYSCTL_UINT(_kern, OID_AUTO, memorystatus_available_pages_critical, CTLFLAG_RD|CTLFLAG_LOCKED, &memorystatus_available_pages_critical, 0, "");
1048 SYSCTL_UINT(_kern, OID_AUTO, memorystatus_available_pages_critical_base, CTLFLAG_RW|CTLFLAG_LOCKED, &memorystatus_available_pages_critical_base, 0, "");
1049 SYSCTL_UINT(_kern, OID_AUTO, memorystatus_available_pages_critical_idle_offset, CTLFLAG_RW|CTLFLAG_LOCKED, &memorystatus_available_pages_critical_idle_offset, 0, "");
1050 SYSCTL_UINT(_kern, OID_AUTO, memorystatus_policy_more_free_offset_pages, CTLFLAG_RW, &memorystatus_policy_more_free_offset_pages, 0, "");
1051
1052 static unsigned int memorystatus_jetsam_panic_debug = 0;
1053 static unsigned int memorystatus_jetsam_policy_offset_pages_diagnostic = 0;
1054
1055 /* Diagnostic code */
1056
1057 enum {
1058 kJetsamDiagnosticModeNone = 0,
1059 kJetsamDiagnosticModeAll = 1,
1060 kJetsamDiagnosticModeStopAtFirstActive = 2,
1061 kJetsamDiagnosticModeCount
1062 } jetsam_diagnostic_mode = kJetsamDiagnosticModeNone;
1063
1064 static int jetsam_diagnostic_suspended_one_active_proc = 0;
1065
1066 static int
1067 sysctl_jetsam_diagnostic_mode SYSCTL_HANDLER_ARGS
1068 {
1069 #pragma unused(arg1, arg2)
1070
1071 const char *diagnosticStrings[] = {
1072 "jetsam: diagnostic mode: resetting critical level.",
1073 "jetsam: diagnostic mode: will examine all processes",
1074 "jetsam: diagnostic mode: will stop at first active process"
1075 };
1076
1077 int error, val = jetsam_diagnostic_mode;
1078 boolean_t changed = FALSE;
1079
1080 error = sysctl_handle_int(oidp, &val, 0, req);
1081 if (error || !req->newptr)
1082 return (error);
1083 if ((val < 0) || (val >= kJetsamDiagnosticModeCount)) {
1084 printf("jetsam: diagnostic mode: invalid value - %d\n", val);
1085 return EINVAL;
1086 }
1087
1088 proc_list_lock();
1089
1090 if ((unsigned int) val != jetsam_diagnostic_mode) {
1091 jetsam_diagnostic_mode = val;
1092
1093 memorystatus_jetsam_policy &= ~kPolicyDiagnoseActive;
1094
1095 switch (jetsam_diagnostic_mode) {
1096 case kJetsamDiagnosticModeNone:
1097 /* Already cleared */
1098 break;
1099 case kJetsamDiagnosticModeAll:
1100 memorystatus_jetsam_policy |= kPolicyDiagnoseAll;
1101 break;
1102 case kJetsamDiagnosticModeStopAtFirstActive:
1103 memorystatus_jetsam_policy |= kPolicyDiagnoseFirst;
1104 break;
1105 default:
1106 /* Already validated */
1107 break;
1108 }
1109
1110 memorystatus_update_levels_locked(FALSE);
1111 changed = TRUE;
1112 }
1113
1114 proc_list_unlock();
1115
1116 if (changed) {
1117 printf("%s\n", diagnosticStrings[val]);
1118 }
1119
1120 return (0);
1121 }
1122
1123 SYSCTL_PROC(_debug, OID_AUTO, jetsam_diagnostic_mode, CTLTYPE_INT|CTLFLAG_RW|CTLFLAG_LOCKED|CTLFLAG_ANYBODY,
1124 &jetsam_diagnostic_mode, 0, sysctl_jetsam_diagnostic_mode, "I", "Jetsam Diagnostic Mode");
1125
1126 SYSCTL_UINT(_kern, OID_AUTO, memorystatus_jetsam_policy_offset_pages_diagnostic, CTLFLAG_RW|CTLFLAG_LOCKED, &memorystatus_jetsam_policy_offset_pages_diagnostic, 0, "");
1127
1128 #if VM_PRESSURE_EVENTS
1129
1130 SYSCTL_UINT(_kern, OID_AUTO, memorystatus_available_pages_pressure, CTLFLAG_RW|CTLFLAG_LOCKED, &memorystatus_available_pages_pressure, 0, "");
1131
1132 #endif /* VM_PRESSURE_EVENTS */
1133
1134 #endif /* CONFIG_JETSAM */
1135
1136 #if CONFIG_FREEZE
1137
1138 SYSCTL_UINT(_kern, OID_AUTO, memorystatus_freeze_daily_mb_max, CTLFLAG_RW|CTLFLAG_LOCKED, &memorystatus_freeze_daily_mb_max, 0, "");
1139
1140 SYSCTL_UINT(_kern, OID_AUTO, memorystatus_freeze_threshold, CTLFLAG_RW|CTLFLAG_LOCKED, &memorystatus_freeze_threshold, 0, "");
1141
1142 SYSCTL_UINT(_kern, OID_AUTO, memorystatus_freeze_pages_min, CTLFLAG_RW|CTLFLAG_LOCKED, &memorystatus_freeze_pages_min, 0, "");
1143 SYSCTL_UINT(_kern, OID_AUTO, memorystatus_freeze_pages_max, CTLFLAG_RW|CTLFLAG_LOCKED, &memorystatus_freeze_pages_max, 0, "");
1144
1145 SYSCTL_QUAD(_kern, OID_AUTO, memorystatus_freeze_count, CTLFLAG_RD|CTLFLAG_LOCKED, &memorystatus_freeze_count, "");
1146 SYSCTL_QUAD(_kern, OID_AUTO, memorystatus_freeze_pageouts, CTLFLAG_RD|CTLFLAG_LOCKED, &memorystatus_freeze_pageouts, "");
1147 SYSCTL_QUAD(_kern, OID_AUTO, memorystatus_freeze_throttle_count, CTLFLAG_RD|CTLFLAG_LOCKED, &memorystatus_freeze_throttle_count, "");
1148 SYSCTL_UINT(_kern, OID_AUTO, memorystatus_freeze_min_processes, CTLFLAG_RW|CTLFLAG_LOCKED, &memorystatus_freeze_suspended_threshold, 0, "");
1149
1150 boolean_t memorystatus_freeze_throttle_enabled = TRUE;
1151 SYSCTL_UINT(_kern, OID_AUTO, memorystatus_freeze_throttle_enabled, CTLFLAG_RW|CTLFLAG_LOCKED, &memorystatus_freeze_throttle_enabled, 0, "");
1152
1153 #define VM_PAGES_FOR_ALL_PROCS (2)
1154 /*
1155 * Manual trigger of freeze and thaw for dev / debug kernels only.
1156 */
1157 static int
1158 sysctl_memorystatus_freeze SYSCTL_HANDLER_ARGS
1159 {
1160 #pragma unused(arg1, arg2)
1161 int error, pid = 0;
1162 proc_t p;
1163
1164 if (memorystatus_freeze_enabled == FALSE) {
1165 return ENOTSUP;
1166 }
1167
1168 error = sysctl_handle_int(oidp, &pid, 0, req);
1169 if (error || !req->newptr)
1170 return (error);
1171
1172 if (pid == VM_PAGES_FOR_ALL_PROCS) {
1173 vm_pageout_anonymous_pages();
1174
1175 return 0;
1176 }
1177
1178 lck_mtx_lock(&freezer_mutex);
1179
1180 p = proc_find(pid);
1181 if (p != NULL) {
1182 uint32_t purgeable, wired, clean, dirty;
1183 boolean_t shared;
1184 uint32_t max_pages = 0;
1185
1186 if (VM_CONFIG_FREEZER_SWAP_IS_ACTIVE) {
1187
1188 unsigned int avail_swap_space = 0; /* in pages. */
1189
1190 /*
1191 * Freezer backed by the compressor and swap file(s)
1192 * while will hold compressed data.
1193 */
1194 avail_swap_space = vm_swap_get_free_space() / PAGE_SIZE_64;
1195
1196 max_pages = MIN(avail_swap_space, memorystatus_freeze_pages_max);
1197
1198 } else {
1199 /*
1200 * We only have the compressor without any swap.
1201 */
1202 max_pages = UINT32_MAX - 1;
1203 }
1204
1205 error = task_freeze(p->task, &purgeable, &wired, &clean, &dirty, max_pages, &shared, FALSE);
1206 proc_rele(p);
1207
1208 if (error)
1209 error = EIO;
1210
1211 lck_mtx_unlock(&freezer_mutex);
1212 return error;
1213 }
1214
1215 lck_mtx_unlock(&freezer_mutex);
1216 return EINVAL;
1217 }
1218
1219 SYSCTL_PROC(_kern, OID_AUTO, memorystatus_freeze, CTLTYPE_INT|CTLFLAG_WR|CTLFLAG_LOCKED|CTLFLAG_MASKED,
1220 0, 0, &sysctl_memorystatus_freeze, "I", "");
1221
1222 static int
1223 sysctl_memorystatus_available_pages_thaw SYSCTL_HANDLER_ARGS
1224 {
1225 #pragma unused(arg1, arg2)
1226
1227 int error, pid = 0;
1228 proc_t p;
1229
1230 if (memorystatus_freeze_enabled == FALSE) {
1231 return ENOTSUP;
1232 }
1233
1234 error = sysctl_handle_int(oidp, &pid, 0, req);
1235 if (error || !req->newptr)
1236 return (error);
1237
1238 if (pid == VM_PAGES_FOR_ALL_PROCS) {
1239 do_fastwake_warmup_all();
1240 return 0;
1241 } else {
1242 p = proc_find(pid);
1243 if (p != NULL) {
1244 error = task_thaw(p->task);
1245 proc_rele(p);
1246
1247 if (error)
1248 error = EIO;
1249 return error;
1250 }
1251 }
1252
1253 return EINVAL;
1254 }
1255
1256 SYSCTL_PROC(_kern, OID_AUTO, memorystatus_thaw, CTLTYPE_INT|CTLFLAG_WR|CTLFLAG_LOCKED|CTLFLAG_MASKED,
1257 0, 0, &sysctl_memorystatus_available_pages_thaw, "I", "");
1258
1259 #endif /* CONFIG_FREEZE */
1260
1261 #endif /* DEVELOPMENT || DEBUG */
1262
1263 extern kern_return_t kernel_thread_start_priority(thread_continue_t continuation,
1264 void *parameter,
1265 integer_t priority,
1266 thread_t *new_thread);
1267
1268 #if DEVELOPMENT || DEBUG
1269
1270 static int
1271 sysctl_memorystatus_disconnect_page_mappings SYSCTL_HANDLER_ARGS
1272 {
1273 #pragma unused(arg1, arg2)
1274 int error = 0, pid = 0;
1275 proc_t p;
1276
1277 error = sysctl_handle_int(oidp, &pid, 0, req);
1278 if (error || !req->newptr)
1279 return (error);
1280
1281 lck_mtx_lock(&disconnect_page_mappings_mutex);
1282
1283 if (pid == -1) {
1284 vm_pageout_disconnect_all_pages();
1285 } else {
1286 p = proc_find(pid);
1287
1288 if (p != NULL) {
1289 error = task_disconnect_page_mappings(p->task);
1290
1291 proc_rele(p);
1292
1293 if (error)
1294 error = EIO;
1295 } else
1296 error = EINVAL;
1297 }
1298 lck_mtx_unlock(&disconnect_page_mappings_mutex);
1299
1300 return error;
1301 }
1302
1303 SYSCTL_PROC(_kern, OID_AUTO, memorystatus_disconnect_page_mappings, CTLTYPE_INT|CTLFLAG_WR|CTLFLAG_LOCKED|CTLFLAG_MASKED,
1304 0, 0, &sysctl_memorystatus_disconnect_page_mappings, "I", "");
1305
1306 #endif /* DEVELOPMENT || DEBUG */
1307
1308
1309 /*
1310 * Picks the sorting routine for a given jetsam priority band.
1311 *
1312 * Input:
1313 * bucket_index - jetsam priority band to be sorted.
1314 * sort_order - JETSAM_SORT_xxx from kern_memorystatus.h
1315 * Currently sort_order is only meaningful when handling
1316 * coalitions.
1317 *
1318 * Return:
1319 * 0 on success
1320 * non-0 on failure
1321 */
1322 static int memorystatus_sort_bucket(unsigned int bucket_index, int sort_order)
1323 {
1324 int coal_sort_order;
1325
1326 /*
1327 * Verify the jetsam priority
1328 */
1329 if (bucket_index >= MEMSTAT_BUCKET_COUNT) {
1330 return(EINVAL);
1331 }
1332
1333 #if DEVELOPMENT || DEBUG
1334 if (sort_order == JETSAM_SORT_DEFAULT) {
1335 coal_sort_order = COALITION_SORT_DEFAULT;
1336 } else {
1337 coal_sort_order = sort_order; /* only used for testing scenarios */
1338 }
1339 #else
1340 /* Verify default */
1341 if (sort_order == JETSAM_SORT_DEFAULT) {
1342 coal_sort_order = COALITION_SORT_DEFAULT;
1343 } else {
1344 return(EINVAL);
1345 }
1346 #endif
1347
1348 proc_list_lock();
1349
1350 if (memstat_bucket[bucket_index].count == 0) {
1351 proc_list_unlock();
1352 return (0);
1353 }
1354
1355 switch (bucket_index) {
1356 case JETSAM_PRIORITY_FOREGROUND:
1357 if (memorystatus_sort_by_largest_coalition_locked(bucket_index, coal_sort_order) == 0) {
1358 /*
1359 * Fall back to per process sorting when zero coalitions are found.
1360 */
1361 memorystatus_sort_by_largest_process_locked(bucket_index);
1362 }
1363 break;
1364 default:
1365 memorystatus_sort_by_largest_process_locked(bucket_index);
1366 break;
1367 }
1368 proc_list_unlock();
1369
1370 return(0);
1371 }
1372
1373 /*
1374 * Sort processes by size for a single jetsam bucket.
1375 */
1376
1377 static void memorystatus_sort_by_largest_process_locked(unsigned int bucket_index)
1378 {
1379 proc_t p = NULL, insert_after_proc = NULL, max_proc = NULL;
1380 proc_t next_p = NULL, prev_max_proc = NULL;
1381 uint32_t pages = 0, max_pages = 0;
1382 memstat_bucket_t *current_bucket;
1383
1384 if (bucket_index >= MEMSTAT_BUCKET_COUNT) {
1385 return;
1386 }
1387
1388 current_bucket = &memstat_bucket[bucket_index];
1389
1390 p = TAILQ_FIRST(&current_bucket->list);
1391
1392 while (p) {
1393 memorystatus_get_task_page_counts(p->task, &pages, NULL, NULL, NULL);
1394 max_pages = pages;
1395 max_proc = p;
1396 prev_max_proc = p;
1397
1398 while ((next_p = TAILQ_NEXT(p, p_memstat_list)) != NULL) {
1399 /* traversing list until we find next largest process */
1400 p=next_p;
1401 memorystatus_get_task_page_counts(p->task, &pages, NULL, NULL, NULL);
1402 if (pages > max_pages) {
1403 max_pages = pages;
1404 max_proc = p;
1405 }
1406 }
1407
1408 if (prev_max_proc != max_proc) {
1409 /* found a larger process, place it in the list */
1410 TAILQ_REMOVE(&current_bucket->list, max_proc, p_memstat_list);
1411 if (insert_after_proc == NULL) {
1412 TAILQ_INSERT_HEAD(&current_bucket->list, max_proc, p_memstat_list);
1413 } else {
1414 TAILQ_INSERT_AFTER(&current_bucket->list, insert_after_proc, max_proc, p_memstat_list);
1415 }
1416 prev_max_proc = max_proc;
1417 }
1418
1419 insert_after_proc = max_proc;
1420
1421 p = TAILQ_NEXT(max_proc, p_memstat_list);
1422 }
1423 }
1424
1425 static proc_t memorystatus_get_first_proc_locked(unsigned int *bucket_index, boolean_t search) {
1426 memstat_bucket_t *current_bucket;
1427 proc_t next_p;
1428
1429 if ((*bucket_index) >= MEMSTAT_BUCKET_COUNT) {
1430 return NULL;
1431 }
1432
1433 current_bucket = &memstat_bucket[*bucket_index];
1434 next_p = TAILQ_FIRST(&current_bucket->list);
1435 if (!next_p && search) {
1436 while (!next_p && (++(*bucket_index) < MEMSTAT_BUCKET_COUNT)) {
1437 current_bucket = &memstat_bucket[*bucket_index];
1438 next_p = TAILQ_FIRST(&current_bucket->list);
1439 }
1440 }
1441
1442 return next_p;
1443 }
1444
1445 static proc_t memorystatus_get_next_proc_locked(unsigned int *bucket_index, proc_t p, boolean_t search) {
1446 memstat_bucket_t *current_bucket;
1447 proc_t next_p;
1448
1449 if (!p || ((*bucket_index) >= MEMSTAT_BUCKET_COUNT)) {
1450 return NULL;
1451 }
1452
1453 next_p = TAILQ_NEXT(p, p_memstat_list);
1454 while (!next_p && search && (++(*bucket_index) < MEMSTAT_BUCKET_COUNT)) {
1455 current_bucket = &memstat_bucket[*bucket_index];
1456 next_p = TAILQ_FIRST(&current_bucket->list);
1457 }
1458
1459 return next_p;
1460 }
1461
1462 __private_extern__ void
1463 memorystatus_init(void)
1464 {
1465 thread_t thread = THREAD_NULL;
1466 kern_return_t result;
1467 int i;
1468
1469 #if CONFIG_FREEZE
1470 memorystatus_freeze_pages_min = FREEZE_PAGES_MIN;
1471 memorystatus_freeze_pages_max = FREEZE_PAGES_MAX;
1472 #endif
1473
1474 #if DEVELOPMENT || DEBUG
1475 disconnect_page_mappings_lck_grp_attr = lck_grp_attr_alloc_init();
1476 disconnect_page_mappings_lck_grp = lck_grp_alloc_init("disconnect_page_mappings", disconnect_page_mappings_lck_grp_attr);
1477
1478 lck_mtx_init(&disconnect_page_mappings_mutex, disconnect_page_mappings_lck_grp, NULL);
1479
1480 if (kill_on_no_paging_space == TRUE) {
1481 max_kill_priority = JETSAM_PRIORITY_MAX;
1482 }
1483 #endif
1484
1485
1486 /* Init buckets */
1487 for (i = 0; i < MEMSTAT_BUCKET_COUNT; i++) {
1488 TAILQ_INIT(&memstat_bucket[i].list);
1489 memstat_bucket[i].count = 0;
1490 }
1491 memorystatus_idle_demotion_call = thread_call_allocate((thread_call_func_t)memorystatus_perform_idle_demotion, NULL);
1492
1493 #if CONFIG_JETSAM
1494 nanoseconds_to_absolutetime((uint64_t)DEFERRED_IDLE_EXIT_TIME_SECS * NSEC_PER_SEC, &memorystatus_sysprocs_idle_delay_time);
1495 nanoseconds_to_absolutetime((uint64_t)DEFERRED_IDLE_EXIT_TIME_SECS * NSEC_PER_SEC, &memorystatus_apps_idle_delay_time);
1496
1497 /* Apply overrides */
1498 PE_get_default("kern.jetsam_delta", &delta_percentage, sizeof(delta_percentage));
1499 if (delta_percentage == 0) {
1500 delta_percentage = 5;
1501 }
1502 assert(delta_percentage < 100);
1503 PE_get_default("kern.jetsam_critical_threshold", &critical_threshold_percentage, sizeof(critical_threshold_percentage));
1504 assert(critical_threshold_percentage < 100);
1505 PE_get_default("kern.jetsam_idle_offset", &idle_offset_percentage, sizeof(idle_offset_percentage));
1506 assert(idle_offset_percentage < 100);
1507 PE_get_default("kern.jetsam_pressure_threshold", &pressure_threshold_percentage, sizeof(pressure_threshold_percentage));
1508 assert(pressure_threshold_percentage < 100);
1509 PE_get_default("kern.jetsam_freeze_threshold", &freeze_threshold_percentage, sizeof(freeze_threshold_percentage));
1510 assert(freeze_threshold_percentage < 100);
1511
1512 if (!PE_parse_boot_argn("jetsam_aging_policy", &jetsam_aging_policy,
1513 sizeof (jetsam_aging_policy))) {
1514
1515 if (!PE_get_default("kern.jetsam_aging_policy", &jetsam_aging_policy,
1516 sizeof(jetsam_aging_policy))) {
1517
1518 jetsam_aging_policy = kJetsamAgingPolicyLegacy;
1519 }
1520 }
1521
1522 if (jetsam_aging_policy > kJetsamAgingPolicyMax) {
1523 jetsam_aging_policy = kJetsamAgingPolicyLegacy;
1524 }
1525
1526 switch (jetsam_aging_policy) {
1527
1528 case kJetsamAgingPolicyNone:
1529 system_procs_aging_band = JETSAM_PRIORITY_IDLE;
1530 applications_aging_band = JETSAM_PRIORITY_IDLE;
1531 break;
1532
1533 case kJetsamAgingPolicyLegacy:
1534 /*
1535 * Legacy behavior where some daemons get a 10s protection once
1536 * AND only before the first clean->dirty->clean transition before
1537 * going into IDLE band.
1538 */
1539 system_procs_aging_band = JETSAM_PRIORITY_AGING_BAND1;
1540 applications_aging_band = JETSAM_PRIORITY_IDLE;
1541 break;
1542
1543 case kJetsamAgingPolicySysProcsReclaimedFirst:
1544 system_procs_aging_band = JETSAM_PRIORITY_AGING_BAND1;
1545 applications_aging_band = JETSAM_PRIORITY_AGING_BAND2;
1546 break;
1547
1548 case kJetsamAgingPolicyAppsReclaimedFirst:
1549 system_procs_aging_band = JETSAM_PRIORITY_AGING_BAND2;
1550 applications_aging_band = JETSAM_PRIORITY_AGING_BAND1;
1551 break;
1552
1553 default:
1554 break;
1555 }
1556
1557 /*
1558 * The aging bands cannot overlap with the JETSAM_PRIORITY_ELEVATED_INACTIVE
1559 * band and must be below it in priority. This is so that we don't have to make
1560 * our 'aging' code worry about a mix of processes, some of which need to age
1561 * and some others that need to stay elevated in the jetsam bands.
1562 */
1563 assert(JETSAM_PRIORITY_ELEVATED_INACTIVE > system_procs_aging_band);
1564 assert(JETSAM_PRIORITY_ELEVATED_INACTIVE > applications_aging_band);
1565
1566 /* Take snapshots for idle-exit kills by default? First check the boot-arg... */
1567 if (!PE_parse_boot_argn("jetsam_idle_snapshot", &memorystatus_idle_snapshot, sizeof (memorystatus_idle_snapshot))) {
1568 /* ...no boot-arg, so check the device tree */
1569 PE_get_default("kern.jetsam_idle_snapshot", &memorystatus_idle_snapshot, sizeof(memorystatus_idle_snapshot));
1570 }
1571
1572 memorystatus_delta = delta_percentage * atop_64(max_mem) / 100;
1573 memorystatus_available_pages_critical_idle_offset = idle_offset_percentage * atop_64(max_mem) / 100;
1574 memorystatus_available_pages_critical_base = (critical_threshold_percentage / delta_percentage) * memorystatus_delta;
1575 memorystatus_policy_more_free_offset_pages = (policy_more_free_offset_percentage / delta_percentage) * memorystatus_delta;
1576
1577 /* Jetsam Loop Detection */
1578 if (max_mem <= (512 * 1024 * 1024)) {
1579 /* 512 MB devices */
1580 memorystatus_jld_eval_period_msecs = 8000; /* 8000 msecs == 8 second window */
1581 } else {
1582 /* 1GB and larger devices */
1583 memorystatus_jld_eval_period_msecs = 6000; /* 6000 msecs == 6 second window */
1584 }
1585
1586 memorystatus_jld_enabled = TRUE;
1587
1588 /* No contention at this point */
1589 memorystatus_update_levels_locked(FALSE);
1590
1591 #endif /* CONFIG_JETSAM */
1592
1593 memorystatus_jetsam_snapshot_max = maxproc;
1594 memorystatus_jetsam_snapshot =
1595 (memorystatus_jetsam_snapshot_t*)kalloc(sizeof(memorystatus_jetsam_snapshot_t) +
1596 sizeof(memorystatus_jetsam_snapshot_entry_t) * memorystatus_jetsam_snapshot_max);
1597 if (!memorystatus_jetsam_snapshot) {
1598 panic("Could not allocate memorystatus_jetsam_snapshot");
1599 }
1600
1601 nanoseconds_to_absolutetime((uint64_t)JETSAM_SNAPSHOT_TIMEOUT_SECS * NSEC_PER_SEC, &memorystatus_jetsam_snapshot_timeout);
1602
1603 memset(&memorystatus_at_boot_snapshot, 0, sizeof(memorystatus_jetsam_snapshot_t));
1604
1605 #if CONFIG_FREEZE
1606 memorystatus_freeze_threshold = (freeze_threshold_percentage / delta_percentage) * memorystatus_delta;
1607 #endif
1608
1609 result = kernel_thread_start_priority(memorystatus_thread, NULL, 95 /* MAXPRI_KERNEL */, &thread);
1610 if (result == KERN_SUCCESS) {
1611 thread_deallocate(thread);
1612 } else {
1613 panic("Could not create memorystatus_thread");
1614 }
1615 }
1616
1617 /* Centralised for the purposes of allowing panic-on-jetsam */
1618 extern void
1619 vm_run_compactor(void);
1620
1621 /*
1622 * The jetsam no frills kill call
1623 * Return: 0 on success
1624 * error code on failure (EINVAL...)
1625 */
1626 static int
1627 jetsam_do_kill(proc_t p, int jetsam_flags, os_reason_t jetsam_reason) {
1628 int error = 0;
1629 error = exit_with_reason(p, W_EXITCODE(0, SIGKILL), (int *)NULL, FALSE, FALSE, jetsam_flags, jetsam_reason);
1630 return(error);
1631 }
1632
1633 /*
1634 * Wrapper for processes exiting with memorystatus details
1635 */
1636 static boolean_t
1637 memorystatus_do_kill(proc_t p, uint32_t cause, os_reason_t jetsam_reason) {
1638
1639 int error = 0;
1640 __unused pid_t victim_pid = p->p_pid;
1641
1642 KERNEL_DEBUG_CONSTANT( (BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_DO_KILL)) | DBG_FUNC_START,
1643 victim_pid, cause, vm_page_free_count, 0, 0);
1644
1645 DTRACE_MEMORYSTATUS3(memorystatus_do_kill, proc_t, p, os_reason_t, jetsam_reason, uint32_t, cause);
1646 #if CONFIG_JETSAM && (DEVELOPMENT || DEBUG)
1647 if (memorystatus_jetsam_panic_debug & (1 << cause)) {
1648 panic("memorystatus_do_kill(): jetsam debug panic (cause: %d)", cause);
1649 }
1650 #else
1651 #pragma unused(cause)
1652 #endif
1653
1654 if (p->p_memstat_effectivepriority >= JETSAM_PRIORITY_FOREGROUND) {
1655 printf("memorystatus: killing process %d [%s] in high band %s (%d) - memorystatus_available_pages: %llu\n", p->p_pid,
1656 (*p->p_name ? p->p_name : "unknown"),
1657 memorystatus_priority_band_name(p->p_memstat_effectivepriority), p->p_memstat_effectivepriority,
1658 (uint64_t)memorystatus_available_pages);
1659 }
1660
1661 int jetsam_flags = P_LTERM_JETSAM;
1662 switch (cause) {
1663 case kMemorystatusKilledHiwat: jetsam_flags |= P_JETSAM_HIWAT; break;
1664 case kMemorystatusKilledVnodes: jetsam_flags |= P_JETSAM_VNODE; break;
1665 case kMemorystatusKilledVMPageShortage: jetsam_flags |= P_JETSAM_VMPAGESHORTAGE; break;
1666 case kMemorystatusKilledVMThrashing: jetsam_flags |= P_JETSAM_VMTHRASHING; break;
1667 case kMemorystatusKilledFCThrashing: jetsam_flags |= P_JETSAM_FCTHRASHING; break;
1668 case kMemorystatusKilledPerProcessLimit: jetsam_flags |= P_JETSAM_PID; break;
1669 case kMemorystatusKilledIdleExit: jetsam_flags |= P_JETSAM_IDLEEXIT; break;
1670 }
1671 error = jetsam_do_kill(p, jetsam_flags, jetsam_reason);
1672
1673 KERNEL_DEBUG_CONSTANT( (BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_DO_KILL)) | DBG_FUNC_END,
1674 victim_pid, cause, vm_page_free_count, error, 0);
1675
1676 vm_run_compactor();
1677
1678 return (error == 0);
1679 }
1680
1681 /*
1682 * Node manipulation
1683 */
1684
1685 static void
1686 memorystatus_check_levels_locked(void) {
1687 #if CONFIG_JETSAM
1688 /* Update levels */
1689 memorystatus_update_levels_locked(TRUE);
1690 #else /* CONFIG_JETSAM */
1691 /*
1692 * Nothing to do here currently since we update
1693 * memorystatus_available_pages in vm_pressure_response.
1694 */
1695 #endif /* CONFIG_JETSAM */
1696 }
1697
1698 /*
1699 * Pin a process to a particular jetsam band when it is in the background i.e. not doing active work.
1700 * For an application: that means no longer in the FG band
1701 * For a daemon: that means no longer in its 'requested' jetsam priority band
1702 */
1703
1704 int
1705 memorystatus_update_inactive_jetsam_priority_band(pid_t pid, uint32_t op_flags, boolean_t effective_now)
1706 {
1707 int error = 0;
1708 boolean_t enable = FALSE;
1709 proc_t p = NULL;
1710
1711 if (op_flags == MEMORYSTATUS_CMD_ELEVATED_INACTIVEJETSAMPRIORITY_ENABLE) {
1712 enable = TRUE;
1713 } else if (op_flags == MEMORYSTATUS_CMD_ELEVATED_INACTIVEJETSAMPRIORITY_DISABLE) {
1714 enable = FALSE;
1715 } else {
1716 return EINVAL;
1717 }
1718
1719 p = proc_find(pid);
1720 if (p != NULL) {
1721
1722 if ((enable && ((p->p_memstat_state & P_MEMSTAT_USE_ELEVATED_INACTIVE_BAND) == P_MEMSTAT_USE_ELEVATED_INACTIVE_BAND)) ||
1723 (!enable && ((p->p_memstat_state & P_MEMSTAT_USE_ELEVATED_INACTIVE_BAND) == 0))) {
1724 /*
1725 * No change in state.
1726 */
1727
1728 } else {
1729
1730 proc_list_lock();
1731
1732 if (enable) {
1733 p->p_memstat_state |= P_MEMSTAT_USE_ELEVATED_INACTIVE_BAND;
1734 memorystatus_invalidate_idle_demotion_locked(p, TRUE);
1735
1736 if (effective_now) {
1737 if (p->p_memstat_effectivepriority < JETSAM_PRIORITY_ELEVATED_INACTIVE) {
1738 if(memorystatus_highwater_enabled) {
1739 /*
1740 * Process is about to transition from
1741 * inactive --> active
1742 * assign active state
1743 */
1744 boolean_t is_fatal;
1745 boolean_t use_active = TRUE;
1746 CACHE_ACTIVE_LIMITS_LOCKED(p, is_fatal);
1747 task_set_phys_footprint_limit_internal(p->task, (p->p_memstat_memlimit > 0) ? p->p_memstat_memlimit : -1, NULL, use_active, is_fatal);
1748 }
1749 memorystatus_update_priority_locked(p, JETSAM_PRIORITY_ELEVATED_INACTIVE, FALSE, FALSE);
1750 }
1751 } else {
1752 if (isProcessInAgingBands(p)) {
1753 memorystatus_update_priority_locked(p, JETSAM_PRIORITY_IDLE, FALSE, TRUE);
1754 }
1755 }
1756 } else {
1757
1758 p->p_memstat_state &= ~P_MEMSTAT_USE_ELEVATED_INACTIVE_BAND;
1759 memorystatus_invalidate_idle_demotion_locked(p, TRUE);
1760
1761 if (effective_now) {
1762 if (p->p_memstat_effectivepriority == JETSAM_PRIORITY_ELEVATED_INACTIVE) {
1763 memorystatus_update_priority_locked(p, JETSAM_PRIORITY_IDLE, FALSE, TRUE);
1764 }
1765 } else {
1766 if (isProcessInAgingBands(p)) {
1767 memorystatus_update_priority_locked(p, JETSAM_PRIORITY_IDLE, FALSE, TRUE);
1768 }
1769 }
1770 }
1771
1772 proc_list_unlock();
1773 }
1774 proc_rele(p);
1775 error = 0;
1776
1777 } else {
1778 error = ESRCH;
1779 }
1780
1781 return error;
1782 }
1783
1784 static void
1785 memorystatus_perform_idle_demotion(__unused void *spare1, __unused void *spare2)
1786 {
1787 proc_t p;
1788 uint64_t current_time = 0, idle_delay_time = 0;
1789 int demote_prio_band = 0;
1790 memstat_bucket_t *demotion_bucket;
1791
1792 MEMORYSTATUS_DEBUG(1, "memorystatus_perform_idle_demotion()\n");
1793
1794 KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_IDLE_DEMOTE) | DBG_FUNC_START, 0, 0, 0, 0, 0);
1795
1796 current_time = mach_absolute_time();
1797
1798 proc_list_lock();
1799
1800 demote_prio_band = JETSAM_PRIORITY_IDLE + 1;
1801
1802 for (; demote_prio_band < JETSAM_PRIORITY_MAX; demote_prio_band++) {
1803
1804 if (demote_prio_band != system_procs_aging_band && demote_prio_band != applications_aging_band)
1805 continue;
1806
1807 demotion_bucket = &memstat_bucket[demote_prio_band];
1808 p = TAILQ_FIRST(&demotion_bucket->list);
1809
1810 while (p) {
1811 MEMORYSTATUS_DEBUG(1, "memorystatus_perform_idle_demotion() found %d\n", p->p_pid);
1812
1813 assert(p->p_memstat_idledeadline);
1814
1815 assert(p->p_memstat_dirty & P_DIRTY_AGING_IN_PROGRESS);
1816
1817 if (current_time >= p->p_memstat_idledeadline) {
1818
1819 if ((isSysProc(p) &&
1820 ((p->p_memstat_dirty & (P_DIRTY_IDLE_EXIT_ENABLED|P_DIRTY_IS_DIRTY)) != P_DIRTY_IDLE_EXIT_ENABLED)) || /* system proc marked dirty*/
1821 task_has_assertions((struct task *)(p->task))) { /* has outstanding assertions which might indicate outstanding work too */
1822 idle_delay_time = (isSysProc(p)) ? memorystatus_sysprocs_idle_delay_time : memorystatus_apps_idle_delay_time;
1823
1824 p->p_memstat_idledeadline += idle_delay_time;
1825 p = TAILQ_NEXT(p, p_memstat_list);
1826
1827 } else {
1828
1829 proc_t next_proc = NULL;
1830
1831 next_proc = TAILQ_NEXT(p, p_memstat_list);
1832 memorystatus_invalidate_idle_demotion_locked(p, TRUE);
1833
1834 memorystatus_update_priority_locked(p, JETSAM_PRIORITY_IDLE, false, true);
1835
1836 p = next_proc;
1837 continue;
1838
1839 }
1840 } else {
1841 // No further candidates
1842 break;
1843 }
1844 }
1845
1846 }
1847
1848 memorystatus_reschedule_idle_demotion_locked();
1849
1850 proc_list_unlock();
1851
1852 KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_IDLE_DEMOTE) | DBG_FUNC_END, 0, 0, 0, 0, 0);
1853 }
1854
1855 static void
1856 memorystatus_schedule_idle_demotion_locked(proc_t p, boolean_t set_state)
1857 {
1858 boolean_t present_in_sysprocs_aging_bucket = FALSE;
1859 boolean_t present_in_apps_aging_bucket = FALSE;
1860 uint64_t idle_delay_time = 0;
1861
1862 if (jetsam_aging_policy == kJetsamAgingPolicyNone) {
1863 return;
1864 }
1865
1866 if (p->p_memstat_state & P_MEMSTAT_USE_ELEVATED_INACTIVE_BAND) {
1867 /*
1868 * This process isn't going to be making the trip to the lower bands.
1869 */
1870 return;
1871 }
1872
1873 if (isProcessInAgingBands(p)){
1874
1875 if (jetsam_aging_policy != kJetsamAgingPolicyLegacy) {
1876 assert((p->p_memstat_dirty & P_DIRTY_AGING_IN_PROGRESS) != P_DIRTY_AGING_IN_PROGRESS);
1877 }
1878
1879 if (isSysProc(p) && system_procs_aging_band) {
1880 present_in_sysprocs_aging_bucket = TRUE;
1881
1882 } else if (isApp(p) && applications_aging_band) {
1883 present_in_apps_aging_bucket = TRUE;
1884 }
1885 }
1886
1887 assert(!present_in_sysprocs_aging_bucket);
1888 assert(!present_in_apps_aging_bucket);
1889
1890 MEMORYSTATUS_DEBUG(1, "memorystatus_schedule_idle_demotion_locked: scheduling demotion to idle band for pid %d (dirty:0x%x, set_state %d, demotions %d).\n",
1891 p->p_pid, p->p_memstat_dirty, set_state, (memorystatus_scheduled_idle_demotions_sysprocs + memorystatus_scheduled_idle_demotions_apps));
1892
1893 if(isSysProc(p)) {
1894 assert((p->p_memstat_dirty & P_DIRTY_IDLE_EXIT_ENABLED) == P_DIRTY_IDLE_EXIT_ENABLED);
1895 }
1896
1897 idle_delay_time = (isSysProc(p)) ? memorystatus_sysprocs_idle_delay_time : memorystatus_apps_idle_delay_time;
1898
1899 if (set_state) {
1900 p->p_memstat_dirty |= P_DIRTY_AGING_IN_PROGRESS;
1901 p->p_memstat_idledeadline = mach_absolute_time() + idle_delay_time;
1902 }
1903
1904 assert(p->p_memstat_idledeadline);
1905
1906 if (isSysProc(p) && present_in_sysprocs_aging_bucket == FALSE) {
1907 memorystatus_scheduled_idle_demotions_sysprocs++;
1908
1909 } else if (isApp(p) && present_in_apps_aging_bucket == FALSE) {
1910 memorystatus_scheduled_idle_demotions_apps++;
1911 }
1912 }
1913
1914 static void
1915 memorystatus_invalidate_idle_demotion_locked(proc_t p, boolean_t clear_state)
1916 {
1917 boolean_t present_in_sysprocs_aging_bucket = FALSE;
1918 boolean_t present_in_apps_aging_bucket = FALSE;
1919
1920 if (!system_procs_aging_band && !applications_aging_band) {
1921 return;
1922 }
1923
1924 if ((p->p_memstat_dirty & P_DIRTY_AGING_IN_PROGRESS) == 0) {
1925 return;
1926 }
1927
1928 if (isProcessInAgingBands(p)) {
1929
1930 if (jetsam_aging_policy != kJetsamAgingPolicyLegacy) {
1931 assert((p->p_memstat_dirty & P_DIRTY_AGING_IN_PROGRESS) == P_DIRTY_AGING_IN_PROGRESS);
1932 }
1933
1934 if (isSysProc(p) && system_procs_aging_band) {
1935 assert(p->p_memstat_effectivepriority == system_procs_aging_band);
1936 assert(p->p_memstat_idledeadline);
1937 present_in_sysprocs_aging_bucket = TRUE;
1938
1939 } else if (isApp(p) && applications_aging_band) {
1940 assert(p->p_memstat_effectivepriority == applications_aging_band);
1941 assert(p->p_memstat_idledeadline);
1942 present_in_apps_aging_bucket = TRUE;
1943 }
1944 }
1945
1946 MEMORYSTATUS_DEBUG(1, "memorystatus_invalidate_idle_demotion(): invalidating demotion to idle band for pid %d (clear_state %d, demotions %d).\n",
1947 p->p_pid, clear_state, (memorystatus_scheduled_idle_demotions_sysprocs + memorystatus_scheduled_idle_demotions_apps));
1948
1949
1950 if (clear_state) {
1951 p->p_memstat_idledeadline = 0;
1952 p->p_memstat_dirty &= ~P_DIRTY_AGING_IN_PROGRESS;
1953 }
1954
1955 if (isSysProc(p) &&present_in_sysprocs_aging_bucket == TRUE) {
1956 memorystatus_scheduled_idle_demotions_sysprocs--;
1957 assert(memorystatus_scheduled_idle_demotions_sysprocs >= 0);
1958
1959 } else if (isApp(p) && present_in_apps_aging_bucket == TRUE) {
1960 memorystatus_scheduled_idle_demotions_apps--;
1961 assert(memorystatus_scheduled_idle_demotions_apps >= 0);
1962 }
1963
1964 assert((memorystatus_scheduled_idle_demotions_sysprocs + memorystatus_scheduled_idle_demotions_apps) >= 0);
1965 }
1966
1967 static void
1968 memorystatus_reschedule_idle_demotion_locked(void) {
1969 if (0 == (memorystatus_scheduled_idle_demotions_sysprocs + memorystatus_scheduled_idle_demotions_apps)) {
1970 if (memstat_idle_demotion_deadline) {
1971 /* Transitioned 1->0, so cancel next call */
1972 thread_call_cancel(memorystatus_idle_demotion_call);
1973 memstat_idle_demotion_deadline = 0;
1974 }
1975 } else {
1976 memstat_bucket_t *demotion_bucket;
1977 proc_t p = NULL, p1 = NULL, p2 = NULL;
1978
1979 if (system_procs_aging_band) {
1980
1981 demotion_bucket = &memstat_bucket[system_procs_aging_band];
1982 p1 = TAILQ_FIRST(&demotion_bucket->list);
1983
1984 p = p1;
1985 }
1986
1987 if (applications_aging_band) {
1988
1989 demotion_bucket = &memstat_bucket[applications_aging_band];
1990 p2 = TAILQ_FIRST(&demotion_bucket->list);
1991
1992 if (p1 && p2) {
1993 p = (p1->p_memstat_idledeadline > p2->p_memstat_idledeadline) ? p2 : p1;
1994 } else {
1995 p = (p1 == NULL) ? p2 : p1;
1996 }
1997
1998 }
1999
2000 assert(p);
2001
2002 if (p != NULL) {
2003 assert(p && p->p_memstat_idledeadline);
2004 if (memstat_idle_demotion_deadline != p->p_memstat_idledeadline){
2005 thread_call_enter_delayed(memorystatus_idle_demotion_call, p->p_memstat_idledeadline);
2006 memstat_idle_demotion_deadline = p->p_memstat_idledeadline;
2007 }
2008 }
2009 }
2010 }
2011
2012 /*
2013 * List manipulation
2014 */
2015
2016 int
2017 memorystatus_add(proc_t p, boolean_t locked)
2018 {
2019 memstat_bucket_t *bucket;
2020
2021 MEMORYSTATUS_DEBUG(1, "memorystatus_list_add(): adding pid %d with priority %d.\n", p->p_pid, p->p_memstat_effectivepriority);
2022
2023 if (!locked) {
2024 proc_list_lock();
2025 }
2026
2027 DTRACE_MEMORYSTATUS2(memorystatus_add, proc_t, p, int32_t, p->p_memstat_effectivepriority);
2028
2029 /* Processes marked internal do not have priority tracked */
2030 if (p->p_memstat_state & P_MEMSTAT_INTERNAL) {
2031 goto exit;
2032 }
2033
2034 bucket = &memstat_bucket[p->p_memstat_effectivepriority];
2035
2036 if (isSysProc(p) && system_procs_aging_band && (p->p_memstat_effectivepriority == system_procs_aging_band)) {
2037 assert(bucket->count == memorystatus_scheduled_idle_demotions_sysprocs - 1);
2038
2039 } else if (isApp(p) && applications_aging_band && (p->p_memstat_effectivepriority == applications_aging_band)) {
2040 assert(bucket->count == memorystatus_scheduled_idle_demotions_apps - 1);
2041
2042 } else if (p->p_memstat_effectivepriority == JETSAM_PRIORITY_IDLE) {
2043 /*
2044 * Entering the idle band.
2045 * Record idle start time.
2046 */
2047 p->p_memstat_idle_start = mach_absolute_time();
2048 }
2049
2050 TAILQ_INSERT_TAIL(&bucket->list, p, p_memstat_list);
2051 bucket->count++;
2052
2053 memorystatus_list_count++;
2054
2055 memorystatus_check_levels_locked();
2056
2057 exit:
2058 if (!locked) {
2059 proc_list_unlock();
2060 }
2061
2062 return 0;
2063 }
2064
2065 /*
2066 * Description:
2067 * Moves a process from one jetsam bucket to another.
2068 * which changes the LRU position of the process.
2069 *
2070 * Monitors transition between buckets and if necessary
2071 * will update cached memory limits accordingly.
2072 *
2073 * skip_demotion_check:
2074 * - if the 'jetsam aging policy' is NOT 'legacy':
2075 * When this flag is TRUE, it means we are going
2076 * to age the ripe processes out of the aging bands and into the
2077 * IDLE band and apply their inactive memory limits.
2078 *
2079 * - if the 'jetsam aging policy' is 'legacy':
2080 * When this flag is TRUE, it might mean the above aging mechanism
2081 * OR
2082 * It might be that we have a process that has used up its 'idle deferral'
2083 * stay that is given to it once per lifetime. And in this case, the process
2084 * won't be going through any aging codepaths. But we still need to apply
2085 * the right inactive limits and so we explicitly set this to TRUE if the
2086 * new priority for the process is the IDLE band.
2087 */
2088 void
2089 memorystatus_update_priority_locked(proc_t p, int priority, boolean_t head_insert, boolean_t skip_demotion_check)
2090 {
2091 memstat_bucket_t *old_bucket, *new_bucket;
2092
2093 assert(priority < MEMSTAT_BUCKET_COUNT);
2094
2095 /* Ensure that exit isn't underway, leaving the proc retained but removed from its bucket */
2096 if ((p->p_listflag & P_LIST_EXITED) != 0) {
2097 return;
2098 }
2099
2100 MEMORYSTATUS_DEBUG(1, "memorystatus_update_priority_locked(): setting %s(%d) to priority %d, inserting at %s\n",
2101 (*p->p_name ? p->p_name : "unknown"), p->p_pid, priority, head_insert ? "head" : "tail");
2102
2103 DTRACE_MEMORYSTATUS3(memorystatus_update_priority, proc_t, p, int32_t, p->p_memstat_effectivepriority, int, priority);
2104
2105 #if DEVELOPMENT || DEBUG
2106 if (priority == JETSAM_PRIORITY_IDLE && /* if the process is on its way into the IDLE band */
2107 skip_demotion_check == FALSE && /* and it isn't via the path that will set the INACTIVE memlimits */
2108 (p->p_memstat_dirty & P_DIRTY_TRACK) && /* and it has 'DIRTY' tracking enabled */
2109 ((p->p_memstat_memlimit != p->p_memstat_memlimit_inactive) || /* and we notice that the current limit isn't the right value (inactive) */
2110 ((p->p_memstat_state & P_MEMSTAT_MEMLIMIT_INACTIVE_FATAL) ? ( ! (p->p_memstat_state & P_MEMSTAT_FATAL_MEMLIMIT)) : (p->p_memstat_state & P_MEMSTAT_FATAL_MEMLIMIT)))) /* OR type (fatal vs non-fatal) */
2111 panic("memorystatus_update_priority_locked: on %s with 0x%x, prio: %d and %d\n", p->p_name, p->p_memstat_state, priority, p->p_memstat_memlimit); /* then we must catch this */
2112 #endif /* DEVELOPMENT || DEBUG */
2113
2114 old_bucket = &memstat_bucket[p->p_memstat_effectivepriority];
2115
2116 if (skip_demotion_check == FALSE) {
2117
2118 if (isSysProc(p)) {
2119 /*
2120 * For system processes, the memorystatus_dirty_* routines take care of adding/removing
2121 * the processes from the aging bands and balancing the demotion counts.
2122 * We can, however, override that if the process has an 'elevated inactive jetsam band' attribute.
2123 */
2124
2125 if (priority <= JETSAM_PRIORITY_ELEVATED_INACTIVE && (p->p_memstat_state & P_MEMSTAT_USE_ELEVATED_INACTIVE_BAND)) {
2126 priority = JETSAM_PRIORITY_ELEVATED_INACTIVE;
2127
2128 assert(! (p->p_memstat_dirty & P_DIRTY_AGING_IN_PROGRESS));
2129 }
2130 } else if (isApp(p)) {
2131
2132 /*
2133 * Check to see if the application is being lowered in jetsam priority. If so, and:
2134 * - it has an 'elevated inactive jetsam band' attribute, then put it in the JETSAM_PRIORITY_ELEVATED_INACTIVE band.
2135 * - it is a normal application, then let it age in the aging band if that policy is in effect.
2136 */
2137
2138 if (priority <= JETSAM_PRIORITY_ELEVATED_INACTIVE && (p->p_memstat_state & P_MEMSTAT_USE_ELEVATED_INACTIVE_BAND)) {
2139 priority = JETSAM_PRIORITY_ELEVATED_INACTIVE;
2140 } else {
2141
2142 if (applications_aging_band) {
2143 if (p->p_memstat_effectivepriority == applications_aging_band) {
2144 assert(old_bucket->count == (memorystatus_scheduled_idle_demotions_apps + 1));
2145 }
2146
2147 if ((jetsam_aging_policy != kJetsamAgingPolicyLegacy) && (priority <= applications_aging_band)) {
2148 assert(! (p->p_memstat_dirty & P_DIRTY_AGING_IN_PROGRESS));
2149 priority = applications_aging_band;
2150 memorystatus_schedule_idle_demotion_locked(p, TRUE);
2151 }
2152 }
2153 }
2154 }
2155 }
2156
2157 if ((system_procs_aging_band && (priority == system_procs_aging_band)) || (applications_aging_band && (priority == applications_aging_band))) {
2158 assert(p->p_memstat_dirty & P_DIRTY_AGING_IN_PROGRESS);
2159 }
2160
2161 TAILQ_REMOVE(&old_bucket->list, p, p_memstat_list);
2162 old_bucket->count--;
2163
2164 new_bucket = &memstat_bucket[priority];
2165 if (head_insert)
2166 TAILQ_INSERT_HEAD(&new_bucket->list, p, p_memstat_list);
2167 else
2168 TAILQ_INSERT_TAIL(&new_bucket->list, p, p_memstat_list);
2169 new_bucket->count++;
2170
2171 if (memorystatus_highwater_enabled) {
2172 boolean_t is_fatal;
2173 boolean_t use_active;
2174
2175 /*
2176 * If cached limit data is updated, then the limits
2177 * will be enforced by writing to the ledgers.
2178 */
2179 boolean_t ledger_update_needed = TRUE;
2180
2181 /*
2182 * Here, we must update the cached memory limit if the task
2183 * is transitioning between:
2184 * active <--> inactive
2185 * FG <--> BG
2186 * but:
2187 * dirty <--> clean is ignored
2188 *
2189 * We bypass non-idle processes that have opted into dirty tracking because
2190 * a move between buckets does not imply a transition between the
2191 * dirty <--> clean state.
2192 */
2193
2194 if (p->p_memstat_dirty & P_DIRTY_TRACK) {
2195
2196 if (skip_demotion_check == TRUE && priority == JETSAM_PRIORITY_IDLE) {
2197 CACHE_INACTIVE_LIMITS_LOCKED(p, is_fatal);
2198 use_active = FALSE;
2199 } else {
2200 ledger_update_needed = FALSE;
2201 }
2202
2203 } else if ((priority >= JETSAM_PRIORITY_FOREGROUND) && (p->p_memstat_effectivepriority < JETSAM_PRIORITY_FOREGROUND)) {
2204 /*
2205 * inactive --> active
2206 * BG --> FG
2207 * assign active state
2208 */
2209 CACHE_ACTIVE_LIMITS_LOCKED(p, is_fatal);
2210 use_active = TRUE;
2211
2212 } else if ((priority < JETSAM_PRIORITY_FOREGROUND) && (p->p_memstat_effectivepriority >= JETSAM_PRIORITY_FOREGROUND)) {
2213 /*
2214 * active --> inactive
2215 * FG --> BG
2216 * assign inactive state
2217 */
2218 CACHE_INACTIVE_LIMITS_LOCKED(p, is_fatal);
2219 use_active = FALSE;
2220 } else {
2221 /*
2222 * The transition between jetsam priority buckets apparently did
2223 * not affect active/inactive state.
2224 * This is not unusual... especially during startup when
2225 * processes are getting established in their respective bands.
2226 */
2227 ledger_update_needed = FALSE;
2228 }
2229
2230 /*
2231 * Enforce the new limits by writing to the ledger
2232 */
2233 if (ledger_update_needed) {
2234 task_set_phys_footprint_limit_internal(p->task, (p->p_memstat_memlimit > 0) ? p->p_memstat_memlimit : -1, NULL, use_active, is_fatal);
2235
2236 MEMORYSTATUS_DEBUG(3, "memorystatus_update_priority_locked: new limit on pid %d (%dMB %s) priority old --> new (%d --> %d) dirty?=0x%x %s\n",
2237 p->p_pid, (p->p_memstat_memlimit > 0 ? p->p_memstat_memlimit : -1),
2238 (p->p_memstat_state & P_MEMSTAT_FATAL_MEMLIMIT ? "F " : "NF"), p->p_memstat_effectivepriority, priority, p->p_memstat_dirty,
2239 (p->p_memstat_dirty ? ((p->p_memstat_dirty & P_DIRTY) ? "isdirty" : "isclean") : ""));
2240 }
2241 }
2242
2243 /*
2244 * Record idle start or idle delta.
2245 */
2246 if (p->p_memstat_effectivepriority == priority) {
2247 /*
2248 * This process is not transitioning between
2249 * jetsam priority buckets. Do nothing.
2250 */
2251 } else if (p->p_memstat_effectivepriority == JETSAM_PRIORITY_IDLE) {
2252 uint64_t now;
2253 /*
2254 * Transitioning out of the idle priority bucket.
2255 * Record idle delta.
2256 */
2257 assert(p->p_memstat_idle_start != 0);
2258 now = mach_absolute_time();
2259 if (now > p->p_memstat_idle_start) {
2260 p->p_memstat_idle_delta = now - p->p_memstat_idle_start;
2261 }
2262 } else if (priority == JETSAM_PRIORITY_IDLE) {
2263 /*
2264 * Transitioning into the idle priority bucket.
2265 * Record idle start.
2266 */
2267 p->p_memstat_idle_start = mach_absolute_time();
2268 }
2269
2270 p->p_memstat_effectivepriority = priority;
2271
2272 #if CONFIG_SECLUDED_MEMORY
2273 if (secluded_for_apps &&
2274 task_could_use_secluded_mem(p->task)) {
2275 task_set_can_use_secluded_mem(
2276 p->task,
2277 (priority >= JETSAM_PRIORITY_FOREGROUND));
2278 }
2279 #endif /* CONFIG_SECLUDED_MEMORY */
2280
2281 memorystatus_check_levels_locked();
2282 }
2283
2284 /*
2285 *
2286 * Description: Update the jetsam priority and memory limit attributes for a given process.
2287 *
2288 * Parameters:
2289 * p init this process's jetsam information.
2290 * priority The jetsam priority band
2291 * user_data user specific data, unused by the kernel
2292 * effective guards against race if process's update already occurred
2293 * update_memlimit When true we know this is the init step via the posix_spawn path.
2294 *
2295 * memlimit_active Value in megabytes; The monitored footprint level while the
2296 * process is active. Exceeding it may result in termination
2297 * based on it's associated fatal flag.
2298 *
2299 * memlimit_active_is_fatal When a process is active and exceeds its memory footprint,
2300 * this describes whether or not it should be immediately fatal.
2301 *
2302 * memlimit_inactive Value in megabytes; The monitored footprint level while the
2303 * process is inactive. Exceeding it may result in termination
2304 * based on it's associated fatal flag.
2305 *
2306 * memlimit_inactive_is_fatal When a process is inactive and exceeds its memory footprint,
2307 * this describes whether or not it should be immediatly fatal.
2308 *
2309 * Returns: 0 Success
2310 * non-0 Failure
2311 */
2312
2313 int
2314 memorystatus_update(proc_t p, int priority, uint64_t user_data, boolean_t effective, boolean_t update_memlimit,
2315 int32_t memlimit_active, boolean_t memlimit_active_is_fatal,
2316 int32_t memlimit_inactive, boolean_t memlimit_inactive_is_fatal)
2317 {
2318 int ret;
2319 boolean_t head_insert = false;
2320
2321 MEMORYSTATUS_DEBUG(1, "memorystatus_update: changing (%s) pid %d: priority %d, user_data 0x%llx\n", (*p->p_name ? p->p_name : "unknown"), p->p_pid, priority, user_data);
2322
2323 KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_UPDATE) | DBG_FUNC_START, p->p_pid, priority, user_data, effective, 0);
2324
2325 if (priority == -1) {
2326 /* Use as shorthand for default priority */
2327 priority = JETSAM_PRIORITY_DEFAULT;
2328 } else if ((priority == system_procs_aging_band) || (priority == applications_aging_band)) {
2329 /* Both the aging bands are reserved for internal use; if requested, adjust to JETSAM_PRIORITY_IDLE. */
2330 priority = JETSAM_PRIORITY_IDLE;
2331 } else if (priority == JETSAM_PRIORITY_IDLE_HEAD) {
2332 /* JETSAM_PRIORITY_IDLE_HEAD inserts at the head of the idle queue */
2333 priority = JETSAM_PRIORITY_IDLE;
2334 head_insert = TRUE;
2335 } else if ((priority < 0) || (priority >= MEMSTAT_BUCKET_COUNT)) {
2336 /* Sanity check */
2337 ret = EINVAL;
2338 goto out;
2339 }
2340
2341 proc_list_lock();
2342
2343 assert(!(p->p_memstat_state & P_MEMSTAT_INTERNAL));
2344
2345 if (effective && (p->p_memstat_state & P_MEMSTAT_PRIORITYUPDATED)) {
2346 ret = EALREADY;
2347 proc_list_unlock();
2348 MEMORYSTATUS_DEBUG(1, "memorystatus_update: effective change specified for pid %d, but change already occurred.\n", p->p_pid);
2349 goto out;
2350 }
2351
2352 if ((p->p_memstat_state & P_MEMSTAT_TERMINATED) || ((p->p_listflag & P_LIST_EXITED) != 0)) {
2353 /*
2354 * This could happen when a process calling posix_spawn() is exiting on the jetsam thread.
2355 */
2356 ret = EBUSY;
2357 proc_list_unlock();
2358 goto out;
2359 }
2360
2361 p->p_memstat_state |= P_MEMSTAT_PRIORITYUPDATED;
2362 p->p_memstat_userdata = user_data;
2363 p->p_memstat_requestedpriority = priority;
2364
2365 if (update_memlimit) {
2366 boolean_t is_fatal;
2367 boolean_t use_active;
2368
2369 /*
2370 * Posix_spawn'd processes come through this path to instantiate ledger limits.
2371 * Forked processes do not come through this path, so no ledger limits exist.
2372 * (That's why forked processes can consume unlimited memory.)
2373 */
2374
2375 MEMORYSTATUS_DEBUG(3, "memorystatus_update(enter): pid %d, priority %d, dirty=0x%x, Active(%dMB %s), Inactive(%dMB, %s)\n",
2376 p->p_pid, priority, p->p_memstat_dirty,
2377 memlimit_active, (memlimit_active_is_fatal ? "F " : "NF"),
2378 memlimit_inactive, (memlimit_inactive_is_fatal ? "F " : "NF"));
2379
2380 if (memlimit_active <= 0) {
2381 /*
2382 * This process will have a system_wide task limit when active.
2383 * System_wide task limit is always fatal.
2384 * It's quite common to see non-fatal flag passed in here.
2385 * It's not an error, we just ignore it.
2386 */
2387
2388 /*
2389 * For backward compatibility with some unexplained launchd behavior,
2390 * we allow a zero sized limit. But we still enforce system_wide limit
2391 * when written to the ledgers.
2392 */
2393
2394 if (memlimit_active < 0) {
2395 memlimit_active = -1; /* enforces system_wide task limit */
2396 }
2397 memlimit_active_is_fatal = TRUE;
2398 }
2399
2400 if (memlimit_inactive <= 0) {
2401 /*
2402 * This process will have a system_wide task limit when inactive.
2403 * System_wide task limit is always fatal.
2404 */
2405
2406 memlimit_inactive = -1;
2407 memlimit_inactive_is_fatal = TRUE;
2408 }
2409
2410 /*
2411 * Initialize the active limit variants for this process.
2412 */
2413 SET_ACTIVE_LIMITS_LOCKED(p, memlimit_active, memlimit_active_is_fatal);
2414
2415 /*
2416 * Initialize the inactive limit variants for this process.
2417 */
2418 SET_INACTIVE_LIMITS_LOCKED(p, memlimit_inactive, memlimit_inactive_is_fatal);
2419
2420 /*
2421 * Initialize the cached limits for target process.
2422 * When the target process is dirty tracked, it's typically
2423 * in a clean state. Non dirty tracked processes are
2424 * typically active (Foreground or above).
2425 * But just in case, we don't make assumptions...
2426 */
2427
2428 if (proc_jetsam_state_is_active_locked(p) == TRUE) {
2429 CACHE_ACTIVE_LIMITS_LOCKED(p, is_fatal);
2430 use_active = TRUE;
2431 } else {
2432 CACHE_INACTIVE_LIMITS_LOCKED(p, is_fatal);
2433 use_active = FALSE;
2434 }
2435
2436 /*
2437 * Enforce the cached limit by writing to the ledger.
2438 */
2439 if (memorystatus_highwater_enabled) {
2440 /* apply now */
2441 task_set_phys_footprint_limit_internal(p->task, ((p->p_memstat_memlimit > 0) ? p->p_memstat_memlimit : -1), NULL, use_active, is_fatal);
2442
2443 MEMORYSTATUS_DEBUG(3, "memorystatus_update: init: limit on pid %d (%dMB %s) targeting priority(%d) dirty?=0x%x %s\n",
2444 p->p_pid, (p->p_memstat_memlimit > 0 ? p->p_memstat_memlimit : -1),
2445 (p->p_memstat_state & P_MEMSTAT_FATAL_MEMLIMIT ? "F " : "NF"), priority, p->p_memstat_dirty,
2446 (p->p_memstat_dirty ? ((p->p_memstat_dirty & P_DIRTY) ? "isdirty" : "isclean") : ""));
2447 }
2448 }
2449
2450 /*
2451 * We can't add to the aging bands buckets here.
2452 * But, we could be removing it from those buckets.
2453 * Check and take appropriate steps if so.
2454 */
2455
2456 if (isProcessInAgingBands(p)) {
2457
2458 memorystatus_invalidate_idle_demotion_locked(p, TRUE);
2459 memorystatus_update_priority_locked(p, JETSAM_PRIORITY_IDLE, FALSE, TRUE);
2460 } else {
2461 if (jetsam_aging_policy == kJetsamAgingPolicyLegacy && priority == JETSAM_PRIORITY_IDLE) {
2462 /*
2463 * Daemons with 'inactive' limits will go through the dirty tracking codepath.
2464 * This path deals with apps that may have 'inactive' limits e.g. WebContent processes.
2465 * If this is the legacy aging policy we explicitly need to apply those limits. If it
2466 * is any other aging policy, then we don't need to worry because all processes
2467 * will go through the aging bands and then the demotion thread will take care to
2468 * move them into the IDLE band and apply the required limits.
2469 */
2470 memorystatus_update_priority_locked(p, priority, head_insert, TRUE);
2471 }
2472 }
2473
2474 memorystatus_update_priority_locked(p, priority, head_insert, FALSE);
2475
2476 proc_list_unlock();
2477 ret = 0;
2478
2479 out:
2480 KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_UPDATE) | DBG_FUNC_END, ret, 0, 0, 0, 0);
2481
2482 return ret;
2483 }
2484
2485 int
2486 memorystatus_remove(proc_t p, boolean_t locked)
2487 {
2488 int ret;
2489 memstat_bucket_t *bucket;
2490 boolean_t reschedule = FALSE;
2491
2492 MEMORYSTATUS_DEBUG(1, "memorystatus_list_remove: removing pid %d\n", p->p_pid);
2493
2494 if (!locked) {
2495 proc_list_lock();
2496 }
2497
2498 assert(!(p->p_memstat_state & P_MEMSTAT_INTERNAL));
2499
2500 bucket = &memstat_bucket[p->p_memstat_effectivepriority];
2501
2502 if (isSysProc(p) && system_procs_aging_band && (p->p_memstat_effectivepriority == system_procs_aging_band)) {
2503
2504 assert(bucket->count == memorystatus_scheduled_idle_demotions_sysprocs);
2505 reschedule = TRUE;
2506
2507 } else if (isApp(p) && applications_aging_band && (p->p_memstat_effectivepriority == applications_aging_band)) {
2508
2509 assert(bucket->count == memorystatus_scheduled_idle_demotions_apps);
2510 reschedule = TRUE;
2511 }
2512
2513 /*
2514 * Record idle delta
2515 */
2516
2517 if (p->p_memstat_effectivepriority == JETSAM_PRIORITY_IDLE) {
2518 uint64_t now = mach_absolute_time();
2519 if (now > p->p_memstat_idle_start) {
2520 p->p_memstat_idle_delta = now - p->p_memstat_idle_start;
2521 }
2522 }
2523
2524 TAILQ_REMOVE(&bucket->list, p, p_memstat_list);
2525 bucket->count--;
2526
2527 memorystatus_list_count--;
2528
2529 /* If awaiting demotion to the idle band, clean up */
2530 if (reschedule) {
2531 memorystatus_invalidate_idle_demotion_locked(p, TRUE);
2532 memorystatus_reschedule_idle_demotion_locked();
2533 }
2534
2535 memorystatus_check_levels_locked();
2536
2537 #if CONFIG_FREEZE
2538 if (p->p_memstat_state & (P_MEMSTAT_FROZEN)) {
2539 memorystatus_frozen_count--;
2540 }
2541
2542 if (p->p_memstat_state & P_MEMSTAT_SUSPENDED) {
2543 memorystatus_suspended_footprint_total -= p->p_memstat_suspendedfootprint;
2544 memorystatus_suspended_count--;
2545 }
2546 #endif
2547
2548 if (!locked) {
2549 proc_list_unlock();
2550 }
2551
2552 if (p) {
2553 ret = 0;
2554 } else {
2555 ret = ESRCH;
2556 }
2557
2558 return ret;
2559 }
2560
2561 /*
2562 * Validate dirty tracking flags with process state.
2563 *
2564 * Return:
2565 * 0 on success
2566 * non-0 on failure
2567 *
2568 * The proc_list_lock is held by the caller.
2569 */
2570
2571 static int
2572 memorystatus_validate_track_flags(struct proc *target_p, uint32_t pcontrol) {
2573 /* See that the process isn't marked for termination */
2574 if (target_p->p_memstat_dirty & P_DIRTY_TERMINATED) {
2575 return EBUSY;
2576 }
2577
2578 /* Idle exit requires that process be tracked */
2579 if ((pcontrol & PROC_DIRTY_ALLOW_IDLE_EXIT) &&
2580 !(pcontrol & PROC_DIRTY_TRACK)) {
2581 return EINVAL;
2582 }
2583
2584 /* 'Launch in progress' tracking requires that process have enabled dirty tracking too. */
2585 if ((pcontrol & PROC_DIRTY_LAUNCH_IN_PROGRESS) &&
2586 !(pcontrol & PROC_DIRTY_TRACK)) {
2587 return EINVAL;
2588 }
2589
2590 /* Deferral is only relevant if idle exit is specified */
2591 if ((pcontrol & PROC_DIRTY_DEFER) &&
2592 !(pcontrol & PROC_DIRTY_ALLOWS_IDLE_EXIT)) {
2593 return EINVAL;
2594 }
2595
2596 return(0);
2597 }
2598
2599 static void
2600 memorystatus_update_idle_priority_locked(proc_t p) {
2601 int32_t priority;
2602
2603 MEMORYSTATUS_DEBUG(1, "memorystatus_update_idle_priority_locked(): pid %d dirty 0x%X\n", p->p_pid, p->p_memstat_dirty);
2604
2605 assert(isSysProc(p));
2606
2607 if ((p->p_memstat_dirty & (P_DIRTY_IDLE_EXIT_ENABLED|P_DIRTY_IS_DIRTY)) == P_DIRTY_IDLE_EXIT_ENABLED) {
2608
2609 priority = (p->p_memstat_dirty & P_DIRTY_AGING_IN_PROGRESS) ? system_procs_aging_band : JETSAM_PRIORITY_IDLE;
2610 } else {
2611 priority = p->p_memstat_requestedpriority;
2612 }
2613
2614 if (priority != p->p_memstat_effectivepriority) {
2615
2616 if ((jetsam_aging_policy == kJetsamAgingPolicyLegacy) &&
2617 (priority == JETSAM_PRIORITY_IDLE)) {
2618
2619 /*
2620 * This process is on its way into the IDLE band. The system is
2621 * using 'legacy' jetsam aging policy. That means, this process
2622 * has already used up its idle-deferral aging time that is given
2623 * once per its lifetime. So we need to set the INACTIVE limits
2624 * explicitly because it won't be going through the demotion paths
2625 * that take care to apply the limits appropriately.
2626 */
2627
2628 if (p->p_memstat_state & P_MEMSTAT_USE_ELEVATED_INACTIVE_BAND) {
2629
2630 /*
2631 * This process has the 'elevated inactive jetsam band' attribute.
2632 * So, there will be no trip to IDLE after all.
2633 * Instead, we pin the process in the elevated band,
2634 * where its ACTIVE limits will apply.
2635 */
2636
2637 priority = JETSAM_PRIORITY_ELEVATED_INACTIVE;
2638 }
2639
2640 memorystatus_update_priority_locked(p, priority, false, true);
2641
2642 } else {
2643 memorystatus_update_priority_locked(p, priority, false, false);
2644 }
2645 }
2646 }
2647
2648 /*
2649 * Processes can opt to have their state tracked by the kernel, indicating when they are busy (dirty) or idle
2650 * (clean). They may also indicate that they support termination when idle, with the result that they are promoted
2651 * to their desired, higher, jetsam priority when dirty (and are therefore killed later), and demoted to the low
2652 * priority idle band when clean (and killed earlier, protecting higher priority procesess).
2653 *
2654 * If the deferral flag is set, then newly tracked processes will be protected for an initial period (as determined by
2655 * memorystatus_sysprocs_idle_delay_time); if they go clean during this time, then they will be moved to a deferred-idle band
2656 * with a slightly higher priority, guarding against immediate termination under memory pressure and being unable to
2657 * make forward progress. Finally, when the guard expires, they will be moved to the standard, lowest-priority, idle
2658 * band. The deferral can be cleared early by clearing the appropriate flag.
2659 *
2660 * The deferral timer is active only for the duration that the process is marked as guarded and clean; if the process
2661 * is marked dirty, the timer will be cancelled. Upon being subsequently marked clean, the deferment will either be
2662 * re-enabled or the guard state cleared, depending on whether the guard deadline has passed.
2663 */
2664
2665 int
2666 memorystatus_dirty_track(proc_t p, uint32_t pcontrol) {
2667 unsigned int old_dirty;
2668 boolean_t reschedule = FALSE;
2669 boolean_t already_deferred = FALSE;
2670 boolean_t defer_now = FALSE;
2671 int ret = 0;
2672
2673 KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_DIRTY_TRACK),
2674 p->p_pid, p->p_memstat_dirty, pcontrol, 0, 0);
2675
2676 proc_list_lock();
2677
2678 if ((p->p_listflag & P_LIST_EXITED) != 0) {
2679 /*
2680 * Process is on its way out.
2681 */
2682 ret = EBUSY;
2683 goto exit;
2684 }
2685
2686 if (p->p_memstat_state & P_MEMSTAT_INTERNAL) {
2687 ret = EPERM;
2688 goto exit;
2689 }
2690
2691 if ((ret = memorystatus_validate_track_flags(p, pcontrol)) != 0) {
2692 /* error */
2693 goto exit;
2694 }
2695
2696 old_dirty = p->p_memstat_dirty;
2697
2698 /* These bits are cumulative, as per <rdar://problem/11159924> */
2699 if (pcontrol & PROC_DIRTY_TRACK) {
2700 p->p_memstat_dirty |= P_DIRTY_TRACK;
2701 }
2702
2703 if (pcontrol & PROC_DIRTY_ALLOW_IDLE_EXIT) {
2704 p->p_memstat_dirty |= P_DIRTY_ALLOW_IDLE_EXIT;
2705 }
2706
2707 if (pcontrol & PROC_DIRTY_LAUNCH_IN_PROGRESS) {
2708 p->p_memstat_dirty |= P_DIRTY_LAUNCH_IN_PROGRESS;
2709 }
2710
2711 if (old_dirty & P_DIRTY_AGING_IN_PROGRESS) {
2712 already_deferred = TRUE;
2713 }
2714
2715
2716 /* This can be set and cleared exactly once. */
2717 if (pcontrol & PROC_DIRTY_DEFER) {
2718
2719 if ( !(old_dirty & P_DIRTY_DEFER)) {
2720 p->p_memstat_dirty |= P_DIRTY_DEFER;
2721 }
2722
2723 defer_now = TRUE;
2724 }
2725
2726 MEMORYSTATUS_DEBUG(1, "memorystatus_on_track_dirty(): set idle-exit %s / defer %s / dirty %s for pid %d\n",
2727 ((p->p_memstat_dirty & P_DIRTY_IDLE_EXIT_ENABLED) == P_DIRTY_IDLE_EXIT_ENABLED) ? "Y" : "N",
2728 defer_now ? "Y" : "N",
2729 p->p_memstat_dirty & P_DIRTY ? "Y" : "N",
2730 p->p_pid);
2731
2732 /* Kick off or invalidate the idle exit deferment if there's a state transition. */
2733 if (!(p->p_memstat_dirty & P_DIRTY_IS_DIRTY)) {
2734 if ((p->p_memstat_dirty & P_DIRTY_IDLE_EXIT_ENABLED) == P_DIRTY_IDLE_EXIT_ENABLED) {
2735
2736 if (defer_now && !already_deferred) {
2737
2738 /*
2739 * Request to defer a clean process that's idle-exit enabled
2740 * and not already in the jetsam deferred band. Most likely a
2741 * new launch.
2742 */
2743 memorystatus_schedule_idle_demotion_locked(p, TRUE);
2744 reschedule = TRUE;
2745
2746 } else if (!defer_now) {
2747
2748 /*
2749 * The process isn't asking for the 'aging' facility.
2750 * Could be that it is:
2751 */
2752
2753 if (already_deferred) {
2754 /*
2755 * already in the aging bands. Traditionally,
2756 * some processes have tried to use this to
2757 * opt out of the 'aging' facility.
2758 */
2759
2760 memorystatus_invalidate_idle_demotion_locked(p, TRUE);
2761 } else {
2762 /*
2763 * agnostic to the 'aging' facility. In that case,
2764 * we'll go ahead and opt it in because this is likely
2765 * a new launch (clean process, dirty tracking enabled)
2766 */
2767
2768 memorystatus_schedule_idle_demotion_locked(p, TRUE);
2769 }
2770
2771 reschedule = TRUE;
2772 }
2773 }
2774 } else {
2775
2776 /*
2777 * We are trying to operate on a dirty process. Dirty processes have to
2778 * be removed from the deferred band. The question is do we reset the
2779 * deferred state or not?
2780 *
2781 * This could be a legal request like:
2782 * - this process had opted into the 'aging' band
2783 * - but it's now dirty and requests to opt out.
2784 * In this case, we remove the process from the band and reset its
2785 * state too. It'll opt back in properly when needed.
2786 *
2787 * OR, this request could be a user-space bug. E.g.:
2788 * - this process had opted into the 'aging' band when clean
2789 * - and, then issues another request to again put it into the band except
2790 * this time the process is dirty.
2791 * The process going dirty, as a transition in memorystatus_dirty_set(), will pull the process out of
2792 * the deferred band with its state intact. So our request below is no-op.
2793 * But we do it here anyways for coverage.
2794 *
2795 * memorystatus_update_idle_priority_locked()
2796 * single-mindedly treats a dirty process as "cannot be in the aging band".
2797 */
2798
2799 if (!defer_now && already_deferred) {
2800 memorystatus_invalidate_idle_demotion_locked(p, TRUE);
2801 reschedule = TRUE;
2802 } else {
2803
2804 boolean_t reset_state = (jetsam_aging_policy != kJetsamAgingPolicyLegacy) ? TRUE : FALSE;
2805
2806 memorystatus_invalidate_idle_demotion_locked(p, reset_state);
2807 reschedule = TRUE;
2808 }
2809 }
2810
2811 memorystatus_update_idle_priority_locked(p);
2812
2813 if (reschedule) {
2814 memorystatus_reschedule_idle_demotion_locked();
2815 }
2816
2817 ret = 0;
2818
2819 exit:
2820 proc_list_unlock();
2821
2822 return ret;
2823 }
2824
2825 int
2826 memorystatus_dirty_set(proc_t p, boolean_t self, uint32_t pcontrol) {
2827 int ret;
2828 boolean_t kill = false;
2829 boolean_t reschedule = FALSE;
2830 boolean_t was_dirty = FALSE;
2831 boolean_t now_dirty = FALSE;
2832
2833 MEMORYSTATUS_DEBUG(1, "memorystatus_dirty_set(): %d %d 0x%x 0x%x\n", self, p->p_pid, pcontrol, p->p_memstat_dirty);
2834 KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_DIRTY_SET), p->p_pid, self, pcontrol, 0, 0);
2835
2836 proc_list_lock();
2837
2838 if ((p->p_listflag & P_LIST_EXITED) != 0) {
2839 /*
2840 * Process is on its way out.
2841 */
2842 ret = EBUSY;
2843 goto exit;
2844 }
2845
2846 if (p->p_memstat_state & P_MEMSTAT_INTERNAL) {
2847 ret = EPERM;
2848 goto exit;
2849 }
2850
2851 if (p->p_memstat_dirty & P_DIRTY_IS_DIRTY)
2852 was_dirty = TRUE;
2853
2854 if (!(p->p_memstat_dirty & P_DIRTY_TRACK)) {
2855 /* Dirty tracking not enabled */
2856 ret = EINVAL;
2857 } else if (pcontrol && (p->p_memstat_dirty & P_DIRTY_TERMINATED)) {
2858 /*
2859 * Process is set to be terminated and we're attempting to mark it dirty.
2860 * Set for termination and marking as clean is OK - see <rdar://problem/10594349>.
2861 */
2862 ret = EBUSY;
2863 } else {
2864 int flag = (self == TRUE) ? P_DIRTY : P_DIRTY_SHUTDOWN;
2865 if (pcontrol && !(p->p_memstat_dirty & flag)) {
2866 /* Mark the process as having been dirtied at some point */
2867 p->p_memstat_dirty |= (flag | P_DIRTY_MARKED);
2868 memorystatus_dirty_count++;
2869 ret = 0;
2870 } else if ((pcontrol == 0) && (p->p_memstat_dirty & flag)) {
2871 if ((flag == P_DIRTY_SHUTDOWN) && (!(p->p_memstat_dirty & P_DIRTY))) {
2872 /* Clearing the dirty shutdown flag, and the process is otherwise clean - kill */
2873 p->p_memstat_dirty |= P_DIRTY_TERMINATED;
2874 kill = true;
2875 } else if ((flag == P_DIRTY) && (p->p_memstat_dirty & P_DIRTY_TERMINATED)) {
2876 /* Kill previously terminated processes if set clean */
2877 kill = true;
2878 }
2879 p->p_memstat_dirty &= ~flag;
2880 memorystatus_dirty_count--;
2881 ret = 0;
2882 } else {
2883 /* Already set */
2884 ret = EALREADY;
2885 }
2886 }
2887
2888 if (ret != 0) {
2889 goto exit;
2890 }
2891
2892 if (p->p_memstat_dirty & P_DIRTY_IS_DIRTY)
2893 now_dirty = TRUE;
2894
2895 if ((was_dirty == TRUE && now_dirty == FALSE) ||
2896 (was_dirty == FALSE && now_dirty == TRUE)) {
2897
2898 /* Manage idle exit deferral, if applied */
2899 if ((p->p_memstat_dirty & P_DIRTY_IDLE_EXIT_ENABLED) == P_DIRTY_IDLE_EXIT_ENABLED) {
2900
2901 /*
2902 * Legacy mode: P_DIRTY_AGING_IN_PROGRESS means the process is in the aging band OR it might be heading back
2903 * there once it's clean again. For the legacy case, this only applies if it has some protection window left.
2904 *
2905 * Non-Legacy mode: P_DIRTY_AGING_IN_PROGRESS means the process is in the aging band. It will always stop over
2906 * in that band on it's way to IDLE.
2907 */
2908
2909 if (p->p_memstat_dirty & P_DIRTY_IS_DIRTY) {
2910 /*
2911 * New dirty process i.e. "was_dirty == FALSE && now_dirty == TRUE"
2912 *
2913 * The process will move from its aging band to its higher requested
2914 * jetsam band.
2915 */
2916 boolean_t reset_state = (jetsam_aging_policy != kJetsamAgingPolicyLegacy) ? TRUE : FALSE;
2917
2918 memorystatus_invalidate_idle_demotion_locked(p, reset_state);
2919 reschedule = TRUE;
2920 } else {
2921
2922 /*
2923 * Process is back from "dirty" to "clean".
2924 */
2925
2926 if (jetsam_aging_policy == kJetsamAgingPolicyLegacy) {
2927 if (mach_absolute_time() >= p->p_memstat_idledeadline) {
2928 /*
2929 * The process' deadline has expired. It currently
2930 * does not reside in any of the aging buckets.
2931 *
2932 * It's on its way to the JETSAM_PRIORITY_IDLE
2933 * bucket via memorystatus_update_idle_priority_locked()
2934 * below.
2935
2936 * So all we need to do is reset all the state on the
2937 * process that's related to the aging bucket i.e.
2938 * the AGING_IN_PROGRESS flag and the timer deadline.
2939 */
2940
2941 memorystatus_invalidate_idle_demotion_locked(p, TRUE);
2942 reschedule = TRUE;
2943 } else {
2944 /*
2945 * It still has some protection window left and so
2946 * we just re-arm the timer without modifying any
2947 * state on the process iff it still wants into that band.
2948 */
2949
2950 if (p->p_memstat_dirty & P_DIRTY_AGING_IN_PROGRESS) {
2951 memorystatus_schedule_idle_demotion_locked(p, FALSE);
2952 reschedule = TRUE;
2953 }
2954 }
2955 } else {
2956
2957 memorystatus_schedule_idle_demotion_locked(p, TRUE);
2958 reschedule = TRUE;
2959 }
2960 }
2961 }
2962
2963 memorystatus_update_idle_priority_locked(p);
2964
2965 if (memorystatus_highwater_enabled) {
2966 boolean_t ledger_update_needed = TRUE;
2967 boolean_t use_active;
2968 boolean_t is_fatal;
2969 /*
2970 * We are in this path because this process transitioned between
2971 * dirty <--> clean state. Update the cached memory limits.
2972 */
2973
2974 if (proc_jetsam_state_is_active_locked(p) == TRUE) {
2975 /*
2976 * process is pinned in elevated band
2977 * or
2978 * process is dirty
2979 */
2980 CACHE_ACTIVE_LIMITS_LOCKED(p, is_fatal);
2981 use_active = TRUE;
2982 ledger_update_needed = TRUE;
2983 } else {
2984 /*
2985 * process is clean...but if it has opted into pressured-exit
2986 * we don't apply the INACTIVE limit till the process has aged
2987 * out and is entering the IDLE band.
2988 * See memorystatus_update_priority_locked() for that.
2989 */
2990
2991 if (p->p_memstat_dirty & P_DIRTY_ALLOW_IDLE_EXIT) {
2992 ledger_update_needed = FALSE;
2993 } else {
2994 CACHE_INACTIVE_LIMITS_LOCKED(p, is_fatal);
2995 use_active = FALSE;
2996 ledger_update_needed = TRUE;
2997 }
2998 }
2999
3000 /*
3001 * Enforce the new limits by writing to the ledger.
3002 *
3003 * This is a hot path and holding the proc_list_lock while writing to the ledgers,
3004 * (where the task lock is taken) is bad. So, we temporarily drop the proc_list_lock.
3005 * We aren't traversing the jetsam bucket list here, so we should be safe.
3006 * See rdar://21394491.
3007 */
3008
3009 if (ledger_update_needed && proc_ref_locked(p) == p) {
3010 int ledger_limit;
3011 if (p->p_memstat_memlimit > 0) {
3012 ledger_limit = p->p_memstat_memlimit;
3013 } else {
3014 ledger_limit = -1;
3015 }
3016 proc_list_unlock();
3017 task_set_phys_footprint_limit_internal(p->task, ledger_limit, NULL, use_active, is_fatal);
3018 proc_list_lock();
3019 proc_rele_locked(p);
3020
3021 MEMORYSTATUS_DEBUG(3, "memorystatus_dirty_set: new limit on pid %d (%dMB %s) priority(%d) dirty?=0x%x %s\n",
3022 p->p_pid, (p->p_memstat_memlimit > 0 ? p->p_memstat_memlimit : -1),
3023 (p->p_memstat_state & P_MEMSTAT_FATAL_MEMLIMIT ? "F " : "NF"), p->p_memstat_effectivepriority, p->p_memstat_dirty,
3024 (p->p_memstat_dirty ? ((p->p_memstat_dirty & P_DIRTY) ? "isdirty" : "isclean") : ""));
3025 }
3026
3027 }
3028
3029 /* If the deferral state changed, reschedule the demotion timer */
3030 if (reschedule) {
3031 memorystatus_reschedule_idle_demotion_locked();
3032 }
3033 }
3034
3035 if (kill) {
3036 if (proc_ref_locked(p) == p) {
3037 proc_list_unlock();
3038 psignal(p, SIGKILL);
3039 proc_list_lock();
3040 proc_rele_locked(p);
3041 }
3042 }
3043
3044 exit:
3045 proc_list_unlock();
3046
3047 return ret;
3048 }
3049
3050 int
3051 memorystatus_dirty_clear(proc_t p, uint32_t pcontrol) {
3052
3053 int ret = 0;
3054
3055 MEMORYSTATUS_DEBUG(1, "memorystatus_dirty_clear(): %d 0x%x 0x%x\n", p->p_pid, pcontrol, p->p_memstat_dirty);
3056
3057 KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_DIRTY_CLEAR), p->p_pid, pcontrol, 0, 0, 0);
3058
3059 proc_list_lock();
3060
3061 if ((p->p_listflag & P_LIST_EXITED) != 0) {
3062 /*
3063 * Process is on its way out.
3064 */
3065 ret = EBUSY;
3066 goto exit;
3067 }
3068
3069 if (p->p_memstat_state & P_MEMSTAT_INTERNAL) {
3070 ret = EPERM;
3071 goto exit;
3072 }
3073
3074 if (!(p->p_memstat_dirty & P_DIRTY_TRACK)) {
3075 /* Dirty tracking not enabled */
3076 ret = EINVAL;
3077 goto exit;
3078 }
3079
3080 if (!pcontrol || (pcontrol & (PROC_DIRTY_LAUNCH_IN_PROGRESS | PROC_DIRTY_DEFER)) == 0) {
3081 ret = EINVAL;
3082 goto exit;
3083 }
3084
3085 if (pcontrol & PROC_DIRTY_LAUNCH_IN_PROGRESS) {
3086 p->p_memstat_dirty &= ~P_DIRTY_LAUNCH_IN_PROGRESS;
3087 }
3088
3089 /* This can be set and cleared exactly once. */
3090 if (pcontrol & PROC_DIRTY_DEFER) {
3091
3092 if (p->p_memstat_dirty & P_DIRTY_DEFER) {
3093
3094 p->p_memstat_dirty &= ~P_DIRTY_DEFER;
3095
3096 memorystatus_invalidate_idle_demotion_locked(p, TRUE);
3097 memorystatus_update_idle_priority_locked(p);
3098 memorystatus_reschedule_idle_demotion_locked();
3099 }
3100 }
3101
3102 ret = 0;
3103 exit:
3104 proc_list_unlock();
3105
3106 return ret;
3107 }
3108
3109 int
3110 memorystatus_dirty_get(proc_t p) {
3111 int ret = 0;
3112
3113 proc_list_lock();
3114
3115 if (p->p_memstat_dirty & P_DIRTY_TRACK) {
3116 ret |= PROC_DIRTY_TRACKED;
3117 if (p->p_memstat_dirty & P_DIRTY_ALLOW_IDLE_EXIT) {
3118 ret |= PROC_DIRTY_ALLOWS_IDLE_EXIT;
3119 }
3120 if (p->p_memstat_dirty & P_DIRTY) {
3121 ret |= PROC_DIRTY_IS_DIRTY;
3122 }
3123 if (p->p_memstat_dirty & P_DIRTY_LAUNCH_IN_PROGRESS) {
3124 ret |= PROC_DIRTY_LAUNCH_IS_IN_PROGRESS;
3125 }
3126 }
3127
3128 proc_list_unlock();
3129
3130 return ret;
3131 }
3132
3133 int
3134 memorystatus_on_terminate(proc_t p) {
3135 int sig;
3136
3137 proc_list_lock();
3138
3139 p->p_memstat_dirty |= P_DIRTY_TERMINATED;
3140
3141 if ((p->p_memstat_dirty & (P_DIRTY_TRACK|P_DIRTY_IS_DIRTY)) == P_DIRTY_TRACK) {
3142 /* Clean; mark as terminated and issue SIGKILL */
3143 sig = SIGKILL;
3144 } else {
3145 /* Dirty, terminated, or state tracking is unsupported; issue SIGTERM to allow cleanup */
3146 sig = SIGTERM;
3147 }
3148
3149 proc_list_unlock();
3150
3151 return sig;
3152 }
3153
3154 void
3155 memorystatus_on_suspend(proc_t p)
3156 {
3157 #if CONFIG_FREEZE
3158 uint32_t pages;
3159 memorystatus_get_task_page_counts(p->task, &pages, NULL, NULL, NULL);
3160 #endif
3161 proc_list_lock();
3162 #if CONFIG_FREEZE
3163 p->p_memstat_suspendedfootprint = pages;
3164 memorystatus_suspended_footprint_total += pages;
3165 memorystatus_suspended_count++;
3166 #endif
3167 p->p_memstat_state |= P_MEMSTAT_SUSPENDED;
3168 proc_list_unlock();
3169 }
3170
3171 void
3172 memorystatus_on_resume(proc_t p)
3173 {
3174 #if CONFIG_FREEZE
3175 boolean_t frozen;
3176 pid_t pid;
3177 #endif
3178
3179 proc_list_lock();
3180
3181 #if CONFIG_FREEZE
3182 frozen = (p->p_memstat_state & P_MEMSTAT_FROZEN);
3183 if (frozen) {
3184 memorystatus_frozen_count--;
3185 p->p_memstat_state |= P_MEMSTAT_PRIOR_THAW;
3186 }
3187
3188 memorystatus_suspended_footprint_total -= p->p_memstat_suspendedfootprint;
3189 memorystatus_suspended_count--;
3190
3191 pid = p->p_pid;
3192 #endif
3193
3194 p->p_memstat_state &= ~(P_MEMSTAT_SUSPENDED | P_MEMSTAT_FROZEN);
3195
3196 proc_list_unlock();
3197
3198 #if CONFIG_FREEZE
3199 if (frozen) {
3200 memorystatus_freeze_entry_t data = { pid, FALSE, 0 };
3201 memorystatus_send_note(kMemorystatusFreezeNote, &data, sizeof(data));
3202 }
3203 #endif
3204 }
3205
3206 void
3207 memorystatus_on_inactivity(proc_t p)
3208 {
3209 #pragma unused(p)
3210 #if CONFIG_FREEZE
3211 /* Wake the freeze thread */
3212 thread_wakeup((event_t)&memorystatus_freeze_wakeup);
3213 #endif
3214 }
3215
3216 /*
3217 * The proc_list_lock is held by the caller.
3218 */
3219 static uint32_t
3220 memorystatus_build_state(proc_t p) {
3221 uint32_t snapshot_state = 0;
3222
3223 /* General */
3224 if (p->p_memstat_state & P_MEMSTAT_SUSPENDED) {
3225 snapshot_state |= kMemorystatusSuspended;
3226 }
3227 if (p->p_memstat_state & P_MEMSTAT_FROZEN) {
3228 snapshot_state |= kMemorystatusFrozen;
3229 }
3230 if (p->p_memstat_state & P_MEMSTAT_PRIOR_THAW) {
3231 snapshot_state |= kMemorystatusWasThawed;
3232 }
3233
3234 /* Tracking */
3235 if (p->p_memstat_dirty & P_DIRTY_TRACK) {
3236 snapshot_state |= kMemorystatusTracked;
3237 }
3238 if ((p->p_memstat_dirty & P_DIRTY_IDLE_EXIT_ENABLED) == P_DIRTY_IDLE_EXIT_ENABLED) {
3239 snapshot_state |= kMemorystatusSupportsIdleExit;
3240 }
3241 if (p->p_memstat_dirty & P_DIRTY_IS_DIRTY) {
3242 snapshot_state |= kMemorystatusDirty;
3243 }
3244
3245 return snapshot_state;
3246 }
3247
3248 static boolean_t
3249 kill_idle_exit_proc(void)
3250 {
3251 proc_t p, victim_p = PROC_NULL;
3252 uint64_t current_time;
3253 boolean_t killed = FALSE;
3254 unsigned int i = 0;
3255 os_reason_t jetsam_reason = OS_REASON_NULL;
3256
3257 /* Pick next idle exit victim. */
3258 current_time = mach_absolute_time();
3259
3260 jetsam_reason = os_reason_create(OS_REASON_JETSAM, JETSAM_REASON_MEMORY_IDLE_EXIT);
3261 if (jetsam_reason == OS_REASON_NULL) {
3262 printf("kill_idle_exit_proc: failed to allocate jetsam reason\n");
3263 }
3264
3265 proc_list_lock();
3266
3267 p = memorystatus_get_first_proc_locked(&i, FALSE);
3268 while (p) {
3269 /* No need to look beyond the idle band */
3270 if (p->p_memstat_effectivepriority != JETSAM_PRIORITY_IDLE) {
3271 break;
3272 }
3273
3274 if ((p->p_memstat_dirty & (P_DIRTY_ALLOW_IDLE_EXIT|P_DIRTY_IS_DIRTY|P_DIRTY_TERMINATED)) == (P_DIRTY_ALLOW_IDLE_EXIT)) {
3275 if (current_time >= p->p_memstat_idledeadline) {
3276 p->p_memstat_dirty |= P_DIRTY_TERMINATED;
3277 victim_p = proc_ref_locked(p);
3278 break;
3279 }
3280 }
3281
3282 p = memorystatus_get_next_proc_locked(&i, p, FALSE);
3283 }
3284
3285 proc_list_unlock();
3286
3287 if (victim_p) {
3288 printf("memorystatus: killing_idle_process pid %d [%s]\n", victim_p->p_pid, (*victim_p->p_name ? victim_p->p_name : "unknown"));
3289 killed = memorystatus_do_kill(victim_p, kMemorystatusKilledIdleExit, jetsam_reason);
3290 proc_rele(victim_p);
3291 } else {
3292 os_reason_free(jetsam_reason);
3293 }
3294
3295 return killed;
3296 }
3297
3298 static void
3299 memorystatus_thread_wake(void) {
3300 thread_wakeup((event_t)&memorystatus_wakeup);
3301 }
3302
3303 extern void vm_pressure_response(void);
3304
3305 static int
3306 memorystatus_thread_block(uint32_t interval_ms, thread_continue_t continuation)
3307 {
3308 if (interval_ms) {
3309 assert_wait_timeout(&memorystatus_wakeup, THREAD_UNINT, interval_ms, 1000 * NSEC_PER_USEC);
3310 } else {
3311 assert_wait(&memorystatus_wakeup, THREAD_UNINT);
3312 }
3313
3314 return thread_block(continuation);
3315 }
3316
3317 static boolean_t
3318 memorystatus_avail_pages_below_pressure(void)
3319 {
3320 #if CONFIG_EMBEDDED
3321 /*
3322 * Instead of CONFIG_EMBEDDED for these *avail_pages* routines, we should
3323 * key off of the system having dynamic swap support. With full swap support,
3324 * the system shouldn't really need to worry about various page thresholds.
3325 */
3326 return (memorystatus_available_pages <= memorystatus_available_pages_pressure);
3327 #else /* CONFIG_EMBEDDED */
3328 return FALSE;
3329 #endif /* CONFIG_EMBEDDED */
3330 }
3331
3332 static boolean_t
3333 memorystatus_avail_pages_below_critical(void)
3334 {
3335 #if CONFIG_EMBEDDED
3336 return (memorystatus_available_pages <= memorystatus_available_pages_critical);
3337 #else /* CONFIG_EMBEDDED */
3338 return FALSE;
3339 #endif /* CONFIG_EMBEDDED */
3340 }
3341
3342 static boolean_t
3343 memorystatus_post_snapshot(int32_t priority, uint32_t cause)
3344 {
3345 #if CONFIG_EMBEDDED
3346 #pragma unused(cause)
3347 /*
3348 * Don't generate logs for steady-state idle-exit kills,
3349 * unless it is overridden for debug or by the device
3350 * tree.
3351 */
3352
3353 return ((priority != JETSAM_PRIORITY_IDLE) || memorystatus_idle_snapshot);
3354
3355 #else /* CONFIG_EMBEDDED */
3356 /*
3357 * Don't generate logs for steady-state idle-exit kills,
3358 * unless
3359 * - it is overridden for debug or by the device
3360 * tree.
3361 * OR
3362 * - the kill causes are important i.e. not kMemorystatusKilledIdleExit
3363 */
3364
3365 boolean_t snapshot_eligible_kill_cause = (is_reason_thrashing(cause) || is_reason_zone_map_exhaustion(cause));
3366 return ((priority != JETSAM_PRIORITY_IDLE) || memorystatus_idle_snapshot || snapshot_eligible_kill_cause);
3367 #endif /* CONFIG_EMBEDDED */
3368 }
3369
3370 static boolean_t
3371 memorystatus_action_needed(void)
3372 {
3373 #if CONFIG_EMBEDDED
3374 return (is_reason_thrashing(kill_under_pressure_cause) ||
3375 is_reason_zone_map_exhaustion(kill_under_pressure_cause) ||
3376 memorystatus_available_pages <= memorystatus_available_pages_pressure);
3377 #else /* CONFIG_EMBEDDED */
3378 return (is_reason_thrashing(kill_under_pressure_cause) ||
3379 is_reason_zone_map_exhaustion(kill_under_pressure_cause));
3380 #endif /* CONFIG_EMBEDDED */
3381 }
3382
3383 static boolean_t
3384 memorystatus_act_on_hiwat_processes(uint32_t *errors, uint32_t *hwm_kill, boolean_t *post_snapshot, __unused boolean_t *is_critical)
3385 {
3386 boolean_t purged = FALSE;
3387 boolean_t killed = memorystatus_kill_hiwat_proc(errors, &purged);
3388
3389 if (killed) {
3390 *hwm_kill = *hwm_kill + 1;
3391 *post_snapshot = TRUE;
3392 return TRUE;
3393 } else {
3394 if (purged == FALSE) {
3395 /* couldn't purge and couldn't kill */
3396 memorystatus_hwm_candidates = FALSE;
3397 }
3398 }
3399
3400 #if CONFIG_JETSAM
3401 /* No highwater processes to kill. Continue or stop for now? */
3402 if (!is_reason_thrashing(kill_under_pressure_cause) &&
3403 !is_reason_zone_map_exhaustion(kill_under_pressure_cause) &&
3404 (memorystatus_available_pages > memorystatus_available_pages_critical)) {
3405 /*
3406 * We are _not_ out of pressure but we are above the critical threshold and there's:
3407 * - no compressor thrashing
3408 * - enough zone memory
3409 * - no more HWM processes left.
3410 * For now, don't kill any other processes.
3411 */
3412
3413 if (*hwm_kill == 0) {
3414 memorystatus_thread_wasted_wakeup++;
3415 }
3416
3417 *is_critical = FALSE;
3418
3419 return TRUE;
3420 }
3421 #endif /* CONFIG_JETSAM */
3422
3423 return FALSE;
3424 }
3425
3426 static boolean_t
3427 memorystatus_act_aggressive(uint32_t cause, os_reason_t jetsam_reason, int *jld_idle_kills, boolean_t *corpse_list_purged, boolean_t *post_snapshot)
3428 {
3429 if (memorystatus_jld_enabled == TRUE) {
3430
3431 boolean_t killed;
3432 uint32_t errors = 0;
3433
3434 /* Jetsam Loop Detection - locals */
3435 memstat_bucket_t *bucket;
3436 int jld_bucket_count = 0;
3437 struct timeval jld_now_tstamp = {0,0};
3438 uint64_t jld_now_msecs = 0;
3439 int elevated_bucket_count = 0;
3440
3441 /* Jetsam Loop Detection - statics */
3442 static uint64_t jld_timestamp_msecs = 0;
3443 static int jld_idle_kill_candidates = 0; /* Number of available processes in band 0,1 at start */
3444 static int jld_eval_aggressive_count = 0; /* Bumps the max priority in aggressive loop */
3445 static int32_t jld_priority_band_max = JETSAM_PRIORITY_UI_SUPPORT;
3446 /*
3447 * Jetsam Loop Detection: attempt to detect
3448 * rapid daemon relaunches in the lower bands.
3449 */
3450
3451 microuptime(&jld_now_tstamp);
3452
3453 /*
3454 * Ignore usecs in this calculation.
3455 * msecs granularity is close enough.
3456 */
3457 jld_now_msecs = (jld_now_tstamp.tv_sec * 1000);
3458
3459 proc_list_lock();
3460 switch (jetsam_aging_policy) {
3461 case kJetsamAgingPolicyLegacy:
3462 bucket = &memstat_bucket[JETSAM_PRIORITY_IDLE];
3463 jld_bucket_count = bucket->count;
3464 bucket = &memstat_bucket[JETSAM_PRIORITY_AGING_BAND1];
3465 jld_bucket_count += bucket->count;
3466 break;
3467 case kJetsamAgingPolicySysProcsReclaimedFirst:
3468 case kJetsamAgingPolicyAppsReclaimedFirst:
3469 bucket = &memstat_bucket[JETSAM_PRIORITY_IDLE];
3470 jld_bucket_count = bucket->count;
3471 bucket = &memstat_bucket[system_procs_aging_band];
3472 jld_bucket_count += bucket->count;
3473 bucket = &memstat_bucket[applications_aging_band];
3474 jld_bucket_count += bucket->count;
3475 break;
3476 case kJetsamAgingPolicyNone:
3477 default:
3478 bucket = &memstat_bucket[JETSAM_PRIORITY_IDLE];
3479 jld_bucket_count = bucket->count;
3480 break;
3481 }
3482
3483 bucket = &memstat_bucket[JETSAM_PRIORITY_ELEVATED_INACTIVE];
3484 elevated_bucket_count = bucket->count;
3485
3486 proc_list_unlock();
3487
3488 /*
3489 * memorystatus_jld_eval_period_msecs is a tunable
3490 * memorystatus_jld_eval_aggressive_count is a tunable
3491 * memorystatus_jld_eval_aggressive_priority_band_max is a tunable
3492 */
3493 if ( (jld_bucket_count == 0) ||
3494 (jld_now_msecs > (jld_timestamp_msecs + memorystatus_jld_eval_period_msecs))) {
3495
3496 /*
3497 * Refresh evaluation parameters
3498 */
3499 jld_timestamp_msecs = jld_now_msecs;
3500 jld_idle_kill_candidates = jld_bucket_count;
3501 *jld_idle_kills = 0;
3502 jld_eval_aggressive_count = 0;
3503 jld_priority_band_max = JETSAM_PRIORITY_UI_SUPPORT;
3504 }
3505
3506 if (*jld_idle_kills > jld_idle_kill_candidates) {
3507 jld_eval_aggressive_count++;
3508
3509 #if DEVELOPMENT || DEBUG
3510 printf("memorystatus: aggressive%d: beginning of window: %lld ms, : timestamp now: %lld ms\n",
3511 jld_eval_aggressive_count,
3512 jld_timestamp_msecs,
3513 jld_now_msecs);
3514 printf("memorystatus: aggressive%d: idle candidates: %d, idle kills: %d\n",
3515 jld_eval_aggressive_count,
3516 jld_idle_kill_candidates,
3517 *jld_idle_kills);
3518 #endif /* DEVELOPMENT || DEBUG */
3519
3520 if ((jld_eval_aggressive_count == memorystatus_jld_eval_aggressive_count) &&
3521 (total_corpses_count() > 0) && (*corpse_list_purged == FALSE)) {
3522 /*
3523 * If we reach this aggressive cycle, corpses might be causing memory pressure.
3524 * So, in an effort to avoid jetsams in the FG band, we will attempt to purge
3525 * corpse memory prior to this final march through JETSAM_PRIORITY_UI_SUPPORT.
3526 */
3527 task_purge_all_corpses();
3528 *corpse_list_purged = TRUE;
3529 }
3530 else if (jld_eval_aggressive_count > memorystatus_jld_eval_aggressive_count) {
3531 /*
3532 * Bump up the jetsam priority limit (eg: the bucket index)
3533 * Enforce bucket index sanity.
3534 */
3535 if ((memorystatus_jld_eval_aggressive_priority_band_max < 0) ||
3536 (memorystatus_jld_eval_aggressive_priority_band_max >= MEMSTAT_BUCKET_COUNT)) {
3537 /*
3538 * Do nothing. Stick with the default level.
3539 */
3540 } else {
3541 jld_priority_band_max = memorystatus_jld_eval_aggressive_priority_band_max;
3542 }
3543 }
3544
3545 /* Visit elevated processes first */
3546 while (elevated_bucket_count) {
3547
3548 elevated_bucket_count--;
3549
3550 /*
3551 * memorystatus_kill_elevated_process() drops a reference,
3552 * so take another one so we can continue to use this exit reason
3553 * even after it returns.
3554 */
3555
3556 os_reason_ref(jetsam_reason);
3557 killed = memorystatus_kill_elevated_process(
3558 cause,
3559 jetsam_reason,
3560 jld_eval_aggressive_count,
3561 &errors);
3562
3563 if (killed) {
3564 *post_snapshot = TRUE;
3565 if (memorystatus_avail_pages_below_pressure()) {
3566 /*
3567 * Still under pressure.
3568 * Find another pinned processes.
3569 */
3570 continue;
3571 } else {
3572 return TRUE;
3573 }
3574 } else {
3575 /*
3576 * No pinned processes left to kill.
3577 * Abandon elevated band.
3578 */
3579 break;
3580 }
3581 }
3582
3583 /*
3584 * memorystatus_kill_top_process_aggressive() allocates its own
3585 * jetsam_reason so the kMemorystatusKilledVMThrashing cause
3586 * is consistent throughout the aggressive march.
3587 */
3588 killed = memorystatus_kill_top_process_aggressive(
3589 kMemorystatusKilledVMThrashing,
3590 jld_eval_aggressive_count,
3591 jld_priority_band_max,
3592 &errors);
3593
3594 if (killed) {
3595 /* Always generate logs after aggressive kill */
3596 *post_snapshot = TRUE;
3597 *jld_idle_kills = 0;
3598 return TRUE;
3599 }
3600 }
3601
3602 return FALSE;
3603 }
3604
3605 return FALSE;
3606 }
3607
3608
3609 static void
3610 memorystatus_thread(void *param __unused, wait_result_t wr __unused)
3611 {
3612 static boolean_t is_vm_privileged = FALSE;
3613
3614 boolean_t post_snapshot = FALSE;
3615 uint32_t errors = 0;
3616 uint32_t hwm_kill = 0;
3617 boolean_t sort_flag = TRUE;
3618 boolean_t corpse_list_purged = FALSE;
3619 int jld_idle_kills = 0;
3620
3621 if (is_vm_privileged == FALSE) {
3622 /*
3623 * It's the first time the thread has run, so just mark the thread as privileged and block.
3624 * This avoids a spurious pass with unset variables, as set out in <rdar://problem/9609402>.
3625 */
3626 thread_wire(host_priv_self(), current_thread(), TRUE);
3627 is_vm_privileged = TRUE;
3628
3629 if (vm_restricted_to_single_processor == TRUE)
3630 thread_vm_bind_group_add();
3631 thread_set_thread_name(current_thread(), "VM_memorystatus");
3632 memorystatus_thread_block(0, memorystatus_thread);
3633 }
3634
3635 KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_SCAN) | DBG_FUNC_START,
3636 memorystatus_available_pages, memorystatus_jld_enabled, memorystatus_jld_eval_period_msecs, memorystatus_jld_eval_aggressive_count,0);
3637
3638 /*
3639 * Jetsam aware version.
3640 *
3641 * The VM pressure notification thread is working it's way through clients in parallel.
3642 *
3643 * So, while the pressure notification thread is targeting processes in order of
3644 * increasing jetsam priority, we can hopefully reduce / stop it's work by killing
3645 * any processes that have exceeded their highwater mark.
3646 *
3647 * If we run out of HWM processes and our available pages drops below the critical threshold, then,
3648 * we target the least recently used process in order of increasing jetsam priority (exception: the FG band).
3649 */
3650 while (memorystatus_action_needed()) {
3651 boolean_t killed;
3652 int32_t priority;
3653 uint32_t cause;
3654 uint64_t jetsam_reason_code = JETSAM_REASON_INVALID;
3655 os_reason_t jetsam_reason = OS_REASON_NULL;
3656
3657 cause = kill_under_pressure_cause;
3658 switch (cause) {
3659 case kMemorystatusKilledFCThrashing:
3660 jetsam_reason_code = JETSAM_REASON_MEMORY_FCTHRASHING;
3661 break;
3662 case kMemorystatusKilledVMThrashing:
3663 jetsam_reason_code = JETSAM_REASON_MEMORY_VMTHRASHING;
3664 break;
3665 case kMemorystatusKilledZoneMapExhaustion:
3666 jetsam_reason_code = JETSAM_REASON_ZONE_MAP_EXHAUSTION;
3667 break;
3668 case kMemorystatusKilledVMPageShortage:
3669 /* falls through */
3670 default:
3671 jetsam_reason_code = JETSAM_REASON_MEMORY_VMPAGESHORTAGE;
3672 cause = kMemorystatusKilledVMPageShortage;
3673 break;
3674 }
3675
3676 /* Highwater */
3677 boolean_t is_critical = TRUE;
3678 if (memorystatus_act_on_hiwat_processes(&errors, &hwm_kill, &post_snapshot, &is_critical)) {
3679 if (is_critical == FALSE) {
3680 /*
3681 * For now, don't kill any other processes.
3682 */
3683 break;
3684 } else {
3685 goto done;
3686 }
3687 }
3688
3689 jetsam_reason = os_reason_create(OS_REASON_JETSAM, jetsam_reason_code);
3690 if (jetsam_reason == OS_REASON_NULL) {
3691 printf("memorystatus_thread: failed to allocate jetsam reason\n");
3692 }
3693
3694 if (memorystatus_act_aggressive(cause, jetsam_reason, &jld_idle_kills, &corpse_list_purged, &post_snapshot)) {
3695 goto done;
3696 }
3697
3698 /*
3699 * memorystatus_kill_top_process() drops a reference,
3700 * so take another one so we can continue to use this exit reason
3701 * even after it returns
3702 */
3703 os_reason_ref(jetsam_reason);
3704
3705 /* LRU */
3706 killed = memorystatus_kill_top_process(TRUE, sort_flag, cause, jetsam_reason, &priority, &errors);
3707 sort_flag = FALSE;
3708
3709 if (killed) {
3710 if (memorystatus_post_snapshot(priority, cause) == TRUE) {
3711
3712 post_snapshot = TRUE;
3713 }
3714
3715 /* Jetsam Loop Detection */
3716 if (memorystatus_jld_enabled == TRUE) {
3717 if ((priority == JETSAM_PRIORITY_IDLE) || (priority == system_procs_aging_band) || (priority == applications_aging_band)) {
3718 jld_idle_kills++;
3719 } else {
3720 /*
3721 * We've reached into bands beyond idle deferred.
3722 * We make no attempt to monitor them
3723 */
3724 }
3725 }
3726
3727 if ((priority >= JETSAM_PRIORITY_UI_SUPPORT) && (total_corpses_count() > 0) && (corpse_list_purged == FALSE)) {
3728 /*
3729 * If we have jetsammed a process in or above JETSAM_PRIORITY_UI_SUPPORT
3730 * then we attempt to relieve pressure by purging corpse memory.
3731 */
3732 task_purge_all_corpses();
3733 corpse_list_purged = TRUE;
3734 }
3735 goto done;
3736 }
3737
3738 if (memorystatus_avail_pages_below_critical()) {
3739 /*
3740 * Still under pressure and unable to kill a process - purge corpse memory
3741 */
3742 if (total_corpses_count() > 0) {
3743 task_purge_all_corpses();
3744 corpse_list_purged = TRUE;
3745 }
3746
3747 if (memorystatus_avail_pages_below_critical()) {
3748 /*
3749 * Still under pressure and unable to kill a process - panic
3750 */
3751 panic("memorystatus_jetsam_thread: no victim! available pages:%llu\n", (uint64_t)memorystatus_available_pages);
3752 }
3753 }
3754
3755 done:
3756
3757 /*
3758 * We do not want to over-kill when thrashing has been detected.
3759 * To avoid that, we reset the flag here and notify the
3760 * compressor.
3761 */
3762 if (is_reason_thrashing(kill_under_pressure_cause)) {
3763 kill_under_pressure_cause = 0;
3764 #if CONFIG_JETSAM
3765 vm_thrashing_jetsam_done();
3766 #endif /* CONFIG_JETSAM */
3767 } else if (is_reason_zone_map_exhaustion(kill_under_pressure_cause)) {
3768 kill_under_pressure_cause = 0;
3769 }
3770
3771 os_reason_free(jetsam_reason);
3772 }
3773
3774 kill_under_pressure_cause = 0;
3775
3776 if (errors) {
3777 memorystatus_clear_errors();
3778 }
3779
3780 if (post_snapshot) {
3781 proc_list_lock();
3782 size_t snapshot_size = sizeof(memorystatus_jetsam_snapshot_t) +
3783 sizeof(memorystatus_jetsam_snapshot_entry_t) * (memorystatus_jetsam_snapshot_count);
3784 uint64_t timestamp_now = mach_absolute_time();
3785 memorystatus_jetsam_snapshot->notification_time = timestamp_now;
3786 memorystatus_jetsam_snapshot->js_gencount++;
3787 if (memorystatus_jetsam_snapshot_count > 0 && (memorystatus_jetsam_snapshot_last_timestamp == 0 ||
3788 timestamp_now > memorystatus_jetsam_snapshot_last_timestamp + memorystatus_jetsam_snapshot_timeout)) {
3789 proc_list_unlock();
3790 int ret = memorystatus_send_note(kMemorystatusSnapshotNote, &snapshot_size, sizeof(snapshot_size));
3791 if (!ret) {
3792 proc_list_lock();
3793 memorystatus_jetsam_snapshot_last_timestamp = timestamp_now;
3794 proc_list_unlock();
3795 }
3796 } else {
3797 proc_list_unlock();
3798 }
3799 }
3800
3801 KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_SCAN) | DBG_FUNC_END,
3802 memorystatus_available_pages, 0, 0, 0, 0);
3803
3804 memorystatus_thread_block(0, memorystatus_thread);
3805 }
3806
3807 /*
3808 * Returns TRUE:
3809 * when an idle-exitable proc was killed
3810 * Returns FALSE:
3811 * when there are no more idle-exitable procs found
3812 * when the attempt to kill an idle-exitable proc failed
3813 */
3814 boolean_t memorystatus_idle_exit_from_VM(void) {
3815
3816 /*
3817 * This routine should no longer be needed since we are
3818 * now using jetsam bands on all platforms and so will deal
3819 * with IDLE processes within the memorystatus thread itself.
3820 *
3821 * But we still use it because we observed that macos systems
3822 * started heavy compression/swapping with a bunch of
3823 * idle-exitable processes alive and doing nothing. We decided
3824 * to rather kill those processes than start swapping earlier.
3825 */
3826
3827 return(kill_idle_exit_proc());
3828 }
3829
3830 /*
3831 * Callback invoked when allowable physical memory footprint exceeded
3832 * (dirty pages + IOKit mappings)
3833 *
3834 * This is invoked for both advisory, non-fatal per-task high watermarks,
3835 * as well as the fatal task memory limits.
3836 */
3837 void
3838 memorystatus_on_ledger_footprint_exceeded(boolean_t warning, boolean_t memlimit_is_active, boolean_t memlimit_is_fatal)
3839 {
3840 os_reason_t jetsam_reason = OS_REASON_NULL;
3841
3842 proc_t p = current_proc();
3843
3844 #if VM_PRESSURE_EVENTS
3845 if (warning == TRUE) {
3846 /*
3847 * This is a warning path which implies that the current process is close, but has
3848 * not yet exceeded its per-process memory limit.
3849 */
3850 if (memorystatus_warn_process(p->p_pid, memlimit_is_active, memlimit_is_fatal, FALSE /* not exceeded */) != TRUE) {
3851 /* Print warning, since it's possible that task has not registered for pressure notifications */
3852 os_log(OS_LOG_DEFAULT, "memorystatus_on_ledger_footprint_exceeded: failed to warn the current task (%d exiting, or no handler registered?).\n", p->p_pid);
3853 }
3854 return;
3855 }
3856 #endif /* VM_PRESSURE_EVENTS */
3857
3858 if (memlimit_is_fatal) {
3859 /*
3860 * If this process has no high watermark or has a fatal task limit, then we have been invoked because the task
3861 * has violated either the system-wide per-task memory limit OR its own task limit.
3862 */
3863 jetsam_reason = os_reason_create(OS_REASON_JETSAM, JETSAM_REASON_MEMORY_PERPROCESSLIMIT);
3864 if (jetsam_reason == NULL) {
3865 printf("task_exceeded footprint: failed to allocate jetsam reason\n");
3866 } else if (corpse_for_fatal_memkill != 0) {
3867 /* Set OS_REASON_FLAG_GENERATE_CRASH_REPORT to generate corpse */
3868 jetsam_reason->osr_flags |= OS_REASON_FLAG_GENERATE_CRASH_REPORT;
3869 }
3870
3871 if (memorystatus_kill_process_sync(p->p_pid, kMemorystatusKilledPerProcessLimit, jetsam_reason) != TRUE) {
3872 printf("task_exceeded_footprint: failed to kill the current task (exiting?).\n");
3873 }
3874 } else {
3875 /*
3876 * HWM offender exists. Done without locks or synchronization.
3877 * See comment near its declaration for more details.
3878 */
3879 memorystatus_hwm_candidates = TRUE;
3880
3881 #if VM_PRESSURE_EVENTS
3882 /*
3883 * The current process is not in the warning path.
3884 * This path implies the current process has exceeded a non-fatal (soft) memory limit.
3885 * Failure to send note is ignored here.
3886 */
3887 (void)memorystatus_warn_process(p->p_pid, memlimit_is_active, memlimit_is_fatal, TRUE /* exceeded */);
3888
3889 #endif /* VM_PRESSURE_EVENTS */
3890 }
3891 }
3892
3893 void
3894 memorystatus_log_exception(const int max_footprint_mb, boolean_t memlimit_is_active, boolean_t memlimit_is_fatal)
3895 {
3896 proc_t p = current_proc();
3897
3898 /*
3899 * The limit violation is logged here, but only once per process per limit.
3900 * Soft memory limit is a non-fatal high-water-mark
3901 * Hard memory limit is a fatal custom-task-limit or system-wide per-task memory limit.
3902 */
3903
3904 os_log_with_startup_serial(OS_LOG_DEFAULT, "EXC_RESOURCE -> %s[%d] exceeded mem limit: %s%s %d MB (%s)\n",
3905 (*p->p_name ? p->p_name : "unknown"), p->p_pid, (memlimit_is_active ? "Active" : "Inactive"),
3906 (memlimit_is_fatal ? "Hard" : "Soft"), max_footprint_mb,
3907 (memlimit_is_fatal ? "fatal" : "non-fatal"));
3908
3909 return;
3910 }
3911
3912
3913 /*
3914 * Description:
3915 * Evaluates process state to determine which limit
3916 * should be applied (active vs. inactive limit).
3917 *
3918 * Processes that have the 'elevated inactive jetsam band' attribute
3919 * are first evaluated based on their current priority band.
3920 * presently elevated ==> active
3921 *
3922 * Processes that opt into dirty tracking are evaluated
3923 * based on clean vs dirty state.
3924 * dirty ==> active
3925 * clean ==> inactive
3926 *
3927 * Process that do not opt into dirty tracking are
3928 * evalulated based on priority level.
3929 * Foreground or above ==> active
3930 * Below Foreground ==> inactive
3931 *
3932 * Return: TRUE if active
3933 * False if inactive
3934 */
3935
3936 static boolean_t
3937 proc_jetsam_state_is_active_locked(proc_t p) {
3938
3939 if ((p->p_memstat_state & P_MEMSTAT_USE_ELEVATED_INACTIVE_BAND) &&
3940 (p->p_memstat_effectivepriority == JETSAM_PRIORITY_ELEVATED_INACTIVE)) {
3941 /*
3942 * process has the 'elevated inactive jetsam band' attribute
3943 * and process is present in the elevated band
3944 * implies active state
3945 */
3946 return TRUE;
3947 } else if (p->p_memstat_dirty & P_DIRTY_TRACK) {
3948 /*
3949 * process has opted into dirty tracking
3950 * active state is based on dirty vs. clean
3951 */
3952 if (p->p_memstat_dirty & P_DIRTY_IS_DIRTY) {
3953 /*
3954 * process is dirty
3955 * implies active state
3956 */
3957 return TRUE;
3958 } else {
3959 /*
3960 * process is clean
3961 * implies inactive state
3962 */
3963 return FALSE;
3964 }
3965 } else if (p->p_memstat_effectivepriority >= JETSAM_PRIORITY_FOREGROUND) {
3966 /*
3967 * process is Foreground or higher
3968 * implies active state
3969 */
3970 return TRUE;
3971 } else {
3972 /*
3973 * process found below Foreground
3974 * implies inactive state
3975 */
3976 return FALSE;
3977 }
3978 }
3979
3980 static boolean_t
3981 memorystatus_kill_process_sync(pid_t victim_pid, uint32_t cause, os_reason_t jetsam_reason) {
3982 boolean_t res;
3983
3984 uint32_t errors = 0;
3985
3986 if (victim_pid == -1) {
3987 /* No pid, so kill first process */
3988 res = memorystatus_kill_top_process(TRUE, TRUE, cause, jetsam_reason, NULL, &errors);
3989 } else {
3990 res = memorystatus_kill_specific_process(victim_pid, cause, jetsam_reason);
3991 }
3992
3993 if (errors) {
3994 memorystatus_clear_errors();
3995 }
3996
3997 if (res == TRUE) {
3998 /* Fire off snapshot notification */
3999 proc_list_lock();
4000 size_t snapshot_size = sizeof(memorystatus_jetsam_snapshot_t) +
4001 sizeof(memorystatus_jetsam_snapshot_entry_t) * memorystatus_jetsam_snapshot_count;
4002 uint64_t timestamp_now = mach_absolute_time();
4003 memorystatus_jetsam_snapshot->notification_time = timestamp_now;
4004 if (memorystatus_jetsam_snapshot_count > 0 && (memorystatus_jetsam_snapshot_last_timestamp == 0 ||
4005 timestamp_now > memorystatus_jetsam_snapshot_last_timestamp + memorystatus_jetsam_snapshot_timeout)) {
4006 proc_list_unlock();
4007 int ret = memorystatus_send_note(kMemorystatusSnapshotNote, &snapshot_size, sizeof(snapshot_size));
4008 if (!ret) {
4009 proc_list_lock();
4010 memorystatus_jetsam_snapshot_last_timestamp = timestamp_now;
4011 proc_list_unlock();
4012 }
4013 } else {
4014 proc_list_unlock();
4015 }
4016 }
4017
4018 return res;
4019 }
4020
4021 /*
4022 * Jetsam a specific process.
4023 */
4024 static boolean_t
4025 memorystatus_kill_specific_process(pid_t victim_pid, uint32_t cause, os_reason_t jetsam_reason) {
4026 boolean_t killed;
4027 proc_t p;
4028 uint64_t killtime = 0;
4029 clock_sec_t tv_sec;
4030 clock_usec_t tv_usec;
4031 uint32_t tv_msec;
4032
4033 /* TODO - add a victim queue and push this into the main jetsam thread */
4034
4035 p = proc_find(victim_pid);
4036 if (!p) {
4037 os_reason_free(jetsam_reason);
4038 return FALSE;
4039 }
4040
4041 proc_list_lock();
4042
4043 if (memorystatus_jetsam_snapshot_count == 0) {
4044 memorystatus_init_jetsam_snapshot_locked(NULL,0);
4045 }
4046
4047 killtime = mach_absolute_time();
4048 absolutetime_to_microtime(killtime, &tv_sec, &tv_usec);
4049 tv_msec = tv_usec / 1000;
4050
4051 memorystatus_update_jetsam_snapshot_entry_locked(p, cause, killtime);
4052
4053 proc_list_unlock();
4054
4055 os_log_with_startup_serial(OS_LOG_DEFAULT, "%lu.%03d memorystatus: killing_specific_process pid %d [%s] (%s %d) - memorystatus_available_pages: %llu\n",
4056 (unsigned long)tv_sec, tv_msec, victim_pid, (*p->p_name ? p->p_name : "unknown"),
4057 memorystatus_kill_cause_name[cause], p->p_memstat_effectivepriority, (uint64_t)memorystatus_available_pages);
4058
4059 killed = memorystatus_do_kill(p, cause, jetsam_reason);
4060 proc_rele(p);
4061
4062 return killed;
4063 }
4064
4065
4066 /*
4067 * Toggle the P_MEMSTAT_TERMINATED state.
4068 * Takes the proc_list_lock.
4069 */
4070 void
4071 proc_memstat_terminated(proc_t p, boolean_t set)
4072 {
4073 #if DEVELOPMENT || DEBUG
4074 if (p) {
4075 proc_list_lock();
4076 if (set == TRUE) {
4077 p->p_memstat_state |= P_MEMSTAT_TERMINATED;
4078 } else {
4079 p->p_memstat_state &= ~P_MEMSTAT_TERMINATED;
4080 }
4081 proc_list_unlock();
4082 }
4083 #else
4084 #pragma unused(p, set)
4085 /*
4086 * do nothing
4087 */
4088 #endif /* DEVELOPMENT || DEBUG */
4089 return;
4090 }
4091
4092
4093 #if CONFIG_JETSAM
4094 /*
4095 * This is invoked when cpulimits have been exceeded while in fatal mode.
4096 * The jetsam_flags do not apply as those are for memory related kills.
4097 * We call this routine so that the offending process is killed with
4098 * a non-zero exit status.
4099 */
4100 void
4101 jetsam_on_ledger_cpulimit_exceeded(void)
4102 {
4103 int retval = 0;
4104 int jetsam_flags = 0; /* make it obvious */
4105 proc_t p = current_proc();
4106 os_reason_t jetsam_reason = OS_REASON_NULL;
4107
4108 printf("task_exceeded_cpulimit: killing pid %d [%s]\n",
4109 p->p_pid, (*p->p_name ? p->p_name : "(unknown)"));
4110
4111 jetsam_reason = os_reason_create(OS_REASON_JETSAM, JETSAM_REASON_CPULIMIT);
4112 if (jetsam_reason == OS_REASON_NULL) {
4113 printf("task_exceeded_cpulimit: unable to allocate memory for jetsam reason\n");
4114 }
4115
4116 retval = jetsam_do_kill(p, jetsam_flags, jetsam_reason);
4117
4118 if (retval) {
4119 printf("task_exceeded_cpulimit: failed to kill current task (exiting?).\n");
4120 }
4121 }
4122
4123 #endif /* CONFIG_JETSAM */
4124
4125 static void
4126 memorystatus_get_task_memory_region_count(task_t task, uint64_t *count)
4127 {
4128 assert(task);
4129 assert(count);
4130
4131 *count = get_task_memory_region_count(task);
4132 }
4133
4134
4135 #define MEMORYSTATUS_VM_MAP_FORK_ALLOWED 0x100000000
4136 #define MEMORYSTATUS_VM_MAP_FORK_NOT_ALLOWED 0x200000000
4137
4138 #if DEVELOPMENT || DEBUG
4139
4140 /*
4141 * Sysctl only used to test memorystatus_allowed_vm_map_fork() path.
4142 * set a new pidwatch value
4143 * or
4144 * get the current pidwatch value
4145 *
4146 * The pidwatch_val starts out with a PID to watch for in the map_fork path.
4147 * Its value is:
4148 * - OR'd with MEMORYSTATUS_VM_MAP_FORK_ALLOWED if we allow the map_fork.
4149 * - OR'd with MEMORYSTATUS_VM_MAP_FORK_NOT_ALLOWED if we disallow the map_fork.
4150 * - set to -1ull if the map_fork() is aborted for other reasons.
4151 */
4152
4153 uint64_t memorystatus_vm_map_fork_pidwatch_val = 0;
4154
4155 static int sysctl_memorystatus_vm_map_fork_pidwatch SYSCTL_HANDLER_ARGS {
4156 #pragma unused(oidp, arg1, arg2)
4157
4158 uint64_t new_value = 0;
4159 uint64_t old_value = 0;
4160 int error = 0;
4161
4162 /*
4163 * The pid is held in the low 32 bits.
4164 * The 'allowed' flags are in the upper 32 bits.
4165 */
4166 old_value = memorystatus_vm_map_fork_pidwatch_val;
4167
4168 error = sysctl_io_number(req, old_value, sizeof(old_value), &new_value, NULL);
4169
4170 if (error || !req->newptr) {
4171 /*
4172 * No new value passed in.
4173 */
4174 return(error);
4175 }
4176
4177 /*
4178 * A new pid was passed in via req->newptr.
4179 * Ignore any attempt to set the higher order bits.
4180 */
4181 memorystatus_vm_map_fork_pidwatch_val = new_value & 0xFFFFFFFF;
4182 printf("memorystatus: pidwatch old_value = 0x%llx, new_value = 0x%llx \n", old_value, new_value);
4183
4184 return(error);
4185 }
4186
4187 SYSCTL_PROC(_kern, OID_AUTO, memorystatus_vm_map_fork_pidwatch, CTLTYPE_QUAD | CTLFLAG_RW | CTLFLAG_LOCKED| CTLFLAG_MASKED,
4188 0, 0, sysctl_memorystatus_vm_map_fork_pidwatch, "Q", "get/set pid watched for in vm_map_fork");
4189
4190
4191 /*
4192 * Record if a watched process fails to qualify for a vm_map_fork().
4193 */
4194 void
4195 memorystatus_abort_vm_map_fork(task_t task)
4196 {
4197 if (memorystatus_vm_map_fork_pidwatch_val != 0) {
4198 proc_t p = get_bsdtask_info(task);
4199 if (p != NULL && memorystatus_vm_map_fork_pidwatch_val == (uint64_t)p->p_pid) {
4200 memorystatus_vm_map_fork_pidwatch_val = -1ull;
4201 }
4202 }
4203 }
4204
4205 static void
4206 set_vm_map_fork_pidwatch(task_t task, uint64_t x)
4207 {
4208 if (memorystatus_vm_map_fork_pidwatch_val != 0) {
4209 proc_t p = get_bsdtask_info(task);
4210 if (p && (memorystatus_vm_map_fork_pidwatch_val == (uint64_t)p->p_pid)) {
4211 memorystatus_vm_map_fork_pidwatch_val |= x;
4212 }
4213 }
4214 }
4215
4216 #else /* DEVELOPMENT || DEBUG */
4217
4218
4219 static void
4220 set_vm_map_fork_pidwatch(task_t task, uint64_t x)
4221 {
4222 #pragma unused(task)
4223 #pragma unused(x)
4224 }
4225
4226 #endif /* DEVELOPMENT || DEBUG */
4227
4228 /*
4229 * Called during EXC_RESOURCE handling when a process exceeds a soft
4230 * memory limit. This is the corpse fork path and here we decide if
4231 * vm_map_fork will be allowed when creating the corpse.
4232 * The task being considered is suspended.
4233 *
4234 * By default, a vm_map_fork is allowed to proceed.
4235 *
4236 * A few simple policy assumptions:
4237 * Desktop platform is not considered in this path.
4238 * The vm_map_fork is always allowed.
4239 *
4240 * If the device has a zero system-wide task limit,
4241 * then the vm_map_fork is allowed.
4242 *
4243 * And if a process's memory footprint calculates less
4244 * than or equal to half of the system-wide task limit,
4245 * then the vm_map_fork is allowed. This calculation
4246 * is based on the assumption that a process can
4247 * munch memory up to the system-wide task limit.
4248 */
4249 boolean_t
4250 memorystatus_allowed_vm_map_fork(task_t task)
4251 {
4252 boolean_t is_allowed = TRUE; /* default */
4253
4254 #if CONFIG_EMBEDDED
4255
4256 uint64_t footprint_in_bytes;
4257 uint64_t purgeable_in_bytes;
4258 uint64_t max_allowed_bytes;
4259
4260 if (max_task_footprint_mb == 0) {
4261 set_vm_map_fork_pidwatch(task, MEMORYSTATUS_VM_MAP_FORK_ALLOWED);
4262 return (is_allowed);
4263 }
4264
4265 purgeable_in_bytes = get_task_purgeable_size(task);
4266 footprint_in_bytes = get_task_phys_footprint(task);
4267
4268 /*
4269 * Maximum is half the system-wide task limit.
4270 */
4271 max_allowed_bytes = ((uint64_t)max_task_footprint_mb * 1024 * 1024) >> 1;
4272
4273 if (footprint_in_bytes > purgeable_in_bytes) {
4274 footprint_in_bytes -= purgeable_in_bytes;
4275 }
4276
4277 if (footprint_in_bytes > max_allowed_bytes) {
4278 printf("memorystatus disallowed vm_map_fork %lld %lld\n", footprint_in_bytes, max_allowed_bytes);
4279 set_vm_map_fork_pidwatch(task, MEMORYSTATUS_VM_MAP_FORK_NOT_ALLOWED);
4280 return (!is_allowed);
4281 }
4282 #endif /* CONFIG_EMBEDDED */
4283
4284 set_vm_map_fork_pidwatch(task, MEMORYSTATUS_VM_MAP_FORK_ALLOWED);
4285 return (is_allowed);
4286
4287 }
4288
4289 static void
4290 memorystatus_get_task_page_counts(task_t task, uint32_t *footprint, uint32_t *max_footprint, uint32_t *max_footprint_lifetime, uint32_t *purgeable_pages)
4291 {
4292 assert(task);
4293 assert(footprint);
4294
4295 uint64_t pages;
4296
4297 pages = (get_task_phys_footprint(task) / PAGE_SIZE_64);
4298 assert(((uint32_t)pages) == pages);
4299 *footprint = (uint32_t)pages;
4300
4301 if (max_footprint) {
4302 pages = (get_task_phys_footprint_recent_max(task) / PAGE_SIZE_64);
4303 assert(((uint32_t)pages) == pages);
4304 *max_footprint = (uint32_t)pages;
4305 }
4306 if (max_footprint_lifetime) {
4307 pages = (get_task_resident_max(task) / PAGE_SIZE_64);
4308 assert(((uint32_t)pages) == pages);
4309 *max_footprint_lifetime = (uint32_t)pages;
4310 }
4311 if (purgeable_pages) {
4312 pages = (get_task_purgeable_size(task) / PAGE_SIZE_64);
4313 assert(((uint32_t)pages) == pages);
4314 *purgeable_pages = (uint32_t)pages;
4315 }
4316 }
4317
4318 static void
4319 memorystatus_get_task_phys_footprint_page_counts(task_t task,
4320 uint64_t *internal_pages, uint64_t *internal_compressed_pages,
4321 uint64_t *purgeable_nonvolatile_pages, uint64_t *purgeable_nonvolatile_compressed_pages,
4322 uint64_t *alternate_accounting_pages, uint64_t *alternate_accounting_compressed_pages,
4323 uint64_t *iokit_mapped_pages, uint64_t *page_table_pages)
4324 {
4325 assert(task);
4326
4327 if (internal_pages) {
4328 *internal_pages = (get_task_internal(task) / PAGE_SIZE_64);
4329 }
4330
4331 if (internal_compressed_pages) {
4332 *internal_compressed_pages = (get_task_internal_compressed(task) / PAGE_SIZE_64);
4333 }
4334
4335 if (purgeable_nonvolatile_pages) {
4336 *purgeable_nonvolatile_pages = (get_task_purgeable_nonvolatile(task) / PAGE_SIZE_64);
4337 }
4338
4339 if (purgeable_nonvolatile_compressed_pages) {
4340 *purgeable_nonvolatile_compressed_pages = (get_task_purgeable_nonvolatile_compressed(task) / PAGE_SIZE_64);
4341 }
4342
4343 if (alternate_accounting_pages) {
4344 *alternate_accounting_pages = (get_task_alternate_accounting(task) / PAGE_SIZE_64);
4345 }
4346
4347 if (alternate_accounting_compressed_pages) {
4348 *alternate_accounting_compressed_pages = (get_task_alternate_accounting_compressed(task) / PAGE_SIZE_64);
4349 }
4350
4351 if (iokit_mapped_pages) {
4352 *iokit_mapped_pages = (get_task_iokit_mapped(task) / PAGE_SIZE_64);
4353 }
4354
4355 if (page_table_pages) {
4356 *page_table_pages = (get_task_page_table(task) / PAGE_SIZE_64);
4357 }
4358 }
4359
4360 /*
4361 * This routine only acts on the global jetsam event snapshot.
4362 * Updating the process's entry can race when the memorystatus_thread
4363 * has chosen to kill a process that is racing to exit on another core.
4364 */
4365 static void
4366 memorystatus_update_jetsam_snapshot_entry_locked(proc_t p, uint32_t kill_cause, uint64_t killtime)
4367 {
4368 memorystatus_jetsam_snapshot_entry_t *entry = NULL;
4369 memorystatus_jetsam_snapshot_t *snapshot = NULL;
4370 memorystatus_jetsam_snapshot_entry_t *snapshot_list = NULL;
4371
4372 unsigned int i;
4373
4374 if (memorystatus_jetsam_snapshot_count == 0) {
4375 /*
4376 * No active snapshot.
4377 * Nothing to do.
4378 */
4379 return;
4380 }
4381
4382 /*
4383 * Sanity check as this routine should only be called
4384 * from a jetsam kill path.
4385 */
4386 assert(kill_cause != 0 && killtime != 0);
4387
4388 snapshot = memorystatus_jetsam_snapshot;
4389 snapshot_list = memorystatus_jetsam_snapshot->entries;
4390
4391 for (i = 0; i < memorystatus_jetsam_snapshot_count; i++) {
4392 if (snapshot_list[i].pid == p->p_pid) {
4393
4394 entry = &snapshot_list[i];
4395
4396 if (entry->killed || entry->jse_killtime) {
4397 /*
4398 * We apparently raced on the exit path
4399 * for this process, as it's snapshot entry
4400 * has already recorded a kill.
4401 */
4402 assert(entry->killed && entry->jse_killtime);
4403 break;
4404 }
4405
4406 /*
4407 * Update the entry we just found in the snapshot.
4408 */
4409
4410 entry->killed = kill_cause;
4411 entry->jse_killtime = killtime;
4412 entry->jse_gencount = snapshot->js_gencount;
4413 entry->jse_idle_delta = p->p_memstat_idle_delta;
4414
4415 /*
4416 * If a process has moved between bands since snapshot was
4417 * initialized, then likely these fields changed too.
4418 */
4419 if (entry->priority != p->p_memstat_effectivepriority) {
4420
4421 strlcpy(entry->name, p->p_name, sizeof(entry->name));
4422 entry->priority = p->p_memstat_effectivepriority;
4423 entry->state = memorystatus_build_state(p);
4424 entry->user_data = p->p_memstat_userdata;
4425 entry->fds = p->p_fd->fd_nfiles;
4426 }
4427
4428 /*
4429 * Always update the page counts on a kill.
4430 */
4431
4432 uint32_t pages = 0;
4433 uint32_t max_pages = 0;
4434 uint32_t max_pages_lifetime = 0;
4435 uint32_t purgeable_pages = 0;
4436
4437 memorystatus_get_task_page_counts(p->task, &pages, &max_pages, &max_pages_lifetime, &purgeable_pages);
4438 entry->pages = (uint64_t)pages;
4439 entry->max_pages = (uint64_t)max_pages;
4440 entry->max_pages_lifetime = (uint64_t)max_pages_lifetime;
4441 entry->purgeable_pages = (uint64_t)purgeable_pages;
4442
4443 uint64_t internal_pages = 0;
4444 uint64_t internal_compressed_pages = 0;
4445 uint64_t purgeable_nonvolatile_pages = 0;
4446 uint64_t purgeable_nonvolatile_compressed_pages = 0;
4447 uint64_t alternate_accounting_pages = 0;
4448 uint64_t alternate_accounting_compressed_pages = 0;
4449 uint64_t iokit_mapped_pages = 0;
4450 uint64_t page_table_pages = 0;
4451
4452 memorystatus_get_task_phys_footprint_page_counts(p->task, &internal_pages, &internal_compressed_pages,
4453 &purgeable_nonvolatile_pages, &purgeable_nonvolatile_compressed_pages,
4454 &alternate_accounting_pages, &alternate_accounting_compressed_pages,
4455 &iokit_mapped_pages, &page_table_pages);
4456
4457 entry->jse_internal_pages = internal_pages;
4458 entry->jse_internal_compressed_pages = internal_compressed_pages;
4459 entry->jse_purgeable_nonvolatile_pages = purgeable_nonvolatile_pages;
4460 entry->jse_purgeable_nonvolatile_compressed_pages = purgeable_nonvolatile_compressed_pages;
4461 entry->jse_alternate_accounting_pages = alternate_accounting_pages;
4462 entry->jse_alternate_accounting_compressed_pages = alternate_accounting_compressed_pages;
4463 entry->jse_iokit_mapped_pages = iokit_mapped_pages;
4464 entry->jse_page_table_pages = page_table_pages;
4465
4466 uint64_t region_count = 0;
4467 memorystatus_get_task_memory_region_count(p->task, &region_count);
4468 entry->jse_memory_region_count = region_count;
4469
4470 goto exit;
4471 }
4472 }
4473
4474 if (entry == NULL) {
4475 /*
4476 * The entry was not found in the snapshot, so the process must have
4477 * launched after the snapshot was initialized.
4478 * Let's try to append the new entry.
4479 */
4480 if (memorystatus_jetsam_snapshot_count < memorystatus_jetsam_snapshot_max) {
4481 /*
4482 * A populated snapshot buffer exists
4483 * and there is room to init a new entry.
4484 */
4485 assert(memorystatus_jetsam_snapshot_count == snapshot->entry_count);
4486
4487 unsigned int next = memorystatus_jetsam_snapshot_count;
4488
4489 if(memorystatus_init_jetsam_snapshot_entry_locked(p, &snapshot_list[next], (snapshot->js_gencount)) == TRUE) {
4490
4491 entry = &snapshot_list[next];
4492 entry->killed = kill_cause;
4493 entry->jse_killtime = killtime;
4494
4495 snapshot->entry_count = ++next;
4496 memorystatus_jetsam_snapshot_count = next;
4497
4498 if (memorystatus_jetsam_snapshot_count >= memorystatus_jetsam_snapshot_max) {
4499 /*
4500 * We just used the last slot in the snapshot buffer.
4501 * We only want to log it once... so we do it here
4502 * when we notice we've hit the max.
4503 */
4504 printf("memorystatus: WARNING snapshot buffer is full, count %d\n",
4505 memorystatus_jetsam_snapshot_count);
4506 }
4507 }
4508 }
4509 }
4510
4511 exit:
4512 if (entry == NULL) {
4513 /*
4514 * If we reach here, the snapshot buffer could not be updated.
4515 * Most likely, the buffer is full, in which case we would have
4516 * logged a warning in the previous call.
4517 *
4518 * For now, we will stop appending snapshot entries.
4519 * When the buffer is consumed, the snapshot state will reset.
4520 */
4521
4522 MEMORYSTATUS_DEBUG(4, "memorystatus_update_jetsam_snapshot_entry_locked: failed to update pid %d, priority %d, count %d\n",
4523 p->p_pid, p->p_memstat_effectivepriority, memorystatus_jetsam_snapshot_count);
4524 }
4525
4526 return;
4527 }
4528
4529 #if CONFIG_JETSAM
4530 void memorystatus_pages_update(unsigned int pages_avail)
4531 {
4532 memorystatus_available_pages = pages_avail;
4533
4534 #if VM_PRESSURE_EVENTS
4535 /*
4536 * Since memorystatus_available_pages changes, we should
4537 * re-evaluate the pressure levels on the system and
4538 * check if we need to wake the pressure thread.
4539 * We also update memorystatus_level in that routine.
4540 */
4541 vm_pressure_response();
4542
4543 if (memorystatus_available_pages <= memorystatus_available_pages_pressure) {
4544
4545 if (memorystatus_hwm_candidates || (memorystatus_available_pages <= memorystatus_available_pages_critical)) {
4546 memorystatus_thread_wake();
4547 }
4548 }
4549 #else /* VM_PRESSURE_EVENTS */
4550
4551 boolean_t critical, delta;
4552
4553 if (!memorystatus_delta) {
4554 return;
4555 }
4556
4557 critical = (pages_avail < memorystatus_available_pages_critical) ? TRUE : FALSE;
4558 delta = ((pages_avail >= (memorystatus_available_pages + memorystatus_delta))
4559 || (memorystatus_available_pages >= (pages_avail + memorystatus_delta))) ? TRUE : FALSE;
4560
4561 if (critical || delta) {
4562 unsigned int total_pages;
4563
4564 total_pages = (unsigned int) atop_64(max_mem);
4565 #if CONFIG_SECLUDED_MEMORY
4566 total_pages -= vm_page_secluded_count;
4567 #endif /* CONFIG_SECLUDED_MEMORY */
4568 memorystatus_level = memorystatus_available_pages * 100 / total_pages;
4569 memorystatus_thread_wake();
4570 }
4571 #endif /* VM_PRESSURE_EVENTS */
4572 }
4573 #endif /* CONFIG_JETSAM */
4574
4575 static boolean_t
4576 memorystatus_init_jetsam_snapshot_entry_locked(proc_t p, memorystatus_jetsam_snapshot_entry_t *entry, uint64_t gencount)
4577 {
4578 clock_sec_t tv_sec;
4579 clock_usec_t tv_usec;
4580 uint32_t pages = 0;
4581 uint32_t max_pages = 0;
4582 uint32_t max_pages_lifetime = 0;
4583 uint32_t purgeable_pages = 0;
4584 uint64_t internal_pages = 0;
4585 uint64_t internal_compressed_pages = 0;
4586 uint64_t purgeable_nonvolatile_pages = 0;
4587 uint64_t purgeable_nonvolatile_compressed_pages = 0;
4588 uint64_t alternate_accounting_pages = 0;
4589 uint64_t alternate_accounting_compressed_pages = 0;
4590 uint64_t iokit_mapped_pages = 0;
4591 uint64_t page_table_pages =0;
4592 uint64_t region_count = 0;
4593 uint64_t cids[COALITION_NUM_TYPES];
4594
4595 memset(entry, 0, sizeof(memorystatus_jetsam_snapshot_entry_t));
4596
4597 entry->pid = p->p_pid;
4598 strlcpy(&entry->name[0], p->p_name, sizeof(entry->name));
4599 entry->priority = p->p_memstat_effectivepriority;
4600
4601 memorystatus_get_task_page_counts(p->task, &pages, &max_pages, &max_pages_lifetime, &purgeable_pages);
4602 entry->pages = (uint64_t)pages;
4603 entry->max_pages = (uint64_t)max_pages;
4604 entry->max_pages_lifetime = (uint64_t)max_pages_lifetime;
4605 entry->purgeable_pages = (uint64_t)purgeable_pages;
4606
4607 memorystatus_get_task_phys_footprint_page_counts(p->task, &internal_pages, &internal_compressed_pages,
4608 &purgeable_nonvolatile_pages, &purgeable_nonvolatile_compressed_pages,
4609 &alternate_accounting_pages, &alternate_accounting_compressed_pages,
4610 &iokit_mapped_pages, &page_table_pages);
4611
4612 entry->jse_internal_pages = internal_pages;
4613 entry->jse_internal_compressed_pages = internal_compressed_pages;
4614 entry->jse_purgeable_nonvolatile_pages = purgeable_nonvolatile_pages;
4615 entry->jse_purgeable_nonvolatile_compressed_pages = purgeable_nonvolatile_compressed_pages;
4616 entry->jse_alternate_accounting_pages = alternate_accounting_pages;
4617 entry->jse_alternate_accounting_compressed_pages = alternate_accounting_compressed_pages;
4618 entry->jse_iokit_mapped_pages = iokit_mapped_pages;
4619 entry->jse_page_table_pages = page_table_pages;
4620
4621 memorystatus_get_task_memory_region_count(p->task, &region_count);
4622 entry->jse_memory_region_count = region_count;
4623
4624 entry->state = memorystatus_build_state(p);
4625 entry->user_data = p->p_memstat_userdata;
4626 memcpy(&entry->uuid[0], &p->p_uuid[0], sizeof(p->p_uuid));
4627 entry->fds = p->p_fd->fd_nfiles;
4628
4629 absolutetime_to_microtime(get_task_cpu_time(p->task), &tv_sec, &tv_usec);
4630 entry->cpu_time.tv_sec = tv_sec;
4631 entry->cpu_time.tv_usec = tv_usec;
4632
4633 assert(p->p_stats != NULL);
4634 entry->jse_starttime = p->p_stats->ps_start; /* abstime process started */
4635 entry->jse_killtime = 0; /* abstime jetsam chose to kill process */
4636 entry->killed = 0; /* the jetsam kill cause */
4637 entry->jse_gencount = gencount; /* indicates a pass through jetsam thread, when process was targeted to be killed */
4638
4639 entry->jse_idle_delta = p->p_memstat_idle_delta; /* Most recent timespan spent in idle-band */
4640
4641 proc_coalitionids(p, cids);
4642 entry->jse_coalition_jetsam_id = cids[COALITION_TYPE_JETSAM];
4643
4644 return TRUE;
4645 }
4646
4647 static void
4648 memorystatus_init_snapshot_vmstats(memorystatus_jetsam_snapshot_t *snapshot)
4649 {
4650 kern_return_t kr = KERN_SUCCESS;
4651 mach_msg_type_number_t count = HOST_VM_INFO64_COUNT;
4652 vm_statistics64_data_t vm_stat;
4653
4654 if ((kr = host_statistics64(host_self(), HOST_VM_INFO64, (host_info64_t)&vm_stat, &count)) != KERN_SUCCESS) {
4655 printf("memorystatus_init_jetsam_snapshot_stats: host_statistics64 failed with %d\n", kr);
4656 memset(&snapshot->stats, 0, sizeof(snapshot->stats));
4657 } else {
4658 snapshot->stats.free_pages = vm_stat.free_count;
4659 snapshot->stats.active_pages = vm_stat.active_count;
4660 snapshot->stats.inactive_pages = vm_stat.inactive_count;
4661 snapshot->stats.throttled_pages = vm_stat.throttled_count;
4662 snapshot->stats.purgeable_pages = vm_stat.purgeable_count;
4663 snapshot->stats.wired_pages = vm_stat.wire_count;
4664
4665 snapshot->stats.speculative_pages = vm_stat.speculative_count;
4666 snapshot->stats.filebacked_pages = vm_stat.external_page_count;
4667 snapshot->stats.anonymous_pages = vm_stat.internal_page_count;
4668 snapshot->stats.compressions = vm_stat.compressions;
4669 snapshot->stats.decompressions = vm_stat.decompressions;
4670 snapshot->stats.compressor_pages = vm_stat.compressor_page_count;
4671 snapshot->stats.total_uncompressed_pages_in_compressor = vm_stat.total_uncompressed_pages_in_compressor;
4672 }
4673
4674 get_zone_map_size(&snapshot->stats.zone_map_size, &snapshot->stats.zone_map_capacity);
4675 get_largest_zone_info(snapshot->stats.largest_zone_name, sizeof(snapshot->stats.largest_zone_name),
4676 &snapshot->stats.largest_zone_size);
4677 }
4678
4679 /*
4680 * Collect vm statistics at boot.
4681 * Called only once (see kern_exec.c)
4682 * Data can be consumed at any time.
4683 */
4684 void
4685 memorystatus_init_at_boot_snapshot() {
4686 memorystatus_init_snapshot_vmstats(&memorystatus_at_boot_snapshot);
4687 memorystatus_at_boot_snapshot.entry_count = 0;
4688 memorystatus_at_boot_snapshot.notification_time = 0; /* updated when consumed */
4689 memorystatus_at_boot_snapshot.snapshot_time = mach_absolute_time();
4690 }
4691
4692 static void
4693 memorystatus_init_jetsam_snapshot_locked(memorystatus_jetsam_snapshot_t *od_snapshot, uint32_t ods_list_count )
4694 {
4695 proc_t p, next_p;
4696 unsigned int b = 0, i = 0;
4697
4698 memorystatus_jetsam_snapshot_t *snapshot = NULL;
4699 memorystatus_jetsam_snapshot_entry_t *snapshot_list = NULL;
4700 unsigned int snapshot_max = 0;
4701
4702 if (od_snapshot) {
4703 /*
4704 * This is an on_demand snapshot
4705 */
4706 snapshot = od_snapshot;
4707 snapshot_list = od_snapshot->entries;
4708 snapshot_max = ods_list_count;
4709 } else {
4710 /*
4711 * This is a jetsam event snapshot
4712 */
4713 snapshot = memorystatus_jetsam_snapshot;
4714 snapshot_list = memorystatus_jetsam_snapshot->entries;
4715 snapshot_max = memorystatus_jetsam_snapshot_max;
4716 }
4717
4718 /*
4719 * Init the snapshot header information
4720 */
4721 memorystatus_init_snapshot_vmstats(snapshot);
4722 snapshot->snapshot_time = mach_absolute_time();
4723 snapshot->notification_time = 0;
4724 snapshot->js_gencount = 0;
4725
4726 next_p = memorystatus_get_first_proc_locked(&b, TRUE);
4727 while (next_p) {
4728 p = next_p;
4729 next_p = memorystatus_get_next_proc_locked(&b, p, TRUE);
4730
4731 if (FALSE == memorystatus_init_jetsam_snapshot_entry_locked(p, &snapshot_list[i], snapshot->js_gencount)) {
4732 continue;
4733 }
4734
4735 MEMORYSTATUS_DEBUG(0, "jetsam snapshot pid %d, uuid = %02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x\n",
4736 p->p_pid,
4737 p->p_uuid[0], p->p_uuid[1], p->p_uuid[2], p->p_uuid[3], p->p_uuid[4], p->p_uuid[5], p->p_uuid[6], p->p_uuid[7],
4738 p->p_uuid[8], p->p_uuid[9], p->p_uuid[10], p->p_uuid[11], p->p_uuid[12], p->p_uuid[13], p->p_uuid[14], p->p_uuid[15]);
4739
4740 if (++i == snapshot_max) {
4741 break;
4742 }
4743 }
4744
4745 snapshot->entry_count = i;
4746
4747 if (!od_snapshot) {
4748 /* update the system buffer count */
4749 memorystatus_jetsam_snapshot_count = i;
4750 }
4751 }
4752
4753 #if DEVELOPMENT || DEBUG
4754
4755 #if CONFIG_JETSAM
4756 static int
4757 memorystatus_cmd_set_panic_bits(user_addr_t buffer, uint32_t buffer_size) {
4758 int ret;
4759 memorystatus_jetsam_panic_options_t debug;
4760
4761 if (buffer_size != sizeof(memorystatus_jetsam_panic_options_t)) {
4762 return EINVAL;
4763 }
4764
4765 ret = copyin(buffer, &debug, buffer_size);
4766 if (ret) {
4767 return ret;
4768 }
4769
4770 /* Panic bits match kMemorystatusKilled* enum */
4771 memorystatus_jetsam_panic_debug = (memorystatus_jetsam_panic_debug & ~debug.mask) | (debug.data & debug.mask);
4772
4773 /* Copyout new value */
4774 debug.data = memorystatus_jetsam_panic_debug;
4775 ret = copyout(&debug, buffer, sizeof(memorystatus_jetsam_panic_options_t));
4776
4777 return ret;
4778 }
4779 #endif /* CONFIG_JETSAM */
4780
4781 /*
4782 * Triggers a sort_order on a specified jetsam priority band.
4783 * This is for testing only, used to force a path through the sort
4784 * function.
4785 */
4786 static int
4787 memorystatus_cmd_test_jetsam_sort(int priority, int sort_order) {
4788
4789 int error = 0;
4790
4791 unsigned int bucket_index = 0;
4792
4793 if (priority == -1) {
4794 /* Use as shorthand for default priority */
4795 bucket_index = JETSAM_PRIORITY_DEFAULT;
4796 } else {
4797 bucket_index = (unsigned int)priority;
4798 }
4799
4800 error = memorystatus_sort_bucket(bucket_index, sort_order);
4801
4802 return (error);
4803 }
4804
4805 #endif /* DEVELOPMENT || DEBUG */
4806
4807 /*
4808 * Prepare the process to be killed (set state, update snapshot) and kill it.
4809 */
4810 static uint64_t memorystatus_purge_before_jetsam_success = 0;
4811
4812 static boolean_t
4813 memorystatus_kill_proc(proc_t p, uint32_t cause, os_reason_t jetsam_reason, boolean_t *killed)
4814 {
4815 pid_t aPid = 0;
4816 uint32_t aPid_ep = 0;
4817
4818 uint64_t killtime = 0;
4819 clock_sec_t tv_sec;
4820 clock_usec_t tv_usec;
4821 uint32_t tv_msec;
4822 boolean_t retval = FALSE;
4823 uint64_t num_pages_purged = 0;
4824
4825 aPid = p->p_pid;
4826 aPid_ep = p->p_memstat_effectivepriority;
4827
4828 if (cause != kMemorystatusKilledVnodes && cause != kMemorystatusKilledZoneMapExhaustion) {
4829 /*
4830 * Genuine memory pressure and not other (vnode/zone) resource exhaustion.
4831 */
4832 boolean_t success = FALSE;
4833
4834 networking_memstatus_callout(p, cause);
4835 num_pages_purged = vm_purgeable_purge_task_owned(p->task);
4836
4837 if (num_pages_purged) {
4838 /*
4839 * We actually purged something and so let's
4840 * check if we need to continue with the kill.
4841 */
4842 if (cause == kMemorystatusKilledHiwat) {
4843 uint64_t footprint_in_bytes = get_task_phys_footprint(p->task);
4844 uint64_t memlimit_in_bytes = (((uint64_t)p->p_memstat_memlimit) * 1024ULL * 1024ULL); /* convert MB to bytes */
4845 success = (footprint_in_bytes <= memlimit_in_bytes);
4846 } else {
4847 success = (memorystatus_avail_pages_below_pressure() == FALSE);
4848 }
4849
4850 if (success) {
4851
4852 memorystatus_purge_before_jetsam_success++;
4853
4854 os_log_with_startup_serial(OS_LOG_DEFAULT, "memorystatus: purged %llu pages from pid %d [%s] and avoided %s\n",
4855 num_pages_purged, aPid, (*p->p_name ? p->p_name : "unknown"), memorystatus_kill_cause_name[cause]);
4856
4857 *killed = FALSE;
4858
4859 return TRUE;
4860 }
4861 }
4862 }
4863
4864 #if CONFIG_JETSAM && (DEVELOPMENT || DEBUG)
4865 MEMORYSTATUS_DEBUG(1, "jetsam: %s pid %d [%s] - %lld Mb > 1 (%d Mb)\n",
4866 (memorystatus_jetsam_policy & kPolicyDiagnoseActive) ? "suspending": "killing",
4867 aPid, (*p->p_name ? p->p_name : "unknown"),
4868 (footprint_in_bytes / (1024ULL * 1024ULL)), /* converted bytes to MB */
4869 p->p_memstat_memlimit);
4870 #endif /* CONFIG_JETSAM && (DEVELOPMENT || DEBUG) */
4871
4872 killtime = mach_absolute_time();
4873 absolutetime_to_microtime(killtime, &tv_sec, &tv_usec);
4874 tv_msec = tv_usec / 1000;
4875
4876 #if CONFIG_JETSAM && (DEVELOPMENT || DEBUG)
4877 if (memorystatus_jetsam_policy & kPolicyDiagnoseActive) {
4878 if (cause == kMemorystatusKilledHiwat) {
4879 MEMORYSTATUS_DEBUG(1, "jetsam: suspending pid %d [%s] for diagnosis - memorystatus_available_pages: %d\n",
4880 aPid, (*p->p_name ? p->p_name: "(unknown)"), memorystatus_available_pages);
4881 } else {
4882 int activeProcess = p->p_memstat_state & P_MEMSTAT_FOREGROUND;
4883 if (activeProcess) {
4884 MEMORYSTATUS_DEBUG(1, "jetsam: suspending pid %d [%s] (active) for diagnosis - memorystatus_available_pages: %d\n",
4885 aPid, (*p->p_name ? p->p_name: "(unknown)"), memorystatus_available_pages);
4886
4887 if (memorystatus_jetsam_policy & kPolicyDiagnoseFirst) {
4888 jetsam_diagnostic_suspended_one_active_proc = 1;
4889 printf("jetsam: returning after suspending first active proc - %d\n", aPid);
4890 }
4891 }
4892 }
4893
4894 memorystatus_update_jetsam_snapshot_entry_locked(p, kMemorystatusKilledDiagnostic, killtime);
4895 p->p_memstat_state |= P_MEMSTAT_DIAG_SUSPENDED;
4896
4897 if (p) {
4898 task_suspend(p->task);
4899 *killed = TRUE;
4900 }
4901 } else
4902 #endif /* CONFIG_JETSAM && (DEVELOPMENT || DEBUG) */
4903 {
4904 memorystatus_update_jetsam_snapshot_entry_locked(p, cause, killtime);
4905
4906 char kill_reason_string[128];
4907
4908 if (cause == kMemorystatusKilledHiwat) {
4909 strlcpy(kill_reason_string, "killing_highwater_process", 128);
4910 } else {
4911 if (aPid_ep == JETSAM_PRIORITY_IDLE) {
4912 strlcpy(kill_reason_string, "killing_idle_process", 128);
4913 } else {
4914 strlcpy(kill_reason_string, "killing_top_process", 128);
4915 }
4916 }
4917
4918 os_log_with_startup_serial(OS_LOG_DEFAULT, "%lu.%03d memorystatus: %s pid %d [%s] (%s %d) - memorystatus_available_pages: %llu\n",
4919 (unsigned long)tv_sec, tv_msec, kill_reason_string,
4920 aPid, (*p->p_name ? p->p_name : "unknown"),
4921 memorystatus_kill_cause_name[cause], aPid_ep, (uint64_t)memorystatus_available_pages);
4922
4923 /*
4924 * memorystatus_do_kill drops a reference, so take another one so we can
4925 * continue to use this exit reason even after memorystatus_do_kill()
4926 * returns
4927 */
4928 os_reason_ref(jetsam_reason);
4929
4930 retval = memorystatus_do_kill(p, cause, jetsam_reason);
4931
4932 *killed = retval;
4933 }
4934
4935 return retval;
4936 }
4937
4938 /*
4939 * Jetsam the first process in the queue.
4940 */
4941 static boolean_t
4942 memorystatus_kill_top_process(boolean_t any, boolean_t sort_flag, uint32_t cause, os_reason_t jetsam_reason,
4943 int32_t *priority, uint32_t *errors)
4944 {
4945 pid_t aPid;
4946 proc_t p = PROC_NULL, next_p = PROC_NULL;
4947 boolean_t new_snapshot = FALSE, force_new_snapshot = FALSE, killed = FALSE, freed_mem = FALSE;
4948 unsigned int i = 0;
4949 uint32_t aPid_ep;
4950 int32_t local_max_kill_prio = JETSAM_PRIORITY_IDLE;
4951
4952 #ifndef CONFIG_FREEZE
4953 #pragma unused(any)
4954 #endif
4955
4956 KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_JETSAM) | DBG_FUNC_START,
4957 memorystatus_available_pages, 0, 0, 0, 0);
4958
4959
4960 #if CONFIG_JETSAM
4961 if (sort_flag == TRUE) {
4962 (void)memorystatus_sort_bucket(JETSAM_PRIORITY_FOREGROUND, JETSAM_SORT_DEFAULT);
4963 }
4964
4965 local_max_kill_prio = max_kill_priority;
4966
4967 force_new_snapshot = FALSE;
4968
4969 #else /* CONFIG_JETSAM */
4970
4971 if (sort_flag == TRUE) {
4972 (void)memorystatus_sort_bucket(JETSAM_PRIORITY_IDLE, JETSAM_SORT_DEFAULT);
4973 }
4974
4975 /*
4976 * On macos, we currently only have 2 reasons to be here:
4977 *
4978 * kMemorystatusKilledZoneMapExhaustion
4979 * AND
4980 * kMemorystatusKilledVMThrashing
4981 *
4982 * If we are here because of kMemorystatusKilledZoneMapExhaustion, we will consider
4983 * any and all processes as eligible kill candidates since we need to avoid a panic.
4984 *
4985 * Since this function can be called async. it is harder to toggle the max_kill_priority
4986 * value before and after a call. And so we use this local variable to set the upper band
4987 * on the eligible kill bands.
4988 */
4989 if (cause == kMemorystatusKilledZoneMapExhaustion) {
4990 local_max_kill_prio = JETSAM_PRIORITY_MAX;
4991 } else {
4992 local_max_kill_prio = max_kill_priority;
4993 }
4994
4995 /*
4996 * And, because we are here under extreme circumstances, we force a snapshot even for
4997 * IDLE kills.
4998 */
4999 force_new_snapshot = TRUE;
5000
5001 #endif /* CONFIG_JETSAM */
5002
5003 proc_list_lock();
5004
5005 next_p = memorystatus_get_first_proc_locked(&i, TRUE);
5006 while (next_p && (next_p->p_memstat_effectivepriority <= local_max_kill_prio)) {
5007 #if DEVELOPMENT || DEBUG
5008 int procSuspendedForDiagnosis;
5009 #endif /* DEVELOPMENT || DEBUG */
5010
5011 p = next_p;
5012 next_p = memorystatus_get_next_proc_locked(&i, p, TRUE);
5013
5014 #if DEVELOPMENT || DEBUG
5015 procSuspendedForDiagnosis = p->p_memstat_state & P_MEMSTAT_DIAG_SUSPENDED;
5016 #endif /* DEVELOPMENT || DEBUG */
5017
5018 aPid = p->p_pid;
5019 aPid_ep = p->p_memstat_effectivepriority;
5020
5021 if (p->p_memstat_state & (P_MEMSTAT_ERROR | P_MEMSTAT_TERMINATED)) {
5022 continue; /* with lock held */
5023 }
5024
5025 #if CONFIG_JETSAM && (DEVELOPMENT || DEBUG)
5026 if ((memorystatus_jetsam_policy & kPolicyDiagnoseActive) && procSuspendedForDiagnosis) {
5027 printf("jetsam: continuing after ignoring proc suspended already for diagnosis - %d\n", aPid);
5028 continue;
5029 }
5030 #endif /* CONFIG_JETSAM && (DEVELOPMENT || DEBUG) */
5031
5032 if (cause == kMemorystatusKilledVnodes)
5033 {
5034 /*
5035 * If the system runs out of vnodes, we systematically jetsam
5036 * processes in hopes of stumbling onto a vnode gain that helps
5037 * the system recover. The process that happens to trigger
5038 * this path has no known relationship to the vnode shortage.
5039 * Deadlock avoidance: attempt to safeguard the caller.
5040 */
5041
5042 if (p == current_proc()) {
5043 /* do not jetsam the current process */
5044 continue;
5045 }
5046 }
5047
5048 #if CONFIG_FREEZE
5049 boolean_t skip;
5050 boolean_t reclaim_proc = !(p->p_memstat_state & (P_MEMSTAT_LOCKED | P_MEMSTAT_NORECLAIM));
5051 if (any || reclaim_proc) {
5052 skip = FALSE;
5053 } else {
5054 skip = TRUE;
5055 }
5056
5057 if (skip) {
5058 continue;
5059 } else
5060 #endif
5061 {
5062 if (proc_ref_locked(p) == p) {
5063 /*
5064 * Mark as terminated so that if exit1() indicates success, but the process (for example)
5065 * is blocked in task_exception_notify(), it'll be skipped if encountered again - see
5066 * <rdar://problem/13553476>. This is cheaper than examining P_LEXIT, which requires the
5067 * acquisition of the proc lock.
5068 */
5069 p->p_memstat_state |= P_MEMSTAT_TERMINATED;
5070
5071 proc_list_unlock();
5072 } else {
5073 /*
5074 * We need to restart the search again because
5075 * proc_ref_locked _can_ drop the proc_list lock
5076 * and we could have lost our stored next_p via
5077 * an exit() on another core.
5078 */
5079 i = 0;
5080 next_p = memorystatus_get_first_proc_locked(&i, TRUE);
5081 continue;
5082 }
5083
5084 /*
5085 * Capture a snapshot if none exists and:
5086 * - we are forcing a new snapshot creation, either because:
5087 * - on a particular platform we need these snapshots every time, OR
5088 * - a boot-arg/embedded device tree property has been set.
5089 * - priority was not requested (this is something other than an ambient kill)
5090 * - the priority was requested *and* the targeted process is not at idle priority
5091 */
5092 if ((memorystatus_jetsam_snapshot_count == 0) &&
5093 (force_new_snapshot || memorystatus_idle_snapshot || ((!priority) || (priority && (aPid_ep != JETSAM_PRIORITY_IDLE))))) {
5094 memorystatus_init_jetsam_snapshot_locked(NULL,0);
5095 new_snapshot = TRUE;
5096 }
5097
5098 freed_mem = memorystatus_kill_proc(p, cause, jetsam_reason, &killed); /* purged and/or killed 'p' */
5099 /* Success? */
5100 if (freed_mem) {
5101 if (killed) {
5102 if (priority) {
5103 *priority = aPid_ep;
5104 }
5105 } else {
5106 /* purged */
5107 proc_list_lock();
5108 p->p_memstat_state &= ~P_MEMSTAT_TERMINATED;
5109 proc_list_unlock();
5110 }
5111 proc_rele(p);
5112 goto exit;
5113 }
5114
5115 /*
5116 * Failure - first unwind the state,
5117 * then fall through to restart the search.
5118 */
5119 proc_list_lock();
5120 proc_rele_locked(p);
5121 p->p_memstat_state &= ~P_MEMSTAT_TERMINATED;
5122 p->p_memstat_state |= P_MEMSTAT_ERROR;
5123 *errors += 1;
5124
5125 i = 0;
5126 next_p = memorystatus_get_first_proc_locked(&i, TRUE);
5127 }
5128 }
5129
5130 proc_list_unlock();
5131
5132 exit:
5133 os_reason_free(jetsam_reason);
5134
5135 /* Clear snapshot if freshly captured and no target was found */
5136 if (new_snapshot && !killed) {
5137 proc_list_lock();
5138 memorystatus_jetsam_snapshot->entry_count = memorystatus_jetsam_snapshot_count = 0;
5139 proc_list_unlock();
5140 }
5141
5142 KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_JETSAM) | DBG_FUNC_END,
5143 memorystatus_available_pages, killed ? aPid : 0, 0, 0, 0);
5144
5145 return killed;
5146 }
5147
5148 /*
5149 * Jetsam aggressively
5150 */
5151 static boolean_t
5152 memorystatus_kill_top_process_aggressive(uint32_t cause, int aggr_count,
5153 int32_t priority_max, uint32_t *errors)
5154 {
5155 pid_t aPid;
5156 proc_t p = PROC_NULL, next_p = PROC_NULL;
5157 boolean_t new_snapshot = FALSE, killed = FALSE;
5158 int kill_count = 0;
5159 unsigned int i = 0;
5160 int32_t aPid_ep = 0;
5161 unsigned int memorystatus_level_snapshot = 0;
5162 uint64_t killtime = 0;
5163 clock_sec_t tv_sec;
5164 clock_usec_t tv_usec;
5165 uint32_t tv_msec;
5166 os_reason_t jetsam_reason = OS_REASON_NULL;
5167
5168 KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_JETSAM) | DBG_FUNC_START,
5169 memorystatus_available_pages, priority_max, 0, 0, 0);
5170
5171 memorystatus_sort_bucket(JETSAM_PRIORITY_FOREGROUND, JETSAM_SORT_DEFAULT);
5172
5173 jetsam_reason = os_reason_create(OS_REASON_JETSAM, cause);
5174 if (jetsam_reason == OS_REASON_NULL) {
5175 printf("memorystatus_kill_top_process_aggressive: failed to allocate exit reason\n");
5176 }
5177
5178 proc_list_lock();
5179
5180 next_p = memorystatus_get_first_proc_locked(&i, TRUE);
5181 while (next_p) {
5182 #if DEVELOPMENT || DEBUG
5183 int activeProcess;
5184 int procSuspendedForDiagnosis;
5185 #endif /* DEVELOPMENT || DEBUG */
5186
5187 if (((next_p->p_listflag & P_LIST_EXITED) != 0) ||
5188 ((unsigned int)(next_p->p_memstat_effectivepriority) != i)) {
5189
5190 /*
5191 * We have raced with next_p running on another core.
5192 * It may be exiting or it may have moved to a different
5193 * jetsam priority band. This means we have lost our
5194 * place in line while traversing the jetsam list. We
5195 * attempt to recover by rewinding to the beginning of the band
5196 * we were already traversing. By doing this, we do not guarantee
5197 * that no process escapes this aggressive march, but we can make
5198 * skipping an entire range of processes less likely. (PR-21069019)
5199 */
5200
5201 MEMORYSTATUS_DEBUG(1, "memorystatus: aggressive%d: rewinding band %d, %s(%d) moved or exiting.\n",
5202 aggr_count, i, (*next_p->p_name ? next_p->p_name : "unknown"), next_p->p_pid);
5203
5204 next_p = memorystatus_get_first_proc_locked(&i, TRUE);
5205 continue;
5206 }
5207
5208 p = next_p;
5209 next_p = memorystatus_get_next_proc_locked(&i, p, TRUE);
5210
5211 if (p->p_memstat_effectivepriority > priority_max) {
5212 /*
5213 * Bail out of this killing spree if we have
5214 * reached beyond the priority_max jetsam band.
5215 * That is, we kill up to and through the
5216 * priority_max jetsam band.
5217 */
5218 proc_list_unlock();
5219 goto exit;
5220 }
5221
5222 #if DEVELOPMENT || DEBUG
5223 activeProcess = p->p_memstat_state & P_MEMSTAT_FOREGROUND;
5224 procSuspendedForDiagnosis = p->p_memstat_state & P_MEMSTAT_DIAG_SUSPENDED;
5225 #endif /* DEVELOPMENT || DEBUG */
5226
5227 aPid = p->p_pid;
5228 aPid_ep = p->p_memstat_effectivepriority;
5229
5230 if (p->p_memstat_state & (P_MEMSTAT_ERROR | P_MEMSTAT_TERMINATED)) {
5231 continue;
5232 }
5233
5234 #if CONFIG_JETSAM && (DEVELOPMENT || DEBUG)
5235 if ((memorystatus_jetsam_policy & kPolicyDiagnoseActive) && procSuspendedForDiagnosis) {
5236 printf("jetsam: continuing after ignoring proc suspended already for diagnosis - %d\n", aPid);
5237 continue;
5238 }
5239 #endif /* CONFIG_JETSAM && (DEVELOPMENT || DEBUG) */
5240
5241 /*
5242 * Capture a snapshot if none exists.
5243 */
5244 if (memorystatus_jetsam_snapshot_count == 0) {
5245 memorystatus_init_jetsam_snapshot_locked(NULL,0);
5246 new_snapshot = TRUE;
5247 }
5248
5249 /*
5250 * Mark as terminated so that if exit1() indicates success, but the process (for example)
5251 * is blocked in task_exception_notify(), it'll be skipped if encountered again - see
5252 * <rdar://problem/13553476>. This is cheaper than examining P_LEXIT, which requires the
5253 * acquisition of the proc lock.
5254 */
5255 p->p_memstat_state |= P_MEMSTAT_TERMINATED;
5256
5257 killtime = mach_absolute_time();
5258 absolutetime_to_microtime(killtime, &tv_sec, &tv_usec);
5259 tv_msec = tv_usec / 1000;
5260
5261 /* Shift queue, update stats */
5262 memorystatus_update_jetsam_snapshot_entry_locked(p, cause, killtime);
5263
5264 /*
5265 * In order to kill the target process, we will drop the proc_list_lock.
5266 * To guaranteee that p and next_p don't disappear out from under the lock,
5267 * we must take a ref on both.
5268 * If we cannot get a reference, then it's likely we've raced with
5269 * that process exiting on another core.
5270 */
5271 if (proc_ref_locked(p) == p) {
5272 if (next_p) {
5273 while (next_p && (proc_ref_locked(next_p) != next_p)) {
5274 proc_t temp_p;
5275
5276 /*
5277 * We must have raced with next_p exiting on another core.
5278 * Recover by getting the next eligible process in the band.
5279 */
5280
5281 MEMORYSTATUS_DEBUG(1, "memorystatus: aggressive%d: skipping %d [%s] (exiting?)\n",
5282 aggr_count, next_p->p_pid, (*next_p->p_name ? next_p->p_name : "(unknown)"));
5283
5284 temp_p = next_p;
5285 next_p = memorystatus_get_next_proc_locked(&i, temp_p, TRUE);
5286 }
5287 }
5288 proc_list_unlock();
5289
5290 printf("%lu.%03d memorystatus: %s%d pid %d [%s] (%s %d) - memorystatus_available_pages: %llu\n",
5291 (unsigned long)tv_sec, tv_msec,
5292 ((aPid_ep == JETSAM_PRIORITY_IDLE) ? "killing_idle_process_aggressive" : "killing_top_process_aggressive"),
5293 aggr_count, aPid, (*p->p_name ? p->p_name : "unknown"),
5294 memorystatus_kill_cause_name[cause], aPid_ep, (uint64_t)memorystatus_available_pages);
5295
5296 memorystatus_level_snapshot = memorystatus_level;
5297
5298 /*
5299 * memorystatus_do_kill() drops a reference, so take another one so we can
5300 * continue to use this exit reason even after memorystatus_do_kill()
5301 * returns.
5302 */
5303 os_reason_ref(jetsam_reason);
5304 killed = memorystatus_do_kill(p, cause, jetsam_reason);
5305
5306 /* Success? */
5307 if (killed) {
5308 proc_rele(p);
5309 kill_count++;
5310 p = NULL;
5311 killed = FALSE;
5312
5313 /*
5314 * Continue the killing spree.
5315 */
5316 proc_list_lock();
5317 if (next_p) {
5318 proc_rele_locked(next_p);
5319 }
5320
5321 if (aPid_ep == JETSAM_PRIORITY_FOREGROUND && memorystatus_aggressive_jetsam_lenient == TRUE) {
5322 if (memorystatus_level > memorystatus_level_snapshot && ((memorystatus_level - memorystatus_level_snapshot) >= AGGRESSIVE_JETSAM_LENIENT_MODE_THRESHOLD)) {
5323 #if DEVELOPMENT || DEBUG
5324 printf("Disabling Lenient mode after one-time deployment.\n");
5325 #endif /* DEVELOPMENT || DEBUG */
5326 memorystatus_aggressive_jetsam_lenient = FALSE;
5327 break;
5328 }
5329 }
5330
5331 continue;
5332 }
5333
5334 /*
5335 * Failure - first unwind the state,
5336 * then fall through to restart the search.
5337 */
5338 proc_list_lock();
5339 proc_rele_locked(p);
5340 if (next_p) {
5341 proc_rele_locked(next_p);
5342 }
5343 p->p_memstat_state &= ~P_MEMSTAT_TERMINATED;
5344 p->p_memstat_state |= P_MEMSTAT_ERROR;
5345 *errors += 1;
5346 p = NULL;
5347 }
5348
5349 /*
5350 * Failure - restart the search at the beginning of
5351 * the band we were already traversing.
5352 *
5353 * We might have raced with "p" exiting on another core, resulting in no
5354 * ref on "p". Or, we may have failed to kill "p".
5355 *
5356 * Either way, we fall thru to here, leaving the proc in the
5357 * P_MEMSTAT_TERMINATED or P_MEMSTAT_ERROR state.
5358 *
5359 * And, we hold the the proc_list_lock at this point.
5360 */
5361
5362 next_p = memorystatus_get_first_proc_locked(&i, TRUE);
5363 }
5364
5365 proc_list_unlock();
5366
5367 exit:
5368 os_reason_free(jetsam_reason);
5369
5370 /* Clear snapshot if freshly captured and no target was found */
5371 if (new_snapshot && (kill_count == 0)) {
5372 memorystatus_jetsam_snapshot->entry_count = memorystatus_jetsam_snapshot_count = 0;
5373 }
5374
5375 KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_JETSAM) | DBG_FUNC_END,
5376 memorystatus_available_pages, killed ? aPid : 0, kill_count, 0, 0);
5377
5378 if (kill_count > 0) {
5379 return(TRUE);
5380 }
5381 else {
5382 return(FALSE);
5383 }
5384 }
5385
5386 static boolean_t
5387 memorystatus_kill_hiwat_proc(uint32_t *errors, boolean_t *purged)
5388 {
5389 pid_t aPid = 0;
5390 proc_t p = PROC_NULL, next_p = PROC_NULL;
5391 boolean_t new_snapshot = FALSE, killed = FALSE, freed_mem = FALSE;
5392 unsigned int i = 0;
5393 uint32_t aPid_ep;
5394 os_reason_t jetsam_reason = OS_REASON_NULL;
5395 KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_JETSAM_HIWAT) | DBG_FUNC_START,
5396 memorystatus_available_pages, 0, 0, 0, 0);
5397
5398 jetsam_reason = os_reason_create(OS_REASON_JETSAM, JETSAM_REASON_MEMORY_HIGHWATER);
5399 if (jetsam_reason == OS_REASON_NULL) {
5400 printf("memorystatus_kill_hiwat_proc: failed to allocate exit reason\n");
5401 }
5402
5403 proc_list_lock();
5404
5405 next_p = memorystatus_get_first_proc_locked(&i, TRUE);
5406 while (next_p) {
5407 uint64_t footprint_in_bytes = 0;
5408 uint64_t memlimit_in_bytes = 0;
5409 boolean_t skip = 0;
5410
5411 p = next_p;
5412 next_p = memorystatus_get_next_proc_locked(&i, p, TRUE);
5413
5414 aPid = p->p_pid;
5415 aPid_ep = p->p_memstat_effectivepriority;
5416
5417 if (p->p_memstat_state & (P_MEMSTAT_ERROR | P_MEMSTAT_TERMINATED)) {
5418 continue;
5419 }
5420
5421 /* skip if no limit set */
5422 if (p->p_memstat_memlimit <= 0) {
5423 continue;
5424 }
5425
5426 footprint_in_bytes = get_task_phys_footprint(p->task);
5427 memlimit_in_bytes = (((uint64_t)p->p_memstat_memlimit) * 1024ULL * 1024ULL); /* convert MB to bytes */
5428 skip = (footprint_in_bytes <= memlimit_in_bytes);
5429
5430 #if CONFIG_JETSAM && (DEVELOPMENT || DEBUG)
5431 if (!skip && (memorystatus_jetsam_policy & kPolicyDiagnoseActive)) {
5432 if (p->p_memstat_state & P_MEMSTAT_DIAG_SUSPENDED) {
5433 continue;
5434 }
5435 }
5436 #endif /* CONFIG_JETSAM && (DEVELOPMENT || DEBUG) */
5437
5438 #if CONFIG_FREEZE
5439 if (!skip) {
5440 if (p->p_memstat_state & P_MEMSTAT_LOCKED) {
5441 skip = TRUE;
5442 } else {
5443 skip = FALSE;
5444 }
5445 }
5446 #endif
5447
5448 if (skip) {
5449 continue;
5450 } else {
5451
5452 if (memorystatus_jetsam_snapshot_count == 0) {
5453 memorystatus_init_jetsam_snapshot_locked(NULL,0);
5454 new_snapshot = TRUE;
5455 }
5456
5457 if (proc_ref_locked(p) == p) {
5458 /*
5459 * Mark as terminated so that if exit1() indicates success, but the process (for example)
5460 * is blocked in task_exception_notify(), it'll be skipped if encountered again - see
5461 * <rdar://problem/13553476>. This is cheaper than examining P_LEXIT, which requires the
5462 * acquisition of the proc lock.
5463 */
5464 p->p_memstat_state |= P_MEMSTAT_TERMINATED;
5465
5466 proc_list_unlock();
5467 } else {
5468 /*
5469 * We need to restart the search again because
5470 * proc_ref_locked _can_ drop the proc_list lock
5471 * and we could have lost our stored next_p via
5472 * an exit() on another core.
5473 */
5474 i = 0;
5475 next_p = memorystatus_get_first_proc_locked(&i, TRUE);
5476 continue;
5477 }
5478
5479 freed_mem = memorystatus_kill_proc(p, kMemorystatusKilledHiwat, jetsam_reason, &killed); /* purged and/or killed 'p' */
5480
5481 /* Success? */
5482 if (freed_mem) {
5483 if (killed == FALSE) {
5484 /* purged 'p'..don't reset HWM candidate count */
5485 *purged = TRUE;
5486
5487 proc_list_lock();
5488 p->p_memstat_state &= ~P_MEMSTAT_TERMINATED;
5489 proc_list_unlock();
5490 }
5491 proc_rele(p);
5492 goto exit;
5493 }
5494 /*
5495 * Failure - first unwind the state,
5496 * then fall through to restart the search.
5497 */
5498 proc_list_lock();
5499 proc_rele_locked(p);
5500 p->p_memstat_state &= ~P_MEMSTAT_TERMINATED;
5501 p->p_memstat_state |= P_MEMSTAT_ERROR;
5502 *errors += 1;
5503
5504 i = 0;
5505 next_p = memorystatus_get_first_proc_locked(&i, TRUE);
5506 }
5507 }
5508
5509 proc_list_unlock();
5510
5511 exit:
5512 os_reason_free(jetsam_reason);
5513
5514 /* Clear snapshot if freshly captured and no target was found */
5515 if (new_snapshot && !killed) {
5516 proc_list_lock();
5517 memorystatus_jetsam_snapshot->entry_count = memorystatus_jetsam_snapshot_count = 0;
5518 proc_list_unlock();
5519 }
5520
5521 KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_JETSAM_HIWAT) | DBG_FUNC_END,
5522 memorystatus_available_pages, killed ? aPid : 0, 0, 0, 0);
5523
5524 return killed;
5525 }
5526
5527 /*
5528 * Jetsam a process pinned in the elevated band.
5529 *
5530 * Return: true -- at least one pinned process was jetsammed
5531 * false -- no pinned process was jetsammed
5532 */
5533 static boolean_t
5534 memorystatus_kill_elevated_process(uint32_t cause, os_reason_t jetsam_reason, int aggr_count, uint32_t *errors)
5535 {
5536 pid_t aPid = 0;
5537 proc_t p = PROC_NULL, next_p = PROC_NULL;
5538 boolean_t new_snapshot = FALSE, killed = FALSE;
5539 int kill_count = 0;
5540 unsigned int i = JETSAM_PRIORITY_ELEVATED_INACTIVE;
5541 uint32_t aPid_ep;
5542 uint64_t killtime = 0;
5543 clock_sec_t tv_sec;
5544 clock_usec_t tv_usec;
5545 uint32_t tv_msec;
5546
5547
5548 KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_JETSAM) | DBG_FUNC_START,
5549 memorystatus_available_pages, 0, 0, 0, 0);
5550
5551 proc_list_lock();
5552
5553 next_p = memorystatus_get_first_proc_locked(&i, FALSE);
5554 while (next_p) {
5555
5556 p = next_p;
5557 next_p = memorystatus_get_next_proc_locked(&i, p, FALSE);
5558
5559 aPid = p->p_pid;
5560 aPid_ep = p->p_memstat_effectivepriority;
5561
5562 /*
5563 * Only pick a process pinned in this elevated band
5564 */
5565 if (!(p->p_memstat_state & P_MEMSTAT_USE_ELEVATED_INACTIVE_BAND)) {
5566 continue;
5567 }
5568
5569 if (p->p_memstat_state & (P_MEMSTAT_ERROR | P_MEMSTAT_TERMINATED)) {
5570 continue;
5571 }
5572
5573 #if CONFIG_FREEZE
5574 if (p->p_memstat_state & P_MEMSTAT_LOCKED) {
5575 continue;
5576 }
5577 #endif
5578
5579 #if DEVELOPMENT || DEBUG
5580 MEMORYSTATUS_DEBUG(1, "jetsam: elevated%d process pid %d [%s] - memorystatus_available_pages: %d\n",
5581 aggr_count,
5582 aPid, (*p->p_name ? p->p_name : "unknown"),
5583 memorystatus_available_pages);
5584 #endif /* DEVELOPMENT || DEBUG */
5585
5586 if (memorystatus_jetsam_snapshot_count == 0) {
5587 memorystatus_init_jetsam_snapshot_locked(NULL,0);
5588 new_snapshot = TRUE;
5589 }
5590
5591 p->p_memstat_state |= P_MEMSTAT_TERMINATED;
5592
5593 killtime = mach_absolute_time();
5594 absolutetime_to_microtime(killtime, &tv_sec, &tv_usec);
5595 tv_msec = tv_usec / 1000;
5596
5597 memorystatus_update_jetsam_snapshot_entry_locked(p, cause, killtime);
5598
5599 if (proc_ref_locked(p) == p) {
5600
5601 proc_list_unlock();
5602
5603 os_log_with_startup_serial(OS_LOG_DEFAULT, "%lu.%03d memorystatus: killing_top_process_elevated%d pid %d [%s] (%s %d) - memorystatus_available_pages: %llu\n",
5604 (unsigned long)tv_sec, tv_msec,
5605 aggr_count,
5606 aPid, (*p->p_name ? p->p_name : "unknown"),
5607 memorystatus_kill_cause_name[cause], aPid_ep, (uint64_t)memorystatus_available_pages);
5608
5609 /*
5610 * memorystatus_do_kill drops a reference, so take another one so we can
5611 * continue to use this exit reason even after memorystatus_do_kill()
5612 * returns
5613 */
5614 os_reason_ref(jetsam_reason);
5615 killed = memorystatus_do_kill(p, cause, jetsam_reason);
5616
5617 /* Success? */
5618 if (killed) {
5619 proc_rele(p);
5620 kill_count++;
5621 goto exit;
5622 }
5623
5624 /*
5625 * Failure - first unwind the state,
5626 * then fall through to restart the search.
5627 */
5628 proc_list_lock();
5629 proc_rele_locked(p);
5630 p->p_memstat_state &= ~P_MEMSTAT_TERMINATED;
5631 p->p_memstat_state |= P_MEMSTAT_ERROR;
5632 *errors += 1;
5633 }
5634
5635 /*
5636 * Failure - restart the search.
5637 *
5638 * We might have raced with "p" exiting on another core, resulting in no
5639 * ref on "p". Or, we may have failed to kill "p".
5640 *
5641 * Either way, we fall thru to here, leaving the proc in the
5642 * P_MEMSTAT_TERMINATED state or P_MEMSTAT_ERROR state.
5643 *
5644 * And, we hold the the proc_list_lock at this point.
5645 */
5646
5647 next_p = memorystatus_get_first_proc_locked(&i, FALSE);
5648 }
5649
5650 proc_list_unlock();
5651
5652 exit:
5653 os_reason_free(jetsam_reason);
5654
5655 /* Clear snapshot if freshly captured and no target was found */
5656 if (new_snapshot && (kill_count == 0)) {
5657 proc_list_lock();
5658 memorystatus_jetsam_snapshot->entry_count = memorystatus_jetsam_snapshot_count = 0;
5659 proc_list_unlock();
5660 }
5661
5662 KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_JETSAM) | DBG_FUNC_END,
5663 memorystatus_available_pages, killed ? aPid : 0, kill_count, 0, 0);
5664
5665 return (killed);
5666 }
5667
5668 static boolean_t
5669 memorystatus_kill_process_async(pid_t victim_pid, uint32_t cause) {
5670 /*
5671 * TODO: allow a general async path
5672 *
5673 * NOTE: If a new async kill cause is added, make sure to update memorystatus_thread() to
5674 * add the appropriate exit reason code mapping.
5675 */
5676 if ((victim_pid != -1) || (cause != kMemorystatusKilledVMPageShortage && cause != kMemorystatusKilledVMThrashing &&
5677 cause != kMemorystatusKilledFCThrashing && cause != kMemorystatusKilledZoneMapExhaustion)) {
5678 return FALSE;
5679 }
5680
5681 kill_under_pressure_cause = cause;
5682 memorystatus_thread_wake();
5683 return TRUE;
5684 }
5685
5686 boolean_t
5687 memorystatus_kill_on_VM_thrashing(boolean_t async) {
5688 if (async) {
5689 return memorystatus_kill_process_async(-1, kMemorystatusKilledVMThrashing);
5690 } else {
5691 os_reason_t jetsam_reason = os_reason_create(OS_REASON_JETSAM, JETSAM_REASON_MEMORY_VMTHRASHING);
5692 if (jetsam_reason == OS_REASON_NULL) {
5693 printf("memorystatus_kill_on_VM_thrashing -- sync: failed to allocate jetsam reason\n");
5694 }
5695
5696 return memorystatus_kill_process_sync(-1, kMemorystatusKilledVMThrashing, jetsam_reason);
5697 }
5698 }
5699
5700 #if CONFIG_JETSAM
5701 boolean_t
5702 memorystatus_kill_on_VM_page_shortage(boolean_t async) {
5703 if (async) {
5704 return memorystatus_kill_process_async(-1, kMemorystatusKilledVMPageShortage);
5705 } else {
5706 os_reason_t jetsam_reason = os_reason_create(OS_REASON_JETSAM, JETSAM_REASON_MEMORY_VMPAGESHORTAGE);
5707 if (jetsam_reason == OS_REASON_NULL) {
5708 printf("memorystatus_kill_on_VM_page_shortage -- sync: failed to allocate jetsam reason\n");
5709 }
5710
5711 return memorystatus_kill_process_sync(-1, kMemorystatusKilledVMPageShortage, jetsam_reason);
5712 }
5713 }
5714
5715 boolean_t
5716 memorystatus_kill_on_FC_thrashing(boolean_t async) {
5717
5718
5719 if (async) {
5720 return memorystatus_kill_process_async(-1, kMemorystatusKilledFCThrashing);
5721 } else {
5722 os_reason_t jetsam_reason = os_reason_create(OS_REASON_JETSAM, JETSAM_REASON_MEMORY_FCTHRASHING);
5723 if (jetsam_reason == OS_REASON_NULL) {
5724 printf("memorystatus_kill_on_FC_thrashing -- sync: failed to allocate jetsam reason\n");
5725 }
5726
5727 return memorystatus_kill_process_sync(-1, kMemorystatusKilledFCThrashing, jetsam_reason);
5728 }
5729 }
5730
5731 boolean_t
5732 memorystatus_kill_on_vnode_limit(void) {
5733 os_reason_t jetsam_reason = os_reason_create(OS_REASON_JETSAM, JETSAM_REASON_VNODE);
5734 if (jetsam_reason == OS_REASON_NULL) {
5735 printf("memorystatus_kill_on_vnode_limit: failed to allocate jetsam reason\n");
5736 }
5737
5738 return memorystatus_kill_process_sync(-1, kMemorystatusKilledVnodes, jetsam_reason);
5739 }
5740
5741 #endif /* CONFIG_JETSAM */
5742
5743 boolean_t
5744 memorystatus_kill_on_zone_map_exhaustion(pid_t pid) {
5745 boolean_t res = FALSE;
5746 if (pid == -1) {
5747 res = memorystatus_kill_process_async(-1, kMemorystatusKilledZoneMapExhaustion);
5748 } else {
5749 os_reason_t jetsam_reason = os_reason_create(OS_REASON_JETSAM, JETSAM_REASON_ZONE_MAP_EXHAUSTION);
5750 if (jetsam_reason == OS_REASON_NULL) {
5751 printf("memorystatus_kill_on_zone_map_exhaustion: failed to allocate jetsam reason\n");
5752 }
5753
5754 res = memorystatus_kill_process_sync(pid, kMemorystatusKilledZoneMapExhaustion, jetsam_reason);
5755 }
5756 return res;
5757 }
5758
5759 #if CONFIG_FREEZE
5760
5761 __private_extern__ void
5762 memorystatus_freeze_init(void)
5763 {
5764 kern_return_t result;
5765 thread_t thread;
5766
5767 freezer_lck_grp_attr = lck_grp_attr_alloc_init();
5768 freezer_lck_grp = lck_grp_alloc_init("freezer", freezer_lck_grp_attr);
5769
5770 lck_mtx_init(&freezer_mutex, freezer_lck_grp, NULL);
5771
5772 result = kernel_thread_start(memorystatus_freeze_thread, NULL, &thread);
5773 if (result == KERN_SUCCESS) {
5774 thread_deallocate(thread);
5775 } else {
5776 panic("Could not create memorystatus_freeze_thread");
5777 }
5778 }
5779
5780 /*
5781 * Synchronously freeze the passed proc. Called with a reference to the proc held.
5782 *
5783 * Returns EINVAL or the value returned by task_freeze().
5784 */
5785 int
5786 memorystatus_freeze_process_sync(proc_t p)
5787 {
5788 int ret = EINVAL;
5789 pid_t aPid = 0;
5790 boolean_t memorystatus_freeze_swap_low = FALSE;
5791
5792 KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_FREEZE) | DBG_FUNC_START,
5793 memorystatus_available_pages, 0, 0, 0, 0);
5794
5795 lck_mtx_lock(&freezer_mutex);
5796
5797 if (p == NULL) {
5798 goto exit;
5799 }
5800
5801 if (memorystatus_freeze_enabled == FALSE) {
5802 goto exit;
5803 }
5804
5805 if (!memorystatus_can_freeze(&memorystatus_freeze_swap_low)) {
5806 goto exit;
5807 }
5808
5809 if (memorystatus_freeze_update_throttle()) {
5810 printf("memorystatus_freeze_process_sync: in throttle, ignorning freeze\n");
5811 memorystatus_freeze_throttle_count++;
5812 goto exit;
5813 }
5814
5815 proc_list_lock();
5816
5817 if (p != NULL) {
5818 uint32_t purgeable, wired, clean, dirty, state;
5819 uint32_t max_pages, pages, i;
5820 boolean_t shared;
5821
5822 aPid = p->p_pid;
5823 state = p->p_memstat_state;
5824
5825 /* Ensure the process is eligible for freezing */
5826 if ((state & (P_MEMSTAT_TERMINATED | P_MEMSTAT_LOCKED | P_MEMSTAT_FROZEN)) || !(state & P_MEMSTAT_SUSPENDED)) {
5827 proc_list_unlock();
5828 goto exit;
5829 }
5830
5831 /* Only freeze processes meeting our minimum resident page criteria */
5832 memorystatus_get_task_page_counts(p->task, &pages, NULL, NULL, NULL);
5833 if (pages < memorystatus_freeze_pages_min) {
5834 proc_list_unlock();
5835 goto exit;
5836 }
5837
5838 if (VM_CONFIG_FREEZER_SWAP_IS_ACTIVE) {
5839
5840 unsigned int avail_swap_space = 0; /* in pages. */
5841
5842 /*
5843 * Freezer backed by the compressor and swap file(s)
5844 * while will hold compressed data.
5845 */
5846 avail_swap_space = vm_swap_get_free_space() / PAGE_SIZE_64;
5847
5848 max_pages = MIN(avail_swap_space, memorystatus_freeze_pages_max);
5849
5850 if (max_pages < memorystatus_freeze_pages_min) {
5851 proc_list_unlock();
5852 goto exit;
5853 }
5854 } else {
5855 /*
5856 * We only have the compressor without any swap.
5857 */
5858 max_pages = UINT32_MAX - 1;
5859 }
5860
5861 /* Mark as locked temporarily to avoid kill */
5862 p->p_memstat_state |= P_MEMSTAT_LOCKED;
5863 proc_list_unlock();
5864
5865 ret = task_freeze(p->task, &purgeable, &wired, &clean, &dirty, max_pages, &shared, FALSE);
5866
5867 DTRACE_MEMORYSTATUS6(memorystatus_freeze, proc_t, p, unsigned int, memorystatus_available_pages, boolean_t, purgeable, unsigned int, wired, uint32_t, clean, uint32_t, dirty);
5868
5869 MEMORYSTATUS_DEBUG(1, "memorystatus_freeze_process_sync: task_freeze %s for pid %d [%s] - "
5870 "memorystatus_pages: %d, purgeable: %d, wired: %d, clean: %d, dirty: %d, max_pages %d, shared %d\n",
5871 (ret == KERN_SUCCESS) ? "SUCCEEDED" : "FAILED", aPid, (*p->p_name ? p->p_name : "(unknown)"),
5872 memorystatus_available_pages, purgeable, wired, clean, dirty, max_pages, shared);
5873
5874 proc_list_lock();
5875 p->p_memstat_state &= ~P_MEMSTAT_LOCKED;
5876
5877 if (ret == KERN_SUCCESS) {
5878 memorystatus_freeze_entry_t data = { aPid, TRUE, dirty };
5879
5880 memorystatus_frozen_count++;
5881
5882 p->p_memstat_state |= (P_MEMSTAT_FROZEN | (shared ? 0: P_MEMSTAT_NORECLAIM));
5883
5884 if (VM_CONFIG_FREEZER_SWAP_IS_ACTIVE) {
5885 /* Update stats */
5886 for (i = 0; i < sizeof(throttle_intervals) / sizeof(struct throttle_interval_t); i++) {
5887 throttle_intervals[i].pageouts += dirty;
5888 }
5889 }
5890
5891 memorystatus_freeze_pageouts += dirty;
5892 memorystatus_freeze_count++;
5893
5894 proc_list_unlock();
5895
5896 memorystatus_send_note(kMemorystatusFreezeNote, &data, sizeof(data));
5897 } else {
5898 proc_list_unlock();
5899 }
5900 }
5901
5902 exit:
5903 lck_mtx_unlock(&freezer_mutex);
5904 KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_FREEZE) | DBG_FUNC_END,
5905 memorystatus_available_pages, aPid, 0, 0, 0);
5906
5907 return ret;
5908 }
5909
5910 static int
5911 memorystatus_freeze_top_process(boolean_t *memorystatus_freeze_swap_low)
5912 {
5913 pid_t aPid = 0;
5914 int ret = -1;
5915 proc_t p = PROC_NULL, next_p = PROC_NULL;
5916 unsigned int i = 0;
5917
5918 KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_FREEZE) | DBG_FUNC_START,
5919 memorystatus_available_pages, 0, 0, 0, 0);
5920
5921 proc_list_lock();
5922
5923 next_p = memorystatus_get_first_proc_locked(&i, TRUE);
5924 while (next_p) {
5925 kern_return_t kr;
5926 uint32_t purgeable, wired, clean, dirty;
5927 boolean_t shared;
5928 uint32_t pages;
5929 uint32_t max_pages = 0;
5930 uint32_t state;
5931
5932 p = next_p;
5933 next_p = memorystatus_get_next_proc_locked(&i, p, TRUE);
5934
5935 aPid = p->p_pid;
5936 state = p->p_memstat_state;
5937
5938 /* Ensure the process is eligible for freezing */
5939 if ((state & (P_MEMSTAT_TERMINATED | P_MEMSTAT_LOCKED | P_MEMSTAT_FROZEN)) || !(state & P_MEMSTAT_SUSPENDED)) {
5940 continue; // with lock held
5941 }
5942
5943 /* Only freeze processes meeting our minimum resident page criteria */
5944 memorystatus_get_task_page_counts(p->task, &pages, NULL, NULL, NULL);
5945 if (pages < memorystatus_freeze_pages_min) {
5946 continue; // with lock held
5947 }
5948
5949 if (VM_CONFIG_FREEZER_SWAP_IS_ACTIVE) {
5950
5951 /* Ensure there's enough free space to freeze this process. */
5952
5953 unsigned int avail_swap_space = 0; /* in pages. */
5954
5955 /*
5956 * Freezer backed by the compressor and swap file(s)
5957 * while will hold compressed data.
5958 */
5959 avail_swap_space = vm_swap_get_free_space() / PAGE_SIZE_64;
5960
5961 max_pages = MIN(avail_swap_space, memorystatus_freeze_pages_max);
5962
5963 if (max_pages < memorystatus_freeze_pages_min) {
5964 *memorystatus_freeze_swap_low = TRUE;
5965 proc_list_unlock();
5966 goto exit;
5967 }
5968 } else {
5969 /*
5970 * We only have the compressor pool.
5971 */
5972 max_pages = UINT32_MAX - 1;
5973 }
5974
5975 /* Mark as locked temporarily to avoid kill */
5976 p->p_memstat_state |= P_MEMSTAT_LOCKED;
5977
5978 p = proc_ref_locked(p);
5979 proc_list_unlock();
5980 if (!p) {
5981 goto exit;
5982 }
5983
5984 kr = task_freeze(p->task, &purgeable, &wired, &clean, &dirty, max_pages, &shared, FALSE);
5985
5986 MEMORYSTATUS_DEBUG(1, "memorystatus_freeze_top_process: task_freeze %s for pid %d [%s] - "
5987 "memorystatus_pages: %d, purgeable: %d, wired: %d, clean: %d, dirty: %d, max_pages %d, shared %d\n",
5988 (kr == KERN_SUCCESS) ? "SUCCEEDED" : "FAILED", aPid, (*p->p_name ? p->p_name : "(unknown)"),
5989 memorystatus_available_pages, purgeable, wired, clean, dirty, max_pages, shared);
5990
5991 proc_list_lock();
5992 p->p_memstat_state &= ~P_MEMSTAT_LOCKED;
5993
5994 /* Success? */
5995 if (KERN_SUCCESS == kr) {
5996 memorystatus_freeze_entry_t data = { aPid, TRUE, dirty };
5997
5998 memorystatus_frozen_count++;
5999
6000 p->p_memstat_state |= (P_MEMSTAT_FROZEN | (shared ? 0: P_MEMSTAT_NORECLAIM));
6001
6002 if (VM_CONFIG_FREEZER_SWAP_IS_ACTIVE) {
6003 /* Update stats */
6004 for (i = 0; i < sizeof(throttle_intervals) / sizeof(struct throttle_interval_t); i++) {
6005 throttle_intervals[i].pageouts += dirty;
6006 }
6007 }
6008
6009 memorystatus_freeze_pageouts += dirty;
6010 memorystatus_freeze_count++;
6011
6012 proc_list_unlock();
6013
6014 memorystatus_send_note(kMemorystatusFreezeNote, &data, sizeof(data));
6015
6016 /* Return KERN_SUCESS */
6017 ret = kr;
6018
6019 } else {
6020 proc_list_unlock();
6021 }
6022
6023 proc_rele(p);
6024 goto exit;
6025 }
6026
6027 proc_list_unlock();
6028
6029 exit:
6030 KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_FREEZE) | DBG_FUNC_END,
6031 memorystatus_available_pages, aPid, 0, 0, 0);
6032
6033 return ret;
6034 }
6035
6036 static inline boolean_t
6037 memorystatus_can_freeze_processes(void)
6038 {
6039 boolean_t ret;
6040
6041 proc_list_lock();
6042
6043 if (memorystatus_suspended_count) {
6044 uint32_t average_resident_pages, estimated_processes;
6045
6046 /* Estimate the number of suspended processes we can fit */
6047 average_resident_pages = memorystatus_suspended_footprint_total / memorystatus_suspended_count;
6048 estimated_processes = memorystatus_suspended_count +
6049 ((memorystatus_available_pages - memorystatus_available_pages_critical) / average_resident_pages);
6050
6051 /* If it's predicted that no freeze will occur, lower the threshold temporarily */
6052 if (estimated_processes <= FREEZE_SUSPENDED_THRESHOLD_DEFAULT) {
6053 memorystatus_freeze_suspended_threshold = FREEZE_SUSPENDED_THRESHOLD_LOW;
6054 } else {
6055 memorystatus_freeze_suspended_threshold = FREEZE_SUSPENDED_THRESHOLD_DEFAULT;
6056 }
6057
6058 MEMORYSTATUS_DEBUG(1, "memorystatus_can_freeze_processes: %d suspended processes, %d average resident pages / process, %d suspended processes estimated\n",
6059 memorystatus_suspended_count, average_resident_pages, estimated_processes);
6060
6061 if ((memorystatus_suspended_count - memorystatus_frozen_count) > memorystatus_freeze_suspended_threshold) {
6062 ret = TRUE;
6063 } else {
6064 ret = FALSE;
6065 }
6066 } else {
6067 ret = FALSE;
6068 }
6069
6070 proc_list_unlock();
6071
6072 return ret;
6073 }
6074
6075 static boolean_t
6076 memorystatus_can_freeze(boolean_t *memorystatus_freeze_swap_low)
6077 {
6078 boolean_t can_freeze = TRUE;
6079
6080 /* Only freeze if we're sufficiently low on memory; this holds off freeze right
6081 after boot, and is generally is a no-op once we've reached steady state. */
6082 if (memorystatus_available_pages > memorystatus_freeze_threshold) {
6083 return FALSE;
6084 }
6085
6086 /* Check minimum suspended process threshold. */
6087 if (!memorystatus_can_freeze_processes()) {
6088 return FALSE;
6089 }
6090 assert(VM_CONFIG_COMPRESSOR_IS_PRESENT);
6091
6092 if ( !VM_CONFIG_FREEZER_SWAP_IS_ACTIVE) {
6093 /*
6094 * In-core compressor used for freezing WITHOUT on-disk swap support.
6095 */
6096 if (vm_compressor_low_on_space()) {
6097 if (*memorystatus_freeze_swap_low) {
6098 *memorystatus_freeze_swap_low = TRUE;
6099 }
6100
6101 can_freeze = FALSE;
6102
6103 } else {
6104 if (*memorystatus_freeze_swap_low) {
6105 *memorystatus_freeze_swap_low = FALSE;
6106 }
6107
6108 can_freeze = TRUE;
6109 }
6110 } else {
6111 /*
6112 * Freezing WITH on-disk swap support.
6113 *
6114 * In-core compressor fronts the swap.
6115 */
6116 if (vm_swap_low_on_space()) {
6117 if (*memorystatus_freeze_swap_low) {
6118 *memorystatus_freeze_swap_low = TRUE;
6119 }
6120
6121 can_freeze = FALSE;
6122 }
6123
6124 }
6125
6126 return can_freeze;
6127 }
6128
6129 static void
6130 memorystatus_freeze_update_throttle_interval(mach_timespec_t *ts, struct throttle_interval_t *interval)
6131 {
6132 unsigned int freeze_daily_pageouts_max = memorystatus_freeze_daily_mb_max * (1024 * 1024 / PAGE_SIZE);
6133 if (CMP_MACH_TIMESPEC(ts, &interval->ts) >= 0) {
6134 if (!interval->max_pageouts) {
6135 interval->max_pageouts = (interval->burst_multiple * (((uint64_t)interval->mins * freeze_daily_pageouts_max) / (24 * 60)));
6136 } else {
6137 printf("memorystatus_freeze_update_throttle_interval: %d minute throttle timeout, resetting\n", interval->mins);
6138 }
6139 interval->ts.tv_sec = interval->mins * 60;
6140 interval->ts.tv_nsec = 0;
6141 ADD_MACH_TIMESPEC(&interval->ts, ts);
6142 /* Since we update the throttle stats pre-freeze, adjust for overshoot here */
6143 if (interval->pageouts > interval->max_pageouts) {
6144 interval->pageouts -= interval->max_pageouts;
6145 } else {
6146 interval->pageouts = 0;
6147 }
6148 interval->throttle = FALSE;
6149 } else if (!interval->throttle && interval->pageouts >= interval->max_pageouts) {
6150 printf("memorystatus_freeze_update_throttle_interval: %d minute pageout limit exceeded; enabling throttle\n", interval->mins);
6151 interval->throttle = TRUE;
6152 }
6153
6154 MEMORYSTATUS_DEBUG(1, "memorystatus_freeze_update_throttle_interval: throttle updated - %d frozen (%d max) within %dm; %dm remaining; throttle %s\n",
6155 interval->pageouts, interval->max_pageouts, interval->mins, (interval->ts.tv_sec - ts->tv_sec) / 60,
6156 interval->throttle ? "on" : "off");
6157 }
6158
6159 static boolean_t
6160 memorystatus_freeze_update_throttle(void)
6161 {
6162 clock_sec_t sec;
6163 clock_nsec_t nsec;
6164 mach_timespec_t ts;
6165 uint32_t i;
6166 boolean_t throttled = FALSE;
6167
6168 #if DEVELOPMENT || DEBUG
6169 if (!memorystatus_freeze_throttle_enabled)
6170 return FALSE;
6171 #endif
6172
6173 clock_get_system_nanotime(&sec, &nsec);
6174 ts.tv_sec = sec;
6175 ts.tv_nsec = nsec;
6176
6177 /* Check freeze pageouts over multiple intervals and throttle if we've exceeded our budget.
6178 *
6179 * This ensures that periods of inactivity can't be used as 'credit' towards freeze if the device has
6180 * remained dormant for a long period. We do, however, allow increased thresholds for shorter intervals in
6181 * order to allow for bursts of activity.
6182 */
6183 for (i = 0; i < sizeof(throttle_intervals) / sizeof(struct throttle_interval_t); i++) {
6184 memorystatus_freeze_update_throttle_interval(&ts, &throttle_intervals[i]);
6185 if (throttle_intervals[i].throttle == TRUE)
6186 throttled = TRUE;
6187 }
6188
6189 return throttled;
6190 }
6191
6192 static void
6193 memorystatus_freeze_thread(void *param __unused, wait_result_t wr __unused)
6194 {
6195 static boolean_t memorystatus_freeze_swap_low = FALSE;
6196
6197 lck_mtx_lock(&freezer_mutex);
6198 if (memorystatus_freeze_enabled) {
6199 if (memorystatus_can_freeze(&memorystatus_freeze_swap_low)) {
6200 /* Only freeze if we've not exceeded our pageout budgets.*/
6201 if (!memorystatus_freeze_update_throttle()) {
6202 memorystatus_freeze_top_process(&memorystatus_freeze_swap_low);
6203 } else {
6204 printf("memorystatus_freeze_thread: in throttle, ignoring freeze\n");
6205 memorystatus_freeze_throttle_count++; /* Throttled, update stats */
6206 }
6207 }
6208 }
6209 lck_mtx_unlock(&freezer_mutex);
6210
6211 assert_wait((event_t) &memorystatus_freeze_wakeup, THREAD_UNINT);
6212 thread_block((thread_continue_t) memorystatus_freeze_thread);
6213 }
6214
6215 static int
6216 sysctl_memorystatus_do_fastwake_warmup_all SYSCTL_HANDLER_ARGS
6217 {
6218 #pragma unused(oidp, req, arg1, arg2)
6219
6220 /* Need to be root or have entitlement */
6221 if (!kauth_cred_issuser(kauth_cred_get()) && !IOTaskHasEntitlement(current_task(), MEMORYSTATUS_ENTITLEMENT)) {
6222 return EPERM;
6223 }
6224
6225 if (memorystatus_freeze_enabled == FALSE) {
6226 return ENOTSUP;
6227 }
6228
6229 do_fastwake_warmup_all();
6230
6231 return 0;
6232 }
6233
6234 SYSCTL_PROC(_kern, OID_AUTO, memorystatus_do_fastwake_warmup_all, CTLTYPE_INT|CTLFLAG_WR|CTLFLAG_LOCKED|CTLFLAG_MASKED,
6235 0, 0, &sysctl_memorystatus_do_fastwake_warmup_all, "I", "");
6236
6237 #endif /* CONFIG_FREEZE */
6238
6239 #if VM_PRESSURE_EVENTS
6240
6241 #if CONFIG_MEMORYSTATUS
6242
6243 static int
6244 memorystatus_send_note(int event_code, void *data, size_t data_length) {
6245 int ret;
6246 struct kev_msg ev_msg;
6247
6248 ev_msg.vendor_code = KEV_VENDOR_APPLE;
6249 ev_msg.kev_class = KEV_SYSTEM_CLASS;
6250 ev_msg.kev_subclass = KEV_MEMORYSTATUS_SUBCLASS;
6251
6252 ev_msg.event_code = event_code;
6253
6254 ev_msg.dv[0].data_length = data_length;
6255 ev_msg.dv[0].data_ptr = data;
6256 ev_msg.dv[1].data_length = 0;
6257
6258 ret = kev_post_msg(&ev_msg);
6259 if (ret) {
6260 printf("%s: kev_post_msg() failed, err %d\n", __func__, ret);
6261 }
6262
6263 return ret;
6264 }
6265
6266 boolean_t
6267 memorystatus_warn_process(pid_t pid, __unused boolean_t is_active, __unused boolean_t is_fatal, boolean_t limit_exceeded) {
6268
6269 boolean_t ret = FALSE;
6270 boolean_t found_knote = FALSE;
6271 struct knote *kn = NULL;
6272 int send_knote_count = 0;
6273
6274 /*
6275 * See comment in sysctl_memorystatus_vm_pressure_send.
6276 */
6277
6278 memorystatus_klist_lock();
6279
6280 SLIST_FOREACH(kn, &memorystatus_klist, kn_selnext) {
6281 proc_t knote_proc = knote_get_kq(kn)->kq_p;
6282 pid_t knote_pid = knote_proc->p_pid;
6283
6284 if (knote_pid == pid) {
6285 /*
6286 * By setting the "fflags" here, we are forcing
6287 * a process to deal with the case where it's
6288 * bumping up into its memory limits. If we don't
6289 * do this here, we will end up depending on the
6290 * system pressure snapshot evaluation in
6291 * filt_memorystatus().
6292 */
6293
6294 #if CONFIG_EMBEDDED
6295 if (!limit_exceeded) {
6296 /*
6297 * Intentionally set either the unambiguous limit warning,
6298 * the system-wide critical or the system-wide warning
6299 * notification bit.
6300 */
6301
6302 if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_WARN) {
6303 kn->kn_fflags = NOTE_MEMORYSTATUS_PROC_LIMIT_WARN;
6304 found_knote = TRUE;
6305 send_knote_count++;
6306 } else if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PRESSURE_CRITICAL) {
6307 kn->kn_fflags = NOTE_MEMORYSTATUS_PRESSURE_CRITICAL;
6308 found_knote = TRUE;
6309 send_knote_count++;
6310 } else if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PRESSURE_WARN) {
6311 kn->kn_fflags = NOTE_MEMORYSTATUS_PRESSURE_WARN;
6312 found_knote = TRUE;
6313 send_knote_count++;
6314 }
6315 } else {
6316 /*
6317 * Send this notification when a process has exceeded a soft limit.
6318 */
6319 if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL) {
6320 kn->kn_fflags = NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL;
6321 found_knote = TRUE;
6322 send_knote_count++;
6323 }
6324 }
6325 #else /* CONFIG_EMBEDDED */
6326 if (!limit_exceeded) {
6327
6328 /*
6329 * Processes on desktop are not expecting to handle a system-wide
6330 * critical or system-wide warning notification from this path.
6331 * Intentionally set only the unambiguous limit warning here.
6332 *
6333 * If the limit is soft, however, limit this to one notification per
6334 * active/inactive limit (per each registered listener).
6335 */
6336
6337 if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_WARN) {
6338 found_knote=TRUE;
6339 if (!is_fatal) {
6340 /*
6341 * Restrict proc_limit_warn notifications when
6342 * non-fatal (soft) limit is at play.
6343 */
6344 if (is_active) {
6345 if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_WARN_ACTIVE) {
6346 /*
6347 * Mark this knote for delivery.
6348 */
6349 kn->kn_fflags = NOTE_MEMORYSTATUS_PROC_LIMIT_WARN;
6350 /*
6351 * And suppress it from future notifications.
6352 */
6353 kn->kn_sfflags &= ~NOTE_MEMORYSTATUS_PROC_LIMIT_WARN_ACTIVE;
6354 send_knote_count++;
6355 }
6356 } else {
6357 if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_WARN_INACTIVE) {
6358 /*
6359 * Mark this knote for delivery.
6360 */
6361 kn->kn_fflags = NOTE_MEMORYSTATUS_PROC_LIMIT_WARN;
6362 /*
6363 * And suppress it from future notifications.
6364 */
6365 kn->kn_sfflags &= ~NOTE_MEMORYSTATUS_PROC_LIMIT_WARN_INACTIVE;
6366 send_knote_count++;
6367 }
6368 }
6369 } else {
6370 /*
6371 * No restriction on proc_limit_warn notifications when
6372 * fatal (hard) limit is at play.
6373 */
6374 kn->kn_fflags = NOTE_MEMORYSTATUS_PROC_LIMIT_WARN;
6375 send_knote_count++;
6376 }
6377 }
6378 } else {
6379 /*
6380 * Send this notification when a process has exceeded a soft limit,
6381 */
6382
6383 if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL) {
6384 found_knote = TRUE;
6385 if (!is_fatal) {
6386 /*
6387 * Restrict critical notifications for soft limits.
6388 */
6389
6390 if (is_active) {
6391 if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL_ACTIVE) {
6392 /*
6393 * Suppress future proc_limit_critical notifications
6394 * for the active soft limit.
6395 */
6396 kn->kn_sfflags &= ~NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL_ACTIVE;
6397 kn->kn_fflags = NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL;
6398 send_knote_count++;
6399
6400 }
6401 } else {
6402 if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL_INACTIVE) {
6403 /*
6404 * Suppress future proc_limit_critical_notifications
6405 * for the inactive soft limit.
6406 */
6407 kn->kn_sfflags &= ~NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL_INACTIVE;
6408 kn->kn_fflags = NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL;
6409 send_knote_count++;
6410 }
6411 }
6412 } else {
6413 /*
6414 * We should never be trying to send a critical notification for
6415 * a hard limit... the process would be killed before it could be
6416 * received.
6417 */
6418 panic("Caught sending pid %d a critical warning for a fatal limit.\n", pid);
6419 }
6420 }
6421 }
6422 #endif /* CONFIG_EMBEDDED */
6423 }
6424 }
6425
6426 if (found_knote) {
6427 if (send_knote_count > 0) {
6428 KNOTE(&memorystatus_klist, 0);
6429 }
6430 ret = TRUE;
6431 }
6432
6433 memorystatus_klist_unlock();
6434
6435 return ret;
6436 }
6437
6438 /*
6439 * Can only be set by the current task on itself.
6440 */
6441 int
6442 memorystatus_low_mem_privileged_listener(uint32_t op_flags)
6443 {
6444 boolean_t set_privilege = FALSE;
6445 /*
6446 * Need an entitlement check here?
6447 */
6448 if (op_flags == MEMORYSTATUS_CMD_PRIVILEGED_LISTENER_ENABLE) {
6449 set_privilege = TRUE;
6450 } else if (op_flags == MEMORYSTATUS_CMD_PRIVILEGED_LISTENER_DISABLE) {
6451 set_privilege = FALSE;
6452 } else {
6453 return EINVAL;
6454 }
6455
6456 return (task_low_mem_privileged_listener(current_task(), set_privilege, NULL));
6457 }
6458
6459 int
6460 memorystatus_send_pressure_note(pid_t pid) {
6461 MEMORYSTATUS_DEBUG(1, "memorystatus_send_pressure_note(): pid %d\n", pid);
6462 return memorystatus_send_note(kMemorystatusPressureNote, &pid, sizeof(pid));
6463 }
6464
6465 void
6466 memorystatus_send_low_swap_note(void) {
6467
6468 struct knote *kn = NULL;
6469
6470 memorystatus_klist_lock();
6471 SLIST_FOREACH(kn, &memorystatus_klist, kn_selnext) {
6472 /* We call is_knote_registered_modify_task_pressure_bits to check if the sfflags for the
6473 * current note contain NOTE_MEMORYSTATUS_LOW_SWAP. Once we find one note in the memorystatus_klist
6474 * that has the NOTE_MEMORYSTATUS_LOW_SWAP flags in its sfflags set, we call KNOTE with
6475 * kMemoryStatusLowSwap as the hint to process and update all knotes on the memorystatus_klist accordingly. */
6476 if (is_knote_registered_modify_task_pressure_bits(kn, NOTE_MEMORYSTATUS_LOW_SWAP, NULL, 0, 0) == TRUE) {
6477 KNOTE(&memorystatus_klist, kMemorystatusLowSwap);
6478 break;
6479 }
6480 }
6481
6482 memorystatus_klist_unlock();
6483 }
6484
6485 boolean_t
6486 memorystatus_bg_pressure_eligible(proc_t p) {
6487 boolean_t eligible = FALSE;
6488
6489 proc_list_lock();
6490
6491 MEMORYSTATUS_DEBUG(1, "memorystatus_bg_pressure_eligible: pid %d, state 0x%x\n", p->p_pid, p->p_memstat_state);
6492
6493 /* Foreground processes have already been dealt with at this point, so just test for eligibility */
6494 if (!(p->p_memstat_state & (P_MEMSTAT_TERMINATED | P_MEMSTAT_LOCKED | P_MEMSTAT_SUSPENDED | P_MEMSTAT_FROZEN))) {
6495 eligible = TRUE;
6496 }
6497
6498 proc_list_unlock();
6499
6500 return eligible;
6501 }
6502
6503 boolean_t
6504 memorystatus_is_foreground_locked(proc_t p) {
6505 return ((p->p_memstat_effectivepriority == JETSAM_PRIORITY_FOREGROUND) ||
6506 (p->p_memstat_effectivepriority == JETSAM_PRIORITY_FOREGROUND_SUPPORT));
6507 }
6508
6509 /*
6510 * This is meant for stackshot and kperf -- it does not take the proc_list_lock
6511 * to access the p_memstat_dirty field.
6512 */
6513 boolean_t
6514 memorystatus_proc_is_dirty_unsafe(void *v)
6515 {
6516 if (!v) {
6517 return FALSE;
6518 }
6519 proc_t p = (proc_t)v;
6520 return (p->p_memstat_dirty & P_DIRTY_IS_DIRTY) != 0;
6521 }
6522
6523 #endif /* CONFIG_MEMORYSTATUS */
6524
6525 /*
6526 * Trigger levels to test the mechanism.
6527 * Can be used via a sysctl.
6528 */
6529 #define TEST_LOW_MEMORY_TRIGGER_ONE 1
6530 #define TEST_LOW_MEMORY_TRIGGER_ALL 2
6531 #define TEST_PURGEABLE_TRIGGER_ONE 3
6532 #define TEST_PURGEABLE_TRIGGER_ALL 4
6533 #define TEST_LOW_MEMORY_PURGEABLE_TRIGGER_ONE 5
6534 #define TEST_LOW_MEMORY_PURGEABLE_TRIGGER_ALL 6
6535
6536 boolean_t memorystatus_manual_testing_on = FALSE;
6537 vm_pressure_level_t memorystatus_manual_testing_level = kVMPressureNormal;
6538
6539 extern struct knote *
6540 vm_pressure_select_optimal_candidate_to_notify(struct klist *, int, boolean_t);
6541
6542 /*
6543 * This value is the threshold that a process must meet to be considered for scavenging.
6544 */
6545 #if CONFIG_EMBEDDED
6546 #define VM_PRESSURE_MINIMUM_RSIZE 1 /* MB */
6547 #else /* CONFIG_EMBEDDED */
6548 #define VM_PRESSURE_MINIMUM_RSIZE 10 /* MB */
6549 #endif /* CONFIG_EMBEDDED */
6550
6551 #define VM_PRESSURE_NOTIFY_WAIT_PERIOD 10000 /* milliseconds */
6552
6553 #if DEBUG
6554 #define VM_PRESSURE_DEBUG(cond, format, ...) \
6555 do { \
6556 if (cond) { printf(format, ##__VA_ARGS__); } \
6557 } while(0)
6558 #else
6559 #define VM_PRESSURE_DEBUG(cond, format, ...)
6560 #endif
6561
6562 #define INTER_NOTIFICATION_DELAY (250000) /* .25 second */
6563
6564 void memorystatus_on_pageout_scan_end(void) {
6565 /* No-op */
6566 }
6567
6568 /*
6569 * kn_max - knote
6570 *
6571 * knote_pressure_level - to check if the knote is registered for this notification level.
6572 *
6573 * task - task whose bits we'll be modifying
6574 *
6575 * pressure_level_to_clear - if the task has been notified of this past level, clear that notification bit so that if/when we revert to that level, the task will be notified again.
6576 *
6577 * pressure_level_to_set - the task is about to be notified of this new level. Update the task's bit notification information appropriately.
6578 *
6579 */
6580
6581 boolean_t
6582 is_knote_registered_modify_task_pressure_bits(struct knote *kn_max, int knote_pressure_level, task_t task, vm_pressure_level_t pressure_level_to_clear, vm_pressure_level_t pressure_level_to_set)
6583 {
6584 if (kn_max->kn_sfflags & knote_pressure_level) {
6585
6586 if (pressure_level_to_clear && task_has_been_notified(task, pressure_level_to_clear) == TRUE) {
6587
6588 task_clear_has_been_notified(task, pressure_level_to_clear);
6589 }
6590
6591 task_mark_has_been_notified(task, pressure_level_to_set);
6592 return TRUE;
6593 }
6594
6595 return FALSE;
6596 }
6597
6598 void
6599 memorystatus_klist_reset_all_for_level(vm_pressure_level_t pressure_level_to_clear)
6600 {
6601 struct knote *kn = NULL;
6602
6603 memorystatus_klist_lock();
6604 SLIST_FOREACH(kn, &memorystatus_klist, kn_selnext) {
6605
6606 proc_t p = PROC_NULL;
6607 struct task* t = TASK_NULL;
6608
6609 p = knote_get_kq(kn)->kq_p;
6610 proc_list_lock();
6611 if (p != proc_ref_locked(p)) {
6612 p = PROC_NULL;
6613 proc_list_unlock();
6614 continue;
6615 }
6616 proc_list_unlock();
6617
6618 t = (struct task *)(p->task);
6619
6620 task_clear_has_been_notified(t, pressure_level_to_clear);
6621
6622 proc_rele(p);
6623 }
6624
6625 memorystatus_klist_unlock();
6626 }
6627
6628 extern kern_return_t vm_pressure_notify_dispatch_vm_clients(boolean_t target_foreground_process);
6629
6630 struct knote *
6631 vm_pressure_select_optimal_candidate_to_notify(struct klist *candidate_list, int level, boolean_t target_foreground_process);
6632
6633 /*
6634 * Used by the vm_pressure_thread which is
6635 * signalled from within vm_pageout_scan().
6636 */
6637 static void vm_dispatch_memory_pressure(void);
6638 void consider_vm_pressure_events(void);
6639
6640 void consider_vm_pressure_events(void)
6641 {
6642 vm_dispatch_memory_pressure();
6643 }
6644 static void vm_dispatch_memory_pressure(void)
6645 {
6646 memorystatus_update_vm_pressure(FALSE);
6647 }
6648
6649 extern vm_pressure_level_t
6650 convert_internal_pressure_level_to_dispatch_level(vm_pressure_level_t);
6651
6652 struct knote *
6653 vm_pressure_select_optimal_candidate_to_notify(struct klist *candidate_list, int level, boolean_t target_foreground_process)
6654 {
6655 struct knote *kn = NULL, *kn_max = NULL;
6656 uint64_t resident_max = 0; /* MB */
6657 struct timeval curr_tstamp = {0, 0};
6658 int elapsed_msecs = 0;
6659 int selected_task_importance = 0;
6660 static int pressure_snapshot = -1;
6661 boolean_t pressure_increase = FALSE;
6662
6663 if (pressure_snapshot == -1) {
6664 /*
6665 * Initial snapshot.
6666 */
6667 pressure_snapshot = level;
6668 pressure_increase = TRUE;
6669 } else {
6670
6671 if (level && (level >= pressure_snapshot)) {
6672 pressure_increase = TRUE;
6673 } else {
6674 pressure_increase = FALSE;
6675 }
6676
6677 pressure_snapshot = level;
6678 }
6679
6680 if (pressure_increase == TRUE) {
6681 /*
6682 * We'll start by considering the largest
6683 * unimportant task in our list.
6684 */
6685 selected_task_importance = INT_MAX;
6686 } else {
6687 /*
6688 * We'll start by considering the largest
6689 * important task in our list.
6690 */
6691 selected_task_importance = 0;
6692 }
6693
6694 microuptime(&curr_tstamp);
6695
6696 SLIST_FOREACH(kn, candidate_list, kn_selnext) {
6697
6698 uint64_t resident_size = 0; /* MB */
6699 proc_t p = PROC_NULL;
6700 struct task* t = TASK_NULL;
6701 int curr_task_importance = 0;
6702 boolean_t consider_knote = FALSE;
6703 boolean_t privileged_listener = FALSE;
6704
6705 p = knote_get_kq(kn)->kq_p;
6706 proc_list_lock();
6707 if (p != proc_ref_locked(p)) {
6708 p = PROC_NULL;
6709 proc_list_unlock();
6710 continue;
6711 }
6712 proc_list_unlock();
6713
6714 #if CONFIG_MEMORYSTATUS
6715 if (target_foreground_process == TRUE && !memorystatus_is_foreground_locked(p)) {
6716 /*
6717 * Skip process not marked foreground.
6718 */
6719 proc_rele(p);
6720 continue;
6721 }
6722 #endif /* CONFIG_MEMORYSTATUS */
6723
6724 t = (struct task *)(p->task);
6725
6726 timevalsub(&curr_tstamp, &p->vm_pressure_last_notify_tstamp);
6727 elapsed_msecs = curr_tstamp.tv_sec * 1000 + curr_tstamp.tv_usec / 1000;
6728
6729 vm_pressure_level_t dispatch_level = convert_internal_pressure_level_to_dispatch_level(level);
6730
6731 if ((kn->kn_sfflags & dispatch_level) == 0) {
6732 proc_rele(p);
6733 continue;
6734 }
6735
6736 #if CONFIG_MEMORYSTATUS
6737 if (target_foreground_process == FALSE && !memorystatus_bg_pressure_eligible(p)) {
6738 VM_PRESSURE_DEBUG(1, "[vm_pressure] skipping process %d\n", p->p_pid);
6739 proc_rele(p);
6740 continue;
6741 }
6742 #endif /* CONFIG_MEMORYSTATUS */
6743
6744 #if CONFIG_EMBEDDED
6745 curr_task_importance = p->p_memstat_effectivepriority;
6746 #else /* CONFIG_EMBEDDED */
6747 curr_task_importance = task_importance_estimate(t);
6748 #endif /* CONFIG_EMBEDDED */
6749
6750 /*
6751 * Privileged listeners are only considered in the multi-level pressure scheme
6752 * AND only if the pressure is increasing.
6753 */
6754 if (level > 0) {
6755
6756 if (task_has_been_notified(t, level) == FALSE) {
6757
6758 /*
6759 * Is this a privileged listener?
6760 */
6761 if (task_low_mem_privileged_listener(t, FALSE, &privileged_listener) == 0) {
6762
6763 if (privileged_listener) {
6764 kn_max = kn;
6765 proc_rele(p);
6766 goto done_scanning;
6767 }
6768 }
6769 } else {
6770 proc_rele(p);
6771 continue;
6772 }
6773 } else if (level == 0) {
6774
6775 /*
6776 * Task wasn't notified when the pressure was increasing and so
6777 * no need to notify it that the pressure is decreasing.
6778 */
6779 if ((task_has_been_notified(t, kVMPressureWarning) == FALSE) && (task_has_been_notified(t, kVMPressureCritical) == FALSE)) {
6780 proc_rele(p);
6781 continue;
6782 }
6783 }
6784
6785 /*
6786 * We don't want a small process to block large processes from
6787 * being notified again. <rdar://problem/7955532>
6788 */
6789 resident_size = (get_task_phys_footprint(t))/(1024*1024ULL); /* MB */
6790
6791 if (resident_size >= VM_PRESSURE_MINIMUM_RSIZE) {
6792
6793 if (level > 0) {
6794 /*
6795 * Warning or Critical Pressure.
6796 */
6797 if (pressure_increase) {
6798 if ((curr_task_importance < selected_task_importance) ||
6799 ((curr_task_importance == selected_task_importance) && (resident_size > resident_max))) {
6800
6801 /*
6802 * We have found a candidate process which is:
6803 * a) at a lower importance than the current selected process
6804 * OR
6805 * b) has importance equal to that of the current selected process but is larger
6806 */
6807
6808 consider_knote = TRUE;
6809 }
6810 } else {
6811 if ((curr_task_importance > selected_task_importance) ||
6812 ((curr_task_importance == selected_task_importance) && (resident_size > resident_max))) {
6813
6814 /*
6815 * We have found a candidate process which is:
6816 * a) at a higher importance than the current selected process
6817 * OR
6818 * b) has importance equal to that of the current selected process but is larger
6819 */
6820
6821 consider_knote = TRUE;
6822 }
6823 }
6824 } else if (level == 0) {
6825 /*
6826 * Pressure back to normal.
6827 */
6828 if ((curr_task_importance > selected_task_importance) ||
6829 ((curr_task_importance == selected_task_importance) && (resident_size > resident_max))) {
6830
6831 consider_knote = TRUE;
6832 }
6833 }
6834
6835 if (consider_knote) {
6836 resident_max = resident_size;
6837 kn_max = kn;
6838 selected_task_importance = curr_task_importance;
6839 consider_knote = FALSE; /* reset for the next candidate */
6840 }
6841 } else {
6842 /* There was no candidate with enough resident memory to scavenge */
6843 VM_PRESSURE_DEBUG(0, "[vm_pressure] threshold failed for pid %d with %llu resident...\n", p->p_pid, resident_size);
6844 }
6845 proc_rele(p);
6846 }
6847
6848 done_scanning:
6849 if (kn_max) {
6850 VM_DEBUG_CONSTANT_EVENT(vm_pressure_event, VM_PRESSURE_EVENT, DBG_FUNC_NONE, knote_get_kq(kn_max)->kq_p->p_pid, resident_max, 0, 0);
6851 VM_PRESSURE_DEBUG(1, "[vm_pressure] sending event to pid %d with %llu resident\n", knote_get_kq(kn_max)->kq_p->p_pid, resident_max);
6852 }
6853
6854 return kn_max;
6855 }
6856
6857 #define VM_PRESSURE_DECREASED_SMOOTHING_PERIOD 5000 /* milliseconds */
6858 #define WARNING_NOTIFICATION_RESTING_PERIOD 25 /* seconds */
6859 #define CRITICAL_NOTIFICATION_RESTING_PERIOD 25 /* seconds */
6860
6861 uint64_t next_warning_notification_sent_at_ts = 0;
6862 uint64_t next_critical_notification_sent_at_ts = 0;
6863
6864 kern_return_t
6865 memorystatus_update_vm_pressure(boolean_t target_foreground_process)
6866 {
6867 struct knote *kn_max = NULL;
6868 struct knote *kn_cur = NULL, *kn_temp = NULL; /* for safe list traversal */
6869 pid_t target_pid = -1;
6870 struct klist dispatch_klist = { NULL };
6871 proc_t target_proc = PROC_NULL;
6872 struct task *task = NULL;
6873 boolean_t found_candidate = FALSE;
6874
6875 static vm_pressure_level_t level_snapshot = kVMPressureNormal;
6876 static vm_pressure_level_t prev_level_snapshot = kVMPressureNormal;
6877 boolean_t smoothing_window_started = FALSE;
6878 struct timeval smoothing_window_start_tstamp = {0, 0};
6879 struct timeval curr_tstamp = {0, 0};
6880 int elapsed_msecs = 0;
6881 uint64_t curr_ts = mach_absolute_time();
6882
6883 #if !CONFIG_JETSAM
6884 #define MAX_IDLE_KILLS 100 /* limit the number of idle kills allowed */
6885
6886 int idle_kill_counter = 0;
6887
6888 /*
6889 * On desktop we take this opportunity to free up memory pressure
6890 * by immediately killing idle exitable processes. We use a delay
6891 * to avoid overkill. And we impose a max counter as a fail safe
6892 * in case daemons re-launch too fast.
6893 */
6894 while ((memorystatus_vm_pressure_level != kVMPressureNormal) && (idle_kill_counter < MAX_IDLE_KILLS)) {
6895 if (memorystatus_idle_exit_from_VM() == FALSE) {
6896 /* No idle exitable processes left to kill */
6897 break;
6898 }
6899 idle_kill_counter++;
6900
6901 if (memorystatus_manual_testing_on == TRUE) {
6902 /*
6903 * Skip the delay when testing
6904 * the pressure notification scheme.
6905 */
6906 } else {
6907 delay(1000000); /* 1 second */
6908 }
6909 }
6910 #endif /* !CONFIG_JETSAM */
6911
6912 if (level_snapshot != kVMPressureNormal) {
6913
6914 /*
6915 * Check to see if we are still in the 'resting' period
6916 * after having notified all clients interested in
6917 * a particular pressure level.
6918 */
6919
6920 level_snapshot = memorystatus_vm_pressure_level;
6921
6922 if (level_snapshot == kVMPressureWarning || level_snapshot == kVMPressureUrgent) {
6923
6924 if (next_warning_notification_sent_at_ts) {
6925 if (curr_ts < next_warning_notification_sent_at_ts) {
6926 delay(INTER_NOTIFICATION_DELAY * 4 /* 1 sec */);
6927 return KERN_SUCCESS;
6928 }
6929
6930 next_warning_notification_sent_at_ts = 0;
6931 memorystatus_klist_reset_all_for_level(kVMPressureWarning);
6932 }
6933 } else if (level_snapshot == kVMPressureCritical) {
6934
6935 if (next_critical_notification_sent_at_ts) {
6936 if (curr_ts < next_critical_notification_sent_at_ts) {
6937 delay(INTER_NOTIFICATION_DELAY * 4 /* 1 sec */);
6938 return KERN_SUCCESS;
6939 }
6940 next_critical_notification_sent_at_ts = 0;
6941 memorystatus_klist_reset_all_for_level(kVMPressureCritical);
6942 }
6943 }
6944 }
6945
6946 while (1) {
6947
6948 /*
6949 * There is a race window here. But it's not clear
6950 * how much we benefit from having extra synchronization.
6951 */
6952 level_snapshot = memorystatus_vm_pressure_level;
6953
6954 if (prev_level_snapshot > level_snapshot) {
6955 /*
6956 * Pressure decreased? Let's take a little breather
6957 * and see if this condition stays.
6958 */
6959 if (smoothing_window_started == FALSE) {
6960
6961 smoothing_window_started = TRUE;
6962 microuptime(&smoothing_window_start_tstamp);
6963 }
6964
6965 microuptime(&curr_tstamp);
6966 timevalsub(&curr_tstamp, &smoothing_window_start_tstamp);
6967 elapsed_msecs = curr_tstamp.tv_sec * 1000 + curr_tstamp.tv_usec / 1000;
6968
6969 if (elapsed_msecs < VM_PRESSURE_DECREASED_SMOOTHING_PERIOD) {
6970
6971 delay(INTER_NOTIFICATION_DELAY);
6972 continue;
6973 }
6974 }
6975
6976 prev_level_snapshot = level_snapshot;
6977 smoothing_window_started = FALSE;
6978
6979 memorystatus_klist_lock();
6980 kn_max = vm_pressure_select_optimal_candidate_to_notify(&memorystatus_klist, level_snapshot, target_foreground_process);
6981
6982 if (kn_max == NULL) {
6983 memorystatus_klist_unlock();
6984
6985 /*
6986 * No more level-based clients to notify.
6987 *
6988 * Start the 'resting' window within which clients will not be re-notified.
6989 */
6990
6991 if (level_snapshot != kVMPressureNormal) {
6992 if (level_snapshot == kVMPressureWarning || level_snapshot == kVMPressureUrgent) {
6993 nanoseconds_to_absolutetime(WARNING_NOTIFICATION_RESTING_PERIOD * NSEC_PER_SEC, &curr_ts);
6994
6995 /* Next warning notification (if nothing changes) won't be sent before...*/
6996 next_warning_notification_sent_at_ts = mach_absolute_time() + curr_ts;
6997 }
6998
6999 if (level_snapshot == kVMPressureCritical) {
7000 nanoseconds_to_absolutetime(CRITICAL_NOTIFICATION_RESTING_PERIOD * NSEC_PER_SEC, &curr_ts);
7001
7002 /* Next critical notification (if nothing changes) won't be sent before...*/
7003 next_critical_notification_sent_at_ts = mach_absolute_time() + curr_ts;
7004 }
7005 }
7006 return KERN_FAILURE;
7007 }
7008
7009 target_proc = knote_get_kq(kn_max)->kq_p;
7010
7011 proc_list_lock();
7012 if (target_proc != proc_ref_locked(target_proc)) {
7013 target_proc = PROC_NULL;
7014 proc_list_unlock();
7015 memorystatus_klist_unlock();
7016 continue;
7017 }
7018 proc_list_unlock();
7019
7020 target_pid = target_proc->p_pid;
7021
7022 task = (struct task *)(target_proc->task);
7023
7024 if (level_snapshot != kVMPressureNormal) {
7025
7026 if (level_snapshot == kVMPressureWarning || level_snapshot == kVMPressureUrgent) {
7027
7028 if (is_knote_registered_modify_task_pressure_bits(kn_max, NOTE_MEMORYSTATUS_PRESSURE_WARN, task, 0, kVMPressureWarning) == TRUE) {
7029 found_candidate = TRUE;
7030 }
7031 } else {
7032 if (level_snapshot == kVMPressureCritical) {
7033
7034 if (is_knote_registered_modify_task_pressure_bits(kn_max, NOTE_MEMORYSTATUS_PRESSURE_CRITICAL, task, 0, kVMPressureCritical) == TRUE) {
7035 found_candidate = TRUE;
7036 }
7037 }
7038 }
7039 } else {
7040 if (kn_max->kn_sfflags & NOTE_MEMORYSTATUS_PRESSURE_NORMAL) {
7041
7042 task_clear_has_been_notified(task, kVMPressureWarning);
7043 task_clear_has_been_notified(task, kVMPressureCritical);
7044
7045 found_candidate = TRUE;
7046 }
7047 }
7048
7049 if (found_candidate == FALSE) {
7050 proc_rele(target_proc);
7051 memorystatus_klist_unlock();
7052 continue;
7053 }
7054
7055 SLIST_FOREACH_SAFE(kn_cur, &memorystatus_klist, kn_selnext, kn_temp) {
7056
7057 int knote_pressure_level = convert_internal_pressure_level_to_dispatch_level(level_snapshot);
7058
7059 if (is_knote_registered_modify_task_pressure_bits(kn_cur, knote_pressure_level, task, 0, level_snapshot) == TRUE) {
7060 proc_t knote_proc = knote_get_kq(kn_cur)->kq_p;
7061 pid_t knote_pid = knote_proc->p_pid;
7062 if (knote_pid == target_pid) {
7063 KNOTE_DETACH(&memorystatus_klist, kn_cur);
7064 KNOTE_ATTACH(&dispatch_klist, kn_cur);
7065 }
7066 }
7067 }
7068
7069 KNOTE(&dispatch_klist, (level_snapshot != kVMPressureNormal) ? kMemorystatusPressure : kMemorystatusNoPressure);
7070
7071 SLIST_FOREACH_SAFE(kn_cur, &dispatch_klist, kn_selnext, kn_temp) {
7072 KNOTE_DETACH(&dispatch_klist, kn_cur);
7073 KNOTE_ATTACH(&memorystatus_klist, kn_cur);
7074 }
7075
7076 memorystatus_klist_unlock();
7077
7078 microuptime(&target_proc->vm_pressure_last_notify_tstamp);
7079 proc_rele(target_proc);
7080
7081 if (memorystatus_manual_testing_on == TRUE && target_foreground_process == TRUE) {
7082 break;
7083 }
7084
7085 if (memorystatus_manual_testing_on == TRUE) {
7086 /*
7087 * Testing out the pressure notification scheme.
7088 * No need for delays etc.
7089 */
7090 } else {
7091
7092 uint32_t sleep_interval = INTER_NOTIFICATION_DELAY;
7093 #if CONFIG_JETSAM
7094 unsigned int page_delta = 0;
7095 unsigned int skip_delay_page_threshold = 0;
7096
7097 assert(memorystatus_available_pages_pressure >= memorystatus_available_pages_critical_base);
7098
7099 page_delta = (memorystatus_available_pages_pressure - memorystatus_available_pages_critical_base) / 2;
7100 skip_delay_page_threshold = memorystatus_available_pages_pressure - page_delta;
7101
7102 if (memorystatus_available_pages <= skip_delay_page_threshold) {
7103 /*
7104 * We are nearing the critcal mark fast and can't afford to wait between
7105 * notifications.
7106 */
7107 sleep_interval = 0;
7108 }
7109 #endif /* CONFIG_JETSAM */
7110
7111 if (sleep_interval) {
7112 delay(sleep_interval);
7113 }
7114 }
7115 }
7116
7117 return KERN_SUCCESS;
7118 }
7119
7120 vm_pressure_level_t
7121 convert_internal_pressure_level_to_dispatch_level(vm_pressure_level_t internal_pressure_level)
7122 {
7123 vm_pressure_level_t dispatch_level = NOTE_MEMORYSTATUS_PRESSURE_NORMAL;
7124
7125 switch (internal_pressure_level) {
7126
7127 case kVMPressureNormal:
7128 {
7129 dispatch_level = NOTE_MEMORYSTATUS_PRESSURE_NORMAL;
7130 break;
7131 }
7132
7133 case kVMPressureWarning:
7134 case kVMPressureUrgent:
7135 {
7136 dispatch_level = NOTE_MEMORYSTATUS_PRESSURE_WARN;
7137 break;
7138 }
7139
7140 case kVMPressureCritical:
7141 {
7142 dispatch_level = NOTE_MEMORYSTATUS_PRESSURE_CRITICAL;
7143 break;
7144 }
7145
7146 default:
7147 break;
7148 }
7149
7150 return dispatch_level;
7151 }
7152
7153 static int
7154 sysctl_memorystatus_vm_pressure_level SYSCTL_HANDLER_ARGS
7155 {
7156 #pragma unused(arg1, arg2, oidp)
7157 #if CONFIG_EMBEDDED
7158 int error = 0;
7159
7160 error = priv_check_cred(kauth_cred_get(), PRIV_VM_PRESSURE, 0);
7161 if (error)
7162 return (error);
7163
7164 #endif /* CONFIG_EMBEDDED */
7165 vm_pressure_level_t dispatch_level = convert_internal_pressure_level_to_dispatch_level(memorystatus_vm_pressure_level);
7166
7167 return SYSCTL_OUT(req, &dispatch_level, sizeof(dispatch_level));
7168 }
7169
7170 #if DEBUG || DEVELOPMENT
7171
7172 SYSCTL_PROC(_kern, OID_AUTO, memorystatus_vm_pressure_level, CTLTYPE_INT|CTLFLAG_RD|CTLFLAG_LOCKED,
7173 0, 0, &sysctl_memorystatus_vm_pressure_level, "I", "");
7174
7175 #else /* DEBUG || DEVELOPMENT */
7176
7177 SYSCTL_PROC(_kern, OID_AUTO, memorystatus_vm_pressure_level, CTLTYPE_INT|CTLFLAG_RD|CTLFLAG_LOCKED|CTLFLAG_MASKED,
7178 0, 0, &sysctl_memorystatus_vm_pressure_level, "I", "");
7179
7180 #endif /* DEBUG || DEVELOPMENT */
7181
7182 extern int memorystatus_purge_on_warning;
7183 extern int memorystatus_purge_on_critical;
7184
7185 static int
7186 sysctl_memorypressure_manual_trigger SYSCTL_HANDLER_ARGS
7187 {
7188 #pragma unused(arg1, arg2)
7189
7190 int level = 0;
7191 int error = 0;
7192 int pressure_level = 0;
7193 int trigger_request = 0;
7194 int force_purge;
7195
7196 error = sysctl_handle_int(oidp, &level, 0, req);
7197 if (error || !req->newptr) {
7198 return (error);
7199 }
7200
7201 memorystatus_manual_testing_on = TRUE;
7202
7203 trigger_request = (level >> 16) & 0xFFFF;
7204 pressure_level = (level & 0xFFFF);
7205
7206 if (trigger_request < TEST_LOW_MEMORY_TRIGGER_ONE ||
7207 trigger_request > TEST_LOW_MEMORY_PURGEABLE_TRIGGER_ALL) {
7208 return EINVAL;
7209 }
7210 switch (pressure_level) {
7211 case NOTE_MEMORYSTATUS_PRESSURE_NORMAL:
7212 case NOTE_MEMORYSTATUS_PRESSURE_WARN:
7213 case NOTE_MEMORYSTATUS_PRESSURE_CRITICAL:
7214 break;
7215 default:
7216 return EINVAL;
7217 }
7218
7219 /*
7220 * The pressure level is being set from user-space.
7221 * And user-space uses the constants in sys/event.h
7222 * So we translate those events to our internal levels here.
7223 */
7224 if (pressure_level == NOTE_MEMORYSTATUS_PRESSURE_NORMAL) {
7225
7226 memorystatus_manual_testing_level = kVMPressureNormal;
7227 force_purge = 0;
7228
7229 } else if (pressure_level == NOTE_MEMORYSTATUS_PRESSURE_WARN) {
7230
7231 memorystatus_manual_testing_level = kVMPressureWarning;
7232 force_purge = memorystatus_purge_on_warning;
7233
7234 } else if (pressure_level == NOTE_MEMORYSTATUS_PRESSURE_CRITICAL) {
7235
7236 memorystatus_manual_testing_level = kVMPressureCritical;
7237 force_purge = memorystatus_purge_on_critical;
7238 }
7239
7240 memorystatus_vm_pressure_level = memorystatus_manual_testing_level;
7241
7242 /* purge according to the new pressure level */
7243 switch (trigger_request) {
7244 case TEST_PURGEABLE_TRIGGER_ONE:
7245 case TEST_LOW_MEMORY_PURGEABLE_TRIGGER_ONE:
7246 if (force_purge == 0) {
7247 /* no purging requested */
7248 break;
7249 }
7250 vm_purgeable_object_purge_one_unlocked(force_purge);
7251 break;
7252 case TEST_PURGEABLE_TRIGGER_ALL:
7253 case TEST_LOW_MEMORY_PURGEABLE_TRIGGER_ALL:
7254 if (force_purge == 0) {
7255 /* no purging requested */
7256 break;
7257 }
7258 while (vm_purgeable_object_purge_one_unlocked(force_purge));
7259 break;
7260 }
7261
7262 if ((trigger_request == TEST_LOW_MEMORY_TRIGGER_ONE) ||
7263 (trigger_request == TEST_LOW_MEMORY_PURGEABLE_TRIGGER_ONE)) {
7264
7265 memorystatus_update_vm_pressure(TRUE);
7266 }
7267
7268 if ((trigger_request == TEST_LOW_MEMORY_TRIGGER_ALL) ||
7269 (trigger_request == TEST_LOW_MEMORY_PURGEABLE_TRIGGER_ALL)) {
7270
7271 while (memorystatus_update_vm_pressure(FALSE) == KERN_SUCCESS) {
7272 continue;
7273 }
7274 }
7275
7276 if (pressure_level == NOTE_MEMORYSTATUS_PRESSURE_NORMAL) {
7277 memorystatus_manual_testing_on = FALSE;
7278 }
7279
7280 return 0;
7281 }
7282
7283 SYSCTL_PROC(_kern, OID_AUTO, memorypressure_manual_trigger, CTLTYPE_INT|CTLFLAG_WR|CTLFLAG_LOCKED|CTLFLAG_MASKED,
7284 0, 0, &sysctl_memorypressure_manual_trigger, "I", "");
7285
7286
7287 extern int memorystatus_purge_on_warning;
7288 extern int memorystatus_purge_on_urgent;
7289 extern int memorystatus_purge_on_critical;
7290
7291 SYSCTL_INT(_kern, OID_AUTO, memorystatus_purge_on_warning, CTLFLAG_RW|CTLFLAG_LOCKED, &memorystatus_purge_on_warning, 0, "");
7292 SYSCTL_INT(_kern, OID_AUTO, memorystatus_purge_on_urgent, CTLTYPE_INT|CTLFLAG_RW|CTLFLAG_LOCKED, &memorystatus_purge_on_urgent, 0, "");
7293 SYSCTL_INT(_kern, OID_AUTO, memorystatus_purge_on_critical, CTLTYPE_INT|CTLFLAG_RW|CTLFLAG_LOCKED, &memorystatus_purge_on_critical, 0, "");
7294
7295
7296 #endif /* VM_PRESSURE_EVENTS */
7297
7298 /* Return both allocated and actual size, since there's a race between allocation and list compilation */
7299 static int
7300 memorystatus_get_priority_list(memorystatus_priority_entry_t **list_ptr, size_t *buffer_size, size_t *list_size, boolean_t size_only)
7301 {
7302 uint32_t list_count, i = 0;
7303 memorystatus_priority_entry_t *list_entry;
7304 proc_t p;
7305
7306 list_count = memorystatus_list_count;
7307 *list_size = sizeof(memorystatus_priority_entry_t) * list_count;
7308
7309 /* Just a size check? */
7310 if (size_only) {
7311 return 0;
7312 }
7313
7314 /* Otherwise, validate the size of the buffer */
7315 if (*buffer_size < *list_size) {
7316 return EINVAL;
7317 }
7318
7319 *list_ptr = (memorystatus_priority_entry_t*)kalloc(*list_size);
7320 if (!*list_ptr) {
7321 return ENOMEM;
7322 }
7323
7324 memset(*list_ptr, 0, *list_size);
7325
7326 *buffer_size = *list_size;
7327 *list_size = 0;
7328
7329 list_entry = *list_ptr;
7330
7331 proc_list_lock();
7332
7333 p = memorystatus_get_first_proc_locked(&i, TRUE);
7334 while (p && (*list_size < *buffer_size)) {
7335 list_entry->pid = p->p_pid;
7336 list_entry->priority = p->p_memstat_effectivepriority;
7337 list_entry->user_data = p->p_memstat_userdata;
7338
7339 if (p->p_memstat_memlimit <= 0) {
7340 task_get_phys_footprint_limit(p->task, &list_entry->limit);
7341 } else {
7342 list_entry->limit = p->p_memstat_memlimit;
7343 }
7344
7345 list_entry->state = memorystatus_build_state(p);
7346 list_entry++;
7347
7348 *list_size += sizeof(memorystatus_priority_entry_t);
7349
7350 p = memorystatus_get_next_proc_locked(&i, p, TRUE);
7351 }
7352
7353 proc_list_unlock();
7354
7355 MEMORYSTATUS_DEBUG(1, "memorystatus_get_priority_list: returning %lu for size\n", (unsigned long)*list_size);
7356
7357 return 0;
7358 }
7359
7360 static int
7361 memorystatus_get_priority_pid(pid_t pid, user_addr_t buffer, size_t buffer_size) {
7362 int error = 0;
7363 memorystatus_priority_entry_t mp_entry;
7364
7365 /* Validate inputs */
7366 if ((pid == 0) || (buffer == USER_ADDR_NULL) || (buffer_size != sizeof(memorystatus_priority_entry_t))) {
7367 return EINVAL;
7368 }
7369
7370 proc_t p = proc_find(pid);
7371 if (!p) {
7372 return ESRCH;
7373 }
7374
7375 memset (&mp_entry, 0, sizeof(memorystatus_priority_entry_t));
7376
7377 mp_entry.pid = p->p_pid;
7378 mp_entry.priority = p->p_memstat_effectivepriority;
7379 mp_entry.user_data = p->p_memstat_userdata;
7380 if (p->p_memstat_memlimit <= 0) {
7381 task_get_phys_footprint_limit(p->task, &mp_entry.limit);
7382 } else {
7383 mp_entry.limit = p->p_memstat_memlimit;
7384 }
7385 mp_entry.state = memorystatus_build_state(p);
7386
7387 proc_rele(p);
7388
7389 error = copyout(&mp_entry, buffer, buffer_size);
7390
7391 return (error);
7392 }
7393
7394 static int
7395 memorystatus_cmd_get_priority_list(pid_t pid, user_addr_t buffer, size_t buffer_size, int32_t *retval) {
7396 int error = 0;
7397 boolean_t size_only;
7398 size_t list_size;
7399
7400 /*
7401 * When a non-zero pid is provided, the 'list' has only one entry.
7402 */
7403
7404 size_only = ((buffer == USER_ADDR_NULL) ? TRUE: FALSE);
7405
7406 if (pid != 0) {
7407 list_size = sizeof(memorystatus_priority_entry_t) * 1;
7408 if (!size_only) {
7409 error = memorystatus_get_priority_pid(pid, buffer, buffer_size);
7410 }
7411 } else {
7412 memorystatus_priority_entry_t *list = NULL;
7413 error = memorystatus_get_priority_list(&list, &buffer_size, &list_size, size_only);
7414
7415 if (error == 0) {
7416 if (!size_only) {
7417 error = copyout(list, buffer, list_size);
7418 }
7419 }
7420
7421 if (list) {
7422 kfree(list, buffer_size);
7423 }
7424 }
7425
7426 if (error == 0) {
7427 *retval = list_size;
7428 }
7429
7430 return (error);
7431 }
7432
7433 static void
7434 memorystatus_clear_errors(void)
7435 {
7436 proc_t p;
7437 unsigned int i = 0;
7438
7439 KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_CLEAR_ERRORS) | DBG_FUNC_START, 0, 0, 0, 0, 0);
7440
7441 proc_list_lock();
7442
7443 p = memorystatus_get_first_proc_locked(&i, TRUE);
7444 while (p) {
7445 if (p->p_memstat_state & P_MEMSTAT_ERROR) {
7446 p->p_memstat_state &= ~P_MEMSTAT_ERROR;
7447 }
7448 p = memorystatus_get_next_proc_locked(&i, p, TRUE);
7449 }
7450
7451 proc_list_unlock();
7452
7453 KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_CLEAR_ERRORS) | DBG_FUNC_END, 0, 0, 0, 0, 0);
7454 }
7455
7456 #if CONFIG_JETSAM
7457 static void
7458 memorystatus_update_levels_locked(boolean_t critical_only) {
7459
7460 memorystatus_available_pages_critical = memorystatus_available_pages_critical_base;
7461
7462 /*
7463 * If there's an entry in the first bucket, we have idle processes.
7464 */
7465
7466 memstat_bucket_t *first_bucket = &memstat_bucket[JETSAM_PRIORITY_IDLE];
7467 if (first_bucket->count) {
7468 memorystatus_available_pages_critical += memorystatus_available_pages_critical_idle_offset;
7469
7470 if (memorystatus_available_pages_critical > memorystatus_available_pages_pressure ) {
7471 /*
7472 * The critical threshold must never exceed the pressure threshold
7473 */
7474 memorystatus_available_pages_critical = memorystatus_available_pages_pressure;
7475 }
7476 }
7477
7478 #if DEBUG || DEVELOPMENT
7479 if (memorystatus_jetsam_policy & kPolicyDiagnoseActive) {
7480 memorystatus_available_pages_critical += memorystatus_jetsam_policy_offset_pages_diagnostic;
7481
7482 if (memorystatus_available_pages_critical > memorystatus_available_pages_pressure ) {
7483 /*
7484 * The critical threshold must never exceed the pressure threshold
7485 */
7486 memorystatus_available_pages_critical = memorystatus_available_pages_pressure;
7487 }
7488 }
7489 #endif /* DEBUG || DEVELOPMENT */
7490
7491 if (memorystatus_jetsam_policy & kPolicyMoreFree) {
7492 memorystatus_available_pages_critical += memorystatus_policy_more_free_offset_pages;
7493 }
7494
7495 if (critical_only) {
7496 return;
7497 }
7498
7499 #if VM_PRESSURE_EVENTS
7500 memorystatus_available_pages_pressure = (pressure_threshold_percentage / delta_percentage) * memorystatus_delta;
7501 #if DEBUG || DEVELOPMENT
7502 if (memorystatus_jetsam_policy & kPolicyDiagnoseActive) {
7503 memorystatus_available_pages_pressure += memorystatus_jetsam_policy_offset_pages_diagnostic;
7504 }
7505 #endif
7506 #endif
7507 }
7508
7509
7510 static int
7511 sysctl_kern_memorystatus_policy_more_free SYSCTL_HANDLER_ARGS
7512 {
7513 #pragma unused(arg1, arg2, oidp)
7514 int error = 0, more_free = 0;
7515
7516 /*
7517 * TODO: Enable this privilege check?
7518 *
7519 * error = priv_check_cred(kauth_cred_get(), PRIV_VM_JETSAM, 0);
7520 * if (error)
7521 * return (error);
7522 */
7523
7524 error = sysctl_handle_int(oidp, &more_free, 0, req);
7525 if (error || !req->newptr)
7526 return (error);
7527
7528 if ((more_free && ((memorystatus_jetsam_policy & kPolicyMoreFree) == kPolicyMoreFree)) ||
7529 (!more_free && ((memorystatus_jetsam_policy & kPolicyMoreFree) == 0))) {
7530
7531 /*
7532 * No change in state.
7533 */
7534 return 0;
7535 }
7536
7537 proc_list_lock();
7538
7539 if (more_free) {
7540 memorystatus_jetsam_policy |= kPolicyMoreFree;
7541 } else {
7542 memorystatus_jetsam_policy &= ~kPolicyMoreFree;
7543 }
7544
7545 memorystatus_update_levels_locked(TRUE);
7546
7547 proc_list_unlock();
7548
7549 return 0;
7550 }
7551 SYSCTL_PROC(_kern, OID_AUTO, memorystatus_policy_more_free, CTLTYPE_INT|CTLFLAG_WR|CTLFLAG_LOCKED|CTLFLAG_MASKED,
7552 0, 0, &sysctl_kern_memorystatus_policy_more_free, "I", "");
7553
7554 #endif /* CONFIG_JETSAM */
7555
7556 /*
7557 * Get the at_boot snapshot
7558 */
7559 static int
7560 memorystatus_get_at_boot_snapshot(memorystatus_jetsam_snapshot_t **snapshot, size_t *snapshot_size, boolean_t size_only) {
7561 size_t input_size = *snapshot_size;
7562
7563 /*
7564 * The at_boot snapshot has no entry list.
7565 */
7566 *snapshot_size = sizeof(memorystatus_jetsam_snapshot_t);
7567
7568 if (size_only) {
7569 return 0;
7570 }
7571
7572 /*
7573 * Validate the size of the snapshot buffer
7574 */
7575 if (input_size < *snapshot_size) {
7576 return EINVAL;
7577 }
7578
7579 /*
7580 * Update the notification_time only
7581 */
7582 memorystatus_at_boot_snapshot.notification_time = mach_absolute_time();
7583 *snapshot = &memorystatus_at_boot_snapshot;
7584
7585 MEMORYSTATUS_DEBUG(7, "memorystatus_get_at_boot_snapshot: returned inputsize (%ld), snapshot_size(%ld), listcount(%d)\n",
7586 (long)input_size, (long)*snapshot_size, 0);
7587 return 0;
7588 }
7589
7590 static int
7591 memorystatus_get_on_demand_snapshot(memorystatus_jetsam_snapshot_t **snapshot, size_t *snapshot_size, boolean_t size_only) {
7592 size_t input_size = *snapshot_size;
7593 uint32_t ods_list_count = memorystatus_list_count;
7594 memorystatus_jetsam_snapshot_t *ods = NULL; /* The on_demand snapshot buffer */
7595
7596 *snapshot_size = sizeof(memorystatus_jetsam_snapshot_t) + (sizeof(memorystatus_jetsam_snapshot_entry_t) * (ods_list_count));
7597
7598 if (size_only) {
7599 return 0;
7600 }
7601
7602 /*
7603 * Validate the size of the snapshot buffer.
7604 * This is inherently racey. May want to revisit
7605 * this error condition and trim the output when
7606 * it doesn't fit.
7607 */
7608 if (input_size < *snapshot_size) {
7609 return EINVAL;
7610 }
7611
7612 /*
7613 * Allocate and initialize a snapshot buffer.
7614 */
7615 ods = (memorystatus_jetsam_snapshot_t *)kalloc(*snapshot_size);
7616 if (!ods) {
7617 return (ENOMEM);
7618 }
7619
7620 memset(ods, 0, *snapshot_size);
7621
7622 proc_list_lock();
7623 memorystatus_init_jetsam_snapshot_locked(ods, ods_list_count);
7624 proc_list_unlock();
7625
7626 /*
7627 * Return the kernel allocated, on_demand buffer.
7628 * The caller of this routine will copy the data out
7629 * to user space and then free the kernel allocated
7630 * buffer.
7631 */
7632 *snapshot = ods;
7633
7634 MEMORYSTATUS_DEBUG(7, "memorystatus_get_on_demand_snapshot: returned inputsize (%ld), snapshot_size(%ld), listcount(%ld)\n",
7635 (long)input_size, (long)*snapshot_size, (long)ods_list_count);
7636
7637 return 0;
7638 }
7639
7640 static int
7641 memorystatus_get_jetsam_snapshot(memorystatus_jetsam_snapshot_t **snapshot, size_t *snapshot_size, boolean_t size_only) {
7642 size_t input_size = *snapshot_size;
7643
7644 if (memorystatus_jetsam_snapshot_count > 0) {
7645 *snapshot_size = sizeof(memorystatus_jetsam_snapshot_t) + (sizeof(memorystatus_jetsam_snapshot_entry_t) * (memorystatus_jetsam_snapshot_count));
7646 } else {
7647 *snapshot_size = 0;
7648 }
7649
7650 if (size_only) {
7651 return 0;
7652 }
7653
7654 if (input_size < *snapshot_size) {
7655 return EINVAL;
7656 }
7657
7658 *snapshot = memorystatus_jetsam_snapshot;
7659
7660 MEMORYSTATUS_DEBUG(7, "memorystatus_get_jetsam_snapshot: returned inputsize (%ld), snapshot_size(%ld), listcount(%ld)\n",
7661 (long)input_size, (long)*snapshot_size, (long)memorystatus_jetsam_snapshot_count);
7662
7663 return 0;
7664 }
7665
7666
7667 static int
7668 memorystatus_cmd_get_jetsam_snapshot(int32_t flags, user_addr_t buffer, size_t buffer_size, int32_t *retval) {
7669 int error = EINVAL;
7670 boolean_t size_only;
7671 boolean_t is_default_snapshot = FALSE;
7672 boolean_t is_on_demand_snapshot = FALSE;
7673 boolean_t is_at_boot_snapshot = FALSE;
7674 memorystatus_jetsam_snapshot_t *snapshot;
7675
7676 size_only = ((buffer == USER_ADDR_NULL) ? TRUE : FALSE);
7677
7678 if (flags == 0) {
7679 /* Default */
7680 is_default_snapshot = TRUE;
7681 error = memorystatus_get_jetsam_snapshot(&snapshot, &buffer_size, size_only);
7682 } else {
7683 if (flags & ~(MEMORYSTATUS_SNAPSHOT_ON_DEMAND | MEMORYSTATUS_SNAPSHOT_AT_BOOT)) {
7684 /*
7685 * Unsupported bit set in flag.
7686 */
7687 return EINVAL;
7688 }
7689
7690 if ((flags & (MEMORYSTATUS_SNAPSHOT_ON_DEMAND | MEMORYSTATUS_SNAPSHOT_AT_BOOT)) ==
7691 (MEMORYSTATUS_SNAPSHOT_ON_DEMAND | MEMORYSTATUS_SNAPSHOT_AT_BOOT)) {
7692 /*
7693 * Can't have both set at the same time.
7694 */
7695 return EINVAL;
7696 }
7697
7698 if (flags & MEMORYSTATUS_SNAPSHOT_ON_DEMAND) {
7699 is_on_demand_snapshot = TRUE;
7700 /*
7701 * When not requesting the size only, the following call will allocate
7702 * an on_demand snapshot buffer, which is freed below.
7703 */
7704 error = memorystatus_get_on_demand_snapshot(&snapshot, &buffer_size, size_only);
7705
7706 } else if (flags & MEMORYSTATUS_SNAPSHOT_AT_BOOT) {
7707 is_at_boot_snapshot = TRUE;
7708 error = memorystatus_get_at_boot_snapshot(&snapshot, &buffer_size, size_only);
7709 } else {
7710 /*
7711 * Invalid flag setting.
7712 */
7713 return EINVAL;
7714 }
7715 }
7716
7717 if (error) {
7718 goto out;
7719 }
7720
7721 /*
7722 * Copy the data out to user space and clear the snapshot buffer.
7723 * If working with the jetsam snapshot,
7724 * clearing the buffer means, reset the count.
7725 * If working with an on_demand snapshot
7726 * clearing the buffer means, free it.
7727 * If working with the at_boot snapshot
7728 * there is nothing to clear or update.
7729 */
7730 if (!size_only) {
7731 if ((error = copyout(snapshot, buffer, buffer_size)) == 0) {
7732 if (is_default_snapshot) {
7733 /*
7734 * The jetsam snapshot is never freed, its count is simply reset.
7735 */
7736 proc_list_lock();
7737 snapshot->entry_count = memorystatus_jetsam_snapshot_count = 0;
7738 memorystatus_jetsam_snapshot_last_timestamp = 0;
7739 proc_list_unlock();
7740 }
7741 }
7742
7743 if (is_on_demand_snapshot) {
7744 /*
7745 * The on_demand snapshot is always freed,
7746 * even if the copyout failed.
7747 */
7748 if(snapshot) {
7749 kfree(snapshot, buffer_size);
7750 }
7751 }
7752 }
7753
7754 if (error == 0) {
7755 *retval = buffer_size;
7756 }
7757 out:
7758 return error;
7759 }
7760
7761 /*
7762 * Routine: memorystatus_cmd_grp_set_properties
7763 * Purpose: Update properties for a group of processes.
7764 *
7765 * Supported Properties:
7766 * [priority]
7767 * Move each process out of its effective priority
7768 * band and into a new priority band.
7769 * Maintains relative order from lowest to highest priority.
7770 * In single band, maintains relative order from head to tail.
7771 *
7772 * eg: before [effectivepriority | pid]
7773 * [18 | p101 ]
7774 * [17 | p55, p67, p19 ]
7775 * [12 | p103 p10 ]
7776 * [ 7 | p25 ]
7777 * [ 0 | p71, p82, ]
7778 *
7779 * after [ new band | pid]
7780 * [ xxx | p71, p82, p25, p103, p10, p55, p67, p19, p101]
7781 *
7782 * Returns: 0 on success, else non-zero.
7783 *
7784 * Caveat: We know there is a race window regarding recycled pids.
7785 * A process could be killed before the kernel can act on it here.
7786 * If a pid cannot be found in any of the jetsam priority bands,
7787 * then we simply ignore it. No harm.
7788 * But, if the pid has been recycled then it could be an issue.
7789 * In that scenario, we might move an unsuspecting process to the new
7790 * priority band. It's not clear how the kernel can safeguard
7791 * against this, but it would be an extremely rare case anyway.
7792 * The caller of this api might avoid such race conditions by
7793 * ensuring that the processes passed in the pid list are suspended.
7794 */
7795
7796
7797 /* This internal structure can expand when we add support for more properties */
7798 typedef struct memorystatus_internal_properties
7799 {
7800 proc_t proc;
7801 int32_t priority; /* see memorytstatus_priority_entry_t : priority */
7802 } memorystatus_internal_properties_t;
7803
7804
7805 static int
7806 memorystatus_cmd_grp_set_properties(int32_t flags, user_addr_t buffer, size_t buffer_size, __unused int32_t *retval) {
7807
7808 #pragma unused (flags)
7809
7810 /*
7811 * We only handle setting priority
7812 * per process
7813 */
7814
7815 int error = 0;
7816 memorystatus_priority_entry_t *entries = NULL;
7817 uint32_t entry_count = 0;
7818
7819 /* This will be the ordered proc list */
7820 memorystatus_internal_properties_t *table = NULL;
7821 size_t table_size = 0;
7822 uint32_t table_count = 0;
7823
7824 uint32_t i = 0;
7825 uint32_t bucket_index = 0;
7826 boolean_t head_insert;
7827 int32_t new_priority;
7828
7829 proc_t p;
7830
7831 /* Verify inputs */
7832 if ((buffer == USER_ADDR_NULL) || (buffer_size == 0) || ((buffer_size % sizeof(memorystatus_priority_entry_t)) != 0)) {
7833 error = EINVAL;
7834 goto out;
7835 }
7836
7837 entry_count = (buffer_size / sizeof(memorystatus_priority_entry_t));
7838 if ((entries = (memorystatus_priority_entry_t *)kalloc(buffer_size)) == NULL) {
7839 error = ENOMEM;
7840 goto out;
7841 }
7842
7843 KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_GRP_SET_PROP) | DBG_FUNC_START, entry_count, 0, 0, 0, 0);
7844
7845 if ((error = copyin(buffer, entries, buffer_size)) != 0) {
7846 goto out;
7847 }
7848
7849 /* Verify sanity of input priorities */
7850 for (i=0; i < entry_count; i++) {
7851 if (entries[i].priority == -1) {
7852 /* Use as shorthand for default priority */
7853 entries[i].priority = JETSAM_PRIORITY_DEFAULT;
7854 } else if ((entries[i].priority == system_procs_aging_band) || (entries[i].priority == applications_aging_band)) {
7855 /* Both the aging bands are reserved for internal use;
7856 * if requested, adjust to JETSAM_PRIORITY_IDLE. */
7857 entries[i].priority = JETSAM_PRIORITY_IDLE;
7858 } else if (entries[i].priority == JETSAM_PRIORITY_IDLE_HEAD) {
7859 /* JETSAM_PRIORITY_IDLE_HEAD inserts at the head of the idle
7860 * queue */
7861 /* Deal with this later */
7862 } else if ((entries[i].priority < 0) || (entries[i].priority >= MEMSTAT_BUCKET_COUNT)) {
7863 /* Sanity check */
7864 error = EINVAL;
7865 goto out;
7866 }
7867 }
7868
7869 table_size = sizeof(memorystatus_internal_properties_t) * entry_count;
7870 if ( (table = (memorystatus_internal_properties_t *)kalloc(table_size)) == NULL) {
7871 error = ENOMEM;
7872 goto out;
7873 }
7874 memset(table, 0, table_size);
7875
7876
7877 /*
7878 * For each jetsam bucket entry, spin through the input property list.
7879 * When a matching pid is found, populate an adjacent table with the
7880 * appropriate proc pointer and new property values.
7881 * This traversal automatically preserves order from lowest
7882 * to highest priority.
7883 */
7884
7885 bucket_index=0;
7886
7887 proc_list_lock();
7888
7889 /* Create the ordered table */
7890 p = memorystatus_get_first_proc_locked(&bucket_index, TRUE);
7891 while (p && (table_count < entry_count)) {
7892 for (i=0; i < entry_count; i++ ) {
7893 if (p->p_pid == entries[i].pid) {
7894 /* Build the table data */
7895 table[table_count].proc = p;
7896 table[table_count].priority = entries[i].priority;
7897 table_count++;
7898 break;
7899 }
7900 }
7901 p = memorystatus_get_next_proc_locked(&bucket_index, p, TRUE);
7902 }
7903
7904 /* We now have ordered list of procs ready to move */
7905 for (i=0; i < table_count; i++) {
7906 p = table[i].proc;
7907 assert(p != NULL);
7908
7909 /* Allow head inserts -- but relative order is now */
7910 if (table[i].priority == JETSAM_PRIORITY_IDLE_HEAD) {
7911 new_priority = JETSAM_PRIORITY_IDLE;
7912 head_insert = true;
7913 } else {
7914 new_priority = table[i].priority;
7915 head_insert = false;
7916 }
7917
7918 /* Not allowed */
7919 if (p->p_memstat_state & P_MEMSTAT_INTERNAL) {
7920 continue;
7921 }
7922
7923 /*
7924 * Take appropriate steps if moving proc out of
7925 * either of the aging bands.
7926 */
7927 if ((p->p_memstat_effectivepriority == system_procs_aging_band) || (p->p_memstat_effectivepriority == applications_aging_band)) {
7928 memorystatus_invalidate_idle_demotion_locked(p, TRUE);
7929 }
7930
7931 memorystatus_update_priority_locked(p, new_priority, head_insert, false);
7932 }
7933
7934 proc_list_unlock();
7935
7936 /*
7937 * if (table_count != entry_count)
7938 * then some pids were not found in a jetsam band.
7939 * harmless but interesting...
7940 */
7941 KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_GRP_SET_PROP) | DBG_FUNC_END, entry_count, table_count, 0, 0, 0);
7942
7943 out:
7944 if (entries)
7945 kfree(entries, buffer_size);
7946 if (table)
7947 kfree(table, table_size);
7948
7949 return (error);
7950 }
7951
7952
7953 /*
7954 * This routine is used to update a process's jetsam priority position and stored user_data.
7955 * It is not used for the setting of memory limits, which is why the last 6 args to the
7956 * memorystatus_update() call are 0 or FALSE.
7957 */
7958
7959 static int
7960 memorystatus_cmd_set_priority_properties(pid_t pid, user_addr_t buffer, size_t buffer_size, __unused int32_t *retval) {
7961 int error = 0;
7962 memorystatus_priority_properties_t mpp_entry;
7963
7964 /* Validate inputs */
7965 if ((pid == 0) || (buffer == USER_ADDR_NULL) || (buffer_size != sizeof(memorystatus_priority_properties_t))) {
7966 return EINVAL;
7967 }
7968
7969 error = copyin(buffer, &mpp_entry, buffer_size);
7970
7971 if (error == 0) {
7972 proc_t p;
7973
7974 p = proc_find(pid);
7975 if (!p) {
7976 return ESRCH;
7977 }
7978
7979 if (p->p_memstat_state & P_MEMSTAT_INTERNAL) {
7980 proc_rele(p);
7981 return EPERM;
7982 }
7983
7984 error = memorystatus_update(p, mpp_entry.priority, mpp_entry.user_data, FALSE, FALSE, 0, 0, FALSE, FALSE);
7985 proc_rele(p);
7986 }
7987
7988 return(error);
7989 }
7990
7991 static int
7992 memorystatus_cmd_set_memlimit_properties(pid_t pid, user_addr_t buffer, size_t buffer_size, __unused int32_t *retval) {
7993 int error = 0;
7994 memorystatus_memlimit_properties_t mmp_entry;
7995
7996 /* Validate inputs */
7997 if ((pid == 0) || (buffer == USER_ADDR_NULL) || (buffer_size != sizeof(memorystatus_memlimit_properties_t))) {
7998 return EINVAL;
7999 }
8000
8001 error = copyin(buffer, &mmp_entry, buffer_size);
8002
8003 if (error == 0) {
8004 error = memorystatus_set_memlimit_properties(pid, &mmp_entry);
8005 }
8006
8007 return(error);
8008 }
8009
8010 /*
8011 * When getting the memlimit settings, we can't simply call task_get_phys_footprint_limit().
8012 * That gets the proc's cached memlimit and there is no guarantee that the active/inactive
8013 * limits will be the same in the no-limit case. Instead we convert limits <= 0 using
8014 * task_convert_phys_footprint_limit(). It computes the same limit value that would be written
8015 * to the task's ledgers via task_set_phys_footprint_limit().
8016 */
8017 static int
8018 memorystatus_cmd_get_memlimit_properties(pid_t pid, user_addr_t buffer, size_t buffer_size, __unused int32_t *retval) {
8019 int error = 0;
8020 memorystatus_memlimit_properties_t mmp_entry;
8021
8022 /* Validate inputs */
8023 if ((pid == 0) || (buffer == USER_ADDR_NULL) || (buffer_size != sizeof(memorystatus_memlimit_properties_t))) {
8024 return EINVAL;
8025 }
8026
8027 memset (&mmp_entry, 0, sizeof(memorystatus_memlimit_properties_t));
8028
8029 proc_t p = proc_find(pid);
8030 if (!p) {
8031 return ESRCH;
8032 }
8033
8034 /*
8035 * Get the active limit and attributes.
8036 * No locks taken since we hold a reference to the proc.
8037 */
8038
8039 if (p->p_memstat_memlimit_active > 0 ) {
8040 mmp_entry.memlimit_active = p->p_memstat_memlimit_active;
8041 } else {
8042 task_convert_phys_footprint_limit(-1, &mmp_entry.memlimit_active);
8043 }
8044
8045 if (p->p_memstat_state & P_MEMSTAT_MEMLIMIT_ACTIVE_FATAL) {
8046 mmp_entry.memlimit_active_attr |= MEMORYSTATUS_MEMLIMIT_ATTR_FATAL;
8047 }
8048
8049 /*
8050 * Get the inactive limit and attributes
8051 */
8052 if (p->p_memstat_memlimit_inactive <= 0) {
8053 task_convert_phys_footprint_limit(-1, &mmp_entry.memlimit_inactive);
8054 } else {
8055 mmp_entry.memlimit_inactive = p->p_memstat_memlimit_inactive;
8056 }
8057 if (p->p_memstat_state & P_MEMSTAT_MEMLIMIT_INACTIVE_FATAL) {
8058 mmp_entry.memlimit_inactive_attr |= MEMORYSTATUS_MEMLIMIT_ATTR_FATAL;
8059 }
8060 proc_rele(p);
8061
8062 error = copyout(&mmp_entry, buffer, buffer_size);
8063
8064 return(error);
8065 }
8066
8067
8068 /*
8069 * SPI for kbd - pr24956468
8070 * This is a very simple snapshot that calculates how much a
8071 * process's phys_footprint exceeds a specific memory limit.
8072 * Only the inactive memory limit is supported for now.
8073 * The delta is returned as bytes in excess or zero.
8074 */
8075 static int
8076 memorystatus_cmd_get_memlimit_excess_np(pid_t pid, uint32_t flags, user_addr_t buffer, size_t buffer_size, __unused int32_t *retval) {
8077 int error = 0;
8078 uint64_t footprint_in_bytes = 0;
8079 uint64_t delta_in_bytes = 0;
8080 int32_t memlimit_mb = 0;
8081 uint64_t memlimit_bytes = 0;
8082
8083 /* Validate inputs */
8084 if ((pid == 0) || (buffer == USER_ADDR_NULL) || (buffer_size != sizeof(uint64_t)) || (flags != 0)) {
8085 return EINVAL;
8086 }
8087
8088 proc_t p = proc_find(pid);
8089 if (!p) {
8090 return ESRCH;
8091 }
8092
8093 /*
8094 * Get the inactive limit.
8095 * No locks taken since we hold a reference to the proc.
8096 */
8097
8098 if (p->p_memstat_memlimit_inactive <= 0) {
8099 task_convert_phys_footprint_limit(-1, &memlimit_mb);
8100 } else {
8101 memlimit_mb = p->p_memstat_memlimit_inactive;
8102 }
8103
8104 footprint_in_bytes = get_task_phys_footprint(p->task);
8105
8106 proc_rele(p);
8107
8108 memlimit_bytes = memlimit_mb * 1024 * 1024; /* MB to bytes */
8109
8110 /*
8111 * Computed delta always returns >= 0 bytes
8112 */
8113 if (footprint_in_bytes > memlimit_bytes) {
8114 delta_in_bytes = footprint_in_bytes - memlimit_bytes;
8115 }
8116
8117 error = copyout(&delta_in_bytes, buffer, sizeof(delta_in_bytes));
8118
8119 return(error);
8120 }
8121
8122
8123 static int
8124 memorystatus_cmd_get_pressure_status(int32_t *retval) {
8125 int error;
8126
8127 /* Need privilege for check */
8128 error = priv_check_cred(kauth_cred_get(), PRIV_VM_PRESSURE, 0);
8129 if (error) {
8130 return (error);
8131 }
8132
8133 /* Inherently racy, so it's not worth taking a lock here */
8134 *retval = (kVMPressureNormal != memorystatus_vm_pressure_level) ? 1 : 0;
8135
8136 return error;
8137 }
8138
8139 int
8140 memorystatus_get_pressure_status_kdp() {
8141 return (kVMPressureNormal != memorystatus_vm_pressure_level) ? 1 : 0;
8142 }
8143
8144 /*
8145 * Every process, including a P_MEMSTAT_INTERNAL process (currently only pid 1), is allowed to set a HWM.
8146 *
8147 * This call is inflexible -- it does not distinguish between active/inactive, fatal/non-fatal
8148 * So, with 2-level HWM preserving previous behavior will map as follows.
8149 * - treat the limit passed in as both an active and inactive limit.
8150 * - treat the is_fatal_limit flag as though it applies to both active and inactive limits.
8151 *
8152 * When invoked via MEMORYSTATUS_CMD_SET_JETSAM_HIGH_WATER_MARK
8153 * - the is_fatal_limit is FALSE, meaning the active and inactive limits are non-fatal/soft
8154 * - so mapping is (active/non-fatal, inactive/non-fatal)
8155 *
8156 * When invoked via MEMORYSTATUS_CMD_SET_JETSAM_TASK_LIMIT
8157 * - the is_fatal_limit is TRUE, meaning the process's active and inactive limits are fatal/hard
8158 * - so mapping is (active/fatal, inactive/fatal)
8159 */
8160
8161 #if CONFIG_JETSAM
8162 static int
8163 memorystatus_cmd_set_jetsam_memory_limit(pid_t pid, int32_t high_water_mark, __unused int32_t *retval, boolean_t is_fatal_limit) {
8164 int error = 0;
8165 memorystatus_memlimit_properties_t entry;
8166
8167 entry.memlimit_active = high_water_mark;
8168 entry.memlimit_active_attr = 0;
8169 entry.memlimit_inactive = high_water_mark;
8170 entry.memlimit_inactive_attr = 0;
8171
8172 if (is_fatal_limit == TRUE) {
8173 entry.memlimit_active_attr |= MEMORYSTATUS_MEMLIMIT_ATTR_FATAL;
8174 entry.memlimit_inactive_attr |= MEMORYSTATUS_MEMLIMIT_ATTR_FATAL;
8175 }
8176
8177 error = memorystatus_set_memlimit_properties(pid, &entry);
8178 return (error);
8179 }
8180 #endif /* CONFIG_JETSAM */
8181
8182 static int
8183 memorystatus_set_memlimit_properties(pid_t pid, memorystatus_memlimit_properties_t *entry) {
8184
8185 int32_t memlimit_active;
8186 boolean_t memlimit_active_is_fatal;
8187 int32_t memlimit_inactive;
8188 boolean_t memlimit_inactive_is_fatal;
8189 uint32_t valid_attrs = 0;
8190 int error = 0;
8191
8192 proc_t p = proc_find(pid);
8193 if (!p) {
8194 return ESRCH;
8195 }
8196
8197 /*
8198 * Check for valid attribute flags.
8199 */
8200 valid_attrs |= (MEMORYSTATUS_MEMLIMIT_ATTR_FATAL);
8201 if ((entry->memlimit_active_attr & (~valid_attrs)) != 0) {
8202 proc_rele(p);
8203 return EINVAL;
8204 }
8205 if ((entry->memlimit_inactive_attr & (~valid_attrs)) != 0) {
8206 proc_rele(p);
8207 return EINVAL;
8208 }
8209
8210 /*
8211 * Setup the active memlimit properties
8212 */
8213 memlimit_active = entry->memlimit_active;
8214 if (entry->memlimit_active_attr & MEMORYSTATUS_MEMLIMIT_ATTR_FATAL) {
8215 memlimit_active_is_fatal = TRUE;
8216 } else {
8217 memlimit_active_is_fatal = FALSE;
8218 }
8219
8220 /*
8221 * Setup the inactive memlimit properties
8222 */
8223 memlimit_inactive = entry->memlimit_inactive;
8224 if (entry->memlimit_inactive_attr & MEMORYSTATUS_MEMLIMIT_ATTR_FATAL) {
8225 memlimit_inactive_is_fatal = TRUE;
8226 } else {
8227 memlimit_inactive_is_fatal = FALSE;
8228 }
8229
8230 /*
8231 * Setting a limit of <= 0 implies that the process has no
8232 * high-water-mark and has no per-task-limit. That means
8233 * the system_wide task limit is in place, which by the way,
8234 * is always fatal.
8235 */
8236
8237 if (memlimit_active <= 0) {
8238 /*
8239 * Enforce the fatal system_wide task limit while process is active.
8240 */
8241 memlimit_active = -1;
8242 memlimit_active_is_fatal = TRUE;
8243 }
8244
8245 if (memlimit_inactive <= 0) {
8246 /*
8247 * Enforce the fatal system_wide task limit while process is inactive.
8248 */
8249 memlimit_inactive = -1;
8250 memlimit_inactive_is_fatal = TRUE;
8251 }
8252
8253 proc_list_lock();
8254
8255 /*
8256 * Store the active limit variants in the proc.
8257 */
8258 SET_ACTIVE_LIMITS_LOCKED(p, memlimit_active, memlimit_active_is_fatal);
8259
8260 /*
8261 * Store the inactive limit variants in the proc.
8262 */
8263 SET_INACTIVE_LIMITS_LOCKED(p, memlimit_inactive, memlimit_inactive_is_fatal);
8264
8265 /*
8266 * Enforce appropriate limit variant by updating the cached values
8267 * and writing the ledger.
8268 * Limit choice is based on process active/inactive state.
8269 */
8270
8271 if (memorystatus_highwater_enabled) {
8272 boolean_t is_fatal;
8273 boolean_t use_active;
8274
8275 if (proc_jetsam_state_is_active_locked(p) == TRUE) {
8276 CACHE_ACTIVE_LIMITS_LOCKED(p, is_fatal);
8277 use_active = TRUE;
8278 } else {
8279 CACHE_INACTIVE_LIMITS_LOCKED(p, is_fatal);
8280 use_active = FALSE;
8281 }
8282
8283 /* Enforce the limit by writing to the ledgers */
8284 error = (task_set_phys_footprint_limit_internal(p->task, ((p->p_memstat_memlimit > 0) ? p->p_memstat_memlimit : -1), NULL, use_active, is_fatal) == 0) ? 0 : EINVAL;
8285
8286 MEMORYSTATUS_DEBUG(3, "memorystatus_set_memlimit_properties: new limit on pid %d (%dMB %s) current priority (%d) dirty_state?=0x%x %s\n",
8287 p->p_pid, (p->p_memstat_memlimit > 0 ? p->p_memstat_memlimit : -1),
8288 (p->p_memstat_state & P_MEMSTAT_FATAL_MEMLIMIT ? "F " : "NF"), p->p_memstat_effectivepriority, p->p_memstat_dirty,
8289 (p->p_memstat_dirty ? ((p->p_memstat_dirty & P_DIRTY) ? "isdirty" : "isclean") : ""));
8290 DTRACE_MEMORYSTATUS2(memorystatus_set_memlimit, proc_t, p, int32_t, (p->p_memstat_memlimit > 0 ? p->p_memstat_memlimit : -1));
8291 }
8292
8293 proc_list_unlock();
8294 proc_rele(p);
8295
8296 return error;
8297 }
8298
8299 /*
8300 * Returns the jetsam priority (effective or requested) of the process
8301 * associated with this task.
8302 */
8303 int
8304 proc_get_memstat_priority(proc_t p, boolean_t effective_priority)
8305 {
8306 if (p) {
8307 if (effective_priority) {
8308 return p->p_memstat_effectivepriority;
8309 } else {
8310 return p->p_memstat_requestedpriority;
8311 }
8312 }
8313 return 0;
8314 }
8315
8316 int
8317 memorystatus_control(struct proc *p __unused, struct memorystatus_control_args *args, int *ret) {
8318 int error = EINVAL;
8319 os_reason_t jetsam_reason = OS_REASON_NULL;
8320
8321 #if !CONFIG_JETSAM
8322 #pragma unused(ret)
8323 #pragma unused(jetsam_reason)
8324 #endif
8325
8326 /* Need to be root or have entitlement */
8327 if (!kauth_cred_issuser(kauth_cred_get()) && !IOTaskHasEntitlement(current_task(), MEMORYSTATUS_ENTITLEMENT)) {
8328 error = EPERM;
8329 goto out;
8330 }
8331
8332 /*
8333 * Sanity check.
8334 * Do not enforce it for snapshots.
8335 */
8336 if (args->command != MEMORYSTATUS_CMD_GET_JETSAM_SNAPSHOT) {
8337 if (args->buffersize > MEMORYSTATUS_BUFFERSIZE_MAX) {
8338 error = EINVAL;
8339 goto out;
8340 }
8341 }
8342
8343 switch (args->command) {
8344 case MEMORYSTATUS_CMD_GET_PRIORITY_LIST:
8345 error = memorystatus_cmd_get_priority_list(args->pid, args->buffer, args->buffersize, ret);
8346 break;
8347 case MEMORYSTATUS_CMD_SET_PRIORITY_PROPERTIES:
8348 error = memorystatus_cmd_set_priority_properties(args->pid, args->buffer, args->buffersize, ret);
8349 break;
8350 case MEMORYSTATUS_CMD_SET_MEMLIMIT_PROPERTIES:
8351 error = memorystatus_cmd_set_memlimit_properties(args->pid, args->buffer, args->buffersize, ret);
8352 break;
8353 case MEMORYSTATUS_CMD_GET_MEMLIMIT_PROPERTIES:
8354 error = memorystatus_cmd_get_memlimit_properties(args->pid, args->buffer, args->buffersize, ret);
8355 break;
8356 case MEMORYSTATUS_CMD_GET_MEMLIMIT_EXCESS:
8357 error = memorystatus_cmd_get_memlimit_excess_np(args->pid, args->flags, args->buffer, args->buffersize, ret);
8358 break;
8359 case MEMORYSTATUS_CMD_GRP_SET_PROPERTIES:
8360 error = memorystatus_cmd_grp_set_properties((int32_t)args->flags, args->buffer, args->buffersize, ret);
8361 break;
8362 case MEMORYSTATUS_CMD_GET_JETSAM_SNAPSHOT:
8363 error = memorystatus_cmd_get_jetsam_snapshot((int32_t)args->flags, args->buffer, args->buffersize, ret);
8364 break;
8365 case MEMORYSTATUS_CMD_GET_PRESSURE_STATUS:
8366 error = memorystatus_cmd_get_pressure_status(ret);
8367 break;
8368 #if CONFIG_JETSAM
8369 case MEMORYSTATUS_CMD_SET_JETSAM_HIGH_WATER_MARK:
8370 /*
8371 * This call does not distinguish between active and inactive limits.
8372 * Default behavior in 2-level HWM world is to set both.
8373 * Non-fatal limit is also assumed for both.
8374 */
8375 error = memorystatus_cmd_set_jetsam_memory_limit(args->pid, (int32_t)args->flags, ret, FALSE);
8376 break;
8377 case MEMORYSTATUS_CMD_SET_JETSAM_TASK_LIMIT:
8378 /*
8379 * This call does not distinguish between active and inactive limits.
8380 * Default behavior in 2-level HWM world is to set both.
8381 * Fatal limit is also assumed for both.
8382 */
8383 error = memorystatus_cmd_set_jetsam_memory_limit(args->pid, (int32_t)args->flags, ret, TRUE);
8384 break;
8385 #endif /* CONFIG_JETSAM */
8386 /* Test commands */
8387 #if DEVELOPMENT || DEBUG
8388 case MEMORYSTATUS_CMD_TEST_JETSAM:
8389 jetsam_reason = os_reason_create(OS_REASON_JETSAM, JETSAM_REASON_GENERIC);
8390 if (jetsam_reason == OS_REASON_NULL) {
8391 printf("memorystatus_control: failed to allocate jetsam reason\n");
8392 }
8393
8394 error = memorystatus_kill_process_sync(args->pid, kMemorystatusKilled, jetsam_reason) ? 0 : EINVAL;
8395 break;
8396 case MEMORYSTATUS_CMD_TEST_JETSAM_SORT:
8397 error = memorystatus_cmd_test_jetsam_sort(args->pid, (int32_t)args->flags);
8398 break;
8399 #if CONFIG_JETSAM
8400 case MEMORYSTATUS_CMD_SET_JETSAM_PANIC_BITS:
8401 error = memorystatus_cmd_set_panic_bits(args->buffer, args->buffersize);
8402 break;
8403 #endif /* CONFIG_JETSAM */
8404 #else /* DEVELOPMENT || DEBUG */
8405 #pragma unused(jetsam_reason)
8406 #endif /* DEVELOPMENT || DEBUG */
8407 case MEMORYSTATUS_CMD_AGGRESSIVE_JETSAM_LENIENT_MODE_ENABLE:
8408 if (memorystatus_aggressive_jetsam_lenient_allowed == FALSE) {
8409 #if DEVELOPMENT || DEBUG
8410 printf("Enabling Lenient Mode\n");
8411 #endif /* DEVELOPMENT || DEBUG */
8412
8413 memorystatus_aggressive_jetsam_lenient_allowed = TRUE;
8414 memorystatus_aggressive_jetsam_lenient = TRUE;
8415 error = 0;
8416 }
8417 break;
8418 case MEMORYSTATUS_CMD_AGGRESSIVE_JETSAM_LENIENT_MODE_DISABLE:
8419 #if DEVELOPMENT || DEBUG
8420 printf("Disabling Lenient mode\n");
8421 #endif /* DEVELOPMENT || DEBUG */
8422 memorystatus_aggressive_jetsam_lenient_allowed = FALSE;
8423 memorystatus_aggressive_jetsam_lenient = FALSE;
8424 error = 0;
8425 break;
8426 case MEMORYSTATUS_CMD_PRIVILEGED_LISTENER_ENABLE:
8427 case MEMORYSTATUS_CMD_PRIVILEGED_LISTENER_DISABLE:
8428 error = memorystatus_low_mem_privileged_listener(args->command);
8429 break;
8430
8431 case MEMORYSTATUS_CMD_ELEVATED_INACTIVEJETSAMPRIORITY_ENABLE:
8432 case MEMORYSTATUS_CMD_ELEVATED_INACTIVEJETSAMPRIORITY_DISABLE:
8433 error = memorystatus_update_inactive_jetsam_priority_band(args->pid, args->command, args->flags ? TRUE : FALSE);
8434 break;
8435
8436 default:
8437 break;
8438 }
8439
8440 out:
8441 return error;
8442 }
8443
8444
8445 static int
8446 filt_memorystatusattach(struct knote *kn, __unused struct kevent_internal_s *kev)
8447 {
8448 int error;
8449
8450 kn->kn_flags |= EV_CLEAR;
8451 error = memorystatus_knote_register(kn);
8452 if (error) {
8453 kn->kn_flags = EV_ERROR;
8454 kn->kn_data = error;
8455 }
8456 return 0;
8457 }
8458
8459 static void
8460 filt_memorystatusdetach(struct knote *kn)
8461 {
8462 memorystatus_knote_unregister(kn);
8463 }
8464
8465 static int
8466 filt_memorystatus(struct knote *kn __unused, long hint)
8467 {
8468 if (hint) {
8469 switch (hint) {
8470 case kMemorystatusNoPressure:
8471 if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PRESSURE_NORMAL) {
8472 kn->kn_fflags = NOTE_MEMORYSTATUS_PRESSURE_NORMAL;
8473 }
8474 break;
8475 case kMemorystatusPressure:
8476 if (memorystatus_vm_pressure_level == kVMPressureWarning || memorystatus_vm_pressure_level == kVMPressureUrgent) {
8477 if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PRESSURE_WARN) {
8478 kn->kn_fflags = NOTE_MEMORYSTATUS_PRESSURE_WARN;
8479 }
8480 } else if (memorystatus_vm_pressure_level == kVMPressureCritical) {
8481
8482 if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PRESSURE_CRITICAL) {
8483 kn->kn_fflags = NOTE_MEMORYSTATUS_PRESSURE_CRITICAL;
8484 }
8485 }
8486 break;
8487 case kMemorystatusLowSwap:
8488 if (kn->kn_sfflags & NOTE_MEMORYSTATUS_LOW_SWAP) {
8489 kn->kn_fflags = NOTE_MEMORYSTATUS_LOW_SWAP;
8490 }
8491 break;
8492
8493 case kMemorystatusProcLimitWarn:
8494 if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_WARN) {
8495 kn->kn_fflags = NOTE_MEMORYSTATUS_PROC_LIMIT_WARN;
8496 }
8497 break;
8498
8499 case kMemorystatusProcLimitCritical:
8500 if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL) {
8501 kn->kn_fflags = NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL;
8502 }
8503 break;
8504
8505 default:
8506 break;
8507 }
8508 }
8509
8510 #if 0
8511 if (kn->kn_fflags != 0) {
8512 proc_t knote_proc = knote_get_kq(kn)->kq_p;
8513 pid_t knote_pid = knote_proc->p_pid;
8514
8515 printf("filt_memorystatus: sending kn 0x%lx (event 0x%x) for pid (%d)\n",
8516 (unsigned long)kn, kn->kn_fflags, knote_pid);
8517 }
8518 #endif
8519
8520 return (kn->kn_fflags != 0);
8521 }
8522
8523 static int
8524 filt_memorystatustouch(struct knote *kn, struct kevent_internal_s *kev)
8525 {
8526 int res;
8527 int prev_kn_sfflags = 0;
8528
8529 memorystatus_klist_lock();
8530
8531 /*
8532 * copy in new kevent settings
8533 * (saving the "desired" data and fflags).
8534 */
8535
8536 prev_kn_sfflags = kn->kn_sfflags;
8537 kn->kn_sfflags = (kev->fflags & EVFILT_MEMORYSTATUS_ALL_MASK);
8538
8539 #if !CONFIG_EMBEDDED
8540 /*
8541 * Only on desktop do we restrict notifications to
8542 * one per active/inactive state (soft limits only).
8543 */
8544 if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_WARN) {
8545 /*
8546 * Is there previous state to preserve?
8547 */
8548 if (prev_kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_WARN) {
8549 /*
8550 * This knote was previously interested in proc_limit_warn,
8551 * so yes, preserve previous state.
8552 */
8553 if (prev_kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_WARN_ACTIVE) {
8554 kn->kn_sfflags |= NOTE_MEMORYSTATUS_PROC_LIMIT_WARN_ACTIVE;
8555 }
8556 if (prev_kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_WARN_INACTIVE) {
8557 kn->kn_sfflags |= NOTE_MEMORYSTATUS_PROC_LIMIT_WARN_INACTIVE;
8558 }
8559 } else {
8560 /*
8561 * This knote was not previously interested in proc_limit_warn,
8562 * but it is now. Set both states.
8563 */
8564 kn->kn_sfflags |= NOTE_MEMORYSTATUS_PROC_LIMIT_WARN_ACTIVE;
8565 kn->kn_sfflags |= NOTE_MEMORYSTATUS_PROC_LIMIT_WARN_INACTIVE;
8566 }
8567 }
8568
8569 if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL) {
8570 /*
8571 * Is there previous state to preserve?
8572 */
8573 if (prev_kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL) {
8574 /*
8575 * This knote was previously interested in proc_limit_critical,
8576 * so yes, preserve previous state.
8577 */
8578 if (prev_kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL_ACTIVE) {
8579 kn->kn_sfflags |= NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL_ACTIVE;
8580 }
8581 if (prev_kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL_INACTIVE) {
8582 kn->kn_sfflags |= NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL_INACTIVE;
8583 }
8584 } else {
8585 /*
8586 * This knote was not previously interested in proc_limit_critical,
8587 * but it is now. Set both states.
8588 */
8589 kn->kn_sfflags |= NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL_ACTIVE;
8590 kn->kn_sfflags |= NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL_INACTIVE;
8591 }
8592 }
8593 #endif /* !CONFIG_EMBEDDED */
8594
8595 if ((kn->kn_status & KN_UDATA_SPECIFIC) == 0)
8596 kn->kn_udata = kev->udata;
8597
8598 /*
8599 * reset the output flags based on a
8600 * combination of the old events and
8601 * the new desired event list.
8602 */
8603 //kn->kn_fflags &= kn->kn_sfflags;
8604
8605 res = (kn->kn_fflags != 0);
8606
8607 memorystatus_klist_unlock();
8608
8609 return res;
8610 }
8611
8612 static int
8613 filt_memorystatusprocess(struct knote *kn, struct filt_process_s *data, struct kevent_internal_s *kev)
8614 {
8615 #pragma unused(data)
8616 int res;
8617
8618 memorystatus_klist_lock();
8619 res = (kn->kn_fflags != 0);
8620 if (res) {
8621 *kev = kn->kn_kevent;
8622 kn->kn_flags |= EV_CLEAR; /* automatic */
8623 kn->kn_fflags = 0;
8624 kn->kn_data = 0;
8625 }
8626 memorystatus_klist_unlock();
8627
8628 return res;
8629 }
8630
8631 static void
8632 memorystatus_klist_lock(void) {
8633 lck_mtx_lock(&memorystatus_klist_mutex);
8634 }
8635
8636 static void
8637 memorystatus_klist_unlock(void) {
8638 lck_mtx_unlock(&memorystatus_klist_mutex);
8639 }
8640
8641 void
8642 memorystatus_kevent_init(lck_grp_t *grp, lck_attr_t *attr) {
8643 lck_mtx_init(&memorystatus_klist_mutex, grp, attr);
8644 klist_init(&memorystatus_klist);
8645 }
8646
8647 int
8648 memorystatus_knote_register(struct knote *kn) {
8649 int error = 0;
8650
8651 memorystatus_klist_lock();
8652
8653 /*
8654 * Support only userspace visible flags.
8655 */
8656 if ((kn->kn_sfflags & EVFILT_MEMORYSTATUS_ALL_MASK) == (unsigned int) kn->kn_sfflags) {
8657
8658 #if !CONFIG_EMBEDDED
8659 if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_WARN) {
8660 kn->kn_sfflags |= NOTE_MEMORYSTATUS_PROC_LIMIT_WARN_ACTIVE;
8661 kn->kn_sfflags |= NOTE_MEMORYSTATUS_PROC_LIMIT_WARN_INACTIVE;
8662 }
8663
8664 if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL) {
8665 kn->kn_sfflags |= NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL_ACTIVE;
8666 kn->kn_sfflags |= NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL_INACTIVE;
8667 }
8668 #endif /* !CONFIG_EMBEDDED */
8669
8670 KNOTE_ATTACH(&memorystatus_klist, kn);
8671
8672 } else {
8673 error = ENOTSUP;
8674 }
8675
8676 memorystatus_klist_unlock();
8677
8678 return error;
8679 }
8680
8681 void
8682 memorystatus_knote_unregister(struct knote *kn __unused) {
8683 memorystatus_klist_lock();
8684 KNOTE_DETACH(&memorystatus_klist, kn);
8685 memorystatus_klist_unlock();
8686 }
8687
8688
8689 #if 0
8690 #if CONFIG_JETSAM && VM_PRESSURE_EVENTS
8691 static boolean_t
8692 memorystatus_issue_pressure_kevent(boolean_t pressured) {
8693 memorystatus_klist_lock();
8694 KNOTE(&memorystatus_klist, pressured ? kMemorystatusPressure : kMemorystatusNoPressure);
8695 memorystatus_klist_unlock();
8696 return TRUE;
8697 }
8698 #endif /* CONFIG_JETSAM && VM_PRESSURE_EVENTS */
8699 #endif /* 0 */
8700
8701 /* Coalition support */
8702
8703 /* sorting info for a particular priority bucket */
8704 typedef struct memstat_sort_info {
8705 coalition_t msi_coal;
8706 uint64_t msi_page_count;
8707 pid_t msi_pid;
8708 int msi_ntasks;
8709 } memstat_sort_info_t;
8710
8711 /*
8712 * qsort from smallest page count to largest page count
8713 *
8714 * return < 0 for a < b
8715 * 0 for a == b
8716 * > 0 for a > b
8717 */
8718 static int memstat_asc_cmp(const void *a, const void *b)
8719 {
8720 const memstat_sort_info_t *msA = (const memstat_sort_info_t *)a;
8721 const memstat_sort_info_t *msB = (const memstat_sort_info_t *)b;
8722
8723 return (int)((uint64_t)msA->msi_page_count - (uint64_t)msB->msi_page_count);
8724 }
8725
8726 /*
8727 * Return the number of pids rearranged during this sort.
8728 */
8729 static int
8730 memorystatus_sort_by_largest_coalition_locked(unsigned int bucket_index, int coal_sort_order)
8731 {
8732 #define MAX_SORT_PIDS 80
8733 #define MAX_COAL_LEADERS 10
8734
8735 unsigned int b = bucket_index;
8736 int nleaders = 0;
8737 int ntasks = 0;
8738 proc_t p = NULL;
8739 coalition_t coal = COALITION_NULL;
8740 int pids_moved = 0;
8741 int total_pids_moved = 0;
8742 int i;
8743
8744 /*
8745 * The system is typically under memory pressure when in this
8746 * path, hence, we want to avoid dynamic memory allocation.
8747 */
8748 memstat_sort_info_t leaders[MAX_COAL_LEADERS];
8749 pid_t pid_list[MAX_SORT_PIDS];
8750
8751 if (bucket_index >= MEMSTAT_BUCKET_COUNT) {
8752 return(0);
8753 }
8754
8755 /*
8756 * Clear the array that holds coalition leader information
8757 */
8758 for (i=0; i < MAX_COAL_LEADERS; i++) {
8759 leaders[i].msi_coal = COALITION_NULL;
8760 leaders[i].msi_page_count = 0; /* will hold total coalition page count */
8761 leaders[i].msi_pid = 0; /* will hold coalition leader pid */
8762 leaders[i].msi_ntasks = 0; /* will hold the number of tasks in a coalition */
8763 }
8764
8765 p = memorystatus_get_first_proc_locked(&b, FALSE);
8766 while (p) {
8767 if (coalition_is_leader(p->task, COALITION_TYPE_JETSAM, &coal)) {
8768 if (nleaders < MAX_COAL_LEADERS) {
8769 int coal_ntasks = 0;
8770 uint64_t coal_page_count = coalition_get_page_count(coal, &coal_ntasks);
8771 leaders[nleaders].msi_coal = coal;
8772 leaders[nleaders].msi_page_count = coal_page_count;
8773 leaders[nleaders].msi_pid = p->p_pid; /* the coalition leader */
8774 leaders[nleaders].msi_ntasks = coal_ntasks;
8775 nleaders++;
8776 } else {
8777 /*
8778 * We've hit MAX_COAL_LEADERS meaning we can handle no more coalitions.
8779 * Abandoned coalitions will linger at the tail of the priority band
8780 * when this sort session ends.
8781 * TODO: should this be an assert?
8782 */
8783 printf("%s: WARNING: more than %d leaders in priority band [%d]\n",
8784 __FUNCTION__, MAX_COAL_LEADERS, bucket_index);
8785 break;
8786 }
8787 }
8788 p=memorystatus_get_next_proc_locked(&b, p, FALSE);
8789 }
8790
8791 if (nleaders == 0) {
8792 /* Nothing to sort */
8793 return(0);
8794 }
8795
8796 /*
8797 * Sort the coalition leader array, from smallest coalition page count
8798 * to largest coalition page count. When inserted in the priority bucket,
8799 * smallest coalition is handled first, resulting in the last to be jetsammed.
8800 */
8801 if (nleaders > 1) {
8802 qsort(leaders, nleaders, sizeof(memstat_sort_info_t), memstat_asc_cmp);
8803 }
8804
8805 #if 0
8806 for (i = 0; i < nleaders; i++) {
8807 printf("%s: coal_leader[%d of %d] pid[%d] pages[%llu] ntasks[%d]\n",
8808 __FUNCTION__, i, nleaders, leaders[i].msi_pid, leaders[i].msi_page_count,
8809 leaders[i].msi_ntasks);
8810 }
8811 #endif
8812
8813 /*
8814 * During coalition sorting, processes in a priority band are rearranged
8815 * by being re-inserted at the head of the queue. So, when handling a
8816 * list, the first process that gets moved to the head of the queue,
8817 * ultimately gets pushed toward the queue tail, and hence, jetsams last.
8818 *
8819 * So, for example, the coalition leader is expected to jetsam last,
8820 * after its coalition members. Therefore, the coalition leader is
8821 * inserted at the head of the queue first.
8822 *
8823 * After processing a coalition, the jetsam order is as follows:
8824 * undefs(jetsam first), extensions, xpc services, leader(jetsam last)
8825 */
8826
8827 /*
8828 * Coalition members are rearranged in the priority bucket here,
8829 * based on their coalition role.
8830 */
8831 total_pids_moved = 0;
8832 for (i=0; i < nleaders; i++) {
8833
8834 /* a bit of bookkeeping */
8835 pids_moved = 0;
8836
8837 /* Coalition leaders are jetsammed last, so move into place first */
8838 pid_list[0] = leaders[i].msi_pid;
8839 pids_moved += memorystatus_move_list_locked(bucket_index, pid_list, 1);
8840
8841 /* xpc services should jetsam after extensions */
8842 ntasks = coalition_get_pid_list (leaders[i].msi_coal, COALITION_ROLEMASK_XPC,
8843 coal_sort_order, pid_list, MAX_SORT_PIDS);
8844
8845 if (ntasks > 0) {
8846 pids_moved += memorystatus_move_list_locked(bucket_index, pid_list,
8847 (ntasks <= MAX_SORT_PIDS ? ntasks : MAX_SORT_PIDS));
8848 }
8849
8850 /* extensions should jetsam after unmarked processes */
8851 ntasks = coalition_get_pid_list (leaders[i].msi_coal, COALITION_ROLEMASK_EXT,
8852 coal_sort_order, pid_list, MAX_SORT_PIDS);
8853
8854 if (ntasks > 0) {
8855 pids_moved += memorystatus_move_list_locked(bucket_index, pid_list,
8856 (ntasks <= MAX_SORT_PIDS ? ntasks : MAX_SORT_PIDS));
8857 }
8858
8859 /* undefined coalition members should be the first to jetsam */
8860 ntasks = coalition_get_pid_list (leaders[i].msi_coal, COALITION_ROLEMASK_UNDEF,
8861 coal_sort_order, pid_list, MAX_SORT_PIDS);
8862
8863 if (ntasks > 0) {
8864 pids_moved += memorystatus_move_list_locked(bucket_index, pid_list,
8865 (ntasks <= MAX_SORT_PIDS ? ntasks : MAX_SORT_PIDS));
8866 }
8867
8868 #if 0
8869 if (pids_moved == leaders[i].msi_ntasks) {
8870 /*
8871 * All the pids in the coalition were found in this band.
8872 */
8873 printf("%s: pids_moved[%d] equal total coalition ntasks[%d] \n", __FUNCTION__,
8874 pids_moved, leaders[i].msi_ntasks);
8875 } else if (pids_moved > leaders[i].msi_ntasks) {
8876 /*
8877 * Apparently new coalition members showed up during the sort?
8878 */
8879 printf("%s: pids_moved[%d] were greater than expected coalition ntasks[%d] \n", __FUNCTION__,
8880 pids_moved, leaders[i].msi_ntasks);
8881 } else {
8882 /*
8883 * Apparently not all the pids in the coalition were found in this band?
8884 */
8885 printf("%s: pids_moved[%d] were less than expected coalition ntasks[%d] \n", __FUNCTION__,
8886 pids_moved, leaders[i].msi_ntasks);
8887 }
8888 #endif
8889
8890 total_pids_moved += pids_moved;
8891
8892 } /* end for */
8893
8894 return(total_pids_moved);
8895 }
8896
8897
8898 /*
8899 * Traverse a list of pids, searching for each within the priority band provided.
8900 * If pid is found, move it to the front of the priority band.
8901 * Never searches outside the priority band provided.
8902 *
8903 * Input:
8904 * bucket_index - jetsam priority band.
8905 * pid_list - pointer to a list of pids.
8906 * list_sz - number of pids in the list.
8907 *
8908 * Pid list ordering is important in that,
8909 * pid_list[n] is expected to jetsam ahead of pid_list[n+1].
8910 * The sort_order is set by the coalition default.
8911 *
8912 * Return:
8913 * the number of pids found and hence moved within the priority band.
8914 */
8915 static int
8916 memorystatus_move_list_locked(unsigned int bucket_index, pid_t *pid_list, int list_sz)
8917 {
8918 memstat_bucket_t *current_bucket;
8919 int i;
8920 int found_pids = 0;
8921
8922 if ((pid_list == NULL) || (list_sz <= 0)) {
8923 return(0);
8924 }
8925
8926 if (bucket_index >= MEMSTAT_BUCKET_COUNT) {
8927 return(0);
8928 }
8929
8930 current_bucket = &memstat_bucket[bucket_index];
8931 for (i=0; i < list_sz; i++) {
8932 unsigned int b = bucket_index;
8933 proc_t p = NULL;
8934 proc_t aProc = NULL;
8935 pid_t aPid;
8936 int list_index;
8937
8938 list_index = ((list_sz - 1) - i);
8939 aPid = pid_list[list_index];
8940
8941 /* never search beyond bucket_index provided */
8942 p = memorystatus_get_first_proc_locked(&b, FALSE);
8943 while (p) {
8944 if (p->p_pid == aPid) {
8945 aProc = p;
8946 break;
8947 }
8948 p = memorystatus_get_next_proc_locked(&b, p, FALSE);
8949 }
8950
8951 if (aProc == NULL) {
8952 /* pid not found in this band, just skip it */
8953 continue;
8954 } else {
8955 TAILQ_REMOVE(&current_bucket->list, aProc, p_memstat_list);
8956 TAILQ_INSERT_HEAD(&current_bucket->list, aProc, p_memstat_list);
8957 found_pids++;
8958 }
8959 }
8960 return(found_pids);
8961 }
8962
8963 int
8964 memorystatus_get_proccnt_upto_priority(int32_t max_bucket_index)
8965 {
8966 int32_t i = JETSAM_PRIORITY_IDLE;
8967 int count = 0;
8968
8969 if (max_bucket_index >= MEMSTAT_BUCKET_COUNT) {
8970 return(-1);
8971 }
8972
8973 while(i <= max_bucket_index) {
8974 count += memstat_bucket[i++].count;
8975 }
8976
8977 return count;
8978 }
8979
8980 int
8981 memorystatus_update_priority_for_appnap(proc_t p, boolean_t is_appnap)
8982 {
8983 #if !CONFIG_JETSAM
8984 if (!p || (!isApp(p)) || (p->p_memstat_state & P_MEMSTAT_INTERNAL)) {
8985 /*
8986 * Ineligible processes OR system processes e.g. launchd.
8987 */
8988 return -1;
8989 }
8990
8991 /*
8992 * For macOS only:
8993 * We would like to use memorystatus_update() here to move the processes
8994 * within the bands. Unfortunately memorystatus_update() calls
8995 * memorystatus_update_priority_locked() which uses any band transitions
8996 * as an indication to modify ledgers. For that it needs the task lock
8997 * and since we came into this function with the task lock held, we'll deadlock.
8998 *
8999 * Unfortunately we can't completely disable ledger updates because we still
9000 * need the ledger updates for a subset of processes i.e. daemons.
9001 * When all processes on all platforms support memory limits, we can simply call
9002 * memorystatus_update().
9003
9004 * It also has some logic to deal with 'aging' which, currently, is only applicable
9005 * on CONFIG_JETSAM configs. So, till every platform has CONFIG_JETSAM we'll need
9006 * to do this explicit band transition.
9007 */
9008
9009 memstat_bucket_t *current_bucket, *new_bucket;
9010 int32_t priority = 0;
9011
9012 proc_list_lock();
9013
9014 if (((p->p_listflag & P_LIST_EXITED) != 0) ||
9015 (p->p_memstat_state & (P_MEMSTAT_ERROR | P_MEMSTAT_TERMINATED))) {
9016 /*
9017 * If the process is on its way out OR
9018 * jetsam has alread tried and failed to kill this process,
9019 * let's skip the whole jetsam band transition.
9020 */
9021 proc_list_unlock();
9022 return(0);
9023 }
9024
9025 if (is_appnap) {
9026 current_bucket = &memstat_bucket[p->p_memstat_effectivepriority];
9027 new_bucket = &memstat_bucket[JETSAM_PRIORITY_IDLE];
9028 priority = JETSAM_PRIORITY_IDLE;
9029 } else {
9030 if (p->p_memstat_effectivepriority != JETSAM_PRIORITY_IDLE) {
9031 /*
9032 * It is possible that someone pulled this process
9033 * out of the IDLE band without updating its app-nap
9034 * parameters.
9035 */
9036 proc_list_unlock();
9037 return (0);
9038 }
9039
9040 current_bucket = &memstat_bucket[JETSAM_PRIORITY_IDLE];
9041 new_bucket = &memstat_bucket[p->p_memstat_requestedpriority];
9042 priority = p->p_memstat_requestedpriority;
9043 }
9044
9045 TAILQ_REMOVE(&current_bucket->list, p, p_memstat_list);
9046 current_bucket->count--;
9047
9048 TAILQ_INSERT_TAIL(&new_bucket->list, p, p_memstat_list);
9049 new_bucket->count++;
9050
9051 /*
9052 * Record idle start or idle delta.
9053 */
9054 if (p->p_memstat_effectivepriority == priority) {
9055 /*
9056 * This process is not transitioning between
9057 * jetsam priority buckets. Do nothing.
9058 */
9059 } else if (p->p_memstat_effectivepriority == JETSAM_PRIORITY_IDLE) {
9060 uint64_t now;
9061 /*
9062 * Transitioning out of the idle priority bucket.
9063 * Record idle delta.
9064 */
9065 assert(p->p_memstat_idle_start != 0);
9066 now = mach_absolute_time();
9067 if (now > p->p_memstat_idle_start) {
9068 p->p_memstat_idle_delta = now - p->p_memstat_idle_start;
9069 }
9070 } else if (priority == JETSAM_PRIORITY_IDLE) {
9071 /*
9072 * Transitioning into the idle priority bucket.
9073 * Record idle start.
9074 */
9075 p->p_memstat_idle_start = mach_absolute_time();
9076 }
9077
9078 p->p_memstat_effectivepriority = priority;
9079
9080 proc_list_unlock();
9081
9082 return (0);
9083
9084 #else /* !CONFIG_JETSAM */
9085 #pragma unused(p)
9086 #pragma unused(is_appnap)
9087 return -1;
9088 #endif /* !CONFIG_JETSAM */
9089 }
mirror of XNU