xnu-4903.221.2.tar.gz

[apple/xnu.git] / bsd / kern / kern_overrides.c

diff --git a/bsd/kern/kern_overrides.c b/bsd/kern/kern_overrides.c
index e2055aae6ac91aa914e7b2855d5f8014be76159d..0d7ece73db85640b179cef3a84c02a73bfb11cd5 100644 (file)
--- a/bsd/kern/kern_overrides.c
+++ b/bsd/kern/kern_overrides.c
@@ -26,10 +26,6 @@
  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 

-/*
- * System Overrides syscall implementation
- */
-

 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>


@@ -39,12 +35,11 @@
 #include <sys/kauth.h>
 #include <sys/unistd.h>
 #include <sys/priv.h>
 #include <sys/kauth.h>
 #include <sys/unistd.h>
 #include <sys/priv.h>

-#include <security/audit/audit.h>

 
 #include <mach/mach_types.h>
 #include <mach/vm_param.h>
 #include <kern/task.h>
 
 #include <mach/mach_types.h>
 #include <mach/vm_param.h>
 #include <kern/task.h>

-#include <kern/lock.h>
+#include <kern/locks.h>

 #include <kern/assert.h>
 #include <kern/sched_prim.h>
 
 #include <kern/assert.h>
 #include <kern/sched_prim.h>
 


@@ -53,6 +48,7 @@
 #include <sys/kdebug.h>
 #include <sys/sysproto.h>
 #include <sys/msgbuf.h>
 #include <sys/kdebug.h>
 #include <sys/sysproto.h>
 #include <sys/msgbuf.h>

+#include <sys/kern_memorystatus.h>

 
 /* Mutex for global system override state */
 static lck_mtx_t       sys_override_lock;
 
 /* Mutex for global system override state */
 static lck_mtx_t       sys_override_lock;


@@ -60,22 +56,46 @@ static lck_grp_t        *sys_override_mtx_grp;
 static lck_attr_t       *sys_override_mtx_attr;
 static lck_grp_attr_t   *sys_override_mtx_grp_attr;
 
 static lck_attr_t       *sys_override_mtx_attr;
 static lck_grp_attr_t   *sys_override_mtx_grp_attr;
 

-/* Assertion counts for system properties */
+/* 
+ * Assertion counts for system properties (add new ones for each new mechanism)
+ *
+ * The assertion count management for system overrides is as follows:
+ *
+ * - All assertion counts are protected by the sys_override_lock.
+ *
+ * - Each caller of system_override() increments the assertion count for the 
+ *   mechanism it specified in the flags. The caller then blocks for the 
+ *   timeout specified in the system call. 
+ *
+ * - At the end of the timeout, the caller thread wakes up and decrements the 
+ *   assertion count for the mechanism it originally took an assertion on.
+ *
+ * - If another caller calls the system_override() to disable the override 
+ *   for a mechanism, it simply disables the mechanism without changing any 
+ *   assertion counts. That way, the assertion counts are properly balanced. 
+ *
+ * One thing to note is that a SYS_OVERRIDE_DISABLE disables the overrides 
+ * for a mechanism irrespective of how many clients requested that override.
+ * That makes the implementation simpler and avoids keeping a lot of process 
+ * specific state in the kernel.
+ *
+ */

 static int64_t         io_throttle_assert_cnt;
 static int64_t         cpu_throttle_assert_cnt;
 static int64_t         io_throttle_assert_cnt;
 static int64_t         cpu_throttle_assert_cnt;

+static int64_t         fast_jetsam_assert_cnt;

 
 /* Wait Channel for system override */
 static uint64_t                sys_override_wait;
 
 
 /* Wait Channel for system override */
 static uint64_t                sys_override_wait;
 

-/* Forward Declarations */
-static void enable_system_override(uint64_t flags);
-static void disable_system_override(uint64_t flags);
-static __attribute__((noinline)) void PROCESS_OVERRIDING_SYSTEM_DEFAULTS(uint64_t timeout);
+/* Global variable to indicate if system_override is enabled */
+int                    sys_override_enabled;

 
 

-/***************************** system_override ********************/
-/*
- * int system_override(uint64_t timeout, uint64_t flags);
- */
+/* Helper routines */
+static void system_override_begin(uint64_t flags);
+static void system_override_end(uint64_t flags);
+static void system_override_abort(uint64_t flags);
+static void system_override_callouts(uint64_t flags, boolean_t enable_override);
+static __attribute__((noinline)) void PROCESS_OVERRIDING_SYSTEM_DEFAULTS(uint64_t timeout);

 
 void
 init_system_override()
 
 void
 init_system_override()


@@ -84,7 +104,8 @@ init_system_override()
        sys_override_mtx_grp = lck_grp_alloc_init("system_override", sys_override_mtx_grp_attr);
        sys_override_mtx_attr = lck_attr_alloc_init();
        lck_mtx_init(&sys_override_lock, sys_override_mtx_grp, sys_override_mtx_attr);
        sys_override_mtx_grp = lck_grp_alloc_init("system_override", sys_override_mtx_grp_attr);
        sys_override_mtx_attr = lck_attr_alloc_init();
        lck_mtx_init(&sys_override_lock, sys_override_mtx_grp, sys_override_mtx_attr);

-       io_throttle_assert_cnt = cpu_throttle_assert_cnt = 0;
+       io_throttle_assert_cnt = cpu_throttle_assert_cnt = fast_jetsam_assert_cnt = 0;
+       sys_override_enabled = 1;

 }
 
 /* system call implementation */
 }
 
 /* system call implementation */


@@ -100,19 +121,28 @@ system_override(__unused struct proc *p, struct system_override_args * uap, __un
                goto out;
        }       
 
                goto out;
        }       
 

-       /* Check to see if some flags are specified. Zero flags are invalid. */
-       if ((flags == 0) || ((flags & ~SYS_OVERRIDE_FLAGS_MASK) != 0)) {
+       /* Check to see if sane flags are specified. */
+       if ((flags & ~SYS_OVERRIDE_FLAGS_MASK) != 0) {

                error = EINVAL;
                goto out;
        }
 
                error = EINVAL;
                goto out;
        }
 

-       lck_mtx_lock(&sys_override_lock);
-
-       enable_system_override(flags);
+       /* Make sure that the system override syscall has been initialized */
+       if (!sys_override_enabled) {
+               error = EINVAL;
+               goto out;
+       }

 
 

-       PROCESS_OVERRIDING_SYSTEM_DEFAULTS(timeout);
+       lck_mtx_lock(&sys_override_lock);

 
 

-       disable_system_override(flags);
+       if (flags & SYS_OVERRIDE_DISABLE) {
+               flags &= ~SYS_OVERRIDE_DISABLE;
+               system_override_abort(flags);
+       } else {
+               system_override_begin(flags);
+               PROCESS_OVERRIDING_SYSTEM_DEFAULTS(timeout);
+               system_override_end(flags);
+       }

 
        lck_mtx_unlock(&sys_override_lock);
 
 
        lck_mtx_unlock(&sys_override_lock);
 


@@ -121,62 +151,164 @@ out:
 }
 
 /*
 }
 
 /*

- * Call for enabling global system override.
- * This should be called only with the sys_override_lock held.
+ * Helper routines for enabling/disabling system overrides for various mechanisms.
+ * These routines should be called with the sys_override_lock held. Each subsystem 
+ * which is hooked into the override service provides two routines:
+ * 
+ * - void sys_override_foo_init(void);
+ * Routine to initialize the subsystem or the data needed for the override to work.
+ * This routine is optional and if a subsystem needs it, it should be invoked from 
+ * init_system_override().
+ * 
+ * - void sys_override_foo(boolean_t enable_override);
+ * Routine to enable/disable the override mechanism for that subsystem. A value of 
+ * true indicates that the mechanism should be overridden and the special behavior 
+ * should begin. A false value indicates that the subsystem should return to default 
+ * behavior. This routine is mandatory and should be invoked as part of the helper 
+ * routines if the flags passed in the syscall match the subsystem. Also, this 
+ * routine should preferably be idempotent.
+ */
+
+static void
+system_override_callouts(uint64_t flags, boolean_t enable_override)
+{
+       switch (flags) {
+               case SYS_OVERRIDE_IO_THROTTLE:
+                       if (enable_override) {
+                               KERNEL_DEBUG_CONSTANT(FSDBG_CODE(DBG_THROTTLE, IO_THROTTLE_DISABLE) | DBG_FUNC_START, 
+                                       current_proc()->p_pid, 0, 0, 0, 0);
+                       } else {
+                               KERNEL_DEBUG_CONSTANT(FSDBG_CODE(DBG_THROTTLE, IO_THROTTLE_DISABLE) | DBG_FUNC_END, 
+                                       current_proc()->p_pid, 0, 0, 0, 0);
+                       }
+                       sys_override_io_throttle(enable_override);
+                       break;
+
+               case SYS_OVERRIDE_CPU_THROTTLE:
+                       if (enable_override) {
+                               KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_CPU_THROTTLE_DISABLE) | DBG_FUNC_START, 
+                                       current_proc()->p_pid, 0, 0, 0, 0);
+                       } else {
+                               KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_CPU_THROTTLE_DISABLE) | DBG_FUNC_END, 
+                                       current_proc()->p_pid, 0, 0, 0, 0);
+                       }
+                       sys_override_cpu_throttle(enable_override);
+                       break;
+
+               case SYS_OVERRIDE_FAST_JETSAM:
+                       if (enable_override) {
+                               KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_FAST_JETSAM) | DBG_FUNC_START, 
+                                       current_proc()->p_pid, 0, 0, 0, 0);
+                       } else {
+                               KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_FAST_JETSAM) | DBG_FUNC_END, 
+                                       current_proc()->p_pid, 0, 0, 0, 0);
+                       }
+#if CONFIG_JETSAM
+                       memorystatus_fast_jetsam_override(enable_override);
+#endif /* CONFIG_JETSAM */
+                       break;
+
+               default:
+                       panic("Unknown option to system_override_callouts(): %llu\n", flags);
+       }
+}
+
+/*
+ * system_override_begin(uint64_t flags)
+ *
+ * Routine to start a system override if the assertion count 
+ * transitions from 0->1 for a specified mechanism.

  */
 static void
  */
 static void

-enable_system_override(uint64_t flags)
+system_override_begin(uint64_t flags)

 {
 {

+       lck_mtx_assert(&sys_override_lock, LCK_MTX_ASSERT_OWNED);

        
        if (flags & SYS_OVERRIDE_IO_THROTTLE) {
                if (io_throttle_assert_cnt == 0) {
        
        if (flags & SYS_OVERRIDE_IO_THROTTLE) {
                if (io_throttle_assert_cnt == 0) {

-                       /* Disable I/O Throttling */
-                       printf("Process %s [%d] disabling system-wide I/O Throttling\n", current_proc()->p_comm, current_proc()->p_pid);
-                       KERNEL_DEBUG_CONSTANT(FSDBG_CODE(DBG_THROTTLE, IO_THROTTLE_DISABLE) | DBG_FUNC_START, current_proc()->p_pid, 0, 0, 0, 0);
-                       sys_override_io_throttle(THROTTLE_IO_DISABLE);
+                       system_override_callouts(SYS_OVERRIDE_IO_THROTTLE, true);

                }
                io_throttle_assert_cnt++;
        }
        
        if (flags & SYS_OVERRIDE_CPU_THROTTLE) {
                if (cpu_throttle_assert_cnt == 0) {
                }
                io_throttle_assert_cnt++;
        }
        
        if (flags & SYS_OVERRIDE_CPU_THROTTLE) {
                if (cpu_throttle_assert_cnt == 0) {

-                       /* Disable CPU Throttling */
-                       printf("Process %s [%d] disabling system-wide CPU Throttling\n", current_proc()->p_comm, current_proc()->p_pid);
-                       KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_CPU_THROTTLE_DISABLE) | DBG_FUNC_START, current_proc()->p_pid, 0, 0, 0, 0);
-                       sys_override_cpu_throttle(CPU_THROTTLE_DISABLE);
+                       system_override_callouts(SYS_OVERRIDE_CPU_THROTTLE, true);

                }
                cpu_throttle_assert_cnt++;
        }
                }
                cpu_throttle_assert_cnt++;
        }

+       
+       if (flags & SYS_OVERRIDE_FAST_JETSAM) {
+               if (fast_jetsam_assert_cnt == 0) {
+                       system_override_callouts(SYS_OVERRIDE_FAST_JETSAM, true);
+               }
+               fast_jetsam_assert_cnt++;
+       }

 
 }
 
 /*
 
 }
 
 /*

- * Call for disabling global system override.
- * This should be called only with the sys_override_lock held.
+ * system_override_end(uint64_t flags)
+ *
+ * Routine to end a system override if the assertion count 
+ * transitions from 1->0 for a specified mechanism.

  */
 static void
  */
 static void

-disable_system_override(uint64_t flags)
+system_override_end(uint64_t flags)

 {
 
 {
 

+       lck_mtx_assert(&sys_override_lock, LCK_MTX_ASSERT_OWNED);
+       

        if (flags & SYS_OVERRIDE_IO_THROTTLE) {
                assert(io_throttle_assert_cnt > 0);
                io_throttle_assert_cnt--;
                if (io_throttle_assert_cnt == 0) {
        if (flags & SYS_OVERRIDE_IO_THROTTLE) {
                assert(io_throttle_assert_cnt > 0);
                io_throttle_assert_cnt--;
                if (io_throttle_assert_cnt == 0) {

-                       /* Enable I/O Throttling */
-                       KERNEL_DEBUG_CONSTANT(FSDBG_CODE(DBG_THROTTLE, IO_THROTTLE_DISABLE) | DBG_FUNC_END, current_proc()->p_pid, 0, 0, 0, 0);
-                       sys_override_io_throttle(THROTTLE_IO_ENABLE);
+                       system_override_callouts(SYS_OVERRIDE_IO_THROTTLE, false);

                }
        }
 
        if (flags & SYS_OVERRIDE_CPU_THROTTLE) {
                assert(cpu_throttle_assert_cnt > 0);
                cpu_throttle_assert_cnt--;
                }
        }
 
        if (flags & SYS_OVERRIDE_CPU_THROTTLE) {
                assert(cpu_throttle_assert_cnt > 0);
                cpu_throttle_assert_cnt--;

-                       if (cpu_throttle_assert_cnt == 0) {
-                       /* Enable CPU Throttling */
-                       KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_CPU_THROTTLE_DISABLE) | DBG_FUNC_END, current_proc()->p_pid, 0, 0, 0, 0);
-                       sys_override_cpu_throttle(CPU_THROTTLE_ENABLE);
+               if (cpu_throttle_assert_cnt == 0) {
+                       system_override_callouts(SYS_OVERRIDE_CPU_THROTTLE, false);
+               }
+       }
+
+       if (flags & SYS_OVERRIDE_FAST_JETSAM) {
+               assert(fast_jetsam_assert_cnt > 0);
+               fast_jetsam_assert_cnt--;
+               if (fast_jetsam_assert_cnt == 0) {
+                       system_override_callouts(SYS_OVERRIDE_FAST_JETSAM, false);

                }
        }
                }
        }

+
+}
+
+/*
+ * system_override_abort(uint64_t flags)
+ *
+ * Routine to abort a system override (if one was active) 
+ * irrespective of the assertion counts and number of blocked 
+ * requestors.
+ */
+static void
+system_override_abort(uint64_t flags)
+{
+       
+       lck_mtx_assert(&sys_override_lock, LCK_MTX_ASSERT_OWNED);
+       
+       if ((flags & SYS_OVERRIDE_IO_THROTTLE) && (io_throttle_assert_cnt > 0)) {
+               system_override_callouts(SYS_OVERRIDE_IO_THROTTLE, false);
+       }
+
+       if ((flags & SYS_OVERRIDE_CPU_THROTTLE) && (cpu_throttle_assert_cnt > 0))  {
+               system_override_callouts(SYS_OVERRIDE_CPU_THROTTLE, false);
+       }
+
+       if ((flags & SYS_OVERRIDE_FAST_JETSAM) && (fast_jetsam_assert_cnt > 0))  {
+               system_override_callouts(SYS_OVERRIDE_FAST_JETSAM, false);
+       }

 }
 
 static __attribute__((noinline)) void
 }
 
 static __attribute__((noinline)) void