#include <kern/macro_help.h>
#include <kern/timer_call.h>
#include <kern/ast.h>
+#include <kern/kalloc.h>
+#include <kern/bits.h>
#define NRQS 128 /* 128 levels per run queue */
-#define NRQBM (NRQS / 32) /* number of words per bit map */
#define MAXPRI (NRQS-1)
-#define MINPRI IDLEPRI /* lowest legal priority schedulable */
-#define IDLEPRI 0 /* idle thread priority */
+#define MINPRI 0 /* lowest legal priority schedulable */
+#define IDLEPRI MINPRI /* idle thread priority */
+#define NOPRI -1
/*
* High-level priority assignments
#define BASEPRI_REALTIME (MAXPRI - (NRQS / 4) + 1) /* 96 */
#define MAXPRI_KERNEL (BASEPRI_REALTIME - 1) /* 95 */
-#define BASEPRI_PREEMPT (MAXPRI_KERNEL - 2) /* 93 */
-#define BASEPRI_KERNEL (MINPRI_KERNEL + 1) /* 81 */
-#define MINPRI_KERNEL (MAXPRI_KERNEL - (NRQS / 8) + 1) /* 80 */
+#define BASEPRI_PREEMPT (MAXPRI_KERNEL - 2) /* 93 */
+#define BASEPRI_KERNEL (MINPRI_KERNEL + 1) /* 81 */
+#define MINPRI_KERNEL (MAXPRI_KERNEL - (NRQS / 8) + 1) /* 80 */
-#define MAXPRI_RESERVED (MINPRI_KERNEL - 1) /* 79 */
+#define MAXPRI_RESERVED (MINPRI_KERNEL - 1) /* 79 */
#define BASEPRI_GRAPHICS (MAXPRI_RESERVED - 3) /* 76 */
-#define MINPRI_RESERVED (MAXPRI_RESERVED - (NRQS / 8) + 1) /* 64 */
+#define MINPRI_RESERVED (MAXPRI_RESERVED - (NRQS / 8) + 1) /* 64 */
-#define MAXPRI_USER (MINPRI_RESERVED - 1) /* 63 */
+#define MAXPRI_USER (MINPRI_RESERVED - 1) /* 63 */
#define BASEPRI_CONTROL (BASEPRI_DEFAULT + 17) /* 48 */
#define BASEPRI_FOREGROUND (BASEPRI_DEFAULT + 16) /* 47 */
#define BASEPRI_BACKGROUND (BASEPRI_DEFAULT + 15) /* 46 */
#define BASEPRI_DEFAULT (MAXPRI_USER - (NRQS / 4)) /* 31 */
#define MAXPRI_SUPPRESSED (BASEPRI_DEFAULT - 3) /* 28 */
#define BASEPRI_UTILITY (BASEPRI_DEFAULT - 11) /* 20 */
-#define MAXPRI_THROTTLE (MINPRI + 4) /* 4 */
-#define MINPRI_USER MINPRI /* 0 */
+#define MAXPRI_THROTTLE (MINPRI + 4) /* 4 */
+#define MINPRI_USER MINPRI /* 0 */
-#define DEPRESSPRI MINPRI /* depress priority */
+#define DEPRESSPRI MINPRI /* depress priority */
#define MAXPRI_PROMOTE (MAXPRI_KERNEL) /* ceiling for mutex promotion */
/* Type used for thread->sched_mode and saved_mode */
TH_MODE_TIMESHARE, /* use timesharing algorithm */
} sched_mode_t;
+/* Buckets used for load calculation */
+typedef enum {
+ TH_BUCKET_RUN = 0, /* All runnable threads */
+ TH_BUCKET_FIXPRI, /* Fixed-priority */
+ TH_BUCKET_SHARE_FG, /* Timeshare thread above BASEPRI_UTILITY */
+ TH_BUCKET_SHARE_UT, /* Timeshare thread between BASEPRI_UTILITY and MAXPRI_THROTTLE */
+ TH_BUCKET_SHARE_BG, /* Timeshare thread between MAXPRI_THROTTLE and MINPRI */
+ TH_BUCKET_MAX,
+} sched_bucket_t;
+
/*
* Macro to check for invalid priorities.
*/
struct run_queue {
int highq; /* highest runnable queue */
- int bitmap[NRQBM]; /* run queue bitmap array */
+ bitmap_t bitmap[BITMAP_LEN(NRQS)]; /* run queue bitmap array */
int count; /* # of threads total */
int urgency; /* level of preemption urgency */
queue_head_t queues[NRQS]; /* one for each priority */
struct runq_stats runq_stats;
};
+inline static void
+rq_bitmap_set(bitmap_t *map, u_int n)
+{
+ assert(n < NRQS);
+ bitmap_set(map, n);
+}
+
+inline static void
+rq_bitmap_clear(bitmap_t *map, u_int n)
+{
+ assert(n < NRQS);
+ bitmap_clear(map, n);
+}
+
#endif /* defined(CONFIG_SCHED_TIMESHARE_CORE) || defined(CONFIG_SCHED_PROTO) */
struct rt_queue {
extern void compute_memory_pressure(
void *arg);
-extern void compute_zone_gc_throttle(
- void *arg);
-
extern void compute_pageout_gc_throttle(
void *arg);
* to priority.
*/
#if defined(CONFIG_SCHED_TIMESHARE_CORE)
-extern uint32_t sched_pri_shift;
-extern uint32_t sched_background_pri_shift;
-extern uint32_t sched_combined_fgbg_pri_shift;
+
+#define MAX_LOAD (NRQS - 1)
+extern uint32_t sched_pri_shifts[TH_BUCKET_MAX];
extern uint32_t sched_fixed_shift;
extern int8_t sched_load_shifts[NRQS];
extern uint32_t sched_decay_usage_age_factor;
-extern uint32_t sched_use_combined_fgbg_decay;
void sched_timeshare_consider_maintenance(uint64_t ctime);
#endif /* CONFIG_SCHED_TIMESHARE_CORE */
extern int32_t sched_poll_yield_shift;
extern uint64_t sched_safe_duration;
-extern uint32_t sched_run_count, sched_share_count, sched_background_count;
extern uint32_t sched_load_average, sched_mach_factor;
extern uint32_t avenrun[3], mach_factor[3];
extern uint64_t max_unsafe_computation;
extern uint64_t max_poll_computation;
-/* TH_RUN & !TH_IDLE controls whether a thread has a run count */
-#define sched_run_incr(th) \
- hw_atomic_add(&sched_run_count, 1) \
-
-#define sched_run_decr(th) \
- hw_atomic_sub(&sched_run_count, 1) \
-
-#if MACH_ASSERT
-extern void sched_share_incr(thread_t thread);
-extern void sched_share_decr(thread_t thread);
-extern void sched_background_incr(thread_t thread);
-extern void sched_background_decr(thread_t thread);
-
-extern void assert_thread_sched_count(thread_t thread);
-
-#else /* MACH_ASSERT */
-/* sched_mode == TH_MODE_TIMESHARE controls whether a thread has a timeshare count when it has a run count */
-#define sched_share_incr(th) \
-MACRO_BEGIN \
- (void)hw_atomic_add(&sched_share_count, 1); \
-MACRO_END
-
-#define sched_share_decr(th) \
-MACRO_BEGIN \
- (void)hw_atomic_sub(&sched_share_count, 1); \
-MACRO_END
-
-/* TH_SFLAG_THROTTLED controls whether a thread has a background count when it has a run count and a share count */
-#define sched_background_incr(th) \
-MACRO_BEGIN \
- hw_atomic_add(&sched_background_count, 1); \
-MACRO_END
-
-#define sched_background_decr(th) \
-MACRO_BEGIN \
- hw_atomic_sub(&sched_background_count, 1); \
-MACRO_END
-
-#define assert_thread_sched_count(th) \
-MACRO_BEGIN \
-MACRO_END
+extern volatile uint32_t sched_run_buckets[TH_BUCKET_MAX];
-#endif /* !MACH_ASSERT */
+extern uint32_t sched_run_incr(thread_t thread);
+extern uint32_t sched_run_decr(thread_t thread);
/*
* thread_timer_delta macro takes care of both thread timers.
projects / apple / xnu.git / blobdiff