X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/490019cf9519204c5fb36b2fba54ceb983bb6b72..008676633c2ad2c325837c2b64915f7ded690a8f:/osfmk/kern/sched.h diff --git a/osfmk/kern/sched.h b/osfmk/kern/sched.h index d8f470150..f1225c9d2 100644 --- a/osfmk/kern/sched.h +++ b/osfmk/kern/sched.h @@ -73,13 +73,15 @@ #include #include #include +#include +#include #define NRQS 128 /* 128 levels per run queue */ -#define NRQBM (NRQS / 32) /* number of words per bit map */ #define MAXPRI (NRQS-1) -#define MINPRI IDLEPRI /* lowest legal priority schedulable */ -#define IDLEPRI 0 /* idle thread priority */ +#define MINPRI 0 /* lowest legal priority schedulable */ +#define IDLEPRI MINPRI /* idle thread priority */ +#define NOPRI -1 /* * High-level priority assignments @@ -142,15 +144,15 @@ #define BASEPRI_REALTIME (MAXPRI - (NRQS / 4) + 1) /* 96 */ #define MAXPRI_KERNEL (BASEPRI_REALTIME - 1) /* 95 */ -#define BASEPRI_PREEMPT (MAXPRI_KERNEL - 2) /* 93 */ -#define BASEPRI_KERNEL (MINPRI_KERNEL + 1) /* 81 */ -#define MINPRI_KERNEL (MAXPRI_KERNEL - (NRQS / 8) + 1) /* 80 */ +#define BASEPRI_PREEMPT (MAXPRI_KERNEL - 2) /* 93 */ +#define BASEPRI_KERNEL (MINPRI_KERNEL + 1) /* 81 */ +#define MINPRI_KERNEL (MAXPRI_KERNEL - (NRQS / 8) + 1) /* 80 */ -#define MAXPRI_RESERVED (MINPRI_KERNEL - 1) /* 79 */ +#define MAXPRI_RESERVED (MINPRI_KERNEL - 1) /* 79 */ #define BASEPRI_GRAPHICS (MAXPRI_RESERVED - 3) /* 76 */ -#define MINPRI_RESERVED (MAXPRI_RESERVED - (NRQS / 8) + 1) /* 64 */ +#define MINPRI_RESERVED (MAXPRI_RESERVED - (NRQS / 8) + 1) /* 64 */ -#define MAXPRI_USER (MINPRI_RESERVED - 1) /* 63 */ +#define MAXPRI_USER (MINPRI_RESERVED - 1) /* 63 */ #define BASEPRI_CONTROL (BASEPRI_DEFAULT + 17) /* 48 */ #define BASEPRI_FOREGROUND (BASEPRI_DEFAULT + 16) /* 47 */ #define BASEPRI_BACKGROUND (BASEPRI_DEFAULT + 15) /* 46 */ @@ -158,10 +160,10 @@ #define BASEPRI_DEFAULT (MAXPRI_USER - (NRQS / 4)) /* 31 */ #define MAXPRI_SUPPRESSED (BASEPRI_DEFAULT - 3) /* 28 */ #define BASEPRI_UTILITY (BASEPRI_DEFAULT - 11) /* 20 */ -#define MAXPRI_THROTTLE (MINPRI + 4) /* 4 */ -#define MINPRI_USER MINPRI /* 0 */ +#define MAXPRI_THROTTLE (MINPRI + 4) /* 4 */ +#define MINPRI_USER MINPRI /* 0 */ -#define DEPRESSPRI MINPRI /* depress priority */ +#define DEPRESSPRI MINPRI /* depress priority */ #define MAXPRI_PROMOTE (MAXPRI_KERNEL) /* ceiling for mutex promotion */ /* Type used for thread->sched_mode and saved_mode */ @@ -172,6 +174,16 @@ typedef enum { TH_MODE_TIMESHARE, /* use timesharing algorithm */ } sched_mode_t; +/* Buckets used for load calculation */ +typedef enum { + TH_BUCKET_RUN = 0, /* All runnable threads */ + TH_BUCKET_FIXPRI, /* Fixed-priority */ + TH_BUCKET_SHARE_FG, /* Timeshare thread above BASEPRI_UTILITY */ + TH_BUCKET_SHARE_UT, /* Timeshare thread between BASEPRI_UTILITY and MAXPRI_THROTTLE */ + TH_BUCKET_SHARE_BG, /* Timeshare thread between MAXPRI_THROTTLE and MINPRI */ + TH_BUCKET_MAX, +} sched_bucket_t; + /* * Macro to check for invalid priorities. */ @@ -186,7 +198,7 @@ struct runq_stats { struct run_queue { int highq; /* highest runnable queue */ - int bitmap[NRQBM]; /* run queue bitmap array */ + bitmap_t bitmap[BITMAP_LEN(NRQS)]; /* run queue bitmap array */ int count; /* # of threads total */ int urgency; /* level of preemption urgency */ queue_head_t queues[NRQS]; /* one for each priority */ @@ -194,6 +206,20 @@ struct run_queue { struct runq_stats runq_stats; }; +inline static void +rq_bitmap_set(bitmap_t *map, u_int n) +{ + assert(n < NRQS); + bitmap_set(map, n); +} + +inline static void +rq_bitmap_clear(bitmap_t *map, u_int n) +{ + assert(n < NRQS); + bitmap_clear(map, n); +} + #endif /* defined(CONFIG_SCHED_TIMESHARE_CORE) || defined(CONFIG_SCHED_PROTO) */ struct rt_queue { @@ -314,9 +340,6 @@ extern void compute_stack_target( extern void compute_memory_pressure( void *arg); -extern void compute_zone_gc_throttle( - void *arg); - extern void compute_pageout_gc_throttle( void *arg); @@ -328,13 +351,12 @@ extern void compute_pmap_gc_throttle( * to priority. */ #if defined(CONFIG_SCHED_TIMESHARE_CORE) -extern uint32_t sched_pri_shift; -extern uint32_t sched_background_pri_shift; -extern uint32_t sched_combined_fgbg_pri_shift; + +#define MAX_LOAD (NRQS - 1) +extern uint32_t sched_pri_shifts[TH_BUCKET_MAX]; extern uint32_t sched_fixed_shift; extern int8_t sched_load_shifts[NRQS]; extern uint32_t sched_decay_usage_age_factor; -extern uint32_t sched_use_combined_fgbg_decay; void sched_timeshare_consider_maintenance(uint64_t ctime); #endif /* CONFIG_SCHED_TIMESHARE_CORE */ @@ -343,7 +365,6 @@ void sched_consider_recommended_cores(uint64_t ctime, thread_t thread); extern int32_t sched_poll_yield_shift; extern uint64_t sched_safe_duration; -extern uint32_t sched_run_count, sched_share_count, sched_background_count; extern uint32_t sched_load_average, sched_mach_factor; extern uint32_t avenrun[3], mach_factor[3]; @@ -351,49 +372,10 @@ extern uint32_t avenrun[3], mach_factor[3]; extern uint64_t max_unsafe_computation; extern uint64_t max_poll_computation; -/* TH_RUN & !TH_IDLE controls whether a thread has a run count */ -#define sched_run_incr(th) \ - hw_atomic_add(&sched_run_count, 1) \ - -#define sched_run_decr(th) \ - hw_atomic_sub(&sched_run_count, 1) \ - -#if MACH_ASSERT -extern void sched_share_incr(thread_t thread); -extern void sched_share_decr(thread_t thread); -extern void sched_background_incr(thread_t thread); -extern void sched_background_decr(thread_t thread); - -extern void assert_thread_sched_count(thread_t thread); - -#else /* MACH_ASSERT */ -/* sched_mode == TH_MODE_TIMESHARE controls whether a thread has a timeshare count when it has a run count */ -#define sched_share_incr(th) \ -MACRO_BEGIN \ - (void)hw_atomic_add(&sched_share_count, 1); \ -MACRO_END - -#define sched_share_decr(th) \ -MACRO_BEGIN \ - (void)hw_atomic_sub(&sched_share_count, 1); \ -MACRO_END - -/* TH_SFLAG_THROTTLED controls whether a thread has a background count when it has a run count and a share count */ -#define sched_background_incr(th) \ -MACRO_BEGIN \ - hw_atomic_add(&sched_background_count, 1); \ -MACRO_END - -#define sched_background_decr(th) \ -MACRO_BEGIN \ - hw_atomic_sub(&sched_background_count, 1); \ -MACRO_END - -#define assert_thread_sched_count(th) \ -MACRO_BEGIN \ -MACRO_END +extern volatile uint32_t sched_run_buckets[TH_BUCKET_MAX]; -#endif /* !MACH_ASSERT */ +extern uint32_t sched_run_incr(thread_t thread); +extern uint32_t sched_run_decr(thread_t thread); /* * thread_timer_delta macro takes care of both thread timers.