osfmk/kern/sched_average.c

   1 /*
   2  * Copyright (c) 2000-2007 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 /*
  29  * @OSF_COPYRIGHT@
  30  */
  31 /*
  32  * Mach Operating System
  33  * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
  34  * All Rights Reserved.
  35  *
  36  * Permission to use, copy, modify and distribute this software and its
  37  * documentation is hereby granted, provided that both the copyright
  38  * notice and this permission notice appear in all copies of the
  39  * software, derivative works or modified versions, and any portions
  40  * thereof, and that both notices appear in supporting documentation.
  41  *
  42  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  43  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
  44  * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
  45  *
  46  * Carnegie Mellon requests users of this software to return to
  47  *
  48  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  49  *  School of Computer Science
  50  *  Carnegie Mellon University
  51  *  Pittsburgh PA 15213-3890
  52  *
  53  * any improvements or extensions that they make and grant Carnegie Mellon
  54  * the rights to redistribute these changes.
  55  */
  56 /*
  57  */
  58 /*
  59  *      Author: Avadis Tevanian, Jr.
  60  *      Date:   1986
  61  *
  62  *      Compute various averages.
  63  */
  64
  65 #include <mach/mach_types.h>
  66
  67 #include <kern/sched.h>
  68 #include <kern/assert.h>
  69 #include <kern/processor.h>
  70 #include <kern/thread.h>
  71 #if CONFIG_TELEMETRY
  72 #include <kern/telemetry.h>
  73 #endif
  74
  75 #include <sys/kdebug.h>
  76
  77 uint32_t        avenrun[3] = {0, 0, 0};
  78 uint32_t        mach_factor[3] = {0, 0, 0};
  79
  80 uint32_t        sched_load_average, sched_mach_factor;
  81
  82 #if defined(CONFIG_SCHED_TIMESHARE_CORE)
  83 /*
  84  * Values are scaled by LOAD_SCALE, defined in processor_info.h
  85  */
  86 #define base(n)         ((n) << SCHED_TICK_SHIFT)
  87 #define frac(n)         (((base(n) - 1) * LOAD_SCALE) / base(n))
  88
  89 static uint32_t         fract[3] = {
  90         frac(5),                /* 5 second average */
  91         frac(30),               /* 30 second average */
  92         frac(60),               /* 1 minute average */
  93 };
  94
  95 #undef base
  96 #undef frac
  97
  98 #endif /* CONFIG_SCHED_TIMESHARE_CORE */
  99
 100 static unsigned int             sched_nrun;
 101
 102 typedef void    (*sched_avg_comp_t)(
 103                                         void                    *param);
 104
 105 static struct sched_average {
 106         sched_avg_comp_t        comp;
 107         void                    *param;
 108         int                     period; /* in seconds */
 109         uint64_t                deadline;
 110 } sched_average[] = {
 111         { compute_averunnable, &sched_nrun, 5, 0 },
 112         { compute_stack_target, NULL, 5, 1 },
 113         { compute_memory_pressure, NULL, 1, 0 },
 114         { compute_pageout_gc_throttle, NULL, 1, 0 },
 115         { compute_pmap_gc_throttle, NULL, 60, 0 },
 116 #if CONFIG_TELEMETRY
 117         { compute_telemetry, NULL, 1, 0 },
 118 #endif
 119         { NULL, NULL, 0, 0 }
 120 };
 121
 122 typedef struct sched_average    *sched_average_t;
 123
 124 uint32_t load_now[TH_BUCKET_MAX];
 125
 126 /* The "stdelta" parameter represents the number of scheduler maintenance
 127  * "ticks" that have elapsed since the last invocation, subject to
 128  * integer division imprecision.
 129  */
 130
 131 void
 132 compute_averages(uint64_t stdelta)
 133 {
 134         /*
 135          * Retrieve a snapshot of the current run counts.
 136          *
 137          * Why not a bcopy()? Because we need atomic word-sized reads of sched_run_buckets,
 138          * not byte-by-byte copy.
 139          */
 140         uint32_t ncpus = processor_avail_count;
 141
 142         load_now[TH_BUCKET_RUN]      = sched_run_buckets[TH_BUCKET_RUN];
 143         load_now[TH_BUCKET_FIXPRI]   = sched_run_buckets[TH_BUCKET_FIXPRI];
 144         load_now[TH_BUCKET_SHARE_FG] = sched_run_buckets[TH_BUCKET_SHARE_FG];
 145         load_now[TH_BUCKET_SHARE_UT] = sched_run_buckets[TH_BUCKET_SHARE_UT];
 146         load_now[TH_BUCKET_SHARE_BG] = sched_run_buckets[TH_BUCKET_SHARE_BG];
 147
 148         assert(load_now[TH_BUCKET_RUN] >= 0);
 149         assert(load_now[TH_BUCKET_FIXPRI] >= 0);
 150
 151         /* Ignore the current thread, which is a running fixpri thread */
 152
 153         uint32_t nthreads = load_now[TH_BUCKET_RUN] - 1;
 154         uint32_t nfixpri  = load_now[TH_BUCKET_FIXPRI] - 1;
 155
 156         KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
 157                 MACHDBG_CODE(DBG_MACH_SCHED, MACH_SCHED_LOAD) | DBG_FUNC_NONE,
 158                 load_now[TH_BUCKET_FIXPRI] - 1, load_now[TH_BUCKET_SHARE_FG],
 159                 load_now[TH_BUCKET_SHARE_BG],   load_now[TH_BUCKET_SHARE_UT], 0);
 160
 161         /*
 162          * Compute the timeshare priority conversion factor based on loading.
 163          * Because our counters may be incremented and accessed
 164          * concurrently with respect to each other, we may have
 165          * windows where the invariant (nthreads - nfixpri) == (fg + bg + ut)
 166          * is broken, so truncate values in these cases.
 167          */
 168
 169         uint32_t timeshare_threads = (nthreads - nfixpri);
 170
 171         for (uint32_t i = TH_BUCKET_SHARE_FG; i <= TH_BUCKET_SHARE_BG ; i++) {
 172                 if (load_now[i] > timeshare_threads)
 173                         load_now[i] = timeshare_threads;
 174         }
 175
 176         /*
 177          * Utility threads contribute up to NCPUS of load to FG threads
 178          */
 179         if (load_now[TH_BUCKET_SHARE_UT] <= ncpus) {
 180                 load_now[TH_BUCKET_SHARE_FG] += load_now[TH_BUCKET_SHARE_UT];
 181         } else {
 182                 load_now[TH_BUCKET_SHARE_FG] += ncpus;
 183         }
 184
 185         /*
 186          * FG and UT should notice there's one thread of competition from BG,
 187          * but no more.
 188          */
 189         if (load_now[TH_BUCKET_SHARE_BG] > 0) {
 190                 load_now[TH_BUCKET_SHARE_FG] += 1;
 191                 load_now[TH_BUCKET_SHARE_UT] += 1;
 192         }
 193
 194         /*
 195          * The conversion factor consists of two components:
 196          * a fixed value based on the absolute time unit (sched_fixed_shift),
 197          * and a dynamic portion based on load (sched_load_shifts).
 198          *
 199          * Zero load results in a out of range shift count.
 200          */
 201
 202         for (uint32_t i = TH_BUCKET_SHARE_FG; i <= TH_BUCKET_SHARE_BG ; i++) {
 203                 uint32_t bucket_load = 0;
 204
 205                 if (load_now[i] > ncpus) {
 206                         if (ncpus > 1)
 207                                 bucket_load = load_now[i] / ncpus;
 208                         else
 209                                 bucket_load = load_now[i];
 210
 211                         if (bucket_load > MAX_LOAD)
 212                                 bucket_load = MAX_LOAD;
 213                 }
 214
 215                 sched_pri_shifts[i] = sched_fixed_shift - sched_load_shifts[bucket_load];
 216         }
 217
 218         /*
 219          * Sample total running threads for the load average calculation.
 220          */
 221         sched_nrun = nthreads;
 222
 223         /*
 224          * Load average and mach factor calculations for
 225          * those which ask about these things.
 226          */
 227         uint32_t average_now = nthreads * LOAD_SCALE;
 228         uint32_t factor_now;
 229
 230         if (nthreads > ncpus)
 231                 factor_now = (ncpus * LOAD_SCALE) / (nthreads + 1);
 232         else
 233                 factor_now = (ncpus - nthreads) * LOAD_SCALE;
 234
 235         /*
 236          * For those statistics that formerly relied on being recomputed
 237          * on timer ticks, advance by the approximate number of corresponding
 238          * elapsed intervals, thus compensating for potential idle intervals.
 239          */
 240         for (uint32_t index = 0; index < stdelta; index++) {
 241                 sched_mach_factor = ((sched_mach_factor << 2) + factor_now) / 5;
 242                 sched_load_average = ((sched_load_average << 2) + average_now) / 5;
 243         }
 244
 245         /*
 246          * Compute old-style Mach load averages.
 247          */
 248         for (uint32_t index = 0; index < stdelta; index++) {
 249                 for (uint32_t i = 0; i < 3; i++) {
 250                         mach_factor[i] = ((mach_factor[i] * fract[i]) +
 251                                                 (factor_now * (LOAD_SCALE - fract[i]))) / LOAD_SCALE;
 252
 253                         avenrun[i] = ((avenrun[i] * fract[i]) +
 254                                                 (average_now * (LOAD_SCALE - fract[i]))) / LOAD_SCALE;
 255                 }
 256         }
 257
 258         /*
 259          * Compute averages in other components.
 260          */
 261         uint64_t abstime = mach_absolute_time();
 262
 263         for (sched_average_t avg = sched_average; avg->comp != NULL; ++avg) {
 264                 if (abstime >= avg->deadline) {
 265                         uint64_t period_abs = (avg->period * sched_one_second_interval);
 266                         uint64_t ninvokes = 1;
 267
 268                         ninvokes += (abstime - avg->deadline) / period_abs;
 269                         ninvokes = MIN(ninvokes, SCHED_TICK_MAX_DELTA);
 270
 271                         for (uint32_t index = 0; index < ninvokes; index++) {
 272                                 (*avg->comp)(avg->param);
 273                         }
 274                         avg->deadline = abstime + period_abs;
 275                 }
 276         }
 277 }