bsd/kern/kern_synch.c

   1 /*
   2  * Copyright (c) 2000-2016 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 /*
  29  * Mach Operating System
  30  * Copyright (c) 1987 Carnegie-Mellon University
  31  * All rights reserved.  The CMU software License Agreement specifies
  32  * the terms and conditions for use and redistribution.
  33  */
  34
  35 #include <sys/param.h>
  36 #include <sys/systm.h>
  37 #include <sys/proc_internal.h>
  38 #include <sys/user.h>
  39 #include <sys/file_internal.h>
  40 #include <sys/vnode.h>
  41 #include <sys/kernel.h>
  42
  43 #include <kern/queue.h>
  44 #include <sys/lock.h>
  45 #include <kern/thread.h>
  46 #include <kern/sched_prim.h>
  47 #include <kern/ast.h>
  48
  49 #include <kern/cpu_number.h>
  50 #include <vm/vm_kern.h>
  51
  52 #include <kern/task.h>
  53 #include <mach/time_value.h>
  54 #include <kern/locks.h>
  55 #include <kern/policy_internal.h>
  56
  57 #include <sys/systm.h>                  /* for unix_syscall_return() */
  58 #include <libkern/OSAtomic.h>
  59
  60 extern void compute_averunnable(void *);        /* XXX */
  61
  62 __attribute__((noreturn))
  63 static void
  64 _sleep_continue( __unused void *parameter, wait_result_t wresult)
  65 {
  66         struct proc *p = current_proc();
  67         thread_t self  = current_thread();
  68         struct uthread * ut;
  69         int sig, catch;
  70         int error = 0;
  71         int dropmutex, spinmutex;
  72
  73         ut = get_bsdthread_info(self);
  74         catch     = ut->uu_pri & PCATCH;
  75         dropmutex = ut->uu_pri & PDROP;
  76         spinmutex = ut->uu_pri & PSPIN;
  77
  78         switch (wresult) {
  79                 case THREAD_TIMED_OUT:
  80                         error = EWOULDBLOCK;
  81                         break;
  82                 case THREAD_AWAKENED:
  83                         /*
  84                          * Posix implies any signal should be delivered
  85                          * first, regardless of whether awakened due
  86                          * to receiving event.
  87                          */
  88                         if (!catch)
  89                                 break;
  90                         /* else fall through */
  91                 case THREAD_INTERRUPTED:
  92                         if (catch) {
  93                                 if (thread_should_abort(self)) {
  94                                         error = EINTR;
  95                                 } else if (SHOULDissignal(p,ut)) {
  96                                         if ((sig = CURSIG(p)) != 0) {
  97                                                 if (p->p_sigacts->ps_sigintr & sigmask(sig))
  98                                                         error = EINTR;
  99                                                 else
 100                                                         error = ERESTART;
 101                                         }
 102                                         if (thread_should_abort(self)) {
 103                                                 error = EINTR;
 104                                         }
 105                                 } else if( (ut->uu_flag & ( UT_CANCELDISABLE | UT_CANCEL | UT_CANCELED)) == UT_CANCEL) {
 106                                         /* due to thread cancel */
 107                                         error = EINTR;
 108                                 }
 109                         }  else
 110                                 error = EINTR;
 111                         break;
 112         }
 113
 114         if (error == EINTR || error == ERESTART)
 115                 act_set_astbsd(self);
 116
 117         if (ut->uu_mtx && !dropmutex) {
 118                 if (spinmutex)
 119                         lck_mtx_lock_spin(ut->uu_mtx);
 120                 else
 121                         lck_mtx_lock(ut->uu_mtx);
 122         }
 123         ut->uu_wchan = NULL;
 124         ut->uu_wmesg = NULL;
 125
 126         unix_syscall_return((*ut->uu_continuation)(error));
 127 }
 128
 129 /*
 130  * Give up the processor till a wakeup occurs
 131  * on chan, at which time the process
 132  * enters the scheduling queue at priority pri.
 133  * The most important effect of pri is that when
 134  * pri<=PZERO a signal cannot disturb the sleep;
 135  * if pri>PZERO signals will be processed.
 136  * If pri&PCATCH is set, signals will cause sleep
 137  * to return 1, rather than longjmp.
 138  * Callers of this routine must be prepared for
 139  * premature return, and check that the reason for
 140  * sleeping has gone away.
 141  *
 142  * if msleep was the entry point, than we have a mutex to deal with
 143  *
 144  * The mutex is unlocked before the caller is blocked, and
 145  * relocked before msleep returns unless the priority includes the PDROP
 146  * flag... if PDROP is specified, _sleep returns with the mutex unlocked
 147  * regardless of whether it actually blocked or not.
 148  */
 149
 150 static int
 151 _sleep(
 152         caddr_t         chan,
 153         int             pri,
 154         const char      *wmsg,
 155         u_int64_t       abstime,
 156         int             (*continuation)(int),
 157         lck_mtx_t       *mtx)
 158 {
 159         struct proc *p;
 160         thread_t self = current_thread();
 161         struct uthread * ut;
 162         int sig, catch;
 163         int dropmutex  = pri & PDROP;
 164         int spinmutex  = pri & PSPIN;
 165         int wait_result;
 166         int error = 0;
 167
 168         ut = get_bsdthread_info(self);
 169
 170         p = current_proc();
 171         p->p_priority = pri & PRIMASK;
 172         /* It can still block in proc_exit() after the teardown. */
 173         if (p->p_stats != NULL)
 174                 OSIncrementAtomicLong(&p->p_stats->p_ru.ru_nvcsw);
 175
 176         if (pri & PCATCH)
 177                 catch = THREAD_ABORTSAFE;
 178         else
 179                 catch = THREAD_UNINT;
 180
 181         /* set wait message & channel */
 182         ut->uu_wchan = chan;
 183         ut->uu_wmesg = wmsg ? wmsg : "unknown";
 184
 185         if (mtx != NULL && chan != NULL && (thread_continue_t)continuation == THREAD_CONTINUE_NULL) {
 186                 int     flags;
 187
 188                 if (dropmutex)
 189                         flags = LCK_SLEEP_UNLOCK;
 190                 else
 191                         flags = LCK_SLEEP_DEFAULT;
 192
 193                 if (spinmutex)
 194                         flags |= LCK_SLEEP_SPIN;
 195
 196                 if (abstime)
 197                         wait_result = lck_mtx_sleep_deadline(mtx, flags, chan, catch, abstime);
 198                 else
 199                         wait_result = lck_mtx_sleep(mtx, flags, chan, catch);
 200         }
 201         else {
 202                 if (chan != NULL)
 203                         assert_wait_deadline(chan, catch, abstime);
 204                 if (mtx)
 205                         lck_mtx_unlock(mtx);
 206
 207                 if (catch == THREAD_ABORTSAFE) {
 208                         if (SHOULDissignal(p,ut)) {
 209                                 if ((sig = CURSIG(p)) != 0) {
 210                                         if (clear_wait(self, THREAD_INTERRUPTED) == KERN_FAILURE)
 211                                                 goto block;
 212                                         if (p->p_sigacts->ps_sigintr & sigmask(sig))
 213                                                 error = EINTR;
 214                                         else
 215                                                 error = ERESTART;
 216                                         if (mtx && !dropmutex) {
 217                                                 if (spinmutex)
 218                                                         lck_mtx_lock_spin(mtx);
 219                                                 else
 220                                                         lck_mtx_lock(mtx);
 221                                         }
 222                                         goto out;
 223                                 }
 224                         }
 225                         if (thread_should_abort(self)) {
 226                                 if (clear_wait(self, THREAD_INTERRUPTED) == KERN_FAILURE)
 227                                         goto block;
 228                                 error = EINTR;
 229
 230                                 if (mtx && !dropmutex) {
 231                                         if (spinmutex)
 232                                                 lck_mtx_lock_spin(mtx);
 233                                         else
 234                                                 lck_mtx_lock(mtx);
 235                                 }
 236                                 goto out;
 237                         }
 238                 }
 239
 240
 241 block:
 242                 if ((thread_continue_t)continuation != THREAD_CONTINUE_NULL) {
 243                         ut->uu_continuation = continuation;
 244                         ut->uu_pri  = pri;
 245                         ut->uu_timo = abstime? 1: 0;
 246                         ut->uu_mtx  = mtx;
 247                         (void) thread_block(_sleep_continue);
 248                         /* NOTREACHED */
 249                 }
 250
 251                 wait_result = thread_block(THREAD_CONTINUE_NULL);
 252
 253                 if (mtx && !dropmutex) {
 254                         if (spinmutex)
 255                                 lck_mtx_lock_spin(mtx);
 256                         else
 257                                 lck_mtx_lock(mtx);
 258                 }
 259         }
 260
 261         switch (wait_result) {
 262                 case THREAD_TIMED_OUT:
 263                         error = EWOULDBLOCK;
 264                         break;
 265                 case THREAD_AWAKENED:
 266                 case THREAD_RESTART:
 267                         /*
 268                          * Posix implies any signal should be delivered
 269                          * first, regardless of whether awakened due
 270                          * to receiving event.
 271                          */
 272                         if (catch != THREAD_ABORTSAFE)
 273                                 break;
 274                         /* else fall through */
 275                 case THREAD_INTERRUPTED:
 276                         if (catch == THREAD_ABORTSAFE) {
 277                                 if (thread_should_abort(self)) {
 278                                         error = EINTR;
 279                                 } else if (SHOULDissignal(p, ut)) {
 280                                         if ((sig = CURSIG(p)) != 0) {
 281                                                 if (p->p_sigacts->ps_sigintr & sigmask(sig))
 282                                                         error = EINTR;
 283                                                 else
 284                                                         error = ERESTART;
 285                                         }
 286                                         if (thread_should_abort(self)) {
 287                                                 error = EINTR;
 288                                         }
 289                                 } else if( (ut->uu_flag & ( UT_CANCELDISABLE | UT_CANCEL | UT_CANCELED)) == UT_CANCEL) {
 290                                         /* due to thread cancel */
 291                                         error = EINTR;
 292                                 }
 293                         }  else
 294                                 error = EINTR;
 295                         break;
 296         }
 297 out:
 298         if (error == EINTR || error == ERESTART)
 299                 act_set_astbsd(self);
 300         ut->uu_wchan = NULL;
 301         ut->uu_wmesg = NULL;
 302
 303         return (error);
 304 }
 305
 306 int
 307 sleep(
 308         void    *chan,
 309         int             pri)
 310 {
 311         return _sleep((caddr_t)chan, pri, (char *)NULL, 0, (int (*)(int))0, (lck_mtx_t *)0);
 312 }
 313
 314 int
 315 msleep0(
 316         void            *chan,
 317         lck_mtx_t       *mtx,
 318         int             pri,
 319         const char      *wmsg,
 320         int             timo,
 321         int             (*continuation)(int))
 322 {
 323         u_int64_t       abstime = 0;
 324
 325         if (timo)
 326                 clock_interval_to_deadline(timo, NSEC_PER_SEC / hz, &abstime);
 327
 328         return _sleep((caddr_t)chan, pri, wmsg, abstime, continuation, mtx);
 329 }
 330
 331 int
 332 msleep(
 333         void            *chan,
 334         lck_mtx_t       *mtx,
 335         int             pri,
 336         const char      *wmsg,
 337         struct timespec         *ts)
 338 {
 339         u_int64_t       abstime = 0;
 340
 341         if (ts && (ts->tv_sec || ts->tv_nsec)) {
 342                 nanoseconds_to_absolutetime((uint64_t)ts->tv_sec * NSEC_PER_SEC + ts->tv_nsec,  &abstime );
 343                 clock_absolutetime_interval_to_deadline( abstime, &abstime );
 344         }
 345
 346         return _sleep((caddr_t)chan, pri, wmsg, abstime, (int (*)(int))0, mtx);
 347 }
 348
 349 int
 350 msleep1(
 351         void            *chan,
 352         lck_mtx_t       *mtx,
 353         int             pri,
 354         const char      *wmsg,
 355         u_int64_t       abstime)
 356 {
 357         return _sleep((caddr_t)chan, pri, wmsg, abstime, (int (*)(int))0, mtx);
 358 }
 359
 360 int
 361 tsleep(
 362         void            *chan,
 363         int             pri,
 364         const char      *wmsg,
 365         int             timo)
 366 {
 367         u_int64_t       abstime = 0;
 368
 369         if (timo)
 370                 clock_interval_to_deadline(timo, NSEC_PER_SEC / hz, &abstime);
 371         return _sleep((caddr_t)chan, pri, wmsg, abstime, (int (*)(int))0, (lck_mtx_t *)0);
 372 }
 373
 374 int
 375 tsleep0(
 376         void            *chan,
 377         int             pri,
 378         const char      *wmsg,
 379         int             timo,
 380         int             (*continuation)(int))
 381 {
 382         u_int64_t       abstime = 0;
 383
 384         if (timo)
 385                 clock_interval_to_deadline(timo, NSEC_PER_SEC / hz, &abstime);
 386         return _sleep((caddr_t)chan, pri, wmsg, abstime, continuation, (lck_mtx_t *)0);
 387 }
 388
 389 int
 390 tsleep1(
 391         void            *chan,
 392         int             pri,
 393         const char      *wmsg,
 394         u_int64_t       abstime,
 395         int             (*continuation)(int))
 396 {
 397         return _sleep((caddr_t)chan, pri, wmsg, abstime, continuation, (lck_mtx_t *)0);
 398 }
 399
 400 /*
 401  * Wake up all processes sleeping on chan.
 402  */
 403 void
 404 wakeup(void *chan)
 405 {
 406         thread_wakeup((caddr_t)chan);
 407 }
 408
 409 /*
 410  * Wake up the first process sleeping on chan.
 411  *
 412  * Be very sure that the first process is really
 413  * the right one to wakeup.
 414  */
 415 void
 416 wakeup_one(caddr_t chan)
 417 {
 418         thread_wakeup_one((caddr_t)chan);
 419 }
 420
 421 /*
 422  * Compute the priority of a process when running in user mode.
 423  * Arrange to reschedule if the resulting priority is better
 424  * than that of the current process.
 425  */
 426 void
 427 resetpriority(struct proc *p)
 428 {
 429         (void)task_importance(p->task, -p->p_nice);
 430 }
 431
 432 struct loadavg averunnable =
 433         { {0, 0, 0}, FSCALE };          /* load average, of runnable procs */
 434 /*
 435  * Constants for averages over 1, 5, and 15 minutes
 436  * when sampling at 5 second intervals.
 437  */
 438 static fixpt_t cexp[3] = {
 439     (fixpt_t)(0.9200444146293232 * FSCALE),    /* exp(-1/12) */
 440     (fixpt_t)(0.9834714538216174 * FSCALE),    /* exp(-1/60) */
 441     (fixpt_t)(0.9944598480048967 * FSCALE),    /* exp(-1/180) */
 442 };
 443
 444 void
 445 compute_averunnable(void *arg)
 446 {
 447         unsigned int            nrun = *(unsigned int *)arg;
 448         struct loadavg          *avg = &averunnable;
 449         int             i;
 450
 451     for (i = 0; i < 3; i++)
 452         avg->ldavg[i] = (cexp[i] * avg->ldavg[i] +
 453             nrun * FSCALE * (FSCALE - cexp[i])) >> FSHIFT;
 454 }