osfmk/kern/syscall_subr.c

   1 /*
   2  * Copyright (c) 2000-2009 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 /*
  29  * @OSF_COPYRIGHT@
  30  */
  31 /*
  32  * Mach Operating System
  33  * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
  34  * All Rights Reserved.
  35  *
  36  * Permission to use, copy, modify and distribute this software and its
  37  * documentation is hereby granted, provided that both the copyright
  38  * notice and this permission notice appear in all copies of the
  39  * software, derivative works or modified versions, and any portions
  40  * thereof, and that both notices appear in supporting documentation.
  41  *
  42  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  43  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
  44  * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
  45  *
  46  * Carnegie Mellon requests users of this software to return to
  47  *
  48  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  49  *  School of Computer Science
  50  *  Carnegie Mellon University
  51  *  Pittsburgh PA 15213-3890
  52  *
  53  * any improvements or extensions that they make and grant Carnegie Mellon
  54  * the rights to redistribute these changes.
  55  */
  56 /*
  57  */
  58
  59 #include <mach/boolean.h>
  60 #include <mach/thread_switch.h>
  61 #include <ipc/ipc_port.h>
  62 #include <ipc/ipc_space.h>
  63 #include <kern/counters.h>
  64 #include <kern/ipc_kobject.h>
  65 #include <kern/processor.h>
  66 #include <kern/sched.h>
  67 #include <kern/sched_prim.h>
  68 #include <kern/spl.h>
  69 #include <kern/task.h>
  70 #include <kern/thread.h>
  71 #include <mach/policy.h>
  72
  73 #include <kern/syscall_subr.h>
  74 #include <mach/mach_host_server.h>
  75 #include <mach/mach_syscalls.h>
  76
  77
  78 #ifdef MACH_BSD
  79 extern void workqueue_thread_yielded(void);
  80 #endif /* MACH_BSD */
  81
  82
  83 /* Called from commpage to take a delayed preemption when exiting
  84  * the "Preemption Free Zone" (PFZ).
  85  */
  86 kern_return_t
  87 pfz_exit(
  88 __unused        struct pfz_exit_args *args)
  89 {
  90         /* For now, nothing special to do.  We'll pick up the ASTs on kernel exit. */
  91
  92         return (KERN_SUCCESS);
  93 }
  94
  95
  96 /*
  97  *      swtch and swtch_pri both attempt to context switch (logic in
  98  *      thread_block no-ops the context switch if nothing would happen).
  99  *      A boolean is returned that indicates whether there is anything
 100  *      else runnable.
 101  *
 102  *      This boolean can be used by a thread waiting on a
 103  *      lock or condition:  If FALSE is returned, the thread is justified
 104  *      in becoming a resource hog by continuing to spin because there's
 105  *      nothing else useful that the processor could do.  If TRUE is
 106  *      returned, the thread should make one more check on the
 107  *      lock and then be a good citizen and really suspend.
 108  */
 109
 110 static void
 111 swtch_continue(void)
 112 {
 113         register processor_t    myprocessor;
 114     boolean_t                           result;
 115
 116     disable_preemption();
 117         myprocessor = current_processor();
 118         result = !SCHED(processor_queue_empty)(myprocessor) || rt_runq.count > 0;
 119         enable_preemption();
 120
 121         thread_syscall_return(result);
 122         /*NOTREACHED*/
 123 }
 124
 125 boolean_t
 126 swtch(
 127         __unused struct swtch_args *args)
 128 {
 129         register processor_t    myprocessor;
 130         boolean_t                               result;
 131
 132         disable_preemption();
 133         myprocessor = current_processor();
 134         if (SCHED(processor_queue_empty)(myprocessor) &&        rt_runq.count == 0) {
 135                 mp_enable_preemption();
 136
 137                 return (FALSE);
 138         }
 139         enable_preemption();
 140
 141         counter(c_swtch_block++);
 142
 143         thread_block_reason((thread_continue_t)swtch_continue, NULL, AST_YIELD);
 144
 145         disable_preemption();
 146         myprocessor = current_processor();
 147         result = !SCHED(processor_queue_empty)(myprocessor) || rt_runq.count > 0;
 148         enable_preemption();
 149
 150         return (result);
 151 }
 152
 153 static void
 154 swtch_pri_continue(void)
 155 {
 156         register processor_t    myprocessor;
 157     boolean_t                           result;
 158
 159         thread_depress_abort_internal(current_thread());
 160
 161     disable_preemption();
 162         myprocessor = current_processor();
 163         result = !SCHED(processor_queue_empty)(myprocessor) || rt_runq.count > 0;
 164         mp_enable_preemption();
 165
 166         thread_syscall_return(result);
 167         /*NOTREACHED*/
 168 }
 169
 170 boolean_t
 171 swtch_pri(
 172 __unused        struct swtch_pri_args *args)
 173 {
 174         register processor_t    myprocessor;
 175         boolean_t                               result;
 176
 177         disable_preemption();
 178         myprocessor = current_processor();
 179         if (SCHED(processor_queue_empty)(myprocessor) && rt_runq.count == 0) {
 180                 mp_enable_preemption();
 181
 182                 return (FALSE);
 183         }
 184         enable_preemption();
 185
 186         counter(c_swtch_pri_block++);
 187
 188         thread_depress_abstime(thread_depress_time);
 189
 190         thread_block_reason((thread_continue_t)swtch_pri_continue, NULL, AST_YIELD);
 191
 192         thread_depress_abort_internal(current_thread());
 193
 194         disable_preemption();
 195         myprocessor = current_processor();
 196         result = !SCHED(processor_queue_empty)(myprocessor) || rt_runq.count > 0;
 197         enable_preemption();
 198
 199         return (result);
 200 }
 201
 202 static void
 203 thread_switch_continue(void)
 204 {
 205         register thread_t       self = current_thread();
 206         int                                     option = self->saved.swtch.option;
 207
 208         if (option == SWITCH_OPTION_DEPRESS)
 209                 thread_depress_abort_internal(self);
 210
 211         thread_syscall_return(KERN_SUCCESS);
 212         /*NOTREACHED*/
 213 }
 214
 215 /*
 216  *      thread_switch:
 217  *
 218  *      Context switch.  User may supply thread hint.
 219  */
 220 kern_return_t
 221 thread_switch(
 222         struct thread_switch_args *args)
 223 {
 224         register thread_t               thread, self = current_thread();
 225         mach_port_name_t                thread_name = args->thread_name;
 226         int                                             option = args->option;
 227         mach_msg_timeout_t              option_time = args->option_time;
 228
 229     /*
 230      *  Process option.
 231      */
 232     switch (option) {
 233
 234         case SWITCH_OPTION_NONE:
 235         case SWITCH_OPTION_DEPRESS:
 236         case SWITCH_OPTION_WAIT:
 237             break;
 238
 239         default:
 240             return (KERN_INVALID_ARGUMENT);
 241     }
 242
 243     workqueue_thread_yielded();
 244
 245         /*
 246          * Translate the port name if supplied.
 247          */
 248     if (thread_name != MACH_PORT_NULL) {
 249                 ipc_port_t                      port;
 250
 251                 if (ipc_port_translate_send(self->task->itk_space,
 252                                                                         thread_name, &port) == KERN_SUCCESS) {
 253                         ip_reference(port);
 254                         ip_unlock(port);
 255
 256                         thread = convert_port_to_thread(port);
 257                         ip_release(port);
 258
 259                         if (thread == self) {
 260                                 (void)thread_deallocate_internal(thread);
 261                                 thread = THREAD_NULL;
 262                         }
 263                 }
 264                 else
 265                         thread = THREAD_NULL;
 266         }
 267         else
 268                 thread = THREAD_NULL;
 269
 270         /*
 271          * Try to handoff if supplied.
 272          */
 273         if (thread != THREAD_NULL) {
 274                 processor_t             processor;
 275                 spl_t                   s;
 276
 277                 s = splsched();
 278                 thread_lock(thread);
 279
 280                 /*
 281                  *      Check that the thread is not bound
 282                  *      to a different processor, and that realtime
 283                  *      is not involved.
 284                  *
 285                  *      Next, pull it off its run queue.  If it
 286                  *      doesn't come, it's not eligible.
 287                  */
 288                 processor = current_processor();
 289                 if (processor->current_pri < BASEPRI_RTQUEUES                   &&
 290                         thread->sched_pri < BASEPRI_RTQUEUES                            &&
 291                         (thread->bound_processor == PROCESSOR_NULL      ||
 292                          thread->bound_processor == processor)                          &&
 293                                 thread_run_queue_remove(thread)                                                 ) {
 294                         /*
 295                          *      Hah, got it!!
 296                          */
 297                         thread_unlock(thread);
 298
 299                         (void)thread_deallocate_internal(thread);
 300
 301                         if (option == SWITCH_OPTION_WAIT)
 302                                 assert_wait_timeout((event_t)assert_wait_timeout, THREAD_ABORTSAFE,
 303                                                                                                                 option_time, 1000*NSEC_PER_USEC);
 304                         else
 305                         if (option == SWITCH_OPTION_DEPRESS)
 306                                 thread_depress_ms(option_time);
 307
 308                         self->saved.swtch.option = option;
 309
 310                         thread_run(self, (thread_continue_t)thread_switch_continue, NULL, thread);
 311                         /* NOTREACHED */
 312                 }
 313
 314                 thread_unlock(thread);
 315                 splx(s);
 316
 317                 thread_deallocate(thread);
 318         }
 319
 320         if (option == SWITCH_OPTION_WAIT)
 321                 assert_wait_timeout((event_t)assert_wait_timeout, THREAD_ABORTSAFE, option_time, 1000*NSEC_PER_USEC);
 322         else
 323         if (option == SWITCH_OPTION_DEPRESS)
 324                 thread_depress_ms(option_time);
 325
 326         self->saved.swtch.option = option;
 327
 328         thread_block_reason((thread_continue_t)thread_switch_continue, NULL, AST_YIELD);
 329
 330         if (option == SWITCH_OPTION_DEPRESS)
 331                 thread_depress_abort_internal(self);
 332
 333     return (KERN_SUCCESS);
 334 }
 335
 336 /*
 337  * Depress thread's priority to lowest possible for the specified interval,
 338  * with a value of zero resulting in no timeout being scheduled.
 339  */
 340 void
 341 thread_depress_abstime(
 342         uint64_t                                interval)
 343 {
 344         register thread_t               self = current_thread();
 345         uint64_t                                deadline;
 346     spl_t                                       s;
 347
 348     s = splsched();
 349     thread_lock(self);
 350         if (!(self->sched_flags & TH_SFLAG_DEPRESSED_MASK)) {
 351                 processor_t             myprocessor = self->last_processor;
 352
 353                 self->sched_pri = DEPRESSPRI;
 354                 myprocessor->current_pri = self->sched_pri;
 355                 self->sched_flags |= TH_SFLAG_DEPRESS;
 356
 357                 if (interval != 0) {
 358                         clock_absolutetime_interval_to_deadline(interval, &deadline);
 359                         if (!timer_call_enter(&self->depress_timer, deadline, TIMER_CALL_CRITICAL))
 360                                 self->depress_timer_active++;
 361                 }
 362         }
 363         thread_unlock(self);
 364     splx(s);
 365 }
 366
 367 void
 368 thread_depress_ms(
 369         mach_msg_timeout_t              interval)
 370 {
 371         uint64_t                abstime;
 372
 373         clock_interval_to_absolutetime_interval(
 374                                                         interval, 1000*NSEC_PER_USEC, &abstime);
 375         thread_depress_abstime(abstime);
 376 }
 377
 378 /*
 379  *      Priority depression expiration.
 380  */
 381 void
 382 thread_depress_expire(
 383         void                    *p0,
 384         __unused void   *p1)
 385 {
 386         thread_t                thread = p0;
 387     spl_t                       s;
 388
 389     s = splsched();
 390     thread_lock(thread);
 391         if (--thread->depress_timer_active == 0) {
 392                 thread->sched_flags &= ~TH_SFLAG_DEPRESSED_MASK;
 393                 SCHED(compute_priority)(thread, FALSE);
 394         }
 395     thread_unlock(thread);
 396     splx(s);
 397 }
 398
 399 /*
 400  *      Prematurely abort priority depression if there is one.
 401  */
 402 kern_return_t
 403 thread_depress_abort_internal(
 404         thread_t                                thread)
 405 {
 406     kern_return_t                       result = KERN_NOT_DEPRESSED;
 407     spl_t                                       s;
 408
 409     s = splsched();
 410     thread_lock(thread);
 411         if (!(thread->sched_flags & TH_SFLAG_POLLDEPRESS)) {
 412                 if (thread->sched_flags & TH_SFLAG_DEPRESSED_MASK) {
 413                         thread->sched_flags &= ~TH_SFLAG_DEPRESSED_MASK;
 414                         SCHED(compute_priority)(thread, FALSE);
 415                         result = KERN_SUCCESS;
 416                 }
 417
 418                 if (timer_call_cancel(&thread->depress_timer))
 419                         thread->depress_timer_active--;
 420         }
 421         thread_unlock(thread);
 422     splx(s);
 423
 424     return (result);
 425 }
 426
 427 void
 428 thread_poll_yield(
 429         thread_t                self)
 430 {
 431         spl_t                   s;
 432
 433         assert(self == current_thread());
 434
 435         s = splsched();
 436         if (self->sched_mode == TH_MODE_FIXED) {
 437                 uint64_t                        total_computation, abstime;
 438
 439                 abstime = mach_absolute_time();
 440                 total_computation = abstime - self->computation_epoch;
 441                 total_computation += self->computation_metered;
 442                 if (total_computation >= max_poll_computation) {
 443                         processor_t             myprocessor = current_processor();
 444                         ast_t                   preempt;
 445
 446                         thread_lock(self);
 447                         if (!(self->sched_flags & TH_SFLAG_DEPRESSED_MASK)) {
 448                                 self->sched_pri = DEPRESSPRI;
 449                                 myprocessor->current_pri = self->sched_pri;
 450                         }
 451                         self->computation_epoch = abstime;
 452                         self->computation_metered = 0;
 453                         self->sched_flags |= TH_SFLAG_POLLDEPRESS;
 454
 455                         abstime += (total_computation >> sched_poll_yield_shift);
 456                         if (!timer_call_enter(&self->depress_timer, abstime, TIMER_CALL_CRITICAL))
 457                                 self->depress_timer_active++;
 458                         thread_unlock(self);
 459
 460                         if ((preempt = csw_check(myprocessor)) != AST_NONE)
 461                                 ast_on(preempt);
 462                 }
 463         }
 464         splx(s);
 465 }
 466
 467
 468 void
 469 thread_yield_internal(
 470         mach_msg_timeout_t      ms)
 471 {
 472         processor_t     myprocessor;
 473
 474         disable_preemption();
 475         myprocessor = current_processor();
 476         if (SCHED(processor_queue_empty)(myprocessor) && rt_runq.count == 0) {
 477                 mp_enable_preemption();
 478
 479                 return;
 480         }
 481         enable_preemption();
 482
 483         thread_depress_ms(ms);
 484
 485         thread_block_reason(THREAD_CONTINUE_NULL, NULL, AST_YIELD);
 486
 487         thread_depress_abort_internal(current_thread());
 488 }
 489