osfmk/kern/syscall_subr.c

   1 /*
   2  * Copyright (c) 2000-2009 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 /*
  29  * @OSF_COPYRIGHT@
  30  */
  31 /*
  32  * Mach Operating System
  33  * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
  34  * All Rights Reserved.
  35  *
  36  * Permission to use, copy, modify and distribute this software and its
  37  * documentation is hereby granted, provided that both the copyright
  38  * notice and this permission notice appear in all copies of the
  39  * software, derivative works or modified versions, and any portions
  40  * thereof, and that both notices appear in supporting documentation.
  41  *
  42  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  43  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
  44  * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
  45  *
  46  * Carnegie Mellon requests users of this software to return to
  47  *
  48  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  49  *  School of Computer Science
  50  *  Carnegie Mellon University
  51  *  Pittsburgh PA 15213-3890
  52  *
  53  * any improvements or extensions that they make and grant Carnegie Mellon
  54  * the rights to redistribute these changes.
  55  */
  56 /*
  57  */
  58
  59 #include <mach/boolean.h>
  60 #include <mach/thread_switch.h>
  61 #include <ipc/ipc_port.h>
  62 #include <ipc/ipc_space.h>
  63 #include <kern/counters.h>
  64 #include <kern/ipc_kobject.h>
  65 #include <kern/processor.h>
  66 #include <kern/sched.h>
  67 #include <kern/sched_prim.h>
  68 #include <kern/spl.h>
  69 #include <kern/task.h>
  70 #include <kern/thread.h>
  71 #include <mach/policy.h>
  72
  73 #include <kern/syscall_subr.h>
  74 #include <mach/mach_host_server.h>
  75 #include <mach/mach_syscalls.h>
  76 #include <sys/kdebug.h>
  77
  78 #ifdef MACH_BSD
  79 extern void workqueue_thread_yielded(void);
  80 extern sched_call_t workqueue_get_sched_callback(void);
  81 #endif /* MACH_BSD */
  82
  83
  84 /* Called from commpage to take a delayed preemption when exiting
  85  * the "Preemption Free Zone" (PFZ).
  86  */
  87 kern_return_t
  88 pfz_exit(
  89 __unused        struct pfz_exit_args *args)
  90 {
  91         /* For now, nothing special to do.  We'll pick up the ASTs on kernel exit. */
  92
  93         return (KERN_SUCCESS);
  94 }
  95
  96
  97 /*
  98  *      swtch and swtch_pri both attempt to context switch (logic in
  99  *      thread_block no-ops the context switch if nothing would happen).
 100  *      A boolean is returned that indicates whether there is anything
 101  *      else runnable.  That's no excuse to spin, though.
 102  */
 103
 104 static void
 105 swtch_continue(void)
 106 {
 107         register processor_t    myprocessor;
 108     boolean_t                           result;
 109
 110     disable_preemption();
 111         myprocessor = current_processor();
 112         result = !SCHED(processor_queue_empty)(myprocessor) || rt_runq.count > 0;
 113         enable_preemption();
 114
 115         thread_syscall_return(result);
 116         /*NOTREACHED*/
 117 }
 118
 119 boolean_t
 120 swtch(
 121         __unused struct swtch_args *args)
 122 {
 123         register processor_t    myprocessor;
 124         boolean_t                               result;
 125
 126         disable_preemption();
 127         myprocessor = current_processor();
 128         if (SCHED(processor_queue_empty)(myprocessor) &&        rt_runq.count == 0) {
 129                 mp_enable_preemption();
 130
 131                 return (FALSE);
 132         }
 133         enable_preemption();
 134
 135         counter(c_swtch_block++);
 136
 137         thread_block_reason((thread_continue_t)swtch_continue, NULL, AST_YIELD);
 138
 139         disable_preemption();
 140         myprocessor = current_processor();
 141         result = !SCHED(processor_queue_empty)(myprocessor) || rt_runq.count > 0;
 142         enable_preemption();
 143
 144         return (result);
 145 }
 146
 147 static void
 148 swtch_pri_continue(void)
 149 {
 150         register processor_t    myprocessor;
 151     boolean_t                           result;
 152
 153         thread_depress_abort_internal(current_thread());
 154
 155     disable_preemption();
 156         myprocessor = current_processor();
 157         result = !SCHED(processor_queue_empty)(myprocessor) || rt_runq.count > 0;
 158         mp_enable_preemption();
 159
 160         thread_syscall_return(result);
 161         /*NOTREACHED*/
 162 }
 163
 164 boolean_t
 165 swtch_pri(
 166 __unused        struct swtch_pri_args *args)
 167 {
 168         register processor_t    myprocessor;
 169         boolean_t                               result;
 170
 171         disable_preemption();
 172         myprocessor = current_processor();
 173         if (SCHED(processor_queue_empty)(myprocessor) && rt_runq.count == 0) {
 174                 mp_enable_preemption();
 175
 176                 return (FALSE);
 177         }
 178         enable_preemption();
 179
 180         counter(c_swtch_pri_block++);
 181
 182         thread_depress_abstime(thread_depress_time);
 183
 184         thread_block_reason((thread_continue_t)swtch_pri_continue, NULL, AST_YIELD);
 185
 186         thread_depress_abort_internal(current_thread());
 187
 188         disable_preemption();
 189         myprocessor = current_processor();
 190         result = !SCHED(processor_queue_empty)(myprocessor) || rt_runq.count > 0;
 191         enable_preemption();
 192
 193         return (result);
 194 }
 195
 196 static int
 197 thread_switch_disable_workqueue_sched_callback(void)
 198 {
 199         sched_call_t callback = workqueue_get_sched_callback();
 200         thread_t self = current_thread();
 201         if (!callback || self->sched_call != callback) {
 202                 return FALSE;
 203         }
 204         spl_t s = splsched();
 205         thread_lock(self);
 206         thread_sched_call(self, NULL);
 207         thread_unlock(self);
 208         splx(s);
 209         return TRUE;
 210 }
 211
 212 static void
 213 thread_switch_enable_workqueue_sched_callback(void)
 214 {
 215         sched_call_t callback = workqueue_get_sched_callback();
 216         thread_t self = current_thread();
 217         spl_t s = splsched();
 218         thread_lock(self);
 219         thread_sched_call(self, callback);
 220         thread_unlock(self);
 221         splx(s);
 222 }
 223
 224 static void
 225 thread_switch_continue(void)
 226 {
 227         register thread_t       self = current_thread();
 228         int                                     option = self->saved.swtch.option;
 229         boolean_t                       reenable_workq_callback = self->saved.swtch.reenable_workq_callback;
 230
 231
 232         if (option == SWITCH_OPTION_DEPRESS || option == SWITCH_OPTION_OSLOCK_DEPRESS)
 233                 thread_depress_abort_internal(self);
 234
 235         if (reenable_workq_callback)
 236                 thread_switch_enable_workqueue_sched_callback();
 237
 238         thread_syscall_return(KERN_SUCCESS);
 239         /*NOTREACHED*/
 240 }
 241
 242 /*
 243  *      thread_switch:
 244  *
 245  *      Context switch.  User may supply thread hint.
 246  */
 247 kern_return_t
 248 thread_switch(
 249         struct thread_switch_args *args)
 250 {
 251         thread_t                        thread = THREAD_NULL;
 252         thread_t                        self = current_thread();
 253         mach_port_name_t                thread_name = args->thread_name;
 254         int                                             option = args->option;
 255         mach_msg_timeout_t              option_time = args->option_time;
 256         uint32_t                                scale_factor = NSEC_PER_MSEC;
 257         boolean_t                               reenable_workq_callback = FALSE;
 258         boolean_t                               depress_option = FALSE;
 259         boolean_t                               wait_option = FALSE;
 260
 261     /*
 262      *  Validate and process option.
 263      */
 264     switch (option) {
 265
 266         case SWITCH_OPTION_NONE:
 267                 workqueue_thread_yielded();
 268                 break;
 269         case SWITCH_OPTION_WAIT:
 270                 wait_option = TRUE;
 271                 workqueue_thread_yielded();
 272                 break;
 273         case SWITCH_OPTION_DEPRESS:
 274                 depress_option = TRUE;
 275                 workqueue_thread_yielded();
 276                 break;
 277         case SWITCH_OPTION_DISPATCH_CONTENTION:
 278                 scale_factor = NSEC_PER_USEC;
 279                 wait_option = TRUE;
 280                 if (thread_switch_disable_workqueue_sched_callback())
 281                         reenable_workq_callback = TRUE;
 282                 break;
 283         case SWITCH_OPTION_OSLOCK_DEPRESS:
 284                 depress_option = TRUE;
 285                 if (thread_switch_disable_workqueue_sched_callback())
 286                         reenable_workq_callback = TRUE;
 287                 break;
 288         case SWITCH_OPTION_OSLOCK_WAIT:
 289                 wait_option = TRUE;
 290                 if (thread_switch_disable_workqueue_sched_callback())
 291                         reenable_workq_callback = TRUE;
 292                 break;
 293         default:
 294             return (KERN_INVALID_ARGUMENT);
 295     }
 296
 297         /*
 298          * Translate the port name if supplied.
 299          */
 300         if (thread_name != MACH_PORT_NULL) {
 301                 ipc_port_t port;
 302
 303                 if (ipc_port_translate_send(self->task->itk_space,
 304                                             thread_name, &port) == KERN_SUCCESS) {
 305                         ip_reference(port);
 306                         ip_unlock(port);
 307
 308                         thread = convert_port_to_thread(port);
 309                         ip_release(port);
 310
 311                         if (thread == self) {
 312                                 thread_deallocate(thread);
 313                                 thread = THREAD_NULL;
 314                         }
 315                 }
 316         }
 317
 318         if (option == SWITCH_OPTION_OSLOCK_DEPRESS || option == SWITCH_OPTION_OSLOCK_WAIT) {
 319                 if (thread != THREAD_NULL) {
 320
 321                         if (thread->task != self->task) {
 322                                 /*
 323                                  * OSLock boosting only applies to other threads
 324                                  * in your same task (even if you have a port for
 325                                  * a thread in another task)
 326                                  */
 327
 328                                 thread_deallocate(thread);
 329                                 thread = THREAD_NULL;
 330                         } else {
 331                                 /*
 332                                  * Attempt to kick the lock owner up to our same IO throttling tier.
 333                                  * If the thread is currently blocked in throttle_lowpri_io(),
 334                                  * it will immediately break out.
 335                                  *
 336                                  * TODO: SFI break out?
 337                                  */
 338                                 int new_policy = proc_get_effective_thread_policy(self, TASK_POLICY_IO);
 339
 340                                 set_thread_iotier_override(thread, new_policy);
 341                         }
 342                 }
 343         }
 344
 345         /*
 346          * Try to handoff if supplied.
 347          */
 348         if (thread != THREAD_NULL) {
 349                 spl_t s = splsched();
 350
 351                 /* This may return a different thread if the target is pushing on something */
 352                 thread_t pulled_thread = thread_run_queue_remove_for_handoff(thread);
 353
 354                 KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED,MACH_SCHED_THREAD_SWITCH)|DBG_FUNC_NONE,
 355                                       thread_tid(thread), thread->state,
 356                                       pulled_thread ? TRUE : FALSE, 0, 0);
 357
 358                 if (pulled_thread != THREAD_NULL) {
 359                         /* We can't be dropping the last ref here */
 360                         thread_deallocate_safe(thread);
 361
 362                         if (wait_option)
 363                                 assert_wait_timeout((event_t)assert_wait_timeout, THREAD_ABORTSAFE,
 364                                                     option_time, scale_factor);
 365                         else if (depress_option)
 366                                 thread_depress_ms(option_time);
 367
 368                         self->saved.swtch.option = option;
 369                         self->saved.swtch.reenable_workq_callback = reenable_workq_callback;
 370
 371                         thread_run(self, (thread_continue_t)thread_switch_continue, NULL, pulled_thread);
 372                         /* NOTREACHED */
 373                         panic("returned from thread_run!");
 374                 }
 375
 376                 splx(s);
 377
 378                 thread_deallocate(thread);
 379         }
 380
 381         if (wait_option)
 382                 assert_wait_timeout((event_t)assert_wait_timeout, THREAD_ABORTSAFE, option_time, scale_factor);
 383         else if (depress_option)
 384                 thread_depress_ms(option_time);
 385
 386         self->saved.swtch.option = option;
 387         self->saved.swtch.reenable_workq_callback = reenable_workq_callback;
 388
 389         thread_block_reason((thread_continue_t)thread_switch_continue, NULL, AST_YIELD);
 390
 391         if (depress_option)
 392                 thread_depress_abort_internal(self);
 393
 394         if (reenable_workq_callback)
 395                 thread_switch_enable_workqueue_sched_callback();
 396
 397     return (KERN_SUCCESS);
 398 }
 399
 400 /*
 401  * Depress thread's priority to lowest possible for the specified interval,
 402  * with a value of zero resulting in no timeout being scheduled.
 403  */
 404 void
 405 thread_depress_abstime(
 406         uint64_t                                interval)
 407 {
 408         register thread_t               self = current_thread();
 409         uint64_t                                deadline;
 410     spl_t                                       s;
 411
 412     s = splsched();
 413     thread_lock(self);
 414         if (!(self->sched_flags & TH_SFLAG_DEPRESSED_MASK)) {
 415                 processor_t             myprocessor = self->last_processor;
 416
 417                 self->sched_pri = DEPRESSPRI;
 418                 myprocessor->current_pri = self->sched_pri;
 419                 self->sched_flags |= TH_SFLAG_DEPRESS;
 420
 421                 if (interval != 0) {
 422                         clock_absolutetime_interval_to_deadline(interval, &deadline);
 423                         if (!timer_call_enter(&self->depress_timer, deadline, TIMER_CALL_USER_CRITICAL))
 424                                 self->depress_timer_active++;
 425                 }
 426         }
 427         thread_unlock(self);
 428     splx(s);
 429 }
 430
 431 void
 432 thread_depress_ms(
 433         mach_msg_timeout_t              interval)
 434 {
 435         uint64_t                abstime;
 436
 437         clock_interval_to_absolutetime_interval(
 438                                                         interval, NSEC_PER_MSEC, &abstime);
 439         thread_depress_abstime(abstime);
 440 }
 441
 442 /*
 443  *      Priority depression expiration.
 444  */
 445 void
 446 thread_depress_expire(
 447         void                    *p0,
 448         __unused void   *p1)
 449 {
 450         thread_t                thread = p0;
 451     spl_t                       s;
 452
 453     s = splsched();
 454     thread_lock(thread);
 455         if (--thread->depress_timer_active == 0) {
 456                 thread->sched_flags &= ~TH_SFLAG_DEPRESSED_MASK;
 457                 thread_recompute_sched_pri(thread, FALSE);
 458         }
 459     thread_unlock(thread);
 460     splx(s);
 461 }
 462
 463 /*
 464  *      Prematurely abort priority depression if there is one.
 465  */
 466 kern_return_t
 467 thread_depress_abort_internal(
 468         thread_t                                thread)
 469 {
 470     kern_return_t                       result = KERN_NOT_DEPRESSED;
 471     spl_t                                       s;
 472
 473     s = splsched();
 474     thread_lock(thread);
 475         if (!(thread->sched_flags & TH_SFLAG_POLLDEPRESS)) {
 476                 if (thread->sched_flags & TH_SFLAG_DEPRESSED_MASK) {
 477                         thread->sched_flags &= ~TH_SFLAG_DEPRESSED_MASK;
 478                         thread_recompute_sched_pri(thread, FALSE);
 479                         result = KERN_SUCCESS;
 480                 }
 481
 482                 if (timer_call_cancel(&thread->depress_timer))
 483                         thread->depress_timer_active--;
 484         }
 485         thread_unlock(thread);
 486     splx(s);
 487
 488     return (result);
 489 }
 490
 491 void
 492 thread_poll_yield(
 493         thread_t                self)
 494 {
 495         spl_t                   s;
 496
 497         assert(self == current_thread());
 498
 499         s = splsched();
 500         if (self->sched_mode == TH_MODE_FIXED) {
 501                 uint64_t                        total_computation, abstime;
 502
 503                 abstime = mach_absolute_time();
 504                 total_computation = abstime - self->computation_epoch;
 505                 total_computation += self->computation_metered;
 506                 if (total_computation >= max_poll_computation) {
 507                         processor_t             myprocessor = current_processor();
 508                         ast_t                   preempt;
 509
 510                         thread_lock(self);
 511                         if (!(self->sched_flags & TH_SFLAG_DEPRESSED_MASK)) {
 512                                 self->sched_pri = DEPRESSPRI;
 513                                 myprocessor->current_pri = self->sched_pri;
 514                         }
 515                         self->computation_epoch = abstime;
 516                         self->computation_metered = 0;
 517                         self->sched_flags |= TH_SFLAG_POLLDEPRESS;
 518
 519                         abstime += (total_computation >> sched_poll_yield_shift);
 520                         if (!timer_call_enter(&self->depress_timer, abstime, TIMER_CALL_USER_CRITICAL))
 521                                 self->depress_timer_active++;
 522
 523                         if ((preempt = csw_check(myprocessor, AST_NONE)) != AST_NONE)
 524                                 ast_on(preempt);
 525
 526                         thread_unlock(self);
 527                 }
 528         }
 529         splx(s);
 530 }
 531
 532
 533 void
 534 thread_yield_internal(
 535         mach_msg_timeout_t      ms)
 536 {
 537         processor_t     myprocessor;
 538
 539         disable_preemption();
 540         myprocessor = current_processor();
 541         if (SCHED(processor_queue_empty)(myprocessor) && rt_runq.count == 0) {
 542                 mp_enable_preemption();
 543
 544                 return;
 545         }
 546         enable_preemption();
 547
 548         thread_depress_ms(ms);
 549
 550         thread_block_reason(THREAD_CONTINUE_NULL, NULL, AST_YIELD);
 551
 552         thread_depress_abort_internal(current_thread());
 553 }
 554