xnu-7195.50.7.100.1.tar.gz

[apple/xnu.git] / osfmk / kern / sched_proto.c

diff --git a/osfmk/kern/sched_proto.c b/osfmk/kern/sched_proto.c
index 6523bc66dd6927c172ab802606293e2c8b864ebb..9693756fdab48e10a5f1fd165b048ccaeb945262 100644 (file)
--- a/osfmk/kern/sched_proto.c
+++ b/osfmk/kern/sched_proto.c
@@ -2,7 +2,7 @@
  * Copyright (c) 2009 Apple Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * Copyright (c) 2009 Apple Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@

- * 
+ *

  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in
  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in


@@ -11,10 +11,10 @@
  * unlawful or unlicensed copies of an Apple operating system, or to
  * circumvent, violate, or enable the circumvention or violation of, any
  * terms of an Apple operating system software license agreement.
  * unlawful or unlicensed copies of an Apple operating system, or to
  * circumvent, violate, or enable the circumvention or violation of, any
  * terms of an Apple operating system software license agreement.

- * 
+ *

  * Please obtain a copy of the License at
  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  * Please obtain a copy of the License at
  * http://www.opensource.apple.com/apsl/ and read it before using this file.

- * 
+ *

  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,


@@ -22,7 +22,7 @@
  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  * Please see the License for the specific language governing rights and
  * limitations under the License.
  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  * Please see the License for the specific language governing rights and
  * limitations under the License.

- * 
+ *

  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 


@@ -52,7 +52,6 @@
 #include <kern/syscall_subr.h>
 #include <kern/task.h>
 #include <kern/thread.h>
 #include <kern/syscall_subr.h>
 #include <kern/task.h>
 #include <kern/thread.h>

-#include <kern/wait_queue.h>

 
 #include <vm/pmap.h>
 #include <vm/vm_kern.h>
 
 #include <vm/pmap.h>
 #include <vm/vm_kern.h>


@@ -78,45 +77,44 @@ static void
 sched_proto_maintenance_continuation(void);
 
 static thread_t
 sched_proto_maintenance_continuation(void);
 
 static thread_t

-sched_proto_choose_thread(processor_t          processor,
-                                                        int                            priority,
-                                                        ast_t                          reason);
+sched_proto_choose_thread(processor_t           processor,
+    int                            priority,
+    ast_t                          reason);

 
 static thread_t
 
 static thread_t

-sched_proto_steal_thread(processor_set_t               pset);
+sched_proto_steal_thread(processor_set_t                pset);

 
 

-static void
-sched_proto_compute_priority(thread_t  thread,
-                                                        boolean_t                      override_depress);
+static int
+sched_proto_compute_priority(thread_t thread);

 
 static processor_t
 
 static processor_t

-sched_proto_choose_processor(  processor_set_t         pset,
-                                                               processor_t                     processor,
-                                                               thread_t                        thread);
+sched_proto_choose_processor(   processor_set_t         pset,
+    processor_t                     processor,
+    thread_t                        thread);

 
 
 static boolean_t
 sched_proto_processor_enqueue(
 
 
 static boolean_t
 sched_proto_processor_enqueue(

-                                                        processor_t                    processor,
-                                                        thread_t                       thread,
-                                                        integer_t                      options);
+       processor_t                    processor,
+       thread_t                       thread,
+       sched_options_t                options);

 
 static void
 sched_proto_processor_queue_shutdown(
 
 static void
 sched_proto_processor_queue_shutdown(

-                                                                        processor_t                    processor);
+       processor_t                    processor);

 
 static boolean_t
 sched_proto_processor_queue_remove(
 
 static boolean_t
 sched_proto_processor_queue_remove(

-                                                   processor_t                 processor,
-                                                       thread_t                thread);
+       processor_t                 processor,
+       thread_t                thread);

 
 static boolean_t
 
 static boolean_t

-sched_proto_processor_queue_empty(processor_t          processor);
+sched_proto_processor_queue_empty(processor_t           processor);

 
 static boolean_t
 
 static boolean_t

-sched_proto_processor_queue_has_priority(processor_t           processor,
-                                                                                int                            priority,
-                                                                                boolean_t              gte);
+sched_proto_processor_queue_has_priority(processor_t            processor,
+    int                            priority,
+    boolean_t              gte);

 
 static boolean_t
 sched_proto_priority_is_urgent(int priority);
 
 static boolean_t
 sched_proto_priority_is_urgent(int priority);


@@ -131,19 +129,16 @@ static sched_mode_t
 sched_proto_initial_thread_sched_mode(task_t parent_task);
 
 static boolean_t
 sched_proto_initial_thread_sched_mode(task_t parent_task);
 
 static boolean_t

-sched_proto_can_update_priority(thread_t       thread);
+sched_proto_can_update_priority(thread_t        thread);

 
 static void
 
 static void

-sched_proto_update_priority(thread_t   thread);
+sched_proto_update_priority(thread_t    thread);

 
 static void
 
 static void

-sched_proto_lightweight_update_priority(thread_t       thread);
+sched_proto_lightweight_update_priority(thread_t        thread);

 
 static void
 
 static void

-sched_proto_quantum_expire(thread_t    thread);
-
-static boolean_t
-sched_proto_should_current_thread_rechoose_processor(processor_t                       processor);
+sched_proto_quantum_expire(thread_t     thread);

 
 static int
 sched_proto_processor_runq_count(processor_t   processor);
 
 static int
 sched_proto_processor_runq_count(processor_t   processor);


@@ -155,18 +150,21 @@ static int
 sched_proto_processor_bound_count(processor_t   processor);
 
 static void
 sched_proto_processor_bound_count(processor_t   processor);
 
 static void

-sched_proto_thread_update_scan(void);
+sched_proto_thread_update_scan(sched_update_scan_context_t scan_context);

 
 
 const struct sched_dispatch_table sched_proto_dispatch = {
 
 
 const struct sched_dispatch_table sched_proto_dispatch = {

+       .sched_name                                     = "proto",

        .init                                           = sched_proto_init,
        .timebase_init                                  = sched_proto_timebase_init,
        .processor_init                                 = sched_proto_processor_init,
        .pset_init                                      = sched_proto_pset_init,
        .maintenance_continuation                       = sched_proto_maintenance_continuation,
        .choose_thread                                  = sched_proto_choose_thread,
        .init                                           = sched_proto_init,
        .timebase_init                                  = sched_proto_timebase_init,
        .processor_init                                 = sched_proto_processor_init,
        .pset_init                                      = sched_proto_pset_init,
        .maintenance_continuation                       = sched_proto_maintenance_continuation,
        .choose_thread                                  = sched_proto_choose_thread,

+       .steal_thread_enabled                           = sched_steal_thread_DISABLED,

        .steal_thread                                   = sched_proto_steal_thread,
        .steal_thread                                   = sched_proto_steal_thread,

-       .compute_priority                               = sched_proto_compute_priority,
+       .compute_timeshare_priority                     = sched_proto_compute_priority,
+       .choose_node                                    = sched_choose_node,

        .choose_processor                               = sched_proto_choose_processor,
        .processor_enqueue                              = sched_proto_processor_enqueue,
        .processor_queue_shutdown                       = sched_proto_processor_queue_shutdown,
        .choose_processor                               = sched_proto_choose_processor,
        .processor_enqueue                              = sched_proto_processor_enqueue,
        .processor_queue_shutdown                       = sched_proto_processor_queue_shutdown,


@@ -181,71 +179,82 @@ const struct sched_dispatch_table sched_proto_dispatch = {
        .update_priority                                = sched_proto_update_priority,
        .lightweight_update_priority                    = sched_proto_lightweight_update_priority,
        .quantum_expire                                 = sched_proto_quantum_expire,
        .update_priority                                = sched_proto_update_priority,
        .lightweight_update_priority                    = sched_proto_lightweight_update_priority,
        .quantum_expire                                 = sched_proto_quantum_expire,

-       .should_current_thread_rechoose_processor       = sched_proto_should_current_thread_rechoose_processor,

        .processor_runq_count                           = sched_proto_processor_runq_count,
        .processor_runq_stats_count_sum                 = sched_proto_processor_runq_stats_count_sum,
        .processor_runq_count                           = sched_proto_processor_runq_count,
        .processor_runq_stats_count_sum                 = sched_proto_processor_runq_stats_count_sum,

-       .fairshare_init                                 = sched_traditional_fairshare_init,
-       .fairshare_runq_count                           = sched_traditional_fairshare_runq_count,
-       .fairshare_runq_stats_count_sum                 = sched_traditional_fairshare_runq_stats_count_sum,
-       .fairshare_enqueue                              = sched_traditional_fairshare_enqueue,
-       .fairshare_dequeue                              = sched_traditional_fairshare_dequeue,
-       .fairshare_queue_remove                         = sched_traditional_fairshare_queue_remove,

        .processor_bound_count                          = sched_proto_processor_bound_count,
        .thread_update_scan                             = sched_proto_thread_update_scan,
        .processor_bound_count                          = sched_proto_processor_bound_count,
        .thread_update_scan                             = sched_proto_thread_update_scan,

-       .direct_dispatch_to_idle_processors             = TRUE,
+       .multiple_psets_enabled                         = TRUE,
+       .sched_groups_enabled                           = FALSE,
+       .avoid_processor_enabled                        = FALSE,
+       .thread_avoid_processor                         = NULL,
+       .processor_balance                              = sched_SMT_balance,
+
+       .rt_runq                                        = sched_rtlocal_runq,
+       .rt_init                                        = sched_rtlocal_init,
+       .rt_queue_shutdown                              = sched_rtlocal_queue_shutdown,
+       .rt_runq_scan                                   = sched_rtlocal_runq_scan,
+       .rt_runq_count_sum                              = sched_rtlocal_runq_count_sum,
+
+       .qos_max_parallelism                            = sched_qos_max_parallelism,
+       .check_spill                                    = sched_check_spill,
+       .ipi_policy                                     = sched_ipi_policy,
+       .thread_should_yield                            = sched_thread_should_yield,
+       .run_count_incr                                 = sched_run_incr,
+       .run_count_decr                                 = sched_run_decr,
+       .update_thread_bucket                           = sched_update_thread_bucket,
+       .pset_made_schedulable                          = sched_pset_made_schedulable,

 };
 
 };
 

-static struct run_queue        *global_runq;
-static struct run_queue        global_runq_storage;
+static struct run_queue *global_runq;
+static struct run_queue global_runq_storage;

 
 

-#define GLOBAL_RUNQ            ((processor_t)-2)
-decl_simple_lock_data(static,global_runq_lock);
+#define GLOBAL_RUNQ             ((processor_t)-2)
+decl_simple_lock_data(static, global_runq_lock);

 
 

-extern int     max_unsafe_quanta;
+extern int      max_unsafe_quanta;

 
 static uint32_t proto_quantum_us;
 static uint32_t proto_quantum;
 
 
 static uint32_t proto_quantum_us;
 static uint32_t proto_quantum;
 

-static uint32_t        runqueue_generation;
+static uint32_t runqueue_generation;

 
 static processor_t proto_processor;
 
 
 static processor_t proto_processor;
 

-static uint64_t                        sched_proto_tick_deadline;
-static uint32_t                        sched_proto_tick;
+static uint64_t                 sched_proto_tick_deadline;
+static uint32_t                 sched_proto_tick;

 
 static void
 sched_proto_init(void)
 {
 
 static void
 sched_proto_init(void)
 {

-       proto_quantum_us = 10*1000;
-       
+       proto_quantum_us = 10 * 1000;
+

        printf("standard proto timeslicing quantum is %d us\n", proto_quantum_us);
 
        simple_lock_init(&global_runq_lock, 0);
        global_runq = &global_runq_storage;
        run_queue_init(global_runq);
        runqueue_generation = 0;
        printf("standard proto timeslicing quantum is %d us\n", proto_quantum_us);
 
        simple_lock_init(&global_runq_lock, 0);
        global_runq = &global_runq_storage;
        run_queue_init(global_runq);
        runqueue_generation = 0;

-       
+

        proto_processor = master_processor;
 }
 
 static void
 sched_proto_timebase_init(void)
 {
        proto_processor = master_processor;
 }
 
 static void
 sched_proto_timebase_init(void)
 {

-       uint64_t        abstime;
+       uint64_t        abstime;

 
        /* standard timeslicing quantum */
        clock_interval_to_absolutetime_interval(
 
        /* standard timeslicing quantum */
        clock_interval_to_absolutetime_interval(

-                                                                                       proto_quantum_us, NSEC_PER_USEC, &abstime);
+               proto_quantum_us, NSEC_PER_USEC, &abstime);

        assert((abstime >> 32) == 0 && (uint32_t)abstime != 0);
        proto_quantum = (uint32_t)abstime;
        assert((abstime >> 32) == 0 && (uint32_t)abstime != 0);
        proto_quantum = (uint32_t)abstime;

-       
+

        thread_depress_time = 1 * proto_quantum;
        default_timeshare_computation = proto_quantum / 2;
        default_timeshare_constraint = proto_quantum;
        thread_depress_time = 1 * proto_quantum;
        default_timeshare_computation = proto_quantum / 2;
        default_timeshare_constraint = proto_quantum;

-       
+

        max_unsafe_computation = max_unsafe_quanta * proto_quantum;
        sched_safe_duration = 2 * max_unsafe_quanta * proto_quantum;
        max_unsafe_computation = max_unsafe_quanta * proto_quantum;
        sched_safe_duration = 2 * max_unsafe_quanta * proto_quantum;

-

 }
 
 static void
 }
 
 static void


@@ -262,55 +271,57 @@ sched_proto_pset_init(processor_set_t pset __unused)
 static void
 sched_proto_maintenance_continuation(void)
 {
 static void
 sched_proto_maintenance_continuation(void)
 {

-       uint64_t                        abstime = mach_absolute_time();
-       
+       uint64_t                        abstime = mach_absolute_time();
+

        sched_proto_tick++;
        sched_proto_tick++;

-       
+

        /* Every 8 seconds, switch to another processor */
        if ((sched_proto_tick & 0x7) == 0) {
                processor_t new_processor;
        /* Every 8 seconds, switch to another processor */
        if ((sched_proto_tick & 0x7) == 0) {
                processor_t new_processor;

-               
+

                new_processor = proto_processor->processor_list;
                new_processor = proto_processor->processor_list;

-               if (new_processor == PROCESSOR_NULL)
+               if (new_processor == PROCESSOR_NULL) {

                        proto_processor = master_processor;
                        proto_processor = master_processor;

-               else
+               } else {

                        proto_processor = new_processor;
                        proto_processor = new_processor;

+               }

        }
        }

-               
-       
+
+

        /*
         *  Compute various averages.
         */
        compute_averages(1);
        /*
         *  Compute various averages.
         */
        compute_averages(1);

-       
-       if (sched_proto_tick_deadline == 0)
+
+       if (sched_proto_tick_deadline == 0) {

                sched_proto_tick_deadline = abstime;
                sched_proto_tick_deadline = abstime;

-       
+       }
+

        clock_deadline_for_periodic_event(sched_one_second_interval, abstime,
        clock_deadline_for_periodic_event(sched_one_second_interval, abstime,

-                                               &sched_proto_tick_deadline);
-       
+           &sched_proto_tick_deadline);
+

        assert_wait_deadline((event_t)sched_proto_maintenance_continuation, THREAD_UNINT, sched_proto_tick_deadline);
        thread_block((thread_continue_t)sched_proto_maintenance_continuation);
        /*NOTREACHED*/
 }
 
 static thread_t
        assert_wait_deadline((event_t)sched_proto_maintenance_continuation, THREAD_UNINT, sched_proto_tick_deadline);
        thread_block((thread_continue_t)sched_proto_maintenance_continuation);
        /*NOTREACHED*/
 }
 
 static thread_t

-sched_proto_choose_thread(processor_t          processor,
-                                                 int                           priority,
-                                                 ast_t                         reason __unused)
+sched_proto_choose_thread(processor_t           processor,
+    int                           priority,
+    ast_t                         reason __unused)

 {
 {

-       run_queue_t             rq = global_runq;
-       queue_t                 queue;
-       int                             pri, count;
-       thread_t                thread;
-       
-       
-       simple_lock(&global_runq_lock);
-       
+       run_queue_t             rq = global_runq;
+       circle_queue_t          queue;
+       int                             pri, count;
+       thread_t                thread;
+
+
+       simple_lock(&global_runq_lock, LCK_GRP_NULL);
+

        queue = rq->queues + rq->highq;
        pri = rq->highq;
        count = rq->count;
        queue = rq->queues + rq->highq;
        pri = rq->highq;
        count = rq->count;

-       
+

        /*
         * Since we don't depress priorities, a high priority thread
         * may get selected over and over again. Put a runqueue
        /*
         * Since we don't depress priorities, a high priority thread
         * may get selected over and over again. Put a runqueue


@@ -320,99 +331,95 @@ sched_proto_choose_thread(processor_t             processor,
         * perfect, especially if the number of runnable threads always
         * stays high, but is a workable approximation
         */
         * perfect, especially if the number of runnable threads always
         * stays high, but is a workable approximation
         */

-        
+

        while (count > 0 && pri >= priority) {
        while (count > 0 && pri >= priority) {

-               thread = (thread_t)queue_first(queue);
-               while (!queue_end(queue, (queue_entry_t)thread)) {
+               cqe_foreach_element_safe(thread, queue, runq_links) {

                        if ((thread->bound_processor == PROCESSOR_NULL ||
                        if ((thread->bound_processor == PROCESSOR_NULL ||

-                               thread->bound_processor == processor) &&
-                               runqueue_generation != thread->runqueue_generation) {
-                               remqueue((queue_entry_t)thread);
-                               
+                           thread->bound_processor == processor) &&
+                           runqueue_generation != thread->runqueue_generation) {
+                               circle_dequeue(queue, &thread->runq_links);
+

                                thread->runq = PROCESSOR_NULL;
                                thread->runqueue_generation = runqueue_generation;
                                SCHED_STATS_RUNQ_CHANGE(&rq->runq_stats, rq->count);
                                rq->count--;
                                thread->runq = PROCESSOR_NULL;
                                thread->runqueue_generation = runqueue_generation;
                                SCHED_STATS_RUNQ_CHANGE(&rq->runq_stats, rq->count);
                                rq->count--;

-                               if (queue_empty(queue)) {
-                                       if (pri != IDLEPRI)
-                                               clrbit(MAXPRI - pri, rq->bitmap);
-                                       rq->highq = MAXPRI - ffsbit(rq->bitmap);
+                               if (circle_queue_empty(queue)) {
+                                       bitmap_clear(rq->bitmap, pri);
+                                       rq->highq = bitmap_first(rq->bitmap, NRQS);

                                }
                                }

-                               
+

                                simple_unlock(&global_runq_lock);
                                simple_unlock(&global_runq_lock);

-                               return (thread);
+                               return thread;

                        }
                        count--;
                        }
                        count--;

-                       
+

                        thread = (thread_t)queue_next((queue_entry_t)thread);
                }
                        thread = (thread_t)queue_next((queue_entry_t)thread);
                }

-               
+

                queue--; pri--;
        }
                queue--; pri--;
        }

-       
+

        runqueue_generation++;
        runqueue_generation++;

-       
+

        simple_unlock(&global_runq_lock);
        simple_unlock(&global_runq_lock);

-       return (THREAD_NULL);
+       return THREAD_NULL;

 }
 
 static thread_t
 }
 
 static thread_t

-sched_proto_steal_thread(processor_set_t               pset)
+sched_proto_steal_thread(processor_set_t                pset)

 {
        pset_unlock(pset);
 {
        pset_unlock(pset);

-       
-       return (THREAD_NULL);
-       
+
+       return THREAD_NULL;

 }
 
 }
 

-static void
-sched_proto_compute_priority(thread_t  thread,
-                                                        boolean_t                      override_depress __unused)
+static int
+sched_proto_compute_priority(thread_t thread)

 {
 {

-       set_sched_pri(thread, thread->priority);
+       return thread->base_pri;

 }
 
 static processor_t
 }
 
 static processor_t

-sched_proto_choose_processor(  processor_set_t         pset,
-                                                        processor_t                    processor,
-                                                        thread_t                       thread __unused)
+sched_proto_choose_processor(   processor_set_t         pset,
+    processor_t                    processor,
+    thread_t                       thread __unused)

 {
        processor = proto_processor;
 {
        processor = proto_processor;

-       
+

        /*
         *      Check that the correct processor set is
         *      returned locked.
         */
        if (pset != processor->processor_set) {
                pset_unlock(pset);
        /*
         *      Check that the correct processor set is
         *      returned locked.
         */
        if (pset != processor->processor_set) {
                pset_unlock(pset);

-               
+

                pset = processor->processor_set;
                pset_lock(pset);
        }
                pset = processor->processor_set;
                pset_lock(pset);
        }

-       
-       return (processor);
+
+       return processor;

 }
 
 static boolean_t
 sched_proto_processor_enqueue(
 }
 
 static boolean_t
 sched_proto_processor_enqueue(

-                                                        processor_t                    processor __unused,
-                                                        thread_t                       thread,
-                                                        integer_t                      options)
+       processor_t                    processor __unused,
+       thread_t                       thread,
+       sched_options_t                options)

 {
 {

-       run_queue_t             rq = global_runq;
-       boolean_t               result;
-       
-       simple_lock(&global_runq_lock);
+       run_queue_t             rq = global_runq;
+       boolean_t               result;
+
+       simple_lock(&global_runq_lock, LCK_GRP_NULL);

        result = run_queue_enqueue(rq, thread, options);
        thread->runq = GLOBAL_RUNQ;
        simple_unlock(&global_runq_lock);
        result = run_queue_enqueue(rq, thread, options);
        thread->runq = GLOBAL_RUNQ;
        simple_unlock(&global_runq_lock);

-       
-       return (result);
+
+       return result;

 }
 
 static void
 sched_proto_processor_queue_shutdown(
 }
 
 static void
 sched_proto_processor_queue_shutdown(

-                                                                        processor_t                    processor)
+       processor_t                    processor)

 {
        /* With a global runqueue, just stop choosing this processor */
        (void)processor;
 {
        /* With a global runqueue, just stop choosing this processor */
        (void)processor;


@@ -420,77 +427,63 @@ sched_proto_processor_queue_shutdown(
 
 static boolean_t
 sched_proto_processor_queue_remove(
 
 static boolean_t
 sched_proto_processor_queue_remove(

-                                                               processor_t                     processor,
-                                                               thread_t                thread)
+       processor_t                     processor,
+       thread_t                thread)

 {
 {

-       void *                  rqlock;
-       run_queue_t             rq;
-       
+       void *                  rqlock;
+       run_queue_t             rq;
+

        rqlock = &global_runq_lock;
        rq = global_runq;
        rqlock = &global_runq_lock;
        rq = global_runq;

-       
-       simple_lock(rqlock);
+
+       simple_lock(rqlock, LCK_GRP_NULL);

        if (processor == thread->runq) {
                /*
                 *      Thread is on a run queue and we have a lock on
                 *      that run queue.
                 */
        if (processor == thread->runq) {
                /*
                 *      Thread is on a run queue and we have a lock on
                 *      that run queue.
                 */

-               remqueue((queue_entry_t)thread);
-               SCHED_STATS_RUNQ_CHANGE(&rq->runq_stats, rq->count);
-               rq->count--;
-               if (SCHED(priority_is_urgent)(thread->sched_pri)) {
-                       rq->urgency--; assert(rq->urgency >= 0);
-               }
-               
-               if (queue_empty(rq->queues + thread->sched_pri)) {
-                       /* update run queue status */
-                       if (thread->sched_pri != IDLEPRI)
-                               clrbit(MAXPRI - thread->sched_pri, rq->bitmap);
-                       rq->highq = MAXPRI - ffsbit(rq->bitmap);
-               }
-               
-               thread->runq = PROCESSOR_NULL;
-       }
-       else {
+               run_queue_remove(rq, thread);
+       } else {

                /*
                 *      The thread left the run queue before we could
                /*
                 *      The thread left the run queue before we could

-                *      lock the run queue.
+                *      lock the run queue.

                 */
                assert(thread->runq == PROCESSOR_NULL);
                processor = PROCESSOR_NULL;
        }
                 */
                assert(thread->runq == PROCESSOR_NULL);
                processor = PROCESSOR_NULL;
        }

-       
+

        simple_unlock(rqlock);
        simple_unlock(rqlock);

-       
-       return (processor != PROCESSOR_NULL);
+
+       return processor != PROCESSOR_NULL;

 }
 
 static boolean_t
 }
 
 static boolean_t

-sched_proto_processor_queue_empty(processor_t          processor __unused)
+sched_proto_processor_queue_empty(processor_t           processor __unused)

 {
        boolean_t result;
 {
        boolean_t result;

-       
+

        result = (global_runq->count == 0);
        result = (global_runq->count == 0);

-       
+

        return result;
 }
 
 static boolean_t
        return result;
 }
 
 static boolean_t

-sched_proto_processor_queue_has_priority(processor_t           processor __unused,
-                                                                                int                            priority,
-                                                                                boolean_t              gte)
+sched_proto_processor_queue_has_priority(processor_t            processor __unused,
+    int                            priority,
+    boolean_t              gte)

 {
        boolean_t result;
 {
        boolean_t result;

-       
-       simple_lock(&global_runq_lock);

 
 

-       if (gte)
-               result = global_runq->highq >= priority;
-       else
+       simple_lock(&global_runq_lock, LCK_GRP_NULL);
+
+       if (gte) {

                result = global_runq->highq >= priority;
                result = global_runq->highq >= priority;

+       } else {
+               result = global_runq->highq > priority;
+       }

 
        simple_unlock(&global_runq_lock);
 
        simple_unlock(&global_runq_lock);

-       
+

        return result;
 }
 
        return result;
 }
 


@@ -498,35 +491,43 @@ sched_proto_processor_queue_has_priority(processor_t              processor __unused,
 static boolean_t
 sched_proto_priority_is_urgent(int priority)
 {
 static boolean_t
 sched_proto_priority_is_urgent(int priority)
 {

-       if (priority <= BASEPRI_FOREGROUND)
+       if (priority <= BASEPRI_FOREGROUND) {

                return FALSE;
                return FALSE;

-       
-       if (priority < MINPRI_KERNEL)
+       }
+
+       if (priority < MINPRI_KERNEL) {

                return TRUE;
                return TRUE;

+       }

 
 

-       if (priority >= BASEPRI_PREEMPT)
+       if (priority >= BASEPRI_PREEMPT) {

                return TRUE;
                return TRUE;

-       
+       }
+

        return FALSE;
 }
 
 static ast_t
        return FALSE;
 }
 
 static ast_t

-sched_proto_processor_csw_check(processor_t processor __unused)
+sched_proto_processor_csw_check(processor_t processor)

 {
 {

-       run_queue_t             runq;
-       int                             count, urgency;
-       
+       run_queue_t             runq;
+       int                             count, urgency;
+

        runq = global_runq;
        count = runq->count;
        urgency = runq->urgency;
        runq = global_runq;
        count = runq->count;
        urgency = runq->urgency;

-       
+

        if (count > 0) {
        if (count > 0) {

-               if (urgency > 0)
-                       return (AST_PREEMPT | AST_URGENT);
-               
+               if (urgency > 0) {
+                       return AST_PREEMPT | AST_URGENT;
+               }
+

                return AST_PREEMPT;
        }
                return AST_PREEMPT;
        }

-       
+
+       if (proto_processor != processor) {
+               return AST_PREEMPT;
+       }
+

        return AST_NONE;
 }
 
        return AST_NONE;
 }
 


@@ -539,40 +540,32 @@ sched_proto_initial_quantum_size(thread_t thread __unused)
 static sched_mode_t
 sched_proto_initial_thread_sched_mode(task_t parent_task)
 {
 static sched_mode_t
 sched_proto_initial_thread_sched_mode(task_t parent_task)
 {

-       if (parent_task == kernel_task)
+       if (parent_task == kernel_task) {

                return TH_MODE_FIXED;
                return TH_MODE_FIXED;

-       else
-               return TH_MODE_TIMESHARE;       
+       } else {
+               return TH_MODE_TIMESHARE;
+       }

 }
 
 static boolean_t
 }
 
 static boolean_t

-sched_proto_can_update_priority(thread_t       thread __unused)
+sched_proto_can_update_priority(thread_t        thread __unused)

 {
        return FALSE;
 }
 
 static void
 {
        return FALSE;
 }
 
 static void

-sched_proto_update_priority(thread_t   thread __unused)
+sched_proto_update_priority(thread_t    thread __unused)

 {
 {

-       

 }
 
 static void
 }
 
 static void

-sched_proto_lightweight_update_priority(thread_t       thread __unused)
+sched_proto_lightweight_update_priority(thread_t        thread __unused)

 {
 {

-       

 }
 
 static void
 }
 
 static void

-sched_proto_quantum_expire(thread_t    thread __unused)
-{
-       
-}
-
-static boolean_t
-sched_proto_should_current_thread_rechoose_processor(processor_t                       processor)
+sched_proto_quantum_expire(thread_t    thread __unused)

 {
 {

-       return (proto_processor != processor);

 }
 
 static int
 }
 
 static int


@@ -582,7 +575,7 @@ sched_proto_processor_runq_count(processor_t   processor)
                return global_runq->count;
        } else {
                return 0;
                return global_runq->count;
        } else {
                return 0;

-       }       
+       }

 }
 
 static uint64_t
 }
 
 static uint64_t


@@ -602,10 +595,6 @@ sched_proto_processor_bound_count(__unused processor_t   processor)
 }
 
 static void
 }
 
 static void

-sched_proto_thread_update_scan(void)
+sched_proto_thread_update_scan(__unused sched_update_scan_context_t scan_context)

 {
 {

-       

 }
 }

-
-
-