X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/6601e61aa18bf4f09af135ff61fc7f4771d23b06..316670eb35587141e969394ae8537d66b9211e80:/osfmk/kern/thread_call.c diff --git a/osfmk/kern/thread_call.c b/osfmk/kern/thread_call.c index 789e1d52f..7d43919ae 100644 --- a/osfmk/kern/thread_call.c +++ b/osfmk/kern/thread_call.c @@ -1,31 +1,36 @@ /* - * Copyright (c) 1993-1995, 1999-2005 Apple Computer, Inc. - * All rights reserved. + * Copyright (c) 1993-1995, 1999-2008 Apple Inc. All rights reserved. * - * @APPLE_LICENSE_HEADER_START@ + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * - * The contents of this file constitute Original Code as defined in and - * are subject to the Apple Public Source License Version 1.1 (the - * "License"). You may not use this file except in compliance with the - * License. Please obtain a copy of the License at - * http://www.apple.com/publicsource and read it before using this file. + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. * - * This Original Code and all software distributed under the License are - * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the - * License for the specific language governing rights and limitations - * under the License. + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. * - * @APPLE_LICENSE_HEADER_END@ + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ #include #include #include -#include +#include #include #include #include @@ -36,154 +41,302 @@ #include #include - #include +#include + #include -#define internal_call_num 768 -#define thread_call_thread_min 4 +static zone_t thread_call_zone; +static struct wait_queue daemon_wqueue; + +struct thread_call_group { + queue_head_t pending_queue; + uint32_t pending_count; + + queue_head_t delayed_queue; + uint32_t delayed_count; + + timer_call_data_t delayed_timer; + timer_call_data_t dealloc_timer; + + struct wait_queue idle_wqueue; + uint32_t idle_count, active_count; + + integer_t pri; + uint32_t target_thread_count; + uint64_t idle_timestamp; + + uint32_t flags; + sched_call_t sched_call; +}; + +typedef struct thread_call_group *thread_call_group_t; + +#define TCG_PARALLEL 0x01 +#define TCG_DEALLOC_ACTIVE 0x02 + +#define THREAD_CALL_GROUP_COUNT 4 +#define THREAD_CALL_THREAD_MIN 4 +#define INTERNAL_CALL_COUNT 768 +#define THREAD_CALL_DEALLOC_INTERVAL_NS (5 * 1000 * 1000) /* 5 ms */ +#define THREAD_CALL_ADD_RATIO 4 +#define THREAD_CALL_MACH_FACTOR_CAP 3 + +static struct thread_call_group thread_call_groups[THREAD_CALL_GROUP_COUNT]; +static boolean_t thread_call_daemon_awake; +static thread_call_data_t internal_call_storage[INTERNAL_CALL_COUNT]; +static queue_head_t thread_call_internal_queue; +static uint64_t thread_call_dealloc_interval_abs; + +static __inline__ thread_call_t _internal_call_allocate(void); +static __inline__ void _internal_call_release(thread_call_t call); +static __inline__ boolean_t _pending_call_enqueue(thread_call_t call, thread_call_group_t group); +static __inline__ boolean_t _delayed_call_enqueue(thread_call_t call, thread_call_group_t group, uint64_t deadline); +static __inline__ boolean_t _call_dequeue(thread_call_t call, thread_call_group_t group); +static __inline__ void thread_call_wake(thread_call_group_t group); +static __inline__ void _set_delayed_call_timer(thread_call_t call, thread_call_group_t group); +static boolean_t _remove_from_pending_queue(thread_call_func_t func, thread_call_param_t param0, boolean_t remove_all); +static boolean_t _remove_from_delayed_queue(thread_call_func_t func, thread_call_param_t param0, boolean_t remove_all); +static void thread_call_daemon(void *arg); +static void thread_call_thread(thread_call_group_t group, wait_result_t wres); +extern void thread_call_delayed_timer(timer_call_param_t p0, timer_call_param_t p1); +static void thread_call_dealloc_timer(timer_call_param_t p0, timer_call_param_t p1); +static void thread_call_group_setup(thread_call_group_t group, thread_call_priority_t pri, uint32_t target_thread_count, boolean_t parallel); +static void sched_call_thread(int type, thread_t thread); +static void thread_call_start_deallocate_timer(thread_call_group_t group); +static void thread_call_wait_locked(thread_call_t call); -static -thread_call_data_t - internal_call_storage[internal_call_num]; +#define qe(x) ((queue_entry_t)(x)) +#define TC(x) ((thread_call_t)(x)) -decl_simple_lock_data(static,thread_call_lock) -static -timer_call_data_t - thread_call_delaytimer; +lck_grp_t thread_call_queues_lck_grp; +lck_grp_t thread_call_lck_grp; +lck_attr_t thread_call_lck_attr; +lck_grp_attr_t thread_call_lck_grp_attr; -static -queue_head_t - thread_call_xxx_queue, - thread_call_pending_queue, thread_call_delayed_queue; +#if defined(__i386__) || defined(__x86_64__) +lck_mtx_t thread_call_lock_data; +#else +lck_spin_t thread_call_lock_data; +#endif -static -struct wait_queue - call_thread_waitqueue; -static -boolean_t - activate_thread_awake; - -static struct { - int pending_num, - pending_hiwat; - int active_num, - active_hiwat, - active_lowat; - int delayed_num, - delayed_hiwat; - int idle_thread_num; - int thread_num, - thread_hiwat, - thread_lowat; -} thread_call_vars; +#define thread_call_lock_spin() \ + lck_mtx_lock_spin_always(&thread_call_lock_data) -static __inline__ thread_call_t - _internal_call_allocate(void); +#define thread_call_unlock() \ + lck_mtx_unlock_always(&thread_call_lock_data) -static __inline__ void -_internal_call_release( - thread_call_t call -); -static __inline__ void -_pending_call_enqueue( - thread_call_t call -), -_pending_call_dequeue( - thread_call_t call -), -_delayed_call_enqueue( - thread_call_t call -), -_delayed_call_dequeue( - thread_call_t call -); +static inline spl_t +disable_ints_and_lock(void) +{ + spl_t s; + + s = splsched(); + thread_call_lock_spin(); + + return s; +} + +static inline void +enable_ints_and_unlock(void) +{ + thread_call_unlock(); + (void)spllo(); +} + + +static inline boolean_t +group_isparallel(thread_call_group_t group) +{ + return ((group->flags & TCG_PARALLEL) != 0); +} -static __inline__ void -_set_delayed_call_timer( - thread_call_t call -); - static boolean_t -_remove_from_pending_queue( - thread_call_func_t func, - thread_call_param_t param0, - boolean_t remove_all -), -_remove_from_delayed_queue( - thread_call_func_t func, - thread_call_param_t param0, - boolean_t remove_all -); +thread_call_group_should_add_thread(thread_call_group_t group) +{ + uint32_t thread_count; -static __inline__ void - _call_thread_wake(void); + if (!group_isparallel(group)) { + if (group->pending_count > 0 && group->active_count == 0) { + return TRUE; + } -static void - _call_thread(void), - _activate_thread(void); + return FALSE; + } + + if (group->pending_count > 0) { + if (group->idle_count > 0) { + panic("Pending work, but threads are idle?"); + } + + thread_count = group->active_count; + + /* + * Add a thread if either there are no threads, + * the group has fewer than its target number of + * threads, or the amount of work is large relative + * to the number of threads. In the last case, pay attention + * to the total load on the system, and back off if + * it's high. + */ + if ((thread_count == 0) || + (thread_count < group->target_thread_count) || + ((group->pending_count > THREAD_CALL_ADD_RATIO * thread_count) && + (sched_mach_factor < THREAD_CALL_MACH_FACTOR_CAP))) { + return TRUE; + } + } + + return FALSE; +} + +static inline integer_t +thread_call_priority_to_sched_pri(thread_call_priority_t pri) +{ + switch (pri) { + case THREAD_CALL_PRIORITY_HIGH: + return BASEPRI_PREEMPT; + case THREAD_CALL_PRIORITY_KERNEL: + return BASEPRI_KERNEL; + case THREAD_CALL_PRIORITY_USER: + return BASEPRI_DEFAULT; + case THREAD_CALL_PRIORITY_LOW: + return DEPRESSPRI; + default: + panic("Invalid priority."); + } + + return 0; +} + +/* Lock held */ +static inline thread_call_group_t +thread_call_get_group( + thread_call_t call) +{ + thread_call_priority_t pri = call->tc_pri; + + assert(pri == THREAD_CALL_PRIORITY_LOW || + pri == THREAD_CALL_PRIORITY_USER || + pri == THREAD_CALL_PRIORITY_KERNEL || + pri == THREAD_CALL_PRIORITY_HIGH); + + return &thread_call_groups[pri]; +} static void -_delayed_call_timer( - timer_call_param_t p0, - timer_call_param_t p1 -); +thread_call_group_setup( + thread_call_group_t group, + thread_call_priority_t pri, + uint32_t target_thread_count, + boolean_t parallel) +{ + queue_init(&group->pending_queue); + queue_init(&group->delayed_queue); -#define qe(x) ((queue_entry_t)(x)) -#define TC(x) ((thread_call_t)(x)) + timer_call_setup(&group->delayed_timer, thread_call_delayed_timer, group); + timer_call_setup(&group->dealloc_timer, thread_call_dealloc_timer, group); + + wait_queue_init(&group->idle_wqueue, SYNC_POLICY_FIFO); + + group->target_thread_count = target_thread_count; + group->pri = thread_call_priority_to_sched_pri(pri); + + group->sched_call = sched_call_thread; + if (parallel) { + group->flags |= TCG_PARALLEL; + group->sched_call = NULL; + } +} /* - * Routine: thread_call_initialize [public] - * - * Description: Initialize this module, called - * early during system initialization. - * - * Preconditions: None. - * - * Postconditions: None. + * Simple wrapper for creating threads bound to + * thread call groups. */ - -void -thread_call_initialize(void) +static kern_return_t +thread_call_thread_create( + thread_call_group_t group) { - kern_return_t result; - thread_t thread; - thread_call_t call; - spl_t s; + thread_t thread; + kern_return_t result; - simple_lock_init(&thread_call_lock, 0); + result = kernel_thread_start_priority((thread_continue_t)thread_call_thread, group, group->pri, &thread); + if (result != KERN_SUCCESS) { + return result; + } - s = splsched(); - simple_lock(&thread_call_lock); + if (group->pri < BASEPRI_PREEMPT) { + /* + * New style doesn't get to run to completion in + * kernel if there are higher priority threads + * available. + */ + thread_set_eager_preempt(thread); + } - queue_init(&thread_call_pending_queue); - queue_init(&thread_call_delayed_queue); + thread_deallocate(thread); + return KERN_SUCCESS; +} - queue_init(&thread_call_xxx_queue); - for ( - call = internal_call_storage; - call < &internal_call_storage[internal_call_num]; +/* + * thread_call_initialize: + * + * Initialize this module, called + * early during system initialization. + */ +void +thread_call_initialize(void) +{ + thread_call_t call; + kern_return_t result; + thread_t thread; + int i; + + i = sizeof (thread_call_data_t); + thread_call_zone = zinit(i, 4096 * i, 16 * i, "thread_call"); + zone_change(thread_call_zone, Z_CALLERACCT, FALSE); + zone_change(thread_call_zone, Z_NOENCRYPT, TRUE); + + lck_attr_setdefault(&thread_call_lck_attr); + lck_grp_attr_setdefault(&thread_call_lck_grp_attr); + lck_grp_init(&thread_call_queues_lck_grp, "thread_call_queues", &thread_call_lck_grp_attr); + lck_grp_init(&thread_call_lck_grp, "thread_call", &thread_call_lck_grp_attr); + +#if defined(__i386__) || defined(__x86_64__) + lck_mtx_init(&thread_call_lock_data, &thread_call_lck_grp, &thread_call_lck_attr); +#else + lck_spin_init(&thread_call_lock_data, &thread_call_lck_grp, &thread_call_lck_attr); +#endif + + nanotime_to_absolutetime(0, THREAD_CALL_DEALLOC_INTERVAL_NS, &thread_call_dealloc_interval_abs); + wait_queue_init(&daemon_wqueue, SYNC_POLICY_FIFO); + + thread_call_group_setup(&thread_call_groups[THREAD_CALL_PRIORITY_LOW], THREAD_CALL_PRIORITY_LOW, 0, TRUE); + thread_call_group_setup(&thread_call_groups[THREAD_CALL_PRIORITY_USER], THREAD_CALL_PRIORITY_USER, 0, TRUE); + thread_call_group_setup(&thread_call_groups[THREAD_CALL_PRIORITY_KERNEL], THREAD_CALL_PRIORITY_KERNEL, 1, TRUE); + thread_call_group_setup(&thread_call_groups[THREAD_CALL_PRIORITY_HIGH], THREAD_CALL_PRIORITY_HIGH, THREAD_CALL_THREAD_MIN, FALSE); + + disable_ints_and_lock(); + + queue_init(&thread_call_internal_queue); + for ( + call = internal_call_storage; + call < &internal_call_storage[INTERNAL_CALL_COUNT]; call++) { - enqueue_tail(&thread_call_xxx_queue, qe(call)); - } - - timer_call_setup(&thread_call_delaytimer, _delayed_call_timer, NULL); - - wait_queue_init(&call_thread_waitqueue, SYNC_POLICY_FIFO); - thread_call_vars.thread_lowat = thread_call_thread_min; + enqueue_tail(&thread_call_internal_queue, qe(call)); + } - activate_thread_awake = TRUE; + thread_call_daemon_awake = TRUE; - simple_unlock(&thread_call_lock); - splx(s); + enable_ints_and_unlock(); - result = kernel_thread_start_priority((thread_continue_t)_activate_thread, NULL, MAXPRI_KERNEL - 2, &thread); + result = kernel_thread_start_priority((thread_continue_t)thread_call_daemon, NULL, BASEPRI_PREEMPT + 1, &thread); if (result != KERN_SUCCESS) panic("thread_call_initialize"); @@ -194,982 +347,1126 @@ void thread_call_setup( thread_call_t call, thread_call_func_t func, - thread_call_param_t param0 -) + thread_call_param_t param0) { - call_entry_setup(call, func, param0); + bzero(call, sizeof(*call)); + call_entry_setup((call_entry_t)call, func, param0); + call->tc_pri = THREAD_CALL_PRIORITY_HIGH; /* Default priority */ } /* - * Routine: _internal_call_allocate [private, inline] + * _internal_call_allocate: * - * Purpose: Allocate an internal callout entry. + * Allocate an internal callout entry. * - * Preconditions: thread_call_lock held. - * - * Postconditions: None. + * Called with thread_call_lock held. */ - static __inline__ thread_call_t _internal_call_allocate(void) { thread_call_t call; - if (queue_empty(&thread_call_xxx_queue)) + if (queue_empty(&thread_call_internal_queue)) panic("_internal_call_allocate"); - call = TC(dequeue_head(&thread_call_xxx_queue)); + call = TC(dequeue_head(&thread_call_internal_queue)); return (call); } /* - * Routine: _internal_call_release [private, inline] + * _internal_call_release: * - * Purpose: Release an internal callout entry which - * is no longer pending (or delayed). + * Release an internal callout entry which + * is no longer pending (or delayed). * - * Preconditions: thread_call_lock held. - * - * Postconditions: None. + * Called with thread_call_lock held. */ - -static __inline__ -void +static __inline__ void _internal_call_release( - thread_call_t call -) + thread_call_t call) { if ( call >= internal_call_storage && - call < &internal_call_storage[internal_call_num] ) - enqueue_head(&thread_call_xxx_queue, qe(call)); + call < &internal_call_storage[INTERNAL_CALL_COUNT] ) + enqueue_head(&thread_call_internal_queue, qe(call)); } /* - * Routine: _pending_call_enqueue [private, inline] + * _pending_call_enqueue: * - * Purpose: Place an entry at the end of the - * pending queue, to be executed soon. + * Place an entry at the end of the + * pending queue, to be executed soon. * - * Preconditions: thread_call_lock held. + * Returns TRUE if the entry was already + * on a queue. * - * Postconditions: None. + * Called with thread_call_lock held. */ - -static __inline__ -void +static __inline__ boolean_t _pending_call_enqueue( - thread_call_t call -) + thread_call_t call, + thread_call_group_t group) { - enqueue_tail(&thread_call_pending_queue, qe(call)); - if (++thread_call_vars.pending_num > thread_call_vars.pending_hiwat) - thread_call_vars.pending_hiwat = thread_call_vars.pending_num; + queue_head_t *old_queue; - call->state = PENDING; -} + old_queue = call_entry_enqueue_tail(CE(call), &group->pending_queue); -/* - * Routine: _pending_call_dequeue [private, inline] - * - * Purpose: Remove an entry from the pending queue, - * effectively unscheduling it. - * - * Preconditions: thread_call_lock held. - * - * Postconditions: None. - */ + if (old_queue == NULL) { + call->tc_submit_count++; + } -static __inline__ -void -_pending_call_dequeue( - thread_call_t call -) -{ - (void)remque(qe(call)); - thread_call_vars.pending_num--; - - call->state = IDLE; + group->pending_count++; + + thread_call_wake(group); + + return (old_queue != NULL); } /* - * Routine: _delayed_call_enqueue [private, inline] + * _delayed_call_enqueue: * - * Purpose: Place an entry on the delayed queue, - * after existing entries with an earlier - * (or identical) deadline. + * Place an entry on the delayed queue, + * after existing entries with an earlier + * (or identical) deadline. * - * Preconditions: thread_call_lock held. + * Returns TRUE if the entry was already + * on a queue. * - * Postconditions: None. + * Called with thread_call_lock held. */ - -static __inline__ -void +static __inline__ boolean_t _delayed_call_enqueue( - thread_call_t call -) + thread_call_t call, + thread_call_group_t group, + uint64_t deadline) { - thread_call_t current; - - current = TC(queue_first(&thread_call_delayed_queue)); - - while (TRUE) { - if ( queue_end(&thread_call_delayed_queue, qe(current)) || - call->deadline < current->deadline ) { - current = TC(queue_prev(qe(current))); - break; - } - - current = TC(queue_next(qe(current))); - } + queue_head_t *old_queue; - insque(qe(call), qe(current)); - if (++thread_call_vars.delayed_num > thread_call_vars.delayed_hiwat) - thread_call_vars.delayed_hiwat = thread_call_vars.delayed_num; - - call->state = DELAYED; + old_queue = call_entry_enqueue_deadline(CE(call), &group->delayed_queue, deadline); + + if (old_queue == &group->pending_queue) + group->pending_count--; + else if (old_queue == NULL) + call->tc_submit_count++; + + return (old_queue != NULL); } /* - * Routine: _delayed_call_dequeue [private, inline] + * _call_dequeue: * - * Purpose: Remove an entry from the delayed queue, - * effectively unscheduling it. + * Remove an entry from a queue. * - * Preconditions: thread_call_lock held. + * Returns TRUE if the entry was on a queue. * - * Postconditions: None. + * Called with thread_call_lock held. */ - -static __inline__ -void -_delayed_call_dequeue( - thread_call_t call -) +static __inline__ boolean_t +_call_dequeue( + thread_call_t call, + thread_call_group_t group) { - (void)remque(qe(call)); - thread_call_vars.delayed_num--; - - call->state = IDLE; + queue_head_t *old_queue; + + old_queue = call_entry_dequeue(CE(call)); + + if (old_queue != NULL) { + call->tc_finish_count++; + if (old_queue == &group->pending_queue) + group->pending_count--; + } + + return (old_queue != NULL); } /* - * Routine: _set_delayed_call_timer [private] + * _set_delayed_call_timer: * - * Purpose: Reset the timer so that it - * next expires when the entry is due. + * Reset the timer so that it + * next expires when the entry is due. * - * Preconditions: thread_call_lock held. - * - * Postconditions: None. + * Called with thread_call_lock held. */ - static __inline__ void _set_delayed_call_timer( - thread_call_t call -) + thread_call_t call, + thread_call_group_t group) { - timer_call_enter(&thread_call_delaytimer, call->deadline); + timer_call_enter(&group->delayed_timer, call->tc_call.deadline, 0); } /* - * Routine: _remove_from_pending_queue [private] + * _remove_from_pending_queue: * - * Purpose: Remove the first (or all) matching - * entries from the pending queue, - * effectively unscheduling them. - * Returns whether any matching entries - * were found. + * Remove the first (or all) matching + * entries from the pending queue. * - * Preconditions: thread_call_lock held. + * Returns TRUE if any matching entries + * were found. * - * Postconditions: None. + * Called with thread_call_lock held. */ - -static -boolean_t +static boolean_t _remove_from_pending_queue( thread_call_func_t func, thread_call_param_t param0, - boolean_t remove_all -) + boolean_t remove_all) { - boolean_t call_removed = FALSE; - thread_call_t call; - - call = TC(queue_first(&thread_call_pending_queue)); - - while (!queue_end(&thread_call_pending_queue, qe(call))) { - if ( call->func == func && - call->param0 == param0 ) { + boolean_t call_removed = FALSE; + thread_call_t call; + thread_call_group_t group = &thread_call_groups[THREAD_CALL_PRIORITY_HIGH]; + + call = TC(queue_first(&group->pending_queue)); + + while (!queue_end(&group->pending_queue, qe(call))) { + if (call->tc_call.func == func && + call->tc_call.param0 == param0) { thread_call_t next = TC(queue_next(qe(call))); - - _pending_call_dequeue(call); + + _call_dequeue(call, group); _internal_call_release(call); - + call_removed = TRUE; if (!remove_all) break; - + call = next; } else call = TC(queue_next(qe(call))); - } - - return (call_removed); + } + + return (call_removed); } /* - * Routine: _remove_from_delayed_queue [private] + * _remove_from_delayed_queue: * - * Purpose: Remove the first (or all) matching - * entries from the delayed queue, - * effectively unscheduling them. - * Returns whether any matching entries - * were found. + * Remove the first (or all) matching + * entries from the delayed queue. * - * Preconditions: thread_call_lock held. + * Returns TRUE if any matching entries + * were found. * - * Postconditions: None. + * Called with thread_call_lock held. */ - -static -boolean_t +static boolean_t _remove_from_delayed_queue( thread_call_func_t func, thread_call_param_t param0, - boolean_t remove_all -) + boolean_t remove_all) { - boolean_t call_removed = FALSE; - thread_call_t call; - - call = TC(queue_first(&thread_call_delayed_queue)); - - while (!queue_end(&thread_call_delayed_queue, qe(call))) { - if ( call->func == func && - call->param0 == param0 ) { + boolean_t call_removed = FALSE; + thread_call_t call; + thread_call_group_t group = &thread_call_groups[THREAD_CALL_PRIORITY_HIGH]; + + call = TC(queue_first(&group->delayed_queue)); + + while (!queue_end(&group->delayed_queue, qe(call))) { + if (call->tc_call.func == func && + call->tc_call.param0 == param0) { thread_call_t next = TC(queue_next(qe(call))); - - _delayed_call_dequeue(call); - + + _call_dequeue(call, group); + _internal_call_release(call); - + call_removed = TRUE; if (!remove_all) break; - + call = next; } else call = TC(queue_next(qe(call))); - } - - return (call_removed); + } + + return (call_removed); } +#ifndef __LP64__ + /* - * Routine: thread_call_func [public] - * - * Purpose: Schedule a function callout. - * Guarantees { function, argument } - * uniqueness if unique_call is TRUE. + * thread_call_func: * - * Preconditions: Callable from an interrupt context - * below splsched. + * Enqueue a function callout. * - * Postconditions: None. + * Guarantees { function, argument } + * uniqueness if unique_call is TRUE. */ - void thread_call_func( thread_call_func_t func, thread_call_param_t param, - boolean_t unique_call -) + boolean_t unique_call) { - thread_call_t call; - spl_t s; - - s = splsched(); - simple_lock(&thread_call_lock); - - call = TC(queue_first(&thread_call_pending_queue)); - - while (unique_call && !queue_end(&thread_call_pending_queue, qe(call))) { - if ( call->func == func && - call->param0 == param ) { + thread_call_t call; + thread_call_group_t group = &thread_call_groups[THREAD_CALL_PRIORITY_HIGH]; + spl_t s; + + s = splsched(); + thread_call_lock_spin(); + + call = TC(queue_first(&group->pending_queue)); + + while (unique_call && !queue_end(&group->pending_queue, qe(call))) { + if (call->tc_call.func == func && call->tc_call.param0 == param) { break; } - + call = TC(queue_next(qe(call))); - } - - if (!unique_call || queue_end(&thread_call_pending_queue, qe(call))) { + } + + if (!unique_call || queue_end(&group->pending_queue, qe(call))) { call = _internal_call_allocate(); - call->func = func; - call->param0 = param; - call->param1 = 0; - - _pending_call_enqueue(call); - - if (thread_call_vars.active_num <= 0) - _call_thread_wake(); - } + call->tc_call.func = func; + call->tc_call.param0 = param; + call->tc_call.param1 = NULL; - simple_unlock(&thread_call_lock); - splx(s); + _pending_call_enqueue(call, group); + } + + thread_call_unlock(); + splx(s); } +#endif /* __LP64__ */ + /* - * Routine: thread_call_func_delayed [public] + * thread_call_func_delayed: * - * Purpose: Schedule a function callout to - * occur at the stated time. - * - * Preconditions: Callable from an interrupt context - * below splsched. - * - * Postconditions: None. + * Enqueue a function callout to + * occur at the stated time. */ - void thread_call_func_delayed( - thread_call_func_t func, - thread_call_param_t param, - uint64_t deadline -) + thread_call_func_t func, + thread_call_param_t param, + uint64_t deadline) { - thread_call_t call; - spl_t s; - - s = splsched(); - simple_lock(&thread_call_lock); - - call = _internal_call_allocate(); - call->func = func; - call->param0 = param; - call->param1 = 0; - call->deadline = deadline; - - _delayed_call_enqueue(call); - - if (queue_first(&thread_call_delayed_queue) == qe(call)) - _set_delayed_call_timer(call); - - simple_unlock(&thread_call_lock); - splx(s); + thread_call_t call; + thread_call_group_t group = &thread_call_groups[THREAD_CALL_PRIORITY_HIGH]; + spl_t s; + + s = splsched(); + thread_call_lock_spin(); + + call = _internal_call_allocate(); + call->tc_call.func = func; + call->tc_call.param0 = param; + call->tc_call.param1 = 0; + + _delayed_call_enqueue(call, group, deadline); + + if (queue_first(&group->delayed_queue) == qe(call)) + _set_delayed_call_timer(call, group); + + thread_call_unlock(); + splx(s); } /* - * Routine: thread_call_func_cancel [public] + * thread_call_func_cancel: * - * Purpose: Unschedule a function callout. - * Removes one (or all) - * { function, argument } - * instance(s) from either (or both) - * the pending and the delayed queue, - * in that order. Returns a boolean - * indicating whether any calls were - * cancelled. + * Dequeue a function callout. * - * Preconditions: Callable from an interrupt context - * below splsched. + * Removes one (or all) { function, argument } + * instance(s) from either (or both) + * the pending and the delayed queue, + * in that order. * - * Postconditions: None. + * Returns TRUE if any calls were cancelled. */ - boolean_t thread_call_func_cancel( - thread_call_func_t func, - thread_call_param_t param, - boolean_t cancel_all -) + thread_call_func_t func, + thread_call_param_t param, + boolean_t cancel_all) { - boolean_t result; - spl_t s; - - s = splsched(); - simple_lock(&thread_call_lock); + boolean_t result; + spl_t s; - if (cancel_all) + s = splsched(); + thread_call_lock_spin(); + + if (cancel_all) result = _remove_from_pending_queue(func, param, cancel_all) | - _remove_from_delayed_queue(func, param, cancel_all); + _remove_from_delayed_queue(func, param, cancel_all); else result = _remove_from_pending_queue(func, param, cancel_all) || - _remove_from_delayed_queue(func, param, cancel_all); - - simple_unlock(&thread_call_lock); - splx(s); + _remove_from_delayed_queue(func, param, cancel_all); + + thread_call_unlock(); + splx(s); return (result); } /* - * Routine: thread_call_allocate [public] - * - * Purpose: Allocate an external callout - * entry. - * - * Preconditions: None. - * - * Postconditions: None. + * Allocate a thread call with a given priority. Importances + * other than THREAD_CALL_PRIORITY_HIGH will be run in threads + * with eager preemption enabled (i.e. may be aggressively preempted + * by higher-priority threads which are not in the normal "urgent" bands). */ +thread_call_t +thread_call_allocate_with_priority( + thread_call_func_t func, + thread_call_param_t param0, + thread_call_priority_t pri) +{ + thread_call_t call; + if (pri > THREAD_CALL_PRIORITY_LOW) { + panic("Invalid pri: %d\n", pri); + } + + call = thread_call_allocate(func, param0); + call->tc_pri = pri; + + return call; +} + +/* + * thread_call_allocate: + * + * Allocate a callout entry. + */ thread_call_t thread_call_allocate( - thread_call_func_t func, - thread_call_param_t param0 -) + thread_call_func_t func, + thread_call_param_t param0) { - thread_call_t call = (void *)kalloc(sizeof (thread_call_data_t)); - - call->func = func; - call->param0 = param0; - call->state = IDLE; - - return (call); + thread_call_t call = zalloc(thread_call_zone); + + thread_call_setup(call, func, param0); + call->tc_refs = 1; + call->tc_flags = THREAD_CALL_ALLOC; + + return (call); } /* - * Routine: thread_call_free [public] + * thread_call_free: * - * Purpose: Free an external callout - * entry. - * - * Preconditions: None. - * - * Postconditions: None. + * Release a callout. If the callout is currently + * executing, it will be freed when all invocations + * finish. */ - boolean_t thread_call_free( - thread_call_t call -) + thread_call_t call) { - spl_t s; - - s = splsched(); - simple_lock(&thread_call_lock); - - if (call->state != IDLE) { - simple_unlock(&thread_call_lock); + spl_t s; + int32_t refs; + + s = splsched(); + thread_call_lock_spin(); + + if (call->tc_call.queue != NULL) { + thread_call_unlock(); splx(s); return (FALSE); - } - - simple_unlock(&thread_call_lock); - splx(s); - - kfree(call, sizeof (thread_call_data_t)); + } + + refs = --call->tc_refs; + if (refs < 0) { + panic("Refcount negative: %d\n", refs); + } + + thread_call_unlock(); + splx(s); + + if (refs == 0) { + zfree(thread_call_zone, call); + } return (TRUE); } /* - * Routine: thread_call_enter [public] + * thread_call_enter: * - * Purpose: Schedule an external callout - * entry to occur "soon". Returns a - * boolean indicating whether the call - * had been already scheduled. + * Enqueue a callout entry to occur "soon". * - * Preconditions: Callable from an interrupt context - * below splsched. - * - * Postconditions: None. + * Returns TRUE if the call was + * already on a queue. */ - boolean_t thread_call_enter( - thread_call_t call -) + thread_call_t call) { boolean_t result = TRUE; - spl_t s; - - s = splsched(); - simple_lock(&thread_call_lock); - - if (call->state != PENDING) { - if (call->state == DELAYED) - _delayed_call_dequeue(call); - else if (call->state == IDLE) - result = FALSE; + thread_call_group_t group; + spl_t s; - _pending_call_enqueue(call); - - if (thread_call_vars.active_num <= 0) - _call_thread_wake(); + group = thread_call_get_group(call); + + s = splsched(); + thread_call_lock_spin(); + + if (call->tc_call.queue != &group->pending_queue) { + result = _pending_call_enqueue(call, group); } - call->param1 = 0; + call->tc_call.param1 = 0; - simple_unlock(&thread_call_lock); - splx(s); + thread_call_unlock(); + splx(s); return (result); } boolean_t thread_call_enter1( - thread_call_t call, - thread_call_param_t param1 -) + thread_call_t call, + thread_call_param_t param1) { - boolean_t result = TRUE; - spl_t s; - - s = splsched(); - simple_lock(&thread_call_lock); - - if (call->state != PENDING) { - if (call->state == DELAYED) - _delayed_call_dequeue(call); - else if (call->state == IDLE) - result = FALSE; + boolean_t result = TRUE; + thread_call_group_t group; + spl_t s; + + group = thread_call_get_group(call); - _pending_call_enqueue(call); + s = splsched(); + thread_call_lock_spin(); - if (thread_call_vars.active_num <= 0) - _call_thread_wake(); - } + if (call->tc_call.queue != &group->pending_queue) { + result = _pending_call_enqueue(call, group); + } - call->param1 = param1; + call->tc_call.param1 = param1; - simple_unlock(&thread_call_lock); - splx(s); + thread_call_unlock(); + splx(s); return (result); } /* - * Routine: thread_call_enter_delayed [public] - * - * Purpose: Schedule an external callout - * entry to occur at the stated time. - * Returns a boolean indicating whether - * the call had been already scheduled. + * thread_call_enter_delayed: * - * Preconditions: Callable from an interrupt context - * below splsched. + * Enqueue a callout entry to occur + * at the stated time. * - * Postconditions: None. + * Returns TRUE if the call was + * already on a queue. */ - boolean_t thread_call_enter_delayed( - thread_call_t call, - uint64_t deadline -) + thread_call_t call, + uint64_t deadline) { boolean_t result = TRUE; - spl_t s; - - s = splsched(); - simple_lock(&thread_call_lock); + thread_call_group_t group; + spl_t s; + + group = thread_call_get_group(call); - if (call->state == PENDING) - _pending_call_dequeue(call); - else if (call->state == DELAYED) - _delayed_call_dequeue(call); - else if (call->state == IDLE) - result = FALSE; + s = splsched(); + thread_call_lock_spin(); - call->param1 = 0; - call->deadline = deadline; + result = _delayed_call_enqueue(call, group, deadline); - _delayed_call_enqueue(call); + if (queue_first(&group->delayed_queue) == qe(call)) + _set_delayed_call_timer(call, group); - if (queue_first(&thread_call_delayed_queue) == qe(call)) - _set_delayed_call_timer(call); + call->tc_call.param1 = 0; - simple_unlock(&thread_call_lock); - splx(s); + thread_call_unlock(); + splx(s); return (result); } boolean_t thread_call_enter1_delayed( - thread_call_t call, - thread_call_param_t param1, - uint64_t deadline -) + thread_call_t call, + thread_call_param_t param1, + uint64_t deadline) { - boolean_t result = TRUE; - spl_t s; + boolean_t result = TRUE; + thread_call_group_t group; + spl_t s; - s = splsched(); - simple_lock(&thread_call_lock); + group = thread_call_get_group(call); - if (call->state == PENDING) - _pending_call_dequeue(call); - else if (call->state == DELAYED) - _delayed_call_dequeue(call); - else if (call->state == IDLE) - result = FALSE; + s = splsched(); + thread_call_lock_spin(); - call->param1 = param1; - call->deadline = deadline; + result = _delayed_call_enqueue(call, group, deadline); - _delayed_call_enqueue(call); + if (queue_first(&group->delayed_queue) == qe(call)) + _set_delayed_call_timer(call, group); - if (queue_first(&thread_call_delayed_queue) == qe(call)) - _set_delayed_call_timer(call); + call->tc_call.param1 = param1; - simple_unlock(&thread_call_lock); - splx(s); + thread_call_unlock(); + splx(s); return (result); } /* - * Routine: thread_call_cancel [public] + * thread_call_cancel: * - * Purpose: Unschedule a callout entry. - * Returns a boolean indicating - * whether the call had actually - * been scheduled. + * Dequeue a callout entry. * - * Preconditions: Callable from an interrupt context - * below splsched. - * - * Postconditions: None. + * Returns TRUE if the call was + * on a queue. */ - boolean_t thread_call_cancel( - thread_call_t call -) + thread_call_t call) { - boolean_t result = TRUE; - spl_t s; - - s = splsched(); - simple_lock(&thread_call_lock); - - if (call->state == PENDING) - _pending_call_dequeue(call); - else if (call->state == DELAYED) - _delayed_call_dequeue(call); - else - result = FALSE; - - simple_unlock(&thread_call_lock); - splx(s); + boolean_t result; + thread_call_group_t group; + spl_t s; + + group = thread_call_get_group(call); + + s = splsched(); + thread_call_lock_spin(); + + result = _call_dequeue(call, group); + + thread_call_unlock(); + splx(s); return (result); } /* - * Routine: thread_call_is_delayed [public] - * - * Purpose: Returns a boolean indicating - * whether a call is currently scheduled - * to occur at a later time. Optionally - * returns the expiration time. + * Cancel a thread call. If it cannot be cancelled (i.e. + * is already in flight), waits for the most recent invocation + * to finish. Note that if clients re-submit this thread call, + * it may still be pending or in flight when thread_call_cancel_wait + * returns, but all requests to execute this work item prior + * to the call to thread_call_cancel_wait will have finished. + */ +boolean_t +thread_call_cancel_wait( + thread_call_t call) +{ + boolean_t result; + thread_call_group_t group; + + if ((call->tc_flags & THREAD_CALL_ALLOC) == 0) { + panic("%s: Can't wait on thread call whose storage I don't own.", __FUNCTION__); + } + + group = thread_call_get_group(call); + + (void) splsched(); + thread_call_lock_spin(); + + result = _call_dequeue(call, group); + if (result == FALSE) { + thread_call_wait_locked(call); + } + + thread_call_unlock(); + (void) spllo(); + + return result; +} + + +#ifndef __LP64__ + +/* + * thread_call_is_delayed: * - * Preconditions: Callable from an interrupt context - * below splsched. + * Returns TRUE if the call is + * currently on a delayed queue. * - * Postconditions: None. + * Optionally returns the expiration time. */ - boolean_t thread_call_is_delayed( thread_call_t call, uint64_t *deadline) { - boolean_t result = FALSE; - spl_t s; + boolean_t result = FALSE; + thread_call_group_t group; + spl_t s; + + group = thread_call_get_group(call); s = splsched(); - simple_lock(&thread_call_lock); + thread_call_lock_spin(); - if (call->state == DELAYED) { + if (call->tc_call.queue == &group->delayed_queue) { if (deadline != NULL) - *deadline = call->deadline; + *deadline = call->tc_call.deadline; result = TRUE; } - simple_unlock(&thread_call_lock); + thread_call_unlock(); splx(s); return (result); } +#endif /* __LP64__ */ + /* - * Routine: _call_thread_wake [private, inline] + * thread_call_wake: * - * Purpose: Wake a callout thread to service - * pending callout entries. May wake - * the activate thread in order to - * create additional callout threads. + * Wake a call thread to service + * pending call entries. May wake + * the daemon thread in order to + * create additional call threads. * - * Preconditions: thread_call_lock held. + * Called with thread_call_lock held. * - * Postconditions: None. + * For high-priority group, only does wakeup/creation if there are no threads + * running. */ - -static __inline__ -void -_call_thread_wake(void) +static __inline__ void +thread_call_wake( + thread_call_group_t group) { - if (wait_queue_wakeup_one(&call_thread_waitqueue, NULL, THREAD_AWAKENED) == KERN_SUCCESS) { - thread_call_vars.idle_thread_num--; - - if (++thread_call_vars.active_num > thread_call_vars.active_hiwat) - thread_call_vars.active_hiwat = thread_call_vars.active_num; - } - else - if (!activate_thread_awake) { - thread_wakeup_one(&activate_thread_awake); - activate_thread_awake = TRUE; + /* + * New behavior: use threads if you've got 'em. + * Traditional behavior: wake only if no threads running. + */ + if (group_isparallel(group) || group->active_count == 0) { + if (wait_queue_wakeup_one(&group->idle_wqueue, NO_EVENT, THREAD_AWAKENED, -1) == KERN_SUCCESS) { + group->idle_count--; group->active_count++; + + if (group->idle_count == 0) { + timer_call_cancel(&group->dealloc_timer); + group->flags &= TCG_DEALLOC_ACTIVE; + } + } else { + if (!thread_call_daemon_awake && thread_call_group_should_add_thread(group)) { + thread_call_daemon_awake = TRUE; + wait_queue_wakeup_one(&daemon_wqueue, NO_EVENT, THREAD_AWAKENED, -1); + } + } } } /* - * Routine: call_thread_block [private] + * sched_call_thread: * - * Purpose: Hook via thread dispatch on - * the occasion of a callout blocking. - * - * Preconditions: splsched. - * - * Postconditions: None. + * Call out invoked by the scheduler. Used only for high-priority + * thread call group. */ - -void -call_thread_block(void) +static void +sched_call_thread( + int type, + __unused thread_t thread) { - simple_lock(&thread_call_lock); + thread_call_group_t group; + + group = &thread_call_groups[THREAD_CALL_PRIORITY_HIGH]; /* XXX */ - if (--thread_call_vars.active_num < thread_call_vars.active_lowat) - thread_call_vars.active_lowat = thread_call_vars.active_num; + thread_call_lock_spin(); - if ( thread_call_vars.active_num <= 0 && - thread_call_vars.pending_num > 0 ) - _call_thread_wake(); + switch (type) { + + case SCHED_CALL_BLOCK: + --group->active_count; + if (group->pending_count > 0) + thread_call_wake(group); + break; + + case SCHED_CALL_UNBLOCK: + group->active_count++; + break; + } - simple_unlock(&thread_call_lock); + thread_call_unlock(); } -/* - * Routine: call_thread_unblock [private] - * - * Purpose: Hook via thread wakeup on - * the occasion of a callout unblocking. - * - * Preconditions: splsched. - * - * Postconditions: None. +/* + * Interrupts disabled, lock held; returns the same way. + * Only called on thread calls whose storage we own. Wakes up + * anyone who might be waiting on this work item and frees it + * if the client has so requested. */ - -void -call_thread_unblock(void) +static void +thread_call_finish(thread_call_t call) { - simple_lock(&thread_call_lock); + boolean_t dowake = FALSE; + + call->tc_finish_count++; + call->tc_refs--; + + if ((call->tc_flags & THREAD_CALL_WAIT) != 0) { + dowake = TRUE; + call->tc_flags &= ~THREAD_CALL_WAIT; + + /* + * Dropping lock here because the sched call for the + * high-pri group can take the big lock from under + * a thread lock. + */ + thread_call_unlock(); + thread_wakeup((event_t)call); + thread_call_lock_spin(); + } - if (++thread_call_vars.active_num > thread_call_vars.active_hiwat) - thread_call_vars.active_hiwat = thread_call_vars.active_num; + if (call->tc_refs == 0) { + if (dowake) { + panic("Someone waiting on a thread call that is scheduled for free: %p\n", call->tc_call.func); + } + + enable_ints_and_unlock(); + + zfree(thread_call_zone, call); + + (void)disable_ints_and_lock(); + } - simple_unlock(&thread_call_lock); } /* - * Routine: _call_thread [private] - * - * Purpose: Executed by a callout thread. - * - * Preconditions: None. - * - * Postconditions: None. + * thread_call_thread: */ - -static -void -_call_thread_continue(void) +static void +thread_call_thread( + thread_call_group_t group, + wait_result_t wres) { - thread_t self = current_thread(); + thread_t self = current_thread(); + boolean_t canwait; + + /* + * A wakeup with THREAD_INTERRUPTED indicates that + * we should terminate. + */ + if (wres == THREAD_INTERRUPTED) { + thread_terminate(self); - (void) splsched(); - simple_lock(&thread_call_lock); + /* NOTREACHED */ + panic("thread_terminate() returned?"); + } - self->options |= TH_OPT_CALLOUT; + (void)disable_ints_and_lock(); - while (thread_call_vars.pending_num > 0) { + thread_sched_call(self, group->sched_call); + + while (group->pending_count > 0) { thread_call_t call; thread_call_func_t func; thread_call_param_t param0, param1; - call = TC(dequeue_head(&thread_call_pending_queue)); - thread_call_vars.pending_num--; + call = TC(dequeue_head(&group->pending_queue)); + group->pending_count--; - func = call->func; - param0 = call->param0; - param1 = call->param1; - - call->state = IDLE; + func = call->tc_call.func; + param0 = call->tc_call.param0; + param1 = call->tc_call.param1; + + call->tc_call.queue = NULL; _internal_call_release(call); - simple_unlock(&thread_call_lock); - (void) spllo(); + /* + * Can only do wakeups for thread calls whose storage + * we control. + */ + if ((call->tc_flags & THREAD_CALL_ALLOC) != 0) { + canwait = TRUE; + call->tc_refs++; /* Delay free until we're done */ + } else + canwait = FALSE; + + enable_ints_and_unlock(); KERNEL_DEBUG_CONSTANT( - MACHDBG_CODE(DBG_MACH_SCHED,MACH_CALLOUT) | DBG_FUNC_NONE, - (int)func, (int)param0, (int)param1, 0, 0); + MACHDBG_CODE(DBG_MACH_SCHED,MACH_CALLOUT) | DBG_FUNC_NONE, + VM_KERNEL_UNSLIDE(func), param0, param1, 0, 0); (*func)(param0, param1); - (void)thread_funnel_set(self->funnel_lock, FALSE); + if (get_preemption_level() != 0) { + int pl = get_preemption_level(); + panic("thread_call_thread: preemption_level %d, last callout %p(%p, %p)", + pl, (void *)VM_KERNEL_UNSLIDE(func), param0, param1); + } - (void) splsched(); - simple_lock(&thread_call_lock); - } + (void)thread_funnel_set(self->funnel_lock, FALSE); /* XXX */ - self->options &= ~TH_OPT_CALLOUT; + (void) disable_ints_and_lock(); + + if (canwait) { + /* Frees if so desired */ + thread_call_finish(call); + } + } - if (--thread_call_vars.active_num < thread_call_vars.active_lowat) - thread_call_vars.active_lowat = thread_call_vars.active_num; - - if (thread_call_vars.idle_thread_num < thread_call_vars.thread_lowat) { - thread_call_vars.idle_thread_num++; + thread_sched_call(self, NULL); + group->active_count--; - wait_queue_assert_wait(&call_thread_waitqueue, NULL, THREAD_UNINT, 0); - - simple_unlock(&thread_call_lock); - (void) spllo(); + if (group_isparallel(group)) { + /* + * For new style of thread group, thread always blocks. + * If we have more than the target number of threads, + * and this is the first to block, and it isn't active + * already, set a timer for deallocating a thread if we + * continue to have a surplus. + */ + group->idle_count++; - thread_block((thread_continue_t)_call_thread_continue); - /* NOTREACHED */ - } - - thread_call_vars.thread_num--; - - simple_unlock(&thread_call_lock); - (void) spllo(); - - thread_terminate(self); - /* NOTREACHED */ -} + if (group->idle_count == 1) { + group->idle_timestamp = mach_absolute_time(); + } -static -void -_call_thread(void) -{ - _call_thread_continue(); - /* NOTREACHED */ -} + if (((group->flags & TCG_DEALLOC_ACTIVE) == 0) && + ((group->active_count + group->idle_count) > group->target_thread_count)) { + group->flags |= TCG_DEALLOC_ACTIVE; + thread_call_start_deallocate_timer(group); + } -/* - * Routine: _activate_thread [private] - * - * Purpose: Executed by the activate thread. - * - * Preconditions: None. - * - * Postconditions: Never terminates. - */ + /* Wait for more work (or termination) */ + wres = wait_queue_assert_wait(&group->idle_wqueue, NO_EVENT, THREAD_INTERRUPTIBLE, 0); + if (wres != THREAD_WAITING) { + panic("kcall worker unable to assert wait?"); + } -static -void -_activate_thread_continue(void) -{ - kern_return_t result; - thread_t thread; + enable_ints_and_unlock(); - (void) splsched(); - simple_lock(&thread_call_lock); - - while ( thread_call_vars.active_num <= 0 && - thread_call_vars.pending_num > 0 ) { + thread_block_parameter((thread_continue_t)thread_call_thread, group); + } else { + if (group->idle_count < group->target_thread_count) { + group->idle_count++; - if (++thread_call_vars.active_num > thread_call_vars.active_hiwat) - thread_call_vars.active_hiwat = thread_call_vars.active_num; + wait_queue_assert_wait(&group->idle_wqueue, NO_EVENT, THREAD_UNINT, 0); /* Interrupted means to exit */ - if (++thread_call_vars.thread_num > thread_call_vars.thread_hiwat) - thread_call_vars.thread_hiwat = thread_call_vars.thread_num; + enable_ints_and_unlock(); - simple_unlock(&thread_call_lock); - (void) spllo(); - - result = kernel_thread_start_priority((thread_continue_t)_call_thread, NULL, MAXPRI_KERNEL - 1, &thread); - if (result != KERN_SUCCESS) - panic("activate_thread"); + thread_block_parameter((thread_continue_t)thread_call_thread, group); + /* NOTREACHED */ + } + } - thread_deallocate(thread); + enable_ints_and_unlock(); - (void) splsched(); - simple_lock(&thread_call_lock); - } - - assert_wait(&activate_thread_awake, THREAD_INTERRUPTIBLE); - activate_thread_awake = FALSE; - - simple_unlock(&thread_call_lock); - (void) spllo(); - - thread_block((thread_continue_t)_activate_thread_continue); + thread_terminate(self); /* NOTREACHED */ } -static -void -_activate_thread(void) +/* + * thread_call_daemon: walk list of groups, allocating + * threads if appropriate (as determined by + * thread_call_group_should_add_thread()). + */ +static void +thread_call_daemon_continue(__unused void *arg) +{ + int i; + kern_return_t kr; + thread_call_group_t group; + + (void)disable_ints_and_lock(); + + /* Starting at zero happens to be high-priority first. */ + for (i = 0; i < THREAD_CALL_GROUP_COUNT; i++) { + group = &thread_call_groups[i]; + while (thread_call_group_should_add_thread(group)) { + group->active_count++; + + enable_ints_and_unlock(); + + kr = thread_call_thread_create(group); + if (kr != KERN_SUCCESS) { + /* + * On failure, just pause for a moment and give up. + * We can try again later. + */ + delay(10000); /* 10 ms */ + (void)disable_ints_and_lock(); + goto out; + } + + (void)disable_ints_and_lock(); + } + } + +out: + thread_call_daemon_awake = FALSE; + wait_queue_assert_wait(&daemon_wqueue, NO_EVENT, THREAD_UNINT, 0); + + enable_ints_and_unlock(); + + thread_block_parameter((thread_continue_t)thread_call_daemon_continue, NULL); + /* NOTREACHED */ +} + +static void +thread_call_daemon( + __unused void *arg) { thread_t self = current_thread(); self->options |= TH_OPT_VMPRIV; vm_page_free_reserve(2); /* XXX */ - - _activate_thread_continue(); - /* NOTREACHED */ + + thread_call_daemon_continue(NULL); + /* NOTREACHED */ +} + +/* + * Schedule timer to deallocate a worker thread if we have a surplus + * of threads (in excess of the group's target) and at least one thread + * is idle the whole time. + */ +static void +thread_call_start_deallocate_timer( + thread_call_group_t group) +{ + uint64_t deadline; + boolean_t onqueue; + + assert(group->idle_count > 0); + + group->flags |= TCG_DEALLOC_ACTIVE; + deadline = group->idle_timestamp + thread_call_dealloc_interval_abs; + onqueue = timer_call_enter(&group->dealloc_timer, deadline, 0); + + if (onqueue) { + panic("Deallocate timer already active?"); + } } -static void -_delayed_call_timer( - __unused timer_call_param_t p0, - __unused timer_call_param_t p1 +thread_call_delayed_timer( + timer_call_param_t p0, + __unused timer_call_param_t p1 ) { - uint64_t timestamp; - thread_call_t call; - boolean_t new_pending = FALSE; - spl_t s; + thread_call_t call; + thread_call_group_t group = p0; + uint64_t timestamp; - s = splsched(); - simple_lock(&thread_call_lock); + thread_call_lock_spin(); - clock_get_uptime(×tamp); - - call = TC(queue_first(&thread_call_delayed_queue)); - - while (!queue_end(&thread_call_delayed_queue, qe(call))) { - if (call->deadline <= timestamp) { - _delayed_call_dequeue(call); + timestamp = mach_absolute_time(); - _pending_call_enqueue(call); - new_pending = TRUE; + call = TC(queue_first(&group->delayed_queue)); + + while (!queue_end(&group->delayed_queue, qe(call))) { + if (call->tc_call.deadline <= timestamp) { + _pending_call_enqueue(call, group); } else break; - - call = TC(queue_first(&thread_call_delayed_queue)); - } - if (!queue_end(&thread_call_delayed_queue, qe(call))) - _set_delayed_call_timer(call); + call = TC(queue_first(&group->delayed_queue)); + } - if (new_pending && thread_call_vars.active_num <= 0) - _call_thread_wake(); + if (!queue_end(&group->delayed_queue, qe(call))) + _set_delayed_call_timer(call, group); - simple_unlock(&thread_call_lock); - splx(s); + thread_call_unlock(); } + +/* + * Timer callback to tell a thread to terminate if + * we have an excess of threads and at least one has been + * idle for a long time. + */ +static void +thread_call_dealloc_timer( + timer_call_param_t p0, + __unused timer_call_param_t p1) +{ + thread_call_group_t group = (thread_call_group_t)p0; + uint64_t now; + kern_return_t res; + boolean_t terminated = FALSE; + + thread_call_lock_spin(); + + now = mach_absolute_time(); + if (group->idle_count > 0) { + if (now > group->idle_timestamp + thread_call_dealloc_interval_abs) { + terminated = TRUE; + group->idle_count--; + res = wait_queue_wakeup_one(&group->idle_wqueue, NO_EVENT, THREAD_INTERRUPTED, -1); + if (res != KERN_SUCCESS) { + panic("Unable to wake up idle thread for termination?"); + } + } + + } + + /* + * If we still have an excess of threads, schedule another + * invocation of this function. + */ + if (group->idle_count > 0 && (group->idle_count + group->active_count > group->target_thread_count)) { + /* + * If we killed someone just now, push out the + * next deadline. + */ + if (terminated) { + group->idle_timestamp = now; + } + + thread_call_start_deallocate_timer(group); + } else { + group->flags &= ~TCG_DEALLOC_ACTIVE; + } + + thread_call_unlock(); +} + +/* + * Wait for all requested invocations of a thread call prior to now + * to finish. Can only be invoked on thread calls whose storage we manage. + * Just waits for the finish count to catch up to the submit count we find + * at the beginning of our wait. + */ +static void +thread_call_wait_locked(thread_call_t call) +{ + uint64_t submit_count; + wait_result_t res; + + assert(call->tc_flags & THREAD_CALL_ALLOC); + + submit_count = call->tc_submit_count; + + while (call->tc_finish_count < submit_count) { + call->tc_flags |= THREAD_CALL_WAIT; + + res = assert_wait(call, THREAD_UNINT); + if (res != THREAD_WAITING) { + panic("Unable to assert wait?"); + } + + thread_call_unlock(); + (void) spllo(); + + res = thread_block(NULL); + if (res != THREAD_AWAKENED) { + panic("Awoken with %d?", res); + } + + (void) splsched(); + thread_call_lock_spin(); + } +} + +/* + * Determine whether a thread call is either on a queue or + * currently being executed. + */ +boolean_t +thread_call_isactive(thread_call_t call) +{ + boolean_t active; + + disable_ints_and_lock(); + active = (call->tc_submit_count > call->tc_finish_count); + enable_ints_and_unlock(); + + return active; +} +