[apple/xnu.git] / osfmk / kern / wait_queue.c

/*
 * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_LICENSE_HEADER_START@
 * 
 * The contents of this file constitute Original Code as defined in and
 * are subject to the Apple Public Source License Version 1.1 (the
 * "License").  You may not use this file except in compliance with the
 * License.  Please obtain a copy of the License at
 * http://www.apple.com/publicsource and read it before using this file.
 * 
 * This Original Code and all software distributed under the License are
 * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
 * License for the specific language governing rights and limitations
 * under the License.
 * 
 * @APPLE_LICENSE_HEADER_END@
 */
/*
 * @OSF_FREE_COPYRIGHT@
 */
/* 
 * Mach Operating System
 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
 * All Rights Reserved.
 * 
 * Permission to use, copy, modify and distribute this software and its
 * documentation is hereby granted, provided that both the copyright
 * notice and this permission notice appear in all copies of the
 * software, derivative works or modified versions, and any portions
 * thereof, and that both notices appear in supporting documentation.
 * 
 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
 * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
 * 
 * Carnegie Mellon requests users of this software to return to
 * 
 *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
 *  School of Computer Science
 *  Carnegie Mellon University
 *  Pittsburgh PA 15213-3890
 * 
 * any improvements or extensions that they make and grant Carnegie Mellon
 * the rights to redistribute these changes.
 */
/*
 */
/*
 *      File:   wait_queue.c (adapted from sched_prim.c)
 *      Author: Avadis Tevanian, Jr.
 *      Date:   1986
 *
 *      Primitives for manipulating wait queues: either global
 *      ones from sched_prim.c, or private ones associated with
 *      particular structures(pots, semaphores, etc..).
 */

#include <kern/kern_types.h>
#include <kern/simple_lock.h>
#include <kern/kalloc.h>
#include <kern/queue.h>
#include <kern/spl.h>
#include <mach/sync_policy.h>

#include <kern/sched_prim.h>
#include <kern/wait_queue.h>

void
wait_queue_init(
        wait_queue_t wq,
        int policy)
{
        wq->wq_fifo = (policy == SYNC_POLICY_FIFO);
        wq->wq_issub = FALSE;
        queue_init(&wq->wq_queue);
        hw_lock_init(&wq->wq_interlock);
}

void
wait_queue_sub_init(
        wait_queue_sub_t wqsub,
        int policy)
{
        wait_queue_init(&wqsub->wqs_wait_queue, policy);
        wqsub->wqs_wait_queue.wq_issub = TRUE;
        queue_init(&wqsub->wqs_sublinks);
}

void
wait_queue_link_init(
        wait_queue_link_t wql)
{
        queue_init(&wql->wql_links);
        queue_init(&wql->wql_sublinks);
        wql->wql_queue = WAIT_QUEUE_NULL;
        wql->wql_subqueue = WAIT_QUEUE_SUB_NULL;
        wql->wql_event = NO_EVENT;
}


/*
 *      Routine:        wait_queue_lock
 *      Purpose:
 *              Lock the wait queue.
 *      Conditions:
 *              the appropriate spl level (if any) is already raised.
 */
void
wait_queue_lock(
        wait_queue_t wq)
{
#ifdef __ppc__
        vm_offset_t pc;

        /*
         * Double the standard lock timeout, because wait queues tend
         * to iterate over a number of threads - locking each.  If there is
         * a problem with a thread lock, it normally times out at the wait
         * queue level first, hiding the real problem.
         */
        pc = GET_RETURN_PC(&wq);
        if (!hw_lock_to(&wq->wq_interlock, LockTimeOut * 2)) {
                panic("wait queue deadlock detection - wq=0x%x, cpu=%d, ret=0x%x\n", wq, cpu_number(), pc);
        }
#else
        hw_lock_lock(&wq->wq_interlock);
#endif
}

/*
 *      Routine:        wait_queue_lock_try
 *      Purpose:
 *              Try to lock the wait queue without waiting
 *      Conditions:
 *              the appropriate spl level (if any) is already raised.
 *  Returns:
 *              TRUE if the lock was acquired
 *              FALSE if we would have needed to wait
 */
boolean_t
wait_queue_lock_try(
        wait_queue_t wq)
{
        return hw_lock_try(&wq->wq_interlock);
}

/*
 *      Routine:        wait_queue_unlock
 *      Purpose:
 *              unlock the wait queue
 *      Conditions:
 *              The wait queue is assumed locked.
 *              appropriate spl level is still maintained
 */
void
wait_queue_unlock(
        wait_queue_t wq)
{
        assert(hw_lock_held(&wq->wq_interlock));

        hw_lock_unlock(&wq->wq_interlock);
}

int _wait_queue_subordinate; /* phoney event for subordinate wait q elements */

        
/*
 *      Routine:        wait_queue_member_locked
 *      Purpose:
 *              Indicate if this sub queue is a member of the queue
 *      Conditions:
 *              The wait queue is locked
 *              The sub queue is just that, a sub queue
 */
boolean_t
wait_queue_member_locked(
        wait_queue_t wq,
        wait_queue_sub_t wq_sub)
{
        wait_queue_element_t wq_element;
        queue_t q;

        assert(wait_queue_held(wq));
        assert(wait_queue_is_sub(wq_sub));

        q = &wq->wq_queue;

        wq_element = (wait_queue_element_t) queue_first(q);
        while (!queue_end(q, (queue_entry_t)wq_element)) {

                if ((wq_element->wqe_event == WAIT_QUEUE_SUBORDINATE)) {
                        wait_queue_link_t wql = (wait_queue_link_t)wq_element;

                        if (wql->wql_subqueue == wq_sub)
                                return TRUE;
                }
                wq_element = (wait_queue_element_t)
                             queue_next((queue_t) wq_element);
        }
        return FALSE;
}
        

/*
 *      Routine:        wait_queue_member
 *      Purpose:
 *              Indicate if this sub queue is a member of the queue
 *      Conditions:
 *              The sub queue is just that, a sub queue
 */
boolean_t
wait_queue_member(
        wait_queue_t wq,
        wait_queue_sub_t wq_sub)
{
        boolean_t ret;
        spl_t s;

        assert(wait_queue_is_sub(wq_sub));

        s = splsched();
        wait_queue_lock(wq);
        ret = wait_queue_member_locked(wq, wq_sub);
        wait_queue_unlock(wq);
        splx(s);

        return ret;
}

/*
 *      Routine:        wait_queue_link
 *      Purpose:
 *              Insert a subordinate wait queue into a wait queue.  This
 *              requires us to link the two together using a wait_queue_link
 *              structure that we allocate.
 *      Conditions:
 *              The wait queue being inserted must be inited as a sub queue
 *              The sub waitq is not already linked
 *
 */
kern_return_t
wait_queue_link(
        wait_queue_t wq,
        wait_queue_sub_t wq_sub)
{
        wait_queue_link_t wql;
        spl_t s;

        assert(wait_queue_is_sub(wq_sub));
        assert(!wait_queue_member(wq, wq_sub));

        wql = (wait_queue_link_t) kalloc(sizeof(struct wait_queue_link));
        if (wql == WAIT_QUEUE_LINK_NULL)
                return KERN_RESOURCE_SHORTAGE;
        
        wait_queue_link_init(wql);

        s = splsched();
        wait_queue_lock(wq);
        wqs_lock(wq_sub);

        wql->wql_queue = wq;
        wql->wql_subqueue = wq_sub;
        wql->wql_event = WAIT_QUEUE_SUBORDINATE;
        queue_enter(&wq->wq_queue, wql, wait_queue_link_t, wql_links);
        queue_enter(&wq_sub->wqs_sublinks, wql, wait_queue_link_t, wql_sublinks);
        
        wqs_unlock(wq_sub);
        wait_queue_unlock(wq);
        splx(s);

        return KERN_SUCCESS;
}       

/*
 *      Routine:        wait_queue_unlink
 *      Purpose:
 *              Remove the linkage between a wait queue and its subordinate.
 *      Conditions:
 *              The wait queue being must be a member sub queue
 */
kern_return_t
wait_queue_unlink(
        wait_queue_t wq,
        wait_queue_sub_t wq_sub)
{
        wait_queue_element_t wq_element;
        queue_t q;
        spl_t s;

        assert(wait_queue_is_sub(wq_sub));
        assert(wait_queue_member(wq, wq_sub));

        s = splsched();
        wait_queue_lock(wq);
        wqs_lock(wq_sub);

        q = &wq->wq_queue;

        wq_element = (wait_queue_element_t) queue_first(q);
        while (!queue_end(q, (queue_entry_t)wq_element)) {

                if (wq_element->wqe_event == WAIT_QUEUE_SUBORDINATE) {
                        wait_queue_link_t wql = (wait_queue_link_t)wq_element;
                        queue_t sq;
                        
                        if (wql->wql_subqueue == wq_sub) {
                                sq = &wq_sub->wqs_sublinks;
                                queue_remove(q, wql, wait_queue_link_t, wql_links);
                                queue_remove(sq, wql, wait_queue_link_t, wql_sublinks);
                                wqs_unlock(wq_sub);
                                wait_queue_unlock(wq);
                                splx(s);
                                kfree((vm_offset_t)wql,sizeof(struct wait_queue_link));
                                return;
                        }
                }

                wq_element = (wait_queue_element_t)
                             queue_next((queue_t) wq_element);
        }
        panic("wait_queue_unlink");
}       

/*
 *      Routine:        wait_queue_unlink_one
 *      Purpose:
 *              Find and unlink one subordinate wait queue
 *      Conditions:
 *              Nothing of interest locked.
 */
void
wait_queue_unlink_one(
        wait_queue_t wq,
        wait_queue_sub_t *wq_subp)
{
        wait_queue_element_t wq_element;
        queue_t q;
        spl_t s;

        s = splsched();
        wait_queue_lock(wq);

        q = &wq->wq_queue;

        wq_element = (wait_queue_element_t) queue_first(q);
        while (!queue_end(q, (queue_entry_t)wq_element)) {

                if (wq_element->wqe_event == WAIT_QUEUE_SUBORDINATE) {
                        wait_queue_link_t wql = (wait_queue_link_t)wq_element;
                        wait_queue_sub_t wq_sub = wql->wql_subqueue;
                        queue_t sq;

                        wqs_lock(wq_sub);
                        sq = &wq_sub->wqs_sublinks;
                        queue_remove(q, wql, wait_queue_link_t, wql_links);
                        queue_remove(sq, wql, wait_queue_link_t, wql_sublinks);
                        wqs_unlock(wq_sub);
                        wait_queue_unlock(wq);
                        splx(s);
                        kfree((vm_offset_t)wql,sizeof(struct wait_queue_link));
                        *wq_subp = wq_sub;
                        return;
                }

                wq_element = (wait_queue_element_t)
                             queue_next((queue_t) wq_element);
        }
        wait_queue_unlock(wq);
        splx(s);
        *wq_subp = WAIT_QUEUE_SUB_NULL;
}       

/*
 *      Routine:        wait_queue_assert_wait_locked
 *      Purpose:
 *              Insert the current thread into the supplied wait queue
 *              waiting for a particular event to be posted to that queue.
 *
 *      Conditions:
 *              The wait queue is assumed locked.
 *
 */
void
wait_queue_assert_wait_locked(
        wait_queue_t wq,
        event_t event,
        int interruptible,
        boolean_t unlock)
{
        thread_t thread = current_thread();

        thread_lock(thread);

        /*
         * This is the extent to which we currently take scheduling attributes
         * into account.  If the thread is vm priviledged, we stick it at
         * the front of the queue.  Later, these queues will honor the policy
         * value set at wait_queue_init time.
         */
        if (thread->vm_privilege)
                enqueue_head(&wq->wq_queue, (queue_entry_t) thread);
        else
                enqueue_tail(&wq->wq_queue, (queue_entry_t) thread);
        thread->wait_event = event;
        thread->wait_queue = wq;
        thread_mark_wait_locked(thread, interruptible);
        thread_unlock(thread);
        if (unlock)
                wait_queue_unlock(wq);
}

/*
 *      Routine:        wait_queue_assert_wait
 *      Purpose:
 *              Insert the current thread into the supplied wait queue
 *              waiting for a particular event to be posted to that queue.
 *
 *      Conditions:
 *              nothing of interest locked.
 */
void
wait_queue_assert_wait(
        wait_queue_t wq,
        event_t event,
        int interruptible)
{
        spl_t s;

        s = splsched();
        wait_queue_lock(wq);
        wait_queue_assert_wait_locked(wq, event, interruptible, TRUE);
        /* wait queue unlocked */
        splx(s);
}


/*
 *      Routine:        wait_queue_select_all
 *      Purpose:
 *              Select all threads off a wait queue that meet the
 *              supplied criteria.
 *
 *      Conditions:
 *              at splsched
 *              wait queue locked
 *              wake_queue initialized and ready for insertion
 *              possibly recursive
 *
 *      Returns:
 *              a queue of locked threads
 */
void
_wait_queue_select_all(
        wait_queue_t wq,
        event_t event,
        queue_t wake_queue)
{
        wait_queue_element_t wq_element;
        wait_queue_element_t wqe_next;
        queue_t q;

        q = &wq->wq_queue;

        wq_element = (wait_queue_element_t) queue_first(q);
        while (!queue_end(q, (queue_entry_t)wq_element)) {
                wqe_next = (wait_queue_element_t)
                           queue_next((queue_t) wq_element);

                /*
                 * We may have to recurse if this is a compound wait queue.
                 */
                if (wq_element->wqe_event == WAIT_QUEUE_SUBORDINATE) {
                        wait_queue_link_t wql = (wait_queue_link_t)wq_element;
                        wait_queue_t sub_queue;

                        /*
                         * We have to check the subordinate wait queue.
                         */
                        sub_queue = (wait_queue_t)wql->wql_subqueue;
                        wait_queue_lock(sub_queue);
                        if (! wait_queue_empty(sub_queue)) 
                                _wait_queue_select_all(sub_queue, event, wake_queue);
                        wait_queue_unlock(sub_queue);
                } else {
                        
                        /*
                         * Otherwise, its a thread.  If it is waiting on
                         * the event we are posting to this queue, pull
                         * it off the queue and stick it in out wake_queue.
                         */
                        thread_t t = (thread_t)wq_element;

                        if (t->wait_event == event) {
                                thread_lock(t);
                                remqueue(q, (queue_entry_t) t);
                                enqueue (wake_queue, (queue_entry_t) t);
                                t->wait_queue = WAIT_QUEUE_NULL;
                                t->wait_event = NO_EVENT;
                                t->at_safe_point = FALSE;
                                /* returned locked */
                        }
                }
                wq_element = wqe_next;
        }
}

/*
 *      Routine:        wait_queue_wakeup_all_locked
 *      Purpose:
 *              Wakeup some number of threads that are in the specified
 *              wait queue and waiting on the specified event.
 *      Conditions:
 *              wait queue already locked (may be released).
 *      Returns:
 *              KERN_SUCCESS - Threads were woken up
 *              KERN_NOT_WAITING - No threads were waiting <wq,event> pair
 */
kern_return_t
wait_queue_wakeup_all_locked(
        wait_queue_t wq,
        event_t event,
        int result,
        boolean_t unlock)
{
        queue_head_t wake_queue_head;
        queue_t q = &wake_queue_head;
        kern_return_t ret = KERN_NOT_WAITING;

        assert(wait_queue_held(wq));

        queue_init(q);

        /*
         * Select the threads that we will wake up.  The threads
         * are returned to us locked and cleanly removed from the
         * wait queue.
         */
        _wait_queue_select_all(wq, event, q);
        if (unlock)
            wait_queue_unlock(wq);

        /*
         * For each thread, set it running.
         */
        while (!queue_empty (q)) {
                thread_t thread = (thread_t) dequeue(q);
                thread_go_locked(thread, result);
                thread_unlock(thread);
                ret = KERN_SUCCESS;
        }
        return ret;
}


/*
 *      Routine:        wait_queue_wakeup_all
 *      Purpose:
 *              Wakeup some number of threads that are in the specified
 *              wait queue and waiting on the specified event.
 *
 *      Conditions:
 *              Nothing locked
 *
 *      Returns:
 *              KERN_SUCCESS - Threads were woken up
 *              KERN_NOT_WAITING - No threads were waiting <wq,event> pair
 */
kern_return_t
wait_queue_wakeup_all(
        wait_queue_t wq,
        event_t event,
        int result)
{
        kern_return_t ret;
        spl_t s;

        s = splsched();
        wait_queue_lock(wq);
        ret = wait_queue_wakeup_all_locked(wq, event, result, TRUE);
        /* lock released */
        splx(s);

        return ret;
}

/*
 *      Routine:        wait_queue_select_one
 *      Purpose:
 *              Select the best thread off a wait queue that meet the
 *              supplied criteria.
 *      Conditions:
 *              at splsched
 *              wait queue locked
 *              possibly recursive
 *      Returns:
 *              a locked thread - if one found
 *      Note:
 *              This is where the sync policy of the wait queue comes
 *              into effect.  For now, we just assume FIFO.
 */
thread_t
_wait_queue_select_one(
        wait_queue_t wq,
        event_t event)
{
        wait_queue_element_t wq_element;
        wait_queue_element_t wqe_next;
        thread_t t = THREAD_NULL;
        queue_t q;

        assert(wq->wq_fifo);

        q = &wq->wq_queue;

        wq_element = (wait_queue_element_t) queue_first(q);
        while (!queue_end(q, (queue_entry_t)wq_element)) {
                wqe_next = (wait_queue_element_t)
                               queue_next((queue_t) wq_element);

                /*
                 * We may have to recurse if this is a compound wait queue.
                 */
                if (wq_element->wqe_event == WAIT_QUEUE_SUBORDINATE) {
                        wait_queue_link_t wql = (wait_queue_link_t)wq_element;
                        wait_queue_t sub_queue;

                        /*
                         * We have to check the subordinate wait queue.
                         */
                        sub_queue = (wait_queue_t)wql->wql_subqueue;
                        wait_queue_lock(sub_queue);
                        if (! wait_queue_empty(sub_queue)) {
                                t = _wait_queue_select_one(sub_queue, event);
                        }
                        wait_queue_unlock(sub_queue);
                        if (t != THREAD_NULL)
                                return t;
                } else {
                        
                        /*
                         * Otherwise, its a thread.  If it is waiting on
                         * the event we are posting to this queue, pull
                         * it off the queue and stick it in out wake_queue.
                         */
                        thread_t t = (thread_t)wq_element;

                        if (t->wait_event == event) {
                                thread_lock(t);
                                remqueue(q, (queue_entry_t) t);
                                t->wait_queue = WAIT_QUEUE_NULL;
                                t->wait_event = NO_EVENT;
                                t->at_safe_point = FALSE;
                                return t;       /* still locked */
                        }
                }
                wq_element = wqe_next;
        }
        return THREAD_NULL;
}

/*
 *      Routine:        wait_queue_peek_locked
 *      Purpose:
 *              Select the best thread from a wait queue that meet the
 *              supplied criteria, but leave it on the queue you it was
 *              found on.  The thread, and the actual wait_queue the
 *              thread was found on are identified.
 *      Conditions:
 *              at splsched
 *              wait queue locked
 *              possibly recursive
 *      Returns:
 *              a locked thread - if one found
 *              a locked waitq - the one the thread was found on
 *      Note:
 *              Only the waitq the thread was actually found on is locked
 *              after this.
 */
void
wait_queue_peek_locked(
        wait_queue_t wq,
        event_t event,
        thread_t *tp,
        wait_queue_t *wqp)
{
        wait_queue_element_t wq_element;
        wait_queue_element_t wqe_next;
        thread_t t;
        queue_t q;

        assert(wq->wq_fifo);

        *tp = THREAD_NULL;

        q = &wq->wq_queue;

        wq_element = (wait_queue_element_t) queue_first(q);
        while (!queue_end(q, (queue_entry_t)wq_element)) {
                wqe_next = (wait_queue_element_t)
                               queue_next((queue_t) wq_element);

                /*
                 * We may have to recurse if this is a compound wait queue.
                 */
                if (wq_element->wqe_event == WAIT_QUEUE_SUBORDINATE) {
                        wait_queue_link_t wql = (wait_queue_link_t)wq_element;
                        wait_queue_t sub_queue;

                        /*
                         * We have to check the subordinate wait queue.
                         */
                        sub_queue = (wait_queue_t)wql->wql_subqueue;
                        wait_queue_lock(sub_queue);
                        if (! wait_queue_empty(sub_queue)) {
                                wait_queue_peek_locked(sub_queue, event, tp, wqp);
                        }
                        if (*tp != THREAD_NULL)
                                return;  /* thread and its waitq locked */

                        wait_queue_unlock(sub_queue);
                } else {
                        
                        /*
                         * Otherwise, its a thread.  If it is waiting on
                         * the event we are posting to this queue, return
                         * it locked, but leave it on the queue.
                         */
                        thread_t t = (thread_t)wq_element;

                        if (t->wait_event == event) {
                                thread_lock(t);
                                *tp = t;
                                *wqp = wq;
                                return;
                        }
                }
                wq_element = wqe_next;
        }
}

/*
 *      Routine:        wait_queue_pull_thread_locked
 *      Purpose:
 *              Pull a thread that was previously "peeked" off the wait
 *              queue and (possibly) unlock the waitq.
 *      Conditions:
 *              at splsched
 *              wait queue locked
 *              thread locked
 *      Returns:
 *              with the thread still locked.
 */
void
wait_queue_pull_thread_locked(
        wait_queue_t waitq,
        thread_t thread,
        boolean_t unlock)
{

        assert(thread->wait_queue == waitq);

        remqueue(&waitq->wq_queue, (queue_entry_t)thread );
        thread->wait_queue = WAIT_QUEUE_NULL;
        thread->wait_event = NO_EVENT;
        thread->at_safe_point = FALSE;
        if (unlock)
                wait_queue_unlock(waitq);
}


/*
 *      Routine:        wait_queue_select_thread
 *      Purpose:
 *              Look for a thread and remove it from the queues, if
 *              (and only if) the thread is waiting on the supplied
 *              <wait_queue, event> pair.
 *      Conditions:
 *              at splsched
 *              wait queue locked
 *              possibly recursive
 *      Returns:
 *              KERN_NOT_WAITING: Thread is not waiting here.
 *              KERN_SUCCESS: It was, and is now removed (returned locked)
 */
kern_return_t
_wait_queue_select_thread(
        wait_queue_t wq,
        event_t event,
        thread_t thread)
{
        wait_queue_element_t wq_element;
        wait_queue_element_t wqe_next;
        kern_return_t res = KERN_NOT_WAITING;
        queue_t q = &wq->wq_queue;

        assert(wq->wq_fifo);

        thread_lock(thread);
        if ((thread->wait_queue == wq) && (thread->wait_event == event)) {
                remqueue(q, (queue_entry_t) thread);
                thread->at_safe_point = FALSE;
                thread->wait_event = NO_EVENT;
                thread->wait_queue = WAIT_QUEUE_NULL;
                /* thread still locked */
                return KERN_SUCCESS;
        }
        thread_unlock(thread);
        
        /*
         * The wait_queue associated with the thread may be one of this
         * wait queue's subordinates.  Go see.  If so, removing it from
         * there is like removing it from here.
         */
        wq_element = (wait_queue_element_t) queue_first(q);
        while (!queue_end(q, (queue_entry_t)wq_element)) {
                wqe_next = (wait_queue_element_t)
                               queue_next((queue_t) wq_element);

                if (wq_element->wqe_event == WAIT_QUEUE_SUBORDINATE) {
                        wait_queue_link_t wql = (wait_queue_link_t)wq_element;
                        wait_queue_t sub_queue;

                        sub_queue = (wait_queue_t)wql->wql_subqueue;
                        wait_queue_lock(sub_queue);
                        if (! wait_queue_empty(sub_queue)) {
                                res = _wait_queue_select_thread(sub_queue,
                                                                event,
                                                                thread);
                        }
                        wait_queue_unlock(sub_queue);
                        if (res == KERN_SUCCESS)
                                return KERN_SUCCESS;
                }
                wq_element = wqe_next;
        }
        return res;
}


/*
 *      Routine:        wait_queue_wakeup_identity_locked
 *      Purpose:
 *              Select a single thread that is most-eligible to run and set
 *              set it running.  But return the thread locked.
 *
 *      Conditions:
 *              at splsched
 *              wait queue locked
 *              possibly recursive
 *      Returns:
 *              a pointer to the locked thread that was awakened
 */
thread_t
wait_queue_wakeup_identity_locked(
        wait_queue_t wq,
        event_t event,
        int result,
        boolean_t unlock)
{
        thread_t thread;

        assert(wait_queue_held(wq));

        thread = _wait_queue_select_one(wq, event);
        if (unlock)
                wait_queue_unlock(wq);

        if (thread)
                thread_go_locked(thread, result);
        return thread;  /* still locked if not NULL */
}


/*
 *      Routine:        wait_queue_wakeup_one_locked
 *      Purpose:
 *              Select a single thread that is most-eligible to run and set
 *              set it runnings.
 *
 *      Conditions:
 *              at splsched
 *              wait queue locked
 *              possibly recursive
 *      Returns:
 *              KERN_SUCCESS: It was, and is, now removed.
 *              KERN_NOT_WAITING - No thread was waiting <wq,event> pair
 */
kern_return_t
wait_queue_wakeup_one_locked(
        wait_queue_t wq,
        event_t event,
        int result,
        boolean_t unlock)
{
        thread_t thread;

        assert(wait_queue_held(wq));

        thread = _wait_queue_select_one(wq, event);
        if (unlock)
                wait_queue_unlock(wq);

        if (thread) {
                thread_go_locked(thread, result);
                thread_unlock(thread);
                return KERN_SUCCESS;
        }

        return KERN_NOT_WAITING;
}

/*
 *      Routine:        wait_queue_wakeup_one
 *      Purpose:
 *              Wakeup the most appropriate thread that is in the specified
 *              wait queue for the specified event.
 *
 *      Conditions:
 *              Nothing locked
 *
 *      Returns:
 *              KERN_SUCCESS - Thread was woken up
 *              KERN_NOT_WAITING - No thread was waiting <wq,event> pair
 */
kern_return_t
wait_queue_wakeup_one(
        wait_queue_t wq,
        event_t event,
        int result)
{
        thread_t thread;
        spl_t s;

        s = splsched();
        wait_queue_lock(wq);
        thread = _wait_queue_select_one(wq, event);
        wait_queue_unlock(wq);

        if (thread) {
                thread_go_locked(thread, result);
                thread_unlock(thread);
                splx(s);
                return KERN_SUCCESS;
        }

        splx(s);
        return KERN_NOT_WAITING;
}



/*
 *      Routine:        wait_queue_wakeup_thread_locked
 *      Purpose:
 *              Wakeup the particular thread that was specified if and only
 *              it was in this wait queue (or one of it's subordinate queues)
 *              and waiting on the specified event.
 *
 *              This is much safer than just removing the thread from
 *              whatever wait queue it happens to be on.  For instance, it
 *              may have already been awoken from the wait you intended to
 *              interrupt and waited on something else (like another
 *              semaphore).
 *      Conditions:
 *              at splsched
 *              wait queue already locked (may be released).
 *      Returns:
 *              KERN_SUCCESS - the thread was found waiting and awakened
 *              KERN_NOT_WAITING - the thread was not waiting here
 */
kern_return_t
wait_queue_wakeup_thread_locked(
        wait_queue_t wq,
        event_t event,
        thread_t thread,
        int result,
        boolean_t unlock)
{
        kern_return_t res;

        assert(wait_queue_held(wq));

        /*
         * See if the thread was still waiting there.  If so, it got
         * dequeued and returned locked.
         */
        res = _wait_queue_select_thread(wq, event, thread);
        if (unlock)
            wait_queue_unlock(wq);

        if (res != KERN_SUCCESS)
                return KERN_NOT_WAITING;

        thread_go_locked(thread, result);
        thread_unlock(thread);
        return KERN_SUCCESS;
}

/*
 *      Routine:        wait_queue_wakeup_thread
 *      Purpose:
 *              Wakeup the particular thread that was specified if and only
 *              it was in this wait queue (or one of it's subordinate queues)
 *              and waiting on the specified event.
 *
 *              This is much safer than just removing the thread from
 *              whatever wait queue it happens to be on.  For instance, it
 *              may have already been awoken from the wait you intended to
 *              interrupt and waited on something else (like another
 *              semaphore).
 *      Conditions:
 *              nothing of interest locked
 *              we need to assume spl needs to be raised
 *      Returns:
 *              KERN_SUCCESS - the thread was found waiting and awakened
 *              KERN_NOT_WAITING - the thread was not waiting here
 */
kern_return_t
wait_queue_wakeup_thread(
        wait_queue_t wq,
        event_t event,
        thread_t thread,
        int result)
{
        kern_return_t res;
        spl_t s;

        s = splsched();
        wait_queue_lock(wq);
        res = _wait_queue_select_thread(wq, event, thread);
        wait_queue_unlock(wq);

        if (res == KERN_SUCCESS) {
                thread_go_locked(thread, result);
                thread_unlock(thread);
                splx(s);
                return KERN_SUCCESS;
        }
        splx(s);
        return KERN_NOT_WAITING;
}


/*
 *      Routine:        wait_queue_remove
 *      Purpose:
 *              Normal removal operations from wait queues drive from the
 *              wait queue to select a thread.  However, if a thread is
 *              interrupted out of a wait, this routine is called to
 *              remove it from whatever wait queue it may be in.
 *
 *      Conditions:
 *              splsched
 *              thread locked on entry and exit, but may be dropped.
 *
 *      Returns:
 *              KERN_SUCCESS - if thread was in a wait queue
 *              KERN_NOT_WAITING - it was not
 */
kern_return_t
wait_queue_remove(
        thread_t thread)
{
        wait_queue_t wq = thread->wait_queue;

        if (wq == WAIT_QUEUE_NULL)
                return KERN_NOT_WAITING;

        /*
         * have to get the locks again in the right order.
         */
        thread_unlock(thread);
        wait_queue_lock(wq);
        thread_lock(thread);
        
        if (thread->wait_queue == wq) {
                remqueue(&wq->wq_queue, (queue_entry_t)thread);
                thread->wait_queue = WAIT_QUEUE_NULL;
                thread->wait_event = NO_EVENT;
                thread->at_safe_point = FALSE;
                wait_queue_unlock(wq);
                return KERN_SUCCESS;
        } else {
                wait_queue_unlock(wq);
                return KERN_NOT_WAITING; /* anymore */
        }
}