xnu-1504.15.3.tar.gz

[apple/xnu.git] / osfmk / kern / thread.c

diff --git a/osfmk/kern/thread.c b/osfmk/kern/thread.c
index 85ebb7dcf091ed366d5f39a50fc7c336ddf9234d..adff14820ad5a5d164e54a4cb459c9c50b58f541 100644 (file)
--- a/osfmk/kern/thread.c
+++ b/osfmk/kern/thread.c
@@ -1,23 +1,29 @@
 /*
 /*

- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2009 Apple Inc. All rights reserved.

  *
  *

- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@

  * 
  * 

- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.

  * 
  * 

- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER

  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,

- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.

  * 
  * 

- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@

  */
 /*
  * @OSF_FREE_COPYRIGHT@
  */
 /*
  * @OSF_FREE_COPYRIGHT@


@@ -54,7 +60,7 @@
  *     Author: Avadis Tevanian, Jr., Michael Wayne Young, David Golub
  *     Date:   1986
  *
  *     Author: Avadis Tevanian, Jr., Michael Wayne Young, David Golub
  *     Date:   1986
  *

- *     Thread/thread_shuttle management primitives implementation.
+ *     Thread management primitives implementation.

  */
 /*
  * Copyright (c) 1993 The University of Utah and
  */
 /*
  * Copyright (c) 1993 The University of Utah and


@@ -75,12 +81,7 @@
  *
  */
 
  *
  */
 

-#include <cpus.h>
-#include <mach_host.h>
-#include <simple_clock.h>
-#include <mach_debug.h>
-#include <mach_prof.h>
-
+#include <mach/mach_types.h>

 #include <mach/boolean.h>
 #include <mach/policy.h>
 #include <mach/thread_info.h>
 #include <mach/boolean.h>
 #include <mach/policy.h>
 #include <mach/thread_info.h>


@@ -88,10 +89,13 @@
 #include <mach/thread_status.h>
 #include <mach/time_value.h>
 #include <mach/vm_param.h>
 #include <mach/thread_status.h>
 #include <mach/time_value.h>
 #include <mach/vm_param.h>

-#include <kern/ast.h>
+
+#include <machine/thread.h>
+
+#include <kern/kern_types.h>
+#include <kern/kalloc.h>

 #include <kern/cpu_data.h>
 #include <kern/counters.h>
 #include <kern/cpu_data.h>
 #include <kern/counters.h>

-#include <kern/etap_macros.h>

 #include <kern/ipc_mig.h>
 #include <kern/ipc_tt.h>
 #include <kern/mach_param.h>
 #include <kern/ipc_mig.h>
 #include <kern/ipc_tt.h>
 #include <kern/mach_param.h>


@@ -101,577 +105,481 @@
 #include <kern/queue.h>
 #include <kern/sched.h>
 #include <kern/sched_prim.h>
 #include <kern/queue.h>
 #include <kern/sched.h>
 #include <kern/sched_prim.h>

-#include <kern/mk_sp.h>        /*** ??? fix so this can be removed ***/
+#include <kern/sync_lock.h>
+#include <kern/syscall_subr.h>

 #include <kern/task.h>
 #include <kern/thread.h>
 #include <kern/task.h>
 #include <kern/thread.h>

-#include <kern/thread_act.h>
-#include <kern/thread_swap.h>

 #include <kern/host.h>
 #include <kern/zalloc.h>
 #include <kern/host.h>
 #include <kern/zalloc.h>

-#include <vm/vm_kern.h>
+#include <kern/assert.h>
+

 #include <ipc/ipc_kmsg.h>
 #include <ipc/ipc_port.h>
 #include <ipc/ipc_kmsg.h>
 #include <ipc/ipc_port.h>

-#include <machine/thread.h>            /* for MACHINE_STACK */
-#include <kern/profile.h>
-#include <kern/assert.h>
+
+#include <vm/vm_kern.h>
+#include <vm/vm_pageout.h>
+

 #include <sys/kdebug.h>
 
 #include <sys/kdebug.h>
 

+#include <mach/sdt.h>
+

 /*
  * Exported interfaces
  */
 /*
  * Exported interfaces
  */

-
+#include <mach/task_server.h>

 #include <mach/thread_act_server.h>
 #include <mach/mach_host_server.h>
 #include <mach/thread_act_server.h>
 #include <mach/mach_host_server.h>

+#include <mach/host_priv_server.h>

 
 static struct zone                     *thread_zone;
 
 static struct zone                     *thread_zone;

+static lck_grp_attr_t          thread_lck_grp_attr;
+lck_attr_t                                     thread_lck_attr;
+lck_grp_t                                      thread_lck_grp;

 
 

-static queue_head_t                    reaper_queue;
-decl_simple_lock_data(static,reaper_lock)
+decl_simple_lock_data(static,thread_stack_lock)
+static queue_head_t            thread_stack_queue;

 
 

-extern int             tick;
+decl_simple_lock_data(static,thread_terminate_lock)
+static queue_head_t            thread_terminate_queue;

 
 

-/* private */

 static struct thread   thread_template, init_thread;
 
 static struct thread   thread_template, init_thread;
 

-#if    MACH_DEBUG
+static void            sched_call_null(
+                                       int                     type,
+                                       thread_t        thread);

 
 

-#ifdef MACHINE_STACK
-extern void    stack_statistics(
-                       unsigned int    *totalp,
-                       vm_size_t       *maxusagep);
-#endif /* MACHINE_STACK */
-#endif /* MACH_DEBUG */
+#ifdef MACH_BSD
+extern void proc_exit(void *);
+extern uint64_t get_dispatchqueue_offset_from_proc(void *);
+#endif /* MACH_BSD */

 
 

-#ifdef MACHINE_STACK
-/*
- *     Machine-dependent code must define:
- *             stack_alloc_try
- *             stack_alloc
- *             stack_free
- *             stack_free_stack
- *             stack_collect
- *     and if MACH_DEBUG:
- *             stack_statistics
- */
-#else  /* MACHINE_STACK */
-/*
- *     We allocate stacks from generic kernel VM.
- *     Machine-dependent code must define:
- *             machine_kernel_stack_init
- *
- *     The stack_free_list can only be accessed at splsched,
- *     because stack_alloc_try/thread_invoke operate at splsched.
- */
+extern int debug_task;
+int thread_max = CONFIG_THREAD_MAX;    /* Max number of threads */
+int task_threadmax = CONFIG_THREAD_MAX;

 
 

-decl_simple_lock_data(static,stack_lock_data)
-#define stack_lock()           simple_lock(&stack_lock_data)
-#define stack_unlock()         simple_unlock(&stack_lock_data)
+static uint64_t                thread_unique_id = 0;

 
 

-static vm_map_t                                stack_map;
-static vm_offset_t                     stack_free_list;
+void
+thread_bootstrap(void)
+{
+       /*
+        *      Fill in a template thread for fast initialization.
+        */

 
 

-static vm_offset_t                     stack_free_cache[NCPUS];
+       thread_template.runq = PROCESSOR_NULL;

 
 

-unsigned int stack_free_max = 0;
-unsigned int stack_free_count = 0;             /* splsched only */
-unsigned int stack_free_limit = 1;             /* Arbitrary  */
+       thread_template.ref_count = 2;

 
 

-unsigned int stack_cache_hits = 0;             /* debugging */
+       thread_template.reason = AST_NONE;
+       thread_template.at_safe_point = FALSE;
+       thread_template.wait_event = NO_EVENT64;
+       thread_template.wait_queue = WAIT_QUEUE_NULL;
+       thread_template.wait_result = THREAD_WAITING;
+       thread_template.options = THREAD_ABORTSAFE;
+       thread_template.state = TH_WAIT | TH_UNINT;
+       thread_template.wake_active = FALSE;
+       thread_template.continuation = THREAD_CONTINUE_NULL;
+       thread_template.parameter = NULL;

 
 

-unsigned int stack_alloc_hits = 0;             /* debugging */
-unsigned int stack_alloc_misses = 0;   /* debugging */
+       thread_template.importance = 0;
+       thread_template.sched_mode = 0;
+       thread_template.safe_mode = 0;
+       thread_template.safe_release = 0;

 
 

-unsigned int stack_alloc_total = 0;
-unsigned int stack_alloc_hiwater = 0;
-unsigned int stack_alloc_bndry = 0;
+       thread_template.priority = 0;
+       thread_template.sched_pri = 0;
+       thread_template.max_priority = 0;
+       thread_template.task_priority = 0;
+       thread_template.promotions = 0;
+       thread_template.pending_promoter_index = 0;
+       thread_template.pending_promoter[0] =
+               thread_template.pending_promoter[1] = NULL;

 
 

+       thread_template.realtime.deadline = UINT64_MAX;

 
 

-/*
- *     The next field is at the base of the stack,
- *     so the low end is left unsullied.
- */
+       thread_template.current_quantum = 0;

 
 

-#define stack_next(stack) (*((vm_offset_t *)((stack) + KERNEL_STACK_SIZE) - 1))
+       thread_template.computation_metered = 0;
+       thread_template.computation_epoch = 0;

 
 

-/*
- *     stack_alloc:
- *
- *     Allocate a kernel stack for a thread.
- *     May block.
- */
-vm_offset_t
-stack_alloc(
-       thread_t thread,
-       void (*start_pos)(thread_t))
-{
-       vm_offset_t     stack = thread->kernel_stack;
-       spl_t                   s;
+       thread_template.sched_stamp = 0;
+       thread_template.sched_usage = 0;
+       thread_template.pri_shift = INT8_MAX;
+       thread_template.cpu_usage = thread_template.cpu_delta = 0;
+       thread_template.c_switch = thread_template.p_switch = thread_template.ps_switch = 0;

 
 

-       if (stack)
-               return (stack);
+       thread_template.bound_processor = PROCESSOR_NULL;
+       thread_template.last_processor = PROCESSOR_NULL;

 
 

-       s = splsched();
-       stack_lock();
-       stack = stack_free_list;
-       if (stack != 0) {
-               stack_free_list = stack_next(stack);
-               stack_free_count--;
-       }
-       stack_unlock();
-       splx(s);
+       thread_template.sched_call = sched_call_null;

 
 

-       if (stack != 0) {
-               machine_stack_attach(thread, stack, start_pos);
-               return (stack);
-       }
-               
-       if (kernel_memory_allocate(
-                                       stack_map, &stack,
-                                               KERNEL_STACK_SIZE, stack_alloc_bndry - 1,
-                                                                               KMA_KOBJECT) != KERN_SUCCESS)
-               panic("stack_alloc: no space left for stack maps");
-
-       stack_alloc_total++;
-       if (stack_alloc_total > stack_alloc_hiwater)
-               stack_alloc_hiwater = stack_alloc_total;
-
-       machine_stack_attach(thread, stack, start_pos);
-       return (stack);
-}
+       timer_init(&thread_template.user_timer);
+       timer_init(&thread_template.system_timer);
+       thread_template.user_timer_save = 0;
+       thread_template.system_timer_save = 0;
+       thread_template.vtimer_user_save = 0;
+       thread_template.vtimer_prof_save = 0;
+       thread_template.vtimer_rlim_save = 0;

 
 

-/*
- *     stack_free:
- *
- *     Free a kernel stack.
- */
+       thread_template.wait_timer_is_set = FALSE;
+       thread_template.wait_timer_active = 0;

 
 

-void
-stack_free(
-       thread_t thread)
-{
-    vm_offset_t stack = machine_stack_detach(thread);
+       thread_template.depress_timer_active = 0;

 
 

-       assert(stack);
-       if (stack != thread->reserved_stack) {
-               spl_t                   s = splsched();
-               vm_offset_t             *cache;
+       thread_template.special_handler.handler = special_handler;
+       thread_template.special_handler.next = NULL;

 
 

-               cache = &stack_free_cache[cpu_number()];
-               if (*cache == 0) {
-                       *cache = stack;
-                       splx(s);
+       thread_template.funnel_lock = THR_FUNNEL_NULL;
+       thread_template.funnel_state = 0;
+       thread_template.recover = (vm_offset_t)NULL;
+       
+       thread_template.map = VM_MAP_NULL;

 
 

-                       return;
-               }
+#if CONFIG_DTRACE
+       thread_template.t_dtrace_predcache = 0;
+       thread_template.t_dtrace_vtime = 0;
+       thread_template.t_dtrace_tracing = 0;
+#endif /* CONFIG_DTRACE */

 
 

-               stack_lock();
-               stack_next(stack) = stack_free_list;
-               stack_free_list = stack;
-               if (++stack_free_count > stack_free_max)
-                       stack_free_max = stack_free_count;
-               stack_unlock();
-               splx(s);
-       }
+       thread_template.t_chud = 0;
+       thread_template.t_page_creation_count = 0;
+       thread_template.t_page_creation_time = 0;
+
+       thread_template.affinity_set = NULL;
+       
+       init_thread = thread_template;
+       machine_set_current_thread(&init_thread);

 }
 
 void
 }
 
 void

-stack_free_stack(
-       vm_offset_t             stack)
+thread_init(void)

 {
 {

-       spl_t                   s = splsched();
-       vm_offset_t             *cache;
-
-       cache = &stack_free_cache[cpu_number()];
-       if (*cache == 0) {
-               *cache = stack;
-               splx(s);
+       thread_zone = zinit(
+                       sizeof(struct thread),
+                       thread_max * sizeof(struct thread),
+                       THREAD_CHUNK * sizeof(struct thread),
+                       "threads");
+       zone_change(thread_zone, Z_NOENCRYPT, TRUE);
+       
+       lck_grp_attr_setdefault(&thread_lck_grp_attr);
+       lck_grp_init(&thread_lck_grp, "thread", &thread_lck_grp_attr);
+       lck_attr_setdefault(&thread_lck_attr);
+       
+       stack_init();

 
 

-               return;
-       }
+       /*
+        *      Initialize any machine-dependent
+        *      per-thread structures necessary.
+        */
+       machine_thread_init();
+}

 
 

-       stack_lock();
-       stack_next(stack) = stack_free_list;
-       stack_free_list = stack;
-       if (++stack_free_count > stack_free_max)
-               stack_free_max = stack_free_count;
-       stack_unlock();
-       splx(s);
+static void
+thread_terminate_continue(void)
+{
+       panic("thread_terminate_continue");
+       /*NOTREACHED*/

 }
 
 /*
 }
 
 /*

- *     stack_collect:
- *
- *     Free excess kernel stacks.
- *     May block.
+ *     thread_terminate_self:

  */
  */

-

 void
 void

-stack_collect(void)
+thread_terminate_self(void)

 {
 {

-       spl_t   s = splsched();
+       thread_t                thread = current_thread();
+       task_t                  task;
+       spl_t                   s;
+       int threadcnt;

 
 

-       stack_lock();
-       while (stack_free_count > stack_free_limit) {
-               vm_offset_t             stack = stack_free_list;
+       DTRACE_PROC(lwp__exit);

 
 

-               stack_free_list = stack_next(stack);
-               stack_free_count--;
-               stack_unlock();
-               splx(s);
+       thread_mtx_lock(thread);

 
 

-               if (vm_map_remove(
-                                       stack_map, stack, stack + KERNEL_STACK_SIZE,
-                                                                       VM_MAP_REMOVE_KUNWIRE) != KERN_SUCCESS)
-                       panic("stack_collect: vm_map_remove failed");
+       ulock_release_all(thread);

 
 

-               s = splsched();
-               stack_lock();
-               stack_alloc_total--;
-       }
-       stack_unlock();
-       splx(s);
-}
-
-/*
- *     stack_alloc_try:
- *
- *     Non-blocking attempt to allocate a kernel stack.
- *     Called at splsched with the thread locked.
- */
+       ipc_thread_disable(thread);
+       
+       thread_mtx_unlock(thread);

 
 

-boolean_t stack_alloc_try(
-       thread_t        thread,
-       void            (*start)(thread_t))
-{
-       register vm_offset_t    stack, *cache;
+       s = splsched();
+       thread_lock(thread);

 
 

-       cache = &stack_free_cache[cpu_number()];
-       if (stack = *cache) {
-               *cache = 0;
-               machine_stack_attach(thread, stack, start);
-               stack_cache_hits++;
+       /*
+        *      Cancel priority depression, wait for concurrent expirations
+        *      on other processors.
+        */
+       if (thread->sched_mode & TH_MODE_ISDEPRESSED) {
+               thread->sched_mode &= ~TH_MODE_ISDEPRESSED;

 
 

-               return (TRUE);
+               if (timer_call_cancel(&thread->depress_timer))
+                       thread->depress_timer_active--;

        }
 
        }
 

-       stack_lock();
-       stack = stack_free_list;
-       if (stack != (vm_offset_t)0) {
-               stack_free_list = stack_next(stack);
-               stack_free_count--;
+       while (thread->depress_timer_active > 0) {
+               thread_unlock(thread);
+               splx(s);
+
+               delay(1);
+
+               s = splsched();
+               thread_lock(thread);

        }
        }

-       stack_unlock();

 
 

-       if (stack == 0)
-               stack = thread->reserved_stack;
+       thread_sched_call(thread, NULL);

 
 

-       if (stack != 0) {
-               machine_stack_attach(thread, stack, start);
-               stack_alloc_hits++;
+       thread_unlock(thread);
+       splx(s);

 
 

-               return (TRUE);
-       }
-       else {
-               stack_alloc_misses++;
+       thread_policy_reset(thread);

 
 

-               return (FALSE);
-       }
-}
+       task = thread->task;
+       uthread_cleanup(task, thread->uthread, task->bsd_info);
+       threadcnt = hw_atomic_sub(&task->active_thread_count, 1);

 
 

-#if    MACH_DEBUG
-/*
- *     stack_statistics:
- *
- *     Return statistics on cached kernel stacks.
- *     *maxusagep must be initialized by the caller.
- */
+       /*
+        * If we are the last thread to terminate and the task is
+        * associated with a BSD process, perform BSD process exit.
+        */
+       if (threadcnt == 0 && task->bsd_info != NULL)
+               proc_exit(task->bsd_info);

 
 

-void
-stack_statistics(
-       unsigned int    *totalp,
-       vm_size_t       *maxusagep)
-{
-       spl_t   s;
+       uthread_cred_free(thread->uthread);

 
        s = splsched();
 
        s = splsched();

-       stack_lock();
+       thread_lock(thread);

 
 

-       *totalp = stack_free_count;
-       *maxusagep = 0;
+       /*
+        *      Cancel wait timer, and wait for
+        *      concurrent expirations.
+        */
+       if (thread->wait_timer_is_set) {
+               thread->wait_timer_is_set = FALSE;

 
 

-       stack_unlock();
-       splx(s);
-}
-#endif /* MACH_DEBUG */
+               if (timer_call_cancel(&thread->wait_timer))
+                       thread->wait_timer_active--;
+       }

 
 

-#endif /* MACHINE_STACK */
+       while (thread->wait_timer_active > 0) {
+               thread_unlock(thread);
+               splx(s);

 
 

+               delay(1);

 
 

-stack_fake_zone_info(int *count, vm_size_t *cur_size, vm_size_t *max_size, vm_size_t *elem_size,
-                    vm_size_t *alloc_size, int *collectable, int *exhaustable)
-{
-        *count      = stack_alloc_total - stack_free_count;
-       *cur_size   = KERNEL_STACK_SIZE * stack_alloc_total;
-       *max_size   = KERNEL_STACK_SIZE * stack_alloc_hiwater;
-       *elem_size  = KERNEL_STACK_SIZE;
-       *alloc_size = KERNEL_STACK_SIZE;
-       *collectable = 1;
-       *exhaustable = 0;
-}
+               s = splsched();
+               thread_lock(thread);
+       }

 
 

-void
-stack_privilege(
-       register thread_t       thread)
-{
-       /* OBSOLETE */
-}
+       /*
+        *      If there is a reserved stack, release it.
+        */
+       if (thread->reserved_stack != 0) {
+               if (thread->reserved_stack != thread->kernel_stack)
+                       stack_free_stack(thread->reserved_stack);
+               thread->reserved_stack = 0;
+       }

 
 

-void
-thread_bootstrap(void)
-{

        /*
        /*

-        *      Fill in a template thread for fast initialization.
+        *      Mark thread as terminating, and block.

         */
         */

+       thread->state |= TH_TERMINATE;
+       thread_mark_wait_locked(thread, THREAD_UNINT);
+       assert(thread->promotions == 0);
+       thread_unlock(thread);
+       /* splsched */

 
 

-       thread_template.runq = RUN_QUEUE_NULL;
+       thread_block((thread_continue_t)thread_terminate_continue);
+       /*NOTREACHED*/
+}

 
 

-       thread_template.ref_count = 1;
+void
+thread_deallocate(
+       thread_t                        thread)
+{
+       task_t                          task;

 
 

-       thread_template.reason = AST_NONE;
-       thread_template.at_safe_point = FALSE;
-       thread_template.wait_event = NO_EVENT64;
-       thread_template.wait_queue = WAIT_QUEUE_NULL;
-       thread_template.wait_result = THREAD_WAITING;
-       thread_template.interrupt_level = THREAD_ABORTSAFE;
-       thread_template.state = TH_STACK_HANDOFF | TH_WAIT | TH_UNINT;
-       thread_template.wake_active = FALSE;
-       thread_template.active_callout = FALSE;
-       thread_template.continuation = (void (*)(void))0;
-       thread_template.top_act = THR_ACT_NULL;
+       if (thread == THREAD_NULL)
+               return;

 
 

-       thread_template.importance = 0;
-       thread_template.sched_mode = 0;
-       thread_template.safe_mode = 0;
+       if (thread_deallocate_internal(thread) > 0)
+               return;

 
 

-       thread_template.priority = 0;
-       thread_template.sched_pri = 0;
-       thread_template.max_priority = 0;
-       thread_template.task_priority = 0;
-       thread_template.promotions = 0;
-       thread_template.pending_promoter_index = 0;
-       thread_template.pending_promoter[0] =
-               thread_template.pending_promoter[1] = NULL;
+       ipc_thread_terminate(thread);

 
 

-       thread_template.realtime.deadline = UINT64_MAX;
+       task = thread->task;

 
 

-       thread_template.current_quantum = 0;
+#ifdef MACH_BSD 
+       {
+               void *ut = thread->uthread;

 
 

-       thread_template.computation_metered = 0;
-       thread_template.computation_epoch = 0;
+               thread->uthread = NULL;
+               uthread_zone_free(ut);
+       }
+#endif  /* MACH_BSD */   

 
 

-       thread_template.cpu_usage = 0;
-       thread_template.cpu_delta = 0;
-       thread_template.sched_usage = 0;
-       thread_template.sched_delta = 0;
-       thread_template.sched_stamp = 0;
-       thread_template.sleep_stamp = 0;
-       thread_template.safe_release = 0;
+       task_deallocate(task);

 
 

-       thread_template.bound_processor = PROCESSOR_NULL;
-       thread_template.last_processor = PROCESSOR_NULL;
-       thread_template.last_switch = 0;
+       if (thread->kernel_stack != 0)
+               stack_free(thread);

 
 

-       thread_template.vm_privilege = FALSE;
+       lck_mtx_destroy(&thread->mutex, &thread_lck_grp);
+       machine_thread_destroy(thread);

 
 

-       timer_init(&(thread_template.user_timer));
-       timer_init(&(thread_template.system_timer));
-       thread_template.user_timer_save.low = 0;
-       thread_template.user_timer_save.high = 0;
-       thread_template.system_timer_save.low = 0;
-       thread_template.system_timer_save.high = 0;
+       zfree(thread_zone, thread);
+}

 
 

-       thread_template.processor_set = PROCESSOR_SET_NULL;
+/*
+ *     thread_terminate_daemon:
+ *
+ *     Perform final clean up for terminating threads.
+ */
+static void
+thread_terminate_daemon(void)
+{
+       thread_t                        thread;
+       task_t                          task;

 
 

-       thread_template.act_ref_count = 2;
+       (void)splsched();
+       simple_lock(&thread_terminate_lock);

 
 

-       thread_template.special_handler.handler = special_handler;
-       thread_template.special_handler.next = 0;
+       while ((thread = (thread_t)dequeue_head(&thread_terminate_queue)) != THREAD_NULL) {
+               simple_unlock(&thread_terminate_lock);
+               (void)spllo();

 
 

-#if    MACH_HOST
-       thread_template.may_assign = TRUE;
-       thread_template.assign_active = FALSE;
-#endif /* MACH_HOST */
-    thread_template.funnel_lock = THR_FUNNEL_NULL;
-       thread_template.funnel_state = 0;
-#if    MACH_LDEBUG
-       thread_template.mutex_count = 0;
-#endif /* MACH_LDEBUG */
+               task = thread->task;

 
 

-       init_thread = thread_template;
+               task_lock(task);
+               task->total_user_time += timer_grab(&thread->user_timer);
+               task->total_system_time += timer_grab(&thread->system_timer);

 
 

-       init_thread.top_act = &init_thread;
-       init_thread.thread = &init_thread;
-       machine_thread_set_current(&init_thread);
-}
+               task->c_switch += thread->c_switch;
+               task->p_switch += thread->p_switch;
+               task->ps_switch += thread->ps_switch;

 
 

-void
-thread_init(void)
-{
-       kern_return_t ret;
-       unsigned int stack;
-       
-       thread_zone = zinit(
-                       sizeof(struct thread),
-                       THREAD_MAX * sizeof(struct thread),
-                       THREAD_CHUNK * sizeof(struct thread),
-                       "threads");
+               queue_remove(&task->threads, thread, thread_t, task_threads);
+               task->thread_count--;

 
 

-       /*
-        *      Initialize other data structures used in
-        *      this module.
-        */
+               /* 
+                * If the task is being halted, and there is only one thread
+                * left in the task after this one, then wakeup that thread.
+                */
+               if (task->thread_count == 1 && task->halting)
+                       thread_wakeup((event_t)&task->halting);

 
 

-       queue_init(&reaper_queue);
-       simple_lock_init(&reaper_lock, ETAP_THREAD_REAPER);
+               task_unlock(task);

 
 

-#ifndef MACHINE_STACK
-       simple_lock_init(&stack_lock_data, ETAP_THREAD_STACK);  /* Initialize the stack lock */
-       
-       if (KERNEL_STACK_SIZE < round_page_32(KERNEL_STACK_SIZE)) {     /* Kernel stacks must be multiples of pages */
-               panic("thread_init: kernel stack size (%08X) must be a multiple of page size (%08X)\n", 
-                       KERNEL_STACK_SIZE, PAGE_SIZE);
-       }
-       
-       for(stack_alloc_bndry = PAGE_SIZE; stack_alloc_bndry <= KERNEL_STACK_SIZE; stack_alloc_bndry <<= 1);    /* Find next power of 2 above stack size */
+               lck_mtx_lock(&tasks_threads_lock);
+               queue_remove(&threads, thread, thread_t, threads);
+               threads_count--;
+               lck_mtx_unlock(&tasks_threads_lock);

 
 

-       ret = kmem_suballoc(kernel_map,                 /* Suballocate from the kernel map */
+               thread_deallocate(thread);

 
 

-               &stack,
-               (stack_alloc_bndry * (2*THREAD_MAX + 64)),      /* Allocate enough for all of it */
-               FALSE,                                                          /* Say not pageable so that it is wired */
-               TRUE,                                                           /* Allocate from anywhere */
-               &stack_map);                                            /* Allocate a submap */
-               
-       if(ret != KERN_SUCCESS) {                               /* Did we get one? */
-               panic("thread_init: kmem_suballoc for stacks failed - ret = %d\n", ret);        /* Die */
-       }       
-       stack = vm_map_min(stack_map);                  /* Make sure we skip the first hunk */
-       ret = vm_map_enter(stack_map, &stack, PAGE_SIZE, 0,     /* Make sure there is nothing at the start */
-               0,                                                                      /* Force it at start */
-               VM_OBJECT_NULL, 0,                                      /* No object yet */
-               FALSE,                                                          /* No copy */
-               VM_PROT_NONE,                                           /* Allow no access */
-               VM_PROT_NONE,                                           /* Allow no access */
-               VM_INHERIT_DEFAULT);                            /* Just be normal */
-               
-       if(ret != KERN_SUCCESS) {                                       /* Did it work? */
-               panic("thread_init: dummy alignment allocation failed; ret = %d\n", ret);
+               (void)splsched();
+               simple_lock(&thread_terminate_lock);

        }
        }

-               
-#endif  /* MACHINE_STACK */

 
 

-       /*
-        *      Initialize any machine-dependent
-        *      per-thread structures necessary.
-        */
-       machine_thread_init();
+       assert_wait((event_t)&thread_terminate_queue, THREAD_UNINT);
+       simple_unlock(&thread_terminate_lock);
+       /* splsched */
+
+       thread_block((thread_continue_t)thread_terminate_daemon);
+       /*NOTREACHED*/

 }
 
 /*
 }
 
 /*

- * Called at splsched.
+ *     thread_terminate_enqueue:
+ *
+ *     Enqueue a terminating thread for final disposition.
+ *
+ *     Called at splsched.

  */
 void
  */
 void

-thread_reaper_enqueue(
+thread_terminate_enqueue(

        thread_t                thread)
 {
        thread_t                thread)
 {

-       simple_lock(&reaper_lock);
-       enqueue_tail(&reaper_queue, (queue_entry_t)thread);
-       simple_unlock(&reaper_lock);
+       simple_lock(&thread_terminate_lock);
+       enqueue_tail(&thread_terminate_queue, (queue_entry_t)thread);
+       simple_unlock(&thread_terminate_lock);

 
 

-       thread_wakeup((event_t)&reaper_queue);
+       thread_wakeup((event_t)&thread_terminate_queue);

 }
 
 }
 

-void
-thread_termination_continue(void)
+/*
+ *     thread_stack_daemon:
+ *
+ *     Perform stack allocation as required due to
+ *     invoke failures.
+ */
+static void
+thread_stack_daemon(void)

 {
 {

-       panic("thread_termination_continue");
+       thread_t                thread;
+
+       simple_lock(&thread_stack_lock);
+
+       while ((thread = (thread_t)dequeue_head(&thread_stack_queue)) != THREAD_NULL) {
+               simple_unlock(&thread_stack_lock);
+
+               stack_alloc(thread);
+               
+               (void)splsched();
+               thread_lock(thread);
+               thread_setrun(thread, SCHED_PREEMPT | SCHED_TAILQ);
+               thread_unlock(thread);
+               (void)spllo();
+
+               simple_lock(&thread_stack_lock);
+       }
+
+       assert_wait((event_t)&thread_stack_queue, THREAD_UNINT);
+       simple_unlock(&thread_stack_lock);
+
+       thread_block((thread_continue_t)thread_stack_daemon);

        /*NOTREACHED*/
 }
 
 /*
        /*NOTREACHED*/
 }
 
 /*

- *     Routine: thread_terminate_self
+ *     thread_stack_enqueue:

  *
  *

- *             This routine is called by a thread which has unwound from
- *             its current RPC and kernel contexts and found that it's
- *             root activation has been marked for extinction.  This lets
- *             it clean up the last few things that can only be cleaned
- *             up in this context and then impale itself on the reaper
- *             queue.
+ *     Enqueue a thread for stack allocation.

  *
  *

- *             When the reaper gets the thread, it will deallocate the
- *             thread_act's reference on itself, which in turn will release
- *             its own reference on this thread. By doing things in that
- *             order, a thread_act will always have a valid thread - but the
- *             thread may persist beyond having a thread_act (but must never
- *             run like that).
+ *     Called at splsched.

  */
 void
  */
 void

-thread_terminate_self(void)
+thread_stack_enqueue(
+       thread_t                thread)

 {
 {

-       thread_act_t    thr_act = current_act();
-       thread_t                thread;
-       task_t                  task = thr_act->task;
-       long                    active_acts;
-       spl_t                   s;
+       simple_lock(&thread_stack_lock);
+       enqueue_tail(&thread_stack_queue, (queue_entry_t)thread);
+       simple_unlock(&thread_stack_lock);

 
 

-       /*
-        * We should be at the base of the inheritance chain.
-        */
-       thread = act_lock_thread(thr_act);
-       assert(thr_act->thread == thread);
-
-       /* This will allow no more control ops on this thr_act. */
-       ipc_thr_act_disable(thr_act);
-
-       /* Clean-up any ulocks that are still owned by the thread
-        * activation (acquired but not released or handed-off).
-        */
-       act_ulock_release_all(thr_act);
-       
-       act_unlock_thread(thr_act);
+       thread_wakeup((event_t)&thread_stack_queue);
+}

 
 

-       _mk_sp_thread_depress_abort(thread, TRUE);
+void
+thread_daemon_init(void)
+{
+       kern_return_t   result;
+       thread_t                thread;

 
 

-       /*
-        * Check to see if this is the last active activation.  By
-        * this we mean the last activation to call thread_terminate_self.
-        * If so, and the task is associated with a BSD process, we
-        * need to call BSD and let them clean up.
-        */
-       active_acts = hw_atomic_sub(&task->active_thread_count, 1);
+       simple_lock_init(&thread_terminate_lock, 0);
+       queue_init(&thread_terminate_queue);

 
 

-       if (active_acts == 0 && task->bsd_info)
-               proc_exit(task->bsd_info);
+       result = kernel_thread_start_priority((thread_continue_t)thread_terminate_daemon, NULL, MINPRI_KERNEL, &thread);
+       if (result != KERN_SUCCESS)
+               panic("thread_daemon_init: thread_terminate_daemon");

 
 

-       /* JMM - for now, no migration */
-       assert(!thr_act->lower);
+       thread_deallocate(thread);

 
 

-       thread_timer_terminate();
+       simple_lock_init(&thread_stack_lock, 0);
+       queue_init(&thread_stack_queue);

 
 

-       ipc_thread_terminate(thread);
-
-       s = splsched();
-       thread_lock(thread);
-       thread->state |= TH_TERMINATE;
-       assert((thread->state & TH_UNINT) == 0);
-       thread_mark_wait_locked(thread, THREAD_UNINT);
-       assert(thread->promotions == 0);
-       thread_unlock(thread);
-       /* splx(s); */
+       result = kernel_thread_start_priority((thread_continue_t)thread_stack_daemon, NULL, BASEPRI_PREEMPT, &thread);
+       if (result != KERN_SUCCESS)
+               panic("thread_daemon_init: thread_stack_daemon");

 
 

-       ETAP_SET_REASON(thread, BLOCKED_ON_TERMINATION);
-       thread_block(thread_termination_continue);
-       /*NOTREACHED*/
+       thread_deallocate(thread);

 }
 
 /*
 }
 
 /*


@@ -682,49 +590,49 @@ static kern_return_t
 thread_create_internal(
        task_t                                  parent_task,
        integer_t                               priority,
 thread_create_internal(
        task_t                                  parent_task,
        integer_t                               priority,

-       void                                    (*start)(void),
+       thread_continue_t               continuation,
+       int                                             options,
+#define TH_OPTION_NONE         0x00
+#define TH_OPTION_NOCRED       0x01
+#define TH_OPTION_NOSUSP       0x02

        thread_t                                *out_thread)
 {
        thread_t                                new_thread;
        thread_t                                *out_thread)
 {
        thread_t                                new_thread;

-       processor_set_t                 pset;

        static thread_t                 first_thread;
 
        /*
         *      Allocate a thread and initialize static fields
         */
        static thread_t                 first_thread;
 
        /*
         *      Allocate a thread and initialize static fields
         */

-       if (first_thread == NULL)
-               new_thread = first_thread = current_act();
+       if (first_thread == THREAD_NULL)
+               new_thread = first_thread = current_thread();

        else
                new_thread = (thread_t)zalloc(thread_zone);
        else
                new_thread = (thread_t)zalloc(thread_zone);

-       if (new_thread == NULL)
+       if (new_thread == THREAD_NULL)

                return (KERN_RESOURCE_SHORTAGE);
 
        if (new_thread != first_thread)
                *new_thread = thread_template;
 
 #ifdef MACH_BSD
                return (KERN_RESOURCE_SHORTAGE);
 
        if (new_thread != first_thread)
                *new_thread = thread_template;
 
 #ifdef MACH_BSD

-    {
-               extern void     *uthread_alloc(task_t, thread_act_t);
-
-               new_thread->uthread = uthread_alloc(parent_task, new_thread);
-               if (new_thread->uthread == NULL) {
-                       zfree(thread_zone, (vm_offset_t)new_thread);
-                       return (KERN_RESOURCE_SHORTAGE);
-               }
+       new_thread->uthread = uthread_alloc(parent_task, new_thread, (options & TH_OPTION_NOCRED) != 0);
+       if (new_thread->uthread == NULL) {
+               zfree(thread_zone, new_thread);
+               return (KERN_RESOURCE_SHORTAGE);

        }
 #endif  /* MACH_BSD */
 
        if (machine_thread_create(new_thread, parent_task) != KERN_SUCCESS) {
 #ifdef MACH_BSD
        }
 #endif  /* MACH_BSD */
 
        if (machine_thread_create(new_thread, parent_task) != KERN_SUCCESS) {
 #ifdef MACH_BSD

-               {
-                       extern void uthread_free(task_t, void *, void *, void *);
-                       void *ut = new_thread->uthread;
+               void *ut = new_thread->uthread;

 
 

-                       new_thread->uthread = NULL;
-                       uthread_free(parent_task, (void *)new_thread, ut, parent_task->bsd_info);
-               }
+               new_thread->uthread = NULL;
+               /* cred free may not be necessary */
+               uthread_cleanup(parent_task, ut, parent_task->bsd_info);
+               uthread_cred_free(ut);
+               uthread_zone_free(ut);

 #endif  /* MACH_BSD */
 #endif  /* MACH_BSD */

-               zfree(thread_zone, (vm_offset_t)new_thread);
+
+               zfree(thread_zone, new_thread);

                return (KERN_FAILURE);
        }
 
                return (KERN_FAILURE);
        }
 


@@ -733,65 +641,75 @@ thread_create_internal(
        thread_lock_init(new_thread);
        wake_lock_init(new_thread);
 
        thread_lock_init(new_thread);
        wake_lock_init(new_thread);
 

-       mutex_init(&new_thread->lock, ETAP_THREAD_ACT);
-
-       ipc_thr_act_init(parent_task, new_thread);
+       lck_mtx_init(&new_thread->mutex, &thread_lck_grp, &thread_lck_attr);

 
        ipc_thread_init(new_thread);
        queue_init(&new_thread->held_ulocks);
 
        ipc_thread_init(new_thread);
        queue_init(&new_thread->held_ulocks);

-       act_prof_init(new_thread, parent_task);
-
-       new_thread->continuation = start;
-       new_thread->sleep_stamp = sched_tick;

 
 

-       pset = parent_task->processor_set;
-       assert(pset == &default_pset);
-       pset_lock(pset);
+       new_thread->continuation = continuation;

 
 

+       lck_mtx_lock(&tasks_threads_lock);

        task_lock(parent_task);
        task_lock(parent_task);

-       assert(parent_task->processor_set == pset);

 
 

-       if (    !parent_task->active                                                    ||
-                       (parent_task->thread_count >= THREAD_MAX        &&
-                        parent_task != kernel_task)) {
+       if (    !parent_task->active || parent_task->halting ||
+                       ((options & TH_OPTION_NOSUSP) != 0 &&
+                               parent_task->suspend_count > 0) ||
+                       (parent_task->thread_count >= task_threadmax &&
+                               parent_task != kernel_task)             ) {

                task_unlock(parent_task);
                task_unlock(parent_task);

-               pset_unlock(pset);
+               lck_mtx_unlock(&tasks_threads_lock);

 
 #ifdef MACH_BSD
                {
 
 #ifdef MACH_BSD
                {

-                       extern void uthread_free(task_t, void *, void *, void *);

                        void *ut = new_thread->uthread;
 
                        new_thread->uthread = NULL;
                        void *ut = new_thread->uthread;
 
                        new_thread->uthread = NULL;

-                       uthread_free(parent_task, (void *)new_thread, ut, parent_task->bsd_info);
+                       uthread_cleanup(parent_task, ut, parent_task->bsd_info);
+                       /* cred free may not be necessary */
+                       uthread_cred_free(ut);
+                       uthread_zone_free(ut);

                }
 #endif  /* MACH_BSD */
                }
 #endif  /* MACH_BSD */

-               act_prof_deallocate(new_thread);
-               ipc_thr_act_terminate(new_thread);
+               ipc_thread_disable(new_thread);
+               ipc_thread_terminate(new_thread);
+               lck_mtx_destroy(&new_thread->mutex, &thread_lck_grp);

                machine_thread_destroy(new_thread);
                machine_thread_destroy(new_thread);

-               zfree(thread_zone, (vm_offset_t) new_thread);
+               zfree(thread_zone, new_thread);

                return (KERN_FAILURE);
        }
 
                return (KERN_FAILURE);
        }
 

-       act_attach(new_thread, new_thread);
+       /* New threads inherit any default state on the task */
+       machine_thread_inherit_taskwide(new_thread, parent_task);

 
 

-       task_reference_locked(parent_task);
+       task_reference_internal(parent_task);

 
        /* Cache the task's map */
        new_thread->map = parent_task->map;
 
        /* Chain the thread onto the task's list */
 
        /* Cache the task's map */
        new_thread->map = parent_task->map;
 
        /* Chain the thread onto the task's list */

-       queue_enter(&parent_task->threads, new_thread, thread_act_t, task_threads);
+       queue_enter(&parent_task->threads, new_thread, thread_t, task_threads);

        parent_task->thread_count++;
        parent_task->thread_count++;

-       parent_task->res_thread_count++;

        
        /* So terminating threads don't need to take the task lock to decrement */
        hw_atomic_add(&parent_task->active_thread_count, 1);
 
        
        /* So terminating threads don't need to take the task lock to decrement */
        hw_atomic_add(&parent_task->active_thread_count, 1);
 

-       /* Associate the thread with the processor set */
-       pset_add_thread(pset, new_thread);
+       /* Protected by the tasks_threads_lock */
+       new_thread->thread_id = ++thread_unique_id;

 
 

-       thread_timer_setup(new_thread);
+       queue_enter(&threads, new_thread, thread_t, threads);
+       threads_count++;
+
+       timer_call_setup(&new_thread->wait_timer, thread_timer_expire, new_thread);
+       timer_call_setup(&new_thread->depress_timer, thread_depress_expire, new_thread);
+
+#if CONFIG_COUNTERS
+       /*
+        * If parent task has any reservations, they need to be propagated to this
+        * thread.
+        */
+       new_thread->t_chud = (TASK_PMC_FLAG == (parent_task->t_chud & TASK_PMC_FLAG)) ? 
+               THREAD_PMC_FLAG : 0U;
+#endif

 
        /* Set the thread's scheduling parameters */
        if (parent_task != kernel_task)
 
        /* Set the thread's scheduling parameters */
        if (parent_task != kernel_task)


@@ -804,13 +722,9 @@ thread_create_internal(
        new_thread->importance =
                                        new_thread->priority - new_thread->task_priority;
        new_thread->sched_stamp = sched_tick;
        new_thread->importance =
                                        new_thread->priority - new_thread->task_priority;
        new_thread->sched_stamp = sched_tick;

+       new_thread->pri_shift = sched_pri_shift;

        compute_priority(new_thread, FALSE);
 
        compute_priority(new_thread, FALSE);
 

-#if    ETAP_EVENT_MONITOR
-       new_thread->etap_reason = 0;
-       new_thread->etap_trace  = FALSE;
-#endif /* ETAP_EVENT_MONITOR */
-

        new_thread->active = TRUE;
 
        *out_thread = new_thread;
        new_thread->active = TRUE;
 
        *out_thread = new_thread;


@@ -822,7 +736,7 @@ thread_create_internal(
 
                KERNEL_DEBUG_CONSTANT(
                                        TRACEDBG_CODE(DBG_TRACE_DATA, 1) | DBG_FUNC_NONE,
 
                KERNEL_DEBUG_CONSTANT(
                                        TRACEDBG_CODE(DBG_TRACE_DATA, 1) | DBG_FUNC_NONE,

-                                                       (vm_address_t)new_thread, dbg_arg2, 0, 0, 0);
+                                                       (vm_address_t)(uintptr_t)thread_tid(new_thread), dbg_arg2, 0, 0, 0);

 
                kdbg_trace_string(parent_task->bsd_info,
                                                        &dbg_arg1, &dbg_arg2, &dbg_arg3, &dbg_arg4);
 
                kdbg_trace_string(parent_task->bsd_info,
                                                        &dbg_arg1, &dbg_arg2, &dbg_arg3, &dbg_arg4);


@@ -832,15 +746,15 @@ thread_create_internal(
                                                        dbg_arg1, dbg_arg2, dbg_arg3, dbg_arg4, 0);
        }
 
                                                        dbg_arg1, dbg_arg2, dbg_arg3, dbg_arg4, 0);
        }
 

+       DTRACE_PROC1(lwp__create, thread_t, *out_thread);
+

        return (KERN_SUCCESS);
 }
 
        return (KERN_SUCCESS);
 }
 

-extern void                    thread_bootstrap_return(void);
-

 kern_return_t
 thread_create(
        task_t                          task,
 kern_return_t
 thread_create(
        task_t                          task,

-       thread_act_t            *new_thread)
+       thread_t                        *new_thread)

 {
        kern_return_t           result;
        thread_t                        thread;
 {
        kern_return_t           result;
        thread_t                        thread;


@@ -848,7 +762,7 @@ thread_create(
        if (task == TASK_NULL || task == kernel_task)
                return (KERN_INVALID_ARGUMENT);
 
        if (task == TASK_NULL || task == kernel_task)
                return (KERN_INVALID_ARGUMENT);
 

-       result = thread_create_internal(task, -1, thread_bootstrap_return, &thread);
+       result = thread_create_internal(task, -1, (thread_continue_t)thread_bootstrap_return, TH_OPTION_NONE, &thread);

        if (result != KERN_SUCCESS)
                return (result);
 
        if (result != KERN_SUCCESS)
                return (result);
 


@@ -857,8 +771,8 @@ thread_create(
        if (task->suspend_count > 0)
                thread_hold(thread);
 
        if (task->suspend_count > 0)
                thread_hold(thread);
 

-       pset_unlock(task->processor_set);

        task_unlock(task);
        task_unlock(task);

+       lck_mtx_unlock(&tasks_threads_lock);

        
        *new_thread = thread;
 
        
        *new_thread = thread;
 


@@ -871,7 +785,7 @@ thread_create_running(
        int                     flavor,
        thread_state_t          new_state,
        mach_msg_type_number_t  new_state_count,
        int                     flavor,
        thread_state_t          new_state,
        mach_msg_type_number_t  new_state_count,

-       thread_act_t                    *new_thread)
+       thread_t                                *new_thread)

 {
        register kern_return_t  result;
        thread_t                                thread;
 {
        register kern_return_t  result;
        thread_t                                thread;


@@ -879,201 +793,165 @@ thread_create_running(
        if (task == TASK_NULL || task == kernel_task)
                return (KERN_INVALID_ARGUMENT);
 
        if (task == TASK_NULL || task == kernel_task)
                return (KERN_INVALID_ARGUMENT);
 

-       result = thread_create_internal(task, -1, thread_bootstrap_return, &thread);
+       result = thread_create_internal(task, -1, (thread_continue_t)thread_bootstrap_return, TH_OPTION_NONE, &thread);

        if (result != KERN_SUCCESS)
                return (result);
 
        if (result != KERN_SUCCESS)
                return (result);
 

-       result = machine_thread_set_state(thread, flavor, new_state, new_state_count);
+       result = machine_thread_set_state(
+                                               thread, flavor, new_state, new_state_count);

        if (result != KERN_SUCCESS) {
        if (result != KERN_SUCCESS) {

-               pset_unlock(task->processor_set);

                task_unlock(task);
                task_unlock(task);

+               lck_mtx_unlock(&tasks_threads_lock);

 
                thread_terminate(thread);
 
                thread_terminate(thread);

-               act_deallocate(thread);
+               thread_deallocate(thread);

                return (result);
        }
 
                return (result);
        }
 

-       act_lock(thread);
-       clear_wait(thread, THREAD_AWAKENED);
-       thread->started = TRUE;
-       act_unlock(thread);
-       pset_unlock(task->processor_set);
+       thread_mtx_lock(thread);
+       thread_start_internal(thread);
+       thread_mtx_unlock(thread);
+

        task_unlock(task);
        task_unlock(task);

+       lck_mtx_unlock(&tasks_threads_lock);

 
        *new_thread = thread;
 
        return (result);
 }
 
 
        *new_thread = thread;
 
        return (result);
 }
 

+kern_return_t
+thread_create_workq(
+       task_t                          task,
+       thread_continue_t               thread_return,
+       thread_t                        *new_thread)
+{
+       kern_return_t           result;
+       thread_t                        thread;
+
+       if (task == TASK_NULL || task == kernel_task)
+               return (KERN_INVALID_ARGUMENT);
+
+       result = thread_create_internal(task, -1, thread_return, TH_OPTION_NOCRED | TH_OPTION_NOSUSP, &thread);
+       if (result != KERN_SUCCESS)
+               return (result);
+
+       thread->user_stop_count = 1;
+       thread_hold(thread);
+       if (task->suspend_count > 0)
+               thread_hold(thread);
+
+       task_unlock(task);
+       lck_mtx_unlock(&tasks_threads_lock);
+       
+       *new_thread = thread;
+
+       return (KERN_SUCCESS);
+}
+

 /*
 /*

- *     kernel_thread:
+ *     kernel_thread_create:

  *
  *     Create a thread in the kernel task
  *     to execute in kernel context.
  */
  *
  *     Create a thread in the kernel task
  *     to execute in kernel context.
  */

-thread_t
+kern_return_t

 kernel_thread_create(
 kernel_thread_create(

-       void                            (*start)(void),
-       integer_t                       priority)
+       thread_continue_t       continuation,
+       void                            *parameter,
+       integer_t                       priority,
+       thread_t                        *new_thread)

 {
        kern_return_t           result;
 {
        kern_return_t           result;

-       task_t                          task = kernel_task;

        thread_t                        thread;
        thread_t                        thread;

+       task_t                          task = kernel_task;

 
 

-       result = thread_create_internal(task, priority, start, &thread);
+       result = thread_create_internal(task, priority, continuation, TH_OPTION_NONE, &thread);

        if (result != KERN_SUCCESS)
        if (result != KERN_SUCCESS)

-               return (THREAD_NULL);
+               return (result);

 
 

-       pset_unlock(task->processor_set);

        task_unlock(task);
        task_unlock(task);

+       lck_mtx_unlock(&tasks_threads_lock);

 
 

-       thread_doswapin(thread);
+       stack_alloc(thread);

        assert(thread->kernel_stack != 0);
        assert(thread->kernel_stack != 0);

+#if CONFIG_EMBEDDED
+       if (priority > BASEPRI_KERNEL)
+#endif

        thread->reserved_stack = thread->kernel_stack;
 
        thread->reserved_stack = thread->kernel_stack;
 

-       act_deallocate(thread);
-
-       return (thread);
-}
+       thread->parameter = parameter;

 
 

-thread_t
-kernel_thread_with_priority(
-       void                    (*start)(void),
-       integer_t               priority)
-{
-       thread_t                thread;
-
-       thread = kernel_thread_create(start, priority);
-       if (thread == THREAD_NULL)
-               return (THREAD_NULL);
-
-       act_lock(thread);
-       clear_wait(thread, THREAD_AWAKENED);
-       thread->started = TRUE;
-       act_unlock(thread);
-
-#ifdef i386
-       thread_bind(thread, master_processor);
-#endif /* i386 */
-       return (thread);
-}
-
-thread_t
-kernel_thread(
-       task_t                  task,
-       void                    (*start)(void))
-{
-       if (task != kernel_task)
-               panic("kernel_thread");
+if(debug_task & 1)
+       kprintf("kernel_thread_create: thread = %p continuation = %p\n", thread, continuation);
+       *new_thread = thread;

 
 

-       return kernel_thread_with_priority(start, -1);
+       return (result);

 }
 
 }
 

-unsigned int c_weird_pset_ref_exit = 0;        /* pset code raced us */
-
-#if    MACH_HOST
-/* Preclude thread processor set assignement */
-#define thread_freeze(thread)  assert((thread)->processor_set == &default_pset)
-
-/* Allow thread processor set assignement */
-#define thread_unfreeze(thread)        assert((thread)->processor_set == &default_pset)
-
-#endif /* MACH_HOST */
-
-void
-thread_deallocate(
-       thread_t                        thread)
+kern_return_t
+kernel_thread_start_priority(
+       thread_continue_t       continuation,
+       void                            *parameter,
+       integer_t                       priority,
+       thread_t                        *new_thread)

 {
 {

-       task_t                          task;
-       processor_set_t         pset;
-       int                                     refs;
-       spl_t                           s;
-
-       if (thread == THREAD_NULL)
-               return;
-
-       /*
-        *      First, check for new count > 0 (the common case).
-        *      Only the thread needs to be locked.
-        */
-       s = splsched();
-       thread_lock(thread);
-       refs = --thread->ref_count;
-       thread_unlock(thread);
-       splx(s);
-
-       if (refs > 0)
-               return;
-
-       if (thread == current_thread())
-           panic("thread_deallocate");
-
-       /*
-        *      There is a dangling pointer to the thread from the
-        *      processor_set.  To clean it up, we freeze the thread
-        *      in the pset (because pset destruction can cause even
-        *      reference-less threads to be reassigned to the default
-        *      pset) and then remove it.
-        */
-
-#if MACH_HOST
-       thread_freeze(thread);
-#endif
-
-       pset = thread->processor_set;
-       pset_lock(pset);
-       pset_remove_thread(pset, thread);
-       pset_unlock(pset);
-
-#if MACH_HOST
-       thread_unfreeze(thread);
-#endif
+       kern_return_t   result;
+       thread_t                thread;

 
 

-       pset_deallocate(pset);
+       result = kernel_thread_create(continuation, parameter, priority, &thread);
+       if (result != KERN_SUCCESS)
+               return (result);

 
 

-       if (thread->reserved_stack != 0) {
-               if (thread->reserved_stack != thread->kernel_stack)
-                       stack_free_stack(thread->reserved_stack);
-               thread->reserved_stack = 0;
-       }
+       *new_thread = thread;   

 
 

-       if (thread->kernel_stack != 0)
-               stack_free(thread);
+       thread_mtx_lock(thread);
+       thread_start_internal(thread);
+       thread_mtx_unlock(thread);

 
 

-       machine_thread_destroy(thread);
+       return (result);
+}

 
 

-       zfree(thread_zone, (vm_offset_t) thread);
+kern_return_t
+kernel_thread_start(
+       thread_continue_t       continuation,
+       void                            *parameter,
+       thread_t                        *new_thread)
+{
+       return kernel_thread_start_priority(continuation, parameter, -1, new_thread);

 }
 
 }
 

-void
-thread_reference(
-       thread_t        thread)
+#ifndef        __LP64__
+
+thread_t
+kernel_thread(
+       task_t                  task,
+       void                    (*start)(void))

 {
 {

-       spl_t           s;
+       kern_return_t   result;
+       thread_t                thread;

 
 

-       if (thread == THREAD_NULL)
-               return;
+       if (task != kernel_task)
+               panic("kernel_thread");

 
 

-       s = splsched();
-       thread_lock(thread);
-       thread_reference_locked(thread);
-       thread_unlock(thread);
-       splx(s);
+       result = kernel_thread_start_priority((thread_continue_t)start, NULL, -1, &thread);
+       if (result != KERN_SUCCESS)
+               return (THREAD_NULL);
+
+       thread_deallocate(thread);
+
+       return (thread);

 }
 
 }
 

-/*
- * Called with "appropriate" thread-related locks held on
- * thread and its top_act for synchrony with RPC (see
- * act_lock_thread()).
- */
+#endif /* __LP64__ */
+

 kern_return_t
 kern_return_t

-thread_info_shuttle(
-       register thread_act_t   thr_act,
+thread_info_internal(
+       register thread_t               thread,

        thread_flavor_t                 flavor,
        thread_info_t                   thread_info_out,        /* ptr to OUT array */
        mach_msg_type_number_t  *thread_info_count)     /*IN/OUT*/
 {
        thread_flavor_t                 flavor,
        thread_info_t                   thread_info_out,        /* ptr to OUT array */
        mach_msg_type_number_t  *thread_info_count)     /*IN/OUT*/
 {

-       register thread_t               thread = thr_act->thread;

        int                                             state, flags;
        spl_t                                   s;
 
        int                                             state, flags;
        spl_t                                   s;
 


@@ -1109,24 +987,21 @@ thread_info_shuttle(
                 *      then for 5/8 ageing.  The correction factor [3/5] is
                 *      (1/(5/8) - 1).
                 */
                 *      then for 5/8 ageing.  The correction factor [3/5] is
                 *      (1/(5/8) - 1).
                 */

-               basic_info->cpu_usage = (thread->cpu_usage << SCHED_TICK_SHIFT) /
-                                                                                               (TIMER_RATE / TH_USAGE_SCALE);
+               basic_info->cpu_usage = (integer_t)(((uint64_t)thread->cpu_usage
+                                                                       * TH_USAGE_SCALE) /     sched_tick_interval);

                basic_info->cpu_usage = (basic_info->cpu_usage * 3) / 5;
                basic_info->cpu_usage = (basic_info->cpu_usage * 3) / 5;

-#if    SIMPLE_CLOCK
-               /*
-                *      Clock drift compensation.
-                */
-               basic_info->cpu_usage = (basic_info->cpu_usage * 1000000) / sched_usec;
-#endif /* SIMPLE_CLOCK */
+
+               if (basic_info->cpu_usage > TH_USAGE_SCALE)
+                       basic_info->cpu_usage = TH_USAGE_SCALE;

 
                basic_info->policy = ((thread->sched_mode & TH_MODE_TIMESHARE)?
                                                                                                POLICY_TIMESHARE: POLICY_RR);
 
            flags = 0;
 
                basic_info->policy = ((thread->sched_mode & TH_MODE_TIMESHARE)?
                                                                                                POLICY_TIMESHARE: POLICY_RR);
 
            flags = 0;

-               if (thread->state & TH_IDLE)
+               if (thread->bound_processor != PROCESSOR_NULL && thread->bound_processor->idle_thread == thread)

                        flags |= TH_FLAGS_IDLE;
 
                        flags |= TH_FLAGS_IDLE;
 

-           if (thread->state & TH_STACK_HANDOFF)
+           if (!thread->kernel_stack)

                        flags |= TH_FLAGS_SWAPPED;
 
            state = 0;
                        flags |= TH_FLAGS_SWAPPED;
 
            state = 0;


@@ -1148,7 +1023,7 @@ thread_info_shuttle(
            basic_info->run_state = state;
            basic_info->flags = flags;
 
            basic_info->run_state = state;
            basic_info->flags = flags;
 

-           basic_info->suspend_count = thr_act->user_stop_count;
+           basic_info->suspend_count = thread->user_stop_count;

 
            thread_unlock(thread);
            splx(s);
 
            thread_unlock(thread);
            splx(s);


@@ -1158,6 +1033,36 @@ thread_info_shuttle(
            return (KERN_SUCCESS);
        }
        else
            return (KERN_SUCCESS);
        }
        else

+       if (flavor == THREAD_IDENTIFIER_INFO) {
+           register thread_identifier_info_t   identifier_info;
+
+           if (*thread_info_count < THREAD_IDENTIFIER_INFO_COUNT)
+                       return (KERN_INVALID_ARGUMENT);
+
+           identifier_info = (thread_identifier_info_t) thread_info_out;
+
+           s = splsched();
+           thread_lock(thread);
+
+           identifier_info->thread_id = thread->thread_id;
+#if defined(__ppc__) || defined(__arm__)
+           identifier_info->thread_handle = thread->machine.cthread_self;
+#else
+           identifier_info->thread_handle = thread->machine.pcb->cthread_self;
+#endif
+           if(thread->task->bsd_info) {
+               identifier_info->dispatch_qaddr =  identifier_info->thread_handle + get_dispatchqueue_offset_from_proc(thread->task->bsd_info);
+           } else {
+                   thread_unlock(thread);
+                   splx(s);
+                   return KERN_INVALID_ARGUMENT;
+           }
+
+           thread_unlock(thread);
+           splx(s);
+           return KERN_SUCCESS;
+       }
+       else

        if (flavor == THREAD_SCHED_TIMESHARE_INFO) {
                policy_timeshare_info_t         ts_info;
 
        if (flavor == THREAD_SCHED_TIMESHARE_INFO) {
                policy_timeshare_info_t         ts_info;
 


@@ -1247,83 +1152,29 @@ thread_info_shuttle(
 }
 
 void
 }
 
 void

-thread_doreap(
-       register thread_t       thread)
-{
-       thread_act_t            thr_act;
-
-
-       thr_act = thread_lock_act(thread);
-       assert(thr_act && thr_act->thread == thread);
-
-       act_reference_locked(thr_act);
-
-       /*
-        * Replace `act_unlock_thread()' with individual
-        * calls.  (`act_detach()' can change fields used
-        * to determine which locks are held, confusing
-        * `act_unlock_thread()'.)
-        */
-       act_unlock(thr_act);
-
-       /* Remove the reference held by a rooted thread */
-       act_deallocate(thr_act);
-
-       /* Remove the reference held by the thread: */
-       act_deallocate(thr_act);
-}
-
-/*
- *     reaper_thread:
- *
- *     This kernel thread runs forever looking for terminating
- *     threads, releasing their "self" references.
- */
-static void
-reaper_thread_continue(void)
+thread_read_times(
+       thread_t                thread,
+       time_value_t    *user_time,
+       time_value_t    *system_time)

 {
 {

-       register thread_t       thread;
-
-       (void)splsched();
-       simple_lock(&reaper_lock);
-
-       while ((thread = (thread_t) dequeue_head(&reaper_queue)) != THREAD_NULL) {
-               simple_unlock(&reaper_lock);
-               (void)spllo();
-
-               thread_doreap(thread);
-
-               (void)splsched();
-               simple_lock(&reaper_lock);
-       }
-
-       assert_wait((event_t)&reaper_queue, THREAD_UNINT);
-       simple_unlock(&reaper_lock);
-       (void)spllo();
-
-       thread_block(reaper_thread_continue);
-       /*NOTREACHED*/
-}
+       clock_sec_t             secs;
+       clock_usec_t    usecs;

 
 

-static void
-reaper_thread(void)
-{
-       reaper_thread_continue();
-       /*NOTREACHED*/
-}
+       absolutetime_to_microtime(timer_grab(&thread->user_timer), &secs, &usecs);
+       user_time->seconds = (typeof(user_time->seconds))secs;
+       user_time->microseconds = usecs;

 
 

-void
-thread_reaper_init(void)
-{
-       kernel_thread_with_priority(reaper_thread, MINPRI_KERNEL);
+       absolutetime_to_microtime(timer_grab(&thread->system_timer), &secs, &usecs);
+       system_time->seconds = (typeof(system_time->seconds))secs;
+       system_time->microseconds = usecs;

 }
 
 kern_return_t
 thread_assign(
 }
 
 kern_return_t
 thread_assign(

-       thread_act_t    thr_act,
-       processor_set_t new_pset)
+       __unused thread_t                       thread,
+       __unused processor_set_t        new_pset)

 {
 {

-       return(KERN_FAILURE);
+       return (KERN_FAILURE);

 }
 
 /*
 }
 
 /*


@@ -1334,9 +1185,9 @@ thread_assign(
  */
 kern_return_t
 thread_assign_default(
  */
 kern_return_t
 thread_assign_default(

-       thread_act_t    thr_act)
+       thread_t                thread)

 {
 {

-       return (thread_assign(thr_act, &default_pset));
+       return (thread_assign(thread, &pset0));

 }
 
 /*
 }
 
 /*


@@ -1346,22 +1197,15 @@ thread_assign_default(
  */        
 kern_return_t
 thread_get_assignment(
  */        
 kern_return_t
 thread_get_assignment(

-       thread_act_t    thr_act,
+       thread_t                thread,

        processor_set_t *pset)
 {
        processor_set_t *pset)
 {

-       thread_t        thread;
-
-       if (thr_act == THR_ACT_NULL)
-               return(KERN_INVALID_ARGUMENT);
-       thread = act_lock_thread(thr_act);
-       if (thread == THREAD_NULL) {
-               act_unlock_thread(thr_act);
-               return(KERN_INVALID_ARGUMENT);
-       }
-       *pset = thread->processor_set;
-       act_unlock_thread(thr_act);
-       pset_reference(*pset);
-       return(KERN_SUCCESS);
+       if (thread == NULL)
+               return (KERN_INVALID_ARGUMENT);
+
+       *pset = &pset0;
+
+       return (KERN_SUCCESS);

 }
 
 /*
 }
 
 /*


@@ -1372,54 +1216,31 @@ thread_get_assignment(
  */
 kern_return_t
 thread_wire_internal(
  */
 kern_return_t
 thread_wire_internal(

-       host_priv_t     host_priv,
-       thread_act_t    thr_act,
-       boolean_t       wired,
-       boolean_t       *prev_state)
+       host_priv_t             host_priv,
+       thread_t                thread,
+       boolean_t               wired,
+       boolean_t               *prev_state)

 {
 {

-       spl_t           s;
-       thread_t        thread;
-       extern void vm_page_free_reserve(int pages);
-
-       if (thr_act == THR_ACT_NULL || host_priv == HOST_PRIV_NULL)
+       if (host_priv == NULL || thread != current_thread())

                return (KERN_INVALID_ARGUMENT);
 
        assert(host_priv == &realhost);
 
                return (KERN_INVALID_ARGUMENT);
 
        assert(host_priv == &realhost);
 

-       thread = act_lock_thread(thr_act);
-       if (thread ==THREAD_NULL) {
-               act_unlock_thread(thr_act);
-               return(KERN_INVALID_ARGUMENT);
-       }
-
-       /*
-        * This implementation only works for the current thread.
-        */
-       if (thr_act != current_act())
-           return KERN_INVALID_ARGUMENT;
-
-       s = splsched();
-       thread_lock(thread);
-
-       if (prev_state) {
-           *prev_state = thread->vm_privilege;
-       }
+       if (prev_state)
+           *prev_state = (thread->options & TH_OPT_VMPRIV) != 0;

        
        if (wired) {
        
        if (wired) {

-           if (thread->vm_privilege == FALSE) 
+           if (!(thread->options & TH_OPT_VMPRIV)) 

                    vm_page_free_reserve(1);    /* XXX */
                    vm_page_free_reserve(1);    /* XXX */

-           thread->vm_privilege = TRUE;
-       } else {
-           if (thread->vm_privilege == TRUE) 
+           thread->options |= TH_OPT_VMPRIV;
+       }
+       else {
+           if (thread->options & TH_OPT_VMPRIV) 

                    vm_page_free_reserve(-1);   /* XXX */
                    vm_page_free_reserve(-1);   /* XXX */

-           thread->vm_privilege = FALSE;
+           thread->options &= ~TH_OPT_VMPRIV;

        }
 
        }
 

-       thread_unlock(thread);
-       splx(s);
-       act_unlock_thread(thr_act);
-
-       return KERN_SUCCESS;
+       return (KERN_SUCCESS);

 }
 
 
 }
 
 


@@ -1431,166 +1252,35 @@ thread_wire_internal(
 kern_return_t
 thread_wire(
        host_priv_t     host_priv,
 kern_return_t
 thread_wire(
        host_priv_t     host_priv,

-       thread_act_t    thr_act,
+       thread_t        thread,

        boolean_t       wired)
        boolean_t       wired)

-

 {
 {

-    return thread_wire_internal(host_priv, thr_act, wired, NULL);
+    return (thread_wire_internal(host_priv, thread, wired, NULL));

 }
 
 }
 

-kern_return_t
-host_stack_usage(
-       host_t          host,
-       vm_size_t       *reservedp,
-       unsigned int    *totalp,
-       vm_size_t       *spacep,
-       vm_size_t       *residentp,
-       vm_size_t       *maxusagep,
-       vm_offset_t     *maxstackp)
-{
-#if !MACH_DEBUG
-        return KERN_NOT_SUPPORTED;
-#else
-       unsigned int total;
-       vm_size_t maxusage;
-
-       if (host == HOST_NULL)
-               return KERN_INVALID_HOST;
-
-       maxusage = 0;
-
-       stack_statistics(&total, &maxusage);
-
-       *reservedp = 0;
-       *totalp = total;
-       *spacep = *residentp = total * round_page_32(KERNEL_STACK_SIZE);
-       *maxusagep = maxusage;
-       *maxstackp = 0;
-       return KERN_SUCCESS;
-
-#endif /* MACH_DEBUG */
-}
+int            split_funnel_off = 0;
+lck_grp_t      *funnel_lck_grp = LCK_GRP_NULL;
+lck_grp_attr_t *funnel_lck_grp_attr;
+lck_attr_t     *funnel_lck_attr;

 
 

-/*
- * Return info on stack usage for threads in a specific processor set
- */
-kern_return_t
-processor_set_stack_usage(
-       processor_set_t pset,
-       unsigned int    *totalp,
-       vm_size_t       *spacep,
-       vm_size_t       *residentp,
-       vm_size_t       *maxusagep,
-       vm_offset_t     *maxstackp)
+funnel_t *
+funnel_alloc(
+       int type)

 {
 {

-#if !MACH_DEBUG
-        return KERN_NOT_SUPPORTED;
-#else
-       unsigned int total;
-       vm_size_t maxusage;
-       vm_offset_t maxstack;
-
-       register thread_t *threads;
-       register thread_t thread;
-
-       unsigned int actual;    /* this many things */
-       unsigned int i;
-
-       vm_size_t size, size_needed;
-       vm_offset_t addr;
-
-       spl_t s;
-
-       if (pset == PROCESSOR_SET_NULL)
-               return KERN_INVALID_ARGUMENT;
-
-       size = 0; addr = 0;
-
-       for (;;) {
-               pset_lock(pset);
-               if (!pset->active) {
-                       pset_unlock(pset);
-                       return KERN_INVALID_ARGUMENT;
-               }
-
-               actual = pset->thread_count;
-
-               /* do we have the memory we need? */
-
-               size_needed = actual * sizeof(thread_t);
-               if (size_needed <= size)
-                       break;
-
-               /* unlock the pset and allocate more memory */
-               pset_unlock(pset);
-
-               if (size != 0)
-                       kfree(addr, size);
-
-               assert(size_needed > 0);
-               size = size_needed;
-
-               addr = kalloc(size);
-               if (addr == 0)
-                       return KERN_RESOURCE_SHORTAGE;
-       }
-
-       /* OK, have memory and the processor_set is locked & active */
-       s = splsched();
-       threads = (thread_t *) addr;
-       for (i = 0, thread = (thread_t) queue_first(&pset->threads);
-            !queue_end(&pset->threads, (queue_entry_t) thread);
-            thread = (thread_t) queue_next(&thread->pset_threads)) {
-               thread_lock(thread);
-               if (thread->ref_count > 0) {
-                       thread_reference_locked(thread);
-                       threads[i++] = thread;
-               }
-               thread_unlock(thread);
-       }
-       splx(s);
-       assert(i <= actual);
-
-       /* can unlock processor set now that we have the thread refs */
-       pset_unlock(pset);
+       lck_mtx_t       *m;
+       funnel_t        *fnl;

 
 

-       /* calculate maxusage and free thread references */
+       if (funnel_lck_grp == LCK_GRP_NULL) {
+               funnel_lck_grp_attr = lck_grp_attr_alloc_init();

 
 

-       total = 0;
-       maxusage = 0;
-       maxstack = 0;
-       while (i > 0) {
-               thread_t thread = threads[--i];
+               funnel_lck_grp = lck_grp_alloc_init("Funnel",  funnel_lck_grp_attr);

 
 

-               if (thread->kernel_stack != 0)
-                       total++;
-
-               thread_deallocate(thread);
+               funnel_lck_attr = lck_attr_alloc_init();

        }
        }

-
-       if (size != 0)
-               kfree(addr, size);
-
-       *totalp = total;
-       *residentp = *spacep = total * round_page_32(KERNEL_STACK_SIZE);
-       *maxusagep = maxusage;
-       *maxstackp = maxstack;
-       return KERN_SUCCESS;
-
-#endif /* MACH_DEBUG */
-}
-
-int split_funnel_off = 0;
-funnel_t *
-funnel_alloc(
-       int type)
-{
-       mutex_t *m;
-       funnel_t * fnl;

        if ((fnl = (funnel_t *)kalloc(sizeof(funnel_t))) != 0){
                bzero((void *)fnl, sizeof(funnel_t));
        if ((fnl = (funnel_t *)kalloc(sizeof(funnel_t))) != 0){
                bzero((void *)fnl, sizeof(funnel_t));

-               if ((m = mutex_alloc(0)) == (mutex_t *)NULL) {
-                       kfree((vm_offset_t)fnl, sizeof(funnel_t));
+               if ((m = lck_mtx_alloc_init(funnel_lck_grp, funnel_lck_attr)) == (lck_mtx_t *)NULL) {
+                       kfree(fnl, sizeof(funnel_t));

                        return(THR_FUNNEL_NULL);
                }
                fnl->fnl_mutex = m;
                        return(THR_FUNNEL_NULL);
                }
                fnl->fnl_mutex = m;


@@ -1603,54 +1293,29 @@ void
 funnel_free(
        funnel_t * fnl)
 {
 funnel_free(
        funnel_t * fnl)
 {

-       mutex_free(fnl->fnl_mutex);
+       lck_mtx_free(fnl->fnl_mutex, funnel_lck_grp);

        if (fnl->fnl_oldmutex)
        if (fnl->fnl_oldmutex)

-               mutex_free(fnl->fnl_oldmutex);
-       kfree((vm_offset_t)fnl, sizeof(funnel_t));
+               lck_mtx_free(fnl->fnl_oldmutex, funnel_lck_grp);
+       kfree(fnl, sizeof(funnel_t));

 }
 
 void 
 funnel_lock(
        funnel_t * fnl)
 {
 }
 
 void 
 funnel_lock(
        funnel_t * fnl)
 {

-       mutex_t * m;
-
-       m = fnl->fnl_mutex;
-restart:
-       mutex_lock(m);
+       lck_mtx_lock(fnl->fnl_mutex);

        fnl->fnl_mtxholder = current_thread();
        fnl->fnl_mtxholder = current_thread();

-       if (split_funnel_off && (m != fnl->fnl_mutex)) {
-               mutex_unlock(m);
-               m = fnl->fnl_mutex;     
-               goto restart;
-       }

 }
 
 void 
 funnel_unlock(
        funnel_t * fnl)
 {
 }
 
 void 
 funnel_unlock(
        funnel_t * fnl)
 {

-       mutex_unlock(fnl->fnl_mutex);
+       lck_mtx_unlock(fnl->fnl_mutex);
+       fnl->fnl_mtxholder = NULL;

        fnl->fnl_mtxrelease = current_thread();
 }
 
        fnl->fnl_mtxrelease = current_thread();
 }
 

-int            refunnel_hint_enabled = 0;
-
-boolean_t
-refunnel_hint(
-       thread_t                thread,
-       wait_result_t   wresult)
-{
-       if (    !(thread->funnel_state & TH_FN_REFUNNEL)        ||
-                               wresult != THREAD_AWAKENED                              )
-               return (FALSE);
-
-       if (!refunnel_hint_enabled)
-               return (FALSE);
-
-       return (mutex_preblock(thread->funnel_lock->fnl_mutex, thread));
-}
-

 funnel_t *
 thread_funnel_get(
        void)
 funnel_t *
 thread_funnel_get(
        void)


@@ -1680,7 +1345,7 @@ thread_funnel_set(
 
                if (funneled == TRUE) {
                        if (cur_thread->funnel_lock)
 
                if (funneled == TRUE) {
                        if (cur_thread->funnel_lock)

-                               panic("Funnel lock called when holding one %x", cur_thread->funnel_lock);
+                               panic("Funnel lock called when holding one %p", cur_thread->funnel_lock);

                        KERNEL_DEBUG(0x6032428 | DBG_FUNC_NONE,
                                                                                        fnl, 1, 0, 0, 0);
                        funnel_lock(fnl);
                        KERNEL_DEBUG(0x6032428 | DBG_FUNC_NONE,
                                                                                        fnl, 1, 0, 0, 0);
                        funnel_lock(fnl);


@@ -1710,78 +1375,193 @@ thread_funnel_set(
        return(funnel_state_prev);
 }
 
        return(funnel_state_prev);
 }
 

-boolean_t
-thread_funnel_merge(
-       funnel_t * fnl,
-       funnel_t * otherfnl)
+static void
+sched_call_null(
+__unused       int                     type,
+__unused       thread_t        thread)

 {
 {

-       mutex_t * m;
-       mutex_t * otherm;
-       funnel_t * gfnl;
-       extern int disable_funnel;
-
-       if ((gfnl = thread_funnel_get()) == THR_FUNNEL_NULL)
-               panic("thread_funnel_merge called with no funnels held");
-
-       if (gfnl->fnl_type != 1)
-               panic("thread_funnel_merge called from non kernel funnel");
-
-       if (gfnl != fnl)
-               panic("thread_funnel_merge incorrect invocation");
-
-       if (disable_funnel || split_funnel_off)
-               return (KERN_FAILURE);
-
-       m = fnl->fnl_mutex;
-       otherm = otherfnl->fnl_mutex;
-
-       /* Acquire other funnel mutex */
-       mutex_lock(otherm);
-       split_funnel_off = 1;
-       disable_funnel = 1;
-       otherfnl->fnl_mutex = m;
-       otherfnl->fnl_type = fnl->fnl_type;
-       otherfnl->fnl_oldmutex = otherm;        /* save this for future use */
+       return;
+}

 
 

-       mutex_unlock(otherm);
-       return(KERN_SUCCESS);
+void
+thread_sched_call(
+       thread_t                thread,
+       sched_call_t    call)
+{
+       thread->sched_call = (call != NULL)? call: sched_call_null;

 }
 
 void
 }
 
 void

-thread_set_cont_arg(
-       int                             arg)
+thread_static_param(
+       thread_t                thread,
+       boolean_t               state)

 {
 {

-       thread_t                self = current_thread();
+       thread_mtx_lock(thread);
+       thread->static_param = state;
+       thread_mtx_unlock(thread);
+}

 
 

-       self->saved.misc = arg; 
+uint64_t
+thread_tid(
+       thread_t        thread)
+{
+       return (thread != THREAD_NULL? thread->thread_id: 0);

 }
 
 }
 

-int
-thread_get_cont_arg(void)
+uint64_t
+thread_dispatchqaddr(
+       thread_t                thread)

 {
 {

-       thread_t                self = current_thread();
+       uint64_t        dispatchqueue_addr = 0;
+       uint64_t        thread_handle = 0;
+
+       if (thread != THREAD_NULL) {
+#if defined(__ppc__) || defined(__arm__)
+               thread_handle = thread->machine.cthread_self;
+#else
+               thread_handle = thread->machine.pcb->cthread_self;
+#endif
+
+               if (thread->task->bsd_info)
+                       dispatchqueue_addr = thread_handle + get_dispatchqueue_offset_from_proc(thread->task->bsd_info);
+       }

 
 

-       return (self->saved.misc); 
+       return (dispatchqueue_addr);

 }
 
 /*
  * Export routines to other components for things that are done as macros
  * within the osfmk component.
  */
 }
 
 /*
  * Export routines to other components for things that are done as macros
  * within the osfmk component.
  */

+
+#undef thread_reference
+void thread_reference(thread_t thread);
+void
+thread_reference(
+       thread_t        thread)
+{
+       if (thread != THREAD_NULL)
+               thread_reference_internal(thread);
+}
+

 #undef thread_should_halt
 #undef thread_should_halt

+

 boolean_t
 thread_should_halt(
        thread_t                th)
 {
 boolean_t
 thread_should_halt(
        thread_t                th)
 {

-       return(thread_should_halt_fast(th));
+       return (thread_should_halt_fast(th));
+}
+
+#if CONFIG_DTRACE
+uint32_t dtrace_get_thread_predcache(thread_t thread)
+{
+       if (thread != THREAD_NULL)
+               return thread->t_dtrace_predcache;
+       else
+               return 0;
+}
+
+int64_t dtrace_get_thread_vtime(thread_t thread)
+{
+       if (thread != THREAD_NULL)
+               return thread->t_dtrace_vtime;
+       else
+               return 0;
+}
+
+int64_t dtrace_get_thread_tracing(thread_t thread)
+{
+       if (thread != THREAD_NULL)
+               return thread->t_dtrace_tracing;
+       else
+               return 0;
+}
+
+boolean_t dtrace_get_thread_reentering(thread_t thread)
+{
+       if (thread != THREAD_NULL)
+               return (thread->options & TH_OPT_DTRACE) ? TRUE : FALSE;
+       else
+               return 0;
+}
+
+vm_offset_t dtrace_get_kernel_stack(thread_t thread)
+{
+       if (thread != THREAD_NULL)
+               return thread->kernel_stack;
+       else
+               return 0;
+}
+
+int64_t dtrace_calc_thread_recent_vtime(thread_t thread)
+{
+#if STAT_TIME
+       if (thread != THREAD_NULL) {
+               return timer_grab(&(thread->system_timer)) + timer_grab(&(thread->user_timer));
+       } else
+               return 0;
+#else
+       if (thread != THREAD_NULL) {
+               processor_t             processor = current_processor();
+               uint64_t                                abstime = mach_absolute_time();
+               timer_t                                 timer;
+
+               timer = PROCESSOR_DATA(processor, thread_timer);
+
+               return timer_grab(&(thread->system_timer)) + timer_grab(&(thread->user_timer)) +
+                               (abstime - timer->tstamp); /* XXX need interrupts off to prevent missed time? */
+       } else
+               return 0;
+#endif
+}
+
+void dtrace_set_thread_predcache(thread_t thread, uint32_t predcache)
+{
+       if (thread != THREAD_NULL)
+               thread->t_dtrace_predcache = predcache;
+}
+
+void dtrace_set_thread_vtime(thread_t thread, int64_t vtime)
+{
+       if (thread != THREAD_NULL)
+               thread->t_dtrace_vtime = vtime;
+}
+
+void dtrace_set_thread_tracing(thread_t thread, int64_t accum)
+{
+       if (thread != THREAD_NULL)
+               thread->t_dtrace_tracing = accum;

 }
 
 }
 

-vm_offset_t min_valid_stack_address(void)
+void dtrace_set_thread_reentering(thread_t thread, boolean_t vbool)

 {
 {

-       return vm_map_min(stack_map);
+       if (thread != THREAD_NULL) {
+               if (vbool)
+                       thread->options |= TH_OPT_DTRACE;
+               else
+                       thread->options &= (~TH_OPT_DTRACE);
+       }
+}
+
+vm_offset_t dtrace_set_thread_recover(thread_t thread, vm_offset_t recover)
+{
+       vm_offset_t prev = 0;
+
+       if (thread != THREAD_NULL) {
+               prev = thread->recover;
+               thread->recover = recover;
+       }
+       return prev;

 }
 
 }
 

-vm_offset_t max_valid_stack_address(void)
+void dtrace_thread_bootstrap(void)

 {
 {

-       return vm_map_max(stack_map);
+       task_t task = current_task();
+       if(task->thread_count == 1) {
+               DTRACE_PROC(start);
+       }
+       DTRACE_PROC(lwp__start);
+

 }
 }

+#endif /* CONFIG_DTRACE */