[apple/xnu.git] / osfmk / kern / stack.c

/*
 * Copyright (c) 2003-2007 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 * 
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */
/*
 *	Kernel stack management routines.
 */

#include <mach/mach_host.h>
#include <mach/mach_types.h>
#include <mach/processor_set.h>

#include <kern/kern_types.h>
#include <kern/mach_param.h>
#include <kern/processor.h>
#include <kern/thread.h>
#include <kern/zalloc.h>
#include <kern/kalloc.h>
#include <kern/ledger.h>

#include <vm/vm_map.h>
#include <vm/vm_kern.h>

#include <mach_debug.h>
#include <san/kasan.h>

/*
 *	We allocate stacks from generic kernel VM.
 *
 *	The stack_free_list can only be accessed at splsched,
 *	because stack_alloc_try/thread_invoke operate at splsched.
 */

decl_simple_lock_data(static,stack_lock_data)
#define stack_lock()		simple_lock(&stack_lock_data)
#define stack_unlock()		simple_unlock(&stack_lock_data)

#define STACK_CACHE_SIZE	2

static vm_offset_t		stack_free_list;

static unsigned int		stack_free_count, stack_free_hiwat;		/* free list count */
static unsigned int		stack_hiwat;
unsigned int			stack_total;				/* current total count */
unsigned long long		stack_allocs;				/* total count of allocations */

static int			stack_fake_zone_index = -1;	/* index in zone_info array */

static unsigned int		stack_free_target;
static int				stack_free_delta;

static unsigned int		stack_new_count;						/* total new stack allocations */

static vm_offset_t		stack_addr_mask;

unsigned int			kernel_stack_pages;
vm_offset_t			kernel_stack_size;
vm_offset_t			kernel_stack_mask;
vm_offset_t			kernel_stack_depth_max;

/*
 *	The next field is at the base of the stack,
 *	so the low end is left unsullied.
 */
#define stack_next(stack)	\
	(*((vm_offset_t *)((stack) + kernel_stack_size) - 1))

static inline int
log2(vm_offset_t size)
{
	int	result;
	for (result = 0; size > 0; result++)
		size >>= 1;
	return result;
}

static inline vm_offset_t
roundup_pow2(vm_offset_t size)
{
	return 1UL << (log2(size - 1) + 1); 
}

static vm_offset_t stack_alloc_internal(void);
static void stack_free_stack(vm_offset_t);

void
stack_init(void)
{
	simple_lock_init(&stack_lock_data, 0);
	
	kernel_stack_pages = KERNEL_STACK_SIZE / PAGE_SIZE;
	kernel_stack_size = KERNEL_STACK_SIZE;
	kernel_stack_mask = -KERNEL_STACK_SIZE;
	kernel_stack_depth_max = 0;

	if (PE_parse_boot_argn("kernel_stack_pages",
			       &kernel_stack_pages,
			       sizeof (kernel_stack_pages))) {
		kernel_stack_size = kernel_stack_pages * PAGE_SIZE;
		printf("stack_init: kernel_stack_pages=%d kernel_stack_size=%p\n",
			kernel_stack_pages, (void *) kernel_stack_size);
	}

	if (kernel_stack_size < round_page(kernel_stack_size))
		panic("stack_init: stack size %p not a multiple of page size %d\n",
			(void *) kernel_stack_size, PAGE_SIZE);
	
	stack_addr_mask = roundup_pow2(kernel_stack_size) - 1;
	kernel_stack_mask = ~stack_addr_mask;
}

/*
 *	stack_alloc:
 *
 *	Allocate a stack for a thread, may
 *	block.
 */

static vm_offset_t 
stack_alloc_internal(void)
{
	vm_offset_t		stack = 0;
	spl_t			s;
	int			flags = 0;
	kern_return_t		kr = KERN_SUCCESS;

	s = splsched();
	stack_lock();
	stack_allocs++;
	stack = stack_free_list;
	if (stack != 0) {
		stack_free_list = stack_next(stack);
		stack_free_count--;
	}
	else {
		if (++stack_total > stack_hiwat)
			stack_hiwat = stack_total;
		stack_new_count++;
	}
	stack_free_delta--;
	stack_unlock();
	splx(s);
		
	if (stack == 0) {

		/*
		 * Request guard pages on either side of the stack.  Ask
		 * kernel_memory_allocate() for two extra pages to account
		 * for these.
		 */

		flags = KMA_GUARD_FIRST | KMA_GUARD_LAST | KMA_KSTACK | KMA_KOBJECT;
		kr = kernel_memory_allocate(kernel_map, &stack,
					   kernel_stack_size + (2*PAGE_SIZE),
					   stack_addr_mask,
					   flags,
					   VM_KERN_MEMORY_STACK);
		if (kr != KERN_SUCCESS) {
			panic("stack_alloc: kernel_memory_allocate(size:0x%llx, mask: 0x%llx, flags: 0x%x) failed with %d\n", (uint64_t)(kernel_stack_size + (2*PAGE_SIZE)), (uint64_t)stack_addr_mask, flags, kr);
		}

		/*
		 * The stack address that comes back is the address of the lower
		 * guard page.  Skip past it to get the actual stack base address.
		 */

		stack += PAGE_SIZE;
	}
	return stack;
}

void
stack_alloc(
	thread_t	thread)
{

	assert(thread->kernel_stack == 0);
	machine_stack_attach(thread, stack_alloc_internal());
}

void
stack_handoff(thread_t from, thread_t to)
{
	assert(from == current_thread());
	machine_stack_handoff(from, to);
}

/*
 *	stack_free:
 *
 *	Detach and free the stack for a thread.
 */
void
stack_free(
	thread_t	thread)
{
    vm_offset_t		stack = machine_stack_detach(thread);

#if KASAN
	kasan_unpoison_stack(stack, kernel_stack_size);
	kasan_unpoison_fakestack(thread);
#endif

	assert(stack);
	if (stack != thread->reserved_stack) {
		stack_free_stack(stack);
	}
}

void
stack_free_reserved(
	thread_t	thread)
{
	if (thread->reserved_stack != thread->kernel_stack) {
#if KASAN
		kasan_unpoison_stack(thread->reserved_stack, kernel_stack_size);
#endif
		stack_free_stack(thread->reserved_stack);
	}
}

static void
stack_free_stack(
	vm_offset_t		stack)
{
	struct stack_cache	*cache;
	spl_t				s;

	s = splsched();
	cache = &PROCESSOR_DATA(current_processor(), stack_cache);
	if (cache->count < STACK_CACHE_SIZE) {
		stack_next(stack) = cache->free;
		cache->free = stack;
		cache->count++;
	}
	else {
		stack_lock();
		stack_next(stack) = stack_free_list;
		stack_free_list = stack;
		if (++stack_free_count > stack_free_hiwat)
			stack_free_hiwat = stack_free_count;
		stack_free_delta++;
		stack_unlock();
	}
	splx(s);
}

/*
 *	stack_alloc_try:
 *
 *	Non-blocking attempt to allocate a
 *	stack for a thread.
 *
 *	Returns TRUE on success.
 *
 *	Called at splsched.
 */
boolean_t
stack_alloc_try(
	thread_t		thread)
{
	struct stack_cache	*cache;
	vm_offset_t			stack;

	cache = &PROCESSOR_DATA(current_processor(), stack_cache);
	stack = cache->free;
	if (stack != 0) {
		cache->free = stack_next(stack);
		cache->count--;
	}
	else {
		if (stack_free_list != 0) {
			stack_lock();
			stack = stack_free_list;
			if (stack != 0) {
				stack_free_list = stack_next(stack);
				stack_free_count--;
				stack_free_delta--;
			}
			stack_unlock();
		}
	}

	if (stack != 0 || (stack = thread->reserved_stack) != 0) {
		machine_stack_attach(thread, stack);
		return (TRUE);
	}

	return (FALSE);
}

static unsigned int		stack_collect_tick, last_stack_tick;

/*
 *	stack_collect:
 *
 *	Free excess kernel stacks, may
 *	block.
 */
void
stack_collect(void)
{
	if (stack_collect_tick != last_stack_tick) {
		unsigned int	target;
		vm_offset_t		stack;
		spl_t			s;

		s = splsched();
		stack_lock();

		target = stack_free_target + (STACK_CACHE_SIZE * processor_count);
		target += (stack_free_delta >= 0)? stack_free_delta: -stack_free_delta;

		while (stack_free_count > target) {
			stack = stack_free_list;
			stack_free_list = stack_next(stack);
			stack_free_count--; stack_total--;
			stack_unlock();
			splx(s);

			/*
			 * Get the stack base address, then decrement by one page
			 * to account for the lower guard page.  Add two extra pages
			 * to the size to account for the guard pages on both ends
			 * that were originally requested when the stack was allocated
			 * back in stack_alloc().
			 */

			stack = (vm_offset_t)vm_map_trunc_page(
				stack,
				VM_MAP_PAGE_MASK(kernel_map));
			stack -= PAGE_SIZE;
			if (vm_map_remove(
				    kernel_map,
				    stack,
				    stack + kernel_stack_size+(2*PAGE_SIZE),
				    VM_MAP_REMOVE_KUNWIRE)
			    != KERN_SUCCESS)
				panic("stack_collect: vm_map_remove");
			stack = 0;

			s = splsched();
			stack_lock();

			target = stack_free_target + (STACK_CACHE_SIZE * processor_count);
			target += (stack_free_delta >= 0)? stack_free_delta: -stack_free_delta;
		}

		last_stack_tick = stack_collect_tick;

		stack_unlock();
		splx(s);
	}
}

/*
 *	compute_stack_target:
 *
 *	Computes a new target free list count
 *	based on recent alloc / free activity.
 *
 *	Limits stack collection to once per
 *	computation period.
 */
void
compute_stack_target(
__unused void		*arg)
{
	spl_t		s;

	s = splsched();
	stack_lock();

	if (stack_free_target > 5)
		stack_free_target = (4 * stack_free_target) / 5;
	else
	if (stack_free_target > 0)
		stack_free_target--;

	stack_free_target += (stack_free_delta >= 0)? stack_free_delta: -stack_free_delta;

	stack_free_delta = 0;
	stack_collect_tick++;

	stack_unlock();
	splx(s);
}

void
stack_fake_zone_init(int zone_index)
{
	stack_fake_zone_index = zone_index;
}

void
stack_fake_zone_info(int *count, 
		     vm_size_t *cur_size, vm_size_t *max_size, vm_size_t *elem_size, vm_size_t *alloc_size,
		     uint64_t *sum_size, int *collectable, int *exhaustable, int *caller_acct)
{
	unsigned int	total, hiwat, free;
	unsigned long long all;
	spl_t			s;

	s = splsched();
	stack_lock();
	all = stack_allocs;
	total = stack_total;
	hiwat = stack_hiwat;
	free = stack_free_count;
	stack_unlock();
	splx(s);

	*count      = total - free;
	*cur_size   = kernel_stack_size * total;
	*max_size   = kernel_stack_size * hiwat;
	*elem_size  = kernel_stack_size;
	*alloc_size = kernel_stack_size;
	*sum_size = all * kernel_stack_size;

	*collectable = 1;
	*exhaustable = 0;
	*caller_acct = 1;
}

/* OBSOLETE */
void	stack_privilege(
			thread_t	thread);

void
stack_privilege(
	__unused thread_t	thread)
{
	/* OBSOLETE */
}

/*
 * Return info on stack usage for threads in a specific processor set
 */
kern_return_t
processor_set_stack_usage(
	processor_set_t	pset,
	unsigned int	*totalp,
	vm_size_t	*spacep,
	vm_size_t	*residentp,
	vm_size_t	*maxusagep,
	vm_offset_t	*maxstackp)
{
#if !MACH_DEBUG
        return KERN_NOT_SUPPORTED;
#else
	unsigned int total;
	vm_size_t maxusage;
	vm_offset_t maxstack;

	thread_t *thread_list;
	thread_t thread;

	unsigned int actual;	/* this many things */
	unsigned int i;

	vm_size_t size, size_needed;
	void *addr;

	if (pset == PROCESSOR_SET_NULL || pset != &pset0)
		return KERN_INVALID_ARGUMENT;

	size = 0;
	addr = NULL;

	for (;;) {
		lck_mtx_lock(&tasks_threads_lock);

		actual = threads_count;

		/* do we have the memory we need? */

		size_needed = actual * sizeof(thread_t);
		if (size_needed <= size)
			break;

		lck_mtx_unlock(&tasks_threads_lock);

		if (size != 0)
			kfree(addr, size);

		assert(size_needed > 0);
		size = size_needed;

		addr = kalloc(size);
		if (addr == 0)
			return KERN_RESOURCE_SHORTAGE;
	}

	/* OK, have memory and list is locked */
	thread_list = (thread_t *) addr;
	for (i = 0, thread = (thread_t)(void *) queue_first(&threads);
					!queue_end(&threads, (queue_entry_t) thread);
					thread = (thread_t)(void *) queue_next(&thread->threads)) {
		thread_reference_internal(thread);
		thread_list[i++] = thread;
	}
	assert(i <= actual);

	lck_mtx_unlock(&tasks_threads_lock);

	/* calculate maxusage and free thread references */

	total = 0;
	maxusage = 0;
	maxstack = 0;
	while (i > 0) {
		thread_t threadref = thread_list[--i];

		if (threadref->kernel_stack != 0)
			total++;

		thread_deallocate(threadref);
	}

	if (size != 0)
		kfree(addr, size);

	*totalp = total;
	*residentp = *spacep = total * round_page(kernel_stack_size);
	*maxusagep = maxusage;
	*maxstackp = maxstack;
	return KERN_SUCCESS;

#endif	/* MACH_DEBUG */
}

vm_offset_t min_valid_stack_address(void)
{
	return (vm_offset_t)vm_map_min(kernel_map);
}

vm_offset_t max_valid_stack_address(void)
{
	return (vm_offset_t)vm_map_max(kernel_map);
}
Commit	Line	Data
91447636	1	/*
2d21ac55	2	* Copyright (c) 2003-2007 Apple Inc. All rights reserved.
91447636	3	*
2d21ac55	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
91447636	5	*
2d21ac55 A	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
8f6c56a5	14	*
2d21ac55 A	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
8f6c56a5 A	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
8f6c56a5 A	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
2d21ac55 A	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
8f6c56a5	25	*
2d21ac55	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
91447636 A	27	*/
	28	/*
	29	* Kernel stack management routines.
	30	*/
	31
	32	#include <mach/mach_host.h>
	33	#include <mach/mach_types.h>
	34	#include <mach/processor_set.h>
	35
	36	#include <kern/kern_types.h>
	37	#include <kern/mach_param.h>
	38	#include <kern/processor.h>
	39	#include <kern/thread.h>
	40	#include <kern/zalloc.h>
	41	#include <kern/kalloc.h>
316670eb	42	#include <kern/ledger.h>
91447636 A	43
	44	#include <vm/vm_map.h>
	45	#include <vm/vm_kern.h>
	46
	47	#include <mach_debug.h>
5ba3f43e	48	#include <san/kasan.h>
91447636 A	49
	50	/*
	51	* We allocate stacks from generic kernel VM.
	52	*
	53	* The stack_free_list can only be accessed at splsched,
	54	* because stack_alloc_try/thread_invoke operate at splsched.
	55	*/
	56
	57	decl_simple_lock_data(static,stack_lock_data)
	58	#define stack_lock() simple_lock(&stack_lock_data)
	59	#define stack_unlock() simple_unlock(&stack_lock_data)
	60
	61	#define STACK_CACHE_SIZE 2
	62
91447636 A	63	static vm_offset_t stack_free_list;
	64
	65	static unsigned int stack_free_count, stack_free_hiwat; /* free list count */
c910b4d9 A	66	static unsigned int stack_hiwat;
c910b4d9 A	67	unsigned int stack_total; /* current total count */
6d2010ae A	68	unsigned long long stack_allocs; /* total count of allocations */
	69
	70	static int stack_fake_zone_index = -1; /* index in zone_info array */
91447636 A	71
	72	static unsigned int stack_free_target;
	73	static int stack_free_delta;
	74
	75	static unsigned int stack_new_count; /* total new stack allocations */
	76
	77	static vm_offset_t stack_addr_mask;
	78
fe8ab488 A	79	unsigned int kernel_stack_pages;
	80	vm_offset_t kernel_stack_size;
	81	vm_offset_t kernel_stack_mask;
	82	vm_offset_t kernel_stack_depth_max;
b0d623f7	83
91447636 A	84	/*
	85	* The next field is at the base of the stack,
	86	* so the low end is left unsullied.
	87	*/
	88	#define stack_next(stack) \
b0d623f7 A	89	(((vm_offset_t )((stack) + kernel_stack_size) - 1))
	90
	91	static inline int
	92	log2(vm_offset_t size)
	93	{
	94	int result;
	95	for (result = 0; size > 0; result++)
	96	size >>= 1;
	97	return result;
	98	}
	99
	100	static inline vm_offset_t
	101	roundup_pow2(vm_offset_t size)
	102	{
	103	return 1UL << (log2(size - 1) + 1);
	104	}
91447636	105
6d2010ae A	106	static vm_offset_t stack_alloc_internal(void);
	107	static void stack_free_stack(vm_offset_t);
	108
91447636 A	109	void
	110	stack_init(void)
	111	{
91447636 A	112	simple_lock_init(&stack_lock_data, 0);
91447636 A	113
fe8ab488 A	114	kernel_stack_pages = KERNEL_STACK_SIZE / PAGE_SIZE;
	115	kernel_stack_size = KERNEL_STACK_SIZE;
	116	kernel_stack_mask = -KERNEL_STACK_SIZE;
	117	kernel_stack_depth_max = 0;
	118
b0d623f7 A	119	if (PE_parse_boot_argn("kernel_stack_pages",
	120	&kernel_stack_pages,
	121	sizeof (kernel_stack_pages))) {
	122	kernel_stack_size = kernel_stack_pages * PAGE_SIZE;
	123	printf("stack_init: kernel_stack_pages=%d kernel_stack_size=%p\n",
	124	kernel_stack_pages, (void *) kernel_stack_size);
	125	}
	126
	127	if (kernel_stack_size < round_page(kernel_stack_size))
	128	panic("stack_init: stack size %p not a multiple of page size %d\n",
	129	(void *) kernel_stack_size, PAGE_SIZE);
91447636	130
b0d623f7 A	131	stack_addr_mask = roundup_pow2(kernel_stack_size) - 1;
b0d623f7 A	132	kernel_stack_mask = ~stack_addr_mask;
91447636 A	133	}
	134
	135	/*
	136	* stack_alloc:
	137	*
	138	* Allocate a stack for a thread, may
	139	* block.
	140	*/
6d2010ae A	141
	142	static vm_offset_t
	143	stack_alloc_internal(void)
91447636	144	{
5ba3f43e	145	vm_offset_t stack = 0;
91447636	146	spl_t s;
5ba3f43e A	147	int flags = 0;
5ba3f43e A	148	kern_return_t kr = KERN_SUCCESS;
91447636	149
91447636 A	150	s = splsched();
91447636 A	151	stack_lock();
6d2010ae	152	stack_allocs++;
91447636 A	153	stack = stack_free_list;
	154	if (stack != 0) {
	155	stack_free_list = stack_next(stack);
	156	stack_free_count--;
	157	}
	158	else {
	159	if (++stack_total > stack_hiwat)
	160	stack_hiwat = stack_total;
	161	stack_new_count++;
	162	}
	163	stack_free_delta--;
	164	stack_unlock();
	165	splx(s);
	166
	167	if (stack == 0) {
2d21ac55 A	168
	169	/*
	170	* Request guard pages on either side of the stack. Ask
	171	* kernel_memory_allocate() for two extra pages to account
	172	* for these.
	173	*/
	174
5ba3f43e A	175	flags = KMA_GUARD_FIRST \| KMA_GUARD_LAST \| KMA_KSTACK \| KMA_KOBJECT;
5ba3f43e A	176	kr = kernel_memory_allocate(kernel_map, &stack,
b0d623f7	177	kernel_stack_size + (2*PAGE_SIZE),
2d21ac55	178	stack_addr_mask,
5ba3f43e A	179	flags,
	180	VM_KERN_MEMORY_STACK);
	181	if (kr != KERN_SUCCESS) {
	182	panic("stack_alloc: kernel_memory_allocate(size:0x%llx, mask: 0x%llx, flags: 0x%x) failed with %d\n", (uint64_t)(kernel_stack_size + (2*PAGE_SIZE)), (uint64_t)stack_addr_mask, flags, kr);
	183	}
2d21ac55 A	184
	185	/*
	186	* The stack address that comes back is the address of the lower
	187	* guard page. Skip past it to get the actual stack base address.
	188	*/
	189
	190	stack += PAGE_SIZE;
91447636	191	}
6d2010ae A	192	return stack;
6d2010ae A	193	}
91447636	194
6d2010ae A	195	void
	196	stack_alloc(
	197	thread_t thread)
	198	{
	199
	200	assert(thread->kernel_stack == 0);
	201	machine_stack_attach(thread, stack_alloc_internal());
6d2010ae A	202	}
	203
	204	void
	205	stack_handoff(thread_t from, thread_t to)
	206	{
	207	assert(from == current_thread());
	208	machine_stack_handoff(from, to);
91447636 A	209	}
	210
	211	/*
	212	* stack_free:
	213	*
	214	* Detach and free the stack for a thread.
	215	*/
	216	void
	217	stack_free(
	218	thread_t thread)
	219	{
	220	vm_offset_t stack = machine_stack_detach(thread);
	221
5ba3f43e A	222	#if KASAN
	223	kasan_unpoison_stack(stack, kernel_stack_size);
	224	kasan_unpoison_fakestack(thread);
	225	#endif
	226
91447636	227	assert(stack);
6d2010ae	228	if (stack != thread->reserved_stack) {
2d21ac55	229	stack_free_stack(stack);
6d2010ae	230	}
91447636 A	231	}
	232
	233	void
6d2010ae A	234	stack_free_reserved(
	235	thread_t thread)
	236	{
	237	if (thread->reserved_stack != thread->kernel_stack) {
5ba3f43e A	238	#if KASAN
	239	kasan_unpoison_stack(thread->reserved_stack, kernel_stack_size);
	240	#endif
6d2010ae	241	stack_free_stack(thread->reserved_stack);
6d2010ae A	242	}
	243	}
	244
	245	static void
91447636 A	246	stack_free_stack(
	247	vm_offset_t stack)
	248	{
	249	struct stack_cache *cache;
	250	spl_t s;
	251
	252	s = splsched();
	253	cache = &PROCESSOR_DATA(current_processor(), stack_cache);
	254	if (cache->count < STACK_CACHE_SIZE) {
	255	stack_next(stack) = cache->free;
	256	cache->free = stack;
	257	cache->count++;
	258	}
	259	else {
	260	stack_lock();
	261	stack_next(stack) = stack_free_list;
	262	stack_free_list = stack;
	263	if (++stack_free_count > stack_free_hiwat)
	264	stack_free_hiwat = stack_free_count;
	265	stack_free_delta++;
	266	stack_unlock();
	267	}
	268	splx(s);
	269	}
	270
	271	/*
	272	* stack_alloc_try:
	273	*
	274	* Non-blocking attempt to allocate a
	275	* stack for a thread.
	276	*
	277	* Returns TRUE on success.
	278	*
	279	* Called at splsched.
	280	*/
	281	boolean_t
	282	stack_alloc_try(
	283	thread_t thread)
	284	{
	285	struct stack_cache *cache;
	286	vm_offset_t stack;
	287
	288	cache = &PROCESSOR_DATA(current_processor(), stack_cache);
	289	stack = cache->free;
	290	if (stack != 0) {
	291	cache->free = stack_next(stack);
	292	cache->count--;
	293	}
	294	else {
	295	if (stack_free_list != 0) {
	296	stack_lock();
	297	stack = stack_free_list;
	298	if (stack != 0) {
	299	stack_free_list = stack_next(stack);
	300	stack_free_count--;
	301	stack_free_delta--;
	302	}
	303	stack_unlock();
	304	}
	305	}
	306
	307	if (stack != 0 \|\| (stack = thread->reserved_stack) != 0) {
	308	machine_stack_attach(thread, stack);
	309	return (TRUE);
310	}
311
312	return (FALSE);
313	}
314
315	static unsigned int stack_collect_tick, last_stack_tick;
316
317	/*
318	* stack_collect:
319	*
320	* Free excess kernel stacks, may
321	* block.
322	*/
323	void
324	stack_collect(void)
325	{
326	if (stack_collect_tick != last_stack_tick) {
327	unsigned int target;
328	vm_offset_t stack;
329	spl_t s;
330
331	s = splsched();
332	stack_lock();
333
334	target = stack_free_target + (STACK_CACHE_SIZE * processor_count);
335	target += (stack_free_delta >= 0)? stack_free_delta: -stack_free_delta;
336
337	while (stack_free_count > target) {
338	stack = stack_free_list;
339	stack_free_list = stack_next(stack);
340	stack_free_count--; stack_total--;
341	stack_unlock();
342	splx(s);
343
2d21ac55 A	344	/*
	345	* Get the stack base address, then decrement by one page
	346	* to account for the lower guard page. Add two extra pages
	347	* to the size to account for the guard pages on both ends
	348	* that were originally requested when the stack was allocated
	349	* back in stack_alloc().
	350	*/
	351
39236c6e A	352	stack = (vm_offset_t)vm_map_trunc_page(
	353	stack,
	354	VM_MAP_PAGE_MASK(kernel_map));
2d21ac55 A	355	stack -= PAGE_SIZE;
	356	if (vm_map_remove(
	357	kernel_map,
	358	stack,
b0d623f7	359	stack + kernel_stack_size+(2*PAGE_SIZE),
2d21ac55 A	360	VM_MAP_REMOVE_KUNWIRE)
2d21ac55 A	361	!= KERN_SUCCESS)
91447636	362	panic("stack_collect: vm_map_remove");
2d21ac55	363	stack = 0;
91447636 A	364
	365	s = splsched();
	366	stack_lock();
	367
	368	target = stack_free_target + (STACK_CACHE_SIZE * processor_count);
	369	target += (stack_free_delta >= 0)? stack_free_delta: -stack_free_delta;
	370	}
	371
	372	last_stack_tick = stack_collect_tick;
	373
	374	stack_unlock();
	375	splx(s);
	376	}
	377	}
	378
	379	/*
	380	* compute_stack_target:
	381	*
	382	* Computes a new target free list count
	383	* based on recent alloc / free activity.
	384	*
	385	* Limits stack collection to once per
	386	* computation period.
	387	*/
	388	void
	389	compute_stack_target(
	390	__unused void *arg)
	391	{
	392	spl_t s;
	393
	394	s = splsched();
	395	stack_lock();
	396
	397	if (stack_free_target > 5)
	398	stack_free_target = (4 * stack_free_target) / 5;
	399	else
	400	if (stack_free_target > 0)
	401	stack_free_target--;
	402
	403	stack_free_target += (stack_free_delta >= 0)? stack_free_delta: -stack_free_delta;
	404
	405	stack_free_delta = 0;
	406	stack_collect_tick++;
	407
	408	stack_unlock();
	409	splx(s);
	410	}
	411
	412	void
6d2010ae A	413	stack_fake_zone_init(int zone_index)
	414	{
	415	stack_fake_zone_index = zone_index;
	416	}
	417
	418	void
	419	stack_fake_zone_info(int *count,
	420	vm_size_t cur_size, vm_size_t max_size, vm_size_t elem_size, vm_size_t alloc_size,
	421	uint64_t sum_size, int collectable, int exhaustable, int caller_acct)
91447636 A	422	{
91447636 A	423	unsigned int total, hiwat, free;
6d2010ae	424	unsigned long long all;
91447636 A	425	spl_t s;
	426
	427	s = splsched();
	428	stack_lock();
6d2010ae	429	all = stack_allocs;
91447636 A	430	total = stack_total;
	431	hiwat = stack_hiwat;
	432	free = stack_free_count;
	433	stack_unlock();
	434	splx(s);
	435
	436	*count = total - free;
b0d623f7 A	437	cur_size = kernel_stack_size total;
	438	max_size = kernel_stack_size hiwat;
	439	*elem_size = kernel_stack_size;
	440	*alloc_size = kernel_stack_size;
6d2010ae A	441	sum_size = all kernel_stack_size;
6d2010ae A	442
91447636 A	443	*collectable = 1;
91447636 A	444	*exhaustable = 0;
6d2010ae	445	*caller_acct = 1;
91447636 A	446	}
	447
	448	/* OBSOLETE */
	449	void stack_privilege(
	450	thread_t thread);
	451
	452	void
	453	stack_privilege(
	454	__unused thread_t thread)
	455	{
	456	/* OBSOLETE */
	457	}
	458
	459	/*
	460	* Return info on stack usage for threads in a specific processor set
	461	*/
	462	kern_return_t
	463	processor_set_stack_usage(
	464	processor_set_t pset,
	465	unsigned int *totalp,
	466	vm_size_t *spacep,
	467	vm_size_t *residentp,
	468	vm_size_t *maxusagep,
	469	vm_offset_t *maxstackp)
	470	{
	471	#if !MACH_DEBUG
	472	return KERN_NOT_SUPPORTED;
	473	#else
	474	unsigned int total;
	475	vm_size_t maxusage;
	476	vm_offset_t maxstack;
	477
39037602 A	478	thread_t *thread_list;
39037602 A	479	thread_t thread;
91447636 A	480
	481	unsigned int actual; /* this many things */
	482	unsigned int i;
	483
	484	vm_size_t size, size_needed;
	485	void *addr;
	486
2d21ac55	487	if (pset == PROCESSOR_SET_NULL \|\| pset != &pset0)
91447636 A	488	return KERN_INVALID_ARGUMENT;
91447636 A	489
2d21ac55 A	490	size = 0;
2d21ac55 A	491	addr = NULL;
91447636 A	492
91447636 A	493	for (;;) {
b0d623f7	494	lck_mtx_lock(&tasks_threads_lock);
91447636	495
2d21ac55	496	actual = threads_count;
91447636 A	497
	498	/* do we have the memory we need? */
	499
	500	size_needed = actual * sizeof(thread_t);
	501	if (size_needed <= size)
	502	break;
	503
b0d623f7	504	lck_mtx_unlock(&tasks_threads_lock);
91447636 A	505
	506	if (size != 0)
	507	kfree(addr, size);
	508
	509	assert(size_needed > 0);
	510	size = size_needed;
	511
	512	addr = kalloc(size);
	513	if (addr == 0)
	514	return KERN_RESOURCE_SHORTAGE;
	515	}
	516
2d21ac55 A	517	/* OK, have memory and list is locked */
2d21ac55 A	518	thread_list = (thread_t *) addr;
39236c6e	519	for (i = 0, thread = (thread_t)(void *) queue_first(&threads);
2d21ac55	520	!queue_end(&threads, (queue_entry_t) thread);
39236c6e	521	thread = (thread_t)(void *) queue_next(&thread->threads)) {
91447636	522	thread_reference_internal(thread);
2d21ac55	523	thread_list[i++] = thread;
91447636 A	524	}
	525	assert(i <= actual);
	526
b0d623f7	527	lck_mtx_unlock(&tasks_threads_lock);
91447636 A	528
	529	/* calculate maxusage and free thread references */
	530
	531	total = 0;
	532	maxusage = 0;
	533	maxstack = 0;
	534	while (i > 0) {
2d21ac55	535	thread_t threadref = thread_list[--i];
91447636 A	536
	537	if (threadref->kernel_stack != 0)
	538	total++;
	539
	540	thread_deallocate(threadref);
	541	}
	542
	543	if (size != 0)
	544	kfree(addr, size);
	545
	546	*totalp = total;
b0d623f7	547	residentp = spacep = total * round_page(kernel_stack_size);
91447636 A	548	*maxusagep = maxusage;
	549	*maxstackp = maxstack;
	550	return KERN_SUCCESS;
	551
	552	#endif /* MACH_DEBUG */
	553	}
	554
	555	vm_offset_t min_valid_stack_address(void)
	556	{
b0d623f7	557	return (vm_offset_t)vm_map_min(kernel_map);
91447636 A	558	}
	559
	560	vm_offset_t max_valid_stack_address(void)
	561	{
b0d623f7	562	return (vm_offset_t)vm_map_max(kernel_map);
91447636	563	}