X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/8ad349bb6ed4a0be06e34c92be0d98b92e078db4..0a7de7458d150b5d4dffc935ba399be265ef0a1a:/osfmk/kern/stack.c diff --git a/osfmk/kern/stack.c b/osfmk/kern/stack.c index b936ccf13..287458364 100644 --- a/osfmk/kern/stack.c +++ b/osfmk/kern/stack.c @@ -1,31 +1,29 @@ /* - * Copyright (c) 2003-2004 Apple Computer, Inc. All rights reserved. + * Copyright (c) 2003-2007 Apple Inc. All rights reserved. * - * @APPLE_LICENSE_OSREFERENCE_HEADER_START@ - * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. The rights granted to you under the - * License may not be used to create, or enable the creation or - * redistribution of, unlawful or unlicensed copies of an Apple operating - * system, or to circumvent, violate, or enable the circumvention or - * violation of, any terms of an Apple operating system software license - * agreement. + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * - * Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this - * file. + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and * limitations under the License. * - * @APPLE_LICENSE_OSREFERENCE_HEADER_END@ + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ /* * Kernel stack management routines. @@ -36,16 +34,19 @@ #include #include +#include #include #include #include #include #include +#include #include #include #include +#include /* * We allocate stacks from generic kernel VM. @@ -54,56 +55,84 @@ * because stack_alloc_try/thread_invoke operate at splsched. */ -decl_simple_lock_data(static,stack_lock_data) -#define stack_lock() simple_lock(&stack_lock_data) -#define stack_unlock() simple_unlock(&stack_lock_data) +decl_simple_lock_data(static, stack_lock_data) +#define stack_lock() simple_lock(&stack_lock_data, LCK_GRP_NULL) +#define stack_unlock() simple_unlock(&stack_lock_data) + +#define STACK_CACHE_SIZE 2 + +static vm_offset_t stack_free_list; -#define STACK_CACHE_SIZE 2 +static unsigned int stack_free_count, stack_free_hiwat; /* free list count */ +static unsigned int stack_hiwat; +unsigned int stack_total; /* current total count */ +unsigned long long stack_allocs; /* total count of allocations */ -static vm_map_t stack_map; -static vm_offset_t stack_free_list; +static int stack_fake_zone_index = -1; /* index in zone_info array */ -static unsigned int stack_free_count, stack_free_hiwat; /* free list count */ -static unsigned int stack_total, stack_hiwat; /* current total count */ +static unsigned int stack_free_target; +static int stack_free_delta; -static unsigned int stack_free_target; -static int stack_free_delta; +static unsigned int stack_new_count; /* total new stack allocations */ -static unsigned int stack_new_count; /* total new stack allocations */ +static vm_offset_t stack_addr_mask; -static vm_offset_t stack_addr_mask; +unsigned int kernel_stack_pages; +vm_offset_t kernel_stack_size; +vm_offset_t kernel_stack_mask; +vm_offset_t kernel_stack_depth_max; /* * The next field is at the base of the stack, * so the low end is left unsullied. */ -#define stack_next(stack) \ - (*((vm_offset_t *)((stack) + KERNEL_STACK_SIZE) - 1)) +#define stack_next(stack) \ + (*((vm_offset_t *)((stack) + kernel_stack_size) - 1)) + +static inline int +log2(vm_offset_t size) +{ + int result; + for (result = 0; size > 0; result++) { + size >>= 1; + } + return result; +} + +static inline vm_offset_t +roundup_pow2(vm_offset_t size) +{ + return 1UL << (log2(size - 1) + 1); +} + +static vm_offset_t stack_alloc_internal(void); +static void stack_free_stack(vm_offset_t); void stack_init(void) { - vm_offset_t stacks, boundary; - vm_map_offset_t map_addr; - simple_lock_init(&stack_lock_data, 0); - - if (KERNEL_STACK_SIZE < round_page(KERNEL_STACK_SIZE)) - panic("stack_init: stack size %d not a multiple of page size %d\n", KERNEL_STACK_SIZE, PAGE_SIZE); - - for (boundary = PAGE_SIZE; boundary <= KERNEL_STACK_SIZE; ) - boundary <<= 1; - - stack_addr_mask = boundary - 1; - - if (kmem_suballoc(kernel_map, &stacks, (boundary * (2 * THREAD_MAX + 64)), - FALSE, VM_FLAGS_ANYWHERE, &stack_map) != KERN_SUCCESS) - panic("stack_init: kmem_suballoc"); - - map_addr = vm_map_min(stack_map); - if (vm_map_enter(stack_map, &map_addr, vm_map_round_page(PAGE_SIZE), 0, VM_FLAGS_FIXED, - VM_OBJECT_NULL, 0, FALSE, VM_PROT_NONE, VM_PROT_NONE, VM_INHERIT_DEFAULT) != KERN_SUCCESS) - panic("stack_init: vm_map_enter"); + + kernel_stack_pages = KERNEL_STACK_SIZE / PAGE_SIZE; + kernel_stack_size = KERNEL_STACK_SIZE; + kernel_stack_mask = -KERNEL_STACK_SIZE; + kernel_stack_depth_max = 0; + + if (PE_parse_boot_argn("kernel_stack_pages", + &kernel_stack_pages, + sizeof(kernel_stack_pages))) { + kernel_stack_size = kernel_stack_pages * PAGE_SIZE; + printf("stack_init: kernel_stack_pages=%d kernel_stack_size=%p\n", + kernel_stack_pages, (void *) kernel_stack_size); + } + + if (kernel_stack_size < round_page(kernel_stack_size)) { + panic("stack_init: stack size %p not a multiple of page size %d\n", + (void *) kernel_stack_size, PAGE_SIZE); + } + + stack_addr_mask = roundup_pow2(kernel_stack_size) - 1; + kernel_stack_mask = ~stack_addr_mask; } /* @@ -112,37 +141,72 @@ stack_init(void) * Allocate a stack for a thread, may * block. */ -void -stack_alloc( - thread_t thread) -{ - vm_offset_t stack; - spl_t s; - assert(thread->kernel_stack == 0); +static vm_offset_t +stack_alloc_internal(void) +{ + vm_offset_t stack = 0; + spl_t s; + int flags = 0; + kern_return_t kr = KERN_SUCCESS; s = splsched(); stack_lock(); + stack_allocs++; stack = stack_free_list; if (stack != 0) { stack_free_list = stack_next(stack); stack_free_count--; - } - else { - if (++stack_total > stack_hiwat) + } else { + if (++stack_total > stack_hiwat) { stack_hiwat = stack_total; + } stack_new_count++; } stack_free_delta--; stack_unlock(); splx(s); - + if (stack == 0) { - if (kernel_memory_allocate(stack_map, &stack, KERNEL_STACK_SIZE, stack_addr_mask, KMA_KOBJECT) != KERN_SUCCESS) - panic("stack_alloc: kernel_memory_allocate"); + /* + * Request guard pages on either side of the stack. Ask + * kernel_memory_allocate() for two extra pages to account + * for these. + */ + + flags = KMA_GUARD_FIRST | KMA_GUARD_LAST | KMA_KSTACK | KMA_KOBJECT | KMA_ZERO; + kr = kernel_memory_allocate(kernel_map, &stack, + kernel_stack_size + (2 * PAGE_SIZE), + stack_addr_mask, + flags, + VM_KERN_MEMORY_STACK); + if (kr != KERN_SUCCESS) { + panic("stack_alloc: kernel_memory_allocate(size:0x%llx, mask: 0x%llx, flags: 0x%x) failed with %d\n", (uint64_t)(kernel_stack_size + (2 * PAGE_SIZE)), (uint64_t)stack_addr_mask, flags, kr); + } + + /* + * The stack address that comes back is the address of the lower + * guard page. Skip past it to get the actual stack base address. + */ + + stack += PAGE_SIZE; } + return stack; +} + +void +stack_alloc( + thread_t thread) +{ + assert(thread->kernel_stack == 0); + machine_stack_attach(thread, stack_alloc_internal()); +} - machine_stack_attach(thread, stack); +void +stack_handoff(thread_t from, thread_t to) +{ + assert(from == current_thread()); + machine_stack_handoff(from, to); } /* @@ -152,41 +216,36 @@ stack_alloc( */ void stack_free( - thread_t thread) + thread_t thread) { - vm_offset_t stack = machine_stack_detach(thread); + vm_offset_t stack = machine_stack_detach(thread); assert(stack); if (stack != thread->reserved_stack) { - struct stack_cache *cache; - spl_t s; - - s = splsched(); - cache = &PROCESSOR_DATA(current_processor(), stack_cache); - if (cache->count < STACK_CACHE_SIZE) { - stack_next(stack) = cache->free; - cache->free = stack; - cache->count++; - } - else { - stack_lock(); - stack_next(stack) = stack_free_list; - stack_free_list = stack; - if (++stack_free_count > stack_free_hiwat) - stack_free_hiwat = stack_free_count; - stack_free_delta++; - stack_unlock(); - } - splx(s); + stack_free_stack(stack); } } void +stack_free_reserved( + thread_t thread) +{ + if (thread->reserved_stack != thread->kernel_stack) { + stack_free_stack(thread->reserved_stack); + } +} + +static void stack_free_stack( - vm_offset_t stack) + vm_offset_t stack) { - struct stack_cache *cache; - spl_t s; + struct stack_cache *cache; + spl_t s; + +#if KASAN_DEBUG + /* Sanity check - stack should be unpoisoned by now */ + assert(kasan_check_shadow(stack, kernel_stack_size, 0)); +#endif s = splsched(); cache = &PROCESSOR_DATA(current_processor(), stack_cache); @@ -194,13 +253,13 @@ stack_free_stack( stack_next(stack) = cache->free; cache->free = stack; cache->count++; - } - else { + } else { stack_lock(); stack_next(stack) = stack_free_list; stack_free_list = stack; - if (++stack_free_count > stack_free_hiwat) + if (++stack_free_count > stack_free_hiwat) { stack_free_hiwat = stack_free_count; + } stack_free_delta++; stack_unlock(); } @@ -219,18 +278,17 @@ stack_free_stack( */ boolean_t stack_alloc_try( - thread_t thread) + thread_t thread) { - struct stack_cache *cache; - vm_offset_t stack; + struct stack_cache *cache; + vm_offset_t stack; cache = &PROCESSOR_DATA(current_processor(), stack_cache); stack = cache->free; if (stack != 0) { cache->free = stack_next(stack); cache->count--; - } - else { + } else { if (stack_free_list != 0) { stack_lock(); stack = stack_free_list; @@ -245,13 +303,13 @@ stack_alloc_try( if (stack != 0 || (stack = thread->reserved_stack) != 0) { machine_stack_attach(thread, stack); - return (TRUE); + return TRUE; } - return (FALSE); + return FALSE; } -static unsigned int stack_collect_tick, last_stack_tick; +static unsigned int stack_collect_tick, last_stack_tick; /* * stack_collect: @@ -263,9 +321,9 @@ void stack_collect(void) { if (stack_collect_tick != last_stack_tick) { - unsigned int target; - vm_offset_t stack; - spl_t s; + unsigned int target; + vm_offset_t stack; + spl_t s; s = splsched(); stack_lock(); @@ -280,9 +338,27 @@ stack_collect(void) stack_unlock(); splx(s); - if (vm_map_remove(stack_map, vm_map_trunc_page(stack), - vm_map_round_page(stack + KERNEL_STACK_SIZE), VM_MAP_REMOVE_KUNWIRE) != KERN_SUCCESS) + /* + * Get the stack base address, then decrement by one page + * to account for the lower guard page. Add two extra pages + * to the size to account for the guard pages on both ends + * that were originally requested when the stack was allocated + * back in stack_alloc(). + */ + + stack = (vm_offset_t)vm_map_trunc_page( + stack, + VM_MAP_PAGE_MASK(kernel_map)); + stack -= PAGE_SIZE; + if (vm_map_remove( + kernel_map, + stack, + stack + kernel_stack_size + (2 * PAGE_SIZE), + VM_MAP_REMOVE_KUNWIRE) + != KERN_SUCCESS) { panic("stack_collect: vm_map_remove"); + } + stack = 0; s = splsched(); stack_lock(); @@ -309,18 +385,18 @@ stack_collect(void) */ void compute_stack_target( -__unused void *arg) + __unused void *arg) { - spl_t s; + spl_t s; s = splsched(); stack_lock(); - if (stack_free_target > 5) + if (stack_free_target > 5) { stack_free_target = (4 * stack_free_target) / 5; - else - if (stack_free_target > 0) + } else if (stack_free_target > 0) { stack_free_target--; + } stack_free_target += (stack_free_delta >= 0)? stack_free_delta: -stack_free_delta; @@ -332,14 +408,23 @@ __unused void *arg) } void -stack_fake_zone_info(int *count, vm_size_t *cur_size, vm_size_t *max_size, vm_size_t *elem_size, - vm_size_t *alloc_size, int *collectable, int *exhaustable) +stack_fake_zone_init(int zone_index) +{ + stack_fake_zone_index = zone_index; +} + +void +stack_fake_zone_info(int *count, + vm_size_t *cur_size, vm_size_t *max_size, vm_size_t *elem_size, vm_size_t *alloc_size, + uint64_t *sum_size, int *collectable, int *exhaustable, int *caller_acct) { - unsigned int total, hiwat, free; - spl_t s; + unsigned int total, hiwat, free; + unsigned long long all; + spl_t s; s = splsched(); stack_lock(); + all = stack_allocs; total = stack_total; hiwat = stack_hiwat; free = stack_free_count; @@ -347,21 +432,24 @@ stack_fake_zone_info(int *count, vm_size_t *cur_size, vm_size_t *max_size, vm_si splx(s); *count = total - free; - *cur_size = KERNEL_STACK_SIZE * total; - *max_size = KERNEL_STACK_SIZE * hiwat; - *elem_size = KERNEL_STACK_SIZE; - *alloc_size = KERNEL_STACK_SIZE; + *cur_size = kernel_stack_size * total; + *max_size = kernel_stack_size * hiwat; + *elem_size = kernel_stack_size; + *alloc_size = kernel_stack_size; + *sum_size = all * kernel_stack_size; + *collectable = 1; *exhaustable = 0; + *caller_acct = 1; } /* OBSOLETE */ -void stack_privilege( - thread_t thread); +void stack_privilege( + thread_t thread); void stack_privilege( - __unused thread_t thread) + __unused thread_t thread) { /* OBSOLETE */ } @@ -371,75 +459,74 @@ stack_privilege( */ kern_return_t processor_set_stack_usage( - processor_set_t pset, - unsigned int *totalp, - vm_size_t *spacep, - vm_size_t *residentp, - vm_size_t *maxusagep, - vm_offset_t *maxstackp) + processor_set_t pset, + unsigned int *totalp, + vm_size_t *spacep, + vm_size_t *residentp, + vm_size_t *maxusagep, + vm_offset_t *maxstackp) { #if !MACH_DEBUG - return KERN_NOT_SUPPORTED; + return KERN_NOT_SUPPORTED; #else unsigned int total; vm_size_t maxusage; vm_offset_t maxstack; - register thread_t *threads; - register thread_t thread; + thread_t *thread_list; + thread_t thread; - unsigned int actual; /* this many things */ + unsigned int actual; /* this many things */ unsigned int i; vm_size_t size, size_needed; void *addr; - if (pset == PROCESSOR_SET_NULL) + if (pset == PROCESSOR_SET_NULL || pset != &pset0) { return KERN_INVALID_ARGUMENT; + } - size = 0; addr = 0; + size = 0; + addr = NULL; for (;;) { - pset_lock(pset); - if (!pset->active) { - pset_unlock(pset); - return KERN_INVALID_ARGUMENT; - } + lck_mtx_lock(&tasks_threads_lock); - actual = pset->thread_count; + actual = threads_count; /* do we have the memory we need? */ size_needed = actual * sizeof(thread_t); - if (size_needed <= size) + if (size_needed <= size) { break; + } - /* unlock the pset and allocate more memory */ - pset_unlock(pset); + lck_mtx_unlock(&tasks_threads_lock); - if (size != 0) + if (size != 0) { kfree(addr, size); + } assert(size_needed > 0); size = size_needed; addr = kalloc(size); - if (addr == 0) + if (addr == 0) { return KERN_RESOURCE_SHORTAGE; + } } - /* OK, have memory and the processor_set is locked & active */ - threads = (thread_t *) addr; - for (i = 0, thread = (thread_t) queue_first(&pset->threads); - !queue_end(&pset->threads, (queue_entry_t) thread); - thread = (thread_t) queue_next(&thread->pset_threads)) { + /* OK, have memory and list is locked */ + thread_list = (thread_t *) addr; + for (i = 0, thread = (thread_t)(void *) queue_first(&threads); + !queue_end(&threads, (queue_entry_t) thread); + thread = (thread_t)(void *) queue_next(&thread->threads)) { thread_reference_internal(thread); - threads[i++] = thread; + thread_list[i++] = thread; } assert(i <= actual); - /* can unlock processor set now that we have the thread refs */ - pset_unlock(pset); + lck_mtx_unlock(&tasks_threads_lock); /* calculate maxusage and free thread references */ @@ -447,32 +534,36 @@ processor_set_stack_usage( maxusage = 0; maxstack = 0; while (i > 0) { - thread_t threadref = threads[--i]; + thread_t threadref = thread_list[--i]; - if (threadref->kernel_stack != 0) + if (threadref->kernel_stack != 0) { total++; + } thread_deallocate(threadref); } - if (size != 0) + if (size != 0) { kfree(addr, size); + } *totalp = total; - *residentp = *spacep = total * round_page(KERNEL_STACK_SIZE); + *residentp = *spacep = total * round_page(kernel_stack_size); *maxusagep = maxusage; *maxstackp = maxstack; return KERN_SUCCESS; -#endif /* MACH_DEBUG */ +#endif /* MACH_DEBUG */ } -vm_offset_t min_valid_stack_address(void) +vm_offset_t +min_valid_stack_address(void) { - return vm_map_min(stack_map); + return (vm_offset_t)vm_map_min(kernel_map); } -vm_offset_t max_valid_stack_address(void) +vm_offset_t +max_valid_stack_address(void) { - return vm_map_max(stack_map); + return (vm_offset_t)vm_map_max(kernel_map); }