xnu-4570.71.2.tar.gz

[apple/xnu.git] / osfmk / kern / stack.c

diff --git a/osfmk/kern/stack.c b/osfmk/kern/stack.c
index ee1200b1057d189011f8f7d7c3fcb4893c029adf..18db3f24b1cb84c8b548accbe34f2b392cabdf24 100644 (file)
--- a/osfmk/kern/stack.c
+++ b/osfmk/kern/stack.c
@@ -1,23 +1,29 @@
 /*
 /*

- * Copyright (c) 2003-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2007 Apple Inc. All rights reserved.

  *
  *

- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@

  * 
  * 

- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.

  * 
  * 

- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER

  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,

- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.

  * 
  * 

- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@

  */
 /*
  *     Kernel stack management routines.
  */
 /*
  *     Kernel stack management routines.


@@ -33,11 +39,13 @@
 #include <kern/thread.h>
 #include <kern/zalloc.h>
 #include <kern/kalloc.h>
 #include <kern/thread.h>
 #include <kern/zalloc.h>
 #include <kern/kalloc.h>

+#include <kern/ledger.h>

 
 #include <vm/vm_map.h>
 #include <vm/vm_kern.h>
 
 #include <mach_debug.h>
 
 #include <vm/vm_map.h>
 #include <vm/vm_kern.h>
 
 #include <mach_debug.h>

+#include <san/kasan.h>

 
 /*
  *     We allocate stacks from generic kernel VM.
 
 /*
  *     We allocate stacks from generic kernel VM.


@@ -52,11 +60,14 @@ decl_simple_lock_data(static,stack_lock_data)
 
 #define STACK_CACHE_SIZE       2
 
 
 #define STACK_CACHE_SIZE       2
 

-static vm_map_t                        stack_map;

 static vm_offset_t             stack_free_list;
 
 static unsigned int            stack_free_count, stack_free_hiwat;             /* free list count */
 static vm_offset_t             stack_free_list;
 
 static unsigned int            stack_free_count, stack_free_hiwat;             /* free list count */

-static unsigned int            stack_total, stack_hiwat;                               /* current total count */
+static unsigned int            stack_hiwat;
+unsigned int                   stack_total;                            /* current total count */
+unsigned long long             stack_allocs;                           /* total count of allocations */
+
+static int                     stack_fake_zone_index = -1;     /* index in zone_info array */

 
 static unsigned int            stack_free_target;
 static int                             stack_free_delta;
 
 static unsigned int            stack_free_target;
 static int                             stack_free_delta;


@@ -65,37 +76,60 @@ static unsigned int         stack_new_count;                                                /* total new stack allocations */
 
 static vm_offset_t             stack_addr_mask;
 
 
 static vm_offset_t             stack_addr_mask;
 

+unsigned int                   kernel_stack_pages;
+vm_offset_t                    kernel_stack_size;
+vm_offset_t                    kernel_stack_mask;
+vm_offset_t                    kernel_stack_depth_max;
+

 /*
  *     The next field is at the base of the stack,
  *     so the low end is left unsullied.
  */
 #define stack_next(stack)      \
 /*
  *     The next field is at the base of the stack,
  *     so the low end is left unsullied.
  */
 #define stack_next(stack)      \

-                       (*((vm_offset_t *)((stack) + KERNEL_STACK_SIZE) - 1))
+       (*((vm_offset_t *)((stack) + kernel_stack_size) - 1))
+
+static inline int
+log2(vm_offset_t size)
+{
+       int     result;
+       for (result = 0; size > 0; result++)
+               size >>= 1;
+       return result;
+}
+
+static inline vm_offset_t
+roundup_pow2(vm_offset_t size)
+{
+       return 1UL << (log2(size - 1) + 1); 
+}
+
+static vm_offset_t stack_alloc_internal(void);
+static void stack_free_stack(vm_offset_t);

 
 void
 stack_init(void)
 {
 
 void
 stack_init(void)
 {

-       vm_offset_t                     stacks, boundary;
-       vm_map_offset_t         map_addr;
-

        simple_lock_init(&stack_lock_data, 0);
        
        simple_lock_init(&stack_lock_data, 0);
        

-       if (KERNEL_STACK_SIZE < round_page(KERNEL_STACK_SIZE))
-               panic("stack_init: stack size %d not a multiple of page size %d\n",     KERNEL_STACK_SIZE, PAGE_SIZE);
-       
-       for (boundary = PAGE_SIZE; boundary <= KERNEL_STACK_SIZE; )
-               boundary <<= 1;
-
-       stack_addr_mask = boundary - 1;
-
-       if (kmem_suballoc(kernel_map, &stacks, (boundary * (2 * THREAD_MAX + 64)),
-                                                               FALSE, VM_FLAGS_ANYWHERE, &stack_map) != KERN_SUCCESS)
-               panic("stack_init: kmem_suballoc");
+       kernel_stack_pages = KERNEL_STACK_SIZE / PAGE_SIZE;
+       kernel_stack_size = KERNEL_STACK_SIZE;
+       kernel_stack_mask = -KERNEL_STACK_SIZE;
+       kernel_stack_depth_max = 0;
+
+       if (PE_parse_boot_argn("kernel_stack_pages",
+                              &kernel_stack_pages,
+                              sizeof (kernel_stack_pages))) {
+               kernel_stack_size = kernel_stack_pages * PAGE_SIZE;
+               printf("stack_init: kernel_stack_pages=%d kernel_stack_size=%p\n",
+                       kernel_stack_pages, (void *) kernel_stack_size);
+       }

 
 

-       map_addr = vm_map_min(stack_map);
-       if (vm_map_enter(stack_map, &map_addr, vm_map_round_page(PAGE_SIZE), 0, (VM_MAKE_TAG(VM_MEMORY_STACK) | VM_FLAGS_FIXED),
-                                               VM_OBJECT_NULL, 0, FALSE, VM_PROT_NONE, VM_PROT_NONE, VM_INHERIT_DEFAULT) != KERN_SUCCESS)
-               panic("stack_init: vm_map_enter");
+       if (kernel_stack_size < round_page(kernel_stack_size))
+               panic("stack_init: stack size %p not a multiple of page size %d\n",
+                       (void *) kernel_stack_size, PAGE_SIZE);
+       
+       stack_addr_mask = roundup_pow2(kernel_stack_size) - 1;
+       kernel_stack_mask = ~stack_addr_mask;

 }
 
 /*
 }
 
 /*


@@ -104,17 +138,18 @@ stack_init(void)
  *     Allocate a stack for a thread, may
  *     block.
  */
  *     Allocate a stack for a thread, may
  *     block.
  */

-void
-stack_alloc(
-       thread_t        thread)
+
+static vm_offset_t 
+stack_alloc_internal(void)

 {
 {

-       vm_offset_t             stack;
+       vm_offset_t             stack = 0;

        spl_t                   s;
        spl_t                   s;

-
-       assert(thread->kernel_stack == 0);
+       int                     flags = 0;
+       kern_return_t           kr = KERN_SUCCESS;

 
        s = splsched();
        stack_lock();
 
        s = splsched();
        stack_lock();

+       stack_allocs++;

        stack = stack_free_list;
        if (stack != 0) {
                stack_free_list = stack_next(stack);
        stack = stack_free_list;
        if (stack != 0) {
                stack_free_list = stack_next(stack);


@@ -130,11 +165,47 @@ stack_alloc(
        splx(s);
                
        if (stack == 0) {
        splx(s);
                
        if (stack == 0) {

-               if (kernel_memory_allocate(stack_map, &stack, KERNEL_STACK_SIZE, stack_addr_mask, KMA_KOBJECT) != KERN_SUCCESS)
-                       panic("stack_alloc: kernel_memory_allocate");
+
+               /*
+                * Request guard pages on either side of the stack.  Ask
+                * kernel_memory_allocate() for two extra pages to account
+                * for these.
+                */
+
+               flags = KMA_GUARD_FIRST | KMA_GUARD_LAST | KMA_KSTACK | KMA_KOBJECT;
+               kr = kernel_memory_allocate(kernel_map, &stack,
+                                          kernel_stack_size + (2*PAGE_SIZE),
+                                          stack_addr_mask,
+                                          flags,
+                                          VM_KERN_MEMORY_STACK);
+               if (kr != KERN_SUCCESS) {
+                       panic("stack_alloc: kernel_memory_allocate(size:0x%llx, mask: 0x%llx, flags: 0x%x) failed with %d\n", (uint64_t)(kernel_stack_size + (2*PAGE_SIZE)), (uint64_t)stack_addr_mask, flags, kr);
+               }
+
+               /*
+                * The stack address that comes back is the address of the lower
+                * guard page.  Skip past it to get the actual stack base address.
+                */
+
+               stack += PAGE_SIZE;

        }
        }

+       return stack;
+}
+
+void
+stack_alloc(
+       thread_t        thread)
+{
+
+       assert(thread->kernel_stack == 0);
+       machine_stack_attach(thread, stack_alloc_internal());
+}

 
 

-       machine_stack_attach(thread, stack);
+void
+stack_handoff(thread_t from, thread_t to)
+{
+       assert(from == current_thread());
+       machine_stack_handoff(from, to);

 }
 
 /*
 }
 
 /*


@@ -148,32 +219,30 @@ stack_free(
 {
     vm_offset_t                stack = machine_stack_detach(thread);
 
 {
     vm_offset_t                stack = machine_stack_detach(thread);
 

+#if KASAN
+       kasan_unpoison_stack(stack, kernel_stack_size);
+       kasan_unpoison_fakestack(thread);
+#endif
+

        assert(stack);
        if (stack != thread->reserved_stack) {
        assert(stack);
        if (stack != thread->reserved_stack) {

-               struct stack_cache      *cache;
-               spl_t                           s;
-
-               s = splsched();
-               cache = &PROCESSOR_DATA(current_processor(), stack_cache);
-               if (cache->count < STACK_CACHE_SIZE) {
-                       stack_next(stack) = cache->free;
-                       cache->free = stack;
-                       cache->count++;
-               }
-               else {
-                       stack_lock();
-                       stack_next(stack) = stack_free_list;
-                       stack_free_list = stack;
-                       if (++stack_free_count > stack_free_hiwat)
-                               stack_free_hiwat = stack_free_count;
-                       stack_free_delta++;
-                       stack_unlock();
-               }
-               splx(s);
+               stack_free_stack(stack);

        }
 }
 
 void
        }
 }
 
 void

+stack_free_reserved(
+       thread_t        thread)
+{
+       if (thread->reserved_stack != thread->kernel_stack) {
+#if KASAN
+               kasan_unpoison_stack(thread->reserved_stack, kernel_stack_size);
+#endif
+               stack_free_stack(thread->reserved_stack);
+       }
+}
+
+static void

 stack_free_stack(
        vm_offset_t             stack)
 {
 stack_free_stack(
        vm_offset_t             stack)
 {


@@ -272,9 +341,26 @@ stack_collect(void)
                        stack_unlock();
                        splx(s);
 
                        stack_unlock();
                        splx(s);
 

-                       if (vm_map_remove(stack_map, vm_map_trunc_page(stack),
-                                                               vm_map_round_page(stack + KERNEL_STACK_SIZE), VM_MAP_REMOVE_KUNWIRE) != KERN_SUCCESS)
+                       /*
+                        * Get the stack base address, then decrement by one page
+                        * to account for the lower guard page.  Add two extra pages
+                        * to the size to account for the guard pages on both ends
+                        * that were originally requested when the stack was allocated
+                        * back in stack_alloc().
+                        */
+
+                       stack = (vm_offset_t)vm_map_trunc_page(
+                               stack,
+                               VM_MAP_PAGE_MASK(kernel_map));
+                       stack -= PAGE_SIZE;
+                       if (vm_map_remove(
+                                   kernel_map,
+                                   stack,
+                                   stack + kernel_stack_size+(2*PAGE_SIZE),
+                                   VM_MAP_REMOVE_KUNWIRE)
+                           != KERN_SUCCESS)

                                panic("stack_collect: vm_map_remove");
                                panic("stack_collect: vm_map_remove");

+                       stack = 0;

 
                        s = splsched();
                        stack_lock();
 
                        s = splsched();
                        stack_lock();


@@ -324,14 +410,23 @@ __unused void             *arg)
 }
 
 void
 }
 
 void

-stack_fake_zone_info(int *count, vm_size_t *cur_size, vm_size_t *max_size, vm_size_t *elem_size,
-                    vm_size_t *alloc_size, int *collectable, int *exhaustable)
+stack_fake_zone_init(int zone_index)
+{
+       stack_fake_zone_index = zone_index;
+}
+
+void
+stack_fake_zone_info(int *count, 
+                    vm_size_t *cur_size, vm_size_t *max_size, vm_size_t *elem_size, vm_size_t *alloc_size,
+                    uint64_t *sum_size, int *collectable, int *exhaustable, int *caller_acct)

 {
        unsigned int    total, hiwat, free;
 {
        unsigned int    total, hiwat, free;

+       unsigned long long all;

        spl_t                   s;
 
        s = splsched();
        stack_lock();
        spl_t                   s;
 
        s = splsched();
        stack_lock();

+       all = stack_allocs;

        total = stack_total;
        hiwat = stack_hiwat;
        free = stack_free_count;
        total = stack_total;
        hiwat = stack_hiwat;
        free = stack_free_count;


@@ -339,12 +434,15 @@ stack_fake_zone_info(int *count, vm_size_t *cur_size, vm_size_t *max_size, vm_si
        splx(s);
 
        *count      = total - free;
        splx(s);
 
        *count      = total - free;

-       *cur_size   = KERNEL_STACK_SIZE * total;
-       *max_size   = KERNEL_STACK_SIZE * hiwat;
-       *elem_size  = KERNEL_STACK_SIZE;
-       *alloc_size = KERNEL_STACK_SIZE;
+       *cur_size   = kernel_stack_size * total;
+       *max_size   = kernel_stack_size * hiwat;
+       *elem_size  = kernel_stack_size;
+       *alloc_size = kernel_stack_size;
+       *sum_size = all * kernel_stack_size;
+

        *collectable = 1;
        *exhaustable = 0;
        *collectable = 1;
        *exhaustable = 0;

+       *caller_acct = 1;

 }
 
 /* OBSOLETE */
 }
 
 /* OBSOLETE */


@@ -377,8 +475,8 @@ processor_set_stack_usage(
        vm_size_t maxusage;
        vm_offset_t maxstack;
 
        vm_size_t maxusage;
        vm_offset_t maxstack;
 

-       register thread_t *threads;
-       register thread_t thread;
+       thread_t *thread_list;
+       thread_t thread;

 
        unsigned int actual;    /* this many things */
        unsigned int i;
 
        unsigned int actual;    /* this many things */
        unsigned int i;


@@ -386,19 +484,16 @@ processor_set_stack_usage(
        vm_size_t size, size_needed;
        void *addr;
 
        vm_size_t size, size_needed;
        void *addr;
 

-       if (pset == PROCESSOR_SET_NULL)
+       if (pset == PROCESSOR_SET_NULL || pset != &pset0)

                return KERN_INVALID_ARGUMENT;
 
                return KERN_INVALID_ARGUMENT;
 

-       size = 0; addr = 0;
+       size = 0;
+       addr = NULL;

 
        for (;;) {
 
        for (;;) {

-               pset_lock(pset);
-               if (!pset->active) {
-                       pset_unlock(pset);
-                       return KERN_INVALID_ARGUMENT;
-               }
+               lck_mtx_lock(&tasks_threads_lock);

 
 

-               actual = pset->thread_count;
+               actual = threads_count;

 
                /* do we have the memory we need? */
 
 
                /* do we have the memory we need? */
 


@@ -406,8 +501,7 @@ processor_set_stack_usage(
                if (size_needed <= size)
                        break;
 
                if (size_needed <= size)
                        break;
 

-               /* unlock the pset and allocate more memory */
-               pset_unlock(pset);
+               lck_mtx_unlock(&tasks_threads_lock);

 
                if (size != 0)
                        kfree(addr, size);
 
                if (size != 0)
                        kfree(addr, size);


@@ -420,18 +514,17 @@ processor_set_stack_usage(
                        return KERN_RESOURCE_SHORTAGE;
        }
 
                        return KERN_RESOURCE_SHORTAGE;
        }
 

-       /* OK, have memory and the processor_set is locked & active */
-       threads = (thread_t *) addr;
-       for (i = 0, thread = (thread_t) queue_first(&pset->threads);
-                                       !queue_end(&pset->threads, (queue_entry_t) thread);
-                                       thread = (thread_t) queue_next(&thread->pset_threads)) {
+       /* OK, have memory and list is locked */
+       thread_list = (thread_t *) addr;
+       for (i = 0, thread = (thread_t)(void *) queue_first(&threads);
+                                       !queue_end(&threads, (queue_entry_t) thread);
+                                       thread = (thread_t)(void *) queue_next(&thread->threads)) {

                thread_reference_internal(thread);
                thread_reference_internal(thread);

-               threads[i++] = thread;
+               thread_list[i++] = thread;

        }
        assert(i <= actual);
 
        }
        assert(i <= actual);
 

-       /* can unlock processor set now that we have the thread refs */
-       pset_unlock(pset);
+       lck_mtx_unlock(&tasks_threads_lock);

 
        /* calculate maxusage and free thread references */
 
 
        /* calculate maxusage and free thread references */
 


@@ -439,7 +532,7 @@ processor_set_stack_usage(
        maxusage = 0;
        maxstack = 0;
        while (i > 0) {
        maxusage = 0;
        maxstack = 0;
        while (i > 0) {

-               thread_t threadref = threads[--i];
+               thread_t threadref = thread_list[--i];

 
                if (threadref->kernel_stack != 0)
                        total++;
 
                if (threadref->kernel_stack != 0)
                        total++;


@@ -451,7 +544,7 @@ processor_set_stack_usage(
                kfree(addr, size);
 
        *totalp = total;
                kfree(addr, size);
 
        *totalp = total;

-       *residentp = *spacep = total * round_page(KERNEL_STACK_SIZE);
+       *residentp = *spacep = total * round_page(kernel_stack_size);

        *maxusagep = maxusage;
        *maxstackp = maxstack;
        return KERN_SUCCESS;
        *maxusagep = maxusage;
        *maxstackp = maxstack;
        return KERN_SUCCESS;


@@ -461,10 +554,10 @@ processor_set_stack_usage(
 
 vm_offset_t min_valid_stack_address(void)
 {
 
 vm_offset_t min_valid_stack_address(void)
 {

-       return vm_map_min(stack_map);
+       return (vm_offset_t)vm_map_min(kernel_map);

 }
 
 vm_offset_t max_valid_stack_address(void)
 {
 }
 
 vm_offset_t max_valid_stack_address(void)
 {

-       return vm_map_max(stack_map);
+       return (vm_offset_t)vm_map_max(kernel_map);

 }
 }