xnu-3789.31.2.tar.gz

[apple/xnu.git] / osfmk / kern / stack.c

diff --git a/osfmk/kern/stack.c b/osfmk/kern/stack.c
index a5912256932b7222172b63e616ae5dfd9f24a259..0cb79328666e504b9883579a07d7177a95f9b34a 100644 (file)
--- a/osfmk/kern/stack.c
+++ b/osfmk/kern/stack.c
@@ -39,6 +39,7 @@
 #include <kern/thread.h>
 #include <kern/zalloc.h>
 #include <kern/kalloc.h>
 #include <kern/thread.h>
 #include <kern/zalloc.h>
 #include <kern/kalloc.h>

+#include <kern/ledger.h>

 
 #include <vm/vm_map.h>
 #include <vm/vm_kern.h>
 
 #include <vm/vm_map.h>
 #include <vm/vm_kern.h>


@@ -63,6 +64,9 @@ static vm_offset_t            stack_free_list;
 static unsigned int            stack_free_count, stack_free_hiwat;             /* free list count */
 static unsigned int            stack_hiwat;
 unsigned int                   stack_total;                            /* current total count */
 static unsigned int            stack_free_count, stack_free_hiwat;             /* free list count */
 static unsigned int            stack_hiwat;
 unsigned int                   stack_total;                            /* current total count */

+unsigned long long             stack_allocs;                           /* total count of allocations */
+
+static int                     stack_fake_zone_index = -1;     /* index in zone_info array */

 
 static unsigned int            stack_free_target;
 static int                             stack_free_delta;
 
 static unsigned int            stack_free_target;
 static int                             stack_free_delta;


@@ -71,10 +75,29 @@ static unsigned int         stack_new_count;                                                /* total new stack allocations */
 
 static vm_offset_t             stack_addr_mask;
 
 
 static vm_offset_t             stack_addr_mask;
 

-unsigned int                   kernel_stack_pages = KERNEL_STACK_SIZE / PAGE_SIZE;
-vm_offset_t                    kernel_stack_size = KERNEL_STACK_SIZE;
-vm_offset_t                    kernel_stack_mask = -KERNEL_STACK_SIZE;
-vm_offset_t                    kernel_stack_depth_max = 0;
+unsigned int                   kernel_stack_pages;
+vm_offset_t                    kernel_stack_size;
+vm_offset_t                    kernel_stack_mask;
+vm_offset_t                    kernel_stack_depth_max;
+
+static inline void
+STACK_ZINFO_PALLOC(thread_t thread)
+{
+       ledger_credit(thread->t_ledger, task_ledgers.tkm_private, kernel_stack_size);
+}
+
+static inline void
+STACK_ZINFO_PFREE(thread_t thread)
+{
+       ledger_debit(thread->t_ledger, task_ledgers.tkm_private, kernel_stack_size);
+}
+
+static inline void
+STACK_ZINFO_HANDOFF(thread_t from, thread_t to)
+{
+       ledger_debit(from->t_ledger, task_ledgers.tkm_private, kernel_stack_size);
+       ledger_credit(to->t_ledger, task_ledgers.tkm_private, kernel_stack_size);
+}

 
 /*
  *     The next field is at the base of the stack,
 
 /*
  *     The next field is at the base of the stack,


@@ -98,11 +121,19 @@ roundup_pow2(vm_offset_t size)
        return 1UL << (log2(size - 1) + 1); 
 }
 
        return 1UL << (log2(size - 1) + 1); 
 }
 

+static vm_offset_t stack_alloc_internal(void);
+static void stack_free_stack(vm_offset_t);
+

 void
 stack_init(void)
 {
        simple_lock_init(&stack_lock_data, 0);
        
 void
 stack_init(void)
 {
        simple_lock_init(&stack_lock_data, 0);
        

+       kernel_stack_pages = KERNEL_STACK_SIZE / PAGE_SIZE;
+       kernel_stack_size = KERNEL_STACK_SIZE;
+       kernel_stack_mask = -KERNEL_STACK_SIZE;
+       kernel_stack_depth_max = 0;
+

        if (PE_parse_boot_argn("kernel_stack_pages",
                               &kernel_stack_pages,
                               sizeof (kernel_stack_pages))) {
        if (PE_parse_boot_argn("kernel_stack_pages",
                               &kernel_stack_pages,
                               sizeof (kernel_stack_pages))) {


@@ -125,18 +156,17 @@ stack_init(void)
  *     Allocate a stack for a thread, may
  *     block.
  */
  *     Allocate a stack for a thread, may
  *     block.
  */

-void
-stack_alloc(
-       thread_t        thread)
+
+static vm_offset_t 
+stack_alloc_internal(void)

 {
        vm_offset_t             stack;
        spl_t                   s;
        int                     guard_flags;
 
 {
        vm_offset_t             stack;
        spl_t                   s;
        int                     guard_flags;
 

-       assert(thread->kernel_stack == 0);
-

        s = splsched();
        stack_lock();
        s = splsched();
        stack_lock();

+       stack_allocs++;

        stack = stack_free_list;
        if (stack != 0) {
                stack_free_list = stack_next(stack);
        stack = stack_free_list;
        if (stack != 0) {
                stack_free_list = stack_next(stack);


@@ -163,7 +193,8 @@ stack_alloc(
                if (kernel_memory_allocate(kernel_map, &stack,
                                           kernel_stack_size + (2*PAGE_SIZE),
                                           stack_addr_mask,
                if (kernel_memory_allocate(kernel_map, &stack,
                                           kernel_stack_size + (2*PAGE_SIZE),
                                           stack_addr_mask,

-                                          KMA_KOBJECT | guard_flags)
+                                          KMA_KSTACK | KMA_KOBJECT | guard_flags,
+                                          VM_KERN_MEMORY_STACK)

                    != KERN_SUCCESS)
                        panic("stack_alloc: kernel_memory_allocate");
 
                    != KERN_SUCCESS)
                        panic("stack_alloc: kernel_memory_allocate");
 


@@ -174,8 +205,25 @@ stack_alloc(
 
                stack += PAGE_SIZE;
        }
 
                stack += PAGE_SIZE;
        }

+       return stack;
+}
+
+void
+stack_alloc(
+       thread_t        thread)
+{
+
+       assert(thread->kernel_stack == 0);
+       machine_stack_attach(thread, stack_alloc_internal());
+       STACK_ZINFO_PALLOC(thread);
+}

 
 

-       machine_stack_attach(thread, stack);
+void
+stack_handoff(thread_t from, thread_t to)
+{
+       assert(from == current_thread());
+       machine_stack_handoff(from, to);
+       STACK_ZINFO_HANDOFF(from, to);

 }
 
 /*
 }
 
 /*


@@ -190,11 +238,23 @@ stack_free(
     vm_offset_t                stack = machine_stack_detach(thread);
 
        assert(stack);
     vm_offset_t                stack = machine_stack_detach(thread);
 
        assert(stack);

-       if (stack != thread->reserved_stack)
+       if (stack != thread->reserved_stack) {
+               STACK_ZINFO_PFREE(thread);

                stack_free_stack(stack);
                stack_free_stack(stack);

+       }

 }
 
 void
 }
 
 void

+stack_free_reserved(
+       thread_t        thread)
+{
+       if (thread->reserved_stack != thread->kernel_stack) {
+               stack_free_stack(thread->reserved_stack);
+               STACK_ZINFO_PFREE(thread);
+       }
+}
+
+static void

 stack_free_stack(
        vm_offset_t             stack)
 {
 stack_free_stack(
        vm_offset_t             stack)
 {


@@ -240,6 +300,7 @@ stack_alloc_try(
        cache = &PROCESSOR_DATA(current_processor(), stack_cache);
        stack = cache->free;
        if (stack != 0) {
        cache = &PROCESSOR_DATA(current_processor(), stack_cache);
        stack = cache->free;
        if (stack != 0) {

+               STACK_ZINFO_PALLOC(thread);

                cache->free = stack_next(stack);
                cache->count--;
        }
                cache->free = stack_next(stack);
                cache->count--;
        }


@@ -248,6 +309,7 @@ stack_alloc_try(
                        stack_lock();
                        stack = stack_free_list;
                        if (stack != 0) {
                        stack_lock();
                        stack = stack_free_list;
                        if (stack != 0) {

+                               STACK_ZINFO_PALLOC(thread);

                                stack_free_list = stack_next(stack);
                                stack_free_count--;
                                stack_free_delta--;
                                stack_free_list = stack_next(stack);
                                stack_free_count--;
                                stack_free_delta--;


@@ -301,7 +363,9 @@ stack_collect(void)
                         * back in stack_alloc().
                         */
 
                         * back in stack_alloc().
                         */
 

-                       stack = (vm_offset_t)vm_map_trunc_page(stack);
+                       stack = (vm_offset_t)vm_map_trunc_page(
+                               stack,
+                               VM_MAP_PAGE_MASK(kernel_map));

                        stack -= PAGE_SIZE;
                        if (vm_map_remove(
                                    kernel_map,
                        stack -= PAGE_SIZE;
                        if (vm_map_remove(
                                    kernel_map,


@@ -360,14 +424,23 @@ __unused void             *arg)
 }
 
 void
 }
 
 void

-stack_fake_zone_info(int *count, vm_size_t *cur_size, vm_size_t *max_size, vm_size_t *elem_size,
-                    vm_size_t *alloc_size, int *collectable, int *exhaustable)
+stack_fake_zone_init(int zone_index)
+{
+       stack_fake_zone_index = zone_index;
+}
+
+void
+stack_fake_zone_info(int *count, 
+                    vm_size_t *cur_size, vm_size_t *max_size, vm_size_t *elem_size, vm_size_t *alloc_size,
+                    uint64_t *sum_size, int *collectable, int *exhaustable, int *caller_acct)

 {
        unsigned int    total, hiwat, free;
 {
        unsigned int    total, hiwat, free;

+       unsigned long long all;

        spl_t                   s;
 
        s = splsched();
        stack_lock();
        spl_t                   s;
 
        s = splsched();
        stack_lock();

+       all = stack_allocs;

        total = stack_total;
        hiwat = stack_hiwat;
        free = stack_free_count;
        total = stack_total;
        hiwat = stack_hiwat;
        free = stack_free_count;


@@ -379,8 +452,11 @@ stack_fake_zone_info(int *count, vm_size_t *cur_size, vm_size_t *max_size, vm_si
        *max_size   = kernel_stack_size * hiwat;
        *elem_size  = kernel_stack_size;
        *alloc_size = kernel_stack_size;
        *max_size   = kernel_stack_size * hiwat;
        *elem_size  = kernel_stack_size;
        *alloc_size = kernel_stack_size;

+       *sum_size = all * kernel_stack_size;
+

        *collectable = 1;
        *exhaustable = 0;
        *collectable = 1;
        *exhaustable = 0;

+       *caller_acct = 1;

 }
 
 /* OBSOLETE */
 }
 
 /* OBSOLETE */


@@ -413,8 +489,8 @@ processor_set_stack_usage(
        vm_size_t maxusage;
        vm_offset_t maxstack;
 
        vm_size_t maxusage;
        vm_offset_t maxstack;
 

-       register thread_t *thread_list;
-       register thread_t thread;
+       thread_t *thread_list;
+       thread_t thread;

 
        unsigned int actual;    /* this many things */
        unsigned int i;
 
        unsigned int actual;    /* this many things */
        unsigned int i;


@@ -454,9 +530,9 @@ processor_set_stack_usage(
 
        /* OK, have memory and list is locked */
        thread_list = (thread_t *) addr;
 
        /* OK, have memory and list is locked */
        thread_list = (thread_t *) addr;

-       for (i = 0, thread = (thread_t) queue_first(&threads);
+       for (i = 0, thread = (thread_t)(void *) queue_first(&threads);

                                        !queue_end(&threads, (queue_entry_t) thread);
                                        !queue_end(&threads, (queue_entry_t) thread);

-                                       thread = (thread_t) queue_next(&thread->threads)) {
+                                       thread = (thread_t)(void *) queue_next(&thread->threads)) {

                thread_reference_internal(thread);
                thread_list[i++] = thread;
        }
                thread_reference_internal(thread);
                thread_list[i++] = thread;
        }