osfmk/kern/stack.c

   1 /*
   2  * Copyright (c) 2003-2007 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 /*
  29  *      Kernel stack management routines.
  30  */
  31
  32 #include <mach/mach_host.h>
  33 #include <mach/mach_types.h>
  34 #include <mach/processor_set.h>
  35
  36 #include <kern/kern_types.h>
  37 #include <kern/mach_param.h>
  38 #include <kern/processor.h>
  39 #include <kern/thread.h>
  40 #include <kern/zalloc.h>
  41 #include <kern/kalloc.h>
  42 #include <kern/ledger.h>
  43
  44 #include <vm/vm_map.h>
  45 #include <vm/vm_kern.h>
  46
  47 #include <mach_debug.h>
  48 #include <san/kasan.h>
  49
  50 /*
  51  *      We allocate stacks from generic kernel VM.
  52  *
  53  *      The stack_free_list can only be accessed at splsched,
  54  *      because stack_alloc_try/thread_invoke operate at splsched.
  55  */
  56
  57 decl_simple_lock_data(static,stack_lock_data)
  58 #define stack_lock()            simple_lock(&stack_lock_data)
  59 #define stack_unlock()          simple_unlock(&stack_lock_data)
  60
  61 #define STACK_CACHE_SIZE        2
  62
  63 static vm_offset_t              stack_free_list;
  64
  65 static unsigned int             stack_free_count, stack_free_hiwat;             /* free list count */
  66 static unsigned int             stack_hiwat;
  67 unsigned int                    stack_total;                            /* current total count */
  68 unsigned long long              stack_allocs;                           /* total count of allocations */
  69
  70 static int                      stack_fake_zone_index = -1;     /* index in zone_info array */
  71
  72 static unsigned int             stack_free_target;
  73 static int                              stack_free_delta;
  74
  75 static unsigned int             stack_new_count;                                                /* total new stack allocations */
  76
  77 static vm_offset_t              stack_addr_mask;
  78
  79 unsigned int                    kernel_stack_pages;
  80 vm_offset_t                     kernel_stack_size;
  81 vm_offset_t                     kernel_stack_mask;
  82 vm_offset_t                     kernel_stack_depth_max;
  83
  84 /*
  85  *      The next field is at the base of the stack,
  86  *      so the low end is left unsullied.
  87  */
  88 #define stack_next(stack)       \
  89         (*((vm_offset_t *)((stack) + kernel_stack_size) - 1))
  90
  91 static inline int
  92 log2(vm_offset_t size)
  93 {
  94         int     result;
  95         for (result = 0; size > 0; result++)
  96                 size >>= 1;
  97         return result;
  98 }
  99
 100 static inline vm_offset_t
 101 roundup_pow2(vm_offset_t size)
 102 {
 103         return 1UL << (log2(size - 1) + 1);
 104 }
 105
 106 static vm_offset_t stack_alloc_internal(void);
 107 static void stack_free_stack(vm_offset_t);
 108
 109 void
 110 stack_init(void)
 111 {
 112         simple_lock_init(&stack_lock_data, 0);
 113
 114         kernel_stack_pages = KERNEL_STACK_SIZE / PAGE_SIZE;
 115         kernel_stack_size = KERNEL_STACK_SIZE;
 116         kernel_stack_mask = -KERNEL_STACK_SIZE;
 117         kernel_stack_depth_max = 0;
 118
 119         if (PE_parse_boot_argn("kernel_stack_pages",
 120                                &kernel_stack_pages,
 121                                sizeof (kernel_stack_pages))) {
 122                 kernel_stack_size = kernel_stack_pages * PAGE_SIZE;
 123                 printf("stack_init: kernel_stack_pages=%d kernel_stack_size=%p\n",
 124                         kernel_stack_pages, (void *) kernel_stack_size);
 125         }
 126
 127         if (kernel_stack_size < round_page(kernel_stack_size))
 128                 panic("stack_init: stack size %p not a multiple of page size %d\n",
 129                         (void *) kernel_stack_size, PAGE_SIZE);
 130
 131         stack_addr_mask = roundup_pow2(kernel_stack_size) - 1;
 132         kernel_stack_mask = ~stack_addr_mask;
 133 }
 134
 135 /*
 136  *      stack_alloc:
 137  *
 138  *      Allocate a stack for a thread, may
 139  *      block.
 140  */
 141
 142 static vm_offset_t
 143 stack_alloc_internal(void)
 144 {
 145         vm_offset_t             stack = 0;
 146         spl_t                   s;
 147         int                     flags = 0;
 148         kern_return_t           kr = KERN_SUCCESS;
 149
 150         s = splsched();
 151         stack_lock();
 152         stack_allocs++;
 153         stack = stack_free_list;
 154         if (stack != 0) {
 155                 stack_free_list = stack_next(stack);
 156                 stack_free_count--;
 157         }
 158         else {
 159                 if (++stack_total > stack_hiwat)
 160                         stack_hiwat = stack_total;
 161                 stack_new_count++;
 162         }
 163         stack_free_delta--;
 164         stack_unlock();
 165         splx(s);
 166
 167         if (stack == 0) {
 168
 169                 /*
 170                  * Request guard pages on either side of the stack.  Ask
 171                  * kernel_memory_allocate() for two extra pages to account
 172                  * for these.
 173                  */
 174
 175                 flags = KMA_GUARD_FIRST | KMA_GUARD_LAST | KMA_KSTACK | KMA_KOBJECT;
 176                 kr = kernel_memory_allocate(kernel_map, &stack,
 177                                            kernel_stack_size + (2*PAGE_SIZE),
 178                                            stack_addr_mask,
 179                                            flags,
 180                                            VM_KERN_MEMORY_STACK);
 181                 if (kr != KERN_SUCCESS) {
 182                         panic("stack_alloc: kernel_memory_allocate(size:0x%llx, mask: 0x%llx, flags: 0x%x) failed with %d\n", (uint64_t)(kernel_stack_size + (2*PAGE_SIZE)), (uint64_t)stack_addr_mask, flags, kr);
 183                 }
 184
 185                 /*
 186                  * The stack address that comes back is the address of the lower
 187                  * guard page.  Skip past it to get the actual stack base address.
 188                  */
 189
 190                 stack += PAGE_SIZE;
 191         }
 192         return stack;
 193 }
 194
 195 void
 196 stack_alloc(
 197         thread_t        thread)
 198 {
 199
 200         assert(thread->kernel_stack == 0);
 201         machine_stack_attach(thread, stack_alloc_internal());
 202 }
 203
 204 void
 205 stack_handoff(thread_t from, thread_t to)
 206 {
 207         assert(from == current_thread());
 208         machine_stack_handoff(from, to);
 209 }
 210
 211 /*
 212  *      stack_free:
 213  *
 214  *      Detach and free the stack for a thread.
 215  */
 216 void
 217 stack_free(
 218         thread_t        thread)
 219 {
 220     vm_offset_t         stack = machine_stack_detach(thread);
 221
 222 #if KASAN
 223         kasan_unpoison_stack(stack, kernel_stack_size);
 224         kasan_unpoison_fakestack(thread);
 225 #endif
 226
 227         assert(stack);
 228         if (stack != thread->reserved_stack) {
 229                 stack_free_stack(stack);
 230         }
 231 }
 232
 233 void
 234 stack_free_reserved(
 235         thread_t        thread)
 236 {
 237         if (thread->reserved_stack != thread->kernel_stack) {
 238 #if KASAN
 239                 kasan_unpoison_stack(thread->reserved_stack, kernel_stack_size);
 240 #endif
 241                 stack_free_stack(thread->reserved_stack);
 242         }
 243 }
 244
 245 static void
 246 stack_free_stack(
 247         vm_offset_t             stack)
 248 {
 249         struct stack_cache      *cache;
 250         spl_t                           s;
 251
 252         s = splsched();
 253         cache = &PROCESSOR_DATA(current_processor(), stack_cache);
 254         if (cache->count < STACK_CACHE_SIZE) {
 255                 stack_next(stack) = cache->free;
 256                 cache->free = stack;
 257                 cache->count++;
 258         }
 259         else {
 260                 stack_lock();
 261                 stack_next(stack) = stack_free_list;
 262                 stack_free_list = stack;
 263                 if (++stack_free_count > stack_free_hiwat)
 264                         stack_free_hiwat = stack_free_count;
 265                 stack_free_delta++;
 266                 stack_unlock();
 267         }
 268         splx(s);
 269 }
 270
 271 /*
 272  *      stack_alloc_try:
 273  *
 274  *      Non-blocking attempt to allocate a
 275  *      stack for a thread.
 276  *
 277  *      Returns TRUE on success.
 278  *
 279  *      Called at splsched.
 280  */
 281 boolean_t
 282 stack_alloc_try(
 283         thread_t                thread)
 284 {
 285         struct stack_cache      *cache;
 286         vm_offset_t                     stack;
 287
 288         cache = &PROCESSOR_DATA(current_processor(), stack_cache);
 289         stack = cache->free;
 290         if (stack != 0) {
 291                 cache->free = stack_next(stack);
 292                 cache->count--;
 293         }
 294         else {
 295                 if (stack_free_list != 0) {
 296                         stack_lock();
 297                         stack = stack_free_list;
 298                         if (stack != 0) {
 299                                 stack_free_list = stack_next(stack);
 300                                 stack_free_count--;
 301                                 stack_free_delta--;
 302                         }
 303                         stack_unlock();
 304                 }
 305         }
 306
 307         if (stack != 0 || (stack = thread->reserved_stack) != 0) {
 308                 machine_stack_attach(thread, stack);
 309                 return (TRUE);
 310         }
 311
 312         return (FALSE);
 313 }
 314
 315 static unsigned int             stack_collect_tick, last_stack_tick;
 316
 317 /*
 318  *      stack_collect:
 319  *
 320  *      Free excess kernel stacks, may
 321  *      block.
 322  */
 323 void
 324 stack_collect(void)
 325 {
 326         if (stack_collect_tick != last_stack_tick) {
 327                 unsigned int    target;
 328                 vm_offset_t             stack;
 329                 spl_t                   s;
 330
 331                 s = splsched();
 332                 stack_lock();
 333
 334                 target = stack_free_target + (STACK_CACHE_SIZE * processor_count);
 335                 target += (stack_free_delta >= 0)? stack_free_delta: -stack_free_delta;
 336
 337                 while (stack_free_count > target) {
 338                         stack = stack_free_list;
 339                         stack_free_list = stack_next(stack);
 340                         stack_free_count--; stack_total--;
 341                         stack_unlock();
 342                         splx(s);
 343
 344                         /*
 345                          * Get the stack base address, then decrement by one page
 346                          * to account for the lower guard page.  Add two extra pages
 347                          * to the size to account for the guard pages on both ends
 348                          * that were originally requested when the stack was allocated
 349                          * back in stack_alloc().
 350                          */
 351
 352                         stack = (vm_offset_t)vm_map_trunc_page(
 353                                 stack,
 354                                 VM_MAP_PAGE_MASK(kernel_map));
 355                         stack -= PAGE_SIZE;
 356                         if (vm_map_remove(
 357                                     kernel_map,
 358                                     stack,
 359                                     stack + kernel_stack_size+(2*PAGE_SIZE),
 360                                     VM_MAP_REMOVE_KUNWIRE)
 361                             != KERN_SUCCESS)
 362                                 panic("stack_collect: vm_map_remove");
 363                         stack = 0;
 364
 365                         s = splsched();
 366                         stack_lock();
 367
 368                         target = stack_free_target + (STACK_CACHE_SIZE * processor_count);
 369                         target += (stack_free_delta >= 0)? stack_free_delta: -stack_free_delta;
 370                 }
 371
 372                 last_stack_tick = stack_collect_tick;
 373
 374                 stack_unlock();
 375                 splx(s);
 376         }
 377 }
 378
 379 /*
 380  *      compute_stack_target:
 381  *
 382  *      Computes a new target free list count
 383  *      based on recent alloc / free activity.
 384  *
 385  *      Limits stack collection to once per
 386  *      computation period.
 387  */
 388 void
 389 compute_stack_target(
 390 __unused void           *arg)
 391 {
 392         spl_t           s;
 393
 394         s = splsched();
 395         stack_lock();
 396
 397         if (stack_free_target > 5)
 398                 stack_free_target = (4 * stack_free_target) / 5;
 399         else
 400         if (stack_free_target > 0)
 401                 stack_free_target--;
 402
 403         stack_free_target += (stack_free_delta >= 0)? stack_free_delta: -stack_free_delta;
 404
 405         stack_free_delta = 0;
 406         stack_collect_tick++;
 407
 408         stack_unlock();
 409         splx(s);
 410 }
 411
 412 void
 413 stack_fake_zone_init(int zone_index)
 414 {
 415         stack_fake_zone_index = zone_index;
 416 }
 417
 418 void
 419 stack_fake_zone_info(int *count,
 420                      vm_size_t *cur_size, vm_size_t *max_size, vm_size_t *elem_size, vm_size_t *alloc_size,
 421                      uint64_t *sum_size, int *collectable, int *exhaustable, int *caller_acct)
 422 {
 423         unsigned int    total, hiwat, free;
 424         unsigned long long all;
 425         spl_t                   s;
 426
 427         s = splsched();
 428         stack_lock();
 429         all = stack_allocs;
 430         total = stack_total;
 431         hiwat = stack_hiwat;
 432         free = stack_free_count;
 433         stack_unlock();
 434         splx(s);
 435
 436         *count      = total - free;
 437         *cur_size   = kernel_stack_size * total;
 438         *max_size   = kernel_stack_size * hiwat;
 439         *elem_size  = kernel_stack_size;
 440         *alloc_size = kernel_stack_size;
 441         *sum_size = all * kernel_stack_size;
 442
 443         *collectable = 1;
 444         *exhaustable = 0;
 445         *caller_acct = 1;
 446 }
 447
 448 /* OBSOLETE */
 449 void    stack_privilege(
 450                         thread_t        thread);
 451
 452 void
 453 stack_privilege(
 454         __unused thread_t       thread)
 455 {
 456         /* OBSOLETE */
 457 }
 458
 459 /*
 460  * Return info on stack usage for threads in a specific processor set
 461  */
 462 kern_return_t
 463 processor_set_stack_usage(
 464         processor_set_t pset,
 465         unsigned int    *totalp,
 466         vm_size_t       *spacep,
 467         vm_size_t       *residentp,
 468         vm_size_t       *maxusagep,
 469         vm_offset_t     *maxstackp)
 470 {
 471 #if !MACH_DEBUG
 472         return KERN_NOT_SUPPORTED;
 473 #else
 474         unsigned int total;
 475         vm_size_t maxusage;
 476         vm_offset_t maxstack;
 477
 478         thread_t *thread_list;
 479         thread_t thread;
 480
 481         unsigned int actual;    /* this many things */
 482         unsigned int i;
 483
 484         vm_size_t size, size_needed;
 485         void *addr;
 486
 487         if (pset == PROCESSOR_SET_NULL || pset != &pset0)
 488                 return KERN_INVALID_ARGUMENT;
 489
 490         size = 0;
 491         addr = NULL;
 492
 493         for (;;) {
 494                 lck_mtx_lock(&tasks_threads_lock);
 495
 496                 actual = threads_count;
 497
 498                 /* do we have the memory we need? */
 499
 500                 size_needed = actual * sizeof(thread_t);
 501                 if (size_needed <= size)
 502                         break;
 503
 504                 lck_mtx_unlock(&tasks_threads_lock);
 505
 506                 if (size != 0)
 507                         kfree(addr, size);
 508
 509                 assert(size_needed > 0);
 510                 size = size_needed;
 511
 512                 addr = kalloc(size);
 513                 if (addr == 0)
 514                         return KERN_RESOURCE_SHORTAGE;
 515         }
 516
 517         /* OK, have memory and list is locked */
 518         thread_list = (thread_t *) addr;
 519         for (i = 0, thread = (thread_t)(void *) queue_first(&threads);
 520                                         !queue_end(&threads, (queue_entry_t) thread);
 521                                         thread = (thread_t)(void *) queue_next(&thread->threads)) {
 522                 thread_reference_internal(thread);
 523                 thread_list[i++] = thread;
 524         }
 525         assert(i <= actual);
 526
 527         lck_mtx_unlock(&tasks_threads_lock);
 528
 529         /* calculate maxusage and free thread references */
 530
 531         total = 0;
 532         maxusage = 0;
 533         maxstack = 0;
 534         while (i > 0) {
 535                 thread_t threadref = thread_list[--i];
 536
 537                 if (threadref->kernel_stack != 0)
 538                         total++;
 539
 540                 thread_deallocate(threadref);
 541         }
 542
 543         if (size != 0)
 544                 kfree(addr, size);
 545
 546         *totalp = total;
 547         *residentp = *spacep = total * round_page(kernel_stack_size);
 548         *maxusagep = maxusage;
 549         *maxstackp = maxstack;
 550         return KERN_SUCCESS;
 551
 552 #endif  /* MACH_DEBUG */
 553 }
 554
 555 vm_offset_t min_valid_stack_address(void)
 556 {
 557         return (vm_offset_t)vm_map_min(kernel_map);
 558 }
 559
 560 vm_offset_t max_valid_stack_address(void)
 561 {
 562         return (vm_offset_t)vm_map_max(kernel_map);
 563 }