osfmk/kern/zalloc_internal.h

   1 /*
   2  * Copyright (c) 2000-2020 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 /*
  29  * @OSF_COPYRIGHT@
  30  */
  31 /*
  32  * Mach Operating System
  33  * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
  34  * All Rights Reserved.
  35  *
  36  * Permission to use, copy, modify and distribute this software and its
  37  * documentation is hereby granted, provided that both the copyright
  38  * notice and this permission notice appear in all copies of the
  39  * software, derivative works or modified versions, and any portions
  40  * thereof, and that both notices appear in supporting documentation.
  41  *
  42  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  43  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
  44  * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
  45  *
  46  * Carnegie Mellon requests users of this software to return to
  47  *
  48  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  49  *  School of Computer Science
  50  *  Carnegie Mellon University
  51  *  Pittsburgh PA 15213-3890
  52  *
  53  * any improvements or extensions that they make and grant Carnegie Mellon
  54  * the rights to redistribute these changes.
  55  */
  56 /*
  57  */
  58
  59 #ifndef _KERN_ZALLOC_INTERNAL_H_
  60 #define _KERN_ZALLOC_INTERNAL_H_
  61
  62 #include <kern/zalloc.h>
  63 #include <kern/locks.h>
  64 #include <kern/btlog.h>
  65 #include <kern/simple_lock.h>
  66 #include <kern/zcache_internal.h>
  67
  68 #include <os/atomic_private.h>
  69
  70 #if KASAN
  71 #include <sys/queue.h>
  72 #include <san/kasan.h>
  73 /*
  74  * Set to 0 to debug poisoning and ZC_ZFREE_CLEARMEM validation under kasan.
  75  * Otherwise they are double-duty with what kasan already does.
  76  */
  77 #define ZALLOC_ENABLE_POISONING 0
  78 #else /* !KASAN */
  79 #define ZALLOC_ENABLE_POISONING 1
  80 #endif /* !KASAN */
  81
  82 #if DEBUG || DEVELOPMENT
  83 #define ZALLOC_DETAILED_STATS  1
  84 #else
  85 #define ZALLOC_DETAILED_STATS  0
  86 #endif
  87
  88 /*!
  89  * @file <kern/zalloc_internal.h>
  90  *
  91  * @abstract
  92  * Exposes some guts of zalloc to interact with the VM, debugging, copyio and
  93  * kalloc subsystems.
  94  */
  95
  96 __BEGIN_DECLS
  97
  98 #pragma GCC visibility push(hidden)
  99
 100 #if CONFIG_GZALLOC
 101 typedef struct gzalloc_data {
 102         uint32_t        gzfc_index;
 103         vm_offset_t     *gzfc;
 104 } gzalloc_data_t;
 105 #endif
 106
 107 /*
 108  *      A zone is a collection of fixed size blocks for which there
 109  *      is fast allocation/deallocation access.  Kernel routines can
 110  *      use zones to manage data structures dynamically, creating a zone
 111  *      for each type of data structure to be managed.
 112  *
 113  */
 114
 115 /*!
 116  * @typedef zone_pva_t
 117  *
 118  * @brief
 119  * Type used to point to a page virtual address in the zone allocator.
 120  *
 121  * @description
 122  * - Valid pages have the top bit set.
 123  * - 0 represents the "NULL" page
 124  * - non 0 values with the top bit cleared do not represent any valid page.
 125  *   the zone freelists use this space to encode "queue" addresses.
 126  */
 127 typedef struct zone_packed_virtual_address {
 128         uint32_t packed_address;
 129 } zone_pva_t;
 130
 131 /*!
 132  * @struct zone_stats
 133  *
 134  * @abstract
 135  * Per-cpu structure used for basic zone stats.
 136  *
 137  * @discussion
 138  * The values aren't scaled for per-cpu zones.
 139  */
 140 struct zone_stats {
 141         uint64_t            zs_mem_allocated;
 142         uint64_t            zs_mem_freed;
 143 #if ZALLOC_DETAILED_STATS
 144         uint64_t            zs_mem_wasted;
 145 #endif /* ZALLOC_DETAILED_STATS */
 146 };
 147
 148 struct zone {
 149         /*
 150          * Readonly / rarely written fields
 151          */
 152
 153         /*
 154          * The first 4 fields match a zone_view.
 155          *
 156          * z_self points back to the zone when the zone is initialized,
 157          * or is NULL else.
 158          */
 159         struct zone        *z_self;
 160         zone_stats_t        z_stats;
 161         const char         *z_name;
 162         struct zone_view   *z_views;
 163 #ifdef CONFIG_ZCACHE
 164         struct zone_cache   zcache;
 165 #endif  /* CONFIG_ZCACHE */
 166
 167         uint16_t            alloc_pages;    /* size used for more memory in pages */
 168         uint16_t            z_elem_size;    /* size of an element */
 169         uint16_t            pcpu_elem_size;
 170         uint16_t            prio_refill_count; /* if !=0 , refill to this count */
 171         uint32_t            page_count_max; /* how large can this zone grow */
 172
 173         uint32_t            page_count_hwm; /* page_count high watermark */
 174         uint32_t            page_count;     /* number of pages used by this zone */
 175         uint32_t            countavail;     /* Number of elements available */
 176
 177         uint64_t
 178         /*
 179          * Lifecycle state (Mutable after creation)
 180          */
 181             destroyed          :1,  /* zone is (being) destroyed */
 182             expanding_no_vm_priv:1, /* zone expanding via a non-vm_privileged thread */
 183             expanding_vm_priv  :1,  /* zone expanding via a vm_privileged thread */
 184             async_pending      :1,  /* asynchronous allocation pending? */
 185             waiting            :1,  /* is thread waiting for expansion? */
 186             zone_replenishing  :1,
 187
 188         /*
 189          * Security sensitive configuration bits
 190          */
 191             allows_foreign     :1,  /* allow non-zalloc space  */
 192             destructible       :1,  /* zone can be zdestroy()ed  */
 193             kalloc_heap        :2,  /* zone_kheap_id_t when part of a kalloc heap */
 194             noencrypt          :1,  /* do not encrypt pages when hibernating */
 195             submap_idx         :2,  /* a Z_SUBMAP_IDX_* value */
 196             va_sequester       :1,  /* page sequester: no VA reuse with other zones */
 197             zfree_clear_mem    :1,  /* clear memory of elements on free and assert on alloc */
 198
 199         /*
 200          * Behavior configuration bits
 201          */
 202             collectable        :1,  /* garbage collect empty pages */
 203             cpu_cache_enabled  :1,
 204             permanent          :1,  /* the zone allocations are permanent */
 205             exhaustible        :1,  /* merely return if empty? */
 206             expandable         :1,  /* expand zone (with message)? */
 207             no_callout         :1,
 208             percpu             :1,  /* the zone is percpu */
 209
 210             _reserved          :26,
 211
 212         /*
 213          * Debugging features
 214          */
 215             alignment_required :1,  /* element alignment needs to be preserved */
 216             gzalloc_tracked    :1,  /* this zone is tracked by gzalloc */
 217             gzalloc_exempt     :1,  /* this zone doesn't participate with gzalloc */
 218             kasan_fakestacks   :1,
 219             kasan_noquarantine :1,  /* whether to use the kasan quarantine */
 220             tag_zone_index     :7,
 221             tags               :1,
 222             tags_inline        :1,
 223             zleak_on           :1,  /* Are we collecting allocation information? */
 224             zone_logging       :1;  /* Enable zone logging for this zone. */
 225
 226         /*
 227          * often mutated fields
 228          */
 229
 230         decl_simple_lock_data(, lock);
 231
 232         /*
 233          * list of metadata structs, which maintain per-page free element lists
 234          *
 235          * Note: Due to the index packing in page metadata,
 236          *       these pointers can't be at the beginning of the zone struct.
 237          */
 238         zone_pva_t          pages_any_free_foreign;     /* foreign pages crammed into zone */
 239         zone_pva_t          pages_all_used_foreign;
 240         zone_pva_t          pages_all_free;
 241         zone_pva_t          pages_intermediate;
 242         zone_pva_t          pages_all_used;
 243         zone_pva_t          pages_sequester;            /* sequestered pages - allocated VA with no populated pages */
 244
 245         uint32_t            zp_count;                   /* counter for poisoning every N frees */
 246         uint32_t            countfree;                  /* Number of free elements */
 247         uint32_t            allfree_page_count;         /* Number of pages collectable by GC */
 248         uint32_t            sequester_page_count;
 249
 250 #if CONFIG_ZLEAKS
 251         uint32_t            zleak_capture;  /* per-zone counter for capturing every N allocations */
 252 #endif
 253 #if CONFIG_GZALLOC
 254         gzalloc_data_t      gz;
 255 #endif
 256 #if KASAN_ZALLOC
 257         vm_size_t           kasan_redzone;
 258 #endif
 259 #if DEBUG || DEVELOPMENT || CONFIG_ZLEAKS
 260         /* zone logging structure to hold stacks and element references to those stacks. */
 261         btlog_t            *zlog_btlog;
 262 #endif
 263 };
 264
 265
 266 __options_decl(zone_security_options_t, uint64_t, {
 267         /*
 268          * Zsecurity option to enable sequestering VA of zones
 269          */
 270         ZSECURITY_OPTIONS_SEQUESTER             = 0x00000001,
 271         /*
 272          * Zsecurity option to enable creating separate kalloc zones for
 273          * bags of bytes
 274          */
 275         ZSECURITY_OPTIONS_SUBMAP_USER_DATA      = 0x00000004,
 276         /*
 277          * Zsecurity option to enable sequestering of kalloc zones used by
 278          * kexts (KHEAP_KEXT heap)
 279          */
 280         ZSECURITY_OPTIONS_SEQUESTER_KEXT_KALLOC = 0x00000008,
 281         /*
 282          * Zsecurity option to enable strict free of iokit objects to zone
 283          * or heap they were allocated from.
 284          */
 285         ZSECURITY_OPTIONS_STRICT_IOKIT_FREE     = 0x00000010,
 286 });
 287
 288 #define KALLOC_MINALIGN     (1 << KALLOC_LOG2_MINALIGN)
 289 #define KALLOC_DLUT_SIZE    (2048 / KALLOC_MINALIGN)
 290
 291 struct kheap_zones {
 292         struct kalloc_zone_cfg         *cfg;
 293         struct kalloc_heap             *views;
 294         zone_kheap_id_t                 heap_id;
 295         uint16_t                        max_k_zone;
 296         uint8_t                         dlut[KALLOC_DLUT_SIZE];   /* table of indices into k_zone[] */
 297         uint8_t                         k_zindex_start;
 298         /* If there's no hit in the DLUT, then start searching from k_zindex_start. */
 299         zone_t                         *k_zone;
 300 };
 301
 302 extern zone_security_options_t zsecurity_options;
 303 extern uint32_t _Atomic        num_zones;
 304 extern uint32_t                zone_view_count;
 305 extern struct zone             zone_array[];
 306 extern lck_grp_t               zone_locks_grp;
 307 extern const char * const      kalloc_heap_names[KHEAP_ID_COUNT];
 308
 309 #define zone_index_foreach(i) \
 310         for (uint32_t i = 1, num_zones_##i = os_atomic_load(&num_zones, acquire); \
 311             i < num_zones_##i; i++)
 312
 313 __pure2
 314 static inline vm_offset_t
 315 zone_elem_size(zone_t zone)
 316 {
 317         return zone->z_elem_size;
 318 }
 319
 320 static inline uint32_t
 321 zone_count_allocated(zone_t zone)
 322 {
 323         return zone->countavail - zone->countfree;
 324 }
 325
 326 static inline vm_size_t
 327 zone_size_wired(zone_t zone)
 328 {
 329         /*
 330          * this either require the zone lock,
 331          * or to be used for statistics purposes only.
 332          */
 333         return ptoa(os_atomic_load(&zone->page_count, relaxed));
 334 }
 335
 336 static inline vm_size_t
 337 zone_size_free(zone_t zone)
 338 {
 339         return (vm_size_t)zone->pcpu_elem_size * zone->countfree;
 340 }
 341
 342 static inline vm_size_t
 343 zone_size_allocated(zone_t zone)
 344 {
 345         return (vm_size_t)zone->pcpu_elem_size * zone_count_allocated(zone);
 346 }
 347
 348 static inline vm_size_t
 349 zone_size_wasted(zone_t zone)
 350 {
 351         return zone_size_wired(zone) -
 352                (vm_size_t)zone->pcpu_elem_size * zone->countavail;
 353 }
 354
 355 /*
 356  * For sysctl kern.zones_collectable_bytes used by memory_maintenance to check if a
 357  * userspace reboot is needed. The only other way to query for this information
 358  * is via mach_memory_info() which is unavailable on release kernels.
 359  */
 360 extern uint64_t get_zones_collectable_bytes(void);
 361
 362 /*
 363  * zone_gc also checks if the zone maps are getting close to full and triggers
 364  * jetsams if needed, provided consider_jetsams is set to TRUE.
 365  *
 366  * To avoid deadlocks, we only pass a value of TRUE from within the
 367  * vm_pageout_garbage_collect thread.
 368  */
 369 extern void     zone_gc(boolean_t consider_jetsams);
 370 extern void     consider_zone_gc(boolean_t consider_jetsams);
 371
 372 /* Debug logging for zone-map-exhaustion jetsams. */
 373 extern void     get_zone_map_size(uint64_t *current_size, uint64_t *capacity);
 374 extern void     get_largest_zone_info(char *zone_name, size_t zone_name_len, uint64_t *zone_size);
 375
 376 /* Bootstrap zone module (create zone zone) */
 377 extern void     zone_bootstrap(void);
 378
 379 /*
 380  * Steal memory from pmap (prior to initialization of zalloc)
 381  * for the special vm zones that allow foreign memory and store
 382  * the range so as to facilitate range checking in zfree/zcram.
 383  */
 384 __startup_func
 385 extern vm_offset_t zone_foreign_mem_init(vm_size_t size);
 386
 387 /*
 388  * Returns size (greater than min_pages) that is a multiple
 389  * of the allocation granule for the zone.
 390  */
 391 __startup_func
 392 extern vm_size_t zone_get_foreign_alloc_size(
 393         const char          *name __unused,
 394         vm_size_t            elem_size,
 395         zone_create_flags_t  flags,
 396         uint16_t             min_pages);
 397
 398 extern bool     zone_maps_owned(
 399         vm_address_t    addr,
 400         vm_size_t       size);
 401
 402 extern void     zone_map_sizes(
 403         vm_map_size_t  *psize,
 404         vm_map_size_t  *pfree,
 405         vm_map_size_t  *plargest_free);
 406
 407 extern boolean_t
 408 is_zone_map_nearing_exhaustion(void);
 409
 410 #if defined(__LP64__)
 411 #define ZONE_POISON       0xdeadbeefdeadbeef
 412 #else
 413 #define ZONE_POISON       0xdeadbeef
 414 #endif
 415
 416 /*
 417  * Used by zalloc_direct_locked() and zcache to mark elements that have been
 418  * cleared or poisoned and need to be checked.
 419  */
 420 #define ZALLOC_ELEMENT_NEEDS_VALIDATION ((vm_offset_t)1)
 421
 422 static inline vm_tag_t
 423 zalloc_flags_get_tag(zalloc_flags_t flags)
 424 {
 425         return (vm_tag_t)((flags & Z_VM_TAG_MASK) >> Z_VM_TAG_SHIFT);
 426 }
 427
 428 extern void    *zalloc_ext(
 429         zone_t          zone,
 430         zone_stats_t    zstats,
 431         zalloc_flags_t  flags,
 432         vm_size_t       wasted);
 433
 434 extern void     zfree_ext(
 435         zone_t          zone,
 436         zone_stats_t    zstats,
 437         void           *addr);
 438
 439 /* free an element with no regard for gzalloc, zleaks, or kasan*/
 440 extern void     zfree_direct_locked(
 441         zone_t          zone,
 442         vm_offset_t     elem,
 443         bool            poison);
 444
 445 /*
 446  * attempts to allocate an element with no regard for gzalloc, zleaks, or kasan
 447  * returns an address possibly tagged with ZALLOC_ELEMENT_NEEDS_VALIDATION.
 448  */
 449 extern vm_offset_t zalloc_direct_locked(
 450         zone_t          zone,
 451         zalloc_flags_t  flags,
 452         vm_size_t       waste);
 453
 454 extern uint32_t zone_poison_count_init(
 455         zone_t          zone);
 456
 457 extern bool     zfree_clear_or_poison(
 458         zone_t          zone,
 459         uint32_t       *zp_count,
 460         vm_address_t    addr);
 461
 462 extern void     zone_clear_freelist_pointers(
 463         zone_t          zone,
 464         vm_offset_t     addr);
 465
 466 #if ZALLOC_ENABLE_POISONING
 467 extern void     zalloc_validate_element(
 468         zone_t          zone,
 469         vm_offset_t     addr,
 470         vm_size_t       size,
 471         bool            validate);
 472 #endif
 473
 474 extern void     zone_allocated_element_validate(
 475         zone_t          zone,
 476         vm_offset_t     addr);
 477
 478 extern void     zone_prio_refill_configure(
 479         zone_t          zone);
 480
 481 extern vm_size_t zone_element_size(
 482         void           *addr,
 483         zone_t         *z);
 484
 485 /*!
 486  * @function zone_owns
 487  *
 488  * @abstract
 489  * This function is a soft version of zone_require that checks if a given
 490  * pointer belongs to the specified zone and should not be used outside
 491  * allocator code.
 492  *
 493  * @discussion
 494  * Note that zone_owns() can only work with:
 495  * - zones not allowing foreign memory
 496  * - zones in the general submap.
 497  *
 498  * @param zone          the zone the address needs to belong to.
 499  * @param addr          the element address to check.
 500  */
 501 extern bool     zone_owns(
 502         zone_t          zone,
 503         void           *addr);
 504
 505 /*
 506  *  Structure for keeping track of a backtrace, used for leak detection.
 507  *  This is in the .h file because it is used during panic, see kern/debug.c
 508  *  A non-zero size indicates that the trace is in use.
 509  */
 510 struct ztrace {
 511         vm_size_t               zt_size;                        /* How much memory are all the allocations referring to this trace taking up? */
 512         uint32_t                zt_depth;                       /* depth of stack (0 to MAX_ZTRACE_DEPTH) */
 513         void*                   zt_stack[MAX_ZTRACE_DEPTH];     /* series of return addresses from OSBacktrace */
 514         uint32_t                zt_collisions;                  /* How many times did a different stack land here while it was occupied? */
 515         uint32_t                zt_hit_count;                   /* for determining effectiveness of hash function */
 516 };
 517
 518 #ifndef VM_MAX_TAG_ZONES
 519 #error MAX_TAG_ZONES
 520 #endif
 521 #if VM_MAX_TAG_ZONES
 522
 523 extern uint32_t zone_index_from_tag_index(
 524         uint32_t        tag_zone_index,
 525         vm_size_t      *elem_size);
 526
 527 #endif /* VM_MAX_TAG_ZONES */
 528
 529 #define lock_zone(zone)   simple_lock(&(zone)->lock, &zone_locks_grp)
 530 #define unlock_zone(zone) simple_unlock(&(zone)->lock)
 531
 532 #if CONFIG_GZALLOC
 533 void gzalloc_init(vm_size_t);
 534 void gzalloc_zone_init(zone_t);
 535 void gzalloc_empty_free_cache(zone_t);
 536 boolean_t gzalloc_enabled(void);
 537
 538 vm_offset_t gzalloc_alloc(zone_t, zone_stats_t zstats, zalloc_flags_t flags);
 539 void gzalloc_free(zone_t, zone_stats_t zstats, void *);
 540 boolean_t gzalloc_element_size(void *, zone_t *, vm_size_t *);
 541 #endif /* CONFIG_GZALLOC */
 542
 543 #define MAX_ZONE_NAME   32      /* max length of a zone name we can take from the boot-args */
 544 int track_this_zone(const char *zonename, const char *logname);
 545
 546 #if DEBUG || DEVELOPMENT
 547 extern boolean_t run_zone_test(void);
 548 extern void zone_gc_replenish_test(void);
 549 extern void zone_alloc_replenish_test(void);
 550 extern vm_size_t zone_element_info(void *addr, vm_tag_t * ptag);
 551 extern bool zalloc_disable_copyio_check;
 552 #else
 553 #define zalloc_disable_copyio_check false
 554 #endif /* DEBUG || DEVELOPMENT */
 555
 556 #pragma GCC visibility pop
 557
 558 __END_DECLS
 559
 560 #endif  /* _KERN_ZALLOC_INTERNAL_H_ */