xnu-4570.1.46.tar.gz

[apple/xnu.git] / osfmk / kern / zalloc.h

diff --git a/osfmk/kern/zalloc.h b/osfmk/kern/zalloc.h
index 787bfc815b7aa6c27360be1e8edbec02591226a4..6a585b83fc87ee92b868e96e37fe1e2ff4d6f91d 100644 (file)
--- a/osfmk/kern/zalloc.h
+++ b/osfmk/kern/zalloc.h
@@ -79,6 +79,11 @@
 #include <kern/thread_call.h>
 #include <kern/btlog.h>
 
 #include <kern/thread_call.h>
 #include <kern/btlog.h>
 

+#if KASAN
+#include <sys/queue.h>
+#include <san/kasan.h>
+#endif
+

 #if    CONFIG_GZALLOC
 typedef struct gzalloc_data {
        uint32_t        gzfc_index;
 #if    CONFIG_GZALLOC
 typedef struct gzalloc_data {
        uint32_t        gzfc_index;


@@ -135,7 +140,12 @@ struct zone {
        /* boolean_t */ alignment_required :1,
        /* boolean_t */ zone_logging       :1,  /* Enable zone logging for this zone. */
        /* boolean_t */ zone_replenishing  :1,
        /* boolean_t */ alignment_required :1,
        /* boolean_t */ zone_logging       :1,  /* Enable zone logging for this zone. */
        /* boolean_t */ zone_replenishing  :1,

-       /* future    */ _reserved          :15;
+       /* boolean_t */ kasan_quarantine   :1,
+       /* boolean_t */ tags               :1,
+       /* boolean_t */ tags_inline        :1,
+       /* future    */ tag_zone_index     :6,
+       /* boolean_t */ zone_valid         :1,
+       /* future    */ _reserved          :5;

 
        int             index;          /* index into zone_info arrays for this zone */
        const char      *zone_name;     /* a name for the zone */
 
        int             index;          /* index into zone_info arrays for this zone */
        const char      *zone_name;     /* a name for the zone */


@@ -150,6 +160,10 @@ struct zone {
        gzalloc_data_t  gz;
 #endif /* CONFIG_GZALLOC */
 
        gzalloc_data_t  gz;
 #endif /* CONFIG_GZALLOC */
 

+#if KASAN_ZALLOC
+       vm_size_t kasan_redzone;
+#endif
+

        btlog_t         *zlog_btlog;            /* zone logging structure to hold stacks and element references to those stacks. */
 };
 
        btlog_t         *zlog_btlog;            /* zone logging structure to hold stacks and element references to those stacks. */
 };
 


@@ -163,8 +177,25 @@ typedef struct zinfo_usage_store_t {
        uint64_t        free __attribute__((aligned(8)));               /* free counter */
 } zinfo_usage_store_t;
 
        uint64_t        free __attribute__((aligned(8)));               /* free counter */
 } zinfo_usage_store_t;
 

-extern void            zone_gc(void);
-extern void            consider_zone_gc(void);
+/*
+ * For sysctl kern.zones_collectable_bytes used by memory_maintenance to check if a
+ * userspace reboot is needed. The only other way to query for this information
+ * is via mach_memory_info() which is unavailable on release kernels.
+ */
+extern uint64_t get_zones_collectable_bytes(void);
+
+/*
+ * zone_gc also checks if the zone_map is getting close to full and triggers jetsams if needed, provided
+ * consider_jetsams is set to TRUE. To avoid deadlocks, we only pass a value of TRUE from within the
+ * vm_pageout_garbage_collect thread.
+ */
+extern void            zone_gc(boolean_t consider_jetsams);
+extern void            consider_zone_gc(boolean_t consider_jetsams);
+extern void            drop_free_elements(zone_t z);
+
+/* Debug logging for zone-map-exhaustion jetsams. */
+extern void            get_zone_map_size(uint64_t *current_size, uint64_t *capacity);
+extern void            get_largest_zone_info(char *zone_name, size_t zone_name_len, uint64_t *zone_size);

 
 /* Bootstrap zone module (create zone zone) */
 extern void            zone_bootstrap(void);
 
 /* Bootstrap zone module (create zone zone) */
 extern void            zone_bootstrap(void);


@@ -202,6 +233,21 @@ extern void                zone_debug_disable(
 extern unsigned int            num_zones;
 extern struct zone             zone_array[];
 
 extern unsigned int            num_zones;
 extern struct zone             zone_array[];
 

+/* zindex and page_count must pack into 16 bits
+ * update tools/lldbmacros/memory.py:GetRealMetadata
+ * when these values change */
+
+#define ZINDEX_BITS              (10U)
+#define PAGECOUNT_BITS           (16U - ZINDEX_BITS)
+#define MULTIPAGE_METADATA_MAGIC ((1UL << ZINDEX_BITS) - 1)
+#define ZONE_CHUNK_MAXPAGES      ((1UL << PAGECOUNT_BITS) - 1)
+
+/*
+ * The max # of elements in a chunk should fit into zone_page_metadata.free_count (uint16_t).
+ * Update this if the type of free_count changes.
+ */
+#define ZONE_CHUNK_MAXELEMENTS   (UINT16_MAX)
+

 #endif /* MACH_KERNEL_PRIVATE */
 
 __BEGIN_DECLS
 #endif /* MACH_KERNEL_PRIVATE */
 
 __BEGIN_DECLS


@@ -217,8 +263,10 @@ __BEGIN_DECLS
 #define Z_NOCALLOUT    7       /* Don't asynchronously replenish the zone via callouts */
 #define Z_ALIGNMENT_REQUIRED 8
 #define Z_GZALLOC_EXEMPT 9     /* Not tracked in guard allocation mode */
 #define Z_NOCALLOUT    7       /* Don't asynchronously replenish the zone via callouts */
 #define Z_ALIGNMENT_REQUIRED 8
 #define Z_GZALLOC_EXEMPT 9     /* Not tracked in guard allocation mode */

-
-
+#define Z_KASAN_QUARANTINE     10 /* Allow zone elements to be quarantined on free */
+#ifdef XNU_KERNEL_PRIVATE
+#define Z_TAGS_ENABLED 11      /* Store tags */
+#endif  /* XNU_KERNEL_PRIVATE */

 
 #ifdef XNU_KERNEL_PRIVATE
 
 
 #ifdef XNU_KERNEL_PRIVATE
 


@@ -230,15 +278,18 @@ extern vm_offset_t     zone_map_max_address;
 extern void *  zalloc_nopagewait(
                                        zone_t          zone);
 
 extern void *  zalloc_nopagewait(
                                        zone_t          zone);
 

-/* Non-blocking version of zalloc */
-extern void *  zalloc_noblock(
-                                       zone_t          zone);
-

 /* selective version of zalloc */
 extern void *  zalloc_canblock(
                                        zone_t          zone,
                                        boolean_t       canblock);
 
 /* selective version of zalloc */
 extern void *  zalloc_canblock(
                                        zone_t          zone,
                                        boolean_t       canblock);
 

+/* selective version of zalloc */
+extern void *  zalloc_canblock_tag(
+                                       zone_t          zone,
+                                       boolean_t       canblock,
+                                       vm_size_t       reqsize,
+                                       vm_tag_t    tag);
+

 /* Get from zone free list */
 extern void *  zget(
                                        zone_t          zone);
 /* Get from zone free list */
 extern void *  zget(
                                        zone_t          zone);


@@ -306,6 +357,17 @@ extern int get_zleak_state(void);
 
 #endif /* CONFIG_ZLEAKS */
 
 
 #endif /* CONFIG_ZLEAKS */
 

+#ifndef VM_MAX_TAG_ZONES
+#error MAX_TAG_ZONES
+#endif
+
+#if VM_MAX_TAG_ZONES
+
+extern boolean_t zone_tagging_on;
+extern uint32_t  zone_index_from_tag_index(uint32_t tag_zone_index, vm_size_t * elem_size);
+
+#endif /* VM_MAX_TAG_ZONES */
+

 /* These functions used for leak detection both in zalloc.c and mbuf.c */
 extern uintptr_t hash_mix(uintptr_t);
 extern uint32_t hashbacktrace(uintptr_t *, uint32_t, uint32_t);
 /* These functions used for leak detection both in zalloc.c and mbuf.c */
 extern uintptr_t hash_mix(uintptr_t);
 extern uint32_t hashbacktrace(uintptr_t *, uint32_t, uint32_t);


@@ -313,7 +375,7 @@ extern uint32_t hashaddr(uintptr_t, uint32_t);
 
 #define lock_zone(zone)                                        \
 MACRO_BEGIN                                            \
 
 #define lock_zone(zone)                                        \
 MACRO_BEGIN                                            \

-       lck_mtx_lock_spin(&(zone)->lock);               \
+       lck_mtx_lock_spin_always(&(zone)->lock);        \

 MACRO_END
 
 #define unlock_zone(zone)                              \
 MACRO_END
 
 #define unlock_zone(zone)                              \


@@ -326,6 +388,7 @@ void gzalloc_init(vm_size_t);
 void gzalloc_zone_init(zone_t);
 void gzalloc_configure(void);
 void gzalloc_reconfigure(zone_t);
 void gzalloc_zone_init(zone_t);
 void gzalloc_configure(void);
 void gzalloc_reconfigure(zone_t);

+void gzalloc_empty_free_cache(zone_t);

 boolean_t gzalloc_enabled(void);
 
 vm_offset_t gzalloc_alloc(zone_t, boolean_t);
 boolean_t gzalloc_enabled(void);
 
 vm_offset_t gzalloc_alloc(zone_t, boolean_t);


@@ -337,12 +400,26 @@ boolean_t gzalloc_element_size(void *, zone_t *, vm_size_t *);
 void zlog_btlog_lock(__unused void *);
 void zlog_btlog_unlock(__unused void *);
 
 void zlog_btlog_lock(__unused void *);
 void zlog_btlog_unlock(__unused void *);
 

+#ifdef MACH_KERNEL_PRIVATE
+#define MAX_ZONE_NAME  32      /* max length of a zone name we can take from the boot-args */
+int track_this_zone(const char *zonename, const char *logname);
+#endif
+
+#if DEBUG || DEVELOPMENT
+extern boolean_t run_zone_test(void);
+extern vm_size_t zone_element_info(void *addr, vm_tag_t * ptag);
+#endif /* DEBUG || DEVELOPMENT */
+

 #endif /* XNU_KERNEL_PRIVATE */
 
 /* Allocate from zone */
 extern void *  zalloc(
                                        zone_t          zone);
 
 #endif /* XNU_KERNEL_PRIVATE */
 
 /* Allocate from zone */
 extern void *  zalloc(
                                        zone_t          zone);
 

+/* Non-blocking version of zalloc */
+extern void *  zalloc_noblock(
+                                       zone_t          zone);
+

 /* Free zone element */
 extern void            zfree(
                                        zone_t          zone,
 /* Free zone element */
 extern void            zfree(
                                        zone_t          zone,


@@ -361,6 +438,9 @@ extern void         zone_change(
                                        unsigned int    item,
                                        boolean_t               value);
 
                                        unsigned int    item,
                                        boolean_t               value);
 

+/* Destroy the zone */
+extern void            zdestroy(
+                                       zone_t          zone);

 
 __END_DECLS
 
 
 __END_DECLS