xnu-3789.51.2.tar.gz

[apple/xnu.git] / bsd / sys / mcache.h

diff --git a/bsd/sys/mcache.h b/bsd/sys/mcache.h
index caa6250314b551e8f6391804f5dfe9e4dde6350c..906363384dd925a8fe35c323bf2e1189573adbc0 100644 (file)
--- a/bsd/sys/mcache.h
+++ b/bsd/sys/mcache.h
@@ -1,8 +1,8 @@
 /*
 /*

- * Copyright (c) 2006-2007 Apple Inc. All rights reserved.
+ * Copyright (c) 2006-2015 Apple Inc. All rights reserved.

  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@

- * 
+ *

  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in
  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in


@@ -11,10 +11,10 @@
  * unlawful or unlicensed copies of an Apple operating system, or to
  * circumvent, violate, or enable the circumvention or violation of, any
  * terms of an Apple operating system software license agreement.
  * unlawful or unlicensed copies of an Apple operating system, or to
  * circumvent, violate, or enable the circumvention or violation of, any
  * terms of an Apple operating system software license agreement.

- * 
+ *

  * Please obtain a copy of the License at
  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  * Please obtain a copy of the License at
  * http://www.opensource.apple.com/apsl/ and read it before using this file.

- * 
+ *

  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,


@@ -22,7 +22,7 @@
  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  * Please see the License for the specific language governing rights and
  * limitations under the License.
  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  * Please see the License for the specific language governing rights and
  * limitations under the License.

- * 
+ *

  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _SYS_MCACHE_H
  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _SYS_MCACHE_H


@@ -37,7 +37,9 @@ extern "C" {
 #include <sys/types.h>
 #include <sys/queue.h>
 #include <mach/boolean.h>
 #include <sys/types.h>
 #include <sys/queue.h>
 #include <mach/boolean.h>

+#include <mach/branch_predicates.h>

 #include <kern/locks.h>
 #include <kern/locks.h>

+#include <libkern/OSAtomic.h>

 
 #ifdef ASSERT
 #undef ASSERT
 
 #ifdef ASSERT
 #undef ASSERT


@@ -50,20 +52,136 @@ extern "C" {
 /*
  * Unlike VERIFY(), ASSERT() is evaluated only in DEBUG build.
  */
 /*
  * Unlike VERIFY(), ASSERT() is evaluated only in DEBUG build.
  */

-#define        VERIFY(EX)      ((void)((EX) || assfail(#EX, __FILE__, __LINE__)))
+#define        VERIFY(EX)      \
+       ((void)(__probable((EX)) || assfail(#EX, __FILE__, __LINE__)))

 #if DEBUG
 #define        ASSERT(EX)      VERIFY(EX)
 #else
 #define        ASSERT(EX)      ((void)0)
 #endif
 
 #if DEBUG
 #define        ASSERT(EX)      VERIFY(EX)
 #else
 #define        ASSERT(EX)      ((void)0)
 #endif
 

-#if defined(__ppc__)
-#define        CPU_CACHE_SIZE  128
-#elif defined(__arm__)
-#define        CPU_CACHE_SIZE  32
+/*
+ * Compile time assert; this should be on its own someday.
+ */
+#define        _CASSERT(x)     _Static_assert(x, "compile-time assertion failed")
+
+/*
+ * Atomic macros; these should be on their own someday.
+ */
+#define        atomic_add_16_ov(a, n)                                          \
+       ((u_int16_t) OSAddAtomic16(n, (volatile SInt16 *)a))
+
+#define        atomic_add_16(a, n)                                             \
+       ((void) atomic_add_16_ov(a, n))
+
+#define        atomic_add_32_ov(a, n)                                          \
+       ((u_int32_t) OSAddAtomic(n, (volatile SInt32 *)a))
+
+#define        atomic_add_32(a, n)                                             \
+       ((void) atomic_add_32_ov(a, n))
+
+#define        atomic_add_64_ov(a, n)                                          \
+       ((u_int64_t) OSAddAtomic64(n, (volatile SInt64 *)a))
+
+#define        atomic_add_64(a, n)                                             \
+       ((void) atomic_add_64_ov(a, n))
+
+#define        atomic_test_set_32(a, o, n)                                     \
+       OSCompareAndSwap(o, n, (volatile UInt32 *)a)
+
+#define        atomic_set_32(a, n) do {                                        \
+       while (!atomic_test_set_32(a, *a, n))                           \
+               ;                                                       \
+} while (0)
+
+#define        atomic_test_set_64(a, o, n)                                     \
+       OSCompareAndSwap64(o, n, (volatile UInt64 *)a)
+
+#define        atomic_set_64(a, n) do {                                        \
+       while (!atomic_test_set_64(a, *a, n))                           \
+               ;                                                       \
+} while (0)
+
+#if defined(__LP64__)
+#define        atomic_get_64(n, a) do {                                        \
+       (n) = *(a);                                                     \
+} while (0)

 #else
 #else

-#define        CPU_CACHE_SIZE  64
-#endif
+#define        atomic_get_64(n, a) do {                                        \
+       (n) = atomic_add_64_ov(a, 0);                                   \
+} while (0)
+#endif /* __LP64__ */
+
+#define        atomic_test_set_ptr(a, o, n)                                    \
+       OSCompareAndSwapPtr(o, n, (void * volatile *)a)
+
+#define        atomic_set_ptr(a, n) do {                                       \
+       while (!atomic_test_set_ptr(a, *a, n))                          \
+               ;                                                       \
+} while (0)
+
+#define        atomic_or_8_ov(a, n)                                            \
+       ((u_int8_t) OSBitOrAtomic8(n, (volatile UInt8 *)a))
+
+#define        atomic_or_8(a, n)                                               \
+       ((void) atomic_or_8_ov(a, n))
+
+#define        atomic_bitset_8(a, n)                                           \
+       atomic_or_8(a, n)
+
+#define        atomic_or_16_ov(a, n)                                           \
+       ((u_int16_t) OSBitOrAtomic16(n, (volatile UInt16 *)a))
+
+#define        atomic_or_16(a, n)                                              \
+       ((void) atomic_or_16_ov(a, n))
+
+#define        atomic_bitset_16(a, n)                                          \
+       atomic_or_16(a, n)
+
+#define        atomic_or_32_ov(a, n)                                           \
+       ((u_int32_t) OSBitOrAtomic(n, (volatile UInt32 *)a))
+
+#define        atomic_or_32(a, n)                                              \
+       ((void) atomic_or_32_ov(a, n))
+
+#define        atomic_bitset_32(a, n)                                          \
+       atomic_or_32(a, n)
+
+#define        atomic_and_8_ov(a, n)                                           \
+       ((u_int8_t) OSBitAndAtomic8(n, (volatile UInt8 *)a))
+
+#define        atomic_and_8(a, n)                                              \
+       ((void) atomic_and_8_ov(a, n))
+
+#define        atomic_bitclear_8(a, n)                                         \
+       atomic_and_8(a, ~(n))
+
+#define        atomic_and_16_ov(a, n)                                          \
+       ((u_int16_t) OSBitAndAtomic16(n, (volatile UInt16 *)a))
+
+#define        atomic_and_16(a, n)                                             \
+       ((void) atomic_and_16_ov(a, n))
+
+#define        atomic_bitclear_16(a, n)                                        \
+       atomic_and_16(a, ~(n))
+
+#define        atomic_and_32_ov(a, n)                                          \
+       ((u_int32_t) OSBitAndAtomic(n, (volatile UInt32 *)a))
+
+#define        atomic_and_32(a, n)                                             \
+       ((void) atomic_and_32_ov(a, n))
+
+#define        atomic_bitclear_32(a, n)                                        \
+       atomic_and_32(a, ~(n))
+
+#define        membar_sync     OSMemoryBarrier
+
+/*
+ * Use CPU_CACHE_LINE_SIZE instead of MAX_CPU_CACHE_LINE_SIZE, unless
+ * wasting space is of no concern.
+ */
+#define        MAX_CPU_CACHE_LINE_SIZE 128
+#define        CPU_CACHE_LINE_SIZE     mcache_cache_line_size()

 
 #ifndef IS_P2ALIGNED
 #define        IS_P2ALIGNED(v, a) \
 
 #ifndef IS_P2ALIGNED
 #define        IS_P2ALIGNED(v, a) \


@@ -72,7 +190,7 @@ extern "C" {
 
 #ifndef P2ROUNDUP
 #define        P2ROUNDUP(x, align) \
 
 #ifndef P2ROUNDUP
 #define        P2ROUNDUP(x, align) \

-       (-(-((uintptr_t)(x)) & -(align)))
+       (-(-((uintptr_t)(x)) & -((uintptr_t)align)))

 #endif /* P2ROUNDUP */
 
 #ifndef P2ROUNDDOWN
 #endif /* P2ROUNDUP */
 
 #ifndef P2ROUNDDOWN


@@ -148,12 +266,13 @@ typedef struct mcache_cpu {
        int             cc_objs;        /* number of objects in filled bkt */
        int             cc_pobjs;       /* number of objects in previous bkt */
        int             cc_bktsize;     /* number of elements in a full bkt */
        int             cc_objs;        /* number of objects in filled bkt */
        int             cc_pobjs;       /* number of objects in previous bkt */
        int             cc_bktsize;     /* number of elements in a full bkt */

-} __attribute__((aligned(CPU_CACHE_SIZE), packed)) mcache_cpu_t;
+} __attribute__((aligned(MAX_CPU_CACHE_LINE_SIZE))) mcache_cpu_t;

 
 typedef unsigned int (*mcache_allocfn_t)(void *, mcache_obj_t ***,
     unsigned int, int);
 typedef void (*mcache_freefn_t)(void *, mcache_obj_t *, boolean_t);
 typedef void (*mcache_auditfn_t)(void *, mcache_obj_t *, boolean_t);
 
 typedef unsigned int (*mcache_allocfn_t)(void *, mcache_obj_t ***,
     unsigned int, int);
 typedef void (*mcache_freefn_t)(void *, mcache_obj_t *, boolean_t);
 typedef void (*mcache_auditfn_t)(void *, mcache_obj_t *, boolean_t);

+typedef void (*mcache_logfn_t)(u_int32_t, mcache_obj_t *, boolean_t);

 typedef void (*mcache_notifyfn_t)(void *, u_int32_t);
 
 typedef struct mcache {
 typedef void (*mcache_notifyfn_t)(void *, u_int32_t);
 
 typedef struct mcache {


@@ -166,6 +285,7 @@ typedef struct mcache {
        mcache_allocfn_t mc_slab_alloc; /* slab layer allocate callback */
        mcache_freefn_t mc_slab_free;   /* slab layer free callback */
        mcache_auditfn_t mc_slab_audit; /* slab layer audit callback */
        mcache_allocfn_t mc_slab_alloc; /* slab layer allocate callback */
        mcache_freefn_t mc_slab_free;   /* slab layer free callback */
        mcache_auditfn_t mc_slab_audit; /* slab layer audit callback */

+       mcache_logfn_t mc_slab_log;     /* slab layer log callback */

        mcache_notifyfn_t mc_slab_notify; /* slab layer notify callback */
        void            *mc_private;    /* opaque arg to callbacks */
        size_t          mc_bufsize;     /* object size */
        mcache_notifyfn_t mc_slab_notify; /* slab layer notify callback */
        void            *mc_private;    /* opaque arg to callbacks */
        size_t          mc_bufsize;     /* object size */


@@ -205,61 +325,70 @@ typedef struct mcache {
        /*
         * Per-CPU layer, aligned at cache line boundary
         */
        /*
         * Per-CPU layer, aligned at cache line boundary
         */

-       mcache_cpu_t    mc_cpu[1];
+       mcache_cpu_t    mc_cpu[1]
+           __attribute__((aligned(MAX_CPU_CACHE_LINE_SIZE)));

 } mcache_t;
 
 #define        MCACHE_ALIGN    8       /* default guaranteed alignment */
 
 /* Valid values for mc_flags */
 #define        MCF_VERIFY      0x00000001      /* enable verification */
 } mcache_t;
 
 #define        MCACHE_ALIGN    8       /* default guaranteed alignment */
 
 /* Valid values for mc_flags */
 #define        MCF_VERIFY      0x00000001      /* enable verification */

-#define        MCF_AUDIT       0x00000002      /* enable transaction auditing */
+#define        MCF_TRACE       0x00000002      /* enable transaction auditing */

 #define        MCF_NOCPUCACHE  0x00000010      /* disable CPU layer caching */
 #define        MCF_NOCPUCACHE  0x00000010      /* disable CPU layer caching */

+#define        MCF_NOLEAKLOG   0x00000100      /* disable leak logging */
+#define        MCF_EXPLEAKLOG  0x00000200      /* expose leak info to user space */

 
 

-#define        MCF_DEBUG       (MCF_VERIFY | MCF_AUDIT)
-#define        MCF_FLAGS_MASK  (MCF_DEBUG | MCF_NOCPUCACHE)
+#define        MCF_DEBUG       (MCF_VERIFY | MCF_TRACE)
+#define        MCF_FLAGS_MASK  \
+       (MCF_DEBUG | MCF_NOCPUCACHE | MCF_NOLEAKLOG | MCF_EXPLEAKLOG)

 
 /* Valid values for notify callback */
 #define        MCN_RETRYALLOC  0x00000001      /* Allocation should be retried */
 
 #define        MCACHE_STACK_DEPTH 16
 
 
 /* Valid values for notify callback */
 #define        MCN_RETRYALLOC  0x00000001      /* Allocation should be retried */
 
 #define        MCACHE_STACK_DEPTH 16
 

+#define        MCA_TRN_MAX     2               /* Number of transactions to record */
+

 typedef struct mcache_audit {
        struct mcache_audit *mca_next;  /* next audit struct */
        void            *mca_addr;      /* address of buffer */
        mcache_t        *mca_cache;     /* parent cache of the buffer */
 typedef struct mcache_audit {
        struct mcache_audit *mca_next;  /* next audit struct */
        void            *mca_addr;      /* address of buffer */
        mcache_t        *mca_cache;     /* parent cache of the buffer */

-       struct thread   *mca_thread;    /* thread doing transaction */
-       struct thread   *mca_pthread;   /* previous transaction thread */
-       size_t          mca_contents_size; /* size of contents */
-       void            *mca_contents;  /* contents at last free */
-       uint16_t        mca_depth;      /* pc stack depth */
-       uint16_t        mca_pdepth;     /* previous transaction pc stack */
-       void            *mca_stack[MCACHE_STACK_DEPTH];
-       void            *mca_pstack[MCACHE_STACK_DEPTH];
+       size_t          mca_contents_size; /* size of saved contents */
+       void            *mca_contents;  /* user-specific saved contents */

        void            *mca_uptr;      /* user-specific pointer */
        uint32_t        mca_uflags;     /* user-specific flags */
        void            *mca_uptr;      /* user-specific pointer */
        uint32_t        mca_uflags;     /* user-specific flags */

+       uint32_t        mca_next_trn;
+       struct mca_trn {
+               struct thread   *mca_thread;    /* thread doing transaction */
+               uint32_t        mca_tstamp;
+               uint16_t        mca_depth;
+               void            *mca_stack[MCACHE_STACK_DEPTH];
+       } mca_trns[MCA_TRN_MAX];

 } mcache_audit_t;
 
 __private_extern__ int assfail(const char *, const char *, int);
 __private_extern__ void mcache_init(void);
 __private_extern__ unsigned int mcache_getflags(void);
 } mcache_audit_t;
 
 __private_extern__ int assfail(const char *, const char *, int);
 __private_extern__ void mcache_init(void);
 __private_extern__ unsigned int mcache_getflags(void);

+__private_extern__ unsigned int mcache_cache_line_size(void);

 __private_extern__ mcache_t *mcache_create(const char *, size_t,
     size_t, u_int32_t, int);
 __private_extern__ void *mcache_alloc(mcache_t *, int);
 __private_extern__ void mcache_free(mcache_t *, void *);
 __private_extern__ mcache_t *mcache_create_ext(const char *, size_t,
 __private_extern__ mcache_t *mcache_create(const char *, size_t,
     size_t, u_int32_t, int);
 __private_extern__ void *mcache_alloc(mcache_t *, int);
 __private_extern__ void mcache_free(mcache_t *, void *);
 __private_extern__ mcache_t *mcache_create_ext(const char *, size_t,

-    mcache_allocfn_t, mcache_freefn_t, mcache_auditfn_t, mcache_notifyfn_t,
-    void *, u_int32_t, int);
+    mcache_allocfn_t, mcache_freefn_t, mcache_auditfn_t, mcache_logfn_t,
+    mcache_notifyfn_t, void *, u_int32_t, int);

 __private_extern__ void mcache_destroy(mcache_t *);
 __private_extern__ unsigned int mcache_alloc_ext(mcache_t *, mcache_obj_t **,
     unsigned int, int);
 __private_extern__ void mcache_free_ext(mcache_t *, mcache_obj_t *);
 __private_extern__ void mcache_reap(void);
 __private_extern__ void mcache_destroy(mcache_t *);
 __private_extern__ unsigned int mcache_alloc_ext(mcache_t *, mcache_obj_t **,
     unsigned int, int);
 __private_extern__ void mcache_free_ext(mcache_t *, mcache_obj_t *);
 __private_extern__ void mcache_reap(void);

-__private_extern__ boolean_t mcache_purge_cache(mcache_t *);
+__private_extern__ boolean_t mcache_purge_cache(mcache_t *, boolean_t);

 __private_extern__ void mcache_waiter_inc(mcache_t *);
 __private_extern__ void mcache_waiter_dec(mcache_t *);
 __private_extern__ boolean_t mcache_bkt_isempty(mcache_t *);
 
 __private_extern__ void mcache_waiter_inc(mcache_t *);
 __private_extern__ void mcache_waiter_dec(mcache_t *);
 __private_extern__ boolean_t mcache_bkt_isempty(mcache_t *);
 

-__private_extern__ void mcache_buffer_log(mcache_audit_t *, void *, mcache_t *);
+__private_extern__ void mcache_buffer_log(mcache_audit_t *, void *, mcache_t *,
+    struct timeval *);

 __private_extern__ void mcache_set_pattern(u_int64_t, void *, size_t);
 __private_extern__ void *mcache_verify_pattern(u_int64_t, void *, size_t);
 __private_extern__ void *mcache_verify_set_pattern(u_int64_t, u_int64_t,
 __private_extern__ void mcache_set_pattern(u_int64_t, void *, size_t);
 __private_extern__ void *mcache_verify_pattern(u_int64_t, void *, size_t);
 __private_extern__ void *mcache_verify_set_pattern(u_int64_t, u_int64_t,


@@ -272,7 +401,11 @@ __private_extern__ char *mcache_dump_mca(mcache_audit_t *);
 __private_extern__ void mcache_audit_panic(mcache_audit_t *, void *, size_t,
     int64_t, int64_t);
 
 __private_extern__ void mcache_audit_panic(mcache_audit_t *, void *, size_t,
     int64_t, int64_t);
 

-__private_extern__ mcache_t *mcache_audit_cache;
+extern int32_t total_sbmb_cnt;
+extern int32_t total_sbmb_cnt_floor;
+extern int32_t total_sbmb_cnt_peak;
+extern int64_t sbmb_limreached;
+extern mcache_t *mcache_audit_cache;

 
 #ifdef  __cplusplus
 }
 
 #ifdef  __cplusplus
 }